Skip to content

Commit

Permalink
Set position_ids name for optimum-intel based modeling. Fix for model…
Browse files Browse the repository at this point in the history
… destroy
  • Loading branch information
slyalin committed Mar 8, 2024
1 parent 1866b77 commit 504704c
Showing 1 changed file with 7 additions and 5 deletions.
12 changes: 7 additions & 5 deletions vllm/worker/model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -467,6 +467,7 @@ def callback(m: Matcher) -> bool:
position_ids_parameter.append(opset13.parameter(shape=[-1, -1], dtype=np.int64, name="position_ids"))
print('CREATED A NEW position_ids PARAMETER')
replace_node(mapping[position_ids].get_node(), position_ids_parameter[0])
position_ids_parameter[0].get_output_tensor(0).set_names({'position_ids'})
print('APPLIED position_ids PARAMETER INSTEAD OF attention_mask-BASED SUB-GRAPH')
return True

Expand Down Expand Up @@ -574,11 +575,12 @@ def load_model(self) -> None:

def __del__(self):
# Order is important
del self.model.ov_request
del self.model.model
if gc: # when app is being destroyed the module may not be available
gc.collect()
del self.model.ov_node_factory
if hasattr(self.model, 'ov_node_factory'):
del self.model.ov_request
del self.model.model
if gc: # when app is being destroyed the module may not be available
gc.collect()
del self.model.ov_node_factory

def set_block_size(self, block_size: int) -> None:
self.block_size = block_size
Expand Down

0 comments on commit 504704c

Please sign in to comment.