Skip to content

Commit

Permalink
fix: all others
Browse files Browse the repository at this point in the history 
Signed-off-by: yihong0618 <zouzou0208@gmail.com>
  • Loading branch information
@yihong0618
yihong0618 committed Feb 11, 2025
1 parent 17deae3 commit 312e4b2
Show file tree
Hide file tree
Showing 10 changed files with 281 additions and 251 deletions.
55 changes: 29 additions & 26 deletions migrations/models/upstage/models/llm/llm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,32 +482,35 @@ def _num_tokens_for_tools(self, tokenizer: Tokenizer, tools: list[PromptMessageT
num_tokens += len(tokenizer.encode("type"))
num_tokens += len(tokenizer.encode("function"))
num_tokens += len(tokenizer.encode("name"))
num_tokens += len(tokenizer.encode(tool.name))
if hasattr(tool, "name"):
num_tokens += len(tokenizer.encode(tool.name))
num_tokens += len(tokenizer.encode("description"))
num_tokens += len(tokenizer.encode(tool.description))
parameters = tool.parameters
num_tokens += len(tokenizer.encode("parameters"))
if "title" in parameters:
num_tokens += len(tokenizer.encode("title"))
num_tokens += len(tokenizer.encode(parameters.get("title")))
num_tokens += len(tokenizer.encode("type"))
num_tokens += len(tokenizer.encode(parameters.get("type")))
if "properties" in parameters:
num_tokens += len(tokenizer.encode("properties"))
for key, value in parameters.get("properties").items():
num_tokens += len(tokenizer.encode(key))
for field_key, field_value in value.items():
num_tokens += len(tokenizer.encode(field_key))
if field_key == "enum":
for enum_field in field_value:
num_tokens += 3
num_tokens += len(tokenizer.encode(enum_field))
else:
if hasattr(tool, "description"):
num_tokens += len(tokenizer.encode(tool.description))
if hasattr(tool, "parameters"):
parameters = tool.parameters
num_tokens += len(tokenizer.encode("parameters"))
if "title" in parameters:
num_tokens += len(tokenizer.encode("title"))
num_tokens += len(tokenizer.encode(parameters.get("title")))
num_tokens += len(tokenizer.encode("type"))
num_tokens += len(tokenizer.encode(parameters.get("type")))
if "properties" in parameters:
num_tokens += len(tokenizer.encode("properties"))
for key, value in parameters.get("properties").items():
num_tokens += len(tokenizer.encode(key))
for field_key, field_value in value.items():
num_tokens += len(tokenizer.encode(field_key))
num_tokens += len(tokenizer.encode(str(field_value)))
if "required" in parameters:
num_tokens += len(tokenizer.encode("required"))
for required_field in parameters["required"]:
num_tokens += 3
num_tokens += len(tokenizer.encode(required_field))
if field_key == "enum":
for enum_field in field_value:
num_tokens += 3
num_tokens += len(tokenizer.encode(enum_field))
else:
num_tokens += len(tokenizer.encode(field_key))
num_tokens += len(tokenizer.encode(str(field_value)))
if "required" in parameters:
num_tokens += len(tokenizer.encode("required"))
for required_field in parameters["required"]:
num_tokens += 3
num_tokens += len(tokenizer.encode(required_field))
return num_tokens
55 changes: 29 additions & 26 deletions models/azure_openai/models/llm/llm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -721,34 +721,37 @@ def _num_tokens_for_tools(
num_tokens += len(encoding.encode("type"))
num_tokens += len(encoding.encode("function"))
num_tokens += len(encoding.encode("name"))
num_tokens += len(encoding.encode(tool.name))
if hasattr(tool, "name"):
num_tokens += len(encoding.encode(tool.name))
num_tokens += len(encoding.encode("description"))
num_tokens += len(encoding.encode(tool.description))
parameters = tool.parameters
num_tokens += len(encoding.encode("parameters"))
if "title" in parameters:
num_tokens += len(encoding.encode("title"))
num_tokens += len(encoding.encode(parameters["title"]))
num_tokens += len(encoding.encode("type"))
num_tokens += len(encoding.encode(parameters["type"]))
if "properties" in parameters:
num_tokens += len(encoding.encode("properties"))
for key, value in parameters["properties"].items():
num_tokens += len(encoding.encode(key))
for field_key, field_value in value.items():
num_tokens += len(encoding.encode(field_key))
if field_key == "enum":
for enum_field in field_value:
num_tokens += 3
num_tokens += len(encoding.encode(enum_field))
else:
if hasattr(tool, "description"):
num_tokens += len(encoding.encode(tool.description))
if hasattr(tool, "parameters"):
parameters = tool.parameters
num_tokens += len(encoding.encode("parameters"))
if "title" in parameters:
num_tokens += len(encoding.encode("title"))
num_tokens += len(encoding.encode(parameters["title"]))
num_tokens += len(encoding.encode("type"))
num_tokens += len(encoding.encode(parameters["type"]))
if "properties" in parameters:
num_tokens += len(encoding.encode("properties"))
for key, value in parameters["properties"].items():
num_tokens += len(encoding.encode(key))
for field_key, field_value in value.items():
num_tokens += len(encoding.encode(field_key))
num_tokens += len(encoding.encode(str(field_value)))
if "required" in parameters:
num_tokens += len(encoding.encode("required"))
for required_field in parameters["required"]:
num_tokens += 3
num_tokens += len(encoding.encode(required_field))
if field_key == "enum":
for enum_field in field_value:
num_tokens += 3
num_tokens += len(encoding.encode(enum_field))
else:
num_tokens += len(encoding.encode(field_key))
num_tokens += len(encoding.encode(str(field_value)))
if "required" in parameters:
num_tokens += len(encoding.encode("required"))
for required_field in parameters["required"]:
num_tokens += 3
num_tokens += len(encoding.encode(required_field))
return num_tokens

@staticmethod
Expand Down
49 changes: 26 additions & 23 deletions models/chatglm/models/llm/llm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,29 +419,32 @@ def tokens(text: str):
num_tokens = 0
for tool in tools:
num_tokens += tokens("name")
num_tokens += tokens(tool.name)
if hasattr(tool, "name"):
num_tokens += tokens(tool.name)
num_tokens += tokens("description")
num_tokens += tokens(tool.description)
parameters = tool.parameters
num_tokens += tokens("parameters")
num_tokens += tokens("type")
num_tokens += tokens(parameters.get("type"))
if "properties" in parameters:
num_tokens += tokens("properties")
for key, value in parameters.get("properties").items():
num_tokens += tokens(key)
for field_key, field_value in value.items():
num_tokens += tokens(field_key)
if field_key == "enum":
for enum_field in field_value:
num_tokens += 3
num_tokens += tokens(enum_field)
else:
if hasattr(tool, "description"):
num_tokens += tokens(tool.description)
if hasattr(tool, "parameters"):
parameters = tool.parameters
num_tokens += tokens("parameters")
num_tokens += tokens("type")
num_tokens += tokens(parameters.get("type"))
if "properties" in parameters:
num_tokens += tokens("properties")
for key, value in parameters.get("properties").items():
num_tokens += tokens(key)
for field_key, field_value in value.items():
num_tokens += tokens(field_key)
num_tokens += tokens(str(field_value))
if "required" in parameters:
num_tokens += tokens("required")
for required_field in parameters["required"]:
num_tokens += 3
num_tokens += tokens(required_field)
if field_key == "enum":
for enum_field in field_value:
num_tokens += 3
num_tokens += tokens(enum_field)
else:
num_tokens += tokens(field_key)
num_tokens += tokens(str(field_value))
if "required" in parameters:
num_tokens += tokens("required")
for required_field in parameters["required"]:
num_tokens += 3
num_tokens += tokens(required_field)
return num_tokens
49 changes: 26 additions & 23 deletions models/localai/models/llm/llm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,31 +158,34 @@ def tokens(text: str):
num_tokens = 0
for tool in tools:
num_tokens += tokens("name")
num_tokens += tokens(tool.name)
if hasattr(tool, "name"):
num_tokens += tokens(tool.name)
num_tokens += tokens("description")
num_tokens += tokens(tool.description)
parameters = tool.parameters
num_tokens += tokens("parameters")
num_tokens += tokens("type")
num_tokens += tokens(parameters.get("type"))
if "properties" in parameters:
num_tokens += tokens("properties")
for key, value in parameters.get("properties").items():
num_tokens += tokens(key)
for field_key, field_value in value.items():
num_tokens += tokens(field_key)
if field_key == "enum":
for enum_field in field_value:
num_tokens += 3
num_tokens += tokens(enum_field)
else:
if hasattr(tool, "description"):
num_tokens += tokens(tool.description)
if hasattr(tool, "parameters"):
parameters = tool.parameters
num_tokens += tokens("parameters")
num_tokens += tokens("type")
num_tokens += tokens(parameters.get("type"))
if "properties" in parameters:
num_tokens += tokens("properties")
for key, value in parameters.get("properties").items():
num_tokens += tokens(key)
for field_key, field_value in value.items():
num_tokens += tokens(field_key)
num_tokens += tokens(str(field_value))
if "required" in parameters:
num_tokens += tokens("required")
for required_field in parameters["required"]:
num_tokens += 3
num_tokens += tokens(required_field)
if field_key == "enum":
for enum_field in field_value:
num_tokens += 3
num_tokens += tokens(enum_field)
else:
num_tokens += tokens(field_key)
num_tokens += tokens(str(field_value))
if "required" in parameters:
num_tokens += tokens("required")
for required_field in parameters["required"]:
num_tokens += 3
num_tokens += tokens(required_field)
return num_tokens

def validate_credentials(self, model: str, credentials: dict) -> None:
Expand Down
55 changes: 29 additions & 26 deletions models/novita/models/oaiapi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -755,34 +755,37 @@ def _num_tokens_for_tools(self, tools: list[PromptMessageTool]) -> int:

# calculate num tokens for function object
num_tokens += self._get_num_tokens_by_gpt2("name")
num_tokens += self._get_num_tokens_by_gpt2(tool.name)
if hasattr(tool, "name"):
num_tokens += self._get_num_tokens_by_gpt2(tool.name)
num_tokens += self._get_num_tokens_by_gpt2("description")
num_tokens += self._get_num_tokens_by_gpt2(tool.description)
parameters = tool.parameters
num_tokens += self._get_num_tokens_by_gpt2("parameters")
if "title" in parameters:
num_tokens += self._get_num_tokens_by_gpt2("title")
num_tokens += self._get_num_tokens_by_gpt2(parameters.get("title"))
num_tokens += self._get_num_tokens_by_gpt2("type")
num_tokens += self._get_num_tokens_by_gpt2(parameters.get("type"))
if "properties" in parameters:
num_tokens += self._get_num_tokens_by_gpt2("properties")
for key, value in parameters.get("properties").items():
num_tokens += self._get_num_tokens_by_gpt2(key)
for field_key, field_value in value.items():
num_tokens += self._get_num_tokens_by_gpt2(field_key)
if field_key == "enum":
for enum_field in field_value:
num_tokens += 3
num_tokens += self._get_num_tokens_by_gpt2(enum_field)
else:
if hasattr(tool, "description"):
num_tokens += self._get_num_tokens_by_gpt2(tool.description)
if hasattr(tool, "parameters"):
parameters = tool.parameters
num_tokens += self._get_num_tokens_by_gpt2("parameters")
if "title" in parameters:
num_tokens += self._get_num_tokens_by_gpt2("title")
num_tokens += self._get_num_tokens_by_gpt2(parameters.get("title"))
num_tokens += self._get_num_tokens_by_gpt2("type")
num_tokens += self._get_num_tokens_by_gpt2(parameters.get("type"))
if "properties" in parameters:
num_tokens += self._get_num_tokens_by_gpt2("properties")
for key, value in parameters.get("properties").items():
num_tokens += self._get_num_tokens_by_gpt2(key)
for field_key, field_value in value.items():
num_tokens += self._get_num_tokens_by_gpt2(field_key)
num_tokens += self._get_num_tokens_by_gpt2(str(field_value))
if "required" in parameters:
num_tokens += self._get_num_tokens_by_gpt2("required")
for required_field in parameters["required"]:
num_tokens += 3
num_tokens += self._get_num_tokens_by_gpt2(required_field)
if field_key == "enum":
for enum_field in field_value:
num_tokens += 3
num_tokens += self._get_num_tokens_by_gpt2(enum_field)
else:
num_tokens += self._get_num_tokens_by_gpt2(field_key)
num_tokens += self._get_num_tokens_by_gpt2(str(field_value))
if "required" in parameters:
num_tokens += self._get_num_tokens_by_gpt2("required")
for required_field in parameters["required"]:
num_tokens += 3
num_tokens += self._get_num_tokens_by_gpt2(required_field)

return num_tokens

Expand Down
Loading

0 comments on commit 312e4b2

Please sign in to comment.