Skip to content

Commit

Permalink
style: fix typo and format code (infiniflow#2618)
Browse files Browse the repository at this point in the history
### What problem does this PR solve?

- Fix typo
- Remove unused import
- Format code

### Type of change

- [x] Other (please describe): typo and format
  • Loading branch information
yqkcn authored Sep 27, 2024
1 parent 20415ab commit 680c516
Show file tree
Hide file tree
Showing 5 changed files with 12 additions and 16 deletions.
5 changes: 2 additions & 3 deletions api/db/services/llm_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,8 +169,8 @@ def increase_usage(cls, tenant_id, llm_type, used_tokens, llm_name=None):

num = 0
try:
for u in cls.query(tenant_id = tenant_id, llm_name=mdlnm):
num += cls.model.update(used_tokens = u.used_tokens + used_tokens)\
for u in cls.query(tenant_id=tenant_id, llm_name=mdlnm):
num += cls.model.update(used_tokens=u.used_tokens + used_tokens)\
.where(cls.model.tenant_id == tenant_id, cls.model.llm_name == mdlnm)\
.execute()
except Exception as e:
Expand Down Expand Up @@ -252,7 +252,6 @@ def tts(self, text):
return
yield chunk


def chat(self, system, history, gen_conf):
txt, used_tokens = self.mdl.chat(system, history, gen_conf)
if not TenantLLMService.increase_usage(
Expand Down
5 changes: 2 additions & 3 deletions graphrag/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import re
from concurrent.futures import ThreadPoolExecutor
import json
from functools import reduce
Expand All @@ -24,7 +23,7 @@
from api.db.services.user_service import TenantService
from graphrag.community_reports_extractor import CommunityReportsExtractor
from graphrag.entity_resolution import EntityResolution
from graphrag.graph_extractor import GraphExtractor
from graphrag.graph_extractor import GraphExtractor, DEFAULT_ENTITY_TYPES
from graphrag.mind_map_extractor import MindMapExtractor
from rag.nlp import rag_tokenizer
from rag.utils import num_tokens_from_string
Expand Down Expand Up @@ -52,7 +51,7 @@ def graph_merge(g1, g2):
return g


def build_knowlege_graph_chunks(tenant_id: str, chunks: List[str], callback, entity_types=["organization", "person", "location", "event", "time"]):
def build_knowledge_graph_chunks(tenant_id: str, chunks: List[str], callback, entity_types=DEFAULT_ENTITY_TYPES):
_, tenant = TenantService.get_by_id(tenant_id)
llm_bdl = LLMBundle(tenant_id, LLMType.CHAT, tenant.llm_id)
ext = GraphExtractor(llm_bdl)
Expand Down
8 changes: 4 additions & 4 deletions rag/app/knowledge_graph.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import re

from graphrag.index import build_knowlege_graph_chunks
from graphrag.index import build_knowledge_graph_chunks
from rag.app import naive
from rag.nlp import rag_tokenizer, tokenize_chunks

Expand All @@ -15,9 +15,9 @@ def chunk(filename, binary, tenant_id, from_page=0, to_page=100000,
parser_config["layout_recognize"] = False
sections = naive.chunk(filename, binary, from_page=from_page, to_page=to_page, section_only=True,
parser_config=parser_config, callback=callback)
chunks = build_knowlege_graph_chunks(tenant_id, sections, callback,
parser_config.get("entity_types", ["organization", "person", "location", "event", "time"])
)
chunks = build_knowledge_graph_chunks(tenant_id, sections, callback,
parser_config.get("entity_types", ["organization", "person", "location", "event", "time"])
)
for c in chunks: c["docnm_kwd"] = filename

doc = {
Expand Down
2 changes: 1 addition & 1 deletion rag/llm/chat_model.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
from openai import OpenAI
import openai
from ollama import Client
from volcengine.maas.v2 import MaasService
from rag.nlp import is_english
from rag.utils import num_tokens_from_string
from groq import Groq
Expand All @@ -29,6 +28,7 @@
import requests
import asyncio


class Base(ABC):
def __init__(self, key, model_name, base_url):
self.client = OpenAI(api_key=key, base_url=base_url)
Expand Down
8 changes: 3 additions & 5 deletions rag/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,9 @@ def findMaxTm(fnm):
def num_tokens_from_string(string: str) -> int:
"""Returns the number of tokens in a text string."""
try:
num_tokens = len(encoder.encode(string))
return num_tokens
except Exception as e:
pass
return 0
return len(encoder.encode(string))
except Exception:
return 0


def truncate(string: str, max_len: int) -> str:
Expand Down

0 comments on commit 680c516

Please sign in to comment.