Skip to content

Commit

Permalink
refine code to prevent exception (infiniflow#1231)
Browse files Browse the repository at this point in the history
### What problem does this PR solve?


### Type of change

- [x] Refactoring
  • Loading branch information
KevinHuSh authored Jun 21, 2024
1 parent 4af83e9 commit 74df0d9
Showing 1 changed file with 4 additions and 4 deletions.
8 changes: 4 additions & 4 deletions rag/app/naive.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
txt += l
sections = []
for sec in txt.split("\n"):
if num_tokens_from_string(sec) > 10 * parser_config.get("chunk_token_num", 128):
if num_tokens_from_string(sec) > 10 * int(parser_config.get("chunk_token_num", 128)):
sections.append((sec[:int(len(sec)/2)], ""))
sections.append((sec[int(len(sec)/2):], ""))
else:
Expand All @@ -169,7 +169,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,

elif re.search(r"\.json$", filename, re.IGNORECASE):
callback(0.1, "Start to parse.")
sections = JsonParser(parser_config.get("chunk_token_num", 128))(binary)
sections = JsonParser(int(parser_config.get("chunk_token_num", 128)))(binary)
sections = [(l, "") for l in sections if l]
callback(0.8, "Finish parsing.")

Expand All @@ -187,8 +187,8 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,

st = timer()
chunks = naive_merge(
sections, parser_config.get(
"chunk_token_num", 128), parser_config.get(
sections, int(parser_config.get(
"chunk_token_num", 128)), parser_config.get(
"delimiter", "\n!?。;!?"))

res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser))
Expand Down

0 comments on commit 74df0d9

Please sign in to comment.