diff --git a/api/utils/file_utils.py b/api/utils/file_utils.py index 2dedcdc7bcd..5ab61b63bc4 100644 --- a/api/utils/file_utils.py +++ b/api/utils/file_utils.py @@ -146,7 +146,7 @@ def rewrite_yaml_conf(conf_path, config): def rewrite_json_file(filepath, json_data): - with open(filepath, "w") as f: + with open(filepath, "w", encoding='utf-8') as f: json.dump(json_data, f, indent=4, separators=(",", ": ")) f.close() diff --git a/deepdoc/parser/resume/entities/schools.py b/deepdoc/parser/resume/entities/schools.py index d90d9fde02f..598d7ae83f5 100644 --- a/deepdoc/parser/resume/entities/schools.py +++ b/deepdoc/parser/resume/entities/schools.py @@ -11,7 +11,10 @@ # limitations under the License. # -import os, json,re,copy +import os +import json +import re +import copy import pandas as pd current_file_path = os.path.dirname(os.path.abspath(__file__)) TBL = pd.read_csv(os.path.join(current_file_path, "res/schools.csv"), sep="\t", header=0).fillna("") @@ -23,7 +26,7 @@ def loadRank(fnm): global TBL TBL["rank"] = 1000000 - with open(fnm, "r",encoding='UTF-8') as f: + with open(fnm, "r", encoding='utf-8') as f: while True: l = f.readline() if not l:break @@ -32,7 +35,7 @@ def loadRank(fnm): nm,rk = l[0].strip(),int(l[1]) #assert len(TBL[((TBL.name_cn == nm) | (TBL.name_en == nm))]),f"<{nm}>" TBL.loc[((TBL.name_cn == nm) | (TBL.name_en == nm)), "rank"] = rk - except Exception as e: + except Exception: pass diff --git a/deepdoc/vision/t_ocr.py b/deepdoc/vision/t_ocr.py index 910b91be1a1..041c7710754 100644 --- a/deepdoc/vision/t_ocr.py +++ b/deepdoc/vision/t_ocr.py @@ -41,7 +41,7 @@ def main(args): "score": 1} for b, t in bxs if b[0][0] <= b[1][0] and b[0][1] <= b[-1][1]] img = draw_box(images[i], bxs, ["ocr"], 1.) img.save(outputs[i], quality=95) - with open(outputs[i] + ".txt", "w+") as f: + with open(outputs[i] + ".txt", "w+", encoding='utf-8') as f: f.write("\n".join([o["text"] for o in bxs])) diff --git a/deepdoc/vision/t_recognizer.py b/deepdoc/vision/t_recognizer.py index e5d7a22ae7e..a5efc5dcdee 100644 --- a/deepdoc/vision/t_recognizer.py +++ b/deepdoc/vision/t_recognizer.py @@ -50,7 +50,7 @@ def main(args): if args.mode.lower() == "tsr": #lyt = [t for t in lyt if t["type"] == "table column"] html = get_table_html(images[i], lyt, ocr) - with open(outputs[i] + ".html", "w+") as f: + with open(outputs[i] + ".html", "w+", encoding='utf-8') as f: f.write(html) lyt = [{ "type": t["label"], diff --git a/docker/README.md b/docker/README.md index bb37a8ef62b..bab35dbe9e7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -14,7 +14,7 @@ - **docker-compose.yml** Sets up environment for RAGFlow and its dependencies. - **docker-compose-base.yml** - Sets up environment for RAGFlow's dependencies: Elasticsearch, MySQL, MinIO, and Redis. + Sets up environment for RAGFlow's dependencies: Elasticsearch/[Infinity](https://github.com/infiniflow/infinity), MySQL, MinIO, and Redis. ## 🐬 Docker environment variables diff --git a/docs/configurations.md b/docs/configurations.md index 286e695caf1..e94f846b597 100644 --- a/docs/configurations.md +++ b/docs/configurations.md @@ -32,7 +32,7 @@ docker compose -f docker/docker-compose.yml up -d - **docker-compose.yml** Sets up environment for RAGFlow and its dependencies. - **docker-compose-base.yml** - Sets up environment for RAGFlow's dependencies: Elasticsearch, MySQL, MinIO, and Redis. + Sets up environment for RAGFlow's dependencies: Elasticsearch/[Infinity](https://github.com/infiniflow/infinity), MySQL, MinIO, and Redis. ## Docker environment variables diff --git a/docs/guides/_category_.json b/docs/guides/_category_.json index 3512565ea1f..895506b000c 100644 --- a/docs/guides/_category_.json +++ b/docs/guides/_category_.json @@ -1,6 +1,6 @@ { "label": "Guides", - "position": 2, + "position": 3, "link": { "type": "generated-index", "description": "Guides for RAGFlow users and developers." diff --git a/docs/references/_category_.json b/docs/references/_category_.json index 75d4e0eff94..fec53356012 100644 --- a/docs/references/_category_.json +++ b/docs/references/_category_.json @@ -1,6 +1,6 @@ { "label": "References", - "position": 4, + "position": 6, "link": { "type": "generated-index", "description": "Miscellaneous References" diff --git a/docs/references/http_api_reference.md b/docs/references/http_api_reference.md index 381b32aa2fa..a904e14c286 100644 --- a/docs/references/http_api_reference.md +++ b/docs/references/http_api_reference.md @@ -2120,7 +2120,7 @@ Failure: --- -## Create session with an agent +## Create session with agent **POST** `/api/v1/agents/{agent_id}/sessions` diff --git a/docs/release_notes.md b/docs/release_notes.md index bad8b6bcd85..8b489a35317 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -13,7 +13,7 @@ Released on November 26, 2024. ### New features -- Supports [Infinity](https://github.com/infiniflow/infinity) or Elasticsearch (default) as document engine for vector storage and full-text indexing. +- Supports [Infinity](https://github.com/infiniflow/infinity) or Elasticsearch (default) as document engine for vector storage and full-text indexing. [#2894](https://github.com/infiniflow/ragflow/pull/2894) - Enhances user experience by adding more variables to the Agent and implementing auto-saving. - Adds a three-step translation agent template, inspired by [Andrew Ng's translation agent](https://github.com/andrewyng/translation-agent). - Adds an SEO-optimized blog writing agent template. @@ -22,16 +22,60 @@ Released on November 26, 2024. - Optimizes term weight calculations, reducing the retrieval time by 50%. - Improves task executor monitoring with additional performance indicators. - Replaces Redis with Valkey. -- Adds three new UI languages (contributed by community): Indonesian, Spanish, and Vietnamese. +- Adds three new UI languages (*contributed by the community*): Indonesian, Spanish, and Vietnamese. + +### Compatability changes + +As of this release, **service_config.yaml.template** replaces **service_config.yaml** for configuring backend services. Upon Docker container startup, the environment variables defined in this template file are automatically populated and a **service_config.yaml** is auto-generated from it. [#3341](https://github.com/infiniflow/ragflow/pull/3341) + +This approach eliminates the need to manually update **service_config.yaml** after making changes to **.env**, facilitating dynamic environment configurations. + +:::danger +Ensure that you [upgrade both your code and Docker image to this release](https://ragflow.io/docs/dev/upgrade_ragflow#upgrade-ragflow-to-the-most-recent-officially-published-release) before trying this new approach. +::: ### Related APIs #### HTTP APIs -- [Create session with agent](https://ragflow.io/docs/dev/http_api_reference#create-session-with-an-agent) +- [Create session with agent](https://ragflow.io/docs/dev/http_api_reference#create-session-with-agent) - [Converse with agent](https://ragflow.io/docs/dev/http_api_reference#converse-with-agent) #### Python APIs - [Create session with agent](https://ragflow.io/docs/dev/python_api_reference#create-session-with-agent) -- [Converse with agent](https://ragflow.io/docs/dev/python_api_reference#create-session-with-agent) \ No newline at end of file +- [Converse with agent](https://ragflow.io/docs/dev/python_api_reference#create-session-with-agent) + +### Documentation + +#### Added documents + +- [Configurations](https://ragflow.io/docs/dev/configurations) +- [Manage team members](https://ragflow.io/docs/dev/manage_team_members) +- [Run health check on RAGFlow's dependencies](https://ragflow.io/docs/dev/run_health_check) + +## v0.13.0 + +- Adds the team management functionality for all users. +- Updates the Agent UI to improve usability. +- Adds support for Markdown chunking in the **General** chunk method. +- Introduces an **invoke** tool within the Agent UI. +- Integrates support for Dify's knowledge base API. +- Adds support for GLM4-9B and Yi-Lightning models. +- Introduces HTTP and Python APIs for dataset management, file management within dataset, and chat assistant management. + +:::tip NOTE +To download RAGFlow's Python SDK: + +```bash +pip install ragflow-sdk==0.13.0 +``` +::: + +### Documentation + +#### Added documents + +- [Acquire a RAGFlow API key](https://ragflow.io/docs/dev/acquire_ragflow_api_key) +- [HTTP API Reference](https://ragflow.io/docs/dev/http_api_reference) +- [Python API Reference](https://ragflow.io/docs/dev/python_api_reference) \ No newline at end of file diff --git a/graphrag/search.py b/graphrag/search.py index 69f150f4dd1..4e9f444817b 100644 --- a/graphrag/search.py +++ b/graphrag/search.py @@ -59,8 +59,8 @@ def merge_into_first(sres, title="") -> dict[str, str]: q_vec = matchDense.embedding_data src = req.get("fields", ["docnm_kwd", "content_ltks", "kb_id", "img_id", "title_tks", "important_kwd", "doc_id", f"q_{len(q_vec)}_vec", "position_list", "name_kwd", - "q_1024_vec", "q_1536_vec", "available_int", "content_with_weight", - "weight_int", "weight_flt", "rank_int" + "available_int", "content_with_weight", + "weight_int", "weight_flt" ]) fusionExpr = FusionExpr("weighted_sum", 32, {"weights": "0.5, 0.5"}) diff --git a/poetry.lock b/poetry.lock index c8727d1af7d..e62c864b1f4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3808,84 +3808,86 @@ i18n = ["Babel (>=2.7)"] [[package]] name = "jiter" -version = "0.7.1" +version = "0.8.0" description = "Fast iterable JSON parser." optional = false python-versions = ">=3.8" files = [ - {file = "jiter-0.7.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:262e96d06696b673fad6f257e6a0abb6e873dc22818ca0e0600f4a1189eb334f"}, - {file = "jiter-0.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be6de02939aac5be97eb437f45cfd279b1dc9de358b13ea6e040e63a3221c40d"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935f10b802bc1ce2b2f61843e498c7720aa7f4e4bb7797aa8121eab017293c3d"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9cd3cccccabf5064e4bb3099c87bf67db94f805c1e62d1aefd2b7476e90e0ee2"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4aa919ebfc5f7b027cc368fe3964c0015e1963b92e1db382419dadb098a05192"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ae2d01e82c94491ce4d6f461a837f63b6c4e6dd5bb082553a70c509034ff3d4"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f9568cd66dbbdab67ae1b4c99f3f7da1228c5682d65913e3f5f95586b3cb9a9"}, - {file = "jiter-0.7.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9ecbf4e20ec2c26512736284dc1a3f8ed79b6ca7188e3b99032757ad48db97dc"}, - {file = "jiter-0.7.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b1a0508fddc70ce00b872e463b387d49308ef02b0787992ca471c8d4ba1c0fa1"}, - {file = "jiter-0.7.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f84c9996664c460f24213ff1e5881530abd8fafd82058d39af3682d5fd2d6316"}, - {file = "jiter-0.7.1-cp310-none-win32.whl", hash = "sha256:c915e1a1960976ba4dfe06551ea87063b2d5b4d30759012210099e712a414d9f"}, - {file = "jiter-0.7.1-cp310-none-win_amd64.whl", hash = "sha256:75bf3b7fdc5c0faa6ffffcf8028a1f974d126bac86d96490d1b51b3210aa0f3f"}, - {file = "jiter-0.7.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:ad04a23a91f3d10d69d6c87a5f4471b61c2c5cd6e112e85136594a02043f462c"}, - {file = "jiter-0.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e47a554de88dff701226bb5722b7f1b6bccd0b98f1748459b7e56acac2707a5"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e44fff69c814a2e96a20b4ecee3e2365e9b15cf5fe4e00869d18396daa91dab"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:df0a1d05081541b45743c965436f8b5a1048d6fd726e4a030113a2699a6046ea"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f22cf8f236a645cb6d8ffe2a64edb5d2b66fb148bf7c75eea0cb36d17014a7bc"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da8589f50b728ea4bf22e0632eefa125c8aa9c38ed202a5ee6ca371f05eeb3ff"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f20de711224f2ca2dbb166a8d512f6ff48c9c38cc06b51f796520eb4722cc2ce"}, - {file = "jiter-0.7.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8a9803396032117b85ec8cbf008a54590644a062fedd0425cbdb95e4b2b60479"}, - {file = "jiter-0.7.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3d8bae77c82741032e9d89a4026479061aba6e646de3bf5f2fc1ae2bbd9d06e0"}, - {file = "jiter-0.7.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3dc9939e576bbc68c813fc82f6620353ed68c194c7bcf3d58dc822591ec12490"}, - {file = "jiter-0.7.1-cp311-none-win32.whl", hash = "sha256:f7605d24cd6fab156ec89e7924578e21604feee9c4f1e9da34d8b67f63e54892"}, - {file = "jiter-0.7.1-cp311-none-win_amd64.whl", hash = "sha256:f3ea649e7751a1a29ea5ecc03c4ada0a833846c59c6da75d747899f9b48b7282"}, - {file = "jiter-0.7.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ad36a1155cbd92e7a084a568f7dc6023497df781adf2390c345dd77a120905ca"}, - {file = "jiter-0.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7ba52e6aaed2dc5c81a3d9b5e4ab95b039c4592c66ac973879ba57c3506492bb"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b7de0b6f6728b678540c7927587e23f715284596724be203af952418acb8a2d"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9463b62bd53c2fb85529c700c6a3beb2ee54fde8bef714b150601616dcb184a6"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:627164ec01d28af56e1f549da84caf0fe06da3880ebc7b7ee1ca15df106ae172"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:25d0e5bf64e368b0aa9e0a559c3ab2f9b67e35fe7269e8a0d81f48bbd10e8963"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c244261306f08f8008b3087059601997016549cb8bb23cf4317a4827f07b7d74"}, - {file = "jiter-0.7.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7ded4e4b75b68b843b7cea5cd7c55f738c20e1394c68c2cb10adb655526c5f1b"}, - {file = "jiter-0.7.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:80dae4f1889b9d09e5f4de6b58c490d9c8ce7730e35e0b8643ab62b1538f095c"}, - {file = "jiter-0.7.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5970cf8ec943b51bce7f4b98d2e1ed3ada170c2a789e2db3cb484486591a176a"}, - {file = "jiter-0.7.1-cp312-none-win32.whl", hash = "sha256:701d90220d6ecb3125d46853c8ca8a5bc158de8c49af60fd706475a49fee157e"}, - {file = "jiter-0.7.1-cp312-none-win_amd64.whl", hash = "sha256:7824c3ecf9ecf3321c37f4e4d4411aad49c666ee5bc2a937071bdd80917e4533"}, - {file = "jiter-0.7.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:097676a37778ba3c80cb53f34abd6943ceb0848263c21bf423ae98b090f6c6ba"}, - {file = "jiter-0.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3298af506d4271257c0a8f48668b0f47048d69351675dd8500f22420d4eec378"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12fd88cfe6067e2199964839c19bd2b422ca3fd792949b8f44bb8a4e7d21946a"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dacca921efcd21939123c8ea8883a54b9fa7f6545c8019ffcf4f762985b6d0c8"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de3674a5fe1f6713a746d25ad9c32cd32fadc824e64b9d6159b3b34fd9134143"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65df9dbae6d67e0788a05b4bad5706ad40f6f911e0137eb416b9eead6ba6f044"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ba9a358d59a0a55cccaa4957e6ae10b1a25ffdabda863c0343c51817610501d"}, - {file = "jiter-0.7.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:576eb0f0c6207e9ede2b11ec01d9c2182973986514f9c60bc3b3b5d5798c8f50"}, - {file = "jiter-0.7.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:e550e29cdf3577d2c970a18f3959e6b8646fd60ef1b0507e5947dc73703b5627"}, - {file = "jiter-0.7.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:81d968dbf3ce0db2e0e4dec6b0a0d5d94f846ee84caf779b07cab49f5325ae43"}, - {file = "jiter-0.7.1-cp313-none-win32.whl", hash = "sha256:f892e547e6e79a1506eb571a676cf2f480a4533675f834e9ae98de84f9b941ac"}, - {file = "jiter-0.7.1-cp313-none-win_amd64.whl", hash = "sha256:0302f0940b1455b2a7fb0409b8d5b31183db70d2b07fd177906d83bf941385d1"}, - {file = "jiter-0.7.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:c65a3ce72b679958b79d556473f192a4dfc5895e8cc1030c9f4e434690906076"}, - {file = "jiter-0.7.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e80052d3db39f9bb8eb86d207a1be3d9ecee5e05fdec31380817f9609ad38e60"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70a497859c4f3f7acd71c8bd89a6f9cf753ebacacf5e3e799138b8e1843084e3"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c1288bc22b9e36854a0536ba83666c3b1fb066b811019d7b682c9cf0269cdf9f"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b096ca72dd38ef35675e1d3b01785874315182243ef7aea9752cb62266ad516f"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8dbbd52c50b605af13dbee1a08373c520e6fcc6b5d32f17738875847fea4e2cd"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af29c5c6eb2517e71ffa15c7ae9509fa5e833ec2a99319ac88cc271eca865519"}, - {file = "jiter-0.7.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f114a4df1e40c03c0efbf974b376ed57756a1141eb27d04baee0680c5af3d424"}, - {file = "jiter-0.7.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:191fbaee7cf46a9dd9b817547bf556facde50f83199d07fc48ebeff4082f9df4"}, - {file = "jiter-0.7.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0e2b445e5ee627fb4ee6bbceeb486251e60a0c881a8e12398dfdff47c56f0723"}, - {file = "jiter-0.7.1-cp38-none-win32.whl", hash = "sha256:47ac4c3cf8135c83e64755b7276339b26cd3c7ddadf9e67306ace4832b283edf"}, - {file = "jiter-0.7.1-cp38-none-win_amd64.whl", hash = "sha256:60b49c245cd90cde4794f5c30f123ee06ccf42fb8730a019a2870cd005653ebd"}, - {file = "jiter-0.7.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:8f212eeacc7203256f526f550d105d8efa24605828382cd7d296b703181ff11d"}, - {file = "jiter-0.7.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d9e247079d88c00e75e297e6cb3a18a039ebcd79fefc43be9ba4eb7fb43eb726"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0aacaa56360139c53dcf352992b0331f4057a0373bbffd43f64ba0c32d2d155"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bc1b55314ca97dbb6c48d9144323896e9c1a25d41c65bcb9550b3e0c270ca560"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f281aae41b47e90deb70e7386558e877a8e62e1693e0086f37d015fa1c102289"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:93c20d2730a84d43f7c0b6fb2579dc54335db742a59cf9776d0b80e99d587382"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e81ccccd8069110e150613496deafa10da2f6ff322a707cbec2b0d52a87b9671"}, - {file = "jiter-0.7.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0a7d5e85766eff4c9be481d77e2226b4c259999cb6862ccac5ef6621d3c8dcce"}, - {file = "jiter-0.7.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f52ce5799df5b6975439ecb16b1e879d7655e1685b6e3758c9b1b97696313bfb"}, - {file = "jiter-0.7.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e0c91a0304373fdf97d56f88356a010bba442e6d995eb7773cbe32885b71cdd8"}, - {file = "jiter-0.7.1-cp39-none-win32.whl", hash = "sha256:5c08adf93e41ce2755970e8aa95262298afe2bf58897fb9653c47cd93c3c6cdc"}, - {file = "jiter-0.7.1-cp39-none-win_amd64.whl", hash = "sha256:6592f4067c74176e5f369228fb2995ed01400c9e8e1225fb73417183a5e635f0"}, - {file = "jiter-0.7.1.tar.gz", hash = "sha256:448cf4f74f7363c34cdef26214da527e8eeffd88ba06d0b80b485ad0667baf5d"}, + {file = "jiter-0.8.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:dee4eeb293ffcd2c3b31ebab684dbf7f7b71fe198f8eddcdf3a042cc6e10205a"}, + {file = "jiter-0.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aad1e6e9b01cf0304dcee14db03e92e0073287a6297caf5caf2e9dbfea16a924"}, + {file = "jiter-0.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:504099fb7acdbe763e10690d560a25d4aee03d918d6a063f3a761d8a09fb833f"}, + {file = "jiter-0.8.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2373487caad7fe39581f588ab5c9262fc1ade078d448626fec93f4ffba528858"}, + {file = "jiter-0.8.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c341ecc3f9bccde952898b0c97c24f75b84b56a7e2f8bbc7c8e38cab0875a027"}, + {file = "jiter-0.8.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0e48e7a336529b9419d299b70c358d4ebf99b8f4b847ed3f1000ec9f320e8c0c"}, + {file = "jiter-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5ee157a8afd2943be690db679f82fafb8d347a8342e8b9c34863de30c538d55"}, + {file = "jiter-0.8.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d7dceae3549b80087f913aad4acc2a7c1e0ab7cb983effd78bdc9c41cabdcf18"}, + {file = "jiter-0.8.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e29e9ecce53d396772590438214cac4ab89776f5e60bd30601f1050b34464019"}, + {file = "jiter-0.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fa1782f22d5f92c620153133f35a9a395d3f3823374bceddd3e7032e2fdfa0b1"}, + {file = "jiter-0.8.0-cp310-none-win32.whl", hash = "sha256:f754ef13b4e4f67a3bf59fe974ef4342523801c48bf422f720bd37a02a360584"}, + {file = "jiter-0.8.0-cp310-none-win_amd64.whl", hash = "sha256:796f750b65f5d605f5e7acaccc6b051675e60c41d7ac3eab40dbd7b5b81a290f"}, + {file = "jiter-0.8.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:f6f4e645efd96b4690b9b6091dbd4e0fa2885ba5c57a0305c1916b75b4f30ff6"}, + {file = "jiter-0.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f61cf6d93c1ade9b8245c9f14b7900feadb0b7899dbe4aa8de268b705647df81"}, + {file = "jiter-0.8.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0396bc5cb1309c6dab085e70bb3913cdd92218315e47b44afe9eace68ee8adaa"}, + {file = "jiter-0.8.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:62d0e42ec5dc772bd8554a304358220be5d97d721c4648b23f3a9c01ccc2cb26"}, + {file = "jiter-0.8.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ec4b711989860705733fc59fb8c41b2def97041cea656b37cf6c8ea8dee1c3f4"}, + {file = "jiter-0.8.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:859cc35bf304ab066d88f10a44a3251a9cd057fb11ec23e00be22206db878f4f"}, + {file = "jiter-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5000195921aa293b39b9b5bc959d7fa658e7f18f938c0e52732da8e3cc70a278"}, + {file = "jiter-0.8.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:36050284c0abde57aba34964d3920f3d6228211b65df7187059bb7c7f143759a"}, + {file = "jiter-0.8.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a88f608e050cfe45c48d771e86ecdbf5258314c883c986d4217cc79e1fb5f689"}, + {file = "jiter-0.8.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:646cf4237665b2e13b4159d8f26d53f59bc9f2e6e135e3a508a2e5dd26d978c6"}, + {file = "jiter-0.8.0-cp311-none-win32.whl", hash = "sha256:21fe5b8345db1b3023052b2ade9bb4d369417827242892051244af8fae8ba231"}, + {file = "jiter-0.8.0-cp311-none-win_amd64.whl", hash = "sha256:30c2161c5493acf6b6c3c909973fb64ae863747def01cc7574f3954e0a15042c"}, + {file = "jiter-0.8.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:d91a52d8f49ada2672a4b808a0c5c25d28f320a2c9ca690e30ebd561eb5a1002"}, + {file = "jiter-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c38cf25cf7862f61410b7a49684d34eb3b5bcbd7ddaf4773eea40e0bd43de706"}, + {file = "jiter-0.8.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6189beb5c4b3117624be6b2e84545cff7611f5855d02de2d06ff68e316182be"}, + {file = "jiter-0.8.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e13fa849c0e30643554add089983caa82f027d69fad8f50acadcb21c462244ab"}, + {file = "jiter-0.8.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7765ca159d0a58e8e0f8ca972cd6d26a33bc97b4480d0d2309856763807cd28"}, + {file = "jiter-0.8.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1b0befe7c6e9fc867d5bed21bab0131dfe27d1fa5cd52ba2bced67da33730b7d"}, + {file = "jiter-0.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7d6363d4c6f1052b1d8b494eb9a72667c3ef5f80ebacfe18712728e85327000"}, + {file = "jiter-0.8.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a873e57009863eeac3e3969e4653f07031d6270d037d6224415074ac17e5505c"}, + {file = "jiter-0.8.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2582912473c0d9940791479fe1bf2976a34f212eb8e0a82ee9e645ac275c5d16"}, + {file = "jiter-0.8.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:646163201af42f55393ee6e8f6136b8df488253a6533f4230a64242ecbfe6048"}, + {file = "jiter-0.8.0-cp312-none-win32.whl", hash = "sha256:96e75c9abfbf7387cba89a324d2356d86d8897ac58c956017d062ad510832dae"}, + {file = "jiter-0.8.0-cp312-none-win_amd64.whl", hash = "sha256:ed6074552b4a32e047b52dad5ab497223721efbd0e9efe68c67749f094a092f7"}, + {file = "jiter-0.8.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:dd5e351cb9b3e676ec3360a85ea96def515ad2b83c8ae3a251ce84985a2c9a6f"}, + {file = "jiter-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ba9f12b0f801ecd5ed0cec29041dc425d1050922b434314c592fc30d51022467"}, + {file = "jiter-0.8.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7ba461c3681728d556392e8ae56fb44a550155a24905f01982317b367c21dd4"}, + {file = "jiter-0.8.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a15ed47ab09576db560dbc5c2c5a64477535beb056cd7d997d5dd0f2798770e"}, + {file = "jiter-0.8.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cef55042816d0737142b0ec056c0356a5f681fb8d6aa8499b158e87098f4c6f8"}, + {file = "jiter-0.8.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:549f170215adeb5e866f10617c3d019d8eb4e6d4e3c6b724b3b8c056514a3487"}, + {file = "jiter-0.8.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f867edeb279d22020877640d2ea728de5817378c60a51be8af731a8a8f525306"}, + {file = "jiter-0.8.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:aef8845f463093799db4464cee2aa59d61aa8edcb3762aaa4aacbec3f478c929"}, + {file = "jiter-0.8.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:d0d6e22e4062c3d3c1bf3594baa2f67fc9dcdda8275abad99e468e0c6540bc54"}, + {file = "jiter-0.8.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:079e62e64696241ac3f408e337aaac09137ed760ccf2b72b1094b48745c13641"}, + {file = "jiter-0.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74d2b56ed3da5760544df53b5f5c39782e68efb64dc3aa0bba4cc08815e6fae8"}, + {file = "jiter-0.8.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:798dafe108cba58a7bb0a50d4d5971f98bb7f3c974e1373e750de6eb21c1a329"}, + {file = "jiter-0.8.0-cp313-none-win32.whl", hash = "sha256:ca6d3064dfc743eb0d3d7539d89d4ba886957c717567adc72744341c1e3573c9"}, + {file = "jiter-0.8.0-cp313-none-win_amd64.whl", hash = "sha256:38caedda64fe1f04b06d7011fc15e86b3b837ed5088657bf778656551e3cd8f9"}, + {file = "jiter-0.8.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:bb5c8a0a8d081c338db22e5b8d53a89a121790569cbb85f7d3cfb1fe0fbe9836"}, + {file = "jiter-0.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:202dbe8970bfb166fab950eaab8f829c505730a0b33cc5e1cfb0a1c9dd56b2f9"}, + {file = "jiter-0.8.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9046812e5671fdcfb9ae02881fff1f6a14d484b7e8b3316179a372cdfa1e8026"}, + {file = "jiter-0.8.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e6ac56425023e52d65150918ae25480d0a1ce2a6bf5ea2097f66a2cc50f6d692"}, + {file = "jiter-0.8.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dfcf97210c6eab9d2a1c6af15dd39e1d5154b96a7145d0a97fa1df865b7b834"}, + {file = "jiter-0.8.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4e3c8444d418686f78c9a547b9b90031faf72a0a1a46bfec7fb31edbd889c0d"}, + {file = "jiter-0.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6507011a299b7f578559084256405a8428875540d8d13530e00b688e41b09493"}, + {file = "jiter-0.8.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0aae4738eafdd34f0f25c2d3668ce9e8fa0d7cb75a2efae543c9a69aebc37323"}, + {file = "jiter-0.8.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7f5d782e790396b13f2a7b36bdcaa3736a33293bdda80a4bf1a3ce0cd5ef9f15"}, + {file = "jiter-0.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cc7f993bc2c4e03015445adbb16790c303282fce2e8d9dc3a3905b1d40e50564"}, + {file = "jiter-0.8.0-cp38-none-win32.whl", hash = "sha256:d4a8a6eda018a991fa58ef707dd51524055d11f5acb2f516d70b1be1d15ab39c"}, + {file = "jiter-0.8.0-cp38-none-win_amd64.whl", hash = "sha256:4cca948a3eda8ea24ed98acb0ee19dc755b6ad2e570ec85e1527d5167f91ff67"}, + {file = "jiter-0.8.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:ef89663678d8257063ce7c00d94638e05bd72f662c5e1eb0e07a172e6c1a9a9f"}, + {file = "jiter-0.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c402ddcba90b4cc71db3216e8330f4db36e0da2c78cf1d8a9c3ed8f272602a94"}, + {file = "jiter-0.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a6dfe795b7a173a9f8ba7421cdd92193d60c1c973bbc50dc3758a9ad0fa5eb6"}, + {file = "jiter-0.8.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ec29a31b9abd6be39453a2c45da067138a3005d65d2c0507c530e0f1fdcd9a4"}, + {file = "jiter-0.8.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a488f8c54bddc3ddefaf3bfd6de4a52c97fc265d77bc2dcc6ee540c17e8c342"}, + {file = "jiter-0.8.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aeb5561adf4d26ca0d01b5811b4d7b56a8986699a473d700757b4758ef787883"}, + {file = "jiter-0.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ab961858d7ad13132328517d29f121ae1b2d94502191d6bcf96bddcc8bb5d1c"}, + {file = "jiter-0.8.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a207e718d114d23acf0850a2174d290f42763d955030d9924ffa4227dbd0018f"}, + {file = "jiter-0.8.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:733bc9dc8ff718a0ae4695239e9268eb93e88b73b367dfac3ec227d8ce2f1e77"}, + {file = "jiter-0.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d1ec27299e22d05e13a06e460bf7f75f26f9aaa0e0fb7d060f40e88df1d81faa"}, + {file = "jiter-0.8.0-cp39-none-win32.whl", hash = "sha256:e8dbfcb46553e6661d3fc1f33831598fcddf73d0f67834bce9fc3e9ebfe5c439"}, + {file = "jiter-0.8.0-cp39-none-win_amd64.whl", hash = "sha256:af2ce2487b3a93747e2cb5150081d4ae1e5874fce5924fc1a12e9e768e489ad8"}, + {file = "jiter-0.8.0.tar.gz", hash = "sha256:86fee98b569d4cc511ff2e3ec131354fafebd9348a487549c31ad371ae730310"}, ] [[package]] @@ -10077,4 +10079,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "aecf86b4252ba40df4553226f3fec3a1130b4bd4a9bae226337de4739a62e799" +content-hash = "393f51d55da83dc829e387a5f08087a2e90a40ea63dd034586b7717143a115ca" diff --git a/pyproject.toml b/pyproject.toml index 47591319924..29f45a0abc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ azure-identity = "1.17.1" azure-storage-file-datalake = "12.16.0" anthropic = "=0.34.1" arxiv = "2.1.3" -aspose-slides = { version = "^24.9.0", markers = "platform_machine == 'x86_64'" } +aspose-slides = { version = "^24.9.0", markers = "platform_machine == 'x86_64' or (sys_platform == 'darwin' and platform_machine == 'arm64') " } beartype = "^0.18.5" bio = "1.7.1" boto3 = "1.34.140" diff --git a/rag/benchmark.py b/rag/benchmark.py index dc48bed3ebc..1146d55bf6a 100644 --- a/rag/benchmark.py +++ b/rag/benchmark.py @@ -237,8 +237,8 @@ def save_results(self, qrels, run, texts, dataset, file_path): scores = sorted(scores, key=lambda kk: kk[1]) for score in scores[:10]: f.write('- text: ' + str(texts[score[0]]) + '\t qrel: ' + str(score[1]) + '\n') - json.dump(qrels, open(os.path.join(file_path, dataset + '.qrels.json'), "w+"), indent=2) - json.dump(run, open(os.path.join(file_path, dataset + '.run.json'), "w+"), indent=2) + json.dump(qrels, open(os.path.join(file_path, dataset + '.qrels.json'), "w+", encoding='utf-8'), indent=2) + json.dump(run, open(os.path.join(file_path, dataset + '.run.json'), "w+", encoding='utf-8'), indent=2) print(os.path.join(file_path, dataset + '_result.md'), 'Saved!') def __call__(self, dataset, file_path, miracl_corpus=''): diff --git a/rag/llm/embedding_model.py b/rag/llm/embedding_model.py index 5ad6e2a92f7..b6e2b887cdb 100644 --- a/rag/llm/embedding_model.py +++ b/rag/llm/embedding_model.py @@ -38,7 +38,7 @@ class Base(ABC): def __init__(self, key, model_name): pass - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): raise NotImplementedError("Please implement encode method!") def encode_queries(self, text: str): @@ -78,7 +78,7 @@ def __init__(self, key, model_name, **kwargs): use_fp16=torch.cuda.is_available()) self._model = DefaultEmbedding._model - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): texts = [truncate(t, 2048) for t in texts] token_count = 0 for t in texts: @@ -101,7 +101,7 @@ def __init__(self, key, model_name="text-embedding-ada-002", self.client = OpenAI(api_key=key, base_url=base_url) self.model_name = model_name - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): texts = [truncate(t, 8191) for t in texts] res = self.client.embeddings.create(input=texts, model=self.model_name) @@ -123,7 +123,7 @@ def __init__(self, key, model_name, base_url): self.client = OpenAI(api_key="empty", base_url=base_url) self.model_name = model_name.split("___")[0] - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): res = self.client.embeddings.create(input=texts, model=self.model_name) return ( np.array([d.embedding for d in res.data]), @@ -200,7 +200,7 @@ def __init__(self, key, model_name="embedding-2", **kwargs): self.client = ZhipuAI(api_key=key) self.model_name = model_name - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): arr = [] tks_num = 0 for txt in texts: @@ -221,7 +221,7 @@ def __init__(self, key, model_name, **kwargs): self.client = Client(host=kwargs["base_url"]) self.model_name = model_name - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): arr = [] tks_num = 0 for txt in texts: @@ -252,7 +252,7 @@ def __init__( from fastembed import TextEmbedding self._model = TextEmbedding(model_name, cache_dir, threads, **kwargs) - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): # Using the internal tokenizer to encode the texts and get the total # number of tokens encodings = self._model.model.tokenizer.encode_batch(texts) @@ -278,7 +278,7 @@ def __init__(self, key, model_name="", base_url=""): self.client = OpenAI(api_key=key, base_url=base_url) self.model_name = model_name - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): res = self.client.embeddings.create(input=texts, model=self.model_name) return np.array([d.embedding for d in res.data] @@ -394,7 +394,7 @@ def __init__(self, key, model_name="mistral-embed", self.client = MistralClient(api_key=key) self.model_name = model_name - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): texts = [truncate(t, 8196) for t in texts] res = self.client.embeddings(input=texts, model=self.model_name) @@ -418,7 +418,7 @@ def __init__(self, key, model_name, self.client = boto3.client(service_name='bedrock-runtime', region_name=self.bedrock_region, aws_access_key_id=self.bedrock_ak, aws_secret_access_key=self.bedrock_sk) - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): texts = [truncate(t, 8196) for t in texts] embeddings = [] token_count = 0 @@ -456,7 +456,7 @@ def __init__(self, key, model_name='models/text-embedding-004', genai.configure(api_key=key) self.model_name = 'models/' + model_name - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): texts = [truncate(t, 2048) for t in texts] token_count = sum(num_tokens_from_string(text) for text in texts) result = genai.embed_content( @@ -541,7 +541,7 @@ def __init__(self, key, model_name, base_url=None): self.client = Client(api_key=key) self.model_name = model_name - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): res = self.client.embed( texts=texts, model=self.model_name, @@ -599,7 +599,7 @@ def __init__( self.base_url = base_url self.model_name = model_name - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): payload = { "model": self.model_name, "input": texts, @@ -628,7 +628,7 @@ def __init__(self, key, model_name, base_url=None): self.model_name = model_name self.client = Client(api_token=key) - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): res = self.client.run(self.model_name, input={"texts": json.dumps(texts)}) return np.array(res), sum([num_tokens_from_string(text) for text in texts]) @@ -647,7 +647,7 @@ def __init__(self, key, model_name, base_url=None): self.client = qianfan.Embedding(ak=ak, sk=sk) self.model_name = model_name - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): res = self.client.do(model=self.model_name, texts=texts).body return ( np.array([r["embedding"] for r in res["data"]]), @@ -669,7 +669,7 @@ def __init__(self, key, model_name, base_url=None): self.client = voyageai.Client(api_key=key) self.model_name = model_name - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): res = self.client.embed( texts=texts, model=self.model_name, input_type="document" ) @@ -691,7 +691,7 @@ def __init__(self, key, model_name, base_url=None): self.model_name = model_name self.base_url = base_url or "http://127.0.0.1:8080" - def encode(self, texts: list, batch_size=32): + def encode(self, texts: list, batch_size=16): embeddings = [] for text in texts: response = requests.post( diff --git a/rag/nlp/query.py b/rag/nlp/query.py index 3da59d31beb..63fed29b8aa 100644 --- a/rag/nlp/query.py +++ b/rag/nlp/query.py @@ -54,7 +54,7 @@ def isChinese(line): def rmWWW(txt): patts = [ ( - r"是*(什么样的|哪家|一下|那家|请问|啥样|咋样了|什么时候|何时|何地|何人|是否|是不是|多少|哪里|怎么|哪儿|怎么样|如何|哪些|是啥|啥是|啊|吗|呢|吧|咋|什么|有没有|呀)是*", + r"是*(什么样的|哪家|一下|那家|请问|啥样|咋样了|什么时候|何时|何地|何人|是否|是不是|多少|哪里|怎么|哪儿|怎么样|如何|哪些|是啥|啥是|啊|吗|呢|吧|咋|什么|有没有|呀|谁|哪位|哪个)是*", "", ), (r"(^| )(what|who|how|which|where|why)('re|'s)? ", " "), diff --git a/rag/nlp/term_weight.py b/rag/nlp/term_weight.py index 374065a0c91..810f6b88176 100644 --- a/rag/nlp/term_weight.py +++ b/rag/nlp/term_weight.py @@ -228,7 +228,7 @@ def idf(s, N): return math.log10(10 + ((N - s + 0.5) / (s + 0.5))) idf2 = np.array([idf(df(t), 1000000000) for t in tks]) wts = (0.3 * idf1 + 0.7 * idf2) * \ np.array([ner(t) * postag(t) for t in tks]) - wts = [math.pow(s, 2) for s in wts] + wts = [s for s in wts] tw = list(zip(tks, wts)) else: for tk in tks: @@ -237,7 +237,7 @@ def idf(s, N): return math.log10(10 + ((N - s + 0.5) / (s + 0.5))) idf2 = np.array([idf(df(t), 1000000000) for t in tt]) wts = (0.3 * idf1 + 0.7 * idf2) * \ np.array([ner(t) * postag(t) for t in tt]) - wts = [math.pow(s, 2) for s in wts] + wts = [s for s in wts] tw.extend(zip(tt, wts)) S = np.sum([s for _, s in tw]) diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py index 9829e68cb05..d7be6fa818c 100644 --- a/rag/svr/task_executor.py +++ b/rag/svr/task_executor.py @@ -114,6 +114,7 @@ def set_progress(task_id, from_page=0, to_page=-1, prog=None, msg="Processing... if prog is not None: d["progress"] = prog try: + logging.info(f"set_progress({task_id}), progress: {prog}, progress_msg: {msg}") TaskService.update_progress(task_id, d) except Exception: logging.exception(f"set_progress({task_id}) got exception") diff --git a/rag/utils/es_conn.py b/rag/utils/es_conn.py index 2e2d7f4ad81..e964a60f714 100644 --- a/rag/utils/es_conn.py +++ b/rag/utils/es_conn.py @@ -148,9 +148,9 @@ def search(self, selectFields: list[str], highlightFields: list[str], condition: vector_similarity_weight = float(weights.split(",")[1]) for m in matchExprs: if isinstance(m, MatchTextExpr): - minimum_should_match = "0%" - if "minimum_should_match" in m.extra_options: - minimum_should_match = str(int(m.extra_options["minimum_should_match"] * 100)) + "%" + minimum_should_match = m.extra_options.get("minimum_should_match", 0.0) + if isinstance(minimum_should_match, float): + minimum_should_match = str(int(minimum_should_match * 100)) + "%" bqry.must.append(Q("query_string", fields=m.fields, type="best_fields", query=m.matching_text, minimum_should_match=minimum_should_match, diff --git a/rag/utils/infinity_conn.py b/rag/utils/infinity_conn.py index 08fb7c3c312..1c0ec8fb29e 100644 --- a/rag/utils/infinity_conn.py +++ b/rag/utils/infinity_conn.py @@ -231,15 +231,10 @@ def search( if len(filter_cond) != 0: filter_fulltext = f"({filter_cond}) AND {filter_fulltext}" logging.debug(f"filter_fulltext: {filter_fulltext}") - minimum_should_match = "0%" - if "minimum_should_match" in matchExpr.extra_options: - minimum_should_match = ( - str(int(matchExpr.extra_options["minimum_should_match"] * 100)) - + "%" - ) - matchExpr.extra_options.update( - {"minimum_should_match": minimum_should_match} - ) + minimum_should_match = matchExpr.extra_options.get("minimum_should_match", 0.0) + if isinstance(minimum_should_match, float): + str_minimum_should_match = str(int(minimum_should_match * 100)) + "%" + matchExpr.extra_options["minimum_should_match"] = str_minimum_should_match for k, v in matchExpr.extra_options.items(): if not isinstance(v, str): matchExpr.extra_options[k] = str(v) diff --git a/web/src/constants/knowledge.ts b/web/src/constants/knowledge.ts index fe82e5e597c..2bcf19a1980 100644 --- a/web/src/constants/knowledge.ts +++ b/web/src/constants/knowledge.ts @@ -4,6 +4,8 @@ export enum KnowledgeRouteKey { Configuration = 'configuration', } +export const DatasetBaseKey = 'dataset'; + export enum RunningStatus { UNSTART = '0', // need to run RUNNING = '1', // need to cancel diff --git a/web/src/layouts/next-header.tsx b/web/src/layouts/next-header.tsx new file mode 100644 index 00000000000..fd4d35a5dbe --- /dev/null +++ b/web/src/layouts/next-header.tsx @@ -0,0 +1,113 @@ +import { Avatar, AvatarFallback, AvatarImage } from '@/components/ui/avatar'; +import { Button } from '@/components/ui/button'; +import { Container } from '@/components/ui/container'; +import { Segmented, SegmentedValue } from '@/components/ui/segmented '; +import { useTranslate } from '@/hooks/common-hooks'; +import { useNavigateWithFromState } from '@/hooks/route-hook'; +import { + ChevronDown, + Cpu, + Github, + Library, + MessageSquareText, + Search, + Star, + Zap, +} from 'lucide-react'; +import { useCallback, useMemo, useState } from 'react'; +import { useLocation } from 'umi'; + +export function Header() { + const { t } = useTranslate('header'); + const { pathname } = useLocation(); + const navigate = useNavigateWithFromState(); + const [currentPath, setCurrentPath] = useState('/home'); + + const tagsData = useMemo( + () => [ + { path: '/home', name: t('knowledgeBase'), icon: Library }, + { path: '/chat', name: t('chat'), icon: MessageSquareText }, + { path: '/search', name: t('search'), icon: Search }, + { path: '/flow', name: t('flow'), icon: Cpu }, + // { path: '/file', name: t('fileManager'), icon: FileIcon }, + ], + [t], + ); + + const options = useMemo(() => { + return tagsData.map((tag) => { + const HeaderIcon = tag.icon; + + return { + label: ( +
+ + {tag.name} +
+ ), + value: tag.path, + }; + }); + }, [tagsData]); + + // const currentPath = useMemo(() => { + // return tagsData.find((x) => pathname.startsWith(x.path))?.name || 'home'; + // }, [pathname, tagsData]); + + const handleChange = (path: SegmentedValue) => { + // navigate(path as string); + setCurrentPath(path as string); + }; + + const handleLogoClick = useCallback(() => { + navigate('/'); + }, [navigate]); + + return ( +
+
+ logo + +
+
+ +
+
+ + V 0.13.0 + + + + + + CN + + yifanwu92@gmail.com + + +
+
+ ); +} diff --git a/web/src/layouts/next.tsx b/web/src/layouts/next.tsx new file mode 100644 index 00000000000..a4fbef17dab --- /dev/null +++ b/web/src/layouts/next.tsx @@ -0,0 +1,11 @@ +import { Outlet } from 'umi'; +import { Header } from './next-header'; + +export default function NextLayout() { + return ( +
+
+ +
+ ); +} diff --git a/web/src/pages/dataset/dataset/index.tsx b/web/src/pages/dataset/dataset/index.tsx new file mode 100644 index 00000000000..914ef81d362 --- /dev/null +++ b/web/src/pages/dataset/dataset/index.tsx @@ -0,0 +1,3 @@ +export default function Dataset() { + return
Outset
; +} diff --git a/web/src/pages/dataset/index.tsx b/web/src/pages/dataset/index.tsx new file mode 100644 index 00000000000..bd4ab0cc98d --- /dev/null +++ b/web/src/pages/dataset/index.tsx @@ -0,0 +1,13 @@ +import { Outlet } from 'umi'; +import { SideBar } from './sidebar'; + +export default function DatasetWrapper() { + return ( +
+ +
+ +
+
+ ); +} diff --git a/web/src/pages/dataset/settings/index.tsx b/web/src/pages/dataset/settings/index.tsx new file mode 100644 index 00000000000..26c27a1ecb8 --- /dev/null +++ b/web/src/pages/dataset/settings/index.tsx @@ -0,0 +1,3 @@ +export default function DatasetSettings() { + return
DatasetSettings
; +} diff --git a/web/src/pages/dataset/sidebar/hooks.tsx b/web/src/pages/dataset/sidebar/hooks.tsx new file mode 100644 index 00000000000..4f90e191645 --- /dev/null +++ b/web/src/pages/dataset/sidebar/hooks.tsx @@ -0,0 +1,16 @@ +import { DatasetBaseKey, KnowledgeRouteKey } from '@/constants/knowledge'; +import { useCallback } from 'react'; +import { useNavigate } from 'umi'; + +export const useHandleMenuClick = () => { + const navigate = useNavigate(); + + const handleMenuClick = useCallback( + (key: KnowledgeRouteKey) => () => { + navigate(`/${DatasetBaseKey}/${key}`); + }, + [navigate], + ); + + return { handleMenuClick }; +}; diff --git a/web/src/pages/dataset/sidebar/index.tsx b/web/src/pages/dataset/sidebar/index.tsx new file mode 100644 index 00000000000..2fb76f97255 --- /dev/null +++ b/web/src/pages/dataset/sidebar/index.tsx @@ -0,0 +1,66 @@ +import { Button } from '@/components/ui/button'; +import { KnowledgeRouteKey } from '@/constants/knowledge'; +import { useSecondPathName } from '@/hooks/route-hook'; +import { cn } from '@/lib/utils'; +import { Banknote, LayoutGrid, User } from 'lucide-react'; +import { useHandleMenuClick } from './hooks'; + +const items = [ + { icon: User, label: 'Dataset', key: KnowledgeRouteKey.Dataset }, + { + icon: LayoutGrid, + label: 'Retrieval testing', + key: KnowledgeRouteKey.Testing, + }, + { icon: Banknote, label: 'Settings', key: KnowledgeRouteKey.Configuration }, +]; + +const dataset = { + id: 1, + title: 'Legal knowledge base', + files: '1,242 files', + size: '152 MB', + created: '12.02.2024', + image: 'https://github.com/shadcn.png', +}; + +export function SideBar() { + const pathName = useSecondPathName(); + const { handleMenuClick } = useHandleMenuClick(); + + return ( + + ); +} diff --git a/web/src/pages/dataset/testing/index.tsx b/web/src/pages/dataset/testing/index.tsx new file mode 100644 index 00000000000..9d89a4073fe --- /dev/null +++ b/web/src/pages/dataset/testing/index.tsx @@ -0,0 +1,3 @@ +export default function RetrievalTesting() { + return
Retrieval testing
; +} diff --git a/web/src/routes.ts b/web/src/routes.ts index b7d68168bb9..ead85a006d4 100644 --- a/web/src/routes.ts +++ b/web/src/routes.ts @@ -134,7 +134,39 @@ const routes = [ { path: '/datasets', layout: false, - component: '@/pages/datasets', + component: '@/layouts/next', + routes: [ + { + path: '/datasets', + component: '@/pages/datasets', + }, + ], + }, + { + path: '/dataset', + layout: false, + component: '@/layouts/next', + routes: [ + { path: '/dataset', redirect: '/dataset/dataset' }, + { + path: '/dataset', + component: '@/pages/dataset', + routes: [ + { + path: '/dataset/dataset', + component: '@/pages/dataset/dataset', + }, + { + path: '/dataset/configuration', + component: '@/pages/dataset/settings', + }, + { + path: '/dataset/testing', + component: '@/pages/dataset/testing', + }, + ], + }, + ], }, { path: '/profile-setting',