From 69a1b56e8950f9fccc98c79626f1ca16848f8079 Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Mon, 15 Jul 2024 17:38:41 +0800
Subject: [PATCH 01/16] Update wikipedia.py
---
graph/component/wikipedia.py | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/graph/component/wikipedia.py b/graph/component/wikipedia.py
index 9e67875961e..48abefaf6e7 100644
--- a/graph/component/wikipedia.py
+++ b/graph/component/wikipedia.py
@@ -30,9 +30,16 @@ class WikipediaParam(ComponentParamBase):
def __init__(self):
super().__init__()
self.top_n = 10
+ self.lang = 'en'
def check(self):
self.check_positive_integer(self.top_n, "Top N")
+ self.check_valid_value(self.lang, "Wikipedia languages",
+ ['af', 'pl', 'ar', 'ast', 'az', 'bg', 'nan', 'bn', 'be', 'ca', 'cs', 'cy', 'da', 'de',
+ 'et', 'el', 'en', 'es', 'eo', 'eu', 'fa', 'fr', 'gl', 'ko', 'hy', 'hi', 'hr', 'id',
+ 'it', 'he', 'ka', 'lld', 'la', 'lv', 'lt', 'hu', 'mk', 'arz', 'ms', 'min', 'my', 'nl',
+ 'ja', 'nb', 'nn', 'ce', 'uz', 'pt', 'kk', 'ro', 'ru', 'ceb', 'sk', 'sl', 'sr', 'sh',
+ 'fi', 'sv', 'ta', 'tt', 'th', 'tg', 'azb', 'tr', 'uk', 'ur', 'vi', 'war', 'zh', 'yue'])
class Wikipedia(ComponentBase, ABC):
@@ -45,9 +52,11 @@ def _run(self, history, **kwargs):
return Wikipedia.be_output(self._param.no)
wiki_res = []
- for wiki_key in wikipedia.search(ans, results=self._param.top_n):
+ wikipedia.set_lang(self._param.lang)
+ wiki_engine = wikipedia
+ for wiki_key in wiki_engine.search(ans, results=self._param.top_n):
try:
- page = wikipedia.page(title=wiki_key, auto_suggest=False)
+ page = wiki_engine.page(title=wiki_key, auto_suggest=False)
wiki_res.append({"content": '' + page.title + ' ' + page.summary})
except Exception as e:
print(e)
From 268c0c7eb7fb7df47600341af51ca37a3241411e Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Wed, 17 Jul 2024 15:37:28 +0800
Subject: [PATCH 02/16] Update requirements.txt
---
requirements.txt | 176 ++++++++++++++---------------------------------
1 file changed, 51 insertions(+), 125 deletions(-)
diff --git a/requirements.txt b/requirements.txt
index 43eddf5b18f..9d02e96486e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,153 +1,79 @@
-accelerate==0.27.2
-aiohttp==3.9.5
-aiosignal==1.3.1
-annotated-types==0.6.0
-anyio==4.3.0
-argon2-cffi==23.1.0
-argon2-cffi-bindings==21.2.0
Aspose.Slides==24.2.0
-attrs==23.2.0
-blinker==1.7.0
-cachelib==0.12.0
+BCEmbedding==0.1.3
+boto3==1.34.140
+botocore==1.34.140
cachetools==5.3.3
-certifi==2024.2.2
-cffi==1.16.0
-charset-normalizer==3.3.2
-click==8.1.7
-coloredlogs==15.0.1
-cryptography==42.0.5
+chardet==5.2.0
+cn2an==0.5.22
dashscope==1.14.1
-datasets==2.17.1
datrie==0.8.2
demjson3==3.0.6
-dill==0.3.8
-distro==1.9.0
-elastic-transport==8.12.0
+discord.py==2.3.2
+duckduckgo_search==6.1.9
+elastic_transport==8.12.0
elasticsearch==8.12.1
-elasticsearch-dsl==8.12.0
-et-xmlfile==1.1.0
-filelock==3.13.1
+elasticsearch_dsl==8.12.0
fastembed==0.2.6
-FlagEmbedding==1.2.5
-Flask==3.0.2
-Flask-Cors==4.0.0
-Flask-Login==0.6.3
-Flask-Session==0.6.0
-flatbuffers==23.5.26
-frozenlist==1.4.1
-fsspec==2023.10.0
-h11==0.14.0
+fasttext==0.9.3
+filelock==3.15.4
+FlagEmbedding==1.2.10
+Flask==3.0.3
+Flask_Cors==4.0.0
+Flask_Login==0.6.3
+flask_session==0.8.0
+groq==0.9.0
hanziconv==0.3.2
-httpcore==1.0.4
+html_text==0.6.2
httpx==0.27.0
-huggingface-hub==0.20.3
-humanfriendly==10.0
-idna==3.6
-install==1.3.5
+huggingface_hub==0.20.3
+infinity_emb==0.0.51
itsdangerous==2.1.2
-Jinja2==3.1.3
-joblib==1.3.2
-lxml==5.1.0
-MarkupSafe==2.1.5
+Markdown==3.6
minio==7.2.4
-mpmath==1.3.0
-multidict==6.0.5
-multiprocess==0.70.16
-networkx==3.2.1
+mistralai==0.4.2
nltk==3.8.1
numpy==1.26.4
-nvidia-cublas-cu12==12.1.3.1
-nvidia-cuda-cupti-cu12==12.1.105
-nvidia-cuda-nvrtc-cu12==12.1.105
-nvidia-cuda-runtime-cu12==12.1.105
-nvidia-cudnn-cu12==8.9.2.26
-nvidia-cufft-cu12==11.0.2.54
-nvidia-curand-cu12==10.3.2.106
-nvidia-cusolver-cu12==11.4.5.107
-nvidia-cusparse-cu12==12.1.0.106
-nvidia-nccl-cu12==2.19.3
-nvidia-nvjitlink-cu12==12.3.101
-nvidia-nvtx-cu12==12.1.105
-ollama==0.1.9
-onnxruntime-gpu==1.17.1
-openai==1.12.0
-opencv-python==4.9.0.80
+ollama==0.2.1
+onnxruntime==1.17.3
+onnxruntime_gpu==1.17.1
+openai==1.35.14
+opencv_python==4.9.0.80
+opencv_python_headless==4.9.0.80
openpyxl==3.1.2
-packaging==23.2
-pandas==2.2.1
-pdfminer.six==20221105
+pandas==2.2.2
pdfplumber==0.10.4
peewee==3.17.1
-pillow==10.3.0
-protobuf==4.25.3
-psutil==5.9.8
-pyarrow==15.0.0
-pyarrow-hotfix==0.6
+Pillow==10.4.0
+pipreqs==0.5.0
+protobuf==5.27.2
pyclipper==1.3.0.post5
-pycparser==2.21
-pycryptodome
-pycryptodome-test-vectors
-pycryptodomex
-pydantic==2.6.2
-pydantic_core==2.16.3
-PyJWT==2.8.0
-PyMySQL==1.1.1
+pycryptodomex==3.20.0
PyPDF2==3.0.1
-pypdfium2==4.27.0
-python-dateutil==2.8.2
-python-docx==1.1.0
+pytest==8.2.2
python-dotenv==1.0.1
-python-pptx==0.6.23
-PyYAML==6.0.1
+python_dateutil==2.8.2
+python_pptx==0.6.23
+readability_lxml==0.8.1
redis==5.0.3
-regex==2023.12.25
-requests==2.31.0
-ruamel.yaml==0.18.6
-ruamel.yaml.clib==0.2.8
-safetensors==0.4.2
-scikit-learn==1.4.1.post1
-scipy==1.12.0
-sentence-transformers==2.4.0
-shapely==2.0.3
+Requests==2.32.3
+roman_numbers==1.0.2
+ruamel.base==1.0.0
+scikit_learn==1.4.1.post1
+selenium==4.22.0
+setuptools==69.5.1
+Shapely==2.0.5
six==1.16.0
-sniffio==1.3.1
StrEnum==0.4.15
-sympy==1.12
-threadpoolctl==3.3.0
tika==2.6.0
tiktoken==0.6.0
-tokenizers==0.15.2
-torch==2.2.1
-tqdm==4.66.2
+torch==2.3.0
transformers==4.38.1
-triton==2.2.0
-typing_extensions==4.10.0
-tzdata==2024.1
-urllib3==2.2.1
+umap==0.1.1
+volcengine==1.0.146
+webdriver_manager==4.0.1
Werkzeug==3.0.3
-xgboost==2.0.3
-XlsxWriter==3.2.0
+wikipedia==1.4.0
+word2number==1.1
+xgboost==2.1.0
xpinyin==0.7.6
-xxhash==3.4.1
-yarl==1.9.4
zhipuai==2.0.1
-BCEmbedding
-loguru==0.7.2
-umap-learn
-fasttext==0.9.2
-pybind11==2.13.1
-volcengine==1.0.141
-readability-lxml==0.8.1
-html_text==0.6.2
-selenium==4.21.0
-webdriver-manager==4.0.1
-cn2an==0.5.22
-roman-numbers==1.0.2
-word2number==1.1
-markdown==3.6
-mistralai==0.4.2
-boto3==1.34.140
-duckduckgo_search==6.1.9
-google-generativeai==0.7.2
-groq==0.9.0
-wikipedia==1.4.0
From 92cf45134adabff305b38d2f7fea9d768a030995 Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Fri, 19 Jul 2024 18:23:24 +0800
Subject: [PATCH 03/16] Update llm_service.py
---
api/db/services/llm_service.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/api/db/services/llm_service.py b/api/db/services/llm_service.py
index 5906927b7bc..a994d6103eb 100644
--- a/api/db/services/llm_service.py
+++ b/api/db/services/llm_service.py
@@ -70,7 +70,7 @@ def model_instance(cls, tenant_id, llm_type,
elif llm_type == LLMType.SPEECH2TEXT.value:
mdlnm = tenant.asr_id
elif llm_type == LLMType.IMAGE2TEXT.value:
- mdlnm = tenant.img2txt_id
+ mdlnm = tenant.img2txt_id if not llm_name else llm_name
elif llm_type == LLMType.CHAT.value:
mdlnm = tenant.llm_id if not llm_name else llm_name
elif llm_type == LLMType.RERANK:
From ad5a5b8bed32a47c073f0484feb235bcff6e94cd Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Fri, 19 Jul 2024 18:24:37 +0800
Subject: [PATCH 04/16] Update index.tsx
---
web/src/components/llm-setting-items/index.tsx | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/web/src/components/llm-setting-items/index.tsx b/web/src/components/llm-setting-items/index.tsx
index a8365d616e8..c31d7aabeea 100644
--- a/web/src/components/llm-setting-items/index.tsx
+++ b/web/src/components/llm-setting-items/index.tsx
@@ -46,7 +46,7 @@ const LlmSettingItems = ({ prefix, formItemLayout = {} }: IProps) => {
{...formItemLayout}
rules={[{ required: true, message: t('modelMessage') }]}
>
-
+
Date: Mon, 22 Jul 2024 18:27:56 +0800
Subject: [PATCH 05/16] Update init_data.py
---
api/db/init_data.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/api/db/init_data.py b/api/db/init_data.py
index 0acfe7b12a5..61fd0e6de01 100644
--- a/api/db/init_data.py
+++ b/api/db/init_data.py
@@ -121,6 +121,8 @@ def init_llm_factory():
LLMFactoriesService.filter_delete([LLMFactoriesService.model.name == "QAnything"])
LLMService.filter_delete([LLMService.model.fid == "QAnything"])
TenantLLMService.filter_update([TenantLLMService.model.llm_factory == "QAnything"], {"llm_factory": "Youdao"})
+ TenantService.filter_update([1 == 1], {
+ "parser_ids": "naive:General,qa:Q&A,resume:Resume,manual:Manual,table:Table,paper:Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio"})
## insert openai two embedding models to the current openai user.
print("Start to insert 2 OpenAI embedding models...")
tenant_ids = set([row["tenant_id"] for row in TenantLLMService.get_openai_models()])
From 08108cb51e0badd560847471612902c01a8bc29d Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:30:48 +0800
Subject: [PATCH 06/16] Add files via upload
---
agent/component/baidufanyi.py | 117 ++++++++++++++++++++++++++++++++++
agent/component/deepl.py | 75 ++++++++++++++++++++++
agent/component/github.py | 64 +++++++++++++++++++
3 files changed, 256 insertions(+)
create mode 100644 agent/component/baidufanyi.py
create mode 100644 agent/component/deepl.py
create mode 100644 agent/component/github.py
diff --git a/agent/component/baidufanyi.py b/agent/component/baidufanyi.py
new file mode 100644
index 00000000000..0a08e671851
--- /dev/null
+++ b/agent/component/baidufanyi.py
@@ -0,0 +1,117 @@
+#
+# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import random
+from abc import ABC
+from functools import partial
+import pandas as pd
+import requests
+import re
+from graph.settings import DEBUG
+from graph.component.base import ComponentBase, ComponentParamBase
+from hashlib import md5
+
+
+class BaiduFanyiParam(ComponentParamBase):
+ """
+ Define the BaiduFanyi component parameters.
+ """
+
+ def __init__(self):
+ super().__init__()
+ self.prompt = ""
+ self.appid = "xxx"
+ self.secret_key = "xxx"
+ self.trans_type = 'translate'
+ self.parameters = []
+ self.source_lang = 'auto'
+ self.target_lang = 'auto'
+ self.domain = 'finance'
+
+ def check(self):
+ self.check_positive_integer(self.top_n, "Top N")
+ self.check_empty(self.appid, "BaiduFanyi APPID")
+ self.check_empty(self.secret_key, "BaiduFanyi Secret Key")
+ self.check_valid_value(self.trans_type, "Translate type", ['translate', 'fieldtranslate'])
+ self.check_valid_value(self.trans_type, "Translate domain",
+ ['it', 'finance', 'machinery', 'senimed', 'novel', 'academic', 'aerospace', 'wiki',
+ 'news', 'law', 'contract'])
+ self.check_valid_value(self.source_lang, "Source language",
+ ['auto', 'zh', 'en', 'yue', 'wyw', 'jp', 'kor', 'fra', 'spa', 'th', 'ara', 'ru', 'pt',
+ 'de', 'it', 'el', 'nl', 'pl', 'bul', 'est', 'dan', 'fin', 'cs', 'rom', 'slo', 'swe',
+ 'hu', 'cht', 'vie'])
+ self.check_valid_value(self.target_lang, "Target language",
+ ['auto', 'zh', 'en', 'yue', 'wyw', 'jp', 'kor', 'fra', 'spa', 'th', 'ara', 'ru', 'pt',
+ 'de', 'it', 'el', 'nl', 'pl', 'bul', 'est', 'dan', 'fin', 'cs', 'rom', 'slo', 'swe',
+ 'hu', 'cht', 'vie'])
+ self.check_valid_value(self.domain, "Translate field",
+ ['it', 'finance', 'machinery', 'senimed', 'novel', 'academic', 'aerospace', 'wiki',
+ 'news', 'law', 'contract'])
+
+
+class BaiduFanyi(ComponentBase, ABC):
+ component_name = "BaiduFanyi"
+
+ def _run(self, history, **kwargs):
+ prompt = self._param.prompt
+
+ ans = self.get_input()
+ ans = " - ".join(ans["content"]) if "content" in ans else ""
+ if not ans:
+ return BaiduFanyi.be_output("")
+
+ for para in self._param.parameters:
+ cpn = self._canvas.get_component(para["component_id"])["obj"]
+ _, out = cpn.output(allow_partial=False)
+ if "content" not in out.columns:
+ kwargs[para["key"]] = "Nothing"
+ else:
+ kwargs[para["key"]] = " - " + "\n - ".join(out["content"])
+
+ kwargs["input"] = ans
+ for n, v in kwargs.items():
+ prompt = re.sub(r"\{%s\}" % n, str(v), prompt)
+
+ try:
+ source_lang = self._param.source_lang
+ target_lang = self._param.target_lang
+ appid = self._param.appid
+ salt = random.randint(32768, 65536)
+ secret_key = self._param.secret_key
+
+ if self._param.trans_type == 'translate':
+ sign = md5((appid + prompt + salt + secret_key).encode('utf-8')).hexdigest()
+ url = 'http://api.fanyi.baidu.com/api/trans/vip/translate?' + 'q=' + prompt + '&from=' + source_lang + '&to=' + target_lang + '&appid=' + appid + '&salt=' + salt + '&sign=' + sign
+ headers = {"Content-Type": "application/x-www-form-urlencoded"}
+ response = requests.post(url=url, headers=headers).json()
+
+ if response.get('error_code'):
+ BaiduFanyi.be_output("**Error**:" + response['error_msg'])
+
+ return BaiduFanyi.be_output(response['trans_result'][0]['dst'])
+ elif self._param.trans_type == 'fieldtranslate':
+ domain = self._param.domain
+ sign = md5((appid + prompt + salt + domain + secret_key).encode('utf-8')).hexdigest()
+ url = 'http://api.fanyi.baidu.com/api/trans/vip/fieldtranslate?' + 'q=' + prompt + '&from=' + source_lang + '&to=' + target_lang + '&appid=' + appid + '&salt=' + salt + '&domain=' + domain + '&sign=' + sign
+ headers = {"Content-Type": "application/x-www-form-urlencoded"}
+ response = requests.post(url=url, headers=headers).json()
+
+ if response.get('error_code'):
+ BaiduFanyi.be_output("**Error**:" + response['error_msg'])
+
+ return BaiduFanyi.be_output(response['trans_result'][0]['dst'])
+
+ except Exception as e:
+ BaiduFanyi.be_output("**Error**:" + str(e))
diff --git a/agent/component/deepl.py b/agent/component/deepl.py
new file mode 100644
index 00000000000..09a7ad6f052
--- /dev/null
+++ b/agent/component/deepl.py
@@ -0,0 +1,75 @@
+#
+# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import random
+from abc import ABC
+from functools import partial
+import pandas as pd
+import requests
+import re
+from graph.settings import DEBUG
+from graph.component.base import ComponentBase, ComponentParamBase
+import deepl
+
+class DeepLParam(ComponentParamBase):
+ """
+ Define the DeepL component parameters.
+ """
+
+ def __init__(self):
+ super().__init__()
+ self.prompt = ""
+ self.auth_key = "xxx"
+ self.parameters = []
+ self.source_lang = 'ZH'
+ self.target_lang = 'EN-GB'
+ def check(self):
+ self.check_positive_integer(self.top_n, "Top N")
+ self.check_valid_value(self.source_lang, "Source language", ['AR', 'BG', 'CS', 'DA', 'DE', 'EL', 'EN', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH'])
+ self.check_valid_value(self.target_lang, "Target language",['AR', 'BG', 'CS', 'DA', 'DE', 'EL', 'EN-GB', 'EN-US', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT-BR', 'PT-PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH'])
+
+
+class DeepL(ComponentBase, ABC):
+ component_name = "GitHub"
+
+ def _run(self, history, **kwargs):
+ prompt = self._param.prompt
+
+
+ ans = self.get_input()
+ ans = " - ".join(ans["content"]) if "content" in ans else ""
+ if not ans:
+ return DeepL.be_output("")
+
+
+ for para in self._param.parameters:
+ cpn = self._canvas.get_component(para["component_id"])["obj"]
+ _, out = cpn.output(allow_partial=False)
+ if "content" not in out.columns:
+ kwargs[para["key"]] = "Nothing"
+ else:
+ kwargs[para["key"]] = " - " + "\n - ".join(out["content"])
+
+ kwargs["input"] = ans
+ for n, v in kwargs.items():
+ prompt = re.sub(r"\{%s\}" % n, str(v), prompt)
+
+ try:
+ translator = deepl.Translator(self._param.auth_key)
+ result = translator.translate_text(prompt,source_lang=self._param.source_lang, target_lang=self._param.target_lang)
+
+ return DeepL.be_output(result.text)
+ except Exception as e:
+ DeepL.be_output("**Error**:"+str(e))
diff --git a/agent/component/github.py b/agent/component/github.py
new file mode 100644
index 00000000000..08201d9b411
--- /dev/null
+++ b/agent/component/github.py
@@ -0,0 +1,64 @@
+#
+# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import random
+from abc import ABC
+from functools import partial
+import pandas as pd
+import requests
+import re
+from graph.settings import DEBUG
+from graph.component.base import ComponentBase, ComponentParamBase
+
+
+class GitHubParam(ComponentParamBase):
+ """
+ Define the GitHub component parameters.
+ """
+
+ def __init__(self):
+ super().__init__()
+ self.top_n = 10
+
+ def check(self):
+ self.check_positive_integer(self.top_n, "Top N")
+
+
+class GitHub(ComponentBase, ABC):
+ component_name = "GitHub"
+
+ def _run(self, history, **kwargs):
+ ans = self.get_input()
+ ans = " - ".join(ans["content"]) if "content" in ans else ""
+ if not ans:
+ return GitHub.be_output("")
+
+ try:
+ url = 'https://api.github.com/search/repositories?q=' + ans + '&sort=stars&order=desc&per_page=' + str(
+ self._param.top_n)
+ headers = {"Content-Type": "application/vnd.github+json", "X-GitHub-Api-Version": '2022-11-28'}
+ response = requests.get(url=url, headers=headers).json()
+
+ github_res = [{"content": '' + i["name"] + '' + str(
+ i["description"]) + '\n stars:' + str(i['watchers'])} for i in response['items']]
+ except Exception as e:
+ return GitHub.be_output("**ERROR**: " + str(e))
+
+ if not github_res:
+ return GitHub.be_output("")
+
+ df = pd.DataFrame(github_res)
+ if DEBUG: print(df, ":::::::::::::::::::::::::::::::::")
+ return df
From 97c0b76fb166421aac2c96d37adf1d4a4f87bb5b Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:31:17 +0800
Subject: [PATCH 07/16] Update __init__.py
---
agent/component/__init__.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/agent/component/__init__.py b/agent/component/__init__.py
index 199015e47f5..43fa3eeb803 100644
--- a/agent/component/__init__.py
+++ b/agent/component/__init__.py
@@ -17,7 +17,9 @@
from .google import Google, GoogleParam
from .bing import Bing, BingParam
from .googlescholar import GoogleScholar, GoogleScholarParam
-
+from .deepl import DeepL, DeepLParam
+from .github import GitHub, GitHubParam
+from .baidufanyi import BaiduFanyi, BaiduFanyiParam
def component_class(class_name):
m = importlib.import_module("agent.component")
From bf260119b136a3905d7306c8157f2924e278a08a Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:31:47 +0800
Subject: [PATCH 08/16] Update requirements.txt
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index d49e1c7e8bb..e36ab11c5d6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,6 +10,7 @@ cn2an==0.5.22
cohere==5.6.2
dashscope==1.14.1
datrie==0.8.2
+deepl==1.18.0
demjson3==3.0.6
discord.py==2.3.2
duckduckgo_search==6.1.9
From 6e5e3dacb087fda33d381dd48417ccaa0240bfab Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:32:05 +0800
Subject: [PATCH 09/16] Update requirements_arm.txt
---
requirements_arm.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements_arm.txt b/requirements_arm.txt
index 34b3db21f49..546c5ba2217 100644
--- a/requirements_arm.txt
+++ b/requirements_arm.txt
@@ -160,3 +160,4 @@ jina==3.27.2
editdistance==0.8.1
markdown_to_json==2.1.1
scholarly==1.7.11
+deepl==1.18.0
From f0d5a896681d7f65747e0f14c6e63967ad5024bc Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Wed, 7 Aug 2024 19:32:20 +0800
Subject: [PATCH 10/16] Update requirements_dev.txt
---
requirements_dev.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements_dev.txt b/requirements_dev.txt
index 41143d67087..8006e6693ba 100644
--- a/requirements_dev.txt
+++ b/requirements_dev.txt
@@ -145,3 +145,4 @@ jina==3.27.2
editdistance==0.8.1
markdown_to_json==2.1.1
scholarly==1.7.11
+deepl==1.18.0
From ced5b20fdd64935a2c8e81554b3db8f25120cb44 Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Thu, 8 Aug 2024 09:42:37 +0800
Subject: [PATCH 11/16] Update deepl.py
---
agent/component/deepl.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/agent/component/deepl.py b/agent/component/deepl.py
index 09a7ad6f052..167fcf407a9 100644
--- a/agent/component/deepl.py
+++ b/agent/component/deepl.py
@@ -42,7 +42,7 @@ def check(self):
class DeepL(ComponentBase, ABC):
- component_name = "GitHub"
+ component_name = "DeepL"
def _run(self, history, **kwargs):
prompt = self._param.prompt
From c02a953576f7802bdcf2c183b93b2562f87e0b0a Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Thu, 8 Aug 2024 12:25:02 +0800
Subject: [PATCH 12/16] Update agent/component/deepl.py
Co-authored-by: Kevin Hu
---
agent/component/deepl.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/agent/component/deepl.py b/agent/component/deepl.py
index 167fcf407a9..b0717f51eb0 100644
--- a/agent/component/deepl.py
+++ b/agent/component/deepl.py
@@ -19,7 +19,8 @@
import pandas as pd
import requests
import re
-from graph.settings import DEBUG
+from agent.settings import DEBUG
+
from graph.component.base import ComponentBase, ComponentParamBase
import deepl
From a88eace6cd61e7a43d177c661cff7b66c3ca3d65 Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Thu, 8 Aug 2024 12:25:10 +0800
Subject: [PATCH 13/16] Update agent/component/github.py
Co-authored-by: Kevin Hu
---
agent/component/github.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/agent/component/github.py b/agent/component/github.py
index 08201d9b411..073841cfd51 100644
--- a/agent/component/github.py
+++ b/agent/component/github.py
@@ -20,7 +20,8 @@
import requests
import re
from graph.settings import DEBUG
-from graph.component.base import ComponentBase, ComponentParamBase
+from agent.component.base import ComponentBase, ComponentParamBase
+
class GitHubParam(ComponentParamBase):
From 79caa10c14a2f7c4e04609d7ad60d67fac79f875 Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Thu, 8 Aug 2024 12:29:53 +0800
Subject: [PATCH 14/16] Update baidufanyi.py
---
agent/component/baidufanyi.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/agent/component/baidufanyi.py b/agent/component/baidufanyi.py
index 0a08e671851..f77fa80d6ea 100644
--- a/agent/component/baidufanyi.py
+++ b/agent/component/baidufanyi.py
@@ -15,12 +15,9 @@
#
import random
from abc import ABC
-from functools import partial
-import pandas as pd
import requests
import re
-from graph.settings import DEBUG
-from graph.component.base import ComponentBase, ComponentParamBase
+from agent.component.base import ComponentBase, ComponentParamBase
from hashlib import md5
From 48fd39553218015aae22370c1d6d40a0323bbf20 Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Thu, 8 Aug 2024 12:30:31 +0800
Subject: [PATCH 15/16] Update deepl.py
---
agent/component/deepl.py | 29 +++++++++++++++--------------
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git a/agent/component/deepl.py b/agent/component/deepl.py
index b0717f51eb0..be518db3efc 100644
--- a/agent/component/deepl.py
+++ b/agent/component/deepl.py
@@ -13,17 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import random
from abc import ABC
-from functools import partial
-import pandas as pd
-import requests
import re
-from agent.settings import DEBUG
-
-from graph.component.base import ComponentBase, ComponentParamBase
+from agent.component.base import ComponentBase, ComponentParamBase
import deepl
+
class DeepLParam(ComponentParamBase):
"""
Define the DeepL component parameters.
@@ -36,25 +31,30 @@ def __init__(self):
self.parameters = []
self.source_lang = 'ZH'
self.target_lang = 'EN-GB'
+
def check(self):
self.check_positive_integer(self.top_n, "Top N")
- self.check_valid_value(self.source_lang, "Source language", ['AR', 'BG', 'CS', 'DA', 'DE', 'EL', 'EN', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH'])
- self.check_valid_value(self.target_lang, "Target language",['AR', 'BG', 'CS', 'DA', 'DE', 'EL', 'EN-GB', 'EN-US', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT-BR', 'PT-PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH'])
+ self.check_valid_value(self.source_lang, "Source language",
+ ['AR', 'BG', 'CS', 'DA', 'DE', 'EL', 'EN', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT',
+ 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR',
+ 'UK', 'ZH'])
+ self.check_valid_value(self.target_lang, "Target language",
+ ['AR', 'BG', 'CS', 'DA', 'DE', 'EL', 'EN-GB', 'EN-US', 'ES', 'ET', 'FI', 'FR', 'HU',
+ 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT-BR', 'PT-PT', 'RO', 'RU',
+ 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH'])
class DeepL(ComponentBase, ABC):
- component_name = "DeepL"
+ component_name = "GitHub"
def _run(self, history, **kwargs):
prompt = self._param.prompt
-
ans = self.get_input()
ans = " - ".join(ans["content"]) if "content" in ans else ""
if not ans:
return DeepL.be_output("")
-
for para in self._param.parameters:
cpn = self._canvas.get_component(para["component_id"])["obj"]
_, out = cpn.output(allow_partial=False)
@@ -69,8 +69,9 @@ def _run(self, history, **kwargs):
try:
translator = deepl.Translator(self._param.auth_key)
- result = translator.translate_text(prompt,source_lang=self._param.source_lang, target_lang=self._param.target_lang)
+ result = translator.translate_text(prompt, source_lang=self._param.source_lang,
+ target_lang=self._param.target_lang)
return DeepL.be_output(result.text)
except Exception as e:
- DeepL.be_output("**Error**:"+str(e))
+ DeepL.be_output("**Error**:" + str(e))
From 538191a646eb5a9e732e9644f8105f7645a72fd7 Mon Sep 17 00:00:00 2001
From: H <43509927+guoyuhao2330@users.noreply.github.com>
Date: Thu, 8 Aug 2024 12:31:09 +0800
Subject: [PATCH 16/16] Update github.py
---
agent/component/github.py | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/agent/component/github.py b/agent/component/github.py
index 073841cfd51..98180431e96 100644
--- a/agent/component/github.py
+++ b/agent/component/github.py
@@ -13,17 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import random
from abc import ABC
-from functools import partial
import pandas as pd
import requests
-import re
-from graph.settings import DEBUG
+from agent.settings import DEBUG
from agent.component.base import ComponentBase, ComponentParamBase
-
class GitHubParam(ComponentParamBase):
"""
Define the GitHub component parameters.