From 4b3efcbb69801917b7d79b418ea1823a28fa19ca Mon Sep 17 00:00:00 2001 From: Zhedong Cen Date: Fri, 2 Aug 2024 14:36:28 +0800 Subject: [PATCH 1/4] add supprot for OpenAI-API-Compatible llm --- api/apps/llm_app.py | 9 ++++++--- conf/llm_factories.json | 7 +++++++ rag/llm/__init__.py | 12 ++++++++---- rag/llm/chat_model.py | 14 ++++++++++++-- rag/llm/cv_model.py | 11 +++++++++++ rag/llm/embedding_model.py | 10 ++++++++++ rag/llm/rerank_model.py | 8 ++++++++ web/src/assets/svg/llm/openai-api.svg | 6 ++++++ web/src/interfaces/request/llm.ts | 1 + web/src/pages/user-setting/constants.tsx | 2 +- .../pages/user-setting/setting-model/constant.ts | 3 ++- .../setting-model/ollama-modal/index.tsx | 7 +++++++ 12 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 web/src/assets/svg/llm/openai-api.svg diff --git a/api/apps/llm_app.py b/api/apps/llm_app.py index d622b24b453..3d8edf8d1ad 100644 --- a/api/apps/llm_app.py +++ b/api/apps/llm_app.py @@ -129,6 +129,9 @@ def add_llm(): elif factory == "LocalAI": llm_name = req["llm_name"]+"___LocalAI" api_key = "xxxxxxxxxxxxxxx" + elif factory == "OpenAI-API": + llm_name = req["llm_name"]+"___OpenAI-API" + api_key = req["api_key"] else: llm_name = req["llm_name"] api_key = "xxxxxxxxxxxxxxx" @@ -145,7 +148,7 @@ def add_llm(): msg = "" if llm["model_type"] == LLMType.EMBEDDING.value: mdl = EmbeddingModel[factory]( - key=llm['api_key'] if factory in ["VolcEngine", "Bedrock"] else None, + key=llm['api_key'] if factory in ["VolcEngine", "Bedrock","OpenAI-API"] else None, model_name=llm["llm_name"], base_url=llm["api_base"]) try: @@ -156,7 +159,7 @@ def add_llm(): msg += f"\nFail to access embedding model({llm['llm_name']})." + str(e) elif llm["model_type"] == LLMType.CHAT.value: mdl = ChatModel[factory]( - key=llm['api_key'] if factory in ["VolcEngine", "Bedrock"] else None, + key=llm['api_key'] if factory in ["VolcEngine", "Bedrock","OpenAI-API"] else None, model_name=llm["llm_name"], base_url=llm["api_base"] ) @@ -181,7 +184,7 @@ def add_llm(): e) elif llm["model_type"] == LLMType.IMAGE2TEXT.value: mdl = CvModel[factory]( - key=None, model_name=llm["llm_name"], base_url=llm["api_base"] + key=llm["api_key"] if factory in ["OpenAI-API"] else None, model_name=llm["llm_name"], base_url=llm["api_base"] ) try: img_url = ( diff --git a/conf/llm_factories.json b/conf/llm_factories.json index 586b81f065b..d050263e733 100644 --- a/conf/llm_factories.json +++ b/conf/llm_factories.json @@ -164,6 +164,13 @@ "status": "1", "llm": [] }, + { + "name": "OpenAI-API", + "logo": "", + "tags": "LLM,TEXT EMBEDDING,SPEECH2TEXT,MODERATION", + "status": "1", + "llm": [] + }, { "name": "Moonshot", "logo": "", diff --git a/rag/llm/__init__.py b/rag/llm/__init__.py index 3ebb230e278..465c3a3717f 100644 --- a/rag/llm/__init__.py +++ b/rag/llm/__init__.py @@ -36,7 +36,8 @@ "Bedrock": BedrockEmbed, "Gemini": GeminiEmbed, "NVIDIA": NvidiaEmbed, - "LM-Studio": LmStudioEmbed + "LM-Studio": LmStudioEmbed, + "OpenAI-API": OpenAI_APIEmbed } @@ -53,7 +54,8 @@ "LocalAI": LocalAICV, "NVIDIA": NvidiaCV, "LM-Studio": LmStudioCV, - "StepFun":StepFunCV + "StepFun":StepFunCV, + "OpenAI-API": OpenAI_APICV } @@ -78,7 +80,8 @@ "OpenRouter": OpenRouterChat, "StepFun": StepFunChat, "NVIDIA": NvidiaChat, - "LM-Studio": LmStudioChat + "LM-Studio": LmStudioChat, + "OpenAI-API": OpenAI_APIChat } @@ -88,7 +91,8 @@ "Youdao": YoudaoRerank, "Xinference": XInferenceRerank, "NVIDIA": NvidiaRerank, - "LM-Studio": LmStudioRerank + "LM-Studio": LmStudioRerank, + "OpenAI_API": OpenAI_APIRerank } diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.py index 303be41c681..40dbff11178 100644 --- a/rag/llm/chat_model.py +++ b/rag/llm/chat_model.py @@ -887,6 +887,16 @@ def __init__(self, key, model_name, base_url): if not base_url: raise ValueError("Local llm url cannot be None") if base_url.split("/")[-1] != "v1": - self.base_url = os.path.join(base_url, "v1") - self.client = OpenAI(api_key="lm-studio", base_url=self.base_url) + base_url = os.path.join(base_url, "v1") + self.client = OpenAI(api_key="lm-studio", base_url=base_url) self.model_name = model_name + + +class OpenAI_APIChat(Base): + def __init__(self, key, model_name, base_url): + if not base_url: + raise ValueError("url cannot be None") + if base_url.split("/")[-1] != "v1": + base_url = os.path.join(base_url, "v1") + model_name = model_name.split("___")[0] + super().__init__(key, model_name, base_url) diff --git a/rag/llm/cv_model.py b/rag/llm/cv_model.py index 1277791ee59..82a7730240d 100644 --- a/rag/llm/cv_model.py +++ b/rag/llm/cv_model.py @@ -638,3 +638,14 @@ def __init__(self, key, model_name, base_url, lang="Chinese"): self.client = OpenAI(api_key="lm-studio", base_url=base_url) self.model_name = model_name self.lang = lang + + +class OpenAI_APICV(GptV4): + def __init__(self, key, model_name, base_url, lang="Chinese"): + if not base_url: + raise ValueError("url cannot be None") + if base_url.split("/")[-1] != "v1": + base_url = os.path.join(base_url, "v1") + self.client = OpenAI(api_key=key, base_url=base_url) + self.model_name = model_name.split("___")[0] + self.lang = lang diff --git a/rag/llm/embedding_model.py b/rag/llm/embedding_model.py index d9eb484c8eb..3c3a018d43a 100644 --- a/rag/llm/embedding_model.py +++ b/rag/llm/embedding_model.py @@ -513,3 +513,13 @@ def __init__(self, key, model_name, base_url): self.base_url = os.path.join(base_url, "v1") self.client = OpenAI(api_key="lm-studio", base_url=self.base_url) self.model_name = model_name + + +class OpenAI_APIEmbed(OpenAIEmbed): + def __init__(self, key, model_name, base_url): + if not base_url: + raise ValueError("url cannot be None") + if base_url.split("/")[-1] != "v1": + self.base_url = os.path.join(base_url, "v1") + self.client = OpenAI(api_key=key, base_url=base_url) + self.model_name = model_name.split("___")[0] \ No newline at end of file diff --git a/rag/llm/rerank_model.py b/rag/llm/rerank_model.py index b6d5421ee70..4d04f949841 100644 --- a/rag/llm/rerank_model.py +++ b/rag/llm/rerank_model.py @@ -212,3 +212,11 @@ def __init__(self, key, model_name, base_url): def similarity(self, query: str, texts: list): raise NotImplementedError("The LmStudioRerank has not been implement") + + +class OpenAI_APIRerank(Base): + def __init__(self, key, model_name, base_url): + pass + + def similarity(self, query: str, texts: list): + raise NotImplementedError("The LmStudioRerank has not been implement") diff --git a/web/src/assets/svg/llm/openai-api.svg b/web/src/assets/svg/llm/openai-api.svg new file mode 100644 index 00000000000..6114c7c7ed6 --- /dev/null +++ b/web/src/assets/svg/llm/openai-api.svg @@ -0,0 +1,6 @@ + + + \ No newline at end of file diff --git a/web/src/interfaces/request/llm.ts b/web/src/interfaces/request/llm.ts index 727af014981..2db66484e35 100644 --- a/web/src/interfaces/request/llm.ts +++ b/web/src/interfaces/request/llm.ts @@ -3,6 +3,7 @@ export interface IAddLlmRequestBody { llm_name: string; model_type: string; api_base?: string; // chat|embedding|speech2text|image2text + api_key: string; } export interface IDeleteLlmRequestBody { diff --git a/web/src/pages/user-setting/constants.tsx b/web/src/pages/user-setting/constants.tsx index 97df5415e08..8b1755cf919 100644 --- a/web/src/pages/user-setting/constants.tsx +++ b/web/src/pages/user-setting/constants.tsx @@ -17,4 +17,4 @@ export const UserSettingIconMap = { export * from '@/constants/setting'; -export const LocalLlmFactories = ['Ollama', 'Xinference','LocalAI','LM-Studio']; +export const LocalLlmFactories = ['Ollama', 'Xinference','LocalAI','LM-Studio',"OpenAI-API"]; diff --git a/web/src/pages/user-setting/setting-model/constant.ts b/web/src/pages/user-setting/setting-model/constant.ts index 9493835d85c..50faa1d9bf9 100644 --- a/web/src/pages/user-setting/setting-model/constant.ts +++ b/web/src/pages/user-setting/setting-model/constant.ts @@ -21,7 +21,8 @@ export const IconMap = { LocalAI: 'local-ai', StepFun: 'stepfun', NVIDIA:'nvidia', - 'LM-Studio':'lm-studio' + 'LM-Studio':'lm-studio', + 'OpenAI-API':'openai-api' }; export const BedrockRegionList = [ diff --git a/web/src/pages/user-setting/setting-model/ollama-modal/index.tsx b/web/src/pages/user-setting/setting-model/ollama-modal/index.tsx index ffa71241e7e..d102d65723f 100644 --- a/web/src/pages/user-setting/setting-model/ollama-modal/index.tsx +++ b/web/src/pages/user-setting/setting-model/ollama-modal/index.tsx @@ -92,6 +92,13 @@ const OllamaModal = ({ > + + label={t('apiKey')} + name="api_key" + rules={[{ required: false, message: t('apiKeyMessage') }]} + > + + {({ getFieldValue }) => getFieldValue('model_type') === 'chat' && ( From ab0c296c37400e5fd1f4ffc111fc4bf440158993 Mon Sep 17 00:00:00 2001 From: Zhedong Cen Date: Fri, 2 Aug 2024 14:39:53 +0800 Subject: [PATCH 2/4] update --- rag/llm/rerank_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rag/llm/rerank_model.py b/rag/llm/rerank_model.py index 4d04f949841..f5e89437f13 100644 --- a/rag/llm/rerank_model.py +++ b/rag/llm/rerank_model.py @@ -219,4 +219,4 @@ def __init__(self, key, model_name, base_url): pass def similarity(self, query: str, texts: list): - raise NotImplementedError("The LmStudioRerank has not been implement") + raise NotImplementedError("The api has not been implement") From e678f9494eb48331bf2abf9c59d9d3d26f6da296 Mon Sep 17 00:00:00 2001 From: Zhedong Cen Date: Fri, 2 Aug 2024 14:54:12 +0800 Subject: [PATCH 3/4] update --- conf/llm_factories.json | 2 +- web/src/assets/svg/llm/openai-api.svg | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/conf/llm_factories.json b/conf/llm_factories.json index d050263e733..1fc35e34046 100644 --- a/conf/llm_factories.json +++ b/conf/llm_factories.json @@ -165,7 +165,7 @@ "llm": [] }, { - "name": "OpenAI-API", + "name": "OpenAI-API-Compatible", "logo": "", "tags": "LLM,TEXT EMBEDDING,SPEECH2TEXT,MODERATION", "status": "1", diff --git a/web/src/assets/svg/llm/openai-api.svg b/web/src/assets/svg/llm/openai-api.svg index 6114c7c7ed6..a0ecf992f46 100644 --- a/web/src/assets/svg/llm/openai-api.svg +++ b/web/src/assets/svg/llm/openai-api.svg @@ -1,6 +1 @@ - - - \ No newline at end of file + \ No newline at end of file From 2102a769389542e131c934f51cccf7655a5b528e Mon Sep 17 00:00:00 2001 From: Zhedong Cen Date: Fri, 2 Aug 2024 14:59:06 +0800 Subject: [PATCH 4/4] update --- api/apps/llm_app.py | 8 ++++---- rag/llm/__init__.py | 8 ++++---- web/src/pages/user-setting/constants.tsx | 2 +- web/src/pages/user-setting/setting-model/constant.ts | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/api/apps/llm_app.py b/api/apps/llm_app.py index 3d8edf8d1ad..c4657c44cf5 100644 --- a/api/apps/llm_app.py +++ b/api/apps/llm_app.py @@ -129,7 +129,7 @@ def add_llm(): elif factory == "LocalAI": llm_name = req["llm_name"]+"___LocalAI" api_key = "xxxxxxxxxxxxxxx" - elif factory == "OpenAI-API": + elif factory == "OpenAI-API-Compatible": llm_name = req["llm_name"]+"___OpenAI-API" api_key = req["api_key"] else: @@ -148,7 +148,7 @@ def add_llm(): msg = "" if llm["model_type"] == LLMType.EMBEDDING.value: mdl = EmbeddingModel[factory]( - key=llm['api_key'] if factory in ["VolcEngine", "Bedrock","OpenAI-API"] else None, + key=llm['api_key'] if factory in ["VolcEngine", "Bedrock","OpenAI-API-Compatible"] else None, model_name=llm["llm_name"], base_url=llm["api_base"]) try: @@ -159,7 +159,7 @@ def add_llm(): msg += f"\nFail to access embedding model({llm['llm_name']})." + str(e) elif llm["model_type"] == LLMType.CHAT.value: mdl = ChatModel[factory]( - key=llm['api_key'] if factory in ["VolcEngine", "Bedrock","OpenAI-API"] else None, + key=llm['api_key'] if factory in ["VolcEngine", "Bedrock","OpenAI-API-Compatible"] else None, model_name=llm["llm_name"], base_url=llm["api_base"] ) @@ -184,7 +184,7 @@ def add_llm(): e) elif llm["model_type"] == LLMType.IMAGE2TEXT.value: mdl = CvModel[factory]( - key=llm["api_key"] if factory in ["OpenAI-API"] else None, model_name=llm["llm_name"], base_url=llm["api_base"] + key=llm["api_key"] if factory in ["OpenAI-API-Compatible"] else None, model_name=llm["llm_name"], base_url=llm["api_base"] ) try: img_url = ( diff --git a/rag/llm/__init__.py b/rag/llm/__init__.py index 465c3a3717f..4c3182cae95 100644 --- a/rag/llm/__init__.py +++ b/rag/llm/__init__.py @@ -37,7 +37,7 @@ "Gemini": GeminiEmbed, "NVIDIA": NvidiaEmbed, "LM-Studio": LmStudioEmbed, - "OpenAI-API": OpenAI_APIEmbed + "OpenAI-API-Compatible": OpenAI_APIEmbed } @@ -55,7 +55,7 @@ "NVIDIA": NvidiaCV, "LM-Studio": LmStudioCV, "StepFun":StepFunCV, - "OpenAI-API": OpenAI_APICV + "OpenAI-API-Compatible": OpenAI_APICV } @@ -81,7 +81,7 @@ "StepFun": StepFunChat, "NVIDIA": NvidiaChat, "LM-Studio": LmStudioChat, - "OpenAI-API": OpenAI_APIChat + "OpenAI-API-Compatible": OpenAI_APIChat } @@ -92,7 +92,7 @@ "Xinference": XInferenceRerank, "NVIDIA": NvidiaRerank, "LM-Studio": LmStudioRerank, - "OpenAI_API": OpenAI_APIRerank + "OpenAI-API-Compatible": OpenAI_APIRerank } diff --git a/web/src/pages/user-setting/constants.tsx b/web/src/pages/user-setting/constants.tsx index 8b1755cf919..cf2bce9ed6b 100644 --- a/web/src/pages/user-setting/constants.tsx +++ b/web/src/pages/user-setting/constants.tsx @@ -17,4 +17,4 @@ export const UserSettingIconMap = { export * from '@/constants/setting'; -export const LocalLlmFactories = ['Ollama', 'Xinference','LocalAI','LM-Studio',"OpenAI-API"]; +export const LocalLlmFactories = ['Ollama', 'Xinference','LocalAI','LM-Studio',"OpenAI-API-Compatible"]; diff --git a/web/src/pages/user-setting/setting-model/constant.ts b/web/src/pages/user-setting/setting-model/constant.ts index 50faa1d9bf9..865eb29bf34 100644 --- a/web/src/pages/user-setting/setting-model/constant.ts +++ b/web/src/pages/user-setting/setting-model/constant.ts @@ -22,7 +22,7 @@ export const IconMap = { StepFun: 'stepfun', NVIDIA:'nvidia', 'LM-Studio':'lm-studio', - 'OpenAI-API':'openai-api' + 'OpenAI-API-Compatible':'openai-api' }; export const BedrockRegionList = [