interface with qwen

request_llm/bridge_chatglmonnx.py

@@ -1,5 +1,5 @@
 model_name = "ChatGLM-ONNX"
-cmd_to_install = "`pip install request_llm/requirements_chatglm_onnx.txt`"
+cmd_to_install = "`pip install -r request_llm/requirements_chatglm_onnx.txt`"
 
 
 from transformers import AutoModel, AutoTokenizer

request_llm/bridge_internlm.py

@@ -1,5 +1,5 @@
 model_name = "InternLM"
-cmd_to_install = "`pip install request_llm/requirements_chatglm.txt`"
+cmd_to_install = "`pip install -r request_llm/requirements_chatglm.txt`"
 
 from transformers import AutoModel, AutoTokenizer
 import time

request_llm/bridge_qwen.py

@@ -0,0 +1,71 @@
+model_name = "Qwen"
+cmd_to_install = "`pip install -r request_llm/requirements_qwen.txt`"
+
+
+from transformers import AutoModel, AutoTokenizer
+import time
+import threading
+import importlib
+from toolbox import update_ui, get_conf
+from multiprocessing import Process, Pipe
+from .local_llm_class import LocalLLMHandle, get_local_llm_predict_fns, SingletonLocalLLM
+
+
+
+# ------------------------------------------------------------------------------------------------------------------------
+# 🔌💻 Local Model
+# ------------------------------------------------------------------------------------------------------------------------
+@SingletonLocalLLM
+class GetONNXGLMHandle(LocalLLMHandle):
+
+    def load_model_info(self):
+        # 🏃‍♂️🏃‍♂️🏃‍♂️ 子进程执行
+        self.model_name = model_name
+        self.cmd_to_install = cmd_to_install
+
+    def load_model_and_tokenizer(self):
+        # 🏃‍♂️🏃‍♂️🏃‍♂️ 子进程执行
+        import os, glob
+        import os
+        import platform
+        from modelscope import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
+
+        model_id = 'qwen/Qwen-7B-Chat'
+        revision = 'v1.0.1'
+        tokenizer = AutoTokenizer.from_pretrained(model_id, revision=revision, trust_remote_code=True)
+        # use fp16
+        model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", revision=revision, 
+                                                    trust_remote_code=True, fp16=True).eval()
+        model.generation_config = GenerationConfig.from_pretrained(model_id,
+                                                                trust_remote_code=True)  # 可指定不同的生成长度、top_p等相关超参
+        self._model = model
+
+        return self._model, None
+
+    def llm_stream_generator(self, **kwargs):
+        # 🏃‍♂️🏃‍♂️🏃‍♂️ 子进程执行
+        def adaptor(kwargs):
+            query = kwargs['query']
+            max_length = kwargs['max_length']
+            top_p = kwargs['top_p']
+            temperature = kwargs['temperature']
+            history = kwargs['history']
+            return query, max_length, top_p, temperature, history
+
+        query, max_length, top_p, temperature, history = adaptor(kwargs)
+
+        prompt = chat_template(history, query)
+        for response in model.chat(tokenizer, query, history=history, stream=True):
+            yield response
+
+    def try_to_import_special_deps(self, **kwargs):
+        # import something that will raise error if the user does not install requirement_*.txt
+        # 🏃‍♂️🏃‍♂️🏃‍♂️ 主进程执行
+        # from modelscope import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
+        pass
+
+
+# ------------------------------------------------------------------------------------------------------------------------
+# 🔌💻 GPT-Academic Interface
+# ------------------------------------------------------------------------------------------------------------------------
+predict_no_ui_long_connection, predict = get_local_llm_predict_fns(GetONNXGLMHandle, model_name)

request_llm/requirements_qwen.txt

@@ -0,0 +1,2 @@
+modelscope
+transformers_stream_generator

request_llm/test_llms.py

@@ -15,7 +15,8 @@ def validate_path():
     # from request_llm.bridge_jittorllms_pangualpha import predict_no_ui_long_connection
     # from request_llm.bridge_jittorllms_llama import predict_no_ui_long_connection
     # from request_llm.bridge_claude import predict_no_ui_long_connection
-    from request_llm.bridge_internlm import predict_no_ui_long_connection
+    # from request_llm.bridge_internlm import predict_no_ui_long_connection
+    from request_llm.bridge_qwen import predict_no_ui_long_connection
 
     llm_kwargs = {
         'max_length': 512,