From 617b5c58bf8a2e910979527c6ae170509f263faa Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 11:03:02 -0800 Subject: [PATCH 01/19] Outline llm chatbot configuration with documentation --- README.md | 17 +++++++++++++++++ neon_llm_core/llm.py | 13 ++++++++++++- neon_llm_core/rmq.py | 10 +++++++++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9542674..7d39a75 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,20 @@ MQ: LLM_: num_parallel_processes: 0> ``` + +## Enabling Chatbot personas +An LLM may be configured to connect to a `/chatbots` vhost and participate in +discussions as described in the [chatbots project](https://github.com/NeonGeckoCom/chatbot-core). +One LLM may define multiple personas to participate as: +```yaml +llm_bots: + : + - name: Assistant + persona: You are a personal assistant who responds in 40 words or less + - name: Author + persona: You are author and expert in literary history + - name: Student + persona: You are a graduate student working in the field of artificial intelligence + enabled: False +``` +> `LLM Name` is defined in the property `NeonLLMMQConnector.name` diff --git a/neon_llm_core/llm.py b/neon_llm_core/llm.py index 390569c..9ea0a95 100644 --- a/neon_llm_core/llm.py +++ b/neon_llm_core/llm.py @@ -31,10 +31,21 @@ class NeonLLM(ABC): mq_to_llm_role = {} - def __init__(self, config): + def __init__(self, config: dict): + """ + @param config: Dict LLM configuration for this specific LLM + """ + self._llm_config = config self._tokenizer = None self._model = None + @property + def llm_config(self): + """ + Get the configuration for this LLM instance + """ + return self._llm_config + @property @abstractmethod def tokenizer(self): diff --git a/neon_llm_core/rmq.py b/neon_llm_core/rmq.py index 6c76ffb..93650bf 100644 --- a/neon_llm_core/rmq.py +++ b/neon_llm_core/rmq.py @@ -51,8 +51,16 @@ def __init__(self): self.register_consumers() self._model = None + if self.ovos_config.get("llm_bots", {}).get(self.name): + LOG.info(f"Chatbot(s) configured for: {self.name}") + for persona in self.ovos_config['llm_bots'][self.name]: + if not persona.get('enabled', True): + LOG.warning(f"Persona disabled: {persona['name']}") + continue + # TODO: Create Chatbot instance for persona + def register_consumers(self): - for idx in range(self.model_config["num_parallel_processes"]): + for idx in range(self.model_config.get("num_parallel_processes", 1)): self.register_consumer(name=f"neon_llm_{self.name}_ask_{idx}", vhost=self.vhost, queue=self.queue_ask, From 41672dbb3567df1b2e7d9582be19958b063837d4 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 13:37:51 -0800 Subject: [PATCH 02/19] Move `LLMBot` from `neon-llm-submind` package Update to support multiple configured subminds for one LLM instance Update documentation --- README.md | 4 +- neon_llm_core/chatbot.py | 148 +++++++++++++++++++++++++++++++++++++++ neon_llm_core/config.py | 9 +++ neon_llm_core/rmq.py | 63 +++++++++++------ 4 files changed, 201 insertions(+), 23 deletions(-) create mode 100644 neon_llm_core/chatbot.py diff --git a/README.md b/README.md index 7d39a75..f15bd23 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,11 @@ MQ: port: server: users: - : + : user: password: LLM_: - num_parallel_processes: 0> + num_parallel_processes: 0> ``` ## Enabling Chatbot personas diff --git a/neon_llm_core/chatbot.py b/neon_llm_core/chatbot.py new file mode 100644 index 0000000..88e0124 --- /dev/null +++ b/neon_llm_core/chatbot.py @@ -0,0 +1,148 @@ +# NEON AI (TM) SOFTWARE, Software Development Kit & Application Development System +# All trademark and other rights reserved by their respective owners +# Copyright 2008-2021 Neongecko.com Inc. +# BSD-3 +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# 3. Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from this +# software without specific prior written permission. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from typing import List +from chatbot_core.v2 import ChatBot +from neon_mq_connector.utils.client_utils import send_mq_request +from ovos_utils.log import LOG + +from neon_llm_core.config import LLMMQConfig + + +class LLMBot(ChatBot): + + def __init__(self, *args, **kwargs): + ChatBot.__init__(self, *args, **kwargs) + self.bot_type = "submind" + self.base_llm = kwargs.get("llm_name") # chatgpt, fastchat, etc. + self.persona = kwargs.get("persona") + self.current_llm_mq_config = self.get_llm_mq_config(self.base_llm) + LOG.info(f'Initialised config for persona={self._bot_id}') + self.prompt_id_to_shout = dict() + + @property + def contextual_api_supported(self): + return True + + def ask_chatbot(self, user: str, shout: str, timestamp: str, + context: dict = None) -> str: + """ + Handles an incoming shout into the current conversation + :param user: user associated with shout + :param shout: text shouted by user + :param timestamp: formatted timestamp of shout + :param context: message context + """ + prompt_id = context.get('prompt_id') + if prompt_id: + self.prompt_id_to_shout[prompt_id] = shout + LOG.debug(f"Getting response to {shout}") + response = self._get_llm_api_response( + shout=shout).get("response", "I have nothing to say here...") + return response + + def ask_discusser(self, options: dict, context: dict = None) -> str: + """ + Provides one discussion response based on the given options + + :param options: proposed responses (botname: response) + :param context: message context + """ + options = {k: v for k, v in options.items() if k != self.service_name} + prompt_sentence = self.prompt_id_to_shout.get(context['prompt_id'], '') + LOG.info(f'prompt_sentence={prompt_sentence}, options={options}') + opinion = self._get_llm_api_opinion(prompt=prompt_sentence, + options=options).get('opinion', '') + return opinion + + def ask_appraiser(self, options: dict, context: dict = None) -> str: + """ + Selects one of the responses to a prompt and casts a vote in the conversation. + :param options: proposed responses (botname: response) + :param context: message context + """ + if options: + options = {k: v for k, v in options.items() if k != self.service_name} + bots = list(options) + bot_responses = list(options.values()) + LOG.info(f'bots={bots}, answers={bot_responses}') + prompt = self.prompt_id_to_shout.pop(context['prompt_id'], '') + answer_data = self._get_llm_api_choice(prompt=prompt, + responses=bot_responses) + LOG.info(f'Received answer_data={answer_data}') + sorted_answer_indexes = answer_data['sorted_answer_indexes'] + if sorted_answer_indexes: + return bots[sorted_answer_indexes[0]] + return "abstain" + + def _get_llm_api_response(self, shout: str) -> dict: + """ + Requests LLM API for response on provided shout + :param shout: provided should string + :returns response string from LLM API + """ + LOG.debug(f"Sending to {self.current_llm_mq_config.vhost}/{self.current_llm_mq_config.ask_response_queue}") + return send_mq_request(vhost=self.current_llm_mq_config.vhost, + request_data={"query": shout, + "history": [], + "persona": self.persona}, + target_queue=self.current_llm_mq_config.ask_response_queue) + + def _get_llm_api_opinion(self, prompt: str, options: dict) -> dict: + """ + Requests LLM API for opinion on provided submind responses + :param prompt: incoming prompt text + :param options: proposed responses (botname: response) + :returns response data from LLM API + """ + return send_mq_request(vhost=self.current_llm_mq_config.vhost, + request_data={"query": prompt, + "options": options, + "persona": self.persona}, + target_queue=self.current_llm_mq_config.ask_discusser_queue) + + def _get_llm_api_choice(self, prompt: str, responses: List[str]) -> dict: + """ + Requests LLM API for choice among provided message list + :param prompt: incoming prompt text + :param responses: list of answers to select from + :returns response data from LLM API + """ + return send_mq_request(vhost=self.current_llm_mq_config.vhost, + request_data={"query": prompt, + "responses": responses, + "persona": self.persona}, + target_queue=self.current_llm_mq_config.ask_appraiser_queue) + + @staticmethod + def get_llm_mq_config(llm_name: str) -> LLMMQConfig: + """ + Get MQ queue names that the LLM service has access to. These are + LLM-oriented, not bot/persona-oriented. + """ + return LLMMQConfig(ask_response_queue=f"{llm_name}_input", + ask_appraiser_queue=f"{llm_name}_score_input", + ask_discusser_queue=f"{llm_name}_discussion_input") diff --git a/neon_llm_core/config.py b/neon_llm_core/config.py index 2787463..a4c2e51 100644 --- a/neon_llm_core/config.py +++ b/neon_llm_core/config.py @@ -26,6 +26,7 @@ import json +from dataclasses import dataclass from os.path import join, dirname, isfile from ovos_utils.log import LOG from ovos_config.config import Configuration @@ -48,3 +49,11 @@ def load_config() -> dict: with open(default_config_path) as f: config = json.load(f) return config + + +@dataclass +class LLMMQConfig: + ask_response_queue: str + ask_appraiser_queue: str + ask_discusser_queue: str + vhost: str = '/llm' diff --git a/neon_llm_core/rmq.py b/neon_llm_core/rmq.py index 93650bf..f51a83d 100644 --- a/neon_llm_core/rmq.py +++ b/neon_llm_core/rmq.py @@ -37,27 +37,34 @@ class NeonLLMMQConnector(MQConnector, ABC): """ Module for processing MQ requests to Fast Chat LLM """ - - opinion_prompt = "" - def __init__(self): self.service_name = f'neon_llm_{self.name}' self.ovos_config = load_config() - mq_config = self.ovos_config.get("MQ", None) + mq_config = self.ovos_config.get("MQ", dict()) super().__init__(config=mq_config, service_name=self.service_name) self.vhost = "/llm" self.register_consumers() self._model = None + self._bots = list() if self.ovos_config.get("llm_bots", {}).get(self.name): + from neon_llm_core.chatbot import LLMBot LOG.info(f"Chatbot(s) configured for: {self.name}") for persona in self.ovos_config['llm_bots'][self.name]: + # Spawn a service for each persona to support @user requests if not persona.get('enabled', True): LOG.warning(f"Persona disabled: {persona['name']}") continue - # TODO: Create Chatbot instance for persona + # Get a configured username to use for LLM submind connections + if mq_config.get("users", {}).get("neon_llm_submind"): + self.ovos_config["MQ"]["users"][persona['name']] = \ + mq_config['users']['neon_llm_submind'] + self._bots.append(LLMBot(llm_name=self.name, + service_id=persona['name'], + persona=persona, + config=self.ovos_config)) def register_consumers(self): for idx in range(self.model_config.get("num_parallel_processes", 1)): @@ -106,21 +113,23 @@ def model(self) -> NeonLLM: @create_mq_callback() def handle_request(self, body: dict): """ - Handles ask requests from MQ to LLM - :param body: request body (dict) + Handles ask requests from MQ to LLM + :param body: request body (dict) """ message_id = body["message_id"] routing_key = body["routing_key"] query = body["query"] history = body["history"] - persona = body.get("persona",{}) + persona = body.get("persona", {}) try: - response = self.model.ask(message=query, chat_history=history, persona=persona) + response = self.model.ask(message=query, chat_history=history, + persona=persona) except ValueError as err: LOG.error(f'ValueError={err}') - response = 'Sorry, but I cannot respond to your message at the moment, please try again later' + response = ('Sorry, but I cannot respond to your message at the ' + 'moment, please try again later') api_response = { "message_id": message_id, "response": response @@ -146,7 +155,8 @@ def handle_score_request(self, body: dict): sorted_answer_indexes = [] else: try: - sorted_answer_indexes = self.model.get_sorted_answer_indexes(question=query, answers=responses, persona=persona) + sorted_answer_indexes = self.model.get_sorted_answer_indexes( + question=query, answers=responses, persona=persona) except ValueError as err: LOG.error(f'ValueError={err}') sorted_answer_indexes = [] @@ -176,15 +186,17 @@ def handle_opinion_request(self, body: dict): opinion = "Sorry, but I got no options to choose from." else: try: - sorted_answer_indexes = self.model.get_sorted_answer_indexes(question=query, answers=responses, persona=persona) - best_respondent_nick, best_response = list(options.items())[sorted_answer_indexes[0]] - opinion = self._ask_model_for_opinion(respondent_nick=best_respondent_nick, - question=query, - answer=best_response, - persona=persona) + sorted_answer_indexes = self.model.get_sorted_answer_indexes( + question=query, answers=responses, persona=persona) + best_respondent_nick, best_response = list(options.items())[ + sorted_answer_indexes[0]] + opinion = self._ask_model_for_opinion( + respondent_nick=best_respondent_nick, + question=query, answer=best_response, persona=persona) except ValueError as err: LOG.error(f'ValueError={err}') - opinion = "Sorry, but I experienced an issue trying to make up an opinion on this topic" + opinion = ("Sorry, but I experienced an issue trying to form " + "an opinion on this topic") api_response = { "message_id": message_id, @@ -195,15 +207,24 @@ def handle_opinion_request(self, body: dict): queue=routing_key) LOG.info(f"Handled ask request for message_id={message_id}") - def _ask_model_for_opinion(self, respondent_nick: str, question: str, answer: str, persona: dict) -> str: + def _ask_model_for_opinion(self, respondent_nick: str, question: str, + answer: str, persona: dict) -> str: prompt = self.compose_opinion_prompt(respondent_nick=respondent_nick, question=question, answer=answer) - opinion = self.model.ask(message=prompt, chat_history=[], persona=persona) + opinion = self.model.ask(message=prompt, chat_history=[], + persona=persona) LOG.info(f'Received LLM opinion={opinion}, prompt={prompt}') return opinion @staticmethod @abstractmethod - def compose_opinion_prompt(respondent_nick: str, question: str, answer: str) -> str: + def compose_opinion_prompt(respondent_nick: str, question: str, + answer: str) -> str: + """ + Format a response into a prompt to evaluate another submind's response + @param respondent_nick: Name of submind providing a response + @param question: Prompt being responded to + @param answer: respondent's response to the question + """ pass From 7ae6db38439c4863fb8e3fa7223f6b4cf4849adc Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 14:04:24 -0800 Subject: [PATCH 03/19] Update LLM config docs to match existing code --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f15bd23..0c64139 100644 --- a/README.md +++ b/README.md @@ -40,11 +40,11 @@ One LLM may define multiple personas to participate as: llm_bots: : - name: Assistant - persona: You are a personal assistant who responds in 40 words or less + description: You are a personal assistant who responds in 40 words or less - name: Author - persona: You are author and expert in literary history + description: You are an author and expert in literary history - name: Student - persona: You are a graduate student working in the field of artificial intelligence + description: You are a graduate student working in the field of artificial intelligence enabled: False ``` > `LLM Name` is defined in the property `NeonLLMMQConnector.name` From 993f7534caf9a630b0dd9b0c989f3228a661060d Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 14:45:39 -0800 Subject: [PATCH 04/19] Add chatbot-core to extra deps Update LLM config handling and warnings --- neon_llm_core/rmq.py | 5 ++++- requirements/chatbots.txt | 1 + setup.py | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 requirements/chatbots.txt diff --git a/neon_llm_core/rmq.py b/neon_llm_core/rmq.py index f51a83d..9bdc519 100644 --- a/neon_llm_core/rmq.py +++ b/neon_llm_core/rmq.py @@ -91,7 +91,10 @@ def name(self): @property def model_config(self): - return self.ovos_config.get(f"LLM_{self.name.upper()}", None) + if f"LLM_{self.name.upper()}" not in self.ovos_config: + LOG.warning(f"No config for {self.name} found in " + f"{list(self.ovos_config.keys())}") + return self.ovos_config.get(f"LLM_{self.name.upper()}", dict()) @property def queue_ask(self): diff --git a/requirements/chatbots.txt b/requirements/chatbots.txt new file mode 100644 index 0000000..ff34f3a --- /dev/null +++ b/requirements/chatbots.txt @@ -0,0 +1 @@ +chatbot-core \ No newline at end of file diff --git a/setup.py b/setup.py index 47b3c05..5a36578 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,7 @@ def get_requirements(requirements_filename: str): license='BSD-3.0', packages=setuptools.find_packages(), install_requires=get_requirements("requirements.txt"), + extras_require={"chatbots": get_requirements("chatbots.txt")}, zip_safe=True, classifiers=[ 'Intended Audience :: Developers', From 086b32681b8b48d6d5d41aa5551e1c6cb30f9ded Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 14:50:59 -0800 Subject: [PATCH 05/19] Fix typo in chatbot-core dependency --- requirements/chatbots.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/chatbots.txt b/requirements/chatbots.txt index ff34f3a..72f84a3 100644 --- a/requirements/chatbots.txt +++ b/requirements/chatbots.txt @@ -1 +1 @@ -chatbot-core \ No newline at end of file +neon-chatbot-core \ No newline at end of file From eae0f2809c464fde6323d73a27a72945cc4b04a7 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 14:59:06 -0800 Subject: [PATCH 06/19] Fix typo in kwarg --- neon_llm_core/rmq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neon_llm_core/rmq.py b/neon_llm_core/rmq.py index 9bdc519..62ba90b 100644 --- a/neon_llm_core/rmq.py +++ b/neon_llm_core/rmq.py @@ -62,7 +62,7 @@ def __init__(self): self.ovos_config["MQ"]["users"][persona['name']] = \ mq_config['users']['neon_llm_submind'] self._bots.append(LLMBot(llm_name=self.name, - service_id=persona['name'], + service_name=persona['name'], persona=persona, config=self.ovos_config)) From 80d68ead8b92893c58f1563fe3075613a15b803b Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 15:27:41 -0800 Subject: [PATCH 07/19] Start submind chatbots with added logging --- neon_llm_core/rmq.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/neon_llm_core/rmq.py b/neon_llm_core/rmq.py index 62ba90b..3e0c079 100644 --- a/neon_llm_core/rmq.py +++ b/neon_llm_core/rmq.py @@ -61,10 +61,11 @@ def __init__(self): if mq_config.get("users", {}).get("neon_llm_submind"): self.ovos_config["MQ"]["users"][persona['name']] = \ mq_config['users']['neon_llm_submind'] - self._bots.append(LLMBot(llm_name=self.name, - service_name=persona['name'], - persona=persona, - config=self.ovos_config)) + bot = LLMBot(llm_name=self.name, service_name=persona['name'], + persona=persona, config=self.ovos_config) + bot.run() + LOG.info(f"Started chatbot: {bot.service_name}") + self._bots.append(bot) def register_consumers(self): for idx in range(self.model_config.get("num_parallel_processes", 1)): From 8e680c721030fd0ab9645117e4df0f8b1ff42916 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 15:32:08 -0800 Subject: [PATCH 08/19] Pass vhost to LLMBot init --- neon_llm_core/rmq.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neon_llm_core/rmq.py b/neon_llm_core/rmq.py index 3e0c079..4d18868 100644 --- a/neon_llm_core/rmq.py +++ b/neon_llm_core/rmq.py @@ -62,7 +62,8 @@ def __init__(self): self.ovos_config["MQ"]["users"][persona['name']] = \ mq_config['users']['neon_llm_submind'] bot = LLMBot(llm_name=self.name, service_name=persona['name'], - persona=persona, config=self.ovos_config) + persona=persona, config=self.ovos_config, + vhost=self.vhost) bot.run() LOG.info(f"Started chatbot: {bot.service_name}") self._bots.append(bot) From e3199ccaec84fca4fc9732f819d0c93f2d6a89d0 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 15:36:07 -0800 Subject: [PATCH 09/19] Fix LLM Chatbot to connecto to `chatbot` vhost --- neon_llm_core/rmq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neon_llm_core/rmq.py b/neon_llm_core/rmq.py index 4d18868..032e571 100644 --- a/neon_llm_core/rmq.py +++ b/neon_llm_core/rmq.py @@ -63,7 +63,7 @@ def __init__(self): mq_config['users']['neon_llm_submind'] bot = LLMBot(llm_name=self.name, service_name=persona['name'], persona=persona, config=self.ovos_config, - vhost=self.vhost) + vhost="/chatbots") bot.run() LOG.info(f"Started chatbot: {bot.service_name}") self._bots.append(bot) From c893c8e613f32d0a52c52a8609653ec085a6a9d0 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 15:59:02 -0800 Subject: [PATCH 10/19] Troubleshooting threaded submind running --- neon_llm_core/chatbot.py | 12 ++++++++---- neon_llm_core/rmq.py | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/neon_llm_core/chatbot.py b/neon_llm_core/chatbot.py index 88e0124..c52bae5 100644 --- a/neon_llm_core/chatbot.py +++ b/neon_llm_core/chatbot.py @@ -104,12 +104,14 @@ def _get_llm_api_response(self, shout: str) -> dict: :param shout: provided should string :returns response string from LLM API """ - LOG.debug(f"Sending to {self.current_llm_mq_config.vhost}/{self.current_llm_mq_config.ask_response_queue}") + LOG.debug(f"Sending to {self.current_llm_mq_config.vhost}/" + f"{self.current_llm_mq_config.ask_response_queue}") return send_mq_request(vhost=self.current_llm_mq_config.vhost, request_data={"query": shout, "history": [], "persona": self.persona}, - target_queue=self.current_llm_mq_config.ask_response_queue) + target_queue=self.current_llm_mq_config. + ask_response_queue) def _get_llm_api_opinion(self, prompt: str, options: dict) -> dict: """ @@ -122,7 +124,8 @@ def _get_llm_api_opinion(self, prompt: str, options: dict) -> dict: request_data={"query": prompt, "options": options, "persona": self.persona}, - target_queue=self.current_llm_mq_config.ask_discusser_queue) + target_queue=self.current_llm_mq_config. + ask_discusser_queue) def _get_llm_api_choice(self, prompt: str, responses: List[str]) -> dict: """ @@ -135,7 +138,8 @@ def _get_llm_api_choice(self, prompt: str, responses: List[str]) -> dict: request_data={"query": prompt, "responses": responses, "persona": self.persona}, - target_queue=self.current_llm_mq_config.ask_appraiser_queue) + target_queue=self.current_llm_mq_config. + ask_appraiser_queue) @staticmethod def get_llm_mq_config(llm_name: str) -> LLMMQConfig: diff --git a/neon_llm_core/rmq.py b/neon_llm_core/rmq.py index 032e571..6c1653e 100644 --- a/neon_llm_core/rmq.py +++ b/neon_llm_core/rmq.py @@ -24,6 +24,7 @@ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from abc import abstractmethod, ABC +from threading import Thread from neon_mq_connector.connector import MQConnector from neon_mq_connector.utils.rabbit_utils import create_mq_callback @@ -64,7 +65,7 @@ def __init__(self): bot = LLMBot(llm_name=self.name, service_name=persona['name'], persona=persona, config=self.ovos_config, vhost="/chatbots") - bot.run() + Thread(target=bot.run, daemon=True).start() LOG.info(f"Started chatbot: {bot.service_name}") self._bots.append(bot) From a3c4c0089cbfbf457ca005ea599ee05bffd2e284 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 16:09:57 -0800 Subject: [PATCH 11/19] Revert threading change --- neon_llm_core/llm.py | 5 +++-- neon_llm_core/rmq.py | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/neon_llm_core/llm.py b/neon_llm_core/llm.py index 9ea0a95..a5ccb2a 100644 --- a/neon_llm_core/llm.py +++ b/neon_llm_core/llm.py @@ -117,5 +117,6 @@ def convert_role(cls, role: str) -> str: """ Maps MQ role to LLM's internal domain """ matching_llm_role = cls.mq_to_llm_role.get(role) if not matching_llm_role: - raise ValueError(f"role={role} is undefined, supported are: {list(cls.mq_to_llm_role)}") - return matching_llm_role \ No newline at end of file + raise ValueError(f"role={role} is undefined, supported are: " + f"{list(cls.mq_to_llm_role)}") + return matching_llm_role diff --git a/neon_llm_core/rmq.py b/neon_llm_core/rmq.py index 6c1653e..01f76d7 100644 --- a/neon_llm_core/rmq.py +++ b/neon_llm_core/rmq.py @@ -23,9 +23,8 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from abc import abstractmethod, ABC -from threading import Thread +from abc import abstractmethod, ABC from neon_mq_connector.connector import MQConnector from neon_mq_connector.utils.rabbit_utils import create_mq_callback from ovos_utils.log import LOG @@ -65,7 +64,7 @@ def __init__(self): bot = LLMBot(llm_name=self.name, service_name=persona['name'], persona=persona, config=self.ovos_config, vhost="/chatbots") - Thread(target=bot.run, daemon=True).start() + bot.run() LOG.info(f"Started chatbot: {bot.service_name}") self._bots.append(bot) From 1567f3c3ccde29af63d893a2d5c233611ef2fd53 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 17:12:44 -0800 Subject: [PATCH 12/19] Refactor `mq_queue_config` variable name Update logging to debug MQ connection errors --- neon_llm_core/chatbot.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/neon_llm_core/chatbot.py b/neon_llm_core/chatbot.py index c52bae5..59de69b 100644 --- a/neon_llm_core/chatbot.py +++ b/neon_llm_core/chatbot.py @@ -37,10 +37,11 @@ class LLMBot(ChatBot): def __init__(self, *args, **kwargs): ChatBot.__init__(self, *args, **kwargs) self.bot_type = "submind" - self.base_llm = kwargs.get("llm_name") # chatgpt, fastchat, etc. + self.base_llm = kwargs.get("llm_name") # chat_gpt, fastchat, etc. self.persona = kwargs.get("persona") - self.current_llm_mq_config = self.get_llm_mq_config(self.base_llm) - LOG.info(f'Initialised config for persona={self._bot_id}') + self.mq_queue_config = self.get_llm_mq_config(self.base_llm) + LOG.info(f'Initialised config for llm={self.base_llm}|' + f'persona={self._bot_id}') self.prompt_id_to_shout = dict() @property @@ -104,14 +105,20 @@ def _get_llm_api_response(self, shout: str) -> dict: :param shout: provided should string :returns response string from LLM API """ - LOG.debug(f"Sending to {self.current_llm_mq_config.vhost}/" - f"{self.current_llm_mq_config.ask_response_queue}") - return send_mq_request(vhost=self.current_llm_mq_config.vhost, - request_data={"query": shout, - "history": [], - "persona": self.persona}, - target_queue=self.current_llm_mq_config. - ask_response_queue) + LOG.info(f"Sending to {self.mq_queue_config.vhost}/" + f"{self.mq_queue_config.ask_response_queue}") + try: + return send_mq_request(vhost=self.mq_queue_config.vhost, + request_data={"query": shout, + "history": [], + "persona": self.persona}, + target_queue=self.mq_queue_config. + ask_response_queue) + except Exception as e: + LOG.exception(f"Failed to get response on " + f"{self.mq_queue_config.vhost}/" + f"{self.mq_queue_config.ask_response_queue}: " + f"{e}") def _get_llm_api_opinion(self, prompt: str, options: dict) -> dict: """ @@ -120,11 +127,11 @@ def _get_llm_api_opinion(self, prompt: str, options: dict) -> dict: :param options: proposed responses (botname: response) :returns response data from LLM API """ - return send_mq_request(vhost=self.current_llm_mq_config.vhost, + return send_mq_request(vhost=self.mq_queue_config.vhost, request_data={"query": prompt, "options": options, "persona": self.persona}, - target_queue=self.current_llm_mq_config. + target_queue=self.mq_queue_config. ask_discusser_queue) def _get_llm_api_choice(self, prompt: str, responses: List[str]) -> dict: @@ -134,11 +141,11 @@ def _get_llm_api_choice(self, prompt: str, responses: List[str]) -> dict: :param responses: list of answers to select from :returns response data from LLM API """ - return send_mq_request(vhost=self.current_llm_mq_config.vhost, + return send_mq_request(vhost=self.mq_queue_config.vhost, request_data={"query": prompt, "responses": responses, "persona": self.persona}, - target_queue=self.current_llm_mq_config. + target_queue=self.mq_queue_config. ask_appraiser_queue) @staticmethod From 628479b5c12b6299e0c99f31fa374bc6911fdb7c Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 17:25:00 -0800 Subject: [PATCH 13/19] Specify response queues for permissions --- neon_llm_core/chatbot.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/neon_llm_core/chatbot.py b/neon_llm_core/chatbot.py index 59de69b..35dd4e6 100644 --- a/neon_llm_core/chatbot.py +++ b/neon_llm_core/chatbot.py @@ -105,15 +105,15 @@ def _get_llm_api_response(self, shout: str) -> dict: :param shout: provided should string :returns response string from LLM API """ - LOG.info(f"Sending to {self.mq_queue_config.vhost}/" - f"{self.mq_queue_config.ask_response_queue}") + queue = self.mq_queue_config.ask_response_queue + LOG.info(f"Sending to {self.mq_queue_config.vhost}/{queue}") try: return send_mq_request(vhost=self.mq_queue_config.vhost, request_data={"query": shout, "history": [], "persona": self.persona}, - target_queue=self.mq_queue_config. - ask_response_queue) + target_queue=queue, + response_queue=f"{queue}.response") except Exception as e: LOG.exception(f"Failed to get response on " f"{self.mq_queue_config.vhost}/" @@ -127,12 +127,13 @@ def _get_llm_api_opinion(self, prompt: str, options: dict) -> dict: :param options: proposed responses (botname: response) :returns response data from LLM API """ + queue = self.mq_queue_config.ask_discusser_queue return send_mq_request(vhost=self.mq_queue_config.vhost, request_data={"query": prompt, "options": options, "persona": self.persona}, - target_queue=self.mq_queue_config. - ask_discusser_queue) + target_queue=queue, + response_queue=f"{queue}.response") def _get_llm_api_choice(self, prompt: str, responses: List[str]) -> dict: """ @@ -141,12 +142,13 @@ def _get_llm_api_choice(self, prompt: str, responses: List[str]) -> dict: :param responses: list of answers to select from :returns response data from LLM API """ + queue = self.mq_queue_config.ask_appraiser_queue return send_mq_request(vhost=self.mq_queue_config.vhost, request_data={"query": prompt, "responses": responses, "persona": self.persona}, - target_queue=self.mq_queue_config. - ask_appraiser_queue) + target_queue=queue, + response_queue=f"{queue}.response") @staticmethod def get_llm_mq_config(llm_name: str) -> LLMMQConfig: From 66e31c9c23e03874bf5f5792f92f1fa93c6b8413 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Tue, 12 Dec 2023 17:46:41 -0800 Subject: [PATCH 14/19] Add log to debug response errors --- neon_llm_core/rmq.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/neon_llm_core/rmq.py b/neon_llm_core/rmq.py index 01f76d7..b78d1a6 100644 --- a/neon_llm_core/rmq.py +++ b/neon_llm_core/rmq.py @@ -139,6 +139,7 @@ def handle_request(self, body: dict): "message_id": message_id, "response": response } + LOG.info(f"Sending response: {response}") self.send_message(request_data=api_response, queue=routing_key) LOG.info(f"Handled ask request for message_id={message_id}") @@ -154,7 +155,7 @@ def handle_score_request(self, body: dict): query = body["query"] responses = body["responses"] - persona = body.get("persona",{}) + persona = body.get("persona", {}) if not responses: sorted_answer_indexes = [] @@ -184,7 +185,7 @@ def handle_opinion_request(self, body: dict): query = body["query"] options = body["options"] - persona = body.get("persona",{}) + persona = body.get("persona", {}) responses = list(options.values()) if not responses: From 3c4b902d66a79de2eca3db26f9bca2fd1461953e Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 14 Dec 2023 14:18:28 -0800 Subject: [PATCH 15/19] Handle LLM request timeouts --- neon_llm_core/chatbot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/neon_llm_core/chatbot.py b/neon_llm_core/chatbot.py index 35dd4e6..4eed1f2 100644 --- a/neon_llm_core/chatbot.py +++ b/neon_llm_core/chatbot.py @@ -119,6 +119,7 @@ def _get_llm_api_response(self, shout: str) -> dict: f"{self.mq_queue_config.vhost}/" f"{self.mq_queue_config.ask_response_queue}: " f"{e}") + return dict() def _get_llm_api_opinion(self, prompt: str, options: dict) -> dict: """ From 6db521b1a1dbb32286d3af5ed596d3f13a1dbf40 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 14 Dec 2023 14:30:14 -0800 Subject: [PATCH 16/19] Handle appraiser no response --- neon_llm_core/chatbot.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/neon_llm_core/chatbot.py b/neon_llm_core/chatbot.py index 4eed1f2..72499d3 100644 --- a/neon_llm_core/chatbot.py +++ b/neon_llm_core/chatbot.py @@ -86,7 +86,8 @@ def ask_appraiser(self, options: dict, context: dict = None) -> str: :param context: message context """ if options: - options = {k: v for k, v in options.items() if k != self.service_name} + options = {k: v for k, v in options.items() + if k != self.service_name} bots = list(options) bot_responses = list(options.values()) LOG.info(f'bots={bots}, answers={bot_responses}') @@ -94,7 +95,7 @@ def ask_appraiser(self, options: dict, context: dict = None) -> str: answer_data = self._get_llm_api_choice(prompt=prompt, responses=bot_responses) LOG.info(f'Received answer_data={answer_data}') - sorted_answer_indexes = answer_data['sorted_answer_indexes'] + sorted_answer_indexes = answer_data.get('sorted_answer_indexes') if sorted_answer_indexes: return bots[sorted_answer_indexes[0]] return "abstain" From d607563bda8391836ac3a57da3fc88b7cfb08876 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 14 Dec 2023 16:14:43 -0800 Subject: [PATCH 17/19] Handle incoming requests asynchronously to support multiple personas more efficiently --- neon_llm_core/llm.py | 8 +++++--- neon_llm_core/rmq.py | 20 ++++++++++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/neon_llm_core/llm.py b/neon_llm_core/llm.py index a5ccb2a..a4c9244 100644 --- a/neon_llm_core/llm.py +++ b/neon_llm_core/llm.py @@ -91,9 +91,11 @@ def get_sorted_answer_indexes(self, question: str, answers: List[str], persona: @abstractmethod def _call_model(self, prompt: str) -> str: """ - Wrapper for Model generation logic - :param prompt: Input text sequence - :returns: Output text sequence generated by model + Wrapper for Model generation logic. This method may be called + asynchronously, so it is up to the extending class to use locks or + queue inputs as necessary. + :param prompt: Input text sequence + :returns: Output text sequence generated by model """ pass diff --git a/neon_llm_core/rmq.py b/neon_llm_core/rmq.py index b78d1a6..c042542 100644 --- a/neon_llm_core/rmq.py +++ b/neon_llm_core/rmq.py @@ -25,6 +25,8 @@ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from abc import abstractmethod, ABC +from threading import Thread + from neon_mq_connector.connector import MQConnector from neon_mq_connector.utils.rabbit_utils import create_mq_callback from ovos_utils.log import LOG @@ -121,12 +123,18 @@ def handle_request(self, body: dict): Handles ask requests from MQ to LLM :param body: request body (dict) """ - message_id = body["message_id"] - routing_key = body["routing_key"] - - query = body["query"] - history = body["history"] - persona = body.get("persona", {}) + # Handle this asynchronously so multiple subminds can be handled + # concurrently + Thread(target=self._handle_request_async, args=(body,), + daemon=True).start() + + def _handle_request_async(self, request: dict): + message_id = request["message_id"] + routing_key = request["routing_key"] + + query = request["query"] + history = request["history"] + persona = request.get("persona", {}) try: response = self.model.ask(message=query, chat_history=history, From 291ebdd0d22b909bc0f6924da7a0f524eadf56c9 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Thu, 14 Dec 2023 17:44:25 -0800 Subject: [PATCH 18/19] Run personas in threads for better logging --- requirements/chatbots.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/chatbots.txt b/requirements/chatbots.txt index 72f84a3..5d38d2d 100644 --- a/requirements/chatbots.txt +++ b/requirements/chatbots.txt @@ -1 +1,2 @@ -neon-chatbot-core \ No newline at end of file +# neon-chatbot-core +neon-chatbot-core@git+https://github.com/neongeckocom/chatbot-core@FEAT_CopyLogPerBot \ No newline at end of file From c9063c3325570289a5ef56e9ffb5aa9bc24e3f38 Mon Sep 17 00:00:00 2001 From: Daniel McKnight Date: Mon, 18 Dec 2023 11:24:12 -0800 Subject: [PATCH 19/19] Update chatbot-core dependency spec --- requirements/chatbots.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/chatbots.txt b/requirements/chatbots.txt index 5d38d2d..72f84a3 100644 --- a/requirements/chatbots.txt +++ b/requirements/chatbots.txt @@ -1,2 +1 @@ -# neon-chatbot-core -neon-chatbot-core@git+https://github.com/neongeckocom/chatbot-core@FEAT_CopyLogPerBot \ No newline at end of file +neon-chatbot-core \ No newline at end of file