Add logic to convert math to python to context in chat completions workflow

agixt/XT.py

@@ -309,6 +309,7 @@ async def execute_command(
         command_args: dict,
         voice_response: bool = False,
         log_output: bool = False,
+        log_activities: bool = False,
     ):
         """
         Execute a command with arguments
@@ -318,14 +319,16 @@ async def execute_command(
             command_args (dict): Arguments for the command
             voice_response (bool): Whether to generate a voice response
             log_output (bool): Whether to log the output
+            log_activities (bool): Whether to log the activities
 
         Returns:
             str: Response from the command
         """
-        self.conversation.log_interaction(
-            role=self.agent_name,
-            message=f"[ACTIVITY] Executing command `{command_name}` with args:\n```json\n{json.dumps(command_args, indent=2)}```",
-        )
+        if log_activities:
+            self.conversation.log_interaction(
+                role=self.agent_name,
+                message=f"[ACTIVITY] Executing command `{command_name}` with args:\n```json\n{json.dumps(command_args, indent=2)}```",
+            )
         try:
             response = await Extensions(
                 agent_name=self.agent_name,
@@ -1394,7 +1397,14 @@ async def chat_completions(self, prompt: ChatCompletions):
             urls=urls,
             summarize_content=False,
         )
-        data_analysis = await self.analyze_csv(user_input=new_prompt)
+        analyze_user_input = True
+        if "analyze_user_input" in self.agent_settings:
+            analyze_user_input = (
+                str(self.agent_settings["analyze_user_input"]).lower() == "true"
+            )
+        data_analysis = ""
+        if analyze_user_input:
+            data_analysis = await self.analyze_data(user_input=new_prompt)
         if mode == "command" and command_name and command_variable:
             try:
                 command_args = (
@@ -1705,7 +1715,90 @@ def generate_markdown_structure(folder_path, indent=0):
 
         return generate_markdown_structure(folder_path=self.agent_workspace)
 
-    async def analyze_csv(
+    async def execute_code(self, user_input: str):
+        code_execution = ""
+        self.conversation.log_interaction(
+            role=self.agent_name,
+            message=f"[ACTIVITY] Analyzing data.",
+        )
+        code_interpreter = await self.inference(
+            user_input=user_input,
+            prompt_category="Default",
+            prompt_name="Write Code",
+            log_user_input=False,
+            log_output=False,
+            browse_links=False,
+            websearch=False,
+            websearch_depth=0,
+            voice_response=False,
+        )
+        if "```python" in code_interpreter:
+            code_interpreter = code_interpreter.split("```python")[1].split("```")[0]
+            if "```python" in code_interpreter:
+                code_interpreter = code_interpreter.split("```python")[1].split("```")[
+                    0
+                ]
+            code_verification = await self.inference(
+                user_input=user_input,
+                prompt_category="Default",
+                prompt_name="Verify Code",
+                code=code_interpreter,
+                log_user_input=False,
+                log_output=False,
+                browse_links=False,
+                websearch=False,
+                websearch_depth=0,
+                voice_response=False,
+            )
+            if "```python" in code_verification:
+                code_verification = code_verification.split("```python")[1].split(
+                    "```"
+                )[0]
+                if "```python" in code_verification:
+                    code_verification = code_verification.split("```python")[1].split(
+                        "```"
+                    )[0]
+                try:
+                    code_execution = await self.execute_command(
+                        command_name="Execute Python Code",
+                        command_args={"code": code_verification, "text": ""},
+                    )
+                except Exception as e:
+                    fixed_code = await self.inference(
+                        user_input=user_input,
+                        prompt_category="Default",
+                        prompt_name="Fix Verified Code",
+                        code=code_verification,
+                        code_error=str(e),
+                        log_user_input=False,
+                        log_output=False,
+                        browse_links=False,
+                        websearch=False,
+                        websearch_depth=0,
+                        voice_response=False,
+                    )
+                    code_verification = fixed_code
+                    if "```python" in code_verification:
+                        code_verification = code_verification.split("```python")[
+                            1
+                        ].split("```")[0]
+                        if "```python" in code_verification:
+                            code_verification = code_verification.split("```python")[
+                                1
+                            ].split("```")[0]
+                        try:
+                            code_execution = await self.execute_command(
+                                command_name="Execute Python Code",
+                                command_args={"code": code_verification, "text": ""},
+                            )
+                        except Exception as e:
+                            code_execution = ""
+                            logging.error(f"Error executing code: {e}")
+        if code_execution != "":
+            return f"Executed the following code expressed to assist the user:\n```python\n{code_verification}\n```\n**REFERENCE THE RESULTS, NOT THE CODE TO THE USER WHEN RESPONDING! THE RESULTS FROM RUNNING THE CODE ARE AS FOLLOWS:**\n{code_execution}"
+        return ""
+
+    async def analyze_data(
         self,
         user_input: str,
         file_content=None,
@@ -1730,11 +1823,11 @@ async def analyze_csv(
             csv_files = [file for file in files if file.endswith(".csv")]
             logging.info(f"CSV files in conversation workspace: {csv_files}")
             if len(csv_files) == 0:
-                return ""
+                return await self.execute_code(user_input=user_input)
             activities = self.conversation.get_activities(limit=20)["activities"]
             logging.info(f"Activities: {activities}")
             if len(activities) == 0:
-                return ""
+                return await self.execute_code(user_input=user_input)
             likely_files = []
             for activity in activities:
                 if ".csv" in activity["message"]:
@@ -1766,7 +1859,7 @@ async def analyze_csv(
                     file_path = os.path.join(self.conversation_workspace, file_name)
                     file_content = open(file_path, "r").read()
             if file_name == "":
-                return ""
+                return await self.execute_code(user_input=user_input)
         if len(file_names) > 1:
             # Found multiple files, do things a little differently.
             previews = []
@@ -1879,7 +1972,7 @@ async def analyze_csv(
                     role=self.agent_name,
                     message=f"[ACTIVITY][WARN] Data analysis failed, trying again ({self.failures}/3).",
                 )
-                return await self.analyze_csv(
+                return await self.analyze_data(
                     user_input=user_input,
                     file_name=file_name,
                     file_content=file_content,

agixt/prompts/Default/Convert Math to Python.txt

@@ -0,0 +1,11 @@
+## Context 
+ {context} 
+
+Recent conversation history for context: 
+ {conversation_history} 
+
+## User
+ {user_input} 
+
+## System
+If the users input contains a problem that is a math problem able to be expressed in Python code, write python code to express answering the question with a print as the final line in the code that would answer the users question. Otherwise, just respond with "None".

agixt/prompts/Default/Fix Verified Code.txt

@@ -0,0 +1,20 @@
+## Code
+```python
+{code}
+```
+
+## Error
+{code_error}
+
+User's last message: {user_input} 
+
+## Rules
+
+**Make sure the output contains visualizations! It must end with a print for the output to show!**
+**Any files saved to {working_directory} will be immediately available at {output_url} with the file name you used at the end. The final output of the code should be a string in markdown format to return to the user once analysis is complete, this should include any images linked to the URL provided. Make sure the output references any visualizations generated.**
+**Make a best effort to ensure that the code is functional and without errors.**
+
+## System
+The assistant is assisting the user with data analysis and ran into an error, rewrite the code to fix the error, then we will try to execute it again.
+
+**Confirm that the code follows the rules. Return full updated code without placeholders that is confirmed.**

agixt/prompts/Default/Verify Code.txt

@@ -0,0 +1,17 @@
+The date today is {date} .
+
+We built python code to build a visualization for the user's input.
+
+User's input: {user_input} 
+
+**Ensure the code does not modify the file system outside of the working directory at {working_directory} , if it does, remove the portion of the code that does.**
+
+**Any files saved to {working_directory} will be immediately available at {output_url} with the file name you used at the end. The final output of the code should be a string in markdown format to return to the user once analysis is complete, this should include any images linked to the URL provided. Make sure the output references any visualizations generated.**
+
+**Make sure the output contains visualizations! It must end with a print for the output to show! Only print what should output to the user.**
+
+**Confirm that the code follows the rules. Return full updated code without placeholders that is confirmed.**
+
+```python
+{code}
+```

agixt/prompts/Default/Write Code.txt

@@ -0,0 +1,38 @@
+## Context 
+ {context} 
+
+Recent conversation history for context: 
+ {conversation_history} 
+
+## System
+The assistant is a very powerful Python Code Interpreter, designed to assist with a wide range of tasks, particularly those related to data science, data analysis, data visualization, and file manipulation.
+
+Unlike many text-based AIs, the assistant has the capability to directly manipulate files, convert images, and perform a variety of other tasks. Here are some examples:
+
+- Image Description and Manipulation: the assistant can directly manipulate images, including zooming, cropping, color grading, and resolution enhancement. It can also convert images from one format to another.
+- QR Code Generation: the assistant can create QR codes for various purposes.
+- Project Management: the assistant can assist in creating Gantt charts and mapping out project steps.
+- Study Scheduling: the assistant can design optimized study schedules for exam preparation.
+- File Conversion: the assistant can directly convert files from one format to another, such as PDF to text or video to audio.
+- Mathematical Computation: the assistant can solve complex math equations and produce graphs.
+- Document Analysis: the assistant can analyze, summarize, or extract information from large documents.
+- Data Visualization: the assistant can analyze datasets, identify trends, and create various types of graphs.
+- Geolocation Visualization: the assistant can provide geolocation maps to showcase specific trends or occurrences.
+- Code Analysis and Creation: the assistant can analyze and critique code, and even create code from scratch.
+- Many other things that can be accomplished running python code in a jupyter environment.
+- Multiple visualizations are allowed as long as the return is a markdown string of the base64 image.
+- The date today is {date} .
+
+The assistant can execute Python code within a sandboxed Jupyter kernel environment. The kernel comes equipped with a variety of pre-installed Python packages including numpy, pandas, matplotlib, seaborn, scikit-learn, yfinance, scipy, statsmodels, sympy, bokeh, plotly, dash, and networkx. Additionally, you have the ability to use other packages which automatically get installed when found in the code, simply comment `# pip install packageName` anywhere in the code to have it automatically installed.
+
+Remember, You are constantly learning and improving. You are capable of generating human-like text based on the input it receives, engaging in natural-sounding conversations, and providing responses that are coherent and relevant to the topic at hand. Enjoy your coding session!
+
+If the user's input doesn't request any specific analysis or asks to surprise them, write code that will to plot something interesting to provide them with insights into the data through visualizations.
+
+**Any files saved to {working_directory} will be immediately available at {output_url} with the file name you used at the end. The final output of the code should be a string in markdown format to return to the user once analysis is complete, this should include any images linked to the URL provided. Make sure the output references any visualizations generated.**
+
+**Make sure the output contains visualizations! It must end with a print for the output to show!**
+
+User's input: {user_input} 
+
+If the users input contains a problem that is able to be expressed and answered using Python code, write python code to express answering the question with a print as the final line in the code that would help answer the users. Otherwise, just respond with "None".