async update

app/app.py

@@ -149,8 +149,7 @@ def open_file(filepath):
 
 # Function to play audio using PyAudio
 async def play_audio(file_path):
-    loop = asyncio.get_event_loop()
-    await loop.run_in_executor(None, sync_play_audio, file_path)
+    await asyncio.to_thread(sync_play_audio, file_path)
 
 def sync_play_audio(file_path):
     print("Starting audio playback")
@@ -215,10 +214,10 @@ async def process_and_play(prompt, audio_file_pth):
             await send_message_to_clients(json.dumps({"action": "ai_stop_speaking"}))
         else:
             print("Error: Audio file not found.")
-    else:
-        tts_model = xtts_model
+    else:  # XTTS
         try:
-            outputs = await tts_model.synthesize(
+            tts_model = xtts_model
+            outputs = await asyncio.to_thread(tts_model.synthesize,
                 prompt,
                 xtts_config,
                 speaker_wav=audio_file_pth,
@@ -389,6 +388,7 @@ def adjust_prompt(mood):
 
 def chatgpt_streamed(user_input, system_message, mood_prompt, conversation_history):
     full_response = ""
+    print(f"Debug: chatgpt_streamed started. MODEL_PROVIDER: {MODEL_PROVIDER}")
 
     if MODEL_PROVIDER == 'ollama':
         headers = {'Content-Type': 'application/json'}
@@ -399,6 +399,7 @@ def chatgpt_streamed(user_input, system_message, mood_prompt, conversation_histo
             "options": {"num_predict": -2, "temperature": 1.0}
         }
         try:
+            print(f"Debug: Sending request to Ollama: {OLLAMA_BASE_URL}/v1/chat/completions")
             response = requests.post(f'{OLLAMA_BASE_URL}/v1/chat/completions', headers=headers, json=payload, stream=True, timeout=30)
             response.raise_for_status()
 
@@ -426,12 +427,14 @@ def chatgpt_streamed(user_input, system_message, mood_prompt, conversation_histo
 
         except requests.exceptions.RequestException as e:
             full_response = f"Error connecting to Ollama model: {e}"
+            print(f"Debug: Ollama error - {e}")
 
     elif MODEL_PROVIDER == 'openai':
         messages = [{"role": "system", "content": system_message + "\n" + mood_prompt}] + conversation_history + [{"role": "user", "content": user_input}]
         headers = {'Authorization': f'Bearer {OPENAI_API_KEY}', 'Content-Type': 'application/json'}
         payload = {"model": OPENAI_MODEL, "messages": messages, "stream": True}
         try:
+            print(f"Debug: Sending request to OpenAI: {OPENAI_BASE_URL}")
             response = requests.post(OPENAI_BASE_URL, headers=headers, json=payload, stream=True, timeout=30)
             response.raise_for_status()
 
@@ -461,7 +464,9 @@ def chatgpt_streamed(user_input, system_message, mood_prompt, conversation_histo
 
         except requests.exceptions.RequestException as e:
             full_response = f"Error connecting to OpenAI model: {e}"
+            print(f"Debug: OpenAI error - {e}")
 
+    print(f"Debug: chatgpt_streamed completed. Response length: {len(full_response)}")
     return full_response
 
 def transcribe_with_whisper(audio_file):
@@ -505,7 +510,7 @@ def record_audio(file_path, silence_threshold=512, silence_duration=4.0, chunk_s
     wf.writeframes(b''.join(frames))
     wf.close()
 
-def execute_once(question_prompt):
+async def execute_once(question_prompt):
     temp_image_path = os.path.join(output_dir, 'temp_img.jpg')
 
     # Determine the audio file format based on the TTS provider
@@ -519,8 +524,8 @@ def execute_once(question_prompt):
         temp_audio_path = os.path.join(output_dir, 'temp_audio.wav')  # Use wav for XTTS
         max_char_length = 250  # Set a lower limit for XTTS
 
-    image_path = take_screenshot(temp_image_path)
-    response = analyze_image(image_path, question_prompt)
+    image_path = await take_screenshot(temp_image_path)
+    response = await analyze_image(image_path, question_prompt)
     text_response = response.get('choices', [{}])[0].get('message', {}).get('content', 'No response received.')
 
     # Truncate response based on the TTS provider's limit
@@ -529,40 +534,40 @@ def execute_once(question_prompt):
 
     print(text_response)
 
-    generate_speech(text_response, temp_audio_path)
+    await generate_speech(text_response, temp_audio_path)
 
     if TTS_PROVIDER == 'elevenlabs':
         # Convert MP3 to WAV if ElevenLabs is used
         temp_wav_path = os.path.join(output_dir, 'temp_output.wav')
         audio = AudioSegment.from_mp3(temp_audio_path)
         audio.export(temp_wav_path, format="wav")
-        play_audio(temp_wav_path)
+        await play_audio(temp_wav_path)
     else:
-        play_audio(temp_audio_path)
+        await play_audio(temp_audio_path)
 
     os.remove(image_path)
 
-
-def execute_screenshot_and_analyze():
+async def execute_screenshot_and_analyze():
     question_prompt = "What do you see in this image? Keep it short but detailed and answer any follow up questions about it"
     print("Taking screenshot and analyzing...")
-    execute_once(question_prompt)
+    await execute_once(question_prompt)
     print("\nReady for the next question....")
-
-def take_screenshot(temp_image_path):
-    time.sleep(5)
+    
+async def take_screenshot(temp_image_path):
+    await asyncio.sleep(5)
     screenshot = ImageGrab.grab()
     screenshot = screenshot.resize((1024, 1024))
     screenshot.save(temp_image_path, 'JPEG')
     return temp_image_path
 
-def encode_image(image_path):
+# Encode Image
+async def encode_image(image_path):
     with open(image_path, "rb") as image_file:
         return base64.b64encode(image_file.read()).decode('utf-8')
 
-def analyze_image(image_path, question_prompt):
-    encoded_image = encode_image(image_path)
-
+# Analyze Image
+async def analyze_image(image_path, question_prompt):
+    encoded_image = await encode_image(image_path)
     if MODEL_PROVIDER == 'ollama':
         headers = {'Content-Type': 'application/json'}
         payload = {
@@ -572,15 +577,17 @@ def analyze_image(image_path, question_prompt):
             "stream": False
         }
         try:
-            response = requests.post(f'{OLLAMA_BASE_URL}/api/generate', headers=headers, json=payload, timeout=30)
-            print(f"Response status code: {response.status_code}")
-            if response.status_code == 200:
-                return {"choices": [{"message": {"content": response.json().get('response', 'No response received.')}}]}
-            elif response.status_code == 404:
-                return {"choices": [{"message": {"content": "The llava model is not available on this server."}}]}
-            else:
-                response.raise_for_status()
-        except requests.exceptions.RequestException as e:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(f'{OLLAMA_BASE_URL}/api/generate', headers=headers, json=payload, timeout=30) as response:
+                    print(f"Response status code: {response.status}")
+                    if response.status == 200:
+                        response_json = await response.json()
+                        return {"choices": [{"message": {"content": response_json.get('response', 'No response received.')}}]}
+                    elif response.status == 404:
+                        return {"choices": [{"message": {"content": "The llava model is not available on this server."}}]}
+                    else:
+                        response.raise_for_status()
+        except aiohttp.ClientError as e:
             print(f"Request failed: {e}")
             return {"choices": [{"message": {"content": "Failed to process the image with the llava model."}}]}
     else:
@@ -594,29 +601,35 @@ def analyze_image(image_path, question_prompt):
         }
         payload = {"model": OPENAI_MODEL, "temperature": 0.5, "messages": [message], "max_tokens": 1000}
         try:
-            response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload, timeout=30)
-            response.raise_for_status()
-            return response.json()
-        except requests.exceptions.RequestException as e:
+            async with aiohttp.ClientSession() as session:
+                async with session.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload, timeout=30) as response:
+                    response.raise_for_status()
+                    return await response.json()
+        except aiohttp.ClientError as e:
             print(f"Request failed: {e}")
             return {"choices": [{"message": {"content": "Failed to process the image with the OpenAI model."}}]}
 
-def generate_speech(text, temp_audio_path):
+
+async def generate_speech(text, temp_audio_path):
     if TTS_PROVIDER == 'openai':
         headers = {"Content-Type": "application/json", "Authorization": f"Bearer {OPENAI_API_KEY}"}
         payload = {"model": "tts-1", "voice": OPENAI_TTS_VOICE, "input": text, "response_format": "wav"}
-        response = requests.post(OPENAI_TTS_URL, headers=headers, json=payload, timeout=30)
-        if response.status_code == 200:
-            with open(temp_audio_path, "wb") as audio_file:
-                audio_file.write(response.content)
-        else:
-            print(f"Failed to generate speech: {response.status_code} - {response.text}")
+        async with aiohttp.ClientSession() as session:
+            async with session.post(OPENAI_TTS_URL, headers=headers, json=payload, timeout=30) as response:
+                if response.status == 200:
+                    with open(temp_audio_path, "wb") as audio_file:
+                        audio_file.write(await response.read())
+                else:
+                    print(f"Failed to generate speech: {response.status} - {await response.text()}")
+
     elif TTS_PROVIDER == 'elevenlabs':
-        elevenlabs_text_to_speech(text, temp_audio_path)
-    else:
+        await elevenlabs_text_to_speech(text, temp_audio_path)
+
+    else:  # XTTS
         tts_model = xtts_model
         try:
-            outputs = tts_model.synthesize(
+            outputs = await asyncio.to_thread(
+                tts_model.synthesize,
                 text,
                 xtts_config,
                 speaker_wav=character_audio_file,
@@ -632,7 +645,7 @@ def generate_speech(text, temp_audio_path):
         except Exception as e:
             print(f"Error during XTTS audio generation: {e}")
 
-def user_chatbot_conversation():
+async def user_chatbot_conversation():
     conversation_history = []
     base_system_message = open_file(character_prompt_file)
     quit_phrases = ["quit", "Quit", "Quit.", "Exit.", "exit", "Exit", "leave", "Leave."]
@@ -659,7 +672,7 @@ def user_chatbot_conversation():
             conversation_history.append({"role": "user", "content": user_input})
 
             if any(phrase in user_input.lower() for phrase in screenshot_phrases):
-                execute_screenshot_and_analyze()
+                await execute_screenshot_and_analyze()  # Note the 'await' here
                 continue
 
             mood = analyze_mood(user_input)
@@ -672,11 +685,11 @@ def user_chatbot_conversation():
             if len(sanitized_response) > 400:
                 sanitized_response = sanitized_response[:400] + "..."
             prompt2 = sanitized_response
-            process_and_play(prompt2, character_audio_file)
+            await process_and_play(prompt2, character_audio_file)  # Note the 'await' here
             if len(conversation_history) > 20:
                 conversation_history = conversation_history[-20:]
     except KeyboardInterrupt:
         print("Quitting the conversation...")
 
 if __name__ == "__main__":
-    user_chatbot_conversation()
+    asyncio.run(user_chatbot_conversation())

app/app_logic.py

@@ -93,7 +93,7 @@ async def conversation_loop():
             break
 
         if any(phrase in user_input.lower() for phrase in screenshot_phrases):
-            execute_screenshot_and_analyze()
+            await execute_screenshot_and_analyze()
             continue
 
         try:

app/main.py

@@ -10,6 +10,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from .shared import clients, get_current_character, set_current_character
 from .app_logic import start_conversation, stop_conversation, set_env_variable
+# from .app import user_chatbot_conversation
 
 app = FastAPI()
 
@@ -25,6 +26,10 @@
     allow_headers=["*"],
 )
 
+# @app.on_event("startup")
+# async def startup_event():
+#     asyncio.create_task(user_chatbot_conversation())
+
 @app.get("/")
 async def get(request: Request):
     model_provider = os.getenv("MODEL_PROVIDER")
@@ -85,7 +90,7 @@ async def websocket_endpoint(websocket: WebSocket):
                 await start_conversation()
             elif message["action"] == "set_character":
                 set_current_character(message["character"])
-                await websocket.send_json({"message": f"Character set to {message['character']}"})
+                await websocket.send_json({"message": f"Character: {message['character']}"})
             elif message["action"] == "set_provider":
                 set_env_variable("MODEL_PROVIDER", message["provider"])
             elif message["action"] == "set_tts":