Skip to content

Commit

Permalink
Merge pull request #49 from rmusser01/main
Browse files Browse the repository at this point in the history 
RAG Notes now work!
  • Loading branch information
@rmusser01
rmusser01 authored Oct 25, 2024
2 parents 05df517 + 79a49cb commit a828788
Show file tree
Hide file tree
Showing 5 changed files with 213 additions and 66 deletions.
18 changes: 18 additions & 0 deletions App_Function_Libraries/DB/RAG_QA_Chat_DB.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -614,6 +614,24 @@ def update_conversation_title(conversation_id, new_title):
logger.error(f"Error updating conversation title: {e}")
raise

def delete_messages_in_conversation(conversation_id):
"""Helper function to delete all messages in a conversation."""
try:
execute_query("DELETE FROM rag_qa_chats WHERE conversation_id = ?", (conversation_id,))
logging.info(f"Messages in conversation '{conversation_id}' deleted successfully.")
except Exception as e:
logging.error(f"Error deleting messages in conversation '{conversation_id}': {e}")
raise

def get_conversation_title(conversation_id):
"""Helper function to get the conversation title."""
query = "SELECT title FROM conversation_metadata WHERE conversation_id = ?"
result = execute_query(query, (conversation_id,))
if result:
return result[0][0]
else:
return "Untitled Conversation"

def delete_conversation(conversation_id):
"""Delete a conversation and its associated messages and notes."""
try:
Expand Down
69 changes: 53 additions & 16 deletions App_Function_Libraries/Gradio_UI/RAG_QA_Chat_tab.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,31 +199,69 @@ def load_selected_note(note_selection):

def save_notes_function(note_title_text, notes_content, keywords_content, note_state_value, state_value):
"""Save the notes and associated keywords to the database."""
conversation_id = state_value.get("conversation_id")
note_id = note_state_value["note_id"]
if conversation_id and notes_content:
logging.info(f"Starting save_notes_function with state: {state_value}")
logging.info(f"Note title: {note_title_text}")
logging.info(f"Notes content length: {len(notes_content) if notes_content else 0}")

try:
# Check current state
conversation_id = state_value.get("conversation_id")
logging.info(f"Current conversation_id: {conversation_id}")

# Create new conversation if none exists
if not conversation_id:
logging.info("No conversation ID found, creating new conversation")
conversation_title = note_title_text if note_title_text else "Untitled Conversation"
conversation_id = start_new_conversation(title=conversation_title)
state_value = state_value.copy() # Create a new copy of the state
state_value["conversation_id"] = conversation_id
logging.info(f"Created new conversation with ID: {conversation_id}")

if not notes_content:
logging.warning("No notes content provided")
return notes_content, note_state_value, state_value, gr.update(
value="<p style='color:red;'>Cannot save empty notes.</p>")

# Save or update note
note_id = note_state_value.get("note_id")
if note_id:
# Update existing note
logging.info(f"Updating existing note with ID: {note_id}")
update_note(note_id, note_title_text, notes_content)
else:
# Save new note
note_id = save_notes(conversation_id, note_title_text, notes_content)
note_state_value["note_id"] = note_id
logging.info(f"Creating new note for conversation: {conversation_id}")
note_id = save_notes(conversation_id, note_title_text or "Untitled Note", notes_content)
note_state_value = {"note_id": note_id}
logging.info(f"Created new note with ID: {note_id}")

# Handle keywords
if keywords_content:
# Clear existing keywords and add new ones
logging.info("Processing keywords")
clear_keywords_from_note(note_id)
add_keywords_to_note(note_id, [kw.strip() for kw in keywords_content.split(',')])
keywords = [kw.strip() for kw in keywords_content.split(',')]
add_keywords_to_note(note_id, keywords)
logging.info(f"Added keywords: {keywords}")

logging.info("Notes and keywords saved successfully!")
return notes_content, note_state_value
else:
logging.warning("No conversation ID or notes to save.")
return "", note_state_value
logging.info("Notes saved successfully")
return (
notes_content,
note_state_value,
state_value,
gr.update(value="<p style='color:green;'>Notes saved successfully!</p>")
)

except Exception as e:
logging.error(f"Error in save_notes_function: {str(e)}", exc_info=True)
return (
notes_content,
note_state_value,
state_value,
gr.update(value=f"<p style='color:red;'>Error saving notes: {str(e)}</p>")
)

save_notes_btn.click(
save_notes_function,
inputs=[note_title, notes, keywords_for_notes, note_state, state],
outputs=[notes, note_state]
outputs=[notes, note_state, state, status_message]
)

def clear_notes_function():
Expand Down Expand Up @@ -562,7 +600,6 @@ def create_rag_qa_notes_management_tab():
# New Management Tab
with gr.TabItem("Notes Management", visible=True):
gr.Markdown("# RAG QA Notes Management")

management_state = gr.State({
"selected_conversation_id": None,
"selected_note_id": None,
Expand Down
19 changes: 11 additions & 8 deletions App_Function_Libraries/Gradio_UI/Video_transcription_tab.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
import logging
import os
from datetime import datetime
from typing import Dict, Any

#
# External Imports
import gradio as gr
Expand Down Expand Up @@ -210,7 +208,7 @@ def process_videos_with_error_handling(inputs, start_time, end_time, diarize, va
progress: gr.Progress = gr.Progress()) -> tuple:
try:
# Start overall processing timer
proc_start_time = datetime.utcnow()
proc_start_time = datetime.now()
# FIXME - summarize_recursively is not being used...
logging.info("Entering process_videos_with_error_handling")
logging.info(f"Received inputs: {inputs}")
Expand Down Expand Up @@ -264,15 +262,15 @@ def process_videos_with_error_handling(inputs, start_time, end_time, diarize, va

# Start timing
# FIXME - utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
start_proc = datetime.utcnow()
start_proc = datetime.now()

for i in range(0, len(all_inputs), batch_size):
batch = all_inputs[i:i + batch_size]
batch_results = []

for input_item in batch:
# Start individual video processing timer
video_start_time = datetime.utcnow()
video_start_time = datetime.now()
try:
start_seconds = convert_to_seconds(start_time)
end_seconds = convert_to_seconds(end_time) if end_time else None
Expand Down Expand Up @@ -377,7 +375,7 @@ def process_videos_with_error_handling(inputs, start_time, end_time, diarize, va
)

# Calculate processing time
video_end_time = datetime.utcnow()
video_end_time = datetime.now()
processing_time = (video_end_time - video_start_time).total_seconds()
log_histogram(
metric_name="video_processing_time_seconds",
Expand Down Expand Up @@ -485,7 +483,7 @@ def process_videos_with_error_handling(inputs, start_time, end_time, diarize, va
total_inputs = len(all_inputs)

# End overall processing timer
proc_end_time = datetime.utcnow()
proc_end_time = datetime.now()
total_processing_time = (proc_end_time - proc_start_time).total_seconds()
log_histogram(
metric_name="total_processing_time_seconds",
Expand Down Expand Up @@ -714,8 +712,9 @@ def process_url_with_metadata(input_item, num_speakers, whisper_model, custom_pr

# Perform transcription
logging.info("process_url_with_metadata: Starting transcription...")
logging.info(f"process_url_with_metadata: overwrite existing?: {overwrite_existing}")
audio_file_path, segments = perform_transcription(video_file_path, offset, whisper_model,
vad_filter, diarize)
vad_filter, diarize, overwrite_existing)

if audio_file_path is None or segments is None:
logging.error("process_url_with_metadata: Transcription failed or segments not available.")
Expand Down Expand Up @@ -871,3 +870,7 @@ def toggle_confabulation_output(checkbox_value):
],
outputs=[progress_output, error_output, results_output, download_transcription, download_summary, confabulation_output]
)

#
# End of Video_transcription_tab.py
#######################################################################################################################
Loading

0 comments on commit a828788

Please sign in to comment.