feat(assistants): Add PDF generation functionality and nice emails

.env.example

@@ -33,6 +33,8 @@ AUTHENTICATE=true
 TELEMETRY_ENABLED=true
 CELERY_BROKER_URL=redis://redis:6379/0
 CELEBRY_BROKER_QUEUE_NAME=quivr-preview.fifo
+QUIVR_DOMAIN=http://localhost:3000/
+
 
 
 #RESEND

.gitignore

@@ -83,3 +83,4 @@ backend/celerybeat-schedule.db
 
 backend/application.log.*
 backend/score.json
+backend/modules/assistant/ito/utils/simple.pdf

Pipfile

@@ -58,6 +58,7 @@ lxml = {extras = ["html_clean"], version = "*"}
 ragas = "*"
 datasets = "*"
 pytest-dotenv = "*"
+fpdf2 = "*"
 
 [dev-packages]
 black = "*"

backend/modules/assistant/ito/ito.py

@@ -1,3 +1,4 @@
+import os
 import random
 from abc import abstractmethod
 from io import BytesIO
@@ -8,6 +9,7 @@
 from logger import get_logger
 from models.user_usage import UserUsage
 from modules.assistant.dto.inputs import InputAssistant
+from modules.assistant.ito.utils.pdf_generator import PDFGenerator, PDFModel
 from modules.chat.controller.chat.utils import update_user_usage
 from modules.contact_support.controller.settings import ContactsSettings
 from modules.upload.controller.upload_routes import upload_file
@@ -57,32 +59,63 @@ def increase_usage_user(self):
     def calculate_pricing(self):
         return 20
 
+    def generate_pdf(self, filename: str, title: str, content: str):
+        pdf_model = PDFModel(title=title, content=content)
+        pdf = PDFGenerator(pdf_model)
+        pdf.print_pdf()
+        pdf.output(filename, "F")
+
     @abstractmethod
     async def process_assistant(self):
         pass
 
     async def send_output_by_email(
-        self, file: UploadFile, name: str, custom_message: str = None
+        self,
+        file: UploadFile,
+        filename: str,
+        task_name: str,
+        custom_message: str,
+        brain_id: str = None,
     ):
         settings = ContactsSettings()
         file = await self.uploadfile_to_file(file)
+        domain_quivr = os.getenv("QUIVR_DOMAIN", "https://chat.quivr.app/")
 
         with open(file.name, "rb") as f:
-
             mail_from = settings.resend_contact_sales_from
             mail_to = self.current_user.email
             body = f"""
-            <p>{custom_message}</p>
+            <div style="text-align: center;">
+                <img src="https://quivr-cms.s3.eu-west-3.amazonaws.com/logo_quivr_white_7e3c72620f.png" alt="Quivr Logo" style="width: 100px; height: 100px; border-radius: 50%; margin: 0 auto; display: block;">
+                
+                <p>Quivr's ingestion process has been completed. The processed file is attached.</p>
+                
+                <p><strong>Task:</strong> {task_name}</p>
+                
+                <p><strong>Output:</strong> {custom_message}</p>
+                <br />
+                
+
+            </div>
+            """
+            if brain_id:
+                body += f"<div style='text-align: center;'>You can find the file <a href='{domain_quivr}studio/{brain_id}'>here</a>.</div> <br />"
+            body += f"""
+            <div style="text-align: center;">
+                <p>Please let us know if you have any questions or need further assistance.</p>
+                
+                <p> The Quivr Team </p>
+            </div>
             """
             params = {
                 "from": mail_from,
                 "to": mail_to,
                 "subject": "Quivr Ingestion Processed",
                 "reply_to": "no-reply@quivr.app",
                 "html": body,
-                "attachments": [{"filename": name, "content": list(f.read())}],
+                "attachments": [{"filename": filename, "content": list(f.read())}],
             }
-            logger.info(f"Sending email to {mail_to} with file {name}")
+            logger.info(f"Sending email to {mail_to} with file {filename}")
             send_email(params)
 
     async def uploadfile_to_file(self, uploadFile: UploadFile):
@@ -96,29 +129,46 @@ async def create_and_upload_processed_file(
         self, processed_content: str, original_filename: str, file_description: str
     ) -> dict:
         """Handles creation and uploading of the processed file."""
-        content_io = BytesIO(processed_content.encode("utf-8"))
-        content_io.seek(0)
 
         new_filename = (
             original_filename.split(".")[0]
             + "_"
             + file_description.lower().replace(" ", "_")
             + "_"
             + str(random.randint(1000, 9999))
-            + ".txt"
+            + ".pdf"
         )
+        self.generate_pdf(
+            new_filename,
+            f"{file_description} of {original_filename}",
+            processed_content,
+        )
+
+        content_io = BytesIO()
+        with open(new_filename, "rb") as f:
+            content_io.write(f.read())
+        content_io.seek(0)
 
         file_to_upload = UploadFile(
             filename=new_filename,
             file=content_io,
-            headers={"content-type": "text/plain"},
+            headers={"content-type": "application/pdf"},
         )
 
         if self.input.outputs.email.activated:
             await self.send_output_by_email(
                 file_to_upload,
                 new_filename,
+                "Summary",
                 f"{file_description} of {original_filename}",
+                brain_id=(
+                    self.input.outputs.brain.value
+                    if (
+                        self.input.outputs.brain.activated
+                        and self.input.outputs.brain.value
+                    )
+                    else None
+                ),
             )
 
         # Reset to start of file before upload
@@ -131,4 +181,6 @@ async def create_and_upload_processed_file(
                 chat_id=None,
             )
 
+        os.remove(new_filename)
+
         return {"message": f"{file_description} generated successfully"}

backend/modules/assistant/ito/summary.py

@@ -98,7 +98,9 @@ async def process_assistant(self):
         reduce_template = """The following is set of summaries for each section of the document:
         {docs}
         Take these and distill it into a final, consolidated summary of the document. Make sure to include the main themes, key points, and important information.
-        Use markdown, headings, bullet points, or any other formatting to make the summary clear and easy to read.
+        Use markdown such as bold, italics, underlined. For example, **bold**, *italics*, and _underlined_ to highlight key points.
+        Please provide the final summary with sections using bold headers. 
+        Sections should be: a short summary of the document called summary, and a list of key points called key points.
         Summary:"""
         reduce_prompt = PromptTemplate.from_template(reduce_template)
 

backend/modules/assistant/ito/utils/pdf_generator.py

@@ -0,0 +1,67 @@
+import os
+
+from fpdf import FPDF
+from pydantic import BaseModel
+
+
+class PDFModel(BaseModel):
+    title: str
+    content: str
+
+
+class PDFGenerator(FPDF):
+    def __init__(self, pdf_model: PDFModel, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.pdf_model = pdf_model
+
+    def header(self):
+        # Logo
+        logo_path = os.path.join(os.path.dirname(__file__), "logo.png")
+        self.image(logo_path, 10, 10, 20)  # Adjust size as needed
+
+        # Move cursor to right of image
+        self.set_xy(20, 15)
+
+        # Title
+        self.set_font("Arial", "B", 12)
+        self.multi_cell(0, 10, self.pdf_model.title, align="C")
+        self.ln(5)  # Padding after title
+
+    def footer(self):
+        self.set_y(-15)
+        self.set_font("helvetica", "I", 8)
+        self.set_text_color(169, 169, 169)
+        self.cell(80, 10, "Generated by Quivr", 0, 0, "C")
+        self.set_font("helvetica", "U", 8)
+        self.set_text_color(0, 0, 255)
+        self.cell(30, 10, "quivr.app", 0, 0, "C", link="https://quivr.app")
+        self.cell(0, 10, "Github", 0, 1, "C", link="https://github.com/quivrhq/quivr")
+
+    def chapter_body(self):
+        self.set_font("Arial", "", 12)
+        self.multi_cell(0, 10, self.pdf_model.content, markdown=True)
+        self.ln()
+
+    def print_pdf(self):
+        self.add_page()
+        self.chapter_body()
+
+
+if __name__ == "__main__":
+    pdf_model = PDFModel(
+        title="Summary of Legal Services Rendered by Orrick",
+        content="""
+**Summary:** 
+The document is an invoice from Quivr Technologies, Inc. for legal services provided to client YC W24, related to initial corporate work. The total fees and disbursements amount to $8,345.00 for services rendered through February 29, 2024. The invoice includes specific instructions for payment remittance and contact information for inquiries. Online payment through e-billexpress.com is also an option.
+
+**Key Points:**
+- Quivr Technologies, Inc., based in France and represented by Stanislas Girard, provided legal services to client YC W24.
+- Services included preparing and completing forms, drafting instructions, reviewing and responding to emails, filing 83(b) elections, and finalizing documents for submission to YC.
+- The timekeepers involved in providing these services were Julien Barbey, Maria T. Coladonato, Michael LaBlanc, Jessy K. Parker, Marisol Sandoval Villasenor, Alexis A. Smith, and Serena Tibrewala.
+- The total hours billed for the services provided was 16.20, with a total cost of $8,345.00.
+- Instructions for payment remittance, contact information, and online payment options through e-billex
+""",
+    )
+    pdf = PDFGenerator(pdf_model)
+    pdf.print_pdf()
+    pdf.output("simple.pdf")

backend/requirements.txt

@@ -1,7 +1,7 @@
 -i https://pypi.org/simple
 # Ensure PyTorch, along with its required dependencies, is explicitly installed in 'CPU-only' mode to optimize compatibility and performance for systems without GPU support.
 --extra-index-url https://download.pytorch.org/whl/cpu
-aiohttp==3.9.4; python_version >= '3.8'
+aiohttp==3.9.5; python_version >= '3.8'
 aiosignal==1.3.1; python_version >= '3.7'
 amqp==5.2.0; python_version >= '3.6'
 annotated-types==0.6.0; python_version >= '3.8'
@@ -17,8 +17,8 @@ backoff==2.2.1; python_version >= '3.7' and python_version < '4.0'
 beautifulsoup4==4.12.3; python_full_version >= '3.6.0'
 billiard==4.2.0; python_version >= '3.7'
 black==24.4.0; python_version >= '3.8'
-boto3==1.34.84; python_version >= '3.8'
-botocore==1.34.84; python_version >= '3.8'
+boto3==1.34.85; python_version >= '3.8'
+botocore==1.34.85; python_version >= '3.8'
 celery[redis,sqs]==5.3.6; python_version >= '3.8'
 certifi==2024.2.2; python_version >= '3.6'
 cffi==1.16.0; platform_python_implementation != 'PyPy'
@@ -39,6 +39,7 @@ dataclasses-json-speakeasy==0.5.11; python_version >= '3.7' and python_version <
 datasets==2.18.0; python_full_version >= '3.8.0'
 debugpy==1.8.1; python_version >= '3.8'
 decorator==5.1.1; python_version >= '3.5'
+defusedxml==0.7.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 deprecated==1.2.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 deprecation==2.1.0
 dill==0.3.8; python_version >= '3.8'
@@ -62,6 +63,7 @@ flake8-black==0.3.6; python_version >= '3.7'
 flatbuffers==24.3.25
 flower==2.0.1; python_version >= '3.7'
 fonttools==4.51.0; python_version >= '3.8'
+fpdf2==2.7.8; python_version >= '3.7'
 frozenlist==1.4.1; python_version >= '3.8'
 fsspec[http]==2024.2.0; python_version >= '3.8'
 gitdb==4.0.11; python_version >= '3.7'
@@ -90,15 +92,15 @@ jsonpointer==2.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3
 kiwisolver==1.4.5; python_version >= '3.7'
 kombu[sqs]==5.3.7; python_version >= '3.8'
 langchain==0.1.16; python_version < '4.0' and python_full_version >= '3.8.1'
-langchain-community==0.0.32; python_version < '4.0' and python_full_version >= '3.8.1'
-langchain-core==0.1.42; python_version < '4.0' and python_full_version >= '3.8.1'
+langchain-community==0.0.33; python_version < '4.0' and python_full_version >= '3.8.1'
+langchain-core==0.1.43; python_version < '4.0' and python_full_version >= '3.8.1'
 langchain-openai==0.1.3; python_version < '4.0' and python_full_version >= '3.8.1'
 langchain-text-splitters==0.0.1; python_version < '4.0' and python_full_version >= '3.8.1'
 langdetect==1.0.9
-langfuse==2.26.1; python_version < '4.0' and python_full_version >= '3.8.1'
-langsmith==0.1.47; python_version < '4.0' and python_full_version >= '3.8.1'
+langfuse==2.26.2; python_version < '4.0' and python_full_version >= '3.8.1'
+langsmith==0.1.48; python_version < '4.0' and python_full_version >= '3.8.1'
 layoutparser[layoutmodels,tesseract]==0.3.4; python_version >= '3.6'
-litellm==1.35.5; python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7' and python_version >= '3.8'
+litellm==1.35.8; python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7' and python_version >= '3.8'
 llama-index==0.10.29; python_version < '4.0' and python_full_version >= '3.8.1'
 llama-index-agent-openai==0.2.2; python_version < '4.0' and python_full_version >= '3.8.1'
 llama-index-cli==0.1.11; python_version < '4.0' and python_full_version >= '3.8.1'
@@ -110,7 +112,7 @@ llama-index-llms-openai==0.1.15; python_version < '4.0' and python_full_version
 llama-index-multi-modal-llms-openai==0.1.5; python_version < '4.0' and python_full_version >= '3.8.1'
 llama-index-program-openai==0.1.5; python_version < '4.0' and python_full_version >= '3.8.1'
 llama-index-question-gen-openai==0.1.3; python_version < '4.0' and python_full_version >= '3.8.1'
-llama-index-readers-file==0.1.18; python_version < '4.0' and python_full_version >= '3.8.1'
+llama-index-readers-file==0.1.19; python_version < '4.0' and python_full_version >= '3.8.1'
 llama-index-readers-llama-parse==0.1.4; python_version < '4.0' and python_full_version >= '3.8.1'
 llama-parse==0.4.1; python_version < '4.0' and python_full_version >= '3.8.1'
 llamaindex-py-client==0.1.18; python_version >= '3.8' and python_version < '4'
@@ -137,18 +139,18 @@ olefile==0.47; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2,
 omegaconf==2.3.0; python_version >= '3.6'
 onnx==1.16.0
 onnxruntime==1.15.1
-openai==1.17.1; python_full_version >= '3.7.1'
+openai==1.20.0; python_full_version >= '3.7.1'
 opencv-python==4.9.0.80; python_version >= '3.6'
 openpyxl==3.1.2
-orjson==3.10.0; python_version >= '3.8'
+orjson==3.10.1; python_version >= '3.8'
 packaging==23.2; python_version >= '3.7'
 pandas==1.5.3; python_version >= '3.8'
-pandasai==2.0.32; python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8' and python_version >= '3.9'
+pandasai==2.0.33; python_version not in '2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8' and python_version >= '3.9'
 pathspec==0.12.1; python_version >= '3.8'
 pdf2image==1.17.0
 pdfminer.six==20231228
 pdfplumber==0.11.0; python_version >= '3.8'
-pikepdf==8.15.0
+pikepdf==8.15.1
 pillow==10.3.0; python_version >= '3.8'
 pillow-heif==0.16.0
 platformdirs==4.2.0; python_version >= '3.8'
@@ -200,14 +202,14 @@ ragas==0.1.7
 rapidfuzz==3.8.1; python_version >= '3.8'
 realtime==1.0.4; python_version >= '3.8' and python_version < '4.0'
 redis==5.0.3; python_version >= '3.7'
-regex==2023.12.25; python_version >= '3.7'
+regex==2024.4.16; python_version >= '3.7'
 requests==2.31.0; python_version >= '3.7'
 requests-file==2.0.0
 resend==0.8.0; python_version >= '3.7'
 retry==0.9.2
 rsa==4.9; python_version >= '3.6' and python_version < '4'
 s3transfer==0.10.1; python_version >= '3.8'
-safetensors==0.4.2; python_version >= '3.7'
+safetensors==0.4.3; python_version >= '3.7'
 scipy==1.13.0; python_version >= '3.9'
 sentry-sdk[fastapi]==1.45.0
 setuptools==69.5.1; python_version >= '3.8'