Merge pull request #22 from ls1intum/integrate-kb-manager

Integrate kb manager

.github/workflows/deploy.yml

@@ -106,8 +106,10 @@ jobs:
           proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }}
           proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }}
           script: |
-            rm /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
+            rm /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod || true
             touch /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
+      
+            # Add relevant environment variables
             echo "WEAVIATE_URL=${{ vars.WEAVIATE_URL }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
             echo "WEAVIATE_PORT=${{ vars.WEAVIATE_PORT }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
             echo "KNOWLEDGE_BASE_FOLDER=${{ vars.KNOWLEDGE_BASE_FOLDER }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
@@ -133,10 +135,6 @@ jobs:
             echo "COHERE_API_KEY_MULTI=${{ secrets.COHERE_API_KEY_MULTI }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
             echo "COHERE_API_KEY_EN=${{ secrets.COHERE_API_KEY_EN }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
             echo "ANGELOS_APP_API_KEY=${{ secrets.ANGELOS_APP_API_KEY }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
-            echo "API_ENDPOINT_KEY=${{ secrets.API_ENDPOINT_KEY }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
-            echo "EXPECTED_USERNAME=${{ secrets.EXPECTED_USERNAME }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
-            echo "EXPECTED_PASSWORD=${{ secrets.EXPECTED_PASSWORD }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
-            echo "WITHOUT_USER_LOGIN=${{ vars.WITHOUT_USER_LOGIN }}" >> /home/${{ vars.VM_USERNAME }}/${{ github.repository }}/.env.prod
 
 
       - name: SSH to VM and Execute Docker-Compose Up

app/api/admin_router.py

@@ -0,0 +1,21 @@
+import logging
+
+from fastapi import APIRouter, Depends
+
+from app.utils.dependencies import auth_handler, model
+from app.utils.environment import config
+
+
+admin_router = APIRouter(prefix="/api/admin", tags=["settings", "admin"],
+                         dependencies=[Depends(auth_handler.verify_token)])
+
+@admin_router.get("/ping")
+async def ping():
+    logging.info(config.GPU_URL)
+    return {"answer": "Server running."}
+
+
+@admin_router.get("/hi")
+async def ping():
+    logging.info("hi")
+    return model.complete([{"role": "user", "content": "Hi"}])

app/api/knowledge_router.py

@@ -0,0 +1,123 @@
+from typing import List
+from fastapi import HTTPException, APIRouter, status, Response, Depends
+from app.data.knowledge_base_requests import AddWebsiteRequest, EditDocumentRequest, EditSampleQuestionRequest, EditWebsiteRequest, AddDocumentRequest, AddSampleQuestionRequest, RefreshContentRequest
+from app.utils.dependencies import injestion_handler, auth_handler
+from app.data.database_requests import DatabaseDocumentMetadata
+
+knowledge_router = APIRouter(prefix="/api/knowledge", tags=["knowledge"])
+
+@knowledge_router.post("/website/add", dependencies=[Depends(auth_handler.verify_api_key)])
+async def add_website(body: AddWebsiteRequest):
+    try:
+        injestion_handler.add_website(body)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+@knowledge_router.post("/website/addBatch", dependencies=[Depends(auth_handler.verify_api_key)])
+async def add_websites(body: List[AddWebsiteRequest]):
+    try:
+        injestion_handler.add_websites(body)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+@knowledge_router.post("/website/{id}/refresh", dependencies=[Depends(auth_handler.verify_api_key)])
+async def refresh_website(id: str, body: RefreshContentRequest):
+    try:
+        injestion_handler.refresh_content(id=id, content=body.content)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+@knowledge_router.post("/website/{id}/update", dependencies=[Depends(auth_handler.verify_api_key)])
+async def update_website(id: str, body: EditWebsiteRequest):
+    try:
+        metadata: DatabaseDocumentMetadata = DatabaseDocumentMetadata(
+            study_programs=body.studyPrograms
+        )
+        injestion_handler.update_database_document(id=id, metadata=metadata)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+@knowledge_router.delete("/website/{id}/delete", dependencies=[Depends(auth_handler.verify_api_key)])
+async def delete_website(id: str):
+    try:
+        injestion_handler.delete_document(id=id)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+
+# === Document Endpoints ===
+
+@knowledge_router.post("/document/add", dependencies=[Depends(auth_handler.verify_api_key)])
+async def add_document(body: AddDocumentRequest):
+    try:
+        injestion_handler.add_document(body)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+@knowledge_router.post("/sample-question/addBatch", dependencies=[Depends(auth_handler.verify_api_key)])
+async def add_sample_questions(body: List[AddSampleQuestionRequest]):
+    try:
+        injestion_handler.add_sample_questions(body)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+@knowledge_router.post("/document/{id}/refresh", dependencies=[Depends(auth_handler.verify_api_key)])
+async def refresh_document(id: str, body: RefreshContentRequest):
+    try:
+        injestion_handler.refresh_content(id=id, content=body.content)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+@knowledge_router.post("/document/{id}/edit", dependencies=[Depends(auth_handler.verify_api_key)])
+async def edit_document(id: str, body: EditDocumentRequest):
+    try:
+        metadata: DatabaseDocumentMetadata = DatabaseDocumentMetadata(
+            study_programs=body.studyPrograms
+        )
+        injestion_handler.update_database_document(id=id, metadata=metadata)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+@knowledge_router.delete("/document/{id}/delete", dependencies=[Depends(auth_handler.verify_api_key)])
+async def delete_document(id: str):
+    try:
+        injestion_handler.delete_document(id=id)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+
+# === Sample Question Endpoints ===
+
+@knowledge_router.post("/sample-question/add", dependencies=[Depends(auth_handler.verify_api_key)])
+async def add_sample_question(body: AddSampleQuestionRequest):
+    try:
+        injestion_handler.add_sample_question(sample_question=body)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+@knowledge_router.post("/sample-question/{id}/edit", dependencies=[Depends(auth_handler.verify_api_key)])
+async def edit_sample_question(id: str, body: EditSampleQuestionRequest):
+    try:
+        injestion_handler.update_sample_question(kb_id=id, sample_question=body)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)
+
+@knowledge_router.delete("/sample-question/{id}/delete", dependencies=[Depends(auth_handler.verify_api_key)])
+async def delete_sample_question(id: str):
+    try:
+        injestion_handler.delete_sample_question(id=id)
+        return Response(status_code=200)
+    except Exception as e:
+        return Response(status_code=500)

app/api/question_router.py

@@ -1,35 +1,21 @@
 import logging
 
-from fastapi import HTTPException, APIRouter, status, Response, Depends
+from fastapi import HTTPException, APIRouter, Depends, Query
 
 from app.data.user_requests import UserChat, UserRequest
-from app.injestion.vector_store_initializer import initialize_vectorstores
-from app.managers.auth_handler import LoginRequest
-from app.utils.dependencies import request_handler, auth_handler, weaviate_manager, model
+from app.utils.dependencies import request_handler, auth_handler
 from app.utils.environment import config
 
-auth_router = APIRouter(prefix="/api", tags=["authorization"], dependencies=[Depends(auth_handler.verify_api_key)])
 question_router = APIRouter(prefix="/api/v1/question", tags=["response"])
-admin_router = APIRouter(prefix="/api/admin", tags=["settings", "admin"],
-                         dependencies=[Depends(auth_handler.verify_token)])
-
-
-@auth_router.post("/token")
-async def login(login_request: LoginRequest):
-    if config.WITHOUT_USER_LOGIN == "true" or (
-            login_request.username == config.EXPECTED_USERNAME and login_request.password == config.EXPECTED_PASSWORD):
-        token_data = {"sub": "angular_app"}
-        access_token = auth_handler.create_access_token(data=token_data)
-        return {"access_token": access_token, "token_type": "bearer"}
-    else:
-        raise HTTPException(status_code=401, detail="Invalid username or password")
 
 
 @question_router.post("/ask", tags=["email"], dependencies=[Depends(auth_handler.verify_api_key)])
 async def ask(request: UserRequest):
     question = request.message
     classification = request.study_program.lower()
     language = request.language.lower()
+    org_id = request.org_id
+
     if not question or not classification:
         raise HTTPException(status_code=400, detail="No question or classification provided")
 
@@ -44,7 +30,8 @@ async def ask(request: UserRequest):
     if config.TEST_MODE:
         answer, used_tokens, general_context, specific_context = request_handler.handle_question_test_mode(question,
                                                                                                            classification,
-                                                                                                           language)
+                                                                                                           language,
+                                                                                                           org_id=org_id)
         if language == "german":
             answer += "\n\n**Diese Antwort wurde automatisch generiert.**"
         else:
@@ -53,7 +40,7 @@ async def ask(request: UserRequest):
         return {"answer": answer, "used_tokens": used_tokens, "general_context": general_context,
                 "specific_context": specific_context}
     else:
-        answer = request_handler.handle_question(question, classification, language)
+        answer = request_handler.handle_question(question, classification, language, org_id=org_id)
         if language == "german":
             answer += "\n\n**Diese Antwort wurde automatisch generiert.**"
         else:
@@ -62,55 +49,22 @@ async def ask(request: UserRequest):
         return {"answer": answer}
 
 
-@question_router.post("/chat", tags=["chatbot"], dependencies=[Depends(auth_handler.verify_token)])
-async def chat(request: UserChat):
+@question_router.post("/chat", tags=["chatbot"], dependencies=[Depends(auth_handler.verify_api_key)])
+async def chat(
+    request: UserChat,
+    filterByOrg: bool = Query(..., description="Indicates whether to filter context by organization")
+):
     messages = request.messages
+    org_id = request.orgId
+
     if not messages:
         raise HTTPException(status_code=400, detail="No messages have been provided")
+
+    answer = request_handler.handle_chat(
+        messages,
+        study_program=request.study_program,
+        org_id=org_id,
+        filter_by_org=filterByOrg
+    )
 
-    last_message = messages[-1].message
-    if len(last_message) > config.MAX_MESSAGE_LENGTH:
-        raise HTTPException(
-            status_code=400,
-            detail=f"Message length exceeds the allowed limit of {config.MAX_MESSAGE_LENGTH} characters"
-        )
-
-    logging.info(f"Received messages.")
-    answer = request_handler.handle_chat(messages, study_program=request.study_program)
     return {"answer": answer}
-
-
-@admin_router.get("/initSchema",
-                  status_code=status.HTTP_202_ACCEPTED, )
-async def initializeDb():
-    initialize_vectorstores(config.KNOWLEDGE_BASE_FOLDER, config.QA_FOLDER, weaviate_manager)
-    return
-
-
-@admin_router.post("/document")
-async def add_document(request: UserRequest):
-    question = request.message
-    classification = request.study_program
-    if not question or not classification:
-        raise HTTPException(status_code=400, detail="No question or classification provided")
-
-    logging.info(f"Received document: {question} with classification: {classification}")
-    try:
-        request_handler.add_document(question, classification)
-        return Response(status_code=status.HTTP_200_OK)
-
-    except Exception as e:
-        logging.error(f"Failed to add document: {e}")
-        raise HTTPException(status_code=500, detail="Failed to add document")
-
-
-@admin_router.get("/ping")
-async def ping():
-    logging.info(config.GPU_URL)
-    return {"answer": "Server running."}
-
-
-@admin_router.get("/hi")
-async def ping():
-    logging.info("hi")
-    return model.complete([{"role": "user", "content": "Hi"}])

app/data/database_requests.py

@@ -0,0 +1,28 @@
+from pydantic import BaseModel
+from typing import List, Optional
+
+class DatabaseDocument(BaseModel):
+    id: str
+    link: Optional[str] = None
+    study_programs: List[str]
+    content: str
+    org_id: int
+
+class DatabaseDocumentMetadata(BaseModel):
+    link: Optional[str] = None
+    study_programs: List[str]
+    org_id: int
+
+class DatabaseSampleQuestion(BaseModel):
+    id: str
+    topic: str
+    question: str
+    answer: str
+    study_programs: List[str]
+    org_id: int
+
+class SampleQuestion(BaseModel):
+    topic: str
+    question: str
+    answer: str
+    study_programs: List[str]

app/data/knowledge_base_requests.py

@@ -0,0 +1,45 @@
+from pydantic import BaseModel
+from typing import List
+
+
+class AddWebsiteRequest(BaseModel):
+    id: str
+    orgId: int
+    title: str
+    link: str
+    studyPrograms: List[str]
+    content: str
+    type: str
+
+class RefreshContentRequest(BaseModel):
+    content: str
+
+class EditWebsiteRequest(BaseModel):
+    title: str
+    studyPrograms: List[str]
+
+class AddDocumentRequest(BaseModel):
+    id: str
+    orgId: int
+    title: str
+    studyPrograms: List[str]
+    content: str
+
+class EditDocumentRequest(BaseModel):
+    title: str
+    studyPrograms: List[str]
+
+class AddSampleQuestionRequest(BaseModel):
+    id: str
+    orgId: int
+    question: str
+    answer: str
+    topic: str
+    studyPrograms: List[str]
+
+class EditSampleQuestionRequest(BaseModel):
+    question: str
+    answer: str
+    topic: str
+    studyPrograms: List[str]
+    orgId: int

app/data/user_requests.py

@@ -8,20 +8,10 @@ class ChatMessage(BaseModel):
 class UserChat(BaseModel):
     messages: List[ChatMessage]
     study_program: Optional[str] = None
-
-class SampleQuestion(BaseModel):
-    topic: str
-    question: str
-    answer: str
-    study_program: str
-
-class WebsiteContent(BaseModel):
-    type: str
-    content: str
-    link: str
-    study_program: str
+    orgId: int
 
 class UserRequest(BaseModel):
+    org_id: int
     message: str
     study_program: str
     language: str