diff --git a/backend/app/models/user_task_execution_inputs_manager.py b/backend/app/models/user_task_execution_inputs_manager.py index cd07dc8f..c736925e 100644 --- a/backend/app/models/user_task_execution_inputs_manager.py +++ b/backend/app/models/user_task_execution_inputs_manager.py @@ -1,5 +1,6 @@ from mojodex_core.user_storage_manager.user_audio_file_manager import UserAudioFileManager from mojodex_core.user_storage_manager.user_images_file_manager import UserImagesFileManager +from mojodex_core.user_storage_manager.user_video_file_manager import UserVideoFileManager class UserTaskExecutionInputsManager: @@ -9,6 +10,7 @@ class UserTaskExecutionInputsManager: image_type = "image" multiple_images_type = "multiple_images" audio_file = "audio_file" + video_file = "video" def construct_inputs_from_request(self, user_task_execution_json_input_values, inputs, files, user_id, session_id): try: @@ -54,6 +56,10 @@ def construct_inputs_from_request(self, user_task_execution_json_input_values, i if input["input_name"] == file_input: filename = input["value"] UserAudioFileManager().store_audio_file_from_form(files[file_input], filename, user_id, session_id) + if input["type"] == self.video_file: + if input["input_name"] == file_input: + filename = input["value"] + UserVideoFileManager().store_video_file(files[file_input], filename, user_id, session_id) return user_task_execution_json_input_values except Exception as e: diff --git a/backend/app/routes/task.json b/backend/app/routes/task.json index 04e49671..9f73b2ae 100644 --- a/backend/app/routes/task.json +++ b/backend/app/routes/task.json @@ -105,7 +105,8 @@ "image", "drop_down_list", "multiple_images", - "audio_file" + "audio_file", + "video" ] }, "input_name": { @@ -348,7 +349,8 @@ "image", "drop_down_list", "multiple_images", - "audio_file" + "audio_file", + "video" ] }, "input_name": { diff --git a/docs/design-principles/products/whats_a_product.md b/docs/design-principles/products/whats_a_product.md index e45b46db..58b37b5c 100644 --- a/docs/design-principles/products/whats_a_product.md +++ b/docs/design-principles/products/whats_a_product.md @@ -76,6 +76,4 @@ Later, a user can: > Important note: A user can't have 2 active subscriptions at the same time. If a user is affected with a new subscription, the previous one is automatically cancelled. -2 payment services have been implemented for now: -- [Stripe](https://stripe.com) -- [Apple in-app purchase](https://developer.apple.com/in-app-purchase/) \ No newline at end of file +Payment service implemented: [Stripe](https://stripe.com) \ No newline at end of file diff --git a/docs/design-principles/to-do_list/how_todo_works.md b/docs/design-principles/to-do_list/how_todo_works.md index 3c1540fa..7f504e00 100644 --- a/docs/design-principles/to-do_list/how_todo_works.md +++ b/docs/design-principles/to-do_list/how_todo_works.md @@ -1,197 +1,3 @@ # How To-Do works? -To-dos are stored in the database. Typical technical workflow of a To-Do item is as follow: -1) extracted from a task execution -2) organized later if not achieved -3) reminded to the user -4) cleared by the user - -## 1. Extract To-Dos from achieved tasks -Mojodex's scheduler is a python module that triggers routes calls at a certain frequency. See `/scheduler/app/main.py` - -One of its trigger checks every 10 minutes if a task has just been achieved. -```python -from scheduled_tasks.extract_todos import ExtractTodos -[...] -ExtractTodos(600) # extract todos every 10 minutes -[...] -``` - -> A task is considered 'just achieved' if it has an associated produced_text which last version's date is between 10 and 20 minutes ago. See `backend/app/routes/extract_todos.py`. - -Each of those tasks are sent to the background through route `/extract_todos` for processing. -As a reminder, when a background route is called, the data is processed as follow: - -```python -class ExtractTodos(Resource): - [...] - def post(self): - [...] - extract_todos_cortex = ExtractTodosCortex(user_task_execution) - - def run_extract_todos_cortex(cortex): - try: - cortex.extract_todos() - except Exception as err: - print("🔮" + str(err)) - - executor.submit(run_extract_todos_cortex, extract_todos_cortex) - return {"success": "Process started"}, 200 - [...] -``` - -- A cortex (here `background/app/models/cortex/extract_todos_cortex.py`) is created. At construction, the cortex collects all database entries needed for further processing. This is done to avoid database connection out of main thread. - -- Then, the main function of the cortex, responsible for processing data (here `extract_todos()`) is called in an asynchronous way. - -The method extract_todos() uses prompt `data/prompts/background/todos/extract_todos.txt`– filled with all tasks data to extract To-Dos out of completed task. Notable instructions of this prompt are: - -- Extraction instruction: To define what is a To-Do. -``` -[...] -Extract any todo the user mentioned in the task as next steps they have to take. -Those todos describes actions the user will have to do in the future. They cannot be passive. -[...] -``` - -- Explicitely-mentioned only instruction: To avoid any hallucination from the agent. -``` -[...] -Extract ONLY next steps the user explicitly mentioned in the task. -[...] -``` - -- Assigned-only instruction: To avoid including To-Dos' that could be assigned to the user's contact in an email task or other participant mentioned in a meeting minutes, for example. -``` -[...] -Extract ONLY next steps assigned to the user. -[...] -``` - -The result of the prompt is a json list of dictionnary defining To-Do items. -```json -{ - "todo_definition": "", - "mentioned_as_todo": , - "due_date": "" -} -``` - -This json is parsed and items are added to the database, related to the task. - -![extract_todos](../../images/to-dos_flow/extract_todos.png) - -## 2. Organize -Another hourly trigger of the scheduler takes care of reorganizing user's To-Do list every night to keep it up-to-date. - -```python -from scheduled_tasks.reschedule_todos import RescheduleTodos -[...] -RescheduleTodos(3600) # reschedule todos every 1 hour -[...] -``` - -This trigger calls Mojodex's backend route `/todo_scheduling` to retrieve all To-Dos items that: -- Belongs to a user whose local time is between 1am and 2am -- Has not been deleted, nor completed -- Was due for the day before - -Each of those To-Dos are sent to the background using route `/reschedule_todos` for processing. - -Here, the route uses cortex `backend/app/models/cortex/reschedule_todo_cortex.py` to process the data. The main function of the cortex, responsible for processing data is `reschedule_todo`. - -It uses prompt `data/prompts/background/todos/reschedule_todo.txt` provided with: -- related task data -- To-Do item along with the number of times it has already been rescheduled and -- User's To-Do list in upcoming days - -``` -[...] -Regarding the TASK, TODO ITEM and USER TODO LIST, decide when to reschedule the TODO ITEM for later. -The task was currently scheduled for yesterday. -Provide the new scheduled date. -[...] -``` - -This prompt outputs a json answer that can be parsed so that a new scheduling can be added to database. - -![reschedule_todos](../../images/to-dos_flow/reschedule_todos.png) - -## 3. Remind the user -Here comes Mojodex's scheduler again with another hourly trigger. - -`/scheduler/app/main.py` -```python -from scheduled_tasks.purchase_expiration_checker import PurchasesExpirationChecker -[...] -emails = 'AWS_ACCESS_KEY_ID' in os.environ and os.environ['AWS_ACCESS_KEY_ID'] -if emails: - [...] - SendTodoDailyEmails(3600) # send todo daily emails every 1 hour (filtered by timezone) -[...] -``` - -> Note that this trigger is only activated if the environment variable `AWS_ACCESS_KEY_ID` is set. This variable is used to send emails through AWS SES, only emails mechanism implemented in Mojodex for now. - - -This triggers calls Mojodex's backend route `/todo_daily_emails` to retrieve all users whose local time is `DAILY_TODO_EMAIL_TIME` (defined in *env vars* see: `.env.example`). - - -For each of those users, the assistant will collect all To-Dos that are due for the coming day + the re-organization work it has done (cf step 4) and send those data to the background using route `events_generation` with parameter `'event_type': 'todo_daily_emails'`. - -The background uses its `TodoDailyEmailsGenerator` (`background/app/models/events/todo_daily_emails_generator.py`) with prompt `data/prompts/engagement/emails/todo_daily_emails_text_prompt.txt` to draft a friendly reminding emails to send to the user from provided data. - -Once an email is ready, the background sends it to the backend using route `/event` with parameters so that the backend sends the email to the user using AWS SES and logs it to the database. - -`backend/app/routes/event.py` -```python -[...] -mojo_mail_client.send_email(subject=subject, - recipients=[email], - html_body=body) -# add notification to db -email_event = MdEvent(creation_date=datetime.now(), event_type=event_type, - user_id=user_id, - message=message) -db.session.add(email_event) -db.session.commit() -[...] -``` - -![remind_user](../../images/to-dos_flow/remind_user.png) - -## 4. User actions -Users can of course also act on their own To-Dos. For now, they can take 2 actions: -- Delete a To-Do item, if it was not relevant to add it or the assistant made any mistake. As any application call, this call is made to the backend and the route is DELETE `/todo`. -> Note: an item is never deleted for real in the database. It is only marked as deleted so that it does not appear in the user's To-Do list anymore. This is to keep track of all the work the assistant has done. -`backend/app/routes/todo.py` -```python -class Todos(Resource): - [...] - def delete(self, user_id): - [...] - todo.deleted_by_user = datetime.now() - db.session.commit() - [...] - -``` - -- Mark a To-Do as completed as soon as they don't need it anymore to remember of the work they have to do. As any application call, this call is made to the backend and the route is POST `/todo`. -`backend/app/routes/todo.py` -```python -class Todos(Resource): - [...] - def post(self, user_id): - [...] - todo.completed = datetime.now() - db.session.commit() - [...] - -``` - - -![user_actions](../../images/to-dos_flow/user_actions.png) - -> Note: find every API specification in the [Backend API documentation](../../openAPI/backend_api.yaml) and the [Background API documentation](../../openAPI/background_api.yaml) \ No newline at end of file +TODO. \ No newline at end of file diff --git a/docs/design-principles/workflows/execute_workflow.md b/docs/design-principles/workflows/execute_workflow.md index e849e0fc..fd7a8bd7 100644 --- a/docs/design-principles/workflows/execute_workflow.md +++ b/docs/design-principles/workflows/execute_workflow.md @@ -117,7 +117,7 @@ The `WorkflowExecution` is the epicenter of workflow execution. The function `ru - Run current step - Ask for user validation once the step is executed -![start_user_workflow_execution_from_form](../../images/workflow_execution/start_user_workflow_execution_from_form.png) +![start_user_workflow_execution_from_form](../../images/task_execution/start_user_task_execution_from_form.png) > The Workflow Execution detailled flow is described in part 3. diff --git a/docs/images/md_task.png b/docs/images/md_task.png new file mode 100644 index 00000000..4fa32eb8 Binary files /dev/null and b/docs/images/md_task.png differ diff --git a/docs/images/md_task_predefined_action_association.png b/docs/images/md_task_predefined_action_association.png new file mode 100644 index 00000000..06ac418e Binary files /dev/null and b/docs/images/md_task_predefined_action_association.png differ diff --git a/docs/images/md_text_edit_action_text_type_association.png b/docs/images/md_text_edit_action_text_type_association.png new file mode 100644 index 00000000..ab90d2f7 Binary files /dev/null and b/docs/images/md_text_edit_action_text_type_association.png differ diff --git a/docs/images/md_todo.png b/docs/images/md_todo.png new file mode 100644 index 00000000..ed60017b Binary files /dev/null and b/docs/images/md_todo.png differ diff --git a/docs/images/md_user.png b/docs/images/md_user.png new file mode 100644 index 00000000..6a591094 Binary files /dev/null and b/docs/images/md_user.png differ diff --git a/docs/images/md_user_task_execution.png b/docs/images/md_user_task_execution.png new file mode 100644 index 00000000..9daf2344 Binary files /dev/null and b/docs/images/md_user_task_execution.png differ diff --git a/docs/technical-architecture/general-doc/image-1.png b/docs/technical-architecture/general-doc/image-1.png new file mode 100644 index 00000000..98c87d78 Binary files /dev/null and b/docs/technical-architecture/general-doc/image-1.png differ diff --git a/docs/technical-architecture/general-doc/image-10.png b/docs/technical-architecture/general-doc/image-10.png new file mode 100644 index 00000000..d039e855 Binary files /dev/null and b/docs/technical-architecture/general-doc/image-10.png differ diff --git a/docs/technical-architecture/general-doc/image-2.png b/docs/technical-architecture/general-doc/image-2.png new file mode 100644 index 00000000..28c30c0e Binary files /dev/null and b/docs/technical-architecture/general-doc/image-2.png differ diff --git a/docs/technical-architecture/general-doc/image-3.png b/docs/technical-architecture/general-doc/image-3.png new file mode 100644 index 00000000..78db0e48 Binary files /dev/null and b/docs/technical-architecture/general-doc/image-3.png differ diff --git a/docs/technical-architecture/general-doc/image-4.png b/docs/technical-architecture/general-doc/image-4.png new file mode 100644 index 00000000..2a4f6ea6 Binary files /dev/null and b/docs/technical-architecture/general-doc/image-4.png differ diff --git a/docs/technical-architecture/general-doc/image-5.png b/docs/technical-architecture/general-doc/image-5.png new file mode 100644 index 00000000..8a8d4642 Binary files /dev/null and b/docs/technical-architecture/general-doc/image-5.png differ diff --git a/docs/technical-architecture/general-doc/image-6.png b/docs/technical-architecture/general-doc/image-6.png new file mode 100644 index 00000000..c7ab18e6 Binary files /dev/null and b/docs/technical-architecture/general-doc/image-6.png differ diff --git a/docs/technical-architecture/general-doc/image-7.png b/docs/technical-architecture/general-doc/image-7.png new file mode 100644 index 00000000..ea77da90 Binary files /dev/null and b/docs/technical-architecture/general-doc/image-7.png differ diff --git a/docs/technical-architecture/general-doc/image-8.png b/docs/technical-architecture/general-doc/image-8.png new file mode 100644 index 00000000..78bb31a9 Binary files /dev/null and b/docs/technical-architecture/general-doc/image-8.png differ diff --git a/docs/technical-architecture/general-doc/image-9.png b/docs/technical-architecture/general-doc/image-9.png new file mode 100644 index 00000000..c4a65164 Binary files /dev/null and b/docs/technical-architecture/general-doc/image-9.png differ diff --git a/docs/technical-architecture/general-doc/image.png b/docs/technical-architecture/general-doc/image.png new file mode 100644 index 00000000..bb3b7646 Binary files /dev/null and b/docs/technical-architecture/general-doc/image.png differ diff --git a/docs/technical-architecture/general-doc/index.md b/docs/technical-architecture/general-doc/index.md new file mode 100644 index 00000000..6e316094 --- /dev/null +++ b/docs/technical-architecture/general-doc/index.md @@ -0,0 +1,810 @@ +# Mojodex Technical Documentation + +> What's this doc for? +> **High level understanding of the architecture of Mojodex and the relationship between its components.** +> *More detailed documentation is available in each components directory.* + +Mojodex is an open source tool, a platform for creating digital assistants. Designed around tasks, Mojodex enables developers and IT teams to create customised digital assistants for their staff in a simple way. The platform can be deployed in the cloud and works with both backend and frontend tools. Administrators can configure tasks for users, and there are several types of tasks and interfaces for using them. Mojodex allows companies to take ownership of the development of their AI tools. + +## 1. Mojodex technical architecture + +### Main components + +![../../images/architecture_overview.png](../../images/architecture_overview.png) + +- **Mobile application**: Developed in Flutter, it accesses the Python backend directly. +- **Web application** : Developed in React, it accesses the Python backend via a Next.js proxy. +- **CLI**: Command-line tool for developers. +- **Backend** : Manages real-time interactions and REST APIs. +- **Background**: Manages long and proactive tasks. +- **Scheduler**: triggers tasks at given times. +- **PostgreSQL database with PGVector**: Stores data and embeddings. + +### Communication flows + +- **REST routes**: Used for HTTP communications, shown in red. +- **Socket.io streams**: Used for real-time communications, shown in green. + +## 2. Backend component + +The backend is a Python application implemented with Flask to manage REST APIs and business logic. It uses SQLAlchemy to access the database and Flask-SocketIO for real-time interactions. + +### Routes and session management + +Routes are defined in a `HTTPRouteManager` class. The backend uses a `db` object provided by Flask-SQLAlchemy for interactions with the database. A `@with_db_session` decorator is used to handle database sessions outside of route calls, to avoid database locks. Models are defined in `mojodex_core/entities/db_base_entities.py` + +### Real-time interactions + +The backend uses Flask-SocketIO to manage real-time interactions, in particular for chat between the user and the assistant. Messages are sent via Socket.io rooms, separated by session ID. Authentication of Socket.io flows is managed in the same way as Flask authentication, using tokens. + +### Message management + +Messages between the backend and the frontend are managed via Socket.io. Important messages, such as Mojo messages and draft messages, use acknowledgement mechanisms to ensure that they are received by the client. + + +## 3. Background component + +The background is designed to manage long and proactive processes. It uses Flask to expose HTTP routes and a thread pool executor to launch processes in parallel. Processes are logged at admin level, and the background handles errors autonomously. + +#### Features + +Features implemented in the background include: +- Extracting and rescheduling to-dos. +- Creating and updating documents from a website. +- Event generation for push notifications and emails. + + +#### Database access + +The background has access to the database via SQLAlchemy and also uses the `@with_db_session` decorator to manage database sessions outside of route calls. + +### Thread management + +The background runs processes in separate threads to avoid blocking the main Flask server. Process results are logged, and the background handles errors autonomously. + + +## 4. PostgreSQL database and PGVector + +### Choice of PostgreSQL + +PostgreSQL was chosen for its flexibility and its ability to manage vector columns via the PGVector extension. Tests were carried out with other vector databases, but PostgreSQL proved sufficient for the needs of Mojodex. + +### Database schema + +The database schema is available on DB Diagram. Table names are prefixed with `md_` for Mojodex. A `create-mojodex-data.sql` script creates the database schema, and an `init-mojodex-data.sql` script initializes the data. + +> See section #23 for a high level overview of the database structure and organisation + +> Also see detailed database documentation here: [https://dbdiagram.io/d/MojodexDatabase-659d0645ac844320ae85b440](https://dbdiagram.io/d/MojodexDatabase-659d0645ac844320ae85b440) + +### Representing entities + +Database entities are represented in Python classes generated by SQLacodegen. These classes can be overloaded to add specific attributes. + +### Using JSON + +Some columns in the database use the JSON type to retain flexibility on certain fields, such as the `message` field in the `md_message` table. + +## 5. Scheduler +The Mojodex scheduler is designed to trigger actions at specific intervals. Currently, there is no implementation for triggering actions at a specific time of day. Actions are triggered according to the interval and the user's timezone. + +### How the scheduler works +The scheduler uses the Python `schedule` module to execute tasks at a specific frequency. These tasks are implemented in the scheduler main, which usually runs in a simple Python container. The scheduler can call either the backend for administrative tasks (such as updating users whose purchases have expired), or the background for proactive actions (such as sending notifications or retrieving tasks). + +## 6. Webapp +The web app gives users access to the Mojodex API. The front-end is developed in React and the proxy in Node.js. REST Flows pass through the proxy, while Socket.IO flows pass directly between the web app and the Python backend. + +### User authentication +Users can be authenticated via login/password or via providers (Google, Apple...). The authentication request arrives on the Next server (Node.js), is rerouted to the Python backend for verification, and the authentication token is sent back to the client on the front end for future calls. + + +## 7. Deployment management +### 7.1 Local deployment +To deploy Mojodex locally, there are 2 Docker Composes: a classic one and one for development (`docker-compose-dev.yml`). Each Docker Compose contains six Docker containers corresponding to different project directories, each with its own Dockerfile. + +- mojodex-db +- mojodex-backend +- mojodex-background +- mojodex-scheduler +- mojodex-webapp +- (optionnal) mojodex-ollama + + +#### Local deployment procedure +1. Clone the repo. +2. Copy and modify the `.env` file from `.env.example`. +3. Copy and modify the `.conf.example` files (models.conf, stt-models.conf, embeddings-models.conf). +4. Start the services with `docker-compose up -d --build`. +5. Access the web app on `localhost:3000` with the default user (email: demo@example.com, password: demo). + +### 7.2 Deployment on Azure +Mojodex can be deployed on Azure using services such as Container Registry, App Services, Container Instances, Storage Account, and a Postgres database server in flexible mode. + +#### Deployment procedure on Azure +1. Install Azure CLI and log in to Azure. +2. Update the template configuration files (.conf). +3. Use the `azure_deployment_script.sh` script to deploy services. +4. Restart the Azure services after deployment. +5. Access the web app via the URL provided by Azure. + +> Caution: Note that this script is intended to be used more as a ‘memory aid’ than as a script to be run all at once without supervision. + + +!!! note "" + 💡 For Azure users, you can find advanced deployment informations in the following resources: + + - [Deployment Guide with Azure](../../guides/azure-deployment.md) + + +## 8. Data management + +### Data types +1. **Database**: Contains all the business data required to run Mojodex. +2. **User files**: Stored in the `/data` directory and organised by session. +3. **Datasets**: Files created to use the data generated by Mojodex, in particular LLM calls. +4. **Cost files** : Consumption logs to analyse the costs and usage of the various services. + +### Data details +#### Database +- Stored locally in a Docker volume. +- In Azure mode, managed by the Azure Postgres server. + +#### User files +- Stored in `/data/users/{user_id}/{session_id}`. +- Contain audio files, audio messages from the user, and uploaded images and videos. + +#### Datasets +- Stored in `/data/prompt_datasets/{type}/{label}/{task_name}/{id}`. +- Contain LLM call parameters and generated responses. + +#### Cost files +- Stored as CSV files. +- Contain information on consumption by user, task and service. + + +## 9. Differences Between Instruct Tasks and Workflows +![alt text](image-2.png) + + +## 10. Task Creation + +### Instruct Tasks + +Instruct tasks are the simpler type of tasks in Mojodex. Here is the process to create an Instruct task: + +1. **Identify Tasks**: Identify the tasks that need to be created based on the job profiles within the company. +2. **Post-Task JSON Route**: Use the `POST /task_json` route to generate a preliminary JSON definition of the task. + - This route takes the task definition as a parameter and returns a JSON template. + - Ensure to review and adapt the JSON to fit your specific needs. +3. **JSON Schema**: The JSON format and the route are detailed in the OpenAPI documentation. +4. **Put-Task Route**: Once the JSON is ready, send it via the `PUT /task` route to create the task in the database. +5. **SQL Script Alternative**: Alternatively, you can write an SQL script to create tasks directly in the database, which can be useful for multi-environment deployments. + +### Workflow Tasks + +Workflow tasks involve a more complex process: + +1. **Create Workflow Directory**: In the `mojodex_core` module, under the `workflows` directory, create a new directory named after the workflow. +2. **Create Step Files**: For each step in the workflow, create a Python file named after the step. + - Each file should contain a class that extends `WorkflowStep` and overrides the `_execute` method. + - Ensure the input format of each step matches the output format of the previous step. +3. **Step Library**: Add each step to the `steps_library` file in the `mojodex_core/workflows` module, mapping step names to their corresponding classes. +4. **Workflow JSON**: Create a JSON definition for the workflow task, similar to Instruct tasks but including the steps with their names, descriptions, and order. +5. **Put-Task Route**: Send the workflow JSON via the `PUT /task` route. + +> Creating a task does not make it available to users. It must be associated with a product through the `md_product_task_association` table. +> For more information on how to associate users to tasks, see section #2 + + +## 11. Task Execution (`instruct`) + +### 11.1 User Interaction + +When a user wants to execute a task, the following steps occur: + +1. **Task Selection**: The user clicks on "New Task" and sees a list of available tasks. +2. **Task Initiation**: + - **Web App**: A form appears with fields that the user must fill out to start the task. These fields are mandatory. + - **Mobile App**: The user sends an initial message, typically an audio message, containing all necessary inputs for the task. +3. **Task Execution**: + - **Web App**: If all fields are filled correctly, the task starts immediately. + - **Mobile App**: The assistant may ask follow-up questions to gather any missing information before starting the task. + +### 11.2 Technical Description + +![docs/images/task_execution/complete_user_task_execution_flow.png](../../images/task_execution/complete_user_task_execution_flow.png) + +#### Step 1: Task Initialization + +- **Route**: `PUT /user_task_execution` +- **Action**: Creates an instance of `UserTaskExecution` with a `JsonInput` field initialized with null values. +- **Details**: + - A user is associated with multiple `UserTasks`. + - When a user starts a task, a new `UserTaskExecution` object is created. + - This route is called before any user inputs are provided. + +#### Step 2: Form Submission (Web App Only) + +- **Route**: `POST /user_task_execution_run` +- **Action**: Sets the start date of the `UserTaskExecution` and begins generating a response from the assistant. +- **Details**: + - The `SessionController` handles chat exchanges and processes form inputs to generate the assistant's response. + +#### Step 3: Message Handling (Mobile App) + +- **Route**: `PUT /user_message` +- **Action**: Sends a user message and ensures the associated `UserTaskExecution` is started. +- **Details**: + - The route verifies if the `UserTaskExecution` is started; if not, it starts it. + - Manages speech-to-text conversion and handles potential timeouts or crashes. + - Uses a frontend-managed message ID to re-query the route in case of crash that would let the route to answer without providing `message_pk`. + +#### Data Storage + +- **Initial User Inputs**: Stored in the `UserTaskExecution` table under `JsonInputValues`. +- **Messages**: Stored in the `md_message` table in a JSON field. +- **Produced Text Versions**: Stored in the `md_produced_text_version` table, containing titles and bodies of each version. + +## 12. Task Execution (`workflow`) + +### 121.1 User Interaction + +1. **Task Selection**: The user clicks on "New Task" and sees a list of available tasks. +2. **Task Initiation**: The user selects a workflow task and fills out the form. +3. **Workflow Execution**: + - The user is presented with a process tab detailing each step of the workflow. + - Steps can start and complete, with the user receiving updates via Socket.io events. + - Some steps may require user validation before proceeding to the next step. + - The user can interact with the assistant through buttons like `restart`, `edit`, and `review`. + +### 12.2 Technical Description + +![../../images/workflow_execution/complete_user_workflow_execution_flow.png](../../images/workflow_execution/complete_user_workflow_execution_flow.png) + +#### Step 1: Task Initialization + +- **Route**: `PUT /user_task_execution` +- **Action**: Creates an instance of `UserTaskExecution` with a `json_input` field initialized with null values. +- **Details**: + - Similar to `instruct` tasks, a new `UserTaskExecution` object is created when a user starts a task. + +#### Step 2: Form Submission + +- **Route**: `POST /user_task_execution_run` +- **Action**: Sets the start date of the `UserTaskExecution` and begins generating a response from the assistant. +- **Details**: + - The `SessionController` handles chat exchanges and processes form inputs to generate the assistant's response. + +#### Step 3: Step Execution + +- **Socket.io Events**: Used to update the web app when a step starts or completes. +- **User Validation**: Some steps require user validation, defined by a database field associated with the workflow step. +- **Buttons**: + - `restart`: Available only on the first step to modify initial inputs and relaunch the workflow. + - `edit`: Allows the user to modify the step result directly before validating it. + - `review`: Opens a chat for the user to send instructions to the assistant to re-execute the step. + +#### Data Storage + +- **Initial User Inputs**: Stored in the `UserTaskExecution` table under `json_input_values`. +- **Messages**: Stored in the `md_message` table in a JSON field. +- **Produced Text Versions**: Stored in the `md_produced_text_version` table, containing titles and bodies of each version. +- **File Storage**: Images and audio files are stored using the `UserFileStorageManager` in the volume data, identified by session and user. + +#### Workflow Specifics + +- **Step Validation**: Each step can be validated by the user before moving to the next step. +- **Final Produced Text**: Generated from the last step or concatenation of the last steps, with optional chat enabled based on a database field. + + + +### 12.3 Workflow Execution: Method `run()` + +When a user initiates a workflow, the following sequence of events occurs: + + +1. **Task Initialization**: + - The user clicks on the task they want to execute. + - The `PUT /user_task_execution` route is called to instantiate a new `UserTaskExecution`. + +2. **Form Submission**: + - After filling out the form and clicking "Go", the `PUT /user_task_execution_run` route is called. + - This starts the workflow execution in a parallel thread. + +3. **Workflow Execution**: + - In the parallel thread, the `WorkflowProcessController` method `run` is invoked. + - The `run` method determines the step to execute and executes it. + +4. **Step Execution**: + - If the step does not require user validation, the `run` method is called again to calculate and execute the next step. + - If user validation is required, information is sent via `Socket.io` to the front end, indicating that user validation is needed. + +5. **User Interaction**: + - If the user is online, they receive a live notification. + - If the user is offline, they can see the pending validation the next time they log in. + +6. **Step Validation**: + - The user can either validate the step or edit the result. + - Validation calls the `POST /user_workflow_step_execution` route, marking the step as valid and re-invoking the `run` method. + - Editing calls the `PUT /user_workflow_step_execution_result` route, updating the step result and re-invoking the `run` method. + +#### Detailed Execution Flow + +![alt text](image.png) + +- The `run` method identifies the current step, executes it, and produces a result. +- If the workflow execution has no title, it generates the title and summary in a parallel process. +- The method retrieves the step to execute. If the step is `null`, the workflow is complete, and the final result is generated. +- If the step is not `null`, it executes the step, generates a result, and sends an event indicating the step completion. +- Depending on whether user validation is required, the `run` method is called again or waits for user input. + +### 12.4 Determining the Next Step + +#### Step Determination Logic + +![alt text](image-1.png) + +1. **Initial Step**: + - If no step has been executed, the first step is executed. + +2. **Subsequent Steps**: + - If one step has been executed, the next step is determined based on its rank. + - If multiple steps have been executed, the number of items in the result list of the previous step is checked. + - The next step is executed for each item in the result list until all items are processed. + +#### Example: Poem Workflow +- **Step 1**: User inputs the poem subject and number of stanzas. +- **Step 2**: Generates subjects for each stanza. +- **Step 3**: Writes each stanza based on the generated subjects. +- The `run` method iterates through these steps, ensuring each step is executed the required number of times based on the results of the previous step. + +### 12.5 Chat and Step Validation + +#### Chat on Step Validation +- If a step result is unsatisfactory, the user can initiate a chat by clicking "Review". +- The `user_workflow_step_execution` route is called with `validated=false`. +- A "snapshot" of the workflow (= parameter and result of each past steps) is taken and stored in a system message. +- The user interacts with the assistant to provide instructions for improving the step result. +- The step is re-executed with the new instructions. + +#### Chat on Workflow Result +- At the end of the workflow, a snapshot of the entire workflow is taken and stored in a system message. +- If the user requests modifications to the final result, the assistant uses the context from the system message to make the necessary changes. + + + +## 13. SessionController Chat Functionality + +### Chat on Instruct Tasks and Workflows + +#### Instruct Tasks +- **Chat on Result**: Users can send instructions about the final result of an instruct task. + +#### Workflows +- **Chat on Steps**: Users can send instructions about individual steps if enabled. +- **Chat on Result**: Users can send instructions about the final result of the workflow. +- **API Call Considerations**: API calls cannot be repeated once the result is generated, enabling chat on workflow result could misguide users of hallucinated further API calls that would never happen. + + +### Entry Points + +The `SessionController` has two main entry points for handling chat interactions: + +1. **process_form_input()**: Called when the user clicks on "Go" on the WebApp in a parallel process by the route `POST /user_task_execution_run`. +2. **process_chat_message()**: Called when a message is received from the user via the `PUT /user_message` route. + +### `@user_input_processor` Decorator + +![alt text](image-3.png) + +Both entry points are decorated with the `@user_input_processor` decorator. This decorator ensures that the functions will call a `ChatAssistant`, return a `response_message`, and handle adding the response to the database and streaming it. + +### Differences between those methods + +- **process_form_input()**: This method is specific to instruct tasks and cannot be called outside the context of an instruct task. +- **process_chat_message()**: This method can be called in various contexts, including outside of a task, such as in the Home Chat context, or in workflow execution context. It includes additional checks to determine the context and handle it accordingly. + +## 14. Chat Assistant Functionality + +#### ChatAssistant Classes + +The `ChatAssistant` is an abstract class responsible for generating assistant responses. It has three implementations: + +1. **HomeChatAssistant** +2. **InstructTaskAssistant** +3. **WorkflowAssistant** + +#### Common Methods + +The `ChatAssistant` class defines several common methods: + +- **_call_llm()**: Makes a call to the language model. +- **_handle_llm_output()**: Handles the output from the language model, including extracting the language and tags. + +#### Abstract Methods + +Each `ChatAssistant` implementation must override the following abstract methods: + +- **generate_message()**: Generates the message specific to the assistant. +- **mpt()**: A property that returns the MPT (Mojodex Prompt Template) specific to the assistant. +- **_manage_response_tag()**: Manages the response tags specific to the assistant. +- **_token_callback()**: Handles the callback for tokens, which can vary between assistants. + +### 14.1 HomeChatAssistant + +![alt text](image-6.png) + +### 14.2 Instruct Task Assistant + +![alt text](image-4.png) + +### 14.3 Workflow Assistant + +![alt text](image-5.png) + +## 15. Home Chat Functionality + +### Overview +The Home Chat Assistant is a feature implemented in the mobile app but is currently not in active use. It is present in the main branch of the mobile app but has not been fully tested and lacks a defined use case. + +### Session Initialization +When a user navigates to the Home Chat page, a session is initiated, allowing the user to exchange messages with the assistant without selecting or starting any specific user task. + +### Task Execution +- The assistant can start an instruct task at any moment based on the user's natural conversation. +- For example, if a user mentions wanting to post on LinkedIn, the assistant can initiate the task to write a LinkedIn post. + +### Maintaining Task Execution Consistency +- The goal is to maintain consistency between tasks started by user clicks and those initiated by the assistant. +- This is managed by detecting during the assistant's response streaming (in the `_token_callback` function) whether a task has been started. + +### Token Callback `_token_callback` Function +- The partially generated text is analyzed to check for `` and `` tags on the fly. +- If these tags are detected, the value between them is extracted to determine the task context. +- If a `task_pk` is present, it indicates a task context; otherwise, it does not. + +### Task Decision Process +- If the assistant decides to start a task, it begins its response with ``, the task number, and ``. i.e. `12` +- The assistant has access to the list of tasks available to the user, including their PKs. +- The function checks if the tags are detected and processes accordingly: + - If `task_pk` is `null` and no task is running, the assistant continues with a general message. + - If `task_pk` is `null` but a task was running, it indicates the task has ended, and the running task is set to `false`. + - If a `task_pk` is detected and no task is running, it signals the start of a new task, and the function returns, stopping further token processing. + +![alt text](image-7.png) + +### Implementation Details +- The assistant's context changes when a task is running, affecting the prompt's templating. +- This way, by stopping the answer generation and adapting the prompt when a task starts, the prompt used to start a task by the assistant will be consistent with the one used when a user clicks to start a task. This ensures that both ways of initializing a task are using the same execution conditions. +- The assistant can switch tasks based on user instructions, maintaining flexibility in task management. + + +## 16. Proper Nouns Resolution with Mojodex + +### Overview +Proper noun resolution is crucial for improving user experience, especially in audio chats where correcting names repeatedly can be frustrating. + +### Tagging Proper Nouns +- Proper nouns are tagged in both assistant messages and user messages. +- For assistant messages, tagging proper nouns instruction is added directly in the prompt for mobile app users. +- For user messages, after transcription, the `POST /vocabulary` route is called to tag proper nouns. + +### Vocabulary Management +- When a user corrects a proper noun, the `PUT /vocabulary` route updates the spelling in all current session messages, task titles and summaries, produced_text last version. +- The corrected word is added to the user's vocabulary table if it doesn't already exist. + +### Whisper Integration +- Whisper, the speech-to-text model, is used to transcribe audio messages. +- Due to Whisper API limitation, only the most recent 20 words from the user's vocabulary are passed to Whisper as prompt to improve transcription accuracy. + +### Future Improvements +- The system may need to handle a large number of proper nouns over time, requiring an algorithm to manage context-specific information. +- The approach may change if a different speech-to-text model is used in the future, or Whisper API changes. + + +## 17. Placeholders Functionality + +### Overview +Placeholders were introduced to help developers avoid excessive token consumption during development, especially when testing UI elements. + +> Note: the recent raise of strong small language models that can run locally may lead to avoid using this feature, as it dramatically changes the overall behavior of the platform. + +### Usage in InstructTaskAssistant +- Placeholders are used to simulate responses without calling the LLM. +- In the `PUT /user_message` route, setting `use_message_placeholder` to `true` returns a "Hello World" response. +- Setting `use_draft_placeholder` to `true` returns a fake produced_text (draft message). + +### Usage and Limitations in WorkflowAssistant +- WorkflowTasks can generate various message types, making the current placeholder system less effective. +- Currently, placeholders can simulate `` and produced_text (draft message) but not other types like ``. + +### Usage and Limitations in HomeChatAssistant +- For the Home Chat Assistant, setting `use_message_placeholder` to `true` returns a "Hello World" message. +- Other types of messages can't be simulated using placeholders. + +### Future Considerations +- The need for placeholders may decrease with the availability of powerful open-source LLMs for local development. +- For WorkflowTasks, creating specific test data might be more practical than implementing a complex placeholder mechanism. + + +## 18. Document Management + +### Overview +The document management system in Mojodex was designed with future functionalities in mind, particularly for Retrieval-Augmented Generation (RAG), although RAG is not currently implemented. The system is equipped to handle documents and is integrated into the Mojodex web application. + +### Web App Integration +In the Mojodex web application, there is a "Resources" tab where users can add documents they wish to save. These documents can later be utilized in various Mojodex functionalities. Currently, users can only add web pages as documents. During the onboarding process, users have the option to add their company's website, which is then transformed into a user document. + + +> When dealing with tasks that require file inputs, the files are stored in the session for task execution and are not added to the document storage. The primary purpose of storing documents is to allow the assistant to dynamically use them to add contextual information during task execution or in a chat. + +### Document Processing +To transform a website into a document, the user provides a URL. The system then fetches all links related to the provided URL, up to a maximum of 100 links. Each of these URLs is scraped for its HTML content, which is then treated as a text document. The system only processes links directly related to the initial URL and does not follow links within those links. + +### Chunking Strategy +The document service in Mojodex Core, a singleton service, handles the addition of new documents through the `add_new_document()` method. This method naively chunks the document text based on token count, punctuation, and new lines. The chunking strategy is simple and may not be optimal for all use cases. The current method does not overlap chunks, meaning each character belongs to only one chunk. + +### Embedding and Storage +Once the text is split into chunks, each chunk is embedded and stored in the `md_document_chunk` table in the database. Each chunk is stored with its text, its order within the document, and its vector. The order is used to concatenate and recreate the document in the correct sequence for the user. + +### Document Retrieval +Although not currently used, the document service includes a method to retrieve documents based on vector distance calculations. This method takes a query, embeds it to form a vector, and uses the pgVector `cosine_distance()` method to find the nearest neighbors to the query. These nearest neighbors can then be used for various functionalities. + + + + +## 19. Mojodex Advanced Features: Predefined Actions + +### Overview +Predefined Actions in Mojodex allow for chaining Task Executions. This feature is currently available only on the mobile app. It enables the execution of a subsequent task using the result of the previous task, maintaining the same context. + +![alt text](image-8.png) + +### Implementation Details +- **Task Association**: Tasks that can be chained are defined at the task level. There is an association table called `md_task_predefined_action_association` that links two tasks. +- **User Task Execution**: When a user clicks on a predefined action, it triggers the creation and execution of a new User Task Execution. The user message for this new task is prefixed with a message defined in the Predefined Action, followed by the result of the previous User Task. +- **Tracking Chain**: Each User Task Execution that is launched from another task can be traced back using the attribute `predefined_action_from_user_task_execution_fk`, which indicates the originating task. + +![docs/technical-architecture/general-doc/md_task_predefined_action_association.png](../../images/md_task_predefined_action_association.png) + +### Developer Instructions +1. **Creating a Predefined Action**: + - First, create the subsequent task. + - Then, establish the association between the tasks. This can be done during the creation or editing of a task. + - Example: After a meeting summary, a follow-up email can be automatically triggered. + +### Common Use Case +- **Meeting Minutes Predefined action**: After a meeting, it is common to send a follow-up email. This feature allows for the automatic chaining of these tasks. + +## 20. Mojodex Advanced Features: Text-Edit Actions + +### Overview +Text-Edit Actions are shortcuts for editing the output of a task without sending a chat message. This feature is also currently available only on the mobile app and helps users discover the editing capabilities of Mojodex. + +![alt text](image-9.png) + +### Implementation Details +- **Text-Type Association**: Text-Edit Actions are defined at the text-type level. Each task output is associated with a text-type (e.g., document, email). +- **User Interaction**: When a user clicks on a Text-Edit Action, it generates a user message in the chat, which is then processed by calling the `text_edit_action` route instead of the classic `user_message` route. +- **Prompt Storage**: The prompt for each Text-Edit Action is stored in a file, and the path to this file is stored in the database. The file itself is typically a `.txt` file and is stored in `mojodex_core`. + +![docs/images/md_text_edit_action_text_type_association.png](../../images/md_text_edit_action_text_type_association.png) + +### Developer Instructions +1. **Defining Text-Edit Actions**: + - Associate the Text-Edit Actions with a text-type. + - Ensure that the prompt files are correctly stored and referenced in the database. + +### Common Use Case +- **Email Editing**: Users can quickly make an email less formal or shorter by using predefined Text-Edit Actions. + +## 21. Mojodex Advanced Features: Todo Management + +### Overview +Todo Management in Mojodex involves the extraction and scheduling of ToDo items based on user task executions. This process runs in the background and helps users keep track of their tasks. + +### Implementation Details +- **Todo Extraction**: + - Every 10 minutes, a scheduler triggers the extraction of todos from user task executions whose latest `md_produced_text_version` is between 10 and 20 minutes old. + - The assistant extracts actionable items mentioned by the user and adds them to the todo list, scheduling them for future dates. +- **Todo List Access**: + - Users can view their todo list in the todo tab of the mobile or web app, or directly from a user task execution. +- **Todo Execution**: + - Todos can be marked as done or deleted by the user. + - Replanning of todos is done by the assistant every hour for todos that are overdue. + +![docs/images/md_todo.png](../../images/md_todo.png) + +### Features +1. **Background Extraction**: + - The scheduler is set to run every 10 minutes to extract todos. + - The assistant is prompted to extract and reformulate todos from user task executions - only ToDo items explicitely mentioned as so by user. +2. **Todo Replanning**: + - The scheduler runs every hour to replan overdue todos. + - The assistant replans these todos and updates the database accordingly. +3. **Email Notifications**: + - Users with the attribute `todo_email_reception` set at true receive a daily email between 8 AM and 9 AM (weekdays only) with their todos for the day and any replanifications. + - The email service can be configured using SMTP or AWS. + +### Common Use Case +- **Meeting Follow-up Actions**: Extracting follow-up actions from meeting summaries and adding them to the todo list. + + +## 22. Administration and Configuration of the platform + +The goal of this section is to describe how to prepare and configure the Mojodex platform with *users*, *tasks* and *how tasks are made usable by users* + +### 22.1 User Creation + +There are two methods to create a new user: + +1. **User Self-Registration**: + - The user visits the web or mobile app and clicks on `Sign Up`. + - They fill out a form with their name, email, and password. + - After signing up, they go through an onboarding process: + 1. Accept Terms and Conditions. + 2. Choose a profile category, which determines the tasks they can access. + 3. Optionally, enter their company's website for additional context. + - The user's account is created and associated with a free trial product of the chosen category. + `Note: a free trial is particular product that is limited wether in terms of days of use or number of tasks to run AND it's free, i.e. product.is_free = true` + +2. **Admin-Created User**: + - An admin calls the `PUT /user` route with the `skip_user_validation=true` parameter. Terms and conditions are "validated" by default, onboarding is skipped. + - The user receives an email to change their default password. + - The admin must assign a role or purchase to the user to grant access to tasks. + +### 22.2 Product and Profile Management + +Products and profiles in Mojodex are used to group tasks and manage user access: + +1. **Product**: + - A product is a collection of tasks associated with a specific user profile. + - Products are grouped into categories (e.g., recruiters, engineers). + - Each product can have different task lists tailored to specific job (e.g., managers, interns). + +2. **Profile**: +`Products` and `Profiles` are represented with the same DB object: `md_product`. +`Profile` is a specialized representation of the `product` concept, better adapted to “admin all set config” where user has no control over the tasks they can access. A profile is: +- free +- has no limit +- has no Stripe ID + +3. **Subscription**: +`Products` and `Subscription` are represented with the same DB object: `md_product`. +A `Subscription` is a particular type of product that has no limit in terms of number of tasks run and duration. + + +### 22.3 Purchase and Role Management + +Purchases and roles link users to products and profiles: + +1. **Purchase**: + - A purchase is the association between a user and a product. + - When a user makes a new purchase, their task access is updated. + +2. **Role**: +`Purchases` and `Roles` are represented with the same DB object: `md_purchase`. +Role is better adapted to “admin all set config” where user has no control over the tasks they can access. +A `role` is making the relationship between a `user` and a `profile` which is a special `product` (as seen in the previous section). + + +### 22.4 Purchase and Subscription Management + +A user can't have 2 active purchases of a `Subscription` at the same time. +When purchasing a subscription, if user already has a `Subscription` purchase, this one will be deactivated before adding the new `purchase`. + +### 22.5 Database Entities Relationship + +![alt text](image-10.png) + +### 22.6 Procedure to create a new Product + +**Creating a New Product**: +1. Create a product category: `PUT /product_category` +2. Within the category, create products for different user types: `PUT /product` (or `PUT /profile`) +3. Associate tasks with each product: `PUT /product_task_association` +4. Link users to products via purchases. `PUT /manual_purchase` (or `PUT /role`) + +Each operation can be done using the appropriate backend routes. + + +## 23. Overview of Mojodex DB Tables + +Mojodex's database is composed of several tables, each serving a specific purpose. The main categories of tables include: + +- **Users**: Configuration of user accounts, profiles, etc. +- **Documents**: Configuration related to user documents and companies. +- **Tasks Configuration**: How tasks are configured. +- **Tasks Execution**: How users execute tasks. + +Access to the database documentation: +- https://dbdiagram.io/d/MojodexDatabase-659d0645ac844320ae85b440 +- https://dbdocs.io/kelly.roussel/Mojodex + +### 23.1 Users + +#### User Table + +The User table is the central table in Mojodex. It contains essential elements for user identification, configuration data, authentication information, foreign keys to configuration tables, company information, time zones, and links to product categories. The product category determines the list of products that can be purchased by the user. + +#### Product and Category Configuration + +Before adding users, products and categories must be configured. This involves defining categories (*i.e recruitement*) for a given deployment, then creating associated products (*i.e manager*). +Note: Each product category defines an implicit goal for the user, providing additional context for task execution. + +#### Purchase and Payment Systems + +A Purchase links a user to a product, validating their access. Mojodex, designed as a SaaS product, integrates with Stripe payment system. It also supports custom B2B relationships, allowing for invoice validation through a custom Purchase ID. + + +![docs/images/md_user.png](../../images/md_user.png) + +### 23.2 Documents + +#### Display Data + +Display data stores translation information, which is practical for backend flexibility despite adding some load to the database structure. + +#### User Documents + +Users are linked to documents like websites. These documents provide context information for task execution. + +#### Vocabulary + +User vocabulary stores proper names used by the user, enhancing the assistant's recognition capabilities. + +#### Events Management + +Events log notifications sent to users, including emails and push notifications. + +#### Devices and Notifications Management + +The Device table stores Firebase Messaging tokens for user devices, ensuring notifications are sent to all valid devices. Invalid tokens are marked as such to maintain accuracy. + +> Note: with this representation, if a user has multiple accounts on the same device, all notifications of all accounts will be sent to this device + +### 23.3 Tasks Configuration + +#### Task + +Tasks are the core of Mojodex, divided into two types: Instruct and Workflow – but same table `md_task`. Task configuration includes defining task types, names, system definitions, display information, output formats, and more. Tasks are associated with platforms (e.g., mobile, web) to control their availability. + +#### Product Tasks + +Product Tasks link products to tasks, determining the list of tasks available for a given product. + +#### Workflow and Steps + +Workflows consist of ordered steps defined in the `md_workflow_step` table. Each step includes information to process the workflow. + +#### Actions + +Tasks can have predefined actions linking them to other tasks, enabling task chaining. For example, after a meeting summary, a follow-up email task can be triggered. + +![](../../images/md_task.png) + +### 23.4 Tasks Executions + +#### Task Execution and Sessions + +Task execution involves the UserTaskExecution table, which links to user tasks, sessions, and purchases. Sessions manage all exchanges during task execution, including messages between the user and the assistant. + +#### Inputs and Messages + +Users can start tasks via forms or messages. Form inputs are stored in `user_task_execution.json_input.values`, while message-based inputs are managed through message exchanges. + +> Note for message-based execution: at the start of a task AND even in the context of a chat over an already existing user_task_execution, inputs are not re-extracted but concatenated in the run task template, as conversation history. + +#### Task Results and Produced Text Versions + +Task execution results are stored in ProducedTextVersion, with multiple versions possible due to user edits or text-edit actions. The most recent version is considered the final result. + +#### ToDos + +ToDos are extracted at the end of a UserTaskExecution and can have various statuses (e.g., completed, pending, deleted). They are associated with a schedule, which can be adjusted over time. + +#### Home Chat Interactions + +The Home Chat table manages chat interactions not necessarly linked to specific tasks, maintaining context for a week. This allows users to chat with an assistant about various topics, with interactions stored in sessions. + +![](../../images/md_user_task_execution.png) \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index a2fc1e13..00e6da6c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -66,11 +66,11 @@ nav: - Why Mojodex?: motivations.md - Docs: - Getting Started: getting-started.md - - Design Principles: + - Technical Architecture: technical-architecture/general-doc/index.md + - Design Principles: - Task-oriented design: - What's a task: design-principles/tasks/whats_a_task.md - Execute a task: design-principles/tasks/execute_task.md - - Background tasks: design-principles/tasks/background_task.md - Organized by products: - What's a product: design-principles/products/whats_a_product.md - How it works: design-principles/products/how_it_works.md @@ -82,7 +82,7 @@ nav: - Workflows: - What's a workflow: design-principles/workflows/whats_a_workflow.md - How it works: design-principles/workflows/execute_workflow.md - - Technical Architecture: + - Advanced Technical Architecture: - Overview: technical-architecture/overview.md - Backend: technical-architecture/backend.md - Database: technical-architecture/database.md @@ -95,19 +95,16 @@ nav: - Azure OpenAI Services: technical-architecture/llm-features/azure.md - Mistral AI: technical-architecture/llm-features/mistral.md - Mojodex Prompt Template: technical-architecture/llm-features/mpt.md - - Tools: - - Google Search: technical-architecture/existing_tools/google_search.md - - Internal Memory: technical-architecture/existing_tools/internal_memory.md - Integration: - Hubspot: technical-architecture/integration/hubspot.md - Build your own Digital Assistants: - Getting Started: guides/index.md + - Create a new user: guides/users/new_user.md - Configure a new product: guides/products/new_product.md - - Configure a new profile: guides/products/new_profile.md - - Configure a new task: guides/tasks/new_task.md - - New Task example: guides/tasks/example.md - - Add a new tool: guides/tools/new_tool.md - - Configure a new workflow: guides/workflows/new_workflow.md + - Configure a new profile: guides/profiles/new_profile.md + - Configure a new Instruct task: guides/tasks/new_task.md + - New Instruct Task example: guides/tasks/example.md + - Configure a new Workflow task: guides/workflows/new_workflow.md - Tutorial - Sales Assistant Example: guides/sales_assistant_example/index.md - Sales Assistant Team Guide: guides/sales_assistant_example/sales_assistant_scope_template.md diff --git a/mojodex_core/languages/en.json b/mojodex_core/languages/en.json index 54e8358e..5efac2f3 100644 --- a/mojodex_core/languages/en.json +++ b/mojodex_core/languages/en.json @@ -125,6 +125,7 @@ "restartError": "Error restarting workflow. Please retry or contact support.", "imageUploadButton": "Upload image", "audioUploadButton": "Upload audio file", + "videoUploadButton": "Upload video file", "missingFieldLabel": "Missing field" }, "chatTab": { diff --git a/mojodex_core/languages/es.json b/mojodex_core/languages/es.json index 15eed397..893847b3 100644 --- a/mojodex_core/languages/es.json +++ b/mojodex_core/languages/es.json @@ -125,6 +125,7 @@ "restartError": "Error al reiniciar el flujo de trabajo. Vuelva a intentarlo o pĂłngase en contacto con el servicio de asistencia.", "imageUploadButton": "Subir imagen", "audioUploadButton": "Subir audio", + "videoUploadButton": "Subir video", "missingFieldLabel": "Campo requerido" }, "chatTab": { diff --git a/mojodex_core/languages/fr.json b/mojodex_core/languages/fr.json index e98bcc3a..340a4942 100644 --- a/mojodex_core/languages/fr.json +++ b/mojodex_core/languages/fr.json @@ -125,6 +125,7 @@ "restartError": "Erreur lors du redĂ©marrage du flux de travail. Veuillez rĂ©essayer ou contacter le support.", "imageUploadButton": "TĂ©lĂ©charger une image", "audioUploadButton": "TĂ©lĂ©charger un fichier audio", + "videoUploadButton": "TĂ©lĂ©charger un fichier video", "missingFieldLabel": "Champ manquant" }, "chatTab": { diff --git a/mojodex_core/stt/openai_stt.py b/mojodex_core/stt/openai_stt.py index 1046468b..7c82d10d 100644 --- a/mojodex_core/stt/openai_stt.py +++ b/mojodex_core/stt/openai_stt.py @@ -1,3 +1,4 @@ +from datetime import timedelta from mojodex_core.costs_manager.whisper_costs_manager import WhisperCostsManager from openai import OpenAI, AzureOpenAI import tiktoken @@ -6,6 +7,8 @@ from mojodex_core.entities.user import User from mojodex_core.logging_handler import log_error from mojodex_core.stt.stt_engine import SttEngine +from mojodex_core.stt.transcription import TranscriptionWithTimeStamps + from pydub import AudioSegment from mojodex_core.user_storage_manager.user_audio_file_manager import UserAudioFileManager @@ -46,7 +49,19 @@ def _num_tokens_from_string(self, string): except Exception as e: raise Exception(f"_num_tokens_from_string: {e}") - def transcribe(self, audio_file_path, user_id, user_task_execution_pk=None, task_name_for_system=None): + def transcribe(self, audio_file_path, user_id, user_task_execution_pk=None, task_name_for_system=None) -> str: + try: + return self.__transcribe(audio_file_path, user_id, user_task_execution_pk, task_name_for_system, with_timestamps=False) + except Exception as e: + raise Exception(f"{self.__class__.__name__} :: transcribe: {e}") + + def transcribe_with_timestamp(self, filepath, user_id, user_task_execution_pk, task_name_for_system) -> list[TranscriptionWithTimeStamps]: + try: + return self.__transcribe(filepath, user_id, user_task_execution_pk, task_name_for_system, with_timestamps=True) + except Exception as e: + raise Exception(f"{self.__class__.__name__}: transcribe_with_timestamp: {e}") + + def __transcribe(self, audio_file_path, user_id, user_task_execution_pk=None, task_name_for_system=None, with_timestamps=False): try: try: vocab = self.__get_user_vocabulary(user_id) @@ -66,11 +81,12 @@ def transcribe(self, audio_file_path, user_id, user_task_execution_pk=None, task file_format = audio_file_path.split('.')[-1] file_name = audio_file_path.split('/')[-1].split('.')[0] audio_file_dir = os.path.dirname(audio_file_path) + transcriptions = [] + text_transcription = "" if audio.duration_seconds > ten_minutes_in_seconds: # split the file into 10 minutes chunks chunks = audio[::ten_minutes_in_seconds * 1000] - transcriptions = [] count = 0 for chunk in chunks: chunk_name = file_name + f"_chunk_{count}." + file_format @@ -79,25 +95,36 @@ def transcribe(self, audio_file_path, user_id, user_task_execution_pk=None, task transcription = self.client.audio.transcriptions.create( model=self.model, file=chunk_file, - prompt=vocab + prompt=vocab, + response_format="verbose_json" if with_timestamps else "text" ) - transcriptions.append(transcription.text) + if with_timestamps: + for t in transcription.segments: + transcriptions.append(TranscriptionWithTimeStamps(t['text'], timedelta(seconds=t['start'] + count * ten_minutes_in_seconds), timedelta(seconds=t['end'] + count * ten_minutes_in_seconds))) + else: + text_transcription += " " + transcription.text count += 1 - transcription = " ".join(transcriptions) else: with open(audio_file_path, "rb") as audio_file: transcription = self.client.audio.transcriptions.create( model=self.model, file=audio_file, - prompt=vocab - ).text + prompt=vocab, + response_format="verbose_json" if with_timestamps else "text" + ) + if with_timestamps: + for t in transcription.segments: + transcriptions.append(TranscriptionWithTimeStamps(t["text"], timedelta(seconds=t["start"]), timedelta(seconds=t["end"]))) + else: + text_transcription += " " + transcription.text whisper_costs_manager.on_seconds_counted(user_id=user_id, n_seconds=audio.duration_seconds, user_task_execution_pk=user_task_execution_pk, task_name_for_system=task_name_for_system, mode=self.model) - return transcription + return text_transcription if not with_timestamps else transcriptions except Exception as e: - raise Exception(f"{self.__class__.__name__} :: transcribe: {e}") + raise Exception(f"{self.__class__.__name__} :: __transcribe: {e}") + diff --git a/mojodex_core/stt/stt_engine.py b/mojodex_core/stt/stt_engine.py index ad98666b..62c0bc22 100644 --- a/mojodex_core/stt/stt_engine.py +++ b/mojodex_core/stt/stt_engine.py @@ -5,4 +5,9 @@ class SttEngine(ABC): @abstractmethod def transcribe(self, filepath, user_id, user_task_execution_pk, task_name_for_system): - raise NotImplementedError("transcribe method is not implemented") \ No newline at end of file + raise NotImplementedError("transcribe method is not implemented") + + + @abstractmethod + def transcribe_with_timestamp(self, filepath, user_id, user_task_execution_pk, task_name_for_system): + raise NotImplementedError("transcribe_with_timestamp method is not implemented") \ No newline at end of file diff --git a/mojodex_core/stt/stt_service.py b/mojodex_core/stt/stt_service.py index fa2524e8..6c9abdc1 100644 --- a/mojodex_core/stt/stt_service.py +++ b/mojodex_core/stt/stt_service.py @@ -2,6 +2,7 @@ import os from mojodex_core.logging_handler import MojodexCoreLogger from mojodex_core.stt.openai_stt import OpenAISTT +from mojodex_core.stt.transcription import TranscriptionWithTimeStamps from mojodex_core.stt.stt_engine import SttEngine from mojodex_core.user_storage_manager.user_audio_file_manager import UserAudioFileManager @@ -110,3 +111,14 @@ def transcribe(self, filepath, user_id, user_task_execution_pk, task_name_for_sy return transcription except Exception as e: raise Exception(f"{self.__class__.__name__} :: transcribe: {e}") + + def transcribe_with_timestamp(self, filepath, user_id, user_task_execution_pk, task_name_for_system) -> list[TranscriptionWithTimeStamps]: + try: + if not self.is_stt_configured: + raise Exception("STT engine is not configured") + if filepath is None: + raise Exception("filepath is None") + transcription = self.stt_engine.transcribe_with_timestamp(filepath, user_id, user_task_execution_pk, task_name_for_system) + return transcription + except Exception as e: + raise Exception(f"{self.__class__.__name__} :: transcribe_with_timestamp: {e}") diff --git a/mojodex_core/stt/transcription.py b/mojodex_core/stt/transcription.py new file mode 100644 index 00000000..c3712c6d --- /dev/null +++ b/mojodex_core/stt/transcription.py @@ -0,0 +1,23 @@ +from datetime import timedelta +import json + +class TranscriptionWithTimeStamps: + def __init__(self, text: str, start_time: timedelta, end_time: timedelta): + self.text = text + self.start_time = start_time + self.end_time = end_time + + def __str__(self) -> str: + start_ms = int(self.start_time.total_seconds() * 1000) + end_ms = int(self.end_time.total_seconds() * 1000) + return f"[{start_ms}ms-{end_ms}ms] {self.text}" + + def to_json(self) -> dict: + return { + "text": self.text, + "start_time": self.start_time.total_seconds(), + "end_time": self.end_time.total_seconds() + } + + def to_json_string(self) -> str: + return json.dumps(self.to_json(), indent=4) diff --git a/mojodex_core/user_storage_manager/user_video_file_manager.py b/mojodex_core/user_storage_manager/user_video_file_manager.py new file mode 100644 index 00000000..3c1f3e81 --- /dev/null +++ b/mojodex_core/user_storage_manager/user_video_file_manager.py @@ -0,0 +1,44 @@ + +import os + +from mojodex_core.user_storage_manager.user_storage_manager import UserStorageManager + + +class UserVideoFileManager(UserStorageManager): + _instance = None + _initialized = False + + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = super(UserVideoFileManager, cls).__new__( + cls, *args, **kwargs) + cls._instance._initialized = False + return cls._instance + + def get_videos_storage_path(self, user_id, session_id): + try: + session_storage = self._get_session_storage(user_id, session_id) + + videos_storage = os.path.join(session_storage, "videos") + + if not os.path.exists(videos_storage): + os.makedirs(videos_storage) + + return videos_storage + except Exception as e: + raise Exception(f"{self.__class__.__name__} :: get_videos_storage_path: {e}") + + def store_video_file(self, file, filename, user_id, session_id): + try: + videos_storage = self.get_videos_storage_path(user_id, session_id) + video_file_path = os.path.join(videos_storage, filename) + file.save(video_file_path) + except Exception as e: + raise Exception(f"{self.__class__.__name__} :: store_video_file: {e}") + + def get_video_file_path(self, video_name, user_id, session_id): + return os.path.join(self.get_videos_storage_path(user_id, session_id), video_name) + + def get_video_file_from_form_storage_path(self, video_name, user_id, session_id): + return os.path.join(self.get_videos_storage_path(user_id, session_id), video_name) + \ No newline at end of file diff --git a/mojodex_core/workflows/doc_gen_workflow/chapter.py b/mojodex_core/workflows/doc_gen_workflow/chapter.py new file mode 100644 index 00000000..d3bec244 --- /dev/null +++ b/mojodex_core/workflows/doc_gen_workflow/chapter.py @@ -0,0 +1,16 @@ +class Chapter: + def __init__(self, title : str, start : int, end : int): + self.title = title + self.start = start + self.end = end + self.content = None + + @classmethod + def from_input_string(cls, input_string : str): + # example string: 0 --> 1871: Introduction et objectifs de la session + parts = input_string.split(":") + title = " ".join(parts[1:]).strip() + + start, end = parts[0].split(" --> ") + + return cls(title, int(start), int(end)) diff --git a/mojodex_core/workflows/doc_gen_workflow/chapters_to_doc.mpt b/mojodex_core/workflows/doc_gen_workflow/chapters_to_doc.mpt new file mode 100644 index 00000000..884bfd41 --- /dev/null +++ b/mojodex_core/workflows/doc_gen_workflow/chapters_to_doc.mpt @@ -0,0 +1,31 @@ +#! gpt-4o +#! open-mistral-nemo-2407 + + +{{mojo_knowledge}} + +You are an expert developper, your goal is to write a documentation for other developers that want to dive deeper into the project. + + +{{user_datetime_context}} + +{{username}} + +This is the documentation chapters outline: + +{{chapters_outline}} + + +This is the transcription of a conversation between the two project creators: + +{{transcription}} + + + +Your task is to prepare a very detailed technical documentation of the project in Markdown format for developers. +You must exclusively use informations that you have in the 'transcription' but nothing else. +Write the documentation in english. +Be exhaustive. Be sure to use all the detailed informations of the 'transcription' in a structured and organized style. + + +No talk, just the documentation in english Markdown format. \ No newline at end of file diff --git a/mojodex_core/workflows/doc_gen_workflow/chapters_to_doc.py b/mojodex_core/workflows/doc_gen_workflow/chapters_to_doc.py new file mode 100644 index 00000000..8c55fc70 --- /dev/null +++ b/mojodex_core/workflows/doc_gen_workflow/chapters_to_doc.py @@ -0,0 +1,50 @@ +from mojodex_core.db import with_db_session +from mojodex_core.entities.workflow_step import WorkflowStep +from mojodex_core.llm_engine.mpt import MPT +from mojodex_core.workflows.doc_gen_workflow.chapter import Chapter + +from mojodex_core.knowledge_manager import KnowledgeManager +from mojodex_core.entities.user import User +class ChaptersToDocStep(WorkflowStep): + + chapters_to_doc_mpt_filename = "mojodex_core/workflows/doc_gen_workflow/chapters_to_doc.mpt" + + def _execute(self, parameter: dict, learned_instructions: dict, initial_parameters: dict, + past_validated_steps_results: list, user_id: str, user_task_execution_pk: int, + task_name_for_system: str, session_id: str): + try: + chapters = parameter['chapters'] + + + username, user_datetime_context = self._load_user_details(user_id) + + + + chapters_to_doc_mpt = MPT(ChaptersToDocStep.chapters_to_doc_mpt_filename, + transcription=past_validated_steps_results[0]['result'][0]['transcription'], + chapters_outline=chapters, + mojo_knowledge=KnowledgeManager().mojodex_knowledge, + user_datetime_context=user_datetime_context, + username=username) + + response = chapters_to_doc_mpt.run( + user_id=user_id, + temperature=0, + max_tokens=4000, + user_task_execution_pk=user_task_execution_pk, + task_name_for_system=task_name_for_system, + ) + + return [{'doc': response.strip()}] + except Exception as e: + raise Exception(f"{self.__class__.__name__} :: _execute: {e}") + + + @with_db_session + def _load_user_details(self, user_id, db_session): + try: + user: User = db_session.query(User).get(user_id) + + return user.name, user.datetime_context + except Exception as e: + raise Exception(f"_load_user_details :: {e}") \ No newline at end of file diff --git a/mojodex_core/workflows/doc_gen_workflow/doc_gen_workflow.json b/mojodex_core/workflows/doc_gen_workflow/doc_gen_workflow.json new file mode 100644 index 00000000..e1732102 --- /dev/null +++ b/mojodex_core/workflows/doc_gen_workflow/doc_gen_workflow.json @@ -0,0 +1,94 @@ +{"datetime": "2024-02-14T17:49:26.545180", + "task_type": "workflow", + "predefined_actions": [], + "name_for_system": "generate_documentation_from_video", + "icon": "📖", + "definition_for_system": "The user wants to generate a document from a video.", + "output_type": "document", + "platforms": ["webapp"], + "steps": [ + { + "name_for_system": "transcribe_recording", + "definition_for_system": "Transcribe the recording", + "rank": 1, + "review_chat_enabled": false, + "user_validation_required": false, + "step_displayed_data":[ + { + "language_code": "en", + "name_for_user": "Video transcription", + "definition_for_user": "Transcribe the video" + }, + { + "language_code": "fr", + "name_for_user": "Transcription de la vidĂ©o", + "definition_for_user": "Transcription de l'audio en texte" + } + + ]}, + { + "name_for_system": "transcription_to_chapters", + "definition_for_system": "Suggest the chapters outline from the transcription", + "rank": 2, + "review_chat_enabled": false, + "user_validation_required": true, + "step_displayed_data":[ + { + "language_code": "en", + "name_for_user": "Extract chapters", + "definition_for_user": "Extract the chapters from the transcription" + }, + { + "language_code": "fr", + "name_for_user": "Extraire les chapitres", + "definition_for_user": "Extraire les chapitres de la transcription" + } + + ]}, + { + "name_for_system": "chapters_to_doc", + "definition_for_system": "Write the document from the transcription based on the chapters outline", + "rank": 3, + "review_chat_enabled": false, + "user_validation_required": false, + "step_displayed_data":[ + { + "language_code": "en", + "name_for_user": "Write the document", + "definition_for_user": "Write the document based on the chapters outline" + }, + { + "language_code": "fr", + "name_for_user": "Écrire le document", + "definition_for_user": "Écrire le document basĂ© sur le plan des chapitres" + } + + ]} + + ], + "task_displayed_data": [ + { + "language_code":"en", + "name_for_user": "Generate documentation from video", + "definition_for_user": "Generate a document from a video", + "json_inputs": [ + {"input_name": "video_recording", + "description_for_user": "The video recording", + "description_for_system": "The video recording", + "type": "video"} + ] + }, + { + "language_code":"fr", + "name_for_user": "GĂ©nĂ©rer un document Ă  partir d'une vidĂ©o", + "definition_for_user": "GĂ©nĂ©rer un document Ă  partir d'une vidĂ©o", + "json_inputs": [ + {"input_name": "video_recording", + "description_for_user": "Enregistrement vidĂ©o", + "description_for_system": "Enregistrement vidĂ©o", + "type": "video"} + ] + } + ], + "result_chat_enabled": true +} \ No newline at end of file diff --git a/mojodex_core/workflows/doc_gen_workflow/extract_chapters_from_transcription.mpt b/mojodex_core/workflows/doc_gen_workflow/extract_chapters_from_transcription.mpt new file mode 100644 index 00000000..a165b68e --- /dev/null +++ b/mojodex_core/workflows/doc_gen_workflow/extract_chapters_from_transcription.mpt @@ -0,0 +1,38 @@ +#! gpt-4o + + +{{mojo_knowledge}} + + +{{user_datetime_context}} + +{{username}} + + + +This is the output format of a CHAPTERS OUTLINE: + --> : +[...] + --> : + +Example of CHAPTERS OUTLINE: +0 --> 124: Introduction to Machine Learning +124 --> 237: Machine Learning Foundations +237 --> 12309: Applied Machine Learning +12309 --> 14562: Use Cases of Machine Learning in Industry + + + + +Your task is to use the 'recording_transcription' to prepare a CHAPTERS OUTLINE in output format. +A chapter is a sequence of transcriptions that consistently talks about a specific topic with a reasonably good quality and quantity of information +You must exclusively use informations that you have in the 'recording_transcription' and nothing else. +Write the CHAPTERS OUTLINE in the same language as the 'recording_transcription' + + +This is the transcription of the meeting: + +{{transcription}} + + +No talk, just the CHAPTERS OUTLINE in correct output format. \ No newline at end of file diff --git a/mojodex_core/workflows/doc_gen_workflow/transcribe_recording.py b/mojodex_core/workflows/doc_gen_workflow/transcribe_recording.py new file mode 100644 index 00000000..a4d7a239 --- /dev/null +++ b/mojodex_core/workflows/doc_gen_workflow/transcribe_recording.py @@ -0,0 +1,32 @@ + +from typing import List +from mojodex_core.entities.workflow_step import WorkflowStep +from mojodex_core.stt.stt_service import STTService +from mojodex_core.user_storage_manager.user_video_file_manager import UserVideoFileManager + +class TranscribeRecordingStep(WorkflowStep): + + + def _execute(self, parameter: dict, learned_instructions: dict, initial_parameters: dict, + past_validated_steps_results: List[dict], user_id: str, user_task_execution_pk: int, + task_name_for_system: str, session_id: str): + try: + """ This step includes: + 1. Loading the recording from the filestorage + 2. Transcribing the content of the recording + 3. Dividing the transcription into chapters + 4. Returning the chapters as a list of strings including the timestamp of the beginning of each chapter in seconds from the start. + """ + + recording_filepath = UserVideoFileManager().get_video_file_from_form_storage_path(initial_parameters['video_recording'], user_id, session_id) + + # Transcribe the audio from STT + transcription_list = STTService().transcribe_with_timestamp(recording_filepath, user_id, user_task_execution_pk, task_name_for_system) + transcription = "" + + for t in transcription_list: + transcription += str(t.start_time.seconds) + " --> " + str(t.end_time.seconds) + ": " + t.text + "\n" + + return [{'transcription': transcription}] + except Exception as e: + raise Exception(f"{self.__class__.__name__} :: _execute: {e}") \ No newline at end of file diff --git a/mojodex_core/workflows/doc_gen_workflow/transcription_to_chapters.py b/mojodex_core/workflows/doc_gen_workflow/transcription_to_chapters.py new file mode 100644 index 00000000..59e5097e --- /dev/null +++ b/mojodex_core/workflows/doc_gen_workflow/transcription_to_chapters.py @@ -0,0 +1,50 @@ + + +from typing import List +from mojodex_core.db import with_db_session +from mojodex_core.llm_engine.mpt import MPT +from mojodex_core.knowledge_manager import KnowledgeManager +from mojodex_core.entities.user import User +from mojodex_core.entities.workflow_step import WorkflowStep + + +class TranscriptionToChaptersStep(WorkflowStep): + + extract_chapters_mpt_filename = "mojodex_core/workflows/doc_gen_workflow/extract_chapters_from_transcription.mpt" + + def _execute(self, parameter: dict, learned_instructions: dict, initial_parameters: dict, past_validated_steps_results: List[dict], user_id: str, user_task_execution_pk: int, task_name_for_system: str, session_id: str): + try: + transcription = parameter['transcription'] + + username, user_datetime_context = self._load_user_details(user_id) + + extract_chapters_mpt = MPT(TranscriptionToChaptersStep.extract_chapters_mpt_filename, + transcription=transcription, + mojo_knowledge=KnowledgeManager().mojodex_knowledge, + user_datetime_context=user_datetime_context, + username=username, + ) + response = extract_chapters_mpt.run( + user_id=user_id, + temperature=0, + max_tokens=2000, + user_task_execution_pk=user_task_execution_pk, + task_name_for_system=task_name_for_system, + ) + + chapters = response.strip() + + return [{'chapters': chapters}] + except Exception as e: + raise Exception(f"{self.__class__.__name__} :: _execute: {e}") + + + + @with_db_session + def _load_user_details(self, user_id, db_session): + try: + user: User = db_session.query(User).get(user_id) + + return user.name, user.datetime_context + except Exception as e: + raise Exception(f"_load_user_details :: {e}") \ No newline at end of file diff --git a/mojodex_core/workflows/steps_library.py b/mojodex_core/workflows/steps_library.py index 81d941f9..cf7e5b24 100644 --- a/mojodex_core/workflows/steps_library.py +++ b/mojodex_core/workflows/steps_library.py @@ -2,13 +2,18 @@ from mojodex_core.workflows.web_research_synthesis.write_synthesis_note import WriteSynthesisNoteStep from mojodex_core.workflows.write_poem.divide_in_stanza import StanzaDividerStep from mojodex_core.workflows.write_poem.write_stanza import StanzaWriterStep - +from mojodex_core.workflows.doc_gen_workflow.chapters_to_doc import ChaptersToDocStep +from mojodex_core.workflows.doc_gen_workflow.transcription_to_chapters import TranscriptionToChaptersStep +from mojodex_core.workflows.doc_gen_workflow.transcribe_recording import TranscribeRecordingStep steps_class = { "stanza_divider": StanzaDividerStep, "stanza_writer": StanzaWriterStep, "search_sources": SearchSourcesStep, - "write_synthesis_note": WriteSynthesisNoteStep + "write_synthesis_note": WriteSynthesisNoteStep, + "transcribe_recording": TranscribeRecordingStep, + "transcription_to_chapters": TranscriptionToChaptersStep, + "chapters_to_doc": ChaptersToDocStep, } diff --git a/webapp/modules/Tasks/components/TaskExecutionDetail/TaskInputs.tsx b/webapp/modules/Tasks/components/TaskExecutionDetail/TaskInputs.tsx index 641da98e..7319dab9 100644 --- a/webapp/modules/Tasks/components/TaskExecutionDetail/TaskInputs.tsx +++ b/webapp/modules/Tasks/components/TaskExecutionDetail/TaskInputs.tsx @@ -7,7 +7,7 @@ import ImagePreview from "../ImagePreview"; import useOnWorkflowRestart from "modules/Tasks/hooks/useOnWorkflowRestart"; import { useTranslation } from "next-i18next"; import useAlert from "helpers/hooks/useAlert"; -import { FaFileAudio } from "react-icons/fa"; +import { FaFileAudio, FaFileVideo } from "react-icons/fa"; interface TaskInputsProps { user_task_execution_pk: number; @@ -110,6 +110,12 @@ const TaskInputs: FunctionComponent = ({ user_task_execution_pk

{input.value}

+ case TaskJsonInputType.VIDEO: + return <> +

{input.input_name}

+

{input.value}

+ + default: return <>

{input.input_name}

diff --git a/webapp/modules/Tasks/components/TaskForm/InputsForm.tsx b/webapp/modules/Tasks/components/TaskForm/InputsForm.tsx index a73e32d4..cd9d972d 100644 --- a/webapp/modules/Tasks/components/TaskForm/InputsForm.tsx +++ b/webapp/modules/Tasks/components/TaskForm/InputsForm.tsx @@ -5,6 +5,7 @@ import ImageArea from "./ImageArea"; import Textarea from "./TextArea"; import MultipleImagesArea from "./MultipleImagesArea"; import AudioFile from "./AudioFile"; +import VideoFile from "./VideoFile"; interface InputsFormProps { jsonInputs: TaskJsonInput[]; @@ -45,6 +46,14 @@ const InputsForm = ({ jsonInputs, setInputArray, sessionId }: InputsFormProps) = setInputArray={setInputArray} /> ); + case TaskJsonInputType.VIDEO: + return ( + + ); case TaskJsonInputType.DROP_DOWN_LIST: return ( >; +} + +const VideoFile = ({ jsonInput, setInputArray }: VideoFileProps) => { + const { input_name, description_for_user } = jsonInput; + const [videoFilePreview, setVideoFilePreview] = useState(null); + const fileInputRef = useRef(null); + const { t } = useTranslation('dynamic'); + + const handleVideoFileChange = (event: React.ChangeEvent) => { + const file = event.target.files ? event.target.files[0] : null; + + if (file && file.type.startsWith('video/')) { + setVideoFilePreview(file.name); + setInputArray((prev) => { + const updatedInputArray = [...prev]; + const existingIndex = updatedInputArray.findIndex( + (input) => input.input_name === input_name + ); + if (existingIndex === -1) { + updatedInputArray.push({ + input_name, + input_value: file, + }); + } else { + updatedInputArray[existingIndex] = { + input_name, + input_value: file, + }; + } + return updatedInputArray; + }); + } + }; + + return ( +
+ +
+
+ + +
+
+
+ ); +}; + +export default VideoFile; \ No newline at end of file diff --git a/webapp/modules/Tasks/interface/index.ts b/webapp/modules/Tasks/interface/index.ts index 838cfce2..d7667dce 100644 --- a/webapp/modules/Tasks/interface/index.ts +++ b/webapp/modules/Tasks/interface/index.ts @@ -50,7 +50,8 @@ export enum TaskJsonInputType { IMAGE = "image", MULTIPLE_IMAGES = "multiple_images", DROP_DOWN_LIST = "drop_down_list", - AUDIO_FILE = "audio_file" + AUDIO_FILE = "audio_file", + VIDEO = "video" } export interface TaskJsonInput { diff --git a/webapp/public/locales/en/dynamic.json b/webapp/public/locales/en/dynamic.json index 43c09982..06ba9016 100644 --- a/webapp/public/locales/en/dynamic.json +++ b/webapp/public/locales/en/dynamic.json @@ -125,6 +125,7 @@ "restartError": "Error restarting workflow. Please retry or contact support.", "imageUploadButton": "Upload image", "audioUploadButton": "Upload audio file", + "videoUploadButton": "Upload video file", "missingFieldLabel": "Missing field" }, "chatTab": { diff --git a/webapp/public/locales/es/dynamic.json b/webapp/public/locales/es/dynamic.json index 5f00a185..de66f996 100644 --- a/webapp/public/locales/es/dynamic.json +++ b/webapp/public/locales/es/dynamic.json @@ -125,6 +125,7 @@ "restartError": "Error al reiniciar el flujo de trabajo. Vuelva a intentarlo o póngase en contacto con el servicio de asistencia.", "imageUploadButton": "Subir imagen", "audioUploadButton": "Subir audio", + "videoUploadButton": "Subir video", "missingFieldLabel": "Campo requerido" }, "chatTab": { diff --git a/webapp/public/locales/fr/dynamic.json b/webapp/public/locales/fr/dynamic.json index 4c91db76..3898afe5 100644 --- a/webapp/public/locales/fr/dynamic.json +++ b/webapp/public/locales/fr/dynamic.json @@ -124,7 +124,7 @@ "saveEditionButton": "Sauvegarder et redémarrer", "restartError": "Erreur lors du redémarrage du flux de travail. Veuillez réessayer ou contacter le support.", "imageUploadButton": "Télécharger une image", - "audioUploadButton": "Télécharger un fichier audio", + "videoUploadButton": "Télécharger une vidéo", "missingFieldLabel": "Champ manquant" }, "chatTab": {