Adds VisionAgentV2 (#302)

* added AgentMessage type for conversations

* added VisionAgentV2

* added VisionAgentV2 to imports

* made timeout 3 minutes

* consolidated types and added AgentMessage to utils

* fixed owl_v2_image name in prompts

* fix flake8

* added abstract classes for coder and planner

* add message to agentmessage functionality

* added documentation

* add make file

* fix chat highlighting

* fix issues with app

* added readme for app

* add matplotlib to deps

* make local app work with conv agent

* update v2 actions, make max_steps argument

* don't add file names to non-user chats

* fix typo

* update action descriptions

examples/chat/README.md

@@ -0,0 +1,25 @@
+# Example App
+This is an example applicaiton to demonstrate how to run VisionAgent locally.
+
+
+# Quick Start
+To get started install the fastapi library:
+```bash
+pip install "fastapi[standard]"
+```
+
+Then cd into `chat-app` and run `make` to install the node dependencies.
+
+To start the server run:
+```bash
+fastapi dev app.py
+```
+
+And to start the front end, in the chat-app folder run:
+```bash
+npm run dev
+```
+
+Then go to `http://localhost:3000` in your browser to see the app running.
+
+![screenshot](https://github.com/landing-ai/vision-agent/blob/main/assets/screenshot.png?raw=true)

examples/chat/app.py

@@ -6,7 +6,8 @@
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
 
-from vision_agent.agent import VisionAgentCoderV2
+from vision_agent.agent import VisionAgentV2
+from vision_agent.agent.types import AgentMessage
 
 app = FastAPI()
 
@@ -36,7 +37,7 @@ def update_callback(message: Dict[str, Any]):
     loop.close()
 
 
-agent = VisionAgentCoderV2(
+agent = VisionAgentV2(
     verbose=True,
     update_callback=update_callback,
 )
@@ -46,7 +47,17 @@ def process_messages_background(messages: List[Dict[str, Any]]):
     for message in messages:
         if "media" in message and message["media"] is None:
             del message["media"]
-    response = agent.generate_code(messages)
+
+    response = agent.chat(
+        [
+            AgentMessage(
+                role=message["role"],
+                content=message["content"],
+                media=message.get("media", None),
+            )
+            for message in messages
+        ]
+    )
     # Here you can handle the response, e.g., send it via WebSocket or store it
     # For now, we'll just print it
     print("Background task completed:", response)

examples/chat/chat-app/Makefile

@@ -0,0 +1,18 @@
+.PHONY: all
+all: setup
+
+.PHONY: add-components
+add-components:
+	npx shadcn@latest add button card scroll-area tabs
+	npx shadcn@latest add collapsible
+
+# Target to install dependencies using package-lock.json
+.PHONY: install-dependencies
+install-dependencies:
+	npm install --legacy-peer-deps react-syntax-highlighter
+	npm config set legacy-peer-deps true
+	npm ci
+
+# Setup target to run all necessary commands
+.PHONY: setup
+setup: add-components install-dependencies

examples/chat/chat-app/package.json

@@ -10,7 +10,7 @@
   },
   "dependencies": {
     "@radix-ui/react-collapsible": "^1.1.1",
-    "@radix-ui/react-scroll-area": "^1.2.0",
+    "@radix-ui/react-scroll-area": "^1.2.1",
     "@radix-ui/react-slot": "^1.1.0",
     "@radix-ui/react-tabs": "^1.1.1",
     "class-variance-authority": "^0.7.0",