Skip to content

Commit bc9cbd4

Browse files
authored
Add support for web asset rendering and virtual file system (#55)
<!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Session workspace file management: create, read, update (append/replace), delete, and list files. * In-page sandboxed web apps: render full-screen apps, capture form data, view/remove rendered app code, and re-render from results. * **Improvements** * Agents (Search, Research, Content Writer, Web Task, Action) coordinate via persistent session files; orchestrator exposes these tools and recommends reading/appending artifacts. * **Tests** * Added unit and integration tests covering new file tools, storage manager, and webapp code viewer. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent e6d2fda commit bc9cbd4

29 files changed

+3128
-22
lines changed

config/gni/devtools_grd_files.gni

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -656,6 +656,7 @@ grd_files_bundled_sources = [
656656
"front_end/panels/ai_chat/ui/mcp/MCPConnectionsDialog.js",
657657
"front_end/panels/ai_chat/ui/mcp/MCPConnectorsCatalogDialog.js",
658658
"front_end/panels/ai_chat/ui/EvaluationDialog.js",
659+
"front_end/panels/ai_chat/ui/WebAppCodeViewer.js",
659660
"front_end/panels/ai_chat/core/AgentService.js",
660661
"front_end/panels/ai_chat/core/State.js",
661662
"front_end/panels/ai_chat/core/Graph.js",
@@ -676,6 +677,7 @@ grd_files_bundled_sources = [
676677
"front_end/panels/ai_chat/core/AgentDescriptorRegistry.js",
677678
"front_end/panels/ai_chat/core/Version.js",
678679
"front_end/panels/ai_chat/core/VersionChecker.js",
680+
"front_end/panels/ai_chat/core/LLMConfigurationManager.js",
679681
"front_end/panels/ai_chat/LLM/LLMTypes.js",
680682
"front_end/panels/ai_chat/LLM/LLMProvider.js",
681683
"front_end/panels/ai_chat/LLM/LLMProviderRegistry.js",
@@ -702,6 +704,15 @@ grd_files_bundled_sources = [
702704
"front_end/panels/ai_chat/tools/BookmarkStoreTool.js",
703705
"front_end/panels/ai_chat/tools/DocumentSearchTool.js",
704706
"front_end/panels/ai_chat/tools/ThinkingTool.js",
707+
"front_end/panels/ai_chat/tools/RenderWebAppTool.js",
708+
"front_end/panels/ai_chat/tools/GetWebAppDataTool.js",
709+
"front_end/panels/ai_chat/tools/RemoveWebAppTool.js",
710+
"front_end/panels/ai_chat/tools/FileStorageManager.js",
711+
"front_end/panels/ai_chat/tools/CreateFileTool.js",
712+
"front_end/panels/ai_chat/tools/UpdateFileTool.js",
713+
"front_end/panels/ai_chat/tools/DeleteFileTool.js",
714+
"front_end/panels/ai_chat/tools/ReadFileTool.js",
715+
"front_end/panels/ai_chat/tools/ListFilesTool.js",
705716
"front_end/panels/ai_chat/common/utils.js",
706717
"front_end/panels/ai_chat/common/log.js",
707718
"front_end/panels/ai_chat/common/context.js",

front_end/panels/ai_chat/BUILD.gn

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ devtools_module("ai_chat") {
4242
"ui/SettingsDialog.ts",
4343
"ui/PromptEditDialog.ts",
4444
"ui/EvaluationDialog.ts",
45+
"ui/WebAppCodeViewer.ts",
4546
"ai_chat_impl.ts",
4647
"models/ChatTypes.ts",
4748
"core/Graph.ts",
@@ -91,8 +92,17 @@ devtools_module("ai_chat") {
9192
"tools/VectorDBClient.ts",
9293
"tools/BookmarkStoreTool.ts",
9394
"tools/DocumentSearchTool.ts",
95+
"tools/FileStorageManager.ts",
96+
"tools/CreateFileTool.ts",
97+
"tools/UpdateFileTool.ts",
98+
"tools/DeleteFileTool.ts",
99+
"tools/ReadFileTool.ts",
100+
"tools/ListFilesTool.ts",
94101
"tools/SequentialThinkingTool.ts",
95102
"tools/ThinkingTool.ts",
103+
"tools/RenderWebAppTool.ts",
104+
"tools/GetWebAppDataTool.ts",
105+
"tools/RemoveWebAppTool.ts",
96106
"agent_framework/ConfigurableAgentTool.ts",
97107
"agent_framework/AgentRunner.ts",
98108
"agent_framework/AgentRunnerEventBus.ts",
@@ -197,6 +207,7 @@ _ai_chat_sources = [
197207
"ui/PromptEditDialog.ts",
198208
"ui/SettingsDialog.ts",
199209
"ui/EvaluationDialog.ts",
210+
"ui/WebAppCodeViewer.ts",
200211
"ui/mcp/MCPConnectionsDialog.ts",
201212
"ui/mcp/MCPConnectorsCatalogDialog.ts",
202213
"ai_chat_impl.ts",
@@ -248,8 +259,17 @@ _ai_chat_sources = [
248259
"tools/VectorDBClient.ts",
249260
"tools/BookmarkStoreTool.ts",
250261
"tools/DocumentSearchTool.ts",
262+
"tools/FileStorageManager.ts",
263+
"tools/CreateFileTool.ts",
264+
"tools/UpdateFileTool.ts",
265+
"tools/DeleteFileTool.ts",
266+
"tools/ReadFileTool.ts",
267+
"tools/ListFilesTool.ts",
251268
"tools/SequentialThinkingTool.ts",
252269
"tools/ThinkingTool.ts",
270+
"tools/RenderWebAppTool.ts",
271+
"tools/GetWebAppDataTool.ts",
272+
"tools/RemoveWebAppTool.ts",
253273
"agent_framework/ConfigurableAgentTool.ts",
254274
"agent_framework/AgentRunner.ts",
255275
"agent_framework/AgentRunnerEventBus.ts",
@@ -393,6 +413,7 @@ ts_library("unittests") {
393413
"ui/__tests__/ChatViewSequentialSessionsTransition.test.ts",
394414
"ui/__tests__/ChatViewInputClear.test.ts",
395415
"ui/__tests__/SettingsDialogOpenRouterCache.test.ts",
416+
"ui/__tests__/WebAppCodeViewer.test.ts",
396417
"ui/input/__tests__/InputBarClear.test.ts",
397418
"ui/message/__tests__/MessageCombiner.test.ts",
398419
"ui/message/__tests__/StructuredResponseController.test.ts",
@@ -413,6 +434,11 @@ ts_library("unittests") {
413434
"ui/__tests__/AIChatPanel.test.ts",
414435
"ui/__tests__/LiveAgentSessionComponent.test.ts",
415436
"ui/message/__tests__/MessageList.test.ts",
437+
"tools/__tests__/CreateFileTool.test.ts",
438+
"tools/__tests__/UpdateFileTool.test.ts",
439+
"tools/__tests__/ReadFileTool.test.ts",
440+
"tools/__tests__/ListFilesTool.test.ts",
441+
"tools/__tests__/FileStorageManager.test.ts",
416442
]
417443

418444
deps = [

front_end/panels/ai_chat/agent_framework/implementation/ConfiguredAgents.ts

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import { SchemaBasedExtractorTool } from '../../tools/SchemaBasedExtractorTool.j
88
import { StreamlinedSchemaExtractorTool } from '../../tools/StreamlinedSchemaExtractorTool.js';
99
import { BookmarkStoreTool } from '../../tools/BookmarkStoreTool.js';
1010
import { DocumentSearchTool } from '../../tools/DocumentSearchTool.js';
11-
import { NavigateURLTool, PerformActionTool, GetAccessibilityTreeTool, SearchContentTool, NavigateBackTool, NodeIDsToURLsTool, TakeScreenshotTool, ScrollPageTool, WaitTool } from '../../tools/Tools.js';
11+
import { NavigateURLTool, PerformActionTool, GetAccessibilityTreeTool, SearchContentTool, NavigateBackTool, NodeIDsToURLsTool, TakeScreenshotTool, ScrollPageTool, WaitTool, RenderWebAppTool, GetWebAppDataTool, RemoveWebAppTool, CreateFileTool, UpdateFileTool, DeleteFileTool, ReadFileTool, ListFilesTool } from '../../tools/Tools.js';
1212
import { HTMLToMarkdownTool } from '../../tools/HTMLToMarkdownTool.js';
1313
import { ConfigurableAgentTool, ToolRegistry } from '../ConfigurableAgentTool.js';
1414
import { ThinkingTool } from '../../tools/ThinkingTool.js';
@@ -49,7 +49,17 @@ export function initializeConfiguredAgents(): void {
4949
ToolRegistry.registerToolFactory('scroll_page', () => new ScrollPageTool());
5050
ToolRegistry.registerToolFactory('wait_for_page_load', () => new WaitTool());
5151
ToolRegistry.registerToolFactory('thinking', () => new ThinkingTool());
52-
52+
ToolRegistry.registerToolFactory('create_file', () => new CreateFileTool());
53+
ToolRegistry.registerToolFactory('update_file', () => new UpdateFileTool());
54+
ToolRegistry.registerToolFactory('delete_file', () => new DeleteFileTool());
55+
ToolRegistry.registerToolFactory('read_file', () => new ReadFileTool());
56+
ToolRegistry.registerToolFactory('list_files', () => new ListFilesTool());
57+
58+
// Register webapp rendering tools
59+
ToolRegistry.registerToolFactory('render_webapp', () => new RenderWebAppTool());
60+
ToolRegistry.registerToolFactory('get_webapp_data', () => new GetWebAppDataTool());
61+
ToolRegistry.registerToolFactory('remove_webapp', () => new RemoveWebAppTool());
62+
5363
// Register bookmark and document search tools
5464
ToolRegistry.registerToolFactory('bookmark_store', () => new BookmarkStoreTool());
5565
ToolRegistry.registerToolFactory('document_search', () => new DocumentSearchTool());

front_end/panels/ai_chat/agent_framework/implementation/agents/ActionAgent.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,14 @@ Conclusion: Fix the args format and retry with proper syntax: { "method": "fill"
9191
'node_ids_to_urls',
9292
'scroll_page',
9393
'take_screenshot',
94+
'render_webapp',
95+
'get_webapp_data',
96+
'remove_webapp',
97+
'create_file',
98+
'update_file',
99+
'delete_file',
100+
'read_file',
101+
'list_files',
94102
],
95103
maxIterations: 10,
96104
modelName: MODEL_SENTINELS.USE_MINI,

front_end/panels/ai_chat/agent_framework/implementation/agents/ContentWriterAgent.ts

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,12 @@ You are specifically designed to collaborate with the research_agent. When you r
2525
- Collected research data, which may include web content, extractions, analysis, and other information
2626
- Your job is to organize this information into a comprehensive, well-structured report
2727
28+
Use the session file workspace as your shared knowledge base:
29+
- Immediately call 'list_files' to discover research artifacts (notes, structured datasets, outstanding questions) created earlier in the session.
30+
- Read the relevant files before outlining to understand what has already been captured, current confidence levels, and any gaps that remain.
31+
- If the handoff references specific files, open them with 'read_file' and incorporate their contents, citing source filenames or URLs when appropriate.
32+
- Persist your outline, intermediate synthesis, and final report with 'create_file'/'update_file' so future revisions or downstream agents can reuse the material.
33+
2834
Your process should follow these steps:
2935
1. Carefully analyze all the research data provided during the handoff
3036
2. Identify key themes, findings, and important information from the data
@@ -49,7 +55,13 @@ Your process should follow these steps:
4955
10. **References**: Properly formatted citations for all sources used
5056
5157
The final output should be in markdown format, and it should be lengthy and detailed. Aim for 5-10 pages of content, at least 1000 words.`,
52-
tools: [],
58+
tools: [
59+
'read_file',
60+
'list_files',
61+
'create_file',
62+
'update_file',
63+
'delete_file',
64+
],
5365
maxIterations: 3,
5466
modelName: MODEL_SENTINELS.USE_MINI,
5567
temperature: 0.3,

front_end/panels/ai_chat/agent_framework/implementation/agents/ResearchAgent.ts

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,13 +56,20 @@ First, think through the task thoroughly:
5656
- **html_to_markdown**: Use when you need high-quality page text in addition to (not instead of) structured extractions.
5757
- **fetcher_tool**: BATCH PROCESS multiple URLs at once - accepts an array of URLs to save tool calls
5858
59+
### 3. Workspace Coordination
60+
- Treat the file management tools as your shared scratchpad with other agents in the session.
61+
- Start each iteration by calling 'list_files' and 'read_file' on any artifacts relevant to your task so you understand existing progress.
62+
- Persist work products incrementally with 'create_file'/'update_file'. Use descriptive names (e.g. 'research/<topic>-sources.json') and include agent name, timestamp, query used, and quality notes so others can audit or extend the work.
63+
- Append to existing files when adding new findings; only delete files if they are obsolete AND all valuable information is captured elsewhere.
64+
- Record open questions or follow-ups in dedicated tracking files so parallel subtasks avoid duplicating effort.
65+
5966
**CRITICAL - Batch URL Fetching**:
6067
- The fetcher_tool accepts an ARRAY of URLs: {urls: [url1, url2, url3], reasoning: "..."}
6168
- ALWAYS batch multiple URLs together instead of calling fetcher_tool multiple times
6269
- Example: After extracting 5 URLs from search results, call fetcher_tool ONCE with all 5 URLs
6370
- This dramatically reduces tool calls and improves efficiency
6471
65-
### 3. Research Loop (OODA)
72+
### 4. Research Loop (OODA)
6673
Execute an excellent Observe-Orient-Decide-Act loop:
6774
6875
**Observe**: What information has been gathered? What's still needed?
@@ -83,7 +90,7 @@ Execute an excellent Observe-Orient-Decide-Act loop:
8390
- NEVER repeat the same query - adapt based on findings
8491
- If hitting diminishing returns, complete the task immediately
8592
86-
### 4. Source Quality Evaluation
93+
### 5. Source Quality Evaluation
8794
Think critically about sources:
8895
- Distinguish facts from speculation (watch for "could", "may", future tense)
8996
- Identify problematic sources (aggregators vs. originals, unconfirmed reports)
@@ -143,7 +150,9 @@ When your research is complete:
143150
3. The handoff tool expects: {query: "research topic", reasoning: "explanation for user"}
144151
4. The content_writer_agent will create the final report from your research data
145152
146-
Remember: You gather data, content_writer_agent writes the report. Always hand off when research is complete.`,
153+
Remember: You gather data, content_writer_agent writes the report. Always hand off when research is complete.
154+
155+
Before handing off, ensure your latest findings are reflected in the shared files (e.g. summaries, raw notes, structured datasets). This enables the orchestrator and content writer to understand what has been completed, reuse your artifacts, and avoid redundant rework.`,
147156
tools: [
148157
'navigate_url',
149158
'navigate_back',
@@ -152,7 +161,12 @@ Remember: You gather data, content_writer_agent writes the report. Always hand o
152161
'node_ids_to_urls',
153162
'bookmark_store',
154163
'document_search',
155-
'html_to_markdown'
164+
'html_to_markdown',
165+
'create_file',
166+
'update_file',
167+
'delete_file',
168+
'read_file',
169+
'list_files',
156170
],
157171
maxIterations: 15,
158172
modelName: MODEL_SENTINELS.USE_MINI,

front_end/panels/ai_chat/agent_framework/implementation/agents/SearchAgent.ts

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ export function createSearchAgentConfig(): AgentToolConfig {
2323
## Operating Principles
2424
- Stay laser-focused on the requested objective; avoid broad reports or narrative summaries.
2525
- Work fast but carefully: prioritize high-signal queries, follow source leads, and stop once the objective is satisfied with high confidence.
26+
- Use the session file workspace to coordinate: list existing files before launching new queries, read relevant artifacts, record harvested leads or verified results with 'create_file'/'update_file', and append incremental progress instead of creating overlapping files.
2627
- Never fabricate data. Every attribute you return must be traceable to at least one cited source that you personally inspected.
2728
2829
## Search Workflow
@@ -32,6 +33,7 @@ export function createSearchAgentConfig(): AgentToolConfig {
3233
- Use navigate_url to reach the most relevant search entry point (search engines, directories, LinkedIn public results, company pages, press releases).
3334
- Use extract_data with an explicit JSON schema every time you capture structured search results. Prefer capturing multiple leads in one call.
3435
- Batch follow-up pages with fetcher_tool, and use html_to_markdown when you need to confirm context inside long documents.
36+
- After each significant batch of new leads or fetcher_tool response, immediately persist the harvested candidates (including query, timestamp, and confidence notes) by appending to a coordination file via 'create_file'/'update_file'. This keeps other subtasks aligned and prevents redundant scraping.
3537
4. **Mandatory Pagination Loop (ENFORCED)**:
3638
- Harvest target per task: collect 30–50 unique candidates before enrichment (unless the user specifies otherwise). Absolute minimum 25 when the request requires it.
3739
- If current unique candidates < target, you MUST navigate to additional result pages and continue extraction.
@@ -47,6 +49,7 @@ export function createSearchAgentConfig(): AgentToolConfig {
4749
5. **Verify**:
4850
- Cross-check critical attributes (e.g. confirm an email’s domain matches the company, confirm a title with two independent sources when possible).
4951
- Flag low-confidence findings explicitly in the output.
52+
- Document verification status in the appropriate coordination file so other agents can see what has been confirmed and which leads still require attention.
5053
6. **Decide completeness**: Stop once required attributes are filled for the requested number of entities or additional searching would be duplicative.
5154
5255
## Tooling Rules
@@ -57,6 +60,7 @@ export function createSearchAgentConfig(): AgentToolConfig {
5760
})
5861
- Use html_to_markdown when you need high-quality page text in addition to (not instead of) structured extractions.
5962
- Never call extract_data or fetcher_tool without a clear plan for how the results will fill gaps in the objective.
63+
- Before starting new queries, call 'list_files'/'read_file' to review previous batches and avoid duplicating work; always append incremental findings to the existing coordination file for the current objective.
6064
6165
### Pagination and Next Page Handling
6266
- Prefer loading additional results directly in the SERP:
@@ -128,7 +132,12 @@ If you absolutely cannot find any reliable leads, return status "failed" with ga
128132
'extract_data',
129133
'scroll_page',
130134
'action_agent',
131-
'html_to_markdown'
135+
'html_to_markdown',
136+
'create_file',
137+
'update_file',
138+
'delete_file',
139+
'read_file',
140+
'list_files',
132141
],
133142
maxIterations: 12,
134143
modelName: MODEL_SENTINELS.USE_MINI,

front_end/panels/ai_chat/agent_framework/implementation/agents/WebTaskAgent.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,14 @@ Remember: **Plan adaptively, execute systematically, validate continuously, and
204204
'take_screenshot',
205205
'wait_for_page_load',
206206
'thinking',
207+
'render_webapp',
208+
'get_webapp_data',
209+
'remove_webapp',
210+
'create_file',
211+
'update_file',
212+
'delete_file',
213+
'read_file',
214+
'list_files',
207215
],
208216
maxIterations: 15,
209217
temperature: 0.3,

front_end/panels/ai_chat/core/BaseOrchestratorAgent.ts

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,16 @@ import {
2424
NodeIDsToURLsTool,
2525
GetVisitsByDomainTool,
2626
GetVisitsByKeywordTool,
27-
SearchVisitHistoryTool, type Tool
27+
SearchVisitHistoryTool,
28+
RenderWebAppTool,
29+
GetWebAppDataTool,
30+
RemoveWebAppTool,
31+
CreateFileTool,
32+
UpdateFileTool,
33+
DeleteFileTool,
34+
ReadFileTool,
35+
ListFilesTool,
36+
type Tool
2837
} from '../tools/Tools.js';
2938
// Imports from their own files
3039

@@ -48,6 +57,7 @@ Always delegate investigative work to the 'search_agent' tool so it can gather v
4857
4958
- Launch search_agent with a clear objective, attribute list, filters, and quantity requirement.
5059
- Review the JSON output, double-check confidence values and citations, and surface the most credible findings.
60+
- Use the file management tools ('create_file', 'update_file', 'read_file', 'list_files') to coordinate multi-step fact-finding. Persist subtask outputs as you go, read existing files before launching overlapping searches, and append incremental findings rather than duplicating effort.
5161
- If the user pivots into broad synthesis or long-form reporting, switch to the 'research_agent'.
5262
- Keep responses concise, cite the strongest sources, and present the structured findings provided by the agent.
5363
@@ -117,6 +127,12 @@ Based on query type, develop a specific research plan:
117127
- Synthesizing findings
118128
- Identifying gaps and deploying additional agents as needed
119129
130+
**Coordinate through session files:**
131+
- Before launching a new subtask, call 'list_files' to inspect existing outputs and avoid duplication.
132+
- Persist each subtask's plan, raw notes, and structured results with 'create_file'/'update_file'. Include timestamps and ownership so other agents can build on the work.
133+
- Encourage sub-agents to read relevant files ('read_file') before acting, and to append updates instead of overwriting unless the instructions explicitly call for replacement.
134+
- Use file summaries to track progress, surface blockers, and keep an audit trail for the final synthesis.
135+
120136
**Clear instructions to research agents must include:**
121137
- Specific research objectives (ideally one core objective per agent)
122138
- Expected output format with emphasis on collecting detailed, comprehensive data
@@ -299,6 +315,14 @@ export const AGENT_CONFIGS: {[key: string]: AgentConfig} = {
299315
ToolRegistry.getToolInstance('research_agent') || (() => { throw new Error('research_agent tool not found'); })(),
300316
new FinalizeWithCritiqueTool(),
301317
new SearchVisitHistoryTool(),
318+
new RenderWebAppTool(),
319+
new GetWebAppDataTool(),
320+
new RemoveWebAppTool(),
321+
new CreateFileTool(),
322+
new UpdateFileTool(),
323+
new DeleteFileTool(),
324+
new ReadFileTool(),
325+
new ListFilesTool(),
302326
]
303327
},
304328
[BaseOrchestratorAgentType.DEEP_RESEARCH]: {
@@ -315,6 +339,14 @@ export const AGENT_CONFIGS: {[key: string]: AgentConfig} = {
315339
ToolRegistry.getToolInstance('bookmark_store') || (() => { throw new Error('bookmark_store tool not found'); })(),
316340
ToolRegistry.getToolInstance('search_agent') || (() => { throw new Error('search_agent tool not found'); })(),
317341
new FinalizeWithCritiqueTool(),
342+
new RenderWebAppTool(),
343+
new GetWebAppDataTool(),
344+
new RemoveWebAppTool(),
345+
new CreateFileTool(),
346+
new UpdateFileTool(),
347+
new DeleteFileTool(),
348+
new ReadFileTool(),
349+
new ListFilesTool(),
318350
]
319351
},
320352
// [BaseOrchestratorAgentType.SHOPPING]: {
@@ -500,6 +532,11 @@ export function getAgentTools(agentType: string): Array<Tool<any, any>> {
500532
ToolRegistry.getToolInstance('research_agent') || (() => { throw new Error('research_agent tool not found'); })(),
501533
new FinalizeWithCritiqueTool(),
502534
new SearchVisitHistoryTool(),
535+
new CreateFileTool(),
536+
new UpdateFileTool(),
537+
new DeleteFileTool(),
538+
new ReadFileTool(),
539+
new ListFilesTool(),
503540
];
504541
}
505542

0 commit comments

Comments
 (0)