From 13a37bcd326e2edae439c8c2aaefd11a54b03bbc Mon Sep 17 00:00:00 2001 From: Jacob Lee Date: Sat, 6 May 2023 06:47:07 -0700 Subject: [PATCH] MRKL prompt fix, integration test (#1138) * MRKL prompt fix, integration test * Lint and format --------- Co-authored-by: Jacob Lee --- docs/docs/modules/agents/tools/webbrowser.mdx | 2 +- examples/src/agents/custom_llm_agent.ts | 4 +- examples/src/agents/custom_llm_agent_chat.ts | 4 +- langchain/src/agents/mrkl/prompt.ts | 2 +- .../src/agents/tests/mrkl_agent.int.test.ts | 62 +++++++++++++++++++ langchain/src/tools/webbrowser.ts | 2 +- 6 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 langchain/src/agents/tests/mrkl_agent.int.test.ts diff --git a/docs/docs/modules/agents/tools/webbrowser.mdx b/docs/docs/modules/agents/tools/webbrowser.mdx index 6da727563810..f846afe2eb9b 100644 --- a/docs/docs/modules/agents/tools/webbrowser.mdx +++ b/docs/docs/modules/agents/tools/webbrowser.mdx @@ -9,7 +9,7 @@ import CodeBlock from "@theme/CodeBlock"; The Webbrowser Tool gives your agent the ability to visit a website and extract information. It is described to the agent as ``` -useful for when you need to find something on or summarize a webpage. input should be a comma seperated list of "valid URL including protocol","what you want to find on the page or empty string for a summary". +useful for when you need to find something on or summarize a webpage. input should be a comma separated list of "valid URL including protocol","what you want to find on the page or empty string for a summary". ``` It exposes two modes of operation: diff --git a/examples/src/agents/custom_llm_agent.ts b/examples/src/agents/custom_llm_agent.ts index 6694d42af302..a26b35e3ac4c 100644 --- a/examples/src/agents/custom_llm_agent.ts +++ b/examples/src/agents/custom_llm_agent.ts @@ -22,7 +22,9 @@ import { SerpAPI, Tool } from "langchain/tools"; import { Calculator } from "langchain/tools/calculator"; const PREFIX = `Answer the following questions as best you can. You have access to the following tools:`; -const formatInstructions = (toolNames: string) => `Use the following format: +const formatInstructions = ( + toolNames: string +) => `Use the following format in your response: Question: the input question you must answer Thought: you should always think about what to do diff --git a/examples/src/agents/custom_llm_agent_chat.ts b/examples/src/agents/custom_llm_agent_chat.ts index e2c08a8fef0c..a2887c5bf6c2 100644 --- a/examples/src/agents/custom_llm_agent_chat.ts +++ b/examples/src/agents/custom_llm_agent_chat.ts @@ -24,7 +24,9 @@ import { SerpAPI, Tool } from "langchain/tools"; import { Calculator } from "langchain/tools/calculator"; const PREFIX = `Answer the following questions as best you can. You have access to the following tools:`; -const formatInstructions = (toolNames: string) => `Use the following format: +const formatInstructions = ( + toolNames: string +) => `Use the following format in your response: Question: the input question you must answer Thought: you should always think about what to do diff --git a/langchain/src/agents/mrkl/prompt.ts b/langchain/src/agents/mrkl/prompt.ts index 0cf84a5c984b..eb75ebdb8953 100644 --- a/langchain/src/agents/mrkl/prompt.ts +++ b/langchain/src/agents/mrkl/prompt.ts @@ -1,5 +1,5 @@ export const PREFIX = `Answer the following questions as best you can. You have access to the following tools:`; -export const FORMAT_INSTRUCTIONS = `Use the following format: +export const FORMAT_INSTRUCTIONS = `Use the following format in your response: Question: the input question you must answer Thought: you should always think about what to do diff --git a/langchain/src/agents/tests/mrkl_agent.int.test.ts b/langchain/src/agents/tests/mrkl_agent.int.test.ts new file mode 100644 index 000000000000..a140c2bb9dfb --- /dev/null +++ b/langchain/src/agents/tests/mrkl_agent.int.test.ts @@ -0,0 +1,62 @@ +/* eslint-disable no-process-env */ +import { ChatOpenAI } from "../../chat_models/openai.js"; +import { SerpAPI } from "../../tools/serpapi.js"; +import { Calculator } from "../../tools/calculator.js"; +import { initializeAgentExecutorWithOptions } from "../initialize.js"; +import { DynamicTool } from "../../tools/dynamic.js"; + +test("Run agent locally with GPT-3.5", async () => { + const model = new ChatOpenAI({ temperature: 0, modelName: "gpt-3.5-turbo" }); + const tools = [ + new SerpAPI(undefined, { + location: "Austin,Texas,United States", + hl: "en", + gl: "us", + }), + new Calculator(), + new DynamicTool({ + name: "foo", + description: "Some other tool that does foo", + func: async () => "bar", + }), + ]; + + const executor = await initializeAgentExecutorWithOptions(tools, model, { + agentType: "zero-shot-react-description", + }); + + const input = `What is the weather like in Washington DC?`; + console.log(`Executing with input "${input}"...`); + + const result = await executor.call({ input }); + + console.log(`Got output ${result.output}`); +}); + +test("Run agent locally with GPT-4", async () => { + const model = new ChatOpenAI({ temperature: 0, modelName: "gpt-4" }); + const tools = [ + new SerpAPI(undefined, { + location: "Austin,Texas,United States", + hl: "en", + gl: "us", + }), + new Calculator(), + new DynamicTool({ + name: "foo", + description: "Some other tool that does foo", + func: async () => "bar", + }), + ]; + + const executor = await initializeAgentExecutorWithOptions(tools, model, { + agentType: "zero-shot-react-description", + }); + + const input = `What is the weather like in Washington DC?`; + console.log(`Executing with input "${input}"...`); + + const result = await executor.call({ input }); + + console.log(`Got output ${result.output}`); +}); diff --git a/langchain/src/tools/webbrowser.ts b/langchain/src/tools/webbrowser.ts index 62fbe84d4914..b10f64611bc7 100644 --- a/langchain/src/tools/webbrowser.ts +++ b/langchain/src/tools/webbrowser.ts @@ -245,5 +245,5 @@ export class WebBrowser extends Tool { name = "web-browser"; - description = `useful for when you need to find something on or summarize a webpage. input should be a comma seperated list of "ONE valid http URL including protocol","what you want to find on the page or empty string for a summary".`; + description = `useful for when you need to find something on or summarize a webpage. input should be a comma separated list of "ONE valid http URL including protocol","what you want to find on the page or empty string for a summary".`; }