diff --git a/src/agent/model_middleware.ts b/src/agent/model_middleware.ts index 36dfae4ca..1ee36bd70 100644 --- a/src/agent/model_middleware.ts +++ b/src/agent/model_middleware.ts @@ -11,10 +11,11 @@ import type { ChatStreamTextHandler } from './types'; import { createLlmModel } from '.'; import { getLogSingleton } from '../log/logDecortor'; import { log } from '../log/logger'; +import { tools } from '../tools'; type Writeable = { -readonly [P in keyof T]: T[P] }; -export function AIMiddleware({ config, tools, activeTools, onStream, toolChoice, messageReferencer, chatModel }: { config: AgentUserConfig; tools: Record; activeTools: string[]; onStream: ChatStreamTextHandler | null; toolChoice: ToolChoice[] | []; messageReferencer: string[]; chatModel: string }): LanguageModelV1Middleware & { onChunk: (data: any) => void; onStepFinish: (data: StepResult, context: AgentUserConfig) => void } { +export function AIMiddleware({ config, activeTools, onStream, toolChoice, messageReferencer, chatModel }: { config: AgentUserConfig; activeTools: string[]; onStream: ChatStreamTextHandler | null; toolChoice: ToolChoice[] | []; messageReferencer: string[]; chatModel: string }): LanguageModelV1Middleware & { onChunk: (data: any) => void; onStepFinish: (data: StepResult, context: AgentUserConfig) => void } { let startTime: number | undefined; let sendToolCall = false; let step = 0; @@ -55,7 +56,7 @@ export function AIMiddleware({ config, tools, activeTools, onStream, toolChoice, onChunk: (data: any) => { const { chunk } = data; - log.debug(`chunk: ${JSON.stringify(chunk)}`); + // log.debug(`chunk: ${JSON.stringify(chunk)}`); if (chunk.type === 'tool-call' && !sendToolCall) { onStream?.send(`${messageReferencer.join('')}...\n` + `tool call will start: ${chunk.toolName}`); sendToolCall = true; @@ -124,7 +125,7 @@ function warpMessages(params: LanguageModelV1CallOptions, tools: Record 0) { if (messages[0].role === 'system') { messages[0].content = `${rawSystemPrompt}\n\nYou can consider using the following tools:\n##TOOLS${activeTools.map(name => - `\n\n### ${name}\n- desc: ${tools[name].description} \n${tools[name].prompt || ''}`, + `\n\n### ${name}\n- desc: ${tools[name]?.schema?.description || ''} \n${tools[name]?.prompt || ''}`, ).join('')}`; } } else { diff --git a/src/agent/request.ts b/src/agent/request.ts index 3401b2ecc..638a82d7a 100644 --- a/src/agent/request.ts +++ b/src/agent/request.ts @@ -187,7 +187,6 @@ export async function requestChatCompletionsV2(params: { model: LanguageModelV1; try { const middleware = AIMiddleware({ config: params.context, - tools: params.tools || {}, activeTools: params.activeTools || [], onStream, toolChoice: params.toolChoice || [], diff --git a/src/plugins/interpolate.ts b/src/plugins/interpolate.ts index 3644f5709..b5eec0c61 100644 --- a/src/plugins/interpolate.ts +++ b/src/plugins/interpolate.ts @@ -1,9 +1,9 @@ /* eslint-disable regexp/no-potentially-useless-backreference */ const INTERPOLATE_LOOP_REGEXP = /\{\{#each(?::(\w+))?\s+(\w+)\s+in\s+([\w.[\]]+)\}\}([\s\S]*?)\{\{\/each(?::\1)?\}\}/g; const INTERPOLATE_CONDITION_REGEXP = /\{\{#if(?::(\w+))?\s+([\w.[\]]+)\}\}([\s\S]*?)(?:\{\{#else(?::\1)?\}\}([\s\S]*?))?\{\{\/if(?::\1)?\}\}/g; -export const INTERPOLATE_VARIABLE_REGEXP = /\{\{([\w.[\]]+)\}\}/g; +const INTERPOLATE_VARIABLE_REGEXP = /\{\{([\w.[\]]+)\}\}/g; -export function evaluateExpression(expr: string, localData: any): undefined | any { +function evaluateExpression(expr: string, localData: any): undefined | any { if (expr === '.') { return localData['.'] ?? localData; } diff --git a/src/tools/external/app_iap.json b/src/tools/external/app_iap.json index 29b014cd8..7bee89586 100644 --- a/src/tools/external/app_iap.json +++ b/src/tools/external/app_iap.json @@ -1,42 +1,46 @@ { "schema": { "name": "app_iap", - "description": "Search for an app in App Store and retrieve its IAP information.", + "description": "Retrive detailed in-app purchase information based on country code, app id.", "parameters": { "type": "object", "properties": { - "app_name": { - "type": "string", - "description": "The name of the app to search for in App Store", - "examples": ["WeChat", "YouTube"] - }, "country": { "type": "string", "description": "The country code for App Store search", "default": "us", - "examples": ["us", "cn", "jp"] + "examples": [ + "us", + "cn", + "tr", + "ng" + ] + }, + "trackId": { + "type": "string", + "description": "The trackId to be queried", + "examples": ["363590051"] } }, - "required": ["app_name", "country"], + "required": [ + "country", + "trackId" + ], "additionalProperties": false } }, "payload": { - "url": "https://itunes.apple.com/search?entity=software&limit=1&term={{app_name}}&country={{country}}&fields=trackId,trackCensoredName", - "method": "GET", - "headers": { - "Accept": "application/json" - } + "url": "https://apps.apple.com/{{country}}/app/id{{trackId}}" }, - "webcrawler": { - "template": "{{results[0].trackViewUrl}}", + "handler": { + "type": "webclean", "patterns": [ { "pattern": "
  • ([\\s\\S]+?)<\\/li>", "group": 1, - "cleanPattern": "<[^>]*>?" + "clean": ["<[^>]*>?", ""] } ] - } - + }, + "prompt": "Please provide the detailed in-app purchase information for the app in the specified country. The default regions that users want to follow include Turkey and Nigeria, two low-cost areas, unless the user explicitly does not need to query these two regions." } diff --git a/src/tools/external/app_lookup.json b/src/tools/external/app_lookup.json new file mode 100644 index 000000000..69aa4c342 --- /dev/null +++ b/src/tools/external/app_lookup.json @@ -0,0 +1,57 @@ +{ + "schema": { + "name": "app_lookup", + "description": "Retrive five sets of related app id and app name through region code and keywords. The app id is consistent across different countries. Only one piece of data is valid.", + "parameters": { + "type": "object", + "properties": { + "app_name": { + "type": "string", + "description": "The name of the app to search for in App Store", + "examples": [ + "WeChat", + "YouTube" + ] + }, + "country": { + "type": "string", + "description": "The country code for App Store search", + "default": "us", + "examples": [ + "us", + "cn", + "jp" + ] + }, + "entity": { + "type": "string", + "description": "Types of search software", + "default": "software", + "enum": [ + "software", + "iPadSoftware", + "macSoftware" + ] + } + }, + "required": [ + "app_name", + "country", + "entity" + ], + "additionalProperties": false + } + }, + "payload": { + "url": "https://itunes.apple.com/search?entity={{entity}}&limit=3&term={{app_name}}&country={{country}}&fields=trackId,trackCensoredName", + "method": "GET", + "headers": { + "Accept": "application/json" + } + }, + "handler": { + "type": "template", + "data": "{{#each i in results}}{\"trackId\":{{i.trackId}},\"trackName\":\"{{i.trackName}}\",\"primaryGenreName\":{{i.primaryGenreName}}}{{/each}}" + }, + "prompt": "This tool should only be called once, and the app ID is the same in different regions. You should select the most relevant one." +} diff --git a/src/tools/external/index.ts b/src/tools/external/index.ts index 54965f256..d90d965b0 100644 --- a/src/tools/external/index.ts +++ b/src/tools/external/index.ts @@ -1,6 +1,7 @@ import app_iap from './app_iap.json'; +import app_lookup from './app_lookup.json'; import jina_reader from './jina.json'; import qw_lookup from './lookup.json'; import qw_weather from './qweather.json'; -export default { app_iap, jina_reader, qw_lookup, qw_weather }; +export default { app_iap, app_lookup, jina_reader, qw_lookup, qw_weather }; diff --git a/src/tools/external/jina.json b/src/tools/external/jina.json index 119760cb7..57347a816 100644 --- a/src/tools/external/jina.json +++ b/src/tools/external/jina.json @@ -1,18 +1,18 @@ { "schema": { "name": "jina_reader", - "description": - "Grab text content from provided URL links. Can be used to retrieve text information for web pages, articles, or other online resources", + "description": "Grab text content from provided URL links. Can be used to retrieve text information for web pages, articles, or other online resources", "parameters": { "type": "object", "properties": { "url": { "type": "string", - "description": - "The full URL address of the content to be crawled. If the user explicitly requests to read/analyze the content of the link, then call the function. If the data provided by the user is web content with links, but the content is sufficient to answer the question, then there is no need to call the function." + "description": "The full URL address of the content to be crawled. If the user explicitly requests to read/analyze the content of the link, then call the function. If the data provided by the user is web content with links, but the content is sufficient to answer the question, then there is no need to call the function." } }, - "required": ["url"], + "required": [ + "url" + ], "additionalProperties": false } }, @@ -25,7 +25,8 @@ "X-Timeout": "10" } }, - "handler": "content => content", "type": "reader", - "required": ["JINA_API_KEY"] + "required": [ + "JINA_API_KEY" + ] } diff --git a/src/tools/external/lookup.json b/src/tools/external/lookup.json index 7a606e2f8..939c13e59 100644 --- a/src/tools/external/lookup.json +++ b/src/tools/external/lookup.json @@ -1,6 +1,6 @@ { "schema": { - "name": "qweather_city_lookup", + "name": "qw_lookup", "description": "Retrieve city information based on location name using QWeather API. This can be used to get city details such as city ID, name, and coordinates.", "parameters": { "type": "object", @@ -10,7 +10,9 @@ "description": "The name of the location to look up. This can be a city name, administrative region, or other geographical name." } }, - "required": ["location"], + "required": [ + "location" + ], "additionalProperties": false } }, @@ -22,5 +24,7 @@ } }, "type": "lookup", - "required": ["QWEATHER_TOKEN"] + "required": [ + "QWEATHER_TOKEN" + ] } diff --git a/src/tools/external/qweather.json b/src/tools/external/qweather.json index 43429a719..414f4fd83 100644 --- a/src/tools/external/qweather.json +++ b/src/tools/external/qweather.json @@ -8,21 +8,34 @@ "location": { "type": "string", "description": "The location ID or coordinates (longitude,latitude) for which to retrieve the current weather. This ID is typically obtained from a city lookup API. For example, location=101010100 or location=116.41,39.92", - "examples": ["101010100", "116.41,39.92"] + "examples": [ + "101010100", + "116.41,39.92" + ] }, "lang": { "type": "string", "description": "Language setting for the response. Based on the language used by the user. Default is zh-hans", - "examples": ["zh-hans", "zh-hant", "en", "de"] + "examples": [ + "zh-hans", + "zh-hant", + "en", + "de" + ] }, "unit": { "type": "string", "description": "Unit of measurement for the response data. Options include unit=m (metric units, default) and unit=i (imperial units). Default is m", - "enum": ["m", "i"], + "enum": [ + "m", + "i" + ], "default": "m" } }, - "required": ["location"], + "required": [ + "location" + ], "additionalProperties": false } }, @@ -34,5 +47,7 @@ } }, "type": "weather", - "required": ["QWEATHER_TOKEN"] + "required": [ + "QWEATHER_TOKEN" + ] } diff --git a/src/tools/internal/web.ts b/src/tools/internal/web.ts new file mode 100644 index 000000000..3cbf486f9 --- /dev/null +++ b/src/tools/internal/web.ts @@ -0,0 +1,39 @@ +import type { PatternInfo } from '../types'; +import { log } from '../../log/logger'; +import { interpolate } from '../../plugins/interpolate'; + +export function processHtmlText(patterns: PatternInfo[], text: string): string { + let results: string[] = [text]; + for (const { pattern, group = 0, clean } of patterns) { + results = results.flatMap((text) => { + const regex = new RegExp(pattern, 'g'); + const matches = Array.from(text.matchAll(regex)); + return matches.map((match) => { + let extractedText = match[group]; + if (clean) { + const cleanRegex = new RegExp(clean[0], 'g'); + extractedText = extractedText.replace(cleanRegex, clean[1] ?? ''); + } + return extractedText.replace(/\s+/g, ' ').trim(); + }); + }); + } + return results.join('\n'); +} + +export interface WebCrawlerInfo { + url: string; + patterns?: PatternInfo[]; +} + +export async function webCrawler(webcrawler: WebCrawlerInfo, data: Record) { + let result: string | Record = ''; + const url = interpolate(webcrawler.url, data); + log.info(`webcrawler url: ${url}`); + const resp = await fetch(url).then(r => r.text()); + result = { + result: processHtmlText(webcrawler.patterns || [], resp), + source: url, + }; + return result; +} diff --git a/src/tools/internal/webclean.ts b/src/tools/internal/webclean.ts deleted file mode 100644 index e037d604f..000000000 --- a/src/tools/internal/webclean.ts +++ /dev/null @@ -1,41 +0,0 @@ -export interface PatternInfo { - pattern: string; - group?: number; - cleanPattern?: string; -} - -class HTMLTextProcessor { - private text: string; - - constructor(text: string) { - this.text = text; - } - - extractAndClean(patterns: PatternInfo[]): string { - let results: string[] = [this.text]; - - for (const { pattern, group = 0, cleanPattern } of patterns) { - results = results.flatMap((text) => { - const regex = new RegExp(pattern, 'g'); - const matches = Array.from(text.matchAll(regex)); - return matches.map((match) => { - let extractedText = match[group]; - if (cleanPattern) { - const cleanRegex = new RegExp(cleanPattern, 'g'); - extractedText = extractedText.replace(cleanRegex, ''); - } - return extractedText.replace(/\s+/g, ' ').trim(); - }); - }); - } - - return results.join('\n'); - } -} - -function processHtmlText(text: string, patterns: PatternInfo[]): string { - const processor = new HTMLTextProcessor(text); - return processor.extractAndClean(patterns); -} - -export default processHtmlText; diff --git a/src/tools/tool.ts b/src/tools/tool.ts index a51ae9dfa..5665ebf2a 100644 --- a/src/tools/tool.ts +++ b/src/tools/tool.ts @@ -4,22 +4,23 @@ import type { ToolCallPart, ToolResultPart } from 'ai'; import type { ResponseMessage } from '../agent/types'; import type { AgentUserConfig } from '../config/env'; import type { MessageSender } from '../telegram/utils/send'; - import type { FuncTool, ToolResult } from './types'; + import { jsonSchema, tool } from 'ai'; import { ENV } from '../config/env'; import { log } from '../log/logger'; -import { evaluateExpression, INTERPOLATE_VARIABLE_REGEXP } from '../plugins/interpolate'; +import { interpolate } from '../plugins/interpolate'; import { sendImages } from '../telegram/handler/chat'; import { isCfWorker } from '../telegram/utils/utils'; import externalTools from './external'; import internalTools from './internal'; -import processHtmlText from './internal/webclean'; +import { processHtmlText, webCrawler } from './internal/web'; export const tools = { ...externalTools, ...internalTools, -} as Record; +} as unknown as Record; + export function executeTool(toolName: string) { return async (args: any, options: Record & { signal?: AbortSignal }): Promise<{ result: any; time: string }> => { const { signal } = options; @@ -43,14 +44,14 @@ export function executeTool(toolName: string) { const startTime = Date.now(); log.info(`tool request start, url: ${parsedPayload.url}`); let result: any = await fetch(parsedPayload.url, { - method: parsedPayload.method, - headers: parsedPayload.headers, - body: JSON.stringify(parsedPayload.body), + method: parsedPayload.method || 'GET', + headers: parsedPayload.headers || {}, + body: parsedPayload.body ? JSON.stringify(parsedPayload.body) : undefined, signal, }); log.info(`tool request end`); if (!result.ok) { - throw new Error(`Tool call error: ${result.statusText}}`); + return { result: `Tool call error: ${result.statusText}`, time: ((Date.now() - startTime) / 1e3).toFixed(1) }; } try { result = await result.clone().json(); @@ -59,33 +60,37 @@ export function executeTool(toolName: string) { } const middleHandler = async (data: any) => { - if (tools[toolName].handler && !isCfWorker) { - const f = eval(tools[toolName].handler); - data = f(data); + let result = data; + const handler = tools[toolName].handler; + switch (handler?.type) { + case 'function': + if (!isCfWorker && handler.data) { + const f = eval(handler.data); + result = f(data); + } + break; + case 'template': + result = interpolate(handler.data, result); + if (handler.patterns) { + result = processHtmlText(handler.patterns, result); + } + break; + case 'webclean': + result = processHtmlText(handler.patterns || [], result); + break; } - if (tools[toolName].webcrawler) { - let url = data; - if (tools[toolName].webcrawler.template) { - url = tools[toolName].webcrawler.template.replace(INTERPOLATE_VARIABLE_REGEXP, (_, expr) => evaluateExpression(expr, data)); - } - if (!url) { - throw new Error('Invalid webcrawler template'); - } - log.info(`webcrawler url: ${url}`); - - const result = await fetch(url).then(r => r.text()); - data = processHtmlText(result, tools[toolName].webcrawler.patterns || []); + result = await webCrawler(tools[toolName].webcrawler, data); } - return data; + return result; }; result = await middleHandler(result); - if (tools[toolName].next_tool) { - const next_tool_alias = tools[toolName].next_tool; - return executeTool(next_tool_alias)(result, options); - } + // if (tools[toolName].next_tool) { + // const next_tool_alias = tools[toolName].next_tool; + // return executeTool(next_tool_alias)(result, options); + // } return { result, time: ((Date.now() - startTime) / 1e3).toFixed(1) }; }; } diff --git a/src/tools/types.ts b/src/tools/types.ts index fa3f70dac..f7c84581f 100644 --- a/src/tools/types.ts +++ b/src/tools/types.ts @@ -1,5 +1,4 @@ import type { AgentUserConfig } from '../config/env'; -import type { PatternInfo } from './internal/webclean'; export interface SchemaData> { name: string; @@ -13,6 +12,22 @@ export interface SchemaData> { }; } +export type ToolHandler = + | { + type: 'function'; + data?: string; + patterns?: PatternInfo[]; + } + | { + type: 'template'; + data: string; + patterns?: PatternInfo[]; + } + | { + type: 'webclean'; + patterns?: PatternInfo[]; + }; + /** * Function ToolType * @schema tool json schema @@ -21,12 +36,12 @@ export interface SchemaData> { * @extra_params tool extra params * @type options: search, web_crawler, command, llm, workflow * @required tool required env variables - * @not_send_to_ai not send tool output to ai, default: fasle - * - @is_stream is tool output stream, default: false + * @not_send_to_ai options: not send tool output to ai, default: fasle + * @is_stream options: is tool output stream, default: false * @scope tool scope, options: private, supergroup, group - * @handler tool handler, default is content => content * @payload tool payload, default is {} * @buildin is internal tool, default: false + * @handler type options: HandlerType * @webcrawler support input template(same as plugin input template but only support variable interpolation not support loop and condition) and patterns */ @@ -40,17 +55,24 @@ export interface FuncTool { not_send_to_ai?: boolean; // is_stream?: boolean; scope?: 'private' | 'supergroup' | 'group'; - handler?: string; payload?: Record; buildin?: boolean; result_type?: 'text' | 'image' | 'audio' | 'file'; - next_tool?: string; + // next_tool?: string; + handler?: ToolHandler; webcrawler?: { - template?: string; + url: string; patterns?: PatternInfo[]; }; } +type cleanPattern = string; +export interface PatternInfo { + pattern: string; + group?: number; + clean?: [cleanPattern, string]; +} + export interface ToolResult { result: any; time: string; diff --git a/src/utils/others/audio.ts b/src/utils/others/audio.ts index 3e2fd94de..f3bbeb8cd 100644 --- a/src/utils/others/audio.ts +++ b/src/utils/others/audio.ts @@ -1,16 +1,21 @@ import type { ReadableStream as WebReadableStream } from 'node:stream/web'; import { Readable } from 'node:stream'; -import { Base64Encode } from 'base64-stream'; -import ffmpeg from 'fluent-ffmpeg'; +// import { Base64Encode } from 'base64-stream'; +// import ffmpeg from 'fluent-ffmpeg'; interface AudioConverter { convert: (target?: 'base64' | 'blob') => Promise; } export abstract class BaseConverter implements AudioConverter { - protected command: ffmpeg.FfmpegCommand; + protected command: any; constructor() { - this.command = ffmpeg(); + this.command = null; + } + + async init() { + const ffmpeg = await import('fluent-ffmpeg'); + this.command = ffmpeg.default(); } abstract convert(target?: 'base64' | 'blob'): Promise; @@ -40,7 +45,9 @@ class OggToMp3Converter extends BaseConverter { } async convert(): Promise { + await this.init(); const input = await this.prepareInput(); + const { Base64Encode } = await import('base64-stream'); return new Promise((resolve, reject) => { let base64Data = ''; this.command @@ -51,14 +58,14 @@ class OggToMp3Converter extends BaseConverter { // .audioChannels(1) // .audioFrequency(16000) .format('mp3') - .on('error', err => reject(err)) + .on('error', (err: Error) => reject(err)) .pipe() .pipe(new Base64Encode()) .on('data', (chunk: Buffer) => { base64Data += chunk.toString(); }) .on('end', () => resolve(this.target === 'base64' ? base64Data : new Blob([Buffer.from(base64Data, 'base64')], { type: 'audio/mp3' }))) - .on('error', err => reject(err)); + .on('error', (err: Error) => reject(err)); }); } } @@ -69,6 +76,7 @@ class Mp3ToOggConverter extends BaseConverter { } async convert(): Promise { + await this.init(); return new Promise((resolve, reject) => { // 创建输入流 const inputStream = typeof this.data === 'string' ? Readable.from(Buffer.from(this.data, 'base64')) : this.data; @@ -83,7 +91,7 @@ class Mp3ToOggConverter extends BaseConverter { // .audioChannels(1) // .audioFrequency(16000) .format('ogg') - .on('error', err => reject(err)) + .on('error', (err: Error) => reject(err)) .pipe() .on('data', (chunk: Buffer) => { chunks.push(chunk); @@ -93,7 +101,7 @@ class Mp3ToOggConverter extends BaseConverter { const blob = new Blob([buffer], { type: 'audio/ogg' }); resolve(this.target === 'base64' ? buffer.toString('base64') : blob); }) - .on('error', err => reject(err)); + .on('error', (err: Error) => reject(err)); }); } }