From 59b3788f57f14428dc18e70c1a01403e87c87046 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Mon, 8 Apr 2024 16:44:48 +0200 Subject: [PATCH 01/82] Initial work on function calling --- .env.template | 27 ++++ package-lock.json | 139 +++++++++++++++++- package.json | 2 + src/lib/buildPrompt.ts | 12 +- src/lib/components/AssistantSettings.svelte | 24 +++ src/lib/components/chat/ChatMessage.svelte | 30 +++- .../server/endpoints/cohere/endpointCohere.ts | 8 +- src/lib/server/endpoints/endpoints.ts | 3 + src/lib/server/endpoints/tgi/endpointTgi.ts | 4 +- src/lib/server/models.ts | 29 +++- src/lib/server/summarize.ts | 5 + src/lib/server/tools/callFromSpecAndParams.ts | 45 ++++++ src/lib/types/Assistant.ts | 1 + src/lib/types/MessageUpdate.ts | 25 +++- src/lib/types/Template.ts | 3 + src/lib/types/Tool.ts | 23 +++ src/lib/utils/getToolsFromFunctionSpec.ts | 26 ++++ src/lib/utils/isValidOpenAPI.ts | 12 ++ src/routes/conversation/[id]/+page.svelte | 3 + src/routes/conversation/[id]/+server.ts | 102 ++++++++++++- .../[assistantId]/edit/+page.server.ts | 3 + .../(nav)/assistants/new/+page.server.ts | 3 + 22 files changed, 510 insertions(+), 19 deletions(-) create mode 100644 src/lib/server/tools/callFromSpecAndParams.ts create mode 100644 src/lib/types/Tool.ts create mode 100644 src/lib/utils/getToolsFromFunctionSpec.ts create mode 100644 src/lib/utils/isValidOpenAPI.ts diff --git a/.env.template b/.env.template index c464b5c08ea..e2a2b5383db 100644 --- a/.env.template +++ b/.env.template @@ -245,6 +245,33 @@ MODELS=`[ "prompt": "How do I make a delicious lemon cheesecake?" } ] + }, + { + "name" : "command-r-plus", + "tokenizer": "nsarrazin/c4ai-command-r-v01-tokenizer", + "displayName": "CohereForAI/c4ai-command-r-plus", + "description": "C4AI Command-R is a research release of a 35 billion parameter highly performant generative model", + "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/cohere-logo.png", + "parameters": { + "stop": ["<|END_OF_TURN_TOKEN|>"] + }, + "functions" : true, + "endpoints": [ + { + "type": "cohere", + "raw": true, + } + ] + }, + { + "name" : "CohereForAI/c4ai-command-r-v01", + "tokenizer": "nsarrazin/c4ai-command-r-v01-tokenizer", + "description": "C4AI Command-R is a research release of a 35 billion parameter highly performant generative model", + "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/cohere-logo.png", + "parameters": { + "stop": ["<|END_OF_TURN_TOKEN|>"] + }, + "functions" : true, } ]` diff --git a/package-lock.json b/package-lock.json index 6f8c1df2025..4b14d275b14 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "chat-ui", "version": "0.8.1", "dependencies": { + "@apidevtools/swagger-parser": "^10.1.0", "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", @@ -58,6 +59,7 @@ "eslint-plugin-svelte": "^2.30.0", "marked-katex-extension": "^3.0.6", "minimist": "^1.2.8", + "openapi-types": "^12.1.3", "prettier": "^2.8.0", "prettier-plugin-svelte": "^2.10.1", "prettier-plugin-tailwindcss": "^0.2.7", @@ -151,6 +153,104 @@ "node": ">= 8" } }, + "node_modules/@apidevtools/json-schema-ref-parser": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.6.tgz", + "integrity": "sha512-M3YgsLjI0lZxvrpeGVk9Ap032W6TPQkH6pRAZz81Ac3WUNF79VQooAFnp8umjvVzUmD93NkogxEwbSce7qMsUg==", + "dependencies": { + "@jsdevtools/ono": "^7.1.3", + "call-me-maybe": "^1.0.1", + "js-yaml": "^3.13.1" + } + }, + "node_modules/@apidevtools/json-schema-ref-parser/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/@apidevtools/json-schema-ref-parser/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/@apidevtools/json-schema-ref-parser/node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" + }, + "node_modules/@apidevtools/openapi-schemas": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.1.0.tgz", + "integrity": "sha512-Zc1AlqrJlX3SlpupFGpiLi2EbteyP7fXmUOGup6/DnkRgjP9bgMM/ag+n91rsv0U1Gpz0H3VILA/o3bW7Ua6BQ==", + "engines": { + "node": ">=10" + } + }, + "node_modules/@apidevtools/swagger-methods": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@apidevtools/swagger-methods/-/swagger-methods-3.0.2.tgz", + "integrity": "sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg==" + }, + "node_modules/@apidevtools/swagger-parser": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/@apidevtools/swagger-parser/-/swagger-parser-10.1.0.tgz", + "integrity": "sha512-9Kt7EuS/7WbMAUv2gSziqjvxwDbFSg3Xeyfuj5laUODX8o/k/CpsAKiQ8W7/R88eXFTMbJYg6+7uAmOWNKmwnw==", + "dependencies": { + "@apidevtools/json-schema-ref-parser": "9.0.6", + "@apidevtools/openapi-schemas": "^2.1.0", + "@apidevtools/swagger-methods": "^3.0.2", + "@jsdevtools/ono": "^7.1.3", + "ajv": "^8.6.3", + "ajv-draft-04": "^1.0.0", + "call-me-maybe": "^1.0.1" + }, + "peerDependencies": { + "openapi-types": ">=7" + } + }, + "node_modules/@apidevtools/swagger-parser/node_modules/ajv": { + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@apidevtools/swagger-parser/node_modules/ajv-draft-04": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ajv-draft-04/-/ajv-draft-04-1.0.0.tgz", + "integrity": "sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==", + "peerDependencies": { + "ajv": "^8.5.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/@apidevtools/swagger-parser/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -1221,6 +1321,11 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@jsdevtools/ono": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", + "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==" + }, "node_modules/@mongodb-js/saslprep": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.0.tgz", @@ -2901,6 +3006,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/call-me-maybe": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.2.tgz", + "integrity": "sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==" + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -3807,6 +3917,18 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/esquery": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.2.tgz", @@ -3913,8 +4035,7 @@ "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "node_modules/fast-diff": { "version": "1.2.0", @@ -5687,6 +5808,11 @@ "node": ">= 8" } }, + "node_modules/openapi-types": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==" + }, "node_modules/openid-client": { "version": "5.4.2", "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.4.2.tgz", @@ -6490,6 +6616,14 @@ "node": ">=8.10.0" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/requires-port": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", @@ -7845,7 +7979,6 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, "dependencies": { "punycode": "^2.1.0" } diff --git a/package.json b/package.json index f4af3d7e41e..4a79b708d9b 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "eslint-plugin-svelte": "^2.30.0", "marked-katex-extension": "^3.0.6", "minimist": "^1.2.8", + "openapi-types": "^12.1.3", "prettier": "^2.8.0", "prettier-plugin-svelte": "^2.10.1", "prettier-plugin-tailwindcss": "^0.2.7", @@ -50,6 +51,7 @@ }, "type": "module", "dependencies": { + "@apidevtools/swagger-parser": "^10.1.0", "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", diff --git a/src/lib/buildPrompt.ts b/src/lib/buildPrompt.ts index 00598c101e9..2fb288f8f64 100644 --- a/src/lib/buildPrompt.ts +++ b/src/lib/buildPrompt.ts @@ -1,8 +1,11 @@ import type { EndpointParameters } from "./server/endpoints/endpoints"; import type { BackendModel } from "./server/models"; +import type { Tool, ToolResult } from "./types/Tool"; type buildPromptOptions = Pick & { model: BackendModel; + tools?: Tool[]; + toolResults?: ToolResult[]; }; export async function buildPrompt({ @@ -10,11 +13,18 @@ export async function buildPrompt({ model, preprompt, continueMessage, + tools, + toolResults, }: buildPromptOptions): Promise { const filteredMessages = messages.filter((m) => m.from !== "system"); let prompt = model - .chatPromptRender({ messages: filteredMessages, preprompt }) + .chatPromptRender({ + messages: filteredMessages, + preprompt, + tools, + toolResults, + }) // Not super precise, but it's truncated in the model's backend anyway .split(" ") .slice(-(model.parameters?.truncate ?? 0)) diff --git a/src/lib/components/AssistantSettings.svelte b/src/lib/components/AssistantSettings.svelte index 7db7e592414..51b9e4e54a0 100644 --- a/src/lib/components/AssistantSettings.svelte +++ b/src/lib/components/AssistantSettings.svelte @@ -38,6 +38,7 @@ let systemPrompt = assistant?.preprompt ?? ""; let dynamicPrompt = assistant?.dynamicPrompt ?? false; let showModelSettings = Object.values(assistant?.generateSettings ?? {}).some((v) => !!v); + let functionSpec = assistant?.functionSpec ?? ""; let compress: typeof readAndCompressImage | null = null; @@ -99,6 +100,19 @@ const regex = /{{\s?url=(.+?)\s?}}/g; $: templateVariables = [...systemPrompt.matchAll(regex)].map((match) => match[1]); $: selectedModel = models.find((m) => m.id === modelId); + + // const getErrorsFromSpec = debounce(async (spec: string) => { + // if (parser && spec) { + // // disable ts + // parser.validate(spec, (err) => { + // if (err && form) { + // form.errors = [{ field: "functionSpec", message: err.message }]; + // } + // }); + // } + // }, 500); + + // $: getErrorsFromSpec(functionSpec);
+ +
+ Function calling + +

{getError("functionSpec", form)}

{/if} diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 6736d473ea3..277e96ef695 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -21,7 +21,7 @@ import type { Model } from "$lib/types/Model"; import OpenWebSearchResults from "../OpenWebSearchResults.svelte"; - import type { WebSearchUpdate } from "$lib/types/MessageUpdate"; + import type { ToolUpdate, WebSearchUpdate } from "$lib/types/MessageUpdate"; import { base } from "$app/paths"; import { useConvTreeStore } from "$lib/stores/convTree"; @@ -131,6 +131,8 @@ $: searchUpdates = (message.updates?.filter(({ type }) => type === "webSearch") ?? []) as WebSearchUpdate[]; + $: tools = (message.updates?.filter(({ type }) => type === "tool") ?? []) as ToolUpdate[]; + $: downloadLink = message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined; @@ -209,6 +211,32 @@ /> {/if} + {#if tools.length > 0} +
+ {#each tools as tool} + {#if tool.messageType === "parameters"} +

+ Calling tool {tool.name} with the + parameters: +
    + {#each Object.entries(tool.parameters ?? {}) as [k, v]} +
  • + {k}: {v} +
    • + {/each} +
+

+ {:else if tool.messageType === "message"} +

+ {" > "}{tool.message} +

+ {/if} + {/each} +
+ {/if} +
{ + return async ({ messages, preprompt, generateSettings, continueMessage, tools, toolResults }) => { let system = preprompt; if (messages?.[0]?.from === "system") { system = messages[0].content; @@ -42,10 +42,12 @@ export async function endpointCohere( if (raw) { const prompt = await buildPrompt({ - messages: messages.filter((message) => message.from !== "system"), + messages, model, preprompt: system, continueMessage, + tools, + toolResults, }); stream = await cohere.chatStream({ @@ -78,6 +80,8 @@ export async function endpointCohere( temperature: parameters?.temperature, stopSequences: parameters?.stop, frequencyPenalty: parameters?.frequency_penalty, + tools, + //TODO: Add toolResults }); } diff --git a/src/lib/server/endpoints/endpoints.ts b/src/lib/server/endpoints/endpoints.ts index fd6dfbdeacd..29d0ae24dee 100644 --- a/src/lib/server/endpoints/endpoints.ts +++ b/src/lib/server/endpoints/endpoints.ts @@ -17,6 +17,7 @@ import endpointCloudflare, { endpointCloudflareParametersSchema, } from "./cloudflare/endpointCloudflare"; import { endpointCohere, endpointCohereParametersSchema } from "./cohere/endpointCohere"; +import type { Tool, ToolResult } from "$lib/types/Tool"; // parameters passed when generating text export interface EndpointParameters { @@ -24,6 +25,8 @@ export interface EndpointParameters { preprompt?: Conversation["preprompt"]; continueMessage?: boolean; // used to signal that the last message will be extended generateSettings?: Partial; + tools?: Tool[]; + toolResults?: ToolResult[]; } interface CommonEndpoint { diff --git a/src/lib/server/endpoints/tgi/endpointTgi.ts b/src/lib/server/endpoints/tgi/endpointTgi.ts index 131d628ae21..ad770f0f661 100644 --- a/src/lib/server/endpoints/tgi/endpointTgi.ts +++ b/src/lib/server/endpoints/tgi/endpointTgi.ts @@ -16,12 +16,14 @@ export const endpointTgiParametersSchema = z.object({ export function endpointTgi(input: z.input): Endpoint { const { url, accessToken, model, authorization } = endpointTgiParametersSchema.parse(input); - return async ({ messages, preprompt, continueMessage, generateSettings }) => { + return async ({ messages, preprompt, continueMessage, generateSettings, tools, toolResults }) => { const prompt = await buildPrompt({ messages, preprompt, model, continueMessage, + tools, + toolResults, }); return textGenerationStream( diff --git a/src/lib/server/models.ts b/src/lib/server/models.ts index 1991fc0c54f..3a99d21cad9 100644 --- a/src/lib/server/models.ts +++ b/src/lib/server/models.ts @@ -67,6 +67,7 @@ const modelConfig = z.object({ .passthrough() .optional(), multimodal: z.boolean().default(false), + functions: z.boolean().default(false), unlisted: z.boolean().default(false), embeddingModel: validateEmbeddingModelByName(embeddingModels).optional(), }); @@ -98,7 +99,7 @@ async function getChatPromptRender( ); } - const renderTemplate = ({ messages, preprompt }: ChatTemplateInput) => { + const renderTemplate = ({ messages, preprompt, tools, toolResults }: ChatTemplateInput) => { let formattedMessages: { role: string; content: string }[] = messages.map((message) => ({ content: message.content, role: message.from, @@ -114,9 +115,35 @@ async function getChatPromptRender( ]; } + if (toolResults && toolResults.length > 0) { + formattedMessages = [ + ...formattedMessages, + { + role: "system", + content: + "" + + toolResults + .map((result, idx) => `\nDocument: ${idx}\n${result.key}\n${result.value}`) + .join("\n") + + "\n", + }, + ]; + tools = []; + } + + let chatTemplate: string | undefined = undefined; + + if ((tools?.length ?? 0) > 0) { + chatTemplate = "tool_use"; + } + const output = tokenizer.apply_chat_template(formattedMessages, { tokenize: false, add_generation_prompt: true, + chat_template: chatTemplate, + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + tools: tools ?? [], }); if (typeof output !== "string") { diff --git a/src/lib/server/summarize.ts b/src/lib/server/summarize.ts index aeeb4bacd07..46357c4a8ff 100644 --- a/src/lib/server/summarize.ts +++ b/src/lib/server/summarize.ts @@ -8,6 +8,11 @@ export async function summarize(prompt: string) { } const messages: Array> = [ + { + from: "system", + content: + "You are a summarization AI. You'll never answer a user's question directly, but instead summarize the user's request into a single short sentence of four words or less. Always start your answer with an emoji relevant to the summary", + }, { from: "user", content: "Who is the president of Gabon?" }, { from: "assistant", content: "🇬🇦 President of Gabon" }, { from: "user", content: "Who is Julien Chaumond?" }, diff --git a/src/lib/server/tools/callFromSpecAndParams.ts b/src/lib/server/tools/callFromSpecAndParams.ts new file mode 100644 index 00000000000..bc9a4f3a986 --- /dev/null +++ b/src/lib/server/tools/callFromSpecAndParams.ts @@ -0,0 +1,45 @@ +import { OpenAPIV3_1 } from "openapi-types"; + +export async function callOperationFromSpecAndParams( + apiSpec: OpenAPIV3_1.Document, + operationId: string, + parameters: Record +): Promise { + for (const [path, pathItem] of Object.entries(apiSpec.paths ?? {})) { + for (const [method, operation] of Object.entries(pathItem ?? {})) { + if (operation?.operationId === operationId) { + const url = new URL(apiSpec?.servers?.[0].url + path); + const headers: HeadersInit = {}; + + // Handle query parameters + if (operation.parameters) { + operation.parameters.forEach((param: any) => { + if (param.in === "query" && parameters[param.name]) { + url.searchParams.append(param.name, parameters[param.name]); + } else if (param.in === "header" && parameters[param.name]) { + headers[param.name] = parameters[param.name]; + } + // Add similar blocks for other parameter locations (path, cookie) + }); + } + + // Handle request body + let body = undefined; + if (operation.requestBody && operation.requestBody.content) { + const contentType = Object.keys(operation.requestBody.content)[0]; + headers["Content-Type"] = contentType; + body = JSON.stringify(parameters.body); // Simplistic handling; adjust based on actual content type + } + + const response = await fetch(url.toString(), { + method: method.toUpperCase(), + headers, + body, + }); + + return response.json(); // Assuming JSON response; adjust as necessary + } + } + } + throw new Error(`Operation ${operationId} not found`); +} diff --git a/src/lib/types/Assistant.ts b/src/lib/types/Assistant.ts index 783ddca0908..bbb0b21fbb7 100644 --- a/src/lib/types/Assistant.ts +++ b/src/lib/types/Assistant.ts @@ -14,6 +14,7 @@ export interface Assistant extends Timestamps { preprompt: string; userCount?: number; featured?: boolean; + functionSpec?: string; // openAPI spec rag?: { allowAllDomains: boolean; allowedDomains: string[]; diff --git a/src/lib/types/MessageUpdate.ts b/src/lib/types/MessageUpdate.ts index 9bfb25667b9..a5dd934234b 100644 --- a/src/lib/types/MessageUpdate.ts +++ b/src/lib/types/MessageUpdate.ts @@ -1,3 +1,4 @@ +import type { Call, Tool } from "./Tool"; import type { WebSearchSource } from "./WebSearch"; export type FinalAnswer = { @@ -10,12 +11,22 @@ export type TextStreamUpdate = { token: string; }; -export type AgentUpdate = { - type: "agent"; - agent: string; - content: string; - binary?: Blob; -}; +interface ToolUpdateBase { + type: "tool"; + name: Tool["name"]; +} + +interface ToolUpdateParams extends ToolUpdateBase { + messageType: "parameters"; + parameters: Call["parameters"]; +} + +interface ToolUpdateMessage extends ToolUpdateBase { + messageType: "message"; + message?: string; +} + +export type ToolUpdate = ToolUpdateParams | ToolUpdateMessage; export type WebSearchUpdate = { type: "webSearch"; @@ -40,7 +51,7 @@ export type ErrorUpdate = { export type MessageUpdate = | FinalAnswer | TextStreamUpdate - | AgentUpdate + | ToolUpdate | WebSearchUpdate | StatusUpdate | ErrorUpdate; diff --git a/src/lib/types/Template.ts b/src/lib/types/Template.ts index 9fb2fd36e33..e855e7b44f4 100644 --- a/src/lib/types/Template.ts +++ b/src/lib/types/Template.ts @@ -1,6 +1,9 @@ import type { Message } from "./Message"; +import type { Tool, ToolResult } from "./Tool"; export type ChatTemplateInput = { messages: Pick[]; preprompt?: string; + tools?: Tool[]; + toolResults?: ToolResult[]; }; diff --git a/src/lib/types/Tool.ts b/src/lib/types/Tool.ts new file mode 100644 index 00000000000..a559dddf957 --- /dev/null +++ b/src/lib/types/Tool.ts @@ -0,0 +1,23 @@ +interface ToolInput { + description: string; + type: string; + required?: boolean; +} + +export interface Tool { + name: string; + description: string; + parameter_definitions: Record; + spec: string; +} + +export interface ToolResult { + key: string; + status: "success" | "error"; + value: string; +} + +export interface Call { + tool_name: string; + parameters: Record; +} diff --git a/src/lib/utils/getToolsFromFunctionSpec.ts b/src/lib/utils/getToolsFromFunctionSpec.ts new file mode 100644 index 00000000000..20ff31f4dde --- /dev/null +++ b/src/lib/utils/getToolsFromFunctionSpec.ts @@ -0,0 +1,26 @@ +import type { Tool } from "$lib/types/Tool"; + +export async function getToolsFromFunctionSpec(spec: string): Promise { + return [ + { + name: "get_weather", + description: + "Get the weather forecast for a given location. The location can be a city, country, or even a zip code.", + parameter_definitions: { + location: { + description: "The location to get the weather for.", + type: "str", + required: true, + }, + }, + spec, + }, + { + name: "directly-answer", + description: + "Use this tool to let the user know you wish to answer directly. Do not try to provide any parameters when using this tool.", + parameter_definitions: {}, + spec, + }, + ]; +} diff --git a/src/lib/utils/isValidOpenAPI.ts b/src/lib/utils/isValidOpenAPI.ts new file mode 100644 index 00000000000..8594e514f48 --- /dev/null +++ b/src/lib/utils/isValidOpenAPI.ts @@ -0,0 +1,12 @@ +import OpenAPIParser from "@apidevtools/swagger-parser"; + +export function isValidOpenAPI(spec: string) { + // OpenAPIParser.validate(spec, (err) => { + // if (err) { + // console.log(err); + // } + return true; + // TODO: BRING THIS BACK + // return err ? err.message : true; + // }); +} diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte index 04888829a06..2bb036fbbc6 100644 --- a/src/routes/conversation/[id]/+page.svelte +++ b/src/routes/conversation/[id]/+page.svelte @@ -243,6 +243,9 @@ } else if (update.status === "error") { $error = update.message ?? "An error has occurred"; } + } else if (update.type === "tool") { + messageToWriteTo.updates = [...(messageToWriteTo.updates ?? []), update]; + messages = [...messages]; } else if (update.type === "error") { error.set(update.message); messageUpdatesAbortController.abort(); diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 760a24e9b41..925e80e0c60 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -23,6 +23,9 @@ import { addSibling } from "$lib/utils/tree/addSibling.js"; import { preprocessMessages } from "$lib/server/preprocessMessages.js"; import { usageLimits } from "$lib/server/usageLimits"; import { isURLLocal } from "$lib/server/isURLLocal.js"; +import { getToolsFromFunctionSpec } from "$lib/utils/getToolsFromFunctionSpec.js"; +import JSON5 from "json5"; +import type { Call, ToolResult } from "$lib/types/Tool.js"; export async function POST({ request, locals, params, getClientAddress }) { const id = z.string().parse(params.id); @@ -339,10 +342,10 @@ export async function POST({ request, locals, params, getClientAddress }) { // check if assistant has a rag const assistant = await collections.assistants.findOne< - Pick + Pick >( { _id: conv.assistantId }, - { projection: { rag: 1, dynamicPrompt: 1, generateSettings: 1 } } + { projection: { rag: 1, dynamicPrompt: 1, generateSettings: 1, functionSpec: 1 } } ); const assistantHasRAG = @@ -397,6 +400,98 @@ export async function POST({ request, locals, params, getClientAddress }) { } } + const endpoint = await model.getEndpoint(); + + // function calls + let toolResults: ToolResult[] | undefined = undefined; + + if (model.functions && assistant?.functionSpec) { + // turn functionSpec into a list of tools + const tools = await getToolsFromFunctionSpec(assistant?.functionSpec); + + let calls: Call[] | undefined = undefined; + + // do the function calling bits here + for await (const output of await endpoint({ + messages: messagesForPrompt, + preprompt, + generateSettings: assistant?.generateSettings, + tools, + })) { + if (output.generated_text) { + console.log({ toolCheck: output.generated_text }); + + // look for a code blocks of ```json and parse them + // if they're valid json, add them to the calls array + const codeBlocks = output.generated_text.match(/```json\n(.*?)```/gs); + if (codeBlocks) { + for (const block of codeBlocks) { + try { + calls = JSON5.parse(block.replace("```json\n", "").slice(0, -3)); + } catch (e) { + console.error(e); + // error parsing the calls, do something here. + } + } + } + } + } + + // skip if direct answer was requested + if (!(calls?.length === 1 && calls[0].tool_name === "directly-answer")) { + toolResults = calls?.map((call) => { + update({ + type: "tool", + name: call.tool_name, + messageType: "parameters", + parameters: call.parameters, + }); + const tool = tools.find((el) => el.name === call.tool_name); + + const toolAnswer: ToolResult = (() => { + if (!tool) { + return { + key: call.tool_name, + status: "error", + value: "Could not find tool", + }; + } + if (call.tool_name === "get_weather") { + if (call.parameters.location === "New York") { + return { + key: call.tool_name, + status: "success", + value: "It's sunny. 26C. 10% chance of rain.", + }; + } else if (call.parameters.location === "London") { + return { + key: call.tool_name, + status: "success", + value: "It's cloudy. 18C. 50% chance of rain.", + }; + } else { + return { + key: call.tool_name, + status: "error", + value: "Location not found", + }; + } + } + return { key: tool.name, status: "error", value: "Not implemented" }; + })(); + + update({ + type: "tool", + name: toolAnswer?.key, + messageType: "message", + message: toolAnswer?.value, + }); + + return toolAnswer; + }); + } + } + // inject websearch result & optionally images into the messages const processedMessages = await preprocessMessages( messagesForPrompt, @@ -412,12 +507,12 @@ export async function POST({ request, locals, params, getClientAddress }) { let buffer = ""; try { - const endpoint = await model.getEndpoint(); for await (const output of await endpoint({ messages: processedMessages, preprompt, continueMessage: isContinue, generateSettings: assistant?.generateSettings, + toolResults, })) { // if not generated_text is here it means the generation is not done if (!output.generated_text) { @@ -465,6 +560,7 @@ export async function POST({ request, locals, params, getClientAddress }) { } catch (e) { hasError = true; update({ type: "status", status: "error", message: (e as Error).message }); + console.error(e); } finally { // check if no output was generated if (!hasError && messageToWriteTo.content === previousText) { diff --git a/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts b/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts index 78c74af50c3..77c2e052cb3 100644 --- a/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts +++ b/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts @@ -10,6 +10,7 @@ import { sha256 } from "$lib/utils/sha256"; import sharp from "sharp"; import { parseStringToList } from "$lib/utils/parseStringToList"; import { generateSearchTokens } from "$lib/utils/searchTokens"; +import { isValidOpenAPI } from "$lib/utils/isValidOpenAPI"; const newAsssistantSchema = z.object({ name: z.string().min(1), @@ -39,6 +40,7 @@ const newAsssistantSchema = z.object({ top_k: z .union([z.literal(""), z.coerce.number().min(5).max(100)]) .transform((v) => (v === "" ? undefined : v)), + functionSpec: z.union([z.literal(""), z.string().refine(isValidOpenAPI, "Invalid OpenAPI spec")]), }); const uploadAvatar = async (avatar: File, assistantId: ObjectId): Promise => { @@ -157,6 +159,7 @@ export const actions: Actions = { }, dynamicPrompt: parse.data.dynamicPrompt, searchTokens: generateSearchTokens(parse.data.name), + functionSpec: parse.data.functionSpec, generateSettings: { temperature: parse.data.temperature, top_p: parse.data.top_p, diff --git a/src/routes/settings/(nav)/assistants/new/+page.server.ts b/src/routes/settings/(nav)/assistants/new/+page.server.ts index 50e4b738372..2069e42d5d6 100644 --- a/src/routes/settings/(nav)/assistants/new/+page.server.ts +++ b/src/routes/settings/(nav)/assistants/new/+page.server.ts @@ -10,6 +10,7 @@ import sharp from "sharp"; import { parseStringToList } from "$lib/utils/parseStringToList"; import { usageLimits } from "$lib/server/usageLimits"; import { generateSearchTokens } from "$lib/utils/searchTokens"; +import { isValidOpenAPI } from "$lib/utils/isValidOpenAPI"; const newAsssistantSchema = z.object({ name: z.string().min(1), @@ -39,6 +40,7 @@ const newAsssistantSchema = z.object({ top_k: z .union([z.literal(""), z.coerce.number().min(5).max(100)]) .transform((v) => (v === "" ? undefined : v)), + functionSpec: z.union([z.literal(""), z.string().refine(isValidOpenAPI, "Invalid OpenAPI spec")]), }); const uploadAvatar = async (avatar: File, assistantId: ObjectId): Promise => { @@ -140,6 +142,7 @@ export const actions: Actions = { dynamicPrompt: parse.data.dynamicPrompt, searchTokens: generateSearchTokens(parse.data.name), last24HoursCount: 0, + functionSpec: parse.data.functionSpec, generateSettings: { temperature: parse.data.temperature, top_p: parse.data.top_p, From 0181574f63051909441df71ade6be5cdf7536ddc Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Tue, 9 Apr 2024 08:32:37 +0200 Subject: [PATCH 02/82] wip --- src/lib/server/tools/callFromSpecAndParams.ts | 44 +------ src/lib/types/Tool.ts | 2 +- src/lib/utils/getToolsFromFunctionSpec.ts | 18 ++- src/routes/conversation/[id]/+server.ts | 117 +++++++++++++----- 4 files changed, 109 insertions(+), 72 deletions(-) diff --git a/src/lib/server/tools/callFromSpecAndParams.ts b/src/lib/server/tools/callFromSpecAndParams.ts index bc9a4f3a986..0f9e22b8f7a 100644 --- a/src/lib/server/tools/callFromSpecAndParams.ts +++ b/src/lib/server/tools/callFromSpecAndParams.ts @@ -1,45 +1,13 @@ -import { OpenAPIV3_1 } from "openapi-types"; +import type { OpenAPIV3_1 } from "openapi-types"; export async function callOperationFromSpecAndParams( apiSpec: OpenAPIV3_1.Document, operationId: string, parameters: Record ): Promise { - for (const [path, pathItem] of Object.entries(apiSpec.paths ?? {})) { - for (const [method, operation] of Object.entries(pathItem ?? {})) { - if (operation?.operationId === operationId) { - const url = new URL(apiSpec?.servers?.[0].url + path); - const headers: HeadersInit = {}; - - // Handle query parameters - if (operation.parameters) { - operation.parameters.forEach((param: any) => { - if (param.in === "query" && parameters[param.name]) { - url.searchParams.append(param.name, parameters[param.name]); - } else if (param.in === "header" && parameters[param.name]) { - headers[param.name] = parameters[param.name]; - } - // Add similar blocks for other parameter locations (path, cookie) - }); - } - - // Handle request body - let body = undefined; - if (operation.requestBody && operation.requestBody.content) { - const contentType = Object.keys(operation.requestBody.content)[0]; - headers["Content-Type"] = contentType; - body = JSON.stringify(parameters.body); // Simplistic handling; adjust based on actual content type - } - - const response = await fetch(url.toString(), { - method: method.toUpperCase(), - headers, - body, - }); - - return response.json(); // Assuming JSON response; adjust as necessary - } - } - } - throw new Error(`Operation ${operationId} not found`); + // find the matching operationId in the spec, validate parameters against its inputs in the apiSPec + // then fill in proper parameters, making sure to support : + // path parameters, such as /users/{id} + // query parameters, such as /users?role=admin + // and the POST } diff --git a/src/lib/types/Tool.ts b/src/lib/types/Tool.ts index a559dddf957..dc1191aa4e5 100644 --- a/src/lib/types/Tool.ts +++ b/src/lib/types/Tool.ts @@ -8,7 +8,7 @@ export interface Tool { name: string; description: string; parameter_definitions: Record; - spec: string; + spec?: string; } export interface ToolResult { diff --git a/src/lib/utils/getToolsFromFunctionSpec.ts b/src/lib/utils/getToolsFromFunctionSpec.ts index 20ff31f4dde..6938ccf8ada 100644 --- a/src/lib/utils/getToolsFromFunctionSpec.ts +++ b/src/lib/utils/getToolsFromFunctionSpec.ts @@ -1,6 +1,8 @@ import type { Tool } from "$lib/types/Tool"; -export async function getToolsFromFunctionSpec(spec: string): Promise { +export async function getToolsFromFunctionSpec(spec?: string): Promise { + if (!spec) return []; + return [ { name: "get_weather", @@ -16,11 +18,17 @@ export async function getToolsFromFunctionSpec(spec: string): Promise { spec, }, { - name: "directly-answer", + name: "calculator", description: - "Use this tool to let the user know you wish to answer directly. Do not try to provide any parameters when using this tool.", - parameter_definitions: {}, - spec, + "A simple calculator, takes a string containing a mathematical expression and returns the answer. Only supports +, -, *, and /, as well as parenthesis ().", + parameter_definitions: { + expression: { + description: + "The expression to evaluate. Do not include function names or anything else other than digits, +, -, *, /, and ().", + type: "str", + required: true, + }, + }, }, ]; } diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 925e80e0c60..9eca30e2d11 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -348,19 +348,22 @@ export async function POST({ request, locals, params, getClientAddress }) { { projection: { rag: 1, dynamicPrompt: 1, generateSettings: 1, functionSpec: 1 } } ); - const assistantHasRAG = + const assistantHasDynamicPrompt = + ENABLE_ASSISTANTS_RAG === "true" && !!assistant && !!assistant?.dynamicPrompt; + + const assistantHasWebSearch = ENABLE_ASSISTANTS_RAG === "true" && - assistant && - ((assistant.rag && - (assistant.rag.allowedLinks.length > 0 || - assistant.rag.allowedDomains.length > 0 || - assistant.rag.allowAllDomains)) || - assistant.dynamicPrompt); + !!assistant && + !!assistant.rag && + (assistant.rag.allowedLinks.length > 0 || + assistant.rag.allowedDomains.length > 0 || + assistant.rag.allowAllDomains); + console.log(assistant?.rag, ENABLE_ASSISTANTS_RAG, assistantHasWebSearch); // perform websearch if needed if ( !isContinue && - ((webSearch && !conv.assistantId) || (assistantHasRAG && !assistant.dynamicPrompt)) + ((webSearch && !conv.assistantId) || (assistantHasWebSearch && !model.functions)) ) { messageToWriteTo.webSearch = await runWebSearch( conv, @@ -372,7 +375,7 @@ export async function POST({ request, locals, params, getClientAddress }) { let preprompt = conv.preprompt; - if (assistant?.dynamicPrompt && preprompt && ENABLE_ASSISTANTS_RAG === "true") { + if (assistantHasDynamicPrompt && preprompt) { // process the preprompt const urlRegex = /{{\s?url=(.*?)\s?}}/g; let match; @@ -405,10 +408,44 @@ export async function POST({ request, locals, params, getClientAddress }) { // function calls let toolResults: ToolResult[] | undefined = undefined; - if (model.functions && assistant?.functionSpec) { + if (model.functions && (assistant?.functionSpec || assistantHasWebSearch)) { // turn functionSpec into a list of tools - const tools = await getToolsFromFunctionSpec(assistant?.functionSpec); + let tools = await getToolsFromFunctionSpec(assistant?.functionSpec); + + // add the directly-answer tool + tools = [ + ...tools, + { + name: "directly-answer", + description: + "Use this tool to let the user know you wish to answer directly. Do not try to provide any parameters when using this tool.", + parameter_definitions: {}, + }, + ]; + + // add websearch tool if relevant + if (assistantHasWebSearch) { + tools = [ + ...tools, + { + name: "websearch", + description: + "Use this tool to search web pages for answers that will help answer the user's query.", + parameter_definitions: + (assistant.rag?.allowedLinks?.length ?? 0) > 0 + ? {} + : { + query: { + required: true, + type: "string", + description: + "A search query which will be used to fetch the most relevant snippets regarding the user's query", + }, + }, + }, + ]; + } let calls: Call[] | undefined = undefined; // do the function calling bits here @@ -419,8 +456,6 @@ export async function POST({ request, locals, params, getClientAddress }) { tools, })) { if (output.generated_text) { - console.log({ toolCheck: output.generated_text }); - // look for a code blocks of ```json and parse them // if they're valid json, add them to the calls array const codeBlocks = output.generated_text.match(/```json\n(.*?)```/gs); @@ -437,9 +472,12 @@ export async function POST({ request, locals, params, getClientAddress }) { } } - // skip if direct answer was requested - if (!(calls?.length === 1 && calls[0].tool_name === "directly-answer")) { - toolResults = calls?.map((call) => { + toolResults = calls + ?.map((call) => { + if (call.tool_name === "directly-answer") { + return null; + } + update({ type: "tool", name: call.tool_name, @@ -448,7 +486,7 @@ export async function POST({ request, locals, params, getClientAddress }) { }); const tool = tools.find((el) => el.name === call.tool_name); - const toolAnswer: ToolResult = (() => { + const toolAnswer: ToolResult | null = (() => { if (!tool) { return { key: call.tool_name, @@ -476,22 +514,45 @@ export async function POST({ request, locals, params, getClientAddress }) { value: "Location not found", }; } + } else if (call.tool_name === "calculator") { + try { + const query = call.parameters.expression.replace(/[^-()\d/*+.]/g, ""); + return { + key: call.tool_name, + status: "success", + value: eval(query), + }; + } catch { + return { + key: call.tool_name, + status: "error", + value: "Invalid expression", + }; + } + } else if (call.tool_name === "websearch") { + // TODO: Handle websearch + return null; + } else { + return { + key: call.tool_name, + status: "error", + value: "Not implemented", + }; } - return { key: tool.name, status: "error", value: "Not implemented" }; })(); - update({ - type: "tool", - name: toolAnswer?.key, - messageType: "message", - message: toolAnswer?.value, - }); - + if (toolAnswer) { + update({ + type: "tool", + name: toolAnswer.key, + messageType: "message", + message: toolAnswer.value, + }); + } return toolAnswer; - }); - } + }) + .flatMap((el) => (el ? [el] : [])); } - // inject websearch result & optionally images into the messages const processedMessages = await preprocessMessages( messagesForPrompt, From 2acbce1fdfc2d3fd9127ede3d0c7860baabb2169 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Tue, 9 Apr 2024 11:51:49 +0200 Subject: [PATCH 03/82] Add websearch as a tool --- src/lib/server/websearch/runWebSearch.ts | 17 +- src/routes/conversation/[id]/+server.ts | 190 +++++++++++++---------- 2 files changed, 122 insertions(+), 85 deletions(-) diff --git a/src/lib/server/websearch/runWebSearch.ts b/src/lib/server/websearch/runWebSearch.ts index 8eba2ddd627..afa862d6a14 100644 --- a/src/lib/server/websearch/runWebSearch.ts +++ b/src/lib/server/websearch/runWebSearch.ts @@ -25,11 +25,16 @@ const listSchema = z.array(z.string()).default([]); const allowList = listSchema.parse(JSON5.parse(WEBSEARCH_ALLOWLIST)); const blockList = listSchema.parse(JSON5.parse(WEBSEARCH_BLOCKLIST)); +export interface RunWebSearchOptions { + ragSettings?: Assistant["rag"]; + query?: string; +} + export async function runWebSearch( conv: Conversation, messages: Message[], updatePad: (upd: MessageUpdate) => void, - ragSettings?: Assistant["rag"] + options?: RunWebSearchOptions ) { const prompt = messages[messages.length - 1].content; const webSearch: WebSearch = { @@ -47,10 +52,10 @@ export async function runWebSearch( try { // if the assistant specified direct links, skip the websearch - if (ragSettings && ragSettings?.allowedLinks.length > 0) { + if (options?.ragSettings && options?.ragSettings?.allowedLinks.length > 0) { appendUpdate("Using links specified in Assistant"); - let linksToUse = [...ragSettings.allowedLinks]; + let linksToUse = [...options.ragSettings.allowedLinks]; if (ENABLE_LOCAL_FETCH !== "true") { const localLinks = await Promise.all( @@ -71,14 +76,14 @@ export async function runWebSearch( return { link, hostname: new URL(link).hostname, title: "", text: "" }; }); } else { - webSearch.searchQuery = await generateQuery(messages); + webSearch.searchQuery = options?.query ?? (await generateQuery(messages)); const searchProvider = getWebSearchProvider(); appendUpdate(`Searching ${searchProvider}`, [webSearch.searchQuery]); let filters = ""; - if (ragSettings && ragSettings?.allowedDomains.length > 0) { + if (options?.ragSettings && options?.ragSettings?.allowedDomains.length > 0) { appendUpdate("Filtering on specified domains"); - filters += ragSettings.allowedDomains.map((item) => "site:" + item).join(" OR "); + filters += options?.ragSettings.allowedDomains.map((item) => "site:" + item).join(" OR "); } // handle the global lists diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 9eca30e2d11..46142d4bbee 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -359,18 +359,18 @@ export async function POST({ request, locals, params, getClientAddress }) { assistant.rag.allowedDomains.length > 0 || assistant.rag.allowAllDomains); - console.log(assistant?.rag, ENABLE_ASSISTANTS_RAG, assistantHasWebSearch); - // perform websearch if needed + // perform websearch if requested + // it can be because the user toggled the webSearch or because the assistant has webSearch enabled + // if the assistant is functions enabled, we don't perform it here + // since we will add the websearch as a tool + if ( !isContinue && ((webSearch && !conv.assistantId) || (assistantHasWebSearch && !model.functions)) ) { - messageToWriteTo.webSearch = await runWebSearch( - conv, - messagesForPrompt, - update, - assistant?.rag - ); + messageToWriteTo.webSearch = await runWebSearch(conv, messagesForPrompt, update, { + ragSettings: assistant?.rag, + }); } let preprompt = conv.preprompt; @@ -404,10 +404,9 @@ export async function POST({ request, locals, params, getClientAddress }) { } const endpoint = await model.getEndpoint(); - - // function calls let toolResults: ToolResult[] | undefined = undefined; + // function calls if (model.functions && (assistant?.functionSpec || assistantHasWebSearch)) { // turn functionSpec into a list of tools @@ -464,94 +463,127 @@ export async function POST({ request, locals, params, getClientAddress }) { try { calls = JSON5.parse(block.replace("```json\n", "").slice(0, -3)); } catch (e) { + update({ + type: "error", + message: (e as Error).message, + name: (e as Error).name, + }); console.error(e); - // error parsing the calls, do something here. + // error parsing the calls } } } } } - toolResults = calls - ?.map((call) => { - if (call.tool_name === "directly-answer") { - return null; - } - - update({ - type: "tool", - name: call.tool_name, - messageType: "parameters", - parameters: call.parameters, - }); - const tool = tools.find((el) => el.name === call.tool_name); + const toolPromises = calls?.map(async (call) => { + if (call.tool_name === "directly-answer") { + return null; + } - const toolAnswer: ToolResult | null = (() => { - if (!tool) { + update({ + type: "tool", + name: call.tool_name, + messageType: "parameters", + parameters: call.parameters, + }); + const tool = tools.find((el) => el.name === call.tool_name); + + let toolAnswer: ToolResult | Promise | null = (async () => { + if (!tool) { + return { + key: call.tool_name, + status: "error", + value: "Could not find tool", + }; + } + if (call.tool_name === "get_weather") { + if (call.parameters.location === "New York") { + return { + key: call.tool_name, + status: "success", + value: "It's sunny. 26C. 10% chance of rain.", + }; + } else if (call.parameters.location === "London") { + return { + key: call.tool_name, + status: "success", + value: "It's cloudy. 18C. 50% chance of rain.", + }; + } else { return { key: call.tool_name, status: "error", - value: "Could not find tool", + value: "Location not found", }; } - if (call.tool_name === "get_weather") { - if (call.parameters.location === "New York") { - return { - key: call.tool_name, - status: "success", - value: "It's sunny. 26C. 10% chance of rain.", - }; - } else if (call.parameters.location === "London") { - return { - key: call.tool_name, - status: "success", - value: "It's cloudy. 18C. 50% chance of rain.", - }; - } else { - return { - key: call.tool_name, - status: "error", - value: "Location not found", - }; - } - } else if (call.tool_name === "calculator") { - try { - const query = call.parameters.expression.replace(/[^-()\d/*+.]/g, ""); - return { - key: call.tool_name, - status: "success", - value: eval(query), - }; - } catch { - return { - key: call.tool_name, - status: "error", - value: "Invalid expression", - }; - } - } else if (call.tool_name === "websearch") { - // TODO: Handle websearch - return null; - } else { + } else if (call.tool_name === "calculator") { + try { + const query = call.parameters.expression.replace(/[^-()\d/*+.]/g, ""); + return { + key: call.tool_name, + status: "success", + value: eval(query), + }; + } catch { return { key: call.tool_name, status: "error", - value: "Not implemented", + value: "Invalid expression", }; } - })(); - - if (toolAnswer) { - update({ - type: "tool", - name: toolAnswer.key, - messageType: "message", - message: toolAnswer.value, - }); + } else if (call.tool_name === "websearch") { + try { + const webSearchToolResults = await runWebSearch(conv, messagesForPrompt, update, { + ragSettings: assistant?.rag, + query: call.parameters?.query, + }); + const chunks = webSearchToolResults?.contextSources + .map(({ context }) => context) + .flat() + .sort((a, b) => a.idx - b.idx) + .map(({ text }) => text) + .join(" "); + + return { + key: call.tool_name, + status: "success", + value: chunks, + }; + } catch (e) { + return { + key: call.tool_name, + status: "error", + value: "Error within the websearch. Try again later or with a different query", + }; + } + } else { + return { + key: call.tool_name, + status: "error", + value: "Not implemented", + }; } - return toolAnswer; - }) - .flatMap((el) => (el ? [el] : [])); + })(); + + if (toolAnswer instanceof Promise) { + toolAnswer = await toolAnswer; + } + + if (toolAnswer) { + update({ + type: "tool", + name: toolAnswer.key, + messageType: "message", + message: toolAnswer.value, + }); + } + return toolAnswer; + }); + + if (toolPromises) { + toolResults = (await Promise.all(toolPromises)).flatMap((el) => (el ? [el] : [])); + } } // inject websearch result & optionally images into the messages const processedMessages = await preprocessMessages( From 4f3716b4331e524a3c50fb4efcff020a82c0a5ab Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Tue, 9 Apr 2024 12:02:26 +0200 Subject: [PATCH 04/82] lint --- src/lib/server/tools/callFromSpecAndParams.ts | 24 +++++++++---------- src/lib/utils/isValidOpenAPI.ts | 22 ++++++++--------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/lib/server/tools/callFromSpecAndParams.ts b/src/lib/server/tools/callFromSpecAndParams.ts index 0f9e22b8f7a..816060f46c6 100644 --- a/src/lib/server/tools/callFromSpecAndParams.ts +++ b/src/lib/server/tools/callFromSpecAndParams.ts @@ -1,13 +1,13 @@ -import type { OpenAPIV3_1 } from "openapi-types"; +// import type { OpenAPIV3_1 } from "openapi-types"; -export async function callOperationFromSpecAndParams( - apiSpec: OpenAPIV3_1.Document, - operationId: string, - parameters: Record -): Promise { - // find the matching operationId in the spec, validate parameters against its inputs in the apiSPec - // then fill in proper parameters, making sure to support : - // path parameters, such as /users/{id} - // query parameters, such as /users?role=admin - // and the POST -} +// export async function callOperationFromSpecAndParams( +// apiSpec: OpenAPIV3_1.Document, +// operationId: string, +// parameters: Record +// ): Promise { +// // find the matching operationId in the spec, validate parameters against its inputs in the apiSPec +// // then fill in proper parameters, making sure to support : +// // path parameters, such as /users/{id} +// // query parameters, such as /users?role=admin +// // and the POST +// } diff --git a/src/lib/utils/isValidOpenAPI.ts b/src/lib/utils/isValidOpenAPI.ts index 8594e514f48..c278fda5b08 100644 --- a/src/lib/utils/isValidOpenAPI.ts +++ b/src/lib/utils/isValidOpenAPI.ts @@ -1,12 +1,12 @@ -import OpenAPIParser from "@apidevtools/swagger-parser"; +// import OpenAPIParser from "@apidevtools/swagger-parser"; -export function isValidOpenAPI(spec: string) { - // OpenAPIParser.validate(spec, (err) => { - // if (err) { - // console.log(err); - // } - return true; - // TODO: BRING THIS BACK - // return err ? err.message : true; - // }); -} +// export function isValidOpenAPI(spec: string) { +// // OpenAPIParser.validate(spec, (err) => { +// // if (err) { +// // console.log(err); +// // } +// return true; +// // TODO: BRING THIS BACK +// // return err ? err.message : true; +// // }); +// } From 6e31d7266844156df5b835cb2023542e1e834f65 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Tue, 9 Apr 2024 12:04:43 +0200 Subject: [PATCH 05/82] smol --- src/lib/utils/isValidOpenAPI.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/lib/utils/isValidOpenAPI.ts b/src/lib/utils/isValidOpenAPI.ts index c278fda5b08..059166953b3 100644 --- a/src/lib/utils/isValidOpenAPI.ts +++ b/src/lib/utils/isValidOpenAPI.ts @@ -1,12 +1,13 @@ // import OpenAPIParser from "@apidevtools/swagger-parser"; -// export function isValidOpenAPI(spec: string) { -// // OpenAPIParser.validate(spec, (err) => { -// // if (err) { -// // console.log(err); -// // } -// return true; -// // TODO: BRING THIS BACK -// // return err ? err.message : true; -// // }); -// } +export function isValidOpenAPI(spec: string) { + console.log(spec); + // OpenAPIParser.validate(spec, (err) => { + // if (err) { + // console.log(err); + // } + return true; + // TODO: BRING THIS BACK + // return err ? err.message : true; + // }); +} From a5cc012db133f2f34e2e13c2a5a0c80ae8f42a9f Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Wed, 10 Apr 2024 17:27:33 +0200 Subject: [PATCH 06/82] Move tools to their own files --- src/lib/server/tools/calculator.ts | 35 +++++++++ src/lib/server/tools/callFromSpecAndParams.ts | 13 ---- src/lib/server/tools/directlyAnswer.ts | 10 +++ src/lib/server/tools/getWeather.ts | 37 +++++++++ src/lib/server/tools/index.ts | 11 +++ src/lib/server/tools/websearch.ts | 17 ++++ src/lib/utils/getToolsFromFunctionSpec.ts | 33 +------- src/routes/conversation/[id]/+server.ts | 77 +++---------------- 8 files changed, 122 insertions(+), 111 deletions(-) create mode 100644 src/lib/server/tools/calculator.ts delete mode 100644 src/lib/server/tools/callFromSpecAndParams.ts create mode 100644 src/lib/server/tools/directlyAnswer.ts create mode 100644 src/lib/server/tools/getWeather.ts create mode 100644 src/lib/server/tools/index.ts create mode 100644 src/lib/server/tools/websearch.ts diff --git a/src/lib/server/tools/calculator.ts b/src/lib/server/tools/calculator.ts new file mode 100644 index 00000000000..83673bfef97 --- /dev/null +++ b/src/lib/server/tools/calculator.ts @@ -0,0 +1,35 @@ +import type { BackendTool } from "."; + +const calculator: BackendTool = { + name: "calculator", + description: + "A simple calculator, takes a string containing a mathematical expression and returns the answer. Only supports +, -, *, and /, as well as parenthesis ().", + parameter_definitions: { + equation: { + description: + "The formula to evaluate. EXACTLY as you would plug into a calculator. No words, no letters, only numbers and operators. Letters will make the tool crash.", + type: "formula", + required: true, + }, + }, + call: async (params) => { + try { + const blocks = params.equation.split("\n"); + const query = blocks[blocks.length - 1].replace(/[^-()\d/*+.]/g, ""); + + return { + key: "calculator", + status: "success", + value: eval(query), + }; + } catch (e) { + return { + key: "calculator", + status: "error", + value: "Invalid expression", + }; + } + }, +}; + +export default calculator; diff --git a/src/lib/server/tools/callFromSpecAndParams.ts b/src/lib/server/tools/callFromSpecAndParams.ts deleted file mode 100644 index 816060f46c6..00000000000 --- a/src/lib/server/tools/callFromSpecAndParams.ts +++ /dev/null @@ -1,13 +0,0 @@ -// import type { OpenAPIV3_1 } from "openapi-types"; - -// export async function callOperationFromSpecAndParams( -// apiSpec: OpenAPIV3_1.Document, -// operationId: string, -// parameters: Record -// ): Promise { -// // find the matching operationId in the spec, validate parameters against its inputs in the apiSPec -// // then fill in proper parameters, making sure to support : -// // path parameters, such as /users/{id} -// // query parameters, such as /users?role=admin -// // and the POST -// } diff --git a/src/lib/server/tools/directlyAnswer.ts b/src/lib/server/tools/directlyAnswer.ts new file mode 100644 index 00000000000..836b8cfb6ae --- /dev/null +++ b/src/lib/server/tools/directlyAnswer.ts @@ -0,0 +1,10 @@ +import type { BackendTool } from "."; + +const directlyAnswer: BackendTool = { + name: "directly-answer", + description: + "Use this tool to let the user know you wish to answer directly. Do not try to provide any parameters when using this tool.", + parameter_definitions: {}, +}; + +export default directlyAnswer; diff --git a/src/lib/server/tools/getWeather.ts b/src/lib/server/tools/getWeather.ts new file mode 100644 index 00000000000..e7e4ac4dc20 --- /dev/null +++ b/src/lib/server/tools/getWeather.ts @@ -0,0 +1,37 @@ +import type { BackendTool } from "."; + +const getWeather: BackendTool = { + name: "get_weather", + description: + "Get the weather forecast for a given location. The location can be a city, country, or even a zip code.", + parameter_definitions: { + location: { + description: "The location to get the weather for.", + type: "str", + required: true, + }, + }, + call: async (params) => { + if (params.location === "New York") { + return { + key: "get_weather", + status: "success", + value: "It's sunny. 26C. 10% chance of rain.", + }; + } else if (params.location === "London") { + return { + key: "get_weather", + status: "success", + value: "It's cloudy. 18C. 50% chance of rain.", + }; + } else { + return { + key: "get_weather", + status: "error", + value: "Location not found", + }; + } + }, +}; + +export default getWeather; diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts new file mode 100644 index 00000000000..a1e56dd0c71 --- /dev/null +++ b/src/lib/server/tools/index.ts @@ -0,0 +1,11 @@ +import type { Tool, ToolResult } from "$lib/types/Tool"; +import calculator from "./calculator"; +import directlyAnswer from "./directlyAnswer"; +import getWeather from "./getWeather"; +import websearch from "./websearch"; + +export interface BackendTool extends Tool { + call?(params: Record): Promise; +} + +export const tools: BackendTool[] = [calculator, getWeather, websearch, directlyAnswer]; diff --git a/src/lib/server/tools/websearch.ts b/src/lib/server/tools/websearch.ts new file mode 100644 index 00000000000..2e53b30c7e4 --- /dev/null +++ b/src/lib/server/tools/websearch.ts @@ -0,0 +1,17 @@ +import type { BackendTool } from "."; + +const websearch: BackendTool = { + name: "websearch", + description: + "Use this tool to search web pages for answers that will help answer the user's query.", + parameter_definitions: { + query: { + required: true, + type: "string", + description: + "A search query which will be used to fetch the most relevant snippets regarding the user's query", + }, + }, +}; + +export default websearch; diff --git a/src/lib/utils/getToolsFromFunctionSpec.ts b/src/lib/utils/getToolsFromFunctionSpec.ts index 6938ccf8ada..f328c21a208 100644 --- a/src/lib/utils/getToolsFromFunctionSpec.ts +++ b/src/lib/utils/getToolsFromFunctionSpec.ts @@ -1,34 +1,7 @@ -import type { Tool } from "$lib/types/Tool"; +import { tools, type BackendTool } from "$lib/server/tools"; -export async function getToolsFromFunctionSpec(spec?: string): Promise { +export async function getToolsFromFunctionSpec(spec?: string): Promise { if (!spec) return []; - return [ - { - name: "get_weather", - description: - "Get the weather forecast for a given location. The location can be a city, country, or even a zip code.", - parameter_definitions: { - location: { - description: "The location to get the weather for.", - type: "str", - required: true, - }, - }, - spec, - }, - { - name: "calculator", - description: - "A simple calculator, takes a string containing a mathematical expression and returns the answer. Only supports +, -, *, and /, as well as parenthesis ().", - parameter_definitions: { - expression: { - description: - "The expression to evaluate. Do not include function names or anything else other than digits, +, -, *, /, and ().", - type: "str", - required: true, - }, - }, - }, - ]; + return tools; } diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 46142d4bbee..d8780713fef 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -412,38 +412,9 @@ export async function POST({ request, locals, params, getClientAddress }) { let tools = await getToolsFromFunctionSpec(assistant?.functionSpec); - // add the directly-answer tool - tools = [ - ...tools, - { - name: "directly-answer", - description: - "Use this tool to let the user know you wish to answer directly. Do not try to provide any parameters when using this tool.", - parameter_definitions: {}, - }, - ]; - - // add websearch tool if relevant - if (assistantHasWebSearch) { - tools = [ - ...tools, - { - name: "websearch", - description: - "Use this tool to search web pages for answers that will help answer the user's query.", - parameter_definitions: - (assistant.rag?.allowedLinks?.length ?? 0) > 0 - ? {} - : { - query: { - required: true, - type: "string", - description: - "A search query which will be used to fetch the most relevant snippets regarding the user's query", - }, - }, - }, - ]; + // remove websearch tool if relevant + if (!assistantHasWebSearch) { + tools = tools.filter(({ name }) => name !== "websearch"); } let calls: Call[] | undefined = undefined; @@ -497,42 +468,12 @@ export async function POST({ request, locals, params, getClientAddress }) { value: "Could not find tool", }; } - if (call.tool_name === "get_weather") { - if (call.parameters.location === "New York") { - return { - key: call.tool_name, - status: "success", - value: "It's sunny. 26C. 10% chance of rain.", - }; - } else if (call.parameters.location === "London") { - return { - key: call.tool_name, - status: "success", - value: "It's cloudy. 18C. 50% chance of rain.", - }; - } else { - return { - key: call.tool_name, - status: "error", - value: "Location not found", - }; - } - } else if (call.tool_name === "calculator") { - try { - const query = call.parameters.expression.replace(/[^-()\d/*+.]/g, ""); - return { - key: call.tool_name, - status: "success", - value: eval(query), - }; - } catch { - return { - key: call.tool_name, - status: "error", - value: "Invalid expression", - }; - } - } else if (call.tool_name === "websearch") { + + if (tool.call) { + return await tool.call(call.parameters); + } + + if (tool.name === "websearch") { try { const webSearchToolResults = await runWebSearch(conv, messagesForPrompt, update, { ragSettings: assistant?.rag, From ba42921e69fff9f2a46f2dfc89690328d5f57d78 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Wed, 10 Apr 2024 17:58:29 +0200 Subject: [PATCH 07/82] directly answer check --- src/routes/conversation/[id]/+server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index d8780713fef..754b364b164 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -448,7 +448,7 @@ export async function POST({ request, locals, params, getClientAddress }) { } const toolPromises = calls?.map(async (call) => { - if (call.tool_name === "directly-answer") { + if (call.tool_name === "directly-answer" || call.tool_name === "directly_answer") { return null; } From 92cc8ef63e588697a8147985b0bffec879cf351a Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Mon, 15 Apr 2024 11:59:34 +0200 Subject: [PATCH 08/82] Add text2img tool --- .env.template | 2 +- src/lib/components/chat/ChatMessage.svelte | 23 ++++++++++++++- src/lib/server/tools/index.ts | 3 +- src/lib/server/tools/text2img.ts | 16 ++++++++++ src/lib/types/MessageUpdate.ts | 8 ++++- src/routes/conversation/[id]/+page.svelte | 3 ++ src/routes/conversation/[id]/+server.ts | 29 ++++++++++++++++++- .../[id]/output/[sha256]/+server.ts | 2 +- vite.config.ts | 2 +- 9 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 src/lib/server/tools/text2img.ts diff --git a/.env.template b/.env.template index 230dcaf2b3b..f216c02b624 100644 --- a/.env.template +++ b/.env.template @@ -3,7 +3,7 @@ MODELS=`[ { "name" : "CohereForAI/c4ai-command-r-plus", - "tokenizer": "Xenova/c4ai-command-r-v01-tokenizer", + "tokenizer": "nsarrazin/c4ai-command-r-v01-tokenizer", "description": "Command R+ is Cohere's latest LLM and is the first open weight model to beat GPT4 in the Chatbot Arena!", "modelUrl": "https://huggingface.co/CohereForAI/c4ai-command-r-plus", "websiteUrl": "https://docs.cohere.com/docs/command-r-plus", diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 277e96ef695..3430413616a 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -204,6 +204,27 @@
+ {#if message.files && message.files.length > 0} +
+ {#each message.files as file} + + {#if file.length === 64} + input from user + {:else} + + input from user + {/if} + {/each} +
+ {/if} {#if searchUpdates && searchUpdates.length > 0} 0}
{#each tools as tool} {#if tool.messageType === "parameters"} diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index a1e56dd0c71..21a47d8d699 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -2,10 +2,11 @@ import type { Tool, ToolResult } from "$lib/types/Tool"; import calculator from "./calculator"; import directlyAnswer from "./directlyAnswer"; import getWeather from "./getWeather"; +import text2img from "./text2img"; import websearch from "./websearch"; export interface BackendTool extends Tool { call?(params: Record): Promise; } -export const tools: BackendTool[] = [calculator, getWeather, websearch, directlyAnswer]; +export const tools: BackendTool[] = [calculator, getWeather, websearch, text2img, directlyAnswer]; diff --git a/src/lib/server/tools/text2img.ts b/src/lib/server/tools/text2img.ts new file mode 100644 index 00000000000..4cae2cdc48f --- /dev/null +++ b/src/lib/server/tools/text2img.ts @@ -0,0 +1,16 @@ +import type { BackendTool } from "."; + +const text2img: BackendTool = { + name: "text2img", + description: "Use this tool to generate an image from a prompt.", + parameter_definitions: { + prompt: { + description: + "A prompt to generate an image from. Describe the image visually in simple terms, separate terms with a comma.", + type: "string", + required: true, + }, + }, +}; + +export default text2img; diff --git a/src/lib/types/MessageUpdate.ts b/src/lib/types/MessageUpdate.ts index a5dd934234b..bed83179d95 100644 --- a/src/lib/types/MessageUpdate.ts +++ b/src/lib/types/MessageUpdate.ts @@ -48,10 +48,16 @@ export type ErrorUpdate = { name: string; }; +export type FileUpdate = { + type: "file"; + sha: string; +}; + export type MessageUpdate = | FinalAnswer | TextStreamUpdate | ToolUpdate | WebSearchUpdate | StatusUpdate - | ErrorUpdate; + | ErrorUpdate + | FileUpdate; diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte index 2bb036fbbc6..e680b41f8c2 100644 --- a/src/routes/conversation/[id]/+page.svelte +++ b/src/routes/conversation/[id]/+page.svelte @@ -246,6 +246,9 @@ } else if (update.type === "tool") { messageToWriteTo.updates = [...(messageToWriteTo.updates ?? []), update]; messages = [...messages]; + } else if (update.type === "file") { + messageToWriteTo.files = [...(messageToWriteTo.files ?? []), update.sha]; + messages = [...messages]; } else if (update.type === "error") { error.set(update.message); messageUpdatesAbortController.abort(); diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index db27d2769a9..b4d478bac2d 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -1,4 +1,9 @@ -import { MESSAGES_BEFORE_LOGIN, ENABLE_ASSISTANTS_RAG } from "$env/static/private"; +import { + MESSAGES_BEFORE_LOGIN, + ENABLE_ASSISTANTS_RAG, + HF_TOKEN, + HF_ACCESS_TOKEN, +} from "$env/static/private"; import { startOfHour } from "date-fns"; import { authCondition, requiresUser } from "$lib/server/auth"; import { collections } from "$lib/server/database"; @@ -26,6 +31,7 @@ import { isURLLocal } from "$lib/server/isURLLocal.js"; import { getToolsFromFunctionSpec } from "$lib/utils/getToolsFromFunctionSpec.js"; import JSON5 from "json5"; import type { Call, ToolResult } from "$lib/types/Tool.js"; +import { HfInference } from "@huggingface/inference"; export async function POST({ request, locals, params, getClientAddress }) { const id = z.string().parse(params.id); @@ -498,6 +504,27 @@ export async function POST({ request, locals, params, getClientAddress }) { value: "Error within the websearch. Try again later or with a different query", }; } + } else if (tool.name === "text2img") { + const inference = new HfInference(HF_TOKEN ?? HF_ACCESS_TOKEN); + const img = await inference.textToImage({ + model: "runwayml/stable-diffusion-v1-5", + inputs: call.parameters?.prompt, + }); + + const sha = await uploadFile(img, conv); + + messageToWriteTo.files = [...(messageToWriteTo.files ?? []), sha]; + + update({ + type: "file", + sha, + }); + + return { + key: call.tool_name, + status: "success", + value: `An image has been generated for the following prompt: ${call.parameters?.prompt}. Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it.`, + }; } else { return { key: call.tool_name, diff --git a/src/routes/conversation/[id]/output/[sha256]/+server.ts b/src/routes/conversation/[id]/output/[sha256]/+server.ts index 79ae37b7585..dad1550aa64 100644 --- a/src/routes/conversation/[id]/output/[sha256]/+server.ts +++ b/src/routes/conversation/[id]/output/[sha256]/+server.ts @@ -29,7 +29,7 @@ export const GET: RequestHandler = async ({ locals, params }) => { throw error(404, "Conversation not found"); } } else { - // check if the user has access to the conversation + // look for the conversation in shared conversations const conv = await collections.sharedConversations.findOne({ _id: params.id, }); diff --git a/vite.config.ts b/vite.config.ts index 0ffbd0482e3..4b5a2526177 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -26,6 +26,6 @@ export default defineConfig({ loadTTFAsArrayBuffer(), ], optimizeDeps: { - include: ["browser-image-resizer", "uuid"], + include: ["browser-image-resizer", "uuid", "@xenova/transformers"], }, }); From 4975d3f68ffb9201f7f2fc3bdc9f37b05247bc27 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Mon, 15 Apr 2024 12:27:22 +0200 Subject: [PATCH 09/82] group tool calls together --- src/lib/components/chat/ChatMessage.svelte | 68 ++++++++++++++-------- src/lib/types/MessageUpdate.ts | 1 + src/routes/conversation/[id]/+server.ts | 5 ++ 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 3430413616a..cc5faf09334 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -131,7 +131,18 @@ $: searchUpdates = (message.updates?.filter(({ type }) => type === "webSearch") ?? []) as WebSearchUpdate[]; - $: tools = (message.updates?.filter(({ type }) => type === "tool") ?? []) as ToolUpdate[]; + // filter all updates with type === "tool" then group them by uuid field + + $: tools = message.updates + ?.filter(({ type }) => type === "tool") + ?.reduce((acc, update) => { + if (update.type !== "tool") { + return acc; + } + acc[update.uuid] = acc[update.uuid] ?? []; + acc[update.uuid].push(update); + return acc; + }, {} as Record); $: downloadLink = message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined; @@ -232,30 +243,39 @@ /> {/if} - {#if tools.length > 0} -
- {#each tools as tool} - {#if tool.messageType === "parameters"} -

- Calling tool {tool.name} with the - parameters: -
    - {#each Object.entries(tool.parameters ?? {}) as [k, v]} -
  • - {k}: {v} -
    • - {/each} -
-

- {:else if tool.messageType === "message"} -

- {" > "}{tool.message} -

+ {#if tools} + {#each Object.values(tools) as tool} + {#if tool.length > 0} + {@const toolName = tool.filter((t) => t.messageType === "parameters")[0].name} + {#if toolName} +
+ + Calling tool {toolName} + + {#each tool as toolUpdate} +
+ {#if toolUpdate.messageType === "parameters"} +

Parameters:

+
    + {#each Object.entries(toolUpdate.parameters ?? {}) as [k, v]} +
  • + {k}: {v} +
    • + {/each} +
+ {:else if toolUpdate.messageType === "message"} +

Result:

+

+ {" > "}{toolUpdate.message} +

+ {/if} + {/each} +
{/if} - {/each} -
+ {/if} + {/each} {/if}
{ + const uuid = v4(); + if (call.tool_name === "directly-answer" || call.tool_name === "directly_answer") { return null; } @@ -463,6 +466,7 @@ export async function POST({ request, locals, params, getClientAddress }) { name: call.tool_name, messageType: "parameters", parameters: call.parameters, + uuid, }); const tool = tools.find((el) => el.name === call.tool_name); @@ -544,6 +548,7 @@ export async function POST({ request, locals, params, getClientAddress }) { name: toolAnswer.key, messageType: "message", message: toolAnswer.value, + uuid, }); } return toolAnswer; From ea801a7dd5be276a44589b93d3dbf80de7aab342 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Mon, 15 Apr 2024 12:33:29 +0200 Subject: [PATCH 10/82] show retry even if no message --- src/lib/components/chat/ChatMessage.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index cc5faf09334..19bc7bbeb73 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -313,7 +313,7 @@
{/if}
- {#if !loading && message.content} + {#if !loading && (message.content || tools)}
); - $: downloadLink = - message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined; + $: downloadLink = urlNotTrailing + `/message/${message.id}/prompt`; let webSearchIsDone = true; @@ -221,7 +222,7 @@ {#if file.length === 64} input from user @@ -380,7 +381,7 @@ {#if file.length === 64} input from user From 19fb04fbe79ae3ab3937c58ebc76e3d8c1a855c3 Mon Sep 17 00:00:00 2001 From: Nathan Date: Tue, 16 Apr 2024 12:08:18 +0000 Subject: [PATCH 12/82] Add image popup --- src/lib/components/chat/ChatMessage.svelte | 84 ++++++++++++++-------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 91cd1cbd4f6..0c611cf22e4 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -24,6 +24,7 @@ import type { ToolUpdate, WebSearchUpdate } from "$lib/types/MessageUpdate"; import { base } from "$app/paths"; import { useConvTreeStore } from "$lib/stores/convTree"; + import Modal from "../Modal.svelte"; function sanitizeMd(md: string) { let ret = md @@ -191,8 +192,30 @@ const convTreeStore = useConvTreeStore(); $: if (message.children?.length === 0) $convTreeStore.leaf = message.id; + + $: modalImageToShow = ""; +{#if modalImageToShow} + + (modalImageToShow = '')}> + {#if modalImageToShow.length === 64} + input from user + {:else} + + input from user + {/if} + +{/if} + {#if message.from === "assistant"}
{#each message.files as file} - {#if file.length === 64} - input from user - {:else} - - input from user - {/if} + {/each}
{/if} @@ -378,21 +403,22 @@ {#if message.files && message.files.length > 0}
{#each message.files as file} - - {#if file.length === 64} - input from user - {:else} - - input from user - {/if} + {/each}
{/if} From ce1575e8fcd39858646362153c1b762e95447510 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Tue, 16 Apr 2024 16:43:14 +0200 Subject: [PATCH 13/82] format --- src/lib/components/chat/ChatMessage.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 84eb752c85d..a348a791ede 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -198,7 +198,7 @@ {#if modalImageToShow} - (modalImageToShow = '')}> + (modalImageToShow = "")}> {#if modalImageToShow.length === 64} Date: Mon, 22 Apr 2024 16:34:44 +0200 Subject: [PATCH 14/82] wip --- .env | 2 ++ src/lib/components/ToolsToggle.svelte | 37 +++++++++++++++++++++++ src/lib/components/chat/ChatWindow.svelte | 5 ++- src/lib/server/models.ts | 2 +- src/lib/server/tools/index.ts | 3 +- src/lib/stores/webSearchParameters.ts | 2 ++ src/lib/types/Model.ts | 1 + src/lib/utils/messageUpdates.ts | 2 ++ src/routes/+layout.server.ts | 1 + src/routes/conversation/[id]/+page.svelte | 6 +++- src/routes/conversation/[id]/+server.ts | 21 ++++++++----- 11 files changed, 69 insertions(+), 13 deletions(-) create mode 100644 src/lib/components/ToolsToggle.svelte diff --git a/.env b/.env index b12666ea1ec..785eaab6e17 100644 --- a/.env +++ b/.env @@ -150,3 +150,5 @@ WEBHOOK_URL_REPORT_ASSISTANT=#provide webhook url to get notified when an assist ALLOWED_USER_EMAILS=`[]` # if it's defined, only these emails will be allowed to use the app USAGE_LIMITS=`{}` + +IMAGE_GENERATION_MODEL=runwayml/stable-diffusion-v1-5 \ No newline at end of file diff --git a/src/lib/components/ToolsToggle.svelte b/src/lib/components/ToolsToggle.svelte new file mode 100644 index 00000000000..bdb96f359b1 --- /dev/null +++ b/src/lib/components/ToolsToggle.svelte @@ -0,0 +1,37 @@ + + +
+ +
Tools
+
+ +
+

+ This model has function calling capabilities. When enabled, the model will try to use tools + such as web search or image generation to give a better answer. +

+
+
+
diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index 48b443c7b3f..ec0e276effa 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -15,6 +15,7 @@ import StopGeneratingBtn from "../StopGeneratingBtn.svelte"; import type { Model } from "$lib/types/Model"; import WebSearchToggle from "../WebSearchToggle.svelte"; + import ToolsToggle from "../ToolsToggle.svelte"; import LoginModal from "../LoginModal.svelte"; import { page } from "$app/stores"; import FileDropzone from "./FileDropzone.svelte"; @@ -256,8 +257,10 @@
- {#if $page.data.settings?.searchEnabled && !assistant} + {#if $page.data.settings?.searchEnabled && !assistant && !currentModel.functions} + {:else if currentModel.functions} + {/if} {#if loading} dispatch("stop")} /> diff --git a/src/lib/server/models.ts b/src/lib/server/models.ts index 40d87bd2246..c038a8a2203 100644 --- a/src/lib/server/models.ts +++ b/src/lib/server/models.ts @@ -253,5 +253,5 @@ export const smallModel = TASK_MODEL export type BackendModel = Optional< typeof defaultModel, - "preprompt" | "parameters" | "multimodal" | "unlisted" + "preprompt" | "parameters" | "multimodal" | "unlisted" | "functions" >; diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index 21a47d8d699..4e89688c442 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -1,7 +1,6 @@ import type { Tool, ToolResult } from "$lib/types/Tool"; import calculator from "./calculator"; import directlyAnswer from "./directlyAnswer"; -import getWeather from "./getWeather"; import text2img from "./text2img"; import websearch from "./websearch"; @@ -9,4 +8,4 @@ export interface BackendTool extends Tool { call?(params: Record): Promise; } -export const tools: BackendTool[] = [calculator, getWeather, websearch, text2img, directlyAnswer]; +export const tools: BackendTool[] = [calculator, websearch, text2img, directlyAnswer]; diff --git a/src/lib/stores/webSearchParameters.ts b/src/lib/stores/webSearchParameters.ts index fd088a60621..a4ee2ab0128 100644 --- a/src/lib/stores/webSearchParameters.ts +++ b/src/lib/stores/webSearchParameters.ts @@ -1,9 +1,11 @@ import { writable } from "svelte/store"; export interface WebSearchParameters { useSearch: boolean; + useTools: boolean; nItems: number; } export const webSearchParameters = writable({ useSearch: false, + useTools: true, // since the model will decide whether or not to use tools, it makes sense to default to true here nItems: 5, }); diff --git a/src/lib/types/Model.ts b/src/lib/types/Model.ts index c58978b9c46..c9082980bbe 100644 --- a/src/lib/types/Model.ts +++ b/src/lib/types/Model.ts @@ -17,4 +17,5 @@ export type Model = Pick< | "preprompt" | "multimodal" | "unlisted" + | "functions" >; diff --git a/src/lib/utils/messageUpdates.ts b/src/lib/utils/messageUpdates.ts index 82b8bb9a2ca..8190bc71dd2 100644 --- a/src/lib/utils/messageUpdates.ts +++ b/src/lib/utils/messageUpdates.ts @@ -7,6 +7,7 @@ type MessageUpdateRequestOptions = { isRetry: boolean; isContinue: boolean; webSearch: boolean; + tools: boolean; files?: string[]; }; export async function fetchMessageUpdates( @@ -26,6 +27,7 @@ export async function fetchMessageUpdates( is_retry: opts.isRetry, is_continue: opts.isContinue, web_search: opts.webSearch, + tools: opts.tools, files: opts.files, }), signal: abortController.signal, diff --git a/src/routes/+layout.server.ts b/src/routes/+layout.server.ts index c353bc4f47c..75b6fa5b4b1 100644 --- a/src/routes/+layout.server.ts +++ b/src/routes/+layout.server.ts @@ -168,6 +168,7 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => { parameters: model.parameters, preprompt: model.preprompt, multimodal: model.multimodal, + functions: model.functions, unlisted: model.unlisted, })), oldModels, diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte index e680b41f8c2..01367111b27 100644 --- a/src/routes/conversation/[id]/+page.svelte +++ b/src/routes/conversation/[id]/+page.svelte @@ -22,7 +22,9 @@ export let data; - $: ({ messages } = data); + $: ({ messages, model: modelId } = data); + + $: currentModel = findCurrentModel([...data.models, ...data.oldModels], modelId); let loading = false; let pending = false; @@ -198,6 +200,8 @@ isRetry, isContinue, webSearch: !hasAssistant && $webSearchParameters.useSearch, + tools: + (!hasAssistant && currentModel.functions && $webSearchParameters.useTools) ?? false, files: isRetry ? undefined : resizedImages, }, messageUpdatesAbortController.signal diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 5f26c9e634c..258c7f9b226 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -3,6 +3,7 @@ import { ENABLE_ASSISTANTS_RAG, HF_TOKEN, HF_ACCESS_TOKEN, + IMAGE_GENERATION_MODEL, } from "$env/static/private"; import { startOfHour } from "date-fns"; import { authCondition, requiresUser } from "$lib/server/auth"; @@ -33,6 +34,10 @@ import JSON5 from "json5"; import type { Call, ToolResult } from "$lib/types/Tool.js"; import { HfInference } from "@huggingface/inference"; import { v4 } from "uuid"; +import directlyAnswer from "$lib/server/tools/directlyAnswer.js"; +import calculator from "$lib/server/tools/calculator.js"; +import websearch from "$lib/server/tools/websearch.js"; +import text2img from "$lib/server/tools/text2img.js"; export async function POST({ request, locals, params, getClientAddress }) { const id = z.string().parse(params.id); @@ -143,6 +148,7 @@ export async function POST({ request, locals, params, getClientAddress }) { is_retry: isRetry, is_continue: isContinue, web_search: webSearch, + tools: toolsFlag, files: b64files, } = z .object({ @@ -157,6 +163,7 @@ export async function POST({ request, locals, params, getClientAddress }) { is_retry: z.optional(z.boolean()), is_continue: z.optional(z.boolean()), web_search: z.optional(z.boolean()), + tools: z.optional(z.boolean()), files: z.optional(z.array(z.string())), }) .parse(json); @@ -414,15 +421,13 @@ export async function POST({ request, locals, params, getClientAddress }) { let toolResults: ToolResult[] | undefined = undefined; // function calls - if (model.functions && (assistant?.functionSpec || assistantHasWebSearch)) { - // turn functionSpec into a list of tools + if (model.functions && (toolsFlag || assistantHasWebSearch)) { + let tools = [directlyAnswer, calculator, websearch]; - let tools = await getToolsFromFunctionSpec(assistant?.functionSpec); - - // remove websearch tool if relevant - if (!assistantHasWebSearch) { - tools = tools.filter(({ name }) => name !== "websearch"); + if (!assistant) { + tools = [...tools, text2img]; } + let calls: Call[] | undefined = undefined; // do the function calling bits here @@ -511,7 +516,7 @@ export async function POST({ request, locals, params, getClientAddress }) { } else if (tool.name === "text2img") { const inference = new HfInference(HF_TOKEN ?? HF_ACCESS_TOKEN); const img = await inference.textToImage({ - model: "runwayml/stable-diffusion-v1-5", + model: IMAGE_GENERATION_MODEL, inputs: call.parameters?.prompt, }); From 21a1e917072c09e0c32b4e8ab49b85084c61fd9a Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Wed, 24 Apr 2024 09:33:30 +0200 Subject: [PATCH 15/82] clean-up --- src/lib/components/AssistantSettings.svelte | 6 +----- src/lib/components/ToolsToggle.svelte | 8 +------- src/lib/components/chat/ChatMessage.svelte | 13 +++++++++---- src/lib/components/chat/ChatWindow.svelte | 10 ++++++---- src/lib/server/tools/websearch.ts | 2 +- src/routes/conversation/[id]/+server.ts | 6 +++--- 6 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/lib/components/AssistantSettings.svelte b/src/lib/components/AssistantSettings.svelte index 77be2a4a292..e79c355fccd 100644 --- a/src/lib/components/AssistantSettings.svelte +++ b/src/lib/components/AssistantSettings.svelte @@ -533,16 +533,12 @@ each inference.

- -
- Function calling -

{getError("functionSpec", form)}

{/if}
diff --git a/src/lib/components/ToolsToggle.svelte b/src/lib/components/ToolsToggle.svelte index bdb96f359b1..e7e52d90f4e 100644 --- a/src/lib/components/ToolsToggle.svelte +++ b/src/lib/components/ToolsToggle.svelte @@ -15,13 +15,7 @@ role="switch" tabindex="0" > - +
Tools
diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index a348a791ede..91b7fdd170d 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -273,16 +273,19 @@ {#each Object.values(tools) as tool} {#if tool.length > 0} {@const toolName = tool.filter((t) => t.messageType === "parameters")[0].name} - {#if toolName} + {#if toolName && toolName !== "websearch"}
Calling tool {toolName} {#each tool as toolUpdate} -
{#if toolUpdate.messageType === "parameters"} +
+

Parameters:

    {#each Object.entries(toolUpdate.parameters ?? {}) as [k, v]} @@ -291,7 +294,9 @@ {/each}
- {:else if toolUpdate.messageType === "message"} + {:else if toolUpdate.messageType === "message" && toolName !== "text2img"} +
+

Result:

{" > "}{toolUpdate.message} diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index ec0e276effa..8a38a33056b 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -257,10 +257,12 @@

- {#if $page.data.settings?.searchEnabled && !assistant && !currentModel.functions} - - {:else if currentModel.functions} - + {#if !assistant} + {#if currentModel.functions} + + {:else if $page.data.settings?.searchEnabled} + + {/if} {/if} {#if loading} dispatch("stop")} /> diff --git a/src/lib/server/tools/websearch.ts b/src/lib/server/tools/websearch.ts index 2e53b30c7e4..ad6d0609f22 100644 --- a/src/lib/server/tools/websearch.ts +++ b/src/lib/server/tools/websearch.ts @@ -3,7 +3,7 @@ import type { BackendTool } from "."; const websearch: BackendTool = { name: "websearch", description: - "Use this tool to search web pages for answers that will help answer the user's query.", + "Use this tool to search web pages for answers that will help answer the user's query. Only use this tool if you need specific resources from the internet.", parameter_definitions: { query: { required: true, diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index f1367dc5f48..eaf0cceb090 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -29,7 +29,6 @@ import { addSibling } from "$lib/utils/tree/addSibling.js"; import { preprocessMessages } from "$lib/server/preprocessMessages.js"; import { usageLimits } from "$lib/server/usageLimits"; import { isURLLocal } from "$lib/server/isURLLocal.js"; -import { getToolsFromFunctionSpec } from "$lib/utils/getToolsFromFunctionSpec.js"; import JSON5 from "json5"; import type { Call, ToolResult } from "$lib/types/Tool.js"; import { HfInference } from "@huggingface/inference"; @@ -422,10 +421,11 @@ export async function POST({ request, locals, params, getClientAddress }) { // function calls if (model.functions && (toolsFlag || assistantHasWebSearch)) { - let tools = [directlyAnswer, calculator, websearch]; + let tools = [directlyAnswer, websearch]; + // if it's an assistant, only support websearch for now if (!assistant) { - tools = [...tools, text2img]; + tools = [...tools, text2img, calculator]; } let calls: Call[] | undefined = undefined; From 0a1db84dfe8a35cec8fd103e930aca001076a548 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Wed, 24 Apr 2024 14:53:34 +0200 Subject: [PATCH 16/82] better loading indicator --- src/lib/components/chat/ChatMessage.svelte | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 91b7fdd170d..66ff4087464 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -16,7 +16,7 @@ import CarbonPen from "~icons/carbon/pen"; import CarbonChevronLeft from "~icons/carbon/chevron-left"; import CarbonChevronRight from "~icons/carbon/chevron-right"; - + import EosIconLoading from "~icons/eos-icons/loading"; import { PUBLIC_SEP_TOKEN } from "$lib/constants/publicSepToken"; import type { Model } from "$lib/types/Model"; @@ -273,14 +273,26 @@ {#each Object.values(tools) as tool} {#if tool.length > 0} {@const toolName = tool.filter((t) => t.messageType === "parameters")[0].name} + {@const toolDone = !!tool.find((t) => t.messageType === "message")} {#if toolName && toolName !== "websearch"}
- - Calling tool {toolName} + + {#if !toolDone} + + {/if} + {toolDone ? "Called" : "Calling"} tool + {toolName} {#each tool as toolUpdate} {#if toolUpdate.messageType === "parameters"} From b4e3170e74ac3027f8f9300fe25b804e22ae00e6 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Wed, 24 Apr 2024 14:59:13 +0200 Subject: [PATCH 17/82] text colors --- src/lib/components/chat/ChatMessage.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 66ff4087464..df8fa66280b 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -284,8 +284,8 @@ class="pb-1 group-open/tool:text-purple-700 group-open/tool:dark:text-purple-300" class:text-purple-700={!toolDone} class:dark:text-purple-300={!toolDone} - class:text-gray-800={toolDone} - class:dark:text-gray-200={toolDone} + class:text-gray-600={toolDone} + class:dark:text-gray-400={toolDone} class:list-none={!toolDone} > {#if !toolDone} From 88f953ef64910f28f5dd5486c0a017959044155b Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 25 Apr 2024 10:15:33 +0200 Subject: [PATCH 18/82] extra example to summarize --- src/lib/server/summarize.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/server/summarize.ts b/src/lib/server/summarize.ts index 5823f760e6e..5da0dae52c8 100644 --- a/src/lib/server/summarize.ts +++ b/src/lib/server/summarize.ts @@ -27,6 +27,8 @@ export async function summarize(prompt: string) { { from: "assistant", content: "🎥 Favorite movie" }, { from: "user", content: "Explain the concept of artificial intelligence in one sentence" }, { from: "assistant", content: "🤖 AI definition" }, + { from: "user", content: "Draw a cute cat" }, + { from: "assistant", content: "🐱 Cute cat drawing" }, { from: "user", content: prompt }, ]; From 61c2beafda56b4cfdb93da4d1d81551caeafdc06 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 25 Apr 2024 10:43:25 +0200 Subject: [PATCH 19/82] switch default model --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index db033eea7e4..eb499d08b54 100644 --- a/.env +++ b/.env @@ -154,4 +154,4 @@ ALLOWED_USER_EMAILS=`[]` # if it's defined, only these emails will be allowed to USAGE_LIMITS=`{}` -IMAGE_GENERATION_MODEL=runwayml/stable-diffusion-v1-5 \ No newline at end of file +IMAGE_GENERATION_MODEL=stabilityai/stable-diffusion-xl-base-1.0 \ No newline at end of file From dc38800ffdab256c9290d891bc2fc4ebddf12ffe Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Fri, 26 Apr 2024 12:10:30 +0200 Subject: [PATCH 20/82] Add fetchUrl tool --- src/lib/components/chat/ChatMessage.svelte | 2 +- src/lib/server/tools/fetchUrl.ts | 37 ++++++++++++++++++++++ src/lib/server/tools/getWeather.ts | 37 ---------------------- src/lib/server/websearch/parseWeb.ts | 5 ++- src/lib/types/MessageUpdate.ts | 1 + src/lib/types/Tool.ts | 1 + src/routes/conversation/[id]/+server.ts | 9 ++++-- 7 files changed, 51 insertions(+), 41 deletions(-) create mode 100644 src/lib/server/tools/fetchUrl.ts delete mode 100644 src/lib/server/tools/getWeather.ts diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index df8fa66280b..8946fb2f019 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -306,7 +306,7 @@ {/each} - {:else if toolUpdate.messageType === "message" && toolName !== "text2img"} + {:else if toolUpdate.messageType === "message" && toolUpdate.display !== false}

Result:

diff --git a/src/lib/server/tools/fetchUrl.ts b/src/lib/server/tools/fetchUrl.ts new file mode 100644 index 00000000000..e15e9d40cda --- /dev/null +++ b/src/lib/server/tools/fetchUrl.ts @@ -0,0 +1,37 @@ +import type { BackendTool } from "."; +import { parseWeb } from "../websearch/parseWeb"; + +const fetchUrl: BackendTool = { + name: "fetchUrl", + description: "A tool that can be used to fetch an URL and return the content directly.", + parameter_definitions: { + url: { + description: "The url that should be fetched.", + type: "str", + required: true, + }, + }, + call: async (params) => { + try { + const blocks = params.url.split("\n"); + const url = blocks[blocks.length - 1]; + + const content = await parseWeb(url); + + return { + key: "fetchUrl", + status: "success", + value: content, + display: false, + }; + } catch (e) { + return { + key: "fetchUrl", + status: "error", + value: "Fetching the webpage failed because :" + (e as Error).message, + }; + } + }, +}; + +export default fetchUrl; diff --git a/src/lib/server/tools/getWeather.ts b/src/lib/server/tools/getWeather.ts deleted file mode 100644 index e7e4ac4dc20..00000000000 --- a/src/lib/server/tools/getWeather.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { BackendTool } from "."; - -const getWeather: BackendTool = { - name: "get_weather", - description: - "Get the weather forecast for a given location. The location can be a city, country, or even a zip code.", - parameter_definitions: { - location: { - description: "The location to get the weather for.", - type: "str", - required: true, - }, - }, - call: async (params) => { - if (params.location === "New York") { - return { - key: "get_weather", - status: "success", - value: "It's sunny. 26C. 10% chance of rain.", - }; - } else if (params.location === "London") { - return { - key: "get_weather", - status: "success", - value: "It's cloudy. 18C. 50% chance of rain.", - }; - } else { - return { - key: "get_weather", - status: "error", - value: "Location not found", - }; - } - }, -}; - -export default getWeather; diff --git a/src/lib/server/websearch/parseWeb.ts b/src/lib/server/websearch/parseWeb.ts index e25253d11fe..7758993997a 100644 --- a/src/lib/server/websearch/parseWeb.ts +++ b/src/lib/server/websearch/parseWeb.ts @@ -30,7 +30,10 @@ export async function parseWeb(url: string) { return text; } else if ( r.headers.get("content-type")?.includes("text/plain") || - r.headers.get("content-type")?.includes("text/markdown") + r.headers.get("content-type")?.includes("text/markdown") || + r.headers.get("content-type")?.includes("application/json") || + r.headers.get("content-type")?.includes("application/xml") || + r.headers.get("content-type")?.includes("text/csv") ) { return r.text(); } else { diff --git a/src/lib/types/MessageUpdate.ts b/src/lib/types/MessageUpdate.ts index 189214b2134..e0718a1d409 100644 --- a/src/lib/types/MessageUpdate.ts +++ b/src/lib/types/MessageUpdate.ts @@ -25,6 +25,7 @@ interface ToolUpdateParams extends ToolUpdateBase { interface ToolUpdateMessage extends ToolUpdateBase { messageType: "message"; message?: string; + display?: boolean; } export type ToolUpdate = ToolUpdateParams | ToolUpdateMessage; diff --git a/src/lib/types/Tool.ts b/src/lib/types/Tool.ts index dc1191aa4e5..7a8e6f54bb7 100644 --- a/src/lib/types/Tool.ts +++ b/src/lib/types/Tool.ts @@ -15,6 +15,7 @@ export interface ToolResult { key: string; status: "success" | "error"; value: string; + display?: boolean; } export interface Call { diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index ed81bd314d2..9ad818e6813 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -4,6 +4,7 @@ import { HF_TOKEN, HF_ACCESS_TOKEN, IMAGE_GENERATION_MODEL, + ENABLE_LOCAL_FETCH, } from "$env/static/private"; import { startOfHour } from "date-fns"; import { authCondition, requiresUser } from "$lib/server/auth"; @@ -37,6 +38,7 @@ import directlyAnswer from "$lib/server/tools/directlyAnswer.js"; import calculator from "$lib/server/tools/calculator.js"; import websearch from "$lib/server/tools/websearch.js"; import text2img from "$lib/server/tools/text2img.js"; +import fetchUrl from "$lib/server/tools/fetchUrl.js"; export async function POST({ request, locals, params, getClientAddress }) { const id = z.string().parse(params.id); @@ -409,7 +411,7 @@ export async function POST({ request, locals, params, getClientAddress }) { while ((match = urlRegex.exec(preprompt)) !== null) { try { const url = new URL(match[1]); - if (await isURLLocal(url)) { + if ((await isURLLocal(url)) && ENABLE_LOCAL_FETCH !== "true") { throw new Error("URL couldn't be fetched, it resolved to a local address."); } @@ -439,7 +441,7 @@ export async function POST({ request, locals, params, getClientAddress }) { // if it's an assistant, only support websearch for now if (!assistant) { - tools = [...tools, text2img, calculator]; + tools = [...tools, text2img, calculator, fetchUrl]; } let calls: Call[] | undefined = undefined; @@ -519,6 +521,7 @@ export async function POST({ request, locals, params, getClientAddress }) { key: call.tool_name, status: "success", value: chunks, + display: false, }; } catch (e) { return { @@ -547,6 +550,7 @@ export async function POST({ request, locals, params, getClientAddress }) { key: call.tool_name, status: "success", value: `An image has been generated for the following prompt: ${call.parameters?.prompt}. Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it.`, + display: false, }; } else { return { @@ -567,6 +571,7 @@ export async function POST({ request, locals, params, getClientAddress }) { name: toolAnswer.key, messageType: "message", message: toolAnswer.value, + display: toolAnswer.display, uuid, }); } From a9400c66c0fbf47f9f4ac674a836181febb81480 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Fri, 26 Apr 2024 17:34:27 +0200 Subject: [PATCH 21/82] wip --- package-lock.json | 48 ++++++++++++++++++++++--- package.json | 1 + src/routes/conversation/[id]/+server.ts | 22 ++++++++---- 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 762abd08c32..6fab34ebbc4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,7 @@ }, "devDependencies": { "@faker-js/faker": "^8.4.1", + "@gradio/client": "https://gitpkg.now.sh/gradio-app/gradio/client/js?2a2a151", "@iconify-json/carbon": "^1.1.16", "@iconify-json/eos-icons": "^1.1.6", "@sveltejs/adapter-node": "^1.3.1", @@ -741,6 +742,23 @@ "node": ">=18.0.0" } }, + "node_modules/@gradio/client": { + "version": "0.16.0", + "resolved": "https://gitpkg.now.sh/gradio-app/gradio/client/js?2a2a151", + "integrity": "sha512-AMMKl3aD4vNwictF8F2jCz2D/jN0OI5qu66PU1ajUckNREgbgeQRXNWZsTCXl8xZ0QTC8NBttUDcKzA3q2aghg==", + "dev": true, + "dependencies": { + "@types/eventsource": "^1.1.15", + "bufferutil": "^4.0.7", + "eventsource": "^2.0.2", + "semiver": "^1.1.0", + "typescript": "^5.0.0", + "ws": "^8.13.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@huggingface/hub": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/@huggingface/hub/-/hub-0.5.1.tgz", @@ -2117,6 +2135,12 @@ "integrity": "sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==", "dev": true }, + "node_modules/@types/eventsource": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@types/eventsource/-/eventsource-1.1.15.tgz", + "integrity": "sha512-XQmGcbnxUNa06HR3VBVkc9+A2Vpi9ZyLJcdS5dwaQQ/4ZMWFO+5c90FnMUpbtMZwB/FChoYHwuVg8TvkECacTA==", + "dev": true + }, "node_modules/@types/jsdom": { "version": "21.1.1", "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.1.tgz", @@ -2948,9 +2972,8 @@ "version": "4.0.7", "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.7.tgz", "integrity": "sha512-kukuqc39WOHtdxtw4UScxF/WVnMFVSQVKhtx3AjZJzhd0RGZZldcrfSEbVsWWe6KNH253574cq5F+wpv0G9pJw==", + "devOptional": true, "hasInstallScript": true, - "optional": true, - "peer": true, "dependencies": { "node-gyp-build": "^4.3.0" }, @@ -3987,6 +4010,15 @@ "node": ">=6" } }, + "node_modules/eventsource": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-2.0.2.tgz", + "integrity": "sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA==", + "dev": true, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -5615,8 +5647,7 @@ "version": "4.6.1", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.1.tgz", "integrity": "sha512-24vnklJmyRS8ViBNI8KbtK/r/DmXQMRiOMXTNz2nrTnAYUwjmEEbnnpB/+kt+yWRv73bPsSPRFddrcIbAxSiMQ==", - "optional": true, - "peer": true, + "devOptional": true, "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", @@ -6822,6 +6853,15 @@ "node": ">=v12.22.7" } }, + "node_modules/semiver": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/semiver/-/semiver-1.1.0.tgz", + "integrity": "sha512-QNI2ChmuioGC1/xjyYwyZYADILWyW6AmS1UH6gDj/SFUUUS4MBAWs/7mxnkRPc/F4iHezDP+O8t0dO8WHiEOdg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/semver": { "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", diff --git a/package.json b/package.json index 3b872144dd8..c2ac9ee9b31 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ }, "devDependencies": { "@faker-js/faker": "^8.4.1", + "@gradio/client": "https://gitpkg.now.sh/gradio-app/gradio/client/js?2a2a151", "@iconify-json/carbon": "^1.1.16", "@iconify-json/eos-icons": "^1.1.6", "@sveltejs/adapter-node": "^1.3.1", diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 9ad818e6813..0f956f4f0b7 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -3,7 +3,6 @@ import { ENABLE_ASSISTANTS_RAG, HF_TOKEN, HF_ACCESS_TOKEN, - IMAGE_GENERATION_MODEL, ENABLE_LOCAL_FETCH, } from "$env/static/private"; import { startOfHour } from "date-fns"; @@ -32,8 +31,10 @@ import { usageLimits } from "$lib/server/usageLimits"; import { isURLLocal } from "$lib/server/isURLLocal.js"; import JSON5 from "json5"; import type { Call, ToolResult } from "$lib/types/Tool.js"; -import { HfInference } from "@huggingface/inference"; import { v4 } from "uuid"; + +import { Client } from "@gradio/client"; + import directlyAnswer from "$lib/server/tools/directlyAnswer.js"; import calculator from "$lib/server/tools/calculator.js"; import websearch from "$lib/server/tools/websearch.js"; @@ -531,13 +532,20 @@ export async function POST({ request, locals, params, getClientAddress }) { }; } } else if (tool.name === "text2img") { - const inference = new HfInference(HF_TOKEN ?? HF_ACCESS_TOKEN); - const img = await inference.textToImage({ - model: IMAGE_GENERATION_MODEL, - inputs: call.parameters?.prompt, + const app = await Client.connect("ByteDance/Hyper-SDXL-1Step-T2I", { + hf_token: (HF_TOKEN ?? HF_ACCESS_TOKEN) as unknown as `hf_${string}`, }); + const res = await app.predict("/process_image", [ + 1, // number (numeric value between 1 and 8) in 'Number of Images' Slider component + 512, // number in 'Image Height' Number component + 512, // number in 'Image Width' Number component + call.parameters?.prompt, // prompt + Math.floor(Math.random() * 1000), // seed random + ]); + + const response = await fetch(res.data[0][0].image.url); - const sha = await uploadFile(img, conv); + const sha = await uploadFile(await response.blob(), conv); messageToWriteTo.files = [...(messageToWriteTo.files ?? []), sha]; From 920c989843f8df04036b8af7ebb7611446036262 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Sun, 5 May 2024 16:15:45 +0200 Subject: [PATCH 22/82] Add latest gradio --- package-lock.json | 438 +++++++++++++++++++++++++++++++++++++++++++--- package.json | 2 +- 2 files changed, 419 insertions(+), 21 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9b4bb91c237..c6d2bb85a84 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.8.3", "dependencies": { "@apidevtools/swagger-parser": "^10.1.0", + "@gradio/client": "^0.18.0", "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", @@ -46,7 +47,6 @@ }, "devDependencies": { "@faker-js/faker": "^8.4.1", - "@gradio/client": "https://gitpkg.now.sh/gradio-app/gradio/client/js?2a2a151", "@iconify-json/carbon": "^1.1.16", "@iconify-json/eos-icons": "^1.1.6", "@sveltejs/adapter-node": "^1.3.1", @@ -250,6 +250,22 @@ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, + "node_modules/@bundled-es-modules/cookie": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@bundled-es-modules/cookie/-/cookie-2.0.0.tgz", + "integrity": "sha512-Or6YHg/kamKHpxULAdSqhGqnWFneIXu1NKvvfBBzKGwpVsYuFIQ5aBPHDnnoR3ghW1nvSkALd+EF9iMtY7Vjxw==", + "dependencies": { + "cookie": "^0.5.0" + } + }, + "node_modules/@bundled-es-modules/statuses": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@bundled-es-modules/statuses/-/statuses-1.0.1.tgz", + "integrity": "sha512-yn7BklA5acgcBr+7w064fGV+SGIFySjCKpqjcWgBAIfrAkY+4GQTJJHQMeT3V/sgz23VTEVV8TtOmkvJAhFVfg==", + "dependencies": { + "statuses": "^2.0.1" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "devOptional": true, @@ -371,15 +387,14 @@ } }, "node_modules/@gradio/client": { - "version": "0.16.0", - "resolved": "https://gitpkg.now.sh/gradio-app/gradio/client/js?2a2a151", - "integrity": "sha512-AMMKl3aD4vNwictF8F2jCz2D/jN0OI5qu66PU1ajUckNREgbgeQRXNWZsTCXl8xZ0QTC8NBttUDcKzA3q2aghg==", - "dev": true, - "license": "ISC", + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/@gradio/client/-/client-0.18.0.tgz", + "integrity": "sha512-Y9c3RLuil+vZHHOD2eLAPnS3u42AoTpVsZyOzVbuQsg575FBpdCel0GKlfoYOGZGGImUPNWHau2tRB+qdpDFUw==", "dependencies": { "@types/eventsource": "^1.1.15", "bufferutil": "^4.0.7", "eventsource": "^2.0.2", + "msw": "^2.2.1", "semiver": "^1.1.0", "typescript": "^5.0.0", "ws": "^8.13.0" @@ -526,6 +541,76 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@inquirer/confirm": { + "version": "3.1.6", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.1.6.tgz", + "integrity": "sha512-Mj4TU29g6Uy+37UtpA8UpEOI2icBfpCwSW1QDtfx60wRhUy90s/kHPif2OXSSvuwDQT1lhAYRWUfkNf9Tecxvg==", + "dependencies": { + "@inquirer/core": "^8.1.0", + "@inquirer/type": "^1.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/core": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-8.1.0.tgz", + "integrity": "sha512-kfx0SU9nWgGe1f03ao/uXc85SFH1v2w3vQVH7QDGjKxdtJz+7vPitFtG++BTyJMYyYgH8MpXigutcXJeiQwVRw==", + "dependencies": { + "@inquirer/figures": "^1.0.1", + "@inquirer/type": "^1.3.1", + "@types/mute-stream": "^0.0.4", + "@types/node": "^20.12.7", + "@types/wrap-ansi": "^3.0.0", + "ansi-escapes": "^4.3.2", + "chalk": "^4.1.2", + "cli-spinners": "^2.9.2", + "cli-width": "^4.1.0", + "mute-stream": "^1.0.0", + "signal-exit": "^4.1.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^6.2.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/core/node_modules/@types/node": { + "version": "20.12.8", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.8.tgz", + "integrity": "sha512-NU0rJLJnshZWdE/097cdCBbyW1h4hEg0xpovcoAQYHl8dnEyp/NAOiE45pvc+Bd1Dt+2r94v2eGFpQJ4R7g+2w==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@inquirer/core/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@inquirer/figures": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.1.tgz", + "integrity": "sha512-mtup3wVKia3ZwULPHcbs4Mor8Voi+iIXEWD7wCNbIO6lYR62oPCTQyrddi5OMYVXHzeCSoneZwJuS8sBvlEwDw==", + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/type": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.3.1.tgz", + "integrity": "sha512-Pe3PFccjPVJV1vtlfVvm9OnlbxqdnP5QcscFEFEnK5quChf1ufZtM0r8mR5ToWHMxZOh0s8o/qp9ANGRTo/DAw==", + "engines": { + "node": ">=18" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.3", "license": "MIT", @@ -577,6 +662,30 @@ "sparse-bitfield": "^3.0.3" } }, + "node_modules/@mswjs/cookies": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@mswjs/cookies/-/cookies-1.1.0.tgz", + "integrity": "sha512-0ZcCVQxifZmhwNBoQIrystCb+2sWBY2Zw8lpfJBPCHGCA/HWqehITeCRVIv4VMy8MPlaHo2w2pTHFV2pFfqKPw==", + "engines": { + "node": ">=18" + } + }, + "node_modules/@mswjs/interceptors": { + "version": "0.26.15", + "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.26.15.tgz", + "integrity": "sha512-HM47Lu1YFmnYHKMBynFfjCp0U/yRskHj/8QEJW0CBEPOlw8Gkmjfll+S9b8M7V5CNDw2/ciRxjjnWeaCiblSIQ==", + "dependencies": { + "@open-draft/deferred-promise": "^2.2.0", + "@open-draft/logger": "^0.3.0", + "@open-draft/until": "^2.0.0", + "is-node-process": "^1.2.0", + "outvariant": "^1.2.1", + "strict-event-emitter": "^0.5.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "license": "MIT", @@ -606,6 +715,25 @@ "node": ">= 8" } }, + "node_modules/@open-draft/deferred-promise": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", + "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==" + }, + "node_modules/@open-draft/logger": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", + "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", + "dependencies": { + "is-node-process": "^1.2.0", + "outvariant": "^1.4.0" + } + }, + "node_modules/@open-draft/until": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", + "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==" + }, "node_modules/@opentelemetry/api": { "version": "1.8.0", "dev": true, @@ -1027,8 +1155,7 @@ "node_modules/@types/eventsource": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/@types/eventsource/-/eventsource-1.1.15.tgz", - "integrity": "sha512-XQmGcbnxUNa06HR3VBVkc9+A2Vpi9ZyLJcdS5dwaQQ/4ZMWFO+5c90FnMUpbtMZwB/FChoYHwuVg8TvkECacTA==", - "dev": true + "integrity": "sha512-XQmGcbnxUNa06HR3VBVkc9+A2Vpi9ZyLJcdS5dwaQQ/4ZMWFO+5c90FnMUpbtMZwB/FChoYHwuVg8TvkECacTA==" }, "node_modules/@types/express": { "version": "4.17.21", @@ -1090,6 +1217,14 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/mute-stream": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.4.tgz", + "integrity": "sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/node": { "version": "18.13.0", "license": "MIT" @@ -1163,6 +1298,11 @@ "@types/send": "*" } }, + "node_modules/@types/statuses": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.5.tgz", + "integrity": "sha512-jmIUGWrAiwu3dZpxntxieC+1n/5c3mjrImkmOSQ2NC5uP6cYO4aAZDdSmRcI5C1oiTmqlZGHC+/NmJrKogbP5A==" + }, "node_modules/@types/tough-cookie": { "version": "4.0.2", "dev": true, @@ -1185,6 +1325,11 @@ "@types/webidl-conversions": "*" } }, + "node_modules/@types/wrap-ansi": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/wrap-ansi/-/wrap-ansi-3.0.0.tgz", + "integrity": "sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==" + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "6.7.4", "dev": true, @@ -1576,9 +1721,33 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-escapes/node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/ansi-regex": { "version": "5.0.1", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -1586,7 +1755,6 @@ }, "node_modules/ansi-styles": { "version": "4.3.0", - "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -1924,7 +2092,6 @@ }, "node_modules/bufferutil": { "version": "4.0.7", - "devOptional": true, "hasInstallScript": true, "license": "MIT", "dependencies": { @@ -2041,7 +2208,6 @@ }, "node_modules/chalk": { "version": "4.1.2", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", @@ -2109,6 +2275,54 @@ "version": "1.1.4", "license": "ISC" }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "engines": { + "node": ">= 12" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/code-red": { "version": "1.0.4", "dev": true, @@ -2240,7 +2454,6 @@ }, "node_modules/cookie": { "version": "0.5.0", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -2950,7 +3163,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-2.0.2.tgz", "integrity": "sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA==", - "dev": true, "engines": { "node": ">=12.0.0" } @@ -3346,6 +3558,14 @@ "node": ">=14" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-func-name": { "version": "2.0.2", "dev": true, @@ -3499,6 +3719,14 @@ "dev": true, "license": "MIT" }, + "node_modules/graphql": { + "version": "16.8.1", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", + "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==", + "engines": { + "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" + } + }, "node_modules/gtoken": { "version": "7.1.0", "license": "MIT", @@ -3536,7 +3764,6 @@ }, "node_modules/has-flag": { "version": "4.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -3586,6 +3813,11 @@ "node": ">= 0.4" } }, + "node_modules/headers-polyfill": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-4.0.3.tgz", + "integrity": "sha512-IScLbePpkvO846sIwOtOTDjutRMWdXdJmXdMvk6gCBHxFO8d+QKOQedyZSxFTTFYRSmlgSTDtXqqq4pcenBXLQ==" + }, "node_modules/help-me": { "version": "5.0.0", "license": "MIT" @@ -3830,6 +4062,14 @@ "node": ">=0.10.0" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, "node_modules/is-glob": { "version": "4.0.3", "license": "MIT", @@ -3845,6 +4085,11 @@ "dev": true, "license": "MIT" }, + "node_modules/is-node-process": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", + "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==" + }, "node_modules/is-number": { "version": "7.0.0", "license": "MIT", @@ -4481,6 +4726,77 @@ "version": "2.1.2", "license": "MIT" }, + "node_modules/msw": { + "version": "2.2.14", + "resolved": "https://registry.npmjs.org/msw/-/msw-2.2.14.tgz", + "integrity": "sha512-64i8rNCa1xzDK8ZYsTrVMli05D687jty8+Th+PU5VTbJ2/4P7fkQFVyDQ6ZFT5FrNR8z2BHhbY47fKNvfHrumA==", + "hasInstallScript": true, + "dependencies": { + "@bundled-es-modules/cookie": "^2.0.0", + "@bundled-es-modules/statuses": "^1.0.1", + "@inquirer/confirm": "^3.0.0", + "@mswjs/cookies": "^1.1.0", + "@mswjs/interceptors": "^0.26.14", + "@open-draft/until": "^2.1.0", + "@types/cookie": "^0.6.0", + "@types/statuses": "^2.0.4", + "chalk": "^4.1.2", + "graphql": "^16.8.1", + "headers-polyfill": "^4.0.2", + "is-node-process": "^1.2.0", + "outvariant": "^1.4.2", + "path-to-regexp": "^6.2.0", + "strict-event-emitter": "^0.5.1", + "type-fest": "^4.9.0", + "yargs": "^17.7.2" + }, + "bin": { + "msw": "cli/index.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/mswjs" + }, + "peerDependencies": { + "typescript": ">= 4.7.x" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/msw/node_modules/@types/cookie": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==" + }, + "node_modules/msw/node_modules/path-to-regexp": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz", + "integrity": "sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==" + }, + "node_modules/msw/node_modules/type-fest": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.18.1.tgz", + "integrity": "sha512-qXhgeNsX15bM63h5aapNFcQid9jRF/l3ojDoDFmekDQEUufZ9U4ErVt6SjDxnHp48Ltrw616R8yNc3giJ3KvVQ==", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mute-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", + "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/mz": { "version": "2.7.0", "license": "MIT", @@ -4598,7 +4914,6 @@ }, "node_modules/node-gyp-build": { "version": "4.6.1", - "devOptional": true, "license": "MIT", "bin": { "node-gyp-build": "bin.js", @@ -4819,6 +5134,11 @@ "node": ">= 0.8.0" } }, + "node_modules/outvariant": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.2.tgz", + "integrity": "sha512-Ou3dJ6bA/UJ5GVHxah4LnqDwZRwAmWxrG3wtrHrbGnP4RnLCtA64A4F+ae7Y8ww660JaddSoArUR5HjipWSHAQ==" + }, "node_modules/p-limit": { "version": "3.1.0", "dev": true, @@ -5737,6 +6057,14 @@ "node": ">= 12.13.0" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -5950,7 +6278,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/semiver/-/semiver-1.1.0.tgz", "integrity": "sha512-QNI2ChmuioGC1/xjyYwyZYADILWyW6AmS1UH6gDj/SFUUUS4MBAWs/7mxnkRPc/F4iHezDP+O8t0dO8WHiEOdg==", - "dev": true, "engines": { "node": ">=6" } @@ -6303,6 +6630,11 @@ "queue-tick": "^1.0.1" } }, + "node_modules/strict-event-emitter": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz", + "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==" + }, "node_modules/string_decoder": { "version": "1.3.0", "license": "MIT", @@ -6310,13 +6642,30 @@ "safe-buffer": "~5.2.0" } }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, "node_modules/string.prototype.codepointat": { "version": "0.2.1", "license": "MIT" }, "node_modules/strip-ansi": { "version": "6.0.1", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" @@ -6405,7 +6754,6 @@ }, "node_modules/supports-color": { "version": "7.2.0", - "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -6989,7 +7337,6 @@ }, "node_modules/typescript": { "version": "5.2.2", - "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -7029,6 +7376,11 @@ "node": ">=14.0" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/unicode-trie": { "version": "2.0.0", "license": "MIT", @@ -7620,6 +7972,19 @@ "version": "1.0.0", "license": "MIT" }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/wrappy": { "version": "1.0.2", "license": "ISC" @@ -7654,6 +8019,14 @@ "version": "2.2.0", "license": "MIT" }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "engines": { + "node": ">=10" + } + }, "node_modules/yallist": { "version": "4.0.0", "license": "ISC" @@ -7666,6 +8039,31 @@ "node": ">= 6" } }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + }, "node_modules/yn": { "version": "3.1.1", "devOptional": true, diff --git a/package.json b/package.json index 339f48dc132..4c85158befa 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,6 @@ }, "devDependencies": { "@faker-js/faker": "^8.4.1", - "@gradio/client": "https://gitpkg.now.sh/gradio-app/gradio/client/js?2a2a151", "@iconify-json/carbon": "^1.1.16", "@iconify-json/eos-icons": "^1.1.6", "@sveltejs/adapter-node": "^1.3.1", @@ -53,6 +52,7 @@ "type": "module", "dependencies": { "@apidevtools/swagger-parser": "^10.1.0", + "@gradio/client": "^0.18.0", "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", From 79865f47abc71f3455b57c9323d78a07004a6f81 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Sun, 5 May 2024 18:18:32 +0200 Subject: [PATCH 23/82] fix types --- src/routes/conversation/[id]/+server.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 8bd7919709e..ec2c8bae191 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -530,15 +530,15 @@ export async function POST({ request, locals, params, getClientAddress }) { const app = await Client.connect("ByteDance/Hyper-SDXL-1Step-T2I", { hf_token: (env.HF_TOKEN ?? env.HF_ACCESS_TOKEN) as unknown as `hf_${string}`, }); - const res = await app.predict("/process_image", [ + const res = (await app.predict("/process_image", [ 1, // number (numeric value between 1 and 8) in 'Number of Images' Slider component 512, // number in 'Image Height' Number component 512, // number in 'Image Width' Number component call.parameters?.prompt, // prompt Math.floor(Math.random() * 1000), // seed random - ]); + ])) as { data: Array> }; - const response = await fetch(res.data[0][0].image.url); + const response = await fetch(res?.data?.[0]?.[0]?.image?.url ?? "error"); const sha = await uploadFile(await response.blob(), conv); From 37a9bc90edd6de02d31b4ef7153d529e4da9f700 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Mon, 6 May 2024 00:56:30 +0200 Subject: [PATCH 24/82] version bump sharp --- package-lock.json | 467 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 433 insertions(+), 34 deletions(-) diff --git a/package-lock.json b/package-lock.json index c6d2bb85a84..685c202c83d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -286,6 +286,15 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@emnapi/runtime": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.1.1.tgz", + "integrity": "sha512-3bfqkzuR1KLx57nZfjr2NLnFOobvyS0aTszaEGCGqmYMVDRaGvgIZbjGSV/MHSSmLgQ/b9JFHQ5xm5WRZYd+XQ==", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/darwin-arm64": { "version": "0.18.20", "cpu": [ @@ -498,11 +507,12 @@ } }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.33.2", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.3.tgz", + "integrity": "sha512-FaNiGX1MrOuJ3hxuNzWgsT/mg5OHG/Izh59WW2mk1UwYHUwtfbhk5QNKYZgxf0pLOhx9ctGiGa2OykD71vOnSw==", "cpu": [ "arm64" ], - "license": "Apache-2.0", "optional": true, "os": [ "darwin" @@ -518,15 +528,41 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.0.1" + "@img/sharp-libvips-darwin-arm64": "1.0.2" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.3.tgz", + "integrity": "sha512-2QeSl7QDK9ru//YBT4sQkoq7L0EAJZA3rtV+v9p8xTKl4U1bUqTIaCnoC7Ctx2kCjQgwFXDasOtPTCT8eCTXvw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "glibc": ">=2.26", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.2" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.0.1", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.2.tgz", + "integrity": "sha512-tcK/41Rq8IKlSaKRCCAuuY3lDJjQnYIW1UXU1kxcEKrfL8WR7N6+rzNoOxoQRJWTAECuKwgAHnPvqXGN8XfkHA==", "cpu": [ "arm64" ], - "license": "LGPL-3.0-or-later", "optional": true, "os": [ "darwin" @@ -541,6 +577,366 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.2.tgz", + "integrity": "sha512-Ofw+7oaWa0HiiMiKWqqaZbaYV3/UGL2wAPeLuJTx+9cXpCRdvQhCLG0IH8YGwM0yGWGLpsF4Su9vM1o6aer+Fw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "macos": ">=10.13", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.2.tgz", + "integrity": "sha512-iLWCvrKgeFoglQxdEwzu1eQV04o8YeYGFXtfWU26Zr2wWT3q3MTzC+QTCO3ZQfWd3doKHT4Pm2kRmLbupT+sZw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.28", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.2.tgz", + "integrity": "sha512-x7kCt3N00ofFmmkkdshwj3vGPCnmiDh7Gwnd4nUwZln2YjqPxV1NlTyZOvoDWdKQVDL911487HOueBvrpflagw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.2.tgz", + "integrity": "sha512-cmhQ1J4qVhfmS6szYW7RT+gLJq9dH2i4maq+qyXayUSn9/3iY2ZeWpbAgSpSVbV2E1JUL2Gg7pwnYQ1h8rQIog==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.28", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.2.tgz", + "integrity": "sha512-E441q4Qdb+7yuyiADVi5J+44x8ctlrqn8XgkDTwr4qPJzWkaHwD489iZ4nGDgcuya4iMN3ULV6NwbhRZJ9Z7SQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.2.tgz", + "integrity": "sha512-3CAkndNpYUrlDqkCM5qhksfE+qSIREVpyoeHIU6jd48SJZViAmznoQQLAv4hVXF7xyUB9zf+G++e2v1ABjCbEQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.2.tgz", + "integrity": "sha512-VI94Q6khIHqHWNOh6LLdm9s2Ry4zdjWJwH56WoiJU7NTeDwyApdZZ8c+SADC8OH98KWNQXnE01UdJ9CSfZvwZw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.3.tgz", + "integrity": "sha512-Q7Ee3fFSC9P7vUSqVEF0zccJsZ8GiiCJYGWDdhEjdlOeS9/jdkyJ6sUSPj+bL8VuOYFSbofrW0t/86ceVhx32w==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.28", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.2" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.3.tgz", + "integrity": "sha512-Zf+sF1jHZJKA6Gor9hoYG2ljr4wo9cY4twaxgFDvlG0Xz9V7sinsPp8pFd1XtlhTzYo0IhDbl3rK7P6MzHpnYA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.2" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.3.tgz", + "integrity": "sha512-vFk441DKRFepjhTEH20oBlFrHcLjPfI8B0pMIxGm3+yilKyYeHEVvrZhYFdqIseSclIqbQ3SnZMwEMWonY5XFA==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.28", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.2" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.3.tgz", + "integrity": "sha512-Q4I++herIJxJi+qmbySd072oDPRkCg/SClLEIDh5IL9h1zjhqjv82H0Seupd+q2m0yOfD+/fJnjSoDFtKiHu2g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.2" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.3.tgz", + "integrity": "sha512-qnDccehRDXadhM9PM5hLvcPRYqyFCBN31kq+ErBSZtZlsAc1U4Z85xf/RXv1qolkdu+ibw64fUDaRdktxTNP9A==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.2" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.3.tgz", + "integrity": "sha512-Jhchim8kHWIU/GZ+9poHMWRcefeaxFIs9EBqf9KtcC14Ojk6qua7ghKiPs0sbeLbLj/2IGBtDcxHyjCdYWkk2w==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.2" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.3.tgz", + "integrity": "sha512-68zivsdJ0koE96stdUfM+gmyaK/NcoSZK5dV5CAjES0FUXS9lchYt8LAB5rTbM7nlWtxaU/2GON0HVN6/ZYJAQ==", + "cpu": [ + "wasm32" + ], + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.1.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.3.tgz", + "integrity": "sha512-CyimAduT2whQD8ER4Ux7exKrtfoaUiVr7HG0zZvO0XTFn2idUWljjxv58GxNTkFb8/J9Ub9AqITGkJD6ZginxQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.3.tgz", + "integrity": "sha512-viT4fUIDKnli3IfOephGnolMzhz5VaTvDRkYqtZxOMIoMQ4MrAziO7pT1nVnOt2FAm7qW5aa+CCc13aEY6Le0g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@inquirer/confirm": { "version": "3.1.6", "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.1.6.tgz", @@ -2724,8 +3120,9 @@ } }, "node_modules/detect-libc": { - "version": "2.0.2", - "license": "Apache-2.0", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", "engines": { "node": ">=8" } @@ -6283,8 +6680,9 @@ } }, "node_modules/semver": { - "version": "7.5.4", - "license": "ISC", + "version": "7.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", + "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", "dependencies": { "lru-cache": "^6.0.0" }, @@ -6377,41 +6775,42 @@ "license": "ISC" }, "node_modules/sharp": { - "version": "0.33.2", + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.3.tgz", + "integrity": "sha512-vHUeXJU1UvlO/BNwTpT0x/r53WkLUVxrmb5JTgW92fdFCFk0ispLMAeu/jPO2vjkXM1fYUi3K7/qcLF47pwM1A==", "hasInstallScript": true, - "license": "Apache-2.0", "dependencies": { "color": "^4.2.3", - "detect-libc": "^2.0.2", - "semver": "^7.5.4" + "detect-libc": "^2.0.3", + "semver": "^7.6.0" }, "engines": { - "libvips": ">=8.15.1", + "libvips": ">=8.15.2", "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.33.2", - "@img/sharp-darwin-x64": "0.33.2", - "@img/sharp-libvips-darwin-arm64": "1.0.1", - "@img/sharp-libvips-darwin-x64": "1.0.1", - "@img/sharp-libvips-linux-arm": "1.0.1", - "@img/sharp-libvips-linux-arm64": "1.0.1", - "@img/sharp-libvips-linux-s390x": "1.0.1", - "@img/sharp-libvips-linux-x64": "1.0.1", - "@img/sharp-libvips-linuxmusl-arm64": "1.0.1", - "@img/sharp-libvips-linuxmusl-x64": "1.0.1", - "@img/sharp-linux-arm": "0.33.2", - "@img/sharp-linux-arm64": "0.33.2", - "@img/sharp-linux-s390x": "0.33.2", - "@img/sharp-linux-x64": "0.33.2", - "@img/sharp-linuxmusl-arm64": "0.33.2", - "@img/sharp-linuxmusl-x64": "0.33.2", - "@img/sharp-wasm32": "0.33.2", - "@img/sharp-win32-ia32": "0.33.2", - "@img/sharp-win32-x64": "0.33.2" + "@img/sharp-darwin-arm64": "0.33.3", + "@img/sharp-darwin-x64": "0.33.3", + "@img/sharp-libvips-darwin-arm64": "1.0.2", + "@img/sharp-libvips-darwin-x64": "1.0.2", + "@img/sharp-libvips-linux-arm": "1.0.2", + "@img/sharp-libvips-linux-arm64": "1.0.2", + "@img/sharp-libvips-linux-s390x": "1.0.2", + "@img/sharp-libvips-linux-x64": "1.0.2", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.2", + "@img/sharp-libvips-linuxmusl-x64": "1.0.2", + "@img/sharp-linux-arm": "0.33.3", + "@img/sharp-linux-arm64": "0.33.3", + "@img/sharp-linux-s390x": "0.33.3", + "@img/sharp-linux-x64": "0.33.3", + "@img/sharp-linuxmusl-arm64": "0.33.3", + "@img/sharp-linuxmusl-x64": "0.33.3", + "@img/sharp-wasm32": "0.33.3", + "@img/sharp-win32-ia32": "0.33.3", + "@img/sharp-win32-x64": "0.33.3" } }, "node_modules/shebang-command": { @@ -7281,7 +7680,7 @@ }, "node_modules/tslib": { "version": "2.5.0", - "dev": true, + "devOptional": true, "license": "0BSD" }, "node_modules/tunnel-agent": { From c8b64f3f9fabdfdd09cc60ffac9d2c74d31599eb Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 9 May 2024 09:31:02 -0600 Subject: [PATCH 25/82] Basic tools menu --- src/lib/components/ToolsMenu.svelte | 41 +++++++++++++++++++ src/lib/components/ToolsToggle.svelte | 31 -------------- src/lib/components/chat/ChatMessage.svelte | 4 +- src/lib/components/chat/ChatWindow.svelte | 7 ++-- .../routines/03-add-tools-in-settings.ts | 24 +++++++++++ src/lib/migrations/routines/index.ts | 7 +++- src/lib/server/tools/calculator.ts | 2 + src/lib/server/tools/directlyAnswer.ts | 3 ++ src/lib/server/tools/fetchUrl.ts | 2 + src/lib/server/tools/text2img.ts | 2 + src/lib/server/tools/websearch.ts | 2 + src/lib/stores/settings.ts | 1 + src/lib/types/Settings.ts | 2 + src/lib/types/Tool.ts | 4 ++ src/routes/+layout.server.ts | 11 +++++ src/routes/settings/(nav)/+server.ts | 1 + 16 files changed, 108 insertions(+), 36 deletions(-) create mode 100644 src/lib/components/ToolsMenu.svelte delete mode 100644 src/lib/components/ToolsToggle.svelte create mode 100644 src/lib/migrations/routines/03-add-tools-in-settings.ts diff --git a/src/lib/components/ToolsMenu.svelte b/src/lib/components/ToolsMenu.svelte new file mode 100644 index 00000000000..161e08a00ec --- /dev/null +++ b/src/lib/components/ToolsMenu.svelte @@ -0,0 +1,41 @@ + + +
+ + Tools + (4) + +
+
+ {#each $page.data.tools as tool} + {@const isChecked = $settings?.tools?.[tool.name] ?? tool.isOnByDefault} +
+ { + await settings.instantSet({ + tools: { + ...$settings.tools, + [tool.name]: !isChecked, + }, + }); + }} + /> + +
+ {/each} +
+
+
diff --git a/src/lib/components/ToolsToggle.svelte b/src/lib/components/ToolsToggle.svelte deleted file mode 100644 index e7e52d90f4e..00000000000 --- a/src/lib/components/ToolsToggle.svelte +++ /dev/null @@ -1,31 +0,0 @@ - - -
- -
Tools
-
- -
-

- This model has function calling capabilities. When enabled, the model will try to use tools - such as web search or image generation to give a better answer. -

-
-
-
diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 7e68548394a..3f81bb94fdb 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -292,7 +292,9 @@ {/if} {toolDone ? "Called" : "Calling"} tool - {toolName} + {$page.data.tools.find((el) => el.name === toolName)?.displayName}
{#each tool as toolUpdate} {#if toolUpdate.messageType === "parameters"} diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index c45ddff8d3e..ec20e87fd3b 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -15,7 +15,7 @@ import StopGeneratingBtn from "../StopGeneratingBtn.svelte"; import type { Model } from "$lib/types/Model"; import WebSearchToggle from "../WebSearchToggle.svelte"; - import ToolsToggle from "../ToolsToggle.svelte"; + import ToolsToggle from "../ToolsMenu.svelte"; import LoginModal from "../LoginModal.svelte"; import { page } from "$app/stores"; import FileDropzone from "./FileDropzone.svelte"; @@ -33,6 +33,7 @@ import SystemPromptModal from "../SystemPromptModal.svelte"; import ChatIntroduction from "./ChatIntroduction.svelte"; import { useConvTreeStore } from "$lib/stores/convTree"; + import ToolsMenu from "../ToolsMenu.svelte"; export let messages: Message[] = []; export let loading = false; @@ -259,7 +260,7 @@
{#if !assistant} {#if currentModel.functions} - + {:else if $page.data.settings?.searchEnabled} {/if} @@ -400,7 +401,7 @@
Link copied to clipboard
{:else} - +
Share this conversation
{/if} diff --git a/src/lib/migrations/routines/03-add-tools-in-settings.ts b/src/lib/migrations/routines/03-add-tools-in-settings.ts new file mode 100644 index 00000000000..716b4741a5c --- /dev/null +++ b/src/lib/migrations/routines/03-add-tools-in-settings.ts @@ -0,0 +1,24 @@ +import type { Migration } from "."; +import { collections } from "$lib/server/database"; +import { ObjectId } from "mongodb"; + +const addToolsToSettings: Migration = { + _id: new ObjectId(3), + name: "Add empty 'tools' record in settings", + up: async () => { + const { settings } = collections; + + // Find all assistants whose modelId is not in modelIds, and update it to use defaultModelId + await settings.updateMany( + { + tools: { $exists: false }, + }, + { $set: { tools: {} } } + ); + + return true; + }, + runEveryTime: false, +}; + +export default addToolsToSettings; diff --git a/src/lib/migrations/routines/index.ts b/src/lib/migrations/routines/index.ts index 0d6eafa8f04..bf6b79218c8 100644 --- a/src/lib/migrations/routines/index.ts +++ b/src/lib/migrations/routines/index.ts @@ -3,6 +3,7 @@ import type { ObjectId } from "mongodb"; import updateSearchAssistant from "./01-update-search-assistants"; import updateAssistantsModels from "./02-update-assistants-models"; import type { Database } from "$lib/server/database"; +import addToolsToSettings from "./03-add-tools-in-settings"; export interface Migration { _id: ObjectId; @@ -14,4 +15,8 @@ export interface Migration { runEveryTime?: boolean; } -export const migrations: Migration[] = [updateSearchAssistant, updateAssistantsModels]; +export const migrations: Migration[] = [ + updateSearchAssistant, + updateAssistantsModels, + addToolsToSettings, +]; diff --git a/src/lib/server/tools/calculator.ts b/src/lib/server/tools/calculator.ts index 83673bfef97..00077d8b7d7 100644 --- a/src/lib/server/tools/calculator.ts +++ b/src/lib/server/tools/calculator.ts @@ -2,8 +2,10 @@ import type { BackendTool } from "."; const calculator: BackendTool = { name: "calculator", + displayName: "Calculator", description: "A simple calculator, takes a string containing a mathematical expression and returns the answer. Only supports +, -, *, and /, as well as parenthesis ().", + isOnByDefault: true, parameter_definitions: { equation: { description: diff --git a/src/lib/server/tools/directlyAnswer.ts b/src/lib/server/tools/directlyAnswer.ts index 836b8cfb6ae..9e6f0ea487a 100644 --- a/src/lib/server/tools/directlyAnswer.ts +++ b/src/lib/server/tools/directlyAnswer.ts @@ -2,6 +2,9 @@ import type { BackendTool } from "."; const directlyAnswer: BackendTool = { name: "directly-answer", + isOnByDefault: true, + isHidden: true, + isLocked: true, description: "Use this tool to let the user know you wish to answer directly. Do not try to provide any parameters when using this tool.", parameter_definitions: {}, diff --git a/src/lib/server/tools/fetchUrl.ts b/src/lib/server/tools/fetchUrl.ts index e15e9d40cda..465ff8ffc8f 100644 --- a/src/lib/server/tools/fetchUrl.ts +++ b/src/lib/server/tools/fetchUrl.ts @@ -3,7 +3,9 @@ import { parseWeb } from "../websearch/parseWeb"; const fetchUrl: BackendTool = { name: "fetchUrl", + displayName: "URL Fetcher", description: "A tool that can be used to fetch an URL and return the content directly.", + isOnByDefault: true, parameter_definitions: { url: { description: "The url that should be fetched.", diff --git a/src/lib/server/tools/text2img.ts b/src/lib/server/tools/text2img.ts index 4cae2cdc48f..77a4fe96263 100644 --- a/src/lib/server/tools/text2img.ts +++ b/src/lib/server/tools/text2img.ts @@ -2,7 +2,9 @@ import type { BackendTool } from "."; const text2img: BackendTool = { name: "text2img", + displayName: "Text to Image", description: "Use this tool to generate an image from a prompt.", + isOnByDefault: true, parameter_definitions: { prompt: { description: diff --git a/src/lib/server/tools/websearch.ts b/src/lib/server/tools/websearch.ts index ad6d0609f22..ce15eb2f5f5 100644 --- a/src/lib/server/tools/websearch.ts +++ b/src/lib/server/tools/websearch.ts @@ -2,6 +2,8 @@ import type { BackendTool } from "."; const websearch: BackendTool = { name: "websearch", + displayName: "Web Search", + isOnByDefault: true, description: "Use this tool to search web pages for answers that will help answer the user's query. Only use this tool if you need specific resources from the internet.", parameter_definitions: { diff --git a/src/lib/stores/settings.ts b/src/lib/stores/settings.ts index b002c05336c..3cbbdaa9e11 100644 --- a/src/lib/stores/settings.ts +++ b/src/lib/stores/settings.ts @@ -15,6 +15,7 @@ type SettingsStore = { customPrompts: Record; recentlySaved: boolean; assistants: Array; + tools?: Record; }; type SettingsStoreWritable = Writable & { diff --git a/src/lib/types/Settings.ts b/src/lib/types/Settings.ts index 5a6804e05a8..a37a842a1db 100644 --- a/src/lib/types/Settings.ts +++ b/src/lib/types/Settings.ts @@ -21,6 +21,7 @@ export interface Settings extends Timestamps { customPrompts?: Record; assistants?: Assistant["_id"][]; + tools?: Record; } // TODO: move this to a constant file along with other constants @@ -30,4 +31,5 @@ export const DEFAULT_SETTINGS = { hideEmojiOnSidebar: false, customPrompts: {}, assistants: [], + tools: {}, }; diff --git a/src/lib/types/Tool.ts b/src/lib/types/Tool.ts index 7a8e6f54bb7..09c7ead52ce 100644 --- a/src/lib/types/Tool.ts +++ b/src/lib/types/Tool.ts @@ -6,9 +6,13 @@ interface ToolInput { export interface Tool { name: string; + displayName?: string; description: string; parameter_definitions: Record; spec?: string; + isOnByDefault?: true; // will it be toggled if the user hasn't tweaked it in settings ? + isLocked?: true; // can the user enable/disable it ? + isHidden?: true; // should it be hidden from the user ? } export interface ToolResult { diff --git a/src/routes/+layout.server.ts b/src/routes/+layout.server.ts index 3c7f761dd7f..811badcad81 100644 --- a/src/routes/+layout.server.ts +++ b/src/routes/+layout.server.ts @@ -8,6 +8,7 @@ import { DEFAULT_SETTINGS } from "$lib/types/Settings"; import { env } from "$env/dynamic/private"; import { ObjectId } from "mongodb"; import type { ConvSidebar } from "$lib/types/ConvSidebar"; +import { tools } from "$lib/server/tools"; export const load: LayoutServerLoad = async ({ locals, depends }) => { depends(UrlDependency.ConversationList); @@ -142,6 +143,7 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => { DEFAULT_SETTINGS.shareConversationsWithModelAuthors, customPrompts: settings?.customPrompts ?? {}, assistants: userAssistants, + tools: settings?.tools ?? {}, }, models: models.map((model) => ({ id: model.id, @@ -162,6 +164,15 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => { unlisted: model.unlisted, })), oldModels, + tools: tools + .filter((tool) => !tool.isHidden) + .map((tool) => ({ + name: tool.name, + displayName: tool.displayName, + description: tool.description, + isOnByDefault: tool.isOnByDefault, + isLocked: tool.isLocked, + })), assistants: assistants .filter((el) => userAssistantsSet.has(el._id.toString())) .map((el) => ({ diff --git a/src/routes/settings/(nav)/+server.ts b/src/routes/settings/(nav)/+server.ts index 81289bacba2..fbc9ce28005 100644 --- a/src/routes/settings/(nav)/+server.ts +++ b/src/routes/settings/(nav)/+server.ts @@ -15,6 +15,7 @@ export async function POST({ request, locals }) { ethicsModalAccepted: z.boolean().optional(), activeModel: z.string().default(DEFAULT_SETTINGS.activeModel), customPrompts: z.record(z.string()).default({}), + tools: z.record(z.boolean()).optional(), }) .parse(body); From 81a1afc81b9eb91440121b0b3735ba2ffb4a1989 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 9 May 2024 22:45:46 -0600 Subject: [PATCH 26/82] working menu --- src/lib/components/ToolsMenu.svelte | 28 +++++++++++++++------- src/lib/components/chat/ChatMessage.svelte | 5 +++- src/lib/components/chat/ChatWindow.svelte | 7 +++--- src/lib/components/icons/IconTool.svelte | 10 ++++++++ src/lib/server/tools/index.ts | 3 ++- src/lib/server/tools/text2img.ts | 2 +- src/lib/types/Tool.ts | 5 ++++ src/lib/utils/messageUpdates.ts | 2 +- src/routes/conversation/[id]/+page.svelte | 9 ++++--- src/routes/conversation/[id]/+server.ts | 26 ++++++++++++++------ 10 files changed, 69 insertions(+), 28 deletions(-) create mode 100644 src/lib/components/icons/IconTool.svelte diff --git a/src/lib/components/ToolsMenu.svelte b/src/lib/components/ToolsMenu.svelte index 161e08a00ec..d599a8854be 100644 --- a/src/lib/components/ToolsMenu.svelte +++ b/src/lib/components/ToolsMenu.svelte @@ -1,27 +1,41 @@ -
+
+ Tools - (4) + ({activeToolCount}) -
+
{#each $page.data.tools as tool} {@const isChecked = $settings?.tools?.[tool.name] ?? tool.isOnByDefault} -
+
{ await settings.instantSet({ tools: { @@ -31,9 +45,7 @@ }); }} /> - +
{/each}
diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 3f81bb94fdb..ade63f0c46f 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -22,6 +22,7 @@ import OpenWebSearchResults from "../OpenWebSearchResults.svelte"; import type { ToolUpdate, WebSearchUpdate } from "$lib/types/MessageUpdate"; + import type { ToolFront } from "$lib/types/Tool"; import { base } from "$app/paths"; import { useConvTreeStore } from "$lib/stores/convTree"; import Modal from "../Modal.svelte"; @@ -194,6 +195,8 @@ $: if (message.children?.length === 0) $convTreeStore.leaf = message.id; $: modalImageToShow = ""; + + const availableTools: ToolFront[] = $page.data.tools; {#if modalImageToShow} @@ -293,7 +296,7 @@ {/if} {toolDone ? "Called" : "Calling"} tool {$page.data.tools.find((el) => el.name === toolName)?.displayName}{availableTools.find((el) => el.name === toolName)?.displayName} {#each tool as toolUpdate} diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index ec20e87fd3b..a834d7f4764 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -15,7 +15,7 @@ import StopGeneratingBtn from "../StopGeneratingBtn.svelte"; import type { Model } from "$lib/types/Model"; import WebSearchToggle from "../WebSearchToggle.svelte"; - import ToolsToggle from "../ToolsMenu.svelte"; + import ToolsMenu from "../ToolsMenu.svelte"; import LoginModal from "../LoginModal.svelte"; import { page } from "$app/stores"; import FileDropzone from "./FileDropzone.svelte"; @@ -33,7 +33,6 @@ import SystemPromptModal from "../SystemPromptModal.svelte"; import ChatIntroduction from "./ChatIntroduction.svelte"; import { useConvTreeStore } from "$lib/stores/convTree"; - import ToolsMenu from "../ToolsMenu.svelte"; export let messages: Message[] = []; export let loading = false; @@ -260,7 +259,7 @@
{#if !assistant} {#if currentModel.functions} - + {:else if $page.data.settings?.searchEnabled} {/if} @@ -401,7 +400,7 @@
Link copied to clipboard
{:else} - +
Share this conversation
{/if} diff --git a/src/lib/components/icons/IconTool.svelte b/src/lib/components/icons/IconTool.svelte new file mode 100644 index 00000000000..f90d2d94ceb --- /dev/null +++ b/src/lib/components/icons/IconTool.svelte @@ -0,0 +1,10 @@ + + + diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index 4e89688c442..6c371208ee3 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -1,6 +1,7 @@ import type { Tool, ToolResult } from "$lib/types/Tool"; import calculator from "./calculator"; import directlyAnswer from "./directlyAnswer"; +import fetchUrl from "./fetchUrl"; import text2img from "./text2img"; import websearch from "./websearch"; @@ -8,4 +9,4 @@ export interface BackendTool extends Tool { call?(params: Record): Promise; } -export const tools: BackendTool[] = [calculator, websearch, text2img, directlyAnswer]; +export const tools: BackendTool[] = [calculator, websearch, text2img, directlyAnswer, fetchUrl]; diff --git a/src/lib/server/tools/text2img.ts b/src/lib/server/tools/text2img.ts index 77a4fe96263..2a590b631a7 100644 --- a/src/lib/server/tools/text2img.ts +++ b/src/lib/server/tools/text2img.ts @@ -2,7 +2,7 @@ import type { BackendTool } from "."; const text2img: BackendTool = { name: "text2img", - displayName: "Text to Image", + displayName: "Text-to-Image", description: "Use this tool to generate an image from a prompt.", isOnByDefault: true, parameter_definitions: { diff --git a/src/lib/types/Tool.ts b/src/lib/types/Tool.ts index 09c7ead52ce..a181c98f21b 100644 --- a/src/lib/types/Tool.ts +++ b/src/lib/types/Tool.ts @@ -15,6 +15,11 @@ export interface Tool { isHidden?: true; // should it be hidden from the user ? } +export type ToolFront = Pick< + Tool, + "name" | "displayName" | "description" | "isOnByDefault" | "isLocked" +>; + export interface ToolResult { key: string; status: "success" | "error"; diff --git a/src/lib/utils/messageUpdates.ts b/src/lib/utils/messageUpdates.ts index 8190bc71dd2..d6d1b402205 100644 --- a/src/lib/utils/messageUpdates.ts +++ b/src/lib/utils/messageUpdates.ts @@ -7,7 +7,7 @@ type MessageUpdateRequestOptions = { isRetry: boolean; isContinue: boolean; webSearch: boolean; - tools: boolean; + tools?: Record; files?: string[]; }; export async function fetchMessageUpdates( diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte index 01367111b27..90b5e9eb6b6 100644 --- a/src/routes/conversation/[id]/+page.svelte +++ b/src/routes/conversation/[id]/+page.svelte @@ -19,12 +19,11 @@ import { fetchMessageUpdates } from "$lib/utils/messageUpdates"; import { createConvTreeStore } from "$lib/stores/convTree"; import type { v4 } from "uuid"; + import { useSettingsStore } from "$lib/stores/settings.js"; export let data; - $: ({ messages, model: modelId } = data); - - $: currentModel = findCurrentModel([...data.models, ...data.oldModels], modelId); + $: ({ messages } = data); let loading = false; let pending = false; @@ -200,8 +199,7 @@ isRetry, isContinue, webSearch: !hasAssistant && $webSearchParameters.useSearch, - tools: - (!hasAssistant && currentModel.functions && $webSearchParameters.useTools) ?? false, + tools: $settings.tools, // preference for tools files: isRetry ? undefined : resizedImages, }, messageUpdatesAbortController.signal @@ -373,6 +371,7 @@ $: title = data.conversations.find((conv) => conv.id === $page.params.id)?.title ?? data.title; const convTreeStore = createConvTreeStore(); + const settings = useSettingsStore(); diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index ec2c8bae191..a3d7705d37f 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -35,6 +35,7 @@ import websearch from "$lib/server/tools/websearch.js"; import text2img from "$lib/server/tools/text2img.js"; import fetchUrl from "$lib/server/tools/fetchUrl.js"; import { logger } from "$lib/server/logger.js"; +import type { BackendTool } from "$lib/server/tools/index.js"; export async function POST({ request, locals, params, getClientAddress }) { const id = z.string().parse(params.id); @@ -145,7 +146,7 @@ export async function POST({ request, locals, params, getClientAddress }) { is_retry: isRetry, is_continue: isContinue, web_search: webSearch, - tools: toolsFlag, + tools: toolsPreferences, files: b64files, } = z .object({ @@ -160,7 +161,7 @@ export async function POST({ request, locals, params, getClientAddress }) { is_retry: z.optional(z.boolean()), is_continue: z.optional(z.boolean()), web_search: z.optional(z.boolean()), - tools: z.optional(z.boolean()), + tools: z.record(z.boolean()).optional(), files: z.optional(z.array(z.string())), }) .parse(json); @@ -391,7 +392,8 @@ export async function POST({ request, locals, params, getClientAddress }) { if ( !isContinue && - ((webSearch && !conv.assistantId) || (assistantHasWebSearch && !model.functions)) + !model.functions && + ((webSearch && !conv.assistantId) || assistantHasWebSearch) ) { messageToWriteTo.webSearch = await runWebSearch(conv, messagesForPrompt, update, { ragSettings: assistant?.rag, @@ -431,13 +433,23 @@ export async function POST({ request, locals, params, getClientAddress }) { const endpoint = await model.getEndpoint(); let toolResults: ToolResult[] | undefined = undefined; - // function calls - if (model.functions && (toolsFlag || assistantHasWebSearch)) { - let tools = [directlyAnswer, websearch]; + // function calls, if no assistant or assistant has websearch enabled + if (model.functions && (!assistant || assistantHasWebSearch)) { + let tools: BackendTool[] = []; // if it's an assistant, only support websearch for now if (!assistant) { - tools = [...tools, text2img, calculator, fetchUrl]; + // filter based on tool preferences, add the tools that are on by default + tools = [directlyAnswer, text2img, calculator, fetchUrl, websearch].filter((el) => { + if (el.isLocked && el.isOnByDefault) { + return true; + } + + return toolsPreferences?.[el.name] ?? el.isOnByDefault; + }); + } else { + // an assistant can currently only use tools for websearch + tools = [directlyAnswer, websearch]; } let calls: Call[] | undefined = undefined; From 0e09837de5645ca91a9a4ecade685f3c2613e9f4 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 9 May 2024 22:48:41 -0600 Subject: [PATCH 27/82] fix menu positioning --- src/lib/components/ToolsMenu.svelte | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/components/ToolsMenu.svelte b/src/lib/components/ToolsMenu.svelte index d599a8854be..780d8086f0b 100644 --- a/src/lib/components/ToolsMenu.svelte +++ b/src/lib/components/ToolsMenu.svelte @@ -14,10 +14,10 @@ }).length; -
+
Tools From 8e24c6a8c3417f226a1439531e9e8a7bd579dac9 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 9 May 2024 23:03:16 -0600 Subject: [PATCH 28/82] deps fix --- package-lock.json | 401 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 2 +- 2 files changed, 399 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index a4d02faab07..75077b39647 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,7 +39,7 @@ "satori": "^0.10.11", "satori-html": "^0.3.2", "serpapi": "^1.1.1", - "sharp": "^0.33.2", + "sharp": "^0.33.3", "tailwind-scrollbar": "^3.0.0", "tailwindcss": "^3.4.0", "uuid": "^9.0.1", @@ -287,6 +287,15 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@emnapi/runtime": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.1.1.tgz", + "integrity": "sha512-3bfqkzuR1KLx57nZfjr2NLnFOobvyS0aTszaEGCGqmYMVDRaGvgIZbjGSV/MHSSmLgQ/b9JFHQ5xm5WRZYd+XQ==", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/darwin-arm64": { "version": "0.18.20", "cpu": [ @@ -521,6 +530,31 @@ "@img/sharp-libvips-darwin-arm64": "1.0.2" } }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.3.tgz", + "integrity": "sha512-2QeSl7QDK9ru//YBT4sQkoq7L0EAJZA3rtV+v9p8xTKl4U1bUqTIaCnoC7Ctx2kCjQgwFXDasOtPTCT8eCTXvw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "glibc": ">=2.26", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.2" + } + }, "node_modules/@img/sharp-libvips-darwin-arm64": { "version": "1.0.2", "cpu": [ @@ -541,6 +575,366 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.2.tgz", + "integrity": "sha512-Ofw+7oaWa0HiiMiKWqqaZbaYV3/UGL2wAPeLuJTx+9cXpCRdvQhCLG0IH8YGwM0yGWGLpsF4Su9vM1o6aer+Fw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "macos": ">=10.13", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.2.tgz", + "integrity": "sha512-iLWCvrKgeFoglQxdEwzu1eQV04o8YeYGFXtfWU26Zr2wWT3q3MTzC+QTCO3ZQfWd3doKHT4Pm2kRmLbupT+sZw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.28", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.2.tgz", + "integrity": "sha512-x7kCt3N00ofFmmkkdshwj3vGPCnmiDh7Gwnd4nUwZln2YjqPxV1NlTyZOvoDWdKQVDL911487HOueBvrpflagw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.2.tgz", + "integrity": "sha512-cmhQ1J4qVhfmS6szYW7RT+gLJq9dH2i4maq+qyXayUSn9/3iY2ZeWpbAgSpSVbV2E1JUL2Gg7pwnYQ1h8rQIog==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.28", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.2.tgz", + "integrity": "sha512-E441q4Qdb+7yuyiADVi5J+44x8ctlrqn8XgkDTwr4qPJzWkaHwD489iZ4nGDgcuya4iMN3ULV6NwbhRZJ9Z7SQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.2.tgz", + "integrity": "sha512-3CAkndNpYUrlDqkCM5qhksfE+qSIREVpyoeHIU6jd48SJZViAmznoQQLAv4hVXF7xyUB9zf+G++e2v1ABjCbEQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.2.tgz", + "integrity": "sha512-VI94Q6khIHqHWNOh6LLdm9s2Ry4zdjWJwH56WoiJU7NTeDwyApdZZ8c+SADC8OH98KWNQXnE01UdJ9CSfZvwZw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.3.tgz", + "integrity": "sha512-Q7Ee3fFSC9P7vUSqVEF0zccJsZ8GiiCJYGWDdhEjdlOeS9/jdkyJ6sUSPj+bL8VuOYFSbofrW0t/86ceVhx32w==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.28", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.2" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.3.tgz", + "integrity": "sha512-Zf+sF1jHZJKA6Gor9hoYG2ljr4wo9cY4twaxgFDvlG0Xz9V7sinsPp8pFd1XtlhTzYo0IhDbl3rK7P6MzHpnYA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.2" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.3.tgz", + "integrity": "sha512-vFk441DKRFepjhTEH20oBlFrHcLjPfI8B0pMIxGm3+yilKyYeHEVvrZhYFdqIseSclIqbQ3SnZMwEMWonY5XFA==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.28", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.2" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.3.tgz", + "integrity": "sha512-Q4I++herIJxJi+qmbySd072oDPRkCg/SClLEIDh5IL9h1zjhqjv82H0Seupd+q2m0yOfD+/fJnjSoDFtKiHu2g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "glibc": ">=2.26", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.2" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.3.tgz", + "integrity": "sha512-qnDccehRDXadhM9PM5hLvcPRYqyFCBN31kq+ErBSZtZlsAc1U4Z85xf/RXv1qolkdu+ibw64fUDaRdktxTNP9A==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.2" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.3.tgz", + "integrity": "sha512-Jhchim8kHWIU/GZ+9poHMWRcefeaxFIs9EBqf9KtcC14Ojk6qua7ghKiPs0sbeLbLj/2IGBtDcxHyjCdYWkk2w==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "musl": ">=1.2.2", + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.2" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.3.tgz", + "integrity": "sha512-68zivsdJ0koE96stdUfM+gmyaK/NcoSZK5dV5CAjES0FUXS9lchYt8LAB5rTbM7nlWtxaU/2GON0HVN6/ZYJAQ==", + "cpu": [ + "wasm32" + ], + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.1.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.3.tgz", + "integrity": "sha512-CyimAduT2whQD8ER4Ux7exKrtfoaUiVr7HG0zZvO0XTFn2idUWljjxv58GxNTkFb8/J9Ub9AqITGkJD6ZginxQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.3.tgz", + "integrity": "sha512-viT4fUIDKnli3IfOephGnolMzhz5VaTvDRkYqtZxOMIoMQ4MrAziO7pT1nVnOt2FAm7qW5aa+CCc13aEY6Le0g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0", + "npm": ">=9.6.5", + "pnpm": ">=7.1.0", + "yarn": ">=3.2.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@inquirer/confirm": { "version": "3.1.6", "license": "MIT", @@ -6344,8 +6738,9 @@ }, "node_modules/sharp": { "version": "0.33.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.3.tgz", + "integrity": "sha512-vHUeXJU1UvlO/BNwTpT0x/r53WkLUVxrmb5JTgW92fdFCFk0ispLMAeu/jPO2vjkXM1fYUi3K7/qcLF47pwM1A==", "hasInstallScript": true, - "license": "Apache-2.0", "dependencies": { "color": "^4.2.3", "detect-libc": "^2.0.3", @@ -7244,7 +7639,7 @@ }, "node_modules/tslib": { "version": "2.5.0", - "dev": true, + "devOptional": true, "license": "0BSD" }, "node_modules/tunnel-agent": { diff --git a/package.json b/package.json index 21b74d2f4e9..133919cea7c 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,7 @@ "satori": "^0.10.11", "satori-html": "^0.3.2", "serpapi": "^1.1.1", - "sharp": "^0.33.2", + "sharp": "^0.33.3", "tailwind-scrollbar": "^3.0.0", "tailwindcss": "^3.4.0", "uuid": "^9.0.1", From a55ddc2dd73447bc568049f19df6566b208c518f Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 9 May 2024 23:12:41 -0600 Subject: [PATCH 29/82] add deps --- package-lock.json | 182 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 183 insertions(+) diff --git a/package-lock.json b/package-lock.json index 75077b39647..83828f3c400 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", "@resvg/resvg-js": "^2.6.0", + "@resvg/resvg-js-linux-x64-gnu": "*", "@xenova/transformers": "^2.16.1", "autoprefixer": "^10.4.14", "browser-image-resizer": "^2.4.1", @@ -84,6 +85,7 @@ "@anthropic-ai/sdk": "^0.17.1", "@anthropic-ai/vertex-sdk": "^0.3.0", "@google-cloud/vertexai": "^1.1.0", + "@resvg/resvg-js-linux-x64-gnu": "^2.6.2", "aws4fetch": "^1.0.17", "cohere-ai": "^7.9.0", "openai": "^4.14.2" @@ -1194,6 +1196,36 @@ "@resvg/resvg-js-win32-x64-msvc": "2.6.0" } }, + "node_modules/@resvg/resvg-js-android-arm-eabi": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm-eabi/-/resvg-js-android-arm-eabi-2.6.0.tgz", + "integrity": "sha512-lJnZ/2P5aMocrFMW7HWhVne5gH82I8xH6zsfH75MYr4+/JOaVcGCTEQ06XFohGMdYRP3v05SSPLPvTM/RHjxfA==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@resvg/resvg-js-android-arm64": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-android-arm64/-/resvg-js-android-arm64-2.6.0.tgz", + "integrity": "sha512-N527f529bjMwYWShZYfBD60dXA4Fux+D695QsHQ93BDYZSHUoOh1CUGUyICevnTxs7VgEl98XpArmUWBZQVMfQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@resvg/resvg-js-darwin-arm64": { "version": "2.6.0", "cpu": [ @@ -1208,6 +1240,156 @@ "node": ">= 10" } }, + "node_modules/@resvg/resvg-js-darwin-x64": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-darwin-x64/-/resvg-js-darwin-x64-2.6.0.tgz", + "integrity": "sha512-zrFetdnSw/suXjmyxSjfDV7i61hahv6DDG6kM7BYN2yJ3Es5+BZtqYZTcIWogPJedYKmzN1YTMWGd/3f0ubFiA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@resvg/resvg-js-linux-arm-gnueabihf": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm-gnueabihf/-/resvg-js-linux-arm-gnueabihf-2.6.0.tgz", + "integrity": "sha512-sH4gxXt7v7dGwjGyzLwn7SFGvwZG6DQqLaZ11MmzbCwd9Zosy1TnmrMJfn6TJ7RHezmQMgBPi18bl55FZ1AT4A==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@resvg/resvg-js-linux-arm64-gnu": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-gnu/-/resvg-js-linux-arm64-gnu-2.6.0.tgz", + "integrity": "sha512-fCyMncqCJtrlANADIduYF4IfnWQ295UKib7DAxFXQhBsM9PLDTpizr0qemZcCNadcwSVHnAIzL4tliZhCM8P6A==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@resvg/resvg-js-linux-arm64-musl": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-arm64-musl/-/resvg-js-linux-arm64-musl-2.6.0.tgz", + "integrity": "sha512-ouLjTgBQHQyxLht4FdMPTvuY8xzJigM9EM2Tlu0llWkN1mKyTQrvYWi6TA6XnKdzDJHy7ZLpWpjZi7F5+Pg+Vg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@resvg/resvg-js-linux-x64-gnu": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-gnu/-/resvg-js-linux-x64-gnu-2.6.2.tgz", + "integrity": "sha512-IVUe+ckIerA7xMZ50duAZzwf1U7khQe2E0QpUxu5MBJNao5RqC0zwV/Zm965vw6D3gGFUl7j4m+oJjubBVoftw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@resvg/resvg-js-linux-x64-musl": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-musl/-/resvg-js-linux-x64-musl-2.6.0.tgz", + "integrity": "sha512-n4tasK1HOlAxdTEROgYA1aCfsEKk0UOFDNd/AQTTZlTmCbHKXPq+O8npaaKlwXquxlVK8vrkcWbksbiGqbCAcw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@resvg/resvg-js-win32-arm64-msvc": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-arm64-msvc/-/resvg-js-win32-arm64-msvc-2.6.0.tgz", + "integrity": "sha512-X2+EoBJFwDI5LDVb51Sk7ldnVLitMGr9WwU/i21i3fAeAXZb3hM16k67DeTy16OYkT2dk/RfU1tP1wG+rWbz2Q==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@resvg/resvg-js-win32-ia32-msvc": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-ia32-msvc/-/resvg-js-win32-ia32-msvc-2.6.0.tgz", + "integrity": "sha512-L7oevWjQoUgK5W1fCKn0euSVemhDXVhrjtwqpc7MwBKKimYeiOshO1Li1pa8bBt5PESahenhWgdB6lav9O0fEg==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@resvg/resvg-js-win32-x64-msvc": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-win32-x64-msvc/-/resvg-js-win32-x64-msvc-2.6.0.tgz", + "integrity": "sha512-8lJlghb+Unki5AyKgsnFbRJwkEj9r1NpwyuBG8yEJiG1W9eEGl03R3I7bsVa3haof/3J1NlWf0rzSa1G++A2iw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@resvg/resvg-js/node_modules/@resvg/resvg-js-linux-x64-gnu": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@resvg/resvg-js-linux-x64-gnu/-/resvg-js-linux-x64-gnu-2.6.0.tgz", + "integrity": "sha512-n3zC8DWsvxC1AwxpKFclIPapDFibs5XdIRoV/mcIlxlh0vseW1F49b97F33BtJQRmlntsqqN6GMMqx8byB7B+Q==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@rollup/plugin-commonjs": { "version": "25.0.7", "dev": true, diff --git a/package.json b/package.json index 133919cea7c..1073d06ec30 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,7 @@ "@anthropic-ai/sdk": "^0.17.1", "@anthropic-ai/vertex-sdk": "^0.3.0", "@google-cloud/vertexai": "^1.1.0", + "@resvg/resvg-js-linux-x64-gnu": "^2.6.2", "aws4fetch": "^1.0.17", "cohere-ai": "^7.9.0", "openai": "^4.14.2" From 49a58f6acfb4332c379d11054551967c54440912 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 9 May 2024 23:16:07 -0600 Subject: [PATCH 30/82] cleanup --- .env | 1 - src/lib/components/AssistantSettings.svelte | 20 ------------------- src/lib/types/Assistant.ts | 1 - src/lib/utils/getToolsFromFunctionSpec.ts | 7 ------- src/routes/conversation/[id]/+server.ts | 4 ++-- .../[assistantId]/edit/+page.server.ts | 2 -- .../(nav)/assistants/new/+page.server.ts | 2 -- 7 files changed, 2 insertions(+), 35 deletions(-) delete mode 100644 src/lib/utils/getToolsFromFunctionSpec.ts diff --git a/.env b/.env index 5915d8723c3..98c57ea578f 100644 --- a/.env +++ b/.env @@ -154,7 +154,6 @@ ALLOWED_USER_EMAILS=`[]` # if it's defined, only these emails will be allowed to USAGE_LIMITS=`{}` -IMAGE_GENERATION_MODEL=stabilityai/stable-diffusion-xl-base-1.0 ALLOW_INSECURE_COOKIES=false # recommended to keep this to false but set to true if you need to run over http without tls METRICS_PORT= LOG_LEVEL=info diff --git a/src/lib/components/AssistantSettings.svelte b/src/lib/components/AssistantSettings.svelte index e79c355fccd..37092116176 100644 --- a/src/lib/components/AssistantSettings.svelte +++ b/src/lib/components/AssistantSettings.svelte @@ -39,7 +39,6 @@ let systemPrompt = assistant?.preprompt ?? ""; let dynamicPrompt = assistant?.dynamicPrompt ?? false; let showModelSettings = Object.values(assistant?.generateSettings ?? {}).some((v) => !!v); - let functionSpec = assistant?.functionSpec ?? ""; let compress: typeof readAndCompressImage | null = null; @@ -97,19 +96,6 @@ const regex = /{{\s?url=(.+?)\s?}}/g; $: templateVariables = [...systemPrompt.matchAll(regex)].map((match) => match[1]); $: selectedModel = models.find((m) => m.id === modelId); - - // const getErrorsFromSpec = debounce(async (spec: string) => { - // if (parser && spec) { - // // disable ts - // parser.validate(spec, (err) => { - // if (err && form) { - // form.errors = [{ field: "functionSpec", message: err.message }]; - // } - // }); - // } - // }, 500); - - // $: getErrorsFromSpec(functionSpec); -
{/if}
diff --git a/src/lib/types/Assistant.ts b/src/lib/types/Assistant.ts index bbb0b21fbb7..783ddca0908 100644 --- a/src/lib/types/Assistant.ts +++ b/src/lib/types/Assistant.ts @@ -14,7 +14,6 @@ export interface Assistant extends Timestamps { preprompt: string; userCount?: number; featured?: boolean; - functionSpec?: string; // openAPI spec rag?: { allowAllDomains: boolean; allowedDomains: string[]; diff --git a/src/lib/utils/getToolsFromFunctionSpec.ts b/src/lib/utils/getToolsFromFunctionSpec.ts deleted file mode 100644 index f328c21a208..00000000000 --- a/src/lib/utils/getToolsFromFunctionSpec.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { tools, type BackendTool } from "$lib/server/tools"; - -export async function getToolsFromFunctionSpec(spec?: string): Promise { - if (!spec) return []; - - return tools; -} diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index c5d8435b9bb..d361101d107 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -367,10 +367,10 @@ export async function POST({ request, locals, params, getClientAddress }) { // check if assistant has a rag const assistant = await collections.assistants.findOne< - Pick + Pick >( { _id: conv.assistantId }, - { projection: { rag: 1, dynamicPrompt: 1, generateSettings: 1, functionSpec: 1 } } + { projection: { rag: 1, dynamicPrompt: 1, generateSettings: 1 } } ); const assistantHasDynamicPrompt = diff --git a/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts b/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts index 77c2e052cb3..93e3e821ab6 100644 --- a/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts +++ b/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts @@ -40,7 +40,6 @@ const newAsssistantSchema = z.object({ top_k: z .union([z.literal(""), z.coerce.number().min(5).max(100)]) .transform((v) => (v === "" ? undefined : v)), - functionSpec: z.union([z.literal(""), z.string().refine(isValidOpenAPI, "Invalid OpenAPI spec")]), }); const uploadAvatar = async (avatar: File, assistantId: ObjectId): Promise => { @@ -159,7 +158,6 @@ export const actions: Actions = { }, dynamicPrompt: parse.data.dynamicPrompt, searchTokens: generateSearchTokens(parse.data.name), - functionSpec: parse.data.functionSpec, generateSettings: { temperature: parse.data.temperature, top_p: parse.data.top_p, diff --git a/src/routes/settings/(nav)/assistants/new/+page.server.ts b/src/routes/settings/(nav)/assistants/new/+page.server.ts index 2069e42d5d6..9ae860a1944 100644 --- a/src/routes/settings/(nav)/assistants/new/+page.server.ts +++ b/src/routes/settings/(nav)/assistants/new/+page.server.ts @@ -40,7 +40,6 @@ const newAsssistantSchema = z.object({ top_k: z .union([z.literal(""), z.coerce.number().min(5).max(100)]) .transform((v) => (v === "" ? undefined : v)), - functionSpec: z.union([z.literal(""), z.string().refine(isValidOpenAPI, "Invalid OpenAPI spec")]), }); const uploadAvatar = async (avatar: File, assistantId: ObjectId): Promise => { @@ -142,7 +141,6 @@ export const actions: Actions = { dynamicPrompt: parse.data.dynamicPrompt, searchTokens: generateSearchTokens(parse.data.name), last24HoursCount: 0, - functionSpec: parse.data.functionSpec, generateSettings: { temperature: parse.data.temperature, top_p: parse.data.top_p, From 392de1e10b92c1123a45e047b4d91da8479c8b6e Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 9 May 2024 23:16:56 -0600 Subject: [PATCH 31/82] more cleanup --- src/lib/utils/isValidOpenAPI.ts | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 src/lib/utils/isValidOpenAPI.ts diff --git a/src/lib/utils/isValidOpenAPI.ts b/src/lib/utils/isValidOpenAPI.ts deleted file mode 100644 index 059166953b3..00000000000 --- a/src/lib/utils/isValidOpenAPI.ts +++ /dev/null @@ -1,13 +0,0 @@ -// import OpenAPIParser from "@apidevtools/swagger-parser"; - -export function isValidOpenAPI(spec: string) { - console.log(spec); - // OpenAPIParser.validate(spec, (err) => { - // if (err) { - // console.log(err); - // } - return true; - // TODO: BRING THIS BACK - // return err ? err.message : true; - // }); -} From 0ea2ce99c105e4b2135b88396a84a14f88e3b136 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 9 May 2024 23:17:25 -0600 Subject: [PATCH 32/82] cleanup --- .../settings/(nav)/assistants/[assistantId]/edit/+page.server.ts | 1 - src/routes/settings/(nav)/assistants/new/+page.server.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts b/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts index 93e3e821ab6..78c74af50c3 100644 --- a/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts +++ b/src/routes/settings/(nav)/assistants/[assistantId]/edit/+page.server.ts @@ -10,7 +10,6 @@ import { sha256 } from "$lib/utils/sha256"; import sharp from "sharp"; import { parseStringToList } from "$lib/utils/parseStringToList"; import { generateSearchTokens } from "$lib/utils/searchTokens"; -import { isValidOpenAPI } from "$lib/utils/isValidOpenAPI"; const newAsssistantSchema = z.object({ name: z.string().min(1), diff --git a/src/routes/settings/(nav)/assistants/new/+page.server.ts b/src/routes/settings/(nav)/assistants/new/+page.server.ts index 9ae860a1944..50e4b738372 100644 --- a/src/routes/settings/(nav)/assistants/new/+page.server.ts +++ b/src/routes/settings/(nav)/assistants/new/+page.server.ts @@ -10,7 +10,6 @@ import sharp from "sharp"; import { parseStringToList } from "$lib/utils/parseStringToList"; import { usageLimits } from "$lib/server/usageLimits"; import { generateSearchTokens } from "$lib/utils/searchTokens"; -import { isValidOpenAPI } from "$lib/utils/isValidOpenAPI"; const newAsssistantSchema = z.object({ name: z.string().min(1), From fc9049fd369f5556b2dc15ef3caf18f2862a7e85 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 9 May 2024 23:18:00 -0600 Subject: [PATCH 33/82] package cleanups --- package-lock.json | 123 +--------------------------------------------- package.json | 2 - 2 files changed, 2 insertions(+), 123 deletions(-) diff --git a/package-lock.json b/package-lock.json index 83828f3c400..9dbe1e5f7d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,13 +8,11 @@ "name": "chat-ui", "version": "0.8.4", "dependencies": { - "@apidevtools/swagger-parser": "^10.1.0", "@gradio/client": "^0.18.0", "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", "@resvg/resvg-js": "^2.6.0", - "@resvg/resvg-js-linux-x64-gnu": "*", "@xenova/transformers": "^2.16.1", "autoprefixer": "^10.4.14", "browser-image-resizer": "^2.4.1", @@ -66,7 +64,6 @@ "eslint-plugin-svelte": "^2.30.0", "js-yaml": "^4.1.0", "minimist": "^1.2.8", - "openapi-types": "^12.1.3", "prettier": "^2.8.0", "prettier-plugin-svelte": "^2.10.1", "prettier-plugin-tailwindcss": "^0.2.7", @@ -167,94 +164,6 @@ "google-auth-library": "^9.4.2" } }, - "node_modules/@apidevtools/json-schema-ref-parser": { - "version": "9.0.6", - "license": "MIT", - "dependencies": { - "@jsdevtools/ono": "^7.1.3", - "call-me-maybe": "^1.0.1", - "js-yaml": "^3.13.1" - } - }, - "node_modules/@apidevtools/json-schema-ref-parser/node_modules/argparse": { - "version": "1.0.10", - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/@apidevtools/json-schema-ref-parser/node_modules/js-yaml": { - "version": "3.14.1", - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/@apidevtools/json-schema-ref-parser/node_modules/sprintf-js": { - "version": "1.0.3", - "license": "BSD-3-Clause" - }, - "node_modules/@apidevtools/openapi-schemas": { - "version": "2.1.0", - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/@apidevtools/swagger-methods": { - "version": "3.0.2", - "license": "MIT" - }, - "node_modules/@apidevtools/swagger-parser": { - "version": "10.1.0", - "license": "MIT", - "dependencies": { - "@apidevtools/json-schema-ref-parser": "9.0.6", - "@apidevtools/openapi-schemas": "^2.1.0", - "@apidevtools/swagger-methods": "^3.0.2", - "@jsdevtools/ono": "^7.1.3", - "ajv": "^8.6.3", - "ajv-draft-04": "^1.0.0", - "call-me-maybe": "^1.0.1" - }, - "peerDependencies": { - "openapi-types": ">=7" - } - }, - "node_modules/@apidevtools/swagger-parser/node_modules/ajv": { - "version": "8.13.0", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.4.1" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/@apidevtools/swagger-parser/node_modules/ajv-draft-04": { - "version": "1.0.0", - "license": "MIT", - "peerDependencies": { - "ajv": "^8.5.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/@apidevtools/swagger-parser/node_modules/json-schema-traverse": { - "version": "1.0.0", - "license": "MIT" - }, "node_modules/@bundled-es-modules/cookie": { "version": "2.0.0", "license": "ISC", @@ -1039,10 +948,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@jsdevtools/ono": { - "version": "7.1.3", - "license": "MIT" - }, "node_modules/@mongodb-js/saslprep": { "version": "1.1.0", "license": "MIT", @@ -2708,10 +2613,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/call-me-maybe": { - "version": "1.0.2", - "license": "MIT" - }, "node_modules/callsites": { "version": "3.1.0", "dev": true, @@ -3634,17 +3535,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/esprima": { - "version": "4.0.1", - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/esquery": { "version": "1.4.2", "dev": true, @@ -3835,6 +3725,7 @@ }, "node_modules/fast-deep-equal": { "version": "3.1.3", + "dev": true, "license": "MIT" }, "node_modules/fast-diff": { @@ -5640,10 +5531,6 @@ "node": ">= 8" } }, - "node_modules/openapi-types": { - "version": "12.1.3", - "license": "MIT" - }, "node_modules/openid-client": { "version": "5.4.2", "license": "MIT", @@ -6609,13 +6496,6 @@ "node": ">=0.10.0" } }, - "node_modules/require-from-string": { - "version": "2.0.2", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/requires-port": { "version": "1.0.0", "license": "MIT" @@ -8016,6 +7896,7 @@ }, "node_modules/uri-js": { "version": "4.4.1", + "dev": true, "license": "BSD-2-Clause", "dependencies": { "punycode": "^2.1.0" diff --git a/package.json b/package.json index 1073d06ec30..573aff322cf 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,6 @@ "eslint-plugin-svelte": "^2.30.0", "js-yaml": "^4.1.0", "minimist": "^1.2.8", - "openapi-types": "^12.1.3", "prettier": "^2.8.0", "prettier-plugin-svelte": "^2.10.1", "prettier-plugin-tailwindcss": "^0.2.7", @@ -53,7 +52,6 @@ }, "type": "module", "dependencies": { - "@apidevtools/swagger-parser": "^10.1.0", "@gradio/client": "^0.18.0", "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", From ab6fbfbd07ac2eac8196160b372d41031007fa48 Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Thu, 9 May 2024 23:19:30 -0600 Subject: [PATCH 34/82] moar cleanup --- src/lib/stores/webSearchParameters.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/stores/webSearchParameters.ts b/src/lib/stores/webSearchParameters.ts index a4ee2ab0128..fd088a60621 100644 --- a/src/lib/stores/webSearchParameters.ts +++ b/src/lib/stores/webSearchParameters.ts @@ -1,11 +1,9 @@ import { writable } from "svelte/store"; export interface WebSearchParameters { useSearch: boolean; - useTools: boolean; nItems: number; } export const webSearchParameters = writable({ useSearch: false, - useTools: true, // since the model will decide whether or not to use tools, it makes sense to default to true here nItems: 5, }); From 98f9009f0d15e8afb41518fa806437f9bedb9e5e Mon Sep 17 00:00:00 2001 From: Victor Mustar Date: Fri, 10 May 2024 14:17:01 +0200 Subject: [PATCH 35/82] ui update --- src/lib/components/ToolsMenu.svelte | 8 +- src/lib/components/chat/ChatMessage.svelte | 92 ++++++++++++++++------ 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/src/lib/components/ToolsMenu.svelte b/src/lib/components/ToolsMenu.svelte index 780d8086f0b..10de6731f58 100644 --- a/src/lib/components/ToolsMenu.svelte +++ b/src/lib/components/ToolsMenu.svelte @@ -17,20 +17,20 @@
Tools - ({activeToolCount}) + ({activeToolCount})
-
+
{#each $page.data.tools as tool} {@const isChecked = $settings?.tools?.[tool.name] ?? tool.isOnByDefault} -
+
t.messageType === "message")} {#if toolName && toolName !== "websearch"}
- {#if !toolDone} - - {/if} - {toolDone ? "Called" : "Calling"} tool - {availableTools.find((el) => el.name === toolName)?.displayName} + + + + +
+ + + {toolDone ? "Called" : "Calling"} tool + {availableTools.find((el) => el.name === toolName)?.displayName} + {#each tool as toolUpdate} {#if toolUpdate.messageType === "parameters"} -
- -

Parameters:

-
    +
    +

    Parameters

    +
    +
    +
      {#each Object.entries(toolUpdate.parameters ?? {}) as [k, v]}
    • - {k}: {v} + {k}: + {v}
      • {/each}
    {:else if toolUpdate.messageType === "message" && toolUpdate.display !== false} -
    - -

    Result:

    -

    - {" > "}{toolUpdate.message} +

    +

    Result

    +
    +
    + +

    + > + {toolUpdate.message}

    {/if} {/each} @@ -576,3 +599,20 @@ {/if} + + From d9c83f337b94d96200200038389c460638300ce2 Mon Sep 17 00:00:00 2001 From: Victor Mustar Date: Fri, 10 May 2024 17:07:43 +0200 Subject: [PATCH 36/82] Update ChatMessage.svelte --- src/lib/components/chat/ChatMessage.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 1cf48dee48f..be7fc30b2cb 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -279,7 +279,7 @@ {@const toolDone = !!tool.find((t) => t.messageType === "message")} {#if toolName && toolName !== "websearch"}
    Date: Fri, 10 May 2024 12:41:46 -0600 Subject: [PATCH 37/82] upgrade gradio dep --- package-lock.json | 152 ++++++++++++++++++++++++++++++---------------- package.json | 2 +- 2 files changed, 100 insertions(+), 54 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9dbe1e5f7d8..0f8aec88436 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,7 @@ "name": "chat-ui", "version": "0.8.4", "dependencies": { - "@gradio/client": "^0.18.0", + "@gradio/client": "^0.19.0", "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", @@ -166,14 +166,16 @@ }, "node_modules/@bundled-es-modules/cookie": { "version": "2.0.0", - "license": "ISC", + "resolved": "https://registry.npmjs.org/@bundled-es-modules/cookie/-/cookie-2.0.0.tgz", + "integrity": "sha512-Or6YHg/kamKHpxULAdSqhGqnWFneIXu1NKvvfBBzKGwpVsYuFIQ5aBPHDnnoR3ghW1nvSkALd+EF9iMtY7Vjxw==", "dependencies": { "cookie": "^0.5.0" } }, "node_modules/@bundled-es-modules/statuses": { "version": "1.0.1", - "license": "ISC", + "resolved": "https://registry.npmjs.org/@bundled-es-modules/statuses/-/statuses-1.0.1.tgz", + "integrity": "sha512-yn7BklA5acgcBr+7w064fGV+SGIFySjCKpqjcWgBAIfrAkY+4GQTJJHQMeT3V/sgz23VTEVV8TtOmkvJAhFVfg==", "dependencies": { "statuses": "^2.0.1" } @@ -308,8 +310,9 @@ } }, "node_modules/@gradio/client": { - "version": "0.18.0", - "license": "ISC", + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@gradio/client/-/client-0.19.0.tgz", + "integrity": "sha512-66ypWC7QoIN6ntBx6FpG1twIXdEHgnVDZWubuFe5YwyrS3pzalAE2zuTmlcYip5qIxmJ0RNbFGY2qYRDog2Szw==", "dependencies": { "@types/eventsource": "^1.1.15", "bufferutil": "^4.0.7", @@ -848,7 +851,8 @@ }, "node_modules/@inquirer/confirm": { "version": "3.1.6", - "license": "MIT", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.1.6.tgz", + "integrity": "sha512-Mj4TU29g6Uy+37UtpA8UpEOI2icBfpCwSW1QDtfx60wRhUy90s/kHPif2OXSSvuwDQT1lhAYRWUfkNf9Tecxvg==", "dependencies": { "@inquirer/core": "^8.1.0", "@inquirer/type": "^1.3.1" @@ -859,7 +863,8 @@ }, "node_modules/@inquirer/core": { "version": "8.1.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-8.1.0.tgz", + "integrity": "sha512-kfx0SU9nWgGe1f03ao/uXc85SFH1v2w3vQVH7QDGjKxdtJz+7vPitFtG++BTyJMYyYgH8MpXigutcXJeiQwVRw==", "dependencies": { "@inquirer/figures": "^1.0.1", "@inquirer/type": "^1.3.1", @@ -880,15 +885,17 @@ } }, "node_modules/@inquirer/core/node_modules/@types/node": { - "version": "20.12.8", - "license": "MIT", + "version": "20.12.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.11.tgz", + "integrity": "sha512-vDg9PZ/zi+Nqp6boSOT7plNuthRugEKixDv5sFTIpkE89MmNtEArAShI4mxuX2+UrLEe9pxC1vm2cjm9YlWbJw==", "dependencies": { "undici-types": "~5.26.4" } }, "node_modules/@inquirer/core/node_modules/signal-exit": { "version": "4.1.0", - "license": "ISC", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", "engines": { "node": ">=14" }, @@ -898,14 +905,16 @@ }, "node_modules/@inquirer/figures": { "version": "1.0.1", - "license": "MIT", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.1.tgz", + "integrity": "sha512-mtup3wVKia3ZwULPHcbs4Mor8Voi+iIXEWD7wCNbIO6lYR62oPCTQyrddi5OMYVXHzeCSoneZwJuS8sBvlEwDw==", "engines": { "node": ">=18" } }, "node_modules/@inquirer/type": { "version": "1.3.1", - "license": "MIT", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.3.1.tgz", + "integrity": "sha512-Pe3PFccjPVJV1vtlfVvm9OnlbxqdnP5QcscFEFEnK5quChf1ufZtM0r8mR5ToWHMxZOh0s8o/qp9ANGRTo/DAw==", "engines": { "node": ">=18" } @@ -958,14 +967,16 @@ }, "node_modules/@mswjs/cookies": { "version": "1.1.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/@mswjs/cookies/-/cookies-1.1.0.tgz", + "integrity": "sha512-0ZcCVQxifZmhwNBoQIrystCb+2sWBY2Zw8lpfJBPCHGCA/HWqehITeCRVIv4VMy8MPlaHo2w2pTHFV2pFfqKPw==", "engines": { "node": ">=18" } }, "node_modules/@mswjs/interceptors": { - "version": "0.26.15", - "license": "MIT", + "version": "0.29.1", + "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.29.1.tgz", + "integrity": "sha512-3rDakgJZ77+RiQUuSK69t1F0m8BQKA8Vh5DCS5V0DWvNY67zob2JhhQrhCO0AKLGINTRSFd1tBaHcJTkhefoSw==", "dependencies": { "@open-draft/deferred-promise": "^2.2.0", "@open-draft/logger": "^0.3.0", @@ -1009,11 +1020,13 @@ }, "node_modules/@open-draft/deferred-promise": { "version": "2.2.0", - "license": "MIT" + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", + "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==" }, "node_modules/@open-draft/logger": { "version": "0.3.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", + "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", "dependencies": { "is-node-process": "^1.2.0", "outvariant": "^1.4.0" @@ -1021,7 +1034,8 @@ }, "node_modules/@open-draft/until": { "version": "2.1.0", - "license": "MIT" + "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", + "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==" }, "node_modules/@opentelemetry/api": { "version": "1.8.0", @@ -1623,7 +1637,8 @@ }, "node_modules/@types/eventsource": { "version": "1.1.15", - "license": "MIT" + "resolved": "https://registry.npmjs.org/@types/eventsource/-/eventsource-1.1.15.tgz", + "integrity": "sha512-XQmGcbnxUNa06HR3VBVkc9+A2Vpi9ZyLJcdS5dwaQQ/4ZMWFO+5c90FnMUpbtMZwB/FChoYHwuVg8TvkECacTA==" }, "node_modules/@types/express": { "version": "4.17.21", @@ -1693,7 +1708,8 @@ }, "node_modules/@types/mute-stream": { "version": "0.0.4", - "license": "MIT", + "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.4.tgz", + "integrity": "sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==", "dependencies": { "@types/node": "*" } @@ -1773,7 +1789,8 @@ }, "node_modules/@types/statuses": { "version": "2.0.5", - "license": "MIT" + "resolved": "https://registry.npmjs.org/@types/statuses/-/statuses-2.0.5.tgz", + "integrity": "sha512-jmIUGWrAiwu3dZpxntxieC+1n/5c3mjrImkmOSQ2NC5uP6cYO4aAZDdSmRcI5C1oiTmqlZGHC+/NmJrKogbP5A==" }, "node_modules/@types/tough-cookie": { "version": "4.0.2", @@ -1799,7 +1816,8 @@ }, "node_modules/@types/wrap-ansi": { "version": "3.0.0", - "license": "MIT" + "resolved": "https://registry.npmjs.org/@types/wrap-ansi/-/wrap-ansi-3.0.0.tgz", + "integrity": "sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==" }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "6.7.4", @@ -2194,7 +2212,8 @@ }, "node_modules/ansi-escapes": { "version": "4.3.2", - "license": "MIT", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", "dependencies": { "type-fest": "^0.21.3" }, @@ -2207,7 +2226,8 @@ }, "node_modules/ansi-escapes/node_modules/type-fest": { "version": "0.21.3", - "license": "(MIT OR CC0-1.0)", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", "engines": { "node": ">=10" }, @@ -2741,7 +2761,8 @@ }, "node_modules/cli-spinners": { "version": "2.9.2", - "license": "MIT", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", "engines": { "node": ">=6" }, @@ -2751,14 +2772,16 @@ }, "node_modules/cli-width": { "version": "4.1.0", - "license": "ISC", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", "engines": { "node": ">= 12" } }, "node_modules/cliui": { "version": "8.0.1", - "license": "ISC", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", @@ -2770,7 +2793,8 @@ }, "node_modules/cliui/node_modules/wrap-ansi": { "version": "7.0.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -3609,7 +3633,8 @@ }, "node_modules/eventsource": { "version": "2.0.2", - "license": "MIT", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-2.0.2.tgz", + "integrity": "sha512-IzUmBGPR3+oUG9dUeXynyNmf91/3zUSJg1lCktzKw47OXuhco54U3r9B7O4XX+Rb1Itm9OZ2b0RkTs10bICOxA==", "engines": { "node": ">=12.0.0" } @@ -4008,7 +4033,8 @@ }, "node_modules/get-caller-file": { "version": "2.0.5", - "license": "ISC", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "engines": { "node": "6.* || 8.* || >= 10.*" } @@ -4168,7 +4194,8 @@ }, "node_modules/graphql": { "version": "16.8.1", - "license": "MIT", + "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.8.1.tgz", + "integrity": "sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw==", "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -4261,7 +4288,8 @@ }, "node_modules/headers-polyfill": { "version": "4.0.3", - "license": "MIT" + "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-4.0.3.tgz", + "integrity": "sha512-IScLbePpkvO846sIwOtOTDjutRMWdXdJmXdMvk6gCBHxFO8d+QKOQedyZSxFTTFYRSmlgSTDtXqqq4pcenBXLQ==" }, "node_modules/help-me": { "version": "5.0.0", @@ -4509,7 +4537,8 @@ }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "engines": { "node": ">=8" } @@ -4531,7 +4560,8 @@ }, "node_modules/is-node-process": { "version": "1.2.0", - "license": "MIT" + "resolved": "https://registry.npmjs.org/is-node-process/-/is-node-process-1.2.0.tgz", + "integrity": "sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==" }, "node_modules/is-number": { "version": "7.0.0", @@ -5170,15 +5200,16 @@ "license": "MIT" }, "node_modules/msw": { - "version": "2.2.14", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/msw/-/msw-2.3.0.tgz", + "integrity": "sha512-cDr1q/QTMzaWhY8n9lpGhceY209k29UZtdTgJ3P8Bzne3TSMchX2EM/ldvn4ATLOktpCefCU2gcEgzHc31GTPw==", "hasInstallScript": true, - "license": "MIT", "dependencies": { "@bundled-es-modules/cookie": "^2.0.0", "@bundled-es-modules/statuses": "^1.0.1", "@inquirer/confirm": "^3.0.0", "@mswjs/cookies": "^1.1.0", - "@mswjs/interceptors": "^0.26.14", + "@mswjs/interceptors": "^0.29.0", "@open-draft/until": "^2.1.0", "@types/cookie": "^0.6.0", "@types/statuses": "^2.0.4", @@ -5212,15 +5243,18 @@ }, "node_modules/msw/node_modules/@types/cookie": { "version": "0.6.0", - "license": "MIT" + "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==" }, "node_modules/msw/node_modules/path-to-regexp": { "version": "6.2.2", - "license": "MIT" + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz", + "integrity": "sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==" }, "node_modules/msw/node_modules/type-fest": { - "version": "4.18.1", - "license": "(MIT OR CC0-1.0)", + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.18.2.tgz", + "integrity": "sha512-+suCYpfJLAe4OXS6+PPXjW3urOS4IoP9waSiLuXfLgqZODKw/aWwASvzqE886wA0kQgGy0mIWyhd87VpqIy6Xg==", "engines": { "node": ">=16" }, @@ -5230,7 +5264,8 @@ }, "node_modules/mute-stream": { "version": "1.0.0", - "license": "ISC", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", + "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", "engines": { "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } @@ -5569,7 +5604,8 @@ }, "node_modules/outvariant": { "version": "1.4.2", - "license": "MIT" + "resolved": "https://registry.npmjs.org/outvariant/-/outvariant-1.4.2.tgz", + "integrity": "sha512-Ou3dJ6bA/UJ5GVHxah4LnqDwZRwAmWxrG3wtrHrbGnP4RnLCtA64A4F+ae7Y8ww660JaddSoArUR5HjipWSHAQ==" }, "node_modules/p-limit": { "version": "3.1.0", @@ -6491,7 +6527,8 @@ }, "node_modules/require-directory": { "version": "2.1.1", - "license": "MIT", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "engines": { "node": ">=0.10.0" } @@ -6699,7 +6736,8 @@ }, "node_modules/semiver": { "version": "1.1.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/semiver/-/semiver-1.1.0.tgz", + "integrity": "sha512-QNI2ChmuioGC1/xjyYwyZYADILWyW6AmS1UH6gDj/SFUUUS4MBAWs/7mxnkRPc/F4iHezDP+O8t0dO8WHiEOdg==", "engines": { "node": ">=6" } @@ -7055,7 +7093,8 @@ }, "node_modules/strict-event-emitter": { "version": "0.5.1", - "license": "MIT" + "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz", + "integrity": "sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==" }, "node_modules/string_decoder": { "version": "1.3.0", @@ -7066,7 +7105,8 @@ }, "node_modules/string-width": { "version": "4.2.3", - "license": "MIT", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -7078,7 +7118,8 @@ }, "node_modules/string-width/node_modules/emoji-regex": { "version": "8.0.0", - "license": "MIT" + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/string.prototype.codepointat": { "version": "0.2.1", @@ -7798,7 +7839,8 @@ }, "node_modules/undici-types": { "version": "5.26.5", - "license": "MIT" + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, "node_modules/unicode-trie": { "version": "2.0.0", @@ -8394,7 +8436,8 @@ }, "node_modules/wrap-ansi": { "version": "6.2.0", - "license": "MIT", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -8440,7 +8483,8 @@ }, "node_modules/y18n": { "version": "5.0.8", - "license": "ISC", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", "engines": { "node": ">=10" } @@ -8459,7 +8503,8 @@ }, "node_modules/yargs": { "version": "17.7.2", - "license": "MIT", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -8475,7 +8520,8 @@ }, "node_modules/yargs-parser": { "version": "21.1.1", - "license": "ISC", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", "engines": { "node": ">=12" } diff --git a/package.json b/package.json index 573aff322cf..ea98769d817 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ }, "type": "module", "dependencies": { - "@gradio/client": "^0.18.0", + "@gradio/client": "^0.19.0", "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", "@iconify-json/bi": "^1.1.21", From 515826a9a2deb0127df819ff33874f3920d44d0c Mon Sep 17 00:00:00 2001 From: Nathan Sarrazin Date: Fri, 10 May 2024 12:55:02 -0600 Subject: [PATCH 38/82] lint --- src/routes/conversation/[id]/+server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index d361101d107..48de1e96e5e 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -547,7 +547,7 @@ export async function POST({ request, locals, params, getClientAddress }) { 512, // number in 'Image Width' Number component call.parameters?.prompt, // prompt Math.floor(Math.random() * 1000), // seed random - ])) as { data: Array> }; + ])) as unknown as { data: Array> }; const response = await fetch(res?.data?.[0]?.[0]?.image?.url ?? "error"); From 0cf2526e62c35d8f3b708c9e411246d512ff9072 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 15 May 2024 15:45:47 -0400 Subject: [PATCH 39/82] refactor and pass results to cohere --- .../server/endpoints/cohere/endpointCohere.ts | 90 +++- src/lib/server/models.ts | 9 +- src/lib/server/textGeneration/assistant.ts | 54 ++ src/lib/server/textGeneration/generate.ts | 67 +++ src/lib/server/textGeneration/index.ts | 72 +++ .../{summarize.ts => textGeneration/title.ts} | 2 +- src/lib/server/textGeneration/tools.ts | 82 +++ src/lib/server/textGeneration/types.ts | 132 +++++ src/lib/server/tools/calculator.ts | 15 +- src/lib/server/tools/directlyAnswer.ts | 10 +- src/lib/server/tools/fetchUrl.ts | 28 +- src/lib/server/tools/index.ts | 18 +- src/lib/server/tools/text2img.ts | 33 ++ src/lib/server/tools/websearch.ts | 20 + src/lib/server/websearch/runWebSearch.ts | 61 ++- src/lib/types/MessageUpdate.ts | 4 +- src/lib/types/Tool.ts | 23 +- src/routes/+layout.server.ts | 4 +- src/routes/conversation/[id]/+server.ts | 481 ++++-------------- 19 files changed, 747 insertions(+), 458 deletions(-) create mode 100644 src/lib/server/textGeneration/assistant.ts create mode 100644 src/lib/server/textGeneration/generate.ts create mode 100644 src/lib/server/textGeneration/index.ts rename src/lib/server/{summarize.ts => textGeneration/title.ts} (97%) create mode 100644 src/lib/server/textGeneration/tools.ts create mode 100644 src/lib/server/textGeneration/types.ts diff --git a/src/lib/server/endpoints/cohere/endpointCohere.ts b/src/lib/server/endpoints/cohere/endpointCohere.ts index 1d4503da938..1c503a41ea9 100644 --- a/src/lib/server/endpoints/cohere/endpointCohere.ts +++ b/src/lib/server/endpoints/cohere/endpointCohere.ts @@ -4,6 +4,8 @@ import type { Endpoint } from "../endpoints"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; import type { Cohere, CohereClient } from "cohere-ai"; import { buildPrompt } from "$lib/buildPrompt"; +import { ToolResultStatus, type Tool, type ToolResult } from "$lib/types/Tool"; +import stream from "stream"; export const endpointCohereParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -34,6 +36,9 @@ export async function endpointCohere( system = messages[0].content; } + tools = tools?.filter((tool) => tool.name !== "directly_answer")?.map(toCohereTool); + console.log(toolResults); + const parameters = { ...model.parameters, ...generateSettings }; return (async function* () { @@ -69,20 +74,34 @@ export async function endpointCohere( message: message.content, })) satisfies Cohere.ChatMessage[]; - stream = await cohere.chatStream({ - model: model.id ?? model.name, - chatHistory: formattedMessages.slice(0, -1), - message: formattedMessages[formattedMessages.length - 1].message, - preamble: system, - p: parameters?.top_p, - k: parameters?.top_k, - maxTokens: parameters?.max_new_tokens, - temperature: parameters?.temperature, - stopSequences: parameters?.stop, - frequencyPenalty: parameters?.frequency_penalty, - tools, - //TODO: Add toolResults - }); + stream = await cohere + .chatStream({ + model: model.id ?? model.name, + chatHistory: formattedMessages.slice(0, -1), + message: formattedMessages[formattedMessages.length - 1].message, + preamble: system, + p: parameters?.top_p, + k: parameters?.top_k, + maxTokens: parameters?.max_new_tokens, + temperature: parameters?.temperature, + stopSequences: parameters?.stop, + frequencyPenalty: parameters?.frequency_penalty, + tools, + toolResults: toolResults?.map((toolResult) => { + if (toolResult.status === ToolResultStatus.Error) { + return { call: toolResult.call, outputs: [{ error: toolResult.message }] }; + } + return { call: toolResult.call, outputs: toolResult.outputs }; + }), + }) + .catch((err) => { + if (err.body) { + convertWebStreamToBuffer(err.body).then(console.log).catch(console.error); + } else { + console.error(err); + } + throw err; + }); } for await (const output of stream) { @@ -97,6 +116,18 @@ export async function endpointCohere( generated_text: null, details: null, } satisfies TextGenerationStreamOutput; + } else if (output.eventType === "tool-calls-generation") { + yield { + token: { + id: tokenId++, + text: "", + logprob: 0, + special: true, + toolCalls: output.toolCalls, + }, + generated_text: null, + details: null, + }; } else if (output.eventType === "stream-end") { if (["ERROR", "ERROR_TOXIC", "ERROR_LIMIT"].includes(output.finishReason)) { throw new Error(output.finishReason); @@ -116,3 +147,34 @@ export async function endpointCohere( })(); }; } + +function toCohereTool(tool: Tool): Cohere.Tool { + return { + name: tool.name, + description: tool.description, + parameterDefinitions: tool.parameter_definitions, + }; +} + +async function convertWebStreamToBuffer(webReadableStream) { + return new Promise((resolve, reject) => { + const chunks = []; + + stream.pipeline( + webReadableStream, + new stream.Writable({ + write(chunk, encoding, callback) { + chunks.push(chunk); + callback(); + }, + }), + (err) => { + if (err) { + reject(err); + } else { + resolve(Buffer.concat(chunks).toString("utf-8")); + } + } + ); + }); +} diff --git a/src/lib/server/models.ts b/src/lib/server/models.ts index b852a609536..a9f68c78b17 100644 --- a/src/lib/server/models.ts +++ b/src/lib/server/models.ts @@ -161,6 +161,11 @@ const processModel = async (m: z.infer) => ({ parameters: { ...m.parameters, stop_sequences: m.parameters?.stop }, }); +// FIXME: temporary +export type ProcessedModel = Awaited> & { + getEndpoint: () => Promise; +}; + const addEndpoint = (m: Awaited>) => ({ ...m, getEndpoint: async (): Promise => { @@ -216,7 +221,9 @@ const addEndpoint = (m: Awaited>) => ({ }, }); -export const models = await Promise.all(modelsRaw.map((e) => processModel(e).then(addEndpoint))); +export const models: ProcessedModel[] = await Promise.all( + modelsRaw.map((e) => processModel(e).then(addEndpoint)) +); export const defaultModel = models[0]; diff --git a/src/lib/server/textGeneration/assistant.ts b/src/lib/server/textGeneration/assistant.ts new file mode 100644 index 00000000000..332f0b2f25e --- /dev/null +++ b/src/lib/server/textGeneration/assistant.ts @@ -0,0 +1,54 @@ +import { isURLLocal } from "../isURLLocal"; +import { env } from "$env/dynamic/private"; +import { collections } from "$lib/server/database"; +import type { Assistant } from "$lib/types/Assistant"; +import type { ObjectId } from "mongodb"; + +export async function processPreprompt(preprompt: string) { + const urlRegex = /{{\s?url=(.*?)\s?}}/g; + + let match; + while ((match = urlRegex.exec(preprompt)) !== null) { + try { + const url = new URL(match[1]); + if ((await isURLLocal(url)) && env.ENABLE_LOCAL_FETCH !== "true") { + throw new Error("URL couldn't be fetched, it resolved to a local address."); + } + + const res = await fetch(url.href); + + if (!res.ok) { + throw new Error("URL couldn't be fetched, error " + res.status); + } + const text = await res.text(); + preprompt = preprompt.replaceAll(match[0], text); + } catch (e) { + preprompt = preprompt.replaceAll(match[0], (e as Error).message); + } + } + + return preprompt; +} + +export async function getAssistantById(id?: ObjectId) { + return collections.assistants + .findOne>( + { _id: id }, + { projection: { rag: 1, dynamicPrompt: 1, generateSettings: 1 } } + ) + .then((a) => a ?? undefined); +} + +export function assistantHasWebSearch(assistant?: Pick | null) { + return ( + env.ENABLE_ASSISTANTS_RAG === "true" && + !!assistant?.rag && + (assistant.rag.allowedLinks.length > 0 || + assistant.rag.allowedDomains.length > 0 || + assistant.rag.allowAllDomains) + ); +} + +export function assistantHasDynamicPrompt(assistant?: Pick) { + return env.ENABLE_ASSISTANTS_RAG === "true" && Boolean(assistant?.dynamicPrompt); +} diff --git a/src/lib/server/textGeneration/generate.ts b/src/lib/server/textGeneration/generate.ts new file mode 100644 index 00000000000..7beb258d2e3 --- /dev/null +++ b/src/lib/server/textGeneration/generate.ts @@ -0,0 +1,67 @@ +import type { ToolResult } from "$lib/types/Tool"; +import { + TextGenerationUpdateType, + type TextGenerationContext, + type TextGenerationUpdate, +} from "./types"; +import { AbortedGenerations } from "../abortedGenerations"; + +export async function* generate( + { model, endpoint, conv, messages, assistant, isContinue, promptedAt }: TextGenerationContext, + toolResults: ToolResult[], + preprompt?: string +): AsyncIterable { + let buffer = ""; + for await (const output of await endpoint({ + messages, + preprompt, + continueMessage: isContinue, + generateSettings: assistant?.generateSettings, + toolResults, + })) { + if (output.generated_text) { + let interrupted = + !output.token.special && !model.parameters.stop?.includes(output.token.text); + + let text = output.generated_text.trimEnd(); + for (const stopToken of model.parameters.stop ?? []) { + if (!text.endsWith(stopToken)) continue; + + interrupted = false; + text = text.slice(0, text.length - stopToken.length); + } + + yield { type: TextGenerationUpdateType.FinalAnswer, text, interrupted }; + } + + // todo: and if there is a special token? + if (!output.token.special) { + buffer += output.token.text; + + // send 5 chars and leave the rest in the buffer + if (buffer.length >= 5) { + yield { + type: TextGenerationUpdateType.Stream, + token: buffer.slice(0, 5), + }; + buffer = buffer.slice(5); + } + + // abort check + const date = AbortedGenerations.getInstance().getList().get(conv._id.toString()); + if (date && date > promptedAt) break; + + // no output check + if (!output) break; + } + } + + // pass down the remaining buffer + // TODO: ensure this happens before the final answer + // if (buffer) { + // yield { + // type: TextGenerationUpdateType.Stream, + // token: buffer, + // }; + // } +} diff --git a/src/lib/server/textGeneration/index.ts b/src/lib/server/textGeneration/index.ts new file mode 100644 index 00000000000..d34039ba3ff --- /dev/null +++ b/src/lib/server/textGeneration/index.ts @@ -0,0 +1,72 @@ +import { runWebSearch } from "$lib/server/websearch/runWebSearch"; +import { preprocessMessages } from "$lib/server/preprocessMessages.js"; + +import { generateTitle } from "./title"; +import { + assistantHasDynamicPrompt, + assistantHasWebSearch, + getAssistantById, + processPreprompt, +} from "./assistant"; +import { pickTools, runTools } from "./tools"; +import type { WebSearch } from "$lib/types/WebSearch"; +import { + type TextGenerationUpdate, + TextGenerationUpdateType, + TextGenerationStatus, + type TextGenerationContext, +} from "./types"; +import { generate } from "./generate"; + +export async function* textGeneration( + ctx: TextGenerationContext +): AsyncIterable { + yield { + type: TextGenerationUpdateType.Status, + status: TextGenerationStatus.Started, + }; + + ctx.assistant ??= await getAssistantById(ctx.conv.assistantId); + const { model, conv, messages, assistant, isContinue, webSearch, toolsPreference } = ctx; + const convId = conv._id; + + // TODO: should be a better way to detect if the conversation is new + // TODO: dont wait for this + if (conv.title === "New Chat" && conv.messages.length === 3) { + const title = (await generateTitle(conv.messages[1].content)) ?? "New Chat"; + yield { + type: TextGenerationUpdateType.Title, + title, + }; + } + + // perform websearch if requested + // it can be because the user toggled the webSearch or because the assistant has webSearch enabled + // if functions are enabled, we don't perform it here since we will add the websearch as a tool + let webSearchResult: WebSearch | undefined; + if ( + !isContinue && + !model.functions && + ((webSearch && !conv.assistantId) || assistantHasWebSearch(assistant)) + ) { + webSearchResult = yield* runWebSearch(conv, messages, { ragSettings: assistant?.rag }); + } + + // process preprompt + let preprompt = conv.preprompt; + if (assistantHasDynamicPrompt(assistant) && preprompt) { + preprompt = await processPreprompt(preprompt); + if (messages[0].from === "system") messages[0].content = preprompt; + } + + const tools = pickTools(toolsPreference, Boolean(assistant)); + const toolResults = yield* runTools(ctx, tools, preprompt); + + const processedMessages = await preprocessMessages( + messages, + webSearchResult, + model.multimodal, + convId + ); + yield* generate({ ...ctx, messages: processedMessages }, toolResults, preprompt); +} diff --git a/src/lib/server/summarize.ts b/src/lib/server/textGeneration/title.ts similarity index 97% rename from src/lib/server/summarize.ts rename to src/lib/server/textGeneration/title.ts index 7f4df69a3a5..3dd792d9a7c 100644 --- a/src/lib/server/summarize.ts +++ b/src/lib/server/textGeneration/title.ts @@ -3,7 +3,7 @@ import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndp import type { Message } from "$lib/types/Message"; import { logger } from "$lib/server/logger"; -export async function summarize(prompt: string) { +export async function generateTitle(prompt: string) { if (!env.LLM_SUMMERIZATION) { return prompt.split(/\s+/g).slice(0, 5).join(" "); } diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts new file mode 100644 index 00000000000..8262bcfa3cc --- /dev/null +++ b/src/lib/server/textGeneration/tools.ts @@ -0,0 +1,82 @@ +import { ToolResultStatus, type ToolCall, type ToolResult } from "$lib/types/Tool"; +import { v4 as uuidV4 } from "uuid"; +import type { BackendTool } from "../tools"; +import { + TextGenerationToolUpdateType, + TextGenerationUpdateType, + type TextGenerationContext, + type TextGenerationUpdate, +} from "./types"; + +import { allTools } from "../tools"; +import directlyAnswer from "../tools/directlyAnswer"; +import websearch from "../tools/websearch"; + +export function pickTools( + toolsPreference: Record, + isAssistant: boolean +): BackendTool[] { + // if it's an assistant, only support websearch for now + if (isAssistant) return [directlyAnswer, websearch]; + + // filter based on tool preferences, add the tools that are on by default + return allTools.filter((el) => { + if (el.isLocked && el.isOnByDefault) return true; + return toolsPreference?.[el.name] ?? el.isOnByDefault; + }); +} + +export async function* runTools( + { endpoint, conv, messages, assistant }: TextGenerationContext, + tools: BackendTool[], + preprompt?: string +): AsyncGenerator { + const calls: ToolCall[] = []; + + // do the function calling bits here + for await (const output of await endpoint({ + messages, + preprompt, + generateSettings: assistant?.generateSettings, + tools, + })) { + calls.push(...(output.token.toolCalls ?? [])); + } + + const toolResults: ToolResult[] = []; + for (const call of calls) { + const uuid = uuidV4(); + + yield { + type: TextGenerationUpdateType.Tool, + subtype: TextGenerationToolUpdateType.Parameters, + name: call.name, + parameters: call.parameters, + uuid, + }; + + const tool = tools.find((el) => el.name === call.name); + if (!tool) { + toolResults.push({ call, status: ToolResultStatus.Error, message: "Could not find tool" }); + continue; + } + try { + const toolResult = yield* tool.call(call.parameters, { + conv, + messages, + preprompt, + assistant, + }); + toolResults.push({ ...toolResult, call } as ToolResult); + } catch (cause) { + console.error(Error(`Failed while running tool ${call.name}`), { cause }); + toolResults.push({ + call, + status: ToolResultStatus.Error, + message: cause instanceof Error ? cause.message : String(cause), + }); + } + } + + return toolResults; +} diff --git a/src/lib/server/textGeneration/types.ts b/src/lib/server/textGeneration/types.ts new file mode 100644 index 00000000000..1de5ffd3b2a --- /dev/null +++ b/src/lib/server/textGeneration/types.ts @@ -0,0 +1,132 @@ +import type { WebSearch, WebSearchSource } from "$lib/types/WebSearch"; +import type { Tool } from "$lib/types/Tool"; +import type { Conversation } from "$lib/types/Conversation"; +import type { Message } from "$lib/types/Message"; +import type { Assistant } from "$lib/types/Assistant"; +import type { ProcessedModel } from "../models"; +import type { Endpoint } from "../endpoints/endpoints"; + +export interface TextGenerationContext { + model: ProcessedModel; + endpoint: Endpoint; + conv: Conversation; + messages: Message[]; + assistant?: Pick; + isContinue: boolean; + webSearch: boolean; + toolsPreference: Record; + promptedAt: Date; +} + +//---------- +// Text Generation Updates + +export type TextGenerationUpdate = + | TextGenerationStatusUpdate + | TextGenerationTitleUpdate + | TextGenerationToolUpdate + | TextGenerationWebSearchUpdate + | TextGenerationStreamUpdate + | TextGenerationFileUpdate + | TextGenerationFinalAnswerUpdate; + +export enum TextGenerationUpdateType { + Status = "status", + Title = "title", + Tool = "tool", + WebSearch = "webSearch", + Stream = "stream", + File = "file", + FinalAnswer = "finalAnswer", +} + +// Status +export enum TextGenerationStatus { + Started = "started", + Error = "error", + Finished = "finished", +} +interface TextGenerationStatusUpdate { + type: TextGenerationUpdateType.Status; + status: TextGenerationStatus; + message?: string; +} + +// Web search +export enum TextGenerationWebSearchUpdateType { + Update = "update", + Error = "error", + Sources = "sources", + FinalAnswer = "finalAnswer", +} +interface TextGenerationWebSearchErrorUpdate { + type: TextGenerationUpdateType.WebSearch; + subtype: TextGenerationWebSearchUpdateType.Error; + message: string; + args?: string[]; +} +interface TextGenerationWebSearchGeneralUpdate { + type: TextGenerationUpdateType.WebSearch; + subtype: TextGenerationWebSearchUpdateType.Update; + message: string; + args?: string[]; +} +interface TextGenerationWebSearchSourcesUpdate { + type: TextGenerationUpdateType.WebSearch; + subtype: TextGenerationWebSearchUpdateType.Sources; + message: string; + sources: WebSearchSource[]; +} +interface TextGenerationWebSearchFinalAnswerUpdate { + type: TextGenerationUpdateType.WebSearch; + subtype: TextGenerationWebSearchUpdateType.FinalAnswer; + webSearch: WebSearch; +} +export type TextGenerationWebSearchUpdate = + | TextGenerationWebSearchErrorUpdate + | TextGenerationWebSearchGeneralUpdate + | TextGenerationWebSearchSourcesUpdate + | TextGenerationWebSearchFinalAnswerUpdate; + +// Tool +export enum TextGenerationToolUpdateType { + Parameters = "parameters", + Message = "message", +} +interface TextGenerationToolBaseUpdate { + type: TextGenerationUpdateType.Tool; + subtype: TSubType; + name: Tool["name"]; + uuid: string; +} +interface TextGenerationToolParametersUpdate + extends TextGenerationToolBaseUpdate { + parameters: Record; +} +interface TextGenerationToolMessageUpdate + extends TextGenerationToolBaseUpdate { + message?: string; + display?: boolean; +} +type TextGenerationToolUpdate = + | TextGenerationToolParametersUpdate + | TextGenerationToolMessageUpdate; + +// Everything else +interface TextGenerationTitleUpdate { + type: TextGenerationUpdateType.Title; + title: string; +} +interface TextGenerationStreamUpdate { + type: TextGenerationUpdateType.Stream; + token: string; +} +interface TextGenerationFileUpdate { + type: TextGenerationUpdateType.File; + sha: string; +} +interface TextGenerationFinalAnswerUpdate { + type: TextGenerationUpdateType.FinalAnswer; + text: string; + interrupted: boolean; +} diff --git a/src/lib/server/tools/calculator.ts b/src/lib/server/tools/calculator.ts index 00077d8b7d7..248fe679e33 100644 --- a/src/lib/server/tools/calculator.ts +++ b/src/lib/server/tools/calculator.ts @@ -1,7 +1,8 @@ +import { ToolResultStatus } from "$lib/types/Tool"; import type { BackendTool } from "."; const calculator: BackendTool = { - name: "calculator", + name: "query_calculator", displayName: "Calculator", description: "A simple calculator, takes a string containing a mathematical expression and returns the answer. Only supports +, -, *, and /, as well as parenthesis ().", @@ -14,21 +15,19 @@ const calculator: BackendTool = { required: true, }, }, - call: async (params) => { + async *call(params) { try { const blocks = params.equation.split("\n"); const query = blocks[blocks.length - 1].replace(/[^-()\d/*+.]/g, ""); return { - key: "calculator", - status: "success", - value: eval(query), + status: ToolResultStatus.Success, + outputs: [{ calculator: eval(query) }], }; } catch (e) { return { - key: "calculator", - status: "error", - value: "Invalid expression", + status: ToolResultStatus.Error, + message: "Invalid expression", }; } }, diff --git a/src/lib/server/tools/directlyAnswer.ts b/src/lib/server/tools/directlyAnswer.ts index 9e6f0ea487a..197508c38f8 100644 --- a/src/lib/server/tools/directlyAnswer.ts +++ b/src/lib/server/tools/directlyAnswer.ts @@ -1,13 +1,21 @@ +import { ToolResultStatus } from "$lib/types/Tool"; import type { BackendTool } from "."; const directlyAnswer: BackendTool = { - name: "directly-answer", + name: "directly_answer", isOnByDefault: true, isHidden: true, isLocked: true, description: "Use this tool to let the user know you wish to answer directly. Do not try to provide any parameters when using this tool.", parameter_definitions: {}, + async *call() { + return { + status: ToolResultStatus.Success, + outputs: [], + display: false, + }; + }, }; export default directlyAnswer; diff --git a/src/lib/server/tools/fetchUrl.ts b/src/lib/server/tools/fetchUrl.ts index 465ff8ffc8f..aea3d3ce790 100644 --- a/src/lib/server/tools/fetchUrl.ts +++ b/src/lib/server/tools/fetchUrl.ts @@ -1,3 +1,4 @@ +import { ToolResultStatus } from "$lib/types/Tool"; import type { BackendTool } from "."; import { parseWeb } from "../websearch/parseWeb"; @@ -13,26 +14,17 @@ const fetchUrl: BackendTool = { required: true, }, }, - call: async (params) => { - try { - const blocks = params.url.split("\n"); - const url = blocks[blocks.length - 1]; + async *call(params) { + const blocks = params.url.split("\n"); + const url = blocks[blocks.length - 1]; - const content = await parseWeb(url); + const content = await parseWeb(url); - return { - key: "fetchUrl", - status: "success", - value: content, - display: false, - }; - } catch (e) { - return { - key: "fetchUrl", - status: "error", - value: "Fetching the webpage failed because :" + (e as Error).message, - }; - } + return { + status: ToolResultStatus.Success, + outputs: [{ fetchUrl: content }], + display: false, + }; }, }; diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index 6c371208ee3..01122fe4577 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -1,12 +1,26 @@ +import type { Assistant } from "$lib/types/Assistant"; +import type { Conversation } from "$lib/types/Conversation"; +import type { Message } from "$lib/types/Message"; import type { Tool, ToolResult } from "$lib/types/Tool"; +import type { TextGenerationUpdate } from "../textGeneration/types"; import calculator from "./calculator"; import directlyAnswer from "./directlyAnswer"; import fetchUrl from "./fetchUrl"; import text2img from "./text2img"; import websearch from "./websearch"; +interface BackendToolContext { + conv: Conversation; + messages: Message[]; + preprompt?: string; + assistant?: Pick; +} + export interface BackendTool extends Tool { - call?(params: Record): Promise; + call( + params: Record, + context: BackendToolContext + ): AsyncGenerator, undefined>; } -export const tools: BackendTool[] = [calculator, websearch, text2img, directlyAnswer, fetchUrl]; +export const allTools: BackendTool[] = [calculator, websearch, text2img, directlyAnswer, fetchUrl]; diff --git a/src/lib/server/tools/text2img.ts b/src/lib/server/tools/text2img.ts index 2a590b631a7..ca528c43480 100644 --- a/src/lib/server/tools/text2img.ts +++ b/src/lib/server/tools/text2img.ts @@ -1,4 +1,9 @@ import type { BackendTool } from "."; +import { uploadFile } from "../files/uploadFile"; +import { TextGenerationUpdateType } from "../textGeneration/types"; +import { env } from "$env/dynamic/private"; +import { Client } from "@gradio/client"; +import { ToolResultStatus } from "$lib/types/Tool"; const text2img: BackendTool = { name: "text2img", @@ -13,6 +18,34 @@ const text2img: BackendTool = { required: true, }, }, + async *call({ prompt }, { conv }) { + const app = await Client.connect("ByteDance/Hyper-SDXL-1Step-T2I", { + hf_token: (env.HF_TOKEN ?? env.HF_ACCESS_TOKEN) as unknown as `hf_${string}`, + }); + const res = (await app.predict("/process_image", [ + 1, // number (numeric value between 1 and 8) in 'Number of Images' Slider component + 512, // number in 'Image Height' Number component + 512, // number in 'Image Width' Number component + prompt, // prompt + Math.floor(Math.random() * 1000), // seed random + ])) as unknown as { data: { image: { url: string } }[][] }; + + const response = await fetch(res?.data?.[0]?.[0]?.image?.url ?? "error"); + + const sha = await uploadFile(await response.blob(), conv); + + yield { type: TextGenerationUpdateType.File, sha }; + + return { + status: ToolResultStatus.Success, + outputs: [ + { + text2img: `An image has been generated for the following prompt: "${prompt}". Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it.`, + }, + ], + display: false, + }; + }, }; export default text2img; diff --git a/src/lib/server/tools/websearch.ts b/src/lib/server/tools/websearch.ts index ce15eb2f5f5..a1022a19086 100644 --- a/src/lib/server/tools/websearch.ts +++ b/src/lib/server/tools/websearch.ts @@ -1,4 +1,6 @@ +import { ToolResultStatus } from "$lib/types/Tool"; import type { BackendTool } from "."; +import { runWebSearch } from "../websearch/runWebSearch"; const websearch: BackendTool = { name: "websearch", @@ -14,6 +16,24 @@ const websearch: BackendTool = { "A search query which will be used to fetch the most relevant snippets regarding the user's query", }, }, + async *call({ query }, { conv, assistant, messages }) { + const webSearchToolResults = yield* runWebSearch(conv, messages, { + ragSettings: assistant?.rag, + query, + }); + const chunks = webSearchToolResults?.contextSources + .map(({ context }) => context) + .flat() + .sort((a, b) => a.idx - b.idx) + .map(({ text }) => text) + .join(" "); + + return { + status: ToolResultStatus.Success, + outputs: [{ websearch: chunks }], + display: false, + }; + }, }; export default websearch; diff --git a/src/lib/server/websearch/runWebSearch.ts b/src/lib/server/websearch/runWebSearch.ts index dbd85c6c075..263350c7af2 100644 --- a/src/lib/server/websearch/runWebSearch.ts +++ b/src/lib/server/websearch/runWebSearch.ts @@ -16,6 +16,12 @@ import type { Assistant } from "$lib/types/Assistant"; import { z } from "zod"; import JSON5 from "json5"; import { isURLLocal } from "../isURLLocal"; +import { + TextGenerationUpdateType, + TextGenerationWebSearchUpdateType, + type TextGenerationUpdate, + type TextGenerationWebSearchUpdate, +} from "../textGeneration/types"; const MAX_N_PAGES_SCRAPE = 10 as const; const MAX_N_PAGES_EMBED = 5 as const; @@ -30,12 +36,12 @@ export interface RunWebSearchOptions { query?: string; } -export async function runWebSearch( +// TODO: probably revert back to an update function because of the async +export async function* runWebSearch( conv: Conversation, messages: Message[], - updatePad: (upd: MessageUpdate) => void, options?: RunWebSearchOptions -) { +): AsyncGenerator { const prompt = messages[messages.length - 1].content; const webSearch: WebSearch = { prompt, @@ -46,14 +52,24 @@ export async function runWebSearch( updatedAt: new Date(), }; - function appendUpdate(message: string, args?: string[], type?: "error" | "update") { - updatePad({ type: "webSearch", messageType: type ?? "update", message, args }); + function makeUpdate( + message: string, + args?: string[], + subtype: TextGenerationWebSearchUpdateType = TextGenerationWebSearchUpdateType.Update + ): TextGenerationWebSearchUpdate { + if (subtype === TextGenerationWebSearchUpdateType.Update) { + return { type: TextGenerationUpdateType.WebSearch, subtype, message, args }; + } + if (subtype === TextGenerationWebSearchUpdateType.Error) { + return { type: TextGenerationUpdateType.WebSearch, subtype, message, args }; + } + throw new Error(`Invalid web search update subtype: ${subtype}`); } try { // if the assistant specified direct links, skip the websearch if (options?.ragSettings && options?.ragSettings?.allowedLinks.length > 0) { - appendUpdate("Using links specified in Assistant"); + yield makeUpdate("Using links specified in Assistant"); let linksToUse = [...options.ragSettings.allowedLinks]; @@ -78,11 +94,11 @@ export async function runWebSearch( } else { webSearch.searchQuery = options?.query ?? (await generateQuery(messages)); const searchProvider = getWebSearchProvider(); - appendUpdate(`Searching ${searchProvider}`, [webSearch.searchQuery]); + yield makeUpdate(`Searching ${searchProvider}`, [webSearch.searchQuery]); let filters = ""; if (options?.ragSettings && options?.ragSettings?.allowedDomains.length > 0) { - appendUpdate("Filtering on specified domains"); + yield makeUpdate("Filtering on specified domains"); filters += options?.ragSettings.allowedDomains.map((item) => "site:" + item).join(" OR "); } @@ -125,16 +141,20 @@ export async function runWebSearch( let paragraphChunks: { source: WebSearchSource; text: string }[] = []; if (webSearch.results.length > 0) { - appendUpdate("Browsing results"); + yield makeUpdate("Browsing results"); const promises = webSearch.results.map(async (result) => { const { link } = result; let text = result.text ?? ""; if (!text) { try { text = await parseWeb(link); - appendUpdate("Browsing webpage", [link]); + makeUpdate("Browsing webpage", [link]); } catch (e) { - appendUpdate("Failed to parse webpage", [(e as Error).message, link], "error"); + makeUpdate( + "Failed to parse webpage", + [(e as Error).message, link], + TextGenerationWebSearchUpdateType.Error + ); // ignore errors } } @@ -151,7 +171,7 @@ export async function runWebSearch( throw new Error("No results found for this search query"); } - appendUpdate("Extracting relevant information"); + yield makeUpdate("Extracting relevant information"); const topKClosestParagraphs = 8; const texts = paragraphChunks.map(({ text }) => text); const indices = await findSimilarSentences(embeddingModel, prompt, texts, { @@ -168,15 +188,20 @@ export async function runWebSearch( webSearch.contextSources.push({ ...source, context: [contextWithId] }); } } - updatePad({ - type: "webSearch", - messageType: "sources", - message: "sources", + yield { + type: TextGenerationUpdateType.WebSearch, + subtype: TextGenerationWebSearchUpdateType.Sources, sources: webSearch.contextSources, - }); + message: "Sources", + }; } catch (searchError) { + console.error(searchError); if (searchError instanceof Error) { - appendUpdate("An error occurred", [JSON.stringify(searchError.message)], "error"); + yield makeUpdate( + "An error occurred", + [JSON.stringify(searchError.message)], + TextGenerationWebSearchUpdateType.Error + ); } } diff --git a/src/lib/types/MessageUpdate.ts b/src/lib/types/MessageUpdate.ts index e0718a1d409..44701c61405 100644 --- a/src/lib/types/MessageUpdate.ts +++ b/src/lib/types/MessageUpdate.ts @@ -1,4 +1,4 @@ -import type { Call, Tool } from "./Tool"; +import type { ToolCall, Tool } from "./Tool"; import type { WebSearchSource } from "./WebSearch"; export type FinalAnswer = { @@ -19,7 +19,7 @@ interface ToolUpdateBase { interface ToolUpdateParams extends ToolUpdateBase { messageType: "parameters"; - parameters: Call["parameters"]; + parameters: ToolCall["parameters"]; } interface ToolUpdateMessage extends ToolUpdateBase { diff --git a/src/lib/types/Tool.ts b/src/lib/types/Tool.ts index a181c98f21b..b400125b5ac 100644 --- a/src/lib/types/Tool.ts +++ b/src/lib/types/Tool.ts @@ -20,14 +20,25 @@ export type ToolFront = Pick< "name" | "displayName" | "description" | "isOnByDefault" | "isLocked" >; -export interface ToolResult { - key: string; - status: "success" | "error"; - value: string; +export enum ToolResultStatus { + Success = "success", + Error = "error", +} +interface ToolResultSuccess { + status: ToolResultStatus.Success; + call: ToolCall; + outputs: Record[]; + display?: boolean; +} +interface ToolResultError { + status: ToolResultStatus.Error; + call: ToolCall; + message: string; display?: boolean; } +export type ToolResult = ToolResultSuccess | ToolResultError; -export interface Call { - tool_name: string; +export interface ToolCall { + name: string; parameters: Record; } diff --git a/src/routes/+layout.server.ts b/src/routes/+layout.server.ts index 811badcad81..1a123c19e25 100644 --- a/src/routes/+layout.server.ts +++ b/src/routes/+layout.server.ts @@ -8,7 +8,7 @@ import { DEFAULT_SETTINGS } from "$lib/types/Settings"; import { env } from "$env/dynamic/private"; import { ObjectId } from "mongodb"; import type { ConvSidebar } from "$lib/types/ConvSidebar"; -import { tools } from "$lib/server/tools"; +import { allTools } from "$lib/server/tools"; export const load: LayoutServerLoad = async ({ locals, depends }) => { depends(UrlDependency.ConversationList); @@ -164,7 +164,7 @@ export const load: LayoutServerLoad = async ({ locals, depends }) => { unlisted: model.unlisted, })), oldModels, - tools: tools + tools: allTools .filter((tool) => !tool.isHidden) .map((tool) => ({ name: tool.name, diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 48de1e96e5e..2b81a3b439c 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -9,33 +9,21 @@ import { error } from "@sveltejs/kit"; import { ObjectId } from "mongodb"; import { z } from "zod"; import type { MessageUpdate } from "$lib/types/MessageUpdate"; -import { runWebSearch } from "$lib/server/websearch/runWebSearch"; -import { AbortedGenerations } from "$lib/server/abortedGenerations"; -import { summarize } from "$lib/server/summarize"; import { uploadFile } from "$lib/server/files/uploadFile"; import sizeof from "image-size"; -import type { Assistant } from "$lib/types/Assistant"; import { convertLegacyConversation } from "$lib/utils/tree/convertLegacyConversation"; import { isMessageId } from "$lib/utils/tree/isMessageId"; import { buildSubtree } from "$lib/utils/tree/buildSubtree.js"; import { addChildren } from "$lib/utils/tree/addChildren.js"; import { addSibling } from "$lib/utils/tree/addSibling.js"; -import { preprocessMessages } from "$lib/server/preprocessMessages.js"; import { usageLimits } from "$lib/server/usageLimits"; -import { isURLLocal } from "$lib/server/isURLLocal.js"; -import JSON5 from "json5"; -import type { Call, ToolResult } from "$lib/types/Tool.js"; -import { v4 } from "uuid"; - -import { Client } from "@gradio/client"; - -import directlyAnswer from "$lib/server/tools/directlyAnswer.js"; -import calculator from "$lib/server/tools/calculator.js"; -import websearch from "$lib/server/tools/websearch.js"; -import text2img from "$lib/server/tools/text2img.js"; -import fetchUrl from "$lib/server/tools/fetchUrl.js"; -import { logger } from "$lib/server/logger.js"; -import type { BackendTool } from "$lib/server/tools/index.js"; +import { textGeneration } from "$lib/server/textGeneration"; +import { + TextGenerationToolUpdateType, + TextGenerationUpdateType, + TextGenerationWebSearchUpdateType, + type TextGenerationContext, +} from "$lib/server/textGeneration/types"; export async function POST({ request, locals, params, getClientAddress }) { const id = z.string().parse(params.id); @@ -314,13 +302,8 @@ export async function POST({ request, locals, params, getClientAddress }) { async start(controller) { messageToWriteTo.updates ??= []; function update(newUpdate: MessageUpdate) { - if (newUpdate.type !== "stream") { - messageToWriteTo?.updates?.push(newUpdate); - } - - if (newUpdate.type === "stream" && newUpdate.token === "") { - return; - } + console.log(newUpdate); + if (newUpdate.type !== "stream") messageToWriteTo?.updates?.push(newUpdate); controller.enqueue(JSON.stringify(newUpdate) + "\n"); if (newUpdate.type === "finalAnswer") { @@ -329,339 +312,98 @@ export async function POST({ request, locals, params, getClientAddress }) { } } - update({ type: "status", status: "started" }); + await collections.conversations.updateOne( + { _id: convId }, + { $set: { title: conv.title, updatedAt: new Date() } } + ); + messageToWriteTo.updatedAt = new Date(); - const summarizeIfNeeded = (async () => { - if (conv.title === "New Chat" && conv.messages.length === 3) { - try { - conv.title = (await summarize(conv.messages[1].content)) ?? conv.title; - update({ type: "status", status: "title", message: conv.title }); + let hasError = false; + const initialMessageContent = messageToWriteTo.content; + try { + const ctx: TextGenerationContext = { + model, + endpoint: await model.getEndpoint(), + conv, + messages: messagesForPrompt, + assistant: undefined, + isContinue: isContinue ?? false, + webSearch: webSearch ?? false, + toolsPreference: toolsPreferences ?? {}, + promptedAt, + }; + for await (const event of textGeneration(ctx)) { + if (event.type === TextGenerationUpdateType.Status) { + update({ type: "status", status: event.status, message: event.message }); + } + if (event.type === TextGenerationUpdateType.Title) { + conv.title = event.title; await collections.conversations.updateOne( - { - _id: convId, - }, - { - $set: { - title: conv?.title, - updatedAt: new Date(), - }, - } + { _id: convId }, + { $set: { title: conv?.title, updatedAt: new Date() } } ); - } catch (e) { - logger.error(e); + update({ type: "status", status: "title", message: event.title }); } - } - })(); - - await collections.conversations.updateOne( - { - _id: convId, - }, - { - $set: { - title: conv.title, - updatedAt: new Date(), - }, - } - ); - - // check if assistant has a rag - const assistant = await collections.assistants.findOne< - Pick - >( - { _id: conv.assistantId }, - { projection: { rag: 1, dynamicPrompt: 1, generateSettings: 1 } } - ); - - const assistantHasDynamicPrompt = - env.ENABLE_ASSISTANTS_RAG === "true" && !!assistant && !!assistant?.dynamicPrompt; - - const assistantHasWebSearch = - env.ENABLE_ASSISTANTS_RAG === "true" && - !!assistant && - !!assistant.rag && - (assistant.rag.allowedLinks.length > 0 || - assistant.rag.allowedDomains.length > 0 || - assistant.rag.allowAllDomains); - - // perform websearch if requested - // it can be because the user toggled the webSearch or because the assistant has webSearch enabled - // if the assistant is functions enabled, we don't perform it here - // since we will add the websearch as a tool - - if ( - !isContinue && - !model.functions && - ((webSearch && !conv.assistantId) || assistantHasWebSearch) - ) { - messageToWriteTo.webSearch = await runWebSearch(conv, messagesForPrompt, update, { - ragSettings: assistant?.rag, - }); - } - - let preprompt = conv.preprompt; - - if (assistantHasDynamicPrompt && preprompt) { - // process the preprompt - const urlRegex = /{{\s?url=(.*?)\s?}}/g; - let match; - while ((match = urlRegex.exec(preprompt)) !== null) { - try { - const url = new URL(match[1]); - if ((await isURLLocal(url)) && env.ENABLE_LOCAL_FETCH !== "true") { - throw new Error("URL couldn't be fetched, it resolved to a local address."); - } - - const res = await fetch(url.href); - - if (!res.ok) { - throw new Error("URL couldn't be fetched, error " + res.status); - } - const text = await res.text(); - preprompt = preprompt.replaceAll(match[0], text); - } catch (e) { - preprompt = preprompt.replaceAll(match[0], (e as Error).message); + if (event.type === TextGenerationUpdateType.FinalAnswer) { + messageToWriteTo.interrupted = event.interrupted; + messageToWriteTo.content = initialMessageContent + event.text; + update({ type: "finalAnswer", text: event.text }); } - } - - if (messagesForPrompt[0].from === "system") { - messagesForPrompt[0].content = preprompt; - } - } - - const endpoint = await model.getEndpoint(); - let toolResults: ToolResult[] | undefined = undefined; - - // function calls, if no assistant or assistant has websearch enabled - if (model.functions && (!assistant || assistantHasWebSearch)) { - let tools: BackendTool[] = []; - - // if it's an assistant, only support websearch for now - if (!assistant) { - // filter based on tool preferences, add the tools that are on by default - tools = [directlyAnswer, text2img, calculator, fetchUrl, websearch].filter((el) => { - if (el.isLocked && el.isOnByDefault) { - return true; - } - - return toolsPreferences?.[el.name] ?? el.isOnByDefault; - }); - } else { - // an assistant can currently only use tools for websearch - tools = [directlyAnswer, websearch]; - } - - let calls: Call[] | undefined = undefined; - - // do the function calling bits here - for await (const output of await endpoint({ - messages: messagesForPrompt, - preprompt, - generateSettings: assistant?.generateSettings, - tools, - })) { - if (output.generated_text) { - // look for a code blocks of ```json and parse them - // if they're valid json, add them to the calls array - const codeBlocks = output.generated_text.match(/```json\n(.*?)```/gs); - if (codeBlocks) { - for (const block of codeBlocks) { - try { - calls = JSON5.parse(block.replace("```json\n", "").slice(0, -3)); - } catch (e) { - update({ - type: "error", - message: (e as Error).message, - name: (e as Error).name, - }); - console.error(e); - // error parsing the calls - } - } - } + if (event.type === TextGenerationUpdateType.Stream) { + if (event.token === "") continue; + messageToWriteTo.content += event.token; + update({ type: "stream", token: event.token }); } - } - - const toolPromises = calls?.map(async (call) => { - const uuid = v4(); - - if (call.tool_name === "directly-answer" || call.tool_name === "directly_answer") { - return null; + if (event.type === TextGenerationUpdateType.File) { + messageToWriteTo.files = [...(messageToWriteTo.files ?? []), event.sha]; + update({ type: "file", sha: event.sha }); } - - update({ - type: "tool", - name: call.tool_name, - messageType: "parameters", - parameters: call.parameters, - uuid, - }); - const tool = tools.find((el) => el.name === call.tool_name); - - let toolAnswer: ToolResult | Promise | null = (async () => { - if (!tool) { - return { - key: call.tool_name, - status: "error", - value: "Could not find tool", - }; - } - - if (tool.call) { - return await tool.call(call.parameters); - } - - if (tool.name === "websearch") { - try { - const webSearchToolResults = await runWebSearch(conv, messagesForPrompt, update, { - ragSettings: assistant?.rag, - query: call.parameters?.query, - }); - const chunks = webSearchToolResults?.contextSources - .map(({ context }) => context) - .flat() - .sort((a, b) => a.idx - b.idx) - .map(({ text }) => text) - .join(" "); - - return { - key: call.tool_name, - status: "success", - value: chunks, - display: false, - }; - } catch (e) { - return { - key: call.tool_name, - status: "error", - value: "Error within the websearch. Try again later or with a different query", - }; - } - } else if (tool.name === "text2img") { - const app = await Client.connect("ByteDance/Hyper-SDXL-1Step-T2I", { - hf_token: (env.HF_TOKEN ?? env.HF_ACCESS_TOKEN) as unknown as `hf_${string}`, - }); - const res = (await app.predict("/process_image", [ - 1, // number (numeric value between 1 and 8) in 'Number of Images' Slider component - 512, // number in 'Image Height' Number component - 512, // number in 'Image Width' Number component - call.parameters?.prompt, // prompt - Math.floor(Math.random() * 1000), // seed random - ])) as unknown as { data: Array> }; - - const response = await fetch(res?.data?.[0]?.[0]?.image?.url ?? "error"); - - const sha = await uploadFile(await response.blob(), conv); - - messageToWriteTo.files = [...(messageToWriteTo.files ?? []), sha]; - + if (event.type === TextGenerationUpdateType.Tool) { + if (event.subtype === TextGenerationToolUpdateType.Message) { update({ - type: "file", - sha, + type: "tool", + name: event.name, + messageType: "message", + message: event.message, + display: event.display, + uuid: event.uuid, }); - - return { - key: call.tool_name, - status: "success", - value: `An image has been generated for the following prompt: ${call.parameters?.prompt}. Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it.`, - display: false, - }; } else { - return { - key: call.tool_name, - status: "error", - value: "Not implemented", - }; + update({ + type: "tool", + name: event.name, + messageType: "parameters", + parameters: event.parameters, + uuid: event.uuid, + }); } - })(); - - if (toolAnswer instanceof Promise) { - toolAnswer = await toolAnswer; - } - - if (toolAnswer) { - update({ - type: "tool", - name: toolAnswer.key, - messageType: "message", - message: toolAnswer.value, - display: toolAnswer.display, - uuid, - }); } - return toolAnswer; - }); - - if (toolPromises) { - toolResults = (await Promise.all(toolPromises)).flatMap((el) => (el ? [el] : [])); - } - } - // inject websearch result & optionally images into the messages - const processedMessages = await preprocessMessages( - messagesForPrompt, - messageToWriteTo.webSearch, - model.multimodal, - convId - ); - - const previousText = messageToWriteTo.content; - - let hasError = false; - - let buffer = ""; - - messageToWriteTo.updatedAt = new Date(); - - try { - for await (const output of await endpoint({ - messages: processedMessages, - preprompt, - continueMessage: isContinue, - generateSettings: assistant?.generateSettings, - toolResults, - })) { - // if not generated_text is here it means the generation is not done - if (!output.generated_text) { - if (!output.token.special) { - buffer += output.token.text; - - // send the first 5 chars - // and leave the rest in the buffer - if (buffer.length >= 5) { - update({ - type: "stream", - token: buffer.slice(0, 5), - }); - buffer = buffer.slice(5); - } - - // abort check - const date = AbortedGenerations.getInstance().getList().get(convId.toString()); - if (date && date > promptedAt) { - break; - } - // no output check - if (!output) { - break; - } - - // otherwise we just concatenate tokens - messageToWriteTo.content += output.token.text; + if (event.type === TextGenerationUpdateType.WebSearch) { + if (event.subtype === TextGenerationWebSearchUpdateType.Update) { + update({ + type: "webSearch", + messageType: "update", + message: event.message, + args: event.args, + }); + } else if (event.subtype === TextGenerationWebSearchUpdateType.Error) { + update({ + type: "webSearch", + messageType: "error", + message: event.message, + args: event.args, + }); + } else if (event.subtype === TextGenerationWebSearchUpdateType.Sources) { + update({ + type: "webSearch", + messageType: "sources", + sources: event.sources, + message: event.message, + }); + } else if (event.subtype === TextGenerationWebSearchUpdateType.FinalAnswer) { + // something else too? + messageToWriteTo.webSearch = event.webSearch; } - } else { - messageToWriteTo.interrupted = - !output.token.special && !model.parameters.stop?.includes(output.token.text); - // add output.generated text to the last message - // strip end tokens from the output.generated_text - const text = (model.parameters.stop ?? []).reduce((acc: string, curr: string) => { - if (acc.endsWith(curr)) { - messageToWriteTo.interrupted = false; - return acc.slice(0, acc.length - curr.length); - } - return acc; - }, output.generated_text.trimEnd()); - - messageToWriteTo.content = previousText + text; } } } catch (e) { @@ -670,62 +412,31 @@ export async function POST({ request, locals, params, getClientAddress }) { console.error(e); } finally { // check if no output was generated - if (!hasError && messageToWriteTo.content === previousText) { + if (!hasError && messageToWriteTo.content === initialMessageContent) { update({ type: "status", status: "error", message: "No output was generated. Something went wrong.", }); } - - if (buffer) { - update({ - type: "stream", - token: buffer, - }); - } } await collections.conversations.updateOne( - { - _id: convId, - }, - { - $set: { - messages: conv.messages, - title: conv?.title, - updatedAt: new Date(), - }, - } + { _id: convId }, + { $set: { messages: conv.messages, title: conv?.title, updatedAt: new Date() } } ); // used to detect if cancel() is called bc of interrupt or just because the connection closes doneStreaming = true; - update({ - type: "finalAnswer", - text: messageToWriteTo.content, - }); - - await summarizeIfNeeded; controller.close(); - return; }, async cancel() { - if (!doneStreaming) { - await collections.conversations.updateOne( - { - _id: convId, - }, - { - $set: { - messages: conv.messages, - title: conv.title, - updatedAt: new Date(), - }, - } - ); - } + if (doneStreaming) return; + await collections.conversations.updateOne( + { _id: convId }, + { $set: { messages: conv.messages, title: conv.title, updatedAt: new Date() } } + ); }, }); From 5b1639886806d44fa3e19399bf4830420a0c1bc9 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 15 May 2024 17:35:17 -0400 Subject: [PATCH 40/82] feat: code interpreter tool --- src/lib/server/tools/codeInterpreter.ts | 80 +++++++++++++++++++++++++ src/lib/server/tools/index.ts | 10 +++- 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 src/lib/server/tools/codeInterpreter.ts diff --git a/src/lib/server/tools/codeInterpreter.ts b/src/lib/server/tools/codeInterpreter.ts new file mode 100644 index 00000000000..2fddab461ad --- /dev/null +++ b/src/lib/server/tools/codeInterpreter.ts @@ -0,0 +1,80 @@ +import { ToolResultStatus } from "$lib/types/Tool"; +import type { BackendTool } from "."; +import { CodeInterpreter } from "@e2b/code-interpreter"; +import { uploadFile } from "../files/uploadFile"; +import { TextGenerationUpdateType } from "../textGeneration/types"; + +const codeInterpreter: BackendTool = { + name: "code_interpreter", + displayName: "Code Interpreter", + description: ` +- the python code runs in jupyter notebook. +- every time you call \`code_interpreter\` tool, the python code is executed in a separate cell. it's okay to multiple calls to \`code_interpreter\`. +- display visualizations using matplotlib or any other visualization library directly in the notebook. don't worry about saving the visualizations to a file. +- you have access to the internet and can make api requests. +- you also have access to the filesystem and can read/write files. +- you can install any pip package (if it exists) if you need to but the usual packages for data analysis are already preinstalled. +- you can run any python code you want, everything is running in a secure sandbox environment. + `, + isOnByDefault: true, + parameter_definitions: { + code: { + description: + "The python code to evaluate in a Jupyter Notebook cell with support for output images such as via matplotlib.", + type: "str", + required: true, + }, + }, + async *call(params, { conv }) { + try { + const code = params.code; + + const sandbox = await CodeInterpreter.create(); + const execution = await sandbox.notebook.execCell(code); + await sandbox.close(); + + if (execution.error) { + throw Error(execution.error.value, { cause: execution.error }); + } + + const outputs: Record[] = []; + for (const result of execution.results) { + if (result.jpeg || result.png || result.svg) { + const mime = result.jpeg ? "image/jpeg" : result.png ? "image/png" : "image/svg+xml"; + const base64Data = result.jpeg ?? result.png ?? result.svg ?? ""; + const buffer = Buffer.from(base64Data, "base64"); + const blob = new Blob([buffer], { type: mime }); + const sha = await uploadFile(blob, conv); + yield { type: TextGenerationUpdateType.File, sha }; + outputs.push({ + image: + "An image has been generated and shown to the user. Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it", + }); + } else if (result.markdown) outputs.push({ markdown: result.markdown }); + else if (result.html) outputs.push({ html: result.html }); + else if (result.text) outputs.push({ text: result.text }); + } + if (execution.text) { + outputs.push({ text: execution.text }); + } + if (execution.logs.stdout.length) { + outputs.push({ stdout: execution.logs.stdout.join("\n") }); + } + if (execution.logs.stderr.length) { + outputs.push({ stderr: execution.logs.stderr.join("\n") }); + } + + return { + status: ToolResultStatus.Success, + outputs, + }; + } catch (e) { + return { + status: ToolResultStatus.Error, + message: `Failed to execute code: ${e.message}`, + }; + } + }, +}; + +export default codeInterpreter; diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index 01122fe4577..6a1afbbc9c0 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -4,6 +4,7 @@ import type { Message } from "$lib/types/Message"; import type { Tool, ToolResult } from "$lib/types/Tool"; import type { TextGenerationUpdate } from "../textGeneration/types"; import calculator from "./calculator"; +import codeInterpreter from "./codeInterpreter"; import directlyAnswer from "./directlyAnswer"; import fetchUrl from "./fetchUrl"; import text2img from "./text2img"; @@ -23,4 +24,11 @@ export interface BackendTool extends Tool { ): AsyncGenerator, undefined>; } -export const allTools: BackendTool[] = [calculator, websearch, text2img, directlyAnswer, fetchUrl]; +export const allTools: BackendTool[] = [ + calculator, + codeInterpreter, + directlyAnswer, + fetchUrl, + text2img, + websearch, +]; From bf6f1cb927f64e883a3a67b081f366b4906eb91e Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 15 May 2024 17:54:42 -0400 Subject: [PATCH 41/82] fix: add e2b dependency --- package-lock.json | 79 ++++++++++++++++++++++++++++++++++++++++++++--- package.json | 1 + 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0f8aec88436..ef34c59b5bc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "chat-ui", "version": "0.8.4", "dependencies": { + "@e2b/code-interpreter": "^0.0.5", "@gradio/client": "^0.19.0", "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", @@ -200,6 +201,19 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@e2b/code-interpreter": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@e2b/code-interpreter/-/code-interpreter-0.0.5.tgz", + "integrity": "sha512-ToFQ6N6EU8t91z3EJzh+mG+zf7uK8I1PRLWeu1f3bPS0pAJfM0puzBWOB3XlF9A9R5zZu/g8iO1gGPQXzXY5vA==", + "dependencies": { + "e2b": "^0.16.1", + "isomorphic-ws": "^5.0.0", + "ws": "^8.15.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@emnapi/runtime": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.1.1.tgz", @@ -2580,9 +2594,10 @@ "optional": true }, "node_modules/bufferutil": { - "version": "4.0.7", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.8.tgz", + "integrity": "sha512-4T53u4PdgsXqKaIctwF8ifXlRTTmEPJ8iEPWFdGZvcf7sbwYo6FKFEX9eNNAnzFZ7EzJAQ3CJeOtCRA4rDp7Pw==", "hasInstallScript": true, - "license": "MIT", "dependencies": { "node-gyp-build": "^4.3.0" }, @@ -3283,6 +3298,26 @@ "node": ">=12" } }, + "node_modules/e2b": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/e2b/-/e2b-0.16.1.tgz", + "integrity": "sha512-2L1R/REEB+EezD4Q4MmcXXNATjvCYov2lv/69+PY6V95+wl1PZblIMTYAe7USxX6P6sqANxNs+kXqZr6RvXkSw==", + "dependencies": { + "isomorphic-ws": "^5.0.0", + "normalize-path": "^3.0.0", + "openapi-typescript-fetch": "^1.1.3", + "path-browserify": "^1.0.1", + "platform": "^1.3.6", + "ws": "^8.15.1" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "bufferutil": "^4.0.8", + "utf-8-validate": "^6.0.3" + } + }, "node_modules/ecdsa-sig-formatter": { "version": "1.0.11", "license": "Apache-2.0", @@ -4606,6 +4641,14 @@ "dev": true, "license": "ISC" }, + "node_modules/isomorphic-ws": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-5.0.0.tgz", + "integrity": "sha512-muId7Zzn9ywDsyXgTIafTry2sV3nySZeUDe6YedVd1Hvuuep5AsIlqK+XefWpYTyJG5e503F2xIuT2lcU6rCSw==", + "peerDependencies": { + "ws": "*" + } + }, "node_modules/jiti": { "version": "1.21.0", "license": "MIT", @@ -5566,6 +5609,15 @@ "node": ">= 8" } }, + "node_modules/openapi-typescript-fetch": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/openapi-typescript-fetch/-/openapi-typescript-fetch-1.1.3.tgz", + "integrity": "sha512-smLZPck4OkKMNExcw8jMgrMOGgVGx2N/s6DbKL2ftNl77g5HfoGpZGFy79RBzU/EkaO0OZpwBnslfdBfh7ZcWg==", + "engines": { + "node": ">= 12.0.0", + "npm": ">= 7.0.0" + } + }, "node_modules/openid-client": { "version": "5.4.2", "license": "MIT", @@ -5701,6 +5753,11 @@ "node": ">= 0.8" } }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==" + }, "node_modules/path-exists": { "version": "4.0.0", "dev": true, @@ -7957,6 +8014,19 @@ "requires-port": "^1.0.0" } }, + "node_modules/utf-8-validate": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-6.0.4.tgz", + "integrity": "sha512-xu9GQDeFp+eZ6LnCywXN/zBancWvOpUMzgjLPSjy4BRHSmTelvn2E0DG0o1sTiw5hkCKBHo8rwSKncfRfv2EEQ==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "license": "MIT" @@ -8452,8 +8522,9 @@ "license": "ISC" }, "node_modules/ws": { - "version": "8.13.0", - "license": "MIT", + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz", + "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==", "engines": { "node": ">=10.0.0" }, diff --git a/package.json b/package.json index ea98769d817..81951fa984e 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ }, "type": "module", "dependencies": { + "@e2b/code-interpreter": "^0.0.5", "@gradio/client": "^0.19.0", "@huggingface/hub": "^0.5.1", "@huggingface/inference": "^2.6.3", From 759f6754b0553e55a4b439149107b0096183f96e Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 15 May 2024 19:18:50 -0400 Subject: [PATCH 42/82] feat: working TGI endpoint with tool results --- .../server/endpoints/cohere/endpointCohere.ts | 13 +---- src/lib/server/models.ts | 38 +++++++++++--- src/lib/server/textGeneration/tools.ts | 52 ++++++++++++++++++- src/lib/server/tools/index.ts | 10 ++-- .../{text2img.ts => multimodal/text2image.ts} | 6 +-- .../tools/{websearch.ts => web/search.ts} | 4 +- .../server/tools/{fetchUrl.ts => web/url.ts} | 4 +- 7 files changed, 94 insertions(+), 33 deletions(-) rename src/lib/server/tools/{text2img.ts => multimodal/text2image.ts} (91%) rename src/lib/server/tools/{websearch.ts => web/search.ts} (91%) rename src/lib/server/tools/{fetchUrl.ts => web/url.ts} (87%) diff --git a/src/lib/server/endpoints/cohere/endpointCohere.ts b/src/lib/server/endpoints/cohere/endpointCohere.ts index 1c503a41ea9..6aff9885eee 100644 --- a/src/lib/server/endpoints/cohere/endpointCohere.ts +++ b/src/lib/server/endpoints/cohere/endpointCohere.ts @@ -4,7 +4,7 @@ import type { Endpoint } from "../endpoints"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; import type { Cohere, CohereClient } from "cohere-ai"; import { buildPrompt } from "$lib/buildPrompt"; -import { ToolResultStatus, type Tool, type ToolResult } from "$lib/types/Tool"; +import { ToolResultStatus } from "$lib/types/Tool"; import stream from "stream"; export const endpointCohereParametersSchema = z.object({ @@ -36,9 +36,6 @@ export async function endpointCohere( system = messages[0].content; } - tools = tools?.filter((tool) => tool.name !== "directly_answer")?.map(toCohereTool); - console.log(toolResults); - const parameters = { ...model.parameters, ...generateSettings }; return (async function* () { @@ -148,14 +145,6 @@ export async function endpointCohere( }; } -function toCohereTool(tool: Tool): Cohere.Tool { - return { - name: tool.name, - description: tool.description, - parameterDefinitions: tool.parameter_definitions, - }; -} - async function convertWebStreamToBuffer(webReadableStream) { return new Promise((resolve, reject) => { const chunks = []; diff --git a/src/lib/server/models.ts b/src/lib/server/models.ts index a9f68c78b17..016e6fe490c 100644 --- a/src/lib/server/models.ts +++ b/src/lib/server/models.ts @@ -12,6 +12,7 @@ import type { PreTrainedTokenizer } from "@xenova/transformers"; import JSON5 from "json5"; import { getTokenizer } from "$lib/utils/getTokenizer"; import { logger } from "$lib/server/logger"; +import { ToolResultStatus } from "$lib/types/Tool"; type Optional = Pick, K> & Omit; @@ -112,27 +113,51 @@ async function getChatPromptRender( } if (toolResults && toolResults.length > 0) { + // todo: should update the command r+ tokenizer to support system messages at any location + // or use the `rag` mode without the citations formattedMessages = [ - ...formattedMessages, { role: "system", content: - "" + - toolResults - .map((result, idx) => `\nDocument: ${idx}\n${result.key}\n${result.value}`) - .join("\n") + + "\n\n\n" + + toolResults.flatMap((result, idx) => { + if (result.status === ToolResultStatus.Error) { + return `Document: ${idx}\n` + `Tool "${result.call.name}" error\n` + result.message; + } + return ( + `Document: ${idx}\n` + + result.outputs + .flatMap((output) => + Object.entries(output).map(([title, text]) => `${title}\n${text}`) + ) + .join("\n") + ); + }) + "\n", }, + ...formattedMessages, ]; tools = []; } let chatTemplate: string | undefined = undefined; - if ((tools?.length ?? 0) > 0) { + if (tools && tools.length > 0) { chatTemplate = "tool_use"; } + const documents = (toolResults ?? []).flatMap((result) => { + if (result.status === ToolResultStatus.Error) { + return [{ title: `Tool "${result.call.name}" error`, text: "\n" + result.message }]; + } + return result.outputs.flatMap((output) => + Object.entries(output).map(([title, text]) => ({ + title: `Tool "${result.call.name}" ${title}`, + text: "\n" + text, + })) + ); + }); + const output = tokenizer.apply_chat_template(formattedMessages, { tokenize: false, add_generation_prompt: true, @@ -140,6 +165,7 @@ async function getChatPromptRender( // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore tools: tools ?? [], + documents, }); if (typeof output !== "string") { diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts index 8262bcfa3cc..da05d2b4902 100644 --- a/src/lib/server/textGeneration/tools.ts +++ b/src/lib/server/textGeneration/tools.ts @@ -1,7 +1,9 @@ import { ToolResultStatus, type ToolCall, type ToolResult } from "$lib/types/Tool"; import { v4 as uuidV4 } from "uuid"; +import JSON5 from "json5"; import type { BackendTool } from "../tools"; import { + TextGenerationStatus, TextGenerationToolUpdateType, TextGenerationUpdateType, type TextGenerationContext, @@ -10,7 +12,8 @@ import { import { allTools } from "../tools"; import directlyAnswer from "../tools/directlyAnswer"; -import websearch from "../tools/websearch"; +import websearch from "../tools/web/search"; +import { z } from "zod"; export function pickTools( toolsPreference: Record, @@ -40,7 +43,36 @@ export async function* runTools( generateSettings: assistant?.generateSettings, tools, })) { - calls.push(...(output.token.toolCalls ?? [])); + // model natively supports tool calls + if (output.token.toolCalls) { + calls.push(...output.token.toolCalls); + continue; + } + + // look for a code blocks of ```json and parse them + // if they're valid json, add them to the calls array + if (output.generated_text) { + console.log(output.generated_text); + const codeBlocks = output.generated_text.match(/```json\n(.*?)```/gs); + if (!codeBlocks) continue; + + for (const block of codeBlocks) { + const trimmedBlock = block.replace("```json\n", "").slice(0, -3); + try { + calls.push( + ...JSON5.parse(trimmedBlock).filter(isExternalToolCall).map(externalToToolCall) + ); + } catch (cause) { + // error parsing the calls + yield { + type: TextGenerationUpdateType.Status, + status: TextGenerationStatus.Error, + message: cause instanceof Error ? cause.message : String(cause), + }; + console.error(cause); + } + } + } } const toolResults: ToolResult[] = []; @@ -80,3 +112,19 @@ export async function* runTools( return toolResults; } + +const externalToolCall = z.object({ + tool_name: z.string(), + parameters: z.record(z.string()), +}); +type ExternalToolCall = z.infer; +function isExternalToolCall(call: unknown): call is ExternalToolCall { + return externalToolCall.safeParse(call).success; +} + +function externalToToolCall(call: ExternalToolCall): ToolCall { + return { + name: call.tool_name, + parameters: call.parameters, + }; +} diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index 6a1afbbc9c0..742f1cf0ae4 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -5,10 +5,9 @@ import type { Tool, ToolResult } from "$lib/types/Tool"; import type { TextGenerationUpdate } from "../textGeneration/types"; import calculator from "./calculator"; import codeInterpreter from "./codeInterpreter"; -import directlyAnswer from "./directlyAnswer"; -import fetchUrl from "./fetchUrl"; -import text2img from "./text2img"; -import websearch from "./websearch"; +import fetchUrl from "./web/url"; +import text2image from "./multimodal/text2image"; +import websearch from "./web/search"; interface BackendToolContext { conv: Conversation; @@ -27,8 +26,7 @@ export interface BackendTool extends Tool { export const allTools: BackendTool[] = [ calculator, codeInterpreter, - directlyAnswer, fetchUrl, - text2img, + text2image, websearch, ]; diff --git a/src/lib/server/tools/text2img.ts b/src/lib/server/tools/multimodal/text2image.ts similarity index 91% rename from src/lib/server/tools/text2img.ts rename to src/lib/server/tools/multimodal/text2image.ts index ca528c43480..3d39027c481 100644 --- a/src/lib/server/tools/text2img.ts +++ b/src/lib/server/tools/multimodal/text2image.ts @@ -1,6 +1,6 @@ -import type { BackendTool } from "."; -import { uploadFile } from "../files/uploadFile"; -import { TextGenerationUpdateType } from "../textGeneration/types"; +import type { BackendTool } from ".."; +import { uploadFile } from "../../files/uploadFile"; +import { TextGenerationUpdateType } from "../../textGeneration/types"; import { env } from "$env/dynamic/private"; import { Client } from "@gradio/client"; import { ToolResultStatus } from "$lib/types/Tool"; diff --git a/src/lib/server/tools/websearch.ts b/src/lib/server/tools/web/search.ts similarity index 91% rename from src/lib/server/tools/websearch.ts rename to src/lib/server/tools/web/search.ts index a1022a19086..395112da1ac 100644 --- a/src/lib/server/tools/websearch.ts +++ b/src/lib/server/tools/web/search.ts @@ -1,6 +1,6 @@ import { ToolResultStatus } from "$lib/types/Tool"; -import type { BackendTool } from "."; -import { runWebSearch } from "../websearch/runWebSearch"; +import type { BackendTool } from ".."; +import { runWebSearch } from "../../websearch/runWebSearch"; const websearch: BackendTool = { name: "websearch", diff --git a/src/lib/server/tools/fetchUrl.ts b/src/lib/server/tools/web/url.ts similarity index 87% rename from src/lib/server/tools/fetchUrl.ts rename to src/lib/server/tools/web/url.ts index aea3d3ce790..acefce2b71a 100644 --- a/src/lib/server/tools/fetchUrl.ts +++ b/src/lib/server/tools/web/url.ts @@ -1,6 +1,6 @@ import { ToolResultStatus } from "$lib/types/Tool"; -import type { BackendTool } from "."; -import { parseWeb } from "../websearch/parseWeb"; +import type { BackendTool } from ".."; +import { parseWeb } from "../../websearch/parseWeb"; const fetchUrl: BackendTool = { name: "fetchUrl", From f33c901ff93f8030546adc6c0abc3467aca055db Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 15 May 2024 19:25:10 -0400 Subject: [PATCH 43/82] feat: allow image model to expand to 90dvw --- src/lib/components/chat/ChatMessage.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index be7fc30b2cb..f31b0ca28d8 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -201,7 +201,7 @@ {#if modalImageToShow} - (modalImageToShow = "")}> + (modalImageToShow = "")}> {#if modalImageToShow.length === 64} Date: Wed, 15 May 2024 19:28:48 -0400 Subject: [PATCH 44/82] fix: dont block on title gen --- src/lib/server/textGeneration/index.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/lib/server/textGeneration/index.ts b/src/lib/server/textGeneration/index.ts index d34039ba3ff..23766d37a35 100644 --- a/src/lib/server/textGeneration/index.ts +++ b/src/lib/server/textGeneration/index.ts @@ -32,12 +32,9 @@ export async function* textGeneration( // TODO: should be a better way to detect if the conversation is new // TODO: dont wait for this + let titlePromise: Promise | undefined; if (conv.title === "New Chat" && conv.messages.length === 3) { - const title = (await generateTitle(conv.messages[1].content)) ?? "New Chat"; - yield { - type: TextGenerationUpdateType.Title, - title, - }; + titlePromise = generateTitle(conv.messages[1].content).then((title) => title ?? "New Chat"); } // perform websearch if requested @@ -69,4 +66,11 @@ export async function* textGeneration( convId ); yield* generate({ ...ctx, messages: processedMessages }, toolResults, preprompt); + + if (titlePromise) { + yield { + type: TextGenerationUpdateType.Title, + title: await titlePromise, + }; + } } From 6d4ee002529f247f735e30fafafdc8bfb862132c Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Thu, 16 May 2024 16:09:50 -0400 Subject: [PATCH 45/82] bump sharp --- package-lock.json | 207 +++++++++++++++++++++++----------------------- package.json | 2 +- 2 files changed, 103 insertions(+), 106 deletions(-) diff --git a/package-lock.json b/package-lock.json index 081438039d8..7c84d1be256 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,7 +42,7 @@ "satori-html": "^0.3.2", "sbd": "^1.0.19", "serpapi": "^1.1.1", - "sharp": "^0.33.2", + "sharp": "^0.33.4", "tailwind-scrollbar": "^3.0.0", "tailwindcss": "^3.4.0", "uuid": "^9.0.1", @@ -254,9 +254,9 @@ } }, "node_modules/@emnapi/runtime": { - "version": "0.45.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-0.45.0.tgz", - "integrity": "sha512-Txumi3td7J4A/xTTwlssKieHKTGl3j4A1tglBx72auZ49YK7ePY6XZricgIg9mnZT4xPfA+UPCUdnhRuEFDL+w==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.1.1.tgz", + "integrity": "sha512-3bfqkzuR1KLx57nZfjr2NLnFOobvyS0aTszaEGCGqmYMVDRaGvgIZbjGSV/MHSSmLgQ/b9JFHQ5xm5WRZYd+XQ==", "optional": true, "dependencies": { "tslib": "^2.4.0" @@ -845,9 +845,9 @@ } }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.2.tgz", - "integrity": "sha512-itHBs1rPmsmGF9p4qRe++CzCgd+kFYktnsoR1sbIAfsRMrJZau0Tt1AH9KVnufc2/tU02Gf6Ibujx+15qRE03w==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.4.tgz", + "integrity": "sha512-p0suNqXufJs9t3RqLBO6vvrgr5OhgbWp76s5gTRvdmxmuv9E1rcaqGUsl3l4mKVmXPkTkTErXediAui4x+8PSA==", "cpu": [ "arm64" ], @@ -866,13 +866,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.0.1" + "@img/sharp-libvips-darwin-arm64": "1.0.2" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.2.tgz", - "integrity": "sha512-/rK/69Rrp9x5kaWBjVN07KixZanRr+W1OiyKdXcbjQD6KbW+obaTeBBtLUAtbBsnlTTmWthw99xqoOS7SsySDg==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.4.tgz", + "integrity": "sha512-0l7yRObwtTi82Z6ebVI2PnHT8EB2NxBgpK2MiKJZJ7cz32R4lxd001ecMhzzsZig3Yv9oclvqqdV93jo9hy+Dw==", "cpu": [ "x64" ], @@ -891,13 +891,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.0.1" + "@img/sharp-libvips-darwin-x64": "1.0.2" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.1.tgz", - "integrity": "sha512-kQyrSNd6lmBV7O0BUiyu/OEw9yeNGFbQhbxswS1i6rMDwBBSX+e+rPzu3S+MwAiGU3HdLze3PanQ4Xkfemgzcw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.2.tgz", + "integrity": "sha512-tcK/41Rq8IKlSaKRCCAuuY3lDJjQnYIW1UXU1kxcEKrfL8WR7N6+rzNoOxoQRJWTAECuKwgAHnPvqXGN8XfkHA==", "cpu": [ "arm64" ], @@ -916,9 +916,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.1.tgz", - "integrity": "sha512-eVU/JYLPVjhhrd8Tk6gosl5pVlvsqiFlt50wotCvdkFGf+mDNBJxMh+bvav+Wt3EBnNZWq8Sp2I7XfSjm8siog==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.2.tgz", + "integrity": "sha512-Ofw+7oaWa0HiiMiKWqqaZbaYV3/UGL2wAPeLuJTx+9cXpCRdvQhCLG0IH8YGwM0yGWGLpsF4Su9vM1o6aer+Fw==", "cpu": [ "x64" ], @@ -937,9 +937,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.1.tgz", - "integrity": "sha512-FtdMvR4R99FTsD53IA3LxYGghQ82t3yt0ZQ93WMZ2xV3dqrb0E8zq4VHaTOuLEAuA83oDawHV3fd+BsAPadHIQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.2.tgz", + "integrity": "sha512-iLWCvrKgeFoglQxdEwzu1eQV04o8YeYGFXtfWU26Zr2wWT3q3MTzC+QTCO3ZQfWd3doKHT4Pm2kRmLbupT+sZw==", "cpu": [ "arm" ], @@ -958,9 +958,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.1.tgz", - "integrity": "sha512-bnGG+MJjdX70mAQcSLxgeJco11G+MxTz+ebxlz8Y3dxyeb3Nkl7LgLI0mXupoO+u1wRNx/iRj5yHtzA4sde1yA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.2.tgz", + "integrity": "sha512-x7kCt3N00ofFmmkkdshwj3vGPCnmiDh7Gwnd4nUwZln2YjqPxV1NlTyZOvoDWdKQVDL911487HOueBvrpflagw==", "cpu": [ "arm64" ], @@ -979,9 +979,9 @@ } }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.1.tgz", - "integrity": "sha512-3+rzfAR1YpMOeA2zZNp+aYEzGNWK4zF3+sdMxuCS3ey9HhDbJ66w6hDSHDMoap32DueFwhhs3vwooAB2MaK4XQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.2.tgz", + "integrity": "sha512-cmhQ1J4qVhfmS6szYW7RT+gLJq9dH2i4maq+qyXayUSn9/3iY2ZeWpbAgSpSVbV2E1JUL2Gg7pwnYQ1h8rQIog==", "cpu": [ "s390x" ], @@ -1000,9 +1000,9 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.1.tgz", - "integrity": "sha512-3NR1mxFsaSgMMzz1bAnnKbSAI+lHXVTqAHgc1bgzjHuXjo4hlscpUxc0vFSAPKI3yuzdzcZOkq7nDPrP2F8Jgw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.2.tgz", + "integrity": "sha512-E441q4Qdb+7yuyiADVi5J+44x8ctlrqn8XgkDTwr4qPJzWkaHwD489iZ4nGDgcuya4iMN3ULV6NwbhRZJ9Z7SQ==", "cpu": [ "x64" ], @@ -1021,9 +1021,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.1.tgz", - "integrity": "sha512-5aBRcjHDG/T6jwC3Edl3lP8nl9U2Yo8+oTl5drd1dh9Z1EBfzUKAJFUDTDisDjUwc7N4AjnPGfCA3jl3hY8uDg==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.2.tgz", + "integrity": "sha512-3CAkndNpYUrlDqkCM5qhksfE+qSIREVpyoeHIU6jd48SJZViAmznoQQLAv4hVXF7xyUB9zf+G++e2v1ABjCbEQ==", "cpu": [ "arm64" ], @@ -1042,9 +1042,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.1.tgz", - "integrity": "sha512-dcT7inI9DBFK6ovfeWRe3hG30h51cBAP5JXlZfx6pzc/Mnf9HFCQDLtYf4MCBjxaaTfjCCjkBxcy3XzOAo5txw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.2.tgz", + "integrity": "sha512-VI94Q6khIHqHWNOh6LLdm9s2Ry4zdjWJwH56WoiJU7NTeDwyApdZZ8c+SADC8OH98KWNQXnE01UdJ9CSfZvwZw==", "cpu": [ "x64" ], @@ -1063,9 +1063,9 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.2.tgz", - "integrity": "sha512-Fndk/4Zq3vAc4G/qyfXASbS3HBZbKrlnKZLEJzPLrXoJuipFNNwTes71+Ki1hwYW5lch26niRYoZFAtZVf3EGA==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.4.tgz", + "integrity": "sha512-RUgBD1c0+gCYZGCCe6mMdTiOFS0Zc/XrN0fYd6hISIKcDUbAW5NtSQW9g/powkrXYm6Vzwd6y+fqmExDuCdHNQ==", "cpu": [ "arm" ], @@ -1084,13 +1084,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.0.1" + "@img/sharp-libvips-linux-arm": "1.0.2" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.2.tgz", - "integrity": "sha512-pz0NNo882vVfqJ0yNInuG9YH71smP4gRSdeL09ukC2YLE6ZyZePAlWKEHgAzJGTiOh8Qkaov6mMIMlEhmLdKew==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.4.tgz", + "integrity": "sha512-2800clwVg1ZQtxwSoTlHvtm9ObgAax7V6MTAB/hDT945Tfyy3hVkmiHpeLPCKYqYR1Gcmv1uDZ3a4OFwkdBL7Q==", "cpu": [ "arm64" ], @@ -1109,13 +1109,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.0.1" + "@img/sharp-libvips-linux-arm64": "1.0.2" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.2.tgz", - "integrity": "sha512-MBoInDXDppMfhSzbMmOQtGfloVAflS2rP1qPcUIiITMi36Mm5YR7r0ASND99razjQUpHTzjrU1flO76hKvP5RA==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.4.tgz", + "integrity": "sha512-h3RAL3siQoyzSoH36tUeS0PDmb5wINKGYzcLB5C6DIiAn2F3udeFAum+gj8IbA/82+8RGCTn7XW8WTFnqag4tQ==", "cpu": [ "s390x" ], @@ -1124,7 +1124,7 @@ "linux" ], "engines": { - "glibc": ">=2.28", + "glibc": ">=2.31", "node": "^18.17.0 || ^20.3.0 || >=21.0.0", "npm": ">=9.6.5", "pnpm": ">=7.1.0", @@ -1134,13 +1134,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.0.1" + "@img/sharp-libvips-linux-s390x": "1.0.2" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.2.tgz", - "integrity": "sha512-xUT82H5IbXewKkeF5aiooajoO1tQV4PnKfS/OZtb5DDdxS/FCI/uXTVZ35GQ97RZXsycojz/AJ0asoz6p2/H/A==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.4.tgz", + "integrity": "sha512-GoR++s0XW9DGVi8SUGQ/U4AeIzLdNjHka6jidVwapQ/JebGVQIpi52OdyxCNVRE++n1FCLzjDovJNozif7w/Aw==", "cpu": [ "x64" ], @@ -1159,13 +1159,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.0.1" + "@img/sharp-libvips-linux-x64": "1.0.2" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.2.tgz", - "integrity": "sha512-F+0z8JCu/UnMzg8IYW1TMeiViIWBVg7IWP6nE0p5S5EPQxlLd76c8jYemG21X99UzFwgkRo5yz2DS+zbrnxZeA==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.4.tgz", + "integrity": "sha512-nhr1yC3BlVrKDTl6cO12gTpXMl4ITBUZieehFvMntlCXFzH2bvKG76tBL2Y/OqhupZt81pR7R+Q5YhJxW0rGgQ==", "cpu": [ "arm64" ], @@ -1184,13 +1184,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.0.1" + "@img/sharp-libvips-linuxmusl-arm64": "1.0.2" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.2.tgz", - "integrity": "sha512-+ZLE3SQmSL+Fn1gmSaM8uFusW5Y3J9VOf+wMGNnTtJUMUxFhv+P4UPaYEYT8tqnyYVaOVGgMN/zsOxn9pSsO2A==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.4.tgz", + "integrity": "sha512-uCPTku0zwqDmZEOi4ILyGdmW76tH7dm8kKlOIV1XC5cLyJ71ENAAqarOHQh0RLfpIpbV5KOpXzdU6XkJtS0daw==", "cpu": [ "x64" ], @@ -1209,19 +1209,19 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.0.1" + "@img/sharp-libvips-linuxmusl-x64": "1.0.2" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.2.tgz", - "integrity": "sha512-fLbTaESVKuQcpm8ffgBD7jLb/CQLcATju/jxtTXR1XCLwbOQt+OL5zPHSDMmp2JZIeq82e18yE0Vv7zh6+6BfQ==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.4.tgz", + "integrity": "sha512-Bmmauh4sXUsUqkleQahpdNXKvo+wa1V9KhT2pDA4VJGKwnKMJXiSTGphn0gnJrlooda0QxCtXc6RX1XAU6hMnQ==", "cpu": [ "wasm32" ], "optional": true, "dependencies": { - "@emnapi/runtime": "^0.45.0" + "@emnapi/runtime": "^1.1.1" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0", @@ -1234,9 +1234,9 @@ } }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.2.tgz", - "integrity": "sha512-okBpql96hIGuZ4lN3+nsAjGeggxKm7hIRu9zyec0lnfB8E7Z6p95BuRZzDDXZOl2e8UmR4RhYt631i7mfmKU8g==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.4.tgz", + "integrity": "sha512-99SJ91XzUhYHbx7uhK3+9Lf7+LjwMGQZMDlO/E/YVJ7Nc3lyDFZPGhjwiYdctoH2BOzW9+TnfqcaMKt0jHLdqw==", "cpu": [ "ia32" ], @@ -1255,9 +1255,9 @@ } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.2.tgz", - "integrity": "sha512-E4magOks77DK47FwHUIGH0RYWSgRBfGdK56kIHSVeB9uIS4pPFr4N2kIVsXdQQo4LzOsENKV5KAhRlRL7eMAdg==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.4.tgz", + "integrity": "sha512-3QLocdTRVIrFNye5YocZl+KKpYKP+fksi1QhmOArgx7GyhIbQp/WrJRu176jm8IxromS7RIkzMiMINVdBtC8Aw==", "cpu": [ "x64" ], @@ -4068,9 +4068,9 @@ } }, "node_modules/detect-libc": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", - "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", "engines": { "node": ">=8" } @@ -8115,12 +8115,9 @@ } }, "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", + "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", "bin": { "semver": "bin/semver.js" }, @@ -8219,42 +8216,42 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, "node_modules/sharp": { - "version": "0.33.2", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.2.tgz", - "integrity": "sha512-WlYOPyyPDiiM07j/UO+E720ju6gtNtHjEGg5vovUk1Lgxyjm2LFO+37Nt/UI3MMh2l6hxTWQWi7qk3cXJTutcQ==", + "version": "0.33.4", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.4.tgz", + "integrity": "sha512-7i/dt5kGl7qR4gwPRD2biwD2/SvBn3O04J77XKFgL2OnZtQw+AG9wnuS/csmu80nPRHLYE9E41fyEiG8nhH6/Q==", "hasInstallScript": true, "dependencies": { "color": "^4.2.3", - "detect-libc": "^2.0.2", - "semver": "^7.5.4" + "detect-libc": "^2.0.3", + "semver": "^7.6.0" }, "engines": { - "libvips": ">=8.15.1", + "libvips": ">=8.15.2", "node": "^18.17.0 || ^20.3.0 || >=21.0.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.33.2", - "@img/sharp-darwin-x64": "0.33.2", - "@img/sharp-libvips-darwin-arm64": "1.0.1", - "@img/sharp-libvips-darwin-x64": "1.0.1", - "@img/sharp-libvips-linux-arm": "1.0.1", - "@img/sharp-libvips-linux-arm64": "1.0.1", - "@img/sharp-libvips-linux-s390x": "1.0.1", - "@img/sharp-libvips-linux-x64": "1.0.1", - "@img/sharp-libvips-linuxmusl-arm64": "1.0.1", - "@img/sharp-libvips-linuxmusl-x64": "1.0.1", - "@img/sharp-linux-arm": "0.33.2", - "@img/sharp-linux-arm64": "0.33.2", - "@img/sharp-linux-s390x": "0.33.2", - "@img/sharp-linux-x64": "0.33.2", - "@img/sharp-linuxmusl-arm64": "0.33.2", - "@img/sharp-linuxmusl-x64": "0.33.2", - "@img/sharp-wasm32": "0.33.2", - "@img/sharp-win32-ia32": "0.33.2", - "@img/sharp-win32-x64": "0.33.2" + "@img/sharp-darwin-arm64": "0.33.4", + "@img/sharp-darwin-x64": "0.33.4", + "@img/sharp-libvips-darwin-arm64": "1.0.2", + "@img/sharp-libvips-darwin-x64": "1.0.2", + "@img/sharp-libvips-linux-arm": "1.0.2", + "@img/sharp-libvips-linux-arm64": "1.0.2", + "@img/sharp-libvips-linux-s390x": "1.0.2", + "@img/sharp-libvips-linux-x64": "1.0.2", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.2", + "@img/sharp-libvips-linuxmusl-x64": "1.0.2", + "@img/sharp-linux-arm": "0.33.4", + "@img/sharp-linux-arm64": "0.33.4", + "@img/sharp-linux-s390x": "0.33.4", + "@img/sharp-linux-x64": "0.33.4", + "@img/sharp-linuxmusl-arm64": "0.33.4", + "@img/sharp-linuxmusl-x64": "0.33.4", + "@img/sharp-wasm32": "0.33.4", + "@img/sharp-win32-ia32": "0.33.4", + "@img/sharp-win32-x64": "0.33.4" } }, "node_modules/shebang-command": { diff --git a/package.json b/package.json index e5fa11969ad..9a3d8e59ba9 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,7 @@ "satori-html": "^0.3.2", "sbd": "^1.0.19", "serpapi": "^1.1.1", - "sharp": "^0.33.2", + "sharp": "^0.33.4", "tailwind-scrollbar": "^3.0.0", "tailwindcss": "^3.4.0", "uuid": "^9.0.1", From e5eeb5b8a13d5566f42e680b1398b24df6fe4b4e Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Thu, 16 May 2024 17:06:16 -0400 Subject: [PATCH 46/82] misc cleanup --- src/lib/server/textGeneration/tools.ts | 13 +++++-------- src/lib/server/tools/calculator.ts | 2 +- src/lib/server/tools/directlyAnswer.ts | 3 +-- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts index da05d2b4902..57e2c2e8b25 100644 --- a/src/lib/server/textGeneration/tools.ts +++ b/src/lib/server/textGeneration/tools.ts @@ -52,16 +52,13 @@ export async function* runTools( // look for a code blocks of ```json and parse them // if they're valid json, add them to the calls array if (output.generated_text) { - console.log(output.generated_text); - const codeBlocks = output.generated_text.match(/```json\n(.*?)```/gs); - if (!codeBlocks) continue; + const codeBlocks = Array.from(output.generated_text.matchAll(/```json\n(.*?)```/gs)); + if (codeBlocks.length === 0) continue; - for (const block of codeBlocks) { - const trimmedBlock = block.replace("```json\n", "").slice(0, -3); + // grab only the capture group from the regex match + for (const [, block] of codeBlocks) { try { - calls.push( - ...JSON5.parse(trimmedBlock).filter(isExternalToolCall).map(externalToToolCall) - ); + calls.push(...JSON5.parse(block).filter(isExternalToolCall).map(externalToToolCall)); } catch (cause) { // error parsing the calls yield { diff --git a/src/lib/server/tools/calculator.ts b/src/lib/server/tools/calculator.ts index 248fe679e33..a36d7dca0a5 100644 --- a/src/lib/server/tools/calculator.ts +++ b/src/lib/server/tools/calculator.ts @@ -22,7 +22,7 @@ const calculator: BackendTool = { return { status: ToolResultStatus.Success, - outputs: [{ calculator: eval(query) }], + outputs: [{ calculator: Function(`return ${query}`)() }], }; } catch (e) { return { diff --git a/src/lib/server/tools/directlyAnswer.ts b/src/lib/server/tools/directlyAnswer.ts index 197508c38f8..4b911a8ec5b 100644 --- a/src/lib/server/tools/directlyAnswer.ts +++ b/src/lib/server/tools/directlyAnswer.ts @@ -6,8 +6,7 @@ const directlyAnswer: BackendTool = { isOnByDefault: true, isHidden: true, isLocked: true, - description: - "Use this tool to let the user know you wish to answer directly. Do not try to provide any parameters when using this tool.", + description: "Use this tool to let the user know you wish to answer directly", parameter_definitions: {}, async *call() { return { From d89d5afab72d7cd4495eff6f65ba841c2972bb11 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Thu, 16 May 2024 20:26:22 -0400 Subject: [PATCH 47/82] fix url fetcher, adjust tool typing --- src/lib/server/textGeneration/tools.ts | 24 ++++++++++------ src/lib/server/textGeneration/types.ts | 26 ++++++++--------- src/lib/server/tools/index.ts | 9 +++++- src/lib/server/tools/web/url.ts | 4 +-- .../websearch/markdown/utils/stringify.ts | 7 +++++ src/routes/conversation/[id]/+server.ts | 28 ++++--------------- .../message/[messageId]/prompt/+server.ts | 5 ++++ 7 files changed, 55 insertions(+), 48 deletions(-) diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts index 57e2c2e8b25..4f937ee815f 100644 --- a/src/lib/server/textGeneration/tools.ts +++ b/src/lib/server/textGeneration/tools.ts @@ -76,19 +76,21 @@ export async function* runTools( for (const call of calls) { const uuid = uuidV4(); - yield { - type: TextGenerationUpdateType.Tool, - subtype: TextGenerationToolUpdateType.Parameters, - name: call.name, - parameters: call.parameters, - uuid, - }; - const tool = tools.find((el) => el.name === call.name); + if (!tool) { toolResults.push({ call, status: ToolResultStatus.Error, message: "Could not find tool" }); continue; } + // Special case for directly_answer tool where we ignore + if (tool.name === "directly_answer") continue; + + yield { + type: TextGenerationUpdateType.Tool, + subtype: TextGenerationToolUpdateType.Call, + uuid, + toolCall: call, + }; try { const toolResult = yield* tool.call(call.parameters, { conv, @@ -96,6 +98,12 @@ export async function* runTools( preprompt, assistant, }); + yield { + type: TextGenerationUpdateType.Tool, + subtype: TextGenerationToolUpdateType.Result, + uuid, + toolResult: { ...toolResult, call } as ToolResult, + }; toolResults.push({ ...toolResult, call } as ToolResult); } catch (cause) { console.error(Error(`Failed while running tool ${call.name}`), { cause }); diff --git a/src/lib/server/textGeneration/types.ts b/src/lib/server/textGeneration/types.ts index cbc487c8de1..bbdfd35b53e 100644 --- a/src/lib/server/textGeneration/types.ts +++ b/src/lib/server/textGeneration/types.ts @@ -1,5 +1,5 @@ import type { WebSearch, WebSearchSource } from "$lib/types/WebSearch"; -import type { Tool } from "$lib/types/Tool"; +import type { Tool, ToolCall, ToolResult } from "$lib/types/Tool"; import type { Conversation } from "$lib/types/Conversation"; import type { Message } from "$lib/types/Message"; import type { Assistant } from "$lib/types/Assistant"; @@ -90,27 +90,25 @@ export type TextGenerationWebSearchUpdate = // Tool export enum TextGenerationToolUpdateType { - Parameters = "parameters", - Message = "message", + /** A request to call a tool alongside it's parameters */ + Call = "call", + /** The result of a tool call */ + Result = "result", } interface TextGenerationToolBaseUpdate { type: TextGenerationUpdateType.Tool; subtype: TSubType; - name: Tool["name"]; uuid: string; } -interface TextGenerationToolParametersUpdate - extends TextGenerationToolBaseUpdate { - parameters: Record; +interface TextGenerationToolCallUpdate + extends TextGenerationToolBaseUpdate { + toolCall: ToolCall; } -interface TextGenerationToolMessageUpdate - extends TextGenerationToolBaseUpdate { - message?: string; - display?: boolean; +interface TextGenerationToolCallResultUpdate + extends TextGenerationToolBaseUpdate { + toolResult: ToolResult; } -type TextGenerationToolUpdate = - | TextGenerationToolParametersUpdate - | TextGenerationToolMessageUpdate; +type TextGenerationToolUpdate = TextGenerationToolCallUpdate | TextGenerationToolCallResultUpdate; // Everything else interface TextGenerationTitleUpdate { diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index 73066385966..5a4f5173adf 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -4,6 +4,7 @@ import type { Message } from "$lib/types/Message"; import type { Tool, ToolResult } from "$lib/types/Tool"; import type { TextGenerationUpdate } from "../textGeneration/types"; import calculator from "./calculator"; +import directlyAnswer from "./directlyAnswer"; // import codeInterpreter from "./codeInterpreter"; import fetchUrl from "./web/url"; import text2image from "./multimodal/text2image"; @@ -23,4 +24,10 @@ export interface BackendTool extends Tool { ): AsyncGenerator, undefined>; } -export const allTools: BackendTool[] = [calculator, fetchUrl, text2image, websearch]; +export const allTools: BackendTool[] = [ + directlyAnswer, + calculator, + fetchUrl, + text2image, + websearch, +]; diff --git a/src/lib/server/tools/web/url.ts b/src/lib/server/tools/web/url.ts index e505eead431..9cb8a25ce9b 100644 --- a/src/lib/server/tools/web/url.ts +++ b/src/lib/server/tools/web/url.ts @@ -1,4 +1,4 @@ -import { stringifyMarkdownElement } from "$lib/server/websearch/markdown/utils/stringify"; +import { stringifyMarkdownElementTree } from "$lib/server/websearch/markdown/utils/stringify"; import { scrapeUrl } from "$lib/server/websearch/scrape/scrape"; import { ToolResultStatus } from "$lib/types/Tool"; import type { BackendTool } from ".."; @@ -23,7 +23,7 @@ const fetchUrl: BackendTool = { return { status: ToolResultStatus.Success, - outputs: [{ title, text: stringifyMarkdownElement(markdownTree) }], + outputs: [{ title, text: stringifyMarkdownElementTree(markdownTree) }], display: false, }; }, diff --git a/src/lib/server/websearch/markdown/utils/stringify.ts b/src/lib/server/websearch/markdown/utils/stringify.ts index ab9e0ad90cf..0e6941f7182 100644 --- a/src/lib/server/websearch/markdown/utils/stringify.ts +++ b/src/lib/server/websearch/markdown/utils/stringify.ts @@ -26,6 +26,13 @@ export function stringifyMarkdownElement(elem: MarkdownElement): string { return `${content}\n\n`; } +/** Converts a tree of markdown elements to a string with formatting */ +export function stringifyMarkdownElementTree(elem: MarkdownElement): string { + const stringified = stringifyMarkdownElement(elem); + if (!("children" in elem)) return stringified; + return stringified + elem.children.map(stringifyMarkdownElementTree).join(""); +} + // ----- HTML Elements ----- /** Ignores all non-inline tag types and grabs their text. Converts inline tags to markdown */ diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 2b81a3b439c..14d5cb94d59 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -283,16 +283,8 @@ export async function POST({ request, locals, params, getClientAddress }) { // update the conversation with the new messages await collections.conversations.updateOne( - { - _id: convId, - }, - { - $set: { - messages: conv.messages, - title: conv.title, - updatedAt: new Date(), - }, - } + { _id: convId }, + { $set: { messages: conv.messages, title: conv.title, updatedAt: new Date() } } ); let doneStreaming = false; @@ -302,7 +294,6 @@ export async function POST({ request, locals, params, getClientAddress }) { async start(controller) { messageToWriteTo.updates ??= []; function update(newUpdate: MessageUpdate) { - console.log(newUpdate); if (newUpdate.type !== "stream") messageToWriteTo?.updates?.push(newUpdate); controller.enqueue(JSON.stringify(newUpdate) + "\n"); @@ -359,21 +350,12 @@ export async function POST({ request, locals, params, getClientAddress }) { update({ type: "file", sha: event.sha }); } if (event.type === TextGenerationUpdateType.Tool) { - if (event.subtype === TextGenerationToolUpdateType.Message) { - update({ - type: "tool", - name: event.name, - messageType: "message", - message: event.message, - display: event.display, - uuid: event.uuid, - }); - } else { + if (event.subtype === TextGenerationToolUpdateType.Call) { update({ type: "tool", - name: event.name, + name: event.toolCall.name, messageType: "parameters", - parameters: event.parameters, + parameters: event.toolCall.parameters, uuid: event.uuid, }); } diff --git a/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts b/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts index 273df6c97b8..84cd152884f 100644 --- a/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts +++ b/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts @@ -44,6 +44,9 @@ export async function GET({ params, locals }) { model, }); + const userMessage = conv.messages[messageIndex]; + const assistantMessage = conv.messages[messageIndex + 1]; + return new Response( JSON.stringify( { @@ -54,6 +57,8 @@ export async function GET({ params, locals }) { ...model.parameters, return_full_text: false, }, + userMessage, + ...(assistantMessage ? { assistantMessage } : {}), }, null, 2 From 4462ba480c2afb4cb11d186c0eb186c72af6d634 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Thu, 16 May 2024 22:16:25 -0400 Subject: [PATCH 48/82] migrate to new MessageUpdate schema --- .../components/OpenWebSearchResults.svelte | 22 ++- src/lib/components/chat/ChatMessage.svelte | 45 ++--- .../routines/04-update-message-updates.ts | 187 ++++++++++++++++++ src/lib/migrations/routines/index.ts | 2 + src/lib/server/textGeneration/generate.ts | 13 +- src/lib/server/textGeneration/index.ts | 16 +- src/lib/server/textGeneration/title.ts | 6 +- src/lib/server/textGeneration/tools.ts | 30 +-- src/lib/server/textGeneration/types.ts | 117 +---------- src/lib/server/tools/index.ts | 6 +- src/lib/server/tools/multimodal/text2image.ts | 4 +- src/lib/server/websearch/runWebSearch.ts | 4 +- src/lib/server/websearch/scrape/scrape.ts | 4 +- src/lib/server/websearch/search/search.ts | 4 +- src/lib/server/websearch/update.ts | 46 ++--- src/lib/types/MessageUpdate.ts | 148 +++++++++----- src/lib/utils/messageUpdates.ts | 40 +++- src/routes/conversation/[id]/+page.svelte | 48 ++--- src/routes/conversation/[id]/+server.ts | 146 ++++++-------- .../message/[messageId]/prompt/+server.ts | 2 +- 20 files changed, 515 insertions(+), 375 deletions(-) create mode 100644 src/lib/migrations/routines/04-update-message-updates.ts diff --git a/src/lib/components/OpenWebSearchResults.svelte b/src/lib/components/OpenWebSearchResults.svelte index bd219ec1046..ef0295a7e1a 100644 --- a/src/lib/components/OpenWebSearchResults.svelte +++ b/src/lib/components/OpenWebSearchResults.svelte @@ -1,16 +1,22 @@
    @@ -61,7 +67,7 @@ {:else}
      {#each webSearchMessages as message} - {#if message.messageType === "update"} + {#if message.subtype === MessageWebSearchUpdateType.Update}
    1. {/if}
      2. - {:else if message.messageType === "error"} + {:else if message.subtype === MessageWebSearchUpdateType.Error}
    2. type === "webSearch") ?? - []) as WebSearchUpdate[]; + []) as MessageWebSearchUpdate[]; // filter all updates with type === "tool" then group them by uuid field @@ -146,18 +155,20 @@ acc[update.uuid] = acc[update.uuid] ?? []; acc[update.uuid].push(update); return acc; - }, {} as Record); + }, {} as Record); $: downloadLink = urlNotTrailing + `/message/${message.id}/prompt`; let webSearchIsDone = true; - $: webSearchIsDone = - searchUpdates.length > 0 && searchUpdates[searchUpdates.length - 1].messageType === "sources"; + $: webSearchIsDone = searchUpdates.some( + (update) => update.subtype === MessageWebSearchUpdateType.Finished + ); - $: webSearchSources = - searchUpdates && - searchUpdates?.filter(({ messageType }) => messageType === "sources")?.[0]?.sources; + $: webSearchSources = searchUpdates?.find( + (update): update is MessageWebSearchSourcesUpdate => + update.subtype === MessageWebSearchUpdateType.Sources + )?.sources; $: if (isCopied) { setTimeout(() => { @@ -275,8 +286,8 @@ {#if tools} {#each Object.values(tools) as tool} {#if tool.length > 0} - {@const toolName = tool.filter((t) => t.messageType === "parameters")[0].name} - {@const toolDone = !!tool.find((t) => t.messageType === "message")} + {@const toolName = tool.find(isMessageToolCallUpdate)?.call.name} + {@const toolDone = tool.some(isMessageToolResultUpdate)} {#if toolName && toolName !== "websearch"}

      Parameters

        - {#each Object.entries(toolUpdate.parameters ?? {}) as [k, v]} + {#each Object.entries(toolUpdate.call.parameters ?? {}) as [k, v]}
      • {k}: {v}
        • {/each}
      - {:else if toolUpdate.messageType === "message" && toolUpdate.display !== false} -
      -

      Result

      -
      -
      - -

      - > - {toolUpdate.message} -

      {/if} {/each}
      diff --git a/src/lib/migrations/routines/04-update-message-updates.ts b/src/lib/migrations/routines/04-update-message-updates.ts new file mode 100644 index 00000000000..8ddd08e4b6d --- /dev/null +++ b/src/lib/migrations/routines/04-update-message-updates.ts @@ -0,0 +1,187 @@ +import type { Migration } from "."; +import { collections } from "$lib/server/database"; +import { ObjectId, type WithId } from "mongodb"; +import type { Conversation } from "$lib/types/Conversation"; +import type { WebSearchSource } from "$lib/types/WebSearch"; +import { + MessageUpdateStatus, + MessageUpdateType, + MessageWebSearchUpdateType, + type MessageUpdate, + type MessageWebSearchFinishedUpdate, +} from "$lib/types/MessageUpdate"; +import type { Message } from "$lib/types/Message"; +import { isMessageWebSearchSourcesUpdate } from "$lib/utils/messageUpdates"; + +// ----------- +// Copy of the previous message update types +export type FinalAnswer = { + type: "finalAnswer"; + text: string; +}; + +export type TextStreamUpdate = { + type: "stream"; + token: string; +}; + +type WebSearchUpdate = { + type: "webSearch"; + messageType: "update" | "error" | "sources"; + message: string; + args?: string[]; + sources?: WebSearchSource[]; +}; + +type StatusUpdate = { + type: "status"; + status: "started" | "pending" | "finished" | "error" | "title"; + message?: string; +}; + +type ErrorUpdate = { + type: "error"; + message: string; + name: string; +}; + +type FileUpdate = { + type: "file"; + sha: string; +}; + +type OldMessageUpdate = + | FinalAnswer + | TextStreamUpdate + | WebSearchUpdate + | StatusUpdate + | ErrorUpdate + | FileUpdate; + +/** Converts the old message update to the new schema */ +function convertMessageUpdate(message: Message, update: OldMessageUpdate): MessageUpdate | null { + try { + // Text and files + if (update.type === "finalAnswer") { + return { + type: MessageUpdateType.FinalAnswer, + text: update.text, + interrupted: message.interrupted ?? false, + }; + } else if (update.type === "stream") { + return { + type: MessageUpdateType.Stream, + token: update.token, + }; + } else if (update.type === "file") { + return { + type: MessageUpdateType.File, + sha: update.sha, + }; + } + + // Status + else if (update.type === "status") { + if (update.status === "title") { + return { + type: MessageUpdateType.Title, + title: update.message ?? "New Chat", + }; + } + if (update.status === "pending") return null; + + const status = + update.status === "started" + ? MessageUpdateStatus.Started + : update.status === "finished" + ? MessageUpdateStatus.Finished + : MessageUpdateStatus.Error; + return { + type: MessageUpdateType.Status, + status, + message: update.message, + }; + } else if (update.type === "error") { + // Treat it as an error status update + return { + type: MessageUpdateType.Status, + status: MessageUpdateStatus.Error, + message: update.message, + }; + } + + // Web Search + else if (update.type === "webSearch") { + if (update.messageType === "update") { + return { + type: MessageUpdateType.WebSearch, + subtype: MessageWebSearchUpdateType.Update, + message: update.message, + args: update.args, + }; + } else if (update.messageType === "error") { + return { + type: MessageUpdateType.WebSearch, + subtype: MessageWebSearchUpdateType.Error, + message: update.message, + args: update.args, + }; + } else if (update.messageType === "sources") { + return { + type: MessageUpdateType.WebSearch, + subtype: MessageWebSearchUpdateType.Sources, + message: update.message, + sources: update.sources ?? [], + }; + } + } + console.warn("Unknown message update during migration:", update); + return null; + } catch (error) { + console.error("Error converting message update during migration. Skipping it... Error:", error); + return null; + } +} + +const updateMessageUpdates: Migration = { + _id: new ObjectId("5f9f4f4f4f4f4f4f4f4f4f4f"), + name: "Convert message updates to the new schema", + up: async () => { + const allConversations = collections.conversations.find({}, { projection: { messages: 1 } }); + + let conversation: WithId> | null = null; + while ((conversation = await allConversations.tryNext())) { + const messages = conversation.messages.map((message) => { + // Convert all of the existing updates to the new schema + const updates = message.updates + ?.map((update) => convertMessageUpdate(message, update as OldMessageUpdate)) + .filter((update): update is MessageUpdate => Boolean(update)); + + // Add the new web search finished update if the sources update exists and webSearch is defined + const webSearchSourcesUpdateIndex = updates?.findIndex(isMessageWebSearchSourcesUpdate); + if ( + message.webSearch && + updates && + webSearchSourcesUpdateIndex && + webSearchSourcesUpdateIndex !== -1 + ) { + const webSearchFinishedUpdate: MessageWebSearchFinishedUpdate = { + type: MessageUpdateType.WebSearch, + subtype: MessageWebSearchUpdateType.Finished, + webSearch: message.webSearch, + }; + updates.splice(webSearchSourcesUpdateIndex + 1, 0, webSearchFinishedUpdate); + } + return { ...message, updates }; + }); + + // Set the new messages array + await collections.conversations.updateOne({ _id: conversation._id }, { $set: { messages } }); + } + + return true; + }, + runEveryTime: false, +}; + +export default updateMessageUpdates; diff --git a/src/lib/migrations/routines/index.ts b/src/lib/migrations/routines/index.ts index bf6b79218c8..5ae35a20373 100644 --- a/src/lib/migrations/routines/index.ts +++ b/src/lib/migrations/routines/index.ts @@ -4,6 +4,7 @@ import updateSearchAssistant from "./01-update-search-assistants"; import updateAssistantsModels from "./02-update-assistants-models"; import type { Database } from "$lib/server/database"; import addToolsToSettings from "./03-add-tools-in-settings"; +import updateMessageUpdates from "./04-update-message-updates"; export interface Migration { _id: ObjectId; @@ -19,4 +20,5 @@ export const migrations: Migration[] = [ updateSearchAssistant, updateAssistantsModels, addToolsToSettings, + updateMessageUpdates, ]; diff --git a/src/lib/server/textGeneration/generate.ts b/src/lib/server/textGeneration/generate.ts index bfa11aa28fb..65e77e94f96 100644 --- a/src/lib/server/textGeneration/generate.ts +++ b/src/lib/server/textGeneration/generate.ts @@ -1,16 +1,13 @@ import type { ToolResult } from "$lib/types/Tool"; -import { - TextGenerationUpdateType, - type TextGenerationContext, - type TextGenerationUpdate, -} from "./types"; +import { MessageUpdateType, type MessageUpdate } from "$lib/types/MessageUpdate"; import { AbortedGenerations } from "../abortedGenerations"; +import type { TextGenerationContext } from "./types"; export async function* generate( { model, endpoint, conv, messages, assistant, isContinue, promptedAt }: TextGenerationContext, toolResults: ToolResult[], preprompt?: string -): AsyncIterable { +): AsyncIterable { for await (const output of await endpoint({ messages, preprompt, @@ -30,7 +27,7 @@ export async function* generate( text = text.slice(0, text.length - stopToken.length); } - yield { type: TextGenerationUpdateType.FinalAnswer, text, interrupted }; + yield { type: MessageUpdateType.FinalAnswer, text, interrupted }; continue; } @@ -38,7 +35,7 @@ export async function* generate( if (output.token.special) continue; // pass down normal token - yield { type: TextGenerationUpdateType.Stream, token: output.token.text }; + yield { type: MessageUpdateType.Stream, token: output.token.text }; // abort check const date = AbortedGenerations.getInstance().getList().get(conv._id.toString()); diff --git a/src/lib/server/textGeneration/index.ts b/src/lib/server/textGeneration/index.ts index da0954fb6d6..c3991fdcef0 100644 --- a/src/lib/server/textGeneration/index.ts +++ b/src/lib/server/textGeneration/index.ts @@ -11,13 +11,13 @@ import { import { pickTools, runTools } from "./tools"; import type { WebSearch } from "$lib/types/WebSearch"; import { - type TextGenerationUpdate, - TextGenerationUpdateType, - TextGenerationStatus, - type TextGenerationContext, -} from "./types"; + type MessageUpdate, + MessageUpdateType, + MessageUpdateStatus, +} from "$lib/types/MessageUpdate"; import { generate } from "./generate"; import { mergeAsyncGenerators } from "$lib/utils/mergeAsyncGenerators"; +import type { TextGenerationContext } from "./types"; export async function* textGeneration(ctx: TextGenerationContext) { yield* mergeAsyncGenerators([ @@ -28,10 +28,10 @@ export async function* textGeneration(ctx: TextGenerationContext) { async function* textGenerationWithoutTitle( ctx: TextGenerationContext -): AsyncGenerator { +): AsyncGenerator { yield { - type: TextGenerationUpdateType.Status, - status: TextGenerationStatus.Started, + type: MessageUpdateType.Status, + status: MessageUpdateStatus.Started, }; ctx.assistant ??= await getAssistantById(ctx.conv.assistantId); diff --git a/src/lib/server/textGeneration/title.ts b/src/lib/server/textGeneration/title.ts index 07ba771a165..366c59514b6 100644 --- a/src/lib/server/textGeneration/title.ts +++ b/src/lib/server/textGeneration/title.ts @@ -2,12 +2,12 @@ import { env } from "$env/dynamic/private"; import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint"; import type { Message } from "$lib/types/Message"; import { logger } from "$lib/server/logger"; -import { TextGenerationUpdateType, type TextGenerationUpdate } from "./types"; +import { MessageUpdateType, type MessageUpdate } from "$lib/types/MessageUpdate"; import type { Conversation } from "$lib/types/Conversation"; export async function* generateTitleForConversation( conv: Conversation -): AsyncGenerator { +): AsyncGenerator { try { const userMessage = conv.messages.find((m) => m.from === "user"); // HACK: detect if the conversation is new @@ -17,7 +17,7 @@ export async function* generateTitleForConversation( const title = (await generateTitle(prompt)) ?? "New Chat"; yield { - type: TextGenerationUpdateType.Title, + type: MessageUpdateType.Title, title, }; } catch (cause) { diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts index 4f937ee815f..2ee7c0fc59c 100644 --- a/src/lib/server/textGeneration/tools.ts +++ b/src/lib/server/textGeneration/tools.ts @@ -3,12 +3,12 @@ import { v4 as uuidV4 } from "uuid"; import JSON5 from "json5"; import type { BackendTool } from "../tools"; import { - TextGenerationStatus, - TextGenerationToolUpdateType, - TextGenerationUpdateType, - type TextGenerationContext, - type TextGenerationUpdate, -} from "./types"; + MessageToolUpdateType, + MessageUpdateStatus, + MessageUpdateType, + type MessageUpdate, +} from "$lib/types/MessageUpdate"; +import type { TextGenerationContext } from "./types"; import { allTools } from "../tools"; import directlyAnswer from "../tools/directlyAnswer"; @@ -33,7 +33,7 @@ export async function* runTools( { endpoint, conv, messages, assistant }: TextGenerationContext, tools: BackendTool[], preprompt?: string -): AsyncGenerator { +): AsyncGenerator { const calls: ToolCall[] = []; // do the function calling bits here @@ -62,8 +62,8 @@ export async function* runTools( } catch (cause) { // error parsing the calls yield { - type: TextGenerationUpdateType.Status, - status: TextGenerationStatus.Error, + type: MessageUpdateType.Status, + status: MessageUpdateStatus.Error, message: cause instanceof Error ? cause.message : String(cause), }; console.error(cause); @@ -86,10 +86,10 @@ export async function* runTools( if (tool.name === "directly_answer") continue; yield { - type: TextGenerationUpdateType.Tool, - subtype: TextGenerationToolUpdateType.Call, + type: MessageUpdateType.Tool, + subtype: MessageToolUpdateType.Call, uuid, - toolCall: call, + call, }; try { const toolResult = yield* tool.call(call.parameters, { @@ -99,10 +99,10 @@ export async function* runTools( assistant, }); yield { - type: TextGenerationUpdateType.Tool, - subtype: TextGenerationToolUpdateType.Result, + type: MessageUpdateType.Tool, + subtype: MessageToolUpdateType.Result, uuid, - toolResult: { ...toolResult, call } as ToolResult, + result: { ...toolResult, call } as ToolResult, }; toolResults.push({ ...toolResult, call } as ToolResult); } catch (cause) { diff --git a/src/lib/server/textGeneration/types.ts b/src/lib/server/textGeneration/types.ts index bbdfd35b53e..a615861fbec 100644 --- a/src/lib/server/textGeneration/types.ts +++ b/src/lib/server/textGeneration/types.ts @@ -1,10 +1,8 @@ -import type { WebSearch, WebSearchSource } from "$lib/types/WebSearch"; -import type { Tool, ToolCall, ToolResult } from "$lib/types/Tool"; +import type { ProcessedModel } from "../models"; +import type { Endpoint } from "../endpoints/endpoints"; import type { Conversation } from "$lib/types/Conversation"; import type { Message } from "$lib/types/Message"; import type { Assistant } from "$lib/types/Assistant"; -import type { ProcessedModel } from "../models"; -import type { Endpoint } from "../endpoints/endpoints"; export interface TextGenerationContext { model: ProcessedModel; @@ -17,114 +15,3 @@ export interface TextGenerationContext { toolsPreference: Record; promptedAt: Date; } - -//---------- -// Text Generation Updates - -export type TextGenerationUpdate = - | TextGenerationStatusUpdate - | TextGenerationTitleUpdate - | TextGenerationToolUpdate - | TextGenerationWebSearchUpdate - | TextGenerationStreamUpdate - | TextGenerationFileUpdate - | TextGenerationFinalAnswerUpdate; - -export enum TextGenerationUpdateType { - Status = "status", - Title = "title", - Tool = "tool", - WebSearch = "webSearch", - Stream = "stream", - File = "file", - FinalAnswer = "finalAnswer", -} - -// Status -export enum TextGenerationStatus { - Started = "started", - Error = "error", - Finished = "finished", -} -interface TextGenerationStatusUpdate { - type: TextGenerationUpdateType.Status; - status: TextGenerationStatus; - message?: string; -} - -// Web search -export enum TextGenerationWebSearchUpdateType { - Update = "update", - Error = "error", - Sources = "sources", - FinalAnswer = "finalAnswer", -} -export interface TextGenerationWebSearchErrorUpdate { - type: TextGenerationUpdateType.WebSearch; - subtype: TextGenerationWebSearchUpdateType.Error; - message: string; - args?: string[]; -} -export interface TextGenerationWebSearchGeneralUpdate { - type: TextGenerationUpdateType.WebSearch; - subtype: TextGenerationWebSearchUpdateType.Update; - message: string; - args?: string[]; -} -export interface TextGenerationWebSearchSourcesUpdate { - type: TextGenerationUpdateType.WebSearch; - subtype: TextGenerationWebSearchUpdateType.Sources; - message: string; - sources: WebSearchSource[]; -} -export interface TextGenerationWebSearchFinalAnswerUpdate { - type: TextGenerationUpdateType.WebSearch; - subtype: TextGenerationWebSearchUpdateType.FinalAnswer; - webSearch: WebSearch; -} -export type TextGenerationWebSearchUpdate = - | TextGenerationWebSearchErrorUpdate - | TextGenerationWebSearchGeneralUpdate - | TextGenerationWebSearchSourcesUpdate - | TextGenerationWebSearchFinalAnswerUpdate; - -// Tool -export enum TextGenerationToolUpdateType { - /** A request to call a tool alongside it's parameters */ - Call = "call", - /** The result of a tool call */ - Result = "result", -} -interface TextGenerationToolBaseUpdate { - type: TextGenerationUpdateType.Tool; - subtype: TSubType; - uuid: string; -} -interface TextGenerationToolCallUpdate - extends TextGenerationToolBaseUpdate { - toolCall: ToolCall; -} -interface TextGenerationToolCallResultUpdate - extends TextGenerationToolBaseUpdate { - toolResult: ToolResult; -} -type TextGenerationToolUpdate = TextGenerationToolCallUpdate | TextGenerationToolCallResultUpdate; - -// Everything else -interface TextGenerationTitleUpdate { - type: TextGenerationUpdateType.Title; - title: string; -} -interface TextGenerationStreamUpdate { - type: TextGenerationUpdateType.Stream; - token: string; -} -interface TextGenerationFileUpdate { - type: TextGenerationUpdateType.File; - sha: string; -} -interface TextGenerationFinalAnswerUpdate { - type: TextGenerationUpdateType.FinalAnswer; - text: string; - interrupted: boolean; -} diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index 5a4f5173adf..da45bcd14e1 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -1,11 +1,11 @@ import type { Assistant } from "$lib/types/Assistant"; import type { Conversation } from "$lib/types/Conversation"; import type { Message } from "$lib/types/Message"; +import type { MessageUpdate } from "$lib/types/MessageUpdate"; import type { Tool, ToolResult } from "$lib/types/Tool"; -import type { TextGenerationUpdate } from "../textGeneration/types"; + import calculator from "./calculator"; import directlyAnswer from "./directlyAnswer"; -// import codeInterpreter from "./codeInterpreter"; import fetchUrl from "./web/url"; import text2image from "./multimodal/text2image"; import websearch from "./web/search"; @@ -21,7 +21,7 @@ export interface BackendTool extends Tool { call( params: Record, context: BackendToolContext - ): AsyncGenerator, undefined>; + ): AsyncGenerator, undefined>; } export const allTools: BackendTool[] = [ diff --git a/src/lib/server/tools/multimodal/text2image.ts b/src/lib/server/tools/multimodal/text2image.ts index 3d39027c481..3e3d05dbe5b 100644 --- a/src/lib/server/tools/multimodal/text2image.ts +++ b/src/lib/server/tools/multimodal/text2image.ts @@ -1,9 +1,9 @@ import type { BackendTool } from ".."; import { uploadFile } from "../../files/uploadFile"; -import { TextGenerationUpdateType } from "../../textGeneration/types"; import { env } from "$env/dynamic/private"; import { Client } from "@gradio/client"; import { ToolResultStatus } from "$lib/types/Tool"; +import { MessageUpdateType } from "$lib/types/MessageUpdate"; const text2img: BackendTool = { name: "text2img", @@ -34,7 +34,7 @@ const text2img: BackendTool = { const sha = await uploadFile(await response.blob(), conv); - yield { type: TextGenerationUpdateType.File, sha }; + yield { type: MessageUpdateType.File, sha }; return { status: ToolResultStatus.Success, diff --git a/src/lib/server/websearch/runWebSearch.ts b/src/lib/server/websearch/runWebSearch.ts index 5aba2eee74a..c57d4a1fc4f 100644 --- a/src/lib/server/websearch/runWebSearch.ts +++ b/src/lib/server/websearch/runWebSearch.ts @@ -4,7 +4,7 @@ import type { Conversation } from "$lib/types/Conversation"; import type { Message } from "$lib/types/Message"; import type { WebSearch, WebSearchScrapedSource } from "$lib/types/WebSearch"; import type { Assistant } from "$lib/types/Assistant"; -import type { TextGenerationWebSearchUpdate } from "../textGeneration/types"; +import type { MessageWebSearchUpdate } from "$lib/types/MessageUpdate"; import { search } from "./search/search"; import { scrape } from "./scrape/scrape"; @@ -26,7 +26,7 @@ export async function* runWebSearch( messages: Message[], ragSettings?: Assistant["rag"], query?: string -): AsyncGenerator { +): AsyncGenerator { const prompt = messages[messages.length - 1].content; const createdAt = new Date(); const updatedAt = new Date(); diff --git a/src/lib/server/websearch/scrape/scrape.ts b/src/lib/server/websearch/scrape/scrape.ts index c19fbe92689..9029e8c96c1 100644 --- a/src/lib/server/websearch/scrape/scrape.ts +++ b/src/lib/server/websearch/scrape/scrape.ts @@ -1,16 +1,16 @@ import type { WebSearchScrapedSource, WebSearchSource } from "$lib/types/WebSearch"; +import type { MessageWebSearchUpdate } from "$lib/types/MessageUpdate"; import { loadPage } from "./playwright"; import { spatialParser } from "./parser"; import { htmlToMarkdownTree } from "../markdown/tree"; import { timeout } from "$lib/utils/timeout"; -import type { TextGenerationWebSearchUpdate } from "$lib/server/textGeneration/types"; import { makeErrorUpdate, makeGeneralUpdate } from "../update"; export const scrape = (maxCharsPerElem: number) => async function* ( source: WebSearchSource - ): AsyncGenerator { + ): AsyncGenerator { try { const page = await scrapeUrl(source.link, maxCharsPerElem); yield makeGeneralUpdate({ message: "Browsing webpage", args: [source.link] }); diff --git a/src/lib/server/websearch/search/search.ts b/src/lib/server/websearch/search/search.ts index 0044e8d8932..9f232a0ea98 100644 --- a/src/lib/server/websearch/search/search.ts +++ b/src/lib/server/websearch/search/search.ts @@ -9,8 +9,8 @@ import { isURL } from "$lib/utils/isUrl"; import z from "zod"; import JSON5 from "json5"; import { env } from "$env/dynamic/private"; -import type { TextGenerationWebSearchUpdate } from "$lib/server/textGeneration/types"; import { makeGeneralUpdate } from "../update"; +import type { MessageWebSearchUpdate } from "$lib/types/MessageUpdate"; const listSchema = z.array(z.string()).default([]); const allowList = listSchema.parse(JSON5.parse(env.WEBSEARCH_ALLOWLIST)); @@ -21,7 +21,7 @@ export async function* search( ragSettings?: Assistant["rag"], query?: string ): AsyncGenerator< - TextGenerationWebSearchUpdate, + MessageWebSearchUpdate, { searchQuery: string; pages: WebSearchSource[] }, undefined > { diff --git a/src/lib/server/websearch/update.ts b/src/lib/server/websearch/update.ts index 0d06e98957d..909e63139cf 100644 --- a/src/lib/server/websearch/update.ts +++ b/src/lib/server/websearch/update.ts @@ -1,50 +1,46 @@ import type { WebSearch, WebSearchSource } from "$lib/types/WebSearch"; import { - TextGenerationUpdateType, - TextGenerationWebSearchUpdateType, - type TextGenerationWebSearchErrorUpdate, - type TextGenerationWebSearchFinalAnswerUpdate, - type TextGenerationWebSearchGeneralUpdate, - type TextGenerationWebSearchSourcesUpdate, -} from "../textGeneration/types"; + MessageUpdateType, + MessageWebSearchUpdateType, + type MessageWebSearchErrorUpdate, + type MessageWebSearchFinishedUpdate, + type MessageWebSearchGeneralUpdate, + type MessageWebSearchSourcesUpdate, +} from "$lib/types/MessageUpdate"; export function makeGeneralUpdate( - update: Pick -): TextGenerationWebSearchGeneralUpdate { + update: Pick +): MessageWebSearchGeneralUpdate { return { - type: TextGenerationUpdateType.WebSearch, - subtype: TextGenerationWebSearchUpdateType.Update, + type: MessageUpdateType.WebSearch, + subtype: MessageWebSearchUpdateType.Update, ...update, }; } export function makeErrorUpdate( - update: Pick -): TextGenerationWebSearchErrorUpdate { + update: Pick +): MessageWebSearchErrorUpdate { return { - type: TextGenerationUpdateType.WebSearch, - subtype: TextGenerationWebSearchUpdateType.Error, + type: MessageUpdateType.WebSearch, + subtype: MessageWebSearchUpdateType.Error, ...update, }; } -export function makeSourcesUpdate( - sources: WebSearchSource[] -): TextGenerationWebSearchSourcesUpdate { +export function makeSourcesUpdate(sources: WebSearchSource[]): MessageWebSearchSourcesUpdate { return { - type: TextGenerationUpdateType.WebSearch, - subtype: TextGenerationWebSearchUpdateType.Sources, + type: MessageUpdateType.WebSearch, + subtype: MessageWebSearchUpdateType.Sources, message: "sources", sources, }; } -export function makeFinalAnswerUpdate( - webSearch: WebSearch -): TextGenerationWebSearchFinalAnswerUpdate { +export function makeFinalAnswerUpdate(webSearch: WebSearch): MessageWebSearchFinishedUpdate { return { - type: TextGenerationUpdateType.WebSearch, - subtype: TextGenerationWebSearchUpdateType.FinalAnswer, + type: MessageUpdateType.WebSearch, + subtype: MessageWebSearchUpdateType.Finished, webSearch, }; } diff --git a/src/lib/types/MessageUpdate.ts b/src/lib/types/MessageUpdate.ts index 44701c61405..c48bb6b49b6 100644 --- a/src/lib/types/MessageUpdate.ts +++ b/src/lib/types/MessageUpdate.ts @@ -1,65 +1,109 @@ -import type { ToolCall, Tool } from "./Tool"; -import type { WebSearchSource } from "./WebSearch"; +import type { WebSearch, WebSearchSource } from "$lib/types/WebSearch"; +import type { ToolCall, ToolResult } from "$lib/types/Tool"; -export type FinalAnswer = { - type: "finalAnswer"; - text: string; -}; - -export type TextStreamUpdate = { - type: "stream"; - token: string; -}; +export type MessageUpdate = + | MessageStatusUpdate + | MessageTitleUpdate + | MessageToolUpdate + | MessageWebSearchUpdate + | MessageStreamUpdate + | MessageFileUpdate + | MessageFinalAnswerUpdate; -interface ToolUpdateBase { - type: "tool"; - name: Tool["name"]; - uuid: string; +export enum MessageUpdateType { + Status = "status", + Title = "title", + Tool = "tool", + WebSearch = "webSearch", + Stream = "stream", + File = "file", + FinalAnswer = "finalAnswer", } -interface ToolUpdateParams extends ToolUpdateBase { - messageType: "parameters"; - parameters: ToolCall["parameters"]; +// Status +export enum MessageUpdateStatus { + Started = "started", + Error = "error", + Finished = "finished", } - -interface ToolUpdateMessage extends ToolUpdateBase { - messageType: "message"; +export interface MessageStatusUpdate { + type: MessageUpdateType.Status; + status: MessageUpdateStatus; message?: string; - display?: boolean; } -export type ToolUpdate = ToolUpdateParams | ToolUpdateMessage; - -export type WebSearchUpdate = { - type: "webSearch"; - messageType: "update" | "error" | "sources"; +// Web search +export enum MessageWebSearchUpdateType { + Update = "update", + Error = "error", + Sources = "sources", + Finished = "finished", +} +export interface MessageWebSearchErrorUpdate { + type: MessageUpdateType.WebSearch; + subtype: MessageWebSearchUpdateType.Error; message: string; args?: string[]; - sources?: WebSearchSource[]; -}; - -export type StatusUpdate = { - type: "status"; - status: "started" | "pending" | "finished" | "error" | "title"; - message?: string; -}; - -export type ErrorUpdate = { - type: "error"; +} +export interface MessageWebSearchGeneralUpdate { + type: MessageUpdateType.WebSearch; + subtype: MessageWebSearchUpdateType.Update; message: string; - name: string; -}; + args?: string[]; +} +export interface MessageWebSearchSourcesUpdate { + type: MessageUpdateType.WebSearch; + subtype: MessageWebSearchUpdateType.Sources; + message: string; + sources: WebSearchSource[]; +} +export interface MessageWebSearchFinishedUpdate { + type: MessageUpdateType.WebSearch; + subtype: MessageWebSearchUpdateType.Finished; + webSearch: WebSearch; +} +export type MessageWebSearchUpdate = + | MessageWebSearchErrorUpdate + | MessageWebSearchGeneralUpdate + | MessageWebSearchSourcesUpdate + | MessageWebSearchFinishedUpdate; -export type FileUpdate = { - type: "file"; - sha: string; -}; +// Tool +export enum MessageToolUpdateType { + /** A request to call a tool alongside it's parameters */ + Call = "call", + /** The result of a tool call */ + Result = "result", +} +interface MessageToolBaseUpdate { + type: MessageUpdateType.Tool; + subtype: TSubType; + uuid: string; +} +export interface MessageToolCallUpdate extends MessageToolBaseUpdate { + call: ToolCall; +} +export interface MessageToolResultUpdate + extends MessageToolBaseUpdate { + result: ToolResult; +} +export type MessageToolUpdate = MessageToolCallUpdate | MessageToolResultUpdate; -export type MessageUpdate = - | FinalAnswer - | TextStreamUpdate - | ToolUpdate - | WebSearchUpdate - | StatusUpdate - | ErrorUpdate - | FileUpdate; +// Everything else +export interface MessageTitleUpdate { + type: MessageUpdateType.Title; + title: string; +} +export interface MessageStreamUpdate { + type: MessageUpdateType.Stream; + token: string; +} +export interface MessageFileUpdate { + type: MessageUpdateType.File; + sha: string; +} +export interface MessageFinalAnswerUpdate { + type: MessageUpdateType.FinalAnswer; + text: string; + interrupted: boolean; +} diff --git a/src/lib/utils/messageUpdates.ts b/src/lib/utils/messageUpdates.ts index d6d1b402205..93b72102a36 100644 --- a/src/lib/utils/messageUpdates.ts +++ b/src/lib/utils/messageUpdates.ts @@ -1,4 +1,38 @@ -import type { MessageUpdate, TextStreamUpdate } from "$lib/types/MessageUpdate"; +import { + type MessageUpdate, + type MessageStreamUpdate, + type MessageToolCallUpdate, + MessageToolUpdateType, + MessageUpdateType, + type MessageToolUpdate, + type MessageWebSearchUpdate, + type MessageWebSearchGeneralUpdate, + type MessageWebSearchSourcesUpdate, + type MessageWebSearchErrorUpdate, + MessageWebSearchUpdateType, +} from "$lib/types/MessageUpdate"; + +export const isMessageWebSearchUpdate = (update: MessageUpdate): update is MessageWebSearchUpdate => + update.type === MessageUpdateType.WebSearch; +export const isMessageWebSearchGeneralUpdate = ( + update: MessageUpdate +): update is MessageWebSearchGeneralUpdate => + isMessageWebSearchUpdate(update) && update.subtype === MessageWebSearchUpdateType.Update; +export const isMessageWebSearchSourcesUpdate = ( + update: MessageUpdate +): update is MessageWebSearchSourcesUpdate => + isMessageWebSearchUpdate(update) && update.subtype === MessageWebSearchUpdateType.Sources; +export const isMessageWebSearchErrorUpdate = ( + update: MessageUpdate +): update is MessageWebSearchErrorUpdate => + isMessageWebSearchUpdate(update) && update.subtype === MessageWebSearchUpdateType.Error; + +export const isMessageToolUpdate = (update: MessageUpdate): update is MessageToolUpdate => + update.type === MessageUpdateType.Tool; +export const isMessageToolCallUpdate = (update: MessageUpdate): update is MessageToolCallUpdate => + isMessageToolUpdate(update) && update.subtype === MessageToolUpdateType.Call; +export const isMessageToolResultUpdate = (update: MessageUpdate): update is MessageToolCallUpdate => + isMessageToolUpdate(update) && update.subtype === MessageToolUpdateType.Result; type MessageUpdateRequestOptions = { base: string; @@ -109,7 +143,7 @@ function parseMessageUpdates(value: string): { async function* streamMessageUpdatesToFullWords( iterator: AsyncGenerator ): AsyncGenerator { - let bufferedStreamUpdates: TextStreamUpdate[] = []; + let bufferedStreamUpdates: MessageStreamUpdate[] = []; const endAlphanumeric = /[a-zA-Z0-9À-ž'`]+$/; const beginnningAlphanumeric = /^[a-zA-Z0-9À-ž'`]+/; @@ -131,7 +165,7 @@ async function* streamMessageUpdatesToFullWords( // Combine tokens together and emit yield { - type: "stream", + type: MessageUpdateType.Stream, token: bufferedStreamUpdates .slice(lastIndexEmitted, i) .map((_) => _.token) diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte index 90b5e9eb6b6..867f740aa44 100644 --- a/src/routes/conversation/[id]/+page.svelte +++ b/src/routes/conversation/[id]/+page.svelte @@ -11,7 +11,11 @@ import { findCurrentModel } from "$lib/utils/models"; import { webSearchParameters } from "$lib/stores/webSearchParameters"; import type { Message } from "$lib/types/Message"; - import type { MessageUpdate } from "$lib/types/MessageUpdate"; + import { + MessageUpdateStatus, + MessageUpdateType, + type MessageUpdate, + } from "$lib/types/MessageUpdate"; import titleUpdate from "$lib/stores/titleUpdate"; import file2base64 from "$lib/utils/file2base64"; import { addChildren } from "$lib/utils/tree/addChildren"; @@ -224,36 +228,34 @@ messageUpdates.push(update); - if (update.type === "stream") { + if (update.type === MessageUpdateType.Stream) { pending = false; messageToWriteTo.content += update.token; messages = [...messages]; - } else if (update.type === "webSearch") { + } else if ( + update.type === MessageUpdateType.WebSearch || + update.type === MessageUpdateType.Tool + ) { messageToWriteTo.updates = [...(messageToWriteTo.updates ?? []), update]; messages = [...messages]; - } else if (update.type === "status") { - if (update.status === "title" && update.message) { - const convInData = data.conversations.find(({ id }) => id === $page.params.id); - if (convInData) { - convInData.title = update.message; - - $titleUpdate = { - title: update.message, - convId: $page.params.id, - }; - } - } else if (update.status === "error") { - $error = update.message ?? "An error has occurred"; + } else if ( + update.type === MessageUpdateType.Status && + update.status === MessageUpdateStatus.Error + ) { + $error = update.message ?? "An error has occurred"; + } else if (update.type === MessageUpdateType.Title) { + const convInData = data.conversations.find(({ id }) => id === $page.params.id); + if (convInData) { + convInData.title = update.title; + + $titleUpdate = { + title: update.title, + convId: $page.params.id, + }; } - } else if (update.type === "tool") { - messageToWriteTo.updates = [...(messageToWriteTo.updates ?? []), update]; - messages = [...messages]; - } else if (update.type === "file") { + } else if (update.type === MessageUpdateType.File) { messageToWriteTo.files = [...(messageToWriteTo.files ?? []), update.sha]; messages = [...messages]; - } else if (update.type === "error") { - error.set(update.message); - messageUpdatesAbortController.abort(); } } diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 14d5cb94d59..6594bb09642 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -8,7 +8,12 @@ import type { Message } from "$lib/types/Message"; import { error } from "@sveltejs/kit"; import { ObjectId } from "mongodb"; import { z } from "zod"; -import type { MessageUpdate } from "$lib/types/MessageUpdate"; +import { + MessageUpdateStatus, + MessageUpdateType, + MessageWebSearchUpdateType, + type MessageUpdate, +} from "$lib/types/MessageUpdate"; import { uploadFile } from "$lib/server/files/uploadFile"; import sizeof from "image-size"; import { convertLegacyConversation } from "$lib/utils/tree/convertLegacyConversation"; @@ -18,12 +23,7 @@ import { addChildren } from "$lib/utils/tree/addChildren.js"; import { addSibling } from "$lib/utils/tree/addSibling.js"; import { usageLimits } from "$lib/server/usageLimits"; import { textGeneration } from "$lib/server/textGeneration"; -import { - TextGenerationToolUpdateType, - TextGenerationUpdateType, - TextGenerationWebSearchUpdateType, - type TextGenerationContext, -} from "$lib/server/textGeneration/types"; +import type { TextGenerationContext } from "$lib/server/textGeneration/types"; export async function POST({ request, locals, params, getClientAddress }) { const id = z.string().parse(params.id); @@ -293,12 +293,55 @@ export async function POST({ request, locals, params, getClientAddress }) { const stream = new ReadableStream({ async start(controller) { messageToWriteTo.updates ??= []; - function update(newUpdate: MessageUpdate) { - if (newUpdate.type !== "stream") messageToWriteTo?.updates?.push(newUpdate); - controller.enqueue(JSON.stringify(newUpdate) + "\n"); + async function update(event: MessageUpdate) { + if (!messageToWriteTo || !conv) { + throw Error("No message or conversation to write events to"); + } + + // Add token to content or skip if empty + if (event.type === MessageUpdateType.Stream) { + if (event.token === "") return; + messageToWriteTo.content += event.token; + } + + // Set the title + else if (event.type === MessageUpdateType.Title) { + conv.title = event.title; + await collections.conversations.updateOne( + { _id: convId }, + { $set: { title: conv?.title, updatedAt: new Date() } } + ); + } + + // Set the final text and the interrupted flag + else if (event.type === MessageUpdateType.FinalAnswer) { + messageToWriteTo.interrupted = event.interrupted; + messageToWriteTo.content = initialMessageContent + event.text; + } - if (newUpdate.type === "finalAnswer") { - // 4096 of spaces to make sure the browser doesn't blocking buffer that holding the response + // Add file + else if (event.type === MessageUpdateType.File) { + messageToWriteTo.files = [...(messageToWriteTo.files ?? []), event.sha]; + } + + // Set web search + else if ( + event.type === MessageUpdateType.WebSearch && + event.subtype === MessageWebSearchUpdateType.Finished + ) { + messageToWriteTo.webSearch = event.webSearch; + } + + // Append to the persistent message updates if it's not a stream update + if (event.type !== "stream") { + messageToWriteTo?.updates?.push(event); + } + + // Send the update to the client + controller.enqueue(JSON.stringify(event) + "\n"); + + // Send 4096 of spaces to make sure the browser doesn't blocking buffer that holding the response + if (event.type === "finalAnswer") { controller.enqueue(" ".repeat(4096)); } } @@ -323,81 +366,22 @@ export async function POST({ request, locals, params, getClientAddress }) { toolsPreference: toolsPreferences ?? {}, promptedAt, }; - for await (const event of textGeneration(ctx)) { - if (event.type === TextGenerationUpdateType.Status) { - update({ type: "status", status: event.status, message: event.message }); - } - if (event.type === TextGenerationUpdateType.Title) { - conv.title = event.title; - await collections.conversations.updateOne( - { _id: convId }, - { $set: { title: conv?.title, updatedAt: new Date() } } - ); - update({ type: "status", status: "title", message: event.title }); - } - if (event.type === TextGenerationUpdateType.FinalAnswer) { - messageToWriteTo.interrupted = event.interrupted; - messageToWriteTo.content = initialMessageContent + event.text; - update({ type: "finalAnswer", text: event.text }); - } - if (event.type === TextGenerationUpdateType.Stream) { - if (event.token === "") continue; - messageToWriteTo.content += event.token; - update({ type: "stream", token: event.token }); - } - if (event.type === TextGenerationUpdateType.File) { - messageToWriteTo.files = [...(messageToWriteTo.files ?? []), event.sha]; - update({ type: "file", sha: event.sha }); - } - if (event.type === TextGenerationUpdateType.Tool) { - if (event.subtype === TextGenerationToolUpdateType.Call) { - update({ - type: "tool", - name: event.toolCall.name, - messageType: "parameters", - parameters: event.toolCall.parameters, - uuid: event.uuid, - }); - } - } - if (event.type === TextGenerationUpdateType.WebSearch) { - if (event.subtype === TextGenerationWebSearchUpdateType.Update) { - update({ - type: "webSearch", - messageType: "update", - message: event.message, - args: event.args, - }); - } else if (event.subtype === TextGenerationWebSearchUpdateType.Error) { - update({ - type: "webSearch", - messageType: "error", - message: event.message, - args: event.args, - }); - } else if (event.subtype === TextGenerationWebSearchUpdateType.Sources) { - update({ - type: "webSearch", - messageType: "sources", - sources: event.sources, - message: event.message, - }); - } else if (event.subtype === TextGenerationWebSearchUpdateType.FinalAnswer) { - // something else too? - messageToWriteTo.webSearch = event.webSearch; - } - } - } + // run the text generation and send updates to the client + for await (const event of textGeneration(ctx)) await update(event); } catch (e) { hasError = true; - update({ type: "status", status: "error", message: (e as Error).message }); + await update({ + type: MessageUpdateType.Status, + status: MessageUpdateStatus.Error, + message: (e as Error).message, + }); console.error(e); } finally { // check if no output was generated if (!hasError && messageToWriteTo.content === initialMessageContent) { - update({ - type: "status", - status: "error", + await update({ + type: MessageUpdateType.Status, + status: MessageUpdateStatus.Error, message: "No output was generated. Something went wrong.", }); } diff --git a/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts b/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts index 84cd152884f..5d0c8ec77d9 100644 --- a/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts +++ b/src/routes/conversation/[id]/message/[messageId]/prompt/+server.ts @@ -15,7 +15,7 @@ export async function GET({ params, locals }) { }) : await collections.conversations.findOne({ _id: new ObjectId(params.id), - ...authCondition(locals), + // ...authCondition(locals), }); if (conv === null) { From 8cbe09f8ead931f826cf0159fea5a472437a7090 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Fri, 17 May 2024 15:13:35 -0400 Subject: [PATCH 49/82] fix lint errors --- src/lib/components/chat/ChatMessage.svelte | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 22edfcdb212..2e3aa5c95c2 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -25,8 +25,6 @@ MessageToolUpdateType, MessageWebSearchUpdateType, type MessageToolUpdate, - type MessageToolCallUpdate, - type MessageToolResultUpdate, type MessageWebSearchSourcesUpdate, type MessageWebSearchUpdate, } from "$lib/types/MessageUpdate"; From dbc0ca76f290ad222f93da5dc7beb3966c011cfc Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Sun, 19 May 2024 22:20:47 -0400 Subject: [PATCH 50/82] image editing, pdf upload + parsing --- src/lib/components/UploadBtn.svelte | 4 +- src/lib/components/chat/ChatMessage.svelte | 51 ++++++------ src/lib/components/chat/ChatWindow.svelte | 2 +- .../routines/04-update-message-updates.ts | 2 + src/lib/server/files/uploadFile.ts | 2 +- src/lib/server/models.ts | 6 +- src/lib/server/textGeneration/index.ts | 9 +- src/lib/server/textGeneration/tools.ts | 27 +++++- src/lib/server/tools/images/editing.ts | 74 +++++++++++++++++ src/lib/server/tools/images/generation.ts | 83 +++++++++++++++++++ src/lib/server/tools/index.ts | 8 +- src/lib/server/tools/multimodal/text2image.ts | 51 ------------ src/lib/server/tools/parse/pdf.ts | 49 +++++++++++ src/lib/server/tools/utils.ts | 26 ++++++ src/lib/types/Tool.ts | 19 +++-- src/routes/conversation/[id]/+page.svelte | 5 +- 16 files changed, 318 insertions(+), 100 deletions(-) create mode 100644 src/lib/server/tools/images/editing.ts create mode 100644 src/lib/server/tools/images/generation.ts delete mode 100644 src/lib/server/tools/multimodal/text2image.ts create mode 100644 src/lib/server/tools/parse/pdf.ts create mode 100644 src/lib/server/tools/utils.ts diff --git a/src/lib/components/UploadBtn.svelte b/src/lib/components/UploadBtn.svelte index cb869443e9b..11bd2044881 100644 --- a/src/lib/components/UploadBtn.svelte +++ b/src/lib/components/UploadBtn.svelte @@ -17,7 +17,7 @@ bind:files={filelist} class="absolute w-full cursor-pointer opacity-0" type="file" - accept="image/*" + accept="image/*,application/pdf" /> - Upload image + Upload file diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 492e20f3349..f82f8c4ec13 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -1,7 +1,8 @@ -{#if modalImageToShow} +{#if modalImageToShow !== null} - (modalImageToShow = "")}> - {#if modalImageToShow.length === 64} + (modalImageToShow = null)}> + {#if modalImageToShow.type === "hash"} input from user {:else} input from user @@ -256,16 +257,16 @@ {#each message.files as file} {/each}
      diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index f83b98846ae..1009da6ffd3 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -281,7 +281,7 @@ /> {:else}
      - {#if currentModel.multimodal} + {#if currentModel.multimodal || currentModel.functions} {/if} {#if messages && lastMessage && lastMessage.interrupted && !isReadOnly} diff --git a/src/lib/migrations/routines/04-update-message-updates.ts b/src/lib/migrations/routines/04-update-message-updates.ts index 8ddd08e4b6d..1a9989c41da 100644 --- a/src/lib/migrations/routines/04-update-message-updates.ts +++ b/src/lib/migrations/routines/04-update-message-updates.ts @@ -77,6 +77,8 @@ function convertMessageUpdate(message: Message, update: OldMessageUpdate): Messa return { type: MessageUpdateType.File, sha: update.sha, + // assume jpeg but could be any image. should be harmless + mime: "image/jpeg", }; } diff --git a/src/lib/server/files/uploadFile.ts b/src/lib/server/files/uploadFile.ts index 339a4b4ea85..5191bcb23e6 100644 --- a/src/lib/server/files/uploadFile.ts +++ b/src/lib/server/files/uploadFile.ts @@ -4,7 +4,7 @@ import { sha256 } from "$lib/utils/sha256"; import { fileTypeFromBuffer } from "file-type"; import { collections } from "$lib/server/database"; -export async function uploadFile(file: File, conv: Conversation): Promise { +export async function uploadFile(file: Blob | File, conv: Conversation): Promise { const sha = await sha256(await file.text()); const buffer = await file.arrayBuffer(); diff --git a/src/lib/server/models.ts b/src/lib/server/models.ts index 69120e0a7da..6330d943c8e 100644 --- a/src/lib/server/models.ts +++ b/src/lib/server/models.ts @@ -164,7 +164,11 @@ async function getChatPromptRender( chat_template: chatTemplate, // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore - tools: tools ?? [], + tools: + tools?.map(({ parameterDefinitions, ...tool }) => ({ + parameter_definitions: parameterDefinitions, + ...tool, + })) ?? [], documents, }); diff --git a/src/lib/server/textGeneration/index.ts b/src/lib/server/textGeneration/index.ts index c3991fdcef0..3e7c1fbe052 100644 --- a/src/lib/server/textGeneration/index.ts +++ b/src/lib/server/textGeneration/index.ts @@ -1,5 +1,5 @@ import { runWebSearch } from "$lib/server/websearch/runWebSearch"; -import { preprocessMessages } from "$lib/server/preprocessMessages.js"; +import { preprocessMessages } from "../endpoints/preprocessMessages"; import { generateTitleForConversation } from "./title"; import { @@ -59,11 +59,6 @@ async function* textGenerationWithoutTitle( const tools = pickTools(toolsPreference, Boolean(assistant)); const toolResults = yield* runTools(ctx, tools, preprompt); - const processedMessages = await preprocessMessages( - messages, - webSearchResult, - model.multimodal, - convId - ); + const processedMessages = await preprocessMessages(messages, webSearchResult, convId); yield* generate({ ...ctx, messages: processedMessages }, toolResults, preprompt); } diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts index 2ee7c0fc59c..6e6aef9cba0 100644 --- a/src/lib/server/textGeneration/tools.ts +++ b/src/lib/server/textGeneration/tools.ts @@ -66,7 +66,6 @@ export async function* runTools( status: MessageUpdateStatus.Error, message: cause instanceof Error ? cause.message : String(cause), }; - console.error(cause); } } } @@ -120,7 +119,7 @@ export async function* runTools( const externalToolCall = z.object({ tool_name: z.string(), - parameters: z.record(z.string()), + parameters: z.record(z.any()), }); type ExternalToolCall = z.infer; function isExternalToolCall(call: unknown): call is ExternalToolCall { @@ -128,8 +127,30 @@ function isExternalToolCall(call: unknown): call is ExternalToolCall { } function externalToToolCall(call: ExternalToolCall): ToolCall { + const tool = allTools.find((el) => el.name === call.tool_name); + if (tool === undefined) { + throw new Error(`Could not find tool ${call.tool_name}`); + } + + const parametersWithDefaults = Object.fromEntries( + Object.entries(tool.parameterDefinitions).map(([key, definition]) => { + const value = call.parameters[key]; + + // Required so ensure it's there + if (definition.required) { + if (value === undefined) { + throw new Error(`Missing required parameter ${key} for tool ${call.tool_name}`); + } + return [key, value]; + } + + // Optional so use default if not there + return [key, value ?? definition.default]; + }) + ); + return { name: call.tool_name, - parameters: call.parameters, + parameters: parametersWithDefaults, }; } diff --git a/src/lib/server/tools/images/editing.ts b/src/lib/server/tools/images/editing.ts new file mode 100644 index 00000000000..3db045b985a --- /dev/null +++ b/src/lib/server/tools/images/editing.ts @@ -0,0 +1,74 @@ +import type { BackendTool } from ".."; +import { uploadFile } from "../../files/uploadFile"; +import { ToolResultStatus } from "$lib/types/Tool"; +import { MessageUpdateType } from "$lib/types/MessageUpdate"; +import { callSpace, type GradioImage } from "../utils"; +import { downloadFile } from "$lib/server/files/downloadFile"; + +type ImageEditingInput = [ + Blob /* image */, + string /* prompt */, + string /* negative prompt */, + number /* guidance scale */, + number /* steps */ +]; +type ImageEditingOutput = [GradioImage]; + +const imageEditing: BackendTool = { + name: "image-editing", + displayName: "Image Editing", + description: "Use this tool to edit an image from a prompt.", + isOnByDefault: true, + parameterDefinitions: { + prompt: { + description: + "A prompt to generate an image from. Describe the image visually in simple terms, separate terms with a comma.", + type: "string", + required: true, + }, + }, + async *call({ prompt }, { conv, messages }) { + const latestUserMessage = messages.findLast((message) => message.from === "user"); + const images = latestUserMessage?.files?.filter((file) => file.mime.startsWith("image/")); + if (!images || images.length === 0) { + return { + status: ToolResultStatus.Error, + message: "User did not provide an image to edit.", + }; + } + + // todo: should handle multiple images + const image = await downloadFile(images[0].value, conv._id) + .then((file) => fetch(`data:${file.mime};base64,${file.value}`)) + .then((res) => res.blob()); + + const outputs = await callSpace( + "multimodalart/cosxl", + "run_edit", + [ + image, + prompt, + "", // negative prompt + 7, // guidance scale + 20, // steps + ] + ); + const outputImage = await fetch(outputs[0].url) + .then((res) => res.blob()) + .then((blob) => uploadFile(blob, conv)); + + yield { type: MessageUpdateType.File, sha: outputImage.value, mime: outputImage.mime }; + + return { + status: ToolResultStatus.Success, + outputs: [ + { + imageEditing: `An image has been generated for the following prompt: "${prompt}". Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it.`, + }, + ], + display: false, + }; + }, +}; + +export default imageEditing; diff --git a/src/lib/server/tools/images/generation.ts b/src/lib/server/tools/images/generation.ts new file mode 100644 index 00000000000..a4e654d183c --- /dev/null +++ b/src/lib/server/tools/images/generation.ts @@ -0,0 +1,83 @@ +import type { BackendTool } from ".."; +import { uploadFile } from "../../files/uploadFile"; +import { ToolResultStatus } from "$lib/types/Tool"; +import { MessageUpdateType } from "$lib/types/MessageUpdate"; +import { callSpace, type GradioImage } from "../utils"; + +type ImageGenerationInput = [ + number /* number (numeric value between 1 and 8) in 'Number of Images' Slider component */, + number /* number in 'Image Height' Number component */, + number /* number in 'Image Width' Number component */, + string /* prompt */, + number /* seed random */ +]; +type ImageGenerationOutput = [{ image: GradioImage }[]]; + +const imageGeneration: BackendTool = { + name: "image-generation", + displayName: "Image Generation", + description: "Use this tool to generate an image from a prompt.", + isOnByDefault: true, + parameterDefinitions: { + prompt: { + description: + "A prompt to generate an image from. Describe the image visually in simple terms, separate terms with a comma.", + type: "string", + required: true, + }, + numberOfImages: { + description: "Number of images to generate, between 1 and 8.", + type: "number", + required: false, + default: 1, + }, + width: { + description: "Width of the generated image.", + type: "number", + required: false, + default: 1024, + }, + height: { + description: "Height of the generated image.", + type: "number", + required: false, + default: 1024, + }, + }, + async *call({ prompt, numberOfImages }, { conv }) { + const outputs = await callSpace( + "ByteDance/Hyper-SDXL-1Step-T2I", + "/process_image", + [ + Number(numberOfImages), // number (numeric value between 1 and 8) in 'Number of Images' Slider component + 512, // number in 'Image Height' Number component + 512, // number in 'Image Width' Number component + prompt, // prompt + Math.floor(Math.random() * 1000), // seed random + ] + ); + const imageBlobs = await Promise.all( + outputs[0].map((output) => + fetch(output.image.url) + .then((res) => res.blob()) + .then((blob) => uploadFile(blob, conv)) + ) + ); + + for (const image of imageBlobs) { + yield { type: MessageUpdateType.File, sha: image.value, mime: image.mime }; + } + + return { + status: ToolResultStatus.Success, + outputs: [ + { + imageGeneration: `An image has been generated for the following prompt: "${prompt}". Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it.`, + }, + ], + display: false, + }; + }, +}; + +export default imageGeneration; diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index da45bcd14e1..1658ebeabbb 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -6,8 +6,10 @@ import type { Tool, ToolResult } from "$lib/types/Tool"; import calculator from "./calculator"; import directlyAnswer from "./directlyAnswer"; +import imageEditing from "./images/editing"; +import imageGeneration from "./images/generation"; +import parsePdf from "./parse/pdf"; import fetchUrl from "./web/url"; -import text2image from "./multimodal/text2image"; import websearch from "./web/search"; interface BackendToolContext { @@ -28,6 +30,8 @@ export const allTools: BackendTool[] = [ directlyAnswer, calculator, fetchUrl, - text2image, + parsePdf, + imageEditing, + imageGeneration, websearch, ]; diff --git a/src/lib/server/tools/multimodal/text2image.ts b/src/lib/server/tools/multimodal/text2image.ts deleted file mode 100644 index 3e3d05dbe5b..00000000000 --- a/src/lib/server/tools/multimodal/text2image.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { BackendTool } from ".."; -import { uploadFile } from "../../files/uploadFile"; -import { env } from "$env/dynamic/private"; -import { Client } from "@gradio/client"; -import { ToolResultStatus } from "$lib/types/Tool"; -import { MessageUpdateType } from "$lib/types/MessageUpdate"; - -const text2img: BackendTool = { - name: "text2img", - displayName: "Text-to-Image", - description: "Use this tool to generate an image from a prompt.", - isOnByDefault: true, - parameter_definitions: { - prompt: { - description: - "A prompt to generate an image from. Describe the image visually in simple terms, separate terms with a comma.", - type: "string", - required: true, - }, - }, - async *call({ prompt }, { conv }) { - const app = await Client.connect("ByteDance/Hyper-SDXL-1Step-T2I", { - hf_token: (env.HF_TOKEN ?? env.HF_ACCESS_TOKEN) as unknown as `hf_${string}`, - }); - const res = (await app.predict("/process_image", [ - 1, // number (numeric value between 1 and 8) in 'Number of Images' Slider component - 512, // number in 'Image Height' Number component - 512, // number in 'Image Width' Number component - prompt, // prompt - Math.floor(Math.random() * 1000), // seed random - ])) as unknown as { data: { image: { url: string } }[][] }; - - const response = await fetch(res?.data?.[0]?.[0]?.image?.url ?? "error"); - - const sha = await uploadFile(await response.blob(), conv); - - yield { type: MessageUpdateType.File, sha }; - - return { - status: ToolResultStatus.Success, - outputs: [ - { - text2img: `An image has been generated for the following prompt: "${prompt}". Answer as if the user can already see the image. Do not try to insert the image or to add space for it. The user can already see the image. Do not try to describe the image as you the model cannot see it.`, - }, - ], - display: false, - }; - }, -}; - -export default text2img; diff --git a/src/lib/server/tools/parse/pdf.ts b/src/lib/server/tools/parse/pdf.ts new file mode 100644 index 00000000000..4afd5c9b085 --- /dev/null +++ b/src/lib/server/tools/parse/pdf.ts @@ -0,0 +1,49 @@ +import type { BackendTool } from ".."; +import { ToolResultStatus } from "$lib/types/Tool"; +import { callSpace, type GradioImage } from "../utils"; +import { downloadFile } from "$lib/server/files/downloadFile"; + +type PdfParserInput = [Blob /* pdf */, boolean /* include images */]; +type PdfParserOutput = [ + string /* markdown */, + Record /* metadata */, + GradioImage[] | undefined /* extracted images if enabled */ +]; + +const pdfParser: BackendTool = { + name: "pdf-parser", + displayName: "PDF Parser", + description: "Use this tool to parse a PDF and get its content in markdown format.", + isOnByDefault: true, + parameterDefinitions: {}, + async *call(_, { conv, messages }) { + const latestUserMessage = messages.findLast((message) => message.from === "user"); + const pdfs = latestUserMessage?.files?.filter((file) => file.mime === "application/pdf"); + if (!pdfs || pdfs.length === 0) { + return { + status: ToolResultStatus.Error, + message: "User did not provide a pdf to parse", + }; + } + + // todo: should handle multiple images + const pdf = await downloadFile(pdfs[0].value, conv._id) + .then((file) => fetch(`data:${file.mime};base64,${file.value}`)) + .then((res) => res.blob()); + + const outputs = await callSpace( + "saghen/pdf-to-markdown", + "predict", + [pdf, false] + ); + + const outputMarkdown = outputs[0]; + return { + status: ToolResultStatus.Success, + outputs: [{ pdfParserMarkdown: outputMarkdown }], + display: false, + }; + }, +}; + +export default pdfParser; diff --git a/src/lib/server/tools/utils.ts b/src/lib/server/tools/utils.ts new file mode 100644 index 00000000000..027c77c256d --- /dev/null +++ b/src/lib/server/tools/utils.ts @@ -0,0 +1,26 @@ +import { env } from "$env/dynamic/private"; +import { Client } from "@gradio/client"; + +export type GradioImage = { + path: string; + url: string; + orig_name: string; + is_stream: boolean; + meta: Record; +}; +type GradioResponse = { + data: unknown[]; +}; + +export async function callSpace( + name: string, + func: string, + parameters: TInput +): Promise { + const client = await Client.connect(name, { + hf_token: (env.HF_TOKEN ?? env.HF_ACCESS_TOKEN) as unknown as `hf_${string}`, + }); + return await client + .predict(func, parameters) + .then((res) => (res as unknown as GradioResponse).data as TOutput); +} diff --git a/src/lib/types/Tool.ts b/src/lib/types/Tool.ts index b400125b5ac..760a1a7cb58 100644 --- a/src/lib/types/Tool.ts +++ b/src/lib/types/Tool.ts @@ -1,14 +1,21 @@ -interface ToolInput { - description: string; - type: string; - required?: boolean; -} +type ToolInput = + | { + description: string; + type: string; + required: true; + } + | { + description: string; + type: string; + required: false; + default: string | number | boolean; + }; export interface Tool { name: string; displayName?: string; description: string; - parameter_definitions: Record; + parameterDefinitions: Record; spec?: string; isOnByDefault?: true; // will it be toggled if the user hasn't tweaked it in settings ? isLocked?: true; // can the user enable/disable it ? diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte index f9fecb5b7c5..b71f898ac69 100644 --- a/src/routes/conversation/[id]/+page.svelte +++ b/src/routes/conversation/[id]/+page.svelte @@ -249,7 +249,10 @@ }; } } else if (update.type === MessageUpdateType.File) { - messageToWriteTo.files = [...(messageToWriteTo.files ?? []), update.sha]; + messageToWriteTo.files = [ + ...(messageToWriteTo.files ?? []), + { type: "hash", value: update.sha, mime: update.mime }, + ]; messages = [...messages]; } } From 42cd8c543677739c9b5445826d22f572c8d1a222 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Mon, 20 May 2024 12:34:07 -0400 Subject: [PATCH 51/82] feat: file preview for non-images --- src/lib/components/chat/ChatWindow.svelte | 26 ++++-------- src/lib/components/chat/UploadedFile.svelte | 44 +++++++++++++++++++++ 2 files changed, 52 insertions(+), 18 deletions(-) create mode 100644 src/lib/components/chat/UploadedFile.svelte diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index 1009da6ffd3..00ab38ae38a 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -5,7 +5,6 @@ import CarbonSendAltFilled from "~icons/carbon/send-alt-filled"; import CarbonExport from "~icons/carbon/export"; import CarbonStopFilledAlt from "~icons/carbon/stop-filled-alt"; - import CarbonClose from "~icons/carbon/close"; import CarbonCheckmark from "~icons/carbon/checkmark"; import CarbonCaretDown from "~icons/carbon/caret-down"; @@ -33,6 +32,7 @@ import SystemPromptModal from "../SystemPromptModal.svelte"; import ChatIntroduction from "./ChatIntroduction.svelte"; import { useConvTreeStore } from "$lib/stores/convTree"; + import UploadedFile from "./UploadedFile.svelte"; export let messages: Message[] = []; export let loading = false; @@ -94,7 +94,7 @@ lastMessage.updates?.findIndex((u) => u.type === "status" && u.status === "error") !== -1); $: sources = files?.map((file) => - file2base64(file).then((value) => ({ type: "base64", value, mime: file.type })) + file2base64(file).then((value) => ({ type: "base64", value, mime: file.type, name: file.name })) ); function onShare() { @@ -236,22 +236,12 @@
      {#each sources as source, index} {#await source then src} -
      - input content - - -
      + { + files = files.filter((_, i) => i !== index); + }} + /> {/await} {/each}
      diff --git a/src/lib/components/chat/UploadedFile.svelte b/src/lib/components/chat/UploadedFile.svelte new file mode 100644 index 00000000000..2c8370ec14a --- /dev/null +++ b/src/lib/components/chat/UploadedFile.svelte @@ -0,0 +1,44 @@ + + +
      + {#if src.mime.startsWith("image/")} +
      + input content +
      + {:else} +
      + +
      +
      + {src.name} +
      +
      + {src.mime.split("/")[1].toUpperCase()} +
      +
      +
      + {/if} + + +
      From f01f1d5fe319f6a23c8878f6025eb823a3dd434b Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Mon, 20 May 2024 13:32:47 -0400 Subject: [PATCH 52/82] feat: image prompting, file ui, file migration, many fixes --- src/lib/components/chat/ChatMessage.svelte | 28 +++------ src/lib/components/chat/ChatWindow.svelte | 6 +- src/lib/components/chat/UploadedFile.svelte | 44 ++++++++------ .../routines/03-add-tools-in-settings.ts | 2 +- .../routines/04-update-message-updates.ts | 1 + .../routines/05-update-message-files.ts | 56 ++++++++++++++++++ src/lib/migrations/routines/index.ts | 2 + src/lib/server/files/downloadFile.ts | 35 +++++------ src/lib/server/files/uploadFile.ts | 6 +- src/lib/server/textGeneration/generate.ts | 5 +- src/lib/server/textGeneration/title.ts | 2 +- src/lib/server/textGeneration/tools.ts | 58 +++++++++++++------ src/lib/server/tools/calculator.ts | 4 +- src/lib/server/tools/directlyAnswer.ts | 2 +- src/lib/server/tools/images/editing.ts | 16 ++++- src/lib/server/tools/images/generation.ts | 15 ++++- src/lib/server/tools/index.ts | 2 +- src/lib/server/tools/utils.ts | 9 +++ src/lib/server/tools/web/search.ts | 4 +- src/lib/server/tools/web/url.ts | 4 +- src/lib/types/Message.ts | 1 + src/lib/types/MessageUpdate.ts | 3 +- src/lib/types/Tool.ts | 2 +- src/routes/conversation/[id]/+page.svelte | 9 ++- src/routes/conversation/[id]/+server.ts | 8 ++- 25 files changed, 221 insertions(+), 103 deletions(-) create mode 100644 src/lib/migrations/routines/05-update-message-files.ts diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index f82f8c4ec13..b8381918bef 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -2,7 +2,6 @@ import { marked, type MarkedOptions } from "marked"; import markedKatex from "marked-katex-extension"; import type { Message, MessageFile } from "$lib/types/Message"; - import type { MessageUpdate } from "$lib/types/MessageUpdate"; import { afterUpdate, createEventDispatcher, tick } from "svelte"; import { deepestChild } from "$lib/utils/deepestChild"; import { page } from "$app/stores"; @@ -20,6 +19,7 @@ import CarbonTools from "~icons/carbon/tools"; import { PUBLIC_SEP_TOKEN } from "$lib/constants/publicSepToken"; import type { Model } from "$lib/types/Model"; + import UploadedFile from "./UploadedFile.svelte"; import OpenWebSearchResults from "../OpenWebSearchResults.svelte"; import { @@ -446,25 +446,15 @@ >
      {#if message.files && message.files.length > 0} -
      +
      {#each message.files as file} - + {#if file.mime.startsWith("image/")} + + {:else} + + {/if} {/each}
      {/if} diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index 00ab38ae38a..54c6f5795b1 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -1,5 +1,5 @@
      - {#if src.mime.startsWith("image/")} -
      + {#if file.mime.startsWith("image/")} +
      input content
      {:else} -
      +
      - {src.name} + {file.name}
      - {src.mime.split("/")[1].toUpperCase()} + {file.mime.split("/")[1].toUpperCase()}
      {/if} - - + + {#if canClose} + + {/if}
      diff --git a/src/lib/migrations/routines/03-add-tools-in-settings.ts b/src/lib/migrations/routines/03-add-tools-in-settings.ts index 716b4741a5c..f3e883586cc 100644 --- a/src/lib/migrations/routines/03-add-tools-in-settings.ts +++ b/src/lib/migrations/routines/03-add-tools-in-settings.ts @@ -3,7 +3,7 @@ import { collections } from "$lib/server/database"; import { ObjectId } from "mongodb"; const addToolsToSettings: Migration = { - _id: new ObjectId(3), + _id: new ObjectId("5c9c4c4c4c4c4c4c4c4c4c4c"), name: "Add empty 'tools' record in settings", up: async () => { const { settings } = collections; diff --git a/src/lib/migrations/routines/04-update-message-updates.ts b/src/lib/migrations/routines/04-update-message-updates.ts index 1a9989c41da..540e3baf28c 100644 --- a/src/lib/migrations/routines/04-update-message-updates.ts +++ b/src/lib/migrations/routines/04-update-message-updates.ts @@ -76,6 +76,7 @@ function convertMessageUpdate(message: Message, update: OldMessageUpdate): Messa } else if (update.type === "file") { return { type: MessageUpdateType.File, + name: "Unknown", sha: update.sha, // assume jpeg but could be any image. should be harmless mime: "image/jpeg", diff --git a/src/lib/migrations/routines/05-update-message-files.ts b/src/lib/migrations/routines/05-update-message-files.ts new file mode 100644 index 00000000000..0a91cb86aa3 --- /dev/null +++ b/src/lib/migrations/routines/05-update-message-files.ts @@ -0,0 +1,56 @@ +import { ObjectId, type WithId } from "mongodb"; +import { collections } from "$lib/server/database"; + +import type { Migration } from "."; +import type { Conversation } from "$lib/types/Conversation"; +import type { MessageFile } from "$lib/types/Message"; + +const updateMessageFiles: Migration = { + _id: new ObjectId("5f9f5f5f5f5f5f5f5f5f5f5f"), + name: "Convert message files to the new schema", + up: async () => { + const allConversations = collections.conversations.find({}, { projection: { messages: 1 } }); + + let conversation: WithId> | null = null; + while ((conversation = await allConversations.tryNext())) { + const messages = conversation.messages.map((message) => { + const files = (message.files as string[] | undefined)?.map((file) => { + // File is already in the new format + if (typeof file !== "string") return file; + + // File was a hash pointing to a file in the bucket + if (file.length === 64) { + return { + type: "hash", + name: "unknown.jpg", + value: file, + mime: "image/jpeg", + }; + } + // File was a base64 string + else { + return { + type: "base64", + name: "unknown.jpg", + value: file, + mime: "image/jpeg", + }; + } + }); + + return { + ...message, + files, + }; + }); + + // Set the new messages array + await collections.conversations.updateOne({ _id: conversation._id }, { $set: { messages } }); + } + + return true; + }, + runEveryTime: false, +}; + +export default updateMessageFiles; diff --git a/src/lib/migrations/routines/index.ts b/src/lib/migrations/routines/index.ts index 5ae35a20373..78bb34d1ad9 100644 --- a/src/lib/migrations/routines/index.ts +++ b/src/lib/migrations/routines/index.ts @@ -5,6 +5,7 @@ import updateAssistantsModels from "./02-update-assistants-models"; import type { Database } from "$lib/server/database"; import addToolsToSettings from "./03-add-tools-in-settings"; import updateMessageUpdates from "./04-update-message-updates"; +import updateMessageFiles from "./05-update-message-files"; export interface Migration { _id: ObjectId; @@ -21,4 +22,5 @@ export const migrations: Migration[] = [ updateAssistantsModels, addToolsToSettings, updateMessageUpdates, + updateMessageFiles, ]; diff --git a/src/lib/server/files/downloadFile.ts b/src/lib/server/files/downloadFile.ts index dac5bbda848..37776452b74 100644 --- a/src/lib/server/files/downloadFile.ts +++ b/src/lib/server/files/downloadFile.ts @@ -9,29 +9,26 @@ export async function downloadFile( convId: Conversation["_id"] | SharedConversation["_id"] ): Promise { const fileId = collections.bucket.find({ filename: `${convId.toString()}-${sha256}` }); - let mime = ""; - const buffer = await fileId.next().then(async (file) => { - if (!file) { - throw error(404, "File not found"); - } - if (file.metadata?.conversation !== convId.toString()) { - throw error(403, "You don't have access to this file."); - } + const file = await fileId.next(); + if (!file) { + throw error(404, "File not found"); + } + if (file.metadata?.conversation !== convId.toString()) { + throw error(403, "You don't have access to this file."); + } - mime = file.metadata?.mime; + const mime = file.metadata?.mime; + const name = file.filename; - const fileStream = collections.bucket.openDownloadStream(file._id); + const fileStream = collections.bucket.openDownloadStream(file._id); - const fileBuffer = await new Promise((resolve, reject) => { - const chunks: Uint8Array[] = []; - fileStream.on("data", (chunk) => chunks.push(chunk)); - fileStream.on("error", reject); - fileStream.on("end", () => resolve(Buffer.concat(chunks))); - }); - - return fileBuffer; + const buffer = await new Promise((resolve, reject) => { + const chunks: Uint8Array[] = []; + fileStream.on("data", (chunk) => chunks.push(chunk)); + fileStream.on("error", reject); + fileStream.on("end", () => resolve(Buffer.concat(chunks))); }); - return { type: "base64", value: buffer.toString("base64"), mime }; + return { type: "base64", name, value: buffer.toString("base64"), mime }; } diff --git a/src/lib/server/files/uploadFile.ts b/src/lib/server/files/uploadFile.ts index 5191bcb23e6..97b335beaf0 100644 --- a/src/lib/server/files/uploadFile.ts +++ b/src/lib/server/files/uploadFile.ts @@ -4,7 +4,7 @@ import { sha256 } from "$lib/utils/sha256"; import { fileTypeFromBuffer } from "file-type"; import { collections } from "$lib/server/database"; -export async function uploadFile(file: Blob | File, conv: Conversation): Promise { +export async function uploadFile(file: File, conv: Conversation): Promise { const sha = await sha256(await file.text()); const buffer = await file.arrayBuffer(); @@ -20,7 +20,9 @@ export async function uploadFile(file: Blob | File, conv: Conversation): Promise // only return the filename when upload throws a finish event or a 20s time out occurs return new Promise((resolve, reject) => { - upload.once("finish", () => resolve({ type: "hash", value: sha, mime: file.type })); + upload.once("finish", () => + resolve({ type: "hash", value: sha, mime: file.type, name: file.name }) + ); upload.once("error", reject); setTimeout(() => reject(new Error("Upload timed out")), 20_000); }); diff --git a/src/lib/server/textGeneration/generate.ts b/src/lib/server/textGeneration/generate.ts index 65e77e94f96..97a38b8bced 100644 --- a/src/lib/server/textGeneration/generate.ts +++ b/src/lib/server/textGeneration/generate.ts @@ -2,9 +2,12 @@ import type { ToolResult } from "$lib/types/Tool"; import { MessageUpdateType, type MessageUpdate } from "$lib/types/MessageUpdate"; import { AbortedGenerations } from "../abortedGenerations"; import type { TextGenerationContext } from "./types"; +import type { EndpointMessage } from "../endpoints/endpoints"; + +type GenerateContext = Omit & { messages: EndpointMessage[] }; export async function* generate( - { model, endpoint, conv, messages, assistant, isContinue, promptedAt }: TextGenerationContext, + { model, endpoint, conv, messages, assistant, isContinue, promptedAt }: GenerateContext, toolResults: ToolResult[], preprompt?: string ): AsyncIterable { diff --git a/src/lib/server/textGeneration/title.ts b/src/lib/server/textGeneration/title.ts index a17bdbec0ed..3d3359d0dc4 100644 --- a/src/lib/server/textGeneration/title.ts +++ b/src/lib/server/textGeneration/title.ts @@ -1,6 +1,6 @@ import { env } from "$env/dynamic/private"; import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint"; -import type { EndpointMessage } from "./endpoints/endpoints"; +import type { EndpointMessage } from "../endpoints/endpoints"; import { logger } from "$lib/server/logger"; import { MessageUpdateType, type MessageUpdate } from "$lib/types/MessageUpdate"; import type { Conversation } from "$lib/types/Conversation"; diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts index 6e6aef9cba0..6ab3559b413 100644 --- a/src/lib/server/textGeneration/tools.ts +++ b/src/lib/server/textGeneration/tools.ts @@ -14,6 +14,16 @@ import { allTools } from "../tools"; import directlyAnswer from "../tools/directlyAnswer"; import websearch from "../tools/web/search"; import { z } from "zod"; +import { logger } from "../logger"; +import { toolHasName } from "../tools/utils"; +import type { MessageFile } from "$lib/types/Message"; + +function makeFilesPrompt(files: MessageFile[]): string { + if (files.length === 0) return ""; + + const stringifiedFiles = files.map((file) => ` - ${file.name} (${file.mime})`).join("\n"); + return `${files.length} file${files.length === 1 ? "" : "s"} attached:\n${stringifiedFiles}`; +} export function pickTools( toolsPreference: Record, @@ -36,6 +46,10 @@ export async function* runTools( ): AsyncGenerator { const calls: ToolCall[] = []; + // inform the model if there are files attached + const userMessage = messages.find((message) => message.from === "user"); + preprompt = `${preprompt ?? ""}\n${makeFilesPrompt(userMessage?.files ?? [])}`.trim(); + // do the function calling bits here for await (const output of await endpoint({ messages, @@ -58,7 +72,9 @@ export async function* runTools( // grab only the capture group from the regex match for (const [, block] of codeBlocks) { try { - calls.push(...JSON5.parse(block).filter(isExternalToolCall).map(externalToToolCall)); + calls.push( + ...JSON5.parse(block).filter(isExternalToolCall).map(externalToToolCall).filter(Boolean) + ); } catch (cause) { // error parsing the calls yield { @@ -82,7 +98,7 @@ export async function* runTools( continue; } // Special case for directly_answer tool where we ignore - if (tool.name === "directly_answer") continue; + if (toolHasName("directly_answer", tool)) continue; yield { type: MessageUpdateType.Tool, @@ -126,28 +142,34 @@ function isExternalToolCall(call: unknown): call is ExternalToolCall { return externalToolCall.safeParse(call).success; } -function externalToToolCall(call: ExternalToolCall): ToolCall { - const tool = allTools.find((el) => el.name === call.tool_name); +function externalToToolCall(call: ExternalToolCall): ToolCall | undefined { + // Convert - to _ since some models insist on using _ instead of - + const tool = allTools.find((tool) => toolHasName(call.tool_name, tool)); if (tool === undefined) { - throw new Error(`Could not find tool ${call.tool_name}`); + logger.debug(`Model requested tool that does not exist: "${call.tool_name}". Skipping tool...`); + return; } - const parametersWithDefaults = Object.fromEntries( - Object.entries(tool.parameterDefinitions).map(([key, definition]) => { - const value = call.parameters[key]; + const parametersWithDefaults: Record = {}; - // Required so ensure it's there - if (definition.required) { - if (value === undefined) { - throw new Error(`Missing required parameter ${key} for tool ${call.tool_name}`); - } - return [key, value]; + for (const [key, definition] of Object.entries(tool.parameterDefinitions)) { + const value = call.parameters[key]; + + // Required so ensure it's there, otherwise return undefined + if (definition.required) { + if (value === undefined) { + logger.debug( + `Model requested tool "${call.tool_name}" but was missing required parameter "${key}". Skipping tool...` + ); + return; } + parametersWithDefaults[key] = value; + continue; + } - // Optional so use default if not there - return [key, value ?? definition.default]; - }) - ); + // Optional so use default if not there + parametersWithDefaults[key] = value ?? definition.default; + } return { name: call.tool_name, diff --git a/src/lib/server/tools/calculator.ts b/src/lib/server/tools/calculator.ts index a36d7dca0a5..d655c0d4e52 100644 --- a/src/lib/server/tools/calculator.ts +++ b/src/lib/server/tools/calculator.ts @@ -7,7 +7,7 @@ const calculator: BackendTool = { description: "A simple calculator, takes a string containing a mathematical expression and returns the answer. Only supports +, -, *, and /, as well as parenthesis ().", isOnByDefault: true, - parameter_definitions: { + parameterDefinitions: { equation: { description: "The formula to evaluate. EXACTLY as you would plug into a calculator. No words, no letters, only numbers and operators. Letters will make the tool crash.", @@ -17,7 +17,7 @@ const calculator: BackendTool = { }, async *call(params) { try { - const blocks = params.equation.split("\n"); + const blocks = String(params.equation).split("\n"); const query = blocks[blocks.length - 1].replace(/[^-()\d/*+.]/g, ""); return { diff --git a/src/lib/server/tools/directlyAnswer.ts b/src/lib/server/tools/directlyAnswer.ts index 4b911a8ec5b..a0a74c20fef 100644 --- a/src/lib/server/tools/directlyAnswer.ts +++ b/src/lib/server/tools/directlyAnswer.ts @@ -7,7 +7,7 @@ const directlyAnswer: BackendTool = { isHidden: true, isLocked: true, description: "Use this tool to let the user know you wish to answer directly", - parameter_definitions: {}, + parameterDefinitions: {}, async *call() { return { status: ToolResultStatus.Success, diff --git a/src/lib/server/tools/images/editing.ts b/src/lib/server/tools/images/editing.ts index 3db045b985a..7d431b5d50e 100644 --- a/src/lib/server/tools/images/editing.ts +++ b/src/lib/server/tools/images/editing.ts @@ -47,17 +47,27 @@ const imageEditing: BackendTool = { "run_edit", [ image, - prompt, + String(prompt), "", // negative prompt 7, // guidance scale 20, // steps ] ); + const outputImage = await fetch(outputs[0].url) .then((res) => res.blob()) - .then((blob) => uploadFile(blob, conv)); + .then( + (blob) => + new File([blob], `${prompt}.${blob.type.split("/")[1] ?? "png"}`, { type: blob.type }) + ) + .then((file) => uploadFile(file, conv)); - yield { type: MessageUpdateType.File, sha: outputImage.value, mime: outputImage.mime }; + yield { + type: MessageUpdateType.File, + name: outputImage.name, + sha: outputImage.value, + mime: outputImage.mime, + }; return { status: ToolResultStatus.Success, diff --git a/src/lib/server/tools/images/generation.ts b/src/lib/server/tools/images/generation.ts index a4e654d183c..1bb24fde123 100644 --- a/src/lib/server/tools/images/generation.ts +++ b/src/lib/server/tools/images/generation.ts @@ -52,7 +52,7 @@ const imageGeneration: BackendTool = { Number(numberOfImages), // number (numeric value between 1 and 8) in 'Number of Images' Slider component 512, // number in 'Image Height' Number component 512, // number in 'Image Width' Number component - prompt, // prompt + String(prompt), // prompt Math.floor(Math.random() * 1000), // seed random ] ); @@ -60,12 +60,21 @@ const imageGeneration: BackendTool = { outputs[0].map((output) => fetch(output.image.url) .then((res) => res.blob()) - .then((blob) => uploadFile(blob, conv)) + .then( + (blob) => + new File([blob], `${prompt}.${blob.type.split("/")[1] ?? "png"}`, { type: blob.type }) + ) + .then((file) => uploadFile(file, conv)) ) ); for (const image of imageBlobs) { - yield { type: MessageUpdateType.File, sha: image.value, mime: image.mime }; + yield { + type: MessageUpdateType.File, + name: image.name, + sha: image.value, + mime: image.mime, + }; } return { diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index 1658ebeabbb..b00421e14bc 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -21,7 +21,7 @@ interface BackendToolContext { export interface BackendTool extends Tool { call( - params: Record, + params: Record, context: BackendToolContext ): AsyncGenerator, undefined>; } diff --git a/src/lib/server/tools/utils.ts b/src/lib/server/tools/utils.ts index 027c77c256d..a0363055633 100644 --- a/src/lib/server/tools/utils.ts +++ b/src/lib/server/tools/utils.ts @@ -1,5 +1,6 @@ import { env } from "$env/dynamic/private"; import { Client } from "@gradio/client"; +import type { BackendTool } from "."; export type GradioImage = { path: string; @@ -24,3 +25,11 @@ export async function callSpace (res as unknown as GradioResponse).data as TOutput); } + +/** + * Checks if a tool's name equals a value. Replaces all hyphens with underscores before comparison + * since some models return underscores even when hyphens are used in the request. + **/ +export function toolHasName(name: string, tool: BackendTool): boolean { + return tool.name.replaceAll("-", "_") === name.replaceAll("-", "_"); +} diff --git a/src/lib/server/tools/web/search.ts b/src/lib/server/tools/web/search.ts index 19b7fa5308f..da5397e3d39 100644 --- a/src/lib/server/tools/web/search.ts +++ b/src/lib/server/tools/web/search.ts @@ -8,7 +8,7 @@ const websearch: BackendTool = { isOnByDefault: true, description: "Use this tool to search web pages for answers that will help answer the user's query. Only use this tool if you need specific resources from the internet.", - parameter_definitions: { + parameterDefinitions: { query: { required: true, type: "string", @@ -17,7 +17,7 @@ const websearch: BackendTool = { }, }, async *call({ query }, { conv, assistant, messages }) { - const webSearchToolResults = yield* runWebSearch(conv, messages, assistant?.rag, query); + const webSearchToolResults = yield* runWebSearch(conv, messages, assistant?.rag, String(query)); const chunks = webSearchToolResults?.contextSources .map(({ context }) => context) .join("\n------------\n"); diff --git a/src/lib/server/tools/web/url.ts b/src/lib/server/tools/web/url.ts index 9cb8a25ce9b..e5f0adb7e7e 100644 --- a/src/lib/server/tools/web/url.ts +++ b/src/lib/server/tools/web/url.ts @@ -8,7 +8,7 @@ const fetchUrl: BackendTool = { displayName: "URL Fetcher", description: "A tool that can be used to fetch an URL and return the content directly.", isOnByDefault: true, - parameter_definitions: { + parameterDefinitions: { url: { description: "The url that should be fetched.", type: "str", @@ -16,7 +16,7 @@ const fetchUrl: BackendTool = { }, }, async *call(params) { - const blocks = params.url.split("\n"); + const blocks = String(params.url).split("\n"); const url = blocks[blocks.length - 1]; const { title, markdownTree } = await scrapeUrl(url, Infinity); diff --git a/src/lib/types/Message.ts b/src/lib/types/Message.ts index 6791164febf..2f108d7f585 100644 --- a/src/lib/types/Message.ts +++ b/src/lib/types/Message.ts @@ -27,6 +27,7 @@ export type Message = Partial & { export type MessageFile = { type: "hash" | "base64"; + name: string; value: string; mime: string; }; diff --git a/src/lib/types/MessageUpdate.ts b/src/lib/types/MessageUpdate.ts index 27b6e7a2d7b..c21c2680db2 100644 --- a/src/lib/types/MessageUpdate.ts +++ b/src/lib/types/MessageUpdate.ts @@ -100,8 +100,9 @@ export interface MessageStreamUpdate { } export interface MessageFileUpdate { type: MessageUpdateType.File; + name: string; sha: string; - mime: string; + mime: string; } export interface MessageFinalAnswerUpdate { type: MessageUpdateType.FinalAnswer; diff --git a/src/lib/types/Tool.ts b/src/lib/types/Tool.ts index 760a1a7cb58..e825da514db 100644 --- a/src/lib/types/Tool.ts +++ b/src/lib/types/Tool.ts @@ -47,5 +47,5 @@ export type ToolResult = ToolResultSuccess | ToolResultError; export interface ToolCall { name: string; - parameters: Record; + parameters: Record; } diff --git a/src/routes/conversation/[id]/+page.svelte b/src/routes/conversation/[id]/+page.svelte index b71f898ac69..a9c762e5995 100644 --- a/src/routes/conversation/[id]/+page.svelte +++ b/src/routes/conversation/[id]/+page.svelte @@ -82,7 +82,12 @@ const base64Files = await Promise.all( (files ?? []).map((file) => - file2base64(file).then((value) => ({ type: "base64" as const, value, mime: file.type })) + file2base64(file).then((value) => ({ + type: "base64" as const, + value, + mime: file.type, + name: file.name, + })) ) ); @@ -251,7 +256,7 @@ } else if (update.type === MessageUpdateType.File) { messageToWriteTo.files = [ ...(messageToWriteTo.files ?? []), - { type: "hash", value: update.sha, mime: update.mime }, + { type: "hash", value: update.sha, mime: update.mime, name: update.name }, ]; messages = [...messages]; } diff --git a/src/routes/conversation/[id]/+server.ts b/src/routes/conversation/[id]/+server.ts index 9612f063933..6b334b71871 100644 --- a/src/routes/conversation/[id]/+server.ts +++ b/src/routes/conversation/[id]/+server.ts @@ -152,6 +152,7 @@ export async function POST({ request, locals, params, getClientAddress }) { z.array( z.object({ type: z.literal("base64").or(z.literal("hash")), + name: z.string(), value: z.string(), mime: z.string(), }) @@ -173,7 +174,7 @@ export async function POST({ request, locals, params, getClientAddress }) { ?.filter((file) => file.type !== "hash") .map((file) => { const blob = Buffer.from(file.value, "base64"); - return new File([blob], "file", { type: file.mime }); + return new File([blob], file.name, { type: file.mime }); }) ?? []; // check sizes @@ -324,7 +325,10 @@ export async function POST({ request, locals, params, getClientAddress }) { // Add file else if (event.type === MessageUpdateType.File) { - messageToWriteTo.files = [...(messageToWriteTo.files ?? []), { type: 'hash', value: event.sha, mime: event.mime }]; + messageToWriteTo.files = [ + ...(messageToWriteTo.files ?? []), + { type: "hash", name: event.name, value: event.sha, mime: event.mime }, + ]; } // Set web search From d38ac95da3f353b2c40e1740a80afa1a8c0e562d Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Mon, 20 May 2024 21:05:24 -0400 Subject: [PATCH 53/82] feat: multiple files and tool file indices --- .../components/OpenWebSearchResults.svelte | 2 +- src/lib/components/UploadBtn.svelte | 17 ++- src/lib/components/chat/ChatMessage.svelte | 18 +-- src/lib/components/chat/ChatWindow.svelte | 4 +- src/lib/components/chat/UploadedFile.svelte | 8 +- src/lib/server/models.ts | 42 +++---- src/lib/server/textGeneration/generate.ts | 1 + src/lib/server/textGeneration/index.ts | 4 +- src/lib/server/textGeneration/tools.ts | 105 ++++++++++-------- src/lib/server/tools/images/editing.ts | 33 ++++-- src/lib/server/tools/index.ts | 2 +- src/lib/server/tools/parse/pdf.ts | 29 ++++- src/lib/types/MessageUpdate.ts | 22 ++-- src/lib/types/Model.ts | 2 +- src/routes/+layout.server.ts | 2 +- .../message/[messageId]/prompt/+server.ts | 2 +- 16 files changed, 173 insertions(+), 120 deletions(-) diff --git a/src/lib/components/OpenWebSearchResults.svelte b/src/lib/components/OpenWebSearchResults.svelte index ef0295a7e1a..c4f01555131 100644 --- a/src/lib/components/OpenWebSearchResults.svelte +++ b/src/lib/components/OpenWebSearchResults.svelte @@ -15,7 +15,7 @@ $: sources = webSearchMessages.find(isMessageWebSearchSourcesUpdate)?.sources; $: lastMessage = webSearchMessages .filter((update) => update.subtype !== MessageWebSearchUpdateType.Sources) - .slice(-1)[0]; + .at(-1) as MessageWebSearchUpdate; $: loading = !sources && lastMessage.subtype !== MessageWebSearchUpdateType.Error; diff --git a/src/lib/components/UploadBtn.svelte b/src/lib/components/UploadBtn.svelte index 11bd2044881..62d0fbff94c 100644 --- a/src/lib/components/UploadBtn.svelte +++ b/src/lib/components/UploadBtn.svelte @@ -3,21 +3,26 @@ export let classNames = ""; export let files: File[]; - let filelist: FileList; - $: if (filelist) { - files = Array.from(filelist); - } + /** + * Due to a bug with Svelte, we cannot use bind:files with multiple + * So we use this workaround + **/ + let onFileChange = (e: Event) => { + if (!e.target) return; + const target = e.target as HTMLInputElement; + files = [...files, ...(target.files ?? [])]; + }; diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index b8381918bef..c0f9c4524a2 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -145,7 +145,7 @@ // filter all updates with type === "tool" then group them by uuid field - $: tools = message.updates + $: toolUpdates = message.updates ?.filter(({ type }) => type === "tool") ?.reduce((acc, update) => { if (update.type !== "tool") { @@ -209,7 +209,7 @@ const availableTools: ToolFront[] = $page.data.tools; -{#if modalImageToShow !== null} +{#if modalImageToShow} (modalImageToShow = null)}> {#if modalImageToShow.type === "hash"} @@ -252,7 +252,7 @@
      - {#if message.files && message.files.length > 0} + {#if message.files?.length}
      {#each message.files as file} @@ -282,8 +282,8 @@ /> {/if} - {#if tools} - {#each Object.values(tools) as tool} + {#if toolUpdates} + {#each Object.values(toolUpdates) as tool} {#if tool.length > 0} {@const toolName = tool.find(isMessageToolCallUpdate)?.call.name} {@const toolDone = tool.some(isMessageToolResultUpdate)} @@ -384,7 +384,7 @@
      {/if}
      - {#if !loading && (message.content || tools)} + {#if !loading && (message.content || toolUpdates)}
      - {#if message.files && message.files.length > 0} -
      +
      + {#if message.files?.length} +
      {#each message.files as file} {#if file.mime.startsWith("image/")} diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index fe0281232c3..fc80868b4a0 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -253,7 +253,7 @@ class="relative min-h-[calc(2rem+theme(spacing[3.5])*2)] min-w-[60px] break-words rounded-2xl border border-gray-100 bg-gradient-to-br from-gray-50 px-5 py-3.5 text-gray-600 prose-pre:my-2 dark:border-gray-800 dark:from-gray-800/40 dark:text-gray-300" > {#if message.files?.length} -
      +
      {#each message.files as file}
    {#each tool as toolUpdate} {#if toolUpdate.subtype === MessageToolUpdateType.Call} -
    +

    Parameters

    diff --git a/src/lib/components/chat/ChatWindow.svelte b/src/lib/components/chat/ChatWindow.svelte index c8cd67635bf..af3b0bafd38 100644 --- a/src/lib/components/chat/ChatWindow.svelte +++ b/src/lib/components/chat/ChatWindow.svelte @@ -140,7 +140,9 @@ use:snapScrollToBottom={messages.length ? [...messages] : false} bind:this={chatContainer} > -
    +
    {#if $page.data?.assistant && !!messages.length} 0} -
    +
    {#if file.mime.startsWith("image/")} -
    +
    {:else}
    - -
    -
    - {file.name} -
    -
    - {file.mime.split("/")[1].toUpperCase()} -
    +
    +
    +
    +
    + {file.name} +
    +
    {file.mime.split("/")[1].toUpperCase()}
    +
    {/if} {#if canClose} {/if}
    diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index a29143bcbf2..89c2e336e5e 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -28,10 +28,10 @@ export interface BackendTool extends Tool { export const allTools: BackendTool[] = [ directlyAnswer, - calculator, + websearch, + imageGeneration, fetchUrl, - parsePdf, imageEditing, - imageGeneration, - websearch, + parsePdf, + calculator, ]; diff --git a/src/lib/server/websearch/scrape/playwright.ts b/src/lib/server/websearch/scrape/playwright.ts index 2994a80098a..2046df0de31 100644 --- a/src/lib/server/websearch/scrape/playwright.ts +++ b/src/lib/server/websearch/scrape/playwright.ts @@ -52,7 +52,7 @@ export async function loadPage(url: string): Promise<{ res?: Response; page: Pag const page = await ctx.newPage(); await blocker.enableBlockingInPage(page); - const res = await page.goto(url, { waitUntil: "load", timeout: 2000 }).catch(() => { + const res = await page.goto(url, { waitUntil: "load", timeout: 3500 }).catch(() => { console.warn(`Failed to load page within 2s: ${url}`); }); From 86df89de7acb843444b2630ab79c2b3d00fe91b6 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 22 May 2024 13:38:30 -0400 Subject: [PATCH 67/82] feat: update pdf to markdown schema --- src/lib/server/tools/parse/pdf.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/lib/server/tools/parse/pdf.ts b/src/lib/server/tools/parse/pdf.ts index 3fd04ff5e60..f63bcf9e122 100644 --- a/src/lib/server/tools/parse/pdf.ts +++ b/src/lib/server/tools/parse/pdf.ts @@ -1,14 +1,10 @@ import type { BackendTool } from ".."; import { ToolResultStatus } from "$lib/types/Tool"; -import { callSpace, type GradioImage } from "../utils"; +import { callSpace } from "../utils"; import { downloadFile } from "$lib/server/files/downloadFile"; -type PdfParserInput = [Blob /* pdf */, boolean /* include images */]; -type PdfParserOutput = [ - string /* markdown */, - Record /* metadata */, - GradioImage[] | undefined /* extracted images if enabled */ -]; +type PdfParserInput = [Blob /* pdf */]; +type PdfParserOutput = [string /* markdown */, Record /* metadata */]; const pdfParser: BackendTool = { name: "pdf-parser", @@ -53,7 +49,7 @@ const pdfParser: BackendTool = { const outputs = await callSpace( "huggingchat/pdf-to-markdown", "predict", - [pdf, false] + [pdf] ); const outputMarkdown = outputs[0]; From 251e7ccb58b6f3dc1eaf3cacf16e1e52b75b2172 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 22 May 2024 13:43:34 -0400 Subject: [PATCH 68/82] fix: uploaded file container width --- src/lib/components/chat/UploadedFile.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/components/chat/UploadedFile.svelte b/src/lib/components/chat/UploadedFile.svelte index 546bf8c4c22..0cb522ec451 100644 --- a/src/lib/components/chat/UploadedFile.svelte +++ b/src/lib/components/chat/UploadedFile.svelte @@ -27,7 +27,7 @@
    {:else}
    Date: Wed, 22 May 2024 13:47:24 -0400 Subject: [PATCH 69/82] fix: tool name check --- src/lib/server/textGeneration/tools.ts | 2 +- src/lib/server/tools/utils.ts | 9 +-------- src/lib/utils/tools.ts | 9 +++++++++ 3 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 src/lib/utils/tools.ts diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts index 1c23b2ae158..cffe26de675 100644 --- a/src/lib/server/textGeneration/tools.ts +++ b/src/lib/server/textGeneration/tools.ts @@ -53,7 +53,7 @@ async function* runTool( ): AsyncGenerator { const uuid = uuidV4(); - const tool = tools.find((el) => el.name === call.name); + const tool = tools.find((el) => toolHasName(call.name, el)); if (!tool) { return { call, status: ToolResultStatus.Error, message: `Could not find tool "${call.name}"` }; } diff --git a/src/lib/server/tools/utils.ts b/src/lib/server/tools/utils.ts index b0c0d7bfe34..24f9d07381d 100644 --- a/src/lib/server/tools/utils.ts +++ b/src/lib/server/tools/utils.ts @@ -1,6 +1,5 @@ import { env } from "$env/dynamic/private"; import { Client } from "@gradio/client"; -import type { BackendTool } from "."; export type GradioImage = { path: string; @@ -27,10 +26,4 @@ export async function callSpace (res as unknown as GradioResponse).data as TOutput); } -/** - * Checks if a tool's name equals a value. Replaces all hyphens with underscores before comparison - * since some models return underscores even when hyphens are used in the request. - **/ -export function toolHasName(name: string, tool: BackendTool): boolean { - return tool.name.replaceAll("-", "_") === name.replaceAll("-", "_"); -} +export { toolHasName } from "$lib/utils/tools"; diff --git a/src/lib/utils/tools.ts b/src/lib/utils/tools.ts new file mode 100644 index 00000000000..9da745fe2a0 --- /dev/null +++ b/src/lib/utils/tools.ts @@ -0,0 +1,9 @@ +import type { BackendTool } from "$lib/server/tools"; + +/** + * Checks if a tool's name equals a value. Replaces all hyphens with underscores before comparison + * since some models return underscores even when hyphens are used in the request. + **/ +export function toolHasName(name: string, tool: BackendTool): boolean { + return tool.name.replaceAll("-", "_") === name.replaceAll("-", "_"); +} From 456e696aa66bee6641c800034f152ed2871c7047 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 22 May 2024 16:22:50 -0400 Subject: [PATCH 70/82] fix: cohere endpoint --- src/lib/server/endpoints/cohere/endpointCohere.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib/server/endpoints/cohere/endpointCohere.ts b/src/lib/server/endpoints/cohere/endpointCohere.ts index e540c1e35fc..6afe593e2c1 100644 --- a/src/lib/server/endpoints/cohere/endpointCohere.ts +++ b/src/lib/server/endpoints/cohere/endpointCohere.ts @@ -6,6 +6,7 @@ import type { Cohere, CohereClient } from "cohere-ai"; import { buildPrompt } from "$lib/buildPrompt"; import { ToolResultStatus } from "$lib/types/Tool"; import { pipeline, Writable, Readable } from "node:stream"; +import { toolHasName } from "$lib/utils/tools"; export const endpointCohereParametersSchema = z.object({ weight: z.number().int().positive().default(1), @@ -36,6 +37,12 @@ export async function endpointCohere( system = messages[0].content; } + // Tools must use [A-z_] for their names and directly_answer is banned + // It's safe to convert the tool names because we treat - and _ the same + tools = tools + ?.filter((tool) => !toolHasName("directly_answer", tool)) + .map((tool) => ({ ...tool, name: tool.name.replaceAll("-", "_") })); + const parameters = { ...model.parameters, ...generateSettings }; return (async function* () { From 6139400892919b4d7612bb62123c596a712e340a Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 22 May 2024 19:57:49 -0400 Subject: [PATCH 71/82] fix: use most recent message for files --- src/lib/server/textGeneration/tools.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts index cffe26de675..dbe8f3dee64 100644 --- a/src/lib/server/textGeneration/tools.ts +++ b/src/lib/server/textGeneration/tools.ts @@ -99,7 +99,7 @@ export async function* runTools( const calls: ToolCall[] = []; // inform the model if there are files attached - const userMessage = messages.find((message) => message.from === "user"); + const userMessage = messages.findLast((message) => message.from === "user"); preprompt = `${preprompt ?? ""}\n${makeFilesPrompt(userMessage?.files ?? [])}`.trim(); // do the function calling bits here From 1cd068356723b4a8a21faf4b885676c3e54ff8ca Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 22 May 2024 20:09:10 -0400 Subject: [PATCH 72/82] feat: allow tools to access previous files --- src/lib/components/chat/ChatMessage.svelte | 3 ++- src/lib/server/textGeneration/tools.ts | 23 +++++++++++++--------- src/lib/server/tools/images/editing.ts | 14 +++++++++---- src/lib/server/tools/images/generation.ts | 2 +- src/lib/server/tools/parse/pdf.ts | 14 +++++++++---- src/lib/utils/tools.ts | 4 ++-- 6 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index fc80868b4a0..1f0e1e31a18 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -34,6 +34,7 @@ import { base } from "$app/paths"; import { useConvTreeStore } from "$lib/stores/convTree"; import Modal from "../Modal.svelte"; + import { toolHasName } from "$lib/utils/tools"; function sanitizeMd(md: string) { let ret = md @@ -322,7 +323,7 @@ {toolDone ? "Called" : "Calling"} tool {availableTools.find((el) => el.name === toolName)?.displayName}{availableTools.find((el) => toolHasName(toolName, el))?.displayName} diff --git a/src/lib/server/textGeneration/tools.ts b/src/lib/server/textGeneration/tools.ts index dbe8f3dee64..6b4e053890f 100644 --- a/src/lib/server/textGeneration/tools.ts +++ b/src/lib/server/textGeneration/tools.ts @@ -19,17 +19,18 @@ import { toolHasName } from "../tools/utils"; import type { MessageFile } from "$lib/types/Message"; import { mergeAsyncGenerators } from "$lib/utils/mergeAsyncGenerators"; -function makeFilesPrompt(files: MessageFile[]): string { +function makeFilesPrompt(files: MessageFile[], fileMessageIndex: number): string { if (files.length === 0) { return "The user has not uploaded any files. Do not attempt to use any tools that require files"; } const stringifiedFiles = files - .map((file, idx) => ` - fileIndex ${idx} | ${file.name} (${file.mime})`) + .map( + (file, fileIndex) => + ` - fileMessageIndex ${fileMessageIndex} | fileIndex ${fileIndex} | ${file.name} (${file.mime})` + ) .join("\n"); - return `The user attached ${files.length} file${ - files.length === 1 ? "" : "s" - }:\n${stringifiedFiles}`; + return `Attached ${files.length} file${files.length === 1 ? "" : "s"}:\n${stringifiedFiles}`; } export function pickTools( @@ -98,13 +99,17 @@ export async function* runTools( ): AsyncGenerator { const calls: ToolCall[] = []; - // inform the model if there are files attached - const userMessage = messages.findLast((message) => message.from === "user"); - preprompt = `${preprompt ?? ""}\n${makeFilesPrompt(userMessage?.files ?? [])}`.trim(); + const messagesWithFilesPrompt = messages.map((message, idx) => { + if (!message.files?.length) return message; + return { + ...message, + content: `${message.content}\n${makeFilesPrompt(message.files, idx)}`, + }; + }); // do the function calling bits here for await (const output of await endpoint({ - messages, + messages: messagesWithFilesPrompt, preprompt, generateSettings: assistant?.generateSettings, tools, diff --git a/src/lib/server/tools/images/editing.ts b/src/lib/server/tools/images/editing.ts index 7fd9a8c827e..6623030e4b4 100644 --- a/src/lib/server/tools/images/editing.ts +++ b/src/lib/server/tools/images/editing.ts @@ -15,7 +15,7 @@ type ImageEditingInput = [ type ImageEditingOutput = [GradioImage]; const imageEditing: BackendTool = { - name: "image-editing", + name: "image_editing", displayName: "Image Editing", description: "Use this tool to edit an image from a prompt.", isOnByDefault: true, @@ -26,18 +26,24 @@ const imageEditing: BackendTool = { type: "string", required: true, }, + fileMessageIndex: { + description: "Index of the message containing the file to edit", + type: "number", + required: true, + }, fileIndex: { description: "Index of the file to edit", type: "number", required: true, }, }, - async *call({ prompt, fileIndex }, { conv, messages }) { + async *call({ prompt, fileMessageIndex, fileIndex }, { conv, messages }) { prompt = String(prompt); + fileMessageIndex = Number(fileMessageIndex); fileIndex = Number(fileIndex); - const latestUserMessage = messages.findLast((message) => message.from === "user"); - const images = latestUserMessage?.files ?? []; + const message = messages[fileMessageIndex]; + const images = message?.files ?? []; if (!images || images.length === 0) { return { status: ToolResultStatus.Error, diff --git a/src/lib/server/tools/images/generation.ts b/src/lib/server/tools/images/generation.ts index 1bb24fde123..0e8d03dc9a6 100644 --- a/src/lib/server/tools/images/generation.ts +++ b/src/lib/server/tools/images/generation.ts @@ -14,7 +14,7 @@ type ImageGenerationInput = [ type ImageGenerationOutput = [{ image: GradioImage }[]]; const imageGeneration: BackendTool = { - name: "image-generation", + name: "image_generation", displayName: "Image Generation", description: "Use this tool to generate an image from a prompt.", isOnByDefault: true, diff --git a/src/lib/server/tools/parse/pdf.ts b/src/lib/server/tools/parse/pdf.ts index f63bcf9e122..74dc7bf7963 100644 --- a/src/lib/server/tools/parse/pdf.ts +++ b/src/lib/server/tools/parse/pdf.ts @@ -7,22 +7,28 @@ type PdfParserInput = [Blob /* pdf */]; type PdfParserOutput = [string /* markdown */, Record /* metadata */]; const pdfParser: BackendTool = { - name: "pdf-parser", + name: "pdf_parser", displayName: "PDF Parser", description: "Use this tool to parse a PDF and get its content in markdown format.", isOnByDefault: true, parameterDefinitions: { + fileMessageIndex: { + description: "Index of the message containing the pdf file to parse", + type: "number", + required: true, + }, fileIndex: { description: "Index of the pdf file to parse", type: "number", required: true, }, }, - async *call({ fileIndex }, { conv, messages }) { + async *call({ fileMessageIndex, fileIndex }, { conv, messages }) { + fileMessageIndex = Number(fileMessageIndex); fileIndex = Number(fileIndex); - const latestUserMessage = messages.findLast((message) => message.from === "user"); - const pdfs = latestUserMessage?.files ?? []; + const message = messages[fileMessageIndex]; + const pdfs = message?.files ?? []; if (!pdfs || pdfs.length === 0) { return { status: ToolResultStatus.Error, diff --git a/src/lib/utils/tools.ts b/src/lib/utils/tools.ts index 9da745fe2a0..3bd909b9042 100644 --- a/src/lib/utils/tools.ts +++ b/src/lib/utils/tools.ts @@ -1,9 +1,9 @@ -import type { BackendTool } from "$lib/server/tools"; +import type { Tool } from "$lib/types/Tool"; /** * Checks if a tool's name equals a value. Replaces all hyphens with underscores before comparison * since some models return underscores even when hyphens are used in the request. **/ -export function toolHasName(name: string, tool: BackendTool): boolean { +export function toolHasName(name: string, tool: Tool): boolean { return tool.name.replaceAll("-", "_") === name.replaceAll("-", "_"); } From 4a0a3a793f36499ad5553c61c18608d1ea9ca61a Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 22 May 2024 21:51:46 -0400 Subject: [PATCH 73/82] feat: support all file types on document parser --- src/lib/components/UploadBtn.svelte | 2 +- .../tools/{parse/pdf.ts => documentParser.ts} | 44 +++++++++---------- 2 files changed, 21 insertions(+), 25 deletions(-) rename src/lib/server/tools/{parse/pdf.ts => documentParser.ts} (53%) diff --git a/src/lib/components/UploadBtn.svelte b/src/lib/components/UploadBtn.svelte index f112c6a6d38..fdb1af4f513 100644 --- a/src/lib/components/UploadBtn.svelte +++ b/src/lib/components/UploadBtn.svelte @@ -22,7 +22,7 @@ class="absolute w-full cursor-pointer opacity-0" type="file" on:change={onFileChange} - accept="image/*,application/pdf" + accept="*/*" /> Upload file diff --git a/src/lib/server/tools/parse/pdf.ts b/src/lib/server/tools/documentParser.ts similarity index 53% rename from src/lib/server/tools/parse/pdf.ts rename to src/lib/server/tools/documentParser.ts index 74dc7bf7963..b5a0d7f8cc3 100644 --- a/src/lib/server/tools/parse/pdf.ts +++ b/src/lib/server/tools/documentParser.ts @@ -1,24 +1,24 @@ -import type { BackendTool } from ".."; +import type { BackendTool } from "."; import { ToolResultStatus } from "$lib/types/Tool"; -import { callSpace } from "../utils"; +import { callSpace } from "./utils"; import { downloadFile } from "$lib/server/files/downloadFile"; -type PdfParserInput = [Blob /* pdf */]; +type PdfParserInput = [Blob /* pdf */, string /* filename */]; type PdfParserOutput = [string /* markdown */, Record /* metadata */]; -const pdfParser: BackendTool = { - name: "pdf_parser", - displayName: "PDF Parser", - description: "Use this tool to parse a PDF and get its content in markdown format.", +const documentParser: BackendTool = { + name: "document_parser", + displayName: "Document Parser", + description: "Use this tool to parse any document and get its content in markdown format.", isOnByDefault: true, parameterDefinitions: { fileMessageIndex: { - description: "Index of the message containing the pdf file to parse", + description: "Index of the message containing the document file to parse", type: "number", required: true, }, fileIndex: { - description: "Index of the pdf file to parse", + description: "Index of the document file to parse", type: "number", required: true, }, @@ -28,43 +28,39 @@ const pdfParser: BackendTool = { fileIndex = Number(fileIndex); const message = messages[fileMessageIndex]; - const pdfs = message?.files ?? []; - if (!pdfs || pdfs.length === 0) { + const files = message?.files ?? []; + if (!files || files.length === 0) { return { status: ToolResultStatus.Error, message: "User did not provide a pdf to parse", }; } - if (fileIndex >= pdfs.length) { + if (fileIndex >= files.length) { return { status: ToolResultStatus.Error, message: "Model provided an invalid file index", }; } - if (!pdfs[fileIndex].mime.startsWith("application/pdf")) { - return { - status: ToolResultStatus.Error, - message: "Model provided a file index which is not a pdf", - }; - } - const pdf = await downloadFile(pdfs[fileIndex].value, conv._id) + const file = files[fileIndex]; + console.log(file); + const fileBlob = await downloadFile(files[fileIndex].value, conv._id) .then((file) => fetch(`data:${file.mime};base64,${file.value}`)) .then((res) => res.blob()); const outputs = await callSpace( - "huggingchat/pdf-to-markdown", + "huggingchat/document-parser", "predict", - [pdf] + [fileBlob, file.name] ); - const outputMarkdown = outputs[0]; + const documentMarkdown = outputs[0]; return { status: ToolResultStatus.Success, - outputs: [{ pdfParserMarkdown: outputMarkdown }], + outputs: [{ [file.name]: documentMarkdown }], display: false, }; }, }; -export default pdfParser; +export default documentParser; From 45798501f8a4dad0151447ae88d43a47da2af5d6 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 22 May 2024 21:54:47 -0400 Subject: [PATCH 74/82] fix: use document parser in default tools --- src/lib/server/tools/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/server/tools/index.ts b/src/lib/server/tools/index.ts index 89c2e336e5e..9751997e89c 100644 --- a/src/lib/server/tools/index.ts +++ b/src/lib/server/tools/index.ts @@ -8,7 +8,7 @@ import calculator from "./calculator"; import directlyAnswer from "./directlyAnswer"; import imageEditing from "./images/editing"; import imageGeneration from "./images/generation"; -import parsePdf from "./parse/pdf"; +import documentParser from "./documentParser"; import fetchUrl from "./web/url"; import websearch from "./web/search"; @@ -32,6 +32,6 @@ export const allTools: BackendTool[] = [ imageGeneration, fetchUrl, imageEditing, - parsePdf, + documentParser, calculator, ]; From ed4b66605da375af258da286e1f424f1cf106309 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Wed, 22 May 2024 21:55:12 -0400 Subject: [PATCH 75/82] feat: rename url fetcher internal name --- src/lib/server/tools/web/url.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/server/tools/web/url.ts b/src/lib/server/tools/web/url.ts index e5f0adb7e7e..b5bc087d593 100644 --- a/src/lib/server/tools/web/url.ts +++ b/src/lib/server/tools/web/url.ts @@ -4,7 +4,7 @@ import { ToolResultStatus } from "$lib/types/Tool"; import type { BackendTool } from ".."; const fetchUrl: BackendTool = { - name: "fetchUrl", + name: "fetch_url", displayName: "URL Fetcher", description: "A tool that can be used to fetch an URL and return the content directly.", isOnByDefault: true, From 4ac302ced4d9a8ddf34d021311a9018b12ad36a2 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Thu, 23 May 2024 10:23:46 -0400 Subject: [PATCH 76/82] feat: resolve type error --- src/lib/utils/tools.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/utils/tools.ts b/src/lib/utils/tools.ts index 3bd909b9042..36462fc74d9 100644 --- a/src/lib/utils/tools.ts +++ b/src/lib/utils/tools.ts @@ -4,6 +4,6 @@ import type { Tool } from "$lib/types/Tool"; * Checks if a tool's name equals a value. Replaces all hyphens with underscores before comparison * since some models return underscores even when hyphens are used in the request. **/ -export function toolHasName(name: string, tool: Tool): boolean { +export function toolHasName(name: string, tool: Pick): boolean { return tool.name.replaceAll("-", "_") === name.replaceAll("-", "_"); } From 5073b921913b642480e443465a2397581f3bd077 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Thu, 23 May 2024 10:25:07 -0400 Subject: [PATCH 77/82] feat: enable tools on prod command r+ --- chart/env/prod.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/chart/env/prod.yaml b/chart/env/prod.yaml index c1e47387611..fa4afc6821f 100644 --- a/chart/env/prod.yaml +++ b/chart/env/prod.yaml @@ -39,6 +39,7 @@ envVars: "modelUrl": "https://huggingface.co/CohereForAI/c4ai-command-r-plus", "websiteUrl": "https://docs.cohere.com/docs/command-r-plus", "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/cohere-logo.png", + "tools": true, "parameters": { "stop": ["<|END_OF_TURN_TOKEN|>"], "truncate" : 28672, From 03fcb59d86cf31d9fcce735e4a3e70c156cef89e Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Thu, 23 May 2024 10:29:48 -0400 Subject: [PATCH 78/82] feat: truncate document markdown --- src/lib/server/tools/documentParser.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lib/server/tools/documentParser.ts b/src/lib/server/tools/documentParser.ts index b5a0d7f8cc3..c0cb6d142a9 100644 --- a/src/lib/server/tools/documentParser.ts +++ b/src/lib/server/tools/documentParser.ts @@ -43,7 +43,6 @@ const documentParser: BackendTool = { } const file = files[fileIndex]; - console.log(file); const fileBlob = await downloadFile(files[fileIndex].value, conv._id) .then((file) => fetch(`data:${file.mime};base64,${file.value}`)) .then((res) => res.blob()); @@ -54,7 +53,11 @@ const documentParser: BackendTool = { [fileBlob, file.name] ); - const documentMarkdown = outputs[0]; + let documentMarkdown = outputs[0]; + // TODO: quick fix for avoiding context limit. eventually should use the tokenizer + if (documentMarkdown.length > 30_000) { + documentMarkdown = documentMarkdown.slice(0, 30_000) + "\n\n... (truncated)"; + } return { status: ToolResultStatus.Success, outputs: [{ [file.name]: documentMarkdown }], From 1e6e1cfdb1c10cdb14368812526563627f11c038 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Thu, 23 May 2024 10:31:07 -0400 Subject: [PATCH 79/82] feat: bump @huggingface/inference --- package-lock.json | 16 ++++++++++++---- package.json | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 734bee6544c..5df45d978d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,7 @@ "@cliqz/adblocker-playwright": "^1.27.2", "@gradio/client": "^0.19.4", "@huggingface/hub": "^0.5.1", - "@huggingface/inference": "^2.6.3", + "@huggingface/inference": "^2.7.0", "@iconify-json/bi": "^1.1.21", "@playwright/browser-chromium": "^1.43.1", "@resvg/resvg-js": "^2.6.2", @@ -752,9 +752,12 @@ } }, "node_modules/@huggingface/inference": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.6.3.tgz", - "integrity": "sha512-KK6xNrEldjjopiGqwaBCkA+4tEyuIz0qHsD5SVYaQ65HSlmBbntJieSw4NRWT+S5bK/Bf/GFCixW0NshAOcBqA==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.7.0.tgz", + "integrity": "sha512-u7Fn637Q3f7nUB1tajM4CgzhvoFQkOQr5W5Fm+2wT9ETgGoLBh25BLlYPTJRjAd2WY01s71v0lqAwNvHHCc3mg==", + "dependencies": { + "@huggingface/tasks": "^0.10.0" + }, "engines": { "node": ">=18" } @@ -767,6 +770,11 @@ "node": ">=18" } }, + "node_modules/@huggingface/tasks": { + "version": "0.10.8", + "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.10.8.tgz", + "integrity": "sha512-oIp9912FwByyyyxkB/CIiW203QxPeYzBG7dydLmqZdcW0ma0if1uklGumqkJ6y/rJlv7AR/jJ9wbee0Wl5YZTw==" + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", diff --git a/package.json b/package.json index 8fc03765daa..90dc4d4b446 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "@cliqz/adblocker-playwright": "^1.27.2", "@gradio/client": "^0.19.4", "@huggingface/hub": "^0.5.1", - "@huggingface/inference": "^2.6.3", + "@huggingface/inference": "^2.7.0", "@iconify-json/bi": "^1.1.21", "@playwright/browser-chromium": "^1.43.1", "@resvg/resvg-js": "^2.6.2", From abceba7cd9142379e3a5b6abcb3b6123588088a1 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Thu, 23 May 2024 10:35:44 -0400 Subject: [PATCH 80/82] feat: resolve type errors --- src/lib/server/endpoints/cohere/endpointCohere.ts | 4 ++-- src/lib/server/endpoints/endpoints.ts | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/lib/server/endpoints/cohere/endpointCohere.ts b/src/lib/server/endpoints/cohere/endpointCohere.ts index 6afe593e2c1..1b20565d1c6 100644 --- a/src/lib/server/endpoints/cohere/endpointCohere.ts +++ b/src/lib/server/endpoints/cohere/endpointCohere.ts @@ -4,7 +4,7 @@ import type { Endpoint } from "../endpoints"; import type { TextGenerationStreamOutput } from "@huggingface/inference"; import type { Cohere, CohereClient } from "cohere-ai"; import { buildPrompt } from "$lib/buildPrompt"; -import { ToolResultStatus } from "$lib/types/Tool"; +import { ToolResultStatus, type ToolCall } from "$lib/types/Tool"; import { pipeline, Writable, Readable } from "node:stream"; import { toolHasName } from "$lib/utils/tools"; @@ -128,7 +128,7 @@ export async function endpointCohere( text: "", logprob: 0, special: true, - toolCalls: output.toolCalls, + toolCalls: output.toolCalls as ToolCall[], }, generated_text: null, details: null, diff --git a/src/lib/server/endpoints/endpoints.ts b/src/lib/server/endpoints/endpoints.ts index 26e0193ba75..e2970d57df1 100644 --- a/src/lib/server/endpoints/endpoints.ts +++ b/src/lib/server/endpoints/endpoints.ts @@ -1,6 +1,6 @@ import type { Conversation } from "$lib/types/Conversation"; import type { Message } from "$lib/types/Message"; -import type { TextGenerationStreamOutput } from "@huggingface/inference"; +import type { TextGenerationStreamOutput, TextGenerationStreamToken } from "@huggingface/inference"; import { endpointTgi, endpointTgiParametersSchema } from "./tgi/endpointTgi"; import { z } from "zod"; import endpointAws, { endpointAwsParametersSchema } from "./aws/endpointAws"; @@ -26,7 +26,7 @@ import endpointLangserve, { endpointLangserveParametersSchema, } from "./langserve/endpointLangserve"; -import type { Tool, ToolResult } from "$lib/types/Tool"; +import type { Tool, ToolCall, ToolResult } from "$lib/types/Tool"; export type EndpointMessage = Omit; @@ -44,10 +44,13 @@ export interface EndpointParameters { interface CommonEndpoint { weight: number; } +type TextGenerationStreamOutputWithTools = TextGenerationStreamOutput & { + token: TextGenerationStreamToken & { toolCalls?: ToolCall[] }; +}; // type signature for the endpoint export type Endpoint = ( params: EndpointParameters -) => Promise>; +) => Promise>; // generator function that takes in parameters for defining the endpoint and return the endpoint export type EndpointGenerator = (parameters: T) => Endpoint; From 4506ffb315bcc50868dc1bbbbb4e57899f626a77 Mon Sep 17 00:00:00 2001 From: Liam Dyer Date: Thu, 23 May 2024 10:36:37 -0400 Subject: [PATCH 81/82] nit --- src/lib/components/chat/ChatMessage.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 1f0e1e31a18..7bf9e560ccd 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -285,7 +285,7 @@ {#if toolUpdates} {#each Object.values(toolUpdates) as tool} - {#if tool.length > 0} + {#if tool.length} {@const toolName = tool.find(isMessageToolCallUpdate)?.call.name} {@const toolDone = tool.some(isMessageToolResultUpdate)} {#if toolName && toolName !== "websearch"} From e85030b8160db8c45507f54212ad33cfff92197f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Victor=20Mu=C5=A1tar?= Date: Thu, 23 May 2024 16:38:16 +0200 Subject: [PATCH 82/82] Feat functions misc update 2 (#1158) * update examples a * add tool indicator * llama 3 description update * image quick fix * mobile * icon purple --- chart/env/prod.yaml | 12 ++++++------ src/lib/components/ToolsMenu.svelte | 2 +- src/lib/components/chat/ChatMessage.svelte | 2 +- src/routes/models/+page.svelte | 10 +++++++++- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/chart/env/prod.yaml b/chart/env/prod.yaml index fa4afc6821f..0652fbf4f05 100644 --- a/chart/env/prod.yaml +++ b/chart/env/prod.yaml @@ -48,20 +48,20 @@ envVars: }, "promptExamples" : [ { - "title": "Write an email from bullet list", - "prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)" + "title": "Generate a mouse portrait", + "prompt": "Generate the portrait of a scientific mouse in its laboratory." + }, { + "title": "Review a pull request", + "prompt": "Review this pull request: https://github.com/huggingface/chat-ui/pull/1131/files" }, { "title": "Code a snake game", "prompt": "Code a basic snake game in python, give explanations for each step." - }, { - "title": "Assist in a task", - "prompt": "How do I make a delicious lemon cheesecake?" } ] }, { "name" : "meta-llama/Meta-Llama-3-70B-Instruct", - "description": "Generation over generation, Meta Llama 3 demonstrates state-of-the-art performance on a wide range of industry benchmarks and offers new capabilities, including improved reasoning.", + "description": "Meta Llama 3 delivers top performance on various benchmarks and introduces new features like better reasoning.", "logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/meta-logo.png", "modelUrl": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct", "websiteUrl": "https://llama.meta.com/llama3/", diff --git a/src/lib/components/ToolsMenu.svelte b/src/lib/components/ToolsMenu.svelte index b4c9222eacf..c4cc6a61634 100644 --- a/src/lib/components/ToolsMenu.svelte +++ b/src/lib/components/ToolsMenu.svelte @@ -31,7 +31,7 @@ class="absolute bottom-0 flex h-8 cursor-pointer select-none items-center gap-1 rounded-lg border bg-white px-2 py-1.5 shadow-sm hover:shadow-none dark:border-gray-800 dark:bg-gray-900" > - + Tools ({activeToolCount}) diff --git a/src/lib/components/chat/ChatMessage.svelte b/src/lib/components/chat/ChatMessage.svelte index 7bf9e560ccd..97f65b0b9d7 100644 --- a/src/lib/components/chat/ChatMessage.svelte +++ b/src/lib/components/chat/ChatMessage.svelte @@ -212,7 +212,7 @@ {#if modalImageToShow} - (modalImageToShow = null)}> + (modalImageToShow = null)}> {#if modalImageToShow.type === "hash"} @@ -43,7 +44,7 @@ href="{base}/models/{model.id}" class="relative flex flex-col gap-2 overflow-hidden rounded-xl border bg-gray-50/50 px-6 py-5 shadow hover:bg-gray-50 hover:shadow-inner dark:border-gray-800/70 dark:bg-gray-950/20 dark:hover:bg-gray-950/40" > -
    +
    {#if model.logoUrl} {/if} + {#if model.tools} +
    + +
    + {/if} {#if index === 0}