From e0b763f162047aaaae533741bc6f909e899a347c Mon Sep 17 00:00:00 2001 From: bracesproul Date: Tue, 6 Aug 2024 18:00:49 -0700 Subject: [PATCH 1/7] openai[minor]: Add support for json schema response format --- libs/langchain-openai/src/chat_models.ts | 41 +++++------------------ libs/langchain-openai/src/types.ts | 21 ++++++++++++ libs/langchain-openai/src/utils/openai.ts | 15 +++++++++ 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index 51488b0ae7c0..dc3255bd0165 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -62,10 +62,18 @@ import type { OpenAIChatInput, OpenAICoreRequestOptions, LegacyOpenAIInput, + OpenAICompletionParam, + OpenAIFnCallOption, + OpenAIFnDef, + OpenAILLMOutput, + OpenAIRoleEnum, + TokenUsage, + ChatOpenAIResponseFormat, } from "./types.js"; import { type OpenAIEndpointConfig, getEndpoint } from "./utils/azure.js"; import { OpenAIToolChoice, + extractGenericMessageCustomRole, formatToOpenAIToolChoice, wrapOpenAIClientError, } from "./utils/openai.js"; @@ -76,37 +84,6 @@ import { export type { AzureOpenAIInput, OpenAICallOptions, OpenAIChatInput }; -interface TokenUsage { - completionTokens?: number; - promptTokens?: number; - totalTokens?: number; -} - -interface OpenAILLMOutput { - tokenUsage: TokenUsage; -} - -// TODO import from SDK when available -type OpenAIRoleEnum = "system" | "assistant" | "user" | "function" | "tool"; - -type OpenAICompletionParam = - OpenAIClient.Chat.Completions.ChatCompletionMessageParam; -type OpenAIFnDef = OpenAIClient.Chat.ChatCompletionCreateParams.Function; -type OpenAIFnCallOption = OpenAIClient.Chat.ChatCompletionFunctionCallOption; - -function extractGenericMessageCustomRole(message: ChatMessage) { - if ( - message.role !== "system" && - message.role !== "assistant" && - message.role !== "user" && - message.role !== "function" && - message.role !== "tool" - ) { - console.warn(`Unknown message role: ${message.role}`); - } - - return message.role as OpenAIRoleEnum; -} export function messageToOpenAIRole(message: BaseMessage): OpenAIRoleEnum { const type = message._getType(); @@ -324,7 +301,7 @@ export interface ChatOpenAICallOptions tools?: ChatOpenAIToolType[]; tool_choice?: OpenAIToolChoice; promptIndex?: number; - response_format?: { type: "json_object" }; + response_format?: ChatOpenAIResponseFormat; seed?: number; /** * Additional options to pass to streamed completions. diff --git a/libs/langchain-openai/src/types.ts b/libs/langchain-openai/src/types.ts index 0d93089619e2..649d6980e505 100644 --- a/libs/langchain-openai/src/types.ts +++ b/libs/langchain-openai/src/types.ts @@ -1,4 +1,5 @@ import type { OpenAI as OpenAIClient } from "openai"; +import type { ResponseFormatText, ResponseFormatJSONObject, ResponseFormatJSONSchema } from "openai/resources/shared"; import { TiktokenModel } from "js-tiktoken/lite"; import type { BaseLanguageModelCallOptions } from "@langchain/core/language_models/base"; @@ -222,3 +223,23 @@ export declare interface AzureOpenAIInput { */ azureADTokenProvider?: () => Promise; } + +export interface TokenUsage { + completionTokens?: number; + promptTokens?: number; + totalTokens?: number; +} + +export interface OpenAILLMOutput { + tokenUsage: TokenUsage; +} + +// TODO import from SDK when available +export type OpenAIRoleEnum = "system" | "assistant" | "user" | "function" | "tool"; + +export type OpenAICompletionParam = + OpenAIClient.Chat.Completions.ChatCompletionMessageParam; +export type OpenAIFnDef = OpenAIClient.Chat.ChatCompletionCreateParams.Function; +export type OpenAIFnCallOption = OpenAIClient.Chat.ChatCompletionFunctionCallOption; + +export type ChatOpenAIResponseFormat = ResponseFormatText | ResponseFormatJSONObject | ResponseFormatJSONSchema; diff --git a/libs/langchain-openai/src/utils/openai.ts b/libs/langchain-openai/src/utils/openai.ts index e95297e56b64..0cc48cd46e77 100644 --- a/libs/langchain-openai/src/utils/openai.ts +++ b/libs/langchain-openai/src/utils/openai.ts @@ -9,6 +9,7 @@ import { convertToOpenAIFunction, convertToOpenAITool, } from "@langchain/core/utils/function_calling"; +import { ChatMessage } from "@langchain/core/messages"; // eslint-disable-next-line @typescript-eslint/no-explicit-any export function wrapOpenAIClientError(e: any) { @@ -68,3 +69,17 @@ export function formatToOpenAIToolChoice( return toolChoice; } } + +export function extractGenericMessageCustomRole(message: ChatMessage) { + if ( + message.role !== "system" && + message.role !== "assistant" && + message.role !== "user" && + message.role !== "function" && + message.role !== "tool" + ) { + console.warn(`Unknown message role: ${message.role}`); + } + + return message.role as OpenAIRoleEnum; +} \ No newline at end of file From d339d1a942d900b411e8e9bcc2c1e242d631b6c6 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Tue, 6 Aug 2024 18:24:03 -0700 Subject: [PATCH 2/7] cr --- libs/langchain-openai/src/chat_models.ts | 3 +- .../chat_models_structured_output.int.test.ts | 32 +++++++++++++++++++ libs/langchain-openai/src/types.ts | 21 +++++++++--- libs/langchain-openai/src/utils/openai.ts | 3 +- 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index dc3255bd0165..ee4b1487aca8 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -84,7 +84,6 @@ import { export type { AzureOpenAIInput, OpenAICallOptions, OpenAIChatInput }; - export function messageToOpenAIRole(message: BaseMessage): OpenAIRoleEnum { const type = message._getType(); switch (type) { @@ -1066,6 +1065,8 @@ export class ChatOpenAI< request, requestOptions ); + console.log("RES"); + console.dir(res, { depth: null }); return res; } catch (e) { const error = wrapOpenAIClientError(e); diff --git a/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts b/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts index bc0328357a73..60f36bb4699c 100644 --- a/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts +++ b/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts @@ -355,3 +355,35 @@ test("Passing strict true forces the model to conform to the schema", async () = expect(result.tool_calls?.[0].args).toHaveProperty("location"); console.log(result.tool_calls?.[0].args); }); + +test("Passing response_format json_schema works", async () => { + const weatherSchema = z.object({ + city: z.string().describe("The city to get the weather for"), + state: z.string().describe("The state to get the weather for"), + zipCode: z.string().describe("The zip code to get the weather for"), + unit: z + .enum(["fahrenheit", "celsius"]) + .describe("The unit to get the weather in"), + }); + + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).bind({ + response_format: { + type: "json_schema", + json_schema: { + name: "get_current_weather", + description: "Get the current weather in a location", + schema: zodToJsonSchema(weatherSchema), + strict: true, + }, + }, + }); + + const response = await model.invoke( + "What is the weather in San Francisco, 91626 CA?" + ); + console.log(response.content); + const parsed = JSON.parse(response.content as string); + console.log(parsed); +}); diff --git a/libs/langchain-openai/src/types.ts b/libs/langchain-openai/src/types.ts index 649d6980e505..b5ff5417c4c5 100644 --- a/libs/langchain-openai/src/types.ts +++ b/libs/langchain-openai/src/types.ts @@ -1,5 +1,9 @@ import type { OpenAI as OpenAIClient } from "openai"; -import type { ResponseFormatText, ResponseFormatJSONObject, ResponseFormatJSONSchema } from "openai/resources/shared"; +import type { + ResponseFormatText, + ResponseFormatJSONObject, + ResponseFormatJSONSchema, +} from "openai/resources/shared"; import { TiktokenModel } from "js-tiktoken/lite"; import type { BaseLanguageModelCallOptions } from "@langchain/core/language_models/base"; @@ -235,11 +239,20 @@ export interface OpenAILLMOutput { } // TODO import from SDK when available -export type OpenAIRoleEnum = "system" | "assistant" | "user" | "function" | "tool"; +export type OpenAIRoleEnum = + | "system" + | "assistant" + | "user" + | "function" + | "tool"; export type OpenAICompletionParam = OpenAIClient.Chat.Completions.ChatCompletionMessageParam; export type OpenAIFnDef = OpenAIClient.Chat.ChatCompletionCreateParams.Function; -export type OpenAIFnCallOption = OpenAIClient.Chat.ChatCompletionFunctionCallOption; +export type OpenAIFnCallOption = + OpenAIClient.Chat.ChatCompletionFunctionCallOption; -export type ChatOpenAIResponseFormat = ResponseFormatText | ResponseFormatJSONObject | ResponseFormatJSONSchema; +export type ChatOpenAIResponseFormat = + | ResponseFormatText + | ResponseFormatJSONObject + | ResponseFormatJSONSchema; diff --git a/libs/langchain-openai/src/utils/openai.ts b/libs/langchain-openai/src/utils/openai.ts index 0cc48cd46e77..f726988b77f1 100644 --- a/libs/langchain-openai/src/utils/openai.ts +++ b/libs/langchain-openai/src/utils/openai.ts @@ -10,6 +10,7 @@ import { convertToOpenAITool, } from "@langchain/core/utils/function_calling"; import { ChatMessage } from "@langchain/core/messages"; +import { OpenAIRoleEnum } from "../types.js"; // eslint-disable-next-line @typescript-eslint/no-explicit-any export function wrapOpenAIClientError(e: any) { @@ -82,4 +83,4 @@ export function extractGenericMessageCustomRole(message: ChatMessage) { } return message.role as OpenAIRoleEnum; -} \ No newline at end of file +} From fc95ccbf6f680eca185d16be0494f1a9d6d61455 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 8 Aug 2024 17:13:40 -0700 Subject: [PATCH 3/7] lowkey I did it? --- libs/langchain-openai/src/chat_models.ts | 22 +++++++++++++-- .../chat_models_structured_output.int.test.ts | 28 +++++++++++++++++-- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index 82e3fa4063e8..f6b20a45ca52 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -276,7 +276,7 @@ function _convertChatOpenAIToolTypeToOpenAITool( export interface ChatOpenAIStructuredOutputMethodOptions< IncludeRaw extends boolean -> extends StructuredOutputMethodOptions { +> extends Omit, "method"> { /** * strict: If `true` and `method` = "function_calling", model output is * guaranteed to exactly match the schema. If `true`, the input schema @@ -292,6 +292,7 @@ export interface ChatOpenAIStructuredOutputMethodOptions< * "function_calling" as of version `0.3.0`. */ strict?: boolean; + method?: "functionCalling" | "jsonMode" | "jsonSchema"; } export interface ChatOpenAICallOptions @@ -1480,8 +1481,6 @@ export class ChatOpenAI< request, requestOptions ); - console.log("RES"); - console.dir(res, { depth: null }); return res; } catch (e) { const error = wrapOpenAIClientError(e); @@ -1632,6 +1631,23 @@ export class ChatOpenAI< } else { outputParser = new JsonOutputParser(); } + } else if (method === "jsonSchema") { + llm = this.bind({ + response_format: { + type: "json_schema", + json_schema: { + name: name ?? "extract", + description: schema.description, + schema: isZodSchema(schema) ? zodToJsonSchema(schema) : schema, + strict: config?.strict, + }, + }, + } as Partial); + if (isZodSchema(schema)) { + outputParser = StructuredOutputParser.fromZodSchema(schema); + } else { + outputParser = new JsonOutputParser(); + } } else { let functionName = name ?? "extract"; // Is function calling diff --git a/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts b/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts index 60f36bb4699c..5c5d1efec509 100644 --- a/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts +++ b/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts @@ -383,7 +383,31 @@ test("Passing response_format json_schema works", async () => { const response = await model.invoke( "What is the weather in San Francisco, 91626 CA?" ); - console.log(response.content); + console.log("content", response.content); const parsed = JSON.parse(response.content as string); - console.log(parsed); + console.log("parsed", parsed); +}); + +test("Passing response_format json_schema works with WSO", async () => { + const weatherSchema = z.object({ + city: z.string().describe("The city to get the weather for"), + state: z.string().describe("The state to get the weather for"), + zipCode: z.string().describe("The zip code to get the weather for"), + unit: z + .enum(["fahrenheit", "celsius"]) + .describe("The unit to get the weather in"), + }); + + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).withStructuredOutput(weatherSchema, { + name: "get_current_weather", + method: "jsonSchema", + strict: true, + }); + + const response = await model.invoke( + "What is the weather in San Francisco, 91626 CA?" + ); + console.log(response); }); From 2507bb0799030343bd526f62493c00a60bd5b7a6 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Thu, 8 Aug 2024 18:26:29 -0700 Subject: [PATCH 4/7] add tests and more implementation details --- libs/langchain-openai/src/chat_models.ts | 131 ++++++++++-- .../chat_models_structured_output.int.test.ts | 192 ++++++++++++++---- libs/langchain-openai/src/types.ts | 17 +- 3 files changed, 287 insertions(+), 53 deletions(-) diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index f6b20a45ca52..13e43aeee6cd 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -56,6 +56,12 @@ import { } from "@langchain/core/output_parsers/openai_tools"; import { zodToJsonSchema } from "zod-to-json-schema"; import { ToolCallChunk } from "@langchain/core/messages/tool"; +import { zodResponseFormat } from "openai/helpers/zod"; +import type { + ResponseFormatText, + ResponseFormatJSONObject, + ResponseFormatJSONSchema, +} from "openai/resources/shared"; import type { AzureOpenAIInput, OpenAICallOptions, @@ -1038,6 +1044,34 @@ export class ChatOpenAI< } as Partial); } + private createResponseFormat( + resFormat?: CallOptions["response_format"] + ): + | ResponseFormatText + | ResponseFormatJSONObject + | ResponseFormatJSONSchema + | undefined { + if ( + resFormat && + resFormat.type === "json_schema" && + resFormat.json_schema.schema && + isZodSchema(resFormat.json_schema.schema) + ) { + return zodResponseFormat( + resFormat.json_schema.schema, + resFormat.json_schema.name, + { + description: resFormat.json_schema.description, + } + ); + } + return resFormat as + | ResponseFormatText + | ResponseFormatJSONObject + | ResponseFormatJSONSchema + | undefined; + } + /** * Get the parameters used to invoke the model */ @@ -1060,6 +1094,7 @@ export class ChatOpenAI< } else if (this.streamUsage && (this.streaming || extra?.streaming)) { streamOptionsConfig = { stream_options: { include_usage: true } }; } + const params: Omit< OpenAIClient.Chat.ChatCompletionCreateParams, "messages" @@ -1086,7 +1121,7 @@ export class ChatOpenAI< ) : undefined, tool_choice: formatToOpenAIToolChoice(options?.tool_choice), - response_format: options?.response_format, + response_format: this.createResponseFormat(options?.response_format), seed: options?.seed, ...streamOptionsConfig, parallel_tool_calls: options?.parallel_tool_calls, @@ -1124,6 +1159,32 @@ export class ChatOpenAI< stream: true as const, }; let defaultRole: OpenAIRoleEnum | undefined; + if ( + params.response_format && + params.response_format.type === "json_schema" + ) { + console.warn( + `OpenAI does not yet support streaming with "response_format" set to "json_schema". Falling back to non-streaming mode.` + ); + const res = await this._generate(messages, options, runManager); + const chunk = new ChatGenerationChunk({ + message: new AIMessageChunk({ + ...res.generations[0].message, + }), + text: res.generations[0].text, + generationInfo: res.generations[0].generationInfo, + }); + yield chunk; + return runManager?.handleLLMNewToken( + res.generations[0].text ?? "", + undefined, + undefined, + undefined, + undefined, + { chunk } + ); + } + const streamIterable = await this.completionWithRetry(params, options); let usage: OpenAIClient.Completions.CompletionUsage | undefined; for await (const data of streamIterable) { @@ -1259,17 +1320,36 @@ export class ChatOpenAI< tokenUsage.totalTokens = promptTokenUsage + completionTokenUsage; return { generations, llmOutput: { estimatedTokenUsage: tokenUsage } }; } else { - const data = await this.completionWithRetry( - { - ...params, - stream: false, - messages: messagesMapped, - }, - { - signal: options?.signal, - ...options?.options, - } - ); + let data; + if ( + options.response_format && + options.response_format.type === "json_schema" + ) { + data = await this.betaParsedCompletionWithRetry( + { + ...params, + stream: false, + messages: messagesMapped, + }, + { + signal: options?.signal, + ...options?.options, + } + ); + } else { + data = await this.completionWithRetry( + { + ...params, + stream: false, + messages: messagesMapped, + }, + { + signal: options?.signal, + ...options?.options, + } + ); + } + const { completion_tokens: completionTokens, prompt_tokens: promptTokens, @@ -1489,6 +1569,31 @@ export class ChatOpenAI< }); } + /** + * Call the beta chat completions parse endpoint. This should only be called if + * response_format is set to "json_object". + * @param {OpenAIClient.Chat.ChatCompletionCreateParamsNonStreaming} request + * @param {OpenAICoreRequestOptions | undefined} options + */ + async betaParsedCompletionWithRetry( + request: OpenAIClient.Chat.ChatCompletionCreateParamsNonStreaming, + options?: OpenAICoreRequestOptions + ) { + const requestOptions = this._getClientOptions(options); + return this.caller.call(async () => { + try { + const res = await this.client.beta.chat.completions.parse( + request, + requestOptions + ); + return res; + } catch (e) { + const error = wrapOpenAIClientError(e); + throw error; + } + }); + } + protected _getClientOptions(options: OpenAICoreRequestOptions | undefined) { if (!this.client) { const openAIEndpointConfig: OpenAIEndpointConfig = { @@ -1638,7 +1743,7 @@ export class ChatOpenAI< json_schema: { name: name ?? "extract", description: schema.description, - schema: isZodSchema(schema) ? zodToJsonSchema(schema) : schema, + schema, strict: config?.strict, }, }, diff --git a/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts b/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts index 5c5d1efec509..650cf0434b5f 100644 --- a/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts +++ b/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts @@ -1,8 +1,9 @@ import { z } from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; import { ChatPromptTemplate } from "@langchain/core/prompts"; -import { AIMessage } from "@langchain/core/messages"; -import { test, expect } from "@jest/globals"; +import { AIMessage, AIMessageChunk } from "@langchain/core/messages"; +import { test, expect, describe, it } from "@jest/globals"; +import { concat } from "@langchain/core/utils/stream"; import { ChatOpenAI } from "../chat_models.js"; test("withStructuredOutput zod schema function calling", async () => { @@ -356,7 +357,7 @@ test("Passing strict true forces the model to conform to the schema", async () = console.log(result.tool_calls?.[0].args); }); -test("Passing response_format json_schema works", async () => { +describe("response_format: json_schema", () => { const weatherSchema = z.object({ city: z.string().describe("The city to get the weather for"), state: z.string().describe("The state to get the weather for"), @@ -366,48 +367,161 @@ test("Passing response_format json_schema works", async () => { .describe("The unit to get the weather in"), }); - const model = new ChatOpenAI({ - model: "gpt-4o-2024-08-06", - }).bind({ - response_format: { - type: "json_schema", - json_schema: { - name: "get_current_weather", - description: "Get the current weather in a location", - schema: zodToJsonSchema(weatherSchema), - strict: true, + it("can invoke", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).bind({ + response_format: { + type: "json_schema", + json_schema: { + name: "get_current_weather", + description: "Get the current weather in a location", + schema: zodToJsonSchema(weatherSchema), + strict: true, + }, }, - }, + }); + + const response = await model.invoke( + "What is the weather in San Francisco, 91626 CA?" + ); + const parsed = JSON.parse(response.content as string); + expect(parsed).toHaveProperty("city"); + expect(parsed).toHaveProperty("state"); + expect(parsed).toHaveProperty("zipCode"); + expect(parsed).toHaveProperty("unit"); }); - const response = await model.invoke( - "What is the weather in San Francisco, 91626 CA?" - ); - console.log("content", response.content); - const parsed = JSON.parse(response.content as string); - console.log("parsed", parsed); -}); + it("can stream", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).bind({ + response_format: { + type: "json_schema", + json_schema: { + name: "get_current_weather", + description: "Get the current weather in a location", + schema: zodToJsonSchema(weatherSchema), + strict: true, + }, + }, + }); + + const stream = await model.stream( + "What is the weather in San Francisco, 91626 CA?" + ); + let full: AIMessageChunk | undefined; + for await (const chunk of stream) { + full = !full ? chunk : concat(full, chunk); + } + expect(full).toBeDefined(); + if (!full) return; + + const parsed = JSON.parse(full.content as string); + expect(parsed).toHaveProperty("city"); + expect(parsed).toHaveProperty("state"); + expect(parsed).toHaveProperty("zipCode"); + expect(parsed).toHaveProperty("unit"); + }); -test("Passing response_format json_schema works with WSO", async () => { - const weatherSchema = z.object({ - city: z.string().describe("The city to get the weather for"), - state: z.string().describe("The state to get the weather for"), - zipCode: z.string().describe("The zip code to get the weather for"), - unit: z - .enum(["fahrenheit", "celsius"]) - .describe("The unit to get the weather in"), + it("can invoke with a zod schema passed in", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).bind({ + response_format: { + type: "json_schema", + json_schema: { + name: "get_current_weather", + description: "Get the current weather in a location", + schema: weatherSchema, + strict: true, + }, + }, + }); + + const response = await model.invoke( + "What is the weather in San Francisco, 91626 CA?" + ); + const parsed = JSON.parse(response.content as string); + expect(parsed).toHaveProperty("city"); + expect(parsed).toHaveProperty("state"); + expect(parsed).toHaveProperty("zipCode"); + expect(parsed).toHaveProperty("unit"); }); - const model = new ChatOpenAI({ - model: "gpt-4o-2024-08-06", - }).withStructuredOutput(weatherSchema, { - name: "get_current_weather", - method: "jsonSchema", - strict: true, + it("can stream with a zod schema passed in", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).bind({ + response_format: { + type: "json_schema", + json_schema: { + name: "get_current_weather", + description: "Get the current weather in a location", + schema: weatherSchema, + strict: true, + }, + }, + }); + + const stream = await model.stream( + "What is the weather in San Francisco, 91626 CA?" + ); + let full: AIMessageChunk | undefined; + for await (const chunk of stream) { + full = !full ? chunk : concat(full, chunk); + } + expect(full).toBeDefined(); + if (!full) return; + + const parsed = JSON.parse(full.content as string); + expect(parsed).toHaveProperty("city"); + expect(parsed).toHaveProperty("state"); + expect(parsed).toHaveProperty("zipCode"); + expect(parsed).toHaveProperty("unit"); }); - const response = await model.invoke( - "What is the weather in San Francisco, 91626 CA?" - ); - console.log(response); + it("can be invoked with WSO", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).withStructuredOutput(weatherSchema, { + name: "get_current_weather", + method: "jsonSchema", + strict: true, + }); + + const response = await model.invoke( + "What is the weather in San Francisco, 91626 CA?" + ); + expect(response).toHaveProperty("city"); + expect(response).toHaveProperty("state"); + expect(response).toHaveProperty("zipCode"); + expect(response).toHaveProperty("unit"); + }); + + it("can be streamed with WSO", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).withStructuredOutput(weatherSchema, { + name: "get_current_weather", + method: "jsonSchema", + strict: true, + }); + + const stream = await model.stream( + "What is the weather in San Francisco, 91626 CA?" + ); + // It should yield a single chunk + let full: z.infer | undefined; + for await (const chunk of stream) { + full = chunk; + } + expect(full).toBeDefined(); + if (!full) return; + + expect(full).toHaveProperty("city"); + expect(full).toHaveProperty("state"); + expect(full).toHaveProperty("zipCode"); + expect(full).toHaveProperty("unit"); + }); }); diff --git a/libs/langchain-openai/src/types.ts b/libs/langchain-openai/src/types.ts index b5ff5417c4c5..44cab6f6438d 100644 --- a/libs/langchain-openai/src/types.ts +++ b/libs/langchain-openai/src/types.ts @@ -7,6 +7,7 @@ import type { import { TiktokenModel } from "js-tiktoken/lite"; import type { BaseLanguageModelCallOptions } from "@langchain/core/language_models/base"; +import type { z } from "zod"; // reexport this type from the included package so we can easily override and extend it if needed in the future // also makes it easier for folks to import this type without digging around into the dependent packages @@ -252,7 +253,21 @@ export type OpenAIFnDef = OpenAIClient.Chat.ChatCompletionCreateParams.Function; export type OpenAIFnCallOption = OpenAIClient.Chat.ChatCompletionFunctionCallOption; +type ChatOpenAIResponseFormatJSONSchema = Omit< + ResponseFormatJSONSchema, + "json_schema" +> & { + json_schema: Omit & { + /** + * The schema for the response format, described as a JSON Schema object + * or a Zod object. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + schema: Record | z.ZodObject; + }; +}; + export type ChatOpenAIResponseFormat = | ResponseFormatText | ResponseFormatJSONObject - | ResponseFormatJSONSchema; + | ChatOpenAIResponseFormatJSONSchema; From 6cf64ced1d3817bbb60ca426d7ce61edb1ea743f Mon Sep 17 00:00:00 2001 From: bracesproul Date: Fri, 9 Aug 2024 10:24:33 -0700 Subject: [PATCH 5/7] fix build yo --- libs/langchain-openai/langchain.config.js | 2 +- libs/langchain-openai/src/chat_models.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/langchain-openai/langchain.config.js b/libs/langchain-openai/langchain.config.js index 416001cb4772..37cd5f682c5d 100644 --- a/libs/langchain-openai/langchain.config.js +++ b/libs/langchain-openai/langchain.config.js @@ -11,7 +11,7 @@ function abs(relativePath) { export const config = { - internals: [/node\:/, /@langchain\/core\//], + internals: [/node\:/, /@langchain\/core\//, "openai/helpers/zod"], entrypoints: { index: "index", }, diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index 13e43aeee6cd..f78b53e87c41 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -62,6 +62,7 @@ import type { ResponseFormatJSONObject, ResponseFormatJSONSchema, } from "openai/resources/shared"; +import { ParsedChatCompletion } from "openai/resources/beta/chat/completions.mjs"; import type { AzureOpenAIInput, OpenAICallOptions, @@ -1578,7 +1579,7 @@ export class ChatOpenAI< async betaParsedCompletionWithRetry( request: OpenAIClient.Chat.ChatCompletionCreateParamsNonStreaming, options?: OpenAICoreRequestOptions - ) { + ): Promise> { const requestOptions = this._getClientOptions(options); return this.caller.call(async () => { try { From 31a0bdf390101e2404d9dbfc71d18a52db238108 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Fri, 9 Aug 2024 14:20:11 -0700 Subject: [PATCH 6/7] cr --- libs/langchain-openai/src/chat_models.ts | 4 +- libs/langchain-openai/src/utils/tools.ts | 65 ++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 libs/langchain-openai/src/utils/tools.ts diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index f78b53e87c41..31d136e55f57 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -36,7 +36,6 @@ import { type StructuredOutputMethodParams, } from "@langchain/core/language_models/base"; import { NewTokenIndices } from "@langchain/core/callbacks/base"; -import { convertToOpenAITool } from "@langchain/core/utils/function_calling"; import { z } from "zod"; import { Runnable, @@ -88,6 +87,7 @@ import { FunctionDef, formatFunctionDefinitions, } from "./utils/openai-format-fndef.js"; +import { _convertToOpenAITool } from "./utils/tools.js"; export type { AzureOpenAIInput, OpenAICallOptions, OpenAIChatInput }; @@ -278,7 +278,7 @@ function _convertChatOpenAIToolTypeToOpenAITool( return tool; } - return convertToOpenAITool(tool, fields); + return _convertToOpenAITool(tool, fields); } export interface ChatOpenAIStructuredOutputMethodOptions< diff --git a/libs/langchain-openai/src/utils/tools.ts b/libs/langchain-openai/src/utils/tools.ts new file mode 100644 index 000000000000..4a9d4e196ad1 --- /dev/null +++ b/libs/langchain-openai/src/utils/tools.ts @@ -0,0 +1,65 @@ +import { ToolDefinition } from "@langchain/core/language_models/base"; +import { BindToolsInput } from "@langchain/core/language_models/chat_models"; +import { + convertToOpenAIFunction, + isLangChainTool, +} from "@langchain/core/utils/function_calling"; +import { zodFunction } from "openai/helpers/zod"; + +/** + * Formats a tool in either OpenAI format, or LangChain structured tool format + * into an OpenAI tool format. If the tool is already in OpenAI format, return without + * any changes. If it is in LangChain structured tool format, convert it to OpenAI tool format + * using OpenAI's `zodFunction` util, falling back to `convertToOpenAIFunction` if the parameters + * returned from the `zodFunction` util are not defined. + * + * @param {BindToolsInput} tool The tool to convert to an OpenAI tool. + * @param {Object} [fields] Additional fields to add to the OpenAI tool. + * @returns {ToolDefinition} The inputted tool in OpenAI tool format. + */ +export function _convertToOpenAITool( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + tool: BindToolsInput, + fields?: { + /** + * If `true`, model output is guaranteed to exactly match the JSON Schema + * provided in the function definition. + */ + strict?: boolean; + } +): ToolDefinition { + let toolDef: ToolDefinition | undefined; + + if (isLangChainTool(tool)) { + const oaiToolDef = zodFunction({ + name: tool.name, + parameters: tool.schema, + description: tool.description, + }); + if (!oaiToolDef.function.parameters) { + // Fallback to the `convertToOpenAIFunction` util if the parameters are not defined. + toolDef = { + type: "function", + function: convertToOpenAIFunction(tool, fields), + }; + } else { + toolDef = { + type: oaiToolDef.type, + function: { + name: oaiToolDef.function.name, + description: oaiToolDef.function.description, + parameters: oaiToolDef.function.parameters, + ...(fields?.strict !== undefined ? { strict: fields.strict } : {}), + }, + }; + } + } else { + toolDef = tool as ToolDefinition; + } + + if (fields?.strict !== undefined) { + toolDef.function.strict = fields.strict; + } + + return toolDef; +} From 8b5d3ec921c769d275e80c31ef3297c49e587c4c Mon Sep 17 00:00:00 2001 From: bracesproul Date: Fri, 9 Aug 2024 14:47:25 -0700 Subject: [PATCH 7/7] fix build --- langchain-core/src/language_models/base.ts | 2 +- libs/langchain-openai/src/chat_models.ts | 42 +++++++++++++++++----- libs/langchain-openai/src/types.ts | 24 ------------- libs/langchain-openai/src/utils/openai.ts | 16 --------- 4 files changed, 34 insertions(+), 50 deletions(-) diff --git a/langchain-core/src/language_models/base.ts b/langchain-core/src/language_models/base.ts index cea8ca2f9ae3..c72fd150dc86 100644 --- a/langchain-core/src/language_models/base.ts +++ b/langchain-core/src/language_models/base.ts @@ -269,7 +269,7 @@ export type StructuredOutputType = z.infer>; export type StructuredOutputMethodOptions = { name?: string; - method?: "functionCalling" | "jsonMode"; + method?: "functionCalling" | "jsonMode" | "jsonSchema" | string; includeRaw?: IncludeRaw; }; diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index 31d136e55f57..467118cb3d9a 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -68,18 +68,11 @@ import type { OpenAIChatInput, OpenAICoreRequestOptions, LegacyOpenAIInput, - OpenAICompletionParam, - OpenAIFnCallOption, - OpenAIFnDef, - OpenAILLMOutput, - OpenAIRoleEnum, - TokenUsage, ChatOpenAIResponseFormat, } from "./types.js"; import { type OpenAIEndpointConfig, getEndpoint } from "./utils/azure.js"; import { OpenAIToolChoice, - extractGenericMessageCustomRole, formatToOpenAIToolChoice, wrapOpenAIClientError, } from "./utils/openai.js"; @@ -91,6 +84,38 @@ import { _convertToOpenAITool } from "./utils/tools.js"; export type { AzureOpenAIInput, OpenAICallOptions, OpenAIChatInput }; +interface TokenUsage { + completionTokens?: number; + promptTokens?: number; + totalTokens?: number; +} + +interface OpenAILLMOutput { + tokenUsage: TokenUsage; +} + +// TODO import from SDK when available +type OpenAIRoleEnum = "system" | "assistant" | "user" | "function" | "tool"; + +type OpenAICompletionParam = + OpenAIClient.Chat.Completions.ChatCompletionMessageParam; +type OpenAIFnDef = OpenAIClient.Chat.ChatCompletionCreateParams.Function; +type OpenAIFnCallOption = OpenAIClient.Chat.ChatCompletionFunctionCallOption; + +function extractGenericMessageCustomRole(message: ChatMessage) { + if ( + message.role !== "system" && + message.role !== "assistant" && + message.role !== "user" && + message.role !== "function" && + message.role !== "tool" + ) { + console.warn(`Unknown message role: ${message.role}`); + } + + return message.role as OpenAIRoleEnum; +} + export function messageToOpenAIRole(message: BaseMessage): OpenAIRoleEnum { const type = message._getType(); switch (type) { @@ -283,7 +308,7 @@ function _convertChatOpenAIToolTypeToOpenAITool( export interface ChatOpenAIStructuredOutputMethodOptions< IncludeRaw extends boolean -> extends Omit, "method"> { +> extends StructuredOutputMethodOptions { /** * strict: If `true` and `method` = "function_calling", model output is * guaranteed to exactly match the schema. If `true`, the input schema @@ -299,7 +324,6 @@ export interface ChatOpenAIStructuredOutputMethodOptions< * "function_calling" as of version `0.3.0`. */ strict?: boolean; - method?: "functionCalling" | "jsonMode" | "jsonSchema"; } export interface ChatOpenAICallOptions diff --git a/libs/langchain-openai/src/types.ts b/libs/langchain-openai/src/types.ts index 44cab6f6438d..04d8865713d4 100644 --- a/libs/langchain-openai/src/types.ts +++ b/libs/langchain-openai/src/types.ts @@ -229,30 +229,6 @@ export declare interface AzureOpenAIInput { azureADTokenProvider?: () => Promise; } -export interface TokenUsage { - completionTokens?: number; - promptTokens?: number; - totalTokens?: number; -} - -export interface OpenAILLMOutput { - tokenUsage: TokenUsage; -} - -// TODO import from SDK when available -export type OpenAIRoleEnum = - | "system" - | "assistant" - | "user" - | "function" - | "tool"; - -export type OpenAICompletionParam = - OpenAIClient.Chat.Completions.ChatCompletionMessageParam; -export type OpenAIFnDef = OpenAIClient.Chat.ChatCompletionCreateParams.Function; -export type OpenAIFnCallOption = - OpenAIClient.Chat.ChatCompletionFunctionCallOption; - type ChatOpenAIResponseFormatJSONSchema = Omit< ResponseFormatJSONSchema, "json_schema" diff --git a/libs/langchain-openai/src/utils/openai.ts b/libs/langchain-openai/src/utils/openai.ts index f726988b77f1..e95297e56b64 100644 --- a/libs/langchain-openai/src/utils/openai.ts +++ b/libs/langchain-openai/src/utils/openai.ts @@ -9,8 +9,6 @@ import { convertToOpenAIFunction, convertToOpenAITool, } from "@langchain/core/utils/function_calling"; -import { ChatMessage } from "@langchain/core/messages"; -import { OpenAIRoleEnum } from "../types.js"; // eslint-disable-next-line @typescript-eslint/no-explicit-any export function wrapOpenAIClientError(e: any) { @@ -70,17 +68,3 @@ export function formatToOpenAIToolChoice( return toolChoice; } } - -export function extractGenericMessageCustomRole(message: ChatMessage) { - if ( - message.role !== "system" && - message.role !== "assistant" && - message.role !== "user" && - message.role !== "function" && - message.role !== "tool" - ) { - console.warn(`Unknown message role: ${message.role}`); - } - - return message.role as OpenAIRoleEnum; -}