diff --git a/langchain-core/src/language_models/base.ts b/langchain-core/src/language_models/base.ts index cea8ca2f9ae3..c72fd150dc86 100644 --- a/langchain-core/src/language_models/base.ts +++ b/langchain-core/src/language_models/base.ts @@ -269,7 +269,7 @@ export type StructuredOutputType = z.infer>; export type StructuredOutputMethodOptions = { name?: string; - method?: "functionCalling" | "jsonMode"; + method?: "functionCalling" | "jsonMode" | "jsonSchema" | string; includeRaw?: IncludeRaw; }; diff --git a/libs/langchain-openai/langchain.config.js b/libs/langchain-openai/langchain.config.js index 416001cb4772..37cd5f682c5d 100644 --- a/libs/langchain-openai/langchain.config.js +++ b/libs/langchain-openai/langchain.config.js @@ -11,7 +11,7 @@ function abs(relativePath) { export const config = { - internals: [/node\:/, /@langchain\/core\//], + internals: [/node\:/, /@langchain\/core\//, "openai/helpers/zod"], entrypoints: { index: "index", }, diff --git a/libs/langchain-openai/src/chat_models.ts b/libs/langchain-openai/src/chat_models.ts index d59937b2fda1..22113d5235ab 100644 --- a/libs/langchain-openai/src/chat_models.ts +++ b/libs/langchain-openai/src/chat_models.ts @@ -36,7 +36,6 @@ import { type StructuredOutputMethodParams, } from "@langchain/core/language_models/base"; import { NewTokenIndices } from "@langchain/core/callbacks/base"; -import { convertToOpenAITool } from "@langchain/core/utils/function_calling"; import { z } from "zod"; import { Runnable, @@ -56,12 +55,20 @@ import { } from "@langchain/core/output_parsers/openai_tools"; import { zodToJsonSchema } from "zod-to-json-schema"; import { ToolCallChunk } from "@langchain/core/messages/tool"; +import { zodResponseFormat } from "openai/helpers/zod"; +import type { + ResponseFormatText, + ResponseFormatJSONObject, + ResponseFormatJSONSchema, +} from "openai/resources/shared"; +import { ParsedChatCompletion } from "openai/resources/beta/chat/completions.mjs"; import type { AzureOpenAIInput, OpenAICallOptions, OpenAIChatInput, OpenAICoreRequestOptions, LegacyOpenAIInput, + ChatOpenAIResponseFormat, } from "./types.js"; import { type OpenAIEndpointConfig, getEndpoint } from "./utils/azure.js"; import { @@ -73,6 +80,7 @@ import { FunctionDef, formatFunctionDefinitions, } from "./utils/openai-format-fndef.js"; +import { _convertToOpenAITool } from "./utils/tools.js"; export type { AzureOpenAIInput, OpenAICallOptions, OpenAIChatInput }; @@ -295,7 +303,7 @@ function _convertChatOpenAIToolTypeToOpenAITool( return tool; } - return convertToOpenAITool(tool, fields); + return _convertToOpenAITool(tool, fields); } export interface ChatOpenAIStructuredOutputMethodOptions< @@ -324,7 +332,7 @@ export interface ChatOpenAICallOptions tools?: ChatOpenAIToolType[]; tool_choice?: OpenAIToolChoice; promptIndex?: number; - response_format?: { type: "json_object" }; + response_format?: ChatOpenAIResponseFormat; seed?: number; /** * Additional options to pass to streamed completions. @@ -1027,6 +1035,34 @@ export class ChatOpenAI< } as Partial); } + private createResponseFormat( + resFormat?: CallOptions["response_format"] + ): + | ResponseFormatText + | ResponseFormatJSONObject + | ResponseFormatJSONSchema + | undefined { + if ( + resFormat && + resFormat.type === "json_schema" && + resFormat.json_schema.schema && + isZodSchema(resFormat.json_schema.schema) + ) { + return zodResponseFormat( + resFormat.json_schema.schema, + resFormat.json_schema.name, + { + description: resFormat.json_schema.description, + } + ); + } + return resFormat as + | ResponseFormatText + | ResponseFormatJSONObject + | ResponseFormatJSONSchema + | undefined; + } + /** * Get the parameters used to invoke the model */ @@ -1049,6 +1085,7 @@ export class ChatOpenAI< } else if (this.streamUsage && (this.streaming || extra?.streaming)) { streamOptionsConfig = { stream_options: { include_usage: true } }; } + const params: Omit< OpenAIClient.Chat.ChatCompletionCreateParams, "messages" @@ -1075,7 +1112,7 @@ export class ChatOpenAI< ) : undefined, tool_choice: formatToOpenAIToolChoice(options?.tool_choice), - response_format: options?.response_format, + response_format: this.createResponseFormat(options?.response_format), seed: options?.seed, ...streamOptionsConfig, parallel_tool_calls: options?.parallel_tool_calls, @@ -1113,6 +1150,32 @@ export class ChatOpenAI< stream: true as const, }; let defaultRole: OpenAIRoleEnum | undefined; + if ( + params.response_format && + params.response_format.type === "json_schema" + ) { + console.warn( + `OpenAI does not yet support streaming with "response_format" set to "json_schema". Falling back to non-streaming mode.` + ); + const res = await this._generate(messages, options, runManager); + const chunk = new ChatGenerationChunk({ + message: new AIMessageChunk({ + ...res.generations[0].message, + }), + text: res.generations[0].text, + generationInfo: res.generations[0].generationInfo, + }); + yield chunk; + return runManager?.handleLLMNewToken( + res.generations[0].text ?? "", + undefined, + undefined, + undefined, + undefined, + { chunk } + ); + } + const streamIterable = await this.completionWithRetry(params, options); let usage: OpenAIClient.Completions.CompletionUsage | undefined; for await (const data of streamIterable) { @@ -1248,17 +1311,36 @@ export class ChatOpenAI< tokenUsage.totalTokens = promptTokenUsage + completionTokenUsage; return { generations, llmOutput: { estimatedTokenUsage: tokenUsage } }; } else { - const data = await this.completionWithRetry( - { - ...params, - stream: false, - messages: messagesMapped, - }, - { - signal: options?.signal, - ...options?.options, - } - ); + let data; + if ( + options.response_format && + options.response_format.type === "json_schema" + ) { + data = await this.betaParsedCompletionWithRetry( + { + ...params, + stream: false, + messages: messagesMapped, + }, + { + signal: options?.signal, + ...options?.options, + } + ); + } else { + data = await this.completionWithRetry( + { + ...params, + stream: false, + messages: messagesMapped, + }, + { + signal: options?.signal, + ...options?.options, + } + ); + } + const { completion_tokens: completionTokens, prompt_tokens: promptTokens, @@ -1478,6 +1560,31 @@ export class ChatOpenAI< }); } + /** + * Call the beta chat completions parse endpoint. This should only be called if + * response_format is set to "json_object". + * @param {OpenAIClient.Chat.ChatCompletionCreateParamsNonStreaming} request + * @param {OpenAICoreRequestOptions | undefined} options + */ + async betaParsedCompletionWithRetry( + request: OpenAIClient.Chat.ChatCompletionCreateParamsNonStreaming, + options?: OpenAICoreRequestOptions + ): Promise> { + const requestOptions = this._getClientOptions(options); + return this.caller.call(async () => { + try { + const res = await this.client.beta.chat.completions.parse( + request, + requestOptions + ); + return res; + } catch (e) { + const error = wrapOpenAIClientError(e); + throw error; + } + }); + } + protected _getClientOptions(options: OpenAICoreRequestOptions | undefined) { if (!this.client) { const openAIEndpointConfig: OpenAIEndpointConfig = { @@ -1620,6 +1727,23 @@ export class ChatOpenAI< } else { outputParser = new JsonOutputParser(); } + } else if (method === "jsonSchema") { + llm = this.bind({ + response_format: { + type: "json_schema", + json_schema: { + name: name ?? "extract", + description: schema.description, + schema, + strict: config?.strict, + }, + }, + } as Partial); + if (isZodSchema(schema)) { + outputParser = StructuredOutputParser.fromZodSchema(schema); + } else { + outputParser = new JsonOutputParser(); + } } else { let functionName = name ?? "extract"; // Is function calling diff --git a/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts b/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts index bc0328357a73..650cf0434b5f 100644 --- a/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts +++ b/libs/langchain-openai/src/tests/chat_models_structured_output.int.test.ts @@ -1,8 +1,9 @@ import { z } from "zod"; import { zodToJsonSchema } from "zod-to-json-schema"; import { ChatPromptTemplate } from "@langchain/core/prompts"; -import { AIMessage } from "@langchain/core/messages"; -import { test, expect } from "@jest/globals"; +import { AIMessage, AIMessageChunk } from "@langchain/core/messages"; +import { test, expect, describe, it } from "@jest/globals"; +import { concat } from "@langchain/core/utils/stream"; import { ChatOpenAI } from "../chat_models.js"; test("withStructuredOutput zod schema function calling", async () => { @@ -355,3 +356,172 @@ test("Passing strict true forces the model to conform to the schema", async () = expect(result.tool_calls?.[0].args).toHaveProperty("location"); console.log(result.tool_calls?.[0].args); }); + +describe("response_format: json_schema", () => { + const weatherSchema = z.object({ + city: z.string().describe("The city to get the weather for"), + state: z.string().describe("The state to get the weather for"), + zipCode: z.string().describe("The zip code to get the weather for"), + unit: z + .enum(["fahrenheit", "celsius"]) + .describe("The unit to get the weather in"), + }); + + it("can invoke", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).bind({ + response_format: { + type: "json_schema", + json_schema: { + name: "get_current_weather", + description: "Get the current weather in a location", + schema: zodToJsonSchema(weatherSchema), + strict: true, + }, + }, + }); + + const response = await model.invoke( + "What is the weather in San Francisco, 91626 CA?" + ); + const parsed = JSON.parse(response.content as string); + expect(parsed).toHaveProperty("city"); + expect(parsed).toHaveProperty("state"); + expect(parsed).toHaveProperty("zipCode"); + expect(parsed).toHaveProperty("unit"); + }); + + it("can stream", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).bind({ + response_format: { + type: "json_schema", + json_schema: { + name: "get_current_weather", + description: "Get the current weather in a location", + schema: zodToJsonSchema(weatherSchema), + strict: true, + }, + }, + }); + + const stream = await model.stream( + "What is the weather in San Francisco, 91626 CA?" + ); + let full: AIMessageChunk | undefined; + for await (const chunk of stream) { + full = !full ? chunk : concat(full, chunk); + } + expect(full).toBeDefined(); + if (!full) return; + + const parsed = JSON.parse(full.content as string); + expect(parsed).toHaveProperty("city"); + expect(parsed).toHaveProperty("state"); + expect(parsed).toHaveProperty("zipCode"); + expect(parsed).toHaveProperty("unit"); + }); + + it("can invoke with a zod schema passed in", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).bind({ + response_format: { + type: "json_schema", + json_schema: { + name: "get_current_weather", + description: "Get the current weather in a location", + schema: weatherSchema, + strict: true, + }, + }, + }); + + const response = await model.invoke( + "What is the weather in San Francisco, 91626 CA?" + ); + const parsed = JSON.parse(response.content as string); + expect(parsed).toHaveProperty("city"); + expect(parsed).toHaveProperty("state"); + expect(parsed).toHaveProperty("zipCode"); + expect(parsed).toHaveProperty("unit"); + }); + + it("can stream with a zod schema passed in", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).bind({ + response_format: { + type: "json_schema", + json_schema: { + name: "get_current_weather", + description: "Get the current weather in a location", + schema: weatherSchema, + strict: true, + }, + }, + }); + + const stream = await model.stream( + "What is the weather in San Francisco, 91626 CA?" + ); + let full: AIMessageChunk | undefined; + for await (const chunk of stream) { + full = !full ? chunk : concat(full, chunk); + } + expect(full).toBeDefined(); + if (!full) return; + + const parsed = JSON.parse(full.content as string); + expect(parsed).toHaveProperty("city"); + expect(parsed).toHaveProperty("state"); + expect(parsed).toHaveProperty("zipCode"); + expect(parsed).toHaveProperty("unit"); + }); + + it("can be invoked with WSO", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).withStructuredOutput(weatherSchema, { + name: "get_current_weather", + method: "jsonSchema", + strict: true, + }); + + const response = await model.invoke( + "What is the weather in San Francisco, 91626 CA?" + ); + expect(response).toHaveProperty("city"); + expect(response).toHaveProperty("state"); + expect(response).toHaveProperty("zipCode"); + expect(response).toHaveProperty("unit"); + }); + + it("can be streamed with WSO", async () => { + const model = new ChatOpenAI({ + model: "gpt-4o-2024-08-06", + }).withStructuredOutput(weatherSchema, { + name: "get_current_weather", + method: "jsonSchema", + strict: true, + }); + + const stream = await model.stream( + "What is the weather in San Francisco, 91626 CA?" + ); + // It should yield a single chunk + let full: z.infer | undefined; + for await (const chunk of stream) { + full = chunk; + } + expect(full).toBeDefined(); + if (!full) return; + + expect(full).toHaveProperty("city"); + expect(full).toHaveProperty("state"); + expect(full).toHaveProperty("zipCode"); + expect(full).toHaveProperty("unit"); + }); +}); diff --git a/libs/langchain-openai/src/types.ts b/libs/langchain-openai/src/types.ts index 0d93089619e2..04d8865713d4 100644 --- a/libs/langchain-openai/src/types.ts +++ b/libs/langchain-openai/src/types.ts @@ -1,7 +1,13 @@ import type { OpenAI as OpenAIClient } from "openai"; +import type { + ResponseFormatText, + ResponseFormatJSONObject, + ResponseFormatJSONSchema, +} from "openai/resources/shared"; import { TiktokenModel } from "js-tiktoken/lite"; import type { BaseLanguageModelCallOptions } from "@langchain/core/language_models/base"; +import type { z } from "zod"; // reexport this type from the included package so we can easily override and extend it if needed in the future // also makes it easier for folks to import this type without digging around into the dependent packages @@ -222,3 +228,22 @@ export declare interface AzureOpenAIInput { */ azureADTokenProvider?: () => Promise; } + +type ChatOpenAIResponseFormatJSONSchema = Omit< + ResponseFormatJSONSchema, + "json_schema" +> & { + json_schema: Omit & { + /** + * The schema for the response format, described as a JSON Schema object + * or a Zod object. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + schema: Record | z.ZodObject; + }; +}; + +export type ChatOpenAIResponseFormat = + | ResponseFormatText + | ResponseFormatJSONObject + | ChatOpenAIResponseFormatJSONSchema; diff --git a/libs/langchain-openai/src/utils/tools.ts b/libs/langchain-openai/src/utils/tools.ts new file mode 100644 index 000000000000..4a9d4e196ad1 --- /dev/null +++ b/libs/langchain-openai/src/utils/tools.ts @@ -0,0 +1,65 @@ +import { ToolDefinition } from "@langchain/core/language_models/base"; +import { BindToolsInput } from "@langchain/core/language_models/chat_models"; +import { + convertToOpenAIFunction, + isLangChainTool, +} from "@langchain/core/utils/function_calling"; +import { zodFunction } from "openai/helpers/zod"; + +/** + * Formats a tool in either OpenAI format, or LangChain structured tool format + * into an OpenAI tool format. If the tool is already in OpenAI format, return without + * any changes. If it is in LangChain structured tool format, convert it to OpenAI tool format + * using OpenAI's `zodFunction` util, falling back to `convertToOpenAIFunction` if the parameters + * returned from the `zodFunction` util are not defined. + * + * @param {BindToolsInput} tool The tool to convert to an OpenAI tool. + * @param {Object} [fields] Additional fields to add to the OpenAI tool. + * @returns {ToolDefinition} The inputted tool in OpenAI tool format. + */ +export function _convertToOpenAITool( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + tool: BindToolsInput, + fields?: { + /** + * If `true`, model output is guaranteed to exactly match the JSON Schema + * provided in the function definition. + */ + strict?: boolean; + } +): ToolDefinition { + let toolDef: ToolDefinition | undefined; + + if (isLangChainTool(tool)) { + const oaiToolDef = zodFunction({ + name: tool.name, + parameters: tool.schema, + description: tool.description, + }); + if (!oaiToolDef.function.parameters) { + // Fallback to the `convertToOpenAIFunction` util if the parameters are not defined. + toolDef = { + type: "function", + function: convertToOpenAIFunction(tool, fields), + }; + } else { + toolDef = { + type: oaiToolDef.type, + function: { + name: oaiToolDef.function.name, + description: oaiToolDef.function.description, + parameters: oaiToolDef.function.parameters, + ...(fields?.strict !== undefined ? { strict: fields.strict } : {}), + }, + }; + } + } else { + toolDef = tool as ToolDefinition; + } + + if (fields?.strict !== undefined) { + toolDef.function.strict = fields.strict; + } + + return toolDef; +}