community[major]: DeepInfra llm and chat

docs/core_docs/.gitignore

@@ -55,6 +55,8 @@ docs/how_to/tools_prompting.md
 docs/how_to/tools_prompting.mdx
 docs/how_to/tools_builtin.md
 docs/how_to/tools_builtin.mdx
+docs/how_to/tool_calls_multimodal.md
+docs/how_to/tool_calls_multimodal.mdx
 docs/how_to/tool_calling.md
 docs/how_to/tool_calling.mdx
 docs/how_to/structured_output.md
@@ -172,4 +174,4 @@ docs/how_to/assign.mdx
 docs/how_to/agent_executor.md
 docs/how_to/agent_executor.mdx
 docs/integrations/llms/mistral.md
-docs/integrations/llms/mistral.mdx
+docs/integrations/llms/mistral.mdx

examples/src/llms/deepinfra.ts

@@ -0,0 +1,18 @@
+import { DeepInfraLLM } from "@langchain/community/llms/deepinfra";
+
+const apiKey = process.env.DEEPINFRA_API_TOKEN;
+const model = "meta-llama/Meta-Llama-3-70B-Instruct";
+
+export const run = async () => {
+  const llm = new DeepInfraLLM({
+    temperature: 0.7,
+    maxTokens: 20,
+    model,
+    apiKey,
+    maxRetries: 5,
+  });
+  const res = await llm.invoke(
+    "Question: What is the next step in the process of making a good game?\nAnswer:"
+  );
+  console.log({ res });
+};

examples/src/models/chat/integration_deepinfra.ts

@@ -0,0 +1,17 @@
+import { ChatDeepInfra } from "@langchain/community/chat_models/deepinfra";
+import { HumanMessage } from "@langchain/core/messages";
+
+const apiKey = process.env.DEEPINFRA_API_TOKEN;
+
+const model = "meta-llama/Meta-Llama-3-70B-Instruct";
+
+const chat = new ChatDeepInfra({
+  model,
+  apiKey,
+});
+
+const messages = [new HumanMessage("Hello")];
+
+chat.invoke(messages).then((response: any) => {
+  console.log(response);
+});

examples/src/models/llm/deepinfra.ts

@@ -0,0 +1,14 @@
+import { DeepInfraLLM } from "@langchain/community/llms/deepinfra";
+
+const apiKey = process.env.DEEPINFRA_API_TOKEN;
+const model = "meta-llama/Meta-Llama-3-70B-Instruct";
+
+const llm = new DeepInfraLLM({
+  maxTokens: 20,
+  model,
+  apiKey,
+});
+const res = await llm.invoke(
+  "What is the next step in the process of making a good game?"
+);
+console.log({ res });

libs/langchain-community/.gitignore

@@ -234,6 +234,10 @@ llms/cohere.cjs
 llms/cohere.js
 llms/cohere.d.ts
 llms/cohere.d.cts
+llms/deepinfra.cjs
+llms/deepinfra.js
+llms/deepinfra.d.ts
+llms/deepinfra.d.cts
 llms/fireworks.cjs
 llms/fireworks.js
 llms/fireworks.d.ts
@@ -510,6 +514,10 @@ chat_models/cloudflare_workersai.cjs
 chat_models/cloudflare_workersai.js
 chat_models/cloudflare_workersai.d.ts
 chat_models/cloudflare_workersai.d.cts
+chat_models/deepinfra.cjs
+chat_models/deepinfra.js
+chat_models/deepinfra.d.ts
+chat_models/deepinfra.d.cts
 chat_models/fireworks.cjs
 chat_models/fireworks.js
 chat_models/fireworks.d.ts

libs/langchain-community/langchain.config.js

@@ -93,6 +93,7 @@ export const config = {
     "llms/bedrock/web": "llms/bedrock/web",
     "llms/cloudflare_workersai": "llms/cloudflare_workersai",
     "llms/cohere": "llms/cohere",
+    "llms/deepinfra": "llms/deepinfra",
     "llms/fireworks": "llms/fireworks",
     "llms/friendli": "llms/friendli",
     "llms/googlepalm": "llms/googlepalm",
@@ -164,6 +165,7 @@ export const config = {
     "chat_models/bedrock": "chat_models/bedrock/index",
     "chat_models/bedrock/web": "chat_models/bedrock/web",
     "chat_models/cloudflare_workersai": "chat_models/cloudflare_workersai",
+    "chat_models/deepinfra": "chat_models/deepinfra",
     "chat_models/fireworks": "chat_models/fireworks",
     "chat_models/friendli": "chat_models/friendli",
     "chat_models/googlevertexai": "chat_models/googlevertexai/index",

libs/langchain-community/package.json

@@ -1231,6 +1231,15 @@
       "import": "./llms/cohere.js",
       "require": "./llms/cohere.cjs"
     },
+    "./llms/deepinfra": {
+      "types": {
+        "import": "./llms/deepinfra.d.ts",
+        "require": "./llms/deepinfra.d.cts",
+        "default": "./llms/deepinfra.d.ts"
+      },
+      "import": "./llms/deepinfra.js",
+      "require": "./llms/deepinfra.cjs"
+    },
     "./llms/fireworks": {
       "types": {
         "import": "./llms/fireworks.d.ts",
@@ -1852,6 +1861,15 @@
       "import": "./chat_models/cloudflare_workersai.js",
       "require": "./chat_models/cloudflare_workersai.cjs"
     },
+    "./chat_models/deepinfra": {
+      "types": {
+        "import": "./chat_models/deepinfra.d.ts",
+        "require": "./chat_models/deepinfra.d.cts",
+        "default": "./chat_models/deepinfra.d.ts"
+      },
+      "import": "./chat_models/deepinfra.js",
+      "require": "./chat_models/deepinfra.cjs"
+    },
     "./chat_models/fireworks": {
       "types": {
         "import": "./chat_models/fireworks.d.ts",
@@ -3235,6 +3253,10 @@
     "llms/cohere.js",
     "llms/cohere.d.ts",
     "llms/cohere.d.cts",
+    "llms/deepinfra.cjs",
+    "llms/deepinfra.js",
+    "llms/deepinfra.d.ts",
+    "llms/deepinfra.d.cts",
     "llms/fireworks.cjs",
     "llms/fireworks.js",
     "llms/fireworks.d.ts",
@@ -3511,6 +3533,10 @@
     "chat_models/cloudflare_workersai.js",
     "chat_models/cloudflare_workersai.d.ts",
     "chat_models/cloudflare_workersai.d.cts",
+    "chat_models/deepinfra.cjs",
+    "chat_models/deepinfra.js",
+    "chat_models/deepinfra.d.ts",
+    "chat_models/deepinfra.d.cts",
     "chat_models/fireworks.cjs",
     "chat_models/fireworks.js",
     "chat_models/fireworks.d.ts",

libs/langchain-community/src/chat_models/deepinfra.ts

@@ -0,0 +1,217 @@
+import {
+  BaseChatModel,
+  type BaseChatModelParams,
+} from "@langchain/core/language_models/chat_models";
+import { AIMessage, type BaseMessage } from "@langchain/core/messages";
+import { type ChatResult } from "@langchain/core/outputs";
+import { getEnvironmentVariable } from "@langchain/core/utils/env";
+
+export const DEFAULT_MODEL = "meta-llama/Meta-Llama-3-70B-Instruct";
+
+export type DeepInfraMessageRole = "system" | "assistant" | "user";
+
+export const API_BASE_URL =
+  "https://api.deepinfra.com/v1/openai/chat/completions";
+
+export const ENV_VARIABLE_API_KEY = "DEEPINFRA_API_TOKEN";
+
+interface DeepInfraMessage {
+  role: DeepInfraMessageRole;
+  content: string;
+}
+
+interface ChatCompletionRequest {
+  model: string;
+  messages?: DeepInfraMessage[];
+  stream?: boolean;
+  max_tokens?: number | null;
+  temperature?: number | null;
+}
+
+interface BaseResponse {
+  code?: string;
+  message?: string;
+}
+
+interface ChoiceMessage {
+  role: string;
+  content: string;
+}
+
+interface ResponseChoice {
+  index: number;
+  finish_reason: "stop" | "length" | "null" | null;
+  delta: ChoiceMessage;
+  message: ChoiceMessage;
+}
+
+interface ChatCompletionResponse extends BaseResponse {
+  choices: ResponseChoice[];
+  usage: {
+    completion_tokens: number;
+    prompt_tokens: number;
+    total_tokens: number;
+  };
+  output: {
+    text: string;
+    finish_reason: "stop" | "length" | "null" | null;
+  };
+}
+
+export interface ChatDeepInfraParams {
+  model: string;
+  apiKey?: string;
+  temperature?: number;
+  maxTokens?: number;
+}
+
+function messageToRole(message: BaseMessage): DeepInfraMessageRole {
+  const type = message._getType();
+  switch (type) {
+    case "ai":
+      return "assistant";
+    case "human":
+      return "user";
+    case "system":
+      return "system";
+    default:
+      throw new Error(`Unknown message type: ${type}`);
+  }
+}
+
+export class ChatDeepInfra
+  extends BaseChatModel
+  implements ChatDeepInfraParams
+{
+  static lc_name() {
+    return "ChatDeepInfra";
+  }
+
+  get callKeys() {
+    return ["stop", "signal", "options"];
+  }
+
+  apiKey?: string;
+
+  model: string;
+
+  apiUrl: string;
+
+  maxTokens?: number;
+
+  temperature?: number;
+
+  constructor(fields: Partial<ChatDeepInfraParams> & BaseChatModelParams = {}) {
+    super(fields);
+
+    this.apiKey =
+      fields?.apiKey ?? getEnvironmentVariable(ENV_VARIABLE_API_KEY);
+    if (!this.apiKey) {
+      throw new Error(
+        "API key is required, set `DEEPINFRA_API_TOKEN` environment variable or pass it as a parameter"
+      );
+    }
+
+    this.apiUrl = API_BASE_URL;
+    this.model = fields.model ?? DEFAULT_MODEL;
+    this.temperature = fields.temperature ?? 0;
+    this.maxTokens = fields.maxTokens;
+  }
+
+  invocationParams(): Omit<ChatCompletionRequest, "messages"> {
+    return {
+      model: this.model,
+      stream: false,
+      temperature: this.temperature,
+      max_tokens: this.maxTokens,
+    };
+  }
+
+  identifyingParams(): Omit<ChatCompletionRequest, "messages"> {
+    return this.invocationParams();
+  }
+
+  async _generate(
+    messages: BaseMessage[],
+    options?: this["ParsedCallOptions"]
+  ): Promise<ChatResult> {
+    const parameters = this.invocationParams();
+
+    const messagesMapped: DeepInfraMessage[] = messages.map((message) => ({
+      role: messageToRole(message),
+      content: message.content as string,
+    }));
+
+    const data = await this.completionWithRetry(
+      { ...parameters, messages: messagesMapped },
+      false,
+      options?.signal
+    ).then<ChatCompletionResponse>((data) => {
+
+      if (data?.code) {
+        throw new Error(data?.message);
+      }
+      const { finish_reason, message } = data.choices[0];
+      const text = message.content;
+      return {
+        ...data,
+        output: { text, finish_reason },
+      };
+    });
+
+    const {
+      prompt_tokens = 0,
+      completion_tokens = 0,
+      total_tokens = 0,
+    } = data.usage ?? {};
+
+    const { text } = data.output;
+
+    return {
+      generations: [{ text, message: new AIMessage(text) }],
+      llmOutput: {
+        tokenUsage: {
+          promptTokens: prompt_tokens,
+          completionTokens: completion_tokens,
+          totalTokens: total_tokens,
+        },
+      },
+    };
+  }
+
+  async completionWithRetry(
+    request: ChatCompletionRequest,
+    stream: boolean,
+    signal?: AbortSignal
+  ) {
+
+    const body = {
+      temperature: this.temperature,
+      max_tokens: this.maxTokens,
+      ...request,
+      model: this.model,
+    };
+
+    const makeCompletionRequest = async () => {
+      const response = await fetch(this.apiUrl, {
+        method: "POST",
+        headers: {
+          Authorization: `Bearer ${this.apiKey}`,
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(body),
+        signal,
+      });
+
+      if (!stream) {
+        return response.json();
+      }
+    };
+
+    return this.caller.call(makeCompletionRequest);
+  }
+
+  _llmType(): string {
+    return "DeepInfra";
+  }
+}

libs/langchain-community/src/chat_models/tests/chatdeepinfra.int.test.ts

@@ -0,0 +1,19 @@
+import { test } from "@jest/globals";
+import { HumanMessage } from "@langchain/core/messages";
+import { ChatDeepInfra } from "../deepinfra.js";
+
+describe("ChatDeepInfra", () => {
+  test("call", async () => {
+    const deepInfraChat = new ChatDeepInfra({ maxTokens: 20 });
+    const message = new HumanMessage("1 + 1 = ");
+    const res = await deepInfraChat.invoke([message]);
+    console.log({ res });
+  });
+
+  test("generate", async () => {
+    const deepInfraChat = new ChatDeepInfra({ maxTokens: 20 });
+    const message = new HumanMessage("1 + 1 = ");
+    const res = await deepInfraChat.generate([[message]]);
+    console.log(JSON.stringify(res, null, 2));
+  });
+});