From 5ba4faf6472e1dbff7a6722a4b88df4ce5594815 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 4 Aug 2024 07:26:52 +0800 Subject: [PATCH 01/19] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20Fireworks=20AI?= =?UTF-8?q?=20Model=20Provider?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../settings/llm/ProviderList/providers.tsx | 7 + src/app/api/chat/agentRuntime.ts | 7 + src/components/ModelProviderIcon/index.tsx | 5 + src/components/ModelTag/ModelIcon.tsx | 2 +- src/config/llm.ts | 8 + src/config/modelProviders/fireworksai.ts | 78 ++++++ src/config/modelProviders/index.ts | 4 + src/const/settings/llm.ts | 5 + .../Error/APIKeyForm/ProviderAvatar.tsx | 5 + src/libs/agent-runtime/AgentRuntime.ts | 7 + .../agent-runtime/fireworksai/index.test.ts | 255 ++++++++++++++++++ src/libs/agent-runtime/fireworksai/index.ts | 10 + src/libs/agent-runtime/types/type.ts | 1 + src/server/globalConfig/index.ts | 14 + src/types/user/settings/keyVaults.ts | 1 + 15 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 src/config/modelProviders/fireworksai.ts create mode 100644 src/libs/agent-runtime/fireworksai/index.test.ts create mode 100644 src/libs/agent-runtime/fireworksai/index.ts diff --git a/src/app/(main)/settings/llm/ProviderList/providers.tsx b/src/app/(main)/settings/llm/ProviderList/providers.tsx index 30aa944167b1..aa8e81b323b0 100644 --- a/src/app/(main)/settings/llm/ProviderList/providers.tsx +++ b/src/app/(main)/settings/llm/ProviderList/providers.tsx @@ -5,6 +5,7 @@ import { Baichuan, Claude, DeepSeek, + Fireworks, Gemini, Google, Groq, @@ -31,6 +32,7 @@ import { AnthropicProviderCard, BaichuanProviderCard, DeepSeekProviderCard, + FireworksAIProviderCard, GoogleProviderCard, GroqProviderCard, MinimaxProviderCard, @@ -138,6 +140,11 @@ export const useProviderList = (): ProviderItem[] => { docUrl: urlJoin(BASE_DOC_URL, 'togetherai'), title: , }, + { + ...FireworksAIProviderCard, + docUrl: urlJoin(BASE_DOC_URL, 'fireworksai'), + title: , + }, { ...QwenProviderCard, docUrl: urlJoin(BASE_DOC_URL, 'qwen'), diff --git a/src/app/api/chat/agentRuntime.ts b/src/app/api/chat/agentRuntime.ts index e5ff047ef3fc..da6a6f964ce2 100644 --- a/src/app/api/chat/agentRuntime.ts +++ b/src/app/api/chat/agentRuntime.ts @@ -151,6 +151,13 @@ const getLlmOptionsFromPayload = (provider: string, payload: JWTPayload) => { return { apiKey }; } + case ModelProvider.FireworksAI: { + const { FIREWORKSAI_API_KEY } = getLLMConfig(); + + const apiKey = apiKeyManager.pick(payload?.apiKey || FIREWORKSAI_API_KEY); + + return { apiKey }; + } case ModelProvider.ZeroOne: { const { ZEROONE_API_KEY } = getLLMConfig(); diff --git a/src/components/ModelProviderIcon/index.tsx b/src/components/ModelProviderIcon/index.tsx index de6270991684..e56620f2e715 100644 --- a/src/components/ModelProviderIcon/index.tsx +++ b/src/components/ModelProviderIcon/index.tsx @@ -6,6 +6,7 @@ import { Baichuan, Bedrock, DeepSeek, + Fireworks, Google, Groq, LobeHub, @@ -110,6 +111,10 @@ const ModelProviderIcon = memo(({ provider }) => { return ; } + case ModelProvider.FireworksAI: { + return ; + } + case ModelProvider.Qwen: { return ; } diff --git a/src/components/ModelTag/ModelIcon.tsx b/src/components/ModelTag/ModelIcon.tsx index 94cc55c2a1ee..7e7b58ba837c 100644 --- a/src/components/ModelTag/ModelIcon.tsx +++ b/src/components/ModelTag/ModelIcon.tsx @@ -97,7 +97,7 @@ const ModelIcon = memo(({ model: originModel, size = 12 }) => { ) return ; - if (model.includes('phi3') || model.includes('phi-3') || model.includes('wizardlm')) return ; + if (model.includes('phi3') || model.includes('phi-3') || model.includes('wizardlm')) return ; if (model.includes('firefly')) return ; if (model.includes('jamba') || model.includes('j2-')) return ; }); diff --git a/src/config/llm.ts b/src/config/llm.ts index b745e7a235bc..3b9c64794990 100644 --- a/src/config/llm.ts +++ b/src/config/llm.ts @@ -61,6 +61,10 @@ export const getLLMConfig = () => { TOGETHERAI_API_KEY: z.string().optional(), TOGETHERAI_MODEL_LIST: z.string().optional(), + ENABLED_FIREWORKSAI: z.boolean(), + FIREWORKSAI_API_KEY: z.string().optional(), + FIREWORKSAI_MODEL_LIST: z.string().optional(), + ENABLED_AWS_BEDROCK: z.boolean(), AWS_REGION: z.string().optional(), AWS_ACCESS_KEY_ID: z.string().optional(), @@ -134,6 +138,10 @@ export const getLLMConfig = () => { TOGETHERAI_API_KEY: process.env.TOGETHERAI_API_KEY, TOGETHERAI_MODEL_LIST: process.env.TOGETHERAI_MODEL_LIST, + ENABLED_FIREWORKSAI: !!process.env.FIREWORKSAI_API_KEY, + FIREWORKSAI_API_KEY: process.env.FIREWORKSAI_API_KEY, + FIREWORKSAI_MODEL_LIST: process.env.FIREWORKSAI_MODEL_LIST, + ENABLED_MOONSHOT: !!process.env.MOONSHOT_API_KEY, MOONSHOT_API_KEY: process.env.MOONSHOT_API_KEY, MOONSHOT_PROXY_URL: process.env.MOONSHOT_PROXY_URL, diff --git a/src/config/modelProviders/fireworksai.ts b/src/config/modelProviders/fireworksai.ts new file mode 100644 index 000000000000..5d0fe0fb4852 --- /dev/null +++ b/src/config/modelProviders/fireworksai.ts @@ -0,0 +1,78 @@ +import { ModelProviderCard } from '@/types/llm'; + +// ref https://fireworks.ai/models?show=Serverless +// ref https://fireworks.ai/pricing +const FireworksAI: ModelProviderCard = { + chatModels: [ + { + displayName: 'Llama3.1 8B Instruct', + enabled: true, + functionCall: false, + id: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + tokens: 131_072, + }, + { + displayName: 'Llama3.1 70B Instruct', + enabled: true, + functionCall: false, + id: 'accounts/fireworks/models/llama-v3p1-70b-instruct', + tokens: 131_072, + }, + { + displayName: 'Llama3.1 405B Instruct', + enabled: true, + functionCall: false, + id: 'accounts/fireworks/models/llama-v3p1-405b-instruct', + tokens: 131_072, + }, + { + displayName: 'Gemma 2 9B Instruct', + enabled: true, + functionCall: false, + id: 'accounts/fireworks/models/gemma2-9b-it', + tokens: 8192, + }, + { + displayName: 'Mixtral MoE 8x7B Instruct', + enabled: true, + functionCall: false, + id: 'accounts/fireworks/models/mixtral-8x7b-instruct', + tokens: 32_768, + }, + { + displayName: 'Mixtral MoE 8x22B Instruct', + enabled: true, + functionCall: false, + id: 'accounts/fireworks/models/mixtral-8x22b-instruct', + tokens: 65_536, + }, + { + displayName: 'Phi 3 Vision 128K Instruct', + enabled: true, + functionCall: false, + id: 'accounts/fireworks/models/phi-3-vision-128k-instruct', + tokens: 131_072, + vision: true, + }, + { + displayName: 'DeepSeek Coder V2 Instruct', + enabled: true, + functionCall: false, + id: 'accounts/fireworks/models/deepseek-coder-v2-instruct', + tokens: 131_072, + }, + { + displayName: 'Qwen2 72b Instruct', + enabled: true, + functionCall: false, + id: 'accounts/fireworks/models/qwen2-72b-instruct', + tokens: 32_768, + }, + ], + checkModel: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + id: 'fireworksai', + modelList: { showModelFetcher: true }, + name: 'Fireworks AI', +}; + +export default FireworksAI; diff --git a/src/config/modelProviders/index.ts b/src/config/modelProviders/index.ts index db5ae1446e8b..0b67305334cb 100644 --- a/src/config/modelProviders/index.ts +++ b/src/config/modelProviders/index.ts @@ -6,6 +6,7 @@ import AzureProvider from './azure'; import BaichuanProvider from './baichuan'; import BedrockProvider from './bedrock'; import DeepSeekProvider from './deepseek'; +import FireworksAIProvider from './fireworksai'; import GoogleProvider from './google'; import GroqProvider from './groq'; import MinimaxProvider from './minimax'; @@ -37,6 +38,7 @@ export const LOBE_DEFAULT_MODEL_LIST: ChatModelCard[] = [ OllamaProvider.chatModels, OpenRouterProvider.chatModels, TogetherAIProvider.chatModels, + FireworksAIProvider.chatModels, PerplexityProvider.chatModels, AnthropicProvider.chatModels, ZeroOneProvider.chatModels, @@ -57,6 +59,7 @@ export const DEFAULT_MODEL_PROVIDER_LIST = [ GoogleProvider, OpenRouterProvider, TogetherAIProvider, + FireworksAIProvider, BedrockProvider, PerplexityProvider, MinimaxProvider, @@ -87,6 +90,7 @@ export { default as AzureProviderCard } from './azure'; export { default as BaichuanProviderCard } from './baichuan'; export { default as BedrockProviderCard } from './bedrock'; export { default as DeepSeekProviderCard } from './deepseek'; +export { default as FireworksAIProviderCard } from './fireworksai'; export { default as GoogleProviderCard } from './google'; export { default as GroqProviderCard } from './groq'; export { default as MinimaxProviderCard } from './minimax'; diff --git a/src/const/settings/llm.ts b/src/const/settings/llm.ts index 1cd98e069ae8..ed57fef116ee 100644 --- a/src/const/settings/llm.ts +++ b/src/const/settings/llm.ts @@ -4,6 +4,7 @@ import { BaichuanProviderCard, BedrockProviderCard, DeepSeekProviderCard, + FireworksAIProviderCard, GoogleProviderCard, GroqProviderCard, MinimaxProviderCard, @@ -49,6 +50,10 @@ export const DEFAULT_LLM_CONFIG: UserModelProviderConfig = { enabled: false, enabledModels: filterEnabledModels(DeepSeekProviderCard), }, + fireworksai: { + enabled: false, + enabledModels: filterEnabledModels(FireworksAIProviderCard), + }, google: { enabled: false, enabledModels: filterEnabledModels(GoogleProviderCard), diff --git a/src/features/Conversation/Error/APIKeyForm/ProviderAvatar.tsx b/src/features/Conversation/Error/APIKeyForm/ProviderAvatar.tsx index 35dfda03f61c..1c2ec81d89a0 100644 --- a/src/features/Conversation/Error/APIKeyForm/ProviderAvatar.tsx +++ b/src/features/Conversation/Error/APIKeyForm/ProviderAvatar.tsx @@ -4,6 +4,7 @@ import { Anthropic, Baichuan, DeepSeek, + Fireworks, Google, Groq, Minimax, @@ -92,6 +93,10 @@ const ProviderAvatar = memo(({ provider }) => { return ; } + case ModelProvider.FireworksAI: { + return ; + } + case ModelProvider.ZeroOne: { return ; } diff --git a/src/libs/agent-runtime/AgentRuntime.ts b/src/libs/agent-runtime/AgentRuntime.ts index fdb28eb25b3f..f657594124b2 100644 --- a/src/libs/agent-runtime/AgentRuntime.ts +++ b/src/libs/agent-runtime/AgentRuntime.ts @@ -9,6 +9,7 @@ import { LobeAzureOpenAI } from './azureOpenai'; import { LobeBaichuanAI } from './baichuan'; import { LobeBedrockAI, LobeBedrockAIParams } from './bedrock'; import { LobeDeepSeekAI } from './deepseek'; +import { LobeFireworksAI } from './fireworksai'; import { LobeGoogleAI } from './google'; import { LobeGroq } from './groq'; import { LobeMinimaxAI } from './minimax'; @@ -111,6 +112,7 @@ class AgentRuntime { baichuan: Partial; bedrock: Partial; deepseek: Partial; + fireworksai: Partial; google: { apiKey?: string; baseURL?: string }; groq: Partial; minimax: Partial; @@ -213,6 +215,11 @@ class AgentRuntime { break; } + case ModelProvider.FireworksAI: { + runtimeModel = new LobeFireworksAI(params.fireworksai); + break + } + case ModelProvider.ZeroOne: { runtimeModel = new LobeZeroOneAI(params.zeroone); break; diff --git a/src/libs/agent-runtime/fireworksai/index.test.ts b/src/libs/agent-runtime/fireworksai/index.test.ts new file mode 100644 index 000000000000..0461a19b9655 --- /dev/null +++ b/src/libs/agent-runtime/fireworksai/index.test.ts @@ -0,0 +1,255 @@ +// @vitest-environment node +import OpenAI from 'openai'; +import { Mock, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + ChatStreamCallbacks, + LobeOpenAICompatibleRuntime, + ModelProvider, +} from '@/libs/agent-runtime'; + +import * as debugStreamModule from '../utils/debugStream'; +import { LobeFireworksAI } from './index'; + +const provider = ModelProvider.FireworksAI; +const defaultBaseURL = 'https://api.fireworks.ai/inference/v1'; + +const bizErrorType = 'ProviderBizError'; +const invalidErrorType = 'InvalidProviderAPIKey'; + +// Mock the console.error to avoid polluting test output +vi.spyOn(console, 'error').mockImplementation(() => {}); + +let instance: LobeOpenAICompatibleRuntime; + +beforeEach(() => { + instance = new LobeFireworksAI({ apiKey: 'test' }); + + // 使用 vi.spyOn 来模拟 chat.completions.create 方法 + vi.spyOn(instance['client'].chat.completions, 'create').mockResolvedValue( + new ReadableStream() as any, + ); +}); + +afterEach(() => { + vi.clearAllMocks(); +}); + +describe('LobeFireworksAI', () => { + describe('init', () => { + it('should correctly initialize with an API key', async () => { + const instance = new LobeFireworksAI({ apiKey: 'test_api_key' }); + expect(instance).toBeInstanceOf(LobeFireworksAI); + expect(instance.baseURL).toEqual(defaultBaseURL); + }); + }); + + describe('chat', () => { + describe('Error', () => { + it('should return OpenAIBizError with an openai error response when OpenAI.APIError is thrown', async () => { + // Arrange + const apiError = new OpenAI.APIError( + 400, + { + status: 400, + error: { + message: 'Bad Request', + }, + }, + 'Error message', + {}, + ); + + vi.spyOn(instance['client'].chat.completions, 'create').mockRejectedValue(apiError); + + // Act + try { + await instance.chat({ + messages: [{ content: 'Hello', role: 'user' }], + model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + temperature: 0, + }); + } catch (e) { + expect(e).toEqual({ + endpoint: defaultBaseURL, + error: { + error: { message: 'Bad Request' }, + status: 400, + }, + errorType: bizErrorType, + provider, + }); + } + }); + + it('should throw AgentRuntimeError with NoOpenAIAPIKey if no apiKey is provided', async () => { + try { + new LobeFireworksAI({}); + } catch (e) { + expect(e).toEqual({ errorType: invalidErrorType }); + } + }); + + it('should return OpenAIBizError with the cause when OpenAI.APIError is thrown with cause', async () => { + // Arrange + const errorInfo = { + stack: 'abc', + cause: { + message: 'api is undefined', + }, + }; + const apiError = new OpenAI.APIError(400, errorInfo, 'module error', {}); + + vi.spyOn(instance['client'].chat.completions, 'create').mockRejectedValue(apiError); + + // Act + try { + await instance.chat({ + messages: [{ content: 'Hello', role: 'user' }], + model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + temperature: 0, + }); + } catch (e) { + expect(e).toEqual({ + endpoint: defaultBaseURL, + error: { + cause: { message: 'api is undefined' }, + stack: 'abc', + }, + errorType: bizErrorType, + provider, + }); + } + }); + + it('should return OpenAIBizError with an cause response with desensitize Url', async () => { + // Arrange + const errorInfo = { + stack: 'abc', + cause: { message: 'api is undefined' }, + }; + const apiError = new OpenAI.APIError(400, errorInfo, 'module error', {}); + + instance = new LobeFireworksAI({ + apiKey: 'test', + + baseURL: 'https://api.abc.com/v1', + }); + + vi.spyOn(instance['client'].chat.completions, 'create').mockRejectedValue(apiError); + + // Act + try { + await instance.chat({ + messages: [{ content: 'Hello', role: 'user' }], + model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + temperature: 0, + }); + } catch (e) { + expect(e).toEqual({ + endpoint: 'https://api.***.com/v1', + error: { + cause: { message: 'api is undefined' }, + stack: 'abc', + }, + errorType: bizErrorType, + provider, + }); + } + }); + + it('should throw an InvalidFireworksAIAPIKey error type on 401 status code', async () => { + // Mock the API call to simulate a 401 error + const error = new Error('Unauthorized') as any; + error.status = 401; + vi.mocked(instance['client'].chat.completions.create).mockRejectedValue(error); + + try { + await instance.chat({ + messages: [{ content: 'Hello', role: 'user' }], + model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + temperature: 0, + }); + } catch (e) { + // Expect the chat method to throw an error with InvalidFireworksAIAPIKey + expect(e).toEqual({ + endpoint: defaultBaseURL, + error: new Error('Unauthorized'), + errorType: invalidErrorType, + provider, + }); + } + }); + + it('should return AgentRuntimeError for non-OpenAI errors', async () => { + // Arrange + const genericError = new Error('Generic Error'); + + vi.spyOn(instance['client'].chat.completions, 'create').mockRejectedValue(genericError); + + // Act + try { + await instance.chat({ + messages: [{ content: 'Hello', role: 'user' }], + model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + temperature: 0, + }); + } catch (e) { + expect(e).toEqual({ + endpoint: defaultBaseURL, + errorType: 'AgentRuntimeError', + provider, + error: { + name: genericError.name, + cause: genericError.cause, + message: genericError.message, + stack: genericError.stack, + }, + }); + } + }); + }); + + describe('DEBUG', () => { + it('should call debugStream and return StreamingTextResponse when DEBUG_FIREWORKSAI_CHAT_COMPLETION is 1', async () => { + // Arrange + const mockProdStream = new ReadableStream() as any; // 模拟的 prod 流 + const mockDebugStream = new ReadableStream({ + start(controller) { + controller.enqueue('Debug stream content'); + controller.close(); + }, + }) as any; + mockDebugStream.toReadableStream = () => mockDebugStream; // 添加 toReadableStream 方法 + + // 模拟 chat.completions.create 返回值，包括模拟的 tee 方法 + (instance['client'].chat.completions.create as Mock).mockResolvedValue({ + tee: () => [mockProdStream, { toReadableStream: () => mockDebugStream }], + }); + + // 保存原始环境变量值 + const originalDebugValue = process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION; + + // 模拟环境变量 + process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION = '1'; + vi.spyOn(debugStreamModule, 'debugStream').mockImplementation(() => Promise.resolve()); + + // 执行测试 + // 运行你的测试函数，确保它会在条件满足时调用 debugStream + // 假设的测试函数调用，你可能需要根据实际情况调整 + await instance.chat({ + messages: [{ content: 'Hello', role: 'user' }], + model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + stream: true, + temperature: 0, + }); + + // 验证 debugStream 被调用 + expect(debugStreamModule.debugStream).toHaveBeenCalled(); + + // 恢复原始环境变量值 + process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION = originalDebugValue; + }); + }); + }); +}); diff --git a/src/libs/agent-runtime/fireworksai/index.ts b/src/libs/agent-runtime/fireworksai/index.ts new file mode 100644 index 000000000000..3f3fe872d8fb --- /dev/null +++ b/src/libs/agent-runtime/fireworksai/index.ts @@ -0,0 +1,10 @@ +import { ModelProvider } from '../types'; +import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory'; + +export const LobeFireworksAI = LobeOpenAICompatibleFactory({ + baseURL: 'https://api.fireworks.ai/inference/v1', + debug: { + chatCompletion: () => process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION === '1', + }, + provider: ModelProvider.FireworksAI, +}); diff --git a/src/libs/agent-runtime/types/type.ts b/src/libs/agent-runtime/types/type.ts index b2ebbc83e82f..eb619ad181a8 100644 --- a/src/libs/agent-runtime/types/type.ts +++ b/src/libs/agent-runtime/types/type.ts @@ -28,6 +28,7 @@ export enum ModelProvider { Baichuan = 'baichuan', Bedrock = 'bedrock', DeepSeek = 'deepseek', + FireworksAI = 'fireworksai', Google = 'google', Groq = 'groq', Minimax = 'minimax', diff --git a/src/server/globalConfig/index.ts b/src/server/globalConfig/index.ts index a12fa0d94f03..46caf5a50c6f 100644 --- a/src/server/globalConfig/index.ts +++ b/src/server/globalConfig/index.ts @@ -7,6 +7,7 @@ import { OpenAIProviderCard, OpenRouterProviderCard, TogetherAIProviderCard, + FireworksAIProviderCard, } from '@/config/modelProviders'; import { enableNextAuth } from '@/const/auth'; import { parseSystemAgent } from '@/server/globalConfig/parseSystemAgent'; @@ -52,6 +53,9 @@ export const getServerGlobalConfig = () => { ENABLED_ZEROONE, ENABLED_TOGETHERAI, TOGETHERAI_MODEL_LIST, + + ENABLED_FIREWORKSAI, + FIREWORKSAI_MODEL_LIST, } = getLLMConfig(); const config: GlobalServerConfig = { @@ -78,6 +82,16 @@ export const getServerGlobalConfig = () => { baichuan: { enabled: ENABLED_BAICHUAN }, bedrock: { enabled: ENABLED_AWS_BEDROCK }, deepseek: { enabled: ENABLED_DEEPSEEK }, + + fireworksai: { + enabled: ENABLED_FIREWORKSAI, + enabledModels: extractEnabledModels(FIREWORKSAI_MODEL_LIST), + serverModelCards: transformToChatModelCards({ + defaultChatModels: FireworksAIProviderCard.chatModels, + modelString: FIREWORKSAI_MODEL_LIST, + }), + }, + google: { enabled: ENABLED_GOOGLE }, groq: { enabled: ENABLED_GROQ }, minimax: { enabled: ENABLED_MINIMAX }, diff --git a/src/types/user/settings/keyVaults.ts b/src/types/user/settings/keyVaults.ts index 46fc0db51254..9f91fde3229a 100644 --- a/src/types/user/settings/keyVaults.ts +++ b/src/types/user/settings/keyVaults.ts @@ -22,6 +22,7 @@ export interface UserKeyVaults { baichuan?: OpenAICompatibleKeyVault; bedrock?: AWSBedrockKeyVault; deepseek?: OpenAICompatibleKeyVault; + fireworksai?: OpenAICompatibleKeyVault; google?: OpenAICompatibleKeyVault; groq?: OpenAICompatibleKeyVault; lobehub?: any; From 04e34361bc5fb8fafb4fa246c7b3f16aaa82f2f9 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 4 Aug 2024 09:35:09 +0800 Subject: [PATCH 02/19] =?UTF-8?q?=F0=9F=92=84=20style:=20add=20FireworksAI?= =?UTF-8?q?=20self-models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../settings/llm/ProviderList/providers.tsx | 2 +- src/components/ModelIcon/index.tsx | 2 ++ src/components/ModelTag/ModelIcon.tsx | 2 ++ src/config/modelProviders/fireworksai.ts | 17 ++++++++++++++++- .../agent-runtime/fireworksai/index.test.ts | 12 ++++++------ 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/app/(main)/settings/llm/ProviderList/providers.tsx b/src/app/(main)/settings/llm/ProviderList/providers.tsx index aa8e81b323b0..2a963e8b27bc 100644 --- a/src/app/(main)/settings/llm/ProviderList/providers.tsx +++ b/src/app/(main)/settings/llm/ProviderList/providers.tsx @@ -143,7 +143,7 @@ export const useProviderList = (): ProviderItem[] => { { ...FireworksAIProviderCard, docUrl: urlJoin(BASE_DOC_URL, 'fireworksai'), - title: , + title: , }, { ...QwenProviderCard, diff --git a/src/components/ModelIcon/index.tsx b/src/components/ModelIcon/index.tsx index ae6ed3281bdb..4ed13503dde3 100644 --- a/src/components/ModelIcon/index.tsx +++ b/src/components/ModelIcon/index.tsx @@ -14,6 +14,7 @@ import { Cohere, Dbrx, DeepSeek, + Fireworks, FishAudio, Gemini, Gemma, @@ -56,6 +57,7 @@ const ModelIcon = memo(({ model: originModel, size = 12 if (model.includes('deepseek')) return ; if (model.includes('claude')) return ; if (model.includes('titan')) return ; + if (model.includes('accounts/fireworks/models/fire')) return ; if (model.includes('llama')) return ; if (model.includes('llava')) return ; if (model.includes('gemini')) return ; diff --git a/src/components/ModelTag/ModelIcon.tsx b/src/components/ModelTag/ModelIcon.tsx index 7e7b58ba837c..4be0d820cbe7 100644 --- a/src/components/ModelTag/ModelIcon.tsx +++ b/src/components/ModelTag/ModelIcon.tsx @@ -14,6 +14,7 @@ import { Cohere, Dbrx, DeepSeek, + Fireworks, FishAudio, Gemini, Gemma, @@ -55,6 +56,7 @@ const ModelIcon = memo(({ model: originModel, size = 12 }) => { if (model.includes('claude')) return ; if (model.includes('deepseek')) return ; if (model.includes('titan')) return ; + if (model.includes('accounts/fireworks/models/fire')) return ; if (model.includes('llama')) return ; if (model.includes('llava')) return ; if (model.includes('gemini')) return ; diff --git a/src/config/modelProviders/fireworksai.ts b/src/config/modelProviders/fireworksai.ts index 5d0fe0fb4852..bb0301296d57 100644 --- a/src/config/modelProviders/fireworksai.ts +++ b/src/config/modelProviders/fireworksai.ts @@ -4,6 +4,21 @@ import { ModelProviderCard } from '@/types/llm'; // ref https://fireworks.ai/pricing const FireworksAI: ModelProviderCard = { chatModels: [ + { + displayName: 'Firefunction V2', + enabled: true, + //functionCall: true, + id: 'accounts/fireworks/models/firefunction-v2', + tokens: 8192, + }, + { + displayName: 'FireLLaVA-13B', + enabled: true, + functionCall: false, + id: 'accounts/fireworks/models/firellava-13b', + tokens: 4096, + vision: true, + }, { displayName: 'Llama3.1 8B Instruct', enabled: true, @@ -69,7 +84,7 @@ const FireworksAI: ModelProviderCard = { tokens: 32_768, }, ], - checkModel: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + checkModel: 'accounts/fireworks/models/firefunction-v2', id: 'fireworksai', modelList: { showModelFetcher: true }, name: 'Fireworks AI', diff --git a/src/libs/agent-runtime/fireworksai/index.test.ts b/src/libs/agent-runtime/fireworksai/index.test.ts index 0461a19b9655..16c4c4e64065 100644 --- a/src/libs/agent-runtime/fireworksai/index.test.ts +++ b/src/libs/agent-runtime/fireworksai/index.test.ts @@ -66,7 +66,7 @@ describe('LobeFireworksAI', () => { try { await instance.chat({ messages: [{ content: 'Hello', role: 'user' }], - model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + model: 'accounts/fireworks/models/firefunction-v2', temperature: 0, }); } catch (e) { @@ -106,7 +106,7 @@ describe('LobeFireworksAI', () => { try { await instance.chat({ messages: [{ content: 'Hello', role: 'user' }], - model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + model: 'accounts/fireworks/models/firefunction-v2', temperature: 0, }); } catch (e) { @@ -142,7 +142,7 @@ describe('LobeFireworksAI', () => { try { await instance.chat({ messages: [{ content: 'Hello', role: 'user' }], - model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + model: 'accounts/fireworks/models/firefunction-v2', temperature: 0, }); } catch (e) { @@ -167,7 +167,7 @@ describe('LobeFireworksAI', () => { try { await instance.chat({ messages: [{ content: 'Hello', role: 'user' }], - model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + model: 'accounts/fireworks/models/firefunction-v2', temperature: 0, }); } catch (e) { @@ -191,7 +191,7 @@ describe('LobeFireworksAI', () => { try { await instance.chat({ messages: [{ content: 'Hello', role: 'user' }], - model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + model: 'accounts/fireworks/models/firefunction-v2', temperature: 0, }); } catch (e) { @@ -239,7 +239,7 @@ describe('LobeFireworksAI', () => { // 假设的测试函数调用，你可能需要根据实际情况调整 await instance.chat({ messages: [{ content: 'Hello', role: 'user' }], - model: 'accounts/fireworks/models/llama-v3p1-8b-instruct', + model: 'accounts/fireworks/models/firefunction-v2', stream: true, temperature: 0, }); From c8422f494ccbd6dbf9542b45a44a6adf01dd1eb1 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 4 Aug 2024 10:54:10 +0800 Subject: [PATCH 03/19] =?UTF-8?q?=F0=9F=90=9B=20fix:=20support=20function?= =?UTF-8?q?=20call?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config/modelProviders/fireworksai.ts | 4 +- .../agent-runtime/fireworksai/index.test.ts | 43 ------------------- src/libs/agent-runtime/fireworksai/index.ts | 10 +++++ 3 files changed, 13 insertions(+), 44 deletions(-) diff --git a/src/config/modelProviders/fireworksai.ts b/src/config/modelProviders/fireworksai.ts index bb0301296d57..8f3781173d4a 100644 --- a/src/config/modelProviders/fireworksai.ts +++ b/src/config/modelProviders/fireworksai.ts @@ -5,13 +5,15 @@ import { ModelProviderCard } from '@/types/llm'; const FireworksAI: ModelProviderCard = { chatModels: [ { + description: 'Fireworks latest and most performant function-calling model. Firefunction-v2 is based on Llama-3 and trained to excel at function-calling as well as chat and instruction-following. See blog post for more details https://fireworks.ai/blog/firefunction-v2-launch-post', displayName: 'Firefunction V2', enabled: true, - //functionCall: true, + functionCall: true, id: 'accounts/fireworks/models/firefunction-v2', tokens: 8192, }, { + description: 'Vision-language model allowing both image and text as inputs (single image is recommended), trained on OSS model generated training data and open sourced on huggingface at fireworks-ai/FireLLaVA-13b', displayName: 'FireLLaVA-13B', enabled: true, functionCall: false, diff --git a/src/libs/agent-runtime/fireworksai/index.test.ts b/src/libs/agent-runtime/fireworksai/index.test.ts index 16c4c4e64065..cbd740279a38 100644 --- a/src/libs/agent-runtime/fireworksai/index.test.ts +++ b/src/libs/agent-runtime/fireworksai/index.test.ts @@ -8,7 +8,6 @@ import { ModelProvider, } from '@/libs/agent-runtime'; -import * as debugStreamModule from '../utils/debugStream'; import { LobeFireworksAI } from './index'; const provider = ModelProvider.FireworksAI; @@ -209,47 +208,5 @@ describe('LobeFireworksAI', () => { } }); }); - - describe('DEBUG', () => { - it('should call debugStream and return StreamingTextResponse when DEBUG_FIREWORKSAI_CHAT_COMPLETION is 1', async () => { - // Arrange - const mockProdStream = new ReadableStream() as any; // 模拟的 prod 流 - const mockDebugStream = new ReadableStream({ - start(controller) { - controller.enqueue('Debug stream content'); - controller.close(); - }, - }) as any; - mockDebugStream.toReadableStream = () => mockDebugStream; // 添加 toReadableStream 方法 - - // 模拟 chat.completions.create 返回值，包括模拟的 tee 方法 - (instance['client'].chat.completions.create as Mock).mockResolvedValue({ - tee: () => [mockProdStream, { toReadableStream: () => mockDebugStream }], - }); - - // 保存原始环境变量值 - const originalDebugValue = process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION; - - // 模拟环境变量 - process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION = '1'; - vi.spyOn(debugStreamModule, 'debugStream').mockImplementation(() => Promise.resolve()); - - // 执行测试 - // 运行你的测试函数，确保它会在条件满足时调用 debugStream - // 假设的测试函数调用，你可能需要根据实际情况调整 - await instance.chat({ - messages: [{ content: 'Hello', role: 'user' }], - model: 'accounts/fireworks/models/firefunction-v2', - stream: true, - temperature: 0, - }); - - // 验证 debugStream 被调用 - expect(debugStreamModule.debugStream).toHaveBeenCalled(); - - // 恢复原始环境变量值 - process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION = originalDebugValue; - }); - }); }); }); diff --git a/src/libs/agent-runtime/fireworksai/index.ts b/src/libs/agent-runtime/fireworksai/index.ts index 3f3fe872d8fb..dbca228437c9 100644 --- a/src/libs/agent-runtime/fireworksai/index.ts +++ b/src/libs/agent-runtime/fireworksai/index.ts @@ -1,8 +1,18 @@ +import OpenAI from 'openai'; + import { ModelProvider } from '../types'; import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory'; export const LobeFireworksAI = LobeOpenAICompatibleFactory({ baseURL: 'https://api.fireworks.ai/inference/v1', + chatCompletion: { + handlePayload: (payload) => { + return { + ...payload, + stream: false, + } as unknown as OpenAI.ChatCompletionCreateParamsStreaming; + }, + }, debug: { chatCompletion: () => process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION === '1', }, From 4f650767041940a9924560e3497349246478915b Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 4 Aug 2024 11:56:08 +0800 Subject: [PATCH 04/19] =?UTF-8?q?Revert=20"=F0=9F=90=9B=20fix:=20support?= =?UTF-8?q?=20function=20call"=20(#48)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit c8422f494ccbd6dbf9542b45a44a6adf01dd1eb1. --- src/config/modelProviders/fireworksai.ts | 4 +- .../agent-runtime/fireworksai/index.test.ts | 43 +++++++++++++++++++ src/libs/agent-runtime/fireworksai/index.ts | 10 ----- 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/config/modelProviders/fireworksai.ts b/src/config/modelProviders/fireworksai.ts index 8f3781173d4a..bb0301296d57 100644 --- a/src/config/modelProviders/fireworksai.ts +++ b/src/config/modelProviders/fireworksai.ts @@ -5,15 +5,13 @@ import { ModelProviderCard } from '@/types/llm'; const FireworksAI: ModelProviderCard = { chatModels: [ { - description: 'Fireworks latest and most performant function-calling model. Firefunction-v2 is based on Llama-3 and trained to excel at function-calling as well as chat and instruction-following. See blog post for more details https://fireworks.ai/blog/firefunction-v2-launch-post', displayName: 'Firefunction V2', enabled: true, - functionCall: true, + //functionCall: true, id: 'accounts/fireworks/models/firefunction-v2', tokens: 8192, }, { - description: 'Vision-language model allowing both image and text as inputs (single image is recommended), trained on OSS model generated training data and open sourced on huggingface at fireworks-ai/FireLLaVA-13b', displayName: 'FireLLaVA-13B', enabled: true, functionCall: false, diff --git a/src/libs/agent-runtime/fireworksai/index.test.ts b/src/libs/agent-runtime/fireworksai/index.test.ts index cbd740279a38..16c4c4e64065 100644 --- a/src/libs/agent-runtime/fireworksai/index.test.ts +++ b/src/libs/agent-runtime/fireworksai/index.test.ts @@ -8,6 +8,7 @@ import { ModelProvider, } from '@/libs/agent-runtime'; +import * as debugStreamModule from '../utils/debugStream'; import { LobeFireworksAI } from './index'; const provider = ModelProvider.FireworksAI; @@ -208,5 +209,47 @@ describe('LobeFireworksAI', () => { } }); }); + + describe('DEBUG', () => { + it('should call debugStream and return StreamingTextResponse when DEBUG_FIREWORKSAI_CHAT_COMPLETION is 1', async () => { + // Arrange + const mockProdStream = new ReadableStream() as any; // 模拟的 prod 流 + const mockDebugStream = new ReadableStream({ + start(controller) { + controller.enqueue('Debug stream content'); + controller.close(); + }, + }) as any; + mockDebugStream.toReadableStream = () => mockDebugStream; // 添加 toReadableStream 方法 + + // 模拟 chat.completions.create 返回值，包括模拟的 tee 方法 + (instance['client'].chat.completions.create as Mock).mockResolvedValue({ + tee: () => [mockProdStream, { toReadableStream: () => mockDebugStream }], + }); + + // 保存原始环境变量值 + const originalDebugValue = process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION; + + // 模拟环境变量 + process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION = '1'; + vi.spyOn(debugStreamModule, 'debugStream').mockImplementation(() => Promise.resolve()); + + // 执行测试 + // 运行你的测试函数，确保它会在条件满足时调用 debugStream + // 假设的测试函数调用，你可能需要根据实际情况调整 + await instance.chat({ + messages: [{ content: 'Hello', role: 'user' }], + model: 'accounts/fireworks/models/firefunction-v2', + stream: true, + temperature: 0, + }); + + // 验证 debugStream 被调用 + expect(debugStreamModule.debugStream).toHaveBeenCalled(); + + // 恢复原始环境变量值 + process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION = originalDebugValue; + }); + }); }); }); diff --git a/src/libs/agent-runtime/fireworksai/index.ts b/src/libs/agent-runtime/fireworksai/index.ts index dbca228437c9..3f3fe872d8fb 100644 --- a/src/libs/agent-runtime/fireworksai/index.ts +++ b/src/libs/agent-runtime/fireworksai/index.ts @@ -1,18 +1,8 @@ -import OpenAI from 'openai'; - import { ModelProvider } from '../types'; import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory'; export const LobeFireworksAI = LobeOpenAICompatibleFactory({ baseURL: 'https://api.fireworks.ai/inference/v1', - chatCompletion: { - handlePayload: (payload) => { - return { - ...payload, - stream: false, - } as unknown as OpenAI.ChatCompletionCreateParamsStreaming; - }, - }, debug: { chatCompletion: () => process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION === '1', }, From d33a5a4a9ccd69575ea6db2c2b8c41637b3d54b4 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 4 Aug 2024 12:17:02 +0800 Subject: [PATCH 05/19] =?UTF-8?q?=F0=9F=92=84=20style:=20update=20model=20?= =?UTF-8?q?info?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config/modelProviders/fireworksai.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/config/modelProviders/fireworksai.ts b/src/config/modelProviders/fireworksai.ts index bb0301296d57..8f3781173d4a 100644 --- a/src/config/modelProviders/fireworksai.ts +++ b/src/config/modelProviders/fireworksai.ts @@ -5,13 +5,15 @@ import { ModelProviderCard } from '@/types/llm'; const FireworksAI: ModelProviderCard = { chatModels: [ { + description: 'Fireworks latest and most performant function-calling model. Firefunction-v2 is based on Llama-3 and trained to excel at function-calling as well as chat and instruction-following. See blog post for more details https://fireworks.ai/blog/firefunction-v2-launch-post', displayName: 'Firefunction V2', enabled: true, - //functionCall: true, + functionCall: true, id: 'accounts/fireworks/models/firefunction-v2', tokens: 8192, }, { + description: 'Vision-language model allowing both image and text as inputs (single image is recommended), trained on OSS model generated training data and open sourced on huggingface at fireworks-ai/FireLLaVA-13b', displayName: 'FireLLaVA-13B', enabled: true, functionCall: false, From 4ff60a369a62075c8f106e6be2c8e265cb5651fa Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 4 Aug 2024 14:17:42 +0800 Subject: [PATCH 06/19] =?UTF-8?q?=F0=9F=90=9B=20fix:=20disable=20stream=20?= =?UTF-8?q?if=20contains=20tools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libs/agent-runtime/fireworksai/index.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libs/agent-runtime/fireworksai/index.ts b/src/libs/agent-runtime/fireworksai/index.ts index 3f3fe872d8fb..3c86b71d9c5b 100644 --- a/src/libs/agent-runtime/fireworksai/index.ts +++ b/src/libs/agent-runtime/fireworksai/index.ts @@ -3,6 +3,14 @@ import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory'; export const LobeFireworksAI = LobeOpenAICompatibleFactory({ baseURL: 'https://api.fireworks.ai/inference/v1', + chatCompletion: { + handlePayload: (payload) => { + return { + ...payload, + stream: !payload.tools, + } as any; + }, + }, debug: { chatCompletion: () => process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION === '1', }, From 85952a0ddf238ae11017d0672c8a31dac689f20f Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 4 Aug 2024 19:06:02 +0800 Subject: [PATCH 07/19] =?UTF-8?q?=F0=9F=91=B7=20build:=20add=20ENV=20for?= =?UTF-8?q?=20Fireworks=20AI=20provider?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 9e4de8373165..b7dc546b829b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -116,6 +116,8 @@ ENV \ BAICHUAN_API_KEY="" \ # DeepSeek DEEPSEEK_API_KEY="" \ + # Fireworks AI + FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" \ # Google GOOGLE_API_KEY="" GOOGLE_PROXY_URL="" \ # Groq From 990d3c5767af2a1435813f543ec7ad985c3d1638 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 18 Aug 2024 00:01:24 +0800 Subject: [PATCH 08/19] =?UTF-8?q?=F0=9F=91=B7=20build:=20update=20ENV?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile.database | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.database b/Dockerfile.database index 95de8b3131ff..18ac8f38f2a8 100644 --- a/Dockerfile.database +++ b/Dockerfile.database @@ -144,6 +144,8 @@ ENV \ BAICHUAN_API_KEY="" \ # DeepSeek DEEPSEEK_API_KEY="" \ + # Fireworks AI + FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" \ # Google GOOGLE_API_KEY="" GOOGLE_PROXY_URL="" \ # Groq From 64932a3e06d6ff19173eab405f48491e008c32fc Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 18 Aug 2024 06:39:28 +0800 Subject: [PATCH 09/19] =?UTF-8?q?=E2=9C=A8=20feat:=20support=20proxy=20url?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- Dockerfile.database | 2 +- src/app/api/chat/agentRuntime.ts | 5 +++-- src/config/llm.ts | 2 ++ src/config/modelProviders/fireworksai.ts | 3 +++ 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index de4bf1ab4a60..8932b3592fa8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -117,7 +117,7 @@ ENV \ # DeepSeek DEEPSEEK_API_KEY="" \ # Fireworks AI - FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" \ + FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" FIREWORKSAI_PROXY_URL="" \ # Google GOOGLE_API_KEY="" GOOGLE_PROXY_URL="" \ # Groq diff --git a/Dockerfile.database b/Dockerfile.database index 18ac8f38f2a8..fdc603639e1c 100644 --- a/Dockerfile.database +++ b/Dockerfile.database @@ -145,7 +145,7 @@ ENV \ # DeepSeek DEEPSEEK_API_KEY="" \ # Fireworks AI - FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" \ + FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" FIREWORKSAI_PROXY_URL="" \ # Google GOOGLE_API_KEY="" GOOGLE_PROXY_URL="" \ # Groq diff --git a/src/app/api/chat/agentRuntime.ts b/src/app/api/chat/agentRuntime.ts index 47928c383b2a..ff296b61d27c 100644 --- a/src/app/api/chat/agentRuntime.ts +++ b/src/app/api/chat/agentRuntime.ts @@ -152,11 +152,12 @@ const getLlmOptionsFromPayload = (provider: string, payload: JWTPayload) => { return { apiKey }; } case ModelProvider.FireworksAI: { - const { FIREWORKSAI_API_KEY } = getLLMConfig(); + const { FIREWORKSAI_API_KEY, FIREWORKSAI_PROXY_URL } = getLLMConfig(); const apiKey = apiKeyManager.pick(payload?.apiKey || FIREWORKSAI_API_KEY); + const baseURL = payload?.endpoint || SILICONCLOUD_PROXY_URL; - return { apiKey }; + return { apiKey, baseURL }; } case ModelProvider.ZeroOne: { const { ZEROONE_API_KEY } = getLLMConfig(); diff --git a/src/config/llm.ts b/src/config/llm.ts index 8d4fd5bdf7a0..9797487e99e5 100644 --- a/src/config/llm.ts +++ b/src/config/llm.ts @@ -64,6 +64,7 @@ export const getLLMConfig = () => { ENABLED_FIREWORKSAI: z.boolean(), FIREWORKSAI_API_KEY: z.string().optional(), FIREWORKSAI_MODEL_LIST: z.string().optional(), + FIREWORKSAI_PROXY_URL: z.string().optional(), ENABLED_AWS_BEDROCK: z.boolean(), AWS_REGION: z.string().optional(), @@ -144,6 +145,7 @@ export const getLLMConfig = () => { ENABLED_FIREWORKSAI: !!process.env.FIREWORKSAI_API_KEY, FIREWORKSAI_API_KEY: process.env.FIREWORKSAI_API_KEY, FIREWORKSAI_MODEL_LIST: process.env.FIREWORKSAI_MODEL_LIST, + FIREWORKSAI_PROXY_URL: process.env.FIREWORKSAI_PROXY_URL, ENABLED_MOONSHOT: !!process.env.MOONSHOT_API_KEY, MOONSHOT_API_KEY: process.env.MOONSHOT_API_KEY, diff --git a/src/config/modelProviders/fireworksai.ts b/src/config/modelProviders/fireworksai.ts index 8f3781173d4a..9c81e8d34ab0 100644 --- a/src/config/modelProviders/fireworksai.ts +++ b/src/config/modelProviders/fireworksai.ts @@ -90,6 +90,9 @@ const FireworksAI: ModelProviderCard = { id: 'fireworksai', modelList: { showModelFetcher: true }, name: 'Fireworks AI', + proxyUrl: { + placeholder: 'https://api.fireworks.ai/inference/v1', + }, }; export default FireworksAI; From 4abe2400cec40f9c58f61d3b8f760b3ebbb7a111 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 18 Aug 2024 06:41:25 +0800 Subject: [PATCH 10/19] =?UTF-8?q?=F0=9F=90=9B=20fix:=20fix=20typo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app/api/chat/agentRuntime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/api/chat/agentRuntime.ts b/src/app/api/chat/agentRuntime.ts index ff296b61d27c..58011ce00195 100644 --- a/src/app/api/chat/agentRuntime.ts +++ b/src/app/api/chat/agentRuntime.ts @@ -155,7 +155,7 @@ const getLlmOptionsFromPayload = (provider: string, payload: JWTPayload) => { const { FIREWORKSAI_API_KEY, FIREWORKSAI_PROXY_URL } = getLLMConfig(); const apiKey = apiKeyManager.pick(payload?.apiKey || FIREWORKSAI_API_KEY); - const baseURL = payload?.endpoint || SILICONCLOUD_PROXY_URL; + const baseURL = payload?.endpoint || FIREWORKSAI_PROXY_URL; return { apiKey, baseURL }; } From 059c6485fc8fe450feaf8d9928383ea040df1091 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Wed, 28 Aug 2024 18:19:40 +0800 Subject: [PATCH 11/19] Update providers.tsx --- .../settings/llm/ProviderList/providers.tsx | 209 +++--------------- 1 file changed, 29 insertions(+), 180 deletions(-) diff --git a/src/app/(main)/settings/llm/ProviderList/providers.tsx b/src/app/(main)/settings/llm/ProviderList/providers.tsx index 4b6e702f2dbb..15b25cad2e68 100644 --- a/src/app/(main)/settings/llm/ProviderList/providers.tsx +++ b/src/app/(main)/settings/llm/ProviderList/providers.tsx @@ -1,32 +1,4 @@ -import { - Ai360, - AiMass, - Anthropic, - Baichuan, - Claude, - DeepSeek, - Fireworks, - Gemini, - Google, - Groq, - Minimax, - Mistral, - Moonshot, - Novita, - OpenRouter, - Perplexity, - SiliconCloud, - Stepfun, - Together, - Tongyi, - ZeroOne, - Zhipu, -} from '@lobehub/icons'; -import { Divider } from 'antd'; -import { useTheme } from 'antd-style'; import { useMemo } from 'react'; -import { Flexbox } from 'react-layout-kit'; -import urlJoin from 'url-join'; import { Ai360ProviderCard, @@ -57,162 +29,39 @@ import { useBedrockProvider } from './Bedrock'; import { useOllamaProvider } from './Ollama'; import { useOpenAIProvider } from './OpenAI'; -const BASE_DOC_URL = 'https://lobehub.com/docs/usage/providers'; - -const AnthropicBrand = () => { - const { isDarkMode } = useTheme(); - return ; -}; - -const MoonshotBrand = () => { - const theme = useTheme(); - return ( - - ); -}; - -const GroqBrand = () => { - const theme = useTheme(); - - return ; -}; - -const GoogleBrand = () => ( - - - - - -); - export const useProviderList = (): ProviderItem[] => { - const azureProvider = useAzureProvider(); - const ollamaProvider = useOllamaProvider(); - const openAIProvider = useOpenAIProvider(); - const bedrockProvider = useBedrockProvider(); + const AzureProvider = useAzureProvider(); + const OllamaProvider = useOllamaProvider(); + const OpenAIProvider = useOpenAIProvider(); + const BedrockProvider = useBedrockProvider(); return useMemo( () => [ - { - ...openAIProvider, - docUrl: urlJoin(BASE_DOC_URL, 'openai'), - }, - { - ...ollamaProvider, - docUrl: urlJoin(BASE_DOC_URL, 'ollama'), - }, - { - ...azureProvider, - docUrl: urlJoin(BASE_DOC_URL, 'azure'), - }, - { - ...GoogleProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'gemini'), - title: , - }, - { - ...AnthropicProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'anthropic'), - title: , - }, - { - ...bedrockProvider, - docUrl: urlJoin(BASE_DOC_URL, 'bedrock'), - }, - { - ...GroqProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'groq'), - title: , - }, - { - ...OpenRouterProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'openrouter'), - title: , - }, - { - ...NovitaProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'novita'), - title: , - }, - { - ...TogetherAIProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'togetherai'), - title: , - }, - { - ...FireworksAIProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'fireworksai'), - title: , - }, - { - ...QwenProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'qwen'), - title: , - }, - { - ...DeepSeekProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'deepseek'), - title: , - }, - { - ...MinimaxProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'minimax'), - title: , - }, - { - ...MistralProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'mistral'), - title: , - }, - { - ...MoonshotProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'moonshot'), - title: , - }, - { - ...PerplexityProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'perplexity'), - title: , - }, - { - ...ZhiPuProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'zhipu'), - title: , - }, - { - ...ZeroOneProviderCard, - docUrl: urlJoin(BASE_DOC_URL, '01ai'), - title: , - }, - { - ...StepfunProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'stepfun'), - title: , - }, - { - ...BaichuanProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'baichuan'), - title: , - }, - { - ...TaichuProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'taichu'), - title: , - }, - { - ...Ai360ProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'ai360'), - title: , - }, - { - ...SiliconCloudProviderCard, - docUrl: urlJoin(BASE_DOC_URL, 'siliconcloud'), - title: , - }, + OpenAIProvider, + OllamaProvider, + AzureProvider, + GoogleProviderCard, + AnthropicProviderCard, + BedrockProvider, + GroqProviderCard, + OpenRouterProviderCard, + NovitaProviderCard, + TogetherAIProviderCard, + FireworksAIProviderCard, + QwenProviderCard, + DeepSeekProviderCard, + MinimaxProviderCard, + MistralProviderCard, + MoonshotProviderCard, + PerplexityProviderCard, + ZhiPuProviderCard, + ZeroOneProviderCard, + StepfunProviderCard, + BaichuanProviderCard, + TaichuProviderCard, + Ai360ProviderCard, + SiliconCloudProviderCard, ], - [azureProvider, ollamaProvider, ollamaProvider, bedrockProvider], + [AzureProvider, OllamaProvider, OpenAIProvider, BedrockProvider], ); }; From 8f3499c306ac61dd255b1615d786e0499eb15fa2 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Wed, 28 Aug 2024 18:25:09 +0800 Subject: [PATCH 12/19] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/ModelIcon/index.tsx | 116 ---------------- src/components/ModelProviderIcon/index.tsx | 153 --------------------- src/components/ModelTag/ModelIcon.tsx | 107 -------------- src/components/ModelTag/index.tsx | 13 -- 4 files changed, 389 deletions(-) delete mode 100644 src/components/ModelIcon/index.tsx delete mode 100644 src/components/ModelProviderIcon/index.tsx delete mode 100644 src/components/ModelTag/ModelIcon.tsx delete mode 100644 src/components/ModelTag/index.tsx diff --git a/src/components/ModelIcon/index.tsx b/src/components/ModelIcon/index.tsx deleted file mode 100644 index b2832f698c53..000000000000 --- a/src/components/ModelIcon/index.tsx +++ /dev/null @@ -1,116 +0,0 @@ -import { - Adobe, - Ai21, - Ai360, - AiMass, - Aws, - Aya, - Azure, - Baichuan, - ByteDance, - ChatGLM, - Claude, - CodeGeeX, - Cohere, - Dbrx, - DeepSeek, - Fireworks, - FishAudio, - Gemini, - Gemma, - Hunyuan, - LLaVA, - Meta, - Minimax, - Mistral, - Moonshot, - OpenAI, - OpenChat, - OpenRouter, - Perplexity, - Rwkv, - Spark, - Stability, - Stepfun, - Tongyi, - Wenxin, - Yi, -} from '@lobehub/icons'; -import { memo } from 'react'; - -interface ModelProviderIconProps { - model?: string; - size?: number; -} - -const ModelIcon = memo(({ model: originModel, size = 12 }) => { - if (!originModel) return; - - // lower case the origin model so to better match more model id case - const model = originModel.toLowerCase(); - - // currently supported models, maybe not in its own provider - if (model.includes('text-embedding-')) return ; - if (model.includes('gpt-3')) return ; - if (model.includes('gpt-4')) return ; - if (model.includes('glm-') || model.includes('chatglm')) return ; - if (model.startsWith('codegeex')) return ; - if (model.includes('deepseek')) return ; - if (model.includes('claude')) return ; - if (model.includes('titan')) return ; - if (model.includes('accounts/fireworks/models/fire')) return ; - if (model.includes('llama')) return ; - if (model.includes('llava')) return ; - if (model.includes('gemini')) return ; - if (model.includes('gemma')) return ; - if (model.includes('moonshot')) return ; - if (model.includes('qwen')) return ; - if (model.includes('minmax') || model.includes('abab')) return ; - if ( - model.includes('mistral') || - model.includes('mixtral') || - model.includes('codestral') || - model.includes('mathstral') - ) - return ; - if (model.includes('pplx') || model.includes('sonar')) return ; - if (model.includes('yi-')) return ; - if (model.startsWith('openrouter')) return ; // only for Cinematika and Auto - if (model.startsWith('openchat')) return ; - if (model.includes('aya')) return ; - if (model.includes('command')) return ; - if (model.includes('dbrx')) return ; - if (model.includes('step')) return ; - if (model.includes('taichu')) return ; - if (model.includes('360gpt')) return ; - - // below: To be supported in providers, move up if supported - if (model.includes('baichuan')) - return ; - if (model.includes('rwkv')) return ; - if (model.includes('ernie')) return ; - if (model.includes('spark')) return ; - if (model.includes('hunyuan')) return ; - // ref https://github.com/fishaudio/Bert-VITS2/blob/master/train_ms.py#L702 - if (model.startsWith('d_') || model.startsWith('g_') || model.startsWith('wd_')) - return ; - if (model.includes('skylark')) return ; - - if ( - model.includes('stable-diffusion') || - model.includes('stable-video') || - model.includes('stable-cascade') || - model.includes('sdxl') || - model.includes('stablelm') || - model.startsWith('stable-') || - model.startsWith('sd3') - ) - return ; - - if (model.includes('phi3') || model.includes('phi-3') || model.includes('wizardlm')) - return ; - if (model.includes('firefly')) return ; - if (model.includes('jamba') || model.includes('j2-')) return ; -}); - -export default ModelIcon; diff --git a/src/components/ModelProviderIcon/index.tsx b/src/components/ModelProviderIcon/index.tsx deleted file mode 100644 index 6684d690431a..000000000000 --- a/src/components/ModelProviderIcon/index.tsx +++ /dev/null @@ -1,153 +0,0 @@ -import { - Ai360, - AiMass, - Anthropic, - Azure, - Baichuan, - Bedrock, - DeepSeek, - Fireworks, - Google, - Groq, - LobeHub, - Minimax, - Mistral, - Moonshot, - Novita, - Ollama, - OpenAI, - OpenRouter, - Perplexity, - SiliconCloud, - Stepfun, - Together, - Tongyi, - ZeroOne, - Zhipu, -} from '@lobehub/icons'; -import { memo } from 'react'; -import { Center } from 'react-layout-kit'; - -import { ModelProvider } from '@/libs/agent-runtime'; - -interface ModelProviderIconProps { - provider?: string; -} - -const ModelProviderIcon = memo(({ provider }) => { - switch (provider) { - case 'lobehub': { - return ; - } - - case ModelProvider.ZhiPu: { - return ; - } - - case ModelProvider.Bedrock: { - return ; - } - - case ModelProvider.DeepSeek: { - return ; - } - - case ModelProvider.Google: { - return ( -
- -
- ); - } - - case ModelProvider.Azure: { - return ( -
- -
- ); - } - - case ModelProvider.Moonshot: { - return ; - } - - case ModelProvider.OpenAI: { - return ; - } - - case ModelProvider.Ollama: { - return ; - } - - case ModelProvider.Perplexity: { - return ; - } - - case ModelProvider.Minimax: { - return ; - } - - case ModelProvider.Mistral: { - return ; - } - - case ModelProvider.Anthropic: { - return ; - } - - case ModelProvider.Groq: { - return ; - } - - case ModelProvider.OpenRouter: { - return ; - } - - case ModelProvider.ZeroOne: { - return ; - } - - case ModelProvider.TogetherAI: { - return ; - } - - case ModelProvider.FireworksAI: { - return ; - } - - case ModelProvider.Qwen: { - return ; - } - - case ModelProvider.Stepfun: { - return ; - } - - case ModelProvider.Novita: { - return ; - } - - case ModelProvider.Baichuan: { - return ; - } - - case ModelProvider.Taichu: { - return ; - } - - case ModelProvider.Ai360: { - return ; - } - - case ModelProvider.SiliconCloud: { - return ; - } - - default: { - return null; - } - } -}); - -export default ModelProviderIcon; diff --git a/src/components/ModelTag/ModelIcon.tsx b/src/components/ModelTag/ModelIcon.tsx deleted file mode 100644 index 4be0d820cbe7..000000000000 --- a/src/components/ModelTag/ModelIcon.tsx +++ /dev/null @@ -1,107 +0,0 @@ -import { - AdobeFirefly, - Ai21, - Ai360, - AiMass, - Aws, - Aya, - Azure, - Baichuan, - ByteDance, - ChatGLM, - Claude, - CodeGeeX, - Cohere, - Dbrx, - DeepSeek, - Fireworks, - FishAudio, - Gemini, - Gemma, - Hunyuan, - LLaVA, - Meta, - Minimax, - Mistral, - Moonshot, - OpenAI, - OpenChat, - OpenRouter, - Perplexity, - Rwkv, - Spark, - Stability, - Stepfun, - Tongyi, - Wenxin, - Yi, -} from '@lobehub/icons'; -import { memo } from 'react'; - -interface ModelIconProps { - model?: string; - size?: number; -} - -const ModelIcon = memo(({ model: originModel, size = 12 }) => { - if (!originModel) return; - - // lower case the origin model so to better match more model id case - const model = originModel.toLowerCase(); - - // currently supported models, maybe not in its own provider - if (model.startsWith('gpt')) return ; - if (model.startsWith('glm') || model.includes('chatglm')) return ; - if (model.includes('codegeex')) return ; - if (model.includes('claude')) return ; - if (model.includes('deepseek')) return ; - if (model.includes('titan')) return ; - if (model.includes('accounts/fireworks/models/fire')) return ; - if (model.includes('llama')) return ; - if (model.includes('llava')) return ; - if (model.includes('gemini')) return ; - if (model.includes('gemma')) return ; - if (model.includes('moonshot')) return ; - if (model.includes('qwen')) return ; - if (model.includes('minmax')) return ; - if (model.includes('abab')) return ; - if (model.includes('mistral') || model.includes('mixtral') || model.includes('codestral') || model.includes('mathstral')) return ; - if (model.includes('pplx') || model.includes('sonar')) return ; - if (model.includes('yi-')) return ; - if (model.startsWith('openrouter')) return ; // only for Cinematika and Auto - if (model.startsWith('openchat')) return ; - if (model.includes('aya')) return ; - if (model.includes('command')) return ; - if (model.includes('dbrx')) return ; - if (model.includes('step')) return ; - if (model.includes('taichu')) return ; - if (model.includes('360gpt')) return ; - - // below: To be supported in providers, move up if supported - if (model.includes('baichuan')) return ; - if (model.includes('rwkv')) return ; - if (model.includes('ernie')) return ; - if (model.includes('spark')) return ; - if (model.includes('hunyuan')) return ; - // ref https://github.com/fishaudio/Bert-VITS2/blob/master/train_ms.py#L702 - if (model.startsWith('d_') || model.startsWith('g_') || model.startsWith('wd_')) - return ; - if (model.includes('skylark')) return ; - - if ( - model.includes('stable-diffusion') || - model.includes('stable-video') || - model.includes('stable-cascade') || - model.includes('sdxl') || - model.includes('stablelm') || - model.startsWith('stable-') || - model.startsWith('sd3') - ) - return ; - - if (model.includes('phi3') || model.includes('phi-3') || model.includes('wizardlm')) return ; - if (model.includes('firefly')) return ; - if (model.includes('jamba') || model.includes('j2-')) return ; -}); - -export default ModelIcon; diff --git a/src/components/ModelTag/index.tsx b/src/components/ModelTag/index.tsx deleted file mode 100644 index 97def18b52e4..000000000000 --- a/src/components/ModelTag/index.tsx +++ /dev/null @@ -1,13 +0,0 @@ -import { Tag } from '@lobehub/ui'; -import { memo } from 'react'; - -import ModelIcon from './ModelIcon'; - -interface ModelTagProps { - model: string; -} -const ModelTag = memo(({ model }) => ( - }>{model} -)); - -export default ModelTag; From adb3354728886ee08c6d299ac32ca23f740406ad Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Fri, 30 Aug 2024 21:49:36 +0800 Subject: [PATCH 13/19] =?UTF-8?q?=F0=9F=94=A8=20chore:=20remove=20proxyUrl?= =?UTF-8?q?=20for=20FireworksAI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 2 +- Dockerfile.database | 2 +- src/app/api/chat/agentRuntime.ts | 5 ++--- src/config/llm.ts | 4 +--- src/config/modelProviders/fireworksai.ts | 3 --- 5 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 228705056a6c..145c0e833db9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -117,7 +117,7 @@ ENV \ # DeepSeek DEEPSEEK_API_KEY="" \ # Fireworks AI - FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" FIREWORKSAI_PROXY_URL="" \ + FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" \ # Google GOOGLE_API_KEY="" GOOGLE_PROXY_URL="" \ # Groq diff --git a/Dockerfile.database b/Dockerfile.database index 01367499bc04..f09b33ee52e2 100644 --- a/Dockerfile.database +++ b/Dockerfile.database @@ -149,7 +149,7 @@ ENV \ # DeepSeek DEEPSEEK_API_KEY="" \ # Fireworks AI - FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" FIREWORKSAI_PROXY_URL="" \ + FIREWORKSAI_API_KEY="" FIREWORKSAI_MODEL_LIST="" \ # Google GOOGLE_API_KEY="" GOOGLE_PROXY_URL="" \ # Groq diff --git a/src/app/api/chat/agentRuntime.ts b/src/app/api/chat/agentRuntime.ts index 85691abad080..817645e3d8b6 100644 --- a/src/app/api/chat/agentRuntime.ts +++ b/src/app/api/chat/agentRuntime.ts @@ -152,12 +152,11 @@ const getLlmOptionsFromPayload = (provider: string, payload: JWTPayload) => { return { apiKey }; } case ModelProvider.FireworksAI: { - const { FIREWORKSAI_API_KEY, FIREWORKSAI_PROXY_URL } = getLLMConfig(); + const { FIREWORKSAI_API_KEY } = getLLMConfig(); const apiKey = apiKeyManager.pick(payload?.apiKey || FIREWORKSAI_API_KEY); - const baseURL = payload?.endpoint || FIREWORKSAI_PROXY_URL; - return { apiKey, baseURL }; + return { apiKey }; } case ModelProvider.ZeroOne: { const { ZEROONE_API_KEY } = getLLMConfig(); diff --git a/src/config/llm.ts b/src/config/llm.ts index bd02e5893bde..59400fee1421 100644 --- a/src/config/llm.ts +++ b/src/config/llm.ts @@ -65,8 +65,7 @@ export const getLLMConfig = () => { ENABLED_FIREWORKSAI: z.boolean(), FIREWORKSAI_API_KEY: z.string().optional(), FIREWORKSAI_MODEL_LIST: z.string().optional(), - FIREWORKSAI_PROXY_URL: z.string().optional(), - + ENABLED_AWS_BEDROCK: z.boolean(), AWS_REGION: z.string().optional(), AWS_ACCESS_KEY_ID: z.string().optional(), @@ -149,7 +148,6 @@ export const getLLMConfig = () => { ENABLED_FIREWORKSAI: !!process.env.FIREWORKSAI_API_KEY, FIREWORKSAI_API_KEY: process.env.FIREWORKSAI_API_KEY, FIREWORKSAI_MODEL_LIST: process.env.FIREWORKSAI_MODEL_LIST, - FIREWORKSAI_PROXY_URL: process.env.FIREWORKSAI_PROXY_URL, ENABLED_MOONSHOT: !!process.env.MOONSHOT_API_KEY, MOONSHOT_API_KEY: process.env.MOONSHOT_API_KEY, diff --git a/src/config/modelProviders/fireworksai.ts b/src/config/modelProviders/fireworksai.ts index 9c81e8d34ab0..8f3781173d4a 100644 --- a/src/config/modelProviders/fireworksai.ts +++ b/src/config/modelProviders/fireworksai.ts @@ -90,9 +90,6 @@ const FireworksAI: ModelProviderCard = { id: 'fireworksai', modelList: { showModelFetcher: true }, name: 'Fireworks AI', - proxyUrl: { - placeholder: 'https://api.fireworks.ai/inference/v1', - }, }; export default FireworksAI; From e12fc05afa9d1322e78180dce009750fb83fc3b7 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 8 Sep 2024 07:45:52 +0800 Subject: [PATCH 14/19] =?UTF-8?q?=F0=9F=90=9B=20fix:=20tools=20calling=20i?= =?UTF-8?q?ssue=20in=20Fireworks=20AI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libs/agent-runtime/utils/streams/openai.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libs/agent-runtime/utils/streams/openai.ts b/src/libs/agent-runtime/utils/streams/openai.ts index 52f81a776f08..ca372eb6249c 100644 --- a/src/libs/agent-runtime/utils/streams/openai.ts +++ b/src/libs/agent-runtime/utils/streams/openai.ts @@ -23,15 +23,14 @@ export const transformOpenAIStream = (chunk: OpenAI.ChatCompletionChunk): Stream return { data: chunk, id: chunk.id, type: 'data' }; } - if (typeof item.delta?.content === 'string') { - return { data: item.delta.content, id: chunk.id, type: 'text' }; - } - if (item.delta?.tool_calls) { return { data: item.delta.tool_calls.map( (value, index): StreamToolCallChunkData => ({ - function: value.function, + function: { + arguments: value.function?.arguments ?? '', + name: value.function?.name ?? null, + }, id: value.id || generateToolCallId(index, value.function?.name), // mistral's tool calling don't have index and function field, it's data like: @@ -55,6 +54,10 @@ export const transformOpenAIStream = (chunk: OpenAI.ChatCompletionChunk): Stream return { data: item.finish_reason, id: chunk.id, type: 'stop' }; } + if (typeof item.delta?.content === 'string') { + return { data: item.delta.content, id: chunk.id, type: 'text' }; + } + if (item.delta?.content === null) { return { data: item.delta, id: chunk.id, type: 'data' }; } From 2ebfd8c7a928256489b9eeaf640da7333a928530 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Sun, 8 Sep 2024 07:50:02 +0800 Subject: [PATCH 15/19] =?UTF-8?q?=F0=9F=94=A8=20chore:=20remove=20work=20a?= =?UTF-8?q?round=20method=20for=20tool=20calling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libs/agent-runtime/fireworksai/index.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/libs/agent-runtime/fireworksai/index.ts b/src/libs/agent-runtime/fireworksai/index.ts index 3c86b71d9c5b..3f3fe872d8fb 100644 --- a/src/libs/agent-runtime/fireworksai/index.ts +++ b/src/libs/agent-runtime/fireworksai/index.ts @@ -3,14 +3,6 @@ import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory'; export const LobeFireworksAI = LobeOpenAICompatibleFactory({ baseURL: 'https://api.fireworks.ai/inference/v1', - chatCompletion: { - handlePayload: (payload) => { - return { - ...payload, - stream: !payload.tools, - } as any; - }, - }, debug: { chatCompletion: () => process.env.DEBUG_FIREWORKSAI_CHAT_COMPLETION === '1', }, From 8c9eb74f12ac99badcfe985d32e8151155058999 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Tue, 10 Sep 2024 19:02:35 +0800 Subject: [PATCH 16/19] =?UTF-8?q?=F0=9F=94=A8=20chore:=20rollback=20change?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agent-runtime/utils/streams/openai.ts | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/libs/agent-runtime/utils/streams/openai.ts b/src/libs/agent-runtime/utils/streams/openai.ts index ca372eb6249c..5580cee70028 100644 --- a/src/libs/agent-runtime/utils/streams/openai.ts +++ b/src/libs/agent-runtime/utils/streams/openai.ts @@ -8,13 +8,17 @@ import { ChatStreamCallbacks } from '../../types'; import { StreamProtocolChunk, StreamProtocolToolCallChunk, + StreamStack, StreamToolCallChunkData, createCallbacksTransformer, createSSEProtocolTransformer, generateToolCallId, } from './protocol'; -export const transformOpenAIStream = (chunk: OpenAI.ChatCompletionChunk): StreamProtocolChunk => { +export const transformOpenAIStream = ( + chunk: OpenAI.ChatCompletionChunk, + stack?: StreamStack, +): StreamProtocolChunk => { // maybe need another structure to add support for multiple choices try { @@ -23,15 +27,23 @@ export const transformOpenAIStream = (chunk: OpenAI.ChatCompletionChunk): Stream return { data: chunk, id: chunk.id, type: 'data' }; } + if (typeof item.delta?.content === 'string' && !item.finish_reason && !item.delta?.tool_calls) { + return { data: item.delta.content, id: chunk.id, type: 'text' }; + } + if (item.delta?.tool_calls) { return { - data: item.delta.tool_calls.map( - (value, index): StreamToolCallChunkData => ({ + data: item.delta.tool_calls.map((value, index): StreamToolCallChunkData => { + if (stack && !stack.tool) { + stack.tool = { id: value.id!, index: value.index, name: value.function!.name! }; + } + + return { function: { arguments: value.function?.arguments ?? '', name: value.function?.name ?? null, }, - id: value.id || generateToolCallId(index, value.function?.name), + id: value.id || stack?.tool?.id || generateToolCallId(index, value.function?.name), // mistral's tool calling don't have index and function field, it's data like: // [{"id":"xbhnmTtY7","function":{"name":"lobe-image-designer____text2image____builtin","arguments":"{\"prompts\": [\"A photo of a small, fluffy dog with a playful expression and wagging tail.\", \"A watercolor painting of a small, energetic dog with a glossy coat and bright eyes.\", \"A vector illustration of a small, adorable dog with a short snout and perky ears.\", \"A drawing of a small, scruffy dog with a mischievous grin and a wagging tail.\"], \"quality\": \"standard\", \"seeds\": [123456, 654321, 111222, 333444], \"size\": \"1024x1024\", \"style\": \"vivid\"}"}}] @@ -42,8 +54,8 @@ export const transformOpenAIStream = (chunk: OpenAI.ChatCompletionChunk): Stream // so we need to add these default values index: typeof value.index !== 'undefined' ? value.index : index, type: value.type || 'function', - }), - ), + }; + }), id: chunk.id, type: 'tool_calls', } as StreamProtocolToolCallChunk; @@ -54,10 +66,6 @@ export const transformOpenAIStream = (chunk: OpenAI.ChatCompletionChunk): Stream return { data: item.finish_reason, id: chunk.id, type: 'stop' }; } - if (typeof item.delta?.content === 'string') { - return { data: item.delta.content, id: chunk.id, type: 'text' }; - } - if (item.delta?.content === null) { return { data: item.delta, id: chunk.id, type: 'data' }; } @@ -100,10 +108,12 @@ export const OpenAIStream = ( stream: Stream | ReadableStream, callbacks?: ChatStreamCallbacks, ) => { + const streamStack: StreamStack = { id: '' }; + const readableStream = stream instanceof ReadableStream ? stream : readableFromAsyncIterable(chatStreamable(stream)); return readableStream - .pipeThrough(createSSEProtocolTransformer(transformOpenAIStream)) + .pipeThrough(createSSEProtocolTransformer(transformOpenAIStream, streamStack)) .pipeThrough(createCallbacksTransformer(callbacks)); }; From d14d30bb1c65f602ce4753c8fe8c12849eff827b Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Tue, 10 Sep 2024 19:56:13 +0800 Subject: [PATCH 17/19] =?UTF-8?q?=F0=9F=90=9B=20fix:=20fix=20CI=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/libs/agent-runtime/utils/streams/openai.test.ts | 6 +++--- src/libs/agent-runtime/utils/streams/openai.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libs/agent-runtime/utils/streams/openai.test.ts b/src/libs/agent-runtime/utils/streams/openai.test.ts index 05eb9ef27d13..7e977be93262 100644 --- a/src/libs/agent-runtime/utils/streams/openai.test.ts +++ b/src/libs/agent-runtime/utils/streams/openai.test.ts @@ -253,7 +253,7 @@ describe('OpenAIStream', () => { expect(chunks).toEqual([ 'id: 2\n', 'event: tool_calls\n', - `data: [{"function":{"name":"tool1","arguments":"{}"},"id":"call_1","index":0,"type":"function"},{"function":{"name":"tool2","arguments":"{}"},"id":"call_2","index":1,"type":"function"}]\n\n`, + `data: [{"function":{"arguments":"{}","name":"tool1"},"id":"call_1","index":0,"type":"function"},{"function":{"arguments":"{}","name":"tool2"},"id":"call_2","index":1,"type":"function"}]\n\n`, ]); expect(onToolCallMock).toHaveBeenCalledTimes(1); @@ -300,7 +300,7 @@ describe('OpenAIStream', () => { expect(chunks).toEqual([ 'id: 5\n', 'event: tool_calls\n', - `data: [{"function":{"name":"tool1","arguments":"{}"},"id":"call_1","index":0,"type":"function"},{"function":{"name":"tool2","arguments":"{}"},"id":"call_2","index":1,"type":"function"}]\n\n`, + `data: [{"function":{"arguments":"{}","name":"tool1"},"id":"call_1","index":0,"type":"function"},{"function":{"arguments":"{}","name":"tool2"},"id":"call_2","index":1,"type":"function"}]\n\n`, ]); }); @@ -394,7 +394,7 @@ describe('OpenAIStream', () => { `data: [{"function":{"arguments":"","name":"realtime-weather____fetchCurrentWeather"},"id":"toolu_01VQtK4W9kqxGGLHgsPPxiBj","index":0,"type":"function"}]\n`, 'id: 1', 'event: tool_calls', - `data: [{"function":{"arguments":"{\\"city\\": \\"杭州\\"}"},"id":"toolu_01VQtK4W9kqxGGLHgsPPxiBj","index":0,"type":"function"}]\n`, + `data: [{"function":{"arguments":"{\\"city\\": \\"杭州\\"}","name":null},"id":"toolu_01VQtK4W9kqxGGLHgsPPxiBj","index":0,"type":"function"}]\n`, 'id: 1', 'event: stop', `data: "tool_calls"\n`, diff --git a/src/libs/agent-runtime/utils/streams/openai.ts b/src/libs/agent-runtime/utils/streams/openai.ts index 5580cee70028..89ac60d831d1 100644 --- a/src/libs/agent-runtime/utils/streams/openai.ts +++ b/src/libs/agent-runtime/utils/streams/openai.ts @@ -40,7 +40,7 @@ export const transformOpenAIStream = ( return { function: { - arguments: value.function?.arguments ?? '', + arguments: value.function?.arguments ?? '{}', name: value.function?.name ?? null, }, id: value.id || stack?.tool?.id || generateToolCallId(index, value.function?.name), From a98e07b51d8ba142df7aff3a145342a05468b542 Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Tue, 10 Sep 2024 20:34:57 +0800 Subject: [PATCH 18/19] =?UTF-8?q?=F0=9F=92=84=20style:=20update=20model=20?= =?UTF-8?q?list?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/config/modelProviders/fireworksai.ts | 72 ++++++++++++++++++++---- 1 file changed, 60 insertions(+), 12 deletions(-) diff --git a/src/config/modelProviders/fireworksai.ts b/src/config/modelProviders/fireworksai.ts index 8f3781173d4a..d34cdff7177c 100644 --- a/src/config/modelProviders/fireworksai.ts +++ b/src/config/modelProviders/fireworksai.ts @@ -12,6 +12,13 @@ const FireworksAI: ModelProviderCard = { id: 'accounts/fireworks/models/firefunction-v2', tokens: 8192, }, + { + description: 'Fireworks open-source function calling model.', + displayName: 'Firefunction V1', + functionCall: true, + id: 'accounts/fireworks/models/firefunction-v1', + tokens: 32_768, + }, { description: 'Vision-language model allowing both image and text as inputs (single image is recommended), trained on OSS model generated training data and open sourced on huggingface at fireworks-ai/FireLLaVA-13b', displayName: 'FireLLaVA-13B', @@ -22,26 +29,50 @@ const FireworksAI: ModelProviderCard = { vision: true, }, { - displayName: 'Llama3.1 8B Instruct', + displayName: 'Llama 3.1 8B Instruct', enabled: true, functionCall: false, id: 'accounts/fireworks/models/llama-v3p1-8b-instruct', tokens: 131_072, }, { - displayName: 'Llama3.1 70B Instruct', + displayName: 'Llama 3.1 70B Instruct', enabled: true, functionCall: false, id: 'accounts/fireworks/models/llama-v3p1-70b-instruct', tokens: 131_072, }, { - displayName: 'Llama3.1 405B Instruct', + displayName: 'Llama 3.1 405B Instruct', enabled: true, functionCall: false, id: 'accounts/fireworks/models/llama-v3p1-405b-instruct', tokens: 131_072, }, + { + displayName: 'Llama 3 8B Instruct', + functionCall: false, + id: 'accounts/fireworks/models/llama-v3-8b-instruct', + tokens: 8192, + }, + { + displayName: 'Llama 3 70B Instruct', + functionCall: false, + id: 'accounts/fireworks/models/llama-v3-70b-instruct', + tokens: 8192, + }, + { + displayName: 'Llama 3 8B Instruct (HF version)', + functionCall: false, + id: 'accounts/fireworks/models/llama-v3-8b-instruct-hf', + tokens: 8192, + }, + { + displayName: 'Llama 3 70B Instruct (HF version)', + functionCall: false, + id: 'accounts/fireworks/models/llama-v3-70b-instruct-hf', + tokens: 8192, + }, { displayName: 'Gemma 2 9B Instruct', enabled: true, @@ -64,26 +95,43 @@ const FireworksAI: ModelProviderCard = { tokens: 65_536, }, { - displayName: 'Phi 3 Vision 128K Instruct', + displayName: 'Mixtral MoE 8x7B Instruct (HF version)', + functionCall: false, + id: 'accounts/fireworks/models/mixtral-8x7b-instruct-hf', + tokens: 32_768, + }, + { + displayName: 'Phi 3 Vision Instruct', enabled: true, functionCall: false, id: 'accounts/fireworks/models/phi-3-vision-128k-instruct', - tokens: 131_072, + tokens: 8192, vision: true, }, { - displayName: 'DeepSeek Coder V2 Instruct', + displayName: 'Yi-Large', enabled: true, functionCall: false, - id: 'accounts/fireworks/models/deepseek-coder-v2-instruct', - tokens: 131_072, + id: 'accounts/yi-01-ai/models/yi-large', + tokens: 32_768, }, { - displayName: 'Qwen2 72b Instruct', - enabled: true, + displayName: 'StarCoder 7B', functionCall: false, - id: 'accounts/fireworks/models/qwen2-72b-instruct', - tokens: 32_768, + id: 'accounts/fireworks/models/starcoder-7b', + tokens: 8192, + }, + { + displayName: 'StarCoder 15.5B', + functionCall: false, + id: 'accounts/fireworks/models/starcoder-16b', + tokens: 8192, + }, + { + displayName: 'MythoMax L2 13b', + functionCall: false, + id: 'accounts/fireworks/models/mythomax-l2-13b', + tokens: 4096, }, ], checkModel: 'accounts/fireworks/models/firefunction-v2', From e3291d400c43ca5455a136f51a8652e3e5fded8b Mon Sep 17 00:00:00 2001 From: Zhijie He Date: Tue, 10 Sep 2024 23:14:06 +0800 Subject: [PATCH 19/19] =?UTF-8?q?=F0=9F=94=A8=20chore:=20fix=20rebase=20co?= =?UTF-8?q?nflicts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Error/APIKeyForm/ProviderAvatar.tsx | 123 ------------------ 1 file changed, 123 deletions(-) delete mode 100644 src/features/Conversation/Error/APIKeyForm/ProviderAvatar.tsx diff --git a/src/features/Conversation/Error/APIKeyForm/ProviderAvatar.tsx b/src/features/Conversation/Error/APIKeyForm/ProviderAvatar.tsx deleted file mode 100644 index 1a02df25a2f2..000000000000 --- a/src/features/Conversation/Error/APIKeyForm/ProviderAvatar.tsx +++ /dev/null @@ -1,123 +0,0 @@ -import { - Ai360, - AiMass, - Anthropic, - Baichuan, - DeepSeek, - Fireworks, - Google, - Groq, - Minimax, - Mistral, - Moonshot, - Novita, - OpenAI, - OpenRouter, - Perplexity, - Stepfun, - Together, - Tongyi, - Upstage, - ZeroOne, - Zhipu, -} from '@lobehub/icons'; -import { useTheme } from 'antd-style'; -import { memo } from 'react'; - -import { ModelProvider } from '@/libs/agent-runtime'; - -interface ProviderAvatarProps { - provider: ModelProvider; -} - -const ProviderAvatar = memo(({ provider }) => { - const theme = useTheme(); - - switch (provider as ModelProvider) { - case ModelProvider.Google: { - return ; - } - - case ModelProvider.ZhiPu: { - return ; - } - - case ModelProvider.Minimax: { - return ; - } - - case ModelProvider.Mistral: { - return ; - } - - case ModelProvider.Moonshot: { - return ; - } - - case ModelProvider.Perplexity: { - return ; - } - - case ModelProvider.Anthropic: { - return ; - } - - case ModelProvider.Baichuan: { - return ; - } - - case ModelProvider.DeepSeek: { - return ; - } - - case ModelProvider.Groq: { - return ; - } - - case ModelProvider.OpenRouter: { - return ; - } - - case ModelProvider.Qwen: { - return ; - } - - case ModelProvider.Stepfun: { - return ; - } - - case ModelProvider.Taichu: { - return ; - } - - case ModelProvider.TogetherAI: { - return ; - } - - case ModelProvider.FireworksAI: { - return ; - } - - case ModelProvider.ZeroOne: { - return ; - } - case ModelProvider.Novita: { - return ; - } - - case ModelProvider.Ai360: { - return ; - } - - case ModelProvider.Upstage: { - return ; - } - - default: - case ModelProvider.OpenAI: { - return ; - } - } -}); - -export default ProviderAvatar;