Merge remote-tracking branch 'upstream/main'

changelog/v1.json

@@ -1,4 +1,18 @@
 [
+  {
+    "children": {
+      "improvements": ["Refactor client mode upload to match server mode."]
+    },
+    "date": "2024-12-21",
+    "version": "1.36.46"
+  },
+  {
+    "children": {
+      "improvements": ["Add o1 model in GitHub models."]
+    },
+    "date": "2024-12-21",
+    "version": "1.36.45"
+  },
   {
     "children": {
       "improvements": ["Add Gemini flash thinking model."]

src/config/modelProviders/github.ts

@@ -4,6 +4,16 @@ import { ModelProviderCard } from '@/types/llm';
 // https://github.com/marketplace/models
 const Github: ModelProviderCard = {
   chatModels: [
+    {
+      description: '专注于高级推理和解决复杂问题，包括数学和科学任务。非常适合需要深入上下文理解和代理工作流程的应用程序。',
+      displayName: 'OpenAI o1',
+      enabled: true,
+      functionCall: false,
+      id: 'o1',
+      maxOutput: 100_000,
+      tokens: 200_000,
+      vision: true,
+    },
     {
       description: '比 o1-preview 更小、更快，成本低80%，在代码生成和小上下文操作方面表现良好。',
       displayName: 'OpenAI o1-mini',

src/database/_deprecated/models/file.ts

@@ -1,5 +1,6 @@
 import { DBModel } from '@/database/_deprecated/core/types/db';
 import { DB_File, DB_FileSchema } from '@/database/_deprecated/schemas/files';
+import { clientS3Storage } from '@/services/file/ClientS3';
 import { nanoid } from '@/utils/uuid';
 
 import { BaseModel } from '../core';
@@ -20,9 +21,15 @@ class _FileModel extends BaseModel<'files'> {
     if (!item) return;
 
     // arrayBuffer to url
-    const base64 = Buffer.from(item.data!).toString('base64');
-
-    return { ...item, url: `data:${item.fileType};base64,${base64}` };
+    let base64;
+    if (!item.data) {
+      const hash = (item.url as string).replace('client-s3://', '');
+      base64 = await this.getBase64ByFileHash(hash);
+    } else {
+      base64 = Buffer.from(item.data).toString('base64');
+    }
+
+    return { ...item, base64, url: `data:${item.fileType};base64,${base64}` };
   }
 
   async delete(id: string) {
@@ -32,6 +39,13 @@ class _FileModel extends BaseModel<'files'> {
   async clear() {
     return this.table.clear();
   }
+
+  private async getBase64ByFileHash(hash: string) {
+    const fileItem = await clientS3Storage.getObject(hash);
+    if (!fileItem) throw new Error('file not found');
+
+    return Buffer.from(await fileItem.arrayBuffer()).toString('base64');
+  }
 }
 
 export const FileModel = new _FileModel();

src/server/routers/lambda/file.ts

@@ -32,9 +32,7 @@ export const fileRouter = router({
     }),
 
   createFile: fileProcedure
-    .input(
-      UploadFileSchema.omit({ data: true, saveMode: true, url: true }).extend({ url: z.string() }),
-    )
+    .input(UploadFileSchema.omit({ url: true }).extend({ url: z.string() }))
     .mutation(async ({ ctx, input }) => {
       const { isExist } = await ctx.fileModel.checkHash(input.hash!);
 

src/services/__tests__/upload.test.ts

@@ -0,0 +1,175 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { fileEnv } from '@/config/file';
+import { edgeClient } from '@/libs/trpc/client';
+import { API_ENDPOINTS } from '@/services/_url';
+import { clientS3Storage } from '@/services/file/ClientS3';
+
+import { UPLOAD_NETWORK_ERROR, uploadService } from '../upload';
+
+// Mock dependencies
+vi.mock('@/libs/trpc/client', () => ({
+  edgeClient: {
+    upload: {
+      createS3PreSignedUrl: {
+        mutate: vi.fn(),
+      },
+    },
+  },
+}));
+
+vi.mock('@/services/file/ClientS3', () => ({
+  clientS3Storage: {
+    putObject: vi.fn(),
+  },
+}));
+
+vi.mock('@/utils/uuid', () => ({
+  uuid: () => 'mock-uuid',
+}));
+
+describe('UploadService', () => {
+  const mockFile = new File(['test'], 'test.png', { type: 'image/png' });
+  const mockPreSignUrl = 'https://example.com/presign';
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    // Mock Date.now
+    vi.spyOn(Date, 'now').mockImplementation(() => 3600000); // 1 hour in milliseconds
+  });
+
+  describe('uploadWithProgress', () => {
+    beforeEach(() => {
+      // Mock XMLHttpRequest
+      const xhrMock = {
+        upload: {
+          addEventListener: vi.fn(),
+        },
+        open: vi.fn(),
+        send: vi.fn(),
+        setRequestHeader: vi.fn(),
+        addEventListener: vi.fn(),
+        status: 200,
+      };
+      global.XMLHttpRequest = vi.fn(() => xhrMock) as any;
+
+      // Mock createS3PreSignedUrl
+      (edgeClient.upload.createS3PreSignedUrl.mutate as any).mockResolvedValue(mockPreSignUrl);
+    });
+
+    it('should upload file successfully with progress', async () => {
+      const onProgress = vi.fn();
+      const xhr = new XMLHttpRequest();
+
+      // Simulate successful upload
+      vi.spyOn(xhr, 'addEventListener').mockImplementation((event, handler) => {
+        if (event === 'load') {
+          // @ts-ignore
+          handler({ target: { status: 200 } });
+        }
+      });
+
+      const result = await uploadService.uploadWithProgress(mockFile, { onProgress });
+
+      expect(result).toEqual({
+        date: '1',
+        dirname: `${fileEnv.NEXT_PUBLIC_S3_FILE_PATH}/1`,
+        filename: 'mock-uuid.png',
+        path: `${fileEnv.NEXT_PUBLIC_S3_FILE_PATH}/1/mock-uuid.png`,
+      });
+    });
+
+    it('should handle network error', async () => {
+      const xhr = new XMLHttpRequest();
+
+      // Simulate network error
+      vi.spyOn(xhr, 'addEventListener').mockImplementation((event, handler) => {
+        if (event === 'error') {
+          Object.assign(xhr, { status: 0 });
+          // @ts-ignore
+          handler({});
+        }
+      });
+
+      await expect(uploadService.uploadWithProgress(mockFile, {})).rejects.toBe(
+        UPLOAD_NETWORK_ERROR,
+      );
+    });
+
+    it('should handle upload error', async () => {
+      const xhr = new XMLHttpRequest();
+
+      // Simulate upload error
+      vi.spyOn(xhr, 'addEventListener').mockImplementation((event, handler) => {
+        if (event === 'load') {
+          Object.assign(xhr, { status: 400, statusText: 'Bad Request' });
+
+          // @ts-ignore
+          handler({});
+        }
+      });
+
+      await expect(uploadService.uploadWithProgress(mockFile, {})).rejects.toBe('Bad Request');
+    });
+  });
+
+  describe('uploadToClientS3', () => {
+    it('should upload file to client S3 successfully', async () => {
+      const hash = 'test-hash';
+      const expectedResult = {
+        date: '1',
+        dirname: '',
+        filename: mockFile.name,
+        path: `client-s3://${hash}`,
+      };
+
+      (clientS3Storage.putObject as any).mockResolvedValue(undefined);
+
+      const result = await uploadService.uploadToClientS3(hash, mockFile);
+
+      expect(clientS3Storage.putObject).toHaveBeenCalledWith(hash, mockFile);
+      expect(result).toEqual(expectedResult);
+    });
+  });
+
+  describe('getImageFileByUrlWithCORS', () => {
+    beforeEach(() => {
+      global.fetch = vi.fn();
+    });
+
+    it('should fetch and create file from URL', async () => {
+      const url = 'https://example.com/image.png';
+      const filename = 'test.png';
+      const mockArrayBuffer = new ArrayBuffer(8);
+
+      (global.fetch as any).mockResolvedValue({
+        arrayBuffer: () => Promise.resolve(mockArrayBuffer),
+      });
+
+      const result = await uploadService.getImageFileByUrlWithCORS(url, filename);
+
+      expect(global.fetch).toHaveBeenCalledWith(API_ENDPOINTS.proxy, {
+        body: url,
+        method: 'POST',
+      });
+      expect(result).toBeInstanceOf(File);
+      expect(result.name).toBe(filename);
+      expect(result.type).toBe('image/png');
+    });
+
+    it('should handle custom file type', async () => {
+      const url = 'https://example.com/image.jpg';
+      const filename = 'test.jpg';
+      const fileType = 'image/jpeg';
+      const mockArrayBuffer = new ArrayBuffer(8);
+
+      (global.fetch as any).mockResolvedValue({
+        arrayBuffer: () => Promise.resolve(mockArrayBuffer),
+      });
+
+      const result = await uploadService.getImageFileByUrlWithCORS(url, filename, fileType);
+
+      expect(result.type).toBe(fileType);
+    });
+  });
+});

src/services/file/ClientS3/index.test.ts

@@ -0,0 +1,115 @@
+import { createStore, del, get, set } from 'idb-keyval';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { BrowserS3Storage } from './index';
+
+// Mock idb-keyval
+vi.mock('idb-keyval', () => ({
+  createStore: vi.fn(),
+  set: vi.fn(),
+  get: vi.fn(),
+  del: vi.fn(),
+}));
+
+let storage: BrowserS3Storage;
+let mockStore = {};
+
+beforeEach(() => {
+  // Reset all mocks before each test
+  vi.clearAllMocks();
+  mockStore = {};
+  (createStore as any).mockReturnValue(mockStore);
+  storage = new BrowserS3Storage();
+});
+
+describe('BrowserS3Storage', () => {
+  describe('constructor', () => {
+    it('should create store when in browser environment', () => {
+      expect(createStore).toHaveBeenCalledWith('lobechat-local-s3', 'objects');
+    });
+  });
+
+  describe('putObject', () => {
+    it('should successfully put a file object', async () => {
+      const mockFile = new File(['test content'], 'test.txt', { type: 'text/plain' });
+      const mockArrayBuffer = new ArrayBuffer(8);
+      vi.spyOn(mockFile, 'arrayBuffer').mockResolvedValue(mockArrayBuffer);
+      (set as any).mockResolvedValue(undefined);
+
+      await storage.putObject('1-test-key', mockFile);
+
+      expect(set).toHaveBeenCalledWith(
+        '1-test-key',
+        {
+          data: mockArrayBuffer,
+          name: 'test.txt',
+          type: 'text/plain',
+        },
+        mockStore,
+      );
+    });
+
+    it('should throw error when put operation fails', async () => {
+      const mockFile = new File(['test content'], 'test.txt', { type: 'text/plain' });
+      const mockError = new Error('Storage error');
+      (set as any).mockRejectedValue(mockError);
+
+      await expect(storage.putObject('test-key', mockFile)).rejects.toThrow(
+        'Failed to put file test.txt: Storage error',
+      );
+    });
+  });
+
+  describe('getObject', () => {
+    it('should successfully get a file object', async () => {
+      const mockData = {
+        data: new ArrayBuffer(8),
+        name: 'test.txt',
+        type: 'text/plain',
+      };
+      (get as any).mockResolvedValue(mockData);
+
+      const result = await storage.getObject('test-key');
+
+      expect(result).toBeInstanceOf(File);
+      expect(result?.name).toBe('test.txt');
+      expect(result?.type).toBe('text/plain');
+    });
+
+    it('should return undefined when file not found', async () => {
+      (get as any).mockResolvedValue(undefined);
+
+      const result = await storage.getObject('test-key');
+
+      expect(result).toBeUndefined();
+    });
+
+    it('should throw error when get operation fails', async () => {
+      const mockError = new Error('Storage error');
+      (get as any).mockRejectedValue(mockError);
+
+      await expect(storage.getObject('test-key')).rejects.toThrow(
+        'Failed to get object (key=test-key): Storage error',
+      );
+    });
+  });
+
+  describe('deleteObject', () => {
+    it('should successfully delete a file object', async () => {
+      (del as any).mockResolvedValue(undefined);
+
+      await storage.deleteObject('test-key2');
+
+      expect(del).toHaveBeenCalledWith('test-key2', {});
+    });
+
+    it('should throw error when delete operation fails', async () => {
+      const mockError = new Error('Storage error');
+      (del as any).mockRejectedValue(mockError);
+
+      await expect(storage.deleteObject('test-key')).rejects.toThrow(
+        'Failed to delete object (key=test-key): Storage error',
+      );
+    });
+  });
+});