fix(page): support image proxy in transformers (#5855)

toeverything · Dec 26, 2023 · 0f8c492 · 0f8c492 · vercel · Dec 26, 2023
1 parent 3822a25
commit 0f8c492
Show file tree

Hide file tree

Showing 12 changed files with 76 additions and 4 deletions.
diff --git a/packages/block-std/src/clipboard/index.ts b/packages/block-std/src/clipboard/index.ts
@@ -64,6 +64,7 @@ export class Clipboard {
     assertExists(adapterItem);
     const { adapter } = adapterItem;
     const snapshot = await job.sliceToSnapshot(slice);
+    adapter.applyConfigs(job.adapterConfigs);
     return (
       await adapter.fromSliceSnapshot({ snapshot, assets: job.assetsManager })
     ).file;
@@ -90,6 +91,7 @@ export class Clipboard {
       }
       if (item) {
         const job = this._getJob();
+        adapter.applyConfigs(job.adapterConfigs);
         const payload = {
           file: item,
           assets: job.assetsManager,

diff --git a/packages/blocks/src/_common/adapters/html.ts b/packages/blocks/src/_common/adapters/html.ts
@@ -49,6 +49,11 @@ type HtmlToSliceSnapshotPayload = {
 export class HtmlAdapter extends BaseAdapter<Html> {
   private _notion = new NotionHtmlAdapter();
 
+  override applyConfigs(configs: Map<string, string>) {
+    this._notion.applyConfigs(configs);
+    super.applyConfigs(configs);
+  }
+
   override async fromPageSnapshot(
     payload: FromPageSnapshotPayload
   ): Promise<FromPageSnapshotResult<string>> {

diff --git a/packages/blocks/src/_common/adapters/markdown.ts b/packages/blocks/src/_common/adapters/markdown.ts
@@ -29,6 +29,7 @@ import type { SerializedCells } from '../../database-block/database-model.js';
 import type { Column } from '../../database-block/types.js';
 import { getFilenameFromContentDisposition } from '../utils/header-value-parser.js';
 import { remarkGfm } from './gfm.js';
+import { fetchImage } from './utils.js';
 
 export type Markdown = string;
 
@@ -806,7 +807,11 @@ export class MarkdownAdapter extends BaseAdapter<Markdown> {
               }
             });
           } else {
-            const res = await fetch(o.node.url);
+            const res = await fetchImage(
+              o.node.url,
+              undefined,
+              this.configs.get('imageProxy') as string
+            );
             const clonedRes = res.clone();
             const file = new File(
               [await res.blob()],

diff --git a/packages/blocks/src/_common/adapters/notion-html.ts b/packages/blocks/src/_common/adapters/notion-html.ts
@@ -27,6 +27,7 @@ import {
   hastQuerySelector,
   type HtmlAST,
 } from './hast.js';
+import { fetchImage } from './utils.js';
 
 export type NotionHtml = string;
 
@@ -254,7 +255,11 @@ export class NotionHtmlAdapter extends BaseAdapter<NotionHtml> {
                 }
               });
             } else {
-              const res = await fetch(imageURL);
+              const res = await fetchImage(
+                imageURL,
+                undefined,
+                this.configs.get('imageProxy') as string
+              );
               const clonedRes = res.clone();
               const name =
                 getFilenameFromContentDisposition(

diff --git a/packages/blocks/src/_common/adapters/utils.ts b/packages/blocks/src/_common/adapters/utils.ts
@@ -0,0 +1,13 @@
+export const fetchImage = async (
+  url: string,
+  init?: RequestInit,
+  proxy?: string
+) => {
+  if (!proxy) {
+    return fetch(url, init);
+  }
+  return fetch(proxy + '?url=' + encodeURIComponent(url), init).catch(() => {
+    console.warn('Failed to fetch image from proxy, fallback to originial url');
+    return fetch(url, init);
+  });
+};
diff --git a/packages/blocks/src/_common/transformers/markdown.ts b/packages/blocks/src/_common/transformers/markdown.ts
@@ -2,6 +2,7 @@ import type { Page } from '@blocksuite/store';
 import { Job } from '@blocksuite/store';
 
 import { MarkdownAdapter } from '../adapters/index.js';
+import { defaultImageProxyMiddleware } from './middlewares.js';
 import { createAssetsArchive, download } from './utils.js';
 
 export async function exportPage(page: Page) {
@@ -44,8 +45,12 @@ export async function importMarkdown({
   markdown,
   noteId,
 }: ImportMarkdownOptions) {
-  const job = new Job({ workspace: page.workspace });
+  const job = new Job({
+    workspace: page.workspace,
+    middlewares: [defaultImageProxyMiddleware],
+  });
   const adapter = new MarkdownAdapter();
+  adapter.applyConfigs(job.adapterConfigs);
   const snapshot = await adapter.toSliceSnapshot({
     file: markdown,
     assets: job.assetsManager,

diff --git a/packages/blocks/src/_common/transformers/middlewares.ts b/packages/blocks/src/_common/transformers/middlewares.ts
@@ -100,3 +100,15 @@ export const replaceIdMiddleware: JobMiddleware = ({ slots, workspace }) => {
     }
   });
 };
+
+export const customImageProxyMiddleware = (
+  imageProxyURL: string
+): JobMiddleware => {
+  return ({ adapterConfigs }) => {
+    adapterConfigs.set('imageProxy', imageProxyURL);
+  };
+};
+
+export const defaultImageProxyMiddleware = customImageProxyMiddleware(
+  'https://workers.toeverything.workers.dev/proxy/image'
+);
diff --git a/packages/blocks/src/page-block/clipboard/index.ts b/packages/blocks/src/page-block/clipboard/index.ts
@@ -5,7 +5,10 @@ import type { BlockSnapshot, Page } from '@blocksuite/store';
 
 import { HtmlAdapter, ImageAdapter } from '../../_common/adapters/index.js';
 import { MarkdownAdapter } from '../../_common/adapters/markdown.js';
-import { replaceIdMiddleware } from '../../_common/transformers/middlewares.js';
+import {
+  defaultImageProxyMiddleware,
+  replaceIdMiddleware,
+} from '../../_common/transformers/middlewares.js';
 import { ClipboardAdapter } from './adapter.js';
 import { copyMiddleware, pasteMiddleware } from './middlewares/index.js';
 
@@ -68,6 +71,7 @@ export class PageClipboard {
     this._std.clipboard.use(copy);
     this._std.clipboard.use(paste);
     this._std.clipboard.use(replaceIdMiddleware);
+    this._std.clipboard.use(defaultImageProxyMiddleware);
 
     this._disposables.add({
       dispose: () => {
@@ -86,6 +90,7 @@ export class PageClipboard {
         this._std.clipboard.unuse(copy);
         this._std.clipboard.unuse(paste);
         this._std.clipboard.unuse(replaceIdMiddleware);
+        this._std.clipboard.unuse(defaultImageProxyMiddleware);
       },
     });
   };

diff --git a/packages/blocks/src/page-block/widgets/linked-page/import-page/import-page.ts b/packages/blocks/src/page-block/widgets/linked-page/import-page/import-page.ts
@@ -17,6 +17,7 @@ import {
   NewIcon,
   NotionIcon,
 } from '../../../../_common/icons/index.js';
+import { defaultImageProxyMiddleware } from '../../../../_common/transformers/middlewares.js';
 import { openFileOrFiles } from '../../../../_common/utils/index.js';
 import { styles } from './styles.js';
 
@@ -30,8 +31,10 @@ const SHOW_LOADING_SIZE = 1024 * 200;
 export async function importMarkDown(workspace: Workspace, text: string) {
   const job = new Job({
     workspace: workspace,
+    middlewares: [defaultImageProxyMiddleware],
   });
   const mdAdapter = new MarkdownAdapter();
+  mdAdapter.applyConfigs(job.adapterConfigs);
   const snapshot = await mdAdapter.toPageSnapshot({ file: text });
   const page = await job.snapshotToPage(snapshot);
   return [page.id];
@@ -40,8 +43,10 @@ export async function importMarkDown(workspace: Workspace, text: string) {
 export async function importHtml(workspace: Workspace, text: string) {
   const job = new Job({
     workspace: workspace,
+    middlewares: [defaultImageProxyMiddleware],
   });
   const htmlAdapter = new NotionHtmlAdapter();
+  htmlAdapter.applyConfigs(job.adapterConfigs);
   const snapshot = await htmlAdapter.toPageSnapshot({ file: text });
   const page = await job.snapshotToPage(snapshot);
   return [page.id];
@@ -95,8 +100,10 @@ export async function importNotion(workspace: Workspace, file: File) {
     const pagePromises = Array.from(pageMap.keys()).map(async file => {
       const job = new Job({
         workspace: workspace,
+        middlewares: [defaultImageProxyMiddleware],
       });
       const htmlAdapter = new NotionHtmlAdapter();
+      htmlAdapter.applyConfigs(job.adapterConfigs);
       const snapshot = await htmlAdapter.toPageSnapshot({
         file: await zipFile.files[file].async('text'),
         pageId: pageMap.get(file),

diff --git a/packages/store/src/adapter/base.ts b/packages/store/src/adapter/base.ts
@@ -46,6 +46,8 @@ export type ToSliceSnapshotPayload<Target> = {
 };
 
 export abstract class BaseAdapter<AdapterTarget = unknown> {
+  protected configs: Map<string, unknown> = new Map();
+
   abstract fromPageSnapshot(
     payload: FromPageSnapshotPayload
   ): Promise<FromPageSnapshotResult<AdapterTarget>>;
@@ -64,6 +66,10 @@ export abstract class BaseAdapter<AdapterTarget = unknown> {
   abstract toSliceSnapshot(
     payload: ToSliceSnapshotPayload<AdapterTarget>
   ): Promise<SliceSnapshot>;
+
+  applyConfigs(configs: Map<string, unknown>) {
+    this.configs = new Map([...configs]);
+  }
 }
 
 type Keyof<T> = T extends unknown ? keyof T : never;

diff --git a/packages/store/src/transformer/job.ts b/packages/store/src/transformer/job.ts
@@ -36,6 +36,7 @@ export type JobConfig = {
 export class Job {
   private readonly _workspace: Workspace;
   private readonly _assetsManager: AssetsManager;
+  private readonly _adapterConfigs: Map<string, string> = new Map();
 
   private readonly _slots: JobSlots = {
     beforeImport: new Slot<BeforeImportPayload>(),
@@ -53,6 +54,7 @@ export class Job {
         slots: this._slots,
         assetsManager: this._assetsManager,
         workspace: this._workspace,
+        adapterConfigs: this._adapterConfigs,
       });
     });
   }
@@ -65,6 +67,10 @@ export class Job {
     return this._assetsManager.getAssets();
   }
 
+  get adapterConfigs() {
+    return this._adapterConfigs;
+  }
+
   reset() {
     this._assetsManager.cleanup();
   }

diff --git a/packages/store/src/transformer/middleware.ts b/packages/store/src/transformer/middleware.ts
@@ -82,6 +82,7 @@ type JobMiddlewareOptions = {
   workspace: Workspace;
   assetsManager: AssetsManager;
   slots: JobSlots;
+  adapterConfigs: Map<string, string>;
 };
 
 export type JobMiddleware = (options: JobMiddlewareOptions) => void;