From 5ea5de7faa716e3f56142da216ddc1129ea6b675 Mon Sep 17 00:00:00 2001 From: 3720 Date: Sat, 16 Dec 2023 01:51:53 +0800 Subject: [PATCH] feat(presets): chat with workspace (#5748) --- .../apps/starter/components/debug-menu.ts | 98 ++---- .../starter/components/left-side-panel.ts | 54 ++++ .../apps/starter/components/pages-panel.ts | 123 +++++++ .../apps/starter/components/side-panel.ts | 8 + packages/playground/apps/starter/main.ts | 8 + .../copilot-panel/chat-with-workspace.ts | 300 ++++++++++++++++++ .../fragments/copilot-panel/copilot-panel.ts | 10 + .../fragments/copilot-panel/utils/request.ts | 29 ++ tests/utils/actions/click.ts | 9 +- tests/utils/actions/misc.ts | 7 + 10 files changed, 567 insertions(+), 79 deletions(-) create mode 100644 packages/playground/apps/starter/components/left-side-panel.ts create mode 100644 packages/playground/apps/starter/components/pages-panel.ts create mode 100644 packages/presets/src/fragments/copilot-panel/chat-with-workspace.ts diff --git a/packages/playground/apps/starter/components/debug-menu.ts b/packages/playground/apps/starter/components/debug-menu.ts index af3469da8b3a..bd52ac451231 100644 --- a/packages/playground/apps/starter/components/debug-menu.ts +++ b/packages/playground/apps/starter/components/debug-menu.ts @@ -14,11 +14,12 @@ import '@shoelace-style/shoelace/dist/components/tab/tab.js'; import '@shoelace-style/shoelace/dist/components/tooltip/tooltip.js'; import '@shoelace-style/shoelace/dist/themes/light.css'; import '@shoelace-style/shoelace/dist/themes/dark.css'; +import './left-side-panel'; +import './side-panel'; import { BlocksUtils, ColorVariables, - createDefaultPage, extractCssVariables, FontFamilyVariables, HtmlTransformer, @@ -51,6 +52,8 @@ import type { Pane } from 'tweakpane'; import { extendFormatBar } from './custom-format-bar.js'; import type { CustomFramePanel } from './custom-frame-panel.js'; import type { CustomTOCOutlinePanel } from './custom-toc-outline-panel.js'; +import type { LeftSidePanel } from './left-side-panel'; +import type { PagesPanel } from './pages-panel'; import type { SidePanel } from './side-panel'; export function getSurfaceElementFromEditor(editor: AffineEditorContainer) { @@ -212,6 +215,10 @@ export class DebugMenu extends ShadowlessElement { @property({ attribute: false }) sidePanel!: SidePanel; + @property({ attribute: false }) + leftSidePanel!: LeftSidePanel; + @property({ attribute: false }) + pagesPanel!: PagesPanel; @state() private _connected = true; @@ -324,11 +331,10 @@ export class DebugMenu extends ShadowlessElement { } private _toggleCopilotPanel() { - if (this.sidePanel.currentContent === this.copilotPanel) { - this.sidePanel.hideContent(); - } else { - this.sidePanel.showContent(this.copilotPanel); - } + this.sidePanel.toggle(this.copilotPanel); + } + private _togglePagesPanel() { + this.leftSidePanel.toggle(this.pagesPanel); } private _createMindMap() { @@ -411,7 +417,9 @@ export class DebugMenu extends ShadowlessElement { } private async _exportSnapshot() { - const file = await ZipTransformer.exportPages(this.workspace, [this.page]); + const file = await ZipTransformer.exportPages(this.workspace, [ + ...this.workspace.pages.values(), + ]); const url = URL.createObjectURL(file); const a = document.createElement('a'); a.setAttribute('href', url); @@ -707,17 +715,6 @@ export class DebugMenu extends ShadowlessElement { - - createPageBlock(this.workspace)} - > - - - - - ${PageList(this.workspace, this.editor, () => this.requestUpdate())} - + + Pages + `; } } -function createPageBlock(workspace: Workspace) { - const id = workspace.idGenerator('page'); - createDefaultPage(workspace, { id }).catch(console.error); -} - -function PageList( - workspace: Workspace, - editor: AffineEditorContainer, - requestUpdate: () => void -) { - workspace.meta.pageMetasUpdated.on(requestUpdate); - - // This function is called when a delete option is clicked - const handleDeletePage = (pageId: string) => { - workspace.removePage(pageId); - // When delete a page, we need to set the editor page to the first remaining page - const pages = Array.from(workspace.pages.values()); - editor.page = pages[0]; - requestUpdate(); - }; - - // Create a dropdown menu for each page with a delete option - return html` - - Pages - - ${workspace.meta.pageMetas.map( - pageMeta => html` - - ${pageMeta.title || 'Untitled'} - - - Open - - - Delete - - - - ` - )} - - - `; -} - declare global { interface HTMLElementTagNameMap { 'debug-menu': DebugMenu; diff --git a/packages/playground/apps/starter/components/left-side-panel.ts b/packages/playground/apps/starter/components/left-side-panel.ts new file mode 100644 index 000000000000..e8e68accf172 --- /dev/null +++ b/packages/playground/apps/starter/components/left-side-panel.ts @@ -0,0 +1,54 @@ +import { ShadowlessElement } from '@blocksuite/lit'; +import { css, html } from 'lit'; +import { customElement } from 'lit/decorators.js'; + +@customElement('left-side-panel') +export class LeftSidePanel extends ShadowlessElement { + static override styles = css` + left-side-panel { + padding-top: 50px; + width: 300px; + position: absolute; + top: 0; + left: 0; + height: 100%; + display: none; + } + `; + currentContent: HTMLElement | null = null; + + showContent(ele: HTMLElement) { + if (this.currentContent) { + this.currentContent.remove(); + } + this.style.display = 'block'; + this.currentContent = ele; + this.append(ele); + } + + hideContent() { + if (this.currentContent) { + this.style.display = 'none'; + this.currentContent.remove(); + this.currentContent = null; + } + } + + toggle(ele: HTMLElement) { + if (this.currentContent !== ele) { + this.showContent(ele); + } else { + this.hideContent(); + } + } + + protected override render(): unknown { + return html``; + } +} + +declare global { + interface HTMLElementTagNameMap { + 'left-side-panel': LeftSidePanel; + } +} diff --git a/packages/playground/apps/starter/components/pages-panel.ts b/packages/playground/apps/starter/components/pages-panel.ts new file mode 100644 index 000000000000..1c109cfe1d5c --- /dev/null +++ b/packages/playground/apps/starter/components/pages-panel.ts @@ -0,0 +1,123 @@ +import { createDefaultPage } from '@blocksuite/blocks'; +import { CloseIcon } from '@blocksuite/blocks/_common/icons'; +import { ShadowlessElement, WithDisposable } from '@blocksuite/lit'; +import type { AffineEditorContainer } from '@blocksuite/presets'; +import type { Workspace } from '@blocksuite/store'; +import { css, html } from 'lit'; +import { customElement, property } from 'lit/decorators.js'; +import { repeat } from 'lit/directives/repeat.js'; +import { styleMap } from 'lit/directives/style-map.js'; + +@customElement('pages-panel') +export class PagesPanel extends WithDisposable(ShadowlessElement) { + static override styles = css` + pages-panel { + display: flex; + flex-direction: column; + width: 100%; + background-color: var(--affine-background-secondary-color); + font-family: var(--affine-font-family); + height: 100%; + padding: 12px; + gap: 4px; + } + .page-item:hover .delete-page-icon { + display: flex; + } + .delete-page-icon { + display: none; + padding: 2px; + border-radius: 4px; + } + .delete-page-icon:hover { + background-color: var(--affine-hover-color); + } + .delete-page-icon svg { + width: 14px; + height: 14px; + color: var(--affine-secondary-color); + fill: var(--affine-secondary-color); + } + .new-page-button { + margin-bottom: 16px; + border: 1px solid var(--affine-border-color); + border-radius: 4px; + height: 32px; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + } + .new-page-button:hover { + background-color: var(--affine-hover-color); + } + `; + @property({ attribute: false }) + editor!: AffineEditorContainer; + + public override connectedCallback() { + super.connectedCallback(); + this.disposables.add( + this.editor.page.workspace.slots.pagesUpdated.on(() => { + this.requestUpdate(); + }) + ); + } + + createPage = () => { + createPageBlock(this.editor.page.workspace); + }; + + protected override render(): unknown { + const workspace = this.editor.page.workspace; + const pages = [...workspace.pages.values()]; + return html` +
New Page
+ ${repeat( + pages, + v => v.id, + page => { + const style = styleMap({ + backgroundColor: + this.editor.page.id === page.id + ? 'var(--affine-hover-color)' + : undefined, + padding: '4px 4px 4px 8px', + borderRadius: '4px', + cursor: 'pointer', + display: 'flex', + justifyContent: 'space-between', + }); + const click = () => { + this.editor.page = page; + this.requestUpdate(); + }; + const deletePage = () => { + workspace.removePage(page.id); + // When delete a page, we need to set the editor page to the first remaining page + const pages = Array.from(workspace.pages.values()); + this.editor.page = pages[0]; + this.requestUpdate(); + }; + return html`
+ ${page.meta.title || 'Untitled'} +
+ ${CloseIcon} +
+
`; + } + )} + `; + } +} + +function createPageBlock(workspace: Workspace) { + const id = workspace.idGenerator('page'); + createDefaultPage(workspace, { id }).catch(console.error); +} + +declare global { + interface HTMLElementTagNameMap { + 'pages-panel': PagesPanel; + } +} diff --git a/packages/playground/apps/starter/components/side-panel.ts b/packages/playground/apps/starter/components/side-panel.ts index 16ab21bbc104..b161c631b185 100644 --- a/packages/playground/apps/starter/components/side-panel.ts +++ b/packages/playground/apps/starter/components/side-panel.ts @@ -35,4 +35,12 @@ export class SidePanel extends ShadowlessElement { protected override render(): unknown { return html``; } + + toggle(ele: HTMLElement) { + if (this.currentContent !== ele) { + this.showContent(ele); + } else { + this.hideContent(); + } + } } diff --git a/packages/playground/apps/starter/main.ts b/packages/playground/apps/starter/main.ts index a2586139a6fc..fbdceda279b6 100644 --- a/packages/playground/apps/starter/main.ts +++ b/packages/playground/apps/starter/main.ts @@ -16,6 +16,8 @@ import { Job, Workspace } from '@blocksuite/store'; import { CustomFramePanel } from './components/custom-frame-panel'; import { CustomTOCOutlinePanel } from './components/custom-toc-outline-panel.js'; import { DebugMenu } from './components/debug-menu.js'; +import { LeftSidePanel } from './components/left-side-panel'; +import { PagesPanel } from './components/pages-panel'; import { SidePanel } from './components/side-panel'; import type { InitFn } from './data'; import { @@ -50,6 +52,8 @@ function subscribePage(workspace: Workspace) { const framePanel = new CustomFramePanel(); const copilotPanelPanel = new CopilotPanel(); const sidePanel = new SidePanel(); + const leftSidePanel = new LeftSidePanel(); + const pagesPanel = new PagesPanel(); debugMenu.workspace = workspace; debugMenu.editor = editor; @@ -59,14 +63,18 @@ function subscribePage(workspace: Workspace) { debugMenu.framePanel = framePanel; debugMenu.copilotPanel = copilotPanelPanel; debugMenu.sidePanel = sidePanel; + debugMenu.leftSidePanel = leftSidePanel; + debugMenu.pagesPanel = pagesPanel; tocOutlinePanel.editor = editor; copilotPanelPanel.editor = editor; framePanel.editor = editor; + pagesPanel.editor = editor; document.body.appendChild(debugMenu); document.body.appendChild(tocOutlinePanel); document.body.appendChild(sidePanel); + document.body.appendChild(leftSidePanel); document.body.appendChild(framePanel); window.editor = editor; diff --git a/packages/presets/src/fragments/copilot-panel/chat-with-workspace.ts b/packages/presets/src/fragments/copilot-panel/chat-with-workspace.ts new file mode 100644 index 000000000000..a01c90138873 --- /dev/null +++ b/packages/presets/src/fragments/copilot-panel/chat-with-workspace.ts @@ -0,0 +1,300 @@ +import { MarkdownAdapter } from '@blocksuite/blocks'; +import { ShadowlessElement, WithDisposable } from '@blocksuite/lit'; +import { Job, type Page } from '@blocksuite/store'; +import { css, html } from 'lit'; +import { customElement, property, state } from 'lit/decorators.js'; +import { repeat } from 'lit/directives/repeat.js'; +import { styleMap } from 'lit/directives/style-map.js'; + +import type { AffineEditorContainer } from '../../editors/index.js'; +import { askGPT3_5turbo_1106, embeddings } from './utils/request.js'; + +type EmbeddedPage = { + id: string; + sections: { + vector: number[]; + text: string; + }[]; +}; + +@customElement('chat-with-workspace-panel') +export class ChatWithWorkspacePanel extends WithDisposable(ShadowlessElement) { + static override styles = css` + chat-with-workspace-panel { + margin-top: 12px; + display: flex; + flex-direction: column; + width: 100%; + font-family: var(--affine-font-family); + height: 100%; + gap: 4px; + } + + .chat-with-workspace-prompt-container { + display: flex; + gap: 8px; + height: 30px; + margin-top: 24px; + } + + .chat-with-workspace-prompt { + flex: 1; + border: none; + border-radius: 4px; + padding: 4px 8px; + outline: none; + background-color: white; + } + + .send-button { + width: 36px; + background-color: var(--affine-primary-color); + border-radius: 4px; + color: white; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + } + + .sync-workspace-button { + border: 1px solid var(--affine-border-color); + height: 32px; + border-radius: 4px; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + } + + .synced-page-list { + margin-bottom: 14px; + color: var(--affine-text-secondary-color); + font-size: 12px; + } + + .history-item { + display: flex; + flex-direction: column; + padding: 4px 8px; + border-radius: 4px; + font-size: 14px; + } + + .history-refs { + font-size: 12px; + color: var(--affine-text-secondary-color); + } + `; + @property({ attribute: false }) + editor!: AffineEditorContainer; + + @state() + history: { + role: 'user' | 'assistant'; + content: string; + sources: { + id: string; + slice: string[]; + }[]; + }[] = []; + + @state() + value = ''; + @state() + syncedPages: EmbeddedPage[] = []; + + public override connectedCallback() { + super.connectedCallback(); + this.disposables.add( + this.editor.page.workspace.slots.pagesUpdated.on(() => { + this.requestUpdate(); + }) + ); + } + + private ask = async () => { + const value = this.value; + this.history.push({ role: 'user', content: value, sources: [] }); + this.value = ''; + const [result] = await embeddings([value]); + const list = this.syncedPages + .flatMap(page => { + return page.sections.map(section => ({ + id: page.id, + distance: distance(result, section.vector), + text: section.text, + })); + }) + .sort((a, b) => a.distance - b.distance) + .filter(v => v.distance < 0.7) + .slice(0, 3); + const r = await askGPT3_5turbo_1106([ + { + role: 'system', + content: `the background is:\n${list.map(v => v.text).join('\n')}`, + }, + ...this.history.map(v => ({ role: v.role, content: v.content })), + ]); + const refs: Record< + string, + { + slice: string[]; + } + > = {}; + list.forEach(v => { + const ref = refs[v.id] ?? (refs[v.id] = { slice: [] }); + ref.slice.push(v.text); + }); + this.history.push({ + role: 'assistant', + content: r ?? '', + sources: Object.entries(refs).map(([id, ref]) => ({ + id, + slice: ref.slice, + })), + }); + this.requestUpdate(); + }; + + splitPage = async (page: Page): Promise => { + const markdown = await pageToMarkdown(page); + return splitText(markdown); + }; + + embeddingPages = async (pageList: Page[]): Promise => { + const result: Record = {}; + const list = ( + await Promise.all( + pageList.map(async page => + (await this.splitPage(page)).map(v => ({ id: page.id, text: v })) + ) + ) + ).flat(); + const vectors = await embeddings(list.map(v => v.text)); + list.forEach((v, i) => { + const page = result[v.id] ?? (result[v.id] = { id: v.id, sections: [] }); + page.sections.push({ vector: vectors[i], text: v.text }); + }); + return Object.values(result); + }; + + syncWorkspace = async () => { + this.syncedPages = await this.embeddingPages([ + ...this.editor.page.workspace.pages.values(), + ]); + }; + + protected override render(): unknown { + return html` +
+ Sync Workspace +
+
+
Synced pages:
+ ${this.syncedPages.length + ? repeat(this.syncedPages, page => { + const title = + this.editor.page.workspace.getPage(page.id)?.meta.title ?? + 'Untitled'; + return html`
${title}
`; + }) + : 'Empty'} +
+ ${repeat(this.history, data => { + const style = styleMap({ + alignItems: data.role === 'user' ? 'flex-end' : 'flex-start', + backgroundColor: + data.role === 'user' ? undefined : 'var(--affine-hover-color)', + }); + return html`
+
${data.content}
+ ${data.sources?.length + ? html`
+
sources:
+
+ ${repeat(data.sources, ref => { + const page = this.editor.page.workspace.getPage(ref.id); + if (!page) { + return; + } + const title = page.meta.title || 'Untitled'; + const jumpTo = () => { + this.editor.page = page; + }; + return html` ${title}`; + })} +
+
` + : null} +
`; + })} +
+ +
+ +
+
+ `; + } +} + +const pageToMarkdown = async (page: Page) => { + const job = new Job({ workspace: page.workspace }); + const snapshot = await job.pageToSnapshot(page); + const result = await new MarkdownAdapter().fromPageSnapshot({ + snapshot, + assets: job.assetsManager, + }); + return result.file; +}; +const distance = (a: number[], b: number[]) => { + let sum = 0; + for (let i = 0; i < a.length; i++) { + sum += (a[i] - b[i]) ** 2; + } + return Math.sqrt(sum); +}; + +const split = (text: string, n: number) => { + const result: string[] = []; + while (text.length) { + result.push(text.slice(0, n)); + text = text.slice(n); + } + return result; +}; +const maxChunk = 300; +const splitText = (text: string) => { + const data = text.split(/(?<=[\n。，.,])/).flatMap(s => { + if (s.length > maxChunk) { + return split(s, Math.ceil(s.length / maxChunk)); + } + return [s]; + }); + const result: string[] = []; + let current = ''; + for (const item of data) { + if (current.length + item.length > maxChunk) { + result.push(current); + current = ''; + } + current += item; + } + if (current.length) { + result.push(current); + } + return result; +}; diff --git a/packages/presets/src/fragments/copilot-panel/copilot-panel.ts b/packages/presets/src/fragments/copilot-panel/copilot-panel.ts index 9413cf0435f6..db963882b646 100644 --- a/packages/presets/src/fragments/copilot-panel/copilot-panel.ts +++ b/packages/presets/src/fragments/copilot-panel/copilot-panel.ts @@ -1,3 +1,5 @@ +import './chat-with-workspace.js'; + import { type EditorHost, ShadowlessElement, @@ -306,6 +308,11 @@ export class CopilotPanel extends WithDisposable(ShadowlessElement) {
${this._ResultArea()}
`; }; + workspace = () => { + return html` `; + }; edgeless = () => { return html`
@@ -359,6 +366,9 @@ export class CopilotPanel extends WithDisposable(ShadowlessElement) { edgeless: { render: this.edgeless, }, + workspace: { + render: this.workspace, + }, }; @state() currentPanel: keyof typeof this.panels = 'config'; diff --git a/packages/presets/src/fragments/copilot-panel/utils/request.ts b/packages/presets/src/fragments/copilot-panel/utils/request.ts index 98f62bc578ab..c40cad81464b 100644 --- a/packages/presets/src/fragments/copilot-panel/utils/request.ts +++ b/packages/presets/src/fragments/copilot-panel/utils/request.ts @@ -68,6 +68,19 @@ export const askGPT4V = async ( }); return result.choices[0].message.content; }; +export const embeddings = async (textList: string[]) => { + const apiKey = getGPTAPIKey(); + const openai = new OpenAI({ + apiKey: apiKey, + dangerouslyAllowBrowser: true, + }); + const result = await openai.embeddings.create({ + input: textList, + model: 'text-embedding-ada-002', + encoding_format: 'float', + }); + return result.data.map(v => v.embedding); +}; const getGPTAPIKey = () => { const apiKey = APIKeys.GPTAPIKey; if (!apiKey) { @@ -100,3 +113,19 @@ export const askGPT3_5turbo = async ( }); return result.choices[0].message; }; +export const askGPT3_5turbo_1106 = async ( + messages: Array +) => { + const apiKey = getGPTAPIKey(); + const openai = new OpenAI({ + apiKey: apiKey, + dangerouslyAllowBrowser: true, + }); + const result = await openai.chat.completions.create({ + messages, + model: 'gpt-3.5-turbo-1106', + temperature: 0, + max_tokens: 4096, + }); + return result.choices[0].message.content; +}; diff --git a/tests/utils/actions/click.ts b/tests/utils/actions/click.ts index adf931274178..ea4721cbc08e 100644 --- a/tests/utils/actions/click.ts +++ b/tests/utils/actions/click.ts @@ -17,7 +17,7 @@ function getDebugMenu(page: Page) { name: 'Test Operations', }), - addNewPageBtn: debugMenu.locator('sl-tooltip[content="Add New Page"]'), + pagesBtn: debugMenu.getByTestId('pages-button'), }; } @@ -64,8 +64,11 @@ export async function addNoteByClick(page: Page) { } export async function addNewPage(page: Page) { - const { addNewPageBtn } = getDebugMenu(page); - await addNewPageBtn.click(); + const { pagesBtn } = getDebugMenu(page); + if (!(await page.locator('pages-panel').isVisible())) { + await pagesBtn.click(); + } + await page.locator('.new-page-button').click(); const pageMetas = await page.evaluate(() => { const { workspace } = window; return workspace.meta.pageMetas; diff --git a/tests/utils/actions/misc.ts b/tests/utils/actions/misc.ts index baf111fe620e..55116c00caab 100644 --- a/tests/utils/actions/misc.ts +++ b/tests/utils/actions/misc.ts @@ -20,6 +20,7 @@ import { type InlineRootElement, } from '../../../packages/inline/src/index.js'; import type { DebugMenu } from '../../../packages/playground/apps/starter/components/debug-menu.js'; +import type { PagesPanel } from '../../../packages/playground/apps/starter/components/pages-panel.js'; import type { BaseBlockModel } from '../../../packages/store/src/index.js'; import { currentEditorIndex, multiEditor } from '../multiple-editor.js'; import { @@ -103,10 +104,16 @@ async function initEmptyEditor({ if (multiEditor) createEditor(); const debugMenu: DebugMenu = document.createElement('debug-menu'); + const pagesPanel: PagesPanel = document.createElement('pages-panel'); + pagesPanel.editor = editor; debugMenu.workspace = workspace; debugMenu.editor = editor; + debugMenu.pagesPanel = pagesPanel; + const leftSidePanel = document.createElement('left-side-panel'); + debugMenu.leftSidePanel = leftSidePanel; debugMenu.contentParser = new window.ContentParser(page); document.body.appendChild(debugMenu); + document.body.appendChild(leftSidePanel); window.debugMenu = debugMenu; window.editor = editor; window.page = page;