From d5a348209b0a5649461821fc55965076b93e9a55 Mon Sep 17 00:00:00 2001 From: Pionxzh Date: Mon, 15 Jan 2024 02:08:54 +0800 Subject: [PATCH 1/4] refactor: manage web worker with threads-es --- packages/playground/package.json | 1 + .../playground/src/composables/useUnminify.ts | 22 ------------ .../playground/src/composables/useUnpacker.ts | 22 ------------ packages/playground/src/pages/CodeEditor.vue | 14 ++++---- packages/playground/src/pages/Uploader.vue | 8 ++--- packages/playground/src/unminify.worker.ts | 36 ++++++++++--------- packages/playground/src/unpacker.worker.ts | 28 ++++++++------- packages/playground/src/worker.ts | 17 +++++++++ packages/playground/vite.config.ts | 9 +++++ pnpm-lock.yaml | 13 +++++++ 10 files changed, 84 insertions(+), 86 deletions(-) delete mode 100644 packages/playground/src/composables/useUnminify.ts delete mode 100644 packages/playground/src/composables/useUnpacker.ts create mode 100644 packages/playground/src/worker.ts diff --git a/packages/playground/package.json b/packages/playground/package.json index a5c16a04..97b7b961 100644 --- a/packages/playground/package.json +++ b/packages/playground/package.json @@ -53,6 +53,7 @@ "os": "^0.1.2", "sortablejs": "^1.15.0", "splitpanes": "^3.1.5", + "threads-es": "^1.0.0", "vue": "^3.3.9", "vue-codemirror": "^6.1.1", "vue-router": "4.2.5" diff --git a/packages/playground/src/composables/useUnminify.ts b/packages/playground/src/composables/useUnminify.ts deleted file mode 100644 index 910a17f2..00000000 --- a/packages/playground/src/composables/useUnminify.ts +++ /dev/null @@ -1,22 +0,0 @@ -import UnminifyWorker from '../unminify.worker?worker' -import type { CodeModParams, TransformedModule } from '../types' - -export function useUnminify() { - return (param: CodeModParams) => { - return new Promise((resolve, reject) => { - const worker = new UnminifyWorker() - - worker.onmessage = ({ data }: MessageEvent) => { - worker.terminate() - resolve(data) - } - - worker.onerror = (error) => { - worker.terminate() - reject(error) - } - - worker.postMessage(param) - }) - } -} diff --git a/packages/playground/src/composables/useUnpacker.ts b/packages/playground/src/composables/useUnpacker.ts deleted file mode 100644 index 8498c2ec..00000000 --- a/packages/playground/src/composables/useUnpacker.ts +++ /dev/null @@ -1,22 +0,0 @@ -import UnpackerWorker from '../unpacker.worker?worker' -import type { UnpackerResult } from '../types' - -export function useUnpacker() { - return (input: string) => { - return new Promise((resolve, reject) => { - const worker = new UnpackerWorker() - - worker.onmessage = ({ data }: MessageEvent) => { - worker.terminate() - resolve(data) - } - - worker.onerror = (error) => { - worker.terminate() - reject(error) - } - - worker.postMessage(input) - }) - } -} diff --git a/packages/playground/src/pages/CodeEditor.vue b/packages/playground/src/pages/CodeEditor.vue index 2edb3472..a6753872 100644 --- a/packages/playground/src/pages/CodeEditor.vue +++ b/packages/playground/src/pages/CodeEditor.vue @@ -13,21 +13,19 @@ import { encodeOption } from '../composables/url' import { useModule } from '../composables/useModule' import { useModuleMapping } from '../composables/useModuleMapping' import { useModuleMeta } from '../composables/useModuleMeta' -import { useUnminify } from '../composables/useUnminify' +import { unminify } from '../worker' + +const [openSideBar, setOpenSideBar] = useState(false) const { params: { id } } = useRoute() const { module, setModule } = useModule(id as string) const { moduleMeta } = useModuleMeta() const { moduleMapping } = useModuleMapping() - -const [openSideBar, setOpenSideBar] = useState(false) +const moduleName = computed(() => moduleMapping.value[module.value.id]) const enabledRuleIds = useAtomValue(enabledRuleIdsAtom) const disabledRuleIds = useAtomValue(disabledRuleIdsAtom) -const unminify = useUnminify() -const moduleName = computed(() => moduleMapping.value[module.value.id]) - watch([enabledRuleIds, () => module.value.code], async () => { const result = await unminify({ name: moduleName.value, @@ -39,7 +37,7 @@ watch([enabledRuleIds, () => module.value.code], async () => { setModule({ ...module.value, transformed: result.transformed }) }, { immediate: true }) -const onClick = () => { +const copySharableUrl = () => { const hash = encodeOption({ code: module.value.code, rules: disabledRuleIds.value.length ? disabledRuleIds.value : undefined, @@ -58,7 +56,7 @@ const onClick = () => {
((module) => { const { id, isEntry, code, tags } = module return { diff --git a/packages/playground/src/unminify.worker.ts b/packages/playground/src/unminify.worker.ts index 12bade5c..a873804d 100644 --- a/packages/playground/src/unminify.worker.ts +++ b/packages/playground/src/unminify.worker.ts @@ -1,20 +1,24 @@ import { runTransformationRules } from '@wakaru/unminify' +import { exposeApi } from 'threads-es/worker' import type { CodeModParams, TransformedModule } from './types' -onmessage = async ( - msg: MessageEvent, -) => { - try { - const { name, module, transformationRuleIds, moduleMeta, moduleMapping } = msg.data - const fileInfo = { path: name, source: module.code } - const params = { moduleMeta, moduleMapping } - const { code } = await runTransformationRules(fileInfo, transformationRuleIds, params) - const transformedDep: TransformedModule = { ...module, transformed: code } - postMessage(transformedDep) - } - catch (e) { - // We print the error here because it will lose the stack trace after being sent to the main thread - console.error(e) - throw e - } +const UnminifyAPI = { + execute: async ({ name, module, transformationRuleIds, moduleMeta, moduleMapping }: CodeModParams) => { + try { + const fileInfo = { path: name, source: module.code } + const params = { moduleMeta, moduleMapping } + const { code } = await runTransformationRules(fileInfo, transformationRuleIds, params) + const transformedDep: TransformedModule = { ...module, transformed: code } + return transformedDep + } + catch (e) { + // We print the error here because it will lose the stack trace after being sent to the main thread + console.error(e) + throw e + } + }, } + +export type UnminifyApiType = typeof UnminifyAPI + +exposeApi(UnminifyAPI) diff --git a/packages/playground/src/unpacker.worker.ts b/packages/playground/src/unpacker.worker.ts index fe4437af..4bc24b22 100644 --- a/packages/playground/src/unpacker.worker.ts +++ b/packages/playground/src/unpacker.worker.ts @@ -1,15 +1,19 @@ import { unpack } from '@wakaru/unpacker' +import { exposeApi } from 'threads-es/worker' -onmessage = ( - input: MessageEvent, -) => { - try { - const result = unpack(input.data) - postMessage(result) - } - catch (e) { - // We print the error here because it will lose the stack trace after being sent to the main thread - console.error(e) - throw e - } +const UnpackerApi = { + execute: async (input: string) => { + try { + return unpack(input) + } + catch (e) { + // We print the error here because it will lose the stack trace after being sent to the main thread + console.error(e) + throw e + } + }, } + +export type UnpackerApiType = typeof UnpackerApi + +exposeApi(UnpackerApi) diff --git a/packages/playground/src/worker.ts b/packages/playground/src/worker.ts new file mode 100644 index 00000000..a458ca28 --- /dev/null +++ b/packages/playground/src/worker.ts @@ -0,0 +1,17 @@ +import { EsThread, EsThreadPool } from 'threads-es/controller' +import UnminifyWorker from './unminify.worker?worker' +import UnpackerWorker from './unpacker.worker?worker' +import type { CodeModParams } from './types' +import type { UnminifyApiType } from './unminify.worker' +import type { UnpackerApiType } from './unpacker.worker' + +const unminifyPool = await EsThreadPool.Spawn(() => EsThread.Spawn(new UnminifyWorker()), { size: 4 }) +const unpackerPool = await EsThreadPool.Spawn(() => EsThread.Spawn(new UnpackerWorker()), { size: 1 }) + +export async function unminify(param: CodeModParams) { + return unminifyPool.queue(thread => thread.methods.execute(param)) +} + +export async function unpack(input: string) { + return unpackerPool.queue(thread => thread.methods.execute(input)) +} diff --git a/packages/playground/vite.config.ts b/packages/playground/vite.config.ts index a7fb9e7f..83e7d022 100644 --- a/packages/playground/vite.config.ts +++ b/packages/playground/vite.config.ts @@ -5,6 +5,15 @@ import { defineConfig } from 'vite' // https://vitejs.dev/config/ export default defineConfig({ plugins: [vue()], + build: { + rollupOptions: { + input: { + main: path.resolve(__dirname, 'index.html'), + unminifyWorker: path.resolve(__dirname, 'src/unminify.worker.ts'), + unpackerWorker: path.resolve(__dirname, 'src/unpacker.worker.ts'), + }, + }, + }, optimizeDeps: { esbuildOptions: { define: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f51bd0c2..e57be301 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -324,6 +324,9 @@ importers: splitpanes: specifier: ^3.1.5 version: 3.1.5 + threads-es: + specifier: ^1.0.0 + version: 1.0.0 vue: specifier: ^3.3.9 version: 3.3.9(typescript@5.3.2) @@ -4373,6 +4376,10 @@ packages: eslint-visitor-keys: 3.4.3 dev: true + /@ungap/event-target@0.2.4: + resolution: {integrity: sha512-u9Fd3k2qfMtn+0dxbCn/y0pzQ9Ucw6lWR984CrHcbxc+WzcMkJE4VjWHWSb9At40MjwMyHCkJNXroS55Osshhw==} + dev: false + /@ungap/structured-clone@1.2.0: resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} dev: true @@ -9046,6 +9053,12 @@ packages: any-promise: 1.3.0 dev: true + /threads-es@1.0.0: + resolution: {integrity: sha512-ytsl7XSKyX0FJR8xwvwnqL+UUVzoArBJYuE2sWNNlC2Lu0hAArTzTpPXBvlaiiw31d+oj/bZblImSaEfIK+aHQ==} + dependencies: + '@ungap/event-target': 0.2.4 + dev: false + /tinybench@2.5.1: resolution: {integrity: sha512-65NKvSuAVDP/n4CqH+a9w2kTlLReS9vhsAP06MWx+/89nMinJyB2icyl58RIcqCmIggpojIGeuJGhjU1aGMBSg==} From 0f2d9151f0fe62e095b1ab10bbd43db91e551081 Mon Sep 17 00:00:00 2001 From: Pionxzh Date: Sun, 21 Jan 2024 10:36:41 +0800 Subject: [PATCH 2/4] feat(playground): persistent data to idb --- .vscode/settings.json | 1 + packages/playground/package.json | 2 + packages/playground/src/App.vue | 7 +- packages/playground/src/Main.vue | 11 + packages/playground/src/atoms/module.ts | 249 ++++++++++++++++++ packages/playground/src/atoms/rule.ts | 6 + .../playground/src/components/SideBar.vue | 57 ++-- .../src/composables/shared/useLocalStorage.ts | 101 ------- .../src/composables/shared/useStorage.ts | 46 ---- packages/playground/src/composables/url.ts | 1 + .../playground/src/composables/useFileIds.ts | 11 - .../playground/src/composables/useModule.ts | 21 -- .../src/composables/useModuleMapping.ts | 12 - .../src/composables/useModuleMeta.ts | 11 - packages/playground/src/const.ts | 2 + packages/playground/src/pages/CodeEditor.vue | 38 ++- .../playground/src/pages/ModuleMapping.vue | 6 +- packages/playground/src/pages/Uploader.vue | 143 ++++------ pnpm-lock.yaml | 15 +- 19 files changed, 378 insertions(+), 362 deletions(-) create mode 100644 packages/playground/src/Main.vue create mode 100644 packages/playground/src/atoms/module.ts delete mode 100644 packages/playground/src/composables/shared/useLocalStorage.ts delete mode 100644 packages/playground/src/composables/shared/useStorage.ts delete mode 100644 packages/playground/src/composables/useFileIds.ts delete mode 100644 packages/playground/src/composables/useModule.ts delete mode 100644 packages/playground/src/composables/useModuleMapping.ts delete mode 100644 packages/playground/src/composables/useModuleMeta.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 19b5d363..f8c0cf8a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -38,6 +38,7 @@ "jotai", "jscodeshift", "jsxs", + "keyval", "lebab", "Minifier", "outro", diff --git a/packages/playground/package.json b/packages/playground/package.json index 97b7b961..1f41c1ea 100644 --- a/packages/playground/package.json +++ b/packages/playground/package.json @@ -43,11 +43,13 @@ "@vueuse/core": "^10.6.1", "@vueuse/integrations": "^10.6.1", "@wakaru/ast-utils": "workspace:*", + "@wakaru/shared": "workspace:*", "@wakaru/unminify": "workspace:*", "@wakaru/unpacker": "workspace:*", "assert": "^2.1.0", "codemirror": "^6.0.1", "fflate": "^0.8.1", + "idb-keyval": "^6.2.1", "jotai": "^2.6.0", "jotai-vue": "^0.1.0", "os": "^0.1.2", diff --git a/packages/playground/src/App.vue b/packages/playground/src/App.vue index bd0b8c56..13019c81 100644 --- a/packages/playground/src/App.vue +++ b/packages/playground/src/App.vue @@ -1,6 +1,7 @@ diff --git a/packages/playground/src/Main.vue b/packages/playground/src/Main.vue new file mode 100644 index 00000000..f33db69a --- /dev/null +++ b/packages/playground/src/Main.vue @@ -0,0 +1,11 @@ + + + diff --git a/packages/playground/src/atoms/module.ts b/packages/playground/src/atoms/module.ts new file mode 100644 index 00000000..99d14e5a --- /dev/null +++ b/packages/playground/src/atoms/module.ts @@ -0,0 +1,249 @@ +import { + delMany as delIdbMany, + get as getIdb, + getMany as getIdbMany, + keys as keysIdb, + set as setIdb, + setMany as setIdbMany } from 'idb-keyval' +import { atom, getDefaultStore } from 'jotai/vanilla' +import { KEY_FILE_PREFIX, KEY_MODULE_MAPPING, KEY_MODULE_META } from '../const' +import { unminify } from '../worker' +import { enabledRuleIdsAtom, prettifyRules } from './rule' +import type { ImportInfo } from '@wakaru/ast-utils/imports' +import type { ModuleMapping, ModuleMeta } from '@wakaru/shared/types' + +export type ModuleId = number | string + +export interface CodeModule { + id: ModuleId + + /** Whether the module is the entry module */ + isEntry: boolean + + code: string + + transformed: string + + /** A list of import meta */ + import: ImportInfo[] + + /** A map of exported name to local identifier */ + export: Record + + /** + * A map of top-level local identifier to a list of tags. + * A tag represents a special meaning of the identifier. + * For example, a function can be marked as a runtime + * function, and be properly transformed by corresponding + * rules. + */ + tags: Record +} + +export type CodeModuleWithName = CodeModule & { name: string } + +export function getDefaultCodeModule(moduleId: ModuleId = -1): CodeModule { + return { + id: moduleId, + isEntry: false, + code: '', + transformed: '', + import: [], + export: {}, + tags: {}, + } +} + +export function getModuleDefaultName(module: CodeModule) { + return module.isEntry ? `entry-${module.id}.js` : `module-${module.id}.js` +} + +const _moduleMappingAtom = atom({}) +export const moduleMappingAtom = atom( + get => get(_moduleMappingAtom), + (_get, set, newMapping: ModuleMapping) => { + set(_moduleMappingAtom, newMapping) + setIdb(KEY_MODULE_MAPPING, newMapping) + }, +) + +const _modulesAtom = atom([]) +export const modulesAtom = atom( + (get) => { + const modules = get(_modulesAtom) + const moduleMapping = get(moduleMappingAtom) + + const moduleWithNames = modules.map((mod) => { + const mappedName = moduleMapping[mod.id] + if (mappedName) return { ...mod, name: mappedName } + + return { ...mod, name: getModuleDefaultName(mod) } + }) + + return [ + ...moduleWithNames.filter(mod => mod.isEntry).sort((a, b) => +a.id - +b.id), + ...moduleWithNames.filter(mod => !mod.isEntry).sort((a, b) => +a.id - +b.id), + ] + }, + (get, set, adds: CodeModule[], updates: CodeModule[], deletes: ModuleId[], skipSync = false) => { + const modules = get(_modulesAtom) + const newModules = [...modules, ...adds].filter(mod => !deletes.includes(mod.id)) + updates.forEach((mod) => { + const idx = newModules.findIndex(m => m.id === mod.id) + if (idx !== -1) newModules[idx] = mod + }) + set(_modulesAtom, newModules) + + if (adds.length > 0) { + const moduleMapping = get(moduleMappingAtom) + const newModuleMapping = { ...moduleMapping } + + let changed = false + adds.forEach((mod) => { + if (moduleMapping[mod.id]) return + changed = true + newModuleMapping[mod.id] = getModuleDefaultName(mod) + }) + if (changed) { + set(moduleMappingAtom, newModuleMapping) + } + } + + if (!skipSync) { + setIdbMany([...adds, ...updates].map(mod => [`${KEY_FILE_PREFIX}${mod.id}`, mod])) + delIdbMany(deletes.map(modId => `${KEY_FILE_PREFIX}${modId}`)) + } + }, +) + +export function getModuleAtom(moduleId: ModuleId) { + moduleId = moduleId.toString() + return atom( + (get) => { + const modules = get(modulesAtom) + return modules.find(mod => mod.id.toString() === moduleId) || { + name: `module-${moduleId}.js`, + ...getDefaultCodeModule(moduleId), + } + }, + (get, set, updateValue: Partial>) => { + const modules = get(modulesAtom) + const moduleIdx = modules.findIndex(mod => mod.id.toString() === moduleId) + if (moduleIdx === -1) return + + const module = modules[moduleIdx] + const updatedModule = { ...module, ...updateValue } + set(modulesAtom, [], [updatedModule], []) + }, + ) +} + +type ModuleAtom = ReturnType + +/** + * Module Meta is a computed result of all modules. + * + * This atom is used to override the computed result. Used by shared url. + */ +const _moduleMetaOverrideAtom = atom(null) +export const moduleMetaOverrideAtom = atom( + get => get(_moduleMetaOverrideAtom), + (_get, set, newMeta: ModuleMeta | null) => { + set(_moduleMetaOverrideAtom, newMeta) + setIdb(KEY_MODULE_META, newMeta) + }, +) + +export const moduleMetaAtom = atom((get) => { + const moduleMetaOverride = get(moduleMetaOverrideAtom) + if (moduleMetaOverride) return moduleMetaOverride + + const modules = get(modulesAtom) + const moduleMeta = modules.reduce((acc, mod) => { + acc[mod.id] = { + import: mod.import, + export: mod.export, + tags: mod.tags, + } + return acc + }, {} as ModuleMeta) + + return moduleMeta +}) + +export const prettifyAllModulesAtom = atom(null, async (get, set) => { + const modules = get(modulesAtom) + + await Promise.all(modules.map(async (mod) => { + const result = await unminify({ + name: mod.name, + module: mod, + transformationRuleIds: prettifyRules, + moduleMeta: {}, + moduleMapping: {}, + }) + + if (mod.code !== result.transformed) { + set(modulesAtom, [], [{ ...mod, code: result.transformed }], []) + } + })) +}) + +// export const prettifyModuleAtom = atom(null, async (get, set, moduleAtom: ModuleAtom) => { +// const module = get(moduleAtom) +// const result = await unminify({ +// name: module.name, +// module, +// transformationRuleIds: prettifyRules, +// moduleMeta: {}, +// moduleMapping: {}, +// }) +// if (module.code === result.transformed) return + +// set(moduleAtom, { code: result.transformed }) +// }) + +export const unminifyModuleAtom = atom(null, async (get, set, moduleAtom: ModuleAtom) => { + const module = get(moduleAtom) + const moduleMeta = get(moduleMetaAtom) + const moduleMapping = get(moduleMappingAtom) + const transformationRuleIds = get(enabledRuleIdsAtom) + + const result = await unminify({ + name: module.name, + module, + transformationRuleIds, + moduleMeta, + moduleMapping, + }) + if (module.code === result.transformed) return + + set(moduleAtom, { transformed: result.transformed }) +}) + +export const resetModulesAtom = atom(null, (get, set) => { + set(moduleMetaOverrideAtom, null) + set(moduleMappingAtom, {}) + + const modules = get(modulesAtom) + set(modulesAtom, [], [], modules.map(mod => mod.id)) +}) + +export async function prepare() { + const keys = await keysIdb() + const moduleKeys = keys.filter(key => typeof key === 'string' && key.startsWith(KEY_FILE_PREFIX)) + if (moduleKeys.length > 0) { + const moduleMapping = await getIdb(KEY_MODULE_MAPPING) + if (moduleMapping) { + getDefaultStore().set(moduleMappingAtom, moduleMapping) + } + + const moduleMeta = await getIdb(KEY_MODULE_META) + if (moduleMeta) { + getDefaultStore().set(moduleMetaOverrideAtom, moduleMeta) + } + + const modules = await getIdbMany(moduleKeys) + getDefaultStore().set(modulesAtom, modules, [], [], true) + } +} diff --git a/packages/playground/src/atoms/rule.ts b/packages/playground/src/atoms/rule.ts index 64197c8d..0c3859c2 100644 --- a/packages/playground/src/atoms/rule.ts +++ b/packages/playground/src/atoms/rule.ts @@ -3,6 +3,12 @@ import { atomWithStorage } from 'jotai/utils' import { atom } from 'jotai/vanilla' import { KEY_DISABLED_RULES, KEY_RULE_ORDER } from '../const' +export const prettifyRules = [ + 'un-sequence-expression1', + 'un-variable-merging', + 'prettier', +] + export const allRulesAtom = atom(() => transformationRules) export const ruleOrderAtom = atomWithStorage(KEY_RULE_ORDER, transformationRules.map(rule => rule.id)) diff --git a/packages/playground/src/components/SideBar.vue b/packages/playground/src/components/SideBar.vue index 7271b78f..54fc3bfc 100644 --- a/packages/playground/src/components/SideBar.vue +++ b/packages/playground/src/components/SideBar.vue @@ -1,38 +1,29 @@ @@ -83,32 +74,32 @@ function rename(fileId: FileId, e: Event) {
  • @@ -117,25 +108,25 @@ function rename(fileId: FileId, e: Event) { icon="fa-solid fa-code" class="flex-shrink-0 w-5 h-5 text-gray-500 dark:text-gray-400" /> -