From 4cced733bf99be244266cca38acf85d4b6b0c495 Mon Sep 17 00:00:00 2001 From: Coder-Van Date: Tue, 5 Mar 2024 22:37:11 +0800 Subject: [PATCH 1/3] feat(molecule): generate ts code from mol type file recursively for issue (#643) --- packages/molecule/package.json | 2 +- packages/molecule/src/cli.ts | 168 ++++++++++++++++++++++++++++--- packages/molecule/src/codegen.ts | 17 +++- packages/molecule/src/type.ts | 5 + 4 files changed, 175 insertions(+), 17 deletions(-) diff --git a/packages/molecule/package.json b/packages/molecule/package.json index 0b4e58ffd..8fc1942e4 100644 --- a/packages/molecule/package.json +++ b/packages/molecule/package.json @@ -38,7 +38,7 @@ "lumos-molecule-codegen": "lib/cli.js" }, "scripts": { - "test": "ava **/*.test.ts --timeout=2m", + "test": "ava **/codegen.test.ts --timeout=2m", "fmt": "prettier --write \"{src,tests,examples}/**/*.ts\" package.json", "lint": "eslint -c ../../.eslintrc.js \"{src,tests,examples}/**/*.ts\"", "build": "pnpm run build:types && pnpm run build:js", diff --git a/packages/molecule/src/cli.ts b/packages/molecule/src/cli.ts index e3f798a8e..c0dca01b8 100644 --- a/packages/molecule/src/cli.ts +++ b/packages/molecule/src/cli.ts @@ -1,8 +1,10 @@ #!/usr/bin/env node -import { codegen } from "./codegen"; +import { ParseResult } from "./type"; +import { codegenReturnMore } from "./codegen"; import * as fs from "node:fs"; +import * as path from 'path'; -const DEFAULT_CONFIG_FILE_NAME = "lumos-molecule-codegen.json"; +const DEFAULT_CONFIG_FILE_NAME = process.env.CONF_PATH || "lumos-molecule-codegen.json"; function camelcase(str: string): string { return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()); @@ -12,9 +14,12 @@ type Config = { objectKeyFormat: "camelcase" | "keep"; prepend: string; schemaFile: string; + output: number; // 0: Default out console, 1: Write file, 2. Just return + dir: string; // }; const fileConfig: Partial = (() => { + console.log(DEFAULT_CONFIG_FILE_NAME) if (fs.existsSync(DEFAULT_CONFIG_FILE_NAME)) { return JSON.parse(fs.readFileSync(DEFAULT_CONFIG_FILE_NAME, "utf8")); } @@ -25,20 +30,157 @@ const config: Config = { objectKeyFormat: fileConfig.objectKeyFormat || "keep", prepend: fileConfig.prepend || "", schemaFile: fileConfig.schemaFile || "schema.mol", + output: fileConfig.output || 0, + dir: fileConfig.dir || __dirname }; // check if the schema file exists -if (!fs.existsSync(config.schemaFile)) { - console.error( - `Schema file ${config.schemaFile} does not exist. Please configure the \`schemaFile\` in ${DEFAULT_CONFIG_FILE_NAME}` - ); - process.exit(1); +// if (!fs.existsSync(config.schemaFile)) { +// console.error( +// `Schema file ${config.schemaFile} does not exist. Please configure the \`schemaFile\` in ${DEFAULT_CONFIG_FILE_NAME}` +// ); +// process.exit(1); +// } + +// const generated = codegen(fs.readFileSync(config.schemaFile, "utf-8"), { +// prepend: config.prepend, +// formatObjectKeys: +// String(config.objectKeyFormat).toLowerCase() === "camelcase" ? camelcase : undefined, +// }); + +// console.log(generated); + +// const scanedFiles: Set = new Set() +const fileImportsMap: Map> = new Map() +const fileStack: Array = [] +const genResult: Map = new Map() + +function scanImports(filename: string): Array { + // check if the file exist + if (!fs.existsSync(filename)) { + console.error( + `Schema file ${filename} does not exist.` + ); + process.exit(1); + } + const schema = fs.readFileSync(filename, "utf-8") + if (!schema) return []; + + const matched = schema.match(/.*import\s+"(.*)".*;/g); + if (!matched) return [filename]; + + // collect all import filenames + const importFileNames = matched.map((item: string) => { + // if is commit bolck, continue + if (item.trim().startsWith('//')) { + return '' + } + const m = item.match(/.*"(.*)".*/); + return m ? m[1] : '' + }).filter(Boolean) + + // push filename to stack + fileStack.push(filename) + // loop all import files + const baseDir = path.dirname(filename) + for (const importFileName of importFileNames) { + const mFilePath = path.join(baseDir, importFileName + '.mol') + + if (!fileImportsMap.has(mFilePath)) { + // mask this file has checked + fileImportsMap.set(mFilePath, []) + // recursive next + scanImports(mFilePath) + } + fileStack.push(mFilePath) + } + // recard import names for ESM import + fileImportsMap.set(filename, importFileNames) + + return fileStack +} + +function loopCodegen(targetFilename: string): Map { + scanImports(targetFilename) + // TODO check all schema can parseable + if (fileStack.length === 0) { + // console.log('Empty file list for codegen') + return genResult + } + + const sourceDir = path.dirname(targetFilename) + + while (fileStack.length > 0) { + const filename = fileStack.pop() + // console.log(filename) + if (!filename) { break } + + const name = path.relative(sourceDir, filename).split('.')[0] + // path.basename(filename).split('.')[0] + // has gen, contine + if (genResult.has(name)) { + continue + } + // erase the import clause from the schema when calling the codegen method + const schema = fs.readFileSync(filename, "utf-8") + const lines = schema.split('\n') + const delImportLines = lines.filter((line: string) => { + return !line.trim().startsWith('import') + }) + + let optionPrepend = config.prepend + // append all ESM import to config.prepend + if (fileImportsMap.has(filename)) { + const imports = fileImportsMap.get(filename) + if (imports && imports?.length > 0) { + for (const impt of imports) { + if (genResult.has(impt)){ + // console.log('ok 4', genResult.get(impt)) + const imptDesc = `\nimport { ${genResult.get(impt)?.fields.join(', ')} } from './${impt}'` + optionPrepend += imptDesc + } + } + } + } + + const result = codegenReturnMore(delImportLines.join('\n'), { + prepend: optionPrepend , // ex: `\nimport { RGB, UTF8String } from './base'`, + formatObjectKeys: + String(config.objectKeyFormat).toLowerCase() === "camelcase" ? camelcase : undefined, + }); + + genResult.set(name, result) + + } + + // console.log(config.dir, config.output) + for (const name of genResult.keys()) { + if (config.output < 2) { + console.log(`// ${String('-').repeat(66)} //`) + console.log(`// ${name}`) + console.log(`// ${String('-').repeat(66)} //`) + console.log(genResult.get(name)?.code) + if (config.output === 1) { + const dir = path.join(config.dir, 'mols') + if (!fs.existsSync(dir)) { + console.log(`Mkdir mols`) + fs.mkdirSync(dir) + } + const targetDir = path.dirname(path.join(dir, name + '.ts')) + if (!fs.existsSync(targetDir)) { + console.log(`Mkdir ${targetDir}`) + fs.mkdirSync(targetDir, { recursive: true }) + } + console.log(`Writing file ${name}.ts`) + fs.writeFileSync(path.join(dir, name + '.ts'), genResult.get(name)?.code || '') + console.log(`Write file ${name}.ts finish`) + } + } + } + + return genResult } -const generated = codegen(fs.readFileSync(config.schemaFile, "utf-8"), { - prepend: config.prepend, - formatObjectKeys: - config.objectKeyFormat === "camelcase" ? camelcase : undefined, -}); +loopCodegen(config.schemaFile); + -console.log(generated); diff --git a/packages/molecule/src/codegen.ts b/packages/molecule/src/codegen.ts index 86d55b564..f9a95cc97 100644 --- a/packages/molecule/src/codegen.ts +++ b/packages/molecule/src/codegen.ts @@ -1,4 +1,4 @@ -import { MolType } from "./type"; +import { MolType, ParseResult } from "./type"; import { Grammar as NearleyGrammar, Parser as NearleyParser } from "nearley"; import { circularIterator } from "./circularIterator"; @@ -33,7 +33,7 @@ export function scanCustomizedTypes(prepend: string): string[] { ); } -export function codegen(schema: string, options: Options = {}): string { +export function codegenReturnMore(schema: string, options: Options = {}): ParseResult { const parser = new NearleyParser(NearleyGrammar.fromCompiled(grammar)); parser.feed(schema); @@ -45,9 +45,11 @@ export function codegen(schema: string, options: Options = {}): string { importedModules ); + const typeNames: Array = [] const codecs = molTypes .map((molType) => { if (importedModules.includes(molType.name)) return ""; + typeNames.push(molType.name) if (molType.type === "array") { if (molType.item === "byte") { @@ -122,7 +124,7 @@ export function codegen(schema: string, options: Options = {}): string { .filter(Boolean) .join("\n\n"); - return `// This file is generated by @ckb-lumos/molecule, please do not modify it manually. + const code = `// This file is generated by @ckb-lumos/molecule, please do not modify it manually. /* eslint-disable */ import { bytes, createBytesCodec, createFixedBytesCodec, molecule } from "@ckb-lumos/codec"; @@ -147,6 +149,15 @@ const byte = createFallbackFixedBytesCodec(1); ${codecs} `; + const result: ParseResult = { + code, fields: typeNames + } + + return result +} + +export function codegen(schema: string, options: Options = {}): string { + return codegenReturnMore(schema, options).code } // sort molecule types by their dependencies, to make sure the known types can be used in the front diff --git a/packages/molecule/src/type.ts b/packages/molecule/src/type.ts index 21f07b7eb..c3f790a5e 100644 --- a/packages/molecule/src/type.ts +++ b/packages/molecule/src/type.ts @@ -59,3 +59,8 @@ export type ParseOptions = { export interface Parser { parse(data: string, option?: ParseOptions): Record; } + +export type ParseResult = { + code: string; + fields: string[]; +}; \ No newline at end of file From 80068eec22ee36b4c5ca50141ecffe32f21cb774 Mon Sep 17 00:00:00 2001 From: Coder-Van Date: Tue, 5 Mar 2024 23:03:40 +0800 Subject: [PATCH 2/3] fix(molecule): delete some console log items --- packages/molecule/package.json | 2 +- packages/molecule/src/cli.ts | 149 ++++++++++++++++--------------- packages/molecule/src/codegen.ts | 18 ++-- packages/molecule/src/type.ts | 2 +- 4 files changed, 89 insertions(+), 82 deletions(-) diff --git a/packages/molecule/package.json b/packages/molecule/package.json index 8fc1942e4..0b4e58ffd 100644 --- a/packages/molecule/package.json +++ b/packages/molecule/package.json @@ -38,7 +38,7 @@ "lumos-molecule-codegen": "lib/cli.js" }, "scripts": { - "test": "ava **/codegen.test.ts --timeout=2m", + "test": "ava **/*.test.ts --timeout=2m", "fmt": "prettier --write \"{src,tests,examples}/**/*.ts\" package.json", "lint": "eslint -c ../../.eslintrc.js \"{src,tests,examples}/**/*.ts\"", "build": "pnpm run build:types && pnpm run build:js", diff --git a/packages/molecule/src/cli.ts b/packages/molecule/src/cli.ts index c0dca01b8..b1025cefc 100644 --- a/packages/molecule/src/cli.ts +++ b/packages/molecule/src/cli.ts @@ -2,9 +2,10 @@ import { ParseResult } from "./type"; import { codegenReturnMore } from "./codegen"; import * as fs from "node:fs"; -import * as path from 'path'; +import * as path from "path"; -const DEFAULT_CONFIG_FILE_NAME = process.env.CONF_PATH || "lumos-molecule-codegen.json"; +const DEFAULT_CONFIG_FILE_NAME = + process.env.CONF_PATH || "lumos-molecule-codegen.json"; function camelcase(str: string): string { return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()); @@ -15,11 +16,10 @@ type Config = { prepend: string; schemaFile: string; output: number; // 0: Default out console, 1: Write file, 2. Just return - dir: string; // + dir: string; // }; const fileConfig: Partial = (() => { - console.log(DEFAULT_CONFIG_FILE_NAME) if (fs.existsSync(DEFAULT_CONFIG_FILE_NAME)) { return JSON.parse(fs.readFileSync(DEFAULT_CONFIG_FILE_NAME, "utf8")); } @@ -31,7 +31,7 @@ const config: Config = { prepend: fileConfig.prepend || "", schemaFile: fileConfig.schemaFile || "schema.mol", output: fileConfig.output || 0, - dir: fileConfig.dir || __dirname + dir: fileConfig.dir || __dirname, }; // check if the schema file exists @@ -51,136 +51,139 @@ const config: Config = { // console.log(generated); // const scanedFiles: Set = new Set() -const fileImportsMap: Map> = new Map() -const fileStack: Array = [] -const genResult: Map = new Map() +const fileImportsMap: Map> = new Map(); +const fileStack: Array = []; +const genResult: Map = new Map(); function scanImports(filename: string): Array { // check if the file exist if (!fs.existsSync(filename)) { - console.error( - `Schema file ${filename} does not exist.` - ); + console.error(`Schema file ${filename} does not exist.`); process.exit(1); } - const schema = fs.readFileSync(filename, "utf-8") + const schema = fs.readFileSync(filename, "utf-8"); if (!schema) return []; const matched = schema.match(/.*import\s+"(.*)".*;/g); if (!matched) return [filename]; // collect all import filenames - const importFileNames = matched.map((item: string) => { - // if is commit bolck, continue - if (item.trim().startsWith('//')) { - return '' - } - const m = item.match(/.*"(.*)".*/); - return m ? m[1] : '' - }).filter(Boolean) + const importFileNames = matched + .map((item: string) => { + // if is commit bolck, continue + if (item.trim().startsWith("//")) { + return ""; + } + const m = item.match(/.*"(.*)".*/); + return m ? m[1] : ""; + }) + .filter(Boolean); // push filename to stack - fileStack.push(filename) + fileStack.push(filename); // loop all import files - const baseDir = path.dirname(filename) + const baseDir = path.dirname(filename); for (const importFileName of importFileNames) { - const mFilePath = path.join(baseDir, importFileName + '.mol') - + const mFilePath = path.join(baseDir, importFileName + ".mol"); + if (!fileImportsMap.has(mFilePath)) { // mask this file has checked - fileImportsMap.set(mFilePath, []) + fileImportsMap.set(mFilePath, []); // recursive next - scanImports(mFilePath) + scanImports(mFilePath); } - fileStack.push(mFilePath) + fileStack.push(mFilePath); } // recard import names for ESM import - fileImportsMap.set(filename, importFileNames) + fileImportsMap.set(filename, importFileNames); - return fileStack + return fileStack; } function loopCodegen(targetFilename: string): Map { - scanImports(targetFilename) + scanImports(targetFilename); // TODO check all schema can parseable if (fileStack.length === 0) { - // console.log('Empty file list for codegen') - return genResult + return genResult; } - const sourceDir = path.dirname(targetFilename) + const sourceDir = path.dirname(targetFilename); while (fileStack.length > 0) { - const filename = fileStack.pop() - // console.log(filename) - if (!filename) { break } + const filename = fileStack.pop(); + + if (!filename) { + break; + } - const name = path.relative(sourceDir, filename).split('.')[0] + const name = path.relative(sourceDir, filename).split(".")[0]; // path.basename(filename).split('.')[0] // has gen, contine if (genResult.has(name)) { - continue + continue; } // erase the import clause from the schema when calling the codegen method - const schema = fs.readFileSync(filename, "utf-8") - const lines = schema.split('\n') - const delImportLines = lines.filter((line: string) => { - return !line.trim().startsWith('import') - }) + const schema = fs.readFileSync(filename, "utf-8"); + const lines = schema.split("\n"); + const delImportLines = lines.filter((line: string) => { + return !line.trim().startsWith("import"); + }); - let optionPrepend = config.prepend + let optionPrepend = config.prepend; // append all ESM import to config.prepend if (fileImportsMap.has(filename)) { - const imports = fileImportsMap.get(filename) + const imports = fileImportsMap.get(filename); if (imports && imports?.length > 0) { for (const impt of imports) { - if (genResult.has(impt)){ - // console.log('ok 4', genResult.get(impt)) - const imptDesc = `\nimport { ${genResult.get(impt)?.fields.join(', ')} } from './${impt}'` - optionPrepend += imptDesc + if (genResult.has(impt)) { + const imptDesc = `\nimport { ${genResult + .get(impt) + ?.fields.join(", ")} } from './${impt}'`; + optionPrepend += imptDesc; } } } } - - const result = codegenReturnMore(delImportLines.join('\n'), { - prepend: optionPrepend , // ex: `\nimport { RGB, UTF8String } from './base'`, + + const result = codegenReturnMore(delImportLines.join("\n"), { + prepend: optionPrepend, // ex: `\nimport { RGB, UTF8String } from './base'`, formatObjectKeys: - String(config.objectKeyFormat).toLowerCase() === "camelcase" ? camelcase : undefined, + String(config.objectKeyFormat).toLowerCase() === "camelcase" + ? camelcase + : undefined, }); - - genResult.set(name, result) + genResult.set(name, result); } - - // console.log(config.dir, config.output) + for (const name of genResult.keys()) { if (config.output < 2) { - console.log(`// ${String('-').repeat(66)} //`) - console.log(`// ${name}`) - console.log(`// ${String('-').repeat(66)} //`) - console.log(genResult.get(name)?.code) + console.log(`// ${String("-").repeat(66)} //`); + console.log(`// ${name}`); + console.log(`// ${String("-").repeat(66)} //`); + console.log(genResult.get(name)?.code); if (config.output === 1) { - const dir = path.join(config.dir, 'mols') + const dir = path.join(config.dir, "mols"); if (!fs.existsSync(dir)) { - console.log(`Mkdir mols`) - fs.mkdirSync(dir) + console.log(`Mkdir mols`); + fs.mkdirSync(dir); } - const targetDir = path.dirname(path.join(dir, name + '.ts')) + const targetDir = path.dirname(path.join(dir, name + ".ts")); if (!fs.existsSync(targetDir)) { - console.log(`Mkdir ${targetDir}`) - fs.mkdirSync(targetDir, { recursive: true }) + console.log(`Mkdir ${targetDir}`); + fs.mkdirSync(targetDir, { recursive: true }); } - console.log(`Writing file ${name}.ts`) - fs.writeFileSync(path.join(dir, name + '.ts'), genResult.get(name)?.code || '') - console.log(`Write file ${name}.ts finish`) + console.log(`Writing file ${name}.ts`); + fs.writeFileSync( + path.join(dir, name + ".ts"), + genResult.get(name)?.code || "" + ); + console.log(`Write file ${name}.ts finish`); } } } - return genResult + return genResult; } loopCodegen(config.schemaFile); - - diff --git a/packages/molecule/src/codegen.ts b/packages/molecule/src/codegen.ts index f9a95cc97..d1eef44a8 100644 --- a/packages/molecule/src/codegen.ts +++ b/packages/molecule/src/codegen.ts @@ -33,7 +33,10 @@ export function scanCustomizedTypes(prepend: string): string[] { ); } -export function codegenReturnMore(schema: string, options: Options = {}): ParseResult { +export function codegenReturnMore( + schema: string, + options: Options = {} +): ParseResult { const parser = new NearleyParser(NearleyGrammar.fromCompiled(grammar)); parser.feed(schema); @@ -45,11 +48,11 @@ export function codegenReturnMore(schema: string, options: Options = {}): ParseR importedModules ); - const typeNames: Array = [] + const typeNames: Array = []; const codecs = molTypes .map((molType) => { if (importedModules.includes(molType.name)) return ""; - typeNames.push(molType.name) + typeNames.push(molType.name); if (molType.type === "array") { if (molType.item === "byte") { @@ -150,14 +153,15 @@ const byte = createFallbackFixedBytesCodec(1); ${codecs} `; const result: ParseResult = { - code, fields: typeNames - } + code, + fields: typeNames, + }; - return result + return result; } export function codegen(schema: string, options: Options = {}): string { - return codegenReturnMore(schema, options).code + return codegenReturnMore(schema, options).code; } // sort molecule types by their dependencies, to make sure the known types can be used in the front diff --git a/packages/molecule/src/type.ts b/packages/molecule/src/type.ts index c3f790a5e..fbc9b3fe1 100644 --- a/packages/molecule/src/type.ts +++ b/packages/molecule/src/type.ts @@ -63,4 +63,4 @@ export interface Parser { export type ParseResult = { code: string; fields: string[]; -}; \ No newline at end of file +}; From deb09f8ae990ec50e3fd7c73ece89a1f4db9765d Mon Sep 17 00:00:00 2001 From: Coder-Van Date: Tue, 5 Mar 2024 23:03:40 +0800 Subject: [PATCH 3/3] fix(molecule): some optimizations for pull request #646 about issue(#643) --- packages/molecule/package.json | 2 +- packages/molecule/src/cli.ts | 188 +----------------- packages/molecule/src/codegen.ts | 18 +- packages/molecule/src/config.ts | 31 +++ packages/molecule/src/resolve.ts | 176 ++++++++++++++++ packages/molecule/src/type.ts | 4 +- packages/molecule/tests/mol/base.mol | 4 + packages/molecule/tests/mol/character.mol | 13 ++ .../molecule/tests/mol/submodule/base.mol | 3 + packages/molecule/tests/resolve.test.ts | 31 +++ 10 files changed, 276 insertions(+), 194 deletions(-) create mode 100644 packages/molecule/src/config.ts create mode 100644 packages/molecule/src/resolve.ts create mode 100644 packages/molecule/tests/mol/base.mol create mode 100644 packages/molecule/tests/mol/character.mol create mode 100644 packages/molecule/tests/mol/submodule/base.mol create mode 100644 packages/molecule/tests/resolve.test.ts diff --git a/packages/molecule/package.json b/packages/molecule/package.json index 8fc1942e4..0b4e58ffd 100644 --- a/packages/molecule/package.json +++ b/packages/molecule/package.json @@ -38,7 +38,7 @@ "lumos-molecule-codegen": "lib/cli.js" }, "scripts": { - "test": "ava **/codegen.test.ts --timeout=2m", + "test": "ava **/*.test.ts --timeout=2m", "fmt": "prettier --write \"{src,tests,examples}/**/*.ts\" package.json", "lint": "eslint -c ../../.eslintrc.js \"{src,tests,examples}/**/*.ts\"", "build": "pnpm run build:types && pnpm run build:js", diff --git a/packages/molecule/src/cli.ts b/packages/molecule/src/cli.ts index c0dca01b8..590b06f8e 100644 --- a/packages/molecule/src/cli.ts +++ b/packages/molecule/src/cli.ts @@ -1,186 +1,6 @@ #!/usr/bin/env node -import { ParseResult } from "./type"; -import { codegenReturnMore } from "./codegen"; -import * as fs from "node:fs"; -import * as path from 'path'; - -const DEFAULT_CONFIG_FILE_NAME = process.env.CONF_PATH || "lumos-molecule-codegen.json"; - -function camelcase(str: string): string { - return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()); -} - -type Config = { - objectKeyFormat: "camelcase" | "keep"; - prepend: string; - schemaFile: string; - output: number; // 0: Default out console, 1: Write file, 2. Just return - dir: string; // -}; - -const fileConfig: Partial = (() => { - console.log(DEFAULT_CONFIG_FILE_NAME) - if (fs.existsSync(DEFAULT_CONFIG_FILE_NAME)) { - return JSON.parse(fs.readFileSync(DEFAULT_CONFIG_FILE_NAME, "utf8")); - } - return {}; -})(); - -const config: Config = { - objectKeyFormat: fileConfig.objectKeyFormat || "keep", - prepend: fileConfig.prepend || "", - schemaFile: fileConfig.schemaFile || "schema.mol", - output: fileConfig.output || 0, - dir: fileConfig.dir || __dirname -}; - -// check if the schema file exists -// if (!fs.existsSync(config.schemaFile)) { -// console.error( -// `Schema file ${config.schemaFile} does not exist. Please configure the \`schemaFile\` in ${DEFAULT_CONFIG_FILE_NAME}` -// ); -// process.exit(1); -// } - -// const generated = codegen(fs.readFileSync(config.schemaFile, "utf-8"), { -// prepend: config.prepend, -// formatObjectKeys: -// String(config.objectKeyFormat).toLowerCase() === "camelcase" ? camelcase : undefined, -// }); - -// console.log(generated); - -// const scanedFiles: Set = new Set() -const fileImportsMap: Map> = new Map() -const fileStack: Array = [] -const genResult: Map = new Map() - -function scanImports(filename: string): Array { - // check if the file exist - if (!fs.existsSync(filename)) { - console.error( - `Schema file ${filename} does not exist.` - ); - process.exit(1); - } - const schema = fs.readFileSync(filename, "utf-8") - if (!schema) return []; - - const matched = schema.match(/.*import\s+"(.*)".*;/g); - if (!matched) return [filename]; - - // collect all import filenames - const importFileNames = matched.map((item: string) => { - // if is commit bolck, continue - if (item.trim().startsWith('//')) { - return '' - } - const m = item.match(/.*"(.*)".*/); - return m ? m[1] : '' - }).filter(Boolean) - - // push filename to stack - fileStack.push(filename) - // loop all import files - const baseDir = path.dirname(filename) - for (const importFileName of importFileNames) { - const mFilePath = path.join(baseDir, importFileName + '.mol') - - if (!fileImportsMap.has(mFilePath)) { - // mask this file has checked - fileImportsMap.set(mFilePath, []) - // recursive next - scanImports(mFilePath) - } - fileStack.push(mFilePath) - } - // recard import names for ESM import - fileImportsMap.set(filename, importFileNames) - - return fileStack -} - -function loopCodegen(targetFilename: string): Map { - scanImports(targetFilename) - // TODO check all schema can parseable - if (fileStack.length === 0) { - // console.log('Empty file list for codegen') - return genResult - } - - const sourceDir = path.dirname(targetFilename) - - while (fileStack.length > 0) { - const filename = fileStack.pop() - // console.log(filename) - if (!filename) { break } - - const name = path.relative(sourceDir, filename).split('.')[0] - // path.basename(filename).split('.')[0] - // has gen, contine - if (genResult.has(name)) { - continue - } - // erase the import clause from the schema when calling the codegen method - const schema = fs.readFileSync(filename, "utf-8") - const lines = schema.split('\n') - const delImportLines = lines.filter((line: string) => { - return !line.trim().startsWith('import') - }) - - let optionPrepend = config.prepend - // append all ESM import to config.prepend - if (fileImportsMap.has(filename)) { - const imports = fileImportsMap.get(filename) - if (imports && imports?.length > 0) { - for (const impt of imports) { - if (genResult.has(impt)){ - // console.log('ok 4', genResult.get(impt)) - const imptDesc = `\nimport { ${genResult.get(impt)?.fields.join(', ')} } from './${impt}'` - optionPrepend += imptDesc - } - } - } - } - - const result = codegenReturnMore(delImportLines.join('\n'), { - prepend: optionPrepend , // ex: `\nimport { RGB, UTF8String } from './base'`, - formatObjectKeys: - String(config.objectKeyFormat).toLowerCase() === "camelcase" ? camelcase : undefined, - }); - - genResult.set(name, result) - - } - - // console.log(config.dir, config.output) - for (const name of genResult.keys()) { - if (config.output < 2) { - console.log(`// ${String('-').repeat(66)} //`) - console.log(`// ${name}`) - console.log(`// ${String('-').repeat(66)} //`) - console.log(genResult.get(name)?.code) - if (config.output === 1) { - const dir = path.join(config.dir, 'mols') - if (!fs.existsSync(dir)) { - console.log(`Mkdir mols`) - fs.mkdirSync(dir) - } - const targetDir = path.dirname(path.join(dir, name + '.ts')) - if (!fs.existsSync(targetDir)) { - console.log(`Mkdir ${targetDir}`) - fs.mkdirSync(targetDir, { recursive: true }) - } - console.log(`Writing file ${name}.ts`) - fs.writeFileSync(path.join(dir, name + '.ts'), genResult.get(name)?.code || '') - console.log(`Write file ${name}.ts finish`) - } - } - } - - return genResult -} - -loopCodegen(config.schemaFile); - +import { initConfig } from "./config"; +import { loopCodegen } from "./resolve"; +const config = initConfig(); +loopCodegen(config); diff --git a/packages/molecule/src/codegen.ts b/packages/molecule/src/codegen.ts index f9a95cc97..ee97d2cf7 100644 --- a/packages/molecule/src/codegen.ts +++ b/packages/molecule/src/codegen.ts @@ -33,7 +33,10 @@ export function scanCustomizedTypes(prepend: string): string[] { ); } -export function codegenReturnMore(schema: string, options: Options = {}): ParseResult { +export function codegenReturnWithElements( + schema: string, + options: Options = {} +): ParseResult { const parser = new NearleyParser(NearleyGrammar.fromCompiled(grammar)); parser.feed(schema); @@ -45,11 +48,11 @@ export function codegenReturnMore(schema: string, options: Options = {}): ParseR importedModules ); - const typeNames: Array = [] + const typeNames: Array = []; const codecs = molTypes .map((molType) => { if (importedModules.includes(molType.name)) return ""; - typeNames.push(molType.name) + typeNames.push(molType.name); if (molType.type === "array") { if (molType.item === "byte") { @@ -150,14 +153,15 @@ const byte = createFallbackFixedBytesCodec(1); ${codecs} `; const result: ParseResult = { - code, fields: typeNames - } + code, + elements: typeNames, + }; - return result + return result; } export function codegen(schema: string, options: Options = {}): string { - return codegenReturnMore(schema, options).code + return codegenReturnWithElements(schema, options).code; } // sort molecule types by their dependencies, to make sure the known types can be used in the front diff --git a/packages/molecule/src/config.ts b/packages/molecule/src/config.ts new file mode 100644 index 000000000..b429f2845 --- /dev/null +++ b/packages/molecule/src/config.ts @@ -0,0 +1,31 @@ +import * as fs from "node:fs"; + +const DEFAULT_CONFIG_FILE_NAME = + process.env.CONF_PATH || "lumos-molecule-codegen.json"; + +export type Config = { + objectKeyFormat: "camelcase" | "keep"; + prepend: string; + schemaFile: string; + output: number; // 0: Default out console, 1: Write file, 2. Just return + dir: string; // +}; + +export function initConfig(): Config { + const fileConfig: Partial = (() => { + if (fs.existsSync(DEFAULT_CONFIG_FILE_NAME)) { + return JSON.parse(fs.readFileSync(DEFAULT_CONFIG_FILE_NAME, "utf8")); + } + return {}; + })(); + + const config: Config = { + objectKeyFormat: fileConfig.objectKeyFormat || "keep", + prepend: fileConfig.prepend || "", + schemaFile: fileConfig.schemaFile || "schema.mol", + output: fileConfig.output || 0, + dir: fileConfig.dir || __dirname, + }; + + return config; +} diff --git a/packages/molecule/src/resolve.ts b/packages/molecule/src/resolve.ts new file mode 100644 index 000000000..6dc1a3df9 --- /dev/null +++ b/packages/molecule/src/resolve.ts @@ -0,0 +1,176 @@ +#!/usr/bin/env node +import { ParseResult } from "./type"; +import { codegenReturnWithElements } from "./codegen"; +import { Config } from "./config"; +import * as fs from "node:fs"; +import * as path from "path"; + +function camelcase(str: string): string { + return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()); +} + +export type RelativePath = string; +export type FileWithDependence = { + relativePath: string; + dependencies: string[]; +}; + +export function resolveDependencies( + importPath: RelativePath, + baseDir: string, + resolved: Set +): FileWithDependence[] { + const dependencies: FileWithDependence[] = []; + // check if the file exist + const realPath = path.join(baseDir, importPath); + if (!fs.existsSync(realPath)) { + console.error(`Schema file ${realPath} does not exist.`); + process.exit(1); + } + + const cur: FileWithDependence = { + relativePath: importPath, + dependencies: [], + }; + + const schema = fs.readFileSync(realPath, "utf-8"); + if (!schema) { + return [cur]; + } + + const matched = schema.match(/.*import\s+"(.*)".*;/g); + if (!matched) { + return [cur]; + } + + // collect all import filenames + const importFileNames = matched + .map((item: string) => { + // if is comment statement, continue + if (item.trim().startsWith("//")) { + return ""; + } + const m = item.match(/.*"(.*)".*/); + return m ? m[1] : ""; + }) + .filter(Boolean); + + // loop all import files + for (const importFileName of importFileNames) { + const mFilePath = path.join(baseDir, importFileName + ".mol"); + const mRelativePath = path.relative(baseDir, mFilePath); + + cur.dependencies.push(importFileName); + if (!resolved.has(mFilePath)) { + // mask this file has resolved + resolved.add(mFilePath); + + const _dependencies = resolveDependencies( + mRelativePath, + baseDir, + resolved + ); + dependencies.push(..._dependencies); + } + } + + dependencies.push(cur); + + return dependencies; +} + +export function extractAndEraseImportClauses(code: string): string { + const lines = code.split("\n"); + const delImportLines = lines.filter((line: string) => { + return !line.trim().startsWith("import"); + }); + return delImportLines.join("\n"); +} + +function printOrWrite(resultMap: Map, config: Config) { + for (const name of resultMap.keys()) { + if (config.output < 2) { + console.log(`// ${String("-").repeat(66)} //`); + console.log(`// generate from ${name}`); + console.log(`// ${String("-").repeat(66)} //`); + console.log(resultMap.get(name)?.code); + if (config.output === 1) { + const dir = path.join(config.dir, "mols"); + if (!fs.existsSync(dir)) { + console.log(`mkdir mols`); + fs.mkdirSync(dir); + } + const tsName = name.replace(".mol", ".ts"); + const targetDir = path.dirname(path.join(dir, tsName)); + if (!fs.existsSync(targetDir)) { + console.log(`mkdir ${targetDir}`); + fs.mkdirSync(targetDir, { recursive: true }); + } + console.log(`writing file ${tsName}`); + fs.writeFileSync( + path.join(dir, tsName), + resultMap.get(name)?.code || "" + ); + console.log(`write file ${tsName} finish`); + } + } + } +} + +export function loopCodegen(config: Config): Map { + const result: Map = new Map(); + const baseDir = path.dirname(config.schemaFile); + const relativePath = path.basename(config.schemaFile); + const dependencies = resolveDependencies(relativePath, baseDir, new Set()); + + if (dependencies.length === 0) { + return result; + } + + const parsed: Set = new Set(); + dependencies.forEach((cur) => { + // has generated, continue + if (parsed.has(cur.relativePath)) { + return; + } + + // erase the import clause from the schema when calling the codegen method + const realPath = path.join(baseDir, cur.relativePath); + const schema = extractAndEraseImportClauses( + fs.readFileSync(realPath, "utf-8") + ); + + let optionPrepend = config.prepend; + // append all ESM import to config.prepend + + for (const importName of cur.dependencies) { + const importAbsolutePath = path.join( + path.dirname(realPath), + importName + ".mol" + ); + const importRelativePath = path.relative(baseDir, importAbsolutePath); + + if (result.has(importRelativePath)) { + const imptDesc = `\nimport { ${result + .get(importRelativePath) + ?.elements.join(", ")} } from './${importName}'`; + optionPrepend += imptDesc; + } + } + + const codegenReturn = codegenReturnWithElements(schema, { + prepend: optionPrepend, + formatObjectKeys: + String(config.objectKeyFormat).toLowerCase() === "camelcase" + ? camelcase + : undefined, + }); + + parsed.add(cur.relativePath); + result.set(cur.relativePath, codegenReturn); + }); + + printOrWrite(result, config); + + return result; +} diff --git a/packages/molecule/src/type.ts b/packages/molecule/src/type.ts index c3f790a5e..d8f7301e9 100644 --- a/packages/molecule/src/type.ts +++ b/packages/molecule/src/type.ts @@ -62,5 +62,5 @@ export interface Parser { export type ParseResult = { code: string; - fields: string[]; -}; \ No newline at end of file + elements: string[]; +}; diff --git a/packages/molecule/tests/mol/base.mol b/packages/molecule/tests/mol/base.mol new file mode 100644 index 000000000..a05c8a1ba --- /dev/null +++ b/packages/molecule/tests/mol/base.mol @@ -0,0 +1,4 @@ +// base.mol + +array RGB [byte;3]; +vector UTF8String ; \ No newline at end of file diff --git a/packages/molecule/tests/mol/character.mol b/packages/molecule/tests/mol/character.mol new file mode 100644 index 000000000..a9914f54e --- /dev/null +++ b/packages/molecule/tests/mol/character.mol @@ -0,0 +1,13 @@ +// character.mol + +import "base"; +import "submodule/base"; + +// array RGB [byte;3]; +// vector UTF8String ; + +table Character { + hair_color: RGB, + hair_color_c: RGB4, + name: UTF8String, +} \ No newline at end of file diff --git a/packages/molecule/tests/mol/submodule/base.mol b/packages/molecule/tests/mol/submodule/base.mol new file mode 100644 index 000000000..483256dc6 --- /dev/null +++ b/packages/molecule/tests/mol/submodule/base.mol @@ -0,0 +1,3 @@ +// base.mol + +array RGB4 [byte;3]; \ No newline at end of file diff --git a/packages/molecule/tests/resolve.test.ts b/packages/molecule/tests/resolve.test.ts new file mode 100644 index 000000000..a62359c67 --- /dev/null +++ b/packages/molecule/tests/resolve.test.ts @@ -0,0 +1,31 @@ +import test from "ava"; +import { + resolveDependencies, + extractAndEraseImportClauses, +} from "../src/resolve"; + +function expectResolveDependencies() { + const generated = resolveDependencies( + "character.mol", + "./tests/mol", + new Set() + ); + + return generated.length === 3 && generated[2].dependencies.length === 2; +} + +test("dependencies length right", (t) => { + t.true(expectResolveDependencies()); +}); + +test("erase import base", (t) => { + const result = extractAndEraseImportClauses(` + import "base"; + // import "submodule/base"; + + table Character { + hair_color: RGB, + } + `); + t.true(!result.includes(`import "base"`)); +});