diff --git a/packages/molecule/src/cli.ts b/packages/molecule/src/cli.ts index e3f798a8e..590b06f8e 100644 --- a/packages/molecule/src/cli.ts +++ b/packages/molecule/src/cli.ts @@ -1,44 +1,6 @@ #!/usr/bin/env node -import { codegen } from "./codegen"; -import * as fs from "node:fs"; +import { initConfig } from "./config"; +import { loopCodegen } from "./resolve"; -const DEFAULT_CONFIG_FILE_NAME = "lumos-molecule-codegen.json"; - -function camelcase(str: string): string { - return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()); -} - -type Config = { - objectKeyFormat: "camelcase" | "keep"; - prepend: string; - schemaFile: string; -}; - -const fileConfig: Partial = (() => { - if (fs.existsSync(DEFAULT_CONFIG_FILE_NAME)) { - return JSON.parse(fs.readFileSync(DEFAULT_CONFIG_FILE_NAME, "utf8")); - } - return {}; -})(); - -const config: Config = { - objectKeyFormat: fileConfig.objectKeyFormat || "keep", - prepend: fileConfig.prepend || "", - schemaFile: fileConfig.schemaFile || "schema.mol", -}; - -// check if the schema file exists -if (!fs.existsSync(config.schemaFile)) { - console.error( - `Schema file ${config.schemaFile} does not exist. Please configure the \`schemaFile\` in ${DEFAULT_CONFIG_FILE_NAME}` - ); - process.exit(1); -} - -const generated = codegen(fs.readFileSync(config.schemaFile, "utf-8"), { - prepend: config.prepend, - formatObjectKeys: - config.objectKeyFormat === "camelcase" ? camelcase : undefined, -}); - -console.log(generated); +const config = initConfig(); +loopCodegen(config); diff --git a/packages/molecule/src/codegen.ts b/packages/molecule/src/codegen.ts index 86d55b564..ee97d2cf7 100644 --- a/packages/molecule/src/codegen.ts +++ b/packages/molecule/src/codegen.ts @@ -1,4 +1,4 @@ -import { MolType } from "./type"; +import { MolType, ParseResult } from "./type"; import { Grammar as NearleyGrammar, Parser as NearleyParser } from "nearley"; import { circularIterator } from "./circularIterator"; @@ -33,7 +33,10 @@ export function scanCustomizedTypes(prepend: string): string[] { ); } -export function codegen(schema: string, options: Options = {}): string { +export function codegenReturnWithElements( + schema: string, + options: Options = {} +): ParseResult { const parser = new NearleyParser(NearleyGrammar.fromCompiled(grammar)); parser.feed(schema); @@ -45,9 +48,11 @@ export function codegen(schema: string, options: Options = {}): string { importedModules ); + const typeNames: Array = []; const codecs = molTypes .map((molType) => { if (importedModules.includes(molType.name)) return ""; + typeNames.push(molType.name); if (molType.type === "array") { if (molType.item === "byte") { @@ -122,7 +127,7 @@ export function codegen(schema: string, options: Options = {}): string { .filter(Boolean) .join("\n\n"); - return `// This file is generated by @ckb-lumos/molecule, please do not modify it manually. + const code = `// This file is generated by @ckb-lumos/molecule, please do not modify it manually. /* eslint-disable */ import { bytes, createBytesCodec, createFixedBytesCodec, molecule } from "@ckb-lumos/codec"; @@ -147,6 +152,16 @@ const byte = createFallbackFixedBytesCodec(1); ${codecs} `; + const result: ParseResult = { + code, + elements: typeNames, + }; + + return result; +} + +export function codegen(schema: string, options: Options = {}): string { + return codegenReturnWithElements(schema, options).code; } // sort molecule types by their dependencies, to make sure the known types can be used in the front diff --git a/packages/molecule/src/config.ts b/packages/molecule/src/config.ts new file mode 100644 index 000000000..b429f2845 --- /dev/null +++ b/packages/molecule/src/config.ts @@ -0,0 +1,31 @@ +import * as fs from "node:fs"; + +const DEFAULT_CONFIG_FILE_NAME = + process.env.CONF_PATH || "lumos-molecule-codegen.json"; + +export type Config = { + objectKeyFormat: "camelcase" | "keep"; + prepend: string; + schemaFile: string; + output: number; // 0: Default out console, 1: Write file, 2. Just return + dir: string; // +}; + +export function initConfig(): Config { + const fileConfig: Partial = (() => { + if (fs.existsSync(DEFAULT_CONFIG_FILE_NAME)) { + return JSON.parse(fs.readFileSync(DEFAULT_CONFIG_FILE_NAME, "utf8")); + } + return {}; + })(); + + const config: Config = { + objectKeyFormat: fileConfig.objectKeyFormat || "keep", + prepend: fileConfig.prepend || "", + schemaFile: fileConfig.schemaFile || "schema.mol", + output: fileConfig.output || 0, + dir: fileConfig.dir || __dirname, + }; + + return config; +} diff --git a/packages/molecule/src/resolve.ts b/packages/molecule/src/resolve.ts new file mode 100644 index 000000000..6dc1a3df9 --- /dev/null +++ b/packages/molecule/src/resolve.ts @@ -0,0 +1,176 @@ +#!/usr/bin/env node +import { ParseResult } from "./type"; +import { codegenReturnWithElements } from "./codegen"; +import { Config } from "./config"; +import * as fs from "node:fs"; +import * as path from "path"; + +function camelcase(str: string): string { + return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()); +} + +export type RelativePath = string; +export type FileWithDependence = { + relativePath: string; + dependencies: string[]; +}; + +export function resolveDependencies( + importPath: RelativePath, + baseDir: string, + resolved: Set +): FileWithDependence[] { + const dependencies: FileWithDependence[] = []; + // check if the file exist + const realPath = path.join(baseDir, importPath); + if (!fs.existsSync(realPath)) { + console.error(`Schema file ${realPath} does not exist.`); + process.exit(1); + } + + const cur: FileWithDependence = { + relativePath: importPath, + dependencies: [], + }; + + const schema = fs.readFileSync(realPath, "utf-8"); + if (!schema) { + return [cur]; + } + + const matched = schema.match(/.*import\s+"(.*)".*;/g); + if (!matched) { + return [cur]; + } + + // collect all import filenames + const importFileNames = matched + .map((item: string) => { + // if is comment statement, continue + if (item.trim().startsWith("//")) { + return ""; + } + const m = item.match(/.*"(.*)".*/); + return m ? m[1] : ""; + }) + .filter(Boolean); + + // loop all import files + for (const importFileName of importFileNames) { + const mFilePath = path.join(baseDir, importFileName + ".mol"); + const mRelativePath = path.relative(baseDir, mFilePath); + + cur.dependencies.push(importFileName); + if (!resolved.has(mFilePath)) { + // mask this file has resolved + resolved.add(mFilePath); + + const _dependencies = resolveDependencies( + mRelativePath, + baseDir, + resolved + ); + dependencies.push(..._dependencies); + } + } + + dependencies.push(cur); + + return dependencies; +} + +export function extractAndEraseImportClauses(code: string): string { + const lines = code.split("\n"); + const delImportLines = lines.filter((line: string) => { + return !line.trim().startsWith("import"); + }); + return delImportLines.join("\n"); +} + +function printOrWrite(resultMap: Map, config: Config) { + for (const name of resultMap.keys()) { + if (config.output < 2) { + console.log(`// ${String("-").repeat(66)} //`); + console.log(`// generate from ${name}`); + console.log(`// ${String("-").repeat(66)} //`); + console.log(resultMap.get(name)?.code); + if (config.output === 1) { + const dir = path.join(config.dir, "mols"); + if (!fs.existsSync(dir)) { + console.log(`mkdir mols`); + fs.mkdirSync(dir); + } + const tsName = name.replace(".mol", ".ts"); + const targetDir = path.dirname(path.join(dir, tsName)); + if (!fs.existsSync(targetDir)) { + console.log(`mkdir ${targetDir}`); + fs.mkdirSync(targetDir, { recursive: true }); + } + console.log(`writing file ${tsName}`); + fs.writeFileSync( + path.join(dir, tsName), + resultMap.get(name)?.code || "" + ); + console.log(`write file ${tsName} finish`); + } + } + } +} + +export function loopCodegen(config: Config): Map { + const result: Map = new Map(); + const baseDir = path.dirname(config.schemaFile); + const relativePath = path.basename(config.schemaFile); + const dependencies = resolveDependencies(relativePath, baseDir, new Set()); + + if (dependencies.length === 0) { + return result; + } + + const parsed: Set = new Set(); + dependencies.forEach((cur) => { + // has generated, continue + if (parsed.has(cur.relativePath)) { + return; + } + + // erase the import clause from the schema when calling the codegen method + const realPath = path.join(baseDir, cur.relativePath); + const schema = extractAndEraseImportClauses( + fs.readFileSync(realPath, "utf-8") + ); + + let optionPrepend = config.prepend; + // append all ESM import to config.prepend + + for (const importName of cur.dependencies) { + const importAbsolutePath = path.join( + path.dirname(realPath), + importName + ".mol" + ); + const importRelativePath = path.relative(baseDir, importAbsolutePath); + + if (result.has(importRelativePath)) { + const imptDesc = `\nimport { ${result + .get(importRelativePath) + ?.elements.join(", ")} } from './${importName}'`; + optionPrepend += imptDesc; + } + } + + const codegenReturn = codegenReturnWithElements(schema, { + prepend: optionPrepend, + formatObjectKeys: + String(config.objectKeyFormat).toLowerCase() === "camelcase" + ? camelcase + : undefined, + }); + + parsed.add(cur.relativePath); + result.set(cur.relativePath, codegenReturn); + }); + + printOrWrite(result, config); + + return result; +} diff --git a/packages/molecule/src/type.ts b/packages/molecule/src/type.ts index 21f07b7eb..d8f7301e9 100644 --- a/packages/molecule/src/type.ts +++ b/packages/molecule/src/type.ts @@ -59,3 +59,8 @@ export type ParseOptions = { export interface Parser { parse(data: string, option?: ParseOptions): Record; } + +export type ParseResult = { + code: string; + elements: string[]; +}; diff --git a/packages/molecule/tests/mol/base.mol b/packages/molecule/tests/mol/base.mol new file mode 100644 index 000000000..a05c8a1ba --- /dev/null +++ b/packages/molecule/tests/mol/base.mol @@ -0,0 +1,4 @@ +// base.mol + +array RGB [byte;3]; +vector UTF8String ; \ No newline at end of file diff --git a/packages/molecule/tests/mol/character.mol b/packages/molecule/tests/mol/character.mol new file mode 100644 index 000000000..a9914f54e --- /dev/null +++ b/packages/molecule/tests/mol/character.mol @@ -0,0 +1,13 @@ +// character.mol + +import "base"; +import "submodule/base"; + +// array RGB [byte;3]; +// vector UTF8String ; + +table Character { + hair_color: RGB, + hair_color_c: RGB4, + name: UTF8String, +} \ No newline at end of file diff --git a/packages/molecule/tests/mol/submodule/base.mol b/packages/molecule/tests/mol/submodule/base.mol new file mode 100644 index 000000000..483256dc6 --- /dev/null +++ b/packages/molecule/tests/mol/submodule/base.mol @@ -0,0 +1,3 @@ +// base.mol + +array RGB4 [byte;3]; \ No newline at end of file diff --git a/packages/molecule/tests/resolve.test.ts b/packages/molecule/tests/resolve.test.ts new file mode 100644 index 000000000..a62359c67 --- /dev/null +++ b/packages/molecule/tests/resolve.test.ts @@ -0,0 +1,31 @@ +import test from "ava"; +import { + resolveDependencies, + extractAndEraseImportClauses, +} from "../src/resolve"; + +function expectResolveDependencies() { + const generated = resolveDependencies( + "character.mol", + "./tests/mol", + new Set() + ); + + return generated.length === 3 && generated[2].dependencies.length === 2; +} + +test("dependencies length right", (t) => { + t.true(expectResolveDependencies()); +}); + +test("erase import base", (t) => { + const result = extractAndEraseImportClauses(` + import "base"; + // import "submodule/base"; + + table Character { + hair_color: RGB, + } + `); + t.true(!result.includes(`import "base"`)); +});