Codegen 643

packages/molecule/src/cli.ts

@@ -1,44 +1,6 @@
 #!/usr/bin/env node
-import { codegen } from "./codegen";
-import * as fs from "node:fs";
+import { initConfig } from "./config";
+import { loopCodegen } from "./resolve";
 
-const DEFAULT_CONFIG_FILE_NAME = "lumos-molecule-codegen.json";
-
-function camelcase(str: string): string {
-  return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
-}
-
-type Config = {
-  objectKeyFormat: "camelcase" | "keep";
-  prepend: string;
-  schemaFile: string;
-};
-
-const fileConfig: Partial<Config> = (() => {
-  if (fs.existsSync(DEFAULT_CONFIG_FILE_NAME)) {
-    return JSON.parse(fs.readFileSync(DEFAULT_CONFIG_FILE_NAME, "utf8"));
-  }
-  return {};
-})();
-
-const config: Config = {
-  objectKeyFormat: fileConfig.objectKeyFormat || "keep",
-  prepend: fileConfig.prepend || "",
-  schemaFile: fileConfig.schemaFile || "schema.mol",
-};
-
-// check if the schema file exists
-if (!fs.existsSync(config.schemaFile)) {
-  console.error(
-    `Schema file ${config.schemaFile} does not exist. Please configure the \`schemaFile\` in ${DEFAULT_CONFIG_FILE_NAME}`
-  );
-  process.exit(1);
-}
-
-const generated = codegen(fs.readFileSync(config.schemaFile, "utf-8"), {
-  prepend: config.prepend,
-  formatObjectKeys:
-    config.objectKeyFormat === "camelcase" ? camelcase : undefined,
-});
-
-console.log(generated);
+const config = initConfig();
+loopCodegen(config);

packages/molecule/src/codegen.ts

@@ -1,4 +1,4 @@
-import { MolType } from "./type";
+import { MolType, ParseResult } from "./type";
 import { Grammar as NearleyGrammar, Parser as NearleyParser } from "nearley";
 import { circularIterator } from "./circularIterator";
 
@@ -33,7 +33,10 @@
   );
 }
 
-export function codegen(schema: string, options: Options = {}): string {
+export function codegenReturnWithElements(
+  schema: string,
+  options: Options = {}
+): ParseResult {
   const parser = new NearleyParser(NearleyGrammar.fromCompiled(grammar));
   parser.feed(schema);
 
@@ -45,9 +48,11 @@
     importedModules
   );
 
+  const typeNames: Array<string> = [];
   const codecs = molTypes
     .map((molType) => {
       if (importedModules.includes(molType.name)) return "";
+      typeNames.push(molType.name);
 
       if (molType.type === "array") {
         if (molType.item === "byte") {
@@ -122,7 +127,7 @@
     .filter(Boolean)
     .join("\n\n");
 
-  return `// This file is generated by @ckb-lumos/molecule, please do not modify it manually.
+  const code = `// This file is generated by @ckb-lumos/molecule, please do not modify it manually.
 /* eslint-disable */
 
 import { bytes, createBytesCodec, createFixedBytesCodec, molecule } from "@ckb-lumos/codec";
@@ -147,6 +152,16 @@
 
 ${codecs}
 `;
+  const result: ParseResult = {
+    code,
+    elements: typeNames,
+  };
+
+  return result;
+}
+
+export function codegen(schema: string, options: Options = {}): string {
+  return codegenReturnWithElements(schema, options).code;
 }
 
 // sort molecule types by their dependencies, to make sure the known types can be used in the front
@@ -219,7 +234,7 @@
   while (iterator.current() != null && scanTimes < maxScanTimes) {
     scanTimes++;
 
     const molType = iterator.current()!;
     if (checkCanParse(molType)) {
       sortedTypes.push(molType);
       availableTypes.add(molType.name);

packages/molecule/src/config.ts

@@ -0,0 +1,31 @@
+import * as fs from "node:fs";
+
+const DEFAULT_CONFIG_FILE_NAME =
+  process.env.CONF_PATH || "lumos-molecule-codegen.json";
+
+export type Config = {
+  objectKeyFormat: "camelcase" | "keep";
+  prepend: string;
+  schemaFile: string;
+  output: number; // 0: Default out console, 1: Write file, 2. Just return
+  dir: string; //
+};
+
+export function initConfig(): Config {
+  const fileConfig: Partial<Config> = (() => {
+    if (fs.existsSync(DEFAULT_CONFIG_FILE_NAME)) {
+      return JSON.parse(fs.readFileSync(DEFAULT_CONFIG_FILE_NAME, "utf8"));
+    }
+    return {};
+  })();
+
+  const config: Config = {
+    objectKeyFormat: fileConfig.objectKeyFormat || "keep",
+    prepend: fileConfig.prepend || "",
+    schemaFile: fileConfig.schemaFile || "schema.mol",
+    output: fileConfig.output || 0,
+    dir: fileConfig.dir || __dirname,
+  };
+
+  return config;
+}

packages/molecule/src/resolve.ts

@@ -0,0 +1,176 @@
+#!/usr/bin/env node
+import { ParseResult } from "./type";
+import { codegenReturnWithElements } from "./codegen";
+import { Config } from "./config";
+import * as fs from "node:fs";
+import * as path from "path";
+
+function camelcase(str: string): string {
+  return str.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
+}
+
+export type RelativePath = string;
+export type FileWithDependence = {
+  relativePath: string;
+  dependencies: string[];
+};
+
+export function resolveDependencies(
+  importPath: RelativePath,
+  baseDir: string,
+  resolved: Set<RelativePath>
+): FileWithDependence[] {
+  const dependencies: FileWithDependence[] = [];
+  // check if the file exist
+  const realPath = path.join(baseDir, importPath);
+  if (!fs.existsSync(realPath)) {
+    console.error(`Schema file ${realPath} does not exist.`);
+    process.exit(1);
+  }
+
+  const cur: FileWithDependence = {
+    relativePath: importPath,
+    dependencies: [],
+  };
+
+  const schema = fs.readFileSync(realPath, "utf-8");
+  if (!schema) {
+    return [cur];
+  }
+
+  const matched = schema.match(/.*import\s+"(.*)".*;/g);
+  if (!matched) {
+    return [cur];
+  }
+
+  // collect all import filenames
+  const importFileNames = matched
+    .map((item: string) => {
+      // if is comment statement, continue
+      if (item.trim().startsWith("//")) {
+        return "";
+      }
+      const m = item.match(/.*"(.*)".*/);
+      return m ? m[1] : "";
+    })
+    .filter(Boolean);
+
+  // loop all import files
+  for (const importFileName of importFileNames) {
+    const mFilePath = path.join(baseDir, importFileName + ".mol");
+    const mRelativePath = path.relative(baseDir, mFilePath);
+
+    cur.dependencies.push(importFileName);
+    if (!resolved.has(mFilePath)) {
+      // mask this file has resolved
+      resolved.add(mFilePath);
+
+      const _dependencies = resolveDependencies(
+        mRelativePath,
+        baseDir,
+        resolved
+      );
+      dependencies.push(..._dependencies);
+    }
+  }
+
+  dependencies.push(cur);
+
+  return dependencies;
+}
+
+export function extractAndEraseImportClauses(code: string): string {
+  const lines = code.split("\n");
+  const delImportLines = lines.filter((line: string) => {
+    return !line.trim().startsWith("import");
+  });
+  return delImportLines.join("\n");
+}
+
+function printOrWrite(resultMap: Map<string, ParseResult>, config: Config) {
+  for (const name of resultMap.keys()) {
+    if (config.output < 2) {
+      console.log(`// ${String("-").repeat(66)} //`);
+      console.log(`// generate from ${name}`);
+      console.log(`// ${String("-").repeat(66)} //`);
+      console.log(resultMap.get(name)?.code);
+      if (config.output === 1) {
+        const dir = path.join(config.dir, "mols");
+        if (!fs.existsSync(dir)) {
+          console.log(`mkdir mols`);
+          fs.mkdirSync(dir);
+        }
+        const tsName = name.replace(".mol", ".ts");
+        const targetDir = path.dirname(path.join(dir, tsName));
+        if (!fs.existsSync(targetDir)) {
+          console.log(`mkdir ${targetDir}`);
+          fs.mkdirSync(targetDir, { recursive: true });
+        }
+        console.log(`writing file ${tsName}`);
+        fs.writeFileSync(
+          path.join(dir, tsName),
+          resultMap.get(name)?.code || ""
+        );
+        console.log(`write file ${tsName} finish`);
+      }
+    }
+  }
+}
+
+export function loopCodegen(config: Config): Map<string, ParseResult> {
+  const result: Map<string, ParseResult> = new Map();
+  const baseDir = path.dirname(config.schemaFile);
+  const relativePath = path.basename(config.schemaFile);
+  const dependencies = resolveDependencies(relativePath, baseDir, new Set());
+
+  if (dependencies.length === 0) {
+    return result;
+  }
+
+  const parsed: Set<string> = new Set();
+  dependencies.forEach((cur) => {
+    // has generated, continue
+    if (parsed.has(cur.relativePath)) {
+      return;
+    }
+
+    // erase the import clause from the schema when calling the codegen method
+    const realPath = path.join(baseDir, cur.relativePath);
+    const schema = extractAndEraseImportClauses(
+      fs.readFileSync(realPath, "utf-8")
+    );
+
+    let optionPrepend = config.prepend;
+    // append all ESM import to config.prepend
+
+    for (const importName of cur.dependencies) {
+      const importAbsolutePath = path.join(
+        path.dirname(realPath),
+        importName + ".mol"
+      );
+      const importRelativePath = path.relative(baseDir, importAbsolutePath);
+
+      if (result.has(importRelativePath)) {
+        const imptDesc = `\nimport { ${result
+          .get(importRelativePath)
+          ?.elements.join(", ")} } from './${importName}'`;
+        optionPrepend += imptDesc;
+      }
+    }
+
+    const codegenReturn = codegenReturnWithElements(schema, {
+      prepend: optionPrepend,
+      formatObjectKeys:
+        String(config.objectKeyFormat).toLowerCase() === "camelcase"
+          ? camelcase
+          : undefined,
+    });
+
+    parsed.add(cur.relativePath);
+    result.set(cur.relativePath, codegenReturn);
+  });
+
+  printOrWrite(result, config);
+
+  return result;
+}

packages/molecule/src/type.ts

@@ -59,3 +59,8 @@ export type ParseOptions = {
 export interface Parser {
   parse(data: string, option?: ParseOptions): Record<string, AnyCodec>;
 }
+
+export type ParseResult = {
+  code: string;
+  elements: string[];
+};

packages/molecule/tests/mol/base.mol

@@ -0,0 +1,4 @@
+// base.mol
+
+array RGB [byte;3];
+vector UTF8String <byte>;

packages/molecule/tests/mol/character.mol

@@ -0,0 +1,13 @@
+// character.mol
+
+import "base";
+import "submodule/base";
+
+// array RGB [byte;3];
+// vector UTF8String <byte>;
+
+table Character {
+  hair_color: RGB,
+  hair_color_c: RGB4, 
+  name: UTF8String,
+}

packages/molecule/tests/mol/submodule/base.mol

@@ -0,0 +1,3 @@
+// base.mol
+
+array RGB4 [byte;3];

packages/molecule/tests/resolve.test.ts

@@ -0,0 +1,31 @@
+import test from "ava";
+import {
+  resolveDependencies,
+  extractAndEraseImportClauses,
+} from "../src/resolve";
+
+function expectResolveDependencies() {
+  const generated = resolveDependencies(
+    "character.mol",
+    "./tests/mol",
+    new Set()
+  );
+
+  return generated.length === 3 && generated[2].dependencies.length === 2;
+}
+
+test("dependencies length right", (t) => {
+  t.true(expectResolveDependencies());
+});
+
+test("erase import base", (t) => {
+  const result = extractAndEraseImportClauses(`
+  import "base";
+  // import "submodule/base";
+
+  table Character {
+    hair_color: RGB,
+  }
+  `);
+  t.true(!result.includes(`import "base"`));
+});