feat(postprocess): add csv-to-json postprocessor

c-3lab · Jan 28, 2023 · 85d70f8 · 85d70f8
1 parent d8c1956
commit 85d70f8
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 0 deletions.
diff --git a/libs/postprocess/csv_to_json_converter.ts b/libs/postprocess/csv_to_json_converter.ts
@@ -0,0 +1,23 @@
+import { parse } from "https://deno.land/std@0.174.0/encoding/csv.ts";
+import { Colors } from "../../deps.ts";
+import { BasePostprocess } from "./base_postprocess.ts";
+
+export class CsvToJsonConverter extends BasePostprocess {
+  async execute(_: string[], targetPath: string): Promise<string> {
+    const text = await Deno.readTextFile(targetPath);
+    const result = await parse(text, { skipFirstRow: true });
+    await Deno.writeTextFileSync(targetPath.replace(/\.csv?$/, ".json"), JSON.stringify(result));
+    return targetPath;
+  }
+  validate(argumentList: string[]) {
+    if (argumentList.length > 0) {
+      console.log(
+        Colors.red("error: Too many arguments:"),
+        Colors.red(this.type + " " + argumentList.join(" ")),
+      );
+      this.printUsage();
+      return false;
+    }
+    return true;
+  }
+}
diff --git a/libs/postprocess/postprocess_dispatcher.ts b/libs/postprocess/postprocess_dispatcher.ts
@@ -3,16 +3,19 @@ import { Command } from "./command.ts";
 import { Encoder } from "./encoder.ts";
 import { Unzipper } from "./unzipper.ts";
 import { XlsxToCsvConverter } from "./xlsx_to_csv_converter.ts";
+import { CsvToJsonConverter } from "./csv_to_json_converter.ts";
 
 export class PostprocessDispatcher {
   private encoder;
   private unzipper;
   private xlsxToCsvConverter;
+  private csvToJsonConverter;
   private command;
   constructor() {
     this.encoder = new Encoder("encode", ["encoding-to"]);
     this.unzipper = new Unzipper("unzip", []);
     this.xlsxToCsvConverter = new XlsxToCsvConverter("xlsx-to-csv", []);
+    this.csvToJsonConverter = new CsvToJsonConverter("csv-to-json", []);
     this.command = new Command("cmd", ["script"]);
   }
   async dispatch(type: string, argumentList: string[], targetPath: string) {
@@ -38,6 +41,13 @@ export class PostprocessDispatcher {
       } else {
         Deno.exit(1);
       }
+    } else if (type === this.csvToJsonConverter.type) {
+      if (this.csvToJsonConverter.validate(argumentList)) {
+        await this.csvToJsonConverter.execute([], targetPath);
+        console.log('Convert csv to json.');
+      } else {
+        Deno.exit(1);
+      }
     } else if (type === this.command.type) {
       const script = argumentList.join(" ");
       try {

diff --git a/tests/libs/actions.install.test.ts b/tests/libs/actions.install.test.ts
@@ -605,6 +605,56 @@ describe("InstallAction", () => {
       }
     });
 
+    it('convert downloaded file from csv to json and record in dim.json and dim-lock.json when specify "csv-to-json" as postProcess', async () => {
+      createEmptyDimJson();
+      const textCsv = Deno.readFileSync("../test_data/valid_csv.csv");
+      const kyGetStub = createKyGetStub(textCsv);
+      try {
+        await new InstallAction().execute(
+          { name: "csv-to-json", postProcesses: ["csv-to-json"] },
+          "https://example.com/dummy.csv",
+        );
+        assert(fileExists("data_files/csv-to-json/dummy.csv"));
+        const testData = Deno.readTextFileSync(
+          "data_files/csv-to-json/dummy.json",
+        );
+        assertEquals(testData, '[{"aaa":"12","bbb":"34","ccc":"56"},{"aaa":"10","bbb":"20","ccc":"30"}]');
+        assertSpyCall(consoleLogStub, 0, { args: ["Convert csv to json."] });
+        const dimJson = JSON.parse(Deno.readTextFileSync("dim.json"));
+        assertEquals(dimJson, {
+          fileVersion: "1.1",
+          contents: [{
+            catalogResourceId: null,
+            catalogUrl: null,
+            headers: {},
+            name: "csv-to-json",
+            postProcesses: ["csv-to-json"],
+            url: "https://example.com/dummy.csv",
+          }],
+        });
+
+        const dimLockJson = JSON.parse(Deno.readTextFileSync("dim-lock.json"));
+        assertEquals(dimLockJson, {
+          lockFileVersion: "1.1",
+          contents: [{
+            catalogResourceId: null,
+            catalogUrl: null,
+            eTag: null,
+            headers: {},
+            integrity: "2c294b4e8b9fd5a0e520f712f108451975cbbded",
+            lastDownloaded: "2022-01-02T03:04:05.678Z",
+            lastModified: null,
+            name: "csv-to-json",
+            path: "./data_files/csv-to-json/dummy.csv",
+            postProcesses: ["csv-to-json"],
+            url: "https://example.com/dummy.csv",
+          }],
+        });
+      } finally {
+        kyGetStub.restore();
+      }
+    });
+
     it('exit with error when specify "xlsx-to-csv a" as postProcesses and download', async () => {
       createEmptyDimJson();
       const kyGetStub = createKyGetStub("dummy");

diff --git a/tests/test_data/valid_csv.csv b/tests/test_data/valid_csv.csv
@@ -0,0 +1,3 @@
+aaa,bbb,ccc
+12,34,56
+10,20,30