list links from specified page

c-3lab · Mar 29, 2023 · 337de17 · 337de17
1 parent 45f09d8
commit 337de17
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 1 deletion.
diff --git a/deps.ts b/deps.ts
@@ -29,6 +29,8 @@ export { Sha1 } from "https://deno.land/std@0.146.0/hash/sha1.ts";
 
 export { parse } from "https://deno.land/std@0.174.0/encoding/csv.ts";
 
+export { DOMParser } from "https://deno.land/x/deno_dom/deno-dom-wasm.ts";
+
 export const zipWrapper = {
   decompress: _decompress,
 };

diff --git a/dim.ts b/dim.ts
@@ -51,6 +51,14 @@ await new Command()
         "-A, --asyncInstall",
         "Execute asyncronous install.",
       )
+      .option(
+        "-P, --pageInstall <pageInstall:string>",
+        "Execute install from links in specified page."
+      )
+      .option(
+        "-e, --expression <expression:string>",
+        "Filter PageInstall result by regular expression."
+      )
       .description(
         "Install the data.\n" +
           "Specify the url of data. If you dont't specify argument, install all data which is not installed dependency.",

diff --git a/libs/actions.ts b/libs/actions.ts
@@ -1,4 +1,4 @@
-import { Colors, Confirm, Input } from "../deps.ts";
+import { Colors, Confirm, Input, DOMParser } from "../deps.ts";
 import { DEFAULT_DATAFILES_PATH, DEFAULT_DIM_FILE_PATH } from "./consts.ts";
 import { DimFileAccessor, DimLockFileAccessor } from "./accessor.ts";
 import { CkanApiClient } from "./ckan_api_client.ts";
@@ -25,9 +25,37 @@ export class InstallAction {
       file?: string;
       force?: boolean;
       asyncInstall?: boolean;
+      pageInstall?: string;
+      expression?: string;
     },
     url: string | undefined,
   ) {
+    if (options.pageInstall !== undefined) {
+      try {
+        const getResult = await fetch(options.pageInstall);
+        if (!getResult.ok) throw new Error("Fetch response error");
+        const html = await getResult.text();
+        const document = new DOMParser().parseFromString(html, "text/html");
+        const linklist = document.getElementsByTagName("a");
+        linklist.forEach((link) => {
+          const re = new RegExp(options.expression, "ig");
+          let href = link.getAttribute("href");
+          if (re.test(href)) {
+            if (!href.startsWith("http")) {
+              href =
+                options.pageInstall.slice(
+                  0,
+                  options.pageInstall.lastIndexOf("/") + 1
+                ) + href;
+            }
+            console.log(href);
+          }
+        });
+      } catch (error) {
+        console.error(error);
+      }
+    }
+
     if (url && options.file) {
       console.log(
         Colors.red("Cannot use -f option and URL at the same time."),