feat: add decorator for router

package.json

@@ -18,8 +18,8 @@
     "build": "tsc && npm run generate && nino koei -c scripts/build",
     "clean:views": "node scripts/pre-start",
     "codecov": "nino test -d",
-    "compile:server": "tsc --target es6 --module commonjs --sourceMap",
-    "d": "tsc --target es6 --module commonjs --sourceMap --watch",
+    "compile:server": "tsc --target ES5 --experimentalDecorators --module commonjs --sourceMap",
+    "d": "npm run compile:server -- --watch",
     "deploy": "node scripts/publish",
     "dev": "nino koei -c scripts/build -w -d -t",
     "generate": "node scripts/generateSider",

server/controller/ExhentaiController.ts

@@ -0,0 +1,277 @@
+import { getTargetResource } from '../utils/resource';
+import fs from 'fs-extra';
+import path from 'path';
+import puppeteer from 'puppeteer-core';
+import { format } from 'date-fns';
+import request from 'request-promise';
+import { success, info, trace, error } from '../utils/log';
+import { Controller, Request } from '../utils/decorator';
+
+const { exHentai: exHentaiCookie } = getTargetResource('cookie');
+const { exHentai } = getTargetResource('server');
+
+export interface ExHentaiInfoItem {
+  name: string;
+  detailUrl: string;
+  postTime: number;
+  thumbnailUrl: string;
+}
+
+const getLastestFileName = () => {
+  const exHentaiInfoPath = path.join(process.cwd(), './src/assets/exhentai/');
+  const exHentaiInfoFiles = fs
+    .readdirSync(exHentaiInfoPath)
+    .filter((item: string) => item !== '.gitkeep')
+    .map((item: string) => parseInt(item, 10));
+  return exHentaiInfoFiles.sort((a: any, b: any) => b - a);
+};
+
+const setExHentaiCookie = async (page: any) => {
+  for (const item of exHentaiCookie) {
+    await page.setCookie(item);
+  }
+};
+
+const getExHentaiInfo = async ({
+  pageIndex,
+  page,
+}: {
+  pageIndex: number;
+  page: any;
+}) => {
+  await page.goto('https://www.google.com/', {
+    waitUntil: 'domcontentloaded',
+  });
+  await page.goto(exHentai.href + pageIndex, {
+    waitUntil: 'domcontentloaded',
+  });
+  const exHentaiInfo = await page.$$eval(
+    'div.gl1t',
+    (wrappers: any[]) =>
+      new Promise(resolve => {
+        const results: ExHentaiInfoItem[] = [];
+        for (const item of wrappers) {
+          const tempPostTime = item.lastChild.innerText.replace(/[^0-9]/gi, '');
+          const year = tempPostTime.substring(0, 4);
+          const month = tempPostTime.substring(5, 6) - 1;
+          const day = tempPostTime.substring(7, 8);
+          const hour = tempPostTime.substring(9, 10);
+          const minute = tempPostTime.substring(11, 12);
+
+          const postTime = new Date(year, month, day, hour, minute).getTime();
+          results.push({
+            name: item.firstChild.innerText,
+            detailUrl: item.firstChild.href,
+            postTime,
+            thumbnailUrl: item.childNodes[1].firstChild.firstChild.src,
+          });
+        }
+        resolve(results);
+      }),
+  );
+  return exHentaiInfo;
+};
+
+const launchExHentaiPage = async () => {
+  const browser = await puppeteer.launch({
+    executablePath: exHentai.executablePath,
+    args: exHentai.launchArgs,
+    devtools: exHentai.devtools,
+  });
+  success('launch puppeteer');
+  const page = await browser.newPage();
+  setExHentaiCookie(page);
+  success('set cookie');
+  return { page, browser };
+};
+
+const getAllThumbnaiUrls = async (page: any) =>
+  await page.$$eval(
+    exHentai.thumbnailClass,
+    (wrappers: any[]) =>
+      new Promise(resolve => {
+        const result: any[] = [];
+        for (const item of wrappers) {
+          result.push(item.href);
+        }
+        resolve(result);
+      }),
+  );
+
+const getUrlFromPaginationInfo = async (page: any) =>
+  await page.$$eval(
+    'table.ptt a',
+    (wrappers: any[]) =>
+      new Promise(resolve => {
+        if (wrappers.length !== 1) {
+          const result: string[] = [];
+          wrappers.pop();
+          wrappers.shift();
+          for (const item of wrappers) {
+            result.push(item.href);
+          }
+          resolve(result);
+        } else {
+          resolve([]);
+        }
+      }),
+  );
+
+@Controller('/exhentai')
+export default class ExhentaiController {
+  @Request({ url: '/', method: 'get' })
+  async getExhentai(ctx: any) {
+    const { page, browser } = await launchExHentaiPage();
+    let results: ExHentaiInfoItem[] = [];
+    const lastestFileName = getLastestFileName()[0];
+    const lastestFilePath = path.join(
+      process.cwd(),
+      `src/assets/exhentai/${lastestFileName}.json`,
+    );
+    const { postTime } = JSON.parse(
+      fs.readFileSync(lastestFilePath).toString(),
+    )[0];
+    for (let i = 0; i < exHentai.maxPageIndex; i++) {
+      info(`fetching pageIndex => ${i + 1}`);
+      const result = await getExHentaiInfo({ pageIndex: i, page });
+      results = [...results, ...result];
+      // compare lastest date of comic, break when current comic has been fetched
+      if (result.length > 0) {
+        if (result[result.length - 1].postTime < postTime) {
+          break;
+        }
+      }
+
+      await page.waitFor(exHentai.waitTime);
+    }
+    await browser.close();
+
+    trace('write into json');
+    const createTime = format(new Date(), 'yyyyMMddHHmmss');
+    fs.outputJSON(
+      path.join(process.cwd(), `src/assets/exhentai/${createTime}.json`),
+      results,
+    ).catch((err: any) => {
+      error('write into json' + err);
+    });
+    success('write into json');
+
+    ctx.response.body = `./assets/${createTime}.json`;
+  }
+
+  @Request({ url: '/getLastestSet', method: 'get' })
+  async getLastestExHentaiSet(ctx: any) {
+    ctx.response.body = `./assets/exhentai/${getLastestFileName()[0]}.json`;
+  }
+
+  @Request({ url: '/download', method: 'post' })
+  async downloadImages(ctx: any) {
+    const { url, name } = ctx.request.body;
+    const subName = name.replace(
+      /[·！#￥（——）：；“”‘、，|《。》？、【】[\]]/gim,
+      '',
+    );
+    info(`download from: ${url}`);
+    const { page, browser } = await launchExHentaiPage();
+    await page.goto('https://www.google.com/', {
+      waitUntil: 'domcontentloaded',
+    });
+    await page.goto(url, { waitUntil: 'domcontentloaded' });
+
+    // prepare for download
+    const datePath = format(new Date(), 'yyyyMMdd');
+    fs.ensureDirSync(
+      path.join(
+        process.cwd(),
+        `${exHentai.downloadPath}/${datePath}/${subName}`,
+      ),
+    );
+
+    const restDetailUrls = await getUrlFromPaginationInfo(page);
+    const firstPageThumbnailUrls = await getAllThumbnaiUrls(page);
+    await page.waitFor(exHentai.waitTime);
+
+    for (const item of restDetailUrls) {
+      await page.goto('https://www.google.com/', {
+        waitUntil: 'domcontentloaded',
+      });
+      await page.goto(item, { waitUntil: 'domcontentloaded' });
+      const thumbnailUrlsFromNextPage = await getAllThumbnaiUrls(page);
+      firstPageThumbnailUrls.push(...thumbnailUrlsFromNextPage);
+      info('image length: ' + firstPageThumbnailUrls.length);
+      await page.waitFor(exHentai.waitTime);
+    }
+
+    const images = [];
+    const targetImgUrls = firstPageThumbnailUrls;
+    // get thumbnail url in detail page
+    for (let i = 0; i < targetImgUrls.length; i++) {
+      await page.goto('https://www.google.com/', {
+        waitUntil: 'domcontentloaded',
+      });
+      await page.goto(targetImgUrls[i], { waitUntil: 'domcontentloaded' });
+      info(`fetching image url => ${targetImgUrls[i]}`);
+      const imgUrl = await page.$eval(
+        '[id=i3] img',
+        (target: any) =>
+          new Promise(resolve => {
+            resolve(target.src);
+          }),
+      );
+      images.push(imgUrl);
+      await page.waitFor(exHentai.waitTime);
+    }
+    success('fetch all images');
+    // save image url into file, for unexpect error
+    fs.outputJSON(
+      path.join(
+        process.cwd(),
+        `${exHentai.downloadPath}/${datePath}/${subName}/restDetailUrls.json`,
+      ),
+      targetImgUrls,
+    ).catch((err: any) => {
+      error('write into json' + err);
+    });
+    fs.outputJSON(
+      path.join(
+        process.cwd(),
+        `${exHentai.downloadPath}/${datePath}/${subName}/detailImageUrls.json`,
+      ),
+      images,
+    ).catch((err: any) => {
+      error('write into json' + err);
+    });
+
+    // fetch and save images
+    for (let i = 0; i < images.length; i++) {
+      const item = images[i];
+      trace('download begin: ' + item);
+      await request
+        .get({ url: item, proxy: exHentai.proxy } as {
+          url: string;
+          proxy: string;
+        })
+        .on('error', (err: any) => {
+          error(err + ' => ' + item);
+        })
+        .pipe(
+          fs
+            .createWriteStream(
+              path.join(
+                process.cwd(),
+                `${exHentai.downloadPath}/${datePath}/${subName}/${i + 1}.jpg`,
+              ),
+            )
+            .on('finish', () => success(`${i + 1}.jpg`))
+            .on('error', (err: any) =>
+              error(`${subName}-${i + 1}.jpg failed, ${err}`),
+            ),
+        );
+      if (i % 4 === 0) {
+        await page.waitFor(exHentai.waitTime);
+      }
+    }
+    await browser.close();
+    ctx.response.body = true;
+  }
+}

server/controller/MainPageController.ts

@@ -0,0 +1,14 @@
+import fs from 'fs-extra';
+import path from 'path';
+import { Controller, Request } from '../utils/decorator';
+
+@Controller('/')
+export default class MainPageController {
+  @Request({ url: '/', method: 'get' })
+  async getMainPage(ctx: any) {
+    ctx.type = 'html';
+    ctx.response.body = fs.createReadStream(
+      path.join(process.cwd(), 'dist', 'index.html'),
+    );
+  }
+}

server/controller/MappingController.ts

@@ -0,0 +1,48 @@
+import fs from 'fs-extra';
+import path from 'path';
+import { updateMappingRouter } from './MarkdownController';
+import { Controller, Request } from '../utils/decorator';
+
+@Controller('/del')
+export default class MappingController {
+  @Request({ url: '/mapping', method: 'delete' })
+  async del(ctx: any) {
+    const { id, category } = ctx.request.body;
+    const mappingPaths = [
+      `src/assets/mapping/${id}.json`,
+      `dist/assets/mapping/${id}.json`,
+    ];
+    const markdownPaths = [
+      `src/assets/markdown/${id}.md`,
+      `dist/assets/markdown/${id}.md`,
+    ];
+    let targetPaths: string[] = [];
+
+    switch (category) {
+      case 'mapping':
+        targetPaths = mappingPaths;
+        break;
+
+      case 'markdown':
+        targetPaths = markdownPaths;
+        break;
+
+      default:
+        break;
+    }
+
+    try {
+      for (const item of targetPaths) {
+        if (fs.existsSync(item)) {
+          fs.unlinkSync(path.join(process.cwd(), item));
+        } else {
+          throw Error(`${item} doesn't exist.`);
+        }
+      }
+      updateMappingRouter({ id } as any, true);
+      ctx.response.body = true;
+    } catch (error) {
+      ctx.response.body = false;
+    }
+  }
+}