diff --git a/package-lock.json b/package-lock.json index db5deb4bd..3b4d3da3a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1215,6 +1215,15 @@ "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==", "dev": true }, + "@types/cheerio": { + "version": "0.22.11", + "resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.11.tgz", + "integrity": "sha512-x0X3kPbholdJZng9wDMhb2swvUi3UYRNAuWAmIPIWlfgAJZp//cql/qblE7181Mg7SjWVwq6ldCPCLn5AY/e7w==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/connect": { "version": "3.4.32", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.32.tgz", @@ -1236,6 +1245,16 @@ "@types/node": "*" } }, + "@types/enzyme": { + "version": "3.9.4", + "resolved": "https://registry.npmjs.org/@types/enzyme/-/enzyme-3.9.4.tgz", + "integrity": "sha512-bQcwt5gcKnekrbci4hcapfE2J6rkkFbHM1l4VobLtSl4ogOfj0lvSxrdS6FftCakmJqqPBqdQCwb5KnlivL6SQ==", + "dev": true, + "requires": { + "@types/cheerio": "*", + "@types/react": "*" + } + }, "@types/eslint-visitor-keys": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", diff --git a/package.json b/package.json index cd76a42ca..65b2a1fc1 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,8 @@ "build": "tsc && npm run generate && nino koei -c scripts/build", "clean:views": "node scripts/pre-start", "codecov": "nino test -d", - "compile:server": "tsc --target es6 --module commonjs --sourceMap", - "d": "tsc --target es6 --module commonjs --sourceMap --watch", + "compile:server": "tsc --target ES5 --experimentalDecorators --module commonjs --sourceMap", + "d": "npm run compile:server -- --watch", "deploy": "node scripts/publish", "dev": "nino koei -c scripts/build -w -d -t", "generate": "node scripts/generateSider", diff --git a/server/controller/ExhentaiController.ts b/server/controller/ExhentaiController.ts new file mode 100644 index 000000000..8e7517f1b --- /dev/null +++ b/server/controller/ExhentaiController.ts @@ -0,0 +1,277 @@ +import { getTargetResource } from '../utils/resource'; +import fs from 'fs-extra'; +import path from 'path'; +import puppeteer from 'puppeteer-core'; +import { format } from 'date-fns'; +import request from 'request-promise'; +import { success, info, trace, error } from '../utils/log'; +import { Controller, Request } from '../utils/decorator'; + +const { exHentai: exHentaiCookie } = getTargetResource('cookie'); +const { exHentai } = getTargetResource('server'); + +export interface ExHentaiInfoItem { + name: string; + detailUrl: string; + postTime: number; + thumbnailUrl: string; +} + +const getLastestFileName = () => { + const exHentaiInfoPath = path.join(process.cwd(), './src/assets/exhentai/'); + const exHentaiInfoFiles = fs + .readdirSync(exHentaiInfoPath) + .filter((item: string) => item !== '.gitkeep') + .map((item: string) => parseInt(item, 10)); + return exHentaiInfoFiles.sort((a: any, b: any) => b - a); +}; + +const setExHentaiCookie = async (page: any) => { + for (const item of exHentaiCookie) { + await page.setCookie(item); + } +}; + +const getExHentaiInfo = async ({ + pageIndex, + page, +}: { + pageIndex: number; + page: any; +}) => { + await page.goto('https://www.google.com/', { + waitUntil: 'domcontentloaded', + }); + await page.goto(exHentai.href + pageIndex, { + waitUntil: 'domcontentloaded', + }); + const exHentaiInfo = await page.$$eval( + 'div.gl1t', + (wrappers: any[]) => + new Promise(resolve => { + const results: ExHentaiInfoItem[] = []; + for (const item of wrappers) { + const tempPostTime = item.lastChild.innerText.replace(/[^0-9]/gi, ''); + const year = tempPostTime.substring(0, 4); + const month = tempPostTime.substring(5, 6) - 1; + const day = tempPostTime.substring(7, 8); + const hour = tempPostTime.substring(9, 10); + const minute = tempPostTime.substring(11, 12); + + const postTime = new Date(year, month, day, hour, minute).getTime(); + results.push({ + name: item.firstChild.innerText, + detailUrl: item.firstChild.href, + postTime, + thumbnailUrl: item.childNodes[1].firstChild.firstChild.src, + }); + } + resolve(results); + }), + ); + return exHentaiInfo; +}; + +const launchExHentaiPage = async () => { + const browser = await puppeteer.launch({ + executablePath: exHentai.executablePath, + args: exHentai.launchArgs, + devtools: exHentai.devtools, + }); + success('launch puppeteer'); + const page = await browser.newPage(); + setExHentaiCookie(page); + success('set cookie'); + return { page, browser }; +}; + +const getAllThumbnaiUrls = async (page: any) => + await page.$$eval( + exHentai.thumbnailClass, + (wrappers: any[]) => + new Promise(resolve => { + const result: any[] = []; + for (const item of wrappers) { + result.push(item.href); + } + resolve(result); + }), + ); + +const getUrlFromPaginationInfo = async (page: any) => + await page.$$eval( + 'table.ptt a', + (wrappers: any[]) => + new Promise(resolve => { + if (wrappers.length !== 1) { + const result: string[] = []; + wrappers.pop(); + wrappers.shift(); + for (const item of wrappers) { + result.push(item.href); + } + resolve(result); + } else { + resolve([]); + } + }), + ); + +@Controller('/exhentai') +export default class ExhentaiController { + @Request({ url: '/', method: 'get' }) + async getExhentai(ctx: any) { + const { page, browser } = await launchExHentaiPage(); + let results: ExHentaiInfoItem[] = []; + const lastestFileName = getLastestFileName()[0]; + const lastestFilePath = path.join( + process.cwd(), + `src/assets/exhentai/${lastestFileName}.json`, + ); + const { postTime } = JSON.parse( + fs.readFileSync(lastestFilePath).toString(), + )[0]; + for (let i = 0; i < exHentai.maxPageIndex; i++) { + info(`fetching pageIndex => ${i + 1}`); + const result = await getExHentaiInfo({ pageIndex: i, page }); + results = [...results, ...result]; + // compare lastest date of comic, break when current comic has been fetched + if (result.length > 0) { + if (result[result.length - 1].postTime < postTime) { + break; + } + } + + await page.waitFor(exHentai.waitTime); + } + await browser.close(); + + trace('write into json'); + const createTime = format(new Date(), 'yyyyMMddHHmmss'); + fs.outputJSON( + path.join(process.cwd(), `src/assets/exhentai/${createTime}.json`), + results, + ).catch((err: any) => { + error('write into json' + err); + }); + success('write into json'); + + ctx.response.body = `./assets/${createTime}.json`; + } + + @Request({ url: '/getLastestSet', method: 'get' }) + async getLastestExHentaiSet(ctx: any) { + ctx.response.body = `./assets/exhentai/${getLastestFileName()[0]}.json`; + } + + @Request({ url: '/download', method: 'post' }) + async downloadImages(ctx: any) { + const { url, name } = ctx.request.body; + const subName = name.replace( + /[·！#￥（——）：；“”‘、，|《。》？、【】[\]]/gim, + '', + ); + info(`download from: ${url}`); + const { page, browser } = await launchExHentaiPage(); + await page.goto('https://www.google.com/', { + waitUntil: 'domcontentloaded', + }); + await page.goto(url, { waitUntil: 'domcontentloaded' }); + + // prepare for download + const datePath = format(new Date(), 'yyyyMMdd'); + fs.ensureDirSync( + path.join( + process.cwd(), + `${exHentai.downloadPath}/${datePath}/${subName}`, + ), + ); + + const restDetailUrls = await getUrlFromPaginationInfo(page); + const firstPageThumbnailUrls = await getAllThumbnaiUrls(page); + await page.waitFor(exHentai.waitTime); + + for (const item of restDetailUrls) { + await page.goto('https://www.google.com/', { + waitUntil: 'domcontentloaded', + }); + await page.goto(item, { waitUntil: 'domcontentloaded' }); + const thumbnailUrlsFromNextPage = await getAllThumbnaiUrls(page); + firstPageThumbnailUrls.push(...thumbnailUrlsFromNextPage); + info('image length: ' + firstPageThumbnailUrls.length); + await page.waitFor(exHentai.waitTime); + } + + const images = []; + const targetImgUrls = firstPageThumbnailUrls; + // get thumbnail url in detail page + for (let i = 0; i < targetImgUrls.length; i++) { + await page.goto('https://www.google.com/', { + waitUntil: 'domcontentloaded', + }); + await page.goto(targetImgUrls[i], { waitUntil: 'domcontentloaded' }); + info(`fetching image url => ${targetImgUrls[i]}`); + const imgUrl = await page.$eval( + '[id=i3] img', + (target: any) => + new Promise(resolve => { + resolve(target.src); + }), + ); + images.push(imgUrl); + await page.waitFor(exHentai.waitTime); + } + success('fetch all images'); + // save image url into file, for unexpect error + fs.outputJSON( + path.join( + process.cwd(), + `${exHentai.downloadPath}/${datePath}/${subName}/restDetailUrls.json`, + ), + targetImgUrls, + ).catch((err: any) => { + error('write into json' + err); + }); + fs.outputJSON( + path.join( + process.cwd(), + `${exHentai.downloadPath}/${datePath}/${subName}/detailImageUrls.json`, + ), + images, + ).catch((err: any) => { + error('write into json' + err); + }); + + // fetch and save images + for (let i = 0; i < images.length; i++) { + const item = images[i]; + trace('download begin: ' + item); + await request + .get({ url: item, proxy: exHentai.proxy } as { + url: string; + proxy: string; + }) + .on('error', (err: any) => { + error(err + ' => ' + item); + }) + .pipe( + fs + .createWriteStream( + path.join( + process.cwd(), + `${exHentai.downloadPath}/${datePath}/${subName}/${i + 1}.jpg`, + ), + ) + .on('finish', () => success(`${i + 1}.jpg`)) + .on('error', (err: any) => + error(`${subName}-${i + 1}.jpg failed, ${err}`), + ), + ); + if (i % 4 === 0) { + await page.waitFor(exHentai.waitTime); + } + } + await browser.close(); + ctx.response.body = true; + } +} diff --git a/server/controller/MainPageController.ts b/server/controller/MainPageController.ts new file mode 100644 index 000000000..805d1ed14 --- /dev/null +++ b/server/controller/MainPageController.ts @@ -0,0 +1,14 @@ +import fs from 'fs-extra'; +import path from 'path'; +import { Controller, Request } from '../utils/decorator'; + +@Controller('/') +export default class MainPageController { + @Request({ url: '/', method: 'get' }) + async getMainPage(ctx: any) { + ctx.type = 'html'; + ctx.response.body = fs.createReadStream( + path.join(process.cwd(), 'dist', 'index.html'), + ); + } +} diff --git a/server/controller/MappingController.ts b/server/controller/MappingController.ts new file mode 100644 index 000000000..846d2f68b --- /dev/null +++ b/server/controller/MappingController.ts @@ -0,0 +1,48 @@ +import fs from 'fs-extra'; +import path from 'path'; +import { updateMappingRouter } from './MarkdownController'; +import { Controller, Request } from '../utils/decorator'; + +@Controller('/del') +export default class MappingController { + @Request({ url: '/mapping', method: 'delete' }) + async del(ctx: any) { + const { id, category } = ctx.request.body; + const mappingPaths = [ + `src/assets/mapping/${id}.json`, + `dist/assets/mapping/${id}.json`, + ]; + const markdownPaths = [ + `src/assets/markdown/${id}.md`, + `dist/assets/markdown/${id}.md`, + ]; + let targetPaths: string[] = []; + + switch (category) { + case 'mapping': + targetPaths = mappingPaths; + break; + + case 'markdown': + targetPaths = markdownPaths; + break; + + default: + break; + } + + try { + for (const item of targetPaths) { + if (fs.existsSync(item)) { + fs.unlinkSync(path.join(process.cwd(), item)); + } else { + throw Error(`${item} doesn't exist.`); + } + } + updateMappingRouter({ id } as any, true); + ctx.response.body = true; + } catch (error) { + ctx.response.body = false; + } + } +} diff --git a/server/controller/MarkdownController.ts b/server/controller/MarkdownController.ts new file mode 100644 index 000000000..621a05e18 --- /dev/null +++ b/server/controller/MarkdownController.ts @@ -0,0 +1,234 @@ +import fs from 'fs-extra'; +import path from 'path'; +import { Controller, Request } from '../utils/decorator'; +import md5 from 'blueimp-md5'; +const { updateSider } = require('../../scripts/generateSider'); + +export interface MappingProps { + createTime: number; + modifyTime: number; + id: string; + title: string; + url: string; + type: string; + subType: string; + category: 'mapping' | 'markdown'; +} + +export const updateMappingRouter = ( + targetItem: MappingProps, + isDelete?: boolean, + callback?: () => void, +) => { + const writeFilesPaths = [ + `src/assets/mapping.json`, + `dist/assets/mapping.json`, + ]; + const mappingPath = path.join(process.cwd(), writeFilesPaths[0]); + if (!fs.existsSync(mappingPath)) { + fs.writeFileSync(mappingPath, []); + } + const mappingFile = fs.readFileSync(mappingPath); + const result = JSON.parse(mappingFile.toString()); + + const isExistTargetIndex = result.findIndex( + (item: MappingProps) => item.id === targetItem.id, + ); + if (isExistTargetIndex > -1) { + if (isDelete) { + result.splice(isExistTargetIndex, 1); + } else { + result[isExistTargetIndex] = targetItem; + } + } else { + result.push(targetItem); + } + + for (const item of writeFilesPaths) { + fs.outputJSON(path.join(process.cwd(), item), result, { + spaces: 2, + }).catch((err: any) => { + console.error(err); + }); + } + if (callback) { + callback(); + } + // tslint:disable-next-line: no-console + console.log(`mapping updated => ${targetItem.id}`); +}; + +@Controller('/save') +export default class MarkdownController { + @Request({ url: '/update', method: 'post' }) + async updateTargetMapping(ctx: any) { + const { + layout, + id, + title, + url, + type, + subType, + category, + } = ctx.request.body; + if (!id) { + throw Error('id is undefined.'); + } + const mappingPaths = [ + `src/assets/mapping/${id}.json`, + `dist/assets/mapping/${id}.json`, + ]; + const markdownPaths = [ + `src/assets/markdown/${id}.md`, + `dist/assets/markdown/${id}.md`, + ]; + let targetPaths: string[] = []; + + // update router + const targetFile = fs + .readFileSync(path.join(process.cwd(), 'src/assets/mapping.json')) + .toString(); + const targetJSONFile = JSON.parse(targetFile); + const targetArr = targetJSONFile.filter( + (item: MappingProps) => item.id === id, + ); + const targetItem = targetArr.length > 0 ? targetArr[0] : {}; + const { + createTime, + title: originTitle, + url: originUrl, + type: originType, + subType: originSubType, + category: originCategory, + } = targetItem; + const targetCategory = category || originCategory; + updateMappingRouter({ + id, + title: title || originTitle, + url: url || originUrl, + createTime, + modifyTime: new Date().getTime(), + type: type || originType, + subType: subType || originSubType, + category: targetCategory, + }); + + switch (targetCategory) { + case 'mapping': + targetPaths = mappingPaths; + break; + + case 'markdown': + targetPaths = markdownPaths; + break; + + default: + break; + } + + // update layout + let originLayout = layout; + if (!layout && id) { + const targetLayoutFile = fs.readFileSync( + path.join(path.join(process.cwd(), targetPaths[0])), + ); + const targetLayout = targetLayoutFile.toString(); + try { + originLayout = JSON.parse(targetLayout); + } catch (error) { + originLayout = targetLayout; + } + } + + try { + for (const item of targetPaths) { + if (targetCategory === 'markdown') { + fs.writeFileSync(path.join(process.cwd(), item), originLayout); + } + if (targetCategory === 'mapping') { + fs.outputJSON(path.join(process.cwd(), item), originLayout, { + spaces: 2, + }).catch((err: any) => { + console.error(err); + }); + } + // tslint:disable-next-line: no-console + console.log(`written completed => ${item}`); + } + ctx.response.body = true; + } catch (error) { + ctx.response.body = error.message; + } + } + + // 1. generate empty file in assets/mapping for mapping info + // 2. update router file + @Request({ url: '/new', method: 'post' }) + async initNewMapping(ctx: any) { + const { title, type, subType, category } = ctx.request.body; + const dateTime = new Date().getTime(); + const id = md5(dateTime as any); + const mappingPaths = [ + `src/assets/mapping/${id}.json`, + `dist/assets/mapping/${id}.json`, + ]; + const markdownPaths = [ + `src/assets/markdown/${id}.md`, + `dist/assets/markdown/${id}.md`, + ]; + let targetPaths: string[] = []; + + switch (category) { + case 'mapping': + targetPaths = mappingPaths; + break; + + case 'markdown': + targetPaths = markdownPaths; + break; + + default: + break; + } + + try { + // generate empty file in assets/mapping for mapping info + for (const item of targetPaths) { + fs.writeFileSync( + path.join(process.cwd(), item), + `{ + "CanvasPosition": { + "x": -2810, + "y": -3323, + "z": 0, + "gap": 1 + }, + "BlockGroup": {}, + "TagGroup": {}, + "LineGroup": {} + } + `, + ); + } + // update router file + updateMappingRouter( + { + id, + title, + url: `./assets/mapping/${id}.json`, + createTime: dateTime, + modifyTime: dateTime, + type, + subType, + category, + }, + false, + updateSider, + ); + ctx.response.body = id; + } catch (error) { + console.error(error); + ctx.response.body = error.message; + } + } +} diff --git a/server/controller/delete.ts b/server/controller/delete.ts deleted file mode 100644 index a3ad0e2bb..000000000 --- a/server/controller/delete.ts +++ /dev/null @@ -1,46 +0,0 @@ -import fs from 'fs-extra'; -import path from 'path'; -import Save from './save'; -const { updateMappingRouter } = Save; - -const del = async (ctx: any) => { - const { id, category } = ctx.request.body; - const mappingPaths = [ - `src/assets/mapping/${id}.json`, - `dist/assets/mapping/${id}.json`, - ]; - const markdownPaths = [ - `src/assets/markdown/${id}.md`, - `dist/assets/markdown/${id}.md`, - ]; - let targetPaths: string[] = []; - - switch (category) { - case 'mapping': - targetPaths = mappingPaths; - break; - - case 'markdown': - targetPaths = markdownPaths; - break; - - default: - break; - } - - try { - for (const item of targetPaths) { - if (fs.existsSync(item)) { - fs.unlinkSync(path.join(process.cwd(), item)); - } else { - throw Error(`${item} doesn't exist.`); - } - } - updateMappingRouter({ id } as any, true); - ctx.response.body = true; - } catch (error) { - ctx.response.body = false; - } -}; - -export default { 'DELETE /del/mapping': del }; diff --git a/server/controller/get.ts b/server/controller/get.ts deleted file mode 100644 index 639f41cb9..000000000 --- a/server/controller/get.ts +++ /dev/null @@ -1,276 +0,0 @@ -import { getTargetResource } from '../utils/resource'; -import fs from 'fs-extra'; -import path from 'path'; -import puppeteer from 'puppeteer-core'; -import { format } from 'date-fns'; -import request from 'request-promise'; -import { success, info, trace, error } from '../utils/log'; - -const { exHentai: exHentaiCookie } = getTargetResource('cookie'); -const { exHentai } = getTargetResource('server'); - -export interface ExHentaiInfoItem { - name: string; - detailUrl: string; - postTime: number; - thumbnailUrl: string; -} - -const getMainPage = async (ctx: any) => { - ctx.type = 'html'; - ctx.response.body = fs.createReadStream( - path.join(process.cwd(), 'dist', 'index.html'), - ); -}; - -const setExHentaiCookie = async (page: any) => { - for (const item of exHentaiCookie) { - await page.setCookie(item); - } -}; - -const getExHentaiInfo = async ({ - pageIndex, - page, -}: { - pageIndex: number; - page: any; -}) => { - await page.goto('https://www.google.com/', { waitUntil: 'domcontentloaded' }); - await page.goto(exHentai.href + pageIndex, { waitUntil: 'domcontentloaded' }); - const exHentaiInfo = await page.$$eval( - 'div.gl1t', - (wrappers: any[]) => - new Promise(resolve => { - const results: ExHentaiInfoItem[] = []; - for (const item of wrappers) { - const tempPostTime = item.lastChild.innerText.replace(/[^0-9]/gi, ''); - const year = tempPostTime.substring(0, 4); - const month = tempPostTime.substring(5, 6) - 1; - const day = tempPostTime.substring(7, 8); - const hour = tempPostTime.substring(9, 10); - const minute = tempPostTime.substring(11, 12); - - const postTime = new Date(year, month, day, hour, minute).getTime(); - results.push({ - name: item.firstChild.innerText, - detailUrl: item.firstChild.href, - postTime, - thumbnailUrl: item.childNodes[1].firstChild.firstChild.src, - }); - } - resolve(results); - }), - ); - return exHentaiInfo; -}; - -const launchExHentaiPage = async () => { - const browser = await puppeteer.launch({ - executablePath: exHentai.executablePath, - args: exHentai.launchArgs, - devtools: exHentai.devtools, - }); - success('launch puppeteer'); - const page = await browser.newPage(); - setExHentaiCookie(page); - success('set cookie'); - return { page, browser }; -}; - -const getLastestFileName = () => { - const exHentaiInfoPath = path.join(process.cwd(), './src/assets/exhentai/'); - const exHentaiInfoFiles = fs - .readdirSync(exHentaiInfoPath) - .filter((item: string) => item !== '.gitkeep') - .map((item: string) => parseInt(item, 10)); - return exHentaiInfoFiles.sort((a: any, b: any) => b - a); -}; - -const getExhentai = async (ctx: any) => { - const { page, browser } = await launchExHentaiPage(); - let results: ExHentaiInfoItem[] = []; - const lastestFileName = getLastestFileName()[0]; - const lastestFilePath = path.join( - process.cwd(), - `src/assets/exhentai/${lastestFileName}.json`, - ); - const { postTime } = JSON.parse( - fs.readFileSync(lastestFilePath).toString(), - )[0]; - for (let i = 0; i < exHentai.maxPageIndex; i++) { - info(`fetching pageIndex => ${i + 1}`); - const result = await getExHentaiInfo({ pageIndex: i, page }); - results = [...results, ...result]; - // compare lastest date of comic, break when current comic has been fetched - if (result.length > 0) { - if (result[result.length - 1].postTime < postTime) { - break; - } - } - - await page.waitFor(exHentai.waitTime); - } - await browser.close(); - - trace('write into json'); - const createTime = format(new Date(), 'yyyyMMddHHmmss'); - fs.outputJSON( - path.join(process.cwd(), `src/assets/exhentai/${createTime}.json`), - results, - ).catch((err: any) => { - error('write into json' + err); - }); - success('write into json'); - - ctx.response.body = `./assets/${createTime}.json`; -}; - -const getLastestExHentaiSet = async (ctx: any) => { - ctx.response.body = `./assets/exhentai/${getLastestFileName()[0]}.json`; -}; - -const getAllThumbnaiUrls = async (page: any) => - await page.$$eval( - exHentai.thumbnailClass, - (wrappers: any[]) => - new Promise(resolve => { - const result: any[] = []; - for (const item of wrappers) { - result.push(item.href); - } - resolve(result); - }), - ); - -const getUrlFromPaginationInfo = async (page: any) => - await page.$$eval( - 'table.ptt a', - (wrappers: any[]) => - new Promise(resolve => { - if (wrappers.length !== 1) { - const result: string[] = []; - wrappers.pop(); - wrappers.shift(); - for (const item of wrappers) { - result.push(item.href); - } - resolve(result); - } else { - resolve([]); - } - }), - ); - -const downloadImages = async (ctx: any) => { - const { url, name } = ctx.request.body; - const subName = name.replace( - /[·！#￥（——）：；“”‘、，|《。》？、【】[\]]/gim, - '', - ); - info(`download from: ${url}`); - const { page, browser } = await launchExHentaiPage(); - await page.goto('https://www.google.com/', { waitUntil: 'domcontentloaded' }); - await page.goto(url, { waitUntil: 'domcontentloaded' }); - - // prepare for download - const datePath = format(new Date(), 'yyyyMMdd'); - fs.ensureDirSync( - path.join(process.cwd(), `${exHentai.downloadPath}/${datePath}/${subName}`), - ); - - const restDetailUrls = await getUrlFromPaginationInfo(page); - const firstPageThumbnailUrls = await getAllThumbnaiUrls(page); - await page.waitFor(exHentai.waitTime); - - for (const item of restDetailUrls) { - await page.goto('https://www.google.com/', { - waitUntil: 'domcontentloaded', - }); - await page.goto(item, { waitUntil: 'domcontentloaded' }); - const thumbnailUrlsFromNextPage = await getAllThumbnaiUrls(page); - firstPageThumbnailUrls.push(...thumbnailUrlsFromNextPage); - info('image length: ' + firstPageThumbnailUrls.length); - await page.waitFor(exHentai.waitTime); - } - - const images = []; - const targetImgUrls = firstPageThumbnailUrls; - // get thumbnail url in detail page - for (let i = 0; i < targetImgUrls.length; i++) { - await page.goto('https://www.google.com/', { - waitUntil: 'domcontentloaded', - }); - await page.goto(targetImgUrls[i], { waitUntil: 'domcontentloaded' }); - info(`fetching image url => ${targetImgUrls[i]}`); - const imgUrl = await page.$eval( - '[id=i3] img', - (target: any) => - new Promise(resolve => { - resolve(target.src); - }), - ); - images.push(imgUrl); - await page.waitFor(exHentai.waitTime); - } - success('fetch all images'); - // save image url into file, for unexpect error - fs.outputJSON( - path.join( - process.cwd(), - `${exHentai.downloadPath}/${datePath}/${subName}/restDetailUrls.json`, - ), - targetImgUrls, - ).catch((err: any) => { - error('write into json' + err); - }); - fs.outputJSON( - path.join( - process.cwd(), - `${exHentai.downloadPath}/${datePath}/${subName}/detailImageUrls.json`, - ), - images, - ).catch((err: any) => { - error('write into json' + err); - }); - - // fetch and save images - for (let i = 0; i < images.length; i++) { - const item = images[i]; - trace('download begin: ' + item); - await request - .get({ url: item, proxy: exHentai.proxy } as { - url: string; - proxy: string; - }) - .on('error', (err: any) => { - error(err + ' => ' + item); - }) - .pipe( - fs - .createWriteStream( - path.join( - process.cwd(), - `${exHentai.downloadPath}/${datePath}/${subName}/${i + 1}.jpg`, - ), - ) - .on('finish', () => success(`${i + 1}.jpg`)) - .on('error', (err: any) => - error(`${subName}-${i + 1}.jpg failed, ${err}`), - ), - ); - if (i % 4 === 0) { - await page.waitFor(exHentai.waitTime); - } - } - await browser.close(); - ctx.response.body = true; -}; - -module.exports = { - 'GET /': getMainPage, - 'GET /exhentai': getExhentai, - 'POST /exhentai/download': downloadImages, - 'GET /exhentai/getLastestSet': getLastestExHentaiSet, -}; -export {}; diff --git a/server/controller/save.ts b/server/controller/save.ts deleted file mode 100644 index 03c41da7e..000000000 --- a/server/controller/save.ts +++ /dev/null @@ -1,227 +0,0 @@ -import fs from 'fs-extra'; -import path from 'path'; -import md5 from 'blueimp-md5'; -const { updateSider } = require('../../scripts/generateSider'); - -export interface MappingProps { - createTime: number; - modifyTime: number; - id: string; - title: string; - url: string; - type: string; - subType: string; - category: 'mapping' | 'markdown'; -} - -const updateMappingRouter = ( - targetItem: MappingProps, - isDelete?: boolean, - callback?: () => void, -) => { - const writeFilesPaths = [ - `src/assets/mapping.json`, - `dist/assets/mapping.json`, - ]; - const mappingPath = path.join(process.cwd(), writeFilesPaths[0]); - if (!fs.existsSync(mappingPath)) { - fs.writeFileSync(mappingPath, []); - } - const mappingFile = fs.readFileSync(mappingPath); - const result = JSON.parse(mappingFile.toString()); - - const isExistTargetIndex = result.findIndex( - (item: MappingProps) => item.id === targetItem.id, - ); - if (isExistTargetIndex > -1) { - if (isDelete) { - result.splice(isExistTargetIndex, 1); - } else { - result[isExistTargetIndex] = targetItem; - } - } else { - result.push(targetItem); - } - - for (const item of writeFilesPaths) { - fs.outputJSON(path.join(process.cwd(), item), result, { - spaces: 2, - }).catch((err: any) => { - console.error(err); - }); - } - if (callback) { - callback(); - } - // tslint:disable-next-line: no-console - console.log(`mapping updated => ${targetItem.id}`); -}; - -// todo: refactor -const updateTargetMapping = async (ctx: any) => { - const { layout, id, title, url, type, subType, category } = ctx.request.body; - if (!id) { - throw Error('id is undefined.'); - } - const mappingPaths = [ - `src/assets/mapping/${id}.json`, - `dist/assets/mapping/${id}.json`, - ]; - const markdownPaths = [ - `src/assets/markdown/${id}.md`, - `dist/assets/markdown/${id}.md`, - ]; - let targetPaths: string[] = []; - - // update router - const targetFile = fs - .readFileSync(path.join(process.cwd(), 'src/assets/mapping.json')) - .toString(); - const targetJSONFile = JSON.parse(targetFile); - const targetArr = targetJSONFile.filter( - (item: MappingProps) => item.id === id, - ); - const targetItem = targetArr.length > 0 ? targetArr[0] : {}; - const { - createTime, - title: originTitle, - url: originUrl, - type: originType, - subType: originSubType, - category: originCategory, - } = targetItem; - const targetCategory = category || originCategory; - updateMappingRouter({ - id, - title: title || originTitle, - url: url || originUrl, - createTime, - modifyTime: new Date().getTime(), - type: type || originType, - subType: subType || originSubType, - category: targetCategory, - }); - - switch (targetCategory) { - case 'mapping': - targetPaths = mappingPaths; - break; - - case 'markdown': - targetPaths = markdownPaths; - break; - - default: - break; - } - - // update layout - let originLayout = layout; - if (!layout && id) { - const targetLayoutFile = fs.readFileSync( - path.join(path.join(process.cwd(), targetPaths[0])), - ); - const targetLayout = targetLayoutFile.toString(); - try { - originLayout = JSON.parse(targetLayout); - } catch (error) { - originLayout = targetLayout; - } - } - - try { - for (const item of targetPaths) { - if (targetCategory === 'markdown') { - fs.writeFileSync(path.join(process.cwd(), item), originLayout); - } - if (targetCategory === 'mapping') { - fs.outputJSON(path.join(process.cwd(), item), originLayout, { - spaces: 2, - }).catch((err: any) => { - console.error(err); - }); - } - // tslint:disable-next-line: no-console - console.log(`written completed => ${item}`); - } - ctx.response.body = true; - } catch (error) { - ctx.response.body = error.message; - } -}; - -// 1. generate empty file in assets/mapping for mapping info -// 2. update router file -const initNewMapping = async (ctx: any) => { - const { title, type, subType, category } = ctx.request.body; - const dateTime = new Date().getTime(); - const id = md5(dateTime as any); - const mappingPaths = [ - `src/assets/mapping/${id}.json`, - `dist/assets/mapping/${id}.json`, - ]; - const markdownPaths = [ - `src/assets/markdown/${id}.md`, - `dist/assets/markdown/${id}.md`, - ]; - let targetPaths: string[] = []; - - switch (category) { - case 'mapping': - targetPaths = mappingPaths; - break; - - case 'markdown': - targetPaths = markdownPaths; - break; - - default: - break; - } - - try { - // generate empty file in assets/mapping for mapping info - for (const item of targetPaths) { - fs.writeFileSync( - path.join(process.cwd(), item), - `{ - "CanvasPosition": { - "x": -2810, - "y": -3323, - "z": 0, - "gap": 1 - }, - "BlockGroup": {}, - "TagGroup": {}, - "LineGroup": {} - } - `, - ); - } - // update router file - updateMappingRouter( - { - id, - title, - url: `./assets/mapping/${id}.json`, - createTime: dateTime, - modifyTime: dateTime, - type, - subType, - category, - }, - false, - updateSider, - ); - ctx.response.body = id; - } catch (error) { - console.error(error); - ctx.response.body = error.message; - } -}; - -export default { - 'POST /save/update': updateTargetMapping, - 'POST /save/new': initNewMapping, - updateMappingRouter, -}; diff --git a/server/index.ts b/server/index.ts index e49b47160..875733357 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1,14 +1,23 @@ import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; import KoaStatic from 'koa-static'; -import Router from './router'; +import fs from 'fs-extra'; +import path from 'path'; import { info } from './utils/log'; +const initRouter = (targetApp: any) => { + fs.readdirSync(path.join(process.cwd(), '/server/controller')) + .filter((filePath: string) => filePath.endsWith('.js')) + .map((controllerPath: any) => { + const controller = path.join(__dirname, 'controller', controllerPath); + targetApp.use(require(controller).default.routes()); + }); +}; + const app = new Koa(); -const { router } = Router; app.use(bodyParser()); -app.use(router.routes()); +initRouter(app); app.use(KoaStatic(process.cwd() + '/dist')); export default app.listen(9099, () => { diff --git a/server/router.ts b/server/router.ts deleted file mode 100644 index 876347a9d..000000000 --- a/server/router.ts +++ /dev/null @@ -1,44 +0,0 @@ -import Router from 'koa-router'; -import fs from 'fs-extra'; -import path from 'path'; - -const KoaRouter = new Router(); -const cwd = process.cwd(); - -const addMapping = (router: any, mapping: any) => { - // tslint:disable-next-line: forin - for (const url in mapping) { - let routerPath = url.substring(4); - if (url.startsWith('GET ')) { - router.get(routerPath, mapping[url]); - } else if (url.startsWith('POST ')) { - routerPath = url.substring(5); - router.post(routerPath, mapping[url]); - } else if (url.startsWith('DELETE ')) { - routerPath = url.substring(7); - router.delete(routerPath, mapping[url]); - } else if (url.startsWith('PUT ')) { - routerPath = url.substring(4); - router.put(routerPath, mapping[url]); - } else { - console.error(`invalid URL: ${url}`); - } - } -}; - -const addControllers = (router: any) => { - const files = fs.readdirSync(path.join(cwd, '/server/controller')); - const jsFiles = files.filter((f: string) => { - return f.endsWith('.js'); - }); - - for (const file of jsFiles) { - const mapping = require(path.join(cwd, '/server/controller/', file)) - .default; - addMapping(router, mapping); - } -}; - -addControllers(KoaRouter); - -export default { router: KoaRouter }; diff --git a/server/utils/decorator.ts b/server/utils/decorator.ts new file mode 100644 index 000000000..c7ec8102d --- /dev/null +++ b/server/utils/decorator.ts @@ -0,0 +1,36 @@ +import KoaRouter from 'koa-router'; + +export function Controller(prefix: string) { + const router: any = new KoaRouter(); + if (prefix) { + router.prefix(prefix); + } + return function(target: any) { + const reqList = Object.getOwnPropertyDescriptors(target.prototype); + for (const v in reqList) { + if (v !== 'constructor') { + const fn = reqList[v].value; + fn(router); + } + } + // router.prototype.routes = null; + return router; + }; +} + +export function Request({ + url, + method, +}: { + url: string; + method: 'get' | 'post' | 'put' | 'delete'; +}) { + return function(_target: any, _name: string, descriptor: any) { + const fn = descriptor.value; + descriptor.value = (router: any) => { + router[method](url, async (ctx: any, next: any) => { + await fn(ctx, next); + }); + }; + }; +} diff --git a/src/controller/ExHentaiListDataController.tsx b/src/controller/ExHentaiListDataController.tsx index 0c4230182..3172c67bc 100644 --- a/src/controller/ExHentaiListDataController.tsx +++ b/src/controller/ExHentaiListDataController.tsx @@ -1,6 +1,6 @@ import React, { Component } from 'react'; import ExHentaiList, { DownloadProps } from '../pages/ExHentaiList'; -import { ExHentaiInfoItem } from '../../server/controller/get'; +import { ExHentaiInfoItem } from '../../server/controller/ExhentaiController'; import { Empty, message } from 'antd'; export interface ExHentaiListDataControllerState { diff --git a/src/controller/MainPageDataController.tsx b/src/controller/MainPageDataController.tsx index e04dfc7ae..ed09dafa8 100644 --- a/src/controller/MainPageDataController.tsx +++ b/src/controller/MainPageDataController.tsx @@ -1,6 +1,6 @@ import React, { Component } from 'react'; import MainPage from '../pages/MainPage'; -import { MappingProps } from '../../server/controller/save'; +import { MappingProps } from '../../server/controller/MarkdownController'; import { FormProps } from '../pages/EditForm'; import { message } from 'antd'; diff --git a/src/pages/ExHentaiList.tsx b/src/pages/ExHentaiList.tsx index b838cfede..3f82acc3b 100644 --- a/src/pages/ExHentaiList.tsx +++ b/src/pages/ExHentaiList.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { ExHentaiInfoItem } from '../../server/controller/get'; +import { ExHentaiInfoItem } from '../../server/controller/ExhentaiController'; import { Row, Col, Card, Dropdown, Menu } from 'antd'; import LazyLoad from 'react-lazyload'; diff --git a/src/pages/MainPage.tsx b/src/pages/MainPage.tsx index 8f3ec8dee..17caf52f4 100644 --- a/src/pages/MainPage.tsx +++ b/src/pages/MainPage.tsx @@ -4,7 +4,7 @@ import { Dropdown, Menu, Layout, Breadcrumb, List, Icon, Button } from 'antd'; const { SubMenu } = Menu; const { Content, Footer, Sider } = Layout; import { SiderProps } from '../controller/MainPageDataController'; -import { MappingProps } from '../../server/controller/save'; +import { MappingProps } from '../../server/controller/MarkdownController'; import { format } from 'date-fns'; export interface MainPageProps { diff --git a/src/pages/__tests__/EditForm.test.tsx b/src/pages/__tests__/EditForm.test.js similarity index 91% rename from src/pages/__tests__/EditForm.test.tsx rename to src/pages/__tests__/EditForm.test.js index e14bbe58e..4c6631685 100644 --- a/src/pages/__tests__/EditForm.test.tsx +++ b/src/pages/__tests__/EditForm.test.js @@ -2,7 +2,6 @@ import React from 'react'; import { mount } from 'enzyme'; import EditForm from '../EditForm'; import 'nino-cli/scripts/setup'; -import { SiderProps } from '../../controller/MainPageDataController'; const menuData = [ { @@ -33,7 +32,7 @@ describe('EditForm', () => { const wrapper = mount( { const wrapper = mount( { it('render correctly', () => { const wrapper = shallow( - , + , ); expect(wrapper).toMatchSnapshot(); }); @@ -76,7 +71,7 @@ describe('MainPage', () => { const onDelete = jest.fn(); const wrapper = mount(