From acdbd11e93be59a3976a782596826edab0cb8bfc Mon Sep 17 00:00:00 2001 From: Tsuyumi <40047364+SnowAgar25@users.noreply.github.com> Date: Thu, 26 Sep 2024 01:14:26 +0800 Subject: [PATCH] feat(route/chikubi): refactor URL fetching and add new routes (#16758) * refactor(route/chikubi): switch to native /feed for article URL fetching * feat: add new routes * refactor: remove decodeURIComponent * fix: Handle unescaped characters in tag URLs * chore: use '@/utils/rss-parser' instead of 'rss-parser' * chore: update description * feat(utils): add getPostById and getPostsByIdList for REST API content fetching * refactor(search): use getPostsByIdList instead of processItems * fix: Handle unescaped characters * refactor: Separate index and navigation * feat: optimize post fetching by id and add category & tag functions * refactor: use api to get posts for tags and categories * chore: directly use url * fix: use API to fetch 'best' * fix: destructure post object in map function * feat: optimize getPosts function for reusability * refactor: remove wrapper functions for getBySlug and getPostsBy --- lib/routes/chikubi/category.ts | 41 +++++++++ lib/routes/chikubi/index.ts | 91 +++++++++---------- lib/routes/chikubi/namespace.ts | 9 ++ lib/routes/chikubi/navigation.ts | 66 ++++++++++++++ lib/routes/chikubi/nipple-video-category.ts | 49 ++++++++++ lib/routes/chikubi/nipple-video-maker.ts | 49 ++++++++++ lib/routes/chikubi/search.ts | 39 ++++++++ lib/routes/chikubi/tag.ts | 41 +++++++++ lib/routes/chikubi/utils.ts | 99 ++++++++++++++++----- 9 files changed, 411 insertions(+), 73 deletions(-) create mode 100644 lib/routes/chikubi/category.ts create mode 100644 lib/routes/chikubi/navigation.ts create mode 100644 lib/routes/chikubi/nipple-video-category.ts create mode 100644 lib/routes/chikubi/nipple-video-maker.ts create mode 100644 lib/routes/chikubi/search.ts create mode 100644 lib/routes/chikubi/tag.ts diff --git a/lib/routes/chikubi/category.ts b/lib/routes/chikubi/category.ts new file mode 100644 index 00000000000000..b640b94f121159 --- /dev/null +++ b/lib/routes/chikubi/category.ts @@ -0,0 +1,41 @@ +import { Route, Data } from '@/types'; +import { getBySlug, getPostsBy } from './utils'; + +export const route: Route = { + path: '/category/:keyword', + categories: ['multimedia'], + example: '/chikubi/category/nipple-lesbian', + parameters: { keyword: 'Keyword' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'Category', + maintainers: ['SnowAgar25'], + handler, + radar: [ + { + title: 'Category', + source: ['chikubi.jp/category/:keyword'], + target: '/category/:keyword', + }, + ], +}; + +async function handler(ctx): Promise { + const baseUrl = 'https://chikubi.jp'; + const { keyword } = ctx.req.param(); + const { id, name } = await getBySlug('category', keyword); + + const items = await getPostsBy('category', id); + + return { + title: `Category: ${name} - chikubi.jp`, + link: `${baseUrl}/category/${keyword}`, + item: items, + }; +} diff --git a/lib/routes/chikubi/index.ts b/lib/routes/chikubi/index.ts index 41ef3cbc60a2a5..444d2fe0da288e 100644 --- a/lib/routes/chikubi/index.ts +++ b/lib/routes/chikubi/index.ts @@ -1,13 +1,10 @@ import { Route, Data } from '@/types'; -import got from '@/utils/got'; -import { load } from 'cheerio'; -import { processItems } from './utils'; +import { getPosts } from './utils'; export const route: Route = { - path: '/:category?', + path: '/', categories: ['multimedia'], example: '/chikubi', - parameters: { category: '分類，見下表，默認爲最新' }, features: { requireConfig: false, requirePuppeteer: false, @@ -16,60 +13,54 @@ export const route: Route = { supportPodcast: false, supportScihub: false, }, - name: 'Category', - maintainers: ['snowagar25'], + name: '最新記事', + maintainers: ['SnowAgar25'], handler, - description: `| 最新 | 殿堂 | 動畫 | VR | 漫畫 | 音聲 | CG | - | ------ | ---- | ----- | -- | ----- | ----- | -- | - | (empty) | best | video | vr | comic | voice | cg |`, radar: [ { - source: ['chikubi.jp/:category', 'chikubi.jp/'], - target: '/:category', + title: '最新記事', + source: ['chikubi.jp/'], + target: '/', + }, + { + title: '殿堂', + source: ['chikubi.jp/best-nipple-article'], + target: '/best', + }, + { + title: '動畫', + source: ['chikubi.jp/nipple-video'], + target: '/video', + }, + { + title: 'VR', + source: ['chikubi.jp/nipple-video-category/cat-nipple-video-vr'], + target: '/vr', + }, + { + title: '漫畫', + source: ['chikubi.jp/comic'], + target: '/comic', + }, + { + title: '音聲', + source: ['chikubi.jp/voice'], + target: '/voice', + }, + { + title: 'CG・イラスト', + source: ['chikubi.jp/cg'], + target: '/cg', }, ], }; -const categoryMap = { - '': { url: '/page/1', title: '最新', selector: '.article_list_area > article > a' }, - best: { url: '/best-nipple-article', title: '殿堂', selector: '.article-list:first .title > a' }, - video: { url: '/nipple-video', title: '動畫', selector: 'ul.video_list > li > a' }, - vr: { url: '/nipple-video-category/cat-nipple-video-vr', title: 'VR', selector: 'ul.video_list > li > a' }, - comic: { url: '/comic', title: '漫畫', selector: '.section:nth-of-type(2) .list-doujin .photo a' }, - voice: { url: '/voice', title: '音聲', selector: 'ul.list-doujin > li > .photo > a' }, - cg: { url: '/cg', title: 'CG', selector: 'ul.list-doujin > li > .photo > a' }, -}; - -async function handler(ctx): Promise { - const category = ctx.req.param('category') ?? ''; - const baseUrl = 'https://chikubi.jp'; - - const { url, title, selector } = categoryMap[category]; - - const response = await got(`${baseUrl}${url}`); - const $ = load(response.data); - - let list = $(selector) - .toArray() - .map((item) => { - const $item = $(item); - return { - title: $item.text().trim(), - link: new URL($item.attr('href') ?? '', baseUrl).href, - }; - }); - - // 限制殿堂最多獲取30個 - if (category === 'best') { - list = list.slice(0, 30); - } - - // 獲取內文 - const items = await processItems(list); +async function handler(): Promise { + const items = await getPosts(); return { - title: `${title} - chikubi.jp`, - link: `${baseUrl}${url}`, + title: '最新記事 - chikubi.jp', + link: 'https://chikubi.jp', item: items, }; } diff --git a/lib/routes/chikubi/namespace.ts b/lib/routes/chikubi/namespace.ts index b63d4cd211799d..08cf95753d2b37 100644 --- a/lib/routes/chikubi/namespace.ts +++ b/lib/routes/chikubi/namespace.ts @@ -3,4 +3,13 @@ import type { Namespace } from '@/types'; export const namespace: Namespace = { name: '乳首ふぇち', url: 'chikubi.jp', + description: `:::tip +The content of 乳首ふぇち is divided into two parts: + +Works: Only reposts official product descriptions. +Posts: Contains the website author's thoughts and additional information. + +Sometimes a product may exist in both posts and works. +Sometimes there might be only a single post without any reposted work, and vice versa. +:::`, }; diff --git a/lib/routes/chikubi/navigation.ts b/lib/routes/chikubi/navigation.ts new file mode 100644 index 00000000000000..09015ea56837d1 --- /dev/null +++ b/lib/routes/chikubi/navigation.ts @@ -0,0 +1,66 @@ +import { Route, Data } from '@/types'; +import { getBySlug, getPostsBy, processItems } from './utils'; +import parser from '@/utils/rss-parser'; + +export const route: Route = { + path: '/:keyword', + categories: ['multimedia'], + example: '/chikubi', + parameters: { keyword: '導覽列，見下表，默認爲最新' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'Navigation', + maintainers: ['SnowAgar25'], + handler, + description: `| 殿堂 | 動畫 | VR | 漫畫 | 音聲 | CG・イラスト | + | ---- | ----- | -- | ----- | ----- | -- | + | best | video | vr | comic | voice | cg |`, +}; + +const navigationItems = { + video: { url: '/nipple-video', title: '動畫' }, + vr: { url: '/nipple-video-category/cat-nipple-video-vr', title: 'VR' }, + comic: { url: '/comic', title: '漫畫' }, + voice: { url: '/voice', title: '音聲' }, + cg: { url: '/cg', title: 'CG' }, +}; + +async function handler(ctx): Promise { + const keyword = ctx.req.param('keyword') ?? ''; + const baseUrl = 'https://chikubi.jp'; + + if (keyword === 'best') { + const { id } = await getBySlug('category', 'nipple-best'); + const items = await getPostsBy('category', id); + + return { + title: '殿堂 - chikubi.jp', + link: `${baseUrl}/best-nipple-article`, + item: items, + }; + } else { + const { url, title } = navigationItems[keyword]; + + const feed = await parser.parseURL(`${baseUrl}${url}/feed`); + + const list = feed.items.map((item) => ({ + title: item.title, + link: item.link, + })); + + // 獲取內文 + const items = await processItems(list); + + return { + title: `${title} - chikubi.jp`, + link: `${baseUrl}${url}`, + item: items, + }; + } +} diff --git a/lib/routes/chikubi/nipple-video-category.ts b/lib/routes/chikubi/nipple-video-category.ts new file mode 100644 index 00000000000000..02042a34ad2212 --- /dev/null +++ b/lib/routes/chikubi/nipple-video-category.ts @@ -0,0 +1,49 @@ +import { Route, Data } from '@/types'; +import { processItems } from './utils'; +import parser from '@/utils/rss-parser'; + +export const route: Route = { + path: '/nipple-video-category/:keyword', + categories: ['multimedia'], + example: '/chikubi/nipple-video-category/cat-nipple-video-god', + parameters: { keyword: 'Keyword' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: '動画カテゴリー', + maintainers: ['SnowAgar25'], + handler, + radar: [ + { + title: '動画カテゴリー', + source: ['chikubi.jp/nipple-video-category/:keyword'], + target: '/nipple-video-category/:keyword', + }, + ], +}; + +async function handler(ctx): Promise { + const { keyword } = ctx.req.param(); + const baseUrl = 'https://chikubi.jp'; + const url = `/nipple-video-category/${encodeURIComponent(keyword)}`; + + const feed = await parser.parseURL(`${baseUrl}${url}/feed`); + + const list = feed.items.map((item) => ({ + title: item.title, + link: item.link, + })); + + const items = await processItems(list); + + return { + title: `動画カテゴリー: ${feed.title?.split('-')[0]} - chikubi.jp`, + link: `${baseUrl}${url}`, + item: items, + }; +} diff --git a/lib/routes/chikubi/nipple-video-maker.ts b/lib/routes/chikubi/nipple-video-maker.ts new file mode 100644 index 00000000000000..d563c8cad8e58b --- /dev/null +++ b/lib/routes/chikubi/nipple-video-maker.ts @@ -0,0 +1,49 @@ +import { Route, Data } from '@/types'; +import { processItems } from './utils'; +import parser from '@/utils/rss-parser'; + +export const route: Route = { + path: '/nipple-video-maker/:keyword', + categories: ['multimedia'], + example: '/chikubi/nipple-video-maker/nipple-video-maker-nh', + parameters: { keyword: 'Keyword' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'AVメーカー', + maintainers: ['SnowAgar25'], + handler, + radar: [ + { + title: 'AVメーカー', + source: ['chikubi.jp/nipple-video-maker/:keyword'], + target: '/nipple-video-maker/:keyword', + }, + ], +}; + +async function handler(ctx): Promise { + const { keyword } = ctx.req.param(); + const baseUrl = 'https://chikubi.jp'; + const url = `/nipple-video-maker/${encodeURIComponent(keyword)}`; + + const feed = await parser.parseURL(`${baseUrl}${url}/feed`); + + const list = feed.items.map((item) => ({ + title: item.title, + link: item.link, + })); + + const items = await processItems(list); + + return { + title: `AVメーカー: ${feed.title?.split('-')[0]} - chikubi.jp`, + link: `${baseUrl}${url}`, + item: items, + }; +} diff --git a/lib/routes/chikubi/search.ts b/lib/routes/chikubi/search.ts new file mode 100644 index 00000000000000..57519077bca051 --- /dev/null +++ b/lib/routes/chikubi/search.ts @@ -0,0 +1,39 @@ +import { Route, Data } from '@/types'; +import { getPosts } from './utils'; +import got from '@/utils/got'; + +export const route: Route = { + path: '/search/:keyword', + categories: ['multimedia'], + example: '/chikubi/search/ギャップ', + parameters: { keyword: 'Keyword' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'Search', + maintainers: ['SnowAgar25'], + handler, +}; + +async function handler(ctx): Promise { + const { keyword } = ctx.req.param(); + const baseUrl = 'https://chikubi.jp'; + const searchUrl = `${baseUrl}/wp-json/wp/v2/search?search=${keyword}`; + + const response = await got.get(searchUrl); + const searchResults = response.data; + + const postIds = searchResults.map((item) => item.id.toString()); + const items = await getPosts(postIds); + + return { + title: `Search: ${keyword} - chikubi.jp`, + link: `${baseUrl}/search/${encodeURIComponent(keyword)}`, + item: items, + }; +} diff --git a/lib/routes/chikubi/tag.ts b/lib/routes/chikubi/tag.ts new file mode 100644 index 00000000000000..148514eb2f3e48 --- /dev/null +++ b/lib/routes/chikubi/tag.ts @@ -0,0 +1,41 @@ +import { Route, Data } from '@/types'; +import { getBySlug, getPostsBy } from './utils'; + +export const route: Route = { + path: '/tag/:keyword', + categories: ['multimedia'], + example: '/chikubi/tag/ドリームチケット', + parameters: { keyword: 'Keyword' }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'Tag', + maintainers: ['SnowAgar25'], + handler, + radar: [ + { + title: 'Tag', + source: ['chikubi.jp/tag/:keyword'], + target: '/tag/:keyword', + }, + ], +}; + +async function handler(ctx): Promise { + const baseUrl = 'https://chikubi.jp'; + const { keyword } = ctx.req.param(); + const { id, name } = await getBySlug('tag', keyword); + + const items = await getPostsBy('tag', id); + + return { + title: `Tag: ${name} - chikubi.jp`, + link: `${baseUrl}/category/${keyword}`, + item: items, + }; +} diff --git a/lib/routes/chikubi/utils.ts b/lib/routes/chikubi/utils.ts index 59321c768e700e..cc5d3a1ce0914d 100644 --- a/lib/routes/chikubi/utils.ts +++ b/lib/routes/chikubi/utils.ts @@ -4,17 +4,7 @@ import got from '@/utils/got'; import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; -interface ListItem { - title: string; - link: string; -} - -interface ContentSelectors { - title: string; - description: string[]; -} - -const contentTypes: Record = { +const CONTENT_TYPES = { doujin: { title: '.doujin-title', description: ['.doujin-detail', '.section', '.area-buy > a.btn'], @@ -29,7 +19,7 @@ const contentTypes: Record = { }, }; -function getContentType(link: string): keyof typeof contentTypes { +function getContentType(link: string): keyof typeof CONTENT_TYPES { const typePatterns = { doujin: ['/cg/', '/comic/', '/voice/'], video: ['/nipple-video/'], @@ -38,14 +28,14 @@ function getContentType(link: string): keyof typeof contentTypes { for (const [type, patterns] of Object.entries(typePatterns)) { if (patterns.some((pattern) => link.includes(pattern))) { - return type as keyof typeof contentTypes; + return type as keyof typeof CONTENT_TYPES; } } throw new Error(`Unknown content type for link: ${link}`); } -export async function processItems(list: ListItem[]): Promise { +export async function processItems(list): Promise { const items = await Promise.all( list.map((item) => cache.tryGet(item.link, async () => { @@ -53,17 +43,10 @@ export async function processItems(list: ListItem[]): Promise { const $ = load(detailResponse.data); const contentType = getContentType(item.link); - const selectors = contentTypes[contentType]; + const selectors = CONTENT_TYPES[contentType]; const title = $(selectors.title).text().trim() || item.title; - const description = selectors.description - .map((selector) => - $(selector) - .map((_, el) => $(el).clone().wrap('
').parent().html()) - .toArray() - .join('') - ) - .join(''); + const description = processDescription(selectors.description.map((selector) => $(selector).prop('outerHTML')).join('')); const pubDateStr = $('meta[property="article:published_time"]').attr('content'); const pubDate = pubDateStr ? parseDate(pubDateStr) : undefined; @@ -80,3 +63,73 @@ export async function processItems(list: ListItem[]): Promise { return items.filter((item): item is DataItem => item !== null); } + +function processDescription(description: string): string { + const $ = load(description); + return $('body') + .children() + .map((_, el) => $(el).clone().wrap('
').parent().html()) + .toArray() + .join(''); +} + +const WP_REST_API_URL = 'https://chikubi.jp/wp-json/wp/v2'; + +export async function getPosts(ids?: string[]): Promise { + const url = `${WP_REST_API_URL}/posts${ids?.length ? `?include=${ids.join(',')}` : ''}`; + + const cachedData = await cache.tryGet(url, async () => { + const response = await got(url); + const data = JSON.parse(response.body); + + if (!Array.isArray(data)) { + throw new TypeError('No posts found for the given IDs'); + } + + return data.map(({ title, link, date, content }) => ({ + title: title.rendered, + link, + pubDate: parseDate(date), + description: processDescription(content.rendered), + })); + }); + + return (Array.isArray(cachedData) ? cachedData : []).filter((item): item is DataItem => item !== null); +} + +const API_TYPES = { + tag: 'tags', + category: 'categories', +}; + +export async function getBySlug(type: T, slug: string): Promise<{ id: number; name: string }> { + const url = `${WP_REST_API_URL}/${API_TYPES[type]}?slug=${encodeURIComponent(slug)}`; + const { body } = await got(url); + const data = JSON.parse(body); + + if (data?.[0]) { + const { id, name } = data[0]; + return { id, name }; + } + throw new Error(`No ${type} found for slug: ${slug}`); +} + +export async function getPostsBy(type: T, id: number): Promise { + const url = `${WP_REST_API_URL}/posts?${API_TYPES[type]}=${id}`; + const cachedData = await cache.tryGet(url, async () => { + const { body } = await got(url); + const data = JSON.parse(body); + + if (Array.isArray(data) && data.length > 0) { + return data.map(({ title, link, date, content }) => ({ + title: title.rendered, + link, + pubDate: parseDate(date), + description: processDescription(content.rendered), + })); + } + return []; + }); + + return (Array.isArray(cachedData) ? cachedData : []).filter((item): item is DataItem => item !== null); +}