Chore/CI: use fs cache to save bandwidth

.github/workflows/main.yml

@@ -15,6 +15,12 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
+      - name: Cache cache.db
+        uses: actions/cache@v3
+        with:
+          path: .cache
+          key: ${{ runner.os }}-v1
+
       - uses: oven-sh/setup-bun@v1
         with:
           bun-version: latest

.gitignore

@@ -2,6 +2,7 @@
 node_modules
 .clinic
 .wireit
+.cache
 public
 
 # $ build output

Build/build-anti-bogus-domain.ts

@@ -1,15 +1,15 @@
 // @ts-check
 import path from 'path';
 import { createRuleset } from './lib/create-file';
-import { fetchRemoteTextAndReadByLine, readFileByLine } from './lib/fetch-text-by-line';
+import { fetchRemoteTextByLine, readFileByLine } from './lib/fetch-text-by-line';
 import { processLine } from './lib/process-line';
 import { task } from './lib/trace-runner';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { isProbablyIpv4, isProbablyIpv6 } from './lib/is-fast-ip';
 
 const getBogusNxDomainIPs = async () => {
   const result: string[] = [];
-  for await (const line of await fetchRemoteTextAndReadByLine('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')) {
+  for await (const line of await fetchRemoteTextByLine('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/bogus-nxdomain.china.conf')) {
     if (line && line.startsWith('bogus-nxdomain=')) {
       const ip = line.slice(15).trim();
       if (isProbablyIpv4(ip)) {

Build/build-cdn-conf.ts

@@ -1,27 +1,15 @@
 import path from 'path';
 import { createRuleset } from './lib/create-file';
-import { fetchRemoteTextAndReadByLine, readFileByLine } from './lib/fetch-text-by-line';
+import { readFileByLine } from './lib/fetch-text-by-line';
 import { createTrie } from './lib/trie';
 import { task } from './lib/trace-runner';
 import { processLine } from './lib/process-line';
 import { SHARED_DESCRIPTION } from './lib/constants';
-
-const publicSuffixPath: string = path.resolve(import.meta.dir, '../node_modules/.cache/public_suffix_list_dat.txt');
-
+import { getPublicSuffixListTextPromise } from './download-publicsuffixlist';
 const getS3OSSDomains = async (): Promise<Set<string>> => {
   const trie = createTrie();
-
-  const publicSuffixFile = Bun.file(publicSuffixPath);
-
-  if (await publicSuffixFile.exists()) {
-    for await (const line of readFileByLine(publicSuffixFile)) {
-      trie.add(line);
-    }
-  } else {
-    console.log('public_suffix_list.dat not found, fetch directly from remote.');
-    for await (const line of await fetchRemoteTextAndReadByLine('https://publicsuffix.org/list/public_suffix_list.dat')) {
-      trie.add(line);
-    }
+  for await (const line of (await getPublicSuffixListTextPromise()).split('\n')) {
+    trie.add(line);
   }
 
   /**

Build/build-chn-cidr.ts

@@ -1,4 +1,4 @@
-import { fetchRemoteTextAndReadByLine } from './lib/fetch-text-by-line';
+import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
 import { resolve as pathResolve } from 'path';
 import { compareAndWriteFile, withBannerArray } from './lib/create-file';
 import { processLineFromReadline } from './lib/process-line';
@@ -21,7 +21,7 @@ const INCLUDE_CIDRS = [
 export const getChnCidrPromise = createMemoizedPromise(async () => {
   const cidr = await traceAsync(
     picocolors.gray('download chnroutes2'),
-    async () => processLineFromReadline(await fetchRemoteTextAndReadByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')),
+    async () => processLineFromReadline(await fetchRemoteTextByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt')),
     picocolors.gray
   );
   return traceSync(

Build/build-internal-reverse-chn-cidr.ts

@@ -1,4 +1,4 @@
-import { fetchRemoteTextAndReadByLine } from './lib/fetch-text-by-line';
+import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
 import { processLineFromReadline } from './lib/process-line';
 import path from 'path';
 import { task } from './lib/trace-runner';
@@ -26,7 +26,7 @@ const RESERVED_IPV4_CIDR = [
 ];
 
 export const buildInternalReverseChnCIDR = task(import.meta.path, async () => {
-  const cidr = await processLineFromReadline(await fetchRemoteTextAndReadByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt'));
+  const cidr = await processLineFromReadline(await fetchRemoteTextByLine('https://raw.githubusercontent.com/misakaio/chnroutes2/master/chnroutes.txt'));
 
   const reversedCidr = merge(
     exclude(

Build/build-microsoft-cdn.ts

@@ -1,7 +1,7 @@
 import path from 'path';
 import { task, traceAsync } from './lib/trace-runner';
 import { createRuleset } from './lib/create-file';
-import { fetchRemoteTextAndReadByLine } from './lib/fetch-text-by-line';
+import { fetchRemoteTextByLine } from './lib/fetch-text-by-line';
 import { createTrie } from './lib/trie';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { createMemoizedPromise } from './lib/memo-promise';
@@ -22,7 +22,7 @@ const BLACKLIST = [
 export const getMicrosoftCdnRulesetPromise = createMemoizedPromise(async () => {
   const set = await traceAsync('fetch accelerated-domains.china.conf', async () => {
     const trie = createTrie();
-    for await (const line of await fetchRemoteTextAndReadByLine('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')) {
+    for await (const line of await fetchRemoteTextByLine('https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf')) {
       if (line.startsWith('server=/') && line.endsWith('/114.114.114.114')) {
         const domain = line.slice(8, -16);
         trie.add(domain);

Build/build-reject-domainset.ts

@@ -32,16 +32,16 @@ export const buildRejectDomainSet = task(import.meta.path, async () => {
     const [gorhill] = await Promise.all([
       getGorhillPublicSuffixPromise(),
       // Parse from remote hosts & domain lists
-      ...HOSTS.map(entry => processHosts(entry[0], entry[1]).then(hosts => {
+      ...HOSTS.map(entry => processHosts(entry[0], entry[1], entry[2], entry[3]).then(hosts => {
         hosts.forEach(host => {
           domainSets.add(host);
         });
       })),
-      ...DOMAIN_LISTS.map(entry => processDomainLists(entry[0], entry[1])),
+      ...DOMAIN_LISTS.map(entry => processDomainLists(entry[0], entry[1], entry[2])),
       ...ADGUARD_FILTERS.map(input => {
         const promise = typeof input === 'string'
           ? processFilterRules(input)
-          : processFilterRules(input[0], input[1]);
+          : processFilterRules(input[0], input[1], input[2]);
 
         return promise.then(({ white, black, foundDebugDomain }) => {
           if (foundDebugDomain) {

Build/build-speedtest-domainset.ts

@@ -9,6 +9,7 @@ import { task } from './lib/trace-runner';
 import { fetchWithRetry } from './lib/fetch-retry';
 import { SHARED_DESCRIPTION } from './lib/constants';
 import { getGorhillPublicSuffixPromise } from './lib/get-gorhill-publicsuffix';
+import { fsCache } from './lib/cache-filesystem';
 
 const s = new Sema(3);
 
@@ -21,32 +22,42 @@ const querySpeedtestApi = async (keyword: string): Promise<Array<string | null>>
     s.acquire()
   ]))[0];
 
-  const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)];
-
   try {
+    const randomUserAgent = topUserAgents[Math.floor(Math.random() * topUserAgents.length)];
     const key = `fetch speedtest endpoints: ${keyword}`;
     console.time(key);
 
-    const res = await fetchWithRetry(`https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`, {
-      headers: {
-        dnt: '1',
-        Referer: 'https://www.speedtest.net/',
-        accept: 'application/json, text/plain, */*',
-        'User-Agent': randomUserAgent,
-        'Accept-Language': 'en-US,en;q=0.9',
-        'Sec-Ch-Ua': '"Not/A)Brand";v="99", "Google Chrome";v="115", "Chromium";v="115"',
-        'Sec-Ch-Ua-Mobile': '?0',
-        'Sec-Fetch-Dest': 'empty',
-        'Sec-Fetch-Mode': 'cors',
-        'Sec-Fetch-Site': 'same-origin',
-        'Sec-Gpc': '1'
+    const url = `https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`;
+    const json = await fsCache.apply<Array<{ url: string }>>(
+      url,
+      async () => {
+        const res = await fetchWithRetry(`https://www.speedtest.net/api/js/servers?engine=js&search=${keyword}&limit=100`, {
+          headers: {
+            dnt: '1',
+            Referer: 'https://www.speedtest.net/',
+            accept: 'application/json, text/plain, */*',
+            'User-Agent': randomUserAgent,
+            'Accept-Language': 'en-US,en;q=0.9',
+            'Sec-Ch-Ua': '"Not/A)Brand";v="99", "Google Chrome";v="115", "Chromium";v="115"',
+            'Sec-Ch-Ua-Mobile': '?0',
+            'Sec-Fetch-Dest': 'empty',
+            'Sec-Fetch-Mode': 'cors',
+            'Sec-Fetch-Site': 'same-origin',
+            'Sec-Gpc': '1'
+          }
+        });
+        if (!res.ok) {
+          throw new Error(`${res.statusText}\n${await res.text()}`);
+        }
+        return res.json() as Promise<Array<{ url: string }>>;
+      },
+      {
+        ttl: 24 * 60 * 60 * 1000,
+        serializer: JSON.stringify,
+        deserializer: JSON.parse
       }
-    });
-    if (!res.ok) {
-      throw new Error(`${res.statusText}\n${await res.text()}`);
-    }
+    );
 
-    const json = await res.json() as Array<{ url: string }>;
     s.release();
 
     console.timeEnd(key);

Build/download-previous-build.ts

@@ -1,7 +1,6 @@
 import fs from 'fs';
 import fsp from 'fs/promises';
 import path from 'path';
-import os from 'os';
 import { Readable } from 'stream';
 import { pipeline } from 'stream/promises';
 import { readFileByLine } from './lib/fetch-text-by-line';
@@ -85,16 +84,6 @@ export const downloadPreviousBuild = task(import.meta.path, async () => {
   );
 });
 
-export const downloadPublicSuffixList = task(import.meta.path, async () => {
-  const publicSuffixPath = path.resolve(import.meta.dir, '../node_modules/.cache/public_suffix_list_dat.txt');
-  const resp = await fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat', defaultRequestInit);
-
-  return Bun.write(publicSuffixPath, resp as Response);
-}, 'download-publicsuffixlist');
-
 if (import.meta.main) {
-  Promise.all([
-    downloadPreviousBuild(),
-    downloadPublicSuffixList()
-  ]);
+  downloadPreviousBuild();
 }

Build/download-publicsuffixlist.ts

@@ -0,0 +1,10 @@
+import { fsCache } from './lib/cache-filesystem';
+import { defaultRequestInit, fetchWithRetry } from './lib/fetch-retry';
+import { createMemoizedPromise } from './lib/memo-promise';
+import { traceAsync } from './lib/trace-runner';
+
+export const getPublicSuffixListTextPromise = createMemoizedPromise(() => traceAsync('obtain public_suffix_list', () => fsCache.apply(
+  'public_suffix_list.dat',
+  () => fetchWithRetry('https://publicsuffix.org/list/public_suffix_list.dat', defaultRequestInit).then(r => r.text()),
+  { ttl: 24 * 60 * 60 * 1000 }
+)));

Build/index.ts

@@ -1,4 +1,4 @@
-import { downloadPreviousBuild, downloadPublicSuffixList } from './download-previous-build';
+import { downloadPreviousBuild } from './download-previous-build';
 import { buildCommon } from './build-common';
 import { buildAntiBogusDomain } from './build-anti-bogus-domain';
 import { buildAppleCdn } from './build-apple-cdn';
@@ -33,23 +33,15 @@ import type { TaskResult } from './lib/trace-runner';
     // const buildInternalReverseChnCIDRWorker = new Worker(new URL('./workers/build-internal-reverse-chn-cidr-worker.ts', import.meta.url));
 
     const downloadPreviousBuildPromise = downloadPreviousBuild();
-    const downloadPublicSuffixListPromise = downloadPublicSuffixList();
     const buildCommonPromise = downloadPreviousBuildPromise.then(() => buildCommon());
     const buildAntiBogusDomainPromise = downloadPreviousBuildPromise.then(() => buildAntiBogusDomain());
     const buildAppleCdnPromise = downloadPreviousBuildPromise.then(() => buildAppleCdn());
-    const buildCdnConfPromise = Promise.all([
-      downloadPreviousBuildPromise,
-      downloadPublicSuffixListPromise
-    ]).then(() => buildCdnConf());
-    const buildRejectDomainSetPromise = Promise.all([
-      downloadPreviousBuildPromise,
-      downloadPublicSuffixListPromise
-    ]).then(() => buildRejectDomainSet());
+    const buildCdnConfPromise = downloadPreviousBuildPromise.then(() => buildCdnConf());
+    const buildRejectDomainSetPromise = downloadPreviousBuildPromise.then(() => buildRejectDomainSet());
     const buildTelegramCIDRPromise = downloadPreviousBuildPromise.then(() => buildTelegramCIDR());
     const buildChnCidrPromise = downloadPreviousBuildPromise.then(() => buildChnCidr());
     const buildSpeedtestDomainSetPromise = downloadPreviousBuildPromise.then(() => buildSpeedtestDomainSet());
     const buildInternalCDNDomainsPromise = Promise.all([
-      downloadPublicSuffixListPromise,
       buildCommonPromise,
       buildCdnConfPromise
     ]).then(() => buildInternalCDNDomains());
@@ -84,7 +76,6 @@ import type { TaskResult } from './lib/trace-runner';
 
     const stats = await Promise.all([
       downloadPreviousBuildPromise,
-      downloadPublicSuffixListPromise,
       buildCommonPromise,
       buildAntiBogusDomainPromise,
       buildAppleCdnPromise,