diff --git a/app/common/constants.ts b/app/common/constants.ts index f77c1515..4271813c 100644 --- a/app/common/constants.ts +++ b/app/common/constants.ts @@ -1,3 +1,13 @@ export const BUG_VERSIONS = 'bug-versions'; export const LATEST_TAG = 'latest'; export const GLOBAL_WORKER = 'GLOBAL_WORKER'; +export enum SyncMode { + none = 'none', + exist = 'exist', + all = 'all', +} +export enum SyncDeleteMode { + ignore = 'ignore', + block = 'block', + delete = 'delete', +} diff --git a/app/core/service/PackageSyncerService.ts b/app/core/service/PackageSyncerService.ts index 9fe90a5b..9e700f78 100644 --- a/app/core/service/PackageSyncerService.ts +++ b/app/core/service/PackageSyncerService.ts @@ -10,7 +10,8 @@ import { } from 'egg'; import { setTimeout } from 'timers/promises'; import { rm } from 'fs/promises'; -import { NPMRegistry } from '../../common/adapter/NPMRegistry'; +import semver from 'semver'; +import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry'; import { detectInstallScript, getScopeAndName } from '../../common/PackageUtil'; import { downloadToTempfile } from '../../common/FileUtil'; import { TaskState, TaskType } from '../../common/enum/Task'; @@ -31,6 +32,16 @@ import { Registry } from '../entity/Registry'; import { BadRequestError } from 'egg-errors'; import { ScopeManagerService } from './ScopeManagerService'; import { EventCorkAdvice } from './EventCorkerAdvice'; +import { SyncDeleteMode } from '../../common/constants'; + +type syncDeletePkgOptions = { + task: Task, + pkg: Package | null, + logUrl: string, + url: string, + logs: string[], + data: any, +}; function isoNow() { return new Date().toISOString(); @@ -209,6 +220,89 @@ export class PackageSyncerService extends AbstractService { await this.taskService.appendTaskLog(task, logs.join('\n')); } + private isRemovedInRemote(remoteFetchResult: RegistryResponse) { + const { status, data } = remoteFetchResult; + + // deleted or blocked + if (status === 404 || status === 451) { + return true; + } + + const hasMaintainers = data?.maintainers && data?.maintainers.length !== 0; + if (hasMaintainers) { + return false; + } + + // unpublished + const timeMap = data.time || {}; + if (timeMap.unpublished) { + return true; + } + + // security holder + // test/fixtures/registry.npmjs.org/security-holding-package.json + let isSecurityHolder = true; + for (const versionInfo of Object.entries<{ _npmUser?: { name: string } }>(data.versions || {})) { + const [ v, info ] = versionInfo; + // >=0.0.1-security <0.0.2-0 + const isSecurityVersion = semver.satisfies(v, '^0.0.1-security'); + const isNpmUser = info?._npmUser?.name === 'npm'; + if (!isSecurityVersion || !isNpmUser) { + isSecurityHolder = false; + break; + } + } + + return isSecurityHolder; + } + + // sync deleted package, deps on the syncDeleteMode + // - ignore: do nothing, just finish the task + // - delete: remove the package from local registry + // - block: block the package, update the manifest.block, instead of delete versions + // 根据 syncDeleteMode 配置，处理删包场景 + // - ignore: 不做任何处理，直接结束任务 + // - delete: 删除包数据，包括 manifest 存储 + // - block: 软删除 将包标记为 block，用户无法直接使用 + private async syncDeletePkg({ task, pkg, logUrl, url, logs, data }: syncDeletePkgOptions) { + const fullname = task.targetName; + const failEnd = `❌❌❌❌❌ ${url || fullname} ❌❌❌❌❌`; + const syncDeleteMode: SyncDeleteMode = this.config.cnpmcore.syncDeleteMode; + logs.push(`[${isoNow()}] 🟢 Package "${fullname}" was removed in remote registry, response data: ${JSON.stringify(data)}, config.syncDeleteMode = ${syncDeleteMode}`); + + // pkg not exists in local registry + if (!pkg) { + task.error = `Package not exists, response data: ${JSON.stringify(data)}`; + logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`); + logs.push(`[${isoNow()}] ${failEnd}`); + await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); + this.logger.info('[PackageSyncerService.executeTask:fail-404] taskId: %s, targetName: %s, %s', + task.taskId, task.targetName, task.error); + return; + } + + if (syncDeleteMode === SyncDeleteMode.ignore) { + // ignore deleted package + logs.push(`[${isoNow()}] 🟢 Skip remove since config.syncDeleteMode = ignore`); + } else if (syncDeleteMode === SyncDeleteMode.block) { + // block deleted package + await this.packageManagerService.blockPackage(pkg, 'Removed in remote registry'); + logs.push(`[${isoNow()}] 🟢 Block the package since config.syncDeleteMode = block`); + } else if (syncDeleteMode === SyncDeleteMode.delete) { + // delete package + await this.packageManagerService.unpublishPackage(pkg); + logs.push(`[${isoNow()}] 🟢 Delete the package since config.syncDeleteMode = delete`); + } + + // update log + logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`); + logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`); + await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); + this.logger.info('[PackageSyncerService.executeTask:remove-package] taskId: %s, targetName: %s', + task.taskId, task.targetName); + + } + // 初始化对应的 Registry // 1. 优先从 pkg.registryId 获取 (registryId 一经设置 不应改变) // 1. 其次从 task.data.registryId (创建单包同步任务时传入) @@ -314,11 +408,11 @@ export class PackageSyncerService extends AbstractService { return; } - let result: any; + let registryFetchResult: RegistryResponse; try { - result = await this.npmRegistry.getFullManifests(fullname); + registryFetchResult = await this.npmRegistry.getFullManifests(fullname); } catch (err: any) { - const status = err.status || 'unknow'; + const status = err.status || 'unknown'; task.error = `request manifests error: ${err}, status: ${status}`; logs.push(`[${isoNow()}] ❌ Synced ${fullname} fail, ${task.error}, log: ${logUrl}`); logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`); @@ -328,7 +422,7 @@ export class PackageSyncerService extends AbstractService { return; } - const { url, data, headers, res, status } = result; + const { url, data, headers, res, status } = registryFetchResult; let readme = data.readme || ''; if (typeof readme !== 'string') { readme = JSON.stringify(readme); @@ -342,33 +436,15 @@ export class PackageSyncerService extends AbstractService { const contentLength = headers['content-length'] || '-'; logs.push(`[${isoNow()}] HTTP [${status}] content-length: ${contentLength}, timing: ${JSON.stringify(res.timing)}`); - // 404 unpublished - // 451 blocked - const shouldRemovePkg = status === 404 || status === 451; - if (shouldRemovePkg) { - if (pkg) { - await this.packageManagerService.unpublishPackage(pkg); - logs.push(`[${isoNow()}] 🟢 Package "${fullname}" was unpublished caused by ${status} response: ${JSON.stringify(data)}`); - logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`); - logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`); - await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); - this.logger.info('[PackageSyncerService.executeTask:remove-package] taskId: %s, targetName: %s', - task.taskId, task.targetName); - } else { - task.error = `Package not exists, response data: ${JSON.stringify(data)}`; - logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${logUrl}`); - logs.push(`[${isoNow()}] ${failEnd}`); - await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); - this.logger.info('[PackageSyncerService.executeTask:fail-404] taskId: %s, targetName: %s, %s', - task.taskId, task.targetName, task.error); - } + if (this.isRemovedInRemote(registryFetchResult)) { + await this.syncDeletePkg({ task, pkg, logs, logUrl, url, data }); return; } const versionMap = data.versions || {}; const distTags = data['dist-tags'] || {}; - // show latest infomations + // show latest information if (distTags.latest) { logs.push(`[${isoNow()}] 📖 ${fullname} latest version: ${distTags.latest ?? '-'}, published time: ${JSON.stringify(timeMap[distTags.latest])}`); } @@ -432,20 +508,6 @@ export class PackageSyncerService extends AbstractService { // } // } // } - if (timeMap.unpublished) { - if (pkg) { - await this.packageManagerService.unpublishPackage(pkg); - logs.push(`[${isoNow()}] 🟢 Sync unpublished package: ${JSON.stringify(timeMap.unpublished)} success`); - } else { - logs.push(`[${isoNow()}] 📖 Ignore unpublished package: ${JSON.stringify(timeMap.unpublished)}`); - } - logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`); - logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 ${url} 🟢🟢🟢🟢🟢`); - await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); - this.logger.info('[PackageSyncerService.executeTask:success] taskId: %s, targetName: %s', - task.taskId, task.targetName); - return; - } // invalid maintainers, sync fail task.error = `invalid maintainers: ${JSON.stringify(maintainers)}`; diff --git a/app/infra/NFSClientAdapter.ts b/app/infra/NFSClientAdapter.ts index 9b8517cc..bfeaf50d 100644 --- a/app/infra/NFSClientAdapter.ts +++ b/app/infra/NFSClientAdapter.ts @@ -1,10 +1,10 @@ import { AccessLevel, EggObjectLifecycle, - InitTypeQualifier, Inject, - ObjectInitType, SingletonProto, + EggQualifier, + EggType, } from '@eggjs/tegg'; import { EggAppConfig, EggLogger } from 'egg'; import FSClient from 'fs-cnpm'; @@ -17,7 +17,7 @@ import { Readable } from 'stream'; }) export class NFSClientAdapter implements EggObjectLifecycle, NFSClient { @Inject() - @InitTypeQualifier(ObjectInitType.SINGLETON) + @EggQualifier(EggType.APP) private logger: EggLogger; @Inject() diff --git a/config/config.default.ts b/config/config.default.ts index 4232ac7e..7e0f42c4 100644 --- a/config/config.default.ts +++ b/config/config.default.ts @@ -3,6 +3,7 @@ import { join } from 'path'; import { EggAppConfig, PowerPartial } from 'egg'; import OSSClient from 'oss-cnpm'; import { patchAjv } from '../app/port/typebox'; +import { SyncDeleteMode, SyncMode } from '../app/common/constants'; export default (appInfo: EggAppConfig) => { const config = {} as PowerPartial; @@ -22,7 +23,8 @@ export default (appInfo: EggAppConfig) => { // - none: don't sync npm package, just redirect it to sourceRegistry // - all: sync all npm packages // - exist: only sync exist packages, effected when `enableCheckRecentlyUpdated` or `enableChangesStream` is enabled - syncMode: 'none', + syncMode: SyncMode.none, + syncDeleteMode: SyncDeleteMode.delete, hookEnable: false, syncPackageWorkerMaxConcurrentTasks: 10, triggerHookWorkerMaxConcurrentTasks: 10, diff --git a/test/common/adapter/binary/ImageminBinary.test.ts b/test/common/adapter/binary/ImageminBinary.test.ts index 2a0372f4..01c4aa24 100644 --- a/test/common/adapter/binary/ImageminBinary.test.ts +++ b/test/common/adapter/binary/ImageminBinary.test.ts @@ -80,7 +80,7 @@ describe('test/common/adapter/binary/ImageminBinary.test.ts', () => { data: await TestUtil.readFixturesFile('registry.npmjs.com/advpng-bin.json'), }); let result = await binary.fetch('/', 'advpng-bin'); - console.log(result?.items.map(_ => _.name)); + // console.log(result?.items.map(_ => _.name)); assert(result); assert(result.items.length > 0); let matchDir1 = false; @@ -308,7 +308,7 @@ describe('test/common/adapter/binary/ImageminBinary.test.ts', () => { data: await TestUtil.readFixturesFile('registry.npmjs.com/guetzli.json'), }); const result = await binary.fetch('/', 'guetzli-bin'); - console.log(result); + // console.log(result); assert(result); // console.log(result.items); assert(result.items.length > 0); diff --git a/test/core/service/PackageSyncerService/executeTask.test.ts b/test/core/service/PackageSyncerService/executeTask.test.ts index c16c70e4..0f175d90 100644 --- a/test/core/service/PackageSyncerService/executeTask.test.ts +++ b/test/core/service/PackageSyncerService/executeTask.test.ts @@ -18,6 +18,7 @@ import { TaskService } from 'app/core/service/TaskService'; import { ScopeManagerService } from 'app/core/service/ScopeManagerService'; import { UserService } from 'app/core/service/UserService'; import { ChangeRepository } from 'app/repository/ChangeRepository'; +import { PackageVersion } from 'app/repository/model/PackageVersion'; describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { let packageSyncerService: PackageSyncerService; @@ -218,7 +219,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { assert(stream); let log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert(log.includes(`] 🟢 Package "${name}" was unpublished caused by 404 response`)); + assert(log.includes(`] 🟢 Package "${name}" was removed in remote registry`)); manifests = await packageManagerService.listPackageFullManifests('', name); assert(manifests.data.time.unpublished); @@ -908,7 +909,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { assert(stream); let log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert(log.includes('] 📖 Ignore unpublished package: {')); + assert(log.includes(`] 🟢 Package "${name}" was removed in remote registry`)); let data = await packageManagerService.listPackageFullManifests('', name); assert(data.data === null); @@ -949,7 +950,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { assert(stream); log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert(log.includes('] 🟢 Sync unpublished package: {')); + assert(log.includes(`] 🟢 Package "${name}" was removed in remote registry`)); data = await packageManagerService.listPackageFullManifests('', name); // console.log(data.data); assert(data.data.time.unpublished); @@ -1565,7 +1566,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { const log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert(log.includes(`🟢 Package "${name}" was unpublished caused by 451 response`)); + assert(log.includes(`] 🟢 Package "${name}" was removed in remote registry`)); }); it('should stop sync by block list', async () => { @@ -1670,9 +1671,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { assert(stream); const log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert(log.includes('🟢🟢🟢🟢🟢')); - assert(log.includes('🟢 [1] Synced version 0.0.1-security success')); - assert(log.includes('Syncing maintainers: [{\"name\":\"npm\",\"email\":\"npm@npmjs.com\"}]')); + assert(log.includes(`] 🟢 Package "${name}" was removed in remote registry`)); }); it('should mock getFullManifests missing tarball error and downloadTarball error', async () => { @@ -2126,5 +2125,99 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { assert(log.includes('skip sync')); }); }); + + describe('syncDeleteMode = ignore', async () => { + + // already synced pkg + beforeEach(async () => { + app.mockHttpclient('https://registry.npmjs.org/foobar', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar.json'), + persist: false, + repeats: 1, + }); + app.mockHttpclient('https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'), + persist: false, + }); + app.mockHttpclient('https://registry.npmjs.org/foobar/-/foobar-1.1.0.tgz', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.1.0.tgz'), + persist: false, + }); + await packageSyncerService.createTask('foobar', { skipDependencies: true }); + const task = await packageSyncerService.findExecuteTask(); + assert(task); + await packageSyncerService.executeTask(task); + assert(!await TaskModel.findOne({ taskId: task.taskId })); + assert(await HistoryTaskModel.findOne({ taskId: task.taskId })); + + }); + + it('should ignore when upstream is removed', async () => { + // removed in remote + mock(app.config.cnpmcore, 'syncDeleteMode', 'ignore'); + app.mockHttpclient('https://registry.npmjs.org/foobar', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/security-holding-package.json'), + }); + await packageSyncerService.createTask('foobar', { skipDependencies: true }); + const task = await packageSyncerService.findExecuteTask(); + assert(task); + await packageSyncerService.executeTask(task); + assert(!await TaskModel.findOne({ taskId: task.taskId })); + assert(await HistoryTaskModel.findOne({ taskId: task.taskId })); + const stream = await packageSyncerService.findTaskLog(task); + assert(stream); + const log = await TestUtil.readStreamToLog(stream); + assert(log); + // console.log(log); + const model = await PackageModel.findOne({ scope: '', name: 'foobar' }); + assert(model); + const versions = await PackageVersion.find({ packageId: model.packageId }); + assert.equal(model!.isPrivate, false); + assert(versions.length === 2); + + }); + + it('should block when upstream is removed', async () => { + // removed in remote + mock(app.config.cnpmcore, 'syncDeleteMode', 'block'); + app.mockHttpclient('https://registry.npmjs.org/foobar', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/security-holding-package.json'), + }); + await packageSyncerService.createTask('foobar', { skipDependencies: true }); + const task = await packageSyncerService.findExecuteTask(); + assert(task); + await packageSyncerService.executeTask(task); + assert(!await TaskModel.findOne({ taskId: task.taskId })); + assert(await HistoryTaskModel.findOne({ taskId: task.taskId })); + const stream = await packageSyncerService.findTaskLog(task); + assert(stream); + const log = await TestUtil.readStreamToLog(stream); + assert(log); + // console.log(log); + const model = await PackageModel.findOne({ scope: '', name: 'foobar' }); + assert(model); + const versions = await PackageVersion.find({ packageId: model.packageId }); + assert.equal(model!.isPrivate, false); + assert(versions.length === 2); + + const manifests = await packageManagerService.listPackageFullManifests('', 'foobar'); + assert(manifests.blockReason === 'Removed in remote registry'); + assert(manifests.data.block === 'Removed in remote registry'); + const pkg = await packageRepository.findPackage('', 'foobar'); + assert(pkg); + + await app.httpRequest() + .get(`/${pkg.name}`) + .expect(451); + + // could resotre + await packageManagerService.unblockPackage(pkg); + + await app.httpRequest() + .get(`/${pkg.name}`) + .expect(200); + + }); + }); }); });