feat(market): use unified registry versions

packages/registry/src/index.ts

@@ -128,7 +128,9 @@ export interface Extension {
   }
 }
 
-export interface SearchObject extends Extension, Pick<RemotePackage, 'deprecated' | 'peerDependencies' | 'peerDependenciesMeta'> {
+export interface DependencyMeta extends Pick<RemotePackage, 'deprecated' | 'peerDependencies' | 'peerDependenciesMeta'> {}
+
+export interface SearchObject extends Extension, DependencyMeta {
   package: SearchPackage
   searchScore: number
   ignored?: boolean

plugins/market/src/browser/index.ts

@@ -1,15 +1,10 @@
 import { Context, Schema } from 'koishi'
-import Dependencies from './deps'
 import MarketProvider from './market'
 
-export * from './deps'
 export * from './market'
 export * from '../shared'
 
-export {
-  Dependencies,
-  MarketProvider,
-}
+export { MarketProvider }
 
 export const filter = false
 export const name = 'market'
@@ -20,7 +15,6 @@ export interface Config {}
 export const Config: Schema<Config> = Schema.object({})
 
 export function apply(ctx: Context, config: Config) {
-  ctx.plugin(Dependencies)
   ctx.plugin(MarketProvider)
 
   ctx.console.addEntry(process.env.KOISHI_BASE ? [

plugins/market/src/browser/market.ts

@@ -15,7 +15,7 @@ export default class MarketProvider extends BaseMarketProvider {
     const market = await this.prepare()
     if (!market) return { data: {}, failed: 0, total: 0, progress: 0 }
     return {
-      data: Object.fromEntries(market.objects.map(item => [item.name, item])),
+      data: Object.fromEntries(market.objects.map(item => [item.package.name, item])),
       failed: 0,
       total: market.objects.length,
       progress: market.objects.length,

plugins/market/src/node/deps.ts

@@ -1,6 +1,6 @@
 import { Context, Dict, Schema } from 'koishi'
 import { DataService } from '@koishijs/plugin-console'
-import { Dependency } from '../shared'
+import { Dependency } from './installer'
 
 declare module '@koishijs/plugin-console' {
   interface Events {

plugins/market/src/node/installer.ts

@@ -1,5 +1,5 @@
 import { Context, defineProperty, Dict, Logger, pick, Quester, Schema, Service, Time, valueMap } from 'koishi'
-import Scanner, { PackageJson, Registry } from '@koishijs/registry'
+import Scanner, { DependencyMeta, PackageJson, Registry, RemotePackage } from '@koishijs/registry'
 import { resolve } from 'path'
 import { promises as fsp, readFileSync } from 'fs'
 import { compare, satisfies, valid } from 'semver'
@@ -12,6 +12,23 @@ import pMap from 'p-map'
 
 const logger = new Logger('market')
 
+export interface Dependency {
+  /**
+   * requested semver range
+   * @example `^1.2.3` -> `1.2.3`
+   */
+  request: string
+  /**
+   * installed package version
+   * @example `1.2.5`
+   */
+  resolved?: string
+  /** whether it is a workspace package */
+  workspace?: boolean
+  /** valid (unsupported) syntax */
+  invalid?: boolean
+}
+
 export interface LocalPackage extends PackageJson {
   private?: boolean
   $workspace?: boolean
@@ -29,6 +46,8 @@ class Installer extends Service {
   public http: Quester
   public registry: string
 
+  private packageTasks: Dict<Promise<Dict<DependencyMeta>>>
+  private packageCache: Dict<Dict<DependencyMeta>>
   private agent = which()?.name || 'npm'
   private manifest: PackageJson
   private task: Promise<Dict<Dependency>>
@@ -73,6 +92,25 @@ class Installer extends Service {
     return entries.find(Boolean)
   }
 
+  private async _getPackage(name: string) {
+    try {
+      const registry = await this.http.get<Registry>(`/${name}`)
+      return this.setPackage(name, Object.values(registry.versions))
+    } catch (e) {
+      logger.warn(e.message)
+    }
+  }
+
+  setPackage(name: string, versions: RemotePackage[]) {
+    return this.packageCache[name] = Object.fromEntries(versions
+      .map(item => [item.version, pick(item, Dependency.keys)] as const)
+      .sort(([a], [b]) => compare(b, a)))
+  }
+
+  getPackage(name: string) {
+    return this.packageTasks[name] ||= this._getPackage(name)
+  }
+
   private async _get() {
     const result = valueMap(this.manifest.dependencies, (request) => {
       return { request: request.replace(/^[~^]/, '') } as Dependency
@@ -88,18 +126,6 @@ class Installer extends Service {
 
       if (!valid(result[name].request)) {
         result[name].invalid = true
-        return
-      }
-
-      try {
-        const registry = await this.http.get<Registry>(`/${name}`)
-        const entries = Object.values(registry.versions)
-          .map(item => [item.version, pick(item, Dependency.keys)] as const)
-          .sort(([a], [b]) => compare(b, a))
-        result[name].latest = entries[0][0]
-        result[name].versions = Object.fromEntries(entries)
-      } catch (e) {
-        logger.warn(e.message)
       }
     }, { concurrency: 10 })
     return result

plugins/market/src/node/market.ts

@@ -1,14 +1,14 @@
-import { Context, Dict, pick, Quester, Schema, Time, valueMap } from 'koishi'
-import Scanner, { AnalyzedPackage, SearchResult } from '@koishijs/registry'
+import { Context, Dict, Quester, Schema, Time } from 'koishi'
+import Scanner, { SearchObject, SearchResult } from '@koishijs/registry'
 import { MarketProvider as BaseMarketProvider } from '../shared'
 import { throttle } from 'throttle-debounce'
 
 class MarketProvider extends BaseMarketProvider {
   private http: Quester
   private failed: string[] = []
   private scanner: Scanner
-  private fullCache: Dict<AnalyzedPackage> = {}
-  private tempCache: Dict<AnalyzedPackage> = {}
+  private fullCache: Dict<SearchObject> = {}
+  private tempCache: Dict<SearchObject> = {}
 
   constructor(ctx: Context, public config: MarketProvider.Config) {
     super(ctx)
@@ -48,36 +48,44 @@ class MarketProvider extends BaseMarketProvider {
       const result = await this.http.get<SearchResult>('')
       this.scanner.objects = result.objects.filter(object => !object.ignored)
       this.scanner.total = this.scanner.objects.length
+      this.scanner.version = result.version
     } else {
       await this.scanner.collect({ timeout })
     }
 
-    this.scanner.analyze({
-      version: '4',
-      onFailure: (name, reason) => {
-        this.failed.push(name)
-        if (registry.config.endpoint.startsWith('https://registry.npmmirror.com')) {
-          if (Quester.isAxiosError(reason) && reason.response?.status === 404) {
-            // ignore 404 error for npmmirror
+    if (!this.scanner.version) {
+      this.scanner.analyze({
+        version: '4',
+        onFailure: (name, reason) => {
+          this.failed.push(name)
+          if (registry.config.endpoint.startsWith('https://registry.npmmirror.com')) {
+            if (Quester.isAxiosError(reason) && reason.response?.status === 404) {
+              // ignore 404 error for npmmirror
+            }
           }
-        }
-      },
-      onSuccess: (item) => {
-        const { name, versions } = item
-        this.tempCache[name] = this.fullCache[name] = {
-          ...item,
-          versions: valueMap(versions, item => pick(item, ['peerDependencies', 'peerDependenciesMeta'])),
-        }
-      },
-      after: () => this.flushData(),
-    })
+        },
+        onRegistry: (registry, versions) => {
+          this.ctx.installer.setPackage(registry.name, versions)
+        },
+        onSuccess: (item, object) => {
+        },
+        after: () => this.flushData(),
+      })
+    }
+
     return null
   }
 
   async get() {
     await this.prepare()
     if (this._error) return { data: {}, failed: 0, total: 0, progress: 0 }
-    return {
+    return this.scanner.version ? {
+      data: Object.fromEntries(this.scanner.objects.map(item => [item.package.name, item])),
+      failed: 0,
+      total: this.scanner.total,
+      progress: this.scanner.total,
+      gravatar: process.env.GRAVATAR_MIRROR,
+    } : {
       data: this.fullCache,
       failed: this.failed.length,
       total: this.scanner.total,

plugins/market/src/shared/index.ts

@@ -1,6 +1,6 @@
 import { Context, Dict, Logger, Time } from 'koishi'
 import { DataService } from '@koishijs/plugin-console'
-import { AnalyzedPackage, MarketResult, RemotePackage } from '@koishijs/registry'
+import { SearchObject, SearchResult } from '@koishijs/registry'
 
 declare module '@koishijs/plugin-console' {
   interface Events {
@@ -14,27 +14,6 @@ declare module '@koishijs/plugin-console' {
   }
 }
 
-export interface Dependency {
-  /**
-   * requested semver range
-   * @example `^1.2.3` -> `1.2.3`
-   */
-  request: string
-  /**
-   * installed package version
-   * @example `1.2.5`
-   */
-  resolved?: string
-  /** whether it is a workspace package */
-  workspace?: boolean
-  /** all available versions */
-  versions?: Dict<Pick<RemotePackage, typeof Dependency.keys[number]>>
-  /** latest version */
-  latest?: string
-  /** valid (unsupported) syntax */
-  invalid?: boolean
-}
-
 export namespace Dependency {
   export const keys = ['peerDependencies', 'peerDependenciesMeta', 'deprecated'] as const
 }
@@ -64,9 +43,9 @@ export abstract class MarketProvider extends DataService<MarketProvider.Payload>
     this._timestamp = Date.now()
   }
 
-  abstract collect(): Promise<void | MarketResult>
+  abstract collect(): Promise<void | SearchResult>
 
-  async prepare(): Promise<MarketResult> {
+  async prepare(): Promise<SearchResult> {
     return this._task ||= this.collect().catch((error) => {
       logger.warn(error)
       this._error = error
@@ -76,7 +55,7 @@ export abstract class MarketProvider extends DataService<MarketProvider.Payload>
 
 export namespace MarketProvider {
   export interface Payload {
-    data: Dict<AnalyzedPackage>
+    data: Dict<SearchObject>
     total: number
     failed: number
     progress: number