From 6d657772ed5eb9e2afcfa17068c26fefc1ee9016 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 14 Dec 2023 14:44:44 -0500 Subject: [PATCH 1/2] Speed up fromconfig --- .../FromConfigAdapter/FromConfigAdapter.ts | 64 +++++++++---------- .../FromConfigRegionsAdapter.ts | 10 +-- 2 files changed, 32 insertions(+), 42 deletions(-) diff --git a/plugins/config/src/FromConfigAdapter/FromConfigAdapter.ts b/plugins/config/src/FromConfigAdapter/FromConfigAdapter.ts index 650d6d5600..f2380af2d5 100644 --- a/plugins/config/src/FromConfigAdapter/FromConfigAdapter.ts +++ b/plugins/config/src/FromConfigAdapter/FromConfigAdapter.ts @@ -12,15 +12,34 @@ import { import PluginManager from '@jbrowse/core/PluginManager' import { getSubAdapterType } from '@jbrowse/core/data_adapters/dataAdapterCache' -/** - * Adapter that just returns the features defined in its `features` configuration - * key, like: - * `"features": [ { "refName": "ctgA", "start":1, "end":20 }, ... ]` - */ +export function makeFeatures(fdata: SimpleFeatureSerialized[]) { + const features = new Map() + for (const entry of fdata) { + if (entry) { + const f = new SimpleFeature(entry) + const refName = f.get('refName') as string + let bucket = features.get(refName) + if (!bucket) { + bucket = [] + features.set(refName, bucket) + } + + bucket.push(f) + } + } + + // sort the features on each reference sequence by start coordinate + for (const refFeatures of features.values()) { + refFeatures.sort((a, b) => a.get('start') - b.get('start')) + } + + return features +} export default class FromConfigAdapter extends BaseFeatureDataAdapter { protected features: Map + protected refNameMap: string[] | undefined constructor( conf: AnyConfigurationModel, getSubAdapter?: getSubAdapterType, @@ -28,39 +47,14 @@ export default class FromConfigAdapter extends BaseFeatureDataAdapter { ) { super(conf, getSubAdapter, pluginManager) const feats = readConfObject(conf, 'features') as SimpleFeatureSerialized[] - this.features = FromConfigAdapter.makeFeatures(feats || []) - } - - static makeFeatures(fdata: SimpleFeatureSerialized[]) { - const features = new Map() - for (const entry of fdata) { - if (entry) { - const f = this.makeFeature(entry) - const refName = f.get('refName') as string - let bucket = features.get(refName) - if (!bucket) { - bucket = [] - features.set(refName, bucket) - } - - bucket.push(f) - } - } - - // sort the features on each reference sequence by start coordinate - for (const refFeatures of features.values()) { - refFeatures.sort((a, b) => a.get('start') - b.get('start')) - } - - return features - } - - static makeFeature(data: SimpleFeatureSerialized) { - return new SimpleFeature(data) + this.features = makeFeatures(feats || []) } async getRefNames() { - return [...this.features.keys()] + if (!this.refNameMap) { + this.refNameMap = [...this.features.keys()] + } + return this.refNameMap } async getRefNameAliases() { diff --git a/plugins/config/src/FromConfigRegionsAdapter/FromConfigRegionsAdapter.ts b/plugins/config/src/FromConfigRegionsAdapter/FromConfigRegionsAdapter.ts index 2e7544b4d8..ac0b33e45a 100644 --- a/plugins/config/src/FromConfigRegionsAdapter/FromConfigRegionsAdapter.ts +++ b/plugins/config/src/FromConfigRegionsAdapter/FromConfigRegionsAdapter.ts @@ -12,8 +12,7 @@ import { } from '@jbrowse/core/configuration' import PluginManager from '@jbrowse/core/PluginManager' import { getSubAdapterType } from '@jbrowse/core/data_adapters/dataAdapterCache' - -import FromConfigAdapter from '../FromConfigAdapter/FromConfigAdapter' +import { makeFeatures } from '../FromConfigAdapter/FromConfigAdapter' /** * Adapter that just returns the features defined in its `features` configuration @@ -32,11 +31,8 @@ export default class FromConfigRegionsAdapter pluginManager?: PluginManager, ) { super(config, getSubAdapter, pluginManager) - const features = readConfObject( - config, - 'features', - ) as SimpleFeatureSerialized[] - this.features = FromConfigAdapter.makeFeatures(features || []) + const f = readConfObject(config, 'features') as SimpleFeatureSerialized[] + this.features = makeFeatures(f || []) } /** From d372be712abd695c49eda7db6185a11dd66e05bc Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 14 Dec 2023 15:13:49 -0500 Subject: [PATCH 2/2] Use adapterConfigCacheKey instead of full json-stable-stringify --- package.json | 1 - packages/core/assemblyManager/assembly.ts | 20 +++++++++---------- .../core/assemblyManager/assemblyManager.ts | 8 +++++--- .../core/data_adapters/dataAdapterCache.ts | 4 ++-- packages/core/package.json | 1 - packages/core/util/index.ts | 2 +- .../FromConfigAdapter/FromConfigAdapter.ts | 6 +----- yarn.lock | 20 ------------------- 8 files changed, 19 insertions(+), 43 deletions(-) diff --git a/package.json b/package.json index 34ba5c0429..d8970f822a 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,6 @@ "@types/jest-image-snapshot": "^6.1.0", "@types/jexl": "^2.3.0", "@types/json-parse-better-errors": "^1.0.0", - "@types/json-stable-stringify": "^1.0.32", "@types/lodash.merge": "^4.6.6", "@types/node": "^14.14.0", "@types/node-fetch": "^2.5.7", diff --git a/packages/core/assemblyManager/assembly.ts b/packages/core/assemblyManager/assembly.ts index 63b47438f0..a54cf36b19 100644 --- a/packages/core/assemblyManager/assembly.ts +++ b/packages/core/assemblyManager/assembly.ts @@ -1,5 +1,4 @@ import { getParent, types, Instance, IAnyType } from 'mobx-state-tree' -import jsonStableStringify from 'json-stable-stringify' import AbortablePromiseCache from 'abortable-promise-cache' // locals @@ -13,6 +12,9 @@ import PluginManager from '../PluginManager' import { when, Region, Feature } from '../util' import QuickLRU from '../util/QuickLRU' import RpcManager from '../rpc/RpcManager' +import { adapterConfigCacheKey } from '../data_adapters/dataAdapterCache' + +type AdapterConf = Record const refNameRegex = new RegExp( '[0-9A-Za-z!#$%&+./:;?@^_|~-][0-9A-Za-z!#$%&*+./:;=?@^_|~-]*', @@ -107,10 +109,6 @@ function checkRefName(refName: string) { } } -function getAdapterId(adapterConf: unknown) { - return jsonStableStringify(adapterConf) -} - type RefNameAliases = Record interface CacheData { @@ -430,14 +428,13 @@ export default function assemblyFactory( /** * #method */ - getAdapterMapEntry(adapterConf: unknown, options: BaseOptions) { + getAdapterMapEntry(adapterConf: AdapterConf, options: BaseOptions) { const { signal, statusCallback, ...rest } = options if (!options.sessionId) { throw new Error('sessionId is required') } - const adapterId = getAdapterId(adapterConf) return adapterLoads.get( - adapterId, + adapterConfigCacheKey(adapterConf), { adapterConf, self: self as Assembly, @@ -455,7 +452,10 @@ export default function assemblyFactory( * #method * get Map of `canonical-name -> adapter-specific-name` */ - async getRefNameMapForAdapter(adapterConf: unknown, opts: BaseOptions) { + async getRefNameMapForAdapter( + adapterConf: AdapterConf, + opts: BaseOptions, + ) { if (!opts?.sessionId) { throw new Error('sessionId is required') } @@ -468,7 +468,7 @@ export default function assemblyFactory( * get Map of `adapter-specific-name -> canonical-name` */ async getReverseRefNameMapForAdapter( - adapterConf: unknown, + adapterConf: AdapterConf, opts: BaseOptions, ) { const map = await this.getAdapterMapEntry(adapterConf, opts) diff --git a/packages/core/assemblyManager/assemblyManager.ts b/packages/core/assemblyManager/assemblyManager.ts index 0ff04bd5f3..60429adc04 100644 --- a/packages/core/assemblyManager/assemblyManager.ts +++ b/packages/core/assemblyManager/assemblyManager.ts @@ -13,6 +13,8 @@ import assemblyFactory, { Assembly } from './assembly' import PluginManager from '../PluginManager' import RpcManager from '../rpc/RpcManager' +type AdapterConf = Record + /** * #stateModel AssemblyManager */ @@ -105,7 +107,7 @@ function assemblyManagerFactory(conf: IAnyType, pm: PluginManager) { await assembly.load() await when( () => - Boolean(assembly?.regions && assembly.refNameAliases) || + !!(assembly?.regions && assembly.refNameAliases) || !!assembly?.error, ) if (assembly.error) { @@ -118,7 +120,7 @@ function assemblyManagerFactory(conf: IAnyType, pm: PluginManager) { * #method */ async getRefNameMapForAdapter( - adapterConf: unknown, + adapterConf: AdapterConf, assemblyName: string | undefined, opts: { signal?: AbortSignal; sessionId: string }, ) { @@ -133,7 +135,7 @@ function assemblyManagerFactory(conf: IAnyType, pm: PluginManager) { * #method */ async getReverseRefNameMapForAdapter( - adapterConf: unknown, + adapterConf: AdapterConf, assemblyName: string | undefined, opts: { signal?: AbortSignal; sessionId: string }, ) { diff --git a/packages/core/data_adapters/dataAdapterCache.ts b/packages/core/data_adapters/dataAdapterCache.ts index 82b0486117..8db7c7948c 100644 --- a/packages/core/data_adapters/dataAdapterCache.ts +++ b/packages/core/data_adapters/dataAdapterCache.ts @@ -6,8 +6,8 @@ import idMaker from '../util/idMaker' type ConfigSnap = SnapshotIn -function adapterConfigCacheKey(adapterConfig: ConfigSnap) { - return `${idMaker(adapterConfig)}` +export function adapterConfigCacheKey(conf: Record = {}) { + return `${idMaker(conf)}` } interface AdapterCacheEntry { diff --git a/packages/core/package.json b/packages/core/package.json index 30749b3ded..7ca273581b 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -47,7 +47,6 @@ "http-range-fetcher": "^2.0.0", "is-object": "^1.0.1", "jexl": "^2.3.0", - "json-stable-stringify": "^1.0.1", "librpc-web-mod": "^1.1.5", "load-script": "^2.0.0", "material-ui-popup-state": "^5.0.0", diff --git a/packages/core/util/index.ts b/packages/core/util/index.ts index 2c0820d566..6ecbaa5e7f 100644 --- a/packages/core/util/index.ts +++ b/packages/core/util/index.ts @@ -780,7 +780,7 @@ export async function renameRegionsIfNeeded< assemblyName?: string regions?: Region[] signal?: AbortSignal - adapterConfig: unknown + adapterConfig: Record sessionId: string statusCallback?: (arg: string) => void }, diff --git a/plugins/config/src/FromConfigAdapter/FromConfigAdapter.ts b/plugins/config/src/FromConfigAdapter/FromConfigAdapter.ts index f2380af2d5..2fa53c7ff4 100644 --- a/plugins/config/src/FromConfigAdapter/FromConfigAdapter.ts +++ b/plugins/config/src/FromConfigAdapter/FromConfigAdapter.ts @@ -39,7 +39,6 @@ export function makeFeatures(fdata: SimpleFeatureSerialized[]) { export default class FromConfigAdapter extends BaseFeatureDataAdapter { protected features: Map - protected refNameMap: string[] | undefined constructor( conf: AnyConfigurationModel, getSubAdapter?: getSubAdapterType, @@ -51,10 +50,7 @@ export default class FromConfigAdapter extends BaseFeatureDataAdapter { } async getRefNames() { - if (!this.refNameMap) { - this.refNameMap = [...this.features.keys()] - } - return this.refNameMap + return [...this.features.keys()] } async getRefNameAliases() { diff --git a/yarn.lock b/yarn.lock index 157c72198d..1ba44907a1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4417,11 +4417,6 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== -"@types/json-stable-stringify@^1.0.32": - version "1.0.36" - resolved "https://registry.yarnpkg.com/@types/json-stable-stringify/-/json-stable-stringify-1.0.36.tgz#fe6c6001a69ff8160a772da08779448a333c7ddd" - integrity sha512-b7bq23s4fgBB76n34m2b3RBf6M369B0Z9uRR8aHTMd8kZISRkmDEpPD8hhpYvDFzr3bJCPES96cm3Q6qRNDbQw== - "@types/keyv@^3.1.4": version "3.1.4" resolved "https://registry.yarnpkg.com/@types/keyv/-/keyv-3.1.4.tgz#3ccdb1c6751b0c7e52300bcdacd5bcbf8faa75b6" @@ -11538,16 +11533,6 @@ json-stable-stringify-without-jsonify@^1.0.1: resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== -json-stable-stringify@^1.0.1: - version "1.1.0" - resolved "https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-1.1.0.tgz#43d39c7c8da34bfaf785a61a56808b0def9f747d" - integrity sha512-zfA+5SuwYN2VWqN1/5HZaDzQKLJHaBVMZIIM+wuYjdptkaQsqzDdqjqf+lZZJUuJq1aanHiY8LhH8LmH+qBYJA== - dependencies: - call-bind "^1.0.5" - isarray "^2.0.5" - jsonify "^0.0.1" - object-keys "^1.1.1" - json-stringify-nice@^1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/json-stringify-nice/-/json-stringify-nice-1.1.4.tgz#2c937962b80181d3f317dd39aa323e14f5a60a67" @@ -11584,11 +11569,6 @@ jsonfile@^6.0.1: optionalDependencies: graceful-fs "^4.1.6" -jsonify@^0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.1.tgz#2aa3111dae3d34a0f151c63f3a45d995d9420978" - integrity sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg== - jsonparse@^1.2.0, jsonparse@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280"