diff --git a/providers/upgrade/process.js b/providers/upgrade/process.js index 7f7dae73..751e7865 100644 --- a/providers/upgrade/process.js +++ b/providers/upgrade/process.js @@ -59,7 +59,7 @@ class DefinitionUpgrader { let queueHandler let defUpgrader -function setup(_queue, _definitionService, _logger, once = false, _defVersionChecker = factory()) { +function setup(_queue, _definitionService, _logger, once = false, _defVersionChecker = factory({ logger: _logger })) { defUpgrader = new DefinitionUpgrader(_definitionService, _logger, _defVersionChecker) queueHandler = new QueueHandler(_queue, _logger, defUpgrader) return queueHandler.work(once) diff --git a/test/business/definitionServiceTest.js b/test/business/definitionServiceTest.js index a91650bc..5b4ac7d3 100644 --- a/test/business/definitionServiceTest.js +++ b/test/business/definitionServiceTest.js @@ -15,6 +15,9 @@ const expect = chai.expect const FileHarvestStore = require('../../providers/stores/fileHarvestStore') const SummaryService = require('../../business/summarizer') const AggregatorService = require('../../business/aggregator') +const DefinitionQueueUpgrader = require('../../providers/upgrade/defUpgradeQueue') +const memoryQueue = require('../../providers/queueing/memoryQueue') +const { DefinitionVersionChecker } = require('../../providers/upgrade/defVersionCheck') describe('Definition Service', () => { it('invalidates single coordinate', async () => { @@ -314,26 +317,129 @@ describe('Definition Service Facet management', () => { }) describe('Integration test', () => { - let fileHarvestStore - beforeEach(() => { - fileHarvestStore = createFileHarvestStore() + describe('compute', () => { + let fileHarvestStore + beforeEach(() => { + fileHarvestStore = createFileHarvestStore() + }) + + it('computes the same definition with latest harvest data', async () => { + const coordinates = EntityCoordinates.fromString('npm/npmjs/-/debug/3.1.0') + const allHarvestData = await fileHarvestStore.getAll(coordinates) + delete allHarvestData['scancode']['2.9.0+b1'] //remove invalid scancode version + let service = setupServiceToCalculateDefinition(allHarvestData) + const baseline_def = await service.compute(coordinates) + + const latestHarvestData = await fileHarvestStore.getAllLatest(coordinates) + service = setupServiceToCalculateDefinition(latestHarvestData) + const comparison_def = await service.compute(coordinates) + + //updated timestamp is not deterministic + expect(comparison_def._meta.updated).to.not.equal(baseline_def._meta.updated) + comparison_def._meta.updated = baseline_def._meta.updated + expect(comparison_def).to.deep.equal(baseline_def) + }) }) - it('computes the same definition with latest harvest data', async () => { - const coordinates = EntityCoordinates.fromString('npm/npmjs/-/debug/3.1.0') - const allHarvestData = await fileHarvestStore.getAll(coordinates) - delete allHarvestData['scancode']['2.9.0+b1'] //remove invalid scancode version - let service = setupDefinitionService(allHarvestData) - const baseline_def = await service.compute(coordinates) - - const latestHarvestData = await fileHarvestStore.getAllLatest(coordinates) - service = setupDefinitionService(latestHarvestData) - const comparison_def = await service.compute(coordinates) - - //updated timestamp is not deterministic - expect(comparison_def._meta.updated).to.not.equal(baseline_def._meta.updated) - comparison_def._meta.updated = baseline_def._meta.updated - expect(comparison_def).to.deep.equal(baseline_def) + describe('Handle schema version upgrade', () => { + const coordinates = EntityCoordinates.fromString('npm/npmjs/-/test/1.0') + const definition = { _meta: { schemaVersion: '1.7.0' }, coordinates } + + let logger, upgradeHandler + beforeEach(() => { + logger = { debug: sinon.stub(), error: sinon.stub(), info: sinon.stub() } + }) + + const handleVersionedDefinition = function () { + describe('verify schema version', () => { + it('logs and harvests new definitions with empty tools', async () => { + const { service } = setupServiceForUpgrade(null, upgradeHandler) + service._harvest = sinon.stub() + await service.get(coordinates) + expect(service._harvest.calledOnce).to.be.true + expect(service._harvest.getCall(0).args[0]).to.eq(coordinates) + }) + + it('computes if definition does not exist', async () => { + const { service } = setupServiceForUpgrade(null, upgradeHandler) + service.computeStoreAndCurate = sinon.stub().resolves(definition) + await service.get(coordinates) + expect(service.computeStoreAndCurate.calledOnce).to.be.true + expect(service.computeStoreAndCurate.getCall(0).args[0]).to.eq(coordinates) + }) + + it('returns the up-to-date definition', async () => { + const { service } = setupServiceForUpgrade(definition, upgradeHandler) + service.computeStoreAndCurate = sinon.stub() + const result = await service.get(coordinates) + expect(service.computeStoreAndCurate.called).to.be.false + expect(result).to.deep.equal(definition) + }) + }) + } + + describe('schema version check', () => { + beforeEach(async () => { + upgradeHandler = new DefinitionVersionChecker({ logger }) + await upgradeHandler.initialize() + }) + + handleVersionedDefinition() + + context('with stale definitions', () => { + it('recomputes a definition with the updated schema version', async () => { + const staleDef = { ...createDefinition(null, null, ['foo']), _meta: { schemaVersion: '1.0.0' }, coordinates } + const { service, store } = setupServiceForUpgrade(staleDef, upgradeHandler) + const result = await service.get(coordinates) + expect(result._meta.schemaVersion).to.eq('1.7.0') + expect(result.coordinates).to.deep.equal(coordinates) + expect(store.store.calledOnce).to.be.true + }) + }) + }) + + describe('queueing schema version updates', () => { + let queue + beforeEach(async () => { + queue = memoryQueue() + const queueFactory = sinon.stub().returns(queue) + upgradeHandler = new DefinitionQueueUpgrader({ logger, queue: queueFactory }) + await upgradeHandler.initialize() + }) + + handleVersionedDefinition() + + context('with stale definitions', () => { + it('returns a stale definition, queues update, recomputes and retrieves the updated definition', async () => { + const staleDef = { ...createDefinition(null, null, ['foo']), _meta: { schemaVersion: '1.0.0' }, coordinates } + const { service, store } = setupServiceForUpgrade(staleDef, upgradeHandler) + const result = await service.get(coordinates) + expect(result).to.deep.equal(staleDef) + expect(queue.data.length).to.eq(1) + await upgradeHandler.setupProcessing(service, logger, true) + const newResult = await service.get(coordinates) + expect(newResult._meta.schemaVersion).to.eq('1.7.0') + expect(store.store.calledOnce).to.be.true + expect(queue.data.length).to.eq(0) + }) + + it('computes once when the same coordinates is queued twice', async () => { + const staleDef = { ...createDefinition(null, null, ['foo']), _meta: { schemaVersion: '1.0.0' }, coordinates } + const { service, store } = setupServiceForUpgrade(staleDef, upgradeHandler) + await service.get(coordinates) + const result = await service.get(coordinates) + expect(result).to.deep.equal(staleDef) + expect(queue.data.length).to.eq(2) + await upgradeHandler.setupProcessing(service, logger, true) + expect(queue.data.length).to.eq(1) + await upgradeHandler.setupProcessing(service, logger, true) + const newResult = await service.get(coordinates) + expect(newResult._meta.schemaVersion).to.eq('1.7.0') + expect(store.store.calledOnce).to.be.true + expect(queue.data.length).to.eq(0) + }) + }) + }) }) }) @@ -348,7 +454,7 @@ function createFileHarvestStore() { return FileHarvestStore(options) } -function setupDefinitionService(rawHarvestData) { +function setupServiceToCalculateDefinition(rawHarvestData) { const harvestStore = { getAllLatest: () => Promise.resolve(rawHarvestData) } const summary = SummaryService({}) @@ -363,13 +469,35 @@ function setupDefinitionService(rawHarvestData) { return setupWithDelegates(curator, harvestStore, summary, aggregator) } -function setupWithDelegates(curator, harvestStore, summary, aggregator) { - const store = { delete: sinon.stub(), get: sinon.stub(), store: sinon.stub() } +function setupServiceForUpgrade(definition, upgradeHandler) { + let storedDef = definition && { ...definition } + const store = { + get: sinon.stub().resolves(storedDef), + store: sinon.stub().callsFake(def => (storedDef = def)) + } + const harvestStore = { getAllLatest: () => Promise.resolve(null) } + const summary = { summarizeAll: () => Promise.resolve(null) } + const aggregator = { process: () => Promise.resolve(definition) } + const curator = { + get: () => Promise.resolve(), + apply: (_coordinates, _curationSpec, definition) => Promise.resolve(definition), + autoCurate: () => {} + } + const service = setupWithDelegates(curator, harvestStore, summary, aggregator, store, upgradeHandler) + return { service, store } +} + +function setupWithDelegates( + curator, + harvestStore, + summary, + aggregator, + store = { delete: sinon.stub(), get: sinon.stub(), store: sinon.stub() }, + upgradeHandler = { validate: def => Promise.resolve(def) } +) { const search = { delete: sinon.stub(), store: sinon.stub() } const cache = { delete: sinon.stub(), get: sinon.stub(), set: sinon.stub() } - const harvestService = { harvest: () => sinon.stub() } - const upgradeHandler = { validate: def => Promise.resolve(def) } const service = DefinitionService( harvestStore, harvestService, @@ -382,7 +510,6 @@ function setupWithDelegates(curator, harvestStore, summary, aggregator) { upgradeHandler ) service.logger = { info: sinon.stub(), debug: () => {} } - service._harvest = sinon.stub() return service }