From 358d8c97f9f194ac23f6f89c16dabf8de097b3d8 Mon Sep 17 00:00:00 2001 From: Davide Aquaro Date: Thu, 2 Jun 2022 13:20:45 +0100 Subject: [PATCH 1/8] feat: implemented changes for partial migrations --- .../__tests__/MIG-109/MIG-109.test.js | 3 +- .../__tests__/TOOLS-50/TOOLS-50.test.js | 2 + .../lib/StudyManifestTools.js | 181 +++++++++++++++--- packages/mdctl-cli/tasks/study.js | 39 +++- 4 files changed, 191 insertions(+), 34 deletions(-) diff --git a/packages/mdctl-axon-tools/__tests__/MIG-109/MIG-109.test.js b/packages/mdctl-axon-tools/__tests__/MIG-109/MIG-109.test.js index 9372d972..d43b0c41 100644 --- a/packages/mdctl-axon-tools/__tests__/MIG-109/MIG-109.test.js +++ b/packages/mdctl-axon-tools/__tests__/MIG-109/MIG-109.test.js @@ -157,7 +157,8 @@ describe('MIG-109 - Test StudyManifestTools ', () => { jest.spyOn(StudyManifestTools.prototype, 'getOrgObjectInfo').mockImplementation(() => dummyReferences) jest.spyOn(StudyManifestTools.prototype, 'validateReferences').mockImplementation(() => entities) jest.spyOn(StudyManifestTools.prototype, 'createManifest').mockImplementation(() => manifest) - + jest.spyOn(StudyManifestTools.prototype, 'getObjectIDsArray').mockImplementation(() => entities.filter(o => o.object === 'c_task')) + jest.spyOn(StudyManifestTools.prototype, 'mapObjectNameToPlural').mockImplementation(() => 'c_tasks') // eslint-disable-next-line one-var const manifestAndDeps = await manifestTools.buildManifestAndDependencies() diff --git a/packages/mdctl-axon-tools/__tests__/TOOLS-50/TOOLS-50.test.js b/packages/mdctl-axon-tools/__tests__/TOOLS-50/TOOLS-50.test.js index d3c6be01..1b8072b6 100644 --- a/packages/mdctl-axon-tools/__tests__/TOOLS-50/TOOLS-50.test.js +++ b/packages/mdctl-axon-tools/__tests__/TOOLS-50/TOOLS-50.test.js @@ -35,6 +35,8 @@ describe('getStudyManifestEntities', () => { ['orac__form_questions'], ['orac__events'] ])('should include %s', async(entity) => { + jest.spyOn(StudyManifestTools.prototype, 'getObjectIDsArray').mockImplementation(() => [1]) + jest.spyOn(StudyManifestTools.prototype, 'mapObjectNameToPlural').mockImplementation(() => entity) const entities = await manifestTools.getStudyManifestEntities({}, {}, {}), objectsRequested = mockGetExportedObjects .mock diff --git a/packages/mdctl-axon-tools/lib/StudyManifestTools.js b/packages/mdctl-axon-tools/lib/StudyManifestTools.js index fd05b2de..ca4e37e8 100644 --- a/packages/mdctl-axon-tools/lib/StudyManifestTools.js +++ b/packages/mdctl-axon-tools/lib/StudyManifestTools.js @@ -1,3 +1,4 @@ +/* eslint-disable max-len */ /* eslint-disable one-var */ /* eslint-disable no-underscore-dangle */ @@ -22,6 +23,26 @@ class StudyManifestTools { }) } + loadManifest(manifestObject) { + let manifestJSON + try { + manifestJSON = JSON.parse(manifestObject) + if (manifestJSON.object !== 'manifest') { + throw Fault.create('kInvalidArgument', { reason: 'The argument is not a valid manifest' }) + } + } catch (e) { + try { + if (fs.existsSync(manifestObject)) { + throw Fault.create('kInvalidArgument', { reason: 'The manifest file does not exists' }) + } + manifestJSON = JSON.parse(fs.readFileSync(manifestObject)) + } catch (err) { + throw Fault.create('kInvalidArgument', { reason: 'The manifest file is not a valid JSON' }) + } + } + return manifestJSON + } + async getTasks() { const { client } = privatesAccessor(this), driver = new Driver(client), @@ -123,9 +144,9 @@ class StudyManifestTools { return privatesAccessor(this).orgObjects.find(v => v.pluralName === pluralName).name } - async getStudyManifest() { + async getStudyManifest(manifestObject) { console.log('Building Manifest') - const manifestAndDeps = await this.buildManifestAndDependencies() + const manifestAndDeps = await this.buildManifestAndDependencies(manifestObject) await this.writeStudyToDisk(manifestAndDeps) return manifestAndDeps } @@ -151,14 +172,32 @@ class StudyManifestTools { return { manifest, removedEntities } } - async buildManifestAndDependencies() { + async buildManifestAndDependencies(manifestObject) { + let allEntities, + mappingScript, + ignoreKeys + const { org, orgReferenceProps } = await this.getOrgAndReferences(), - study = await this.getFirstStudy(org), - allEntities = [study, ...await this.getStudyManifestEntities(org, study, orgReferenceProps)], - { manifest, removedEntities } = this.validateAndCreateManifest(allEntities, orgReferenceProps), - mappingScript = await getMappingScript(org), ingestTransform = fs.readFileSync(`${__dirname}/../packageScripts/ingestTransform.js`) + if (manifestObject) { + const manifestJSON = this.loadManifest(manifestObject) + + ignoreKeys = Object.keys(manifestJSON).filter(key => typeof manifestJSON[key] === 'object') + .map(key => this.mapObjectNameToPlural(key)) + + allEntities = await this.getStudyManifestEntities(org, null, manifestJSON, orgReferenceProps) + } else { + const study = await this.getFirstStudy(org) + ignoreKeys = [] + allEntities = [study, + ...await this.getStudyManifestEntities(org, study, null, orgReferenceProps)] + mappingScript = await getMappingScript(org) + } + + const { manifest, removedEntities } = this + .validateAndCreateManifest(allEntities, orgReferenceProps, ignoreKeys) + return { manifest, removedEntities, @@ -632,15 +671,18 @@ class StudyManifestTools { .reduce((acc, curr) => acc.concat(curr), []) } + /* async getStudyManifestEntities(org, study, orgReferenceProps) { + const taskIds = (await org.objects.c_tasks.find({ c_study: study._id }).limit(false).toArray()).map(v => v._id), + consentIds = (await org.objects.ec__document_templates.find({ c_study: study._id }).limit(false).toArray()).map(v => v._id), + visitsIds = (await org.objects.c_visit_schedules.find({ c_study: study._id }).limit(false).toArray()).map(v => v._id), + groupsIds = (await org.objects.c_groups.find({ c_study: study._id }).limit(false).toArray()).map(v => v._id), + + tasksAndDependencies = await this.getTaskManifestEntities(org, taskIds, orgReferenceProps), + templatesAndDependencies = await this.getConsentManifestEntities(org, consentIds, orgReferenceProps), + visitSchedulesAndDependencies = await this.getVisitManifestEntities(org, visitsIds, orgReferenceProps), + groupsAndDependencies = await this.getGroupManifestEntities(org, groupsIds, orgReferenceProps), - const tasks = await this.getExportObjects(org, 'c_tasks', { c_study: study._id }, orgReferenceProps), - steps = await this.getExportObjects(org, 'c_steps', { c_task: { $in: tasks.map(v => v._id) } }, orgReferenceProps), - branches = await this.getExportObjects(org, 'c_branches', { c_task: { $in: tasks.map(v => v._id) } }, orgReferenceProps), - visitSchedules = await this.getExportObjects(org, 'c_visit_schedules', { c_study: study._id }, orgReferenceProps), - visits = await this.getExportObjects(org, 'c_visits', { c_visit_schedules: { $in: visitSchedules.map(v => v._id) } }, orgReferenceProps), - groups = await this.getExportObjects(org, 'c_groups', { c_study: study._id }, orgReferenceProps), - groupTasks = await this.getExportObjects(org, 'c_group_tasks', { c_group: { $in: groups.map(v => v._id) } }, orgReferenceProps), faults = await this.getExportObjects(org, 'c_faults', null, orgReferenceProps), reports = await this.getExportObjects(org, 'c_dmweb_reports', null, orgReferenceProps), sites = await this.getExportObjects(org, 'c_sites', { c_study: study._id }, orgReferenceProps), @@ -650,10 +692,6 @@ class StudyManifestTools { participantSchedules = await this.getExportObjects(org, 'c_participant_schedules', null, orgReferenceProps), patientFlags = await this.getExportObjects(org, 'c_patient_flags', null, orgReferenceProps), - documentTemplates = await this.getExportObjects(org, 'ec__document_templates', { ec__study: study._id }, orgReferenceProps), - knowledgeChecks = await this.getExportObjects(org, 'ec__knowledge_checks', { _id: { $in: documentTemplates.map(v => v._id) } }, orgReferenceProps), - defaultDoc = await this.getExportObjects(org, 'ec__default_document_csses', null, orgReferenceProps), - // looker lookerIntegrationRecords = await this.getExportObjects(org, 'c_looker_integration_records', null, orgReferenceProps), @@ -669,18 +707,95 @@ class StudyManifestTools { oracleQuestions = await this.getExportObjects(org, 'orac__form_questions', null, orgReferenceProps), oracleEvents = await this.getExportObjects(org, 'orac__events', null, orgReferenceProps) - return [ - ...tasks, ...steps, ...branches, - ...visitSchedules, ...visits, ...groups, ...faults, ...reports, ...sites, - ...groupTasks, ...taskAssignments, ...participantSchedules, ...anchorDateTemplates, - ...patientFlags, ...documentTemplates, ...knowledgeChecks, ...defaultDoc, - ...lookerIntegrationRecords, + ...tasksAndDependencies, ...visitSchedulesAndDependencies, ...faults, ...reports, ...sites, + ...groupsAndDependencies, ...taskAssignments, ...participantSchedules, ...anchorDateTemplates, + ...patientFlags, ...templatesAndDependencies, ...lookerIntegrationRecords, ...vendorIntegrationRecords, ...integrationMappings, ...integrationPipelines, - ...oracleStudies, ...oracleSites, - ...oracleForms, ...oracleQuestions, ...oracleEvents + ...oracleStudies, ...oracleSites, ...oracleForms, ...oracleQuestions, ...oracleEvents ] } + */ + + async getStudyManifestEntities(org, study, manifestObject, orgReferenceProps) { + const manifestEntities = [], + // Define the available entities to export or get them from the manifest in input + manifestKeys = (study) + ? ['c_task', 'c_visit_schedule', 'ec__document_template', 'c_group', + 'c_anchor_date_template', 'c_fault', 'c_dmweb_report', 'c_site', 'c_task_assignment', 'c_participant_schedule', + 'c_patient_flag', 'c_looker_integration_record', 'int__vendor_integration_record', 'int__model_mapping', + 'int__pipeline', 'orac__studies', 'orac__sites', 'orac__forms', 'orac__form_questions', 'orac__events'] + : Object.keys(manifestObject).filter(key => typeof manifestObject[key] === 'object') + + // eslint-disable-next-line no-restricted-syntax + for await (const key of manifestKeys) { + // Determine whether queriying by c_study or c_key + const property = (study) ? 'c_study' : 'c_key', + // Use the study ID or the entities inside the "includes" array in the manifest + values = (study) ? [study._id] : manifestObject[key].includes + + let ids, + pluralName, + objectAndDependencies + + switch (key) { + case 'c_task': { + // Get the Tasks ID's from the study or the manifest + ids = (await this.getObjectIDsArray(org, key, property, values)).map(v => v._id) + // Load the manifest for the current ID's and their dependencies + objectAndDependencies = await this.getTaskManifestEntities(org, ids, orgReferenceProps) + break + } + case 'ec__document_template': { + // Get the eConsents ID's from the study or the manifest + ids = (await this.getObjectIDsArray(org, key, property, values)).map(v => v._id) + // Load the manifest for the current ID's and their dependencies + objectAndDependencies = await this.getConsentManifestEntities(org, ids, orgReferenceProps) + break + } + case 'c_visit_schedule': { + // Get the Visit Schedules ID's from the study or the manifest + ids = (await this.getObjectIDsArray(org, key, property, values)).map(v => v._id) + // Load the manifest for the current ID's and their dependencies + objectAndDependencies = await this.getVisitManifestEntities(org, ids, orgReferenceProps) + break + } + case 'c_group': { + // Get the Groups ID's from the study or the manifest + ids = (await this.getObjectIDsArray(org, key, property, values)).map(v => v._id) + // Load the manifest for the current ID's and their dependencies + objectAndDependencies = await this.getGroupManifestEntities(org, ids, orgReferenceProps) + break + } + case 'c_site': + case 'c_anchor_date_template': { + pluralName = this.mapObjectNameToPlural(key) + // These objects seem not to have dependencies so we'll load them directly + objectAndDependencies = await this.getExportObjects(org, pluralName, { [property]: { $in: values } }, orgReferenceProps) + break + } + default: { + try { + // If there's no plural form for current key, use the key itself + pluralName = this.mapObjectNameToPlural(key) + } catch (e) { + pluralName = key + } + // Export the entire entity (no 'where' filter) if present + objectAndDependencies = await this.getExportObjects(org, pluralName, null, orgReferenceProps) + break + } + } + // Push the deconstructed object + manifestEntities.push(...objectAndDependencies) + } + + return manifestEntities + } + + async getObjectIDsArray(org, key, property, values) { + return org.objects[key].find({ [property]: { $in: values } }).limit(false).toArray() + } async getTaskManifestEntities(org, taskIds, orgReferenceProps) { @@ -700,6 +815,20 @@ class StudyManifestTools { return [...documentTemplates, ...knowledgeChecks, ...defaultCSS] } + async getVisitManifestEntities(org, visitIds, orgReferenceProps) { + const visitSchedules = await this.getExportObjects(org, 'c_visit_schedules', { _id: { $in: visitIds } }, orgReferenceProps), + visits = await this.getExportObjects(org, 'c_visits', { c_visit_schedules: { $in: visitSchedules.map(v => v._id) } }, orgReferenceProps) + + return [...visitSchedules, ...visits] + } + + async getGroupManifestEntities(org, groupIds, orgReferenceProps) { + const groups = await this.getExportObjects(org, 'c_groups', { _id: { $in: groupIds } }, orgReferenceProps), + groupTasks = await this.getExportObjects(org, 'c_group_tasks', { c_group: { $in: groups.map(v => v._id) } }, orgReferenceProps) + + return [...groups, ...groupTasks] + } + writeIssues(removedEntities) { const { options } = privatesAccessor(this), outputDir = options.dir || process.cwd(), diff --git a/packages/mdctl-cli/tasks/study.js b/packages/mdctl-cli/tasks/study.js index af0b1c21..83a0e9c1 100644 --- a/packages/mdctl-cli/tasks/study.js +++ b/packages/mdctl-cli/tasks/study.js @@ -41,6 +41,10 @@ class Study extends Task { manifestOnly: { type: 'boolean', default: false + }, + manifestObject: { + type: 'string', + default: '' } } @@ -74,10 +78,11 @@ class Study extends Task { async 'study@export'(cli) { const client = await cli.getApiClient({ credentials: await cli.getAuthOptions() }), params = await cli.getArguments(this.optionKeys), - studyTools = new StudyManifestTools(client, params) + studyTools = new StudyManifestTools(client, params), + manifestObj = params.manifestObject try { - const { manifest } = await studyTools.getStudyManifest() + const { manifest } = await studyTools.getStudyManifest(manifestObj) if (!params.manifestOnly) { const options = { @@ -99,13 +104,12 @@ class Study extends Task { async 'study@import'(cli) { console.log('Starting Study Import') - const params = await cli.getArguments(this.optionKeys) + const params = await cli.getArguments(this.optionKeys), + env = new Env() params.triggers = false params.backup = false - const env = new Env() - await env['env@import'](cli) } @@ -233,7 +237,7 @@ class Study extends Task { Usage: - mdctl study [command] + mdctl study [command] --manifestObject Arguments: @@ -241,7 +245,28 @@ class Study extends Task { export - Exports the study from the current org import - Imports the study into the current org task [action] - Allows the select of tasks to export from the current org - consent [action] - Allows the select of consent templates to export from the current org + consent [action] - Allows the select of consent templates to export from the current org + + Options + + --manifestObject - receives a valid manifest JSON object or the path to a manifest file + + Notes + + --manifestObject is \x1b[4monly available for "export" command\x1b[0m, and it is expected to have the following format: + { + "": { + "includes": [ + "key_1", "key_2", etc... + ] + }, + "": { + "includes": [ + "key_N", "key_N+1", etc... + ] + } + "object": "manifest" + } ` } From bebc57ad6eceac9c45942cd8e3b035cc13af4b4a Mon Sep 17 00:00:00 2001 From: Davide Aquaro Date: Thu, 2 Jun 2022 17:49:42 +0100 Subject: [PATCH 2/8] MIG-85 : added unit tests and improved exception handling in loadManifest --- .../__tests__/MIG-85/MIG-85.test.js | 310 ++++++++++++++++++ .../__tests__/MIG-85/wrongManifest.json | 1 + .../lib/StudyManifestTools.js | 39 ++- 3 files changed, 343 insertions(+), 7 deletions(-) create mode 100644 packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js create mode 100644 packages/mdctl-axon-tools/__tests__/MIG-85/wrongManifest.json diff --git a/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js b/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js new file mode 100644 index 00000000..e536f7d9 --- /dev/null +++ b/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js @@ -0,0 +1,310 @@ +/* eslint-disable import/order */ + +jest.mock('@medable/mdctl-api-driver', () => ({ Driver: class {} }), { virtual: true }) +jest.mock('@medable/mdctl-api-driver/lib/cortex.object', () => ({ Object: class {}, Org: class {} }), { virtual: true }) +jest.mock('../../lib/mappings') + +const fs = require('fs'), + StudyManifestTools = require('../../lib/StudyManifestTools') + +describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { + + let manifestTools + const mockGetExportedObjects = jest.fn(() => []), + existingStudy = { + _id: '1', + c_name: 'Study', + c_key: 'abc' + }, + hasNextStudyMock = jest.fn(() => true), + nextStudyMock = jest.fn(() => existingStudy), + hasNextStudySchema = jest.fn(() => true), + nextStudySchemaMock = jest.fn(() => ({ _id: '1', object: 'object', properties: [{ name: 'c_no_pii' }] })), + entities = [{ + _id: '615bcd016631cc0100d2766c', + object: 'c_study', + c_key: 'key-001' + }, + { + _id: '615b60d1bf2e4301008f4d68', + object: 'c_dummy_object', + c_key: 'key-002' + }, + { + _id: '619aaaafe44c6e01003f7313', + object: 'c_task', + c_key: 'key-003' + }, + { + _id: '61981246ca9563010037bfa8', + object: 'c_task', + c_key: 'key-004' + }, + { + _id: '61981246ca95714c14e61a8c', + object: 'c_step', + c_key: 'key-005' + }, + { + _id: '61981246ca966caef6108f28', + object: 'c_step', + c_key: 'key-006' + }, + { + _id: '61981246ca9592ee0e41a3dd', + object: 'ec__document_template', + c_key: 'key-007' + }, + { + _id: '61980eb292466ea32e087378', + object: 'ec__document_template', + c_key: 'key-008' + }, + { + _id: '6d525cf2e328e7300d97c399', + object: 'ec__default_document_css', + c_key: 'key-009' + }, + { + _id: '6d525cfe328e64ac0833baef', + object: 'ec__knowledge_check', + c_key: 'key-010' + }, + { + _id: '6d525f2e328e7f1e48262523', + object: 'ec__knowledge_check', + c_key: 'key-011' + }, + { + _id: '6d525gbed28e7f1e4826bb76', + object: 'c_visit_schedule', + c_key: 'key-012' + }, + { + _id: '6d525gc1408e7f1e4826bb11', + object: 'c_visit', + c_key: 'key-013' + }, + { + _id: '6d525gbe28e7fc4ff43c310', + object: 'c_group', + c_key: 'key-014' + }, + { + _id: '67725gbe28e7f98ee3c8667', + object: 'c_group_task', + c_key: 'key-015' + }], + dummyReferences = [ + { + name: 'c_study', + array: false, + object: 'c_study', + type: 'Reference', + required: false + } + ], + org = { + objects: { + c_study: { + readOne: () => ({ + skipAcl: () => ({ + grant: () => ({ + paths: () => ({ + hasNext: hasNextStudyMock, + next: nextStudyMock + }) + }) + }) + }) + }, + object: { + find: () => ({ + skipAcl: () => ({ + grant: () => ({ + paths: () => ({ + hasNext: hasNextStudySchema, + next: nextStudySchemaMock + }) + }) + }) + }) + } + } + } + + beforeAll(async() => { + manifestTools = new StudyManifestTools({}) + manifestTools.getExportObjects = mockGetExportedObjects + }) + + afterEach(() => { + jest.clearAllMocks() + }) + + it('Test visit schedules manifest creation', async() => { + const manifest = { + object: 'manifest', + dependencies: false, + exportOwner: false, + importOwner: false, + c_visit_schedule: { + includes: [ + 'key-012' + ] + }, + c_visit: { + includes: [ + 'key-013' + ] + } + } + + jest.spyOn(StudyManifestTools.prototype, 'getOrgObjectInfo').mockImplementation(() => dummyReferences) + jest.spyOn(StudyManifestTools.prototype, 'validateReferences').mockImplementation(() => entities) + jest.spyOn(StudyManifestTools.prototype, 'createManifest').mockImplementation(() => manifest) + + // eslint-disable-next-line one-var + const manifestAndDeps = await manifestTools.buildVisitManifestAndDependencies(['6d525gbev28e7f1e4826bb76', '6d525g12328e7f1e4826bb11']) + + expect(manifestAndDeps.manifest) + .toStrictEqual(manifest) + expect(manifestAndDeps.removedEntities) + .toBeUndefined() + }) + + it('Test groups manifest creation', async() => { + const manifest = { + object: 'manifest', + dependencies: false, + exportOwner: false, + importOwner: false, + c_group: { + includes: [ + 'key-014' + ] + }, + c_group_task: { + includes: [ + 'key-015' + ] + } + } + jest.spyOn(StudyManifestTools.prototype, 'getOrgObjectInfo').mockImplementation(() => dummyReferences) + jest.spyOn(StudyManifestTools.prototype, 'validateReferences').mockImplementation(() => entities) + jest.spyOn(StudyManifestTools.prototype, 'createManifest').mockImplementation(() => manifest) + + // eslint-disable-next-line one-var + const manifestAndDeps = await manifestTools.buildGroupManifestAndDependencies(['6d525gbe28e7fc4ff43c310', '67725gbe28e7f98ee3c8667']) + + expect(manifestAndDeps.manifest) + .toStrictEqual(manifest) + expect(manifestAndDeps.removedEntities) + .toBeUndefined() + }) + + it('Test loadManifest function - Invalid manifest', async() => { + const manifest = `{ + "object": "something", + "c_group": { + "includes": [ + "key-014" + ] + }, + "c_group_task": { + "includes": [ + "key-015" + ] + } + }` + + let res + try { + res = manifestTools.loadManifest(manifest) + } catch (err) { + res = err + } + + expect(res.errCode) + .toBe('mdctl.invalidArgument.unspecified') + expect(res.code) + .toBe('kInvalidArgument') + expect(res.reason) + .toBe('The argument is not a valid manifest') + }) + + it('Test loadManifest function - Invalid file path', async() => { + const manifest = `${__dirname}/wrongPath.json` + + let res + try { + res = manifestTools.loadManifest(manifest) + } catch (err) { + res = err + } + + expect(res.errCode) + .toBe('mdctl.invalidArgument.unspecified') + expect(res.code) + .toBe('kInvalidArgument') + expect(res.reason) + .toBe('The manifest file does not exists') + }) + + it('Test loadManifest function - Invalid JSON file', async() => { + const manifest = `${__dirname}/wrongManifest.json` + + let res + try { + res = manifestTools.loadManifest(manifest) + } catch (err) { + res = err + } + + expect(res.errCode) + .toBe('mdctl.invalidArgument.unspecified') + expect(res.code) + .toBe('kInvalidArgument') + expect(res.reason) + .toBe('The manifest file is not a valid JSON') + }) + + it('Test study manifest creation from manifest', async() => { + const manifest = { + object: 'manifest', + c_task: { + includes: [ + 'key-003', + 'key-004' + ] + }, + c_step: { + includes: [ + 'key-005', + 'key-006' + ] + } + }, + ingestTransform = fs.readFileSync(`${__dirname}/../../packageScripts/ingestTransform.js`).toString(), + stringifiedManifest = JSON.stringify(manifest) + + + // jest.spyOn(StudyManifestTools.prototype, 'getFirstStudy').mockImplementation(() => org) + jest.spyOn(StudyManifestTools.prototype, 'getOrgObjectInfo').mockImplementation(() => dummyReferences) + jest.spyOn(StudyManifestTools.prototype, 'validateReferences').mockImplementation(() => entities) + jest.spyOn(StudyManifestTools.prototype, 'createManifest').mockImplementation(() => manifest) + jest.spyOn(StudyManifestTools.prototype, 'getObjectIDsArray').mockImplementation(() => entities.filter(o => o.object === 'c_task')) + jest.spyOn(StudyManifestTools.prototype, 'mapObjectNameToPlural').mockImplementation(() => 'c_tasks') + // eslint-disable-next-line one-var + const manifestAndDeps = await manifestTools.buildManifestAndDependencies(stringifiedManifest) + + expect(manifestAndDeps.manifest) + .toStrictEqual(manifest) + expect(manifestAndDeps.removedEntities) + .toBeUndefined() + expect(manifestAndDeps.mappingScript) + .toBeUndefined() + expect(manifestAndDeps.ingestTransform) + .toStrictEqual(ingestTransform) + }) +}) diff --git a/packages/mdctl-axon-tools/__tests__/MIG-85/wrongManifest.json b/packages/mdctl-axon-tools/__tests__/MIG-85/wrongManifest.json new file mode 100644 index 00000000..6b7f8497 --- /dev/null +++ b/packages/mdctl-axon-tools/__tests__/MIG-85/wrongManifest.json @@ -0,0 +1 @@ +invalid json \ No newline at end of file diff --git a/packages/mdctl-axon-tools/lib/StudyManifestTools.js b/packages/mdctl-axon-tools/lib/StudyManifestTools.js index ca4e37e8..57195583 100644 --- a/packages/mdctl-axon-tools/lib/StudyManifestTools.js +++ b/packages/mdctl-axon-tools/lib/StudyManifestTools.js @@ -27,17 +27,24 @@ class StudyManifestTools { let manifestJSON try { manifestJSON = JSON.parse(manifestObject) - if (manifestJSON.object !== 'manifest') { + if (!manifestJSON.object || manifestJSON.object !== 'manifest') { throw Fault.create('kInvalidArgument', { reason: 'The argument is not a valid manifest' }) } } catch (e) { - try { - if (fs.existsSync(manifestObject)) { - throw Fault.create('kInvalidArgument', { reason: 'The manifest file does not exists' }) + if (!(e instanceof SyntaxError)) { + throw e + } else { + try { + if (!fs.existsSync(manifestObject)) { + throw Fault.create('kInvalidArgument', { reason: 'The manifest file does not exists' }) + } + manifestJSON = JSON.parse(fs.readFileSync(manifestObject)) + } catch (err) { + if (err instanceof SyntaxError) { + throw Fault.create('kInvalidArgument', { reason: 'The manifest file is not a valid JSON' }) + } + throw err } - manifestJSON = JSON.parse(fs.readFileSync(manifestObject)) - } catch (err) { - throw Fault.create('kInvalidArgument', { reason: 'The manifest file is not a valid JSON' }) } } return manifestJSON @@ -273,6 +280,24 @@ class StudyManifestTools { return { manifest, removedEntities } } + // This function is not used but it's handy to have for unit tests + async buildVisitManifestAndDependencies(visitIds) { + const { org, orgReferenceProps } = await this.getOrgAndReferences(), + allEntities = await this.getVisitManifestEntities(org, visitIds, orgReferenceProps), + { manifest, removedEntities } = this.validateAndCreateManifest(allEntities, orgReferenceProps, ['c_visit_schedules']) + + return { manifest, removedEntities } + } + + // This function is not used but it's handy to have for unit tests + async buildGroupManifestAndDependencies(groupIds) { + const { org, orgReferenceProps } = await this.getOrgAndReferences(), + allEntities = await this.getGroupManifestEntities(org, groupIds, orgReferenceProps), + { manifest, removedEntities } = this.validateAndCreateManifest(allEntities, orgReferenceProps, ['c_groups']) + + return { manifest, removedEntities } + } + createManifest(entities) { const manifest = { object: 'manifest', From d08fd8745e056aeb8529a9fb62e4eca76a912996 Mon Sep 17 00:00:00 2001 From: Davide Aquaro Date: Fri, 3 Jun 2022 08:50:41 +0100 Subject: [PATCH 3/8] feat: test for getObjectIDsArray function and code cleanup --- .../__tests__/MIG-85/MIG-85.test.js | 42 +++++-------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js b/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js index e536f7d9..dcea21c8 100644 --- a/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js +++ b/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js @@ -11,15 +11,6 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { let manifestTools const mockGetExportedObjects = jest.fn(() => []), - existingStudy = { - _id: '1', - c_name: 'Study', - c_key: 'abc' - }, - hasNextStudyMock = jest.fn(() => true), - nextStudyMock = jest.fn(() => existingStudy), - hasNextStudySchema = jest.fn(() => true), - nextStudySchemaMock = jest.fn(() => ({ _id: '1', object: 'object', properties: [{ name: 'c_no_pii' }] })), entities = [{ _id: '615bcd016631cc0100d2766c', object: 'c_study', @@ -106,27 +97,10 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { ], org = { objects: { - c_study: { - readOne: () => ({ - skipAcl: () => ({ - grant: () => ({ - paths: () => ({ - hasNext: hasNextStudyMock, - next: nextStudyMock - }) - }) - }) - }) - }, - object: { + c_task: { find: () => ({ - skipAcl: () => ({ - grant: () => ({ - paths: () => ({ - hasNext: hasNextStudySchema, - next: nextStudySchemaMock - }) - }) + limit: () => ({ + toArray: () => entities.filter(e => e.object === 'c_task') }) }) } @@ -288,8 +262,6 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { ingestTransform = fs.readFileSync(`${__dirname}/../../packageScripts/ingestTransform.js`).toString(), stringifiedManifest = JSON.stringify(manifest) - - // jest.spyOn(StudyManifestTools.prototype, 'getFirstStudy').mockImplementation(() => org) jest.spyOn(StudyManifestTools.prototype, 'getOrgObjectInfo').mockImplementation(() => dummyReferences) jest.spyOn(StudyManifestTools.prototype, 'validateReferences').mockImplementation(() => entities) jest.spyOn(StudyManifestTools.prototype, 'createManifest').mockImplementation(() => manifest) @@ -307,4 +279,12 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { expect(manifestAndDeps.ingestTransform) .toStrictEqual(ingestTransform) }) + + it('Test getObjectIDsArray function', async() => { + const filteredEntities = entities.filter(e => e.object === 'c_task'), + res = await manifestTools.getObjectIDsArray(org, 'c_task', 'c_key', ['key-003', 'key-004']) + + expect(res) + .toStrictEqual(filteredEntities) + }) }) From c34dd8a938cd070264ae7cd27923b57309716113 Mon Sep 17 00:00:00 2001 From: Davide Aquaro Date: Fri, 3 Jun 2022 12:26:28 +0100 Subject: [PATCH 4/8] refactor: code cleanup --- .../__tests__/MIG-85/MIG-85.test.js | 12 ++++++------ .../lib/StudyManifestTools.js | 19 ++++++++----------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js b/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js index dcea21c8..85730cca 100644 --- a/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js +++ b/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js @@ -177,7 +177,7 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { .toBeUndefined() }) - it('Test loadManifest function - Invalid manifest', async() => { + it('Test validateManifest function - Invalid manifest', async() => { const manifest = `{ "object": "something", "c_group": { @@ -194,7 +194,7 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { let res try { - res = manifestTools.loadManifest(manifest) + res = manifestTools.validateManifest(manifest) } catch (err) { res = err } @@ -207,12 +207,12 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { .toBe('The argument is not a valid manifest') }) - it('Test loadManifest function - Invalid file path', async() => { + it('Test validateManifest function - Invalid file path', async() => { const manifest = `${__dirname}/wrongPath.json` let res try { - res = manifestTools.loadManifest(manifest) + res = manifestTools.validateManifest(manifest) } catch (err) { res = err } @@ -225,12 +225,12 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { .toBe('The manifest file does not exists') }) - it('Test loadManifest function - Invalid JSON file', async() => { + it('Test validateManifest function - Invalid JSON file', async() => { const manifest = `${__dirname}/wrongManifest.json` let res try { - res = manifestTools.loadManifest(manifest) + res = manifestTools.validateManifest(manifest) } catch (err) { res = err } diff --git a/packages/mdctl-axon-tools/lib/StudyManifestTools.js b/packages/mdctl-axon-tools/lib/StudyManifestTools.js index 57195583..6b8fb8d6 100644 --- a/packages/mdctl-axon-tools/lib/StudyManifestTools.js +++ b/packages/mdctl-axon-tools/lib/StudyManifestTools.js @@ -23,7 +23,7 @@ class StudyManifestTools { }) } - loadManifest(manifestObject) { + validateManifest(manifestObject) { let manifestJSON try { manifestJSON = JSON.parse(manifestObject) @@ -180,28 +180,25 @@ class StudyManifestTools { } async buildManifestAndDependencies(manifestObject) { - let allEntities, + let ignoreKeys = [], mappingScript, - ignoreKeys + study, + manifestJSON const { org, orgReferenceProps } = await this.getOrgAndReferences(), ingestTransform = fs.readFileSync(`${__dirname}/../packageScripts/ingestTransform.js`) if (manifestObject) { - const manifestJSON = this.loadManifest(manifestObject) + manifestJSON = this.validateManifest(manifestObject) ignoreKeys = Object.keys(manifestJSON).filter(key => typeof manifestJSON[key] === 'object') .map(key => this.mapObjectNameToPlural(key)) - - allEntities = await this.getStudyManifestEntities(org, null, manifestJSON, orgReferenceProps) } else { - const study = await this.getFirstStudy(org) - ignoreKeys = [] - allEntities = [study, - ...await this.getStudyManifestEntities(org, study, null, orgReferenceProps)] + study = await this.getFirstStudy(org) mappingScript = await getMappingScript(org) } + const allEntities = await this.getStudyManifestEntities(org, study, manifestJSON, orgReferenceProps) const { manifest, removedEntities } = this .validateAndCreateManifest(allEntities, orgReferenceProps, ignoreKeys) @@ -746,7 +743,7 @@ class StudyManifestTools { const manifestEntities = [], // Define the available entities to export or get them from the manifest in input manifestKeys = (study) - ? ['c_task', 'c_visit_schedule', 'ec__document_template', 'c_group', + ? ['c_study', 'c_task', 'c_visit_schedule', 'ec__document_template', 'c_group', 'c_anchor_date_template', 'c_fault', 'c_dmweb_report', 'c_site', 'c_task_assignment', 'c_participant_schedule', 'c_patient_flag', 'c_looker_integration_record', 'int__vendor_integration_record', 'int__model_mapping', 'int__pipeline', 'orac__studies', 'orac__sites', 'orac__forms', 'orac__form_questions', 'orac__events'] From 46693d9cca3aa36d46ab7c72785be3920b14500f Mon Sep 17 00:00:00 2001 From: Davide Aquaro Date: Fri, 3 Jun 2022 17:38:03 +0100 Subject: [PATCH 5/8] refactor: applied suggestions as per code review feedback --- .../lib/StudyManifestTools.js | 105 ++++-------------- packages/mdctl-cli/tasks/study.js | 34 +++++- 2 files changed, 52 insertions(+), 87 deletions(-) diff --git a/packages/mdctl-axon-tools/lib/StudyManifestTools.js b/packages/mdctl-axon-tools/lib/StudyManifestTools.js index 6b8fb8d6..22ab5e46 100644 --- a/packages/mdctl-axon-tools/lib/StudyManifestTools.js +++ b/packages/mdctl-axon-tools/lib/StudyManifestTools.js @@ -23,31 +23,20 @@ class StudyManifestTools { }) } - validateManifest(manifestObject) { - let manifestJSON - try { - manifestJSON = JSON.parse(manifestObject) - if (!manifestJSON.object || manifestJSON.object !== 'manifest') { - throw Fault.create('kInvalidArgument', { reason: 'The argument is not a valid manifest' }) - } - } catch (e) { - if (!(e instanceof SyntaxError)) { - throw e - } else { - try { - if (!fs.existsSync(manifestObject)) { - throw Fault.create('kInvalidArgument', { reason: 'The manifest file does not exists' }) - } - manifestJSON = JSON.parse(fs.readFileSync(manifestObject)) - } catch (err) { - if (err instanceof SyntaxError) { - throw Fault.create('kInvalidArgument', { reason: 'The manifest file is not a valid JSON' }) - } - throw err - } - } + getAvailableObjectNames() { + return ['c_study', 'c_task', 'c_visit_schedule', 'ec__document_template', 'c_group', + 'c_anchor_date_template', 'c_fault', 'c_dmweb_report', 'c_site', 'c_task_assignment', 'c_participant_schedule', + 'c_patient_flag', 'c_looker_integration_record', 'int__vendor_integration_record', 'int__model_mapping', + 'int__pipeline', 'orac__studies', 'orac__sites', 'orac__forms', 'orac__form_questions', 'orac__events'] + } + + validateAndCleanManifest(manifestJSON) { + if (!manifestJSON.object || manifestJSON.object !== 'manifest') { + throw Fault.create('kInvalidArgument', { reason: 'The argument is not a valid manifest' }) } - return manifestJSON + return Object.keys(manifestJSON) + .filter(key => this.getAvailableObjectNames().includes(key)) + .reduce((curr, key) => Object.assign(curr, { [key]: manifestJSON[key] }), {}) } async getTasks() { @@ -179,26 +168,25 @@ class StudyManifestTools { return { manifest, removedEntities } } - async buildManifestAndDependencies(manifestObject) { + async buildManifestAndDependencies(manifestJSON) { let ignoreKeys = [], mappingScript, - study, - manifestJSON + cleanManifest, + study const { org, orgReferenceProps } = await this.getOrgAndReferences(), ingestTransform = fs.readFileSync(`${__dirname}/../packageScripts/ingestTransform.js`) - if (manifestObject) { - manifestJSON = this.validateManifest(manifestObject) + if (manifestJSON) { + cleanManifest = this.validateAndCleanManifest(manifestJSON) - ignoreKeys = Object.keys(manifestJSON).filter(key => typeof manifestJSON[key] === 'object') - .map(key => this.mapObjectNameToPlural(key)) + ignoreKeys = Object.keys(cleanManifest).map(key => this.mapObjectNameToPlural(key)) } else { study = await this.getFirstStudy(org) mappingScript = await getMappingScript(org) } - const allEntities = await this.getStudyManifestEntities(org, study, manifestJSON, orgReferenceProps) + const allEntities = await this.getStudyManifestEntities(org, study, cleanManifest, orgReferenceProps) const { manifest, removedEntities } = this .validateAndCreateManifest(allEntities, orgReferenceProps, ignoreKeys) @@ -693,61 +681,10 @@ class StudyManifestTools { .reduce((acc, curr) => acc.concat(curr), []) } - /* - async getStudyManifestEntities(org, study, orgReferenceProps) { - const taskIds = (await org.objects.c_tasks.find({ c_study: study._id }).limit(false).toArray()).map(v => v._id), - consentIds = (await org.objects.ec__document_templates.find({ c_study: study._id }).limit(false).toArray()).map(v => v._id), - visitsIds = (await org.objects.c_visit_schedules.find({ c_study: study._id }).limit(false).toArray()).map(v => v._id), - groupsIds = (await org.objects.c_groups.find({ c_study: study._id }).limit(false).toArray()).map(v => v._id), - - tasksAndDependencies = await this.getTaskManifestEntities(org, taskIds, orgReferenceProps), - templatesAndDependencies = await this.getConsentManifestEntities(org, consentIds, orgReferenceProps), - visitSchedulesAndDependencies = await this.getVisitManifestEntities(org, visitsIds, orgReferenceProps), - groupsAndDependencies = await this.getGroupManifestEntities(org, groupsIds, orgReferenceProps), - - faults = await this.getExportObjects(org, 'c_faults', null, orgReferenceProps), - reports = await this.getExportObjects(org, 'c_dmweb_reports', null, orgReferenceProps), - sites = await this.getExportObjects(org, 'c_sites', { c_study: study._id }, orgReferenceProps), - anchorDateTemplates = await this.getExportObjects(org, 'c_anchor_date_templates', { c_study: study._id }, orgReferenceProps), - - taskAssignments = await this.getExportObjects(org, 'c_task_assignments', null, orgReferenceProps), - participantSchedules = await this.getExportObjects(org, 'c_participant_schedules', null, orgReferenceProps), - patientFlags = await this.getExportObjects(org, 'c_patient_flags', null, orgReferenceProps), - - // looker - lookerIntegrationRecords = await this.getExportObjects(org, 'c_looker_integration_records', null, orgReferenceProps), - - // integrations - vendorIntegrationRecords = await this.getExportObjects(org, 'int__vendor_integration_records', null, orgReferenceProps), - integrationMappings = await this.getExportObjects(org, 'int__model_mappings', null, orgReferenceProps), - integrationPipelines = await this.getExportObjects(org, 'int__pipelines', null, orgReferenceProps), - - // oracle - oracleStudies = await this.getExportObjects(org, 'orac__studies', null, orgReferenceProps), - oracleSites = await this.getExportObjects(org, 'orac__sites', null, orgReferenceProps), - oracleForms = await this.getExportObjects(org, 'orac__forms', null, orgReferenceProps), - oracleQuestions = await this.getExportObjects(org, 'orac__form_questions', null, orgReferenceProps), - oracleEvents = await this.getExportObjects(org, 'orac__events', null, orgReferenceProps) - - return [ - ...tasksAndDependencies, ...visitSchedulesAndDependencies, ...faults, ...reports, ...sites, - ...groupsAndDependencies, ...taskAssignments, ...participantSchedules, ...anchorDateTemplates, - ...patientFlags, ...templatesAndDependencies, ...lookerIntegrationRecords, - ...vendorIntegrationRecords, ...integrationMappings, ...integrationPipelines, - ...oracleStudies, ...oracleSites, ...oracleForms, ...oracleQuestions, ...oracleEvents - ] - } - */ - async getStudyManifestEntities(org, study, manifestObject, orgReferenceProps) { const manifestEntities = [], // Define the available entities to export or get them from the manifest in input - manifestKeys = (study) - ? ['c_study', 'c_task', 'c_visit_schedule', 'ec__document_template', 'c_group', - 'c_anchor_date_template', 'c_fault', 'c_dmweb_report', 'c_site', 'c_task_assignment', 'c_participant_schedule', - 'c_patient_flag', 'c_looker_integration_record', 'int__vendor_integration_record', 'int__model_mapping', - 'int__pipeline', 'orac__studies', 'orac__sites', 'orac__forms', 'orac__form_questions', 'orac__events'] - : Object.keys(manifestObject).filter(key => typeof manifestObject[key] === 'object') + manifestKeys = study ? this.getAvailableObjectNames() : Object.keys(manifestObject) // eslint-disable-next-line no-restricted-syntax for await (const key of manifestKeys) { diff --git a/packages/mdctl-cli/tasks/study.js b/packages/mdctl-cli/tasks/study.js index 83a0e9c1..b5d76d08 100644 --- a/packages/mdctl-cli/tasks/study.js +++ b/packages/mdctl-cli/tasks/study.js @@ -1,6 +1,7 @@ /* eslint-disable class-methods-use-this */ const _ = require('lodash'), + fs = require('fs'), jsyaml = require('js-yaml'), { rString, isSet } = require('@medable/mdctl-core-utils/values'), { StudyManifestTools } = require('@medable/mdctl-axon-tools'), @@ -82,7 +83,11 @@ class Study extends Task { manifestObj = params.manifestObject try { - const { manifest } = await studyTools.getStudyManifest(manifestObj) + let manifestJSON + if (manifestObj) { + manifestJSON = this.validateManifest(manifestObj) + } + const { manifest } = await studyTools.getStudyManifest(manifestJSON) if (!params.manifestOnly) { const options = { @@ -191,6 +196,26 @@ class Study extends Task { } + validateManifest(manifestObject) { + let manifestJSON + try { + manifestJSON = JSON.parse(manifestObject) + } catch (e) { + try { + if (!fs.existsSync(manifestObject)) { + throw Fault.create('kInvalidArgument', { reason: 'The manifest file does not exists' }) + } + manifestJSON = JSON.parse(fs.readFileSync(manifestObject)) + } catch (err) { + if (err instanceof SyntaxError) { + throw Fault.create('kInvalidArgument', { reason: 'The manifest is not a valid JSON' }) + } + throw err + } + } + return manifestJSON + } + mergeJsonArgIf(options, arg) { const value = this.args(arg) @@ -249,8 +274,11 @@ class Study extends Task { Options - --manifestObject - receives a valid manifest JSON object or the path to a manifest file - + --manifestObject - receives a valid manifest JSON object \x1b[4OR\x1b[0m the path to a manifest file to + specify the entities to export (e.g. tasks and consents, etc...). + The manifest can only contain object instances, other org config objects + can be exported through "mdctl env export" command + Notes --manifestObject is \x1b[4monly available for "export" command\x1b[0m, and it is expected to have the following format: From 0cafdcdb66ab3416e5b98ca65fe241e75e5c6c61 Mon Sep 17 00:00:00 2001 From: Davide Aquaro Date: Tue, 7 Jun 2022 09:39:08 +0100 Subject: [PATCH 6/8] refactor: fixed unit tests --- .../__tests__/MIG-85/MIG-85.test.js | 57 ++++++------------- .../test/tasks/data}/wrongManifest.json | 0 packages/mdctl-cli/test/tasks/study.js | 48 ++++++++++++++++ 3 files changed, 64 insertions(+), 41 deletions(-) rename packages/{mdctl-axon-tools/__tests__/MIG-85 => mdctl-cli/test/tasks/data}/wrongManifest.json (100%) create mode 100644 packages/mdctl-cli/test/tasks/study.js diff --git a/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js b/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js index 85730cca..54db5873 100644 --- a/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js +++ b/packages/mdctl-axon-tools/__tests__/MIG-85/MIG-85.test.js @@ -5,7 +5,8 @@ jest.mock('@medable/mdctl-api-driver/lib/cortex.object', () => ({ Object: class jest.mock('../../lib/mappings') const fs = require('fs'), - StudyManifestTools = require('../../lib/StudyManifestTools') + StudyManifestTools = require('../../lib/StudyManifestTools'), + study = require('../../../mdctl-cli/tasks/study') describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { @@ -194,7 +195,7 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { let res try { - res = manifestTools.validateManifest(manifest) + res = manifestTools.validateAndCleanManifest(manifest) } catch (err) { res = err } @@ -207,42 +208,6 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { .toBe('The argument is not a valid manifest') }) - it('Test validateManifest function - Invalid file path', async() => { - const manifest = `${__dirname}/wrongPath.json` - - let res - try { - res = manifestTools.validateManifest(manifest) - } catch (err) { - res = err - } - - expect(res.errCode) - .toBe('mdctl.invalidArgument.unspecified') - expect(res.code) - .toBe('kInvalidArgument') - expect(res.reason) - .toBe('The manifest file does not exists') - }) - - it('Test validateManifest function - Invalid JSON file', async() => { - const manifest = `${__dirname}/wrongManifest.json` - - let res - try { - res = manifestTools.validateManifest(manifest) - } catch (err) { - res = err - } - - expect(res.errCode) - .toBe('mdctl.invalidArgument.unspecified') - expect(res.code) - .toBe('kInvalidArgument') - expect(res.reason) - .toBe('The manifest file is not a valid JSON') - }) - it('Test study manifest creation from manifest', async() => { const manifest = { object: 'manifest', @@ -259,8 +224,7 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { ] } }, - ingestTransform = fs.readFileSync(`${__dirname}/../../packageScripts/ingestTransform.js`).toString(), - stringifiedManifest = JSON.stringify(manifest) + ingestTransform = fs.readFileSync(`${__dirname}/../../packageScripts/ingestTransform.js`).toString() jest.spyOn(StudyManifestTools.prototype, 'getOrgObjectInfo').mockImplementation(() => dummyReferences) jest.spyOn(StudyManifestTools.prototype, 'validateReferences').mockImplementation(() => entities) @@ -268,7 +232,7 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { jest.spyOn(StudyManifestTools.prototype, 'getObjectIDsArray').mockImplementation(() => entities.filter(o => o.object === 'c_task')) jest.spyOn(StudyManifestTools.prototype, 'mapObjectNameToPlural').mockImplementation(() => 'c_tasks') // eslint-disable-next-line one-var - const manifestAndDeps = await manifestTools.buildManifestAndDependencies(stringifiedManifest) + const manifestAndDeps = await manifestTools.buildManifestAndDependencies(manifest) expect(manifestAndDeps.manifest) .toStrictEqual(manifest) @@ -287,4 +251,15 @@ describe('MIG-85 - Test partial migrations in StudyManifestTools', () => { expect(res) .toStrictEqual(filteredEntities) }) + + it('Test getAvailableObjectNames', () => { + const availableObjects = manifestTools.getAvailableObjectNames(), + expectedObjects = ['c_study', 'c_task', 'c_visit_schedule', 'ec__document_template', 'c_group', + 'c_anchor_date_template', 'c_fault', 'c_dmweb_report', 'c_site', 'c_task_assignment', 'c_participant_schedule', + 'c_patient_flag', 'c_looker_integration_record', 'int__vendor_integration_record', 'int__model_mapping', + 'int__pipeline', 'orac__studies', 'orac__sites', 'orac__forms', 'orac__form_questions', 'orac__events'] + + expect(availableObjects) + .toStrictEqual(expectedObjects) + }) }) diff --git a/packages/mdctl-axon-tools/__tests__/MIG-85/wrongManifest.json b/packages/mdctl-cli/test/tasks/data/wrongManifest.json similarity index 100% rename from packages/mdctl-axon-tools/__tests__/MIG-85/wrongManifest.json rename to packages/mdctl-cli/test/tasks/data/wrongManifest.json diff --git a/packages/mdctl-cli/test/tasks/study.js b/packages/mdctl-cli/test/tasks/study.js new file mode 100644 index 00000000..24d161e1 --- /dev/null +++ b/packages/mdctl-cli/test/tasks/study.js @@ -0,0 +1,48 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +const { assert } = require('chai'), + Study = require('../../tasks/study') + +let study + +describe('MIG-85 - Test partial migrations in study.js module', () => { + before(() => { + study = new Study() + }) + + it('Test validateManifest function - Invalid file path', () => { + const manifest = `${__dirname}/wrongPath.json` + + let res + try { + res = study.validateManifest(manifest) + } catch (err) { + res = err + } + + assert + .equal(res.errCode, 'mdctl.invalidArgument.unspecified') + assert + .equal(res.code, 'kInvalidArgument') + assert + .equal(res.reason, 'The manifest file does not exists') + }) + + it('Test validateManifest function - Invalid JSON file', () => { + const manifest = `${__dirname}/data/wrongManifest.json` + + let res + try { + res = study.validateManifest(manifest) + } catch (err) { + res = err + } + + assert + .equal(res.errCode, 'mdctl.invalidArgument.unspecified') + assert + .equal(res.code, 'kInvalidArgument') + assert + .equal(res.reason, 'The manifest is not a valid JSON') + }) + +}) From e44f2f37fda8dd8bd4711727e122d327aa0c0450 Mon Sep 17 00:00:00 2001 From: Davide Aquaro Date: Tue, 7 Jun 2022 15:37:57 +0100 Subject: [PATCH 7/8] feat: ignore keys in manifest other than tasks and templates --- packages/mdctl-cli/tasks/study.js | 9 ++++ packages/mdctl-cli/test/tasks/study.js | 61 +++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/packages/mdctl-cli/tasks/study.js b/packages/mdctl-cli/tasks/study.js index b5d76d08..c70f7c20 100644 --- a/packages/mdctl-cli/tasks/study.js +++ b/packages/mdctl-cli/tasks/study.js @@ -213,6 +213,15 @@ class Study extends Task { throw err } } + /* + Ignore any keys passed in other than Assignments and eConsents. + In future this will be removed but for now we will only support those 2 objects together + */ + manifestJSON = _.pick(manifestJSON, ['c_task', 'ec__document_template', 'object']) + if (_.isEqual(manifestJSON, { object: 'manifest' })) { + // This means that the manifest passed does not contain Assignments or eConsents + throw Fault.create('kInvalidArgument', { reason: 'No Assignments or eConsents to export' }) + } return manifestJSON } diff --git a/packages/mdctl-cli/test/tasks/study.js b/packages/mdctl-cli/test/tasks/study.js index 24d161e1..8fcbbd97 100644 --- a/packages/mdctl-cli/test/tasks/study.js +++ b/packages/mdctl-cli/test/tasks/study.js @@ -1,5 +1,5 @@ // eslint-disable-next-line import/no-extraneous-dependencies -const { assert } = require('chai'), +const { assert, expect } = require('chai'), Study = require('../../tasks/study') let study @@ -45,4 +45,63 @@ describe('MIG-85 - Test partial migrations in study.js module', () => { .equal(res.reason, 'The manifest is not a valid JSON') }) + it('Test validateManifest function - No entities to export', () => { + const manifest = `{ + "object": "manifest", + "c_group": { + "includes": [ + "key-014" + ] + }, + "c_group_task": { + "includes": [ + "key-015" + ] + } + }` + + let res + try { + res = study.validateManifest(manifest) + } catch (err) { + res = err + } + + assert + .equal(res.errCode, 'mdctl.invalidArgument.unspecified') + assert + .equal(res.code, 'kInvalidArgument') + assert + .equal(res.reason, 'No Assignments or eConsents to export') + + }) + + it('Test validateManifest function - Removed entities other than Assignments and eConsents', () => { + const manifest = `{ + "object": "manifest", + "c_task": { + "includes": [ + "key-001" + ] + }, + "c_group_task": { + "includes": [ + "key-002" + ] + } + }`, + res = study.validateManifest(manifest) + + expect(res) + .to.deep.equal({ + object: 'manifest', + c_task: { + includes: [ + 'key-001' + ] + } + }) + + }) + }) From 2cc6e5db7e273ab60ee798903c677c0feee76c88 Mon Sep 17 00:00:00 2001 From: Davide Aquaro Date: Tue, 7 Jun 2022 17:53:50 +0100 Subject: [PATCH 8/8] refactor: changed 'help' menu text for manifestObject flag --- packages/mdctl-cli/tasks/study.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/mdctl-cli/tasks/study.js b/packages/mdctl-cli/tasks/study.js index c70f7c20..7bb7ff04 100644 --- a/packages/mdctl-cli/tasks/study.js +++ b/packages/mdctl-cli/tasks/study.js @@ -290,7 +290,8 @@ class Study extends Task { Notes - --manifestObject is \x1b[4monly available for "export" command\x1b[0m, and it is expected to have the following format: + --manifestObject is \x1b[4monly available for "export" command and it currently supports ONLY Assignments and eConsents\x1b[0m; + it is expected to have the following format: { "": { "includes": [