diff --git a/client/src/app/device/components/device-sync/device-sync.component.ts b/client/src/app/device/components/device-sync/device-sync.component.ts index c0db00b891..6b55e17fd4 100644 --- a/client/src/app/device/components/device-sync/device-sync.component.ts +++ b/client/src/app/device/components/device-sync/device-sync.component.ts @@ -25,7 +25,7 @@ export class DeviceSyncComponent implements OnInit { this.syncInProgress = true this.syncService.syncMessage$.subscribe({ next: (progress) => { - this.syncMessage = progress.docs_written + ' docs saved.' + this.syncMessage = progress.docs_written + ' docs saved; ' + progress.pending + ' pending' console.log('Sync Progress: ' + JSON.stringify(progress)) } }) diff --git a/client/src/app/shared/_classes/app-config.class.ts b/client/src/app/shared/_classes/app-config.class.ts index ba563538ad..16cd80c959 100644 --- a/client/src/app/shared/_classes/app-config.class.ts +++ b/client/src/app/shared/_classes/app-config.class.ts @@ -23,5 +23,6 @@ export class AppConfig { p2pSync = 'false' passwordPolicy:string passwordRecipe:string + couchdbSync4All:boolean } diff --git a/client/src/app/sync/components/sync/sync.component.ts b/client/src/app/sync/components/sync/sync.component.ts index cc67e41184..f307a4b2d3 100644 --- a/client/src/app/sync/components/sync/sync.component.ts +++ b/client/src/app/sync/components/sync/sync.component.ts @@ -29,7 +29,7 @@ export class SyncComponent implements OnInit { this.status = STATUS_IN_PROGRESS this.syncService.syncMessage$.subscribe({ next: (progress) => { - this.syncMessage = progress.docs_written + ' docs saved.' + this.syncMessage = progress.docs_written + ' docs saved; ' + progress.pending + ' pending' console.log('Sync Progress: ' + JSON.stringify(progress)) } }) diff --git a/client/src/app/sync/sync-couchdb.service.ts b/client/src/app/sync/sync-couchdb.service.ts index 10ce4033e5..04c8e1dc4a 100644 --- a/client/src/app/sync/sync-couchdb.service.ts +++ b/client/src/app/sync/sync-couchdb.service.ts @@ -7,6 +7,8 @@ import { UserDatabase } from './../shared/_classes/user-database.class'; import { Injectable } from '@angular/core'; import PouchDB from 'pouchdb' import {Subject} from 'rxjs'; +import {VariableService} from '../shared/_services/variable.service'; +import {AppConfigService} from '../shared/_services/app-config.service'; export interface LocationQuery { level:string @@ -31,7 +33,9 @@ export class SyncCouchdbService { public readonly syncMessage$: Subject = new Subject(); constructor( - private http:HttpClient + private http: HttpClient, + private variableService: VariableService, + private appConfigService: AppConfigService ) { } async uploadQueue(userDb:UserDatabase, syncDetails:SyncCouchdbDetails) { @@ -47,18 +51,58 @@ export class SyncCouchdbService { .map(row => row.id) } - // Note that if you run this with no forms configured to CouchDB sync, that will result in no filter query and everything will be synced. Use carefully. + // Note that this ignores the sync config settings. async sync(userDb:UserDatabase, syncDetails:SyncCouchdbDetails): Promise { const syncSessionUrl = await this.http.get(`${syncDetails.serverUrl}sync-session/start/${syncDetails.groupId}/${syncDetails.deviceId}/${syncDetails.deviceToken}`, {responseType:'text'}).toPromise() const remoteDb = new PouchDB(syncSessionUrl) - const pouchDbSyncOptions ={ - selector: { - "$or": syncDetails.formInfos.reduce(($or, formInfo) => { - if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled) { - $or = [ - ...$or, - ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation - ? syncDetails.deviceSyncLocations.map(locationConfig => { + let pull_last_seq = await this.variableService.get('sync-pull-last_seq') + let push_last_seq = await this.variableService.get('sync-push-last_seq') + if (typeof pull_last_seq === 'undefined') { + pull_last_seq = 0; + } + if (typeof push_last_seq === 'undefined') { + push_last_seq = 0; + } + const remotePouchOptions = { + "since": pull_last_seq + } + + const localPouchOptions = { + "since": push_last_seq + } + + if (syncDetails.deviceSyncLocations.length > 0) { + const locationConfig = syncDetails.deviceSyncLocations[0] + // Get last value, that's the focused sync point. + const location = locationConfig.value.slice(-1).pop() + const selector = { + [`location.${location.level}`]: { + '$eq' : location.value + } + } + remotePouchOptions['selector'] = selector + } + + let pouchOptions; + + const appConfig = await this.appConfigService.getAppConfig(); + + if (appConfig.couchdbSync4All) { + // Passing false prevents the changes feed from keeping all the documents in memory + pouchOptions = { + "return_docs": false, + "push": localPouchOptions, + "pull": remotePouchOptions + } + } else { + pouchOptions = { + selector: { + "$or": syncDetails.formInfos.reduce(($or, formInfo) => { + if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled) { + $or = [ + ...$or, + ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation + ? syncDetails.deviceSyncLocations.map(locationConfig => { // Get last value, that's the focused sync point. let location = locationConfig.value.slice(-1).pop() return { @@ -66,38 +110,48 @@ export class SyncCouchdbService { [`location.${location.level}`]: location.value } }) - : [ + : [ { "form.id": formInfo.id } ] - ] - } - return $or - }, []) + ] + } + return $or + }, []) + } } } + const replicationStatus = await new Promise((resolve, reject) => { - userDb.sync(remoteDb, pouchDbSyncOptions).on('complete', async (info) => { + userDb.sync(remoteDb, pouchOptions).on('complete', async (info) => { + await this.variableService.set('sync-push-last_seq', info.push.last_seq) + await this.variableService.set('sync-pull-last_seq', info.pull.last_seq) const conflictsQuery = await userDb.query('sync-conflicts'); resolve({ pulled: info.pull.docs_written, pushed: info.push.docs_written, conflicts: conflictsQuery.rows.map(row => row.id) }) - }).on('change', (info) => { - const docs_read = info.docs_read - const docs_written = info.docs_written - const doc_write_failures = info.doc_write_failures - // const errors = JSON.stringify(info.errors) + }).on('change', async (info) => { + if (typeof info.direction !== 'undefined') { + if (info.direction === 'push') { + await this.variableService.set('sync-push-last_seq', info.change.last_seq) + } else { + await this.variableService.set('sync-pull-last_seq', info.change.last_seq) + } + } + let pending = info.change.pending + if (typeof info.change.pending === 'undefined') { + pending = 0; + } const progress = { 'docs_read': info.change.docs_read, 'docs_written': info.change.docs_written, - 'doc_write_failures': info.change.doc_write_failures + 'doc_write_failures': info.change.doc_write_failures, + 'pending': pending }; this.syncMessage$.next(progress) - }).on('paused', function (err) { - console.log('Sync paused; error: ' + JSON.stringify(err)) }).on('error', function (errorMessage) { console.log('boo, something went wrong! error: ' + errorMessage) reject(errorMessage) diff --git a/client/src/app/sync/sync.service.ts b/client/src/app/sync/sync.service.ts index ece0d5ad82..060c7740a7 100644 --- a/client/src/app/sync/sync.service.ts +++ b/client/src/app/sync/sync.service.ts @@ -60,15 +60,18 @@ export class SyncService { deviceSyncLocations: device.syncLocations, formInfos }) - // console.log('this.syncMessage: ' + JSON.stringify(this.syncMessage)) - await this.syncCustomService.sync(userDb, { - appConfig: appConfig, - serverUrl: appConfig.serverUrl, - groupId: appConfig.groupId, - deviceId: device._id, - deviceToken: device.token, - formInfos - }) + console.log('this.syncMessage: ' + JSON.stringify(this.syncMessage)) + + if (!appConfig.couchdbSync4All) { + await this.syncCustomService.sync(userDb, { + appConfig: appConfig, + serverUrl: appConfig.serverUrl, + groupId: appConfig.groupId, + deviceId: device._id, + deviceToken: device.token, + formInfos + }) + } await this.deviceService.didSync() } diff --git a/config.defaults.sh b/config.defaults.sh index 6c0da27b82..432a59de02 100755 --- a/config.defaults.sh +++ b/config.defaults.sh @@ -94,3 +94,6 @@ T_HIDE_SKIP_IF="true" # In CSV output, set cell value to this when something is skipped. Set to "ORIGINAL_VALUE" if you want the actual value stored. T_REPORTING_MARK_SKIPPED_WITH="SKIPPED" +# Set to true if you want Tangerine to ignore any settings in Sync Configuration and use Couchdb Sync for replication. +T_COUCHDB_SYNC_4_ALL="false" + diff --git a/server/src/scripts/generate-cases/bin.js b/server/src/scripts/generate-cases/bin.js index e1a61da6bf..5d4748b397 100755 --- a/server/src/scripts/generate-cases/bin.js +++ b/server/src/scripts/generate-cases/bin.js @@ -28,7 +28,39 @@ async function go() { const caseDoc = templateDocs.find(doc => doc.type === 'case') // Change the case's ID. const caseId = uuidv1() - caseDoc._id = caseId + caseDoc._id = caseId + const participant_id = Math.round(Math.random() * 1000000) + let firstname = random_name({ first: true, gender: "female" }) + let surname = random_name({ last: true }) + let barcode_data = { "participant_id": participant_id, "treatment_assignment": "Experiment", "bin-mother": "A", "bin-infant": "B", "sub-studies": { "S1": true, "S2": false, "S3": false, "S4": true } } + let tangerineModifiedOn = new Date(); + // tangerineModifiedOn is set to numberOfCasesCompleted days before today, and its time is set based upon numberOfCasesCompleted. + tangerineModifiedOn.setDate( tangerineModifiedOn.getDate() - numberOfCasesCompleted ); + tangerineModifiedOn.setTime( tangerineModifiedOn.getTime() - ( numberOfCases - numberOfCasesCompleted ) ) + const day = String(tangerineModifiedOn.getDate()).padStart(2, '0'); + const month = String(tangerineModifiedOn.getMonth() + 1).padStart(2, '0'); + const year = tangerineModifiedOn.getFullYear(); + const screening_date = year + '-' + month + '-' + day; + const enrollment_date = screening_date; + let caseMother = { + _id: caseId, + tangerineModifiedOn: tangerineModifiedOn, + "participants": [{ + "id": participant_id, + "caseRoleId": "mother-role", + "data": { + "firstname": firstname, + "surname": surname, + "participant_id": participant_id + } + }], + } + console.log("motherId: " + caseId + " participantId: " + participant_id); + doc = Object.assign({}, caseDoc, caseMother); + caseDoc.items[0].inputs[1].value = participant_id; + caseDoc.items[0].inputs[2].value = enrollment_date; + caseDoc.items[0].inputs[8].value = firstname; + caseDoc.items[0].inputs[10].value = surname; for (let caseEvent of caseDoc.events) { const caseEventId = uuidv1() caseEvent.id = caseEventId @@ -50,6 +82,17 @@ async function go() { } } } + // modify the demographics form - s01a-participant-information-f254b9 + const demoDoc = templateDocs.find(doc => doc.form.id === 's01a-participant-information-f254b9') + demoDoc.items[0].inputs[4].value = screening_date; + // "id": "randomization", + demoDoc.items[10].inputs[1].value = barcode_data; + demoDoc.items[10].inputs[2].value = participant_id; + demoDoc.items[10].inputs[7].value = enrollment_date; + // "id": "participant_information", + demoDoc.items[12].inputs[2].value = surname; + demoDoc.items[12].inputs[3].value = firstname; + // Upload the profiles first // now upload the others for (let doc of templateDocs) { diff --git a/server/src/shared/classes/tangerine-config.ts b/server/src/shared/classes/tangerine-config.ts index de255b42e2..158a6e3ea7 100644 --- a/server/src/shared/classes/tangerine-config.ts +++ b/server/src/shared/classes/tangerine-config.ts @@ -11,4 +11,5 @@ export interface TangerineConfig { uploadToken: string reportingDelay: number hideSkipIf: boolean -} \ No newline at end of file + couchdbSync4All: boolean +} diff --git a/server/src/shared/services/tangerine-config/tangerine-config.service.ts b/server/src/shared/services/tangerine-config/tangerine-config.service.ts index 55e06e7d7b..60feae5921 100644 --- a/server/src/shared/services/tangerine-config/tangerine-config.service.ts +++ b/server/src/shared/services/tangerine-config/tangerine-config.service.ts @@ -21,7 +21,8 @@ export class TangerineConfigService { syncUsername: process.env.T_SYNC_USERNAME, syncPassword: process.env.T_SYNC_PASSWORD, hideSkipIf: process.env.T_HIDE_SKIP_IF === 'true' ? true : false, - reportingDelay: parseInt(process.env.T_REPORTING_DELAY) + reportingDelay: parseInt(process.env.T_REPORTING_DELAY), + couchdbSync4All: process.env.T_COUCHDB_SYNC_4_ALL === 'true' ? true : false } } }