WIP: Fix for sync crash; Simplified pouchDbSyncOptions

client/src/app/device/components/device-sync/device-sync.component.ts

@@ -25,7 +25,7 @@ export class DeviceSyncComponent implements OnInit {
     this.syncInProgress = true
     this.syncService.syncMessage$.subscribe({
       next: (progress) => {
-        this.syncMessage =  progress.docs_written + ' docs saved.'
+        this.syncMessage =  progress.docs_written + ' docs saved; ' + progress.pending + ' pending'
         console.log('Sync Progress: ' + JSON.stringify(progress))
       }
     })

client/src/app/shared/_classes/app-config.class.ts

@@ -23,5 +23,6 @@ export class AppConfig {
   p2pSync = 'false'
   passwordPolicy:string
   passwordRecipe:string
+  couchdbSync4All:boolean
 }
 

client/src/app/sync/components/sync/sync.component.ts

@@ -29,7 +29,7 @@ export class SyncComponent implements OnInit {
     this.status = STATUS_IN_PROGRESS
     this.syncService.syncMessage$.subscribe({
       next: (progress) => {
-        this.syncMessage =  progress.docs_written + ' docs saved.'
+        this.syncMessage =  progress.docs_written + ' docs saved; ' + progress.pending + ' pending'
         console.log('Sync Progress: ' + JSON.stringify(progress))
       }
     })

client/src/app/sync/sync-couchdb.service.ts

@@ -7,6 +7,8 @@ import { UserDatabase } from './../shared/_classes/user-database.class';
 import { Injectable } from '@angular/core';
 import PouchDB from 'pouchdb'
 import {Subject} from 'rxjs';
+import {VariableService} from '../shared/_services/variable.service';
+import {AppConfigService} from '../shared/_services/app-config.service';
 
 export interface LocationQuery {
   level:string
@@ -31,7 +33,9 @@ export class SyncCouchdbService {
   public readonly syncMessage$: Subject<any> = new Subject();
 
   constructor(
-    private http:HttpClient
+    private http: HttpClient,
+    private variableService: VariableService,
+    private appConfigService: AppConfigService
   ) { }
 
   async uploadQueue(userDb:UserDatabase, syncDetails:SyncCouchdbDetails) {
@@ -47,57 +51,107 @@ export class SyncCouchdbService {
       .map(row => row.id)
   }
 
-  // Note that if you run this with no forms configured to CouchDB sync, that will result in no filter query and everything will be synced. Use carefully.
+  // Note that this ignores the sync config settings.
   async sync(userDb:UserDatabase, syncDetails:SyncCouchdbDetails): Promise<ReplicationStatus> {
     const syncSessionUrl = await this.http.get(`${syncDetails.serverUrl}sync-session/start/${syncDetails.groupId}/${syncDetails.deviceId}/${syncDetails.deviceToken}`, {responseType:'text'}).toPromise()
     const remoteDb = new PouchDB(syncSessionUrl)
-    const pouchDbSyncOptions ={
-      selector: {
-        "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
-          if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled) {
-            $or = [
-              ...$or,
-              ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
-                ? syncDetails.deviceSyncLocations.map(locationConfig => {
+    let pull_last_seq = await this.variableService.get('sync-pull-last_seq')
+    let push_last_seq = await this.variableService.get('sync-push-last_seq')
+    if (typeof pull_last_seq === 'undefined') {
+      pull_last_seq = 0;
+    }
+    if (typeof push_last_seq === 'undefined') {
+      push_last_seq = 0;
+    }
+    const remotePouchOptions = {
+      "since": pull_last_seq
+    }
+
+    const localPouchOptions = {
+      "since": push_last_seq
+    }
+
+    if (syncDetails.deviceSyncLocations.length > 0) {
+      const locationConfig = syncDetails.deviceSyncLocations[0]
+      // Get last value, that's the focused sync point.
+      const location = locationConfig.value.slice(-1).pop()
+      const selector = {
+          [`location.${location.level}`]: {
+            '$eq' : location.value
+          }
+        }
+      remotePouchOptions['selector'] = selector
+    }
+
+    let pouchOptions;
+
+    const appConfig = await this.appConfigService.getAppConfig();
+
+    if (appConfig.couchdbSync4All) {
+      // Passing false prevents the changes feed from keeping all the documents in memory
+      pouchOptions = {
+        "return_docs": false,
+        "push": localPouchOptions,
+        "pull": remotePouchOptions
+      }
+    } else {
+      pouchOptions = {
+        selector: {
+          "$or": syncDetails.formInfos.reduce(($or, formInfo) => {
+            if (formInfo.couchdbSyncSettings && formInfo.couchdbSyncSettings.enabled) {
+              $or = [
+                ...$or,
+                ...syncDetails.deviceSyncLocations.length > 0 && formInfo.couchdbSyncSettings.filterByLocation
+                  ? syncDetails.deviceSyncLocations.map(locationConfig => {
                     // Get last value, that's the focused sync point.
                     let location = locationConfig.value.slice(-1).pop()
                     return {
                       "form.id": formInfo.id,
                       [`location.${location.level}`]: location.value
                     }
                   })
-                : [
+                  : [
                     {
                       "form.id": formInfo.id
                     }
                   ]
-            ]
-          }
-          return $or
-        }, [])
+              ]
+            }
+            return $or
+          }, [])
+        }
       }
     }
+
     const replicationStatus = <ReplicationStatus>await new Promise((resolve, reject) => {
-      userDb.sync(remoteDb, pouchDbSyncOptions).on('complete', async  (info) => {
+      userDb.sync(remoteDb, pouchOptions).on('complete', async  (info) => {
+        await this.variableService.set('sync-push-last_seq', info.push.last_seq)
+        await this.variableService.set('sync-pull-last_seq', info.pull.last_seq)
         const conflictsQuery = await userDb.query('sync-conflicts');
         resolve(<ReplicationStatus>{
           pulled: info.pull.docs_written,
           pushed: info.push.docs_written,
           conflicts: conflictsQuery.rows.map(row => row.id)
         })
-      }).on('change', (info) => {
-        const docs_read = info.docs_read
-        const docs_written = info.docs_written
-        const doc_write_failures = info.doc_write_failures
-        // const errors = JSON.stringify(info.errors)
+      }).on('change', async (info) => {
+        if (typeof info.direction !== 'undefined') {
+          if (info.direction === 'push') {
+            await this.variableService.set('sync-push-last_seq', info.change.last_seq)
+          } else {
+            await this.variableService.set('sync-pull-last_seq', info.change.last_seq)
+          }
+        }
+        let pending = info.change.pending
+        if (typeof info.change.pending === 'undefined') {
+          pending = 0;
+        }
         const progress = {
           'docs_read': info.change.docs_read,
           'docs_written': info.change.docs_written,
-          'doc_write_failures': info.change.doc_write_failures
+          'doc_write_failures': info.change.doc_write_failures,
+          'pending': pending
         };
         this.syncMessage$.next(progress)
-      }).on('paused', function (err) {
-        console.log('Sync paused; error: ' + JSON.stringify(err))
       }).on('error', function (errorMessage) {
         console.log('boo, something went wrong! error: ' + errorMessage)
         reject(errorMessage)

client/src/app/sync/sync.service.ts

@@ -60,15 +60,18 @@ export class SyncService {
       deviceSyncLocations: device.syncLocations,
       formInfos
     })
-    // console.log('this.syncMessage: ' + JSON.stringify(this.syncMessage))
-    await this.syncCustomService.sync(userDb, <SyncCustomDetails>{
-      appConfig: appConfig,
-      serverUrl: appConfig.serverUrl,
-      groupId: appConfig.groupId,
-      deviceId: device._id,
-      deviceToken: device.token,
-      formInfos
-    })
+    console.log('this.syncMessage: ' + JSON.stringify(this.syncMessage))
+
+    if (!appConfig.couchdbSync4All) {
+      await this.syncCustomService.sync(userDb, <SyncCustomDetails>{
+        appConfig: appConfig,
+        serverUrl: appConfig.serverUrl,
+        groupId: appConfig.groupId,
+        deviceId: device._id,
+        deviceToken: device.token,
+        formInfos
+      })
+    }
     await this.deviceService.didSync()
   }
 

config.defaults.sh

@@ -94,3 +94,6 @@ T_HIDE_SKIP_IF="true"
 # In CSV output, set cell value to this when something is skipped. Set to "ORIGINAL_VALUE" if you want the actual value stored.
 T_REPORTING_MARK_SKIPPED_WITH="SKIPPED"
 
+# Set to true if you want Tangerine to ignore any settings in Sync Configuration and use Couchdb Sync for replication.
+T_COUCHDB_SYNC_4_ALL="false"
+

server/src/scripts/generate-cases/bin.js

@@ -28,7 +28,39 @@ async function go() {
     const caseDoc = templateDocs.find(doc => doc.type === 'case')
     // Change the case's ID.
     const caseId = uuidv1()
-    caseDoc._id = caseId 
+    caseDoc._id = caseId
+    const participant_id = Math.round(Math.random() * 1000000)
+    let firstname = random_name({ first: true, gender: "female" })
+    let surname = random_name({ last: true })
+    let barcode_data =  { "participant_id": participant_id, "treatment_assignment": "Experiment", "bin-mother": "A", "bin-infant": "B", "sub-studies": { "S1": true, "S2": false, "S3": false, "S4": true } }
+    let tangerineModifiedOn = new Date();
+    // tangerineModifiedOn is set to numberOfCasesCompleted days before today, and its time is set based upon numberOfCasesCompleted.
+    tangerineModifiedOn.setDate( tangerineModifiedOn.getDate() - numberOfCasesCompleted );
+    tangerineModifiedOn.setTime( tangerineModifiedOn.getTime() - ( numberOfCases - numberOfCasesCompleted ) )
+    const day = String(tangerineModifiedOn.getDate()).padStart(2, '0');
+    const month = String(tangerineModifiedOn.getMonth() + 1).padStart(2, '0');
+    const year = tangerineModifiedOn.getFullYear();
+    const screening_date = year + '-' + month + '-' + day;
+    const enrollment_date = screening_date;
+    let caseMother = {
+      _id: caseId,
+      tangerineModifiedOn: tangerineModifiedOn,
+      "participants": [{
+        "id": participant_id,
+        "caseRoleId": "mother-role",
+        "data": {
+          "firstname": firstname,
+          "surname": surname,
+          "participant_id": participant_id
+        }
+      }],
+    }
+    console.log("motherId: " + caseId + " participantId: " + participant_id);
+    doc = Object.assign({}, caseDoc, caseMother);
+    caseDoc.items[0].inputs[1].value = participant_id;
+    caseDoc.items[0].inputs[2].value = enrollment_date;
+    caseDoc.items[0].inputs[8].value = firstname;
+    caseDoc.items[0].inputs[10].value = surname;
     for (let caseEvent of caseDoc.events) {
       const caseEventId = uuidv1()
       caseEvent.id = caseEventId
@@ -50,6 +82,17 @@ async function go() {
         }
       }
     }
+    // modify the demographics form - s01a-participant-information-f254b9
+    const demoDoc = templateDocs.find(doc => doc.form.id === 's01a-participant-information-f254b9')
+    demoDoc.items[0].inputs[4].value = screening_date;
+    // "id": "randomization",
+    demoDoc.items[10].inputs[1].value = barcode_data;
+    demoDoc.items[10].inputs[2].value = participant_id;
+    demoDoc.items[10].inputs[7].value = enrollment_date;
+    // "id": "participant_information",
+    demoDoc.items[12].inputs[2].value = surname;
+    demoDoc.items[12].inputs[3].value = firstname;
+
     // Upload the profiles first
     // now upload the others
     for (let doc of templateDocs) {

server/src/shared/classes/tangerine-config.ts

@@ -11,4 +11,5 @@ export interface TangerineConfig {
   uploadToken: string
   reportingDelay: number
   hideSkipIf: boolean
-}
+  couchdbSync4All: boolean
+}

server/src/shared/services/tangerine-config/tangerine-config.service.ts

@@ -21,7 +21,8 @@ export class TangerineConfigService {
       syncUsername: process.env.T_SYNC_USERNAME,
       syncPassword: process.env.T_SYNC_PASSWORD,
       hideSkipIf: process.env.T_HIDE_SKIP_IF === 'true' ? true : false,
-      reportingDelay: parseInt(process.env.T_REPORTING_DELAY)
+      reportingDelay: parseInt(process.env.T_REPORTING_DELAY),
+      couchdbSync4All: process.env.T_COUCHDB_SYNC_4_ALL === 'true' ? true : false
     }
   }
 }