K6 test for finding duplicates (#6524)

Co-authored-by: Ruben <vandervalk@geoit.nl>

.github/workflows/test_k6.yml

@@ -55,10 +55,10 @@ jobs:
         run: |
           curl https://github.com/grafana/k6/releases/download/v0.51.0/k6-v0.51.0-linux-amd64.tar.gz -L | tar xvz --strip-components 1
 
-      - name: Run k6 tests
+      - name: Run all k6 tests
         working-directory: ${{ env.k6TestsPath }}
         run: |
-          npx dotenv -e ../services/.env -- ./k6 run tests/retryFailedJobsOnStartupDuringQueueProcessing.js
+          ./run-all-tests.sh tests/retryFailedJobsOnStartupDuringQueueProcessing.js tests/findDuplicates100kRegistrations.js
 
       - name: Docker logs
         if: always()

k6/helpers/registration-default.data.js

@@ -19,6 +19,17 @@ export const registrationVisa = {
   [CustomDataAttributes.whatsappPhoneNumber]: '14155238887',
 };
 
+export const registrationPV = {
+  referenceId: '44e62864557597e0d',
+  preferredLanguage: 'nl',
+  paymentAmountMultiplier: 1,
+  fullName: 'Gemma Houtenbos',
+  phoneNumber: '14155235556',
+  programFinancialServiceProviderConfigurationName:
+    FinancialServiceProviders.intersolveVoucherWhatsapp,
+  whatsappPhoneNumber: '14155235555',
+};
+
 export const registrationSafaricom = {
   referenceId: '01dc9451-1273-484c-b2e8-ae21b51a96ab',
   programFinancialServiceProviderConfigurationName:

k6/models/registrations.js

@@ -35,4 +35,19 @@ export default class RegistrationsModel {
 
     return res;
   }
+
+  getRegistrations(programId, filter) {
+    let queryParams = '';
+    if (filter) {
+      queryParams = Object.entries(filter)
+        .map(
+          ([key, value]) =>
+            `${encodeURIComponent(key)}=${encodeURIComponent(value)}`,
+        )
+        .join('&');
+    }
+
+    const url = `${baseUrl}api/programs/${programId}/registrations?${queryParams}`;
+    return http.get(url);
+  }
 }

k6/run-all-tests.sh

@@ -1,6 +1,18 @@
+#!/bin/bash
+
 # Array to collect failed tests
 failed_tests=()
-for file in tests/*.js; do
+
+# Check if filenames are provided as arguments
+if [ "$#" -eq 0 ]; then
+  # No filenames provided, run all tests
+  test_files=(tests/*.js)
+else
+  # Use provided filenames
+  test_files=("$@")
+fi
+
+for file in "${test_files[@]}"; do
   echo "Test: $file"
   echo "Starting services"
   (cd ../services ; docker --log-level 'warn' compose -f docker-compose.yml up -d --quiet-pull --wait --wait-timeout 300)
@@ -18,6 +30,7 @@ for file in tests/*.js; do
   echo "Stopping services"
   (cd ../services ; docker compose -f docker-compose.yml down)
 done
+
 # Check if there were any failed tests
 if [ ${#failed_tests[@]} -ne 0 ]; then
   echo "The following tests failed:"

k6/tests/findDuplicates100kRegistrations.js

@@ -0,0 +1,62 @@
+import { check, fail, sleep } from 'k6';
+import { Counter } from 'k6/metrics';
+
+import { registrationPV } from '../helpers/registration-default.data.js';
+import loginModel from '../models/login.js';
+import RegistrationsModel from '../models/registrations.js';
+import resetModel from '../models/reset.js';
+
+const registrationsModel = new RegistrationsModel();
+const resetPage = new resetModel();
+const loginPage = new loginModel();
+
+const duplicateNumber = 17; // '17' leads to 131k registrations
+const resetScript = 'nlrc-multiple';
+const programId = 2;
+
+// At the time of implementation, the request duration was 12s on the server and 3s on the local machine for 130k registrations and about 8k duplicates
+const maxRequestDuration = 12000;
+
+const failedChecks = new Counter('failed_checks');
+
+function checkAndFail(response, checks) {
+  const result = check(response, checks);
+  if (!result) {
+    failedChecks.add(1);
+    fail('One or more checks failed');
+  }
+}
+
+export default function () {
+  resetPage.resetDB(resetScript);
+  // login
+  loginPage.login();
+  // Upload registration
+  registrationsModel.importRegistrations(programId, registrationPV);
+  // Duplicate registration to be more then 100k
+  resetPage.duplicateRegistrations(duplicateNumber);
+
+  const queryParams = {
+    'filter.duplicateStatus': 'duplicate',
+  };
+
+  const getRegistration = registrationsModel.getRegistrations(
+    programId,
+    queryParams,
+  );
+  console.log('🚀 ~ getRegistration:', getRegistration.timings.duration);
+
+  const getRegistrationBody = JSON.parse(getRegistration.body);
+  checkAndFail(getRegistrationBody, {
+    'totalItems between 3000 and 10000': (r) =>
+      r.meta.totalItems >= 3000 && r.meta.totalItems <= 10000,
+  });
+
+  // Check if the request duration is less than 5 seconds
+  checkAndFail(getRegistration, {
+    'smaller than max request duration': (r) =>
+      r.timings.duration < maxRequestDuration,
+  });
+
+  sleep(1);
+}

services/121-service/src/scripts/scripts.service.ts

@@ -83,5 +83,6 @@ export class ScriptsService {
 
     await this.seedMockHelper.multiplyRegistrations(powerNrRegistrations);
     await this.seedMockHelper.updateSequenceNumbers();
+    await this.seedMockHelper.introduceDuplicates();
   }
 }