Pivot table

data-serving/data-service/src/controllers/case.ts

@@ -6,6 +6,7 @@ import {
     CuratorsDocument,
 } from '../model/day0-case';
 import caseFields from '../model/fields.json';
+import { CaseStatus } from '../types/enums';
 import { Error, Query } from 'mongoose';
 import { ObjectId } from 'mongodb';
 import { GeocodeOptions, Geocoder, Resolution } from '../geocoding/geocoder';
@@ -500,14 +501,25 @@ export class CasesController {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const countriesData: any = {};
             // Get total case cardinality
-            const grandTotalCount = await Day0Case.countDocuments({});
+            const grandTotalCount = await Day0Case.countDocuments({
+                caseStatus: {
+                    $nin: [CaseStatus.OmitError, CaseStatus.Discarded]
+                }
+            });
             if (grandTotalCount === 0) {
                 res.status(200).json({});
                 return;
             }
 
             // Get cardinality of case statuses by country
             const countriesStatusCounts = await Day0Case.aggregate([
+                {
+                    $match: {
+                        caseStatus: {
+                            $nin: [CaseStatus.OmitError, CaseStatus.Discarded]
+                        }
+                    }
+                },
                 {
                     $group: {
                         _id: {
@@ -558,7 +570,10 @@ export class CasesController {
                             $exists: true,
                             $ne: null,
                         },
-                    },
+                        caseStatus: {
+                            $nin: [CaseStatus.OmitError, CaseStatus.Discarded]
+                        }
+                    }
                 },
                 {
                     $group: {
@@ -625,6 +640,13 @@ export class CasesController {
             // Get cardinality of case statuses total
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const totalStatusCount: any = await Day0Case.aggregate([
+                {
+                    $match: {
+                        caseStatus: {
+                            $nin: [CaseStatus.OmitError, CaseStatus.Discarded]
+                        }
+                    }
+                },
                 {
                     $group: {
                         _id: '$caseStatus',
@@ -652,6 +674,13 @@ export class CasesController {
 
             // Get cardinality of outcomes total
             const totalOutcomeCount = await Day0Case.aggregate([
+                {
+                    $match: {
+                        caseStatus: {
+                            $nin: [CaseStatus.OmitError, CaseStatus.Discarded]
+                        }
+                    }
+                },
                 {
                     $match: {
                         'events.outcome': {

dev/Dockerfile-test

@@ -65,8 +65,6 @@ WORKDIR /app
 
 COPY ./run_e2e_tests.sh ./
 COPY ./setup_localstack.py ./
-COPY ./setup_ingestion.py ./
-COPY ./parsing.py ./
 COPY ./test_e2e.py ./
 
 CMD [ "./run_e2e_tests.sh" ]

dev/docker-compose.dev.full.test.yml

@@ -4,6 +4,7 @@ services:
     environment:
       DEBUG: "True"
       ENABLE_FAKE_GEOCODER: "True"
+      DB: "${DISEASE_NAME}"
   curator:
     command: "npm run dev"
     volumes:
@@ -74,7 +75,6 @@ services:
       dockerfile: Dockerfile-test
     depends_on:
       - mongo
-      - mock-source-data
       - localstack
     ports:
       - "4444:4444"

dev/run_e2e_tests.sh

@@ -4,12 +4,6 @@ set -eou pipefail
 
 echo "Setting up localstack"
 python3 setup_localstack.py
-echo "Setting up ingestion environment"
-python3 setup_ingestion.py
-echo "Waiting for batch jobs"
-sleep 10
-echo "Running all parser batch jobs"
-python3 parsing.py run
 
 echo "Running end-to-end tests"
 python3 -m pytest -rs -v .