Removing unused parsers

globaldothealth · Mar 21, 2024 · ee88bd6 · ee88bd6
1 parent 082476f
commit ee88bd6
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 69 deletions.
diff --git a/data-serving/data-service/src/controllers/case.ts b/data-serving/data-service/src/controllers/case.ts
@@ -6,6 +6,7 @@ import {
     CuratorsDocument,
 } from '../model/day0-case';
 import caseFields from '../model/fields.json';
+import { CaseStatus } from '../types/enums';
 import { Error, Query } from 'mongoose';
 import { ObjectId } from 'mongodb';
 import { GeocodeOptions, Geocoder, Resolution } from '../geocoding/geocoder';
@@ -500,14 +501,21 @@ export class CasesController {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const countriesData: any = {};
             // Get total case cardinality
-            const grandTotalCount = await Day0Case.countDocuments({});
+            const grandTotalCount = await Day0Case.countDocuments({
+                caseStatus: { $ne: CaseStatus.OmitError }
+            });
             if (grandTotalCount === 0) {
                 res.status(200).json({});
                 return;
             }
 
             // Get cardinality of case statuses by country
             const countriesStatusCounts = await Day0Case.aggregate([
+                {
+                    $match: {
+                        caseStatus: { $ne: CaseStatus.OmitError }
+                    }
+                },
                 {
                     $group: {
                         _id: {
@@ -558,7 +566,8 @@ export class CasesController {
                             $exists: true,
                             $ne: null,
                         },
-                    },
+                        caseStatus: { $ne: CaseStatus.OmitError }
+                    }
                 },
                 {
                     $group: {
@@ -625,6 +634,11 @@ export class CasesController {
             // Get cardinality of case statuses total
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const totalStatusCount: any = await Day0Case.aggregate([
+                {
+                    $match: {
+                        caseStatus: { $ne: CaseStatus.OmitError }
+                    }
+                },
                 {
                     $group: {
                         _id: '$caseStatus',
@@ -652,6 +666,11 @@ export class CasesController {
 
             // Get cardinality of outcomes total
             const totalOutcomeCount = await Day0Case.aggregate([
+                {
+                    $match: {
+                        caseStatus: { $ne: CaseStatus.OmitError }
+                    }
+                },
                 {
                     $match: {
                         'events.outcome': {

diff --git a/dev/Dockerfile-test b/dev/Dockerfile-test
@@ -65,8 +65,6 @@ WORKDIR /app
 
 COPY ./run_e2e_tests.sh ./
 COPY ./setup_localstack.py ./
-COPY ./setup_ingestion.py ./
-COPY ./parsing.py ./
 COPY ./test_e2e.py ./
 
 CMD [ "./run_e2e_tests.sh" ]
diff --git a/dev/docker-compose.dev.full.test.yml b/dev/docker-compose.dev.full.test.yml
@@ -4,6 +4,7 @@ services:
     environment:
       DEBUG: "True"
       ENABLE_FAKE_GEOCODER: "True"
+      DB: "${DISEASE_NAME}"
   curator:
     command: "npm run dev"
     volumes:
@@ -74,7 +75,6 @@ services:
       dockerfile: Dockerfile-test
     depends_on:
       - mongo
-      - mock-source-data
       - localstack
     ports:
       - "4444:4444"

diff --git a/dev/run_e2e_tests.sh b/dev/run_e2e_tests.sh
@@ -4,12 +4,6 @@ set -eou pipefail
 
 echo "Setting up localstack"
 python3 setup_localstack.py
-echo "Setting up ingestion environment"
-python3 setup_ingestion.py
-echo "Waiting for batch jobs"
-sleep 10
-echo "Running all parser batch jobs"
-python3 parsing.py run
 
 echo "Running end-to-end tests"
 python3 -m pytest -rs -v .