GenSpectrum · Taepper · Oct 16, 2024 · Sep 13, 2024 · Oct 1, 2024 · Oct 4, 2024
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -163,7 +163,6 @@ jobs:
     strategy:
       matrix:
         preprocessing-docker-compose: [
-          docker-compose-for-tests-preprocessing-from-tsv.yml,
           docker-compose-for-tests-preprocessing-from-ndjson.yml
         ]
     steps:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -54,7 +54,7 @@ jobs:
           MINOR=${{ steps.release.outputs.minor }}
           PATCH=${{ steps.release.outputs.patch }}
           TAGS=("$MAJOR.$MINOR" "$MAJOR.$MINOR.$PATCH")
-          # TODO (#405) include $MAJOR only for releases >= 1.0
+          # TODO(#405) include $MAJOR only for releases >= 1.0
           # TAGS=("$MAJOR" "$MAJOR.$MINOR" "$MAJOR.$MINOR.$PATCH")
           for TAG in "${TAGS[@]}"; do
             docker buildx imagetools create --tag $DOCKER_IMAGE_NAME:$TAG $DOCKER_IMAGE_NAME:latest

diff --git a/docker-compose-for-tests-preprocessing-from-ndjson.yml b/docker-compose-for-tests-preprocessing-from-ndjson.yml
@@ -3,10 +3,10 @@ services:
   silo:
     image: ${SILO_IMAGE}
     volumes:
-      - ./testBaseData/exampleDatasetAsNdjson:/preprocessing/input
+      - ./testBaseData/exampleDataset:/preprocessing/input
       - ./testBaseData/output:/preprocessing/output
-      - ./testBaseData/exampleDatasetAsNdjson/preprocessing_config.yaml:/app/preprocessing_config.yaml
-      - ./testBaseData/exampleDatasetAsNdjson/database_config.yaml:/app/database_config.yaml
+      - ./testBaseData/exampleDataset/preprocessing_config.yaml:/app/preprocessing_config.yaml
+      - ./testBaseData/exampleDataset/database_config.yaml:/app/database_config.yaml
       - ./logs:/app/logs
     command:
       - "--preprocessing"
diff --git a/docker-compose-for-tests-preprocessing-from-tsv.yml b/docker-compose-for-tests-preprocessing-from-tsv.yml
diff --git a/endToEndTests/test/info.test.js b/endToEndTests/test/info.test.js
@@ -9,7 +9,7 @@ describe('The /info endpoint', () => {
       .expect(200)
       .expect('Content-Type', 'application/json')
       .expect(expectHeaderToHaveDataVersion)
-      .expect({ nBitmapsSize: 3931, sequenceCount: 100, totalSize: 26589508, numberOfPartitions: 11 });
+      .expect({ nBitmapsSize: 3931, sequenceCount: 100, totalSize: 24198036, numberOfPartitions: 10 });
   });
 
   it('should return detailed info about the current state of the database', { timeout: 5000 }, async () => {
@@ -26,13 +26,13 @@ describe('The /info endpoint', () => {
           'bitmapContainerSizeStatistic'
         );
         expect(returnedInfo.bitmapContainerSizePerGenomeSection.bitmapContainerSizeStatistic).to.deep.equal({
-          numberOfArrayContainers: 48524,
+          numberOfArrayContainers: 48579,
           numberOfBitsetContainers: 0,
           numberOfRunContainers: 284,
-          numberOfValuesStoredInArrayContainers: 66620,
+          numberOfValuesStoredInArrayContainers: 66892,
           numberOfValuesStoredInBitsetContainers: 0,
           numberOfValuesStoredInRunContainers: 2875,
-          totalBitmapSizeArrayContainers: 133240,
+          totalBitmapSizeArrayContainers: 133784,
           totalBitmapSizeBitsetContainers: 0,
           totalBitmapSizeRunContainers: 4824,
         });
@@ -61,22 +61,22 @@ describe('The /info endpoint', () => {
 
         expect(returnedInfo).to.have.property('bitmapSizePerSymbol');
         expect(returnedInfo.bitmapSizePerSymbol).to.deep.equal({
-          '-': 2661831,
-          'A': 2775910,
-          'B': 2631464,
-          'C': 2725728,
-          'D': 2631464,
-          'G': 2728118,
-          'H': 2631464,
-          'K': 2631594,
-          'M': 2631554,
-          'N': 2631464,
-          'R': 2631514,
-          'S': 2631464,
-          'T': 2791923,
-          'V': 2631464,
-          'W': 2631514,
-          'Y': 2631494,
+          '-': 2422869,
+          'A': 2536840,
+          'B': 2392240,
+          'C': 2486656,
+          'D': 2392240,
+          'G': 2489072,
+          'H': 2392240,
+          'K': 2392370,
+          'M': 2392330,
+          'N': 2392240,
+          'R': 2392290,
+          'S': 2392240,
+          'T': 2552937,
+          'V': 2392240,
+          'W': 2392290,
+          'Y': 2392270,
         });
       })
       .expect(expectHeaderToHaveDataVersion);

diff --git a/endToEndTests/test/invalidQueries/OffsetNegative.json b/endToEndTests/test/invalidQueries/OffsetNegative.json
@@ -15,7 +15,7 @@
           "value": "Switzerland"
         },
         {
-          "type": "PangoLineage",
+          "type": "Lineage",
           "column": "pango_lineage",
           "value": "B.1.1.7",
           "includeSublineages": true

diff --git a/endToEndTests/test/queries/And.json b/endToEndTests/test/queries/And.json
@@ -13,7 +13,7 @@
           "value": "Switzerland"
         },
         {
-          "type": "PangoLineage",
+          "type": "Lineage",
           "column": "pango_lineage",
           "value": "B.1.1.7",
           "includeSublineages": true

diff --git a/endToEndTests/test/queries/DetailsOrderBy.json b/endToEndTests/test/queries/DetailsOrderBy.json
@@ -14,7 +14,7 @@
           "value": "Switzerland"
         },
         {
-          "type": "PangoLineage",
+          "type": "Lineage",
           "column": "pango_lineage",
           "value": "B.1.1.7",
           "includeSublineages": true

diff --git a/endToEndTests/test/queries/DetailsOrderByLimit.json b/endToEndTests/test/queries/DetailsOrderByLimit.json
@@ -16,7 +16,7 @@
           "value": "Switzerland"
         },
         {
-          "type": "PangoLineage",
+          "type": "Lineage",
           "column": "pango_lineage",
           "value": "B.1.1.7",
           "includeSublineages": true

diff --git a/endToEndTests/test/queries/GroupByLineage.json b/endToEndTests/test/queries/GroupByLineage.json
@@ -99,17 +99,17 @@
       "count": 1,
       "pango_lineage": "GD.1"
     },
+    {
+      "count": 1,
+      "pango_lineage": "GL.1"
+    },
     {
       "count": 1,
       "pango_lineage": "P.1"
     },
     {
       "count": 3,
       "pango_lineage": "Q.7"
-    },
-    {
-      "count": 1,
-      "pango_lineage": "XA.1"
     }
   ]
 }
diff --git a/endToEndTests/test/queries/LimitLargerThanTable.json b/endToEndTests/test/queries/LimitLargerThanTable.json
@@ -15,7 +15,7 @@
           "value": "Switzerland"
         },
         {
-          "type": "PangoLineage",
+          "type": "Lineage",
           "column": "pango_lineage",
           "value": "B.1.1.7",
           "includeSublineages": true

diff --git a/endToEndTests/test/queries/Offset0.json b/endToEndTests/test/queries/Offset0.json
@@ -15,7 +15,7 @@
           "value": "Switzerland"
         },
         {
-          "type": "PangoLineage",
+          "type": "Lineage",
           "column": "pango_lineage",
           "value": "B.1.1.7",
           "includeSublineages": true

diff --git a/endToEndTests/test/queries/OffsetLargerThanTable.json b/endToEndTests/test/queries/OffsetLargerThanTable.json
@@ -15,7 +15,7 @@
           "value": "Switzerland"
         },
         {
-          "type": "PangoLineage",
+          "type": "Lineage",
           "column": "pango_lineage",
           "value": "B.1.1.7",
           "includeSublineages": true

diff --git a/endToEndTests/test/queries/Or.json b/endToEndTests/test/queries/Or.json
@@ -8,13 +8,13 @@
       "type": "Or",
       "children": [
         {
-          "type": "PangoLineage",
+          "type": "Lineage",
           "column": "pango_lineage",
           "value": "B.1.1.7",
           "includeSublineages": false
         },
         {
-          "type": "PangoLineage",
+          "type": "Lineage",
           "column": "pango_lineage",
           "value": "B.1.1.7",
           "includeSublineages": true

diff --git a/endToEndTests/test/queries/PangoLineageAlias.json b/endToEndTests/test/queries/PangoLineageAlias.json
@@ -104,17 +104,17 @@
       "count": 1,
       "pango_lineage": "GD.1"
     },
+    {
+      "count": 1,
+      "pango_lineage": "GL.1"
+    },
     {
       "count": 1,
       "pango_lineage": "P.1"
     },
     {
       "count": 3,
       "pango_lineage": "Q.7"
-    },
-    {
-      "count": 1,
-      "pango_lineage": "XA.1"
     }
   ]
 }
diff --git a/endToEndTests/test/queries/aaMutDistribution_all.json b/endToEndTests/test/queries/aaMutDistribution_all.json
@@ -3,13 +3,23 @@
   "query": {
     "action": {
       "type": "AminoAcidMutations",
-      "minProportion": 0.4
+      "minProportion": 0.4,
+      "orderByFields": ["position", "mutationFrom"]
     },
     "filterExpression": {
       "type": "True"
     }
   },
   "expectedQueryResult": [
+    {
+      "count": 37,
+      "mutation": "G142D",
+      "mutationFrom": "G",
+      "mutationTo": "D",
+      "position": 142,
+      "proportion": 0.4065934065934066,
+      "sequenceName": "S"
+    },
     {
       "count": 46,
       "mutation": "R203K",
@@ -28,33 +38,6 @@
       "proportion": 0.46,
       "sequenceName": "N"
     },
-    {
-      "count": 64,
-      "mutation": "T3255I",
-      "mutationFrom": "T",
-      "mutationTo": "I",
-      "position": 3255,
-      "proportion": 0.6464646464646465,
-      "sequenceName": "ORF1a"
-    },
-    {
-      "count": 44,
-      "mutation": "S3675-",
-      "mutationFrom": "S",
-      "mutationTo": "-",
-      "position": 3675,
-      "proportion": 0.4489795918367347,
-      "sequenceName": "ORF1a"
-    },
-    {
-      "count": 44,
-      "mutation": "G3676-",
-      "mutationFrom": "G",
-      "mutationTo": "-",
-      "position": 3676,
-      "proportion": 0.4444444444444444,
-      "sequenceName": "ORF1a"
-    },
     {
       "count": 98,
       "mutation": "P314L",
@@ -64,15 +47,6 @@
       "proportion": 0.98,
       "sequenceName": "ORF1b"
     },
-    {
-      "count": 37,
-      "mutation": "G142D",
-      "mutationFrom": "G",
-      "mutationTo": "D",
-      "position": 142,
-      "proportion": 0.4065934065934066,
-      "sequenceName": "S"
-    },
     {
       "count": 38,
       "mutation": "L452R",
@@ -117,6 +91,33 @@
       "position": 681,
       "proportion": 0.42,
       "sequenceName": "S"
+    },
+    {
+      "count": 64,
+      "mutation": "T3255I",
+      "mutationFrom": "T",
+      "mutationTo": "I",
+      "position": 3255,
+      "proportion": 0.6464646464646465,
+      "sequenceName": "ORF1a"
+    },
+    {
+      "count": 44,
+      "mutation": "S3675-",
+      "mutationFrom": "S",
+      "mutationTo": "-",
+      "position": 3675,
+      "proportion": 0.4489795918367347,
+      "sequenceName": "ORF1a"
+    },
+    {
+      "count": 44,
+      "mutation": "G3676-",
+      "mutationFrom": "G",
+      "mutationTo": "-",
+      "position": 3676,
+      "proportion": 0.4444444444444444,
+      "sequenceName": "ORF1a"
     }
   ]
 }
diff --git a/endToEndTests/test/queries/booleanEquals_And.json b/endToEndTests/test/queries/booleanEquals_And.json
@@ -13,7 +13,7 @@
           "value": false
         },
         {
-          "type": "PangoLineage",
+          "type": "Lineage",
           "column": "pango_lineage",
           "value": "B.1",
           "includeSublineages": true

diff --git a/endToEndTests/test/queries/booleanEquals_Or.json b/endToEndTests/test/queries/booleanEquals_Or.json
@@ -13,7 +13,7 @@
           "value": null
         },
         {
-          "type": "PangoLineage",
+          "type": "Lineage",
           "column": "pango_lineage",
           "value": "B.1.1",
           "includeSublineages": true

diff --git a/endToEndTests/test/queries/complexQuery.json b/endToEndTests/test/queries/complexQuery.json
@@ -100,7 +100,7 @@
               "column": "pango_lineage",
               "value": "B",
               "includeSublineages": true,
-              "type": "PangoLineage"
+              "type": "Lineage"
             }
           ],
           "type": "And"