Updated NTSB queries and ground truth for CIDR-25 paper.

apps/query-eval/data/ntsb-full.yaml

@@ -423,15 +423,6 @@ queries:
     tags:
       - verified_ground_truth
 
-  - query: "What was the breakdown of incidents by month, expressed as MM/YY?"
-    expected: |
-      06/24: 42
-      07/24: 36
-      08/24: 17
-      09/24: 5
-    tags:
-      - verified_ground_truth
-
   - query: "What was the aircraft type with the most incidents?"
     expected: "Cessna 172"
     tags:
@@ -565,22 +556,49 @@ queries:
     tags:
       - need_ground_truth
 
-  - query: "How many documents have 194 in the pathname?"
-    expected: "91"
-    tags:
-      - verified_ground_truth
-
   - query: "How many incidents occurred in the first 10 days of July 2024?"
     expected: "11"
     tags:
-      - mdw
+      - verified_ground_truth
 
   - query: "How many states did incidents in the first 10 days of July 2024 occur in?"
     expected: "9"
     tags:
-      - mdw
+      - verified_ground_truth
 
   - query: "How many more incidents happened in Florida compared to Oklahoma?"
     expected: "7"
     tags:
       - verified_ground_truth
+
+  - query: "How many incidents occurred that involved serious injuries when the wind speed was greater than four knots?"
+    expected: "12"
+    tags:
+      - need_ground_truth
+
+  - query: "In incidents involving Piper aircraft, what was the most commonly damaged part of the aircraft?"
+    expected: "Wing, with 7 incidents in which it was damaged."
+    tags:
+      - need_ground_truth
+
+  - query: "In incidents involving damage to the fuselage, what are the number of incidents by aircraft type?"
+    expected: |
+      Cessna 172: 3
+      AIR TRACTOR INC AT-402: 2
+      CIRRUS DESIGN CORP SR20: 2
+      Cessna 172R: 2
+      Cessna 180: 2 are 2 incidents reported.
+      Cessna 305: 2
+      Cessna A185F: 2
+      Piper PA28: 2
+      Others: 1 each
+    tags:
+      - need_ground_truth
+
+  - query: "How many incidents were there, broken down by number of engines?"
+    expected: |
+      0 engines: 5
+      1 Engine: 87
+      2 Engines: 9
+    tags:
+      - need_ground_truth

apps/query-eval/queryeval/driver.py

@@ -12,7 +12,7 @@
 from sycamore.query.client import SycamoreQueryClient, configure_logging
 from yaml import safe_load
 
-from queryeval.types import (
+from queryeval.queryeval_types import (
     QueryEvalConfig,
     QueryEvalInputFile,
     QueryEvalMetrics,
@@ -189,11 +189,26 @@ def format_doclist(self, doclist: List[Document]) -> DocSetSummary:
                 if hasattr(doc.data, "model_dump"):
                     results.append(doc.data.model_dump())
                 else:
+                    BASE_PROPS = [
+                        "filename",
+                        "filetype",
+                        "page_number",
+                        "page_numbers",
+                        "links",
+                        "element_id",
+                        "parent_id",
+                        "_schema",
+                        "_schema_class",
+                        "entity",
+                    ]
+                    props_dict = doc.properties.get("entity", {})
+                    props_dict.update({p: doc.properties[p] for p in set(doc.properties) - set(BASE_PROPS)})
                     results.append(
                         DocumentSummary(
                             doc_id=doc.doc_id,
                             path=doc.properties.get("path"),
                             text_representation=doc.text_representation,
+                            properties=props_dict,
                         )
                     )
         return DocSetSummary(docs=results)
@@ -212,7 +227,9 @@ def get_result(self, query: QueryEvalQuery) -> Optional[QueryEvalResult]:
     def _check_tags_match(self, query):
         if not self.config.config or not self.config.config.tags:
             return True
-        return set(self.config.config.tags) == set(query.tags or [])
+        if not query.tags:
+            return False
+        return len(set.intersection(set(self.config.config.tags), set(query.tags or []))) > 0
 
     def do_plan(self, query: QueryEvalQuery, result: QueryEvalResult) -> QueryEvalResult:
         """Generate or return an existing query plan."""

apps/query-eval/queryeval/types.py → apps/query-eval/queryeval/queryeval_types.py

@@ -74,6 +74,7 @@ class DocumentSummary(BaseModel):
     doc_id: Optional[str] = None
     text_representation: Optional[str] = None
     path: Optional[str] = None
+    properties: Optional[Dict[str, Any]] = None
 
 
 class DocSetSummary(BaseModel):

apps/query-eval/queryeval/test_driver.py

@@ -4,7 +4,7 @@
 from sycamore.query.schema import OpenSearchSchema, OpenSearchSchemaField
 
 from queryeval.driver import QueryEvalDriver
-from queryeval.types import (
+from queryeval.queryeval_types import (
     QueryEvalQuery,
     QueryEvalResult,
     QueryEvalMetrics,