Merge branch 'main' of https://github.com/cagov/caldata-mdsa-caltrans…

…-pems into summer_grain_testing

transform/models/_sources.yml

@@ -169,14 +169,6 @@ sources:
     description: |
       Vehicle Detector Station (VDS) (aka Station) related data obtained via data relay from
       db96 within the Caltrans network.
-    freshness:
-      warn_after:
-        count: 2
-        period: day
-      error_after:
-        count: 3
-        period: day
-    loaded_at_field: time_id
     tables:
       - name: vds30sec
         description: |
@@ -288,7 +280,10 @@ sources:
           stations hooked up to a single controller. For example, the same
           controller might have a station for the mainline as well as a station
           for an onramp.
-        freshness: null
+        data_tests:
+          - not_empty
+          - unique:
+              column_name: controller_id
         columns:
           - name: meta
             description: Metadata from the data relay scripts.
@@ -310,6 +305,20 @@ sources:
         description: |
           A log table showing updates to the controller config. This can be joined
           with the `controller_config` table to get a full history of controller metadata.
+        data_tests:
+          - not_empty
+          - dbt_utils.unique_combination_of_columns:
+              combination_of_columns:
+                - controller_id
+                - time_id
+        freshness:
+          warn_after:
+            count: 30
+            period: day
+          error_after:
+            count: 60
+            period: day
+        loaded_at_field: time_id
         columns:
           - name: meta
             description: Metadata from the data relay scripts.
@@ -341,7 +350,10 @@ sources:
         description: |
           Metadata for a single VDS station. Multiple stations may be connected to
           a single controller, and multiple detectors may be connected to a single station.
-        freshness: null
+        data_tests:
+          - not_empty
+          - unique:
+              column_name: station_id
         columns:
           - name: meta
             description: Metadata from the data relay scripts.
@@ -363,6 +375,20 @@ sources:
         description: |
           A log table showing updates to the station config. This can be joined
           with the `station_config` table to get a full history of station metadata.
+        data_tests:
+          - not_empty
+          - dbt_utils.unique_combination_of_columns:
+              combination_of_columns:
+                - station_id
+                - time_id
+        freshness:
+          warn_after:
+            count: 15
+            period: day
+          error_after:
+            count: 30
+            period: day
+        loaded_at_field: time_id
         columns:
           - name: meta
             description: Metadata from the data relay scripts.
@@ -400,7 +426,10 @@ sources:
           Metadata for a single loop detector. This is the device that actually records
           flow, occupancy, and speed, and is typically installed in a single lane.
           Multiple detectors across a set of lanes constitute a station.
-        freshness: null
+        data_tests:
+          - not_empty
+          - unique:
+              column_name: detector_id
         columns:
           - name: meta
             description: Metadata from the data relay scripts.
@@ -412,6 +441,20 @@ sources:
         description: |
           A log table showing updates to the detector config. This can be joined
           with the `detector_config` table to get a full history of detector metadata.
+        data_tests:
+          - not_empty
+          - dbt_utils.unique_combination_of_columns:
+              combination_of_columns:
+                - detector_id
+                - time_id
+        freshness:
+          warn_after:
+            count: 15
+            period: day
+          error_after:
+            count: 30
+            period: day
+        loaded_at_field: time_id
         columns:
           - name: meta
             description: Metadata from the data relay scripts.

transform/tests/generic/not_empty.sql

@@ -0,0 +1,16 @@
+{% test not_empty(model) %}
+
+with validation as (
+   select count(0) as num_rows
+   from {{ model }}
+),
+
+validation_errors as (
+   select num_rows
+   from validation
+   where num_rows = 0
+)
+
+select * from validation_errors
+
+{% endtest %}