From c7ea303c162670d504b7a4f42375f8c3de41370f Mon Sep 17 00:00:00 2001 From: pingpingxiu-DOT-ca-gov <149613649+pingpingxiu-DOT-ca-gov@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:46:43 -0700 Subject: [PATCH 1/5] Update _sources.yml Per discussion with @ian-r-rose , loosen the monitor threshold on the Station config's freshness since it may occasionally lag behind more than 3 days. --- transform/models/_sources.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transform/models/_sources.yml b/transform/models/_sources.yml index 0c34f480..c736845f 100644 --- a/transform/models/_sources.yml +++ b/transform/models/_sources.yml @@ -59,10 +59,10 @@ sources: depending on the measurement capabilities of the detectors. freshness: warn_after: - count: 2 + count: 14 period: day error_after: - count: 3 + count: 28 period: day loaded_at_field: SAMPLE_TIMESTAMP columns: From 103f24e44c64b9890995834aa33bb8bc5ae6916d Mon Sep 17 00:00:00 2001 From: pingpingxiu-DOT-ca-gov <149613649+pingpingxiu-DOT-ca-gov@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:55:46 -0700 Subject: [PATCH 2/5] Update _sources.yml fix bug --- transform/models/_sources.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transform/models/_sources.yml b/transform/models/_sources.yml index c736845f..0c34f480 100644 --- a/transform/models/_sources.yml +++ b/transform/models/_sources.yml @@ -59,10 +59,10 @@ sources: depending on the measurement capabilities of the detectors. freshness: warn_after: - count: 14 + count: 2 period: day error_after: - count: 28 + count: 3 period: day loaded_at_field: SAMPLE_TIMESTAMP columns: From 1d5553efdb05e7961368bdae55eb973ca8206efa Mon Sep 17 00:00:00 2001 From: pingpingxiu-DOT-ca-gov <149613649+pingpingxiu-DOT-ca-gov@users.noreply.github.com> Date: Mon, 21 Oct 2024 11:57:57 -0700 Subject: [PATCH 3/5] Update _sources.yml fix bug --- transform/models/_sources.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/transform/models/_sources.yml b/transform/models/_sources.yml index 0c34f480..f7535789 100644 --- a/transform/models/_sources.yml +++ b/transform/models/_sources.yml @@ -412,6 +412,13 @@ sources: description: | A log table showing updates to the detector config. This can be joined with the `detector_config` table to get a full history of detector metadata. + freshness: + warn_after: + count: 14 + period: day + error_after: + count: 28 + period: day columns: - name: meta description: Metadata from the data relay scripts. From 2ef2cca3bebc32aa2bbb4a894104be3b9c04e3a2 Mon Sep 17 00:00:00 2001 From: "pingping.xiu@dot.ca.gov" Date: Mon, 21 Oct 2024 19:07:36 +0000 Subject: [PATCH 4/5] Fix trailing --- transform/models/_sources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transform/models/_sources.yml b/transform/models/_sources.yml index f7535789..c61f3cd0 100644 --- a/transform/models/_sources.yml +++ b/transform/models/_sources.yml @@ -412,7 +412,7 @@ sources: description: | A log table showing updates to the detector config. This can be joined with the `detector_config` table to get a full history of detector metadata. - freshness: + freshness: warn_after: count: 14 period: day From 06f09202a8d8e15f53111bacac89598bbab9c225 Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Fri, 10 Jan 2025 09:05:18 -0800 Subject: [PATCH 5/5] Adjust freshness check values based on rough update frequencies of these tables. These checks are not 100% reliable, as it is possible that no stations have updates in, e.g., a month, but if one of them trips it is suggestive that something is amiss. --- transform/models/_sources.yml | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/transform/models/_sources.yml b/transform/models/_sources.yml index c61f3cd0..8bec0724 100644 --- a/transform/models/_sources.yml +++ b/transform/models/_sources.yml @@ -169,14 +169,6 @@ sources: description: | Vehicle Detector Station (VDS) (aka Station) related data obtained via data relay from db96 within the Caltrans network. - freshness: - warn_after: - count: 2 - period: day - error_after: - count: 3 - period: day - loaded_at_field: time_id tables: - name: vds30sec description: | @@ -288,7 +280,6 @@ sources: stations hooked up to a single controller. For example, the same controller might have a station for the mainline as well as a station for an onramp. - freshness: null columns: - name: meta description: Metadata from the data relay scripts. @@ -310,6 +301,14 @@ sources: description: | A log table showing updates to the controller config. This can be joined with the `controller_config` table to get a full history of controller metadata. + freshness: + warn_after: + count: 30 + period: day + error_after: + count: 60 + period: day + loaded_at_field: time_id columns: - name: meta description: Metadata from the data relay scripts. @@ -341,7 +340,6 @@ sources: description: | Metadata for a single VDS station. Multiple stations may be connected to a single controller, and multiple detectors may be connected to a single station. - freshness: null columns: - name: meta description: Metadata from the data relay scripts. @@ -363,6 +361,14 @@ sources: description: | A log table showing updates to the station config. This can be joined with the `station_config` table to get a full history of station metadata. + freshness: + warn_after: + count: 15 + period: day + error_after: + count: 30 + period: day + loaded_at_field: time_id columns: - name: meta description: Metadata from the data relay scripts. @@ -400,7 +406,6 @@ sources: Metadata for a single loop detector. This is the device that actually records flow, occupancy, and speed, and is typically installed in a single lane. Multiple detectors across a set of lanes constitute a station. - freshness: null columns: - name: meta description: Metadata from the data relay scripts. @@ -414,11 +419,12 @@ sources: with the `detector_config` table to get a full history of detector metadata. freshness: warn_after: - count: 14 + count: 15 period: day error_after: - count: 28 + count: 30 period: day + loaded_at_field: time_id columns: - name: meta description: Metadata from the data relay scripts.