feat(imports/occhab): preprocess transient data

PnX-SI · Jan 10, 2024 · 92d8df5 · 92d8df5
1 parent 120bc89
commit 92d8df5
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 9 deletions.
diff --git a/contrib/gn_module_occhab/backend/gn_module_occhab/imports/__init__.py b/contrib/gn_module_occhab/backend/gn_module_occhab/imports/__init__.py
@@ -41,6 +41,28 @@
 )
 
 
+def preprocess_transient_data(imprt, df):
+    updated_cols = set()
+    date_min_field = db.session.execute(
+        db.select(BibFields)
+        .where(BibFields.destination == imprt.destination)
+        .where(BibFields.name_field == "date_min")
+    ).scalar_one()
+    date_max_field = db.session.execute(
+        db.select(BibFields)
+        .where(BibFields.destination == imprt.destination)
+        .where(BibFields.name_field == "date_max")
+    ).scalar_one()
+    updated_cols |= concat_dates(
+        df,
+        datetime_min_col=date_min_field.source_field,
+        datetime_max_col=date_max_field.source_field,
+        date_min_col=date_min_field.source_field,
+        date_max_col=date_max_field.source_field,
+    )
+    return updated_cols
+
+
 def check_transient_data(task, logger, imprt):
     task.update_state(state="PROGRESS", meta={"progress": 0})
     transient_table = imprt.destination.get_transient_table()
@@ -69,15 +91,6 @@ def check_transient_data(task, logger, imprt):
 
         df = load_transient_data_in_dataframe(imprt, entity, source_cols)
 
-        if entity.code == "station":
-            updated_cols |= concat_dates(
-                df,
-                datetime_min_field=fields["date_min"],
-                datetime_max_field=fields["date_max"],
-                date_min_field=fields["date_min"],
-                date_max_field=fields["date_max"],
-            )
-
         updated_cols |= check_types(
             imprt, entity, df, fields
         )  # FIXME do not check station uuid twice

diff --git a/contrib/gn_module_occhab/backend/gn_module_occhab/module.py b/contrib/gn_module_occhab/backend/gn_module_occhab/module.py
@@ -1,5 +1,6 @@
 from geonature.core.gn_commons.models import TModules
 from .imports import (
+    preprocess_transient_data,
     check_transient_data,
     import_data_to_occhab,
     remove_data_from_occhab,
@@ -10,6 +11,7 @@ class OcchabModule(TModules):
     __mapper_args__ = {"polymorphic_identity": "occhab"}
 
     _imports_ = {
+        "preprocess_transient_data": preprocess_transient_data,
         "check_transient_data": check_transient_data,
         "import_data_to_destination": import_data_to_occhab,
         "remove_data_from_destination": remove_data_from_occhab,