feat(report,synthese): move sources to id_import for imported obs + g…

…eneric link to destination module in the import report (#3207)

* feat(report) link from import report to occhab, and filter on id_import

---------

Co-authored-by: jacquesfize <jacques.fize@ecrins-parcnational.fr>

backend/geonature/core/gn_meta/routes.py

@@ -26,7 +26,6 @@
 from geonature.utils.env import DB, db
 from geonature.core.gn_synthese.models import (
     Synthese,
-    TSources,
     CorAreaSynthese,
 )
 from geonature.core.gn_permissions.decorators import login_required
@@ -244,14 +243,9 @@ def uuid_report():
         select(Synthese)
         .where(Synthese.id_module == id_module if id_module is not None else True)
         .where(Synthese.id_dataset == ds_id if ds_id is not None else True)
+        .where(Synthese.id_import == id_import if id_import is not None else True)
     )
 
-    # TODO test in module import ?
-    if id_import:
-        query = query.outerjoin(TSources, TSources.id_source == Synthese.id_source).where(
-            TSources.name_source == f"Import(id={id_import})"
-        )
-
     query = query.order_by(Synthese.id_synthese)
 
     data = [
@@ -323,13 +317,9 @@ def sensi_report(ds_id=None):
         .where(LAreas.id_type == func.ref_geo.get_id_area_type("DEP"))
         .where(Synthese.id_module == id_module if id_module else True)
         .where(Synthese.id_dataset == ds_id)
+        .where(Synthese.id_import == id_import if id_import else True)
     )
 
-    if id_import:
-        query = query.outerjoin(TSources, TSources.id_source == Synthese.id_source).where(
-            TSources.name_source == "Import(id={})".format(id_import)
-        )
-
     query = query.group_by(
         Synthese.id_synthese, TNomenclatures.cd_nomenclature, TNomenclatures.label_fr
     )

backend/geonature/core/gn_synthese/imports/actions.py

@@ -303,21 +303,8 @@ def update_batch_progress(batch, step):
     @staticmethod
     def import_data_to_destination(imprt: TImports) -> None:
         module = TModules.query.filter_by(module_code="IMPORT").one()
-        name_source = f"Import(id={imprt.id_import})"
+        name_source = "Import"
         source = TSources.query.filter_by(module=module, name_source=name_source).one_or_none()
-        if source is None:
-            entity_source_pk_field = BibFields.query.filter_by(
-                destination=imprt.destination,
-                name_field="entity_source_pk_value",
-            ).one()
-            source = TSources(
-                module=module,
-                name_source=name_source,
-                desc_source=f"Imported data from import module (id={imprt.id_import})",
-                entity_source_pk_field=entity_source_pk_field.dest_field,
-            )
-            db.session.add(source)
-            db.session.flush()  # force id_source definition
         transient_table = imprt.destination.get_transient_table()
 
         fields = {
@@ -364,6 +351,7 @@ def import_data_to_destination(imprt: TImports) -> None:
                 sa.literal(source.id_source),
                 sa.literal(source.module.id_module),
                 sa.literal(imprt.id_dataset),
+                sa.literal(imprt.id_import),
                 sa.literal("I"),
             )
             .where(transient_table.c.id_import == imprt.id_import)
@@ -373,6 +361,7 @@ def import_data_to_destination(imprt: TImports) -> None:
             "id_source",
             "id_module",
             "id_dataset",
+            "id_import",
             "last_action",
         ]
         insert_stmt = sa.insert(Synthese).from_select(
@@ -384,25 +373,21 @@ def import_data_to_destination(imprt: TImports) -> None:
         # TODO: Improve this
         imprt.statistics = {
             "observation_count": (
-                db.session.query(func.count(Synthese.cd_nom)).filter_by(source=source).scalar()
+                db.session.query(func.count(Synthese.cd_nom))
+                .filter_by(id_import=imprt.id_import)
+                .scalar()
             ),
             "taxa_count": (
                 db.session.query(func.count(distinct(Synthese.cd_nom)))
-                .filter_by(source=source)
+                .filter_by(id_import=imprt.id_import)
                 .scalar()
             ),
         }
 
     @staticmethod
     def remove_data_from_destination(imprt: TImports) -> None:
-        source = TSources.query.filter(
-            TSources.module.has(TModules.module_code == "IMPORT"),
-            TSources.name_source == f"Import(id={imprt.id_import})",
-        ).one_or_none()
-        if source is not None:
-            with start_sentry_child(op="task", description="clean imported data"):
-                Synthese.query.filter(Synthese.source == source).delete()
-            db.session.delete(source)
+        with start_sentry_child(op="task", description="clean imported data"):
+            Synthese.query.filter(Synthese.id_import == imprt.id_import).delete()
 
     @staticmethod
     def report_plot(imprt: TImports) -> StandaloneEmbedJson:
@@ -431,12 +416,6 @@ def report_plot(imprt: TImports) -> StandaloneEmbedJson:
             Returns a dict containing data required to generate the plot
         """
 
-        # Get the source of the import
-        source = TSources.query.filter(
-            TSources.module.has(TModules.module_code == "IMPORT"),
-            TSources.name_source == f"Import(id={imprt.id_import})",
-        ).one_or_none()
-
         # Define the taxonomic rank to consider
         taxon_ranks = "regne phylum classe ordre famille sous_famille tribu group1_inpn group2_inpn group3_inpn".split()
         figures = []
@@ -452,7 +431,7 @@ def report_plot(imprt: TImports) -> StandaloneEmbedJson:
                 )
                 .select_from(Synthese)
                 .outerjoin(Taxref, Taxref.cd_nom == Synthese.cd_nom)
-                .where(Synthese.source == source)
+                .where(Synthese.id_import == imprt.id_import)
                 .group_by(c_rank_taxref)
             )
             data = np.asarray(
@@ -567,12 +546,7 @@ def compute_bounding_box(imprt: TImports):
         # The destination where clause will be called only when the import is finished,
         # avoiding looking for unexisting source when the import is still in progress.
         destination_where_clause = (
-            lambda imprt, destination_table: db.session.scalar(
-                select(TSources.id_source).where(
-                    TSources.name_source == f"Import(id={imprt.id_import})"
-                )
-            )
-            == destination_table.c.id_source
+            lambda imprt, destination_table: imprt.id_import == destination_table.c.id_import
         )
         return compute_bounding_box(
             imprt, "observation", "the_geom_4326", destination_where_clause=destination_where_clause

backend/geonature/core/gn_synthese/models.py

@@ -40,6 +40,7 @@
 from utils_flask_sqla_geo.mixins import GeoFeatureCollectionMixin
 from pypn_habref_api.models import Habref
 from apptax.taxonomie.models import Taxref
+from geonature.core.imports.models import TImports as Import
 from ref_geo.models import LAreas
 
 from geonature.core.gn_meta.models import TDatasets, TAcquisitionFramework
@@ -264,6 +265,7 @@ class Synthese(DB.Model):
     id_source = DB.Column(DB.Integer, ForeignKey(TSources.id_source), nullable=False)
     source = relationship(TSources)
     id_module = DB.Column(DB.Integer, ForeignKey(TModules.id_module))
+    id_import = db.Column(db.Integer, ForeignKey(Import.id_import), nullable=True)
     module = DB.relationship(TModules)
     entity_source_pk_value = DB.Column(DB.Unicode)
     id_dataset = DB.Column(DB.Integer, ForeignKey(TDatasets.id_dataset))
@@ -583,6 +585,7 @@ class VSyntheseForWebApp(DB.Model):
     unique_id_sinp = DB.Column(UUID(as_uuid=True))
     unique_id_sinp_grp = DB.Column(UUID(as_uuid=True))
     id_source = DB.Column(DB.Integer, nullable=False)
+    id_import = DB.Column(DB.Integer, nullable=False)
     id_module = DB.Column(DB.Integer)
     entity_source_pk_value = DB.Column(DB.Integer)
     id_dataset = DB.Column(DB.Integer)