feat(import): move import count to statistics (#3235)

* feat(import): move import count to statistics
* feat(import) : add migration to drop import_count
* feat(migration): modify revision to include import_count transfer data to statistics + fix notification template revision
* fix(test): impact the deplacement of `import_count` into `TImports.statistics`
* feat(import,synthese): remove unused value computation + increase timeout for forntend test

backend/geonature/core/gn_synthese/imports/actions.py

@@ -69,7 +69,7 @@ class SyntheseImportActions(ImportActions):
     @staticmethod
     def statistics_labels() -> typing.List[ImportStatisticsLabels]:
         return [
-            {"key": "observation_count", "value": "Nombre d'observations importées"},
+            {"key": "import_count", "value": "Nombre d'observations importées"},
             {"key": "taxa_count", "value": "Nombre de taxons"},
         ]
 
@@ -372,11 +372,6 @@ def import_data_to_destination(imprt: TImports) -> None:
 
         # TODO: Improve this
         imprt.statistics = {
-            "observation_count": (
-                db.session.query(func.count(Synthese.cd_nom))
-                .filter_by(id_import=imprt.id_import)
-                .scalar()
-            ),
             "taxa_count": (
                 db.session.query(func.count(distinct(Synthese.cd_nom)))
                 .filter_by(id_import=imprt.id_import)

backend/geonature/core/imports/models.py

@@ -373,9 +373,6 @@ class TImports(InstancePermissionMixin, db.Model):
     date_end_import = db.Column(db.DateTime, nullable=True)
     source_count = db.Column(db.Integer, nullable=True)
     erroneous_rows = deferred(db.Column(ARRAY(db.Integer), nullable=True))
-    # TODO: integrate this in statistics
-    # statistics: common and destination (line count, etc. etc. dedans)
-    import_count = db.Column(db.Integer, nullable=True)
     statistics = db.Column(
         MutableDict.as_mutable(JSON), nullable=False, server_default="'{}'::jsonb"
     )

backend/geonature/core/imports/tasks.py

@@ -99,7 +99,7 @@ def do_import_in_destination(self, import_id):
             .where(transient_table.c[entity.validity_column] == True)
         ).scalar()
         count_entities += n_valid_data
-    imprt.import_count = count_entities
+    imprt.statistics["import_count"] = count_entities
 
     # Clear transient data
     stmt = delete(transient_table).where(transient_table.c.id_import == imprt.id_import)

backend/geonature/core/imports/utils.py

@@ -69,8 +69,8 @@ def clean_import(imprt: TImports, step: ImportStep) -> None:
         imprt.processed = False
     if step <= ImportStep.IMPORT:
         imprt.date_end_import = None
-        imprt.import_count = None
-        imprt.statistics = {}
+
+        imprt.statistics = {"import_count": None}
         imprt.destination.actions.remove_data_from_destination(imprt)
 
 

backend/geonature/migrations/data/imports/sample_data.sql

@@ -902,7 +902,6 @@ INSERT INTO gn_imports.t_imports (
         encoding,
         full_file_name,
         source_count,
-        import_count,
         uuid_autogenerated,
         altitude_autogenerated,
         date_min_data,
@@ -931,7 +930,6 @@ VALUES (
         'UTF-8',
         'valid_file_test_link_list_import_synthese.csv',
         100,
-        95,
         true,
         true,
         '2022-01-01 00:00:00',
@@ -960,7 +958,6 @@ VALUES (
         'UTF-8',
         'valid_file_test_import_occhab.csv',
         100,
-        95,
         true,
         true,
         '2022-01-01 00:00:00',
@@ -989,7 +986,6 @@ VALUES (
         'UTF-8',
         'valid_file_test_import_synthese.csv',
         100,
-        95,
         true,
         true,
         '2022-01-01 00:00:00',
@@ -1018,7 +1014,6 @@ VALUES (
         'UTF-8',
         'valid_file_test_import_synthese.csv',
         100,
-        95,
         true,
         true,
         '2022-01-01 00:00:00',

backend/geonature/migrations/versions/02e9b8758709_update_notifications_import_done.py

@@ -19,10 +19,10 @@
 ORIGINAL_TEMPLATE_DB = (
     "<b>Import n° {{ import.id_import }}</b> correctement terminé et intégré dans la synthèse"
 )
-ORIGINAL_TEMPLATE_MAIL = '<p>Bonjour <i>{{ role.nom_complet }}</i> !</p> <p>Votre <a href="{{ url }}">import <b>n°{{ import.id_import }}</b></a> s’est terminé correctement {% if import.import_count == import.source_count %} 👌 et a été bien {% else %} 👍 mais a été partiellement {% endif %} intégré dans la synthèse.</p><p> {{ import.import_count }} / {{ import.source_count }} données ont pu être effectivement intégrées dans la synthèse.</p><hr><p><i>Vous recevez cet email automatiquement via le service de notification de GeoNature. <a href="{{url_notification_rules}}">Gestion de vos règles de notification</a>.</i></p>'
+ORIGINAL_TEMPLATE_MAIL = """<p>Bonjour <i>{{ role.nom_complet }}</i> !</p> <p>Votre <a href="{{ url }}">import <b>n°{{ import.id_import }}</b></a> s’est terminé correctement {% if import.statistics["import_count"] == import.source_count %} 👌 et a été bien {% else %} 👍 mais a été partiellement {% endif %} intégré dans la synthèse.</p><p> {{ import.statistics["import_count"] }} / {{ import.source_count }} données ont pu être effectivement intégrées dans la synthèse.</p><hr><p><i>Vous recevez cet email automatiquement via le service de notification de GeoNature. <a href="{{url_notification_rules}}">Gestion de vos règles de notification</a>.</i></p>"""
 
-NEW_TEMPLATE_DB = "<b>Import n° {{ import.id_import }}</b> terminé{% if import.errors|length > 0 %} 👍 et partiellement {% else %} 👌 et correctement {% endif %}intégré : {{ import.import_count }} entité{% if import.import_count > 1 %}s{% endif %} importée{% if import.import_count > 1 %}s{% endif %} dans la destination {{ destination.label }}"
-NEW_TEMPLATE_MAIL = '<p>Bonjour <i>{{ role.nom_complet }}</i> !</p> <p>Votre <a href="{{ url }}">import <b>n°{{ import.id_import }}</b></a> est terminé {% if import.errors|length > 0 %} 👍 mais a été partiellement {% else %} 👌 et a été correctement {% endif %} intégré dans la destination {{ destination.label }}.</p><p> {{ import.import_count }} entité{% if import.import_count > 1 %}s valides ont{% else %} valide a{% endif %} pu être effectivement intégrée{% if import.import_count > 1 %}s{% endif %}.</p><hr><p><i>Vous recevez cet email automatiquement via le service de notification de GeoNature. <a href="{{url_notification_rules}}">Gestion de vos règles de notification</a>.</i></p>'
+NEW_TEMPLATE_DB = """<b>Import n° {{ import.id_import }}</b> terminé{% if import.errors|length > 0 %} 👍 et partiellement {% else %} 👌 et correctement {% endif %}intégré : {{ import.statistics["import_count"] }} entité{% if import.statistics["import_count"] > 1 %}s{% endif %} importée{% if import.statistics["import_count"] > 1 %}s{% endif %} dans la destination {{ destination.label }}"""
+NEW_TEMPLATE_MAIL = """<p>Bonjour <i>{{ role.nom_complet }}</i> !</p> <p>Votre <a href="{{ url }}">import <b>n°{{ import.id_import }}</b></a> est terminé {% if import.errors|length > 0 %} 👍 mais a été partiellement {% else %} 👌 et a été correctement {% endif %} intégré dans la destination {{ destination.label }}.</p><p> {{ import.statistics["import_count"] }} entité{% if import.statistics["import_count"] > 1 %}s valides ont{% else %} valide a{% endif %} pu être effectivement intégrée{% if import.statistics["import_count"] > 1 %}s{% endif %}.</p><hr><p><i>Vous recevez cet email automatiquement via le service de notification de GeoNature. <a href="{{url_notification_rules}}">Gestion de vos règles de notification</a>.</i></p>"""
 
 
 def upgrade():

backend/geonature/migrations/versions/7b6a578eccd7_drop_import_count_column.py

@@ -0,0 +1,66 @@
+"""drop import_count column
+
+Revision ID: 7b6a578eccd7
+Revises: c49474d2f1f7
+Create Date: 2024-10-18 16:24:44.145501
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "7b6a578eccd7"
+down_revision = "c49474d2f1f7"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute(
+        """
+        UPDATE gn_imports.t_imports
+        SET statistics = statistics::jsonb || jsonb_build_object('import_count', import_count);
+    """
+    )
+    op.drop_column(
+        schema="gn_imports",
+        table_name="t_imports",
+        column_name="import_count",
+    )
+
+
+def downgrade():
+    op.add_column(
+        schema="gn_imports",
+        table_name="t_imports",
+        column=sa.Column(
+            "import_count",
+            sa.Integer,
+        ),
+    )
+
+    op.execute(
+        """
+        WITH count_ AS (
+            SELECT
+                id_import as id_import,
+                (statistics->>'import_count')::integer as import_count
+            FROM
+                gn_imports.t_imports
+            WHERE
+                statistics->>'import_count' IS NOT NULL
+        )
+        UPDATE gn_imports.t_imports as GN
+        SET import_count = count_.import_count
+        FROM count_
+        where GN.id_import = count_.id_import;
+    """
+    )
+    op.execute(
+        """
+        UPDATE gn_imports.t_imports as GN
+        SET statistics = statistics::jsonb #- '{import_count}';
+    """
+    )

backend/geonature/tests/imports/jsonschema_definitions.py

@@ -204,7 +204,6 @@
             "date_create_import": {"type": "string"},
             "date_update_import": {"type": "string"},
             "source_count": {"type": ["integer", "null"]},
-            "import_count": {"type": ["integer", "null"]},
             "statistics": {"type": ["object", "null"]},
             "date_min_data": {"type": ["string", "null"]},
             "date_max_data": {"type": ["string", "null"]},
@@ -241,7 +240,6 @@
             "date_create_import",
             "date_update_import",
             "source_count",
-            "import_count",
             "statistics",
             "date_min_data",
             "date_max_data",

backend/geonature/tests/imports/test_imports_occhab.py

@@ -384,7 +384,11 @@ def test_import_valid_file(self, imported_import):
                 ),
             },
         )
-        assert imported_import.statistics == {"station_count": 7, "habitat_count": 11}
+        assert imported_import.statistics == {
+            "import_count": 18,
+            "station_count": 7,
+            "habitat_count": 11,
+        }
         assert (
             db.session.scalar(
                 sa.select(sa.func.count()).where(Station.id_import == imported_import.id_import)
@@ -506,7 +510,11 @@ def test_import_without_default_uuid(self, no_default_uuid, imported_import):
                 ),
             },
         )
-        assert imported_import.statistics == {"station_count": 3, "habitat_count": 6}
+        assert imported_import.statistics == {
+            "import_count": 9,
+            "station_count": 3,
+            "habitat_count": 6,
+        }
         assert (
             db.session.scalar(
                 sa.select(sa.func.count()).where(Station.id_import == imported_import.id_import)

backend/geonature/tests/imports/test_imports_synthese.py

@@ -967,9 +967,14 @@ def test_import_valid_file(self, users, datasets):
             .where(transient_table.c.id_import == imprt.id_import)
         ).scalar()
         assert transient_rows_count == 0
-        assert valid_file_line_count - len(valid_file_invalid_rows) == imprt.import_count
+        assert (
+            valid_file_line_count - len(valid_file_invalid_rows) == imprt.statistics["import_count"]
+        )
         assert valid_file_taxa_count == imprt.statistics["taxa_count"]
-        assert Synthese.query.filter_by(id_import=imprt.id_import).count() == imprt.import_count
+        assert (
+            Synthese.query.filter_by(id_import=imprt.id_import).count()
+            == imprt.statistics["import_count"]
+        )
 
         # Delete step
         r = self.client.delete(url_for("import.delete_import", import_id=imprt.id_import))

frontend/cypress/e2e/import/constants/common.js

@@ -5,4 +5,4 @@ export const VIEWPORTS = [
   },
 ];
 
-export const TIMEOUT_WAIT = 1000;
+export const TIMEOUT_WAIT = 2000;