From bfc16e2fcc7b29aec4385099343408dfe1ec8d06 Mon Sep 17 00:00:00 2001 From: Jacques Fize <4259846+jacquesfize@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:55:30 +0200 Subject: [PATCH] feat(import): move import count to statistics (#3235) * feat(import): move import count to statistics * feat(import) : add migration to drop import_count * feat(migration): modify revision to include import_count transfer data to statistics + fix notification template revision * fix(test): impact the deplacement of `import_count` into `TImports.statistics` * feat(import,synthese): remove unused value computation + increase timeout for forntend test --- .../core/gn_synthese/imports/actions.py | 7 +- backend/geonature/core/imports/models.py | 3 - backend/geonature/core/imports/tasks.py | 2 +- backend/geonature/core/imports/utils.py | 4 +- .../migrations/data/imports/sample_data.sql | 5 - ...758709_update_notifications_import_done.py | 6 +- .../7b6a578eccd7_drop_import_count_column.py | 66 ++ .../tests/imports/jsonschema_definitions.py | 2 - .../tests/imports/test_imports_occhab.py | 12 +- .../tests/imports/test_imports_synthese.py | 9 +- .../cypress/e2e/import/constants/common.js | 2 +- .../import/synthese/liste_import.json | 1046 ++++++++--------- .../modules/imports/models/import.model.ts | 7 +- .../modules/imports/utils/format-row-count.ts | 4 +- 14 files changed, 611 insertions(+), 564 deletions(-) create mode 100644 backend/geonature/migrations/versions/7b6a578eccd7_drop_import_count_column.py diff --git a/backend/geonature/core/gn_synthese/imports/actions.py b/backend/geonature/core/gn_synthese/imports/actions.py index 082dbb3d55..222cef5fda 100644 --- a/backend/geonature/core/gn_synthese/imports/actions.py +++ b/backend/geonature/core/gn_synthese/imports/actions.py @@ -69,7 +69,7 @@ class SyntheseImportActions(ImportActions): @staticmethod def statistics_labels() -> typing.List[ImportStatisticsLabels]: return [ - {"key": "observation_count", "value": "Nombre d'observations importées"}, + {"key": "import_count", "value": "Nombre d'observations importées"}, {"key": "taxa_count", "value": "Nombre de taxons"}, ] @@ -372,11 +372,6 @@ def import_data_to_destination(imprt: TImports) -> None: # TODO: Improve this imprt.statistics = { - "observation_count": ( - db.session.query(func.count(Synthese.cd_nom)) - .filter_by(id_import=imprt.id_import) - .scalar() - ), "taxa_count": ( db.session.query(func.count(distinct(Synthese.cd_nom))) .filter_by(id_import=imprt.id_import) diff --git a/backend/geonature/core/imports/models.py b/backend/geonature/core/imports/models.py index e3bab66fad..de6c5d53cc 100644 --- a/backend/geonature/core/imports/models.py +++ b/backend/geonature/core/imports/models.py @@ -373,9 +373,6 @@ class TImports(InstancePermissionMixin, db.Model): date_end_import = db.Column(db.DateTime, nullable=True) source_count = db.Column(db.Integer, nullable=True) erroneous_rows = deferred(db.Column(ARRAY(db.Integer), nullable=True)) - # TODO: integrate this in statistics - # statistics: common and destination (line count, etc. etc. dedans) - import_count = db.Column(db.Integer, nullable=True) statistics = db.Column( MutableDict.as_mutable(JSON), nullable=False, server_default="'{}'::jsonb" ) diff --git a/backend/geonature/core/imports/tasks.py b/backend/geonature/core/imports/tasks.py index b2d967ee17..e1ae433ba6 100644 --- a/backend/geonature/core/imports/tasks.py +++ b/backend/geonature/core/imports/tasks.py @@ -99,7 +99,7 @@ def do_import_in_destination(self, import_id): .where(transient_table.c[entity.validity_column] == True) ).scalar() count_entities += n_valid_data - imprt.import_count = count_entities + imprt.statistics["import_count"] = count_entities # Clear transient data stmt = delete(transient_table).where(transient_table.c.id_import == imprt.id_import) diff --git a/backend/geonature/core/imports/utils.py b/backend/geonature/core/imports/utils.py index d33ada6475..77b0e8e3dc 100644 --- a/backend/geonature/core/imports/utils.py +++ b/backend/geonature/core/imports/utils.py @@ -69,8 +69,8 @@ def clean_import(imprt: TImports, step: ImportStep) -> None: imprt.processed = False if step <= ImportStep.IMPORT: imprt.date_end_import = None - imprt.import_count = None - imprt.statistics = {} + + imprt.statistics = {"import_count": None} imprt.destination.actions.remove_data_from_destination(imprt) diff --git a/backend/geonature/migrations/data/imports/sample_data.sql b/backend/geonature/migrations/data/imports/sample_data.sql index 022a370b97..bc81e25297 100644 --- a/backend/geonature/migrations/data/imports/sample_data.sql +++ b/backend/geonature/migrations/data/imports/sample_data.sql @@ -902,7 +902,6 @@ INSERT INTO gn_imports.t_imports ( encoding, full_file_name, source_count, - import_count, uuid_autogenerated, altitude_autogenerated, date_min_data, @@ -931,7 +930,6 @@ VALUES ( 'UTF-8', 'valid_file_test_link_list_import_synthese.csv', 100, - 95, true, true, '2022-01-01 00:00:00', @@ -960,7 +958,6 @@ VALUES ( 'UTF-8', 'valid_file_test_import_occhab.csv', 100, - 95, true, true, '2022-01-01 00:00:00', @@ -989,7 +986,6 @@ VALUES ( 'UTF-8', 'valid_file_test_import_synthese.csv', 100, - 95, true, true, '2022-01-01 00:00:00', @@ -1018,7 +1014,6 @@ VALUES ( 'UTF-8', 'valid_file_test_import_synthese.csv', 100, - 95, true, true, '2022-01-01 00:00:00', diff --git a/backend/geonature/migrations/versions/02e9b8758709_update_notifications_import_done.py b/backend/geonature/migrations/versions/02e9b8758709_update_notifications_import_done.py index a1177d7193..3051059ce7 100644 --- a/backend/geonature/migrations/versions/02e9b8758709_update_notifications_import_done.py +++ b/backend/geonature/migrations/versions/02e9b8758709_update_notifications_import_done.py @@ -19,10 +19,10 @@ ORIGINAL_TEMPLATE_DB = ( "Import n° {{ import.id_import }} correctement terminé et intégré dans la synthèse" ) -ORIGINAL_TEMPLATE_MAIL = '

Bonjour {{ role.nom_complet }} !

Votre import n°{{ import.id_import }} s’est terminé correctement {% if import.import_count == import.source_count %} 👌 et a été bien {% else %} 👍 mais a été partiellement {% endif %} intégré dans la synthèse.

{{ import.import_count }} / {{ import.source_count }} données ont pu être effectivement intégrées dans la synthèse.

Vous recevez cet email automatiquement via le service de notification de GeoNature. Gestion de vos règles de notification.

' +ORIGINAL_TEMPLATE_MAIL = """

Bonjour {{ role.nom_complet }} !

Votre import n°{{ import.id_import }} s’est terminé correctement {% if import.statistics["import_count"] == import.source_count %} 👌 et a été bien {% else %} 👍 mais a été partiellement {% endif %} intégré dans la synthèse.

{{ import.statistics["import_count"] }} / {{ import.source_count }} données ont pu être effectivement intégrées dans la synthèse.

Vous recevez cet email automatiquement via le service de notification de GeoNature. Gestion de vos règles de notification.

""" -NEW_TEMPLATE_DB = "Import n° {{ import.id_import }} terminé{% if import.errors|length > 0 %} 👍 et partiellement {% else %} 👌 et correctement {% endif %}intégré : {{ import.import_count }} entité{% if import.import_count > 1 %}s{% endif %} importée{% if import.import_count > 1 %}s{% endif %} dans la destination {{ destination.label }}" -NEW_TEMPLATE_MAIL = '

Bonjour {{ role.nom_complet }} !

Votre import n°{{ import.id_import }} est terminé {% if import.errors|length > 0 %} 👍 mais a été partiellement {% else %} 👌 et a été correctement {% endif %} intégré dans la destination {{ destination.label }}.

{{ import.import_count }} entité{% if import.import_count > 1 %}s valides ont{% else %} valide a{% endif %} pu être effectivement intégrée{% if import.import_count > 1 %}s{% endif %}.

Vous recevez cet email automatiquement via le service de notification de GeoNature. Gestion de vos règles de notification.

' +NEW_TEMPLATE_DB = """Import n° {{ import.id_import }} terminé{% if import.errors|length > 0 %} 👍 et partiellement {% else %} 👌 et correctement {% endif %}intégré : {{ import.statistics["import_count"] }} entité{% if import.statistics["import_count"] > 1 %}s{% endif %} importée{% if import.statistics["import_count"] > 1 %}s{% endif %} dans la destination {{ destination.label }}""" +NEW_TEMPLATE_MAIL = """

Bonjour {{ role.nom_complet }} !

Votre import n°{{ import.id_import }} est terminé {% if import.errors|length > 0 %} 👍 mais a été partiellement {% else %} 👌 et a été correctement {% endif %} intégré dans la destination {{ destination.label }}.

{{ import.statistics["import_count"] }} entité{% if import.statistics["import_count"] > 1 %}s valides ont{% else %} valide a{% endif %} pu être effectivement intégrée{% if import.statistics["import_count"] > 1 %}s{% endif %}.

Vous recevez cet email automatiquement via le service de notification de GeoNature. Gestion de vos règles de notification.

""" def upgrade(): diff --git a/backend/geonature/migrations/versions/7b6a578eccd7_drop_import_count_column.py b/backend/geonature/migrations/versions/7b6a578eccd7_drop_import_count_column.py new file mode 100644 index 0000000000..08f2d43248 --- /dev/null +++ b/backend/geonature/migrations/versions/7b6a578eccd7_drop_import_count_column.py @@ -0,0 +1,66 @@ +"""drop import_count column + +Revision ID: 7b6a578eccd7 +Revises: c49474d2f1f7 +Create Date: 2024-10-18 16:24:44.145501 + +""" + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "7b6a578eccd7" +down_revision = "c49474d2f1f7" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + """ + UPDATE gn_imports.t_imports + SET statistics = statistics::jsonb || jsonb_build_object('import_count', import_count); + """ + ) + op.drop_column( + schema="gn_imports", + table_name="t_imports", + column_name="import_count", + ) + + +def downgrade(): + op.add_column( + schema="gn_imports", + table_name="t_imports", + column=sa.Column( + "import_count", + sa.Integer, + ), + ) + + op.execute( + """ + WITH count_ AS ( + SELECT + id_import as id_import, + (statistics->>'import_count')::integer as import_count + FROM + gn_imports.t_imports + WHERE + statistics->>'import_count' IS NOT NULL + ) + UPDATE gn_imports.t_imports as GN + SET import_count = count_.import_count + FROM count_ + where GN.id_import = count_.id_import; + """ + ) + op.execute( + """ + UPDATE gn_imports.t_imports as GN + SET statistics = statistics::jsonb #- '{import_count}'; + """ + ) diff --git a/backend/geonature/tests/imports/jsonschema_definitions.py b/backend/geonature/tests/imports/jsonschema_definitions.py index 89e58a161d..42a94a4520 100644 --- a/backend/geonature/tests/imports/jsonschema_definitions.py +++ b/backend/geonature/tests/imports/jsonschema_definitions.py @@ -204,7 +204,6 @@ "date_create_import": {"type": "string"}, "date_update_import": {"type": "string"}, "source_count": {"type": ["integer", "null"]}, - "import_count": {"type": ["integer", "null"]}, "statistics": {"type": ["object", "null"]}, "date_min_data": {"type": ["string", "null"]}, "date_max_data": {"type": ["string", "null"]}, @@ -241,7 +240,6 @@ "date_create_import", "date_update_import", "source_count", - "import_count", "statistics", "date_min_data", "date_max_data", diff --git a/backend/geonature/tests/imports/test_imports_occhab.py b/backend/geonature/tests/imports/test_imports_occhab.py index 44d1869a8d..ad0237bbd3 100644 --- a/backend/geonature/tests/imports/test_imports_occhab.py +++ b/backend/geonature/tests/imports/test_imports_occhab.py @@ -384,7 +384,11 @@ def test_import_valid_file(self, imported_import): ), }, ) - assert imported_import.statistics == {"station_count": 7, "habitat_count": 11} + assert imported_import.statistics == { + "import_count": 18, + "station_count": 7, + "habitat_count": 11, + } assert ( db.session.scalar( sa.select(sa.func.count()).where(Station.id_import == imported_import.id_import) @@ -506,7 +510,11 @@ def test_import_without_default_uuid(self, no_default_uuid, imported_import): ), }, ) - assert imported_import.statistics == {"station_count": 3, "habitat_count": 6} + assert imported_import.statistics == { + "import_count": 9, + "station_count": 3, + "habitat_count": 6, + } assert ( db.session.scalar( sa.select(sa.func.count()).where(Station.id_import == imported_import.id_import) diff --git a/backend/geonature/tests/imports/test_imports_synthese.py b/backend/geonature/tests/imports/test_imports_synthese.py index 9206b16774..cac65221c4 100644 --- a/backend/geonature/tests/imports/test_imports_synthese.py +++ b/backend/geonature/tests/imports/test_imports_synthese.py @@ -967,9 +967,14 @@ def test_import_valid_file(self, users, datasets): .where(transient_table.c.id_import == imprt.id_import) ).scalar() assert transient_rows_count == 0 - assert valid_file_line_count - len(valid_file_invalid_rows) == imprt.import_count + assert ( + valid_file_line_count - len(valid_file_invalid_rows) == imprt.statistics["import_count"] + ) assert valid_file_taxa_count == imprt.statistics["taxa_count"] - assert Synthese.query.filter_by(id_import=imprt.id_import).count() == imprt.import_count + assert ( + Synthese.query.filter_by(id_import=imprt.id_import).count() + == imprt.statistics["import_count"] + ) # Delete step r = self.client.delete(url_for("import.delete_import", import_id=imprt.id_import)) diff --git a/frontend/cypress/e2e/import/constants/common.js b/frontend/cypress/e2e/import/constants/common.js index d26ebd87d6..7bacfa87c1 100644 --- a/frontend/cypress/e2e/import/constants/common.js +++ b/frontend/cypress/e2e/import/constants/common.js @@ -5,4 +5,4 @@ export const VIEWPORTS = [ }, ]; -export const TIMEOUT_WAIT = 1000; +export const TIMEOUT_WAIT = 2000; diff --git a/frontend/cypress/fixtures/import/synthese/liste_import.json b/frontend/cypress/fixtures/import/synthese/liste_import.json index 7c663204ab..5c2da9d0cd 100644 --- a/frontend/cypress/fixtures/import/synthese/liste_import.json +++ b/frontend/cypress/fixtures/import/synthese/liste_import.json @@ -1,541 +1,519 @@ { - "count": 2, - "imports": [ - { - "altitude_autogenerated": null, - "authors": [ + "count": 2, + "imports": [ + { + "altitude_autogenerated": null, + "authors": [ + { + "nom_complet": "User Test - 1" + } + ], + "authors_name": "User Test - 1", + "available_encodings": ["iso-8859-1", "iso-8859-15", "utf-8"], + "available_formats": ["csv", "geojson"], + "available_separators": [",", ";"], + "columns": [ + "error", + "id_synthese", + "id_origine", + "comment_releve", + "comment_occurrence", + "date_debut", + "date_fin", + "heure_debut", + "heure_fin", + "cd_nom", + "cd_ref", + "nom_valide", + "nom_vernaculaire", + "nom_cite", + "regne", + "group1_inpn", + "group2_inpn", + "classe", + "ordre", + "famille", + "rang_taxo", + "nombre_min", + "nombre_max", + "alti_min", + "alti_max", + "prof_min", + "prof_max", + "observateurs", + "determinateur", + "communes", + "geometrie_wkt_4326", + "x_centroid_4326", + "y_centroid_4326", + "nom_lieu", + "validateur", + "niveau_validation", + "date_validation", + "comment_validation", + "preuve_numerique_url", + "preuve_non_numerique", + "jdd_nom", + "jdd_uuid", + "jdd_id", + "ca_nom", + "ca_uuid", + "ca_id", + "cd_habref", + "cd_habitat", + "nom_habitat", + "precision_geographique", + "nature_objet_geo", + "type_regroupement", + "methode_regroupement", + "technique_observation", + "biologique_statut", + "etat_biologique", + "biogeographique_statut", + "naturalite", + "preuve_existante", + "niveau_precision_diffusion", + "stade_vie", + "sexe", + "objet_denombrement", + "type_denombrement", + "niveau_sensibilite", + "statut_observation", + "floutage_dee", + "statut_source", + "type_info_geo", + "methode_determination", + "comportement", + "reference_biblio", + "uuid_perm_sinp", + "uuid_perm_grp_sinp", + "date_creation", + "date_modification" + ], + "contentmapping": { + "DEE_FLOU": { + "Non": "NON" + }, + "ETA_BIO": { + "Non renseign\u00e9": "1" + }, + "METH_DETERMIN": { + "Autre m\u00e9thode de d\u00e9termination": "2" + }, + "METH_OBS": { + "Galerie/terrier": "23" + }, + "NATURALITE": { + "Sauvage": "1" + }, + "NAT_OBJ_GEO": { + "Inventoriel": "In" + }, + "NIV_PRECIS": { + "Pr\u00e9cise": "5" + }, + "OBJ_DENBR": { + "Individu": "IND" + }, + "OCC_COMPORTEMENT": { + "Non renseign\u00e9": "1" + }, + "PREUVE_EXIST": { + "Oui": "1" + }, + "SENSIBILITE": { + "Non sensible - Diffusion pr\u00e9cise": "0" + }, + "SEXE": { + "Femelle": "2" + }, + "STADE_VIE": { + "Adulte": "2", + "Immature": "4", + "Juv\u00e9nile": "3" + }, + "STATUT_BIO": { + "Non renseign\u00e9": "1" + }, + "STATUT_OBS": { + "Pr\u00e9sent": "Pr" + }, + "STATUT_SOURCE": { + "Terrain": "Te" + }, + "STATUT_VALID": { + "En attente de validation": "0" + }, + "STAT_BIOGEO": { + "Non renseign\u00e9": "1" + }, + "TYP_DENBR": { + "Compt\u00e9": "Co" + }, + "TYP_GRP": { + "OBS": "OBS" + } + }, + "cruved": { + "C": true, + "D": true, + "E": false, + "R": true, + "U": true, + "V": false + }, + "dataset": { + "active": true, + "dataset_name": "Contact al\u00e9atoire tous r\u00e8gnes confondus" + }, + "date_create_import": "2024-05-13 23:55:51.167003", + "date_end_import": "2024-05-13 23:56:46.793605", + "date_max_data": null, + "date_min_data": null, + "date_update_import": "2024-05-13 23:56:46.796963", + "destination": { + "code": "synthese", + "label": "Synth\u00e8se", + "statistics_labels": [ { - "nom_complet": "User Test - 1" - } - ], - "authors_name": "User Test - 1" , - "available_encodings": [ - "iso-8859-1", - "iso-8859-15", - "utf-8" - ], - "available_formats": [ - "csv", - "geojson" - ], - "available_separators": [ - ",", - ";" - ], - "columns": [ - "error", - "id_synthese", - "id_origine", - "comment_releve", - "comment_occurrence", - "date_debut", - "date_fin", - "heure_debut", - "heure_fin", - "cd_nom", - "cd_ref", - "nom_valide", - "nom_vernaculaire", - "nom_cite", - "regne", - "group1_inpn", - "group2_inpn", - "classe", - "ordre", - "famille", - "rang_taxo", - "nombre_min", - "nombre_max", - "alti_min", - "alti_max", - "prof_min", - "prof_max", - "observateurs", - "determinateur", - "communes", - "geometrie_wkt_4326", - "x_centroid_4326", - "y_centroid_4326", - "nom_lieu", - "validateur", - "niveau_validation", - "date_validation", - "comment_validation", - "preuve_numerique_url", - "preuve_non_numerique", - "jdd_nom", - "jdd_uuid", - "jdd_id", - "ca_nom", - "ca_uuid", - "ca_id", - "cd_habref", - "cd_habitat", - "nom_habitat", - "precision_geographique", - "nature_objet_geo", - "type_regroupement", - "methode_regroupement", - "technique_observation", - "biologique_statut", - "etat_biologique", - "biogeographique_statut", - "naturalite", - "preuve_existante", - "niveau_precision_diffusion", - "stade_vie", - "sexe", - "objet_denombrement", - "type_denombrement", - "niveau_sensibilite", - "statut_observation", - "floutage_dee", - "statut_source", - "type_info_geo", - "methode_determination", - "comportement", - "reference_biblio", - "uuid_perm_sinp", - "uuid_perm_grp_sinp", - "date_creation", - "date_modification" - ], - "contentmapping": { - "DEE_FLOU": { - "Non": "NON" - }, - "ETA_BIO": { - "Non renseign\u00e9": "1" - }, - "METH_DETERMIN": { - "Autre m\u00e9thode de d\u00e9termination": "2" - }, - "METH_OBS": { - "Galerie/terrier": "23" - }, - "NATURALITE": { - "Sauvage": "1" - }, - "NAT_OBJ_GEO": { - "Inventoriel": "In" - }, - "NIV_PRECIS": { - "Pr\u00e9cise": "5" - }, - "OBJ_DENBR": { - "Individu": "IND" - }, - "OCC_COMPORTEMENT": { - "Non renseign\u00e9": "1" - }, - "PREUVE_EXIST": { - "Oui": "1" - }, - "SENSIBILITE": { - "Non sensible - Diffusion pr\u00e9cise": "0" - }, - "SEXE": { - "Femelle": "2" - }, - "STADE_VIE": { - "Adulte": "2", - "Immature": "4", - "Juv\u00e9nile": "3" - }, - "STATUT_BIO": { - "Non renseign\u00e9": "1" - }, - "STATUT_OBS": { - "Pr\u00e9sent": "Pr" - }, - "STATUT_SOURCE": { - "Terrain": "Te" - }, - "STATUT_VALID": { - "En attente de validation": "0" - }, - "STAT_BIOGEO": { - "Non renseign\u00e9": "1" - }, - "TYP_DENBR": { - "Compt\u00e9": "Co" - }, - "TYP_GRP": { - "OBS": "OBS" + "key": "taxa_count", + "value": "Nombre de taxons import\u00e9s" } + ] + }, + "detected_encoding": "utf-8", + "detected_format": "csv", + "detected_separator": ";", + "encoding": "utf-8", + "errors_count": 3, + "fieldmapping": { + "WKT": "geometrie_wkt_4326", + "altitude_max": "alti_max", + "altitude_min": "alti_min", + "cd_hab": "cd_habref", + "cd_nom": "cd_nom", + "comment_context": "comment_releve", + "comment_description": "comment_occurrence", + "count_max": "nombre_max", + "count_min": "nombre_min", + "date_max": "date_fin", + "date_min": "date_debut", + "depth_max": "prof_max", + "depth_min": "prof_min", + "determiner": "determinateur", + "digital_proof": "preuve_numerique_url", + "entity_source_pk_value": "id_synthese", + "grp_method": "methode_regroupement", + "hour_max": "heure_fin", + "hour_min": "heure_debut", + "id_nomenclature_behaviour": "comportement", + "id_nomenclature_bio_condition": "etat_biologique", + "id_nomenclature_bio_status": "biologique_statut", + "id_nomenclature_biogeo_status": "biogeographique_statut", + "id_nomenclature_blurring": "floutage_dee", + "id_nomenclature_determination_method": "methode_determination", + "id_nomenclature_diffusion_level": "niveau_precision_diffusion", + "id_nomenclature_exist_proof": "preuve_existante", + "id_nomenclature_geo_object_nature": "nature_objet_geo", + "id_nomenclature_grp_typ": "type_regroupement", + "id_nomenclature_life_stage": "stade_vie", + "id_nomenclature_naturalness": "naturalite", + "id_nomenclature_obj_count": "objet_denombrement", + "id_nomenclature_obs_technique": "technique_observation", + "id_nomenclature_observation_status": "statut_observation", + "id_nomenclature_sensitivity": "niveau_sensibilite", + "id_nomenclature_sex": "sexe", + "id_nomenclature_source_status": "statut_source", + "id_nomenclature_type_count": "type_denombrement", + "id_nomenclature_valid_status": "niveau_validation", + "meta_create_date": "date_creation", + "meta_update_date": "date_modification", + "meta_validation_date": "date_validation", + "nom_cite": "nom_cite", + "non_digital_proof": "preuve_non_numerique", + "observers": "observateurs", + "place_name": "nom_lieu", + "precision": "precision_geographique", + "reference_biblio": "reference_biblio", + "unique_id_sinp": "uuid_perm_sinp", + "unique_id_sinp_grp": "uuid_perm_grp_sinp", + "validation_comment": "comment_validation", + "validator": "validateur" + }, + "format_source_file": "csv", + "full_file_name": "fichier_import_1.csv", + "id_dataset": 1, + "id_destination": 1, + "id_import": 1, + "loaded": false, + "processed": true, + "separator": ";", + "source_count": 6, + "srid": 4326, + "statistics": { + "taxa_count": 2 + }, + "task_id": null, + "task_progress": null, + "uuid_autogenerated": null + }, + { + "altitude_autogenerated": null, + "authors": [ + { + "nom_complet": "User Test - 2" + } + ], + "authors_name": "User Test - 2", + "available_encodings": ["iso-8859-1", "iso-8859-15", "utf-8"], + "available_formats": ["csv", "geojson"], + "available_separators": [",", ";"], + "columns": [ + "error", + "id_synthese", + "id_origine", + "comment_releve", + "comment_occurrence", + "date_debut", + "date_fin", + "heure_debut", + "heure_fin", + "cd_nom", + "cd_ref", + "nom_valide", + "nom_vernaculaire", + "nom_cite", + "regne", + "group1_inpn", + "group2_inpn", + "classe", + "ordre", + "famille", + "rang_taxo", + "nombre_min", + "nombre_max", + "alti_min", + "alti_max", + "prof_min", + "prof_max", + "observateurs", + "determinateur", + "communes", + "geometrie_wkt_4326", + "x_centroid_4326", + "y_centroid_4326", + "nom_lieu", + "validateur", + "niveau_validation", + "date_validation", + "comment_validation", + "preuve_numerique_url", + "preuve_non_numerique", + "jdd_nom", + "jdd_uuid", + "jdd_id", + "ca_nom", + "ca_uuid", + "ca_id", + "cd_habref", + "cd_habitat", + "nom_habitat", + "precision_geographique", + "nature_objet_geo", + "type_regroupement", + "methode_regroupement", + "technique_observation", + "biologique_statut", + "etat_biologique", + "biogeographique_statut", + "naturalite", + "preuve_existante", + "niveau_precision_diffusion", + "stade_vie", + "sexe", + "objet_denombrement", + "type_denombrement", + "niveau_sensibilite", + "statut_observation", + "floutage_dee", + "statut_source", + "type_info_geo", + "methode_determination", + "comportement", + "reference_biblio", + "uuid_perm_sinp", + "uuid_perm_grp_sinp", + "date_creation", + "date_modification" + ], + "contentmapping": { + "DEE_FLOU": { + "Non": "NON" }, - "cruved": { - "C": true, - "D": true, - "E": false, - "R": true, - "U": true, - "V": false - }, - "dataset": { - "active": true, - "dataset_name": "Contact al\u00e9atoire tous r\u00e8gnes confondus" - }, - "date_create_import": "2024-05-13 23:55:51.167003", - "date_end_import": "2024-05-13 23:56:46.793605", - "date_max_data": null, - "date_min_data": null, - "date_update_import": "2024-05-13 23:56:46.796963", - "destination": { - "code": "synthese", - "label": "Synth\u00e8se", - "statistics_labels": [ - { - "key": "taxa_count", - "value": "Nombre de taxons import\u00e9s" - } - ] - }, - "detected_encoding": "utf-8", - "detected_format": "csv", - "detected_separator": ";", - "encoding": "utf-8", - "errors_count": 3, - "fieldmapping": { - "WKT": "geometrie_wkt_4326", - "altitude_max": "alti_max", - "altitude_min": "alti_min", - "cd_hab": "cd_habref", - "cd_nom": "cd_nom", - "comment_context": "comment_releve", - "comment_description": "comment_occurrence", - "count_max": "nombre_max", - "count_min": "nombre_min", - "date_max": "date_fin", - "date_min": "date_debut", - "depth_max": "prof_max", - "depth_min": "prof_min", - "determiner": "determinateur", - "digital_proof": "preuve_numerique_url", - "entity_source_pk_value": "id_synthese", - "grp_method": "methode_regroupement", - "hour_max": "heure_fin", - "hour_min": "heure_debut", - "id_nomenclature_behaviour": "comportement", - "id_nomenclature_bio_condition": "etat_biologique", - "id_nomenclature_bio_status": "biologique_statut", - "id_nomenclature_biogeo_status": "biogeographique_statut", - "id_nomenclature_blurring": "floutage_dee", - "id_nomenclature_determination_method": "methode_determination", - "id_nomenclature_diffusion_level": "niveau_precision_diffusion", - "id_nomenclature_exist_proof": "preuve_existante", - "id_nomenclature_geo_object_nature": "nature_objet_geo", - "id_nomenclature_grp_typ": "type_regroupement", - "id_nomenclature_life_stage": "stade_vie", - "id_nomenclature_naturalness": "naturalite", - "id_nomenclature_obj_count": "objet_denombrement", - "id_nomenclature_obs_technique": "technique_observation", - "id_nomenclature_observation_status": "statut_observation", - "id_nomenclature_sensitivity": "niveau_sensibilite", - "id_nomenclature_sex": "sexe", - "id_nomenclature_source_status": "statut_source", - "id_nomenclature_type_count": "type_denombrement", - "id_nomenclature_valid_status": "niveau_validation", - "meta_create_date": "date_creation", - "meta_update_date": "date_modification", - "meta_validation_date": "date_validation", - "nom_cite": "nom_cite", - "non_digital_proof": "preuve_non_numerique", - "observers": "observateurs", - "place_name": "nom_lieu", - "precision": "precision_geographique", - "reference_biblio": "reference_biblio", - "unique_id_sinp": "uuid_perm_sinp", - "unique_id_sinp_grp": "uuid_perm_grp_sinp", - "validation_comment": "comment_validation", - "validator": "validateur" - }, - "format_source_file": "csv", - "full_file_name": "fichier_import_1.csv", - "id_dataset": 1, - "id_destination": 1, - "id_import": 1, - "import_count": 3, - "loaded": false, - "processed": true, - "separator": ";", - "source_count": 6, - "srid": 4326, - "statistics": { - "taxa_count": 2 - }, - "task_id": null, - "task_progress": null, - "uuid_autogenerated": null + "ETA_BIO": { + "Non renseign\u00e9": "1" + }, + "METH_DETERMIN": { + "Autre m\u00e9thode de d\u00e9termination": "2" + }, + "METH_OBS": { + "Galerie/terrier": "23" + }, + "NATURALITE": { + "Sauvage": "1" + }, + "NAT_OBJ_GEO": { + "Inventoriel": "In" + }, + "NIV_PRECIS": { + "Pr\u00e9cise": "5" + }, + "OBJ_DENBR": { + "Individu": "IND" + }, + "OCC_COMPORTEMENT": { + "Non renseign\u00e9": "1" + }, + "PREUVE_EXIST": { + "Oui": "1" + }, + "SENSIBILITE": { + "Non sensible - Diffusion pr\u00e9cise": "0" + }, + "SEXE": { + "Femelle": "2" + }, + "STADE_VIE": { + "Adulte": "2", + "Immature": "4", + "Juv\u00e9nile": "3" + }, + "STATUT_BIO": { + "Non renseign\u00e9": "1" + }, + "STATUT_OBS": { + "Pr\u00e9sent": "Pr" + }, + "STATUT_SOURCE": { + "Terrain": "Te" + }, + "STATUT_VALID": { + "En attente de validation": "0" + }, + "STAT_BIOGEO": { + "Non renseign\u00e9": "1" + }, + "TYP_DENBR": { + "Compt\u00e9": "Co" + }, + "TYP_GRP": { + "OBS": "OBS" + } + }, + "cruved": { + "C": true, + "D": true, + "E": false, + "R": true, + "U": true, + "V": false }, - { - "altitude_autogenerated": null, - "authors": [ + "dataset": { + "active": true, + "dataset_name": "JDD - TEST FRONTEND" + }, + "date_create_import": "2024-05-13 23:55:51.167003", + "date_end_import": "2024-05-13 23:56:46.793605", + "date_max_data": null, + "date_min_data": null, + "date_update_import": "2024-05-13 23:56:46.796963", + "destination": { + "code": "destination", + "label": "Destination", + "statistics_labels": [ { - "nom_complet": "User Test - 2" - } - ], - "authors_name": "User Test - 2", - "available_encodings": [ - "iso-8859-1", - "iso-8859-15", - "utf-8" - ], - "available_formats": [ - "csv", - "geojson" - ], - "available_separators": [ - ",", - ";" - ], - "columns": [ - "error", - "id_synthese", - "id_origine", - "comment_releve", - "comment_occurrence", - "date_debut", - "date_fin", - "heure_debut", - "heure_fin", - "cd_nom", - "cd_ref", - "nom_valide", - "nom_vernaculaire", - "nom_cite", - "regne", - "group1_inpn", - "group2_inpn", - "classe", - "ordre", - "famille", - "rang_taxo", - "nombre_min", - "nombre_max", - "alti_min", - "alti_max", - "prof_min", - "prof_max", - "observateurs", - "determinateur", - "communes", - "geometrie_wkt_4326", - "x_centroid_4326", - "y_centroid_4326", - "nom_lieu", - "validateur", - "niveau_validation", - "date_validation", - "comment_validation", - "preuve_numerique_url", - "preuve_non_numerique", - "jdd_nom", - "jdd_uuid", - "jdd_id", - "ca_nom", - "ca_uuid", - "ca_id", - "cd_habref", - "cd_habitat", - "nom_habitat", - "precision_geographique", - "nature_objet_geo", - "type_regroupement", - "methode_regroupement", - "technique_observation", - "biologique_statut", - "etat_biologique", - "biogeographique_statut", - "naturalite", - "preuve_existante", - "niveau_precision_diffusion", - "stade_vie", - "sexe", - "objet_denombrement", - "type_denombrement", - "niveau_sensibilite", - "statut_observation", - "floutage_dee", - "statut_source", - "type_info_geo", - "methode_determination", - "comportement", - "reference_biblio", - "uuid_perm_sinp", - "uuid_perm_grp_sinp", - "date_creation", - "date_modification" - ], - "contentmapping": { - "DEE_FLOU": { - "Non": "NON" - }, - "ETA_BIO": { - "Non renseign\u00e9": "1" - }, - "METH_DETERMIN": { - "Autre m\u00e9thode de d\u00e9termination": "2" - }, - "METH_OBS": { - "Galerie/terrier": "23" - }, - "NATURALITE": { - "Sauvage": "1" - }, - "NAT_OBJ_GEO": { - "Inventoriel": "In" - }, - "NIV_PRECIS": { - "Pr\u00e9cise": "5" - }, - "OBJ_DENBR": { - "Individu": "IND" - }, - "OCC_COMPORTEMENT": { - "Non renseign\u00e9": "1" - }, - "PREUVE_EXIST": { - "Oui": "1" - }, - "SENSIBILITE": { - "Non sensible - Diffusion pr\u00e9cise": "0" - }, - "SEXE": { - "Femelle": "2" - }, - "STADE_VIE": { - "Adulte": "2", - "Immature": "4", - "Juv\u00e9nile": "3" - }, - "STATUT_BIO": { - "Non renseign\u00e9": "1" - }, - "STATUT_OBS": { - "Pr\u00e9sent": "Pr" - }, - "STATUT_SOURCE": { - "Terrain": "Te" - }, - "STATUT_VALID": { - "En attente de validation": "0" - }, - "STAT_BIOGEO": { - "Non renseign\u00e9": "1" - }, - "TYP_DENBR": { - "Compt\u00e9": "Co" - }, - "TYP_GRP": { - "OBS": "OBS" + "key": "taxa_count", + "value": "Nombre de taxons import\u00e9s" } - }, - "cruved": { - "C": true, - "D": true, - "E": false, - "R": true, - "U": true, - "V": false - }, - "dataset": { - "active": true, - "dataset_name": "JDD - TEST FRONTEND" - }, - "date_create_import": "2024-05-13 23:55:51.167003", - "date_end_import": "2024-05-13 23:56:46.793605", - "date_max_data": null, - "date_min_data": null, - "date_update_import": "2024-05-13 23:56:46.796963", - "destination": { - "code": "destination", - "label": "Destination", - "statistics_labels": [ - { - "key": "taxa_count", - "value": "Nombre de taxons import\u00e9s" - } - ] - }, - "detected_encoding": "utf-8", - "detected_format": "csv", - "detected_separator": ";", - "encoding": "utf-8", - "errors_count": 3, - "fieldmapping": { - "WKT": "geometrie_wkt_4326", - "altitude_max": "alti_max", - "altitude_min": "alti_min", - "cd_hab": "cd_habref", - "cd_nom": "cd_nom", - "comment_context": "comment_releve", - "comment_description": "comment_occurrence", - "count_max": "nombre_max", - "count_min": "nombre_min", - "date_max": "date_fin", - "date_min": "date_debut", - "depth_max": "prof_max", - "depth_min": "prof_min", - "determiner": "determinateur", - "digital_proof": "preuve_numerique_url", - "entity_source_pk_value": "id_synthese", - "grp_method": "methode_regroupement", - "hour_max": "heure_fin", - "hour_min": "heure_debut", - "id_nomenclature_behaviour": "comportement", - "id_nomenclature_bio_condition": "etat_biologique", - "id_nomenclature_bio_status": "biologique_statut", - "id_nomenclature_biogeo_status": "biogeographique_statut", - "id_nomenclature_blurring": "floutage_dee", - "id_nomenclature_determination_method": "methode_determination", - "id_nomenclature_diffusion_level": "niveau_precision_diffusion", - "id_nomenclature_exist_proof": "preuve_existante", - "id_nomenclature_geo_object_nature": "nature_objet_geo", - "id_nomenclature_grp_typ": "type_regroupement", - "id_nomenclature_life_stage": "stade_vie", - "id_nomenclature_naturalness": "naturalite", - "id_nomenclature_obj_count": "objet_denombrement", - "id_nomenclature_obs_technique": "technique_observation", - "id_nomenclature_observation_status": "statut_observation", - "id_nomenclature_sensitivity": "niveau_sensibilite", - "id_nomenclature_sex": "sexe", - "id_nomenclature_source_status": "statut_source", - "id_nomenclature_type_count": "type_denombrement", - "id_nomenclature_valid_status": "niveau_validation", - "meta_create_date": "date_creation", - "meta_update_date": "date_modification", - "meta_validation_date": "date_validation", - "nom_cite": "nom_cite", - "non_digital_proof": "preuve_non_numerique", - "observers": "observateurs", - "place_name": "nom_lieu", - "precision": "precision_geographique", - "reference_biblio": "reference_biblio", - "unique_id_sinp": "uuid_perm_sinp", - "unique_id_sinp_grp": "uuid_perm_grp_sinp", - "validation_comment": "comment_validation", - "validator": "validateur" - }, - "format_source_file": "csv", - "full_file_name": "fichier_import_2.csv", - "id_dataset": 2, - "id_destination": 2, - "id_import": 2, - "import_count": 3, - "loaded": false, - "processed": true, - "separator": ";", - "source_count": 6, - "srid": 4326, - "statistics": { - "taxa_count": 2 - }, - "task_id": null, - "task_progress": null, - "uuid_autogenerated": null - } - ], - "limit": 15, - "offset": 0 - } \ No newline at end of file + ] + }, + "detected_encoding": "utf-8", + "detected_format": "csv", + "detected_separator": ";", + "encoding": "utf-8", + "errors_count": 3, + "fieldmapping": { + "WKT": "geometrie_wkt_4326", + "altitude_max": "alti_max", + "altitude_min": "alti_min", + "cd_hab": "cd_habref", + "cd_nom": "cd_nom", + "comment_context": "comment_releve", + "comment_description": "comment_occurrence", + "count_max": "nombre_max", + "count_min": "nombre_min", + "date_max": "date_fin", + "date_min": "date_debut", + "depth_max": "prof_max", + "depth_min": "prof_min", + "determiner": "determinateur", + "digital_proof": "preuve_numerique_url", + "entity_source_pk_value": "id_synthese", + "grp_method": "methode_regroupement", + "hour_max": "heure_fin", + "hour_min": "heure_debut", + "id_nomenclature_behaviour": "comportement", + "id_nomenclature_bio_condition": "etat_biologique", + "id_nomenclature_bio_status": "biologique_statut", + "id_nomenclature_biogeo_status": "biogeographique_statut", + "id_nomenclature_blurring": "floutage_dee", + "id_nomenclature_determination_method": "methode_determination", + "id_nomenclature_diffusion_level": "niveau_precision_diffusion", + "id_nomenclature_exist_proof": "preuve_existante", + "id_nomenclature_geo_object_nature": "nature_objet_geo", + "id_nomenclature_grp_typ": "type_regroupement", + "id_nomenclature_life_stage": "stade_vie", + "id_nomenclature_naturalness": "naturalite", + "id_nomenclature_obj_count": "objet_denombrement", + "id_nomenclature_obs_technique": "technique_observation", + "id_nomenclature_observation_status": "statut_observation", + "id_nomenclature_sensitivity": "niveau_sensibilite", + "id_nomenclature_sex": "sexe", + "id_nomenclature_source_status": "statut_source", + "id_nomenclature_type_count": "type_denombrement", + "id_nomenclature_valid_status": "niveau_validation", + "meta_create_date": "date_creation", + "meta_update_date": "date_modification", + "meta_validation_date": "date_validation", + "nom_cite": "nom_cite", + "non_digital_proof": "preuve_non_numerique", + "observers": "observateurs", + "place_name": "nom_lieu", + "precision": "precision_geographique", + "reference_biblio": "reference_biblio", + "unique_id_sinp": "uuid_perm_sinp", + "unique_id_sinp_grp": "uuid_perm_grp_sinp", + "validation_comment": "comment_validation", + "validator": "validateur" + }, + "format_source_file": "csv", + "full_file_name": "fichier_import_2.csv", + "id_dataset": 2, + "id_destination": 2, + "id_import": 2, + "loaded": false, + "processed": true, + "separator": ";", + "source_count": 6, + "srid": 4326, + "statistics": { + "taxa_count": 2 + }, + "task_id": null, + "task_progress": null, + "uuid_autogenerated": null + } + ], + "limit": 15, + "offset": 0 +} diff --git a/frontend/src/app/modules/imports/models/import.model.ts b/frontend/src/app/modules/imports/models/import.model.ts index 53a61ff364..0a67f06c72 100644 --- a/frontend/src/app/modules/imports/models/import.model.ts +++ b/frontend/src/app/modules/imports/models/import.model.ts @@ -25,6 +25,10 @@ export interface Dataset { dataset_name: string; active: boolean; } +export interface ImportStatistics { + import_count: number; + [propName: string]: any; +} export interface Import { id_import: number; @@ -41,8 +45,7 @@ export interface Import { date_update_import: string; date_end_import: null | string; source_count: number; - import_count: number; - statistics: object; + statistics: ImportStatistics; date_min_data: string; date_max_data: string; step: Step; diff --git a/frontend/src/app/modules/imports/utils/format-row-count.ts b/frontend/src/app/modules/imports/utils/format-row-count.ts index c188514001..8625aa04d2 100644 --- a/frontend/src/app/modules/imports/utils/format-row-count.ts +++ b/frontend/src/app/modules/imports/utils/format-row-count.ts @@ -1,5 +1,7 @@ import { Import } from '../models/import.model'; export function formatRowCount(imprt: Import): string { - return imprt && imprt.source_count ? `${imprt.import_count ?? 0} / ${imprt.source_count}` : ''; + return imprt && imprt.source_count + ? `${imprt.statistics['import_count'] ?? 0} / ${imprt['source_count']}` + : ''; }