WIP taxhub api lists in backoffice

backend/gncitizen/core/commons/admin.py

@@ -14,30 +14,12 @@
     CustomTileView,
     json_formatter,
 )
-from gncitizen.utils.env import MEDIA_DIR, taxhub_lists_url
+from gncitizen.utils.env import MEDIA_DIR
+from gncitizen.utils.taxonomy import taxonomy_lists
 
 logger = current_app.logger
 
 
-def taxonomy_lists():
-    taxonomy_lists = []
-
-    taxa_lists = requests.get(taxhub_lists_url)
-    logger.debug(taxa_lists)
-    if taxa_lists.status_code == 200:
-        try:
-            taxa_lists = taxa_lists.json()["data"]
-            logger.debug(taxa_lists)
-            for taxa_list in taxa_lists:
-                taxonomy_lists.append(
-                    (taxa_list["id_liste"], taxa_list["nom_liste"])
-                )
-        except Exception as e:
-            logger.critical(str(e))
-    logger.debug(taxonomy_lists)
-    return taxonomy_lists
-
-
 class CorProgramSiteTypeModelInlineForm(InlineFormAdmin):
     form_columns = ("site_type",)
 
@@ -52,7 +34,7 @@ class ProjectView(ModelView):
 
 class ProgramView(ModelView):
     form_overrides = {"long_desc": CKEditorField, "taxonomy_list": SelectField}
-    form_args = {"taxonomy_list": {"choices": taxonomy_lists(), "coerce": int}}
+    form_args = {"taxonomy_list": {"choices": taxonomy_lists, "coerce": int}}
     create_template = "edit.html"
     edit_template = "edit.html"
     form_excluded_columns = [

backend/gncitizen/core/observations/routes.py

@@ -22,7 +22,7 @@
 from gncitizen.utils.geo import get_municipality_id_from_wkb
 from gncitizen.utils.jwt import get_id_role_if_exists, get_user_if_exists
 from gncitizen.utils.media import save_upload_files
-from gncitizen.utils.taxonomy import get_specie_from_cd_nom, taxhub_lists
+from gncitizen.utils.taxonomy import get_specie_from_cd_nom, taxhub_full_lists
 from server import db
 
 from .admin import ObservationView
@@ -121,7 +121,7 @@ def generate_observation_geojson(id_observation):
         .one()
         .taxonomy_list
     )
-    taxon_repository = taxhub_lists[taxhub_list_id]
+    taxon_repository = taxhub_full_lists[taxhub_list_id]
     try:
         taxon = next(
             taxon
@@ -512,7 +512,7 @@ def get_program_observations(
                 .one()
                 .taxonomy_list
             )
-            taxon_repository = taxhub_lists[taxhub_list_id]
+            taxon_repository = taxhub_full_lists[taxhub_list_id]
 
         features = []
         for observation in observations:
@@ -640,7 +640,7 @@ def get_all_observations() -> Union[FeatureCollection, Tuple[Dict, int]]:
                     .one()
                     .taxonomy_list
                 )
-                taxon_data = taxhub_lists[taxhub_list_id]
+                taxon_data = taxhub_full_lists[taxhub_list_id]
                 try:
                     for taxon in taxon_data:
                         if taxon not in taxon_repository:
@@ -792,7 +792,7 @@ def get_observations_by_user_id(user_id):
                             observation.ProgramsModel.taxonomy_list
                         )
                 for tax_list in taxhub_list_id:
-                    taxon_repository.append(taxhub_lists[tax_list])
+                    taxon_repository.append(taxhub_full_lists[tax_list])
 
             features = []
         except Exception as e:

backend/gncitizen/core/taxonomy/routes.py

@@ -4,7 +4,7 @@
 from gncitizen.utils.taxonomy import (
     get_specie_from_cd_nom,
     refresh_taxonlist,
-    taxhub_lists,
+    taxhub_full_lists,
     taxhub_rest_get_all_lists,
 )
 
@@ -81,7 +81,7 @@ def get_list(id):
     """
 
     try:
-        r = taxhub_lists[id]
+        r = taxhub_full_lists[id]
         return r
     except Exception as e:
         return {"message": str(e)}, 400

backend/gncitizen/utils/admin.py

@@ -26,4 +26,8 @@ def _value(self):
 
 class CustomTileView(ModelView):
     tile_layer_url = "a.tile.openstreetmap.org/{z}/{x}/{y}.png"
-    tile_layer_attribution = "some string or html goes here"
+    tile_layer_attribution = """
+    © <a href="https://www.openstreetmap.org/copyright" target="_blank">
+    Les Contributeurs d’OpenStreetMap
+    </a>
+    """

backend/gncitizen/utils/env.py

@@ -15,7 +15,6 @@
 
 # from datetime import timedelta
 
-
 ROOT_DIR = Path(__file__).absolute().parent.parent.parent.parent
 BACKEND_DIR = ROOT_DIR / "backend"
 DEFAULT_VIRTUALENV_DIR = BACKEND_DIR / "venv"
@@ -40,6 +39,12 @@ def get_config_file_path(config_file=None):
     return Path(config_file or DEFAULT_CONFIG_FILE)
 
 
+def valid_api_url(url):
+    """Return a valid API URL ending with /"""
+    url = url if url[-1:] == "/" else url + "/"
+    return url
+
+
 def load_config(config_file=None):
     """Load the geonature-citizen configuration from a given file"""
     config_gnc = load_toml(get_config_file_path())
@@ -55,16 +60,13 @@ def load_config(config_file=None):
     return config_gnc
 
 
-def valid_api_url(url):
-    """Return a valid API URL ending with /"""
-    url = url if url[-1:] == "/" else url + "/"
-    return url
-
-
 app_conf = load_config()
 MEDIA_DIR = str(ROOT_DIR / app_conf["MEDIA_FOLDER"])
 SQLALCHEMY_DATABASE_URI = app_conf["SQLALCHEMY_DATABASE_URI"]
 
+
+taxhub_url = valid_api_url(app_conf.get("API_TAXHUB", ""))
+
 db = SQLAlchemy()
 jwt = JWTManager()
 ckeditor = CKEditor()
@@ -115,21 +117,17 @@ def valid_api_url(url):
 class HomeView(AdminIndexView):
     @expose("/")
     def index(self):
-        return self.render("home.html")
+        return self.render("home.html", taxhub_url=taxhub_url)
 
 
 admin = Admin(
-    name=f"GN-Citizen: Backoffice d'administration (version: {__version__})",
+    name=f"GnCitizen: Backoffice (version: {__version__})",
     index_view=HomeView(name="Home", url="/api/admin"),
     template_mode="bootstrap4",
     url="/api/admin",
 )
 
 
-taxhub_url = valid_api_url(app_conf.get("API_TAXHUB", ""))
-
-taxhub_lists_url = taxhub_url + "biblistes/"
-
 API_CITY = app_conf.get(
     "API_CITY", "https://nominatim.openstreetmap.org/reverse"
 )

backend/gncitizen/utils/taxonomy.py

@@ -27,7 +27,8 @@
 
 Taxon = Dict[str, Union[str, Dict[str, str], List[Dict]]]
 
-taxhub_lists = {}
+taxhub_full_lists = {}
+taxonomy_lists = []
 
 
 def taxhub_rest_get_taxon_list(taxhub_list_id: int) -> Dict:
@@ -42,16 +43,27 @@ def taxhub_rest_get_taxon_list(taxhub_list_id: int) -> Dict:
         params=params,
         timeout=5,
     )
-    logger.debug(f"<taxhub_rest_get_taxon_list> URL {res.url}")
     res.raise_for_status()
     return res.json()
 
 
 def taxhub_rest_get_all_lists() -> Dict:
-    res = session.get("{}biblistes".format(TAXHUB_API))
-    logger.debug(f"<taxhub_rest_get_all_lists> URL {res.url}")
+    url = f"{TAXHUB_API}biblistes"
+    res = session.get(
+        url,
+        timeout=5,
+    )
     res.raise_for_status()
-    return res.json().get("data", [])
+    if res.status_code == 200:
+        try:
+            taxa_lists = res.json()["data"]
+            for taxa_list in taxa_lists:
+                taxonomy_lists.append(
+                    (taxa_list["id_liste"], taxa_list["nom_liste"])
+                )
+        except Exception as e:
+            logger.critical(str(e))
+        return res.json().get("data", [])
 
 
 def taxhub_rest_get_taxon(taxhub_id: int) -> Taxon:
@@ -65,12 +77,10 @@ def taxhub_rest_get_taxon(taxhub_id: int) -> Taxon:
         except requests.exceptions.ReadTimeout:
             continue
 
-    logger.debug(f"<taxhub_rest_get_taxon> URL {res.url}")
     res.raise_for_status()
     data = res.json()
     data.pop("listes", None)
     data.pop("attributs", None)
-    logger.debug(f"MEDIAS Length = {len(data['medias'])}")
     if len(data["medias"]) > 0:
         media_types = ("Photo_gncitizen", "Photo_principale", "Photo")
         i = 0
@@ -84,20 +94,19 @@ def taxhub_rest_get_taxon(taxhub_id: int) -> Taxon:
                 break
             i += 1
         medias = filtered_medias[:1]
-        logger.debug(f"MEDIAS Filtered {medias}")
         data["medias"] = medias
 
     return data
 
 
-def mkTaxonRepository(taxhub_list_id: int) -> List[Taxon]:
+def make_taxon_repository(taxhub_list_id: int) -> List[Taxon]:
     taxa = taxhub_rest_get_taxon_list(taxhub_list_id)
     taxon_ids = [item["id_nom"] for item in taxa.get("items")]
     r = [taxhub_rest_get_taxon(taxon_id) for taxon_id in taxon_ids]
     return r
 
 
-def get_specie_from_cd_nom(cd_nom):
+def get_specie_from_cd_nom(cd_nom) -> Dict:
     """get specie datas from taxref id (cd_nom)
 
     :param cd_nom: taxref unique id (cd_nom)
@@ -130,17 +139,20 @@ def get_specie_from_cd_nom(cd_nom):
     return taxref
 
 
-def refresh_taxonlist():
+def refresh_taxonlist() -> Dict:
     """refresh taxon list"""
+    logger.info("Pre loading taxhub data (taxa lists and medias)")
     lists = taxhub_rest_get_all_lists()
     if lists:
+        count = 0
         for list in lists:
-            print(f"LIST {list}")
-            r = mkTaxonRepository(list["id_liste"])
-            taxhub_lists[list["id_liste"]] = r
-            print(r)
-    print(f"taxhub_lists {taxhub_lists}")
-    return taxhub_lists
+            count += 1
+            logger.info(f"loading list {count}/{len(lists)}")
+            r = make_taxon_repository(list["id_liste"])
+            taxhub_full_lists[list["id_liste"]] = r
+    else:
+        logger.warning("ERROR: No taxhub lists available")
+    return taxhub_full_lists
 
 
 refresh_taxonlist()

backend/templates/home.html

@@ -2,21 +2,37 @@
 
 {% block body %}
   <div class="container">
-    <h2>Refresh taxonomy lists</h2>
-    <div>
-      <button class="btn btn-success" onclick="refreshData()">Refresh</button>
-      <span id="status"></span>
+    <h2>Refresh TaxHub lists</h2>
+    <p>
+      <i class="fa fa-info-circle"></i> Force taxhub lists refresh from <a href="{{ taxhub_url }}" target="_blank" title="TaxHub API URL">{{ taxhub_url }}</a>
+    </p>
+
+    <div class="row">
+      <div class="col col-3">
+        <button class="btn btn-success btn-block" onclick="refreshData()">Refresh</button>
+      </div>
+      <div class="col col-9">
+        <div class="d-flex align-items-center">
+          <strong id="status"></strong>
+          <div id="statusSpinner" class="spinner-border ml-auto text-primary" role="status" aria-hidden="true" style="visibility: hidden;"></div>
+        </div>
+      </div>
     </div>
   </div>
 {% endblock %}
 
 {% block tail %}
   <script>
     const refreshData = async () => {
-      const div = document.getElementById('status')
-      div.innerHTML = 'In progress...'
+      const statusText = document.getElementById('status')
+      const statusSpinner = document.getElementById('statusSpinner')
+      statusSpinner.style.visibility = 'visible'
+      statusText.innerHTML = 'Loading...'
       const reponse = await fetch('/api/taxonomy/refresh')
-      div.innerHTML = 'Success'
+      statusText.innerHTML = 'Success !'
+      statusText.className = 'text-success'
+      statusSpinner.style.visibility = 'hidden'
+
       // do something
     }
   </script>