Fix test

Rename test files for latest hdx python scraper changes (uses resource names for filenames where possible)

pyproject.toml

@@ -35,9 +35,9 @@ requires-python = ">=3.8"
 
 dependencies = [
     "hapi-schema>= 0.4.0",
-    "hdx-python-country>= 3.5.9",
+    "hdx-python-country>= 3.6.3",
     "hdx-python-database[postgresql]>= 1.2.9",
-    "hdx-python-scraper>= 2.2.5",
+    "hdx-python-scraper>= 2.3.0",
     "libhxl",
     "sqlalchemy"
 ]

requirements.txt

@@ -13,7 +13,7 @@ attrs==23.1.0
     #   jsonschema
 cachetools==5.3.2
     # via google-auth
-certifi==2023.7.22
+certifi==2023.11.17
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -31,11 +31,11 @@ colorama==0.4.6
     # via typer
 coverage[toml]==7.3.2
     # via pytest-cov
-cryptography==41.0.5
+cryptography==41.0.7
     # via pyopenssl
 defopt==6.4.0
     # via hdx-python-api
-distlib==0.3.7
+distlib==0.3.8
     # via virtualenv
 dnspython==2.4.2
     # via email-validator
@@ -53,35 +53,38 @@ filelock==3.13.1
     # via virtualenv
 frictionless==5.16.0
     # via hdx-python-utilities
-google-auth==2.23.4
+google-auth==2.25.2
     # via
     #   google-auth-oauthlib
     #   gspread
-google-auth-oauthlib==1.1.0
+google-auth-oauthlib==1.2.0
     # via gspread
-greenlet==3.0.1
+greenlet==3.0.2
     # via sqlalchemy
-gspread==5.12.0
+gspread==5.12.2
     # via hdx-python-scraper
 hapi-schema==0.4.0
     # via hapi-pipelines (pyproject.toml)
-hdx-python-api==6.1.3
+hdx-python-api==6.1.4
     # via hdx-python-scraper
-hdx-python-country==3.5.9
+hdx-python-country==3.6.3
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
+    #   hdx-python-scraper
 hdx-python-database[postgresql]==1.2.9
     # via hapi-pipelines (pyproject.toml)
-hdx-python-scraper==2.2.5
+hdx-python-scraper==2.3.0
     # via hapi-pipelines (pyproject.toml)
 hdx-python-utilities==3.6.2
-    # via hdx-python-country
-humanize==4.8.0
+    # via
+    #   hdx-python-api
+    #   hdx-python-country
+humanize==4.9.0
     # via frictionless
-identify==2.5.31
+identify==2.5.33
     # via pre-commit
-idna==3.4
+idna==3.6
     # via
     #   email-validator
     #   requests
@@ -106,14 +109,15 @@ jsonschema==4.17.3
 libhxl==5.1
     # via
     #   hapi-pipelines (pyproject.toml)
+    #   hdx-python-api
     #   hdx-python-country
 loguru==0.7.2
     # via hdx-python-utilities
-makefun==1.15.1
+makefun==1.15.2
     # via hdx-python-api
 markdown-it-py==3.0.0
     # via rich
-marko==2.0.1
+marko==2.0.2
     # via frictionless
 markupsafe==2.1.3
     # via jinja2
@@ -133,7 +137,7 @@ packaging==23.2
     # via pytest
 petl==1.7.14
     # via frictionless
-platformdirs==3.11.0
+platformdirs==4.1.0
     # via virtualenv
 pluggy==1.3.0
     # via pytest
@@ -143,13 +147,13 @@ ply==3.11
     #   libhxl
 pockets==0.9.1
     # via sphinxcontrib-napoleon
-pre-commit==3.5.0
+pre-commit==3.6.0
     # via hapi-pipelines (pyproject.toml)
-psycopg[binary]==3.1.12
+psycopg[binary]==3.1.14
     # via hdx-python-database
-psycopg-binary==3.1.12
+psycopg-binary==3.1.14
     # via psycopg
-pyasn1==0.5.0
+pyasn1==0.5.1
     # via
     #   hdx-python-api
     #   ndg-httpsclient
@@ -159,13 +163,13 @@ pyasn1-modules==0.3.0
     # via google-auth
 pycparser==2.21
     # via cffi
-pydantic==2.4.2
+pydantic==2.5.2
     # via
     #   frictionless
     #   inflect
-pydantic-core==2.10.1
+pydantic-core==2.14.5
     # via pydantic
-pygments==2.16.1
+pygments==2.17.2
     # via rich
 pyopenssl==23.3.0
     # via
@@ -207,6 +211,7 @@ requests==2.31.0
     # via
     #   ckanapi
     #   frictionless
+    #   hdx-python-api
     #   libhxl
     #   requests-file
     #   requests-oauthlib
@@ -216,7 +221,7 @@ requests-oauthlib==1.3.1
     # via google-auth-oauthlib
 rfc3986==2.0.0
     # via frictionless
-rich==13.6.0
+rich==13.7.0
     # via typer
 rsa==4.9
     # via google-auth
@@ -255,7 +260,7 @@ text-unidecode==1.3
     # via python-slugify
 typer[all]==0.9.0
     # via frictionless
-typing-extensions==4.8.0
+typing-extensions==4.9.0
     # via
     #   frictionless
     #   inflect
@@ -268,15 +273,15 @@ unidecode==1.3.7
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.0.7
+urllib3==2.1.0
     # via
     #   libhxl
     #   requests
 validators==0.22.0
     # via frictionless
-virtualenv==20.24.6
+virtualenv==20.25.0
     # via pre-commit
-wheel==0.41.3
+wheel==0.42.0
     # via libhxl
 xlrd==2.0.1
     # via hdx-python-utilities

src/hapi/pipelines/database/metadata.py

@@ -1,10 +1,8 @@
 import logging
 
-import hxl
 from hapi_schema.db_dataset import DBDataset
 from hapi_schema.db_resource import DBResource
 from hdx.scraper.runner import Runner
-from hxl import InputOptions
 from sqlalchemy.orm import Session
 
 from .base_uploader import BaseUploader
@@ -44,23 +42,14 @@ def populate(self):
             resources = dataset["resources"]
             for resource_id, resource in resources.items():
                 # Then add the resources
-                download_url = resource["download_url"]
-                hxl_info = hxl.info(
-                    download_url, InputOptions(encoding="utf-8")
-                )
-                is_hxlated = False
-                for sheet in hxl_info["sheets"]:
-                    if sheet["is_hxlated"]:
-                        is_hxlated = True
-                        break
                 resource_row = DBResource(
                     hdx_id=resource_id,
                     dataset_ref=dataset_row.id,
                     name=resource["name"],
                     format=resource["format"],
                     update_date=resource["update_date"],
-                    is_hxl=is_hxlated,
-                    download_url=download_url,
+                    is_hxl=resource["is_hxl"],
+                    download_url=resource["download_url"],
                 )
                 self._session.add(resource_row)
                 self._session.commit()