From bad69007784dc071604759903ae99256b975627a Mon Sep 17 00:00:00 2001 From: Mike Date: Wed, 5 Jun 2024 14:22:41 +1200 Subject: [PATCH] Add use_env=False to all Download() calls (#68) --- documentation/main.md | 4 +-- pyproject.toml | 4 +-- requirements.txt | 73 ++++++++++++++++++++------------------ src/hdx/data/resource.py | 7 ++-- src/hdx/data/vocabulary.py | 7 ++-- 5 files changed, 53 insertions(+), 42 deletions(-) diff --git a/documentation/main.md b/documentation/main.md index 16ca193..2b7a720 100755 --- a/documentation/main.md +++ b/documentation/main.md @@ -52,8 +52,8 @@ The library has detailed API documentation which can be found in the menu at the ## Breaking Changes -From 6.2.8, fix mark_data_updated which was broken due to an error in -dataset_update_filestore_resource in which timezone information was +From 6.2.8, fix mark_data_updated which was broken due to an error in +dataset_update_filestore_resource in which timezone information was incorrectly added to the iso formatted string From 6.2.7, generate_resource_from_iterator renamed to diff --git a/pyproject.toml b/pyproject.toml index 588a5db..1a72af6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,8 +37,8 @@ dependencies = [ "ckanapi>=4.8", "defopt>=6.4.0", "email_validator", - "hdx-python-country>=3.7.0", - "hdx-python-utilities>=3.6.8", + "hdx-python-country>=3.7.2", + "hdx-python-utilities>=3.6.9", "libhxl>=5.2.1", "makefun", "ndg-httpsclient", diff --git a/requirements.txt b/requirements.txt index 8938f9b..c1e27a6 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,16 @@ # This file was autogenerated by uv via the following command: # uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt -annotated-types==0.6.0 +annotated-types==0.7.0 # via pydantic attrs==23.2.0 # via # frictionless # jsonlines # jsonschema + # referencing cachetools==5.3.3 # via google-auth -certifi==2024.2.2 +certifi==2024.6.2 # via requests cffi==1.16.0 # via cryptography @@ -22,9 +23,9 @@ charset-normalizer==3.3.2 ckanapi==4.8 click==8.1.7 # via typer -coverage==7.5.0 +coverage==7.5.3 # via pytest-cov -cryptography==42.0.5 +cryptography==42.0.8 # via pyopenssl defopt==6.4.0 distlib==0.3.8 @@ -40,9 +41,9 @@ docutils==0.21.2 email-validator==2.1.1 et-xmlfile==1.1.0 # via openpyxl -filelock==3.13.4 +filelock==3.14.0 # via virtualenv -frictionless==5.16.1 +frictionless==5.17.0 # via hdx-python-utilities google-auth==2.29.0 # via @@ -50,9 +51,9 @@ google-auth==2.29.0 # gspread google-auth-oauthlib==1.2.0 # via gspread -gspread==6.1.0 -hdx-python-country==3.7.0 -hdx-python-utilities==3.6.8 +gspread==6.1.2 +hdx-python-country==3.7.2 +hdx-python-utilities==3.6.9 # via hdx-python-country humanize==4.9.0 # via frictionless @@ -70,16 +71,18 @@ iniconfig==2.0.0 # via pytest isodate==0.6.1 # via frictionless -jinja2==3.1.3 +jinja2==3.1.4 # via frictionless jsonlines==4.0.0 # via hdx-python-utilities jsonpath-ng==1.6.1 # via libhxl -jsonschema==4.17.3 +jsonschema==4.22.0 # via # frictionless # tableschema-to-template +jsonschema-specifications==2023.12.1 + # via jsonschema libhxl==5.2.1 # via hdx-python-country loguru==0.7.2 @@ -96,19 +99,19 @@ mdurl==0.1.2 more-itertools==10.2.0 # via inflect ndg-httpsclient==0.5.1 -nodeenv==1.8.0 +nodeenv==1.9.1 # via pre-commit num2words==0.5.13 # via quantulum3 oauthlib==3.2.2 # via requests-oauthlib -openpyxl==3.1.2 +openpyxl==3.1.3 # via hdx-python-utilities packaging==24.0 # via pytest petl==1.7.15 # via frictionless -platformdirs==4.2.1 +platformdirs==4.2.2 # via virtualenv pluggy==1.5.0 # via pytest @@ -118,7 +121,7 @@ ply==3.11 # libhxl pockets==0.9.1 # via sphinxcontrib-napoleon -pre-commit==3.7.0 +pre-commit==3.7.1 pyasn1==0.6.0 # via # ndg-httpsclient @@ -128,19 +131,17 @@ pyasn1-modules==0.4.0 # via google-auth pycparser==2.22 # via cffi -pydantic==2.7.1 +pydantic==2.7.3 # via frictionless -pydantic-core==2.18.2 +pydantic-core==2.18.4 # via pydantic -pygments==2.17.2 +pygments==2.18.0 # via rich pyopenssl==24.1.0 # via ndg-httpsclient pyphonetics==0.5.3 # via hdx-python-country -pyrsistent==0.20.0 - # via jsonschema -pytest==8.1.1 +pytest==8.2.2 # via pytest-cov pytest-cov==5.0.0 python-dateutil==2.8.2 @@ -162,14 +163,18 @@ pyyaml==6.0.1 quantulum3==0.9.1 ratelimit==2.2.1 # via hdx-python-utilities -requests==2.31.0 +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 # via # ckanapi # frictionless # libhxl # requests-file # requests-oauthlib -requests-file==2.0.0 +requests-file==2.1.0 # via hdx-python-utilities requests-oauthlib==2.0.0 # via google-auth-oauthlib @@ -177,16 +182,18 @@ rfc3986==2.0.0 # via frictionless rich==13.7.1 # via typer +rpds-py==0.18.1 + # via + # jsonschema + # referencing rsa==4.9 # via google-auth ruamel-yaml==0.18.6 # via hdx-python-utilities ruamel-yaml-clib==0.2.8 # via ruamel-yaml -setuptools==69.5.1 - # via - # ckanapi - # nodeenv +setuptools==70.0.0 + # via ckanapi shellingham==1.5.4 # via typer simpleeval==0.9.13 @@ -202,11 +209,9 @@ six==1.16.0 # sphinxcontrib-napoleon sphinxcontrib-napoleon==0.7 # via defopt -strenum==0.4.15 - # via gspread stringcase==1.2.0 # via frictionless -structlog==24.1.0 +structlog==24.2.0 # via libhxl tableschema-to-template==0.0.13 # via hdx-python-utilities @@ -214,11 +219,11 @@ tabulate==0.9.0 # via frictionless text-unidecode==1.3 # via python-slugify -typeguard==4.2.1 +typeguard==4.3.0 # via inflect typer==0.12.3 # via frictionless -typing-extensions==4.11.0 +typing-extensions==4.12.1 # via # frictionless # inflect @@ -234,9 +239,9 @@ urllib3==2.2.1 # via # libhxl # requests -validators==0.28.1 +validators==0.28.3 # via frictionless -virtualenv==20.26.0 +virtualenv==20.26.2 # via pre-commit wheel==0.43.0 # via libhxl diff --git a/src/hdx/data/resource.py b/src/hdx/data/resource.py index 8b94e5c..9e77d19 100755 --- a/src/hdx/data/resource.py +++ b/src/hdx/data/resource.py @@ -172,7 +172,8 @@ def read_formats_mappings( if configuration is None: configuration = Configuration.read() with Download( - full_agent=configuration.get_user_agent() + full_agent=configuration.get_user_agent(), + use_env=False, ) as downloader: if url is None: url = configuration["formats_mapping_url"] @@ -534,7 +535,9 @@ def download(self, folder: Optional[str] = None) -> Tuple[str, str]: else: headers = None with Download( - full_agent=self.configuration.get_user_agent(), headers=headers + full_agent=self.configuration.get_user_agent(), + use_env=False, + headers=headers, ) as downloader: path = downloader.download_file( url, folder=folder, filename=filename diff --git a/src/hdx/data/vocabulary.py b/src/hdx/data/vocabulary.py index 6860ba6..14949a6 100755 --- a/src/hdx/data/vocabulary.py +++ b/src/hdx/data/vocabulary.py @@ -258,7 +258,9 @@ def _read_approved_tags( Returns: List[str]: List of approved tags """ - with Download(full_agent=configuration.get_user_agent()) as downloader: + with Download( + full_agent=configuration.get_user_agent(), use_env=False + ) as downloader: if url is None: url = configuration["tags_list_url"] return list( @@ -402,7 +404,8 @@ def read_tags_mappings( if configuration is None: configuration = Configuration.read() with Download( - full_agent=configuration.get_user_agent() + full_agent=configuration.get_user_agent(), + use_env=False, ) as downloader: if url is None: url = configuration["tags_mapping_url"]