Skip to content

Commit

Permalink
enable all lints in ruff
Browse files Browse the repository at this point in the history
  • Loading branch information
ocefpaf committed Mar 5, 2024
1 parent 3012d18 commit c806b6c
Show file tree
Hide file tree
Showing 22 changed files with 4,399 additions and 31,872 deletions.
3 changes: 1 addition & 2 deletions erddapy/core/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
"""
Core subpackage for the erddapy parent package.
"""Core subpackage for the erddapy parent package.
This package contains the URL and data handling functionalities.
"""
Expand Down
41 changes: 22 additions & 19 deletions erddapy/core/griddap.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,22 @@
"""Griddap handling."""

from __future__ import annotations

import functools

import pandas as pd

from erddapy.core.url import urlopen

ListLike = list[str] | tuple[str]
OptionalList = list[str] | tuple[str] | None


@functools.lru_cache(maxsize=128)
def _griddap_get_constraints(
dataset_url: str,
step: int,
) -> tuple[dict, list, list]:
"""
Fetch metadata of griddap dataset and set initial constraints.
"""Fetch metadata of griddap dataset and set initial constraints.
Step size is applied to all dimensions.
"""
Expand All @@ -37,11 +38,8 @@ def _griddap_get_constraints(
)
for dim in dim_names:
url = f"{dataset_url}.csvp?{dim}"
data = pd.read_csv(url).values
if dim == "time":
data_start = data[-1][0]
else:
data_start = data[0][0]
data = pd.read_csv(url).to_numpy()
data_start = data[-1][0] if dim == "time" else data[0][0]

meta = pd.DataFrame(
[
Expand All @@ -67,24 +65,29 @@ def _griddap_get_constraints(
def _griddap_check_constraints(
user_constraints: dict,
original_constraints: dict,
):
"""Check that constraints changed by user match those expected by dataset."""
) -> None:
"""Validate user constraints against the dataset."""
if user_constraints.keys() != original_constraints.keys():
raise ValueError(
msg = (
"keys in e.constraints have changed. Re-run e.griddap_initialize",
)
raise ValueError(msg)


def _griddap_check_variables(
user_variables: ListLike,
original_variables: ListLike,
):
user_variables: OptionalList = None,
original_variables: OptionalList = None,
) -> None:
"""Check user has not requested variables that do not exist in dataset."""
invalid_variables = []
for variable in user_variables:
if variable not in original_variables:
invalid_variables.append(variable)
invalid_variables.extend(
variable
for variable in user_variables
if variable not in original_variables
)
if invalid_variables:
raise ValueError(
f"variables {invalid_variables} are not present in dataset. Re-run e.griddap_initialize",
msg = (
f"variables {invalid_variables} are not present in dataset. "
"Re-run e.griddap_initialize"
)
raise ValueError(msg)
72 changes: 38 additions & 34 deletions erddapy/core/interfaces.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
"""
Interface between URL responses and third-party libraries.
"""Interface between URL responses and third-party libraries.
This module takes an URL or the bytes response of a request and converts it to Pandas,
XArray, Iris, etc. objects.
This module takes an URL or the bytes response of a request and converts it
to Pandas, XArray, Iris, etc. objects.
"""

from __future__ import annotations

from typing import TYPE_CHECKING

import pandas as pd
Expand All @@ -13,17 +14,21 @@
from erddapy.core.url import urlopen

if TYPE_CHECKING:
import iris.cube
import netCDF4
import xarray as xr
from netCDF4 import Dataset


OptionalStr = str | None
OptionalDict = dict | None


def to_pandas(
url: str,
requests_kwargs: dict | None = None,
pandas_kwargs: dict | None = None,
) -> "pd.DataFrame":
"""
Convert a URL to Pandas DataFrame.
) -> pd.DataFrame:
"""Convert a URL to Pandas DataFrame.
url: URL to request data from.
requests_kwargs: arguments to be passed to urlopen method.
Expand All @@ -32,39 +37,39 @@ def to_pandas(
data = urlopen(url, requests_kwargs or {})
try:
return pd.read_csv(data, **(pandas_kwargs or {}))
except Exception as e:
raise ValueError(
f"Could not read url {url} with Pandas.read_csv.",
) from e
except Exception as e: # noqa: BLE001
msg = f"Could not read url {url} with Pandas.read_csv."
raise ValueError(msg) from e


def to_ncCF(
def to_ncCF( # noqa: N802
url: str,
protocol: str = None,
requests_kwargs: dict | None = None,
) -> "Dataset":
"""
Convert a URL to a netCDF4 Dataset.
protocol: OptionalStr = None,
requests_kwargs: OptionalDict = None,
) -> netCDF4.Dataset:
"""Convert a URL to a netCDF4 Dataset.
url: URL to request data from.
protocol: 'griddap' or 'tabledap'.
requests_kwargs: arguments to be passed to urlopen method (including auth).
"""
msg = (
f"Cannot use .ncCF with griddap protocol."
f"The URL you tried to access is: '{url}'."
)
if protocol == "griddap":
raise ValueError(
f"Cannot use .ncCF with griddap protocol. The URL you tried to access is: '{url}'.",
)
raise ValueError(msg)
return _nc_dataset(url, requests_kwargs)


def to_xarray(
url: str,
response="opendap",
response: OptionalStr = "opendap",
requests_kwargs: dict | None = None,
xarray_kwargs: dict | None = None,
) -> "xr.Dataset":
"""
Convert a URL to an xarray dataset.
) -> xr.Dataset:
"""Convert a URL to an xarray dataset.
url: URL to request data from.
response: type of response to be requested from the server.
Expand All @@ -75,21 +80,20 @@ def to_xarray(

if response == "opendap":
return xr.open_dataset(url, **(xarray_kwargs or {}))
else:
nc = _nc_dataset(url, requests_kwargs)
return xr.open_dataset(
xr.backends.NetCDF4DataStore(nc),
**(xarray_kwargs or {}),
)

nc = _nc_dataset(url, requests_kwargs)
return xr.open_dataset(
xr.backends.NetCDF4DataStore(nc),
**(xarray_kwargs or {}),
)


def to_iris(
url: str,
requests_kwargs: dict | None = None,
iris_kwargs: dict | None = None,
):
"""
Convert a URL to an iris CubeList.
) -> iris.cube.CubeList:
"""Convert a URL to an iris CubeList.
url: URL to request data from.
requests_kwargs: arguments to be passed to urlopen method.
Expand Down
22 changes: 17 additions & 5 deletions erddapy/core/netcdf.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,36 @@
"""Handles netCDF responses."""

from __future__ import annotations

import platform
from collections.abc import Generator
from contextlib import contextmanager
from pathlib import Path
from typing import BinaryIO
from typing import TYPE_CHECKING, BinaryIO
from urllib.parse import urlparse

from erddapy.core.url import urlopen

if TYPE_CHECKING:
from collections.abc import Generator

import netCDF4


def _nc_dataset(
url: str,
requests_kwargs: dict | None = None,
) -> netCDF4.Dataset:
"""Return a netCDF4-python Dataset from memory
and fallbacks to disk if that fails.
def _nc_dataset(url, requests_kwargs: dict | None = None):
"""Return a netCDF4-python Dataset from memory and fallbacks to disk if that fails."""
"""
from netCDF4 import Dataset

data = urlopen(url, requests_kwargs)
try:
return Dataset(Path(urlparse(url).path).name, memory=data.read())
except OSError:
# if libnetcdf is not compiled with in-memory support fallback to a local tmp file
# if libnetcdf is not compiled with in-memory support fallback tmp file
data.seek(0)
with _tempnc(data) as _nc:
return Dataset(_nc)
Expand Down
Loading

0 comments on commit c806b6c

Please sign in to comment.