Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enable all lints in ruff #337

Merged
merged 2 commits into from
Mar 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions erddapy/core/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
"""
Core subpackage for the erddapy parent package.
"""Core subpackage for the erddapy parent package.

This package contains the URL and data handling functionalities.
"""
Expand Down
41 changes: 22 additions & 19 deletions erddapy/core/griddap.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,22 @@
"""Griddap handling."""

from __future__ import annotations

import functools

import pandas as pd

from erddapy.core.url import urlopen

ListLike = list[str] | tuple[str]
OptionalList = list[str] | tuple[str] | None


@functools.lru_cache(maxsize=128)
def _griddap_get_constraints(
dataset_url: str,
step: int,
) -> tuple[dict, list, list]:
"""
Fetch metadata of griddap dataset and set initial constraints.
"""Fetch metadata of griddap dataset and set initial constraints.

Step size is applied to all dimensions.
"""
Expand All @@ -37,11 +38,8 @@ def _griddap_get_constraints(
)
for dim in dim_names:
url = f"{dataset_url}.csvp?{dim}"
data = pd.read_csv(url).values
if dim == "time":
data_start = data[-1][0]
else:
data_start = data[0][0]
data = pd.read_csv(url).to_numpy()
data_start = data[-1][0] if dim == "time" else data[0][0]

meta = pd.DataFrame(
[
Expand All @@ -67,24 +65,29 @@ def _griddap_get_constraints(
def _griddap_check_constraints(
user_constraints: dict,
original_constraints: dict,
):
"""Check that constraints changed by user match those expected by dataset."""
) -> None:
"""Validate user constraints against the dataset."""
if user_constraints.keys() != original_constraints.keys():
raise ValueError(
msg = (
"keys in e.constraints have changed. Re-run e.griddap_initialize",
)
raise ValueError(msg)


def _griddap_check_variables(
user_variables: ListLike,
original_variables: ListLike,
):
user_variables: OptionalList = None,
original_variables: OptionalList = None,
) -> None:
"""Check user has not requested variables that do not exist in dataset."""
invalid_variables = []
for variable in user_variables:
if variable not in original_variables:
invalid_variables.append(variable)
invalid_variables.extend(
variable
for variable in user_variables
if variable not in original_variables
)
if invalid_variables:
raise ValueError(
f"variables {invalid_variables} are not present in dataset. Re-run e.griddap_initialize",
msg = (
f"variables {invalid_variables} are not present in dataset. "
"Re-run e.griddap_initialize"
)
raise ValueError(msg)
72 changes: 38 additions & 34 deletions erddapy/core/interfaces.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
"""
Interface between URL responses and third-party libraries.
"""Interface between URL responses and third-party libraries.

This module takes an URL or the bytes response of a request and converts it to Pandas,
XArray, Iris, etc. objects.
This module takes an URL or the bytes response of a request and converts it
to Pandas, XArray, Iris, etc. objects.
"""

from __future__ import annotations

from typing import TYPE_CHECKING

import pandas as pd
Expand All @@ -13,17 +14,21 @@
from erddapy.core.url import urlopen

if TYPE_CHECKING:
import iris.cube
import netCDF4
import xarray as xr
from netCDF4 import Dataset


OptionalStr = str | None
OptionalDict = dict | None


def to_pandas(
url: str,
requests_kwargs: dict | None = None,
pandas_kwargs: dict | None = None,
) -> "pd.DataFrame":
"""
Convert a URL to Pandas DataFrame.
) -> pd.DataFrame:
"""Convert a URL to Pandas DataFrame.

url: URL to request data from.
requests_kwargs: arguments to be passed to urlopen method.
Expand All @@ -32,39 +37,39 @@ def to_pandas(
data = urlopen(url, requests_kwargs or {})
try:
return pd.read_csv(data, **(pandas_kwargs or {}))
except Exception as e:
raise ValueError(
f"Could not read url {url} with Pandas.read_csv.",
) from e
except Exception as e: # noqa: BLE001
msg = f"Could not read url {url} with Pandas.read_csv."
raise ValueError(msg) from e


def to_ncCF(
def to_ncCF( # noqa: N802
url: str,
protocol: str = None,
requests_kwargs: dict | None = None,
) -> "Dataset":
"""
Convert a URL to a netCDF4 Dataset.
protocol: OptionalStr = None,
requests_kwargs: OptionalDict = None,
) -> netCDF4.Dataset:
"""Convert a URL to a netCDF4 Dataset.

url: URL to request data from.
protocol: 'griddap' or 'tabledap'.
requests_kwargs: arguments to be passed to urlopen method (including auth).

"""
msg = (
f"Cannot use .ncCF with griddap protocol."
f"The URL you tried to access is: '{url}'."
)
if protocol == "griddap":
raise ValueError(
f"Cannot use .ncCF with griddap protocol. The URL you tried to access is: '{url}'.",
)
raise ValueError(msg)
return _nc_dataset(url, requests_kwargs)


def to_xarray(
url: str,
response="opendap",
response: OptionalStr = "opendap",
requests_kwargs: dict | None = None,
xarray_kwargs: dict | None = None,
) -> "xr.Dataset":
"""
Convert a URL to an xarray dataset.
) -> xr.Dataset:
"""Convert a URL to an xarray dataset.

url: URL to request data from.
response: type of response to be requested from the server.
Expand All @@ -75,21 +80,20 @@ def to_xarray(

if response == "opendap":
return xr.open_dataset(url, **(xarray_kwargs or {}))
else:
nc = _nc_dataset(url, requests_kwargs)
return xr.open_dataset(
xr.backends.NetCDF4DataStore(nc),
**(xarray_kwargs or {}),
)

nc = _nc_dataset(url, requests_kwargs)
return xr.open_dataset(
xr.backends.NetCDF4DataStore(nc),
**(xarray_kwargs or {}),
)


def to_iris(
url: str,
requests_kwargs: dict | None = None,
iris_kwargs: dict | None = None,
):
"""
Convert a URL to an iris CubeList.
) -> iris.cube.CubeList:
"""Convert a URL to an iris CubeList.

url: URL to request data from.
requests_kwargs: arguments to be passed to urlopen method.
Expand Down
22 changes: 17 additions & 5 deletions erddapy/core/netcdf.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,36 @@
"""Handles netCDF responses."""

from __future__ import annotations

import platform
from collections.abc import Generator
from contextlib import contextmanager
from pathlib import Path
from typing import BinaryIO
from typing import TYPE_CHECKING, BinaryIO
from urllib.parse import urlparse

from erddapy.core.url import urlopen

if TYPE_CHECKING:
from collections.abc import Generator

import netCDF4


def _nc_dataset(
url: str,
requests_kwargs: dict | None = None,
) -> netCDF4.Dataset:
"""Return a netCDF4-python Dataset from memory
and fallbacks to disk if that fails.

def _nc_dataset(url, requests_kwargs: dict | None = None):
"""Return a netCDF4-python Dataset from memory and fallbacks to disk if that fails."""
"""
from netCDF4 import Dataset

data = urlopen(url, requests_kwargs)
try:
return Dataset(Path(urlparse(url).path).name, memory=data.read())
except OSError:
# if libnetcdf is not compiled with in-memory support fallback to a local tmp file
# if libnetcdf is not compiled with in-memory support fallback tmp file
data.seek(0)
with _tempnc(data) as _nc:
return Dataset(_nc)
Expand Down
Loading
Loading