diff --git a/_parameter.py b/_parameter.py deleted file mode 100644 index 272cf9fa6..000000000 --- a/_parameter.py +++ /dev/null @@ -1,191 +0,0 @@ -from pathlib import Path -from typing import Iterator - -import jsonref -from pydantic import BaseModel, field_validator, Field, RootModel - -FILE = Path("./dwd.json") - - -class _Parameter(BaseModel): - name: str - original: str - resolution: str - dataset: str - - -class _Dataset(BaseModel): - name: str - resolution: str - parameters_: dict[str, _Parameter] = Field(alias="parameters", serialization_alias="parameters") - - @property - def parameters(self): - return list(self.parameters_.values()) - - @field_validator("parameters_", mode="before") - @classmethod - def validate_parameters(cls, v, validation_info): - resolution_name = validation_info.data["resolution"] - dataset_name = validation_info.data["name"] - for name, parameter in v.items(): - parameter["name"] = name - parameter["resolution"] = resolution_name - parameter["dataset"] = dataset_name - return v - - def __getitem__(self, item): - if isinstance(item, int): - return self.parameters[item] - try: - return self.parameters_[item] - except KeyError as e: - for parameter in self.parameters: - if parameter.original == item: - return parameter - raise KeyError(item) from e - - def __getattr__(self, item): - try: - return self.parameters_[item] - except KeyError as e: - for parameter in self.parameters: - if parameter.original == item: - return parameter - raise KeyError(item) from e - - def __iter__(self) -> Iterator[_Parameter]: - return iter(self.parameters) - - def __str__(self): - return f"name='{self.name}' resolution='{self.resolution}' parameters={self.parameters}" - - def __repr__(self): - return f"_Dataset({self.__str__()})" - - -class _Resolution(BaseModel): - name: str - datasets_: dict[str, _Dataset] = Field(alias="datasets", serialization_alias="datasets") - parameters_: dict[str, _Parameter] = Field(alias="parameters", serialization_alias="parameters") - - @property - def datasets(self): - return list(self.datasets_.values()) - - @property - def parameters(self): - return list(self.parameters_.values()) - - @field_validator("datasets_", mode="before") - @classmethod - def validate_datasets(cls, v, validation_info): - resolution_name = validation_info.data["name"] - for name, dataset in v.items(): - dataset["name"] = name - dataset["resolution"] = resolution_name - return v - - @field_validator("parameters_", mode="before") - @classmethod - def validate_parameters(cls, v): - for name, parameter in v.items(): - parameter["name"] = name - return v - - def __getitem__(self, item): - if isinstance(item, int): - return self.datasets[item] - try: - return self.datasets_[item] - except KeyError as e: - try: - return self.parameters_[item] - except KeyError as e: - raise KeyError(item) from e - - def __getattr__(self, item): - return self.datasets_[item] - - def __iter__(self) -> Iterator[_Dataset]: - return iter(self.datasets) - - def __str__(self): - return f"name='{self.name}' datasets={self.datasets} parameters={self.parameters}" - - def __repr__(self): - return f"_Resolution({self.__str__()})" - - -class _Metadata(BaseModel): - resolutions: dict[str, _Resolution] - - @field_validator("resolutions", mode="before") - @classmethod - def validate_parameters(cls, v): - for name, resolution in v.items(): - resolution["name"] = name - return v - - def __getitem__(self, item): - if isinstance(item, int): - return list(self.resolutions.values())[item] - return self.resolutions[item] - - def __getattr__(self, item): - return self.resolutions[item] - - def __iter__(self) -> Iterator[_Resolution]: - return iter(self.resolutions.values()) - - def __str__(self): - return f"resolutions={list(self.resolutions.values())}" - - def __repr__(self): - return f"_Metadata({self.__str__()})" - - -parameters_raw = jsonref.loads(FILE.read_text()) -parameters = _Metadata.model_validate(parameters_raw) - -# print(parameters["minute_1"]["precipitation"]["precipitation_height"]) -# print(parameters.minute_1.precipitation.precipitation_height) -# print(parameters["minute_1"].precipitation["precipitation_height"]) -# print(parameters.minute_1["precipitation"].rs_01) -# -# print(parameters.minute_1[0][1]) -# -# print(parameters.minute_1.parameters[0]) - -precipitation_height = parameters.minute_1.precipitation.precipitation_height - -print(precipitation_height) - -# print(parameters.minute_1.precipitation[precipitation_height]) - - -def parse_parameter(parameter: str | tuple[str, str] | list[str] | _Parameter) -> _Parameter: - resolution = "minute_1" - dataset = None - try: - parameter, dataset = parameter - except ValueError: - pass - try: - parameter = parameter.name - dataset = parameter.dataset - except AttributeError: - pass - datasets = parameters[resolution] - if dataset: - return datasets[dataset][parameter] - return datasets[parameter] - -par = parse_parameter("precipitation_height") -print(par) - -par = parse_parameter(("precipitation_height", "precipitation")) -print(par) - -par = parse_parameter(par) -print(par) diff --git a/dwd.json b/dwd.json deleted file mode 100644 index ab10d1333..000000000 --- a/dwd.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "resolutions": { - "minute_1": { - "datasets": { - "precipitation": { - "parameters": { - "quality": { - "original": "qn" - }, - "precipitation_height": { - "original": "rs_01" - }, - "precipitation_height_droplet": { - "original": "rth_01" - }, - "precipitation_height_rocker": { - "original": "rwh_01" - }, - "precipitation_index": { - "original": "rs_ind_01" - } - } - } - }, - "parameters": { - "precipitation_height": { - "$ref": "#/resolutions/minute_1/datasets/precipitation/parameters/precipitation_height" - }, - "precipitation_height_droplet": { - "$ref": "#/resolutions/minute_1/datasets/precipitation/parameters/precipitation_height_droplet" - }, - "precipitation_height_rocker": { - "$ref": "#/resolutions/minute_1/datasets/precipitation/parameters/precipitation_height_rocker" - }, - "precipitation_index": { - "$ref": "#/resolutions/minute_1/datasets/precipitation/parameters/precipitation_index" - } - } - } - } -} \ No newline at end of file diff --git a/wetterdienst/metadata/metadata_model.py b/wetterdienst/metadata/metadata_model.py new file mode 100644 index 000000000..4c32fd0f3 --- /dev/null +++ b/wetterdienst/metadata/metadata_model.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Iterator + +import jsonref +from pydantic import BaseModel, field_validator, RootModel, model_validator, ValidationError + + +class Parameter(BaseModel): + name: str + original: str + resolution: str + dataset: str + + +class Dataset(BaseModel): + name: str + resolution: str + parameters: list[Parameter] + + @field_validator("parameters", mode="before") + @classmethod + def validate_parameters(cls, v, validation_info): + resolution_name = validation_info.data["resolution"] + dataset_name = validation_info.data["name"] + for parameter in v: + parameter["resolution"] = resolution_name + parameter["dataset"] = dataset_name + return v + + def __getitem__(self, item): + if isinstance(item, int): + return self.parameters[item] + for parameter in self.parameters: + if parameter.name == item or parameter.original == item: + return parameter + raise KeyError(item) + + def __getattr__(self, item): + for parameter in self.parameters: + if parameter.name == item or parameter.original == item: + return parameter + raise AttributeError(item) + + def __iter__(self) -> Iterator[Parameter]: + return iter(self.parameters) + + def __str__(self): + return f"name='{self.name}' resolution='{self.resolution}' parameters={self.parameters}" + + def __repr__(self): + return f"Dataset({self.__str__()})" + + +# parameter reference $ref which consists of "dataset/parameter" +# should be resolved into dataset and parameter before validation of pydanitc model +class ParameterRef(BaseModel): + dataset: str + parameter: str + + @model_validator(mode="before") + def resolve_ref(cls, v): + if v.keys() != {"$ref"}: + raise ValueError("Only $ref is allowed") + dataset, parameter = v["$ref"].split("/") + return {"dataset": dataset, "parameter": parameter} + + +class Resolution(BaseModel): + name: str + datasets: list[Dataset] + parameters: list[Parameter] + + @field_validator("datasets", mode="before") + @classmethod + def validate_datasets(cls, v, validation_info): + resolution_name = validation_info.data["name"] + for dataset in v: + dataset["resolution"] = resolution_name + return v + + @field_validator("parameters", mode="before") + @classmethod + def validate_parameters(cls, v, validation_info): + # resolve references + resolved_parameters = [] + for parameter_ref in v: + parameter_ref = ParameterRef.model_validate(parameter_ref) + for dataset in validation_info.data["datasets"]: + if dataset.name == parameter_ref.dataset: + for parameter in dataset.parameters: + if parameter.name == parameter_ref.parameter: + resolved_parameters.append(parameter) + break + else: + raise KeyError(parameter_ref) + break + else: + raise KeyError(parameter_ref) + return resolved_parameters + + def __getitem__(self, item): + if isinstance(item, int): + return self.datasets[item] + for dataset in self.datasets: + if dataset.name == item: + return dataset + for parameter in self.parameters: + if parameter.name == item: + return parameter + raise KeyError(item) + + def __getattr__(self, item): + for dataset in self.datasets: + if dataset.name == item: + return dataset + for parameter in self.parameters: + if parameter.name == item: + return parameter + raise AttributeError(item) + + def __iter__(self) -> Iterator[Dataset]: + return iter(self.datasets) + + def __str__(self): + return f"name='{self.name}' datasets={self.datasets} parameters={self.parameters}" + + def __repr__(self): + return f"Resolution({self.__str__()})" + + +class MetadataModel(RootModel): + root: list[Resolution] + + def __getitem__(self, item): + if isinstance(item, int): + return self.root[item] + for resolution in self.root: + if resolution.name == item: + return resolution + raise KeyError(item) + + def __getattr__(self, item): + for resolution in self.root: + if resolution.name == item: + return resolution + raise AttributeError(item) + + + def __iter__(self) -> Iterator[Resolution]: + return iter(self.root) + + def __str__(self): + return f"[{self.root}]" + + def __repr__(self): + return f"Metadata({self.__str__()})" + + def search_parameter(self, parameter_search: "ParameterTemplate") -> list[Parameter]: + for resolution in self: + if resolution.name == parameter_search.resolution: + break + else: + raise KeyError(parameter_search.resolution) + for dataset in resolution: + if dataset.name == parameter_search.dataset: + break + else: + raise KeyError(parameter_search.dataset) + if parameter_search.parameter is None: + return [*dataset] + for parameter in dataset: + if parameter.name == parameter_search.parameter: + return [parameter] + +@dataclass +class ParameterTemplate: + resolution: str + dataset: str + parameter: str | None + + @classmethod + def parse(cls, value: str | Iterable[str]) -> "ParameterTemplate": + resolution = None + dataset = None + parameter = None + if isinstance(value, str): + if value.count("/") == 0: + raise ValueError("expected 'resolution/dataset/parameter' or 'resolution/dataset'") + value = value.split("/") + try: + resolution, dataset, parameter = value + except ValueError: + try: + resolution, dataset = value + except ValueError: + pass + return ParameterTemplate(resolution, dataset, parameter) diff --git a/wetterdienst/provider/dwd/observation/metadata/__init__.py b/wetterdienst/provider/dwd/observation/_metadata/__init__.py similarity index 100% rename from wetterdienst/provider/dwd/observation/metadata/__init__.py rename to wetterdienst/provider/dwd/observation/_metadata/__init__.py diff --git a/wetterdienst/provider/dwd/observation/metadata/dataset.py b/wetterdienst/provider/dwd/observation/_metadata/dataset.py similarity index 100% rename from wetterdienst/provider/dwd/observation/metadata/dataset.py rename to wetterdienst/provider/dwd/observation/_metadata/dataset.py diff --git a/wetterdienst/provider/dwd/observation/metadata/parameter.py b/wetterdienst/provider/dwd/observation/_metadata/parameter.py similarity index 100% rename from wetterdienst/provider/dwd/observation/metadata/parameter.py rename to wetterdienst/provider/dwd/observation/_metadata/parameter.py diff --git a/wetterdienst/provider/dwd/observation/metadata/period.py b/wetterdienst/provider/dwd/observation/_metadata/period.py similarity index 100% rename from wetterdienst/provider/dwd/observation/metadata/period.py rename to wetterdienst/provider/dwd/observation/_metadata/period.py diff --git a/wetterdienst/provider/dwd/observation/metadata/resolution.py b/wetterdienst/provider/dwd/observation/_metadata/resolution.py similarity index 100% rename from wetterdienst/provider/dwd/observation/metadata/resolution.py rename to wetterdienst/provider/dwd/observation/_metadata/resolution.py diff --git a/wetterdienst/provider/dwd/observation/metadata/unit.py b/wetterdienst/provider/dwd/observation/_metadata/unit.py similarity index 100% rename from wetterdienst/provider/dwd/observation/metadata/unit.py rename to wetterdienst/provider/dwd/observation/_metadata/unit.py diff --git a/wetterdienst/provider/dwd/observation/api.py b/wetterdienst/provider/dwd/observation/api.py index 15ae4a7c5..46a069e9c 100644 --- a/wetterdienst/provider/dwd/observation/api.py +++ b/wetterdienst/provider/dwd/observation/api.py @@ -17,6 +17,7 @@ from wetterdienst.metadata.columns import Columns from wetterdienst.metadata.datarange import DataRange from wetterdienst.metadata.kind import Kind +from wetterdienst.metadata.metadata_model import MetadataModel from wetterdienst.metadata.period import Period, PeriodType from wetterdienst.metadata.provider import Provider from wetterdienst.metadata.resolution import Resolution, ResolutionType @@ -29,19 +30,19 @@ create_file_index_for_climate_observations, create_file_list_for_climate_observations, ) -from wetterdienst.provider.dwd.observation.metadata.dataset import ( - DwdObservationDataset, +from wetterdienst.provider.dwd.observation.metadata import ( + DwdObservationMetadata, ) -from wetterdienst.provider.dwd.observation.metadata.parameter import ( - DwdObservationParameter, -) -from wetterdienst.provider.dwd.observation.metadata.period import DwdObservationPeriod -from wetterdienst.provider.dwd.observation.metadata.resolution import ( - HIGH_RESOLUTIONS, - RESOLUTION_TO_DATETIME_FORMAT_MAPPING, - DwdObservationResolution, -) -from wetterdienst.provider.dwd.observation.metadata.unit import DwdObservationUnit +# from wetterdienst.provider.dwd.observation.metadata.parameter import ( +# DwdObservationParameter, +# ) +# from wetterdienst.provider.dwd.observation.metadata.period import DwdObservationPeriod +# from wetterdienst.provider.dwd.observation.metadata.resolution import ( +# HIGH_RESOLUTIONS, +# RESOLUTION_TO_DATETIME_FORMAT_MAPPING, +# DwdObservationResolution, +# ) +# from wetterdienst.provider.dwd.observation.metadata.unit import DwdObservationUnit from wetterdienst.provider.dwd.observation.metaindex import ( create_meta_index_for_climate_observations, ) @@ -70,7 +71,7 @@ class DwdObservationValues(TimeseriesValues): _resolution_type = ResolutionType.MULTI _resolution_base = DwdObservationResolution _period_type = PeriodType.MULTI - _period_base = DwdObservationPeriod + # _period_base = DwdObservationPeriod @property def _datetime_format(self): @@ -359,13 +360,14 @@ class DwdObservationRequest(TimeseriesRequest): _provider = Provider.DWD _kind = Kind.OBSERVATION _tz = Timezone.GERMANY - _dataset_base = DwdObservationDataset - _parameter_base = DwdObservationParameter - _unit_base = DwdObservationUnit + # _dataset_base = DwdObservationDataset + # _parameter_base = DwdObservationParameter + # _unit_base = DwdObservationUnit + _metadata = DwdObservationMetadata _resolution_type = ResolutionType.MULTI _resolution_base = DwdObservationResolution _period_type = PeriodType.MULTI - _period_base = DwdObservationPeriod + # _period_base = DwdObservationPeriod _has_datasets = True _unique_dataset = False _data_range = DataRange.FIXED @@ -449,13 +451,18 @@ def _parse_station_id(series: pl.Series) -> pl.Series: def __init__( self, parameter: str - | DwdObservationDataset - | DwdObservationParameter + # | DwdObservationDataset + # | DwdObservationParameter + | MetadataModel + | tuple[str, str] + | tuple[str, str, str] | Sequence[ str - | DwdObservationDataset - | DwdObservationParameter - | tuple[str | DwdObservationParameter, str | DwdObservationDataset] + # | DwdObservationDataset + # | DwdObservationParameter + | MetadataModel + | tuple[str, str] + | tuple[str, str, str] ], resolution: str | DwdObservationResolution | Resolution, period: str | DwdObservationPeriod | Period | Sequence[str | DwdObservationPeriod | Period] = None, diff --git a/wetterdienst/provider/dwd/observation/fileindex.py b/wetterdienst/provider/dwd/observation/fileindex.py index 5aff5848d..26cb6bcb9 100644 --- a/wetterdienst/provider/dwd/observation/fileindex.py +++ b/wetterdienst/provider/dwd/observation/fileindex.py @@ -12,11 +12,11 @@ from wetterdienst.metadata.period import Period from wetterdienst.metadata.resolution import Resolution from wetterdienst.provider.dwd.metadata.datetime import DatetimeFormat -from wetterdienst.provider.dwd.observation.metadata.dataset import ( - DWD_URBAN_DATASETS, - DwdObservationDataset, -) -from wetterdienst.provider.dwd.observation.metadata.resolution import HIGH_RESOLUTIONS +# from wetterdienst.provider.dwd.observation.metadata.dataset import ( +# DWD_URBAN_DATASETS, +# DwdObservationDataset, +# ) +# from wetterdienst.provider.dwd.observation.metadata.resolution import HIGH_RESOLUTIONS from wetterdienst.util.cache import CacheExpiry from wetterdienst.util.network import list_remote_files_fsspec diff --git a/wetterdienst/provider/dwd/observation/metadata.py b/wetterdienst/provider/dwd/observation/metadata.py new file mode 100644 index 000000000..3fbfe8436 --- /dev/null +++ b/wetterdienst/provider/dwd/observation/metadata.py @@ -0,0 +1,51 @@ +from wetterdienst.metadata.metadata_model import MetadataModel + +DwdObservationMetadata = [ + { + "name": "minute_1", + "datasets": [ + { + "name": "precipitation", + "parameters": [ + { + "name": "quality", + "original": "qn", + "unit": "dimensionless", + "unit_original": "dimensionless" + }, + { + "name": "precipitation_height", + "original": "rs_01" + }, + { + "name": "precipitation_height_droplet", + "original": "rth_01" + }, + { + "name": "precipitation_height_rocker", + "original": "rwh_01" + }, + { + "name": "precipitation_index", + "original": "rs_ind_01" + } + ] + } + ], + "parameters": [ + { + "$ref": "precipitation/precipitation_height" + }, + { + "$ref": "precipitation/precipitation_height_droplet" + }, + { + "$ref": "precipitation/precipitation_height_rocker" + }, + { + "$ref": "precipitation/precipitation_index" + } + ] + } +] +DwdObservationMetadata = MetadataModel.model_validate(DwdObservationMetadata) \ No newline at end of file diff --git a/wetterdienst/provider/dwd/observation/metaindex.py b/wetterdienst/provider/dwd/observation/metaindex.py index a355d805b..c2a845ed0 100644 --- a/wetterdienst/provider/dwd/observation/metaindex.py +++ b/wetterdienst/provider/dwd/observation/metaindex.py @@ -17,10 +17,10 @@ from wetterdienst.metadata.period import Period from wetterdienst.metadata.resolution import Resolution from wetterdienst.provider.dwd.observation.fileindex import build_path_to_parameter -from wetterdienst.provider.dwd.observation.metadata.dataset import ( - DWD_URBAN_DATASETS, - DwdObservationDataset, -) +# from wetterdienst.provider.dwd.observation.metadata.dataset import ( +# DWD_URBAN_DATASETS, +# DwdObservationDataset, +# ) from wetterdienst.util.cache import CacheExpiry from wetterdienst.util.network import download_file, list_remote_files_fsspec