diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 805ce058..0cafdb3d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: - test_requirements.txt - repo: https://github.com/psf/black - rev: 24.4.0 + rev: 24.4.2 hooks: - id: black language_version: python3 @@ -31,12 +31,12 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.7 + rev: v0.4.4 hooks: - id: ruff - repo: https://github.com/tox-dev/pyproject-fmt - rev: 1.7.0 + rev: 2.0.4 hooks: - id: pyproject-fmt diff --git a/cchecker.py b/cchecker.py index fd3ef88b..4e5ed5d4 100755 --- a/cchecker.py +++ b/cchecker.py @@ -232,7 +232,7 @@ def main(): check_suite.load_generated_checkers(args) if args.version: - print("IOOS compliance checker version %s" % __version__) + print(f"IOOS compliance checker version {__version__}") sys.exit(0) options_dict = parse_options(args.option) if args.option else defaultdict(set) diff --git a/compliance_checker/acdd.py b/compliance_checker/acdd.py index 9b0abdc7..73d681c1 100644 --- a/compliance_checker/acdd.py +++ b/compliance_checker/acdd.py @@ -626,7 +626,7 @@ def check_time_extents(self, ds): BaseCheck.MEDIUM, False, "time_coverage_extents_match", - ["Failed to retrieve and convert times for variables %s." % timevar], + [f"Failed to retrieve and convert times for variables {timevar}."], ) start_dt = abs(time0 - t_min) diff --git a/compliance_checker/base.py b/compliance_checker/base.py index dbad8518..933d5a5a 100644 --- a/compliance_checker/base.py +++ b/compliance_checker/base.py @@ -19,7 +19,6 @@ from owslib.swe.observation.sos100 import SensorObservationService_1_0_0 from owslib.swe.sensor.sml import SensorML -import compliance_checker.cfutil as cfutil from compliance_checker import __version__ from compliance_checker.util import kvp_convert @@ -185,17 +184,6 @@ def get_test_ctx(self, severity, name, variable=None): ) return self._defined_results[name][variable][severity] - def __del__(self): - """ - Finalizer. Ensure any caches shared by multiple checkers - are cleared before the next checker uses it. Some caches were - inadvertently mutated by other functions. - """ - - if cfutil is not None: - cfutil.get_geophysical_variables.cache_clear() - cfutil.get_time_variables.cache_clear() - class BaseNCCheck: """ diff --git a/compliance_checker/cf/cf_1_6.py b/compliance_checker/cf/cf_1_6.py index f059e0f7..4b45b3e9 100644 --- a/compliance_checker/cf/cf_1_6.py +++ b/compliance_checker/cf/cf_1_6.py @@ -302,7 +302,7 @@ def check_names_unique(self, ds): names[k.lower()] += 1 fails = [ - "Variables are not case sensitive. Duplicate variables named: %s" % k + f"Variables are not case sensitive. Duplicate variables named: {k}" for k, v in names.items() if v > 1 ] @@ -1822,7 +1822,7 @@ def check_time_coordinate(self, ds): BaseCheck.HIGH, False, self.section_titles["4.4"], - ["%s does not have units" % name], + [f"{name} does not have units"], ) ret_val.append(result) continue @@ -1833,7 +1833,7 @@ def check_time_coordinate(self, ds): correct_units = util.units_temporal(variable.units) reasoning = None if not correct_units: - reasoning = ["%s does not have correct time units" % name] + reasoning = [f"{name} does not have correct time units"] result = Result( BaseCheck.HIGH, correct_units, diff --git a/compliance_checker/cf/cf_base.py b/compliance_checker/cf/cf_base.py index 7fd1d978..f7dbffdd 100644 --- a/compliance_checker/cf/cf_base.py +++ b/compliance_checker/cf/cf_base.py @@ -495,11 +495,12 @@ def _find_aux_coord_vars(self, ds, refresh=False): :return: List of variable names (str) that are defined to be auxiliary coordinate variables. """ + ds_str = ds.__str__() if self._aux_coords.get(ds, None) and refresh is False: - return self._aux_coords[ds] + return self._aux_coords[ds_str] - self._aux_coords[ds] = cfutil.get_auxiliary_coordinate_variables(ds) - return self._aux_coords[ds] + self._aux_coords[ds_str] = cfutil.get_auxiliary_coordinate_variables(ds) + return self._aux_coords[ds_str] def _find_boundary_vars(self, ds, refresh=False): """ @@ -512,12 +513,13 @@ def _find_boundary_vars(self, ds, refresh=False): :rtype: list :return: A list containing strings with boundary variable names. """ + ds_str = ds.__str__() if self._boundary_vars.get(ds, None) and refresh is False: - return self._boundary_vars[ds] + return self._boundary_vars[ds_str] - self._boundary_vars[ds] = cfutil.get_cell_boundary_variables(ds) + self._boundary_vars[ds_str] = cfutil.get_cell_boundary_variables(ds) - return self._boundary_vars[ds] + return self._boundary_vars[ds_str] def _find_ancillary_vars(self, ds, refresh=False): """ @@ -541,26 +543,26 @@ def _find_ancillary_vars(self, ds, refresh=False): :return: List of variable names (str) that are defined as ancillary variables in the dataset ds. """ - + ds_str = ds.__str__() # Used the cached version if it exists and is not empty if self._ancillary_vars.get(ds, None) and refresh is False: - return self._ancillary_vars[ds] + return self._ancillary_vars[ds_str] # Invalidate the cache at all costs - self._ancillary_vars[ds] = [] + self._ancillary_vars[ds_str] = [] for var in ds.variables.values(): if hasattr(var, "ancillary_variables"): for anc_name in var.ancillary_variables.split(" "): if anc_name in ds.variables: - self._ancillary_vars[ds].append(anc_name) + self._ancillary_vars[ds_str].append(anc_name) if hasattr(var, "grid_mapping"): gm_name = var.grid_mapping if gm_name in ds.variables: - self._ancillary_vars[ds].append(gm_name) + self._ancillary_vars[ds_str].append(gm_name) - return self._ancillary_vars[ds] + return self._ancillary_vars[ds_str] def _find_clim_vars(self, ds, refresh=False): """ @@ -573,15 +575,15 @@ def _find_clim_vars(self, ds, refresh=False): :return: A list containing strings with geophysical variable names. """ - + ds_str = ds.__str__() if self._clim_vars.get(ds, None) and refresh is False: - return self._clim_vars[ds] + return self._clim_vars[ds_str] climatology_variable = cfutil.get_climatology_variable(ds) if climatology_variable: - self._clim_vars[ds].append(climatology_variable) + self._clim_vars[ds_str].append(climatology_variable) - return self._clim_vars[ds] + return self._clim_vars[ds_str] def _find_cf_standard_name_table(self, ds): """ @@ -693,12 +695,13 @@ def _find_coord_vars(self, ds, refresh=False): :return: A list of variables names (str) that are defined as coordinate variables in the dataset ds. """ - if ds in self._coord_vars and refresh is False: - return self._coord_vars[ds] + ds_str = ds.__str__() + if ds_str in self._coord_vars and refresh is False: + return self._coord_vars[ds_str] - self._coord_vars[ds] = cfutil.get_coordinate_variables(ds) + self._coord_vars[ds_str] = cfutil.get_coordinate_variables(ds) - return self._coord_vars[ds] + return self._coord_vars[ds_str] def _find_geophysical_vars(self, ds, refresh=False): """ @@ -712,12 +715,13 @@ def _find_geophysical_vars(self, ds, refresh=False): :return: A list containing strings with geophysical variable names. """ + ds_str = ds.__str__() if self._geophysical_vars.get(ds, None) and refresh is False: - return self._geophysical_vars[ds] + return self._geophysical_vars[ds_str] - self._geophysical_vars[ds] = cfutil.get_geophysical_variables(ds) + self._geophysical_vars[ds_str] = cfutil.get_geophysical_variables(ds) - return self._geophysical_vars[ds] + return self._geophysical_vars[ds_str] def _find_metadata_vars(self, ds, refresh=False): """ @@ -731,10 +735,11 @@ def _find_metadata_vars(self, ds, refresh=False): variable candidates. """ + ds_str = ds.__str__() if self._metadata_vars.get(ds, None) and refresh is False: - return self._metadata_vars[ds] + return self._metadata_vars[ds_str] - self._metadata_vars[ds] = [] + self._metadata_vars[ds_str] = [] for name, var in ds.variables.items(): if name in self._find_ancillary_vars(ds) or name in self._find_coord_vars( ds, @@ -749,17 +754,17 @@ def _find_metadata_vars(self, ds, refresh=False): "platform_id", "surface_altitude", ): - self._metadata_vars[ds].append(name) + self._metadata_vars[ds_str].append(name) elif getattr(var, "cf_role", "") != "": - self._metadata_vars[ds].append(name) + self._metadata_vars[ds_str].append(name) elif ( getattr(var, "standard_name", None) is None and len(var.dimensions) == 0 ): - self._metadata_vars[ds].append(name) + self._metadata_vars[ds_str].append(name) - return self._metadata_vars[ds] + return self._metadata_vars[ds_str] def _get_coord_axis_map(self, ds): """ diff --git a/compliance_checker/cf/util.py b/compliance_checker/cf/util.py index 6f100653..591a7b3e 100644 --- a/compliance_checker/cf/util.py +++ b/compliance_checker/cf/util.py @@ -199,9 +199,9 @@ def __init__(self, entrynode): def _get(self, entrynode, attrname, required=False): vals = entrynode.xpath(attrname) if len(vals) > 1: - raise Exception("Multiple attrs (%s) found" % attrname) + raise Exception(f"Multiple attrs ({attrname}) found") elif required and len(vals) == 0: - raise Exception("Required attr (%s) not found" % attrname) + raise Exception(f"Required attr ({attrname}) not found") return vals[0].text @@ -236,7 +236,7 @@ def __len__(self): def __getitem__(self, key): if not (key in self._names or key in self._aliases): - raise KeyError("%s not found in standard name table" % key) + raise KeyError(f"{key} not found in standard name table") if key in self._aliases: idx = self._aliases.index(key) @@ -244,14 +244,13 @@ def __getitem__(self, key): if len(entryids) != 1: raise Exception( - "Inconsistency in standard name table, could not lookup alias for %s" - % key, + f"Inconsistency in standard name table, could not lookup alias for {key}", ) key = entryids[0].text if key not in self._names: - raise KeyError("%s not found in standard name table" % key) + raise KeyError(f"{key} not found in standard name table") idx = self._names.index(key) entry = self.NameEntry(self._root.xpath("entry")[idx]) diff --git a/compliance_checker/cfutil.py b/compliance_checker/cfutil.py index 245c6fd3..8bcce331 100644 --- a/compliance_checker/cfutil.py +++ b/compliance_checker/cfutil.py @@ -137,28 +137,28 @@ def get_sea_names(): return _SEA_NAMES -def is_unitless(ds, variable): +def is_unitless(nc, variable): """ Returns true if the variable is unitless Note units of '1' are considered whole numbers or parts but still represent physical units and not the absence of units. - :param netCDF4.Dataset ds: An open netCDF dataset + :param netCDF4.Dataset nc: An open netCDF dataset :param str variable: Name of the variable """ - units = getattr(ds.variables[variable], "units", None) + units = getattr(nc.variables[variable], "units", None) return units is None or units == "" -def is_geophysical(ds, variable): +def is_geophysical(nc, variable): """ Returns true if the dataset's variable is likely a geophysical variable - :param netCDF4.Dataset ds: An open netCDF dataset + :param netCDF4.Dataset nc: An open netCDF dataset :param str variable: Name of the variable """ - ncvar = ds.variables[variable] + ncvar = nc.variables[variable] if getattr(ncvar, "cf_role", None): return False @@ -168,7 +168,7 @@ def is_geophysical(ds, variable): return False standard_name_test = getattr(ncvar, "standard_name", "") - unitless = is_unitless(ds, variable) + unitless = is_unitless(nc, variable) if not isinstance(standard_name_test, str): warnings.warn( @@ -196,13 +196,13 @@ def is_geophysical(ds, variable): }: return False - if variable in get_coordinate_variables(ds): + if variable in get_coordinate_variables(nc): return False - if variable in get_auxiliary_coordinate_variables(ds): + if variable in get_auxiliary_coordinate_variables(nc): return False - if variable in get_forecast_metadata_variables(ds): + if variable in get_forecast_metadata_variables(nc): return False # Is it dimensionless and unitless? @@ -214,10 +214,10 @@ def is_geophysical(ds, variable): return False # Is it a ยง7.1 Cell Boundaries variable - if variable in get_cell_boundary_variables(ds): + if variable in get_cell_boundary_variables(nc): return False - if variable == get_climatology_variable(ds): + if variable == get_climatology_variable(nc): return False # Is it a string but with no defined units? @@ -227,11 +227,11 @@ def is_geophysical(ds, variable): return False # Is it an instrument descriptor? - if variable in get_instrument_variables(ds): + if variable in get_instrument_variables(nc): return False # What about a platform descriptor? - if variable in get_platform_variables(ds): + if variable in get_platform_variables(nc): return False # Skip count/index variables too @@ -241,8 +241,7 @@ def is_geophysical(ds, variable): return True -@lru_cache(128) -def get_coordinate_variables(ds): +def get_coordinate_variables(nc): """ Returns a list of variable names that identify as coordinate variables. @@ -256,17 +255,17 @@ def get_coordinate_variables(ds): ordered monotonically. Missing values are not allowed in coordinate variables. - :param netCDF4.Dataset ds: An open netCDF dataset + :param netCDF4.Dataset nc: An open netCDF dataset """ coord_vars = [] - for dimension in ds.dimensions: - if dimension in ds.variables: - if ds.variables[dimension].dimensions == (dimension,): + for dimension in nc.dimensions: + if dimension in nc.variables: + if nc.variables[dimension].dimensions == (dimension,): coord_vars.append(dimension) return coord_vars -def get_auxiliary_coordinate_variables(ds): +def get_auxiliary_coordinate_variables(nc): """ Returns a list of auxiliary coordinate variables @@ -274,11 +273,11 @@ def get_auxiliary_coordinate_variables(ds): coordinate data, but is not a coordinate variable (in the sense of the term defined by CF). - :param netCDf4.Dataset ds: An open netCDF dataset + :param netCDf4.Dataset nc: An open netCDF dataset """ aux_vars = [] # get any variables referenced by the coordinates attribute - for ncvar in ds.get_variables_by_attributes( + for ncvar in nc.get_variables_by_attributes( coordinates=lambda x: isinstance(x, str), ): # split the coordinates into individual variable names @@ -286,13 +285,13 @@ def get_auxiliary_coordinate_variables(ds): # if the variable names exist, add them for referenced_variable in referenced_variables: if ( - referenced_variable in ds.variables + referenced_variable in nc.variables and referenced_variable not in aux_vars ): aux_vars.append(referenced_variable) # axis variables are automatically in - for variable in get_axis_variables(ds): + for variable in get_axis_variables(nc): if variable not in aux_vars: aux_vars.append(variable) @@ -308,7 +307,7 @@ def get_auxiliary_coordinate_variables(ds): coordinate_standard_names += DIMENSIONLESS_VERTICAL_COORDINATES # Some datasets like ROMS use multiple variables to define coordinates - for ncvar in ds.get_variables_by_attributes( + for ncvar in nc.get_variables_by_attributes( standard_name=lambda x: x in coordinate_standard_names, ): if ncvar.name not in aux_vars: @@ -317,19 +316,19 @@ def get_auxiliary_coordinate_variables(ds): # Remove any that are purely coordinate variables ret_val = [] for aux_var in aux_vars: - if ds.variables[aux_var].dimensions == (aux_var,): + if nc.variables[aux_var].dimensions == (aux_var,): continue ret_val.append(aux_var) return ret_val -def get_forecast_metadata_variables(ds): +def get_forecast_metadata_variables(nc): """ Returns a list of variables that represent forecast reference time metadata. - :param netCDF4.Dataset ds: An open netCDF4 Dataset. + :param netCDF4.Dataset nc: An open netCDF4 Dataset. :rtype: list """ forecast_metadata_standard_names = { @@ -337,14 +336,14 @@ def get_forecast_metadata_variables(ds): "forecast_reference_time", } forecast_metadata_variables = [] - for varname in ds.variables: - standard_name = getattr(ds.variables[varname], "standard_name", None) + for varname in nc.variables: + standard_name = getattr(nc.variables[varname], "standard_name", None) if standard_name in forecast_metadata_standard_names: forecast_metadata_variables.append(varname) return forecast_metadata_variables -def get_cell_boundary_map(ds): +def get_cell_boundary_map(nc): """ Returns a dictionary mapping a variable to its boundary variable. The returned dictionary maps a string variable name to the name of the boundary @@ -353,13 +352,13 @@ def get_cell_boundary_map(ds): :param netCDF4.Dataset nc: netCDF dataset """ boundary_map = {} - for variable in ds.get_variables_by_attributes(bounds=lambda x: x is not None): - if variable.bounds in ds.variables: + for variable in nc.get_variables_by_attributes(bounds=lambda x: x is not None): + if variable.bounds in nc.variables: boundary_map[variable.name] = variable.bounds return boundary_map -def get_cell_boundary_variables(ds): +def get_cell_boundary_variables(nc): """ Returns a list of variable names for variables that represent cell boundaries through the `bounds` attribute @@ -367,21 +366,19 @@ def get_cell_boundary_variables(ds): :param netCDF4.Dataset nc: netCDF dataset """ boundary_variables = [] - has_bounds = ds.get_variables_by_attributes(bounds=lambda x: x is not None) + has_bounds = nc.get_variables_by_attributes(bounds=lambda x: x is not None) for var in has_bounds: - if var.bounds in ds.variables: + if var.bounds in nc.variables: boundary_variables.append(var.bounds) return boundary_variables -@lru_cache(128) -def get_bounds_variables(ds): - contains_bounds = ds.get_variables_by_attributes(bounds=lambda s: s in ds.variables) - return {ds.variables[parent_var.bounds] for parent_var in contains_bounds} +def get_bounds_variables(nc): + contains_bounds = nc.get_variables_by_attributes(bounds=lambda s: s in nc.variables) + return {nc.variables[parent_var.bounds] for parent_var in contains_bounds} -@lru_cache(128) -def get_geophysical_variables(ds): +def get_geophysical_variables(nc): """ Returns a list of variable names for the variables detected as geophysical variables. @@ -389,13 +386,12 @@ def get_geophysical_variables(ds): :param netCDF4.Dataset nc: An open netCDF dataset """ parameters = [] - for variable in ds.variables: - if is_geophysical(ds, variable) and variable not in get_bounds_variables(ds): + for variable in nc.variables: + if is_geophysical(nc, variable) and variable not in get_bounds_variables(nc): parameters.append(variable) return parameters -@lru_cache(128) def get_z_variable(nc): """ Returns the name of the variable that defines the Z axis or height/depth @@ -472,7 +468,6 @@ def get_z_variables(nc): return z_variables -@lru_cache(128) def get_lat_variable(nc): """ Returns the first variable matching latitude @@ -539,7 +534,6 @@ def get_true_latitude_variables(nc): return true_lats -@lru_cache(128) def get_lon_variable(nc): """ Returns the variable for longitude @@ -606,57 +600,57 @@ def get_true_longitude_variables(nc): return true_lons -def get_platform_variables(ds): +def get_platform_variables(nc): """ Returns a list of platform variable NAMES - :param netCDF4.Dataset ds: An open netCDF4 Dataset + :param netCDF4.Dataset nc: An open netCDF4 Dataset """ candidates = [] - for variable in ds.variables: - platform = getattr(ds.variables[variable], "platform", "") - if platform and platform in ds.variables: + for variable in nc.variables: + platform = getattr(nc.variables[variable], "platform", "") + if platform and platform in nc.variables: if platform not in candidates: candidates.append(platform) - platform = getattr(ds, "platform", "") - if platform and platform in ds.variables: + platform = getattr(nc, "platform", "") + if platform and platform in nc.variables: if platform not in candidates: candidates.append(platform) return candidates -def get_instrument_variables(ds): +def get_instrument_variables(nc): """ Returns a list of instrument variables - :param netCDF4.Dataset ds: An open netCDF4 Dataset + :param netCDF4.Dataset nc: An open netCDF4 Dataset """ candidates = [] - for variable in ds.variables: - instrument = getattr(ds.variables[variable], "instrument", "") - if instrument and instrument in ds.variables: + for variable in nc.variables: + instrument = getattr(nc.variables[variable], "instrument", "") + if instrument and instrument in nc.variables: if instrument not in candidates: candidates.append(instrument) - instrument = getattr(ds, "instrument", "") - if instrument and instrument in ds.variables: + instrument = getattr(nc, "instrument", "") + if instrument and instrument in nc.variables: if instrument not in candidates: candidates.append(instrument) return candidates -def get_time_variable(ds): +def get_time_variable(nc): """ Returns the likeliest variable to be the time coordinate variable - :param netCDF4.Dataset ds: An open netCDF4 Dataset + :param netCDF4.Dataset nc: An open netCDF4 Dataset """ - for var in ds.variables: - if getattr(ds.variables[var], "axis", "") == "T": + for var in nc.variables: + if getattr(nc.variables[var], "axis", "") == "T": return var else: - candidates = ds.get_variables_by_attributes(standard_name="time") + candidates = nc.get_variables_by_attributes(standard_name="time") if len(candidates) == 1: return candidates[0].name else: # Look for a coordinate variable time @@ -665,78 +659,76 @@ def get_time_variable(ds): return candidate.name # If we still haven't found the candidate - time_variables = set(get_time_variables(ds)) - coordinate_variables = set(get_coordinate_variables(ds)) + time_variables = set(get_time_variables(nc)) + coordinate_variables = set(get_coordinate_variables(nc)) if len(time_variables.intersection(coordinate_variables)) == 1: return list(time_variables.intersection(coordinate_variables))[0] - auxiliary_coordinates = set(get_auxiliary_coordinate_variables(ds)) + auxiliary_coordinates = set(get_auxiliary_coordinate_variables(nc)) if len(time_variables.intersection(auxiliary_coordinates)) == 1: return list(time_variables.intersection(auxiliary_coordinates))[0] return None -@lru_cache(128) -def get_time_variables(ds): +def get_time_variables(nc): """ Returns a list of variables describing the time coordinate - :param netCDF4.Dataset ds: An open netCDF4 Dataset + :param netCDF4.Dataset nc: An open netCDF4 Dataset """ time_variables = set() - for variable in ds.get_variables_by_attributes(standard_name="time"): + for variable in nc.get_variables_by_attributes(standard_name="time"): time_variables.add(variable.name) - for variable in ds.get_variables_by_attributes(axis="T"): + for variable in nc.get_variables_by_attributes(axis="T"): if variable.name not in time_variables: time_variables.add(variable.name) regx = r"^(?:day|d|hour|hr|h|minute|min|second|s)s? since .*$" - for variable in ds.get_variables_by_attributes(units=lambda x: isinstance(x, str)): + for variable in nc.get_variables_by_attributes(units=lambda x: isinstance(x, str)): if re.match(regx, variable.units) and variable.name not in time_variables: time_variables.add(variable.name) return time_variables -def get_axis_variables(ds): +def get_axis_variables(nc): """ Returns a list of variables that define an axis of the dataset - :param netCDF4.Dataset ds: An open netCDF4 Dataset + :param netCDF4.Dataset nc: An open netCDF4 Dataset """ axis_variables = [] - for ncvar in ds.get_variables_by_attributes(axis=lambda x: x is not None): + for ncvar in nc.get_variables_by_attributes(axis=lambda x: x is not None): axis_variables.append(ncvar.name) return axis_variables -def get_climatology_variable(ds): +def get_climatology_variable(nc): """ Returns the variable describing climatology bounds if it exists. Climatology variables are similar to cell boundary variables that describe - the climatology bounds. + the climatology bounnc. See Example 7.8 in CF 1.6 - :param netCDF4.Dataset ds: An open netCDF4 Dataset + :param netCDF4.Dataset nc: An open netCDF4 Dataset :rtype: str or None """ - time = get_time_variable(ds) + time = get_time_variable(nc) # If there's no time dimension there's no climatology bounds if not time: return None # Climatology variable is simply whatever time points to under the # `climatology` attribute. - if hasattr(ds.variables[time], "climatology"): - if ds.variables[time].climatology in ds.variables: - return ds.variables[time].climatology + if hasattr(nc.variables[time], "climatology"): + if nc.variables[time].climatology in nc.variables: + return nc.variables[time].climatology return None -@lru_cache(128) -def _find_standard_name_modifier_variables(ds, return_deprecated=False): +def _find_standard_name_modifier_variables(nc, return_deprecated=False): def match_modifier_variables(standard_name_str): if standard_name_str is None: return False @@ -751,20 +743,20 @@ def match_modifier_variables(standard_name_str): return [ var.name - for var in ds.get_variables_by_attributes( + for var in nc.get_variables_by_attributes( standard_name=match_modifier_variables, ) ] -def get_flag_variables(ds): +def get_flag_variables(nc): """ Returns a list of variables that are defined as flag variables - :param netCDF4.Dataset ds: An open netCDF4 Dataset + :param netCDF4.Dataset nc: An open netCDF4 Dataset """ flag_variables = [] - for name, ncvar in ds.variables.items(): + for name, ncvar in nc.variables.items(): standard_name = getattr(ncvar, "standard_name", None) if isinstance(standard_name, str) and "status_flag" in standard_name: flag_variables.append(name) @@ -773,21 +765,20 @@ def get_flag_variables(ds): return flag_variables -def get_grid_mapping_variables(ds): +def get_grid_mapping_variables(nc): """ Returns a list of grid mapping variables - :param netCDF4.Dataset ds: An open netCDF4 Dataset + :param netCDF4.Dataset nc: An open netCDF4 Dataset """ grid_mapping_variables = set() - for ncvar in ds.get_variables_by_attributes(grid_mapping=lambda x: x is not None): - if ncvar.grid_mapping in ds.variables: + for ncvar in nc.get_variables_by_attributes(grid_mapping=lambda x: x is not None): + if ncvar.grid_mapping in nc.variables: grid_mapping_variables.add(ncvar.grid_mapping) return grid_mapping_variables -@lru_cache(128) -def get_axis_map(ds, variable): +def get_axis_map(nc, variable): """ Returns an axis_map dictionary that contains an axis key and the coordinate names as values. @@ -803,14 +794,14 @@ def get_axis_map(ds, variable): :param netCDF4.Dataset nc: An open netCDF dataset :param str variable: Variable name """ - all_coords = get_coordinate_variables(ds) + get_auxiliary_coordinate_variables(ds) + all_coords = get_coordinate_variables(nc) + get_auxiliary_coordinate_variables(nc) - latitudes = get_latitude_variables(ds) - longitudes = get_longitude_variables(ds) - times = get_time_variables(ds) - heights = get_z_variables(ds) + latitudes = get_latitude_variables(nc) + longitudes = get_longitude_variables(nc) + times = get_time_variables(nc) + heights = get_z_variables(nc) - coordinates = getattr(ds.variables[variable], "coordinates", None) + coordinates = getattr(nc.variables[variable], "coordinates", None) if not isinstance(coordinates, str): coordinates = [] else: @@ -820,9 +811,9 @@ def get_axis_map(ds, variable): # {'x': ['longitude'], 'y': ['latitude'], 't': ['time']} axis_map = defaultdict(list) for coord_name in all_coords: - axis = getattr(ds.variables[coord_name], "axis", None) + axis = getattr(nc.variables[coord_name], "axis", None) if not axis or axis not in ("X", "Y", "Z", "T"): - if is_compression_coordinate(ds, coord_name): + if is_compression_coordinate(nc, coord_name): axis = "C" elif coord_name in times: axis = "T" @@ -835,7 +826,7 @@ def get_axis_map(ds, variable): else: axis = "U" - if coord_name in ds.variables[variable].dimensions: + if coord_name in nc.variables[variable].dimensions: if coord_name not in axis_map[axis]: axis_map[axis].append(coord_name) @@ -845,19 +836,19 @@ def get_axis_map(ds, variable): return axis_map -def is_coordinate_variable(ds, variable): +def is_coordinate_variable(nc, variable): """ Returns True if the variable is a coordinate variable :param netCDF4.Dataset nc: An open netCDF dataset :param str variable: Variable name """ - if variable not in ds.variables: + if variable not in nc.variables: return False - return ds.variables[variable].dimensions == (variable,) + return nc.variables[variable].dimensions == (variable,) -def is_compression_coordinate(ds, variable): +def is_compression_coordinate(nc, variable): """ Returns True if the variable is a coordinate variable that defines a compression scheme. @@ -866,10 +857,10 @@ def is_compression_coordinate(ds, variable): :param str variable: Variable name """ # Must be a coordinate variable - if not is_coordinate_variable(ds, variable): + if not is_coordinate_variable(nc, variable): return False # must have a string attribute compress - compress = getattr(ds.variables[variable], "compress", None) + compress = getattr(nc.variables[variable], "compress", None) if not isinstance(compress, str): return False if not compress: @@ -879,7 +870,7 @@ def is_compression_coordinate(ds, variable): return False # Must point to dimensions for dim in compress.split(): - if dim not in ds.dimensions: + if dim not in nc.dimensions: return False return True diff --git a/compliance_checker/runner.py b/compliance_checker/runner.py index 0a8c6d2f..536888f5 100644 --- a/compliance_checker/runner.py +++ b/compliance_checker/runner.py @@ -125,7 +125,7 @@ def run_checker( cls.json_output(cs, score_dict, output_filename, ds_loc, limit, out_fmt) else: - raise TypeError("Invalid format %s" % out_fmt) + raise TypeError(f"Invalid format {out_fmt}") errors_occurred = cls.check_errors(score_groups, verbose) @@ -259,8 +259,7 @@ def check_errors(cls, score_groups, verbose): if len(errors): errors_occurred = True print( - "WARNING: The following exceptions occurred during the %s checker (possibly indicate compliance checker issues):" - % checker, + f"WARNING: The following exceptions occurred during the {checker} checker (possibly indicate compliance checker issues):", file=sys.stderr, ) for check_name, epair in errors.items(): diff --git a/compliance_checker/tests/conftest.py b/compliance_checker/tests/conftest.py index 2c662c16..d78fcbb1 100644 --- a/compliance_checker/tests/conftest.py +++ b/compliance_checker/tests/conftest.py @@ -91,7 +91,7 @@ def new_nc_file(tmpdir): """ nc_file_path = os.path.join(tmpdir, "example.nc") if os.path.exists(nc_file_path): - raise OSError("File Exists: %s" % nc_file_path) + raise OSError(f"File Exists: {nc_file_path}") nc = Dataset(nc_file_path, "w") # no need for cleanup, built-in tmpdir fixture will handle it return nc @@ -101,7 +101,7 @@ def new_nc_file(tmpdir): def tmp_txt_file(tmpdir): file_path = os.path.join(tmpdir, "output.txt") if os.path.exists(file_path): - raise OSError("File Exists: %s" % file_path) + raise OSError(f"File Exists: {file_path}") return file_path diff --git a/compliance_checker/tests/helpers.py b/compliance_checker/tests/helpers.py index 3642e123..a07c1aa3 100644 --- a/compliance_checker/tests/helpers.py +++ b/compliance_checker/tests/helpers.py @@ -1,6 +1,6 @@ import tempfile -from netCDF4._netCDF4 import Dataset +from netCDF4 import Dataset class MockNetCDF(Dataset): @@ -23,14 +23,6 @@ def __init__(self, filename=None): persist=False, ) - # suppress usual dealloc routine to prevent caught exception messages - # from printing - def __dealloc__(self): - try: - super().__dealloc__() - except AttributeError: - pass - class MockTimeSeries(MockNetCDF): """ diff --git a/compliance_checker/tests/test_base.py b/compliance_checker/tests/test_base.py index 18cc6c7b..08401127 100644 --- a/compliance_checker/tests/test_base.py +++ b/compliance_checker/tests/test_base.py @@ -65,7 +65,7 @@ def test_attr_in_valid_choices(self): priority, (1, 2), "test", - ["test present, but not in expected value list (%s)" % valid_choices], + [f"test present, but not in expected value list ({valid_choices})"], ) self.ds.test = "a" base.attr_check(attr, self.ds, priority, rv3) diff --git a/compliance_checker/tests/test_cf.py b/compliance_checker/tests/test_cf.py index 292bfd82..5e13bb7b 100644 --- a/compliance_checker/tests/test_cf.py +++ b/compliance_checker/tests/test_cf.py @@ -86,7 +86,7 @@ def new_nc_file(self): """ nc_file_path = os.path.join(gettempdir(), "example.nc") if os.path.exists(nc_file_path): - raise OSError("File Exists: %s" % nc_file_path) + raise OSError(f"File Exists: {nc_file_path}") nc = Dataset(nc_file_path, "w") self.addCleanup(os.remove, nc_file_path) self.addCleanup(nc.close) diff --git a/pyproject.toml b/pyproject.toml index a78aad24..65bc57f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ build-backend = "setuptools.build_meta" requires = [ "setuptools>=42", - "setuptools_scm", + "setuptools-scm", "wheel", ] @@ -10,15 +10,15 @@ requires = [ name = "compliance-checker" description = "Checks Datasets and SOS endpoints for standards compliance" readme = "README.md" -license = {text = "Apache-2.0"} +license = { text = "Apache-2.0" } maintainers = [ - {name = "Dave Foster", email = "dave@axiomdatascience.com"}, - {name = "Benjamin Adams"}, - {name = "Luke Campbell"}, - {name = "Filipe Fernandes"}, + { name = "Dave Foster", email = "dave@axiomdatascience.com" }, + { name = "Benjamin Adams" }, + { name = "Luke Campbell" }, + { name = "Filipe Fernandes" }, ] requires-python = ">=3.8" -classifiers=[ +classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Science/Research", @@ -39,29 +39,32 @@ dynamic = [ "dependencies", "version", ] -[project.urls] -documentation = "https://ioos.github.io/compliance-checker" -homepage = "https://compliance.ioos.us/index.html" -repository = "https://github.com/ioos/compliance-checker" -[project.scripts] -compliance-checker = "cchecker:main" -[project.entry-points."compliance_checker.suites"] -"acdd-1.1" = "compliance_checker.acdd:ACDD1_1Check" -"acdd-1.3" = "compliance_checker.acdd:ACDD1_3Check" -"cf-1.6" = "compliance_checker.cf.cf:CF1_6Check" -"cf-1.7" = "compliance_checker.cf.cf:CF1_7Check" -"cf-1.8" = "compliance_checker.cf.cf:CF1_8Check" -"ioos-0.1" = "compliance_checker.ioos:IOOS0_1Check" -"ioos-1.1" = "compliance_checker.ioos:IOOS1_1Check" -"ioos-1.2" = "compliance_checker.ioos:IOOS1_2Check" -"ioos_sos" = "compliance_checker.ioos:IOOSBaseSOSCheck" +urls.documentation = "https://ioos.github.io/compliance-checker" +urls.homepage = "https://compliance.ioos.us/index.html" +urls.repository = "https://github.com/ioos/compliance-checker" +scripts.compliance-checker = "cchecker:main" +entry-points."compliance_checker.suites"."acdd-1.1" = "compliance_checker.acdd:ACDD1_1Check" +entry-points."compliance_checker.suites"."acdd-1.3" = "compliance_checker.acdd:ACDD1_3Check" +entry-points."compliance_checker.suites"."cf-1.6" = "compliance_checker.cf.cf:CF1_6Check" +entry-points."compliance_checker.suites"."cf-1.7" = "compliance_checker.cf.cf:CF1_7Check" +entry-points."compliance_checker.suites"."cf-1.8" = "compliance_checker.cf.cf:CF1_8Check" +entry-points."compliance_checker.suites"."ioos-0.1" = "compliance_checker.ioos:IOOS0_1Check" +entry-points."compliance_checker.suites"."ioos-1.1" = "compliance_checker.ioos:IOOS1_1Check" +entry-points."compliance_checker.suites"."ioos-1.2" = "compliance_checker.ioos:IOOS1_2Check" +entry-points."compliance_checker.suites"."ioos_sos" = "compliance_checker.ioos:IOOSBaseSOSCheck" [tool.setuptools] -packages = ["compliance_checker"] -license-files = ["LICENSE"] +packages = [ + "compliance_checker", +] +license-files = [ + "LICENSE", +] zip-safe = false include-package-data = true -script-files = ["cchecker.py"] +script-files = [ + "cchecker.py", +] [tool.setuptools.package-data] compliance_checker = [ @@ -73,8 +76,10 @@ compliance_checker = [ ] [tool.setuptools.dynamic] -dependencies = {file = ["requirements.txt"]} -readme = {file = "README.md", content-type = "text/markdown"} +dependencies = { file = [ + "requirements.txt", +] } +readme = { file = "README.md", content-type = "text/markdown" } [tool.setuptools_scm] write_to = "compliance_checker/_version.py" @@ -83,14 +88,14 @@ tag_regex = "^(?Pv)?(?P[^\\+]+)(?P.*)?$" [tool.ruff] lint.select = [ - "A", # flake8-builtins - "B", # flake8-bugbear - "C4", # flake8-comprehensions - "E", # pycodecstyle - "F", # flakes - "I", # import sorting - "W", # pydocstyle - "UP", # upgrade + "A", # flake8-builtins + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "E", # pycodecstyle + "F", # flakes + "I", # import sorting + "W", # pydocstyle + "UP", # upgrade ] target-version = "py38" line-length = 200 @@ -108,16 +113,20 @@ lint.ignore = [ "E402", "A001", ] -"compliance_checker/cfutil.py" = ["B028"] -"compliance_checker/cf/appendix_f.py" = ["B033"] # ignore duplicates items in the set +"compliance_checker/cfutil.py" = [ + "B028", +] +"compliance_checker/cf/appendix_f.py" = [ + "B033", +] # ignore duplicates items in the set [tool.pytest.ini_options] markers = [ - "integration: marks integration tests (deselect with '-m \"not integration\"')", - "slowtest: marks slow tests (deselect with '-m \"not slowtest\"')" + "integration: marks integration tests (deselect with '-m \"not integration\"')", + "slowtest: marks slow tests (deselect with '-m \"not slowtest\"')", ] filterwarnings = [ - "error:::compliance-checker.*", - "ignore::UserWarning", - "ignore::RuntimeWarning", + "error:::compliance-checker.*", + "ignore::UserWarning", + "ignore::RuntimeWarning", ]