From b2c059b0fc4492e047ad139e20d8e0dd08b913f6 Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Wed, 2 Aug 2023 17:36:52 -0600 Subject: [PATCH 01/17] create grid_proj and unit tests --- python/idsse_common/idsse/common/grid_proj.py | 82 ++++++++++++ python/idsse_common/test/test_grid_proj.py | 126 ++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 python/idsse_common/idsse/common/grid_proj.py create mode 100644 python/idsse_common/test/test_grid_proj.py diff --git a/python/idsse_common/idsse/common/grid_proj.py b/python/idsse_common/idsse/common/grid_proj.py new file mode 100644 index 00000000..e15aeb3b --- /dev/null +++ b/python/idsse_common/idsse/common/grid_proj.py @@ -0,0 +1,82 @@ +"""Module that wraps pyproj (cartographic/geodetic) library and transforms objects into other data forms""" +# ------------------------------------------------------------------------------- +# Created on Mon Jul 31 2023 +# +# Copyright (c) 2023 Colorado State University. All rights reserved. (1) +# +# Contributors: +# Mackenzie Grimes (1) +# +# ------------------------------------------------------------------------------- +# pylint: disable=invalid-name + +from typing import Self, Tuple, Any, Optional + +from pyproj import CRS, Transformer +from pyproj.enums import TransformDirection + + +class GridProj: + """Wrapper for pyproj instance with methods to convert to and from geographic coordinates, pixels, etc.""" + def __init__(self, + crs: CRS, + latitude: float, + longitude: float, + width: float, + height: float, + dx: float, + dy: Optional[float] = None): + # pylint: disable=too-many-arguments + self._trans = Transformer.from_crs(crs.geodetic_crs, crs) + self._x_offset, self._y_offset = self._trans.transform(longitude, latitude) # pylint: disable=unpacking-non-sequence + self._w = width + self._h = height + self._dx = dx + self._dy = dy if dy else dx + + @classmethod + def from_proj_grid_spec(cls, proj_string: str, grid_string: str) -> Self: + """ Create GridProj instance from projection grid specs + + Args: + proj_string (str): pyproj projection string + grid_string (str): Grid string + + Returns: + Self: New instance of GridProj which can then be converted to any format + """ + crs = CRS.from_proj4(proj_string) + grid_args = { + key[1:]: float(value) + for key, value in ( + pair.split('=') for pair in grid_string.split(' ') + ) + } + return GridProj(crs, + grid_args['lat_ll'], grid_args['lon_ll'], + int(grid_args['w']), int(grid_args['h']), + grid_args['dx'], grid_args['dy']) + + def map_proj_to_pixel(self, x, y) -> Tuple[float, float]: + """Map projection to a pixel""" + return self.map_geo_to_pixel(*self._trans.transform(x, y)) # pylint: disable=not-an-iterable + + def map_pixel_to_proj(self, x: float, y: float) -> Tuple[Any, Any]: + """Map pixel to a projection""" + return self._trans.transform(*self.map_pixel_to_geo(x, y), direction=TransformDirection.INVERSE) + + def map_proj_to_geo(self, x, y) -> Tuple[float, float]: + """Map projection to geographic coordinates""" + return self._trans.transform(x, y) + + def map_pixel_to_geo(self, x: float, y: float) -> Tuple[float, float]: + """Map pixel to geographic coordinates""" + return x * self._dx + self._x_offset, y * self._dy + self._y_offset + + def map_geo_to_proj(self, x: float, y: float) -> Tuple[Any, Any]: + """Map geographic coordinates to a projection""" + return self._trans.transform(x, y, direction=TransformDirection.INVERSE) + + def map_geo_to_pixel(self, x: float, y: float) -> Tuple[float, float]: + """Map geographic coordinates to a pixel""" + return (x - self._x_offset) / self._dx, (y - self._y_offset) / self._dy diff --git a/python/idsse_common/test/test_grid_proj.py b/python/idsse_common/test/test_grid_proj.py new file mode 100644 index 00000000..2a67e3b7 --- /dev/null +++ b/python/idsse_common/test/test_grid_proj.py @@ -0,0 +1,126 @@ +"""Test suite for grid_proj.py""" +# -------------------------------------------------------------------------------- +# Created on Wed Aug 2 2023 +# +# Copyright (c) 2023 Colorado State University. All rights reserved. (1) +# +# Contributors: +# Mackenzie Grimes (1) +# +# -------------------------------------------------------------------------------- +# pylint: disable=missing-function-docstring,redefined-outer-name,invalid-name,protected-access + +from pytest import fixture, approx + +from idsse.common.grid_proj import GridProj + +# example data +EXAMPLE_PROJ_SPEC = '+proj=lcc +lat_0=25.0 +lon_0=-95.0 +lat_1=25.0 +r=6371200' +PROJ_SPEC_WITH_OFFSET = ('+proj=lcc +lat_0=25.0 +lon_0=-95.0 +lat_1=25.0 +x_0=3271.152832031251 ' + '+y_0=263.7934687500001 +r=6371.2, +units=km') +EXAMPLE_GRID_SPEC = "+dx=2.539703 +dy=2.539703 +w=2345 +h=1597 +lat_ll=19.229 +lon_ll=-126.2766" +EXAMPLE_PROJECTION = (-126.2766, 19.229) +EXAMPLE_PIXEL = (0, 1) +EXAMPLE_GEO = (-3275807.35073, -260554.63043) + + +# fixtures +@fixture +def grid_proj() -> GridProj: + return GridProj.from_proj_grid_spec(EXAMPLE_PROJ_SPEC, EXAMPLE_GRID_SPEC) + + +# test class methods +def test_from_proj_grid_spec(grid_proj: GridProj): + assert isinstance(grid_proj, GridProj) + + assert grid_proj._dx == approx(2.539703) + assert grid_proj._dy == grid_proj._dx + assert grid_proj._x_offset == approx(-3275807.3507) + assert grid_proj._y_offset == approx(-260554.6304) + assert grid_proj._h == 1597 + assert grid_proj._w == 2345 + + transformer = grid_proj._trans + assert transformer.source_crs is not None + assert transformer.source_crs.type_name == 'Geographic 2D CRS' + assert transformer.target_crs is not None + assert transformer.target_crs.type_name == 'Projected CRS' + + +def test_from_proj_grid_spec_with_offset(): + proj_with_offset = GridProj.from_proj_grid_spec(PROJ_SPEC_WITH_OFFSET, EXAMPLE_GRID_SPEC) + assert proj_with_offset._x_offset == approx(-3272536.1979) + assert proj_with_offset._y_offset == approx(-260290.8369) + + +# transformation methods testing +def test_map_proj_to_pixel(grid_proj: GridProj): + pixel_x, pixel_y = grid_proj.map_proj_to_pixel(*EXAMPLE_PROJECTION) + assert (pixel_x, pixel_y) == (0, 0) + + pixel_x, pixel_y = grid_proj.map_proj_to_pixel(-126.2666, 19.2333) + assert (pixel_x, pixel_y) == (approx(448.1133842), approx(88.24123)) + + +def test_map_proj_to_geo(grid_proj: GridProj): + geo_x, geo_y = grid_proj.map_proj_to_geo(*EXAMPLE_PROJECTION) + assert (geo_x, geo_y) == (approx(EXAMPLE_GEO[0]), approx(EXAMPLE_GEO[1])) + + geo_x, geo_y = grid_proj.map_proj_to_geo(-126.2666, 19.2333) + assert (geo_x, geo_y) == (approx(-3274669.2758), approx(-260330.52389)) + + +def test_map_pixel_to_geo(grid_proj: GridProj): + geo_x, geo_y = grid_proj.map_pixel_to_geo(*EXAMPLE_PIXEL) + assert (geo_x, geo_y) == (approx(-3275807.35073), approx(-260552.090732)) + + geo_x, geo_y = grid_proj.map_pixel_to_geo(448.1133842, 8824124) + assert (geo_x, geo_y) == (approx(-3274669.2758), approx(22150099.56473)) + + +def test_map_pixel_to_proj(grid_proj: GridProj): + proj_x, proj_y = grid_proj.map_pixel_to_proj(0, 0) + assert (proj_x, proj_y) == (approx(EXAMPLE_PROJECTION[0]), approx(EXAMPLE_PROJECTION[1])) + + proj_x, proj_y = grid_proj.map_pixel_to_proj(2000, 1500) + assert (proj_x, proj_y) == (approx(-126.238035), approx(19.272773)) + + +def test_map_geo_to_proj(grid_proj: GridProj): + proj_x, proj_y = grid_proj.map_geo_to_proj(*EXAMPLE_GEO) + assert (proj_x, proj_y) == (approx(EXAMPLE_PROJECTION[0]), approx(EXAMPLE_PROJECTION[1])) + + proj_x, proj_y = grid_proj.map_geo_to_proj(-3274669.2758, 22150099.56473) + assert (proj_x, proj_y) == (approx(-110.868173), approx(62.982259)) + + +def test_geo_to_pixel(grid_proj: GridProj): + pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*EXAMPLE_GEO) + assert (round(pixel_x, 5), round(pixel_y, 5)) == (0, 0) + + pixel_x, pixel_y = grid_proj.map_geo_to_pixel(-3274669.2758, 22150099.56473) + assert (pixel_x, pixel_y) == (approx(448.1133948), approx(8824124)) + + +def test_transformations_between_all_formats(grid_proj: GridProj): + # start with pixel, convert to projection + initial_pixel = (2000, 1500) + proj_x, proj_y = grid_proj.map_pixel_to_proj(*initial_pixel) + initial_proj = (proj_x, proj_y) + + # convert projection to geographic coordinates + geo_x, geo_y = grid_proj.map_proj_to_geo(proj_x, proj_y) + initial_geo = geo_x, geo_y + + # convert geographic coordinates back to pixel, full circle, and data should be unchanged + pixel_x, pixel_y = grid_proj.map_geo_to_pixel(geo_x, geo_y) + assert (pixel_x, pixel_y) == (approx(initial_pixel[0]), approx(initial_pixel[1])) + + # convert pixel back to geographic coordinates + geo_x, geo_y = grid_proj.map_pixel_to_geo(pixel_x, pixel_y) + assert (geo_x, geo_y) == (approx(initial_geo[0]), approx(initial_geo[1])) + + # convert geographic coordinates back to projection + proj_x, proj_y = grid_proj.map_geo_to_proj(geo_x, geo_y) + assert (proj_x, proj_y) == (approx(initial_proj[0]), approx(initial_proj[1])) From 6fdc41cf0acbb3ce09495a4efaf0f98c1e332fa9 Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Wed, 2 Aug 2023 17:45:35 -0600 Subject: [PATCH 02/17] cleanup flake8 warnings --- python/idsse_common/idsse/common/aws_utils.py | 7 ++- python/idsse_common/idsse/common/grid_proj.py | 4 +- .../idsse_common/idsse/common/path_builder.py | 4 +- .../idsse/common/publish_confirm.py | 2 +- python/idsse_common/test/test_config.py | 5 +- python/idsse_common/test/test_grid_proj.py | 6 ++- python/idsse_common/test/test_path_builder.py | 49 ++++++++++++++----- 7 files changed, 50 insertions(+), 27 deletions(-) diff --git a/python/idsse_common/idsse/common/aws_utils.py b/python/idsse_common/idsse/common/aws_utils.py index aa5fb891..d9bb80d7 100644 --- a/python/idsse_common/idsse/common/aws_utils.py +++ b/python/idsse_common/idsse/common/aws_utils.py @@ -90,7 +90,6 @@ def aws_cp(self, path: str, dest: str) -> bool: finally: pass - def check_for(self, issue: datetime, valid: datetime) -> Optional[Tuple[datetime, str]]: """Checks if an object passed issue/valid exists @@ -99,8 +98,8 @@ def check_for(self, issue: datetime, valid: datetime) -> Optional[Tuple[datetime valid (datetime): The valid date/time used to format the path to the object's location Returns: - Optional[Tuple[datetime, str]]: A tuple of the valid date/time (indicated by object's - location) and location (path) of a object, or None + Optional[Tuple[datetime, str]]: A tuple of the valid date/time (indicated by object's + location) and location (path) of a object, or None if object does not exist """ lead = TimeDelta(valid-issue) @@ -161,7 +160,7 @@ def get_valids(self, Returns: Sequence[Tuple[datetime, str]]: A sequence of tuples with valid date/time (indicated by - object's location) and the object's location (path). + object's location) and the object's location (path). Empty Sequence if no valids found for given time range. """ if valid_start and valid_start == valid_end: diff --git a/python/idsse_common/idsse/common/grid_proj.py b/python/idsse_common/idsse/common/grid_proj.py index e15aeb3b..8a6681c5 100644 --- a/python/idsse_common/idsse/common/grid_proj.py +++ b/python/idsse_common/idsse/common/grid_proj.py @@ -26,9 +26,9 @@ def __init__(self, height: float, dx: float, dy: Optional[float] = None): - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,unpacking-non-sequence self._trans = Transformer.from_crs(crs.geodetic_crs, crs) - self._x_offset, self._y_offset = self._trans.transform(longitude, latitude) # pylint: disable=unpacking-non-sequence + self._x_offset, self._y_offset = self._trans.transform(longitude, latitude) self._w = width self._h = height self._dx = dx diff --git a/python/idsse_common/idsse/common/path_builder.py b/python/idsse_common/idsse/common/path_builder.py index 209ad81c..bb931d2d 100644 --- a/python/idsse_common/idsse/common/path_builder.py +++ b/python/idsse_common/idsse/common/path_builder.py @@ -1,6 +1,6 @@ """Module for building (and parsing) paths that are dependent on issue, valid, and lead. -In this weather forecasting data context, +In this weather forecasting data context, - issue: the datetime when a given forecast was generated (or at least when generation began) - valid: the datetime when a given forecast "expires" or is superceded by newer forecasts - lead: the time duration (often in hours) between issue and valid, a sort of forecast "lifespan" @@ -49,7 +49,7 @@ def from_dir_filename(cls, dir_fmt: str, filename_fmt: str) -> Self: """Construct a PathBuilder object from specified directory and filename formats. Format string args must follow Python format specifications to define expected fields: https://docs.python.org/3/library/string.html#format-specification-mini-language - + For example, filepaths could be expected to contain "issue" and "lead" fields using format: 'blend.{issue.year:04d}{issue.month:02d}{issue.day:02d}/{issue.hour:02d}/{lead.hour:03d}/' diff --git a/python/idsse_common/idsse/common/publish_confirm.py b/python/idsse_common/idsse/common/publish_confirm.py index 160e8eef..bb571d1c 100644 --- a/python/idsse_common/idsse/common/publish_confirm.py +++ b/python/idsse_common/idsse/common/publish_confirm.py @@ -11,7 +11,7 @@ import pika from pika.exchange_type import ExchangeType -from idsse.common.log_util import get_default_log_config, set_corr_id_context_var +from idsse.common.log_util import get_default_log_config logger = logging.getLogger(__name__) diff --git a/python/idsse_common/test/test_config.py b/python/idsse_common/test/test_config.py index 208c05d5..bf6b3b8c 100644 --- a/python/idsse_common/test/test_config.py +++ b/python/idsse_common/test/test_config.py @@ -116,9 +116,8 @@ def __init__(self, config: dict) -> None: self.b_key = None super().__init__(config, '', ignore_missing=True) - config = WithoutKeyConfig( - [{'a_key': 'value for a'}, {'b_key': 'value for b'}]) - + config = WithoutKeyConfig([{'a_key': 'value for a'}, {'b_key': 'value for b'}]) + assert config.a_key == 'value for a' assert config.next.b_key == 'value for b' diff --git a/python/idsse_common/test/test_grid_proj.py b/python/idsse_common/test/test_grid_proj.py index 2a67e3b7..71197eba 100644 --- a/python/idsse_common/test/test_grid_proj.py +++ b/python/idsse_common/test/test_grid_proj.py @@ -16,8 +16,10 @@ # example data EXAMPLE_PROJ_SPEC = '+proj=lcc +lat_0=25.0 +lon_0=-95.0 +lat_1=25.0 +r=6371200' -PROJ_SPEC_WITH_OFFSET = ('+proj=lcc +lat_0=25.0 +lon_0=-95.0 +lat_1=25.0 +x_0=3271.152832031251 ' - '+y_0=263.7934687500001 +r=6371.2, +units=km') +PROJ_SPEC_WITH_OFFSET = ( + '+proj=lcc +lat_0=25.0 +lon_0=-95.0 +lat_1=25.0 +x_0=3271.152832031251 ' + '+y_0=263.7934687500001 +r=6371.2, +units=km' +) EXAMPLE_GRID_SPEC = "+dx=2.539703 +dy=2.539703 +w=2345 +h=1597 +lat_ll=19.229 +lon_ll=-126.2766" EXAMPLE_PROJECTION = (-126.2766, 19.229) EXAMPLE_PIXEL = (0, 1) diff --git a/python/idsse_common/test/test_path_builder.py b/python/idsse_common/test/test_path_builder.py index 2d096277..747c6603 100644 --- a/python/idsse_common/test/test_path_builder.py +++ b/python/idsse_common/test/test_path_builder.py @@ -8,26 +8,26 @@ # Mackenzie Grimes # # -------------------------------------------------------------------------------- +# pylint: disable=missing-function-docstring,invalid-name,redefined-outer-name,protected-access -import pytest # pylint: disable=import-error from datetime import datetime, timedelta -from typing import Union + +import pytest from idsse.common.utils import TimeDelta from idsse.common.path_builder import PathBuilder -# pylint: disable=missing-function-docstring -# pylint: disable=invalid-name -def test_from_dir_filename_creates_valid_pathbuilder(): +def test_from_dir_filename_creates_valid_pathbuilder(): directory = './test_directory' - filename ='some_file.txt' + filename = 'some_file.txt' path_builder = PathBuilder.from_dir_filename(directory, filename) assert isinstance(path_builder, PathBuilder) assert path_builder._basedir == directory assert path_builder._file_ext == '' + def test_from_path_creates_valid_pathbuilder(): base_dir = './test_directory' path_builder = PathBuilder.from_path(f'{base_dir}/some_file.txt') @@ -37,58 +37,71 @@ def test_from_path_creates_valid_pathbuilder(): assert path_builder._file_base == base_dir assert path_builder._file_ext == '' + # properties EXAMPLE_BASE_DIR = './some/directory' EXAMPLE_SUB_DIR = 'another/directory' EXAMPLE_FILE = 'my_file' EXAMPLE_FILE_EXT = '.txt' + @pytest.fixture def local_path_builder() -> PathBuilder: # create example Pa†hBuilder instance using test strings return PathBuilder(EXAMPLE_BASE_DIR, EXAMPLE_SUB_DIR, EXAMPLE_FILE, EXAMPLE_FILE_EXT) + def test_dir_fmt(local_path_builder: PathBuilder): assert local_path_builder.dir_fmt == f'{EXAMPLE_BASE_DIR}/{EXAMPLE_SUB_DIR}' + def test_filename_fmt(local_path_builder: PathBuilder): assert local_path_builder.filename_fmt == f'{EXAMPLE_FILE}{EXAMPLE_FILE_EXT}' + def test_file_ext(local_path_builder: PathBuilder): assert local_path_builder.file_ext == EXAMPLE_FILE_EXT + def test_path_fmt(local_path_builder: PathBuilder): - assert local_path_builder.path_fmt == f'{EXAMPLE_BASE_DIR}/{EXAMPLE_SUB_DIR}/{EXAMPLE_FILE}{EXAMPLE_FILE_EXT}' + assert local_path_builder.path_fmt == f'{EXAMPLE_BASE_DIR}/{EXAMPLE_SUB_DIR}/{EXAMPLE_FILE}{EXAMPLE_FILE_EXT}' + # methods -EXAMPLE_ISSUE = datetime(1970, 10, 3, 12) # a.k.a. issued at -EXAMPLE_VALID = datetime(1970, 10, 3, 14) # a.k.a. valid until -EXAMPLE_LEAD = TimeDelta(EXAMPLE_VALID - EXAMPLE_ISSUE) # a.k.a. duration of time that issue lasts +EXAMPLE_ISSUE = datetime(1970, 10, 3, 12) # a.k.a. issued at +EXAMPLE_VALID = datetime(1970, 10, 3, 14) # a.k.a. valid until +EXAMPLE_LEAD = TimeDelta(EXAMPLE_VALID - EXAMPLE_ISSUE) # a.k.a. duration of time that issue lasts EXAMPLE_FULL_PATH = '~/blend.19701003/12/core/blend.t12z.core.f002.co.grib2.idx' + @pytest.fixture def path_builder() -> PathBuilder: subdirectory_pattern = 'blend.{issue.year:04d}{issue.month:02d}{issue.day:02d}/{issue.hour:02d}/core/' file_base_pattern = 'blend.t{issue.hour:02d}z.core.f{lead.hour:03d}.co' return PathBuilder('~', subdirectory_pattern, file_base_pattern, 'grib2.idx') -def test_build_dir_gets_issue_valid_and_lead(path_builder: PathBuilder): + +def test_build_dir_gets_issue_valid_and_lead(path_builder: PathBuilder): result_dict = path_builder.build_dir(issue=EXAMPLE_ISSUE) assert result_dict == '~/blend.19701003/12/core/' + def test_build_dir_fails_without_issue(path_builder: PathBuilder): result_dict = path_builder.build_dir(issue=None) assert result_dict is None + def test_build_filename(path_builder: PathBuilder): result_filename = path_builder.build_filename(issue=EXAMPLE_ISSUE, lead=EXAMPLE_LEAD) assert result_filename == 'blend.t12z.core.f002.co.grib2.idx' + def test_build_path(path_builder: PathBuilder): result_filepath = path_builder.build_path(issue=EXAMPLE_ISSUE, valid=EXAMPLE_VALID) assert result_filepath == '~/blend.19701003/12/core/blend.t12z.core.f002.co.grib2.idx' + def test_parse_dir(path_builder: PathBuilder): result_dict = path_builder.parse_dir(EXAMPLE_FULL_PATH) @@ -96,27 +109,32 @@ def test_parse_dir(path_builder: PathBuilder): assert result_dict['issue.year'] == 1970 assert result_dict['issue.day'] == 3 + def test_parse_filename(path_builder: PathBuilder): result_dict = path_builder.parse_filename(EXAMPLE_FULL_PATH) assert result_dict.keys() != [] assert result_dict['lead.hour'] == 2 + def test_get_issue(path_builder: PathBuilder): actual_issue: datetime = path_builder.get_issue(EXAMPLE_FULL_PATH) assert actual_issue == EXAMPLE_ISSUE + def test_get_valid_from_issue_and_lead(path_builder: PathBuilder): # verify valid timestamp gets successfully constructed based on issue and lead embedded into path result_valid: datetime = path_builder.get_valid(EXAMPLE_FULL_PATH) assert result_valid is not None assert result_valid == EXAMPLE_VALID + def test_get_valid_returns_none_when_issue_or_lead_failed(path_builder: PathBuilder): path_with_invalid_lead = '~/blend.19701003/12/core/blend.t12z.core.f000.co.grib2.idx' result_valid = path_builder.get_valid(path_with_invalid_lead) assert result_valid is None + # static methods def test_get_issue_from_time_args(path_builder: PathBuilder): parsed_dict = path_builder.parse_path(EXAMPLE_FULL_PATH) @@ -124,10 +142,12 @@ def test_get_issue_from_time_args(path_builder: PathBuilder): assert issue_result == EXAMPLE_ISSUE + def test_get_issue_returns_none_if_args_empty(): issue_result = PathBuilder.get_issue_from_time_args({}) assert issue_result is None + def test_get_valid_from_time_args(): parsed_dict = {} parsed_dict['valid.year'] = 1970 @@ -135,19 +155,22 @@ def test_get_valid_from_time_args(): parsed_dict['valid.day'] = 3 parsed_dict['valid.hour'] = 14 - valid_result= PathBuilder.get_valid_from_time_args(parsed_dict) + valid_result = PathBuilder.get_valid_from_time_args(parsed_dict) assert valid_result == EXAMPLE_VALID + def test_get_valid_returns_none_if_args_empty(): valid_result = PathBuilder.get_valid_from_time_args({}) assert valid_result is None + def test_get_valid_from_time_args_calculates_based_on_lead(path_builder: PathBuilder): parsed_dict = path_builder.parse_path(EXAMPLE_FULL_PATH) result_valid: datetime = PathBuilder.get_valid_from_time_args(parsed_args=parsed_dict) - + assert result_valid == EXAMPLE_VALID + def test_get_lead_from_time_args(path_builder: PathBuilder): parsed_dict = path_builder.parse_path(EXAMPLE_FULL_PATH) lead_result: timedelta = PathBuilder.get_lead_from_time_args(parsed_dict) From 58bbd455d3da29379fc5823cb5408095e7b77364 Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Thu, 3 Aug 2023 12:31:22 -0600 Subject: [PATCH 03/17] cleanup scattered literal numbers in test_grid_proj, round pixels to int --- python/idsse_common/idsse/common/grid_proj.py | 10 +- python/idsse_common/test/test_grid_proj.py | 99 ++++++++++--------- 2 files changed, 59 insertions(+), 50 deletions(-) diff --git a/python/idsse_common/idsse/common/grid_proj.py b/python/idsse_common/idsse/common/grid_proj.py index 8a6681c5..e81ee41e 100644 --- a/python/idsse_common/idsse/common/grid_proj.py +++ b/python/idsse_common/idsse/common/grid_proj.py @@ -77,6 +77,10 @@ def map_geo_to_proj(self, x: float, y: float) -> Tuple[Any, Any]: """Map geographic coordinates to a projection""" return self._trans.transform(x, y, direction=TransformDirection.INVERSE) - def map_geo_to_pixel(self, x: float, y: float) -> Tuple[float, float]: - """Map geographic coordinates to a pixel""" - return (x - self._x_offset) / self._dx, (y - self._y_offset) / self._dy + def map_geo_to_pixel(self, x: float, y: float) -> Tuple[int, int]: + """Map geographic coordinates to a pixel + + Pixels are only identified by whole numbers when graphically rendered, + so the transformed numerical values (floats) can be safely rounded to ints + """ + return round((x - self._x_offset) / self._dx), round((y - self._y_offset) / self._dy) diff --git a/python/idsse_common/test/test_grid_proj.py b/python/idsse_common/test/test_grid_proj.py index 71197eba..888fccee 100644 --- a/python/idsse_common/test/test_grid_proj.py +++ b/python/idsse_common/test/test_grid_proj.py @@ -10,6 +10,8 @@ # -------------------------------------------------------------------------------- # pylint: disable=missing-function-docstring,redefined-outer-name,invalid-name,protected-access +from typing import Tuple + from pytest import fixture, approx from idsse.common.grid_proj import GridProj @@ -21,9 +23,27 @@ '+y_0=263.7934687500001 +r=6371.2, +units=km' ) EXAMPLE_GRID_SPEC = "+dx=2.539703 +dy=2.539703 +w=2345 +h=1597 +lat_ll=19.229 +lon_ll=-126.2766" -EXAMPLE_PROJECTION = (-126.2766, 19.229) -EXAMPLE_PIXEL = (0, 1) -EXAMPLE_GEO = (-3275807.35073, -260554.63043) + +EXAMPLE_PIXELS = [ + (0, 0), + (0, 1), + (2000, 1500) +] +EXAMPLE_PROJECTIONS = [ + (-126.2766, 19.229), + (-126.27660549554, 19.229022224607), + (-126.23803580508, 19.272773488997) +] +EXAMPLE_GEOS = [ + (-3275807.350733, -260554.63043505), + (-3275807.350733, -260552.09073205), + (-3270727.944733, -256745.07593505) +] + + +# utility to roughly compare tuples of floats with less floating point precision +def approx_tuple(values: Tuple[float, float]) -> Tuple: + return (approx(values[0]), approx(values[1])) # fixtures @@ -36,18 +56,15 @@ def grid_proj() -> GridProj: def test_from_proj_grid_spec(grid_proj: GridProj): assert isinstance(grid_proj, GridProj) - assert grid_proj._dx == approx(2.539703) - assert grid_proj._dy == grid_proj._dx - assert grid_proj._x_offset == approx(-3275807.3507) - assert grid_proj._y_offset == approx(-260554.6304) + assert (grid_proj._x_offset, grid_proj._y_offset) == approx_tuple(EXAMPLE_GEOS[0]) assert grid_proj._h == 1597 assert grid_proj._w == 2345 + assert grid_proj._dx == approx(2.539703) + assert grid_proj._dy == grid_proj._dx - transformer = grid_proj._trans - assert transformer.source_crs is not None - assert transformer.source_crs.type_name == 'Geographic 2D CRS' - assert transformer.target_crs is not None - assert transformer.target_crs.type_name == 'Projected CRS' + t = grid_proj._trans + assert t.source_crs is not None and t.source_crs.type_name == 'Geographic 2D CRS' + assert t.target_crs is not None and t.target_crs.type_name == 'Projected CRS' def test_from_proj_grid_spec_with_offset(): @@ -58,56 +75,44 @@ def test_from_proj_grid_spec_with_offset(): # transformation methods testing def test_map_proj_to_pixel(grid_proj: GridProj): - pixel_x, pixel_y = grid_proj.map_proj_to_pixel(*EXAMPLE_PROJECTION) - assert (pixel_x, pixel_y) == (0, 0) - - pixel_x, pixel_y = grid_proj.map_proj_to_pixel(-126.2666, 19.2333) - assert (pixel_x, pixel_y) == (approx(448.1133842), approx(88.24123)) + for index, proj in enumerate(EXAMPLE_PROJECTIONS): + pixel_x, pixel_y = grid_proj.map_proj_to_pixel(*proj) + assert (pixel_x, pixel_y) == EXAMPLE_PIXELS[index] def test_map_proj_to_geo(grid_proj: GridProj): - geo_x, geo_y = grid_proj.map_proj_to_geo(*EXAMPLE_PROJECTION) - assert (geo_x, geo_y) == (approx(EXAMPLE_GEO[0]), approx(EXAMPLE_GEO[1])) - - geo_x, geo_y = grid_proj.map_proj_to_geo(-126.2666, 19.2333) - assert (geo_x, geo_y) == (approx(-3274669.2758), approx(-260330.52389)) + for index, proj in enumerate(EXAMPLE_PROJECTIONS): + geo_x, geo_y = grid_proj.map_proj_to_geo(*proj) + assert (geo_x, geo_y) == approx_tuple(EXAMPLE_GEOS[index]) def test_map_pixel_to_geo(grid_proj: GridProj): - geo_x, geo_y = grid_proj.map_pixel_to_geo(*EXAMPLE_PIXEL) - assert (geo_x, geo_y) == (approx(-3275807.35073), approx(-260552.090732)) - - geo_x, geo_y = grid_proj.map_pixel_to_geo(448.1133842, 8824124) - assert (geo_x, geo_y) == (approx(-3274669.2758), approx(22150099.56473)) + for index, pixel in enumerate(EXAMPLE_PIXELS): + geo_x, geo_y = grid_proj.map_pixel_to_geo(*pixel) + assert (geo_x, geo_y) == approx_tuple(EXAMPLE_GEOS[index]) def test_map_pixel_to_proj(grid_proj: GridProj): - proj_x, proj_y = grid_proj.map_pixel_to_proj(0, 0) - assert (proj_x, proj_y) == (approx(EXAMPLE_PROJECTION[0]), approx(EXAMPLE_PROJECTION[1])) - - proj_x, proj_y = grid_proj.map_pixel_to_proj(2000, 1500) - assert (proj_x, proj_y) == (approx(-126.238035), approx(19.272773)) + for index, pixel in enumerate(EXAMPLE_PIXELS): + proj_x, proj_y = grid_proj.map_pixel_to_proj(*pixel) + assert (proj_x, proj_y) == approx_tuple(EXAMPLE_PROJECTIONS[index]) def test_map_geo_to_proj(grid_proj: GridProj): - proj_x, proj_y = grid_proj.map_geo_to_proj(*EXAMPLE_GEO) - assert (proj_x, proj_y) == (approx(EXAMPLE_PROJECTION[0]), approx(EXAMPLE_PROJECTION[1])) - - proj_x, proj_y = grid_proj.map_geo_to_proj(-3274669.2758, 22150099.56473) - assert (proj_x, proj_y) == (approx(-110.868173), approx(62.982259)) + for index, geo in enumerate(EXAMPLE_GEOS): + proj_x, proj_y = grid_proj.map_geo_to_proj(*geo) + assert (proj_x, proj_y) == approx_tuple(EXAMPLE_PROJECTIONS[index]) def test_geo_to_pixel(grid_proj: GridProj): - pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*EXAMPLE_GEO) - assert (round(pixel_x, 5), round(pixel_y, 5)) == (0, 0) - - pixel_x, pixel_y = grid_proj.map_geo_to_pixel(-3274669.2758, 22150099.56473) - assert (pixel_x, pixel_y) == (approx(448.1133948), approx(8824124)) + for index, geo in enumerate(EXAMPLE_GEOS): + pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*geo) + assert (pixel_x, pixel_y) == EXAMPLE_PIXELS[index] -def test_transformations_between_all_formats(grid_proj: GridProj): +def test_compound_tranformations_stay_consistent(grid_proj: GridProj): # start with pixel, convert to projection - initial_pixel = (2000, 1500) + initial_pixel = (EXAMPLE_PIXELS[2][0], EXAMPLE_PIXELS[2][1]) proj_x, proj_y = grid_proj.map_pixel_to_proj(*initial_pixel) initial_proj = (proj_x, proj_y) @@ -117,12 +122,12 @@ def test_transformations_between_all_formats(grid_proj: GridProj): # convert geographic coordinates back to pixel, full circle, and data should be unchanged pixel_x, pixel_y = grid_proj.map_geo_to_pixel(geo_x, geo_y) - assert (pixel_x, pixel_y) == (approx(initial_pixel[0]), approx(initial_pixel[1])) + assert (pixel_x, pixel_y) == initial_pixel # convert pixel back to geographic coordinates geo_x, geo_y = grid_proj.map_pixel_to_geo(pixel_x, pixel_y) - assert (geo_x, geo_y) == (approx(initial_geo[0]), approx(initial_geo[1])) + assert (geo_x, geo_y) == approx_tuple(initial_geo) # convert geographic coordinates back to projection proj_x, proj_y = grid_proj.map_geo_to_proj(geo_x, geo_y) - assert (proj_x, proj_y) == (approx(initial_proj[0]), approx(initial_proj[1])) + assert (proj_x, proj_y) == approx_tuple(initial_proj) From 5d216eb77648e214e119ca0e15127137a28c83ab Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Thu, 3 Aug 2023 16:02:41 -0600 Subject: [PATCH 04/17] add optional rounding param to map_geo_to_pixel which controls rounding (none by default) --- python/idsse_common/idsse/common/grid_proj.py | 40 ++++++++++++++++--- python/idsse_common/test/test_grid_proj.py | 21 ++++++++-- 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/python/idsse_common/idsse/common/grid_proj.py b/python/idsse_common/idsse/common/grid_proj.py index e81ee41e..0c1e2fba 100644 --- a/python/idsse_common/idsse/common/grid_proj.py +++ b/python/idsse_common/idsse/common/grid_proj.py @@ -10,12 +10,20 @@ # ------------------------------------------------------------------------------- # pylint: disable=invalid-name -from typing import Self, Tuple, Any, Optional +from typing import Self, Tuple, Union, Any, Optional +from enum import Enum, auto +from math import floor from pyproj import CRS, Transformer from pyproj.enums import TransformDirection +class PixelRounding(Enum): + """Transformations to apply to calculated pixels when casting to ints""" + ROUND = auto() + FLOOR = auto() + + class GridProj: """Wrapper for pyproj instance with methods to convert to and from geographic coordinates, pixels, etc.""" def __init__(self, @@ -77,10 +85,32 @@ def map_geo_to_proj(self, x: float, y: float) -> Tuple[Any, Any]: """Map geographic coordinates to a projection""" return self._trans.transform(x, y, direction=TransformDirection.INVERSE) - def map_geo_to_pixel(self, x: float, y: float) -> Tuple[int, int]: + def map_geo_to_pixel( + self, x: float, + y: float, + rounding: Optional[PixelRounding] = None + ) -> Tuple[Union[int, float], Union[int, float]]: """Map geographic coordinates to a pixel - - Pixels are only identified by whole numbers when graphically rendered, + + Pixels are only identified by whole numbers when graphically rendered, so the transformed numerical values (floats) can be safely rounded to ints + + Args: + x (float): x geographic coordinate + y (float): y geographic coordinate + rounding (Optional[PixelRounding]): + ROUND to apply round() to pixel values, FLOOR to apply math.floor(). + By default, pixels are not rounded and will be returned as floats + + Returns: + Tuple[Union[int, float], Union[int, float]): + x, y values for pixel, rounded to ints if rounding parameter passed, otherwise floats """ - return round((x - self._x_offset) / self._dx), round((y - self._y_offset) / self._dy) + pixel_x = (x - self._x_offset) / self._dx + pixel_y = (y - self._y_offset) / self._dy + + if rounding is PixelRounding.ROUND: + return (round(pixel_x), round(pixel_y)) + if rounding is PixelRounding.FLOOR: + return (floor(pixel_x), floor(pixel_y)) + return (pixel_x, pixel_y) diff --git a/python/idsse_common/test/test_grid_proj.py b/python/idsse_common/test/test_grid_proj.py index 888fccee..46e74ec6 100644 --- a/python/idsse_common/test/test_grid_proj.py +++ b/python/idsse_common/test/test_grid_proj.py @@ -14,7 +14,7 @@ from pytest import fixture, approx -from idsse.common.grid_proj import GridProj +from idsse.common.grid_proj import GridProj, PixelRounding # example data EXAMPLE_PROJ_SPEC = '+proj=lcc +lat_0=25.0 +lon_0=-95.0 +lat_1=25.0 +r=6371200' @@ -77,7 +77,7 @@ def test_from_proj_grid_spec_with_offset(): def test_map_proj_to_pixel(grid_proj: GridProj): for index, proj in enumerate(EXAMPLE_PROJECTIONS): pixel_x, pixel_y = grid_proj.map_proj_to_pixel(*proj) - assert (pixel_x, pixel_y) == EXAMPLE_PIXELS[index] + assert (round(pixel_x, 6), round(pixel_y, 6)) == EXAMPLE_PIXELS[index] def test_map_proj_to_geo(grid_proj: GridProj): @@ -104,9 +104,22 @@ def test_map_geo_to_proj(grid_proj: GridProj): assert (proj_x, proj_y) == approx_tuple(EXAMPLE_PROJECTIONS[index]) -def test_geo_to_pixel(grid_proj: GridProj): +def test_geo_to_pixel_no_rounding(grid_proj: GridProj): for index, geo in enumerate(EXAMPLE_GEOS): pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*geo) + # round result, which will not be precisely the integer that was passed + assert (round(pixel_x, 6), round(pixel_y, 6)) == EXAMPLE_PIXELS[index] + + +def test_geo_to_pixel_floor(grid_proj: GridProj): + for index, geo in enumerate(EXAMPLE_GEOS): + pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*geo, PixelRounding.FLOOR) + assert (pixel_x, pixel_y) == EXAMPLE_PIXELS[index] + + +def test_geo_to_pixel_round(grid_proj: GridProj): + for index, geo in enumerate(EXAMPLE_GEOS): + pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*geo, PixelRounding.ROUND) assert (pixel_x, pixel_y) == EXAMPLE_PIXELS[index] @@ -122,7 +135,7 @@ def test_compound_tranformations_stay_consistent(grid_proj: GridProj): # convert geographic coordinates back to pixel, full circle, and data should be unchanged pixel_x, pixel_y = grid_proj.map_geo_to_pixel(geo_x, geo_y) - assert (pixel_x, pixel_y) == initial_pixel + assert (round(pixel_x, 6), round(pixel_y, 6)) == initial_pixel # convert pixel back to geographic coordinates geo_x, geo_y = grid_proj.map_pixel_to_geo(pixel_x, pixel_y) From c60d13b0288b9fa6461eacc7160a8eb1cb428c6f Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Thu, 3 Aug 2023 16:04:55 -0600 Subject: [PATCH 05/17] fix docstring in map_geo_to_pixel --- python/idsse_common/idsse/common/grid_proj.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/idsse_common/idsse/common/grid_proj.py b/python/idsse_common/idsse/common/grid_proj.py index 0c1e2fba..2d8c570b 100644 --- a/python/idsse_common/idsse/common/grid_proj.py +++ b/python/idsse_common/idsse/common/grid_proj.py @@ -90,10 +90,7 @@ def map_geo_to_pixel( y: float, rounding: Optional[PixelRounding] = None ) -> Tuple[Union[int, float], Union[int, float]]: - """Map geographic coordinates to a pixel - - Pixels are only identified by whole numbers when graphically rendered, - so the transformed numerical values (floats) can be safely rounded to ints + """Map geographic coordinates to pixel x and y Args: x (float): x geographic coordinate From 89d76119d207faf4168bdbc71b528c8d0b7013db Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Fri, 11 Aug 2023 14:39:33 -0600 Subject: [PATCH 06/17] add round_half_up() utility function, unit tests --- python/idsse_common/idsse/common/utils.py | 18 +++++++++++++++- python/idsse_common/test/test_utils.py | 25 +++++++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/python/idsse_common/idsse/common/utils.py b/python/idsse_common/idsse/common/utils.py index 56c66b8c..11f425e8 100644 --- a/python/idsse_common/idsse/common/utils.py +++ b/python/idsse_common/idsse/common/utils.py @@ -13,7 +13,8 @@ import logging from datetime import datetime, timedelta from subprocess import Popen, PIPE, TimeoutExpired -from typing import Sequence, Optional, Generator, Any +from typing import Sequence, Optional, Generator, Union, Any +from decimal import Decimal, ROUND_HALF_UP logger = logging.getLogger(__name__) @@ -166,3 +167,18 @@ def datetime_gen(dt_: datetime, for i in range(0, max_num): logger.debug('dt generator %d/%d', i, max_num) yield dt_ + time_delta * i + +def round_half_up(number: float, precision: int = 0) -> Union[int, float]: + """ + Round a float to a set number of decimal places, using "ties away from zero" method, + in contrast with Python 3's built-in round() or numpy.round() functions, both which + use "ties to even" method. For example, this function will round 2.500000 to 3, not 2. + + Args: + precision (int): number of decimal places to preserve. + + Returns: + Union[int, float]: rounded number as int if precision is 0 decimal places, otherwise as float + """ + rounded_number = Decimal(number).quantize(Decimal(10) ** -precision, rounding=ROUND_HALF_UP) + return int(rounded_number) if precision == 0 else float(rounded_number) diff --git a/python/idsse_common/test/test_utils.py b/python/idsse_common/test/test_utils.py index 5758aa8b..a11cb33c 100644 --- a/python/idsse_common/test/test_utils.py +++ b/python/idsse_common/test/test_utils.py @@ -9,12 +9,12 @@ # # -------------------------------------------------------------------------------- -import pytest # pylint: disable=import-error from datetime import datetime, timedelta +import pytest from idsse.common.utils import TimeDelta -from idsse.common.utils import datetime_gen, hash_code, to_compact, to_iso +from idsse.common.utils import datetime_gen, hash_code, to_compact, to_iso, round_half_up # pylint: disable=missing-function-docstring @@ -85,3 +85,24 @@ def test_datetime_gen_bound(): assert dts_found == [datetime(2021, 1, 2, 3, 0), datetime(2021, 1, 16, 3, 0), datetime(2021, 1, 30, 3, 0)] + + +@pytest.mark.parametrize('number, expected', [(2.50000, 3), (-14.5000, -15), (3.49999, 3)]) +def test_round_half_up_int(number: float, expected: int): + result = round_half_up(number) + assert isinstance(result, int) + assert result == expected + + +@pytest.mark.parametrize('number, expected', [(9.5432, 9.5), (-0.8765, -0.9)]) +def test_round_half_up_float(number: float, expected: float): + result = round_half_up(number, precision=1) + assert isinstance(result, float) + assert result == expected + + +@pytest.mark.parametrize('number, expected', [(100.987654321, 100.988), (-43.21098, -43.211)]) +def test_round_half_up_with_precision(number: float, expected: float): + result = round_half_up(number, precision=3) + assert isinstance(result, float) + assert result == expected From 9a4c6eb3fd519c004e09a01dfd8933c7bfdae5cd Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Fri, 11 Aug 2023 16:19:18 -0600 Subject: [PATCH 07/17] add tests for Map class, exec_cmd, dict_copy_with --- python/idsse_common/idsse/common/utils.py | 3 +- python/idsse_common/test/test_utils.py | 52 ++++++++++++++++++++++- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/python/idsse_common/idsse/common/utils.py b/python/idsse_common/idsse/common/utils.py index 11f425e8..d08664aa 100644 --- a/python/idsse_common/idsse/common/utils.py +++ b/python/idsse_common/idsse/common/utils.py @@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs): super(Map, self).__init__(*args, **kwargs) for arg in args: if isinstance(arg, dict): - for k, v in arg.iteritems(): + for k, v in arg.items(): self[k] = v if kwargs: @@ -168,6 +168,7 @@ def datetime_gen(dt_: datetime, logger.debug('dt generator %d/%d', i, max_num) yield dt_ + time_delta * i + def round_half_up(number: float, precision: int = 0) -> Union[int, float]: """ Round a float to a set number of decimal places, using "ties away from zero" method, diff --git a/python/idsse_common/test/test_utils.py b/python/idsse_common/test/test_utils.py index a11cb33c..da85adb6 100644 --- a/python/idsse_common/test/test_utils.py +++ b/python/idsse_common/test/test_utils.py @@ -9,12 +9,14 @@ # # -------------------------------------------------------------------------------- +from copy import deepcopy from datetime import datetime, timedelta +from os import path import pytest -from idsse.common.utils import TimeDelta -from idsse.common.utils import datetime_gen, hash_code, to_compact, to_iso, round_half_up +from idsse.common.utils import TimeDelta, Map +from idsse.common.utils import datetime_gen, hash_code, exec_cmd, to_compact, to_iso, dict_copy_with, round_half_up # pylint: disable=missing-function-docstring @@ -35,6 +37,32 @@ def test_timedelta_day(): assert td.day == 15 +def test_map_dict_args(): + example_dict = {'value': 123, 'metadata': { 'other_data': [100, 200]}} + example_map = Map(example_dict) + + # pylint: disable=no-member + assert example_map.value == 123 + assert example_map.metadata == {'other_data': [100, 200]} + + +def test_map_dict_kwargs(): + example_map = Map(value=123, metadata={'other_data': [100, 200]}) + + # pylint: disable=no-member + assert example_map.value == 123 + assert example_map.metadata == {'other_data': [100, 200]} + + +def test_exec_cmd(): + current_dir = path.dirname(__file__) + result = exec_cmd(['ls', current_dir]) + + # verify that at least __init__ and this file were found + assert '__init__.py' in result + assert __file__.split(path.sep, maxsplit=-1)[-1] in result + + def test_to_iso(): dt = datetime(2013, 12, 11, 10, 9, 8) assert to_iso(dt) == '2013-12-11T10:09:08Z' @@ -51,6 +79,26 @@ def test_hash_code(string, hash_code_): assert hash_code(string) == hash_code_ +def test_dict_copy_with(): + starting_dict = { 'value': 123, 'units': 'mph'} + copied_dict = deepcopy(starting_dict) + + # run copy + result = dict_copy_with(copied_dict, source='speedometer', metadata={'some': ('other', 'data')}) + + # original dict should be unchanged + assert copied_dict == starting_dict + assert 'source' not in copied_dict + + # starting values should exist in result + assert result['value'] == starting_dict['value'] + assert result['units'] == starting_dict['units'] + + # new values should also have been added to result + assert result['source'] == 'speedometer' + assert result['metadata'] == {'some': ('other', 'data')} + + def test_datetime_gen_forward(): dt_ = datetime(2021, 1, 2, 3) time_delta = timedelta(hours=1) From e6776d93674c9c0cec6e39184cbc1d54da938ba7 Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Fri, 11 Aug 2023 16:51:10 -0600 Subject: [PATCH 08/17] add unit test for Map __setattr__ --- python/idsse_common/test/test_utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/python/idsse_common/test/test_utils.py b/python/idsse_common/test/test_utils.py index da85adb6..cc911071 100644 --- a/python/idsse_common/test/test_utils.py +++ b/python/idsse_common/test/test_utils.py @@ -37,7 +37,7 @@ def test_timedelta_day(): assert td.day == 15 -def test_map_dict_args(): +def test_map_dict_init_with_args(): example_dict = {'value': 123, 'metadata': { 'other_data': [100, 200]}} example_map = Map(example_dict) @@ -46,7 +46,7 @@ def test_map_dict_args(): assert example_map.metadata == {'other_data': [100, 200]} -def test_map_dict_kwargs(): +def test_map_dict_init_with_kwargs(): example_map = Map(value=123, metadata={'other_data': [100, 200]}) # pylint: disable=no-member @@ -54,6 +54,13 @@ def test_map_dict_kwargs(): assert example_map.metadata == {'other_data': [100, 200]} +def test_map_dict_set_value(): + example_map = Map(value=123, metadata={'other_data': [100, 200]}) + example_map.value = 321 # change value + # pylint: disable=no-member + assert example_map.value == 321 + + def test_exec_cmd(): current_dir = path.dirname(__file__) result = exec_cmd(['ls', current_dir]) From e7c7be47c9282ddc490f90cd5c16fbae53d1a5a2 Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Fri, 11 Aug 2023 17:24:13 -0600 Subject: [PATCH 09/17] add _round_pixel_maybe to grid_proj to standardize pixel rounding --- python/idsse_common/idsse/common/grid_proj.py | 79 +++++++++++++------ 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/python/idsse_common/idsse/common/grid_proj.py b/python/idsse_common/idsse/common/grid_proj.py index 2d8c570b..f2615b24 100644 --- a/python/idsse_common/idsse/common/grid_proj.py +++ b/python/idsse_common/idsse/common/grid_proj.py @@ -11,17 +11,23 @@ # pylint: disable=invalid-name from typing import Self, Tuple, Union, Any, Optional -from enum import Enum, auto +from enum import Enum +from decimal import ROUND_HALF_UP, ROUND_FLOOR from math import floor from pyproj import CRS, Transformer from pyproj.enums import TransformDirection +from utils import round_half_up -class PixelRounding(Enum): - """Transformations to apply to calculated pixels when casting to ints""" - ROUND = auto() - FLOOR = auto() + +Pixel = Tuple[Union[int, float], Union[int, float]] + + +class RoundingMethod(Enum): + """Transformations to apply to calculated pixel values when casting to ints""" + ROUND_HALF_UP = ROUND_HALF_UP + ROUND_FLOOR = ROUND_FLOOR class GridProj: @@ -65,9 +71,40 @@ def from_proj_grid_spec(cls, proj_string: str, grid_string: str) -> Self: int(grid_args['w']), int(grid_args['h']), grid_args['dx'], grid_args['dy']) - def map_proj_to_pixel(self, x, y) -> Tuple[float, float]: - """Map projection to a pixel""" - return self.map_geo_to_pixel(*self._trans.transform(x, y)) # pylint: disable=not-an-iterable + + def _round_pixel_maybe(self, pixel: Tuple[float, float], rounding: Optional[Union[str, RoundingMethod]]) -> Pixel: + """Round pixel values if caller requested, or return unchanged if no rounding passed""" + x, y = pixel + # cast str to RoundingMethod enum + if isinstance(rounding, str): + rounding = RoundingMethod[rounding] + + if rounding is RoundingMethod.ROUND_HALF_UP: + return (round_half_up(x), round_half_up(y)) + if rounding is RoundingMethod.ROUND_FLOOR: + return (floor(x), floor(y)) + return x, y + + def map_proj_to_pixel( + self, + x: float, + y: float, + rounding: Optional[Union[str, RoundingMethod]] = None + ) -> Pixel: + """Map projection to a pixel. + + Args: + x (float): x geographic coordinate + y (float): y geographic coordinate + rounding (Optional[Union[str, RoundingMethod]]): + ROUND_HALF_UP to apply round_half_up() to pixel values, ROUND_FLOOR to apply math.floor(). + By default, pixels are not rounded and will be returned as floats + + Returns: + Pixel: x, y values of pixel, rounded to ints if rounding parameter passed, otherwise floats + """ + i, j = self.map_geo_to_pixel(*self._trans.transform(x, y)) # pylint: disable=not-an-iterable + return self._round_pixel_maybe((i, j), rounding) def map_pixel_to_proj(self, x: float, y: float) -> Tuple[Any, Any]: """Map pixel to a projection""" @@ -86,28 +123,24 @@ def map_geo_to_proj(self, x: float, y: float) -> Tuple[Any, Any]: return self._trans.transform(x, y, direction=TransformDirection.INVERSE) def map_geo_to_pixel( - self, x: float, + self, + x: float, y: float, - rounding: Optional[PixelRounding] = None - ) -> Tuple[Union[int, float], Union[int, float]]: + rounding: Optional[Union[str, RoundingMethod]] = None + ) -> Pixel: """Map geographic coordinates to pixel x and y Args: x (float): x geographic coordinate y (float): y geographic coordinate - rounding (Optional[PixelRounding]): - ROUND to apply round() to pixel values, FLOOR to apply math.floor(). + rounding (Optional[Union[str, RoundingMethod]]): + ROUND_HALF_UP to apply round_half_up() to pixel values, ROUND_FLOOR to apply math.floor(). + Can provide the RoundingMethod enum value or str value. By default, pixels are not rounded and will be returned as floats Returns: - Tuple[Union[int, float], Union[int, float]): - x, y values for pixel, rounded to ints if rounding parameter passed, otherwise floats + Pixel: x, y values of pixel, rounded to ints if rounding parameter passed, otherwise floats """ - pixel_x = (x - self._x_offset) / self._dx - pixel_y = (y - self._y_offset) / self._dy - - if rounding is PixelRounding.ROUND: - return (round(pixel_x), round(pixel_y)) - if rounding is PixelRounding.FLOOR: - return (floor(pixel_x), floor(pixel_y)) - return (pixel_x, pixel_y) + i: float = (x - self._x_offset) / self._dx + j: float = (y - self._y_offset) / self._dy + return self._round_pixel_maybe((i, j), rounding) From 957886141b68a6611f81aa6a5430d848df70ba7a Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Fri, 11 Aug 2023 17:26:24 -0600 Subject: [PATCH 10/17] fix grid_proj unit tests --- python/idsse_common/idsse/common/grid_proj.py | 2 +- python/idsse_common/test/test_grid_proj.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/idsse_common/idsse/common/grid_proj.py b/python/idsse_common/idsse/common/grid_proj.py index f2615b24..8b912595 100644 --- a/python/idsse_common/idsse/common/grid_proj.py +++ b/python/idsse_common/idsse/common/grid_proj.py @@ -18,7 +18,7 @@ from pyproj import CRS, Transformer from pyproj.enums import TransformDirection -from utils import round_half_up +from .utils import round_half_up Pixel = Tuple[Union[int, float], Union[int, float]] diff --git a/python/idsse_common/test/test_grid_proj.py b/python/idsse_common/test/test_grid_proj.py index 46e74ec6..bd716d65 100644 --- a/python/idsse_common/test/test_grid_proj.py +++ b/python/idsse_common/test/test_grid_proj.py @@ -14,7 +14,7 @@ from pytest import fixture, approx -from idsse.common.grid_proj import GridProj, PixelRounding +from idsse.common.grid_proj import GridProj, RoundingMethod # example data EXAMPLE_PROJ_SPEC = '+proj=lcc +lat_0=25.0 +lon_0=-95.0 +lat_1=25.0 +r=6371200' @@ -113,13 +113,13 @@ def test_geo_to_pixel_no_rounding(grid_proj: GridProj): def test_geo_to_pixel_floor(grid_proj: GridProj): for index, geo in enumerate(EXAMPLE_GEOS): - pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*geo, PixelRounding.FLOOR) + pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*geo, RoundingMethod.ROUND_FLOOR) assert (pixel_x, pixel_y) == EXAMPLE_PIXELS[index] def test_geo_to_pixel_round(grid_proj: GridProj): for index, geo in enumerate(EXAMPLE_GEOS): - pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*geo, PixelRounding.ROUND) + pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*geo, RoundingMethod.ROUND_HALF_UP) assert (pixel_x, pixel_y) == EXAMPLE_PIXELS[index] From 8aa16854b34eda6a7207f2db5b591c0f7a32045a Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Tue, 15 Aug 2023 14:27:00 -0600 Subject: [PATCH 11/17] round_half_away() now uses only math ceil/floor, no Decimal --- python/idsse_common/idsse/common/utils.py | 29 +++++++++++++++++++---- python/idsse_common/test/test_utils.py | 17 ++++++------- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/python/idsse_common/idsse/common/utils.py b/python/idsse_common/idsse/common/utils.py index d08664aa..91a7a353 100644 --- a/python/idsse_common/idsse/common/utils.py +++ b/python/idsse_common/idsse/common/utils.py @@ -11,10 +11,10 @@ import copy import logging +import math from datetime import datetime, timedelta from subprocess import Popen, PIPE, TimeoutExpired from typing import Sequence, Optional, Generator, Union, Any -from decimal import Decimal, ROUND_HALF_UP logger = logging.getLogger(__name__) @@ -169,11 +169,25 @@ def datetime_gen(dt_: datetime, yield dt_ + time_delta * i -def round_half_up(number: float, precision: int = 0) -> Union[int, float]: +def _round_away_from_zero(number: float) -> int: + func = math.floor if number < 0 else math.ceil + return func(number) + +def _round_toward_zero(number: float) -> int: + func = math.ceil if number < 0 else math.floor + return func(number) + + +def round_half_away(number: float, precision: int = 0) -> Union[int, float]: """ Round a float to a set number of decimal places, using "ties away from zero" method, in contrast with Python 3's built-in round() or numpy.round() functions, both which - use "ties to even" method. For example, this function will round 2.500000 to 3, not 2. + use "ties to even" method. + + | Input | round() | round_half_away() | + | ----- | ------- | ----------------- | + | 2.5 | 2 | 3 | + | -14.5 | -14 | -15 | Args: precision (int): number of decimal places to preserve. @@ -181,5 +195,12 @@ def round_half_up(number: float, precision: int = 0) -> Union[int, float]: Returns: Union[int, float]: rounded number as int if precision is 0 decimal places, otherwise as float """ - rounded_number = Decimal(number).quantize(Decimal(10) ** -precision, rounding=ROUND_HALF_UP) + factor = 10 ** precision + factored_number = number * factor + is_less_than_half = abs(factored_number - math.trunc(factored_number)) < 0.5 + + rounded_number = ( + _round_toward_zero(factored_number) if is_less_than_half + else _round_away_from_zero(factored_number) + ) / factor return int(rounded_number) if precision == 0 else float(rounded_number) diff --git a/python/idsse_common/test/test_utils.py b/python/idsse_common/test/test_utils.py index cc911071..3de140eb 100644 --- a/python/idsse_common/test/test_utils.py +++ b/python/idsse_common/test/test_utils.py @@ -12,11 +12,12 @@ from copy import deepcopy from datetime import datetime, timedelta from os import path +from math import pi import pytest from idsse.common.utils import TimeDelta, Map -from idsse.common.utils import datetime_gen, hash_code, exec_cmd, to_compact, to_iso, dict_copy_with, round_half_up +from idsse.common.utils import datetime_gen, hash_code, exec_cmd, to_compact, to_iso, dict_copy_with, round_half_away # pylint: disable=missing-function-docstring @@ -143,21 +144,21 @@ def test_datetime_gen_bound(): @pytest.mark.parametrize('number, expected', [(2.50000, 3), (-14.5000, -15), (3.49999, 3)]) -def test_round_half_up_int(number: float, expected: int): - result = round_half_up(number) +def test_round_half_away_int(number: float, expected: int): + result = round_half_away(number) assert isinstance(result, int) assert result == expected @pytest.mark.parametrize('number, expected', [(9.5432, 9.5), (-0.8765, -0.9)]) -def test_round_half_up_float(number: float, expected: float): - result = round_half_up(number, precision=1) +def test_round_half_away_float(number: float, expected: float): + result = round_half_away(number, precision=1) assert isinstance(result, float) assert result == expected -@pytest.mark.parametrize('number, expected', [(100.987654321, 100.988), (-43.21098, -43.211)]) -def test_round_half_up_with_precision(number: float, expected: float): - result = round_half_up(number, precision=3) +@pytest.mark.parametrize('number, expected', [(100.987654321, 100.988), (-43.21098, -43.211), (pi, 3.142)]) +def test_round_half_away_with_precision(number: float, expected: float): + result = round_half_away(number, precision=3) assert isinstance(result, float) assert result == expected From 252f9c9514c662b633839776e0e1943f90338696 Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Tue, 15 Aug 2023 14:31:52 -0600 Subject: [PATCH 12/17] cleanup pylint warnings that are my fault --- python/idsse_common/idsse/common/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/idsse_common/idsse/common/utils.py b/python/idsse_common/idsse/common/utils.py index 4d84f506..e23e7704 100644 --- a/python/idsse_common/idsse/common/utils.py +++ b/python/idsse_common/idsse/common/utils.py @@ -184,9 +184,9 @@ def _round_toward_zero(number: float) -> int: def round_half_away(number: float, precision: int = 0) -> Union[int, float]: """ Round a float to a set number of decimal places, using "ties away from zero" method, - in contrast with Python 3's built-in round() or numpy.round() functions, both which + in contrast with Python 3's built-in round() or numpy.round() functions, both which use "ties to even" method. - + | Input | round() | round_half_away() | | ----- | ------- | ----------------- | | 2.5 | 2 | 3 | From d60c7cee1d43c6fa657327fc562fcfd3d112889e Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Tue, 15 Aug 2023 16:11:27 -0600 Subject: [PATCH 13/17] fix rounding tests in test_grid_proj, pylitn warnings --- python/idsse_common/idsse/common/grid_proj.py | 73 ++++++++++++------- python/idsse_common/idsse/common/utils.py | 19 +++-- .../idsse/common/validate_schema.py | 1 + python/idsse_common/test/test_grid_proj.py | 42 ++++++++--- python/idsse_common/test/test_utils.py | 2 - 5 files changed, 91 insertions(+), 46 deletions(-) diff --git a/python/idsse_common/idsse/common/grid_proj.py b/python/idsse_common/idsse/common/grid_proj.py index 8b912595..845355de 100644 --- a/python/idsse_common/idsse/common/grid_proj.py +++ b/python/idsse_common/idsse/common/grid_proj.py @@ -1,4 +1,4 @@ -"""Module that wraps pyproj (cartographic/geodetic) library and transforms objects into other data forms""" +"""Module that wraps pyproj (cartographic) library and transforms objects into other data forms""" # ------------------------------------------------------------------------------- # Created on Mon Jul 31 2023 # @@ -12,13 +12,12 @@ from typing import Self, Tuple, Union, Any, Optional from enum import Enum -from decimal import ROUND_HALF_UP, ROUND_FLOOR -from math import floor +import math from pyproj import CRS, Transformer from pyproj.enums import TransformDirection -from .utils import round_half_up +from .utils import round_half_away Pixel = Tuple[Union[int, float], Union[int, float]] @@ -26,12 +25,15 @@ class RoundingMethod(Enum): """Transformations to apply to calculated pixel values when casting to ints""" - ROUND_HALF_UP = ROUND_HALF_UP - ROUND_FLOOR = ROUND_FLOOR + ROUND = 'ROUND' + FLOOR = 'FLOOR' class GridProj: - """Wrapper for pyproj instance with methods to convert to and from geographic coordinates, pixels, etc.""" + """ + Wrapper for pyproj instance with methods to convert + to and from geographic coordinates, pixels, etc. + """ def __init__(self, crs: CRS, latitude: float, @@ -72,43 +74,60 @@ def from_proj_grid_spec(cls, proj_string: str, grid_string: str) -> Self: grid_args['dx'], grid_args['dy']) - def _round_pixel_maybe(self, pixel: Tuple[float, float], rounding: Optional[Union[str, RoundingMethod]]) -> Pixel: + def _round_pixel_maybe( + self, + pixel: Tuple[float, float], + rounding: Optional[Union[str, RoundingMethod]] = None, + precision: int = 0 + ) -> Pixel: """Round pixel values if caller requested, or return unchanged if no rounding passed""" x, y = pixel # cast str to RoundingMethod enum if isinstance(rounding, str): - rounding = RoundingMethod[rounding] + rounding = RoundingMethod[rounding.upper()] - if rounding is RoundingMethod.ROUND_HALF_UP: - return (round_half_up(x), round_half_up(y)) - if rounding is RoundingMethod.ROUND_FLOOR: - return (floor(x), floor(y)) + if rounding is RoundingMethod.ROUND: + return (round_half_away(x, precision), round_half_away(y, precision)) + if rounding is RoundingMethod.FLOOR: + return (math.floor(x), math.floor(y)) return x, y def map_proj_to_pixel( self, x: float, y: float, - rounding: Optional[Union[str, RoundingMethod]] = None + rounding: Optional[Union[str, RoundingMethod]] = None, + precision: int = 0 ) -> Pixel: """Map projection to a pixel. - + Args: x (float): x geographic coordinate y (float): y geographic coordinate rounding (Optional[Union[str, RoundingMethod]]): - ROUND_HALF_UP to apply round_half_up() to pixel values, ROUND_FLOOR to apply math.floor(). + ROUND to apply round_half_away() to pixel values, + FLOOR to apply math.floor(). + Supports RoundingMethod enum value or str value (case insensitive). By default, pixels are not rounded and will be returned as floats + precision (int): number of decimal places to round to. If rounding parameter + is None, this will be ignored Returns: - Pixel: x, y values of pixel, rounded to ints if rounding parameter passed, otherwise floats + Pixel: x, y values of pixel, rounded to ints if rounding is passed, otherwise floats """ - i, j = self.map_geo_to_pixel(*self._trans.transform(x, y)) # pylint: disable=not-an-iterable - return self._round_pixel_maybe((i, j), rounding) + # pylint: disable=not-an-iterable + return self.map_geo_to_pixel( + *self._trans.transform(x, y), + rounding=rounding, + precision=precision + ) def map_pixel_to_proj(self, x: float, y: float) -> Tuple[Any, Any]: """Map pixel to a projection""" - return self._trans.transform(*self.map_pixel_to_geo(x, y), direction=TransformDirection.INVERSE) + return self._trans.transform( + *self.map_pixel_to_geo(x, y), + direction=TransformDirection.INVERSE + ) def map_proj_to_geo(self, x, y) -> Tuple[float, float]: """Map projection to geographic coordinates""" @@ -126,7 +145,8 @@ def map_geo_to_pixel( self, x: float, y: float, - rounding: Optional[Union[str, RoundingMethod]] = None + rounding: Optional[Union[str, RoundingMethod]] = None, + precision: int = 0 ) -> Pixel: """Map geographic coordinates to pixel x and y @@ -134,13 +154,16 @@ def map_geo_to_pixel( x (float): x geographic coordinate y (float): y geographic coordinate rounding (Optional[Union[str, RoundingMethod]]): - ROUND_HALF_UP to apply round_half_up() to pixel values, ROUND_FLOOR to apply math.floor(). - Can provide the RoundingMethod enum value or str value. + ROUND to apply round_half_away() to pixel values, + FLOOR to apply math.floor(). + Supports RoundingMethod enum value or str value (case insensitive). By default, pixels are not rounded and will be returned as floats + precision (int): number of decimal places to round to. If rounding parameter + is None, this will be ignored Returns: - Pixel: x, y values of pixel, rounded to ints if rounding parameter passed, otherwise floats + Pixel: x, y values of pixel, rounded to ints if rounding passed, otherwise floats """ i: float = (x - self._x_offset) / self._dx j: float = (y - self._y_offset) / self._dy - return self._round_pixel_maybe((i, j), rounding) + return self._round_pixel_maybe((i, j), rounding, precision) diff --git a/python/idsse_common/idsse/common/utils.py b/python/idsse_common/idsse/common/utils.py index e23e7704..1ef7e953 100644 --- a/python/idsse_common/idsse/common/utils.py +++ b/python/idsse_common/idsse/common/utils.py @@ -43,15 +43,15 @@ def day(self): class Map(dict): """Wrapper class for python dictionary with dot access""" def __init__(self, *args, **kwargs): - super(Map, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) for arg in args: if isinstance(arg, dict): - for k, v in arg.items(): - self[k] = v + for key, value in arg.items(): + self[key] = value if kwargs: - for k, v in kwargs.items(): - self[k] = v + for key, value in kwargs.items(): + self[key] = value def __getattr__(self, attr): return self.get(attr) @@ -60,14 +60,14 @@ def __setattr__(self, key, value): self.__setitem__(key, value) def __setitem__(self, key, value): - super(Map, self).__setitem__(key, value) + super().__setitem__(key, value) self.__dict__.update({key: value}) def __delattr__(self, item): self.__delitem__(item) def __delitem__(self, key): - super(Map, self).__delitem__(key) + super().__delitem__(key) del self.__dict__[key] @@ -84,8 +84,7 @@ def exec_cmd(commands: Sequence[str], timeout: Optional[int] = None) -> Sequence Sequence[str]: Result of executing the commands """ logger.debug('Making system call %s', commands) - # with Popen(commands, stdout=PIPE, stderr=PIPE) as proc: - # out = proc.readlines() + # pylint: disable=consider-using-with process = Popen(commands, stdout=PIPE, stderr=PIPE) try: outs, errs = process.communicate(timeout=timeout) @@ -196,7 +195,7 @@ def round_half_away(number: float, precision: int = 0) -> Union[int, float]: precision (int): number of decimal places to preserve. Returns: - Union[int, float]: rounded number as int if precision is 0 decimal places, otherwise as float + Union[int, float]: rounded number as int if precision is 0, otherwise as float """ factor = 10 ** precision factored_number = number * factor diff --git a/python/idsse_common/idsse/common/validate_schema.py b/python/idsse_common/idsse/common/validate_schema.py index 92c45916..5a8492e3 100644 --- a/python/idsse_common/idsse/common/validate_schema.py +++ b/python/idsse_common/idsse/common/validate_schema.py @@ -12,6 +12,7 @@ import os from typing import Optional, Union +# pylint: disable=no-name-in-module from jsonschema import Validator, FormatChecker, RefResolver from jsonschema.validators import validator_for diff --git a/python/idsse_common/test/test_grid_proj.py b/python/idsse_common/test/test_grid_proj.py index bd716d65..ac6feeb8 100644 --- a/python/idsse_common/test/test_grid_proj.py +++ b/python/idsse_common/test/test_grid_proj.py @@ -10,11 +10,13 @@ # -------------------------------------------------------------------------------- # pylint: disable=missing-function-docstring,redefined-outer-name,invalid-name,protected-access +import math from typing import Tuple from pytest import fixture, approx from idsse.common.grid_proj import GridProj, RoundingMethod +from idsse.common.utils import round_half_away # example data EXAMPLE_PROJ_SPEC = '+proj=lcc +lat_0=25.0 +lon_0=-95.0 +lat_1=25.0 +r=6371200' @@ -74,10 +76,24 @@ def test_from_proj_grid_spec_with_offset(): # transformation methods testing -def test_map_proj_to_pixel(grid_proj: GridProj): +def test_map_proj_to_pixel_round_half_away(grid_proj: GridProj): for index, proj in enumerate(EXAMPLE_PROJECTIONS): - pixel_x, pixel_y = grid_proj.map_proj_to_pixel(*proj) - assert (round(pixel_x, 6), round(pixel_y, 6)) == EXAMPLE_PIXELS[index] + pixel_x, pixel_y = grid_proj.map_proj_to_pixel( + *proj, + rounding=RoundingMethod.ROUND + ) + assert (round_half_away(pixel_x), round_half_away(pixel_y)) == EXAMPLE_PIXELS[index] + + +def test_map_proj_to_pixel_round_floor(grid_proj: GridProj): + for index, proj in enumerate(EXAMPLE_PROJECTIONS): + i, j = grid_proj.map_proj_to_pixel( + *proj, + rounding=RoundingMethod.FLOOR + ) + # due to math imprecision internal to pyproj.transform(), some test results are a bit + # unpredictable. E.g. returns 0.999994 which is floored to 0, but expected value was 1 + assert (approx(i, abs=1), approx(j, abs=1)) == EXAMPLE_PIXELS[index] def test_map_proj_to_geo(grid_proj: GridProj): @@ -106,21 +122,29 @@ def test_map_geo_to_proj(grid_proj: GridProj): def test_geo_to_pixel_no_rounding(grid_proj: GridProj): for index, geo in enumerate(EXAMPLE_GEOS): - pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*geo) + i, j = grid_proj.map_geo_to_pixel(*geo) # round result, which will not be precisely the integer that was passed - assert (round(pixel_x, 6), round(pixel_y, 6)) == EXAMPLE_PIXELS[index] + assert (round(i, 6), round(j, 6)) == EXAMPLE_PIXELS[index] def test_geo_to_pixel_floor(grid_proj: GridProj): for index, geo in enumerate(EXAMPLE_GEOS): - pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*geo, RoundingMethod.ROUND_FLOOR) - assert (pixel_x, pixel_y) == EXAMPLE_PIXELS[index] + i, j = grid_proj.map_geo_to_pixel(*geo, RoundingMethod.FLOOR) + assert (math.floor(i), math.floor(j)) == EXAMPLE_PIXELS[index] def test_geo_to_pixel_round(grid_proj: GridProj): for index, geo in enumerate(EXAMPLE_GEOS): - pixel_x, pixel_y = grid_proj.map_geo_to_pixel(*geo, RoundingMethod.ROUND_HALF_UP) - assert (pixel_x, pixel_y) == EXAMPLE_PIXELS[index] + i, j = grid_proj.map_geo_to_pixel(*geo, RoundingMethod.ROUND) + assert (i, j) == EXAMPLE_PIXELS[index] + + +def test_geo_to_pixel_round_str(grid_proj: GridProj): + i, j = grid_proj.map_geo_to_pixel(*EXAMPLE_GEOS[0], 'round') + assert (i, j) == EXAMPLE_PIXELS[0] + + i, j = grid_proj.map_geo_to_pixel(*EXAMPLE_GEOS[1], 'ROUND') + assert (i, j) == EXAMPLE_PIXELS[1] def test_compound_tranformations_stay_consistent(grid_proj: GridProj): diff --git a/python/idsse_common/test/test_utils.py b/python/idsse_common/test/test_utils.py index 9f1b10b7..ed5f80f2 100644 --- a/python/idsse_common/test/test_utils.py +++ b/python/idsse_common/test/test_utils.py @@ -15,8 +15,6 @@ from math import pi import pytest -import pytest # pylint: disable=import-error - from idsse.common.utils import TimeDelta, Map from idsse.common.utils import datetime_gen, hash_code, exec_cmd, to_compact, to_iso, dict_copy_with, round_half_away From 9de285e9f527b6706280bb0a71e250afbc8492ae Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Tue, 15 Aug 2023 16:15:00 -0600 Subject: [PATCH 14/17] more specific type hints in grid_prjo --- python/idsse_common/idsse/common/grid_proj.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/idsse_common/idsse/common/grid_proj.py b/python/idsse_common/idsse/common/grid_proj.py index 845355de..d2fba971 100644 --- a/python/idsse_common/idsse/common/grid_proj.py +++ b/python/idsse_common/idsse/common/grid_proj.py @@ -122,7 +122,7 @@ def map_proj_to_pixel( precision=precision ) - def map_pixel_to_proj(self, x: float, y: float) -> Tuple[Any, Any]: + def map_pixel_to_proj(self, x: float, y: float) -> Tuple[float, float]: """Map pixel to a projection""" return self._trans.transform( *self.map_pixel_to_geo(x, y), @@ -137,7 +137,7 @@ def map_pixel_to_geo(self, x: float, y: float) -> Tuple[float, float]: """Map pixel to geographic coordinates""" return x * self._dx + self._x_offset, y * self._dy + self._y_offset - def map_geo_to_proj(self, x: float, y: float) -> Tuple[Any, Any]: + def map_geo_to_proj(self, x: float, y: float) -> Tuple[float, float]: """Map geographic coordinates to a projection""" return self._trans.transform(x, y, direction=TransformDirection.INVERSE) From 63618b0e5a952d2102fa96e6bbcc100553ce82dc Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Thu, 17 Aug 2023 19:42:30 -0600 Subject: [PATCH 15/17] make _round_pixel_maybe a static method --- python/idsse_common/idsse/common/grid_proj.py | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/python/idsse_common/idsse/common/grid_proj.py b/python/idsse_common/idsse/common/grid_proj.py index d2fba971..fb35017f 100644 --- a/python/idsse_common/idsse/common/grid_proj.py +++ b/python/idsse_common/idsse/common/grid_proj.py @@ -73,25 +73,6 @@ def from_proj_grid_spec(cls, proj_string: str, grid_string: str) -> Self: int(grid_args['w']), int(grid_args['h']), grid_args['dx'], grid_args['dy']) - - def _round_pixel_maybe( - self, - pixel: Tuple[float, float], - rounding: Optional[Union[str, RoundingMethod]] = None, - precision: int = 0 - ) -> Pixel: - """Round pixel values if caller requested, or return unchanged if no rounding passed""" - x, y = pixel - # cast str to RoundingMethod enum - if isinstance(rounding, str): - rounding = RoundingMethod[rounding.upper()] - - if rounding is RoundingMethod.ROUND: - return (round_half_away(x, precision), round_half_away(y, precision)) - if rounding is RoundingMethod.FLOOR: - return (math.floor(x), math.floor(y)) - return x, y - def map_proj_to_pixel( self, x: float, @@ -167,3 +148,21 @@ def map_geo_to_pixel( i: float = (x - self._x_offset) / self._dx j: float = (y - self._y_offset) / self._dy return self._round_pixel_maybe((i, j), rounding, precision) + + @staticmethod + def _round_pixel_maybe( + pixel: Tuple[float, float], + rounding: Optional[Union[str, RoundingMethod]] = None, + precision: int = 0 + ) -> Pixel: + """Round pixel values if caller requested, or return unchanged if no rounding passed""" + x, y = pixel + # cast str to RoundingMethod enum + if isinstance(rounding, str): + rounding = RoundingMethod[rounding.upper()] + + if rounding is RoundingMethod.ROUND: + return (round_half_away(x, precision), round_half_away(y, precision)) + if rounding is RoundingMethod.FLOOR: + return (math.floor(x), math.floor(y)) + return x, y From ce4c78d5a05b1d5f708ea4f97cc95a2965ad6243 Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Thu, 17 Aug 2023 19:42:44 -0600 Subject: [PATCH 16/17] remove unused imports --- python/idsse_common/idsse/common/grid_proj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/idsse_common/idsse/common/grid_proj.py b/python/idsse_common/idsse/common/grid_proj.py index fb35017f..c4c68a60 100644 --- a/python/idsse_common/idsse/common/grid_proj.py +++ b/python/idsse_common/idsse/common/grid_proj.py @@ -10,7 +10,7 @@ # ------------------------------------------------------------------------------- # pylint: disable=invalid-name -from typing import Self, Tuple, Union, Any, Optional +from typing import Self, Tuple, Union, Optional from enum import Enum import math From b287da4c54b0f6eea2429ff0d2a4ca7c440c605c Mon Sep 17 00:00:00 2001 From: Mackenzie Grimes - NOAA Affiliate Date: Fri, 18 Aug 2023 11:00:47 -0600 Subject: [PATCH 17/17] fix all test_grid_proj rounding to use correct round function --- python/idsse_common/idsse/common/utils.py | 18 +++++++++--------- python/idsse_common/test/test_grid_proj.py | 6 +++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/python/idsse_common/idsse/common/utils.py b/python/idsse_common/idsse/common/utils.py index 1ef7e953..4df72995 100644 --- a/python/idsse_common/idsse/common/utils.py +++ b/python/idsse_common/idsse/common/utils.py @@ -84,16 +84,16 @@ def exec_cmd(commands: Sequence[str], timeout: Optional[int] = None) -> Sequence Sequence[str]: Result of executing the commands """ logger.debug('Making system call %s', commands) - # pylint: disable=consider-using-with - process = Popen(commands, stdout=PIPE, stderr=PIPE) - try: - outs, errs = process.communicate(timeout=timeout) - except TimeoutExpired: - process.kill() + with Popen(commands, stdout=PIPE, stderr=PIPE) as process: + try: + outs, errs = process.communicate(timeout=timeout) + except TimeoutExpired: + process.kill() outs, errs = process.communicate() - if process.returncode != 0: - # the process was not successful - raise OSError(process.returncode, errs.decode()) + + if process.returncode != 0: + # the process was not successful + raise OSError(process.returncode, errs.decode()) try: ans = outs.decode().splitlines() except Exception as exc: # pylint: disable=broad-exception-caught diff --git a/python/idsse_common/test/test_grid_proj.py b/python/idsse_common/test/test_grid_proj.py index ac6feeb8..5999a56a 100644 --- a/python/idsse_common/test/test_grid_proj.py +++ b/python/idsse_common/test/test_grid_proj.py @@ -92,7 +92,7 @@ def test_map_proj_to_pixel_round_floor(grid_proj: GridProj): rounding=RoundingMethod.FLOOR ) # due to math imprecision internal to pyproj.transform(), some test results are a bit - # unpredictable. E.g. returns 0.999994 which is floored to 0, but expected value was 1 + # unpredictable. E.g. returns 0.999994 which is floored to 0, but expected pixel value is 1 assert (approx(i, abs=1), approx(j, abs=1)) == EXAMPLE_PIXELS[index] @@ -124,7 +124,7 @@ def test_geo_to_pixel_no_rounding(grid_proj: GridProj): for index, geo in enumerate(EXAMPLE_GEOS): i, j = grid_proj.map_geo_to_pixel(*geo) # round result, which will not be precisely the integer that was passed - assert (round(i, 6), round(j, 6)) == EXAMPLE_PIXELS[index] + assert (round_half_away(i, 6), round_half_away(j, 6)) == EXAMPLE_PIXELS[index] def test_geo_to_pixel_floor(grid_proj: GridProj): @@ -159,7 +159,7 @@ def test_compound_tranformations_stay_consistent(grid_proj: GridProj): # convert geographic coordinates back to pixel, full circle, and data should be unchanged pixel_x, pixel_y = grid_proj.map_geo_to_pixel(geo_x, geo_y) - assert (round(pixel_x, 6), round(pixel_y, 6)) == initial_pixel + assert (round_half_away(pixel_x, 6), round_half_away(pixel_y, 6)) == initial_pixel # convert pixel back to geographic coordinates geo_x, geo_y = grid_proj.map_pixel_to_geo(pixel_x, pixel_y)