From da389cd55e6965b256663c43ed858eb442316713 Mon Sep 17 00:00:00 2001 From: Alex Barreto Date: Sat, 8 May 2021 15:37:16 -0400 Subject: [PATCH 1/3] Dockerfile_alpine: fix broken pip six installation (#1568) Fixes docker hub error: ``` Step 32/49 : RUN pip3 install --upgrade pip six grass-session ---> Running in 1c63e0d56789 Collecting pip Downloading pip-21.1.1-py3-none-any.whl (1.5 MB) Collecting six Downloading six-1.16.0-py2.py3-none-any.whl (11 kB) Collecting grass-session Downloading grass_session-0.5-py2.py3-none-any.whl (31 kB) Installing collected packages: pip, six, grass-session Attempting uninstall: pip Found existing installation: pip 20.0.2 Uninstalling pip-20.0.2: Successfully uninstalled pip-20.0.2 Attempting uninstall: six Found existing installation: six 1.15.0 ERROR: Cannot uninstall 'six'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall. ``` --- CONTRIBUTING.md | 21 +++++++++++++++++++++ docker/alpine/Dockerfile_alpine | 4 +++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 90b2fa82d93..c305ab05f1c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,24 @@ +# Introduction + +GRASS GIS is written in more than one programming language. While most +of the source code is written in C, about 30% is written in Python. A +compiler is needed to convert the C/C++ source code into executable +files ("binaries"). In contrast, Python is an interpreted language that +can only be executed with Python software. + +Now, in order to create an installable binary package from a source +code package, the so-called "compilation step" is required. While the +source code consists of thousands of C and Python files (plus HTML +documentation), the included "makefiles" tell the build system to +generate binaries from the source code in the correct order, render the +manual pages, etc. + +The way to install the compiler tools and Python depends on the operating +system. To make this easier, we have collected copy-paste instructions +for most operating systems in our wiki: + +[Compile and install instructions](https://grasswiki.osgeo.org/wiki/Compile_and_Install) + # Contributing ## Contributions other than code diff --git a/docker/alpine/Dockerfile_alpine b/docker/alpine/Dockerfile_alpine index 5589762de15..e82497091e8 100644 --- a/docker/alpine/Dockerfile_alpine +++ b/docker/alpine/Dockerfile_alpine @@ -193,7 +193,9 @@ COPY --from=build /usr/local/bin/grass /usr/local/bin/grass COPY --from=build /usr/local/grass* /usr/local/grass/ # pip 20.0.0 fix RUN apk add curl && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py pip==20.0.2 && rm get-pip.py -RUN pip3 install --upgrade pip six grass-session + +# install external Python API +RUN pip3 install --upgrade grass-session RUN ln -s /usr/local/grass /usr/local/grass7 RUN ln -s /usr/local/grass `grass --config path` From 7e8f036e2d3fb05186cfb3fe2f8fd7e629a4675e Mon Sep 17 00:00:00 2001 From: Alex Barreto Date: Tue, 11 May 2021 10:18:28 -0400 Subject: [PATCH 2/3] pythonlib: Remove star imports (#1546) Replaces star import (from x import *) by regular imports (from x import y). Enables the relevant Flake8 warnings. Adds the warning to per-file ignores for init files where we expect and allow using star imports (unfortunately some init files contain actual code suffers from these warning being disabled, but that's a different issue). --- python/grass/.flake8 | 14 +++---- python/grass/exceptions/__init__.py | 42 +++++++++++++------ python/grass/gunittest/invoker.py | 37 ++++++++-------- .../grass/pygrass/modules/interface/module.py | 4 -- python/grass/pygrass/raster/history.py | 2 +- python/grass/pygrass/rpc/__init__.py | 18 ++++---- python/grass/pygrass/vector/abstract.py | 2 +- python/grass/pygrass/vector/table.py | 2 +- python/grass/script/core.py | 31 +++++++++----- python/grass/script/db.py | 11 ++++- python/grass/script/raster.py | 12 +++++- python/grass/script/raster3d.py | 4 +- python/grass/script/task.py | 5 ++- python/grass/script/utils.py | 4 +- python/grass/script/vector.py | 12 +++++- .../grass/temporal/c_libraries_interface.py | 2 +- python/grass/temporal/core.py | 2 +- python/grass/temporal/temporal_granularity.py | 2 +- python/grass/temporal/unit_tests.py | 18 ++++++-- 19 files changed, 142 insertions(+), 82 deletions(-) diff --git a/python/grass/.flake8 b/python/grass/.flake8 index 98cd061f4c0..d2d3831bb84 100644 --- a/python/grass/.flake8 +++ b/python/grass/.flake8 @@ -3,8 +3,6 @@ ignore = E203, # whitespace before ':' (Black) W503, # line break before binary operator (Black) E722, # do not use bare 'except' - F403, # 'from ctypes import *' used; unable to detect undefined names - F405, # 'RasterRow' may be undefined, or defined from star imports: ctypes, grass.pygrass.raster, grass.pygrass.vector per-file-ignores = # C wrappers call libgis.G_gisinit before importing other modules. @@ -26,7 +24,7 @@ per-file-ignores = pygrass/modules/grid/*.py: E501, F401 pygrass/raster/*.py: E501 pygrass/raster/rowio.py: E741 - pygrass/rpc/__init__.py: E501, F401 + pygrass/rpc/__init__.py: E501, F401, F403 pygrass/utils.py: E402, E501 script/db.py: E501 script/task.py: W605 @@ -45,10 +43,12 @@ per-file-ignores = docs/conf.py: E402, E501, # Files not managed by Black imaging/images2gif.py: E226, E501 - # Unused imports - */__init__.py: F401, - */*/__init__.py: F401, - */*/*/__init__.py: F401 + # Unused imports in init files + # F401 imported but unused + # F403 star import used; unable to detect undefined names + */__init__.py: F401, F403 + */*/__init__.py: F401, F403 + */*/*/__init__.py: F401, F403 max-line-length = 88 diff --git a/python/grass/exceptions/__init__.py b/python/grass/exceptions/__init__.py index d2185aa8f5b..76116ac089b 100644 --- a/python/grass/exceptions/__init__.py +++ b/python/grass/exceptions/__init__.py @@ -58,25 +58,43 @@ class Usage(Exception): class CalledModuleError(subprocess.CalledProcessError): """Raised when a called module ends with error (non-zero return code) - :param module: module name - :param code: some code snipped which contains parameters - :param rc: process returncode - :param error: errors provided by the module (stderr) + Used for failures of modules called as subprocesses from Python code. """ def __init__(self, module, code, returncode, errors=None): + """Create an exception with a full error message based on the parameters. + + :param module: module name + :param code: some code snipped which contains parameters + :param returncode: process returncode (assuming non-zero) + :param errors: errors provided by the module (e.g., stderr) + """ # CalledProcessError has undocumented constructor super(CalledModuleError, self).__init__(returncode, module) - msg = _("Module run %s %s ended with error") % (module, code) - msg += _("\nProcess ended with non-zero return code %s") % returncode + if not module or module in code: + # No need to include module name if it is directly in code + # of if it is not set. + executed = code + else: + # Make sure module name is there if provided and not in code. + executed = f"{module} {code}" if errors: - msg += _(". See the following errors:\n%s") % errors + # We assume actual errors, e.g., captured stderr. + err = _("See the following errors:\n{errors}").format(errors=errors) else: - # here could be written "above" but it wouldn't work in some cases - # e.g., for testing framework - msg += _(". See errors in the (error) output.") - self.msg = msg - # TODO: handle other parameters + # In command line, the errors will be above, but in testing framework + # or notebooks, the errors will be somewhere else than the traceback. + err = _("See errors above the traceback or in the error output.") + # The full message + self.msg = _( + "Module run `{executed}` ended with an error.\n" + "The subprocess ended with a non-zero return code: {returncode}." + " {see_errors}" + ).format( + executed=executed, + returncode=returncode, + see_errors=err, + ) def __str__(self): return self.msg diff --git a/python/grass/gunittest/invoker.py b/python/grass/gunittest/invoker.py index 4a7cb04abcd..5caffd180dc 100644 --- a/python/grass/gunittest/invoker.py +++ b/python/grass/gunittest/invoker.py @@ -216,25 +216,24 @@ def _run_test_module(self, module, results_dir, gisdbase, location): stdout, stderr = p.communicate() returncode = p.returncode encodings = [_get_encoding(), "utf8", "latin-1", "ascii"] - detected = False - idx = 0 - while not detected: - try: - stdout = decode(stdout, encoding=encodings[idx]) - detected = True - except: - idx += 1 - pass - - detected = False - idx = 0 - while not detected: - try: - stderr = decode(stderr, encoding=encodings[idx]) - detected = True - except: - idx += 1 - pass + + def try_decode(data, encodings): + """Try to decode data (bytes) using one of encodings + + Falls back to decoding as UTF-8 with replacement for bytes. + Strings are returned unmodified. + """ + for encoding in encodings: + try: + return decode(stdout, encoding=encoding) + except UnicodeError: + pass + if isinstance(data, bytes): + return data.decode(encoding="utf-8", errors="replace") + return data + + stdout = try_decode(stdout, encodings=encodings) + stderr = try_decode(stderr, encodings=encodings) with open(stdout_path, "w") as stdout_file: stdout_file.write(stdout) diff --git a/python/grass/pygrass/modules/interface/module.py b/python/grass/pygrass/modules/interface/module.py index a43bb84ccc1..4dddcebba94 100644 --- a/python/grass/pygrass/modules/interface/module.py +++ b/python/grass/pygrass/modules/interface/module.py @@ -1035,8 +1035,6 @@ def run_modules_in_temp_region(module_list, q): for proc in module_list: proc.run() proc.wait() - except: - raise finally: q.put(module_list) del_temp_region() @@ -1055,8 +1053,6 @@ def run_modules(module_list, q): for proc in module_list: proc.run() proc.wait() - except: - raise finally: q.put(module_list) diff --git a/python/grass/pygrass/raster/history.py b/python/grass/pygrass/raster/history.py index 98c8e91e55d..1ca7a5e0a83 100644 --- a/python/grass/pygrass/raster/history.py +++ b/python/grass/pygrass/raster/history.py @@ -137,7 +137,7 @@ def _get_date(self): if date_str: try: return datetime.datetime.strptime(date_str, self.date_fmt) - except: + except ValueError: return date_str def _set_date(self, datetimeobj): diff --git a/python/grass/pygrass/rpc/__init__.py b/python/grass/pygrass/rpc/__init__.py index 5385cf15949..536cc0a5e8b 100644 --- a/python/grass/pygrass/rpc/__init__.py +++ b/python/grass/pygrass/rpc/__init__.py @@ -14,11 +14,12 @@ import threading import sys from multiprocessing import Process, Lock, Pipe -from ctypes import * +from ctypes import CFUNCTYPE, c_void_p from grass.exceptions import FatalError -from grass.pygrass.vector import * -from grass.pygrass.raster import * +from grass.pygrass.vector import VectorTopo +from grass.pygrass.vector.basic import Bbox +from grass.pygrass.raster import RasterRow, raster2numpy_img import grass.lib.gis as libgis from .base import RPCServerBase from grass.pygrass.gis.region import Region @@ -81,9 +82,8 @@ def _get_raster_image_as_np(lock, conn, data): reg.adjust() array = raster2numpy_img(name, reg, color) - except: - raise finally: + # Send even if an exception was raised. conn.send(array) @@ -120,9 +120,8 @@ def _get_vector_table_as_dict(lock, conn, data): ret = {} ret["table"] = table ret["columns"] = columns - except: - raise finally: + # Send even if an exception was raised. conn.send(ret) @@ -156,7 +155,7 @@ def _get_vector_features_as_wkb_list(lock, conn, data): if layer.exist() is True: if extent is not None: - bbox = basic.Bbox( + bbox = Bbox( north=extent["north"], south=extent["south"], east=extent["east"], @@ -171,9 +170,8 @@ def _get_vector_features_as_wkb_list(lock, conn, data): bbox=bbox, feature_type=feature_type, field=field ) layer.close() - except: - raise finally: + # Send even if an exception was raised. conn.send(wkb_list) diff --git a/python/grass/pygrass/vector/abstract.py b/python/grass/pygrass/vector/abstract.py index d1ac6bfef65..72fedb0191c 100644 --- a/python/grass/pygrass/vector/abstract.py +++ b/python/grass/pygrass/vector/abstract.py @@ -182,7 +182,7 @@ def _get_map_date(self): date_str = utils.decode(libvect.Vect_get_map_date(self.c_mapinfo)) try: return datetime.datetime.strptime(date_str, self.date_fmt) - except: + except ValueError: return date_str def _set_map_date(self, datetimeobj): diff --git a/python/grass/pygrass/vector/table.py b/python/grass/pygrass/vector/table.py index a28439ddf1f..a2bc0fd6a41 100644 --- a/python/grass/pygrass/vector/table.py +++ b/python/grass/pygrass/vector/table.py @@ -22,7 +22,7 @@ try: from collections import OrderedDict -except: +except ImportError: from grass.pygrass.orderdict import OrderedDict import grass.lib.vector as libvect diff --git a/python/grass/script/core.py b/python/grass/script/core.py index d939b4bfc04..ba9fd939f32 100644 --- a/python/grass/script/core.py +++ b/python/grass/script/core.py @@ -896,11 +896,18 @@ def _parse_opts(lines): if not line: break try: - [var, val] = line.split(b"=", 1) - [var, val] = [decode(var), decode(val)] - except: - raise SyntaxError("invalid output from g.parser: %s" % line) - + var, val = line.split(b"=", 1) + except ValueError: + raise SyntaxError("invalid output from g.parser: {}".format(line)) + try: + var = decode(var) + val = decode(val) + except UnicodeError as error: + raise SyntaxError( + "invalid output from g.parser ({error}): {line}".format( + error=error, line=line + ) + ) if var.startswith("flag_"): flags[var[5:]] = bool(int(val)) elif var.startswith("opt_"): @@ -908,8 +915,9 @@ def _parse_opts(lines): elif var in ["GRASS_OVERWRITE", "GRASS_VERBOSE"]: os.environ[var] = val else: - raise SyntaxError("invalid output from g.parser: %s" % line) - + raise SyntaxError( + "unexpected output variable from g.parser: {}".format(line) + ) return (options, flags) @@ -1112,12 +1120,12 @@ def _text_to_key_value_dict( # We first try integer then float try: value_converted = int(value) - except: + except ValueError: not_int = True if not_int: try: value_converted = float(value) - except: + except ValueError: not_float = True if not_int and not_float: @@ -1388,7 +1396,8 @@ def del_temp_region(): try: name = os.environ.pop("WIND_OVERRIDE") run_command("g.remove", flags="f", quiet=True, type="region", name=name) - except: + except (KeyError, CalledModuleError): + # The function succeeds even when called more than once. pass @@ -1687,7 +1696,7 @@ def find_program(pgm, *args): # TODO: the doc or impl is not correct, any return code is accepted call([pgm] + list(args), stdin=nuldev, stdout=nuldev, stderr=nuldev) found = True - except: + except Exception: found = False nuldev.close() diff --git a/python/grass/script/db.py b/python/grass/script/db.py index c33a759af28..7d1d8d911cd 100644 --- a/python/grass/script/db.py +++ b/python/grass/script/db.py @@ -19,7 +19,16 @@ .. sectionauthor:: Martin Landa """ from __future__ import absolute_import -from .core import * + +import os +from .core import ( + run_command, + parse_command, + read_command, + tempfile, + fatal, + list_strings, +) from .utils import try_remove from grass.exceptions import CalledModuleError diff --git a/python/grass/script/raster.py b/python/grass/script/raster.py index aad1f9fd9ce..e81dfe53f3b 100644 --- a/python/grass/script/raster.py +++ b/python/grass/script/raster.py @@ -24,7 +24,17 @@ import string import time -from .core import * +from .core import ( + gisenv, + find_file, + tempfile, + run_command, + read_command, + write_command, + feed_command, + warning, + fatal, +) from grass.exceptions import CalledModuleError from .utils import encode, float_or_dms, parse_key_val, try_remove diff --git a/python/grass/script/raster3d.py b/python/grass/script/raster3d.py index f20218aec7c..4a046e4f092 100644 --- a/python/grass/script/raster3d.py +++ b/python/grass/script/raster3d.py @@ -20,9 +20,11 @@ """ from __future__ import absolute_import +import os +import time import string -from .core import * +from .core import read_command, write_command, fatal from .utils import float_or_dms, parse_key_val from grass.exceptions import CalledModuleError diff --git a/python/grass/script/task.py b/python/grass/script/task.py index e7640e6abe8..370454de223 100644 --- a/python/grass/script/task.py +++ b/python/grass/script/task.py @@ -17,12 +17,13 @@ .. sectionauthor:: Martin Landa """ +import os import re import sys +from grass.exceptions import ScriptError from .utils import decode, split -from .core import * - +from .core import Popen, PIPE, get_real_command try: import xml.etree.ElementTree as etree diff --git a/python/grass/script/utils.py b/python/grass/script/utils.py index be4c3718e60..4507799c491 100644 --- a/python/grass/script/utils.py +++ b/python/grass/script/utils.py @@ -108,7 +108,7 @@ def try_remove(path): """ try: os.remove(path) - except: + except Exception: pass @@ -120,7 +120,7 @@ def try_rmdir(path): """ try: os.rmdir(path) - except: + except Exception: shutil.rmtree(path, ignore_errors=True) diff --git a/python/grass/script/vector.py b/python/grass/script/vector.py index 4b04ae83042..6c54fe3b626 100644 --- a/python/grass/script/vector.py +++ b/python/grass/script/vector.py @@ -18,10 +18,18 @@ """ from __future__ import absolute_import import os +import sys from .utils import parse_key_val -from .core import * -from grass.exceptions import CalledModuleError +from .core import ( + run_command, + read_command, + error, + fatal, + debug, +) + +from grass.exceptions import CalledModuleError, ScriptError unicode = str diff --git a/python/grass/temporal/c_libraries_interface.py b/python/grass/temporal/c_libraries_interface.py index 3cbd71f065f..f6026558072 100644 --- a/python/grass/temporal/c_libraries_interface.py +++ b/python/grass/temporal/c_libraries_interface.py @@ -14,7 +14,7 @@ import sys from multiprocessing import Process, Lock, Pipe import logging -from ctypes import * +from ctypes import byref, cast, c_char_p, c_int, c_void_p, CFUNCTYPE, POINTER from datetime import datetime import grass.lib.gis as libgis import grass.lib.raster as libraster diff --git a/python/grass/temporal/core.py b/python/grass/temporal/core.py index 048ef9161ce..2c9aeb28260 100644 --- a/python/grass/temporal/core.py +++ b/python/grass/temporal/core.py @@ -33,7 +33,7 @@ import sys import grass.script as gscript -from .c_libraries_interface import * +from .c_libraries_interface import CLibrariesInterface from grass.pygrass import messages from grass.script.utils import decode diff --git a/python/grass/temporal/temporal_granularity.py b/python/grass/temporal/temporal_granularity.py index 0b32c146067..97112b4bdc7 100644 --- a/python/grass/temporal/temporal_granularity.py +++ b/python/grass/temporal/temporal_granularity.py @@ -18,7 +18,7 @@ :authors: Soeren Gebbert """ from __future__ import print_function -from .datetime_math import * +from .datetime_math import compute_datetime_delta from functools import reduce from collections import OrderedDict import ast diff --git a/python/grass/temporal/unit_tests.py b/python/grass/temporal/unit_tests.py index 7fb01543f67..9e08fc6d2d5 100644 --- a/python/grass/temporal/unit_tests.py +++ b/python/grass/temporal/unit_tests.py @@ -13,14 +13,24 @@ import copy from datetime import datetime import grass.script.core as core -from .temporal_granularity import * -from .datetime_math import * -from .space_time_datasets import * +from .abstract_dataset import ( + AbstractDatasetComparisonKeyStartTime, + AbstractDatasetComparisonKeyEndTime, +) +from .core import init +from .datetime_math import increment_datetime_by_string, compute_datetime_delta +from .space_time_datasets import RasterDataset +from .spatial_extent import SpatialExtent +from .spatio_temporal_relationships import SpatioTemporalTopologyBuilder +from .temporal_granularity import ( + adjust_datetime_to_granularity, + compute_absolute_time_granularity, +) import grass.lib.vector as vector import grass.lib.rtree as rtree import grass.lib.gis as gis -from ctypes import * +from ctypes import byref # Uncomment this to detect the error core.set_raise_on_error(True) From 7c10386e82dd641a1f519e73520ba3c6bf287502 Mon Sep 17 00:00:00 2001 From: Alex Barreto Date: Mon, 17 May 2021 11:45:55 -0400 Subject: [PATCH 3/3] g.proj: fix reading input WKT (#1582) properly terminate input WKT string Co-authored-by: Marc Jansen --- docker/alpine/Dockerfile_alpine | 2 +- general/g.proj/input.c | 10 ++++- gui/wxpython/dbmgr/base.py | 3 +- gui/wxpython/gui_core/prompt.py | 8 ++++ python/grass/.flake8 | 26 +++++++++++-- python/grass/temporal/stds_import.py | 3 -- python/grass/temporal/temporal_operator.py | 2 +- .../temporal/temporal_raster3d_algebra.py | 2 +- .../grass/temporal/temporal_raster_algebra.py | 2 +- .../grass/temporal/temporal_vector_algebra.py | 2 +- scripts/g.extension/g.extension.py | 39 +++++++++++++------ 11 files changed, 73 insertions(+), 26 deletions(-) diff --git a/docker/alpine/Dockerfile_alpine b/docker/alpine/Dockerfile_alpine index e82497091e8..c93e6902158 100644 --- a/docker/alpine/Dockerfile_alpine +++ b/docker/alpine/Dockerfile_alpine @@ -195,7 +195,7 @@ COPY --from=build /usr/local/grass* /usr/local/grass/ RUN apk add curl && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py pip==20.0.2 && rm get-pip.py # install external Python API -RUN pip3 install --upgrade grass-session +RUN pip3 install --upgrade pip six grass-session --ignore-installed six RUN ln -s /usr/local/grass /usr/local/grass7 RUN ln -s /usr/local/grass `grass --config path` diff --git a/general/g.proj/input.c b/general/g.proj/input.c index d351a51b8aa..5d95b5e0dfa 100644 --- a/general/g.proj/input.c +++ b/general/g.proj/input.c @@ -106,7 +106,7 @@ static void set_authnamecode(OGRSpatialReferenceH); int input_wkt(char *wktfile) { FILE *infd; - char buff[8000], *tmpwkt; + char buff[8192], *tmpwkt; OGRSpatialReferenceH hSRS; char *papszOptions[3]; int ret; @@ -117,11 +117,17 @@ int input_wkt(char *wktfile) infd = fopen(wktfile, "r"); if (infd) { - fread(buff, sizeof(buff), 1, infd); + size_t wktlen; + + wktlen = fread(buff, 1, sizeof(buff), infd); + if (wktlen == sizeof(buff)) + G_fatal_error(_("Input WKT definition is too long")); if (ferror(infd)) G_fatal_error(_("Error reading WKT definition")); else fclose(infd); + /* terminate WKT string */ + buff[wktlen] = '\0'; /* Get rid of newlines */ G_squeeze(buff); } diff --git a/gui/wxpython/dbmgr/base.py b/gui/wxpython/dbmgr/base.py index ea84aadf322..11f474202ff 100644 --- a/gui/wxpython/dbmgr/base.py +++ b/gui/wxpython/dbmgr/base.py @@ -4047,7 +4047,8 @@ def Update(self, driver, database, table, column): ) varSum = 0 for var in decode(dataVar).splitlines(): - varSum += float(var) + if var: + varSum += float(var) stddev = math.sqrt(varSum / count) self.SetTitle(_("Field statistics <%s>") % column) diff --git a/gui/wxpython/gui_core/prompt.py b/gui/wxpython/gui_core/prompt.py index 127d09556d3..956e0edc242 100644 --- a/gui/wxpython/gui_core/prompt.py +++ b/gui/wxpython/gui_core/prompt.py @@ -70,6 +70,11 @@ def __init__(self, parent, giface, menuModel): # list of traced commands self.commands = list() + # reload map lists when needed + if giface: + giface.currentMapsetChanged.connect(self._reloadListOfMaps) + giface.grassdbChanged.connect(self._reloadListOfMaps) + def _readHistory(self): """Get list of commands from history file""" hist = list() @@ -110,6 +115,9 @@ def _getListOfMaps(self): return result + def _reloadListOfMaps(self): + self.mapList = self._getListOfMaps() + def _runCmd(self, cmdString): """Run command diff --git a/python/grass/.flake8 b/python/grass/.flake8 index d2d3831bb84..bcecb594efa 100644 --- a/python/grass/.flake8 +++ b/python/grass/.flake8 @@ -2,7 +2,6 @@ ignore = E203, # whitespace before ':' (Black) W503, # line break before binary operator (Black) - E722, # do not use bare 'except' per-file-ignores = # C wrappers call libgis.G_gisinit before importing other modules. @@ -29,12 +28,31 @@ per-file-ignores = script/db.py: E501 script/task.py: W605 script/vector.py: E501 # Long doctest lines which need review anyway - temporal/*.py: E501, F841 - temporal/abstract_space_time_dataset.py: W605, E501, F841 - temporal/temporal_algebra.py: E741, E501, F841 + temporal/abstract_map_dataset.py: E501 + temporal/abstract_space_time_dataset.py: W605, E501, F841, E722 + temporal/aggregation.py: E501 + temporal/base.py: E501 + temporal/c_libraries_interface.py: E501, F841, E722 + temporal/core.py: E501, E722 + temporal/datetime_math.py: E501, F841, E722 + temporal/list_stds.py: E501 + temporal/metadata.py: E501 + temporal/open_stds.py: F841 + temporal/register.py: E501 + temporal/space_time_datasets.py: E501 + temporal/spatial_extent.py: E501 + temporal/spatial_topology_dataset_connector.py: E501, E722 + temporal/spatio_temporal_relationships.py: E501 + temporal/temporal_algebra.py: E741, E501, F841, E722 + temporal/temporal_extent.py: E501 + temporal/temporal_granularity.py: E501, F841, E722 + temporal/temporal_operator.py: E501 temporal/temporal_raster_algebra.py: E741 + temporal/temporal_raster_base_algebra.py: E501, F841, E722 temporal/temporal_raster3d_algebra.py: E741 + temporal/temporal_topology_dataset_connector.py: E501, E722 temporal/temporal_vector_algebra.py: E741, E501, F841 + temporal/univar_statistics.py: E501 # Current benchmarks/tests are changing sys.path before import. # Possibly, a different approach should be taken there anyway. pygrass/tests/benchmark.py: E501, E402, F401, F821 diff --git a/python/grass/temporal/stds_import.py b/python/grass/temporal/stds_import.py index 7dcd4a2af3a..b7f680e7936 100644 --- a/python/grass/temporal/stds_import.py +++ b/python/grass/temporal/stds_import.py @@ -582,9 +582,6 @@ def import_stds( ) os.chdir(old_cwd) - except: - raise - # Make sure the location is switched back correctly finally: if location: diff --git a/python/grass/temporal/temporal_operator.py b/python/grass/temporal/temporal_operator.py index faa82f23dfb..44a72981be7 100644 --- a/python/grass/temporal/temporal_operator.py +++ b/python/grass/temporal/temporal_operator.py @@ -144,7 +144,7 @@ try: import ply.lex as lex import ply.yacc as yacc -except: +except ImportError: pass diff --git a/python/grass/temporal/temporal_raster3d_algebra.py b/python/grass/temporal/temporal_raster3d_algebra.py index 31d55010a15..175699dd3ce 100644 --- a/python/grass/temporal/temporal_raster3d_algebra.py +++ b/python/grass/temporal/temporal_raster3d_algebra.py @@ -14,7 +14,7 @@ try: import ply.yacc as yacc -except: +except ImportError: pass from .temporal_raster_base_algebra import ( diff --git a/python/grass/temporal/temporal_raster_algebra.py b/python/grass/temporal/temporal_raster_algebra.py index 4908de2eb1d..1c77721e009 100644 --- a/python/grass/temporal/temporal_raster_algebra.py +++ b/python/grass/temporal/temporal_raster_algebra.py @@ -55,7 +55,7 @@ try: import ply.yacc as yacc -except: +except ImportError: pass from .temporal_raster_base_algebra import ( diff --git a/python/grass/temporal/temporal_vector_algebra.py b/python/grass/temporal/temporal_vector_algebra.py index cb861c1ad67..b71db7ea81c 100644 --- a/python/grass/temporal/temporal_vector_algebra.py +++ b/python/grass/temporal/temporal_vector_algebra.py @@ -45,7 +45,7 @@ try: import ply.yacc as yacc -except: +except ImportError: pass import grass.pygrass.modules as pygrass diff --git a/scripts/g.extension/g.extension.py b/scripts/g.extension/g.extension.py index c6b97c9a7ec..bf25db6f1b1 100644 --- a/scripts/g.extension/g.extension.py +++ b/scripts/g.extension/g.extension.py @@ -154,6 +154,7 @@ import fileinput import http import os +import codecs import sys import re import atexit @@ -190,6 +191,29 @@ HTTP_STATUS_CODES = list(http.HTTPStatus) +def replace_shebang_win(python_file): + """ + Replaces "python" with "python3" in python files + using UTF8 encoding on MS Windows + """ + + cur_dir = os.path.dirname(python_file) + tmp_name = os.path.join(cur_dir, gscript.tempname(12)) + + with codecs.open(python_file, "r", encoding="utf8") as in_file, codecs.open( + tmp_name, "w", encoding="utf8" + ) as out_file: + + for line in in_file: + new_line = line.replace( + "#!/usr/bin/env python\n", "#!/usr/bin/env python3\n" + ) + out_file.write(new_line) + + os.remove(python_file) # remove original + os.rename(tmp_name, python_file) # rename temp to original name + + def urlretrieve(url, filename, *args, **kwargs): """Same function as 'urlretrieve', but with the ability to define headers. @@ -1291,11 +1315,9 @@ def install_extension_win(name): module_list = list() for r, d, f in os.walk(srcdir): for file in f: - if file.endswith(".py"): - modulename = file.rsplit(".py")[0] - module_list.append(modulename) - if file.endswith(".exe"): - modulename = file.rsplit(".exe")[0] + # Filter GRASS module name patterns + if re.search(r"^[d,db,g,i,m,p,ps,r,r3,s,t,v,wx]\..*[\.py,\.exe]$", file): + modulename = os.path.splitext(file)[0] module_list.append(modulename) # remove duplicates in case there are .exe wrappers for python scripts module_list = set(module_list) @@ -1308,12 +1330,7 @@ def install_extension_win(name): pyfiles.append(os.path.join(r, file)) for filename in pyfiles: - with fileinput.FileInput(filename, inplace=True) as file: - for line in file: - print( - line.replace("#!/usr/bin/env python\n", "#!/usr/bin/env python3\n"), - end="", - ) + replace_shebang_win(filename) # collect old files old_file_list = list()