From 2ba53266d3b357a28c2087e426bc926ba556dae0 Mon Sep 17 00:00:00 2001 From: Alain Lichnewsky Date: Fri, 9 Dec 2022 21:59:32 +0100 Subject: [PATCH 01/19] Preparing for integration of PR Recovered 1) tests/test_path_ext_py from PR submission 2) some github workflows used to validate PR submission --- .github/workflows/linterTest.yml | 56 ++++ .github/workflows/python3-pypy-Test.yml | 105 +++++++ .github/workflows/python3Test.yml | 92 ++++++ .gitignore | 12 +- IntegrateDec22.md | 41 +++ tests/test_path_ext.py | 354 ++++++++++++++++++++++++ 6 files changed, 659 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/linterTest.yml create mode 100644 .github/workflows/python3-pypy-Test.yml create mode 100644 .github/workflows/python3Test.yml create mode 100644 IntegrateDec22.md create mode 100755 tests/test_path_ext.py diff --git a/.github/workflows/linterTest.yml b/.github/workflows/linterTest.yml new file mode 100644 index 0000000..05ff1b8 --- /dev/null +++ b/.github/workflows/linterTest.yml @@ -0,0 +1,56 @@ +name: LinterTest + # ------------------------------------------------------------ + # (C) Alain Lichnewsky, 2021 + # + # For running under Github's Actions + # + # Script performs basic static test of the package under Python-3, + # including added functionality. + # ------------------------------------------------------------ + +# Controls when the action will run. +on: + # + ## Not enabled, would triggers the workflow on push or pull request events but + ## only for the AL-addRegexp branch. + #push: + # branches: [ AL-addRegexp ] + + # Allows to run this workflow manually from the Github Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in +# parallel +jobs: + # This workflow contains a single job called "super-lint" + super-lint: + # Steps represent a sequence of tasks that will be executed by the job + # Name the Job + name: Lint code base + # Set the type of machine to run on + runs-on: ubuntu-latest + + steps: + # Checks out a copy of your repository on the ubuntu-latest machine + - name: Checkout code + uses: actions/checkout@v2 + + # Runs a single command using the runners shell, in practice it is useful + # to figure out some of the environment setup + - name: Use shell to figure out environment + run: echo Hello from Github Actions !!; + bash --version | head -1 ; + echo LANG=${LANG} SHELL=${SHELL} ; + echo PATH=${PATH} ; + pwd; + ls -ltha; + + # Runs the Super-Linter action + - name: Run Super-Linter + uses: github/super-linter@v3 + # + # this script requires some environment variables + # + env: + DEFAULT_BRANCH: AL-addRegexp + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/python3-pypy-Test.yml b/.github/workflows/python3-pypy-Test.yml new file mode 100644 index 0000000..e62d423 --- /dev/null +++ b/.github/workflows/python3-pypy-Test.yml @@ -0,0 +1,105 @@ +name: Test python package dpath-python + # ------------------------------------------------------------ + # (C) Alain Lichnewsky, 2021 + # + # For running under Github's Actions + # + # Here the idea is to use tox for testing and test on python 3.8 and + # pypy-3.7. + # + # There are numerous issues that must be understood with the predefined + # features of Github's preloaded containers. + # Here : + # - try and load in 2 separate steps + # - probably not optimal in viexw of preloaded configurations + # + # ------------------------------------------------------------ + +on: + # manual dispatch, this script will not be started automagically + workflow_dispatch: + +jobs: + test-python3: + + timeout-minutes: 60 + + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + + - name: Set up Pypy 3.7 + uses: actions/setup-python@v2 + with: + python-version: 'pypy-3.7' + architecture: 'x64' + + + - name: Ascertain configuration + # + # Collect information concerning $HOME and the location of + # file(s) loaded from Github/ + run: | + echo Working dir: $(pwd) + echo Files at this location: + ls -ltha + echo HOME: ${HOME} + echo LANG: ${LANG} SHELL: ${SHELL} + which python + echo LD_LIBRARY_PATH: ${LD_LIBRARY_PATH} + echo PYTHONPATH: \'${PYTHONPATH}\' + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # ** here (it is expected that) ** + # pythonLocation: /opt/hostedtoolcache/Python/3.8.8/x64 + # LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.8/x64/lib + # Working dir /home/runner/work/dpath-python/dpath-python + # HOME: /home/runner + # LANG: C.UTF-8 + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + - name: Install dependencies + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # requirements install the test framework, which is not + # required by the package in setup.py + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + python -m pip install --upgrade pip setuptools wheel + if [ -f requirements.txt ]; then + pip install -r requirements.txt; + fi + python setup.py install + echo which nose :$(which nose) + echo which nose2: $(which nose2) + echo which nose2-3.6: $(which nose2-3.6) + echo which nose2-3.8: $(which nose2-3.8) + + - name: Tox testing + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # move to tox testing, otherwise will have to parametrize + # nose in more details; here tox.ini will apply + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + pip install tox + echo "Installed tox" + tox + echo "Ran tox" + + - name: Nose2 testing + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # Last try... with a nose2.cfg file + # + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + nose2 -c nose2.cfg diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml new file mode 100644 index 0000000..701d2d0 --- /dev/null +++ b/.github/workflows/python3Test.yml @@ -0,0 +1,92 @@ +name: Test python package dpath-python + # ------------------------------------------------------------ + # (C) Alain Lichnewsky, 2021 + # + # For running under Github's Actions + # + # Script performs basic test of the Python-3 version of the package + # including added functionality. + # ------------------------------------------------------------ + +on: + # [push] + + workflow_dispatch: + +jobs: + test-python3: + + timeout-minutes: 60 + + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: [ '3.8' , 'pypy-3.7' ] + architecture: 'x64' + + - name: Ascertain configuration + # + # Collect information concerning $HOME and the location of + # file(s) loaded from Github/ + run: | + echo Working dir: $(pwd) + echo Files at this location: + ls -ltha + echo HOME: ${HOME} + echo LANG: ${LANG} SHELL: ${SHELL} + which python + echo LD_LIBRARY_PATH: ${LD_LIBRARY_PATH} + echo PYTHONPATH: \'${PYTHONPATH}\' + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # ** here (it is expected that) ** + # pythonLocation: /opt/hostedtoolcache/Python/3.8.8/x64 + # LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.8.8/x64/lib + # Working dir /home/runner/work/dpath-python/dpath-python + # HOME: /home/runner + # LANG: C.UTF-8 + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + - name: Install dependencies + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # requirements install the test framework, which is not + # required by the package in setup.py + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + python -m pip install --upgrade pip setuptools wheel + if [ -f requirements.txt ]; then + pip install -r requirements.txt; + fi + python setup.py install + echo which nose :$(which nose) + echo which nose2: $(which nose2) + echo which nose2-3.6: $(which nose2-3.6) + echo which nose2-3.8: $(which nose2-3.8) + + - name: Tox testing + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # move to tox testing, otherwise will have to parametrize + # nose in more details; here tox.ini will apply + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + pip install tox + echo "Installed tox" + tox + echo "Ran tox" + + - name: Nose2 testing + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # Last try... with a nose2.cfg file + # + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + run: | + nose2 -c nose2.cfg diff --git a/.gitignore b/.gitignore index 52fd738..93ad9ce 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,14 @@ venv_39 .idea/ dpath.egg-info/ dist/ -tests/.hypothesis \ No newline at end of file +tests/.hypothesis + +# Github logs +.githubLogs + +# Test etc +.coverage + +# Editor temporaries +*~ + \ No newline at end of file diff --git a/IntegrateDec22.md b/IntegrateDec22.md new file mode 100644 index 0000000..590ef4b --- /dev/null +++ b/IntegrateDec22.md @@ -0,0 +1,41 @@ +# Attempt to integrate in December 2022 + +## Resynchronize with `origin/master` + 1. synchronized by githup + 1. synchronized my local files + - fetched / pulled into master + - created new branch `AL-master4merge` for trying (brutal) merge with my changes + + 1. to list differences at time of PR (June 2021) + +> git diff 9F7C35 C29643 --name-status +> 9F7C35: their changes +> C29643: my PR + +Not as bad as it looks (changes in dpath/*.py are the real features) + +> git diff 9F7C35 C29643 --name-status +> A .github/workflows/linterTest.yml +> A .github/workflows/python3-pypy-Test.yml +> A .github/workflows/python3Test.yml +> M .gitignore +> M .travis.yml +> M MAINTAINERS.md +> M README.rst +> M dpath/options.py +> M dpath/segments.py +> M dpath/util.py +> M dpath/version.py +> A issues/err_walk.py +> A maintainers_log.md +> A nose2.cfg +> A requirements-2.7.txt +> A requirements.txt +> A tests/test_path_ext.py +> M tests/test_segments.py +> M tests/test_unicode.py +> M tests/test_util_get_values.py +> M tox.ini + + +## diff --git a/tests/test_path_ext.py b/tests/test_path_ext.py new file mode 100755 index 0000000..db2e1de --- /dev/null +++ b/tests/test_path_ext.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# -*- mode: Python -*- +# +# (C) Alain Lichnewsky, 2021 +# +# Much code copied from test_segments.py +# +import os +import sys +import re + + +import unittest +from hypothesis import given, assume, settings, HealthCheck +import hypothesis.strategies as st + +from dpath import options +import dpath.segments as api +import dpath.options +dpath.options.DPATH_ACCEPT_RE_REGEXP = True # enable re.regexp support in path expr. + +# enables to modify some globals +MAX_SAMPLES = None +if __name__ == "__main__": + if "-v" in sys.argv: + MAX_SAMPLES = 20 + MAX_LEAVES = 7 + + +settings.register_profile("default", suppress_health_check=(HealthCheck.too_slow,)) +settings.load_profile(os.getenv(u'HYPOTHESIS_PROFILE', 'default')) +if MAX_SAMPLES is None: + MAX_LEAVES = 20 + MAX_SAMPLES = 300 + +ALPHABET = ('A', 'B', 'C', ' ') +ALPHABETK = ('a', 'b', 'c', '-') + +random_key_int = st.integers(0, 10) +random_key_str = st.text(alphabet=ALPHABETK, min_size=2) +random_key = random_key_str | random_key_int +random_segments = st.lists(random_key, max_size=4) +random_leaf = random_key_int | st.text(alphabet=ALPHABET, min_size=2) + + +if options.ALLOW_EMPTY_STRING_KEYS: + random_thing = st.recursive( + random_leaf, + lambda children: ( st.lists(children, max_size=3) + | st.dictionaries( st.binary(max_size=5) + | st.text(alphabet=ALPHABET), children)), + max_leaves=MAX_LEAVES) +else: + random_thing = st.recursive( + random_leaf, + lambda children: ( st.lists(children, max_size=3) + | st.dictionaries( st.binary(min_size=1, max_size=5) + | st.text(min_size=1, alphabet=ALPHABET), + children)), + max_leaves=MAX_LEAVES) + +random_node = random_thing.filter(lambda thing: isinstance(thing, (list, dict))) + +if options.ALLOW_EMPTY_STRING_KEYS: + random_mutable_thing = st.recursive( + random_leaf, + lambda children: ( st.lists(children, max_size=3) | st.text(alphabet=ALPHABET), + children), + max_leaves=MAX_LEAVES) +else: + random_mutable_thing = st.recursive( + random_leaf, + lambda children: ( st.lists(children, max_size=3) + | st.dictionaries(st.text(alphabet=ALPHABET, min_size=1), + children)), + max_leaves=MAX_LEAVES) + + +random_mutable_node = random_mutable_thing.filter(lambda thing: isinstance(thing, + (list, dict))) + + +@st.composite +def mutate(draw, segment): + # Convert number segments. + segment = api.int_str(segment) + + # Infer the type constructor for the result. + kind = type(segment) + + # Produce a valid kind conversion for our wildcards. + if isinstance(segment, bytes): + def to_kind(v): + try: + return bytes(v, 'utf-8') + except: + return kind(v) + else: + def to_kind(v): + return kind(v) + + # Convert to an list of single values. + converted = [] + for i in range(len(segment)): + # This carefully constructed nonsense to get a single value + # is necessary to work around limitations in the bytes type + # iteration returning integers instead of byte strings of + # length 1. + c = segment[i:i + 1] + + # Check for values that need to be escaped. + if c in tuple(map(to_kind, ('*', '?', '[', ']'))): + c = to_kind('[') + c + to_kind(']') + + converted.append(c) + + # Start with a non-mutated result. + result = converted + + # 50/50 chance we will attempt any mutation. + change = draw(st.sampled_from((True, False))) + if change: + result = [] + + # For every value in segment maybe mutate, maybe not. + for c in converted: + # If the length isn't 1 then, we know this value is already + # an escaped special character. We will not mutate these. + if len(c) != 1: + result.append(c) + else: + # here the character is mutated to ? or * with 1/3 proba + result.append(draw(st.sampled_from((c, to_kind('?'), to_kind('*'))))) + + combined = kind().join(result) + + # If we by chance produce the star-star result, then just revert + # back to the original converted segment. This is not the mutation + # you are looking for. + if combined == to_kind('**'): + combined = kind().join(converted) + + return combined + + +@st.composite +def random_segments_with_glob(draw): + segments = draw(random_segments) + glob = list(map(lambda x: draw(mutate(x)), segments)) + + # 50/50 chance we will attempt to add a star-star to the glob. + use_ss = draw(st.sampled_from((True, False))) + if use_ss: + # Decide if we are inserting a new segment or replacing a range. + insert_ss = draw(st.sampled_from((True, False))) + if insert_ss: + index = draw(st.integers(0, len(glob))) + glob.insert(index, '**') + else: + start = draw(st.integers(0, len(glob))) + stop = draw(st.integers(start, len(glob))) + glob[start:stop] = ['**'] + + return (segments, glob) + + +@st.composite +def random_segments_with_nonmatching_glob(draw): + (segments, glob) = draw(random_segments_with_glob()) + + # Generate a segment that is not in segments. + invalid = draw(random_key.filter(lambda x: x not in segments and x not in ('*', '**'))) + + # Do we just have a star-star glob? It matches everything, so we + # need to replace it entirely. + if len(glob) == 1 and glob[0] == '**': + glob = [invalid] + # Do we have a star glob and only one segment? It matches anything + # in the segment, so we need to replace it entirely. + elif len(glob) == 1 and glob[0] == '*' and len(segments) == 1: + glob = [invalid] + # Otherwise we can add something we know isn't in the segments to + # the glob. + else: + index = draw(st.integers(0, len(glob))) + glob.insert(index, invalid) + + return (segments, glob) + + + +def checkSegGlob(segments, glob): + """ simple minded check that the translation done in random_segments_with_re_glob + does not suppress matching; just do not inspect in the case where a "**" has been + put in glob. + """ + if "**" in glob: + return + zipped = zip(segments, glob) + for (s, g) in zipped: + if isinstance(s, int): + continue + if isinstance(g, re.Pattern): + m = g.match(s) + elif isinstance(g, str) and not g == "**": + m = re.match(g, s) + else: + raise NotImplementedError(f"unexpected type for g=({type(g)}){g}") + if not m: + print(f"Failure in checkSegGlob {(s,g)} type(g)={type(g)}", file=sys.stderr) + raise RuntimeError("{repr(s)}' does not match regexp:{repr(g)}") + +# exclude translation if too many *, to avoid too large cost in matching +# '*' -> '.*' # see glob +# '?' -> '.' # see glob +# Recall that bash globs are described at URL: +# https://man7.org/linux/man-pages/man7/glob.7.html + + +rex_translate = re.compile("([*])[*]*") +rex_translate2 = re.compile("([?])") +rex_isnumber = re.compile(r"\d+") + + +@st.composite +def random_segments_with_re_glob(draw): + """ Transform some globs with equivalent re.regexprs, to test the use of regexprs + """ + (segments, glob) = draw(random_segments_with_glob()) + glob1 = [] + for g in glob: + if g == "**" or not isinstance(g, str) or rex_isnumber.match(g): + glob1.append(g) + continue + try: + g0 = rex_translate.sub(".\\1", g) + g0 = rex_translate2.sub(".", g0) + g1 = re.compile("^" + g0 + "$") + if not g1.match(g): + g1 = g + except Exception: + sys.stderr.write("Unable to re.compile:({})'{}' from '{}'\n".format(type(g1), g1, g)) + g1 = g + glob1.append(g1) + + checkSegGlob(segments, glob1) + return (segments, glob1) + + +@st.composite +def random_segments_with_nonmatching_re_glob(draw): + """ Transform some globs with equivalent re.regexprs, to test the use of regexprs + """ + (segments, glob) = draw(random_segments_with_nonmatching_glob()) + glob1 = [] + for g in glob: + if g == "**" or not isinstance(g, str): + glob1.append(g) + continue + try: + g0 = rex_translate.sub(".\\1", g) + g0 = rex_translate2.sub(".", g0) + g1 = re.compile("^" + g0 + "$") + except Exception: + sys.stderr.write("(non-matching):Unable to re.compile:({}){}".format(type(g), g)) + g1 = g + glob1.append(g1) + + return (segments, glob1) + + +@st.composite +def random_walk(draw): + """ return a (node, (segment, value)) + where node is arbitrary tree, + (segment, value) is a valid pair drawn from the return of + api.walk(node), wich generates them all. + """ + node = draw(random_mutable_node) + found = tuple(api.walk(node)) + assume(len(found) > 0) + (cr, dr) = draw(st.sampled_from(found)) + if dr in (int, str): + dr = (dr,) + return (node, (cr, dr)) + + +def setup(): + # Allow empty strings in segments. + options.ALLOW_EMPTY_STRING_KEYS = True + + +def teardown(): + # Revert back to default. + options.ALLOW_EMPTY_STRING_KEYS = False + + +# +# Run under unittest +# +class TestEncoding(unittest.TestCase): + DO_DEBUG_PRINT = False + + @settings(max_examples=MAX_SAMPLES) + @given(random_segments_with_re_glob()) + def test_match_re(self, pair): + ''' + Given segments and a known good glob, match should be True. + ''' + (segments, glob) = pair + assert api.match(segments, glob) is True + if TestEncoding.DO_DEBUG_PRINT: + sys.stderr.write("api.match: segments:{} , glob:{}\n".format(segments, glob)) + + + @settings(max_examples=MAX_SAMPLES) + @given(random_segments_with_nonmatching_re_glob()) + def test_match_nonmatching_re(self, pair): + ''' + Given segments and a known bad glob, match should be False. + ''' + (segments, glob) = pair + assert api.match(segments, glob) is False + if TestEncoding.DO_DEBUG_PRINT: + sys.stderr.write("api.match:non match OK: segments:{}, glob:{}\n".format(segments, glob)) + + +if __name__ == "__main__": + if "-h" in sys.argv: + description = """\ +This may run either under tox or standalone. When standalone +flags -h and -v are recognized, other flags are dealt with by unittest.main +and may select test cases. + +Flags: + -h print this help and quit + -V print information messages on stderr; also reduces MAX_SAMPLES to 50 + -v handled by unittest framework +Autonomous CLI syntax: + python3 [-h] [-v] [TestEncoding[.]] + + e.g. python3 TestEncoding.test_match_re +""" + print(description) + sys.exit(0) + + if "-V" in sys.argv: + sys.argv = [x for x in sys.argv if x != "-V"] + TestEncoding.DO_DEBUG_PRINT = True + sys.stderr.write("Set verbose mode\n") + + sys.stderr.write(f"Starting tests in test_path_exts with args {sys.argv}") + unittest.main() From 0a9992d76ef9a4f850381c1d5727067c6e1c440a Mon Sep 17 00:00:00 2001 From: Alain Lichnewsky Date: Mon, 12 Dec 2022 10:43:48 +0100 Subject: [PATCH 02/19] After integration of regexp extention to paths Follow-up PR: "Add ability to select path segments using Python re regexprs" #146 on June 1, 2021 Includes: - integration of extension on dpath-python - improvement of tests (under nose2, with scripts test/noseRunner to control code options) - improvement of documentation in README.rst Baseline for integration: commit ea64635e5e55a7b44398db54068f162e5cd45e10 (tag: v2.1.1, origin/master, origin/HEAD, master, AL-master4merge) Merge: 702b5dd 38007df Author: moomoohk Date: Wed Nov 30 17:16:05 2022 +0200 Note: at this time, no test wrt. Github.Actions has been made (yet) --- IntegrateDec22.md | 41 ----- README.rst | 64 +++++++ dpath/__init__.py | 20 ++- dpath/options.py | 4 + dpath/segments.py | 15 +- dpath/version.py | 2 +- tests/nose_runner | 73 ++++++++ tests/test_path_ext.py | 8 +- tests/test_various_exts.py | 332 +++++++++++++++++++++++++++++++++++++ 9 files changed, 513 insertions(+), 46 deletions(-) delete mode 100644 IntegrateDec22.md create mode 100755 tests/nose_runner create mode 100644 tests/test_various_exts.py diff --git a/IntegrateDec22.md b/IntegrateDec22.md deleted file mode 100644 index 590ef4b..0000000 --- a/IntegrateDec22.md +++ /dev/null @@ -1,41 +0,0 @@ -# Attempt to integrate in December 2022 - -## Resynchronize with `origin/master` - 1. synchronized by githup - 1. synchronized my local files - - fetched / pulled into master - - created new branch `AL-master4merge` for trying (brutal) merge with my changes - - 1. to list differences at time of PR (June 2021) - -> git diff 9F7C35 C29643 --name-status -> 9F7C35: their changes -> C29643: my PR - -Not as bad as it looks (changes in dpath/*.py are the real features) - -> git diff 9F7C35 C29643 --name-status -> A .github/workflows/linterTest.yml -> A .github/workflows/python3-pypy-Test.yml -> A .github/workflows/python3Test.yml -> M .gitignore -> M .travis.yml -> M MAINTAINERS.md -> M README.rst -> M dpath/options.py -> M dpath/segments.py -> M dpath/util.py -> M dpath/version.py -> A issues/err_walk.py -> A maintainers_log.md -> A nose2.cfg -> A requirements-2.7.txt -> A requirements.txt -> A tests/test_path_ext.py -> M tests/test_segments.py -> M tests/test_unicode.py -> M tests/test_util_get_values.py -> M tox.ini - - -## diff --git a/README.rst b/README.rst index 0ad3ad2..05efb49 100644 --- a/README.rst +++ b/README.rst @@ -111,6 +111,9 @@ elements in ``x['a']['b']`` where the key is equal to the glob ``'[cd]'``. Okay. } } +Using Python's `re` regular expressions instead of globs is also possible +see below re_regexp_. + ... Wow that was easy. What if I want to iterate over the results, and not get a merged view? @@ -438,6 +441,67 @@ To get around this, you can sidestep the whole "filesystem path" style, and aban >>> dpath.get(['a', 'b/c']) 0 +.. _re_regexp: + +Globs too imprecise? Use Python's `re` Regular Expressions +========================================================== + +Python's `re` regular expressions PythonRe_ may be used as follows: + + .. _PythonRe: https://docs.python.org/3/library/re.html + + - This facility must be enabled (for backwards compatibility): + + .. code-block:: python + + >>> import dpath + >>> dpath.options.DPATH_ACCEPT_RE_REGEXP = True + + - Now a path component may also be specified as: + - in a path expression, as {} where `` is a regular expression accepted by the + standard Python module `re`. For example: + + .. code-block:: python + + >>> selPath = 'Config/{(Env|Cmd)}' + >>> x = dpath.util.search(js.lod, selPath) + + .. code-block:: python + + >>> selPath = '{(Config|Graph)}/{(Env|Cmd|Data)}' + >>> x = dpath.util.search(js.lod, selPath) + + - When using the list form for a path, a list element can also + be expressed as + + - a string as above + - the output of :: `re.compile( args )`` + + An example: + + .. code-block:: python + + >>> selPath = [ re.compile('(Config|Graph)') , re.compile('(Env|Cmd|Data)') ] + >>> x = dpath.util.search(js.lod, selPath) + + More examples from a realistic json context: + + +-----------------------------------------+--------------------------------------+ + + **Extended path glob** | **Designates** + + +-----------------------------------------+--------------------------------------+ + + "\*\*/{[^A-Za-z]{2}$}" | "Id" + + +-----------------------------------------+--------------------------------------+ + + "\*/{[A-Z][A-Za-z\d]*$}" | "Name","Id","Created", "Scope",... + + +-----------------------------------------+--------------------------------------+ + + "\*\*/{[A-Z][A-Za-z\d]*\d$}" | EnableIPv6" + + +-----------------------------------------+--------------------------------------+ + + "\*\*/{[A-Z][A-Za-z\d]*Address$}" | "Containers/199c5/MacAddress" + + +-----------------------------------------+--------------------------------------+ + + + + + dpath.segments : The Low-Level Backend ====================================== diff --git a/dpath/__init__.py b/dpath/__init__.py index 9f56e6b..d243d90 100644 --- a/dpath/__init__.py +++ b/dpath/__init__.py @@ -29,6 +29,9 @@ _DEFAULT_SENTINEL = object() +import sys +import dpath.options +import re def _split_path(path: Path, separator: Optional[str] = "/") -> Union[List[PathSegment], PathSegment]: """ @@ -45,7 +48,22 @@ def _split_path(path: Path, separator: Optional[str] = "/") -> Union[List[PathSe else: split_segments = path.lstrip(separator).split(separator) - return split_segments + final = [] + for segment in split_segments: + if (options.DPATH_ACCEPT_RE_REGEXP and isinstance(segment, str) + and segment[0] == '{' and segment[-1] == '}'): + try: + rs = segment[1:-1] + rex = re.compile(rs) + except Exception as reErr: + print(f"Error in segment '{segment}' string '{rs}' not accepted" + + f"as re.regexp:\n\t{reErr}", + file=sys.stderr) + raise reErr + final.append(rex) + else: + final.append(segment) + return final def new(obj: MutableMapping, path: Path, value, separator="/", creator: Creator = None) -> MutableMapping: diff --git a/dpath/options.py b/dpath/options.py index 41f35c4..2076ea0 100644 --- a/dpath/options.py +++ b/dpath/options.py @@ -1 +1,5 @@ ALLOW_EMPTY_STRING_KEYS = False + +# extension disabled by default to preserve backwards compatibility +# +DPATH_ACCEPT_RE_REGEXP = False diff --git a/dpath/segments.py b/dpath/segments.py index faa763f..3a6a118 100644 --- a/dpath/segments.py +++ b/dpath/segments.py @@ -7,6 +7,13 @@ from dpath.types import PathSegment, Creator, Hints +import sys +import re +try: + RE_PATTERN_TYPE = re.Pattern +except AttributeError: + RE_PATTERN_TYPE = re._pattern_type + def make_walkable(node) -> Iterator[Tuple[PathSegment, Any]]: """ Returns an iterator which yields tuple pairs of (node index, node value), regardless of node type. @@ -227,8 +234,12 @@ def match(segments: Sequence[PathSegment], glob: Sequence[str]): # exception while attempting to match into a False for the # match. try: - if not fnmatchcase(s, g): - return False + if isinstance(g, RE_PATTERN_TYPE): + mobj = g.match(s) + if mobj is None: + return False + elif not fnmatchcase(s, g): + return False except: return False diff --git a/dpath/version.py b/dpath/version.py index 5b0431e..7270398 100644 --- a/dpath/version.py +++ b/dpath/version.py @@ -1 +1 @@ -VERSION = "2.1.1" +VERSION = "2.1.1b" diff --git a/tests/nose_runner b/tests/nose_runner new file mode 100755 index 0000000..17c44db --- /dev/null +++ b/tests/nose_runner @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Allow to run nose2 tests with predefined variables as defined in options.py +# +# (C) A.Lichnewsky, 2022 + +import nose2 +import sys +from dpath import options as OPTS +import argparse + + +def do_main(): + description = """ + This runs nose2 testframe giving the user the ability to set options + and in particular DPATH_ACCEPT_RE_REGEXP + + Usage: + PYTHONPATH= nose_runner [-d] [-E] [-K] [-s ] + + Flags: + """ + + argLineParser = argparse.ArgumentParser(description=description, + formatter_class=argparse.RawTextHelpFormatter) + argLineParser.add_argument("-d", "--debug", action="store_true", dest="doDebug", + help="Debug messages on stderr"), + + argLineParser.add_argument("-E", "--extended", action="store_true", dest="doExtendedPath", + help="Enable regexp extended paths") + + argLineParser.add_argument("-K", "--emptyStringKeys", action="store_true", dest="doEmptyStringKeys", + help="Enable regexp extended paths") + + argLineParser.add_argument("-v", "--verbose", action="store_true", dest="noseVerbose", + help="Request nose prints tests entered and result") + + argLineParser.add_argument("-s", "--StartDir", dest="startDir", metavar="FILE", + help="Directory to start nose2 discovery") + + argLineParser.add_argument('test_specs',nargs="?", help="test specifications for nose2") + + try: + options = argLineParser.parse_args() + if options.doDebug: + print(repr(options), file=sys.stderr) + + if options.doExtendedPath: + OPTS.DPATH_ACCEPT_RE_REGEXP = True + + if options.doEmptyStringKeys: + OPTS.ALLOW_EMPTY_STRING_KEYS = True + + except Exception: + print("Quitting because of error(s)", file=sys.stderr) + + sys.exit(1) + + sys.argv = [el for el in sys.argv if not (el in ("-d", "-K", "-E"))] + + if options.doDebug: + print(f"Accept extended regexp:{OPTS.DPATH_ACCEPT_RE_REGEXP}", file=sys.stderr) + print(f"Allow empty string keys:{OPTS.ALLOW_EMPTY_STRING_KEYS}", file=sys.stderr) + print(f"Start discovery at:{options.startDir}", file=sys.stderr) + + print(f"Nose2.discover arglist:{sys.argv}") + + ret = nose2.discover() + print("Tests return cc={ret}", file=sys.stderr) + + +if __name__ == '__main__': + do_main() diff --git a/tests/test_path_ext.py b/tests/test_path_ext.py index db2e1de..a299ee6 100755 --- a/tests/test_path_ext.py +++ b/tests/test_path_ext.py @@ -18,7 +18,13 @@ from dpath import options import dpath.segments as api import dpath.options -dpath.options.DPATH_ACCEPT_RE_REGEXP = True # enable re.regexp support in path expr. + +# check that how the options have been set +print(f"At entry in test_path_ext DPATH_ACCEPT_RE_REGEXP = {dpath.options.DPATH_ACCEPT_RE_REGEXP}", file=sys.stderr) + +if not dpath.options.DPATH_ACCEPT_RE_REGEXP : + print("This test doesn't make sense with DPATH_ACCEPT_RE_REGEXP = False", file=sys.stderr) + dpath.options.DPATH_ACCEPT_RE_REGEXP = True # enable re.regexp support in path expr. # enables to modify some globals MAX_SAMPLES = None diff --git a/tests/test_various_exts.py b/tests/test_various_exts.py new file mode 100644 index 0000000..2a7f3fc --- /dev/null +++ b/tests/test_various_exts.py @@ -0,0 +1,332 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# -*- mode: Python -*- +# +# (C) Alain Lichnewsky, 2022 +# +# Test many functionalities that use path specifications for support of extended specs with re.regex +# +import os +import sys +import re + +from copy import copy + +import unittest + +from dpath import options +import dpath as DP + +# check that how the options have been set +print(f"At entry in test_path_ext DPATH_ACCEPT_RE_REGEXP = {DP.options.DPATH_ACCEPT_RE_REGEXP}", file=sys.stderr) + +if not DP.options.DPATH_ACCEPT_RE_REGEXP : + print("This test doesn't make sense with DPATH_ACCEPT_RE_REGEXP = False", file=sys.stderr) + DP.options.DPATH_ACCEPT_RE_REGEXP = True # enable re.regexp support in path expr. + +# one class per function to be tested +class SampleDicts(): + + def build(self): + self.d1= { + "a001": { + "b2": { + "c1.2": { + "d.dd": 0, + "e.ee": 1, + "f.f0": 2, + }, + }, + }, + } + + + self.specs1 = ( [re.compile(".*")] , + [re.compile("[a-z]+$")] , + ["*", re.compile(".*")], + ["*", "*",re.compile(".*")], + ["*", re.compile("[a-z]+\d+$")], + ["*", re.compile("[a-z]+[.][a-z]+$")], + ["**", re.compile(".*")], + ["**", re.compile("[a-z]+\d+$")], + ["**", re.compile("[a-z]+[.][a-z]+$")] + ) + + self.specs1Pairs = ( ( [re.compile(".*")] , ("a001",)), + ( [re.compile("[a-z]+$")], None), + ( ["*", re.compile(".*")], ("a001/b2",)), + ( ["*", "*",re.compile(".*")], ("a001/b2/c1.2",)), + ( ["*", re.compile("[a-z]+\d+$")], ("a001/b2",)) , + ( ["*", re.compile("[a-z]+[.][a-z]+$")], None), + ( ["**", re.compile(".*")], + ("a001","a001/b2", "a001/b2/c1.2","a001/b2/c1.2/d.dd", + "a001/b2/c1.2/e.ee", "a001/b2/c1.2/f.f0")), + ( ["**", re.compile("[a-z]+\d+$")], ("a001", "a001/b2")), + ( ["**", re.compile("[a-z]+[.][a-z]+$")], + ("a001/b2/c1.2/d.dd","a001/b2/c1.2/e.ee")) + ) + + + self.specs1GetPairs = ( + ( [re.compile(".*")] , {'b2': {'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}}), + ([re.compile("[a-z]+$")] , ('*NONE*',)), + (["*", re.compile(".*")],{'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}), + (["*", "*",re.compile(".*")],{'d.dd': 0, 'e.ee': 1, 'f.f0': 2} ), + (["*", re.compile("[a-z]+\d+$")],{'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}), + (["*", re.compile("[a-z]+[.][a-z]+$")],('*NONE*',)), + (["**", re.compile(".*")], "*FAIL*"), + (["**", re.compile("[a-z]+\d+$")],"*FAIL*"), + (["**", re.compile("[a-z]+[.][a-z]+$")],"*FAIL*"), + ) + + + self.d2 = { + "Name": "bridge", + "Id": "333d22b3724", + "Created": "2022-12-08T09:02:33.360812052+01:00", + "Scope": "local", + "Driver": "bridge", + "EnableIPv6": False, + "IPAM": { + "Driver": "default", + "Options": None, + "Config": + { + "Subnet": "172.17.0.0/16", + "Gateway": "172.17.0.1" + } + }, + "Internal": False, + "Attachable": False, + "Ingress": False, + "ConfigFrom": { + "Network": "" + }, + "ConfigOnly": False, + "Containers": { + "199c590e8f13477": { + "Name": "al_dpath", + "EndpointID": "3042bbe16160a63b7", + "MacAddress": "02:34:0a:11:10:22", + "IPv4Address": "172.17.0.2/16", + "IPv6Address": "" + } + }, + "Options": { + "com.docker.network.bridge.default_bridge": "true", + "com.docker.network.bridge.enable_icc": "true", + "com.docker.network.bridge.enable_ip_masquerade": "true", + "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0", + "com.docker.network.bridge.name": "docker0", + "com.docker.network.driver.mtu": "1500" + }, + "Labels": {} + } + + self.specs2 = ( + ["*", re.compile("[A-Z][a-z\d]*$")], + ["**", re.compile("[A-Z][a-z\d]*$")], + ["**", re.compile("[A-Z][A-Za-z\d]*Address$")], + ["**", re.compile("[A-Za-z]+\d+$")], + ["**", re.compile("\d+[.]\d+")] + ) + + self.specs2Pairs = ( + (["*", re.compile("[A-Z][a-z\d]*$")], + ("IPAM/Driver","IPAM/Options","IPAM/Config","ConfigFrom/Network")), + (["**", re.compile("[A-Z][a-z\d]*$")], + ("Name","Id","Created", "Scope","Driver", "Internal", "Attachable", + "Ingress", "Containers", "Options", "Labels", "IPAM/Driver", "IPAM/Options", + "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", + "ConfigFrom/Network","Containers/199c590e8f13477/Name", + "Containers/199c590e8f13477/MacAddress" )), + (["**", re.compile("[A-Z][A-Za-z\d]*Address$")], + ("Containers/199c590e8f13477/MacAddress","Containers/199c590e8f13477/IPv4Address", + "Containers/199c590e8f13477/IPv6Address")), + (["**", re.compile("[A-Za-z]+\d+$")],("EnableIPv6",)), + (["**", re.compile("\d+[.]\d+")],None) + ) + + self.specs3 = ( ["*", re.compile("[A-Z][a-z\d]*$")], + "**/{.*}", + "*/{[A-Z][A-Za-z\d]*$}", + "**/{[A-Z][A-Za-z\d]*$}", + "**/{[A-Z][A-Za-z\d]*Address$}" + ) + + self.specs3Pairs = ( + ("**/{[^A-Za-z]{2}$}",("Id",)), + ("*/{[A-Z][A-Za-z\d]*$}",("Name","Id","Created", "Scope","Driver", "Internal", "Attachable", + "Ingress", "Containers", "Options", "Labels", "IPAM/Driver", "IPAM/Options", + "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", + "ConfigFrom/Network","Containers/199c590e8f13477/Name", + "Containers/199c590e8f13477/MacAddress" )), + ("**/{[A-Z][A-Za-z\d]*\d$}", ("EnableIPv6",)), + ( "**/{[A-Z][A-Za-z\d]*Address$}", ("Containers/199c590e8f13477/MacAddress","Containers/199c590e8f13477/IPv4Address", + "Containers/199c590e8f13477/IPv6Address")), + ) + + + return self + + +class TestSearch(unittest.TestCase): + + def test1(self): + print("Entered test1", file=sys.__stderr__) + dicts = SampleDicts().build() + dict1 = dicts.d1 + specs = dicts.specs1Pairs + for ( spec, expect) in specs: + print(f"Spec={spec}", file=sys.stderr) + for (path,value) in DP.search(dict1, spec, yielded=True): + print( f"\tpath={path}\tv={value}\n\texpected={expect}", + file=sys.stderr) + if path is None: + assert expect is None + else: + assert (path in expect) + print("\n", file=sys.stderr) + + def test2(self): + print("Entered test2", file=sys.__stderr__) + + def afilter(x): + # print(f"In afilter x = {x}({type(x)})", file=sys.stderr) + if isinstance( x, int): + return True + return False + + dicts = SampleDicts().build() + dict1 = dicts.d1 + specs = dicts.specs1 + for spec in specs: + print(f"Spec={spec}", file=sys.stderr) + for ret in DP.search(dict1, spec, yielded=True, afilter=afilter): + print( f"\tret={ret}", file=sys.stderr) + assert (isinstance(ret[1],int)) + + def test3(self): + print("Entered test3", file=sys.__stderr__) + dicts = SampleDicts().build() + dict1 = dicts.d2 + specs = dicts.specs2Pairs + for (spec, expect) in specs: + print(f"Spec={spec}", file=sys.stderr) + for (path,value) in DP.search(dict1, spec, yielded=True): + print( f"\tpath={path}\tv={value}", file=sys.stderr) + if path is None: + assert expect is None + else: + assert (path in expect) + + def test4(self): + print("Entered test4", file=sys.__stderr__) + dicts = SampleDicts().build() + dict1 = dicts.d2 + specs = dicts.specs3Pairs + for (spec, expect) in specs: + print(f"Spec={spec}", file=sys.stderr) + for (path,value) in DP.search(dict1, spec, yielded=True): + print( f"\tpath={path}\tv={value}", file=sys.stderr) + if path is None: + assert expect is None + else: + assert (path in expect) + +class TestGet(unittest.TestCase): + + def test1(self): + print("Entered test1", file=sys.__stderr__) + + dicts = SampleDicts().build() + dict1 = dicts.d1 + specs = dicts.specs1GetPairs + for ( spec, expect) in specs: + print(f"Spec={spec}", file=sys.stderr) + try: + ret = DP.get(dict1, spec, default=("*NONE*",)) + print( f"\tret={ret}", file=sys.stderr) + assert( ret == expect) + except Exception as err: + print("\t get fails:", err, type(err), file=sys.stderr) + assert( expect == "*FAIL*") + +class TestDelete(unittest.TestCase): + def test1(self): + print("This is test1", file=sys.stderr) + dict1 = { + "a": { + "b": 0, + "12": 0, + }, + "a0": { + "b": 0, + }, + } + + specs = (re.compile("[a-z]+$"), re.compile("[a-z]+\d+$"), + "{[a-z]+\d+$}") + i=0 + for spec in specs: + dict = copy(dict1) + print(f"spec={spec}") + print(f"Before deletion dict={dict}", file=sys.stderr) + DP.delete(dict, [spec]) + print(f"After deletion dict={dict}", file=sys.stderr) + if i == 0: + assert(dict == {"a0": {"b": 0,},}) + else: + assert(dict == {"a": { "b": 0, "12": 0, }}) + i+=1 + + +class TestView(unittest.TestCase): + def test1(self): + print("Entered test1", file=sys.__stderr__) + + dicts = SampleDicts().build() + dict1 = dicts.d1 + specs = dicts.specs1 + for spec in specs: + print(f"Spec={spec}", file=sys.stderr) + for ret in DP.segments.view(dict1, spec): + print( f"\tview returns:{ret}", file=sys.stderr) + +class TestMatch(unittest.TestCase): + def test1(self): + print("Entered test1", file=sys.__stderr__) + + dicts = SampleDicts().build() + dict1 = dicts.d1 + specs = dicts.specs1 + for spec in specs: + print(f"Spec={spec}", file=sys.stderr) + if DP.segments.match(dict1, spec): + print( f"Spec={spec} matches", file=sys.stderr) + +if __name__ == "__main__": + + # To debug under the Python debugger: + # A) Use a command like: + # PYTHONPATH="../dpath-source" python3 -m pdb \ + # ../dpath-source/tests/test_various_exts.py + # B) Adapt and uncomment the following + # ts=TestSearch() + # ts.test4() + + + + print(""" + a) This is intended to be run under nose2 and not standalone ! + b) Python script nose_runner adds to nose2 capability to set dpath.options + + Exiting + """, + file=sys.stderr + ) + sys.exit(2) + + + + From 23f491f223457055beac71604b2d46bd9c51b9e7 Mon Sep 17 00:00:00 2001 From: Alain Lichnewsky Date: Mon, 12 Dec 2022 13:54:42 +0100 Subject: [PATCH 03/19] After integration into with update of master on Dec 5, 2022 - tests improved and running locally - documentation updated Follow-up PR: "Add ability to select path segments using Python re regexprs" #146 on June 1, 2021 Baseline for merge: commit 45b34882d8f815299a0be936cac66e8b4587c6e4 (tag: v2.1.2, origin/master, origin/HEAD, master) Author: moomoohk <2220203+moomoohk@users.noreply.github.com> Date: Mon Dec 5 09:52:05 2022 +0200 --- .github/workflows/linterTest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linterTest.yml b/.github/workflows/linterTest.yml index 05ff1b8..f699493 100644 --- a/.github/workflows/linterTest.yml +++ b/.github/workflows/linterTest.yml @@ -1,6 +1,6 @@ name: LinterTest # ------------------------------------------------------------ - # (C) Alain Lichnewsky, 2021 + # (C) Alain Lichnewsky, 2021, 2022 # # For running under Github's Actions # @@ -12,9 +12,9 @@ name: LinterTest on: # ## Not enabled, would triggers the workflow on push or pull request events but - ## only for the AL-addRegexp branch. + ## only for the AL-master4merge branch. #push: - # branches: [ AL-addRegexp ] + # branches: [ AL-master4merge ] # Allows to run this workflow manually from the Github Actions tab workflow_dispatch: From 98cf08a98969ef3c1563d48d715f9e8f7cb39fdb Mon Sep 17 00:00:00 2001 From: Alain Lichnewsky Date: Mon, 12 Dec 2022 17:46:51 +0100 Subject: [PATCH 04/19] Modifications in python3Test.yml for current Python version and running action manually --- .github/workflows/python3Test.yml | 36 ++++++++++++++++++++++--------- tox.ini | 6 ++---- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml index 701d2d0..43c462e 100644 --- a/.github/workflows/python3Test.yml +++ b/.github/workflows/python3Test.yml @@ -1,17 +1,18 @@ -name: Test python package dpath-python +name: Test python package dpath-python with regexp extension # ------------------------------------------------------------ - # (C) Alain Lichnewsky, 2021 + # (C) Alain Lichnewsky, 2021, 2022 # # For running under Github's Actions # # Script performs basic test of the Python-3 version of the package - # including added functionality. + # including added functionality (regexp in search paths). # ------------------------------------------------------------ on: # [push] workflow_dispatch: + # Allows manual dispatch from the Actions tab jobs: test-python3: @@ -20,14 +21,19 @@ jobs: runs-on: ubuntu-latest + strategy: + matrix: + # Match versions specified in tox.ini + python-version: ['3.10', 'pypy-3.7'] + steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@main - - name: Set up Python 3.8 - uses: actions/setup-python@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@main with: - python-version: [ '3.8' , 'pypy-3.7' ] + python-version: ${{ matrix.python-version }} architecture: 'x64' - name: Ascertain configuration @@ -60,7 +66,7 @@ jobs: # required by the package in setup.py # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ run: | - python -m pip install --upgrade pip setuptools wheel + python -m pip install --upgrade pip setuptools wheel nose2 hypothesis if [ -f requirements.txt ]; then pip install -r requirements.txt; fi @@ -70,6 +76,16 @@ jobs: echo which nose2-3.6: $(which nose2-3.6) echo which nose2-3.8: $(which nose2-3.8) + - name: nose_runner testing + + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + # This is the nose2 wrapper that allows to set pgm. parms + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + run: | + echo running the nose2 wrapper + PYTHONPATH="." tests/nose_runner -E -s tests + - name: Tox testing # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -85,8 +101,8 @@ jobs: - name: Nose2 testing # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # Last try... with a nose2.cfg file + # Last try... without a cfg file # # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ run: | - nose2 -c nose2.cfg + nose2 --start-dir tests diff --git a/tox.ini b/tox.ini index d613837..d7ecdff 100644 --- a/tox.ini +++ b/tox.ini @@ -7,13 +7,11 @@ ignore = E501,E722 [tox] -envlist = pypy37, py38, py39, py310 +envlist = pypy37, py310 [gh-actions] python = - pypy-3.7: pypy37 - 3.8: py38 - 3.9: py39 + pypy-3.7: pypy37 3.10: py310 [testenv] From 18fc279f60ab6f33b307d26cdf272f262e563d18 Mon Sep 17 00:00:00 2001 From: Alain Lichnewsky Date: Tue, 13 Dec 2022 08:49:50 +0100 Subject: [PATCH 05/19] Corrects issues found running Github Action test-python3, documentation enhancements 1) suppress dependency on mock now integrated as unittest.mock 2) avoids warnings concerning invalid escape sequences: either use raw strings or use double backslash: testing both 3) detailed the issue of such escape sequences in re.regexp in README.rst, giving example of both techniques No change in .yml: rerunning should resolve identified issues --- README.rst | 17 ++++++++-- tests/test_get_values.py | 4 +-- tests/test_various_exts.py | 69 +++++++++++++++++++++----------------- 3 files changed, 55 insertions(+), 35 deletions(-) diff --git a/README.rst b/README.rst index 05efb49..dfc10d8 100644 --- a/README.rst +++ b/README.rst @@ -491,13 +491,24 @@ Python's `re` regular expressions PythonRe_ may be used as follows: +-----------------------------------------+--------------------------------------+ + "\*\*/{[^A-Za-z]{2}$}" | "Id" + +-----------------------------------------+--------------------------------------+ - + "\*/{[A-Z][A-Za-z\d]*$}" | "Name","Id","Created", "Scope",... + + + r"\*/{[A-Z][A-Za-z\d]*$}" | "Name","Id","Created", "Scope",... + +-----------------------------------------+--------------------------------------+ - + "\*\*/{[A-Z][A-Za-z\d]*\d$}" | EnableIPv6" + + + r"\*\*/{[A-Z][A-Za-z\d]*\d$}" | EnableIPv6" + +-----------------------------------------+--------------------------------------+ - + "\*\*/{[A-Z][A-Za-z\d]*Address$}" | "Containers/199c5/MacAddress" + + + r"\*\*/{[A-Z][A-Za-z\d]*Address$}" | "Containers/199c5/MacAddress" + +-----------------------------------------+--------------------------------------+ + With Python's chararcter string conventions, required backslashes in the `re` syntax + can be entered either in raw strings or using double backslashes, thus + the following are equivalent: + + +-----------------------------------------+----------------------------------------+ + + *with raw strings* | *equivalent* with double backslash + + +-----------------------------------------+----------------------------------------+ + + r"\*\*/{[A-Z][A-Za-z\\d]*\\d$}" | "\*\*/{[A-Z][A-Za-z\\\\d]*\\\\d$}" + + +-----------------------------------------+----------------------------------------+ + + r"\*\*/{[A-Z][A-Za-z\\d]*Address$}" | "\*\*/{[A-Z][A-Za-z\\\\d]*Address$}" + + +-----------------------------------------+----------------------------------------+ diff --git a/tests/test_get_values.py b/tests/test_get_values.py index 9eeef82..5972ccd 100644 --- a/tests/test_get_values.py +++ b/tests/test_get_values.py @@ -2,7 +2,7 @@ import decimal import time -import mock +from unittest.mock import patch from nose2.tools.such import helper import dpath @@ -109,7 +109,7 @@ def test_values(): assert 2 in ret -@mock.patch('dpath.search') +@patch('dpath.search') def test_values_passes_through(searchfunc): searchfunc.return_value = [] diff --git a/tests/test_various_exts.py b/tests/test_various_exts.py index 2a7f3fc..924be06 100644 --- a/tests/test_various_exts.py +++ b/tests/test_various_exts.py @@ -45,10 +45,10 @@ def build(self): [re.compile("[a-z]+$")] , ["*", re.compile(".*")], ["*", "*",re.compile(".*")], - ["*", re.compile("[a-z]+\d+$")], + ["*", re.compile("[a-z]+\\d+$")], ["*", re.compile("[a-z]+[.][a-z]+$")], ["**", re.compile(".*")], - ["**", re.compile("[a-z]+\d+$")], + ["**", re.compile("[a-z]+\\d+$")], ["**", re.compile("[a-z]+[.][a-z]+$")] ) @@ -56,12 +56,12 @@ def build(self): ( [re.compile("[a-z]+$")], None), ( ["*", re.compile(".*")], ("a001/b2",)), ( ["*", "*",re.compile(".*")], ("a001/b2/c1.2",)), - ( ["*", re.compile("[a-z]+\d+$")], ("a001/b2",)) , + ( ["*", re.compile("[a-z]+\\d+$")], ("a001/b2",)) , ( ["*", re.compile("[a-z]+[.][a-z]+$")], None), ( ["**", re.compile(".*")], ("a001","a001/b2", "a001/b2/c1.2","a001/b2/c1.2/d.dd", "a001/b2/c1.2/e.ee", "a001/b2/c1.2/f.f0")), - ( ["**", re.compile("[a-z]+\d+$")], ("a001", "a001/b2")), + ( ["**", re.compile("[a-z]+\\d+$")], ("a001", "a001/b2")), ( ["**", re.compile("[a-z]+[.][a-z]+$")], ("a001/b2/c1.2/d.dd","a001/b2/c1.2/e.ee")) ) @@ -72,10 +72,10 @@ def build(self): ([re.compile("[a-z]+$")] , ('*NONE*',)), (["*", re.compile(".*")],{'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}), (["*", "*",re.compile(".*")],{'d.dd': 0, 'e.ee': 1, 'f.f0': 2} ), - (["*", re.compile("[a-z]+\d+$")],{'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}), + (["*", re.compile("[a-z]+\\d+$")],{'c1.2': {'d.dd': 0, 'e.ee': 1, 'f.f0': 2}}), (["*", re.compile("[a-z]+[.][a-z]+$")],('*NONE*',)), (["**", re.compile(".*")], "*FAIL*"), - (["**", re.compile("[a-z]+\d+$")],"*FAIL*"), + (["**", re.compile("[a-z]+\\d+$")],"*FAIL*"), (["**", re.compile("[a-z]+[.][a-z]+$")],"*FAIL*"), ) @@ -123,46 +123,55 @@ def build(self): "Labels": {} } - self.specs2 = ( - ["*", re.compile("[A-Z][a-z\d]*$")], - ["**", re.compile("[A-Z][a-z\d]*$")], - ["**", re.compile("[A-Z][A-Za-z\d]*Address$")], - ["**", re.compile("[A-Za-z]+\d+$")], - ["**", re.compile("\d+[.]\d+")] - ) self.specs2Pairs = ( - (["*", re.compile("[A-Z][a-z\d]*$")], + (["*", re.compile("[A-Z][a-z\\d]*$")], ("IPAM/Driver","IPAM/Options","IPAM/Config","ConfigFrom/Network")), - (["**", re.compile("[A-Z][a-z\d]*$")], + (["**", re.compile("[A-Z][a-z\\d]*$")], ("Name","Id","Created", "Scope","Driver", "Internal", "Attachable", "Ingress", "Containers", "Options", "Labels", "IPAM/Driver", "IPAM/Options", "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", "ConfigFrom/Network","Containers/199c590e8f13477/Name", "Containers/199c590e8f13477/MacAddress" )), - (["**", re.compile("[A-Z][A-Za-z\d]*Address$")], + (["**", re.compile("[A-Z][A-Za-z\\d]*Address$")], ("Containers/199c590e8f13477/MacAddress","Containers/199c590e8f13477/IPv4Address", "Containers/199c590e8f13477/IPv6Address")), - (["**", re.compile("[A-Za-z]+\d+$")],("EnableIPv6",)), - (["**", re.compile("\d+[.]\d+")],None) - ) + (["**", re.compile("[A-Za-z]+\\d+$")],("EnableIPv6",)), + (["**", re.compile("\\d+[.]\\d+")],None), + + # repeated intentionally using raw strings rather than '\\' escapes + + (["*", re.compile(r"[A-Z][a-z\d]*$")], + ("IPAM/Driver","IPAM/Options","IPAM/Config","ConfigFrom/Network")), + (["**", re.compile(r"[A-Z][a-z\d]*$")], + ("Name","Id","Created", "Scope","Driver", "Internal", "Attachable", + "Ingress", "Containers", "Options", "Labels", "IPAM/Driver", "IPAM/Options", + "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", + "ConfigFrom/Network","Containers/199c590e8f13477/Name", + "Containers/199c590e8f13477/MacAddress" )), + (["**", re.compile(r"[A-Z][A-Za-z\d]*Address$")], + ("Containers/199c590e8f13477/MacAddress","Containers/199c590e8f13477/IPv4Address", + "Containers/199c590e8f13477/IPv6Address")), + (["**", re.compile(r"[A-Za-z]+\d+$")],("EnableIPv6",)), + (["**", re.compile(r"\d+[.]\d+")],None) + ) - self.specs3 = ( ["*", re.compile("[A-Z][a-z\d]*$")], - "**/{.*}", - "*/{[A-Z][A-Za-z\d]*$}", - "**/{[A-Z][A-Za-z\d]*$}", - "**/{[A-Z][A-Za-z\d]*Address$}" - ) self.specs3Pairs = ( ("**/{[^A-Za-z]{2}$}",("Id",)), - ("*/{[A-Z][A-Za-z\d]*$}",("Name","Id","Created", "Scope","Driver", "Internal", "Attachable", + ("*/{[A-Z][A-Za-z\\d]*$}",("Name","Id","Created", "Scope","Driver", "Internal", "Attachable", "Ingress", "Containers", "Options", "Labels", "IPAM/Driver", "IPAM/Options", "IPAM/Config", "IPAM/Config/Subnet", "IPAM/Config/Gateway", "ConfigFrom/Network","Containers/199c590e8f13477/Name", "Containers/199c590e8f13477/MacAddress" )), - ("**/{[A-Z][A-Za-z\d]*\d$}", ("EnableIPv6",)), - ( "**/{[A-Z][A-Za-z\d]*Address$}", ("Containers/199c590e8f13477/MacAddress","Containers/199c590e8f13477/IPv4Address", + ("**/{[A-Z][A-Za-z\\d]*\\d$}", ("EnableIPv6",)), + ("**/{[A-Z][A-Za-z\\d]*Address$}", ("Containers/199c590e8f13477/MacAddress","Containers/199c590e8f13477/IPv4Address", + "Containers/199c590e8f13477/IPv6Address")), + + # repeated intentionally using raw strings rather than '\\' escapes + + (r"**/{[A-Z][A-Za-z\d]*\d$}", ("EnableIPv6",)), + (r"**/{[A-Z][A-Za-z\d]*Address$}", ("Containers/199c590e8f13477/MacAddress","Containers/199c590e8f13477/IPv4Address", "Containers/199c590e8f13477/IPv6Address")), ) @@ -265,8 +274,8 @@ def test1(self): }, } - specs = (re.compile("[a-z]+$"), re.compile("[a-z]+\d+$"), - "{[a-z]+\d+$}") + specs = (re.compile("[a-z]+$"), re.compile("[a-z]+\\d+$"), + "{[a-z]+\\d+$}") i=0 for spec in specs: dict = copy(dict1) From 1a36acc1ef2e18138800aca82540f92e02fa4f11 Mon Sep 17 00:00:00 2001 From: Alain Lichnewsky Date: Tue, 13 Dec 2022 11:23:42 +0100 Subject: [PATCH 06/19] Correction in README.rst, bumped Pypy version in tox.ini --- README.rst | 13 +++++++------ tox.ini | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index dfc10d8..6b67b84 100644 --- a/README.rst +++ b/README.rst @@ -457,9 +457,10 @@ Python's `re` regular expressions PythonRe_ may be used as follows: >>> import dpath >>> dpath.options.DPATH_ACCEPT_RE_REGEXP = True - - Now a path component may also be specified as: - - in a path expression, as {} where `` is a regular expression accepted by the - standard Python module `re`. For example: + - Now a path component may also be specified : + + - in a path expression, as {} where `` is a regular expression + accepted by the standard Python module `re`. For example: .. code-block:: python @@ -491,11 +492,11 @@ Python's `re` regular expressions PythonRe_ may be used as follows: +-----------------------------------------+--------------------------------------+ + "\*\*/{[^A-Za-z]{2}$}" | "Id" + +-----------------------------------------+--------------------------------------+ - + r"\*/{[A-Z][A-Za-z\d]*$}" | "Name","Id","Created", "Scope",... + + + r"\*/{[A-Z][A-Za-z\\d]*$}" | "Name","Id","Created", "Scope",... + +-----------------------------------------+--------------------------------------+ - + r"\*\*/{[A-Z][A-Za-z\d]*\d$}" | EnableIPv6" + + + r"\*\*/{[A-Z][A-Za-z\\d]*\d$}" | EnableIPv6" + +-----------------------------------------+--------------------------------------+ - + r"\*\*/{[A-Z][A-Za-z\d]*Address$}" | "Containers/199c5/MacAddress" + + + r"\*\*/{[A-Z][A-Za-z\\d]*Address$}" | "Containers/199c5/MacAddress" + +-----------------------------------------+--------------------------------------+ With Python's chararcter string conventions, required backslashes in the `re` syntax diff --git a/tox.ini b/tox.ini index d7ecdff..c985caa 100644 --- a/tox.ini +++ b/tox.ini @@ -7,11 +7,11 @@ ignore = E501,E722 [tox] -envlist = pypy37, py310 +envlist = pypy38, py310 [gh-actions] python = - pypy-3.7: pypy37 + pypy-3.8: pypy38 3.10: py310 [testenv] From 932e9e6affc2d1c84d7ce2d922e9ce8d3860d896 Mon Sep 17 00:00:00 2001 From: Alain Lichnewsky Date: Mon, 19 Dec 2022 22:24:19 +0100 Subject: [PATCH 07/19] Resolved Flake8 issues (in full module), added tool to help with sys.path for pypy - Improved Github actions (clean up, test under pypy3) - formatting in accordance with Flake8, deal with errors diagnosed by Flake but accepted by python3 - moved test tools to dir test-utils - tox.ini, flake.ini: avoid spurious diagnostics --- .github/workflows/linterTest.yml | 6 +- .github/workflows/python3-pypy-Test.yml | 55 ++-- .github/workflows/python3Test.yml | 14 +- buildTools/pypy-test | 125 +++++++ dpath/__init__.py | 17 +- dpath/options.py | 2 +- dpath/segments.py | 12 +- flake8.ini | 7 + {tests => test-utils}/nose_runner | 0 test-utils/py_path_checker | 186 +++++++++++ tests/test_get_values.py | 2 +- tests/test_path_ext.py | 20 +- tests/test_segments.py | 2 +- tests/test_various_exts.py | 415 ++++++++++++------------ tox.ini | 6 +- 15 files changed, 588 insertions(+), 281 deletions(-) create mode 100755 buildTools/pypy-test rename {tests => test-utils}/nose_runner (100%) create mode 100755 test-utils/py_path_checker diff --git a/.github/workflows/linterTest.yml b/.github/workflows/linterTest.yml index f699493..05ff1b8 100644 --- a/.github/workflows/linterTest.yml +++ b/.github/workflows/linterTest.yml @@ -1,6 +1,6 @@ name: LinterTest # ------------------------------------------------------------ - # (C) Alain Lichnewsky, 2021, 2022 + # (C) Alain Lichnewsky, 2021 # # For running under Github's Actions # @@ -12,9 +12,9 @@ name: LinterTest on: # ## Not enabled, would triggers the workflow on push or pull request events but - ## only for the AL-master4merge branch. + ## only for the AL-addRegexp branch. #push: - # branches: [ AL-master4merge ] + # branches: [ AL-addRegexp ] # Allows to run this workflow manually from the Github Actions tab workflow_dispatch: diff --git a/.github/workflows/python3-pypy-Test.yml b/.github/workflows/python3-pypy-Test.yml index e62d423..d4154a5 100644 --- a/.github/workflows/python3-pypy-Test.yml +++ b/.github/workflows/python3-pypy-Test.yml @@ -1,18 +1,12 @@ -name: Test python package dpath-python +name: Test python package dpath-python with regexp ext and pypy # ------------------------------------------------------------ - # (C) Alain Lichnewsky, 2021 + # (C) Alain Lichnewsky, 2021, 2022 # # For running under Github's Actions # # Here the idea is to use tox for testing and test on python 3.8 and # pypy-3.7. # - # There are numerous issues that must be understood with the predefined - # features of Github's preloaded containers. - # Here : - # - try and load in 2 separate steps - # - probably not optimal in viexw of preloaded configurations - # # ------------------------------------------------------------ on: @@ -28,22 +22,23 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@main - - name: Set up Python 3.8 - uses: actions/setup-python@v2 + - name: Set up Python 3.10 + uses: actions/setup-python@main with: - python-version: '3.8' + python-version: '3.10' architecture: 'x64' - - name: Set up Pypy 3.7 - uses: actions/setup-python@v2 + - name: Set up Pypy + uses: actions/setup-python@main with: - python-version: 'pypy-3.7' + python-version: 'pypy3.9' architecture: 'x64' - name: Ascertain configuration + shell: bash # # Collect information concerning $HOME and the location of # file(s) loaded from Github/ @@ -72,6 +67,7 @@ jobs: # requirements install the test framework, which is not # required by the package in setup.py # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + shell: bash run: | python -m pip install --upgrade pip setuptools wheel if [ -f requirements.txt ]; then @@ -83,23 +79,38 @@ jobs: echo which nose2-3.6: $(which nose2-3.6) echo which nose2-3.8: $(which nose2-3.8) + - name: Check path preparing for pypy + shell: bash + + run: | + test_utils/py_path_checker -M nose2 -U + pypy3 test_utils/py_path_checker -M nose2 -U + + - name: Perform python tests + shell: bash + if : always() + run: | + PYTHONPATH="." test-utils/nose_runner -E -s tests + + - name: Perform pypy tests + shell: bash + if : always() + run: | + PYTHONPATH=".:/usr/local/lib/python3.10/dist-packages/" pypy3 test-utils/nose_runner -E -s tests + + - name: Tox testing # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ # move to tox testing, otherwise will have to parametrize # nose in more details; here tox.ini will apply # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + shell: bash + if : always() run: | pip install tox echo "Installed tox" tox echo "Ran tox" - - name: Nose2 testing - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # Last try... with a nose2.cfg file - # - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - run: | - nose2 -c nose2.cfg diff --git a/.github/workflows/python3Test.yml b/.github/workflows/python3Test.yml index 43c462e..46a0e38 100644 --- a/.github/workflows/python3Test.yml +++ b/.github/workflows/python3Test.yml @@ -9,8 +9,6 @@ name: Test python package dpath-python with regexp extension # ------------------------------------------------------------ on: - # [push] - workflow_dispatch: # Allows manual dispatch from the Actions tab @@ -24,7 +22,7 @@ jobs: strategy: matrix: # Match versions specified in tox.ini - python-version: ['3.10', 'pypy-3.7'] + python-version: ['3.10', 'pypy3.9'] steps: - name: Checkout code @@ -84,7 +82,7 @@ jobs: run: | echo running the nose2 wrapper - PYTHONPATH="." tests/nose_runner -E -s tests + PYTHONPATH="." test-utils/nose_runner -E -s tests - name: Tox testing @@ -98,11 +96,5 @@ jobs: tox echo "Ran tox" - - name: Nose2 testing - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - # Last try... without a cfg file - # - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - run: | - nose2 --start-dir tests + \ No newline at end of file diff --git a/buildTools/pypy-test b/buildTools/pypy-test new file mode 100755 index 0000000..b059b42 --- /dev/null +++ b/buildTools/pypy-test @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +# ...................................................................... +# Function : +# - check pypy installation +# - run pypy test +# +# +# Normally used/called by devel-Steps -y. +# +# Generalization; +# This proc does not know about installation parameters... files +# and directories need to be passes explicitely +# ...................................................................... +# +# (C) Alain Lichnewsky, 2022 +# +# ...................................................................... +trap 'status=$?; echo "error status is $status"; trap - EXIT; exit $status' ERR +SCRIPTPATH=$(dirname $(realpath $0)) +SCRIPTNAME=$(basename $(realpath $0)) + +# ...................................................................... +# Configurables +# ...................................................................... +PYPY=pypy3 +# ...................................................................... + + +function requireApproval () { + question="$1" + ok="YyOo" + notOK="nN" + pattern="[oOyY]" + prompt="Answer [${ok}]? " + count=0 + + echo "$question" + read -p "${prompt}" -n 1 inchar + while (( $count < 4 )) ; do + if [[ "$inchar" =~ $pattern ]] ; then + echo "" + return + else + exit 1 + fi + count=$(( 1 + $count)) + done + exit 1 +} + +function run_pypy () { + declare -r target="$1" + declare -r pythonpath="$2" + + declare -r pythonpath2="${pythonpath}:${HOME}/.local/lib/python3.10/site-packages/:/usr/local/lib/python3.10/dist-packages/" + export PYTHONPATH="${pythonpath2}" + + echo ${ECHO} PYTHONPATH="${pythonpath2}" + ${ECHO} ${PYPY} ${target} ${@: 3} +} + +function usage() { + less >/dev/stderr <] [-r