diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8b6eca9..1d4918d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,6 +69,8 @@ jobs: py310-pipgit-toml, py310-piplatest-pytoml, py310-pipgit-pytoml, + py310-piplatest-tomli, + py310-pipgit-tomli, mypy, ] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d2d9c95..e4791a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,6 +29,7 @@ repos: rev: v3.3.0 hooks: - id: check-toml + exclude: 'tests/data/install/invalid_*' - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace diff --git a/micropipenv.py b/micropipenv.py index 9a2942a..cb93402 100755 --- a/micropipenv.py +++ b/micropipenv.py @@ -185,9 +185,22 @@ def _check_pip_version(raise_on_incompatible=False): # type: (bool) -> bool def _import_toml(): # type: () -> Any """Import and return toml or pytoml module (in this order).""" - for module in "toml", "pytoml": + exception_names = { + "toml": "TomlDecodeError", + "pytoml": "TomlError", + "tomli": "TOMLDecodeError", + } + + # Only tomli requires TOML files to be opened + # in binary mode: https://github.com/hukkin/tomli#parse-a-toml-file + open_kwargs = defaultdict(dict) # type: Dict[str, Dict[str, str]] + open_kwargs["tomli"] = {"mode": "rb"} + + for module_name in "toml", "pytoml", "tomli": try: - return import_module(module) + module = import_module(module_name) + exception = getattr(module, exception_names[module_name]) + return module, exception, open_kwargs[module_name] except ImportError: pass else: @@ -232,14 +245,14 @@ def _read_pipfile_lock(): # type: () -> Any def _read_pipfile(): # type: () -> Any """Find and read Pipfile.""" - toml = _import_toml() + toml, toml_exception, open_kwargs = _import_toml() pipfile_path = _traverse_up_find_file("Pipfile") try: - with open(pipfile_path) as input_file: + with open(pipfile_path, **open_kwargs) as input_file: return toml.load(input_file) - except toml.TomlDecodeError as exc: + except toml_exception as exc: raise FileReadError("Failed to parse Pipfile: {}".format(str(exc))) from exc except Exception as exc: raise FileReadError(str(exc)) from exc @@ -247,23 +260,23 @@ def _read_pipfile(): # type: () -> Any def _read_poetry(): # type: () -> Tuple[MutableMapping[str, Any], MutableMapping[str, Any]] """Find and read poetry.lock and pyproject.toml.""" - toml = _import_toml() + toml, toml_exception, open_kwargs = _import_toml() poetry_lock_path = _traverse_up_find_file("poetry.lock") pyproject_toml_path = _traverse_up_find_file("pyproject.toml") try: - with open(poetry_lock_path) as input_file: + with open(poetry_lock_path, **open_kwargs) as input_file: poetry_lock = toml.load(input_file) - except toml.TomlDecodeError as exc: + except toml_exception as exc: raise FileReadError("Failed to parse poetry.lock: {}".format(str(exc))) from exc except Exception as exc: raise FileReadError(str(exc)) from exc try: - with open(pyproject_toml_path) as input_file: + with open(pyproject_toml_path, **open_kwargs) as input_file: pyproject_toml = toml.load(input_file) - except toml.TomlDecodeError as exc: + except toml_exception as exc: raise FileReadError("Failed to parse pyproject.toml: {}".format(str(exc))) from exc except Exception as exc: raise FileReadError(str(exc)) from exc diff --git a/setup.cfg b/setup.cfg index 67227eb..87f367c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,7 +26,7 @@ classifiers = [options] -py_modules = +py_modules = micropipenv install_requires = pip>=9 diff --git a/tests/conftest.py b/tests/conftest.py index 87ddbc2..4c52a0a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,8 @@ # Version of pip to test micropipenv with # the default is the pip wheel bundled in virtualenv package MICROPIPENV_TEST_PIP_VERSION = os.getenv("MICROPIPENV_TEST_PIP_VERSION") +# For some very old pips, we have to limit also setuptools version +MICROPIPENV_TEST_SETUPTOOLS_VERSION = os.getenv("MICROPIPENV_TEST_SETUPTOOLS_VERSION") # pip version used in tests, assigned using pytest_configure PIP_VERSION = None @@ -46,6 +48,9 @@ def _venv_install_pip(venv): else: venv.install(f"pip{MICROPIPENV_TEST_PIP_VERSION}") + if MICROPIPENV_TEST_SETUPTOOLS_VERSION is not None: + venv.install(f"setuptools{MICROPIPENV_TEST_SETUPTOOLS_VERSION}") + def pytest_configure(config): """Configure tests before pytest collects tests.""" diff --git a/tests/data/install/invalid_pipfile/Pipfile b/tests/data/install/invalid_pipfile/Pipfile new file mode 100644 index 0000000..7eed2a9 --- /dev/null +++ b/tests/data/install/invalid_pipfile/Pipfile @@ -0,0 +1 @@ +completely invalid pipfile diff --git a/tests/data/install/invalid_pipfile/Pipfile.lock b/tests/data/install/invalid_pipfile/Pipfile.lock new file mode 100644 index 0000000..d7e24dd --- /dev/null +++ b/tests/data/install/invalid_pipfile/Pipfile.lock @@ -0,0 +1,35 @@ +{ + "_meta": { + "hash": { + "sha256": "ddaa9d41aa9d6aa9adf294fc44af2baf00bb452085d043566f6609237273ab4f" + }, + "pipfile-spec": 6, + "sources": [ + { + "name": "5a06749b2297be54ac5699f6f2761716adc5001a2d5f8b915ab2172922dd5706", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "attrs": { + "hashes": [ + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + ], + "index": "5a06749b2297be54ac5699f6f2761716adc5001a2d5f8b915ab2172922dd5706", + "version": "==19.3.0" + } + }, + "develop": { + "hexsticker": { + "hashes": [ + "sha256:28aac200b57dd4eabf102c78e0d6254cc893b5061fe11d7f157ee9c72b497de4", + "sha256:1e328c8d652a05ed829e66f56b7dc8a9f20fa91a16c5b0fa914f7f423692696b" + ], + "index": "5a06749b2297be54ac5699f6f2761716adc5001a2d5f8b915ab2172922dd5706", + "version": "==1.2.0" + } + } +} diff --git a/tests/data/install/invalid_poetry_lock/poetry.lock b/tests/data/install/invalid_poetry_lock/poetry.lock new file mode 100644 index 0000000..2456967 --- /dev/null +++ b/tests/data/install/invalid_poetry_lock/poetry.lock @@ -0,0 +1 @@ +completely invalid poetry.lock file diff --git a/tests/data/install/invalid_poetry_lock/pyproject.toml b/tests/data/install/invalid_poetry_lock/pyproject.toml new file mode 100644 index 0000000..0e21720 --- /dev/null +++ b/tests/data/install/invalid_poetry_lock/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "test-micropipenv" +version = "0.1.0" +description = "" +authors = ["Fridolin Pokorny "] + +[tool.poetry.dependencies] +python = "^3.7" +micropipenv = {version = "0.0.1", optional = true} +selinon = {version = "1.0.0", extras = ["s3", "postgres"]} +daiquiri = {git = "https://github.com/jd/daiquiri", rev = "2.1.0"} + +[tool.poetry.dev-dependencies] +hexsticker = "^1.2.0" + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" diff --git a/tests/data/install/invalid_pyproject_toml/poetry.lock b/tests/data/install/invalid_pyproject_toml/poetry.lock new file mode 100644 index 0000000..5c4ef1a --- /dev/null +++ b/tests/data/install/invalid_pyproject_toml/poetry.lock @@ -0,0 +1,23 @@ +[[package]] +category = "main" +description = "Classes Without Boilerplate" +name = "attrs" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "19.3.0" + +[package.extras] +azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] +dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"] +docs = ["sphinx", "zope.interface"] +tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] + +[metadata] +content-hash = "ddaa9d41aa9d6aa9adf294fc44af2baf00bb452085d043566f6609237273ab4f" +python-versions = "^3.7" + +[metadata.files] +attrs = [ + {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, + {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, +] diff --git a/tests/data/install/invalid_pyproject_toml/pyproject.toml b/tests/data/install/invalid_pyproject_toml/pyproject.toml new file mode 100644 index 0000000..81e417b --- /dev/null +++ b/tests/data/install/invalid_pyproject_toml/pyproject.toml @@ -0,0 +1 @@ +completely invalid pyproject.toml file diff --git a/tests/data/install/pipenv_vcs/Pipfile.lock b/tests/data/install/pipenv_vcs/Pipfile.lock index 56d5a3c..70930fc 100644 --- a/tests/data/install/pipenv_vcs/Pipfile.lock +++ b/tests/data/install/pipenv_vcs/Pipfile.lock @@ -15,7 +15,7 @@ }, "default": { "daiquiri": { - "git": "git://github.com/jd/daiquiri", + "git": "https://github.com/jd/daiquiri", "ref": "66933d792171e2c1362ce036b797da56c9b0ea45", "version": "==2.0.0" } diff --git a/tests/data/install/requirements_vcs/requirements.txt b/tests/data/install/requirements_vcs/requirements.txt index 8c31582..0bfede9 100644 --- a/tests/data/install/requirements_vcs/requirements.txt +++ b/tests/data/install/requirements_vcs/requirements.txt @@ -1,4 +1,4 @@ --index-url https://pypi.org/simple # No hashes, raw pip method is used. -git+git://github.com/jd/daiquiri@2.1.0#egg=daiquiri +git+https://github.com/jd/daiquiri@2.1.0#egg=daiquiri diff --git a/tests/test_micropipenv.py b/tests/test_micropipenv.py index 97f09fd..7ee9735 100644 --- a/tests/test_micropipenv.py +++ b/tests/test_micropipenv.py @@ -452,6 +452,26 @@ def test_install_pipenv_iter_index(venv): assert str(venv.get_version("requests")) == "2.22.0" +@pytest.mark.parametrize( + "directory, filename, method", + [ + ["invalid_pipfile", "Pipfile", "pipenv"], + ["invalid_poetry_lock", "poetry.lock", "poetry"], + ["invalid_pyproject_toml", "pyproject.toml", "poetry"], + ], +) +def test_install_invalid_toml_file(venv, directory, filename, method): + """Test exception when a Pipfile is not a valid TOML.""" + venv.install(MICROPIPENV_TEST_TOML_MODULE) + cmd = [os.path.join(venv.path, BIN_DIR, "python"), micropipenv.__file__, "install", "--deploy", "--method", method] + work_dir = os.path.join(_DATA_DIR, "install", directory) + with cwd(work_dir): + with pytest.raises(subprocess.CalledProcessError) as exception: + subprocess.check_output(cmd, env=get_updated_env(venv), stderr=subprocess.PIPE, universal_newlines=True) + + assert f"FileReadError: Failed to parse {filename}: " in exception.value.stderr + + def test_parse_requirements2pipfile_lock(): """Test parsing of requirements.txt into their Pipfile.lock representation.""" work_dir = os.path.join(_DATA_DIR, "parse", "pip-tools") diff --git a/tox.ini b/tox.ini index 728e939..62ea835 100644 --- a/tox.ini +++ b/tox.ini @@ -9,6 +9,7 @@ envlist = py{38,39}-pip{90,latest,git}-pytoml py310-pip{192,193,203,213,latest,git}-toml py310-pip{latest,git}-pytoml + py310-pip{latest,git}-tomli mypy skipsdist = True @@ -22,9 +23,13 @@ deps = packaging toml: toml pytoml: pytoml + tomli: tomli setenv = # platform-python-pip and python27-pip in RHEL8 - pip90: MICROPIPENV_TEST_PIP_VERSION = >=9.0,<10.0 + # pip 9 seems not to be compatible with the latest setuptools + # so we are using the same version we have in RHEL 8 + pip90: MICROPIPENV_TEST_PIP_VERSION = >=9.0,<10.0 + pip90: MICROPIPENV_TEST_SETUPTOOLS_VERSION = <60 # older version in Python 3.8 module pip192: MICROPIPENV_TEST_PIP_VERSION = >=19.2,<19.3 # first version with manylinux2014 support, Python 3.8 module, Fedora 32 @@ -40,6 +45,7 @@ setenv = # Two implementations of toml format toml: MICROPIPENV_TEST_TOML_MODULE = toml pytoml: MICROPIPENV_TEST_TOML_MODULE = pytoml + tomli: MICROPIPENV_TEST_TOML_MODULE = tomli [testenv:mypy] commands = mypy micropipenv.py