Test with tomli

.github/workflows/main.yml

@@ -69,6 +69,8 @@ jobs:
           py310-pipgit-toml,
           py310-piplatest-pytoml,
           py310-pipgit-pytoml,
+          py310-piplatest-tomli,
+          py310-pipgit-tomli,
           mypy,
         ]
 

.pre-commit-config.yaml

@@ -29,6 +29,7 @@ repos:
     rev: v3.3.0
     hooks:
       - id: check-toml
+        exclude: 'tests/data/install/invalid_*'
       - id: check-yaml
       - id: end-of-file-fixer
       - id: trailing-whitespace

micropipenv.py

@@ -185,9 +185,22 @@ def _check_pip_version(raise_on_incompatible=False):  # type: (bool) -> bool
 
 def _import_toml():  # type: () -> Any
     """Import and return toml or pytoml module (in this order)."""
-    for module in "toml", "pytoml":
+    exception_names = {
+        "toml": "TomlDecodeError",
+        "pytoml": "TomlError",
+        "tomli": "TOMLDecodeError",
+    }
+
+    # Only tomli requires TOML files to be opened
+    # in binary mode: https://github.com/hukkin/tomli#parse-a-toml-file
+    open_kwargs = defaultdict(dict)  # type: Dict[str, Dict[str, str]]
+    open_kwargs["tomli"] = {"mode": "rb"}
+
+    for module_name in "toml", "pytoml", "tomli":
         try:
-            return import_module(module)
+            module = import_module(module_name)
+            exception = getattr(module, exception_names[module_name])
+            return module, exception, open_kwargs[module_name]
         except ImportError:
             pass
     else:
@@ -232,38 +245,38 @@ def _read_pipfile_lock():  # type: () -> Any
 
 def _read_pipfile():  # type: () -> Any
     """Find and read Pipfile."""
-    toml = _import_toml()
+    toml, toml_exception, open_kwargs = _import_toml()
 
     pipfile_path = _traverse_up_find_file("Pipfile")
 
     try:
-        with open(pipfile_path) as input_file:
+        with open(pipfile_path, **open_kwargs) as input_file:
             return toml.load(input_file)
-    except toml.TomlDecodeError as exc:
+    except toml_exception as exc:
         raise FileReadError("Failed to parse Pipfile: {}".format(str(exc))) from exc
     except Exception as exc:
         raise FileReadError(str(exc)) from exc
 
 
 def _read_poetry():  # type: () -> Tuple[MutableMapping[str, Any], MutableMapping[str, Any]]
     """Find and read poetry.lock and pyproject.toml."""
-    toml = _import_toml()
+    toml, toml_exception, open_kwargs = _import_toml()
 
     poetry_lock_path = _traverse_up_find_file("poetry.lock")
     pyproject_toml_path = _traverse_up_find_file("pyproject.toml")
 
     try:
-        with open(poetry_lock_path) as input_file:
+        with open(poetry_lock_path, **open_kwargs) as input_file:
             poetry_lock = toml.load(input_file)
-    except toml.TomlDecodeError as exc:
+    except toml_exception as exc:
         raise FileReadError("Failed to parse poetry.lock: {}".format(str(exc))) from exc
     except Exception as exc:
         raise FileReadError(str(exc)) from exc
 
     try:
-        with open(pyproject_toml_path) as input_file:
+        with open(pyproject_toml_path, **open_kwargs) as input_file:
             pyproject_toml = toml.load(input_file)
-    except toml.TomlDecodeError as exc:
+    except toml_exception as exc:
         raise FileReadError("Failed to parse pyproject.toml: {}".format(str(exc))) from exc
     except Exception as exc:
         raise FileReadError(str(exc)) from exc

setup.cfg

@@ -26,7 +26,7 @@ classifiers =
 
 
 [options]
-py_modules = 
+py_modules =
     micropipenv
 install_requires =
     pip>=9

tests/conftest.py

@@ -31,6 +31,8 @@
 # Version of pip to test micropipenv with
 # the default is the pip wheel bundled in virtualenv package
 MICROPIPENV_TEST_PIP_VERSION = os.getenv("MICROPIPENV_TEST_PIP_VERSION")
+# For some very old pips, we have to limit also setuptools version
+MICROPIPENV_TEST_SETUPTOOLS_VERSION = os.getenv("MICROPIPENV_TEST_SETUPTOOLS_VERSION")
 # pip version used in tests, assigned using pytest_configure
 PIP_VERSION = None
 
@@ -46,6 +48,9 @@ def _venv_install_pip(venv):
         else:
             venv.install(f"pip{MICROPIPENV_TEST_PIP_VERSION}")
 
+    if MICROPIPENV_TEST_SETUPTOOLS_VERSION is not None:
+        venv.install(f"setuptools{MICROPIPENV_TEST_SETUPTOOLS_VERSION}")
+
 
 def pytest_configure(config):
     """Configure tests before pytest collects tests."""

tests/data/install/invalid_pipfile/Pipfile

@@ -0,0 +1 @@
+completely invalid pipfile

tests/data/install/invalid_poetry_lock/pyproject.toml

@@ -0,0 +1,18 @@
+[tool.poetry]
+name = "test-micropipenv"
+version = "0.1.0"
+description = ""
+authors = ["Fridolin Pokorny <fridolin@gmail.com>"]
+
+[tool.poetry.dependencies]
+python = "^3.7"
+micropipenv = {version = "0.0.1", optional = true}
+selinon = {version = "1.0.0", extras = ["s3", "postgres"]}
+daiquiri = {git = "https://github.com/jd/daiquiri", rev = "2.1.0"}
+
+[tool.poetry.dev-dependencies]
+hexsticker = "^1.2.0"
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"

tests/data/install/invalid_pyproject_toml/pyproject.toml

@@ -0,0 +1 @@
+completely invalid pyproject.toml file

tests/data/install/requirements_vcs/requirements.txt

@@ -1,4 +1,4 @@
 --index-url https://pypi.org/simple
 
 # No hashes, raw pip method is used.
-git+git://github.com/jd/daiquiri@2.1.0#egg=daiquiri
+git+https://github.com/jd/daiquiri@2.1.0#egg=daiquiri

tests/test_micropipenv.py

@@ -452,6 +452,26 @@ def test_install_pipenv_iter_index(venv):
         assert str(venv.get_version("requests")) == "2.22.0"
 
 
+@pytest.mark.parametrize(
+    "directory, filename, method",
+    [
+        ["invalid_pipfile", "Pipfile", "pipenv"],
+        ["invalid_poetry_lock", "poetry.lock", "poetry"],
+        ["invalid_pyproject_toml", "pyproject.toml", "poetry"],
+    ],
+)
+def test_install_invalid_toml_file(venv, directory, filename, method):
+    """Test exception when a Pipfile is not a valid TOML."""
+    venv.install(MICROPIPENV_TEST_TOML_MODULE)
+    cmd = [os.path.join(venv.path, BIN_DIR, "python"), micropipenv.__file__, "install", "--deploy", "--method", method]
+    work_dir = os.path.join(_DATA_DIR, "install", directory)
+    with cwd(work_dir):
+        with pytest.raises(subprocess.CalledProcessError) as exception:
+            subprocess.check_output(cmd, env=get_updated_env(venv), stderr=subprocess.PIPE, universal_newlines=True)
+
+    assert f"FileReadError: Failed to parse {filename}: " in exception.value.stderr
+
+
 def test_parse_requirements2pipfile_lock():
     """Test parsing of requirements.txt into their Pipfile.lock representation."""
     work_dir = os.path.join(_DATA_DIR, "parse", "pip-tools")

tox.ini

@@ -9,6 +9,7 @@ envlist =
     py{38,39}-pip{90,latest,git}-pytoml
     py310-pip{192,193,203,213,latest,git}-toml
     py310-pip{latest,git}-pytoml
+    py310-pip{latest,git}-tomli
     mypy
 skipsdist = True
 
@@ -22,9 +23,13 @@ deps =
     packaging
     toml: toml
     pytoml: pytoml
+    tomli: tomli
 setenv =
     # platform-python-pip and python27-pip in RHEL8
-    pip90:  MICROPIPENV_TEST_PIP_VERSION = >=9.0,<10.0
+    # pip 9 seems not to be compatible with the latest setuptools
+    # so we are using the same version we have in RHEL 8
+    pip90: MICROPIPENV_TEST_PIP_VERSION = >=9.0,<10.0
+    pip90: MICROPIPENV_TEST_SETUPTOOLS_VERSION = <60
     # older version in Python 3.8 module
     pip192: MICROPIPENV_TEST_PIP_VERSION = >=19.2,<19.3
     # first version with manylinux2014 support, Python 3.8 module, Fedora 32
@@ -40,6 +45,7 @@ setenv =
     # Two implementations of toml format
     toml: MICROPIPENV_TEST_TOML_MODULE = toml
     pytoml: MICROPIPENV_TEST_TOML_MODULE = pytoml
+    tomli:  MICROPIPENV_TEST_TOML_MODULE = tomli
 
 [testenv:mypy]
 commands = mypy micropipenv.py