Merge pull request #1080 from AndydeCleyre/bugfix/321

Pass arbitrary args to 'pip' from pip-compile and pip-sync, with '--pip-args'

README.rst

@@ -201,6 +201,16 @@ Or to output to standard output, use ``--output-file=-``:
     $ pip-compile --output-file=- > requirements.txt
     $ pip-compile - --output-file=- < requirements.in > requirements.txt
 
+Forwarding options to ``pip``
+-----------------------------
+
+Any valid ``pip`` flags or arguments may be passed on with ``pip-compile``'s
+``--pip-args`` option, e.g.
+
+.. code-block:: bash
+
+    $ pip-compile requirements.in --pip-args '--retries 10 --timeout 30'
+
 Configuration
 -------------
 
@@ -368,6 +378,13 @@ line arguments, e.g.
 
 Passing in empty arguments would cause it to default to ``requirements.txt``.
 
+Any valid ``pip install`` flags or arguments may be passed with ``pip-sync``'s
+``--pip-args`` option, e.g.
+
+.. code-block:: bash
+
+    $ pip-sync requirements.txt --pip-args '--no-cache-dir --no-deps'
+
 If you use multiple Python versions, you can run ``pip-sync`` as
 ``py -X.Y -m piptools sync ...`` on Windows and
 ``pythonX.Y -m piptools sync ...`` on other systems.

piptools/scripts/compile.py

@@ -2,6 +2,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import os
+import shlex
 import sys
 import tempfile
 
@@ -177,6 +178,7 @@
     show_envvar=True,
     type=click.Path(file_okay=False, writable=True),
 )
+@click.option("--pip-args", help="Arguments to pass directly to the pip command.")
 def cli(
     ctx,
     verbose,
@@ -204,6 +206,7 @@ def cli(
     build_isolation,
     emit_find_links,
     cache_dir,
+    pip_args,
 ):
     """Compiles requirements.txt from requirements.in specs."""
     log.verbosity = verbose - quiet
@@ -247,6 +250,7 @@ def cli(
     # Setup
     ###
 
+    right_args = shlex.split(pip_args or "")
     pip_args = []
     if find_links:
         for link in find_links:
@@ -268,6 +272,7 @@ def cli(
 
     if not build_isolation:
         pip_args.append("--no-build-isolation")
+    pip_args.extend(right_args)
 
     repository = PyPIRepository(pip_args, cache_dir=cache_dir)
 

piptools/scripts/sync.py

@@ -3,6 +3,7 @@
 
 import itertools
 import os
+import shlex
 import sys
 
 from pip._internal.commands import create_command
@@ -73,6 +74,7 @@
     "the private key and the certificate in PEM format.",
 )
 @click.argument("src_files", required=False, type=click.Path(exists=True), nargs=-1)
+@click.option("--pip-args", help="Arguments to pass directly to pip install.")
 def cli(
     ask,
     dry_run,
@@ -87,6 +89,7 @@ def cli(
     cert,
     client_cert,
     src_files,
+    pip_args,
 ):
     """Synchronize virtual environment with requirements.txt."""
     if not src_files:
@@ -139,7 +142,7 @@ def cli(
         user_only=user_only,
         cert=cert,
         client_cert=client_cert,
-    )
+    ) + shlex.split(pip_args or "")
     sys.exit(
         sync.sync(
             to_install,

piptools/utils.py

@@ -328,6 +328,10 @@ def get_compile_command(click_ctx):
         # Collect variadic args separately, they will be added
         # at the end of the command later
         if option.nargs < 0:
+            # These will necessarily be src_files
+            # Re-add click-stripped '--' if any start with '-'
+            if any(val.startswith("-") and val != "-" for val in value):
+                right_args.append("--")
             right_args.extend([shlex_quote(force_text(val)) for val in value])
             continue
 
@@ -366,10 +370,20 @@ def get_compile_command(click_ctx):
                 left_args.append(shlex_quote(arg))
             # Append to args the option with a value
             else:
-                left_args.append(
-                    "{option}={value}".format(
-                        option=option_long_name, value=shlex_quote(force_text(val))
+                if option.name == "pip_args":
+                    # shlex_quote would produce functional but noisily quoted results,
+                    # e.g. --pip-args='--cache-dir='"'"'/tmp/with spaces'"'"''
+                    # Instead, we try to get more legible quoting via repr:
+                    left_args.append(
+                        "{option}={value}".format(
+                            option=option_long_name, value=repr(fs_str(force_text(val)))
+                        )
+                    )
+                else:
+                    left_args.append(
+                        "{option}={value}".format(
+                            option=option_long_name, value=shlex_quote(force_text(val))
+                        )
                     )
-                )
 
     return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args))

tests/test_cli_compile.py

@@ -730,7 +730,7 @@ def test_allow_unsafe_option(pip_conf, monkeypatch, runner, option, expected):
 @mock.patch("piptools.scripts.compile.parse_requirements")
 def test_cert_option(parse_requirements, runner, option, attr, expected):
     """
-    The options --cert and --client-crt have to be passed to the PyPIRepository.
+    The options --cert and --client-cert have to be passed to the PyPIRepository.
     """
     with open("requirements.in", "w"):
         pass
@@ -759,6 +759,20 @@ def test_build_isolation_option(parse_requirements, runner, option, expected):
     assert parse_requirements.call_args.kwargs["options"].build_isolation is expected
 
 
+@mock.patch("piptools.scripts.compile.PyPIRepository")
+def test_forwarded_args(PyPIRepository, runner):
+    """
+    Test the forwarded cli args (--pip-args 'arg...') are passed to the pip command.
+    """
+    with open("requirements.in", "w"):
+        pass
+
+    cli_args = ("--no-annotate", "--generate-hashes")
+    pip_args = ("--no-color", "--isolated", "--disable-pip-version-check")
+    runner.invoke(cli, cli_args + ("--pip-args", " ".join(pip_args)))
+    assert set(pip_args).issubset(set(PyPIRepository.call_args.args[0]))
+
+
 @pytest.mark.parametrize(
     "cli_option, infile_option, expected_package",
     [

tests/test_cli_sync.py

@@ -111,31 +111,52 @@ def test_merge_error(runner):
 
 
 @pytest.mark.parametrize(
-    "install_flags",
+    ("cli_flags", "expected_install_flags"),
     [
-        ["--find-links", "./libs1", "--find-links", "./libs2"],
-        ["--no-index"],
-        ["--index-url", "https://example.com"],
-        ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
-        ["--trusted-host", "foo", "--trusted-host", "bar"],
-        ["--user"],
-        ["--cert", "foo.crt"],
-        ["--client-cert", "foo.pem"],
+        (
+            ["--find-links", "./libs1", "--find-links", "./libs2"],
+            ["--find-links", "./libs1", "--find-links", "./libs2"],
+        ),
+        (["--no-index"], ["--no-index"]),
+        (
+            ["--index-url", "https://example.com"],
+            ["--index-url", "https://example.com"],
+        ),
+        (
+            ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
+            ["--extra-index-url", "https://foo", "--extra-index-url", "https://bar"],
+        ),
+        (
+            ["--trusted-host", "foo", "--trusted-host", "bar"],
+            ["--trusted-host", "foo", "--trusted-host", "bar"],
+        ),
+        (["--user"], ["--user"]),
+        (["--cert", "foo.crt"], ["--cert", "foo.crt"]),
+        (["--client-cert", "foo.pem"], ["--client-cert", "foo.pem"]),
+        (
+            ["--pip-args", "--no-cache-dir --no-deps --no-warn-script-location"],
+            ["--no-cache-dir", "--no-deps", "--no-warn-script-location"],
+        ),
+        (["--pip-args='--cache-dir=/tmp'"], ["--cache-dir=/tmp"]),
+        (
+            ["--pip-args=\"--cache-dir='/tmp/cache dir with spaces/'\""],
+            ["--cache-dir='/tmp/cache dir with spaces/'"],
+        ),
     ],
 )
 @mock.patch("piptools.sync.check_call")
-def test_pip_install_flags(check_call, install_flags, runner):
+def test_pip_install_flags(check_call, cli_flags, expected_install_flags, runner):
     """
     Test the cli flags have to be passed to the pip install command.
     """
     with open("requirements.txt", "w") as req_in:
         req_in.write("six==1.10.0")
 
-    runner.invoke(cli, install_flags)
+    runner.invoke(cli, cli_flags)
 
     call_args = [call[0][0] for call in check_call.call_args_list]
     called_install_options = [args[6:] for args in call_args if args[3] == "install"]
-    assert called_install_options == [install_flags], "Called args: {}".format(
+    assert called_install_options == [expected_install_flags], "Called args: {}".format(
         call_args
     )
 

tests/test_utils.py

@@ -265,6 +265,15 @@ def test_force_text(value, expected_text):
         (["-f", "συνδέσεις"], "pip-compile --find-links='συνδέσεις'"),
         (["-o", "my file.txt"], "pip-compile --output-file='my file.txt'"),
         (["-o", "απαιτήσεις.txt"], "pip-compile --output-file='απαιτήσεις.txt'"),
+        # Check '--pip-args' (forwarded) arguments
+        (
+            ["--pip-args", "--disable-pip-version-check"],
+            "pip-compile --pip-args='--disable-pip-version-check'",
+        ),
+        (
+            ["--pip-args", "--disable-pip-version-check --isolated"],
+            "pip-compile --pip-args='--disable-pip-version-check --isolated'",
+        ),
     ],
 )
 def test_get_compile_command(tmpdir_cwd, cli_args, expected_command):
@@ -275,6 +284,16 @@ def test_get_compile_command(tmpdir_cwd, cli_args, expected_command):
         assert get_compile_command(ctx) == expected_command
 
 
+def test_get_compile_command_escaped_filenames(tmpdir_cwd):
+    """
+    Test that get_compile_command output (re-)escapes ' -- '-escaped filenames.
+    """
+    with open("--requirements.in", "w"):
+        pass
+    with compile_cli.make_context("pip-compile", ["--", "--requirements.in"]) as ctx:
+        assert get_compile_command(ctx) == "pip-compile -- --requirements.in"
+
+
 @mark.parametrize(
     "filename", ["requirements.in", "my requirements.in", "απαιτήσεις.txt"]
 )