Add distribution constraint options to pip download

CHANGES.txt

@@ -2,6 +2,11 @@
 
 * Added optional column formatting to ``pip list`` (:issue:`3651`).
 
+* Add --platform, --python-version, --implementation and --abi parameters to
+  ``pip download``. These allow utilities and advanced users to gather
+  distributions for interpreters other than the one pip is being run on.
+  (:pull:`3092`)
+
 
 **8.1.2 (2016-05-10)**
 

docs/reference/pip_download.rst

@@ -26,9 +26,22 @@ which is now deprecated and will be removed in pip 10.
 ``pip download`` does the same resolution and downloading as ``pip install``,
 but instead of installing the dependencies, it collects the downloaded
 distributions into the directory provided (defaulting to the current
-directory). This directory can later be passed as the value to
-``pip install --find-links`` to facilitate offline or locked down package
-installation.
+directory). This directory can later be passed as the value to ``pip install
+--find-links`` to facilitate offline or locked down package installation.
+
+``pip download`` with the ``--platform``, ``--python-version``,
+``--implementation``, and ``--abi`` options provides the ability to fetch
+dependencies for an interpreter and system other than the ones that pip is
+running on.  ``--only-binary=:all:`` is required when using any of these
+options.  It is important to note that these options all default to the
+current system/interpreter, and not to the most restrictive constraints (e.g.
+platform any, abi none, etc). To avoid fetching dependencies that happen to
+match the constraint of the current interpreter (but not your target one), it
+is recommended to specify all of these options if you are specifying one of
+them. Generic dependencies (e.g. universal wheels, or dependencies with no
+platform, abi, or implementation constraints) will still match an over-
+constrained download requirement.
+
 
 
 Options
@@ -50,4 +63,58 @@ Examples
     $ pip download -d . SomePackage  # equivalent to above
     $ pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage
 
+2. Download a package and all of its dependencies with OSX specific interpreter constraints.
+   This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible,
+   this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``,
+   etc.
+   It will also match deps with platform ``any``.  Also force the interpreter version to ``27``
+   (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``).
+
+  ::
+
+    $ pip download \
+        --only-binary=:all: \
+        --platform macosx-10_10_x86_64 \
+        --python-version 27 \
+        --implementation cp \
+        SomePackage
+
+3.  Download a package and its dependencies with linux specific constraints, including
+    packages that support the ``manylinux1`` platform.  Force the interpreter to be any
+    minor version of py3k, and only accept ``cp34m`` or ``none`` as the abi.
+
+  ::
+
+    $ pip download \
+        --only-binary=:all: \
+        --platform linux_x86_64 --manylinux \
+        --python-version 3 \
+        --implementation cp \
+        --abi cp34m \
+        SomePackage
+
+4.  Force platform, implementation, and abi agnostic deps.
+
+  ::
+
+    $ pip download \
+        --only-binary=:all: \
+        --platform any \
+        --python-version 3 \
+        --implementation py \
+        --abi none \
+        SomePackage
+
+5.  Even when overconstrained, this will still correctly fetch the pip universal wheel.
+
+  ::
 
+    $ pip download \
+        --only-binary=:all: \
+        --platform linux_x86_64 --manylinux \
+        --python-version 33 \
+        --implementation cp \
+        --abi cp34m \
+        pip>=8
+    $ ls pip-8.1.1-py2.py3-none-any.whl
+    pip-8.1.1-py2.py3-none-any.whl

pip/basecommand.py

@@ -311,7 +311,9 @@ def populate_requirement_set(requirement_set, args, options, finder,
                        'to %(name)s (see "pip help %(name)s")' % opts)
             logger.warning(msg)
 
-    def _build_package_finder(self, options, session):
+    def _build_package_finder(self, options, session,
+                              platform=None, python_versions=None,
+                              abi=None, implementation=None):
         """
         Create a package finder appropriate to this requirement command.
         """
@@ -328,4 +330,8 @@ def _build_package_finder(self, options, session):
             allow_all_prereleases=options.pre,
             process_dependency_links=options.process_dependency_links,
             session=session,
+            platform=platform,
+            versions=python_versions,
+            abi=abi,
+            implementation=implementation,
         )

pip/commands/download.py

@@ -3,6 +3,8 @@
 import logging
 import os
 
+from pip.exceptions import CommandError
+from pip.index import FormatControl
 from pip.req import RequirementSet
 from pip.basecommand import RequirementCommand
 from pip import cmdoptions
@@ -63,6 +65,53 @@ def __init__(self, *args, **kw):
             help=("Download packages into <dir>."),
         )
 
+        cmd_opts.add_option(
+            '--platform',
+            dest='platform',
+            metavar='platform',
+            default=None,
+            help=("Only download wheels compatible with <platform>. "
+                  "Defaults to the platform of the local computer."),
+        )
+
+        cmd_opts.add_option(
+            '--python-version',
+            dest='python_version',
+            metavar='python_version',
+            default=None,
+            help=("Only download wheels compatible with Python "
+                  "interpreter version <version>. If not specified, then the "
+                  "current system interpreter minor version is used. A major "
+                  "version (e.g. '2') can be specified to match all "
+                  "minor revs of that major version.  A minor version "
+                  "(e.g. '34') can also be specified."),
+        )
+
+        cmd_opts.add_option(
+            '--implementation',
+            dest='implementation',
+            metavar='implementation',
+            default=None,
+            help=("Only download wheels compatible with Python "
+                  "implementation <implementation>, e.g. 'pp', 'jy', 'cp', "
+                  " or 'ip'. If not specified, then the current "
+                  "interpreter implementation is used.  Use 'py' to force "
+                  "implementation-agnostic wheels."),
+        )
+
+        cmd_opts.add_option(
+            '--abi',
+            dest='abi',
+            metavar='abi',
+            default=None,
+            help=("Only download wheels compatible with Python "
+                  "abi <abi>, e.g. 'pypy_41'.  If not specified, then the "
+                  "current interpreter abi tag is used.  Generally "
+                  "you will need to specify --implementation, "
+                  "--platform, and --python-version when using "
+                  "this option."),
+        )
+
         index_opts = cmdoptions.make_option_group(
             cmdoptions.non_deprecated_index_group,
             self.parser,
@@ -73,14 +122,41 @@ def __init__(self, *args, **kw):
 
     def run(self, options, args):
         options.ignore_installed = True
+
+        if options.python_version:
+            python_versions = [options.python_version]
+        else:
+            python_versions = None
+
+        dist_restriction_set = any([
+            options.python_version,
+            options.platform,
+            options.abi,
+            options.implementation,
+        ])
+        binary_only = FormatControl(set(), set([':all:']))
+        if dist_restriction_set and options.format_control != binary_only:
+            raise CommandError(
+                "--only-binary=:all: must be set and --no-binary must not "
+                "be set (or must be set to :none:) when restricting platform "
+                "and interpreter constraints using --python-version, "
+                "--platform, --abi, or --implementation."
+            )
+
         options.src_dir = os.path.abspath(options.src_dir)
         options.download_dir = normalize_path(options.download_dir)
 
         ensure_dir(options.download_dir)
 
         with self._build_session(options) as session:
-
-            finder = self._build_package_finder(options, session)
+            finder = self._build_package_finder(
+                options=options,
+                session=session,
+                platform=options.platform,
+                python_versions=python_versions,
+                abi=options.abi,
+                implementation=options.implementation,
+            )
             build_delete = (not (options.no_clean or options.build_dir))
             if options.cache_dir and not check_path_owner(options.cache_dir):
                 logger.warning(

pip/index.py

@@ -28,7 +28,7 @@
 )
 from pip.download import HAS_TLS, is_url, path_to_url, url_to_path
 from pip.wheel import Wheel, wheel_ext
-from pip.pep425tags import supported_tags
+from pip.pep425tags import get_supported
 from pip._vendor import html5lib, requests, six
 from pip._vendor.packaging.version import parse as parse_version
 from pip._vendor.packaging.utils import canonicalize_name
@@ -104,12 +104,26 @@ class PackageFinder(object):
 
     def __init__(self, find_links, index_urls, allow_all_prereleases=False,
                  trusted_hosts=None, process_dependency_links=False,
-                 session=None, format_control=None):
+                 session=None, format_control=None, platform=None,
+                 versions=None, abi=None, implementation=None):
         """Create a PackageFinder.
 
         :param format_control: A FormatControl object or None. Used to control
             the selection of source packages / binary packages when consulting
             the index and links.
+        :param platform: A string or None. If None, searches for packages
+            that are supported by the current system. Otherwise, will find
+            packages that can be built on the platform passed in. It is
+            understood that these packages will only be downloaded for
+            distribution: they will not be built locally.
+        :param versions: A list of strings or None. This is passed directly
+            to pep425tags.py in the get_supported() method.
+        :param abi: A string or None. This is passed directly
+            to pep425tags.py in the get_supported() method.
+        :param implementation: A string or None. This is passed directly
+            to pep425tags.py in the get_supported() method.
+        :param manylinux1: A boolean or None. This is passed directly
+            to pep425tags.py in the get_supported() method.
         """
         if session is None:
             raise TypeError(
@@ -153,6 +167,18 @@ def __init__(self, find_links, index_urls, allow_all_prereleases=False,
         # The Session we'll use to make requests
         self.session = session
 
+        # The valid tags to check potential found wheel candidates against
+        self.valid_tags = get_supported(
+            versions=versions,
+            platform=platform,
+            abi=abi,
+            impl=implementation,
+        )
+        self.valid_tags_noarch = get_supported(
+            versions=versions,
+            noarch=True
+        )
+
         # If we don't have TLS enabled, then WARN if anyplace we're looking
         # relies on TLS.
         if not HAS_TLS:
@@ -236,22 +262,22 @@ def _candidate_sort_key(self, candidate):
         If not finding wheels, then sorted by version only.
         If finding wheels, then the sort order is by version, then:
           1. existing installs
-          2. wheels ordered via Wheel.support_index_min()
+          2. wheels ordered via Wheel.support_index_min(self.valid_tags)
           3. source archives
         Note: it was considered to embed this logic into the Link
               comparison operators, but then different sdist links
               with the same version, would have to be considered equal
         """
-        support_num = len(supported_tags)
+        support_num = len(self.valid_tags)
         if candidate.location.is_wheel:
             # can raise InvalidWheelFilename
             wheel = Wheel(candidate.location.filename)
-            if not wheel.supported():
+            if not wheel.supported(self.valid_tags):
                 raise UnsupportedWheel(
                     "%s is not a supported wheel for this platform. It "
                     "can't be sorted." % wheel.filename
                 )
-            pri = -(wheel.support_index_min())
+            pri = -(wheel.support_index_min(self.valid_tags))
         else:  # sdist
             pri = -(support_num)
         return (candidate.version, pri)
@@ -581,7 +607,6 @@ def _log_skipped_link(self, link, reason):
 
     def _link_package_versions(self, link, search):
         """Return an InstallationCandidate or None"""
-
         version = None
         if link.egg_fragment:
             egg_info = link.egg_fragment
@@ -612,7 +637,8 @@ def _link_package_versions(self, link, search):
                     self._log_skipped_link(
                         link, 'wrong project name (not %s)' % search.supplied)
                     return
-                if not wheel.supported():
+
+                if not wheel.supported(self.valid_tags):
                     self._log_skipped_link(
                         link, 'it is not compatible with this Python')
                     return

pip/pep425tags.py

@@ -264,12 +264,19 @@ def _supports_arch(major, minor, arch):
     return arches
 
 
-def get_supported(versions=None, noarch=False):
+def get_supported(versions=None, noarch=False, platform=None,
+                  impl=None, abi=None):
     """Return a list of supported tags for each version specified in
     `versions`.
 
     :param versions: a list of string versions, of the form ["33", "32"],
         or None. The first version will be assumed to support our ABI.
+    :param platform: specify the exact platform you want valid
+        tags for, or None. If None, use the local system platform.
+    :param impl: specify the exact implementation you want valid
+        tags for, or None. If None, use the local interpreter impl.
+    :param abi: specify the exact abi you want valid
+        tags for, or None. If None, use the local interpreter abi.
     """
     supported = []
 
@@ -282,11 +289,11 @@ def get_supported(versions=None, noarch=False):
         for minor in range(version_info[-1], -1, -1):
             versions.append(''.join(map(str, major + (minor,))))
 
-    impl = get_abbr_impl()
+    impl = impl or get_abbr_impl()
 
     abis = []
 
-    abi = get_abi_tag()
+    abi = abi or get_abi_tag()
     if abi:
         abis[0:0] = [abi]
 
@@ -301,8 +308,8 @@ def get_supported(versions=None, noarch=False):
     abis.append('none')
 
     if not noarch:
-        arch = get_platform()
-        if sys.platform == 'darwin':
+        arch = platform or get_platform()
+        if arch.startswith('macosx'):
             # support macosx-10.6-intel on macosx-10.9-x86_64
             match = _osx_arch_pat.match(arch)
             if match:
@@ -315,7 +322,7 @@ def get_supported(versions=None, noarch=False):
             else:
                 # arch pattern didn't match (?!)
                 arches = [arch]
-        elif is_manylinux1_compatible():
+        elif platform is None and is_manylinux1_compatible():
             arches = [arch.replace('linux', 'manylinux1'), arch]
         else:
             arches = [arch]