Add {pip,whl}tool_wrapper.py as a replacement for {pip,whl}tool.par

python/pip.bzl

@@ -43,7 +43,7 @@ pip_import = repository_rule(
         ),
         "_script": attr.label(
             executable = True,
-            default = Label("//tools:piptool.par"),
+            default = Label("//tools:piptool_wrapper.py"),
             cfg = "host",
         ),
     },

python/whl.bzl

@@ -44,7 +44,7 @@ whl_library = repository_rule(
         "extras": attr.string_list(),
         "_script": attr.label(
             executable = True,
-            default = Label("//tools:whltool.par"),
+            default = Label("//tools:whltool_wrapper.py"),
             cfg = "host",
         ),
     },

rules_python/piptool.py

@@ -25,76 +25,36 @@
 import tempfile
 import zipfile
 
-# Note: We carefully import the following modules in a particular
-# order, since these modules modify the import path and machinery.
 import pkg_resources
-
-
-def extract_packages(package_names):
-    """Extract zipfile contents to disk and add to import path"""
-
-    # Set a safe extraction dir
-    extraction_tmpdir = tempfile.mkdtemp()
-    atexit.register(lambda: shutil.rmtree(
-        extraction_tmpdir, ignore_errors=True))
-    pkg_resources.set_extraction_path(extraction_tmpdir)
-
-    # Extract each package to disk
-    dirs_to_add = []
-    for package_name in package_names:
-        req = pkg_resources.Requirement.parse(package_name)
-        extraction_dir = pkg_resources.resource_filename(req, '')
-        dirs_to_add.append(extraction_dir)
-
-    # Add extracted directories to import path ahead of their zip file
-    # counterparts.
-    sys.path[0:0] = dirs_to_add
-    existing_pythonpath = os.environ.get('PYTHONPATH')
-    if existing_pythonpath:
-        dirs_to_add.extend(existing_pythonpath.split(':'))
-    os.environ['PYTHONPATH'] = ':'.join(dirs_to_add)
-
-
-# Wheel, pip, and setuptools are much happier running from actual
-# files on disk, rather than entries in a zipfile.  Extract zipfile
-# contents, add those contents to the path, then import them.
-extract_packages(['pip', 'setuptools', 'wheel'])
-
-# Defeat pip's attempt to mangle sys.path
-saved_sys_path = sys.path
-sys.path = sys.path[:]
 import pip
-sys.path = saved_sys_path
-
 import setuptools
 import wheel
 
 
 def pip_main(argv):
-    # Extract the certificates from the PAR following the example of get-pip.py
-    # https://github.com/pypa/get-pip/blob/430ba37776ae2ad89/template.py#L164-L168
-    cert_path = os.path.join(tempfile.mkdtemp(), "cacert.pem")
-    with open(cert_path, "wb") as cert:
-      cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem"))
-    argv = ["--disable-pip-version-check", "--cert", cert_path] + argv
+    argv = ["--disable-pip-version-check"] + argv
     return pip.main(argv)
 
-from rules_python.whl import Wheel
+from whl import Wheel
 
 parser = argparse.ArgumentParser(
     description='Import Python dependencies into Bazel.')
 
 parser.add_argument('--name', action='store',
-                    help=('The namespace of the import.'))
+                    help=('The namespace of the import.'),
+                    required=True)
 
 parser.add_argument('--input', action='store',
-                    help=('The requirements.txt file to import.'))
+                    help=('The requirements.txt file to import.'),
+                    required=True)
 
 parser.add_argument('--output', action='store',
-                    help=('The requirements.bzl file to export.'))
+                    help=('The requirements.bzl file to export.'),
+                    required=True)
 
 parser.add_argument('--directory', action='store',
-                    help=('The directory into which to put .whl files.'))
+                    help=('The directory into which to put .whl files.'),
+                    required=True)
 
 def determine_possible_extras(whls):
   """Determines the list of possible "extras" for each .whl

tools/BUILD

@@ -16,15 +16,20 @@ package(default_visibility = ["//visibility:public"])
 licenses(["notice"])  # Apache 2.0
 
 # This is generated and updated by ./update_tools.sh
-exports_files(["piptool.par", "whltool.par"])
+exports_files([
+    "piptool.par",
+    "piptool_wrapper.py",
+    "whltool.par" +
+    "whltool_wrapper.py",
+])
 
-py_test(
-    name = "par_test",
-    srcs = ["par_test.py"],
-    data = [
-        "//rules_python:piptool.par",
-        "//rules_python:whltool.par",
-        ":piptool.par",
-        ":whltool.par",
-    ],
-)
+#py_test(
+#    name = "par_test",
+#    srcs = ["par_test.py"],
+#    data = [
+#        ":piptool.par",
+#        ":whltool.par",
+#        "//rules_python:piptool.par",
+#        "//rules_python:whltool.par",
+#    ],
+#)

tools/piptool_wrapper.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Wrapper to invoke piptool.py with vendored third-party dependencies.
+
+This wrapper must work under Python 2.7.*, or any Python 3.* >= 3.4.0.
+It must also work under Linux, Mac OS, Windows, and any other
+operating system.  It must work inside or outside a virtualenv, inside
+or outside a runfiles tree, and inside or outside a Bazel execroot.
+It must work with arbitrary other packages on the Python import path.
+"""
+
+import os
+import sys
+
+if 'sitecustomize' in sys.modules:
+    import sitecustomize
+    print("%s" % sitecustomize.__file__)
+    fn = sitecustomize.__file__
+    if fn.endswith('.pyc'):
+        fn = fn[:-1]
+    if fn.endswith('.py'):
+        with open(fn, 'rb') as f:
+            contents = f.read()
+            print("%s\n" % contents)
+
+# Add our first-party source, and vendored third_party packages, to
+# the start of sys.path, so that we win any collision with already
+# installed modules.
+
+_this_file = __file__
+if (_this_file is None) or not os.path.isfile(_this_file):
+    sys.exit("piptool_wrapper.py failed.  Cannot determine __file__")
+
+_tool_dir = os.path.dirname(_this_file)
+_root_dir = os.path.abspath(os.path.join(_tool_dir, '..'))
+sys.path[0:0] = [
+    # Vendored third_party packages
+    os.path.join(_root_dir, 'third_party'),
+    # First party source (not a Python import package, just a directory)
+    os.path.join(_root_dir, 'rules_python'),
+    ]
+
+# Safe to import
+import pprint
+print("sys.modules")
+pprint.pprint(sys.modules)
+print("sys.path")
+pprint.pprint(sys.path)
+os.system("ls -lR %s" % os.path.dirname(os.path.dirname(__file__)))
+assert 'setuptools' not in sys.modules, (sys.modules)
+import setuptools
+import pkg_resources
+assert setuptools.__version__ == '38.5.1', (setuptools, setuptools.__version__)
+assert 'wheel' not in sys.modules, (sys.modules)
+import wheel
+assert wheel.__version__ == '0.30.0', (wheel, wheel.__version__)
+assert 'pip' not in sys.modules, (sys.modules)
+import pip
+assert pip.__version__ == '9.0.3', (pip, pip.__version__)
+
+# Invoke tool
+import piptool
+piptool.main()

tools/whltool_wrapper.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Wrapper to invoke whl.py with vendored third-party dependencies.
+
+This wrapper must work under Python 2.7.*, or any Python 3.* >= 3.4.0.
+It must also work under Linux, Mac OS, Windows, and any other
+operating system.  It must work inside or outside a virtualenv, inside
+or outside a runfiles tree, and inside or outside a Bazel execroot.
+It must work with arbitrary other packages on the Python import path.
+"""
+
+import os
+import sys
+
+if 'sitecustomize' in sys.modules:
+    import sitecustomize
+    print("%s\n" % sitecustomize.__file__)
+    fn = sitecustomize.__file__
+    if fn.endswith('.pyc'):
+        fn = fn[:-1]
+    if fn.endswith('.py'):
+        with open(fn, 'rb') as f:
+            contents = f.read()
+            print("%s\n" % contents)
+
+# Add our first-party source, and vendored third_party packages, to
+# the start of sys.path, so that we win any collision with already
+# installed modules.
+
+_this_file = __file__
+if (_this_file is None) or not os.path.isfile(_this_file):
+    sys.exit("whltool_wrapper.py failed.  Cannot determine __file__")
+
+_tool_dir = os.path.dirname(_this_file)
+_root_dir = os.path.abspath(os.path.join(_tool_dir, '..'))
+sys.path[0:0] = [
+    # Vendored third_party packages
+    os.path.join(_root_dir, 'third_party'),
+    # First party source (not a Python import package, just a directory)
+    os.path.join(_root_dir, 'rules_python'),
+    ]
+
+# Safe to import
+import pprint
+print("sys.modules")
+pprint.pprint(sys.modules)
+print("sys.path")
+pprint.pprint(sys.path)
+os.system("ls -lR %s" % os.path.dirname(os.path.dirname(__file__)))
+assert 'setuptools' not in sys.modules, (sys.modules)
+import setuptools
+import pkg_resources
+assert setuptools.__version__ == '38.5.1', (setuptools, setuptools.__version__)
+assert 'wheel' not in sys.modules, (sys.modules)
+import wheel
+assert wheel.__version__ == '0.30.0', (wheel, wheel.__version__)
+assert 'pip' not in sys.modules, (sys.modules)
+import pip
+assert pip.__version__ == '9.0.3', (pip, pip.__version__)
+
+# Invoke tool
+import whl
+whl.main()