Allow alternate binaries download urls generation and convert GoDistribution and LLVM subsystems to use it

build-support/bin/native/bootstrap.sh

@@ -107,9 +107,9 @@ function ensure_native_build_prerequisites() {
   if [[ ! -x "${CARGO_HOME}/bin/cargo-ensure-installed" ]]; then
     "${CARGO_HOME}/bin/cargo" install cargo-ensure-installed >&2
   fi
-  "${CARGO_HOME}/bin/cargo" ensure-installed --package=cargo-ensure-installed --version=0.1.0 >&2
+  "${CARGO_HOME}/bin/cargo" ensure-installed --package=cargo-ensure-installed --version=0.2.1 >&2
   "${CARGO_HOME}/bin/cargo" ensure-installed --package=protobuf --version=1.4.2 >&2
-  "${CARGO_HOME}/bin/cargo" ensure-installed --package=grpcio-compiler --version=0.2.0 >&2
+  "${CARGO_HOME}/bin/cargo" ensure-installed --package=grpcio-compiler --version=0.2.0 --git-url=https://github.com/illicitonion/grpc-rs.git --git-rev=1147866fda52ca1b53de8d80272a68e3b7bd8b93 >&2
 
   local download_binary="${REPO_ROOT}/build-support/bin/download_binary.sh"
   local -r cmakeroot="$("${download_binary}" "binaries.pantsbuild.org" "cmake" "3.9.5" "cmake.tar.gz")" || die "Failed to fetch cmake"

contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py

@@ -10,10 +10,25 @@
 
 from pants.base.workunit import WorkUnit, WorkUnitLabel
 from pants.binaries.binary_tool import NativeTool
+from pants.binaries.binary_util import BinaryToolUrlGenerator
 from pants.util.memo import memoized_property
 from pants.util.process_handler import subprocess
 
 
+class GoReleaseUrlGenerator(BinaryToolUrlGenerator):
+
+  _DIST_URL_FMT = 'https://storage.googleapis.com/golang/go{version}.{system_id}.tar.gz'
+
+  _SYSTEM_ID = {
+    'darwin': 'darwin-amd64',
+    'linux': 'linux-amd64',
+  }
+
+  def generate_urls(self, version, host_platform):
+    system_id = self._SYSTEM_ID[host_platform.os_name]
+    return [self._DIST_URL_FMT.format(version=version, system_id=system_id)]
+
+
 class GoDistribution(NativeTool):
   """Represents a self-bootstrapping Go distribution."""
 
@@ -22,6 +37,9 @@ class GoDistribution(NativeTool):
   default_version = '1.8.3'
   archive_type = 'tgz'
 
+  def get_external_url_generator(self):
+    return GoReleaseUrlGenerator()
+
   @memoized_property
   def goroot(self):
     """Returns the $GOROOT for this go distribution.

contrib/node/src/python/pants/contrib/node/tasks/node_bundle.py

@@ -37,7 +37,7 @@ def execute(self):
 
     for target in self.context.target_roots:
       if self.is_node_bundle(target):
-        archiver = archive.archiver(target.payload.archive)
+        archiver = archive.create_archiver(target.payload.archive)
         for _, abs_paths in bundleable_js[target.node_module].abs_paths():
           for abs_path in abs_paths:
             # build_dir is a symlink.  Since dereference option for tar is set to False, we need to

contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_bundle_integration.py

@@ -8,7 +8,7 @@
 import os
 from contextlib import contextmanager
 
-from pants.fs.archive import archiver, archiver_for_path
+from pants.fs.archive import archiver_for_path, create_archiver
 from pants.util.contextutil import temporary_dir
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest
 
@@ -139,7 +139,7 @@ def _extract_archive(self, archive_path):
       _, extension = os.path.splitext(archive_path)
       print (extension)
       if extension == '.jar':
-        extraction_archiver = archiver('zip')
+        extraction_archiver = create_archiver('zip')
       else:
         extraction_archiver = archiver_for_path(os.path.basename(archive_path))
       extraction_archiver.extract(archive_path, temp_dir)

src/docs/setup_repo.md

@@ -231,9 +231,9 @@ proxy for the `dl.bintray.com` repo, or create your own repo of build
 tools:
 
     :::ini
-    pants_support_baseurls = [
+    binaries_baseurls = [
         "https://nexus.example.com/content/repositories/dl.bintray.com/pantsbuild/bin/build-support"
-	  ]
+      ]
 
 ### Redirecting python requirements to other servers
 
@@ -257,4 +257,3 @@ You can also reference a local repo relative to your project's build root with t
     repos: [
         "%(buildroot)s/repo_path"
       ]
-

src/python/pants/backend/jvm/tasks/bundle_create.py

@@ -96,7 +96,7 @@ def execute(self):
         app = self.App.create_app(vt.target,
                                   self.resolved_option(self.get_options(), vt.target, 'deployjar'),
                                   self.resolved_option(self.get_options(), vt.target, 'archive'))
-        archiver = archive.archiver(app.archive) if app.archive else None
+        archiver = archive.create_archiver(app.archive) if app.archive else None
 
         bundle_dir = self.get_bundle_dir(app.id, vt.results_dir)
         ext = archive.archive_extensions.get(app.archive, app.archive)

src/python/pants/backend/native/subsystems/llvm.py

@@ -0,0 +1,53 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pants.binaries.binary_tool import ExecutablePathProvider, NativeTool
+from pants.binaries.binary_util import BinaryToolUrlGenerator
+from pants.util.memo import memoized_method
+
+
+class LLVMReleaseUrlGenerator(BinaryToolUrlGenerator):
+
+  _DIST_URL_FMT = 'https://releases.llvm.org/{version}/{base}.tar.xz'
+
+  _ARCHIVE_BASE_FMT = 'clang+llvm-{version}-x86_64-{system_id}'
+
+  # TODO(cosmicexplorer): Give a more useful error message than KeyError if the host platform was
+  # not recognized (and make it easy for other BinaryTool subclasses to do this as well).
+  _SYSTEM_ID = {
+    'darwin': 'apple-darwin',
+    'linux': 'linux-gnu-ubuntu-16.04',
+  }
+
+  def generate_urls(self, version, host_platform):
+    system_id = self._SYSTEM_ID[host_platform.os_name]
+    archive_basename = self._ARCHIVE_BASE_FMT.format(version=version, system_id=system_id)
+    return [self._DIST_URL_FMT.format(version=version, base=archive_basename)]
+
+
+class LLVM(NativeTool, ExecutablePathProvider):
+  options_scope = 'llvm'
+  default_version = '6.0.0'
+  archive_type = 'txz'
+
+  def get_external_url_generator(self):
+    return LLVMReleaseUrlGenerator()
+
+  @memoized_method
+  def select(self):
+    unpacked_path = super(LLVM, self).select()
+    children = os.listdir(unpacked_path)
+    # The archive from releases.llvm.org wraps the extracted content into a directory one level
+    # deeper, but the one from our S3 does not.
+    if len(children) == 1 and os.path.isdir(children[0]):
+      return os.path.join(unpacked_path, children[0])
+    return unpacked_path
+
+  def path_entries(self):
+    return [os.path.join(self.select(), 'bin')]

src/python/pants/backend/native/subsystems/native_toolchain.py

@@ -5,8 +5,8 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
-from pants.backend.native.subsystems.clang import Clang
 from pants.backend.native.subsystems.gcc import GCC
+from pants.backend.native.subsystems.llvm import LLVM
 from pants.backend.native.subsystems.platform_specific.darwin.xcode_cli_tools import XCodeCLITools
 from pants.backend.native.subsystems.platform_specific.linux.binutils import Binutils
 from pants.binaries.binary_tool import ExecutablePathProvider
@@ -41,7 +41,7 @@ class NativeToolchain(Subsystem, ExecutablePathProvider):
 
   # This is a list of subsystems which implement `ExecutablePathProvider` and
   # can be provided for all supported platforms.
-  _CROSS_PLATFORM_SUBSYSTEMS = [Clang, GCC]
+  _CROSS_PLATFORM_SUBSYSTEMS = [LLVM, GCC]
 
   # This is a map of {<platform> -> [<subsystem_cls>, ...]}; the key is the
   # normalized OS name, and the value is a list of subsystem class objects that

src/python/pants/backend/python/tasks/setup_py.py

@@ -83,6 +83,8 @@ class SetupPyInvocationEnvironment(datatype(['joined_path'])):
 
   def as_env_dict(self):
     return {
+      'CC': 'gcc',
+      'CXX': 'g++',
       'PATH': self.joined_path,
     }
 

src/python/pants/binaries/binary_tool.py

@@ -6,15 +6,18 @@
                         unicode_literals, with_statement)
 
 import logging
+import os
 
-from pants.binaries.binary_util import BinaryUtilPrivate
+from pants.binaries.binary_util import BinaryRequest, BinaryUtilPrivate
+from pants.fs.archive import XZCompressedTarArchiver, create_archiver
 from pants.subsystem.subsystem import Subsystem
 from pants.util.memo import memoized_method, memoized_property
 
 
 logger = logging.getLogger(__name__)
 
 
+# TODO(cosmicexplorer): Add integration tests for this file.
 class BinaryToolBase(Subsystem):
   """Base class for subsytems that configure binary tools.
 
@@ -46,7 +49,48 @@ class BinaryToolBase(Subsystem):
 
   @classmethod
   def subsystem_dependencies(cls):
-    return super(BinaryToolBase, cls).subsystem_dependencies() + (BinaryUtilPrivate.Factory,)
+    sub_deps = super(BinaryToolBase, cls).subsystem_dependencies() + (BinaryUtilPrivate.Factory,)
+
+    # TODO(cosmicexplorer): if we need to do more conditional subsystem dependencies, do it
+    # declaratively with a dict class field so that we only try to create or access it if we
+    # declared a dependency on it.
+    if cls.archive_type == 'txz':
+      sub_deps = sub_deps + (XZ.scoped(cls),)
+
+    return sub_deps
+
+  @memoized_property
+  def _xz(self):
+    if self.archive_type == 'txz':
+      return XZ.scoped_instance(self)
+    return None
+
+  @memoized_method
+  def _get_archiver(self):
+    if not self.archive_type:
+      return None
+
+    if self.archive_type == 'txz':
+      return self._xz.tar_xz_extractor
+
+    return create_archiver(self.archive_type)
+
+  def get_external_url_generator(self):
+    """Override and return an instance of BinaryToolUrlGenerator to download from those urls.
+
+    If this method returns None, urls to download the tool will be constructed from
+    --binaries-baseurls. Otherwise, generate_urls() will be invoked on the result with the requested
+    version and host platform.
+
+    If the bootstrap option --allow-external-binary-tool-downloads is False, the result of this
+    method will be ignored. In general, implementors of BinaryTool subclasses should try to ensure
+    (within reason) that the rest of their code can handle the result of downloading from different
+    urls if get_external_url_generator() is overridden yet again by a subclass, or if it is ignored
+    with --no-allow-external-binary-tool-downloads.
+
+    See the :class:`LLVM` subsystem for an example of usage.
+    """
+    return None
 
   @classmethod
   def register_options(cls, register):
@@ -107,22 +151,30 @@ def version(self, context=None):
   def _binary_util(self):
     return BinaryUtilPrivate.Factory.create()
 
+  @classmethod
+  def _get_name(cls):
+    return cls.name or cls.options_scope
+
   @classmethod
   def get_support_dir(cls):
     return 'bin/{}'.format(cls._get_name())
 
-  @memoized_method
-  def _select_for_version(self, version):
-    return self._binary_util.select(
+  @classmethod
+  def _name_to_fetch(cls):
+    return '{}{}'.format(cls._get_name(), cls.suffix)
+
+  def _make_binary_request(self, version):
+    return BinaryRequest(
       supportdir=self.get_support_dir(),
       version=version,
-      name='{}{}'.format(self._get_name(), self.suffix),
+      name=self._name_to_fetch(),
       platform_dependent=self.platform_dependent,
-      archive_type=self.archive_type)
+      external_url_generator=self.get_external_url_generator(),
+      archiver=self._get_archiver())
 
-  @classmethod
-  def _get_name(cls):
-    return cls.name or cls.options_scope
+  def _select_for_version(self, version):
+    binary_request = self._make_binary_request(version)
+    return self._binary_util.select(binary_request)
 
 
 class NativeTool(BinaryToolBase):
@@ -151,3 +203,19 @@ class ExecutablePathProvider(object):
 
   def path_entries(self):
     return []
+
+
+class XZ(NativeTool):
+  options_scope = 'xz'
+  default_version = '5.2.4'
+  archive_type = 'tgz'
+
+  @memoized_property
+  def tar_xz_extractor(self):
+    return XZCompressedTarArchiver(self._executable_location(), self._lib_dir())
+
+  def _executable_location(self):
+    return os.path.join(self.select(), 'bin', 'xz')
+
+  def _lib_dir(self):
+    return os.path.join(self.select(), 'lib')