diff --git a/build-support/bin/native/bootstrap.sh b/build-support/bin/native/bootstrap.sh
index 5e08e61f1b8..50d8ec4442e 100644
--- a/build-support/bin/native/bootstrap.sh
+++ b/build-support/bin/native/bootstrap.sh
@@ -107,9 +107,9 @@ function ensure_native_build_prerequisites() {
   if [[ ! -x "${CARGO_HOME}/bin/cargo-ensure-installed" ]]; then
     "${CARGO_HOME}/bin/cargo" install cargo-ensure-installed >&2
   fi
-  "${CARGO_HOME}/bin/cargo" ensure-installed --package=cargo-ensure-installed --version=0.1.0 >&2
+  "${CARGO_HOME}/bin/cargo" ensure-installed --package=cargo-ensure-installed --version=0.2.1 >&2
   "${CARGO_HOME}/bin/cargo" ensure-installed --package=protobuf --version=1.4.2 >&2
-  "${CARGO_HOME}/bin/cargo" ensure-installed --package=grpcio-compiler --version=0.2.0 >&2
+  "${CARGO_HOME}/bin/cargo" ensure-installed --package=grpcio-compiler --version=0.2.0 --git-url=https://github.com/illicitonion/grpc-rs.git --git-rev=1147866fda52ca1b53de8d80272a68e3b7bd8b93 >&2
 
   local download_binary="${REPO_ROOT}/build-support/bin/download_binary.sh"
   local -r cmakeroot="$("${download_binary}" "binaries.pantsbuild.org" "cmake" "3.9.5" "cmake.tar.gz")" || die "Failed to fetch cmake"
diff --git a/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py b/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py
index f658cb662bb..032efd4bd71 100644
--- a/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py
+++ b/contrib/go/src/python/pants/contrib/go/subsystems/go_distribution.py
@@ -10,10 +10,25 @@
 
 from pants.base.workunit import WorkUnit, WorkUnitLabel
 from pants.binaries.binary_tool import NativeTool
+from pants.binaries.binary_util import BinaryToolUrlGenerator
 from pants.util.memo import memoized_property
 from pants.util.process_handler import subprocess
 
 
+class GoReleaseUrlGenerator(BinaryToolUrlGenerator):
+
+  _DIST_URL_FMT = 'https://storage.googleapis.com/golang/go{version}.{system_id}.tar.gz'
+
+  _SYSTEM_ID = {
+    'darwin': 'darwin-amd64',
+    'linux': 'linux-amd64',
+  }
+
+  def generate_urls(self, version, host_platform):
+    system_id = self._SYSTEM_ID[host_platform.os_name]
+    return [self._DIST_URL_FMT.format(version=version, system_id=system_id)]
+
+
 class GoDistribution(NativeTool):
   """Represents a self-bootstrapping Go distribution."""
 
@@ -22,6 +37,9 @@ class GoDistribution(NativeTool):
   default_version = '1.8.3'
   archive_type = 'tgz'
 
+  def get_external_url_generator(self):
+    return GoReleaseUrlGenerator()
+
   @memoized_property
   def goroot(self):
     """Returns the $GOROOT for this go distribution.
diff --git a/contrib/node/src/python/pants/contrib/node/tasks/node_bundle.py b/contrib/node/src/python/pants/contrib/node/tasks/node_bundle.py
index 9f1c9c95336..8d736a34580 100644
--- a/contrib/node/src/python/pants/contrib/node/tasks/node_bundle.py
+++ b/contrib/node/src/python/pants/contrib/node/tasks/node_bundle.py
@@ -37,7 +37,7 @@ def execute(self):
 
     for target in self.context.target_roots:
       if self.is_node_bundle(target):
-        archiver = archive.archiver(target.payload.archive)
+        archiver = archive.create_archiver(target.payload.archive)
         for _, abs_paths in bundleable_js[target.node_module].abs_paths():
           for abs_path in abs_paths:
             # build_dir is a symlink.  Since dereference option for tar is set to False, we need to
diff --git a/contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_bundle_integration.py b/contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_bundle_integration.py
index 07bb111a029..6aa9cbad62d 100644
--- a/contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_bundle_integration.py
+++ b/contrib/node/tests/python/pants_test/contrib/node/tasks/test_node_bundle_integration.py
@@ -8,7 +8,7 @@
 import os
 from contextlib import contextmanager
 
-from pants.fs.archive import archiver, archiver_for_path
+from pants.fs.archive import archiver_for_path, create_archiver
 from pants.util.contextutil import temporary_dir
 from pants_test.pants_run_integration_test import PantsRunIntegrationTest
 
@@ -139,7 +139,7 @@ def _extract_archive(self, archive_path):
       _, extension = os.path.splitext(archive_path)
       print (extension)
       if extension == '.jar':
-        extraction_archiver = archiver('zip')
+        extraction_archiver = create_archiver('zip')
       else:
         extraction_archiver = archiver_for_path(os.path.basename(archive_path))
       extraction_archiver.extract(archive_path, temp_dir)
diff --git a/src/docs/setup_repo.md b/src/docs/setup_repo.md
index f7d7ecca8ee..4de1bd297c5 100644
--- a/src/docs/setup_repo.md
+++ b/src/docs/setup_repo.md
@@ -231,9 +231,9 @@ proxy for the `dl.bintray.com` repo, or create your own repo of build
 tools:
 
     :::ini
-    pants_support_baseurls = [
+    binaries_baseurls = [
         "https://nexus.example.com/content/repositories/dl.bintray.com/pantsbuild/bin/build-support"
-	  ]
+      ]
 
 ### Redirecting python requirements to other servers
 
@@ -257,4 +257,3 @@ You can also reference a local repo relative to your project's build root with t
     repos: [
         "%(buildroot)s/repo_path"
       ]
-
diff --git a/src/python/pants/backend/jvm/tasks/bundle_create.py b/src/python/pants/backend/jvm/tasks/bundle_create.py
index 29ce174490f..5e24d363fc2 100644
--- a/src/python/pants/backend/jvm/tasks/bundle_create.py
+++ b/src/python/pants/backend/jvm/tasks/bundle_create.py
@@ -96,7 +96,7 @@ def execute(self):
         app = self.App.create_app(vt.target,
                                   self.resolved_option(self.get_options(), vt.target, 'deployjar'),
                                   self.resolved_option(self.get_options(), vt.target, 'archive'))
-        archiver = archive.archiver(app.archive) if app.archive else None
+        archiver = archive.create_archiver(app.archive) if app.archive else None
 
         bundle_dir = self.get_bundle_dir(app.id, vt.results_dir)
         ext = archive.archive_extensions.get(app.archive, app.archive)
diff --git a/src/python/pants/backend/native/subsystems/clang.py b/src/python/pants/backend/native/subsystems/clang.py
deleted file mode 100644
index 1afe82cc875..00000000000
--- a/src/python/pants/backend/native/subsystems/clang.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# coding=utf-8
-# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
-# Licensed under the Apache License, Version 2.0 (see LICENSE).
-
-from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
-                        unicode_literals, with_statement)
-
-import os
-
-from pants.binaries.binary_tool import ExecutablePathProvider, NativeTool
-
-
-class Clang(NativeTool, ExecutablePathProvider):
-  options_scope = 'clang'
-  default_version = '6.0.0'
-  archive_type = 'tgz'
-
-  def path_entries(self):
-    return [os.path.join(self.select(), 'bin')]
diff --git a/src/python/pants/backend/native/subsystems/llvm.py b/src/python/pants/backend/native/subsystems/llvm.py
new file mode 100644
index 00000000000..43c3c757a80
--- /dev/null
+++ b/src/python/pants/backend/native/subsystems/llvm.py
@@ -0,0 +1,53 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pants.binaries.binary_tool import ExecutablePathProvider, NativeTool
+from pants.binaries.binary_util import BinaryToolUrlGenerator
+from pants.util.memo import memoized_method
+
+
+class LLVMReleaseUrlGenerator(BinaryToolUrlGenerator):
+
+  _DIST_URL_FMT = 'https://releases.llvm.org/{version}/{base}.tar.xz'
+
+  _ARCHIVE_BASE_FMT = 'clang+llvm-{version}-x86_64-{system_id}'
+
+  # TODO(cosmicexplorer): Give a more useful error message than KeyError if the host platform was
+  # not recognized (and make it easy for other BinaryTool subclasses to do this as well).
+  _SYSTEM_ID = {
+    'darwin': 'apple-darwin',
+    'linux': 'linux-gnu-ubuntu-16.04',
+  }
+
+  def generate_urls(self, version, host_platform):
+    system_id = self._SYSTEM_ID[host_platform.os_name]
+    archive_basename = self._ARCHIVE_BASE_FMT.format(version=version, system_id=system_id)
+    return [self._DIST_URL_FMT.format(version=version, base=archive_basename)]
+
+
+class LLVM(NativeTool, ExecutablePathProvider):
+  options_scope = 'llvm'
+  default_version = '6.0.0'
+  archive_type = 'txz'
+
+  def get_external_url_generator(self):
+    return LLVMReleaseUrlGenerator()
+
+  @memoized_method
+  def select(self):
+    unpacked_path = super(LLVM, self).select()
+    children = os.listdir(unpacked_path)
+    # The archive from releases.llvm.org wraps the extracted content into a directory one level
+    # deeper, but the one from our S3 does not.
+    if len(children) == 1 and os.path.isdir(children[0]):
+      return os.path.join(unpacked_path, children[0])
+    return unpacked_path
+
+  def path_entries(self):
+    return [os.path.join(self.select(), 'bin')]
diff --git a/src/python/pants/backend/native/subsystems/native_toolchain.py b/src/python/pants/backend/native/subsystems/native_toolchain.py
index 119b1cf943a..4851e1f2cbd 100644
--- a/src/python/pants/backend/native/subsystems/native_toolchain.py
+++ b/src/python/pants/backend/native/subsystems/native_toolchain.py
@@ -5,8 +5,8 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
-from pants.backend.native.subsystems.clang import Clang
 from pants.backend.native.subsystems.gcc import GCC
+from pants.backend.native.subsystems.llvm import LLVM
 from pants.backend.native.subsystems.platform_specific.darwin.xcode_cli_tools import XCodeCLITools
 from pants.backend.native.subsystems.platform_specific.linux.binutils import Binutils
 from pants.binaries.binary_tool import ExecutablePathProvider
@@ -41,7 +41,7 @@ class NativeToolchain(Subsystem, ExecutablePathProvider):
 
   # This is a list of subsystems which implement `ExecutablePathProvider` and
   # can be provided for all supported platforms.
-  _CROSS_PLATFORM_SUBSYSTEMS = [Clang, GCC]
+  _CROSS_PLATFORM_SUBSYSTEMS = [LLVM, GCC]
 
   # This is a map of {<platform> -> [<subsystem_cls>, ...]}; the key is the
   # normalized OS name, and the value is a list of subsystem class objects that
diff --git a/src/python/pants/backend/python/tasks/setup_py.py b/src/python/pants/backend/python/tasks/setup_py.py
index a5582f59ddb..8670442ff56 100644
--- a/src/python/pants/backend/python/tasks/setup_py.py
+++ b/src/python/pants/backend/python/tasks/setup_py.py
@@ -83,6 +83,8 @@ class SetupPyInvocationEnvironment(datatype(['joined_path'])):
 
   def as_env_dict(self):
     return {
+      'CC': 'gcc',
+      'CXX': 'g++',
       'PATH': self.joined_path,
     }
 
diff --git a/src/python/pants/binaries/binary_tool.py b/src/python/pants/binaries/binary_tool.py
index cbe53cafa49..768127a5d06 100644
--- a/src/python/pants/binaries/binary_tool.py
+++ b/src/python/pants/binaries/binary_tool.py
@@ -6,8 +6,10 @@
                         unicode_literals, with_statement)
 
 import logging
+import os
 
-from pants.binaries.binary_util import BinaryUtilPrivate
+from pants.binaries.binary_util import BinaryRequest, BinaryUtilPrivate
+from pants.fs.archive import XZCompressedTarArchiver, create_archiver
 from pants.subsystem.subsystem import Subsystem
 from pants.util.memo import memoized_method, memoized_property
 
@@ -15,6 +17,7 @@
 logger = logging.getLogger(__name__)
 
 
+# TODO(cosmicexplorer): Add integration tests for this file.
 class BinaryToolBase(Subsystem):
   """Base class for subsytems that configure binary tools.
 
@@ -46,7 +49,46 @@ class BinaryToolBase(Subsystem):
 
   @classmethod
   def subsystem_dependencies(cls):
-    return super(BinaryToolBase, cls).subsystem_dependencies() + (BinaryUtilPrivate.Factory,)
+    sub_deps = super(BinaryToolBase, cls).subsystem_dependencies() + (BinaryUtilPrivate.Factory,)
+
+    # TODO(cosmicexplorer): if we need to do more conditional subsystem dependencies, do it
+    # declaratively with a dict class field so that we only try to create or access it if we
+    # declared a dependency on it.
+    if cls.archive_type == 'txz':
+      sub_deps = sub_deps + (XZ.scoped(cls),)
+
+    return sub_deps
+
+  @memoized_property
+  def _xz(self):
+    if self.archive_type == 'txz':
+      return XZ.scoped_instance(self)
+    return None
+
+  @memoized_method
+  def _get_archiver(self):
+    if not self.archive_type:
+      return None
+
+    if self.archive_type == 'txz':
+      return self._xz.tar_xz_extractor
+
+    return create_archiver(self.archive_type)
+
+  def get_external_url_generator(self):
+    """Override and return an instance of BinaryToolUrlGenerator to download from those urls.
+
+    If this method returns None, urls to download the tool will be constructed from
+    --binaries-baseurls. Otherwise, generate_urls() will be invoked on the result with the requested
+    version and host platform.
+
+    If the bootstrap option --allow-external-binary-tool-downloads is False, the result of this
+    method will be ignored. Implementations of BinaryTool must be aware of differences (e.g., in
+    archive structure) between the external and internal versions of the downloaded tool, if any.
+
+    See the :class:`LLVM` subsystem for an example of usage.
+    """
+    return None
 
   @classmethod
   def register_options(cls, register):
@@ -107,22 +149,30 @@ def version(self, context=None):
   def _binary_util(self):
     return BinaryUtilPrivate.Factory.create()
 
+  @classmethod
+  def _get_name(cls):
+    return cls.name or cls.options_scope
+
   @classmethod
   def get_support_dir(cls):
     return 'bin/{}'.format(cls._get_name())
 
-  @memoized_method
-  def _select_for_version(self, version):
-    return self._binary_util.select(
+  @classmethod
+  def _name_to_fetch(cls):
+    return '{}{}'.format(cls._get_name(), cls.suffix)
+
+  def _make_binary_request(self, version):
+    return BinaryRequest(
       supportdir=self.get_support_dir(),
       version=version,
-      name='{}{}'.format(self._get_name(), self.suffix),
+      name=self._name_to_fetch(),
       platform_dependent=self.platform_dependent,
-      archive_type=self.archive_type)
+      external_url_generator=self.get_external_url_generator(),
+      archiver=self._get_archiver())
 
-  @classmethod
-  def _get_name(cls):
-    return cls.name or cls.options_scope
+  def _select_for_version(self, version):
+    binary_request = self._make_binary_request(version)
+    return self._binary_util.select(binary_request)
 
 
 class NativeTool(BinaryToolBase):
@@ -151,3 +201,19 @@ class ExecutablePathProvider(object):
 
   def path_entries(self):
     return []
+
+
+class XZ(NativeTool):
+  options_scope = 'xz'
+  default_version = '5.2.4'
+  archive_type = 'tgz'
+
+  @memoized_property
+  def tar_xz_extractor(self):
+    return XZCompressedTarArchiver(self._executable_location(), self._lib_dir())
+
+  def _executable_location(self):
+    return os.path.join(self.select(), 'bin', 'xz')
+
+  def _lib_dir(self):
+    return os.path.join(self.select(), 'lib')
diff --git a/src/python/pants/binaries/binary_util.py b/src/python/pants/binaries/binary_util.py
index bbdb709d004..c58379f8a90 100644
--- a/src/python/pants/binaries/binary_util.py
+++ b/src/python/pants/binaries/binary_util.py
@@ -8,6 +8,8 @@
 import logging
 import os
 import posixpath
+import shutil
+from abc import abstractmethod
 from contextlib import contextmanager
 
 from twitter.common.collections import OrderedSet
@@ -15,34 +17,226 @@
 from pants.base.build_environment import get_buildroot
 from pants.base.deprecated import deprecated
 from pants.base.exceptions import TaskError
-from pants.fs.archive import archiver as create_archiver
 from pants.net.http.fetcher import Fetcher
 from pants.subsystem.subsystem import Subsystem
 from pants.util.contextutil import temporary_file
 from pants.util.dirutil import chmod_plus_x, safe_concurrent_creation, safe_open
-from pants.util.osutil import get_os_id
-
-
-_DEFAULT_PATH_BY_ID = {
-  ('linux', 'x86_64'): ('linux', 'x86_64'),
-  ('linux', 'amd64'): ('linux', 'x86_64'),
-  ('linux', 'i386'): ('linux', 'i386'),
-  ('linux', 'i686'): ('linux', 'i386'),
-  ('darwin', '9'): ('mac', '10.5'),
-  ('darwin', '10'): ('mac', '10.6'),
-  ('darwin', '11'): ('mac', '10.7'),
-  ('darwin', '12'): ('mac', '10.8'),
-  ('darwin', '13'): ('mac', '10.9'),
-  ('darwin', '14'): ('mac', '10.10'),
-  ('darwin', '15'): ('mac', '10.11'),
-  ('darwin', '16'): ('mac', '10.12'),
-  ('darwin', '17'): ('mac', '10.13'),
-}
+from pants.util.memo import memoized_method, memoized_property
+from pants.util.objects import datatype
+from pants.util.osutil import SUPPORTED_PLATFORM_NORMALIZED_NAMES
 
 
 logger = logging.getLogger(__name__)
 
 
+class HostPlatform(datatype(['os_name', 'arch_or_version'])):
+  """Describes a platform to resolve binaries for. Determines the binary's location on disk.
+
+  :class:`BinaryToolUrlGenerator` instances receive this to generate download urls.
+  """
+
+  def binary_path_components(self):
+    """These strings are used as consecutive components of the path where a binary is fetched.
+
+    This is also used in generating urls from --binaries-baseurls in PantsHosted."""
+    return [self.os_name, self.arch_or_version]
+
+
+class BinaryToolUrlGenerator(object):
+  """Encapsulates the selection of urls to download for some binary tool.
+
+  :API: public
+
+  :class:`BinaryTool` subclasses can return an instance of a class mixing this in to
+  get_external_url_generator(self) to download their file or archive from some specified url or set
+  of urls.
+  """
+
+  @abstractmethod
+  def generate_urls(self, version, host_platform):
+    """Return a list of urls to download some binary tool from given a version and platform.
+
+    Each url is tried in order to resolve the binary -- if the list of urls is empty, or downloading
+    from each of the urls fails, Pants will raise an exception when the binary tool is fetched which
+    should describe why the urls failed to work.
+
+    :param str version: version string for the requested binary (e.g. '2.0.1').
+    :param host_platform: description of the platform to fetch binaries for.
+    :type host_platform: :class:`HostPlatform`
+    :returns: a list of urls to download the binary tool from.
+    :rtype: list
+    """
+    pass
+
+
+class PantsHosted(BinaryToolUrlGenerator):
+  """Given a binary request and --binaries-baseurls, generate urls to download the binary from.
+
+  This url generator is used if get_external_url_generator(self) is not overridden by a BinaryTool
+  subclass, or if --allow-external-binary-tool-downloads is False.
+
+  NB: "pants-hosted" is referring to the organization of the urls being specific to pants. It also
+  happens that most binaries are downloaded from S3 hosting at binaries.pantsbuild.org by default --
+  but setting --binaries-baseurls to anything else will only download binaries from the baseurls
+  given, not from binaries.pantsbuild.org.
+  """
+
+  class NoBaseUrlsError(ValueError): pass
+
+  def __init__(self, binary_request, baseurls):
+    self._binary_request = binary_request
+
+    if not baseurls:
+      raise self.NoBaseUrlsError(
+        "Error constructing pants-hosted urls for the {} binary: no baseurls were provided."
+        .format(binary_request.name))
+    self._baseurls = baseurls
+
+  def generate_urls(self, _version, host_platform):
+    """Append the file's download path to each of --binaries-baseurls.
+
+    This assumes that the urls in --binaries-baseurls point somewhere that mirrors Pants's
+    organization of the downloaded binaries on disk. Each url is tried in order until a request
+    succeeds.
+    """
+    binary_path = self._binary_request.get_download_path(host_platform)
+    return [posixpath.join(baseurl, binary_path) for baseurl in self._baseurls]
+
+
+# TODO: Deprecate passing in an explicit supportdir? Seems like we should be able to
+# organize our binary hosting so that it's not needed. It's also used to calculate the binary
+# download location, though.
+class BinaryRequest(datatype([
+    'supportdir',
+    'version',
+    'name',
+    'platform_dependent',
+    # NB: this can be None!
+    'external_url_generator',
+    # NB: this can be None!
+    'archiver',
+])):
+  """Describes a request for a binary to download."""
+
+  def _full_name(self):
+    if self.archiver:
+      return '{}.{}'.format(self.name, self.archiver.extension)
+    return self.name
+
+  def get_download_path(self, host_platform):
+    binary_path_components = [self.supportdir]
+    if self.platform_dependent:
+      # TODO(John Sirois): finish doc of the path structure expected under base_path.
+      binary_path_components.extend(host_platform.binary_path_components())
+    binary_path_components.extend([self.version, self._full_name()])
+    return os.path.join(*binary_path_components)
+
+
+class BinaryFetchRequest(datatype(['download_path', 'urls'])):
+  """Describes a request to download a file."""
+
+  @memoized_property
+  def file_name(self):
+    return os.path.basename(self.download_path)
+
+  class NoDownloadUrlsError(ValueError): pass
+
+  def __new__(cls, download_path, urls):
+    this_object = super(BinaryFetchRequest, cls).__new__(
+      cls, download_path, tuple(urls))
+
+    if not this_object.urls:
+      raise cls.NoDownloadUrlsError(
+        "No urls were provided to {cls_name}: {obj!r}."
+        .format(cls_name=cls.__name__, obj=this_object))
+
+    return this_object
+
+
+class BinaryToolFetcher(object):
+
+  @classmethod
+  def _default_http_fetcher(cls):
+    """Return a fetcher that resolves local file paths against the build root.
+
+    Currently this is used everywhere except in testing.
+    """
+    return Fetcher(get_buildroot())
+
+  def __init__(self, bootstrap_dir, timeout_secs, fetcher=None, ignore_cached_download=False):
+    """
+    :param str bootstrap_dir: The root directory where Pants downloads binaries to.
+    :param int timeout_secs: The number of seconds to wait before timing out on a request for some
+                             url.
+    :param fetcher: object to fetch urls with, overridden in testing.
+    :type fetcher: :class:`pants.net.http.fetcher.Fetcher`
+    :param bool ignore_cached_download: whether to fetch a binary even if it already exists on disk.
+    """
+    self._bootstrap_dir = bootstrap_dir
+    self._timeout_secs = timeout_secs
+    self._fetcher = fetcher or self._default_http_fetcher()
+    self._ignore_cached_download = ignore_cached_download
+
+  class BinaryNotFound(TaskError):
+
+    def __init__(self, name, accumulated_errors):
+      super(BinaryToolFetcher.BinaryNotFound, self).__init__(
+        'Failed to fetch {name} binary from any source: ({error_msgs})'
+        .format(name=name, error_msgs=', '.join(accumulated_errors)))
+
+  @contextmanager
+  def _select_binary_stream(self, name, urls):
+    """Download a file from a list of urls, yielding a stream after downloading the file.
+
+    URLs are tried in order until they succeed.
+
+    :raises: :class:`BinaryToolFetcher.BinaryNotFound` if requests to all the given urls fail.
+    """
+    downloaded_successfully = False
+    accumulated_errors = []
+    for url in OrderedSet(urls):  # De-dup URLS: we only want to try each URL once.
+      logger.info('Attempting to fetch {name} binary from: {url} ...'.format(name=name, url=url))
+      try:
+        with temporary_file() as dest:
+          logger.debug("in BinaryToolFetcher: url={}, timeout_secs={}"
+                       .format(url, self._timeout_secs))
+          self._fetcher.download(url,
+                                 listener=Fetcher.ProgressListener(),
+                                 path_or_fd=dest,
+                                 timeout_secs=self._timeout_secs)
+          logger.info('Fetched {name} binary from: {url} .'.format(name=name, url=url))
+          downloaded_successfully = True
+          dest.seek(0)
+          yield dest
+          break
+      except (IOError, Fetcher.Error, ValueError) as e:
+        accumulated_errors.append('Failed to fetch binary from {url}: {error}'
+                                  .format(url=url, error=e))
+    if not downloaded_successfully:
+      raise self.BinaryNotFound(name, accumulated_errors)
+
+  def _do_fetch(self, download_path, file_name, urls):
+    with safe_concurrent_creation(download_path) as downloadpath:
+      with self._select_binary_stream(file_name, urls) as binary_tool_stream:
+        with safe_open(downloadpath, 'wb') as bootstrapped_binary:
+          shutil.copyfileobj(binary_tool_stream, bootstrapped_binary)
+
+  def fetch_binary(self, fetch_request):
+    """Fulfill a binary fetch request."""
+    bootstrap_dir = os.path.realpath(os.path.expanduser(self._bootstrap_dir))
+    bootstrapped_binary_path = os.path.join(bootstrap_dir, fetch_request.download_path)
+    logger.debug("bootstrapped_binary_path: {}".format(bootstrapped_binary_path))
+    file_name = fetch_request.file_name
+    urls = fetch_request.urls
+
+    if self._ignore_cached_download or not os.path.exists(bootstrapped_binary_path):
+      self._do_fetch(bootstrapped_binary_path, file_name, urls)
+
+    logger.debug('Selected {binary} binary bootstrapped to: {path}'
+                 .format(binary=file_name, path=bootstrapped_binary_path))
+    return bootstrapped_binary_path
+
+
 class BinaryUtilPrivate(object):
   """Wraps utility methods for finding binary executables."""
 
@@ -63,52 +257,33 @@ def create(cls):
     def _create_for_cls(cls, binary_util_cls):
       # NB: create is a class method to ~force binary fetch location to be global.
       options = cls.global_instance().get_options()
+      binary_tool_fetcher = BinaryToolFetcher(
+        bootstrap_dir=options.pants_bootstrapdir,
+        timeout_secs=options.binaries_fetch_timeout_secs)
       return binary_util_cls(
-        options.binaries_baseurls,
-        options.binaries_fetch_timeout_secs,
-        options.pants_bootstrapdir,
-        options.binaries_path_by_id
-      )
+        baseurls=options.binaries_baseurls,
+        binary_tool_fetcher=binary_tool_fetcher,
+        path_by_id=options.binaries_path_by_id,
+        allow_external_binary_tool_downloads=options.allow_external_binary_tool_downloads)
 
   class MissingMachineInfo(TaskError):
     """Indicates that pants was unable to map this machine's OS to a binary path prefix."""
     pass
 
-  class BinaryNotFound(TaskError):
-
-    def __init__(self, binary, accumulated_errors):
-      super(BinaryUtil.BinaryNotFound, self).__init__(
-          'Failed to fetch binary {binary} from any source: ({sources})'
-          .format(binary=binary, sources=', '.join(accumulated_errors)))
-
   class NoBaseUrlsError(TaskError):
     """Indicates that no urls were specified in pants.ini."""
     pass
 
-  def _select_binary_base_path(self, supportdir, version, name, uname_func=None):
-    """Calculate the base path.
-
-    Exposed for associated unit tests.
-    :param supportdir: the path used to make a path under --pants_bootstrapdir.
-    :param version: the version number of the tool used to make a path under --pants-bootstrapdir.
-    :param name: name of the binary to search for. (e.g 'protoc')
-    :param uname_func: method to use to emulate os.uname() in testing
-    :returns: Base path used to select the binary file.
-    """
-    uname_func = uname_func or os.uname
-    os_id = get_os_id(uname_func=uname_func)
-    if not os_id:
-      raise self.MissingMachineInfo('Pants has no binaries for {}'.format(' '.join(uname_func())))
+  class BinaryResolutionError(TaskError):
+    """Raised to wrap other exceptions raised in the select() method to provide context."""
 
-    try:
-      middle_path = self._path_by_id[os_id]
-    except KeyError:
-      raise self.MissingMachineInfo('Unable to find binary {name} version {version}. '
-                                    'Update --binaries-path-by-id to find binaries for {os_id!r}'
-                                    .format(name=name, version=version, os_id=os_id))
-    return os.path.join(supportdir, *(middle_path + (version, name)))
+    def __init__(self, binary_request, base_exception):
+      super(BinaryUtilPrivate.BinaryResolutionError, self).__init__(
+        "Error resolving binary request {}: {}".format(binary_request, base_exception),
+        base_exception)
 
-  def __init__(self, baseurls, timeout_secs, bootstrapdir, path_by_id=None):
+  def __init__(self, baseurls, binary_tool_fetcher, path_by_id=None,
+               allow_external_binary_tool_downloads=True, uname_func=None):
     """Creates a BinaryUtil with the given settings to define binary lookup behavior.
 
     This constructor is primarily used for testing.  Production code will usually initialize
@@ -121,125 +296,151 @@ def __init__(self, baseurls, timeout_secs, bootstrapdir, path_by_id=None):
       search for binaries in, or download binaries to if needed.
     :param dict path_by_id: Additional mapping from (sysname, id) -> (os, arch) for tool
       directory naming
+    :param bool allow_external_binary_tool_downloads: If False, use --binaries-baseurls to download
+                                                      all binaries, regardless of whether an
+                                                      external_url_generator field is provided.
+    :param function uname_func: method to use to emulate os.uname() in testing
     """
     self._baseurls = baseurls
-    self._timeout_secs = timeout_secs
-    self._pants_bootstrapdir = bootstrapdir
-    self._path_by_id = _DEFAULT_PATH_BY_ID.copy()
+    self._binary_tool_fetcher = binary_tool_fetcher
+
+    self._path_by_id = SUPPORTED_PLATFORM_NORMALIZED_NAMES.copy()
     if path_by_id:
       self._path_by_id.update((tuple(k), tuple(v)) for k, v in path_by_id.items())
 
-  # TODO: Deprecate passing in an explicit supportdir? Seems like we should be able to
-  # organize our binary hosting so that it's not needed.
-  def select(self, supportdir, version, name, platform_dependent, archive_type):
+    self._allow_external_binary_tool_downloads = allow_external_binary_tool_downloads
+    self._uname_func = uname_func or os.uname
+
+  _ID_BY_OS = {
+    'linux': lambda release, machine: ('linux', machine),
+    'darwin': lambda release, machine: ('darwin', release.split('.')[0]),
+  }
+
+  # FIXME(cosmicexplorer): we create a HostPlatform in this class instead of in the constructor
+  # because we don't want to fail until a binary is requested. The HostPlatform should be a
+  # parameter that gets lazily resolved by the v2 engine.
+  @memoized_method
+  def _host_platform(self):
+    uname_result = self._uname_func()
+    sysname, _, release, _, machine = uname_result
+    os_id_key = sysname.lower()
+    try:
+      os_id_fun = self._ID_BY_OS[os_id_key]
+      os_id_tuple = os_id_fun(release, machine)
+    except KeyError:
+      # TODO: test this!
+      raise self.MissingMachineInfo(
+        "Pants could not resolve binaries for the current host: platform '{}' was not recognized. "
+        "Recognized platforms are: {}."
+        .format(os_id_key, self._ID_BY_OS.keys()))
+    try:
+      os_name, arch_or_version = self._path_by_id[os_id_tuple]
+      host_platform = HostPlatform(os_name, arch_or_version)
+    except KeyError:
+      # We fail early here because we need the host_platform to identify where to download binaries
+      # to.
+      raise self.MissingMachineInfo(
+        "Pants could not resolve binaries for the current host. Update --binaries-path-by-id to "
+        "find binaries for the current host platform {}.\n"
+        "--binaries-path-by-id was: {}."
+        .format(os_id_tuple, self._path_by_id))
+
+    return host_platform
+
+  def _get_download_path(self, binary_request):
+    return binary_request.get_download_path(self._host_platform())
+
+  def _get_url_generator(self, binary_request):
+
+    external_url_generator = binary_request.external_url_generator
+
+    logger.debug("self._allow_external_binary_tool_downloads: {}"
+                 .format(self._allow_external_binary_tool_downloads))
+    logger.debug("external_url_generator: {}".format(external_url_generator))
+
+    if external_url_generator and self._allow_external_binary_tool_downloads:
+      url_generator = external_url_generator
+    else:
+      if not self._baseurls:
+        raise self.NoBaseUrlsError("--binaries-baseurls is empty.")
+      url_generator = PantsHosted(binary_request=binary_request, baseurls=self._baseurls)
+
+    return url_generator
+
+  def _get_urls(self, url_generator, binary_request):
+    return url_generator.generate_urls(binary_request.version, self._host_platform())
+
+  def select(self, binary_request):
     """Fetches a file, unpacking it if necessary."""
-    if archive_type is None:
-      return self._select_file(supportdir, version, name, platform_dependent)
-    archiver = create_archiver(archive_type)
-    return self._select_archive(supportdir, version, name, platform_dependent, archiver)
-
-  def _select_file(self, supportdir, version, name, platform_dependent):
-    """Generates a path to request a file and fetches the file located at that path.
-
-    :param string supportdir: The path the `name` binaries are stored under.
-    :param string version: The version number of the binary to select.
-    :param string name: The name of the file to fetch.
-    :param bool platform_dependent: Whether the file content differs depending
-      on the current platform.
-    :raises: :class:`pants.binary_util.BinaryUtil.BinaryNotFound` if no file of the given version
-      and name could be found for the current platform.
-    """
-    binary_path = self._binary_path_to_fetch(supportdir, version, name, platform_dependent)
-    return self._fetch_binary(name=name, binary_path=binary_path)
-
-  def _select_archive(self, supportdir, version, name, platform_dependent, archiver):
-    """Generates a path to fetch, fetches the archive file, and unpacks the archive.
-
-    :param string supportdir: The path the `name` binaries are stored under.
-    :param string version: The version number of the binary to select.
-    :param string name: The name of the file to fetch.
-    :param bool platform_dependent: Whether the file content differs depending
-      on the current platform.
-    :param archiver: The archiver object which provides the file extension and
-      unpacks the archive.
-    :type: :class:`pants.fs.archive.Archiver`
-    :raises: :class:`pants.binary_util.BinaryUtil.BinaryNotFound` if no file of the given version
-      and name could be found for the current platform.
-    """
-    full_name = '{}.{}'.format(name, archiver.extension)
-    downloaded_file = self._select_file(supportdir, version, full_name, platform_dependent)
-    # Use filename without rightmost extension as the directory name.
-    unpacked_dirname, _ = os.path.splitext(downloaded_file)
-    if not os.path.exists(unpacked_dirname):
+
+    logger.debug("binary_request: {!r}".format(binary_request))
+
+    try:
+      download_path = self._get_download_path(binary_request)
+    except self.MissingMachineInfo as e:
+      raise self.BinaryResolutionError(binary_request, e)
+
+    try:
+      url_generator = self._get_url_generator(binary_request)
+    except self.NoBaseUrlsError as e:
+      raise self.BinaryResolutionError(binary_request, e)
+
+    urls = self._get_urls(url_generator, binary_request)
+    if not isinstance(urls, list):
+      # TODO: add test for this error!
+      raise self.BinaryResolutionError(
+        binary_request,
+        TypeError("urls must be a list: was '{}'.".format(urls)))
+    fetch_request = BinaryFetchRequest(
+      download_path=download_path,
+      urls=urls)
+
+    logger.debug("fetch_request: {!r}".format(fetch_request))
+
+    try:
+      downloaded_file = self._binary_tool_fetcher.fetch_binary(fetch_request)
+    except BinaryToolFetcher.BinaryNotFound as e:
+      raise self.BinaryResolutionError(binary_request, e)
+
+    # NB: we mark the downloaded file executable if it is not an archive.
+    archiver = binary_request.archiver
+    if archiver is None:
+      chmod_plus_x(downloaded_file)
+      return downloaded_file
+
+    download_dir = os.path.dirname(downloaded_file)
+    # Use the 'name' given in the request as the directory name to extract to.
+    unpacked_dirname = os.path.join(download_dir, binary_request.name)
+    if not os.path.isdir(unpacked_dirname):
+      logger.info("Extracting {} to {} .".format(downloaded_file, unpacked_dirname))
       archiver.extract(downloaded_file, unpacked_dirname, concurrency_safe=True)
     return unpacked_dirname
 
-  def _binary_path_to_fetch(self, supportdir, version, name, platform_dependent):
-    if platform_dependent:
-      # TODO(John Sirois): finish doc of the path structure expected under base_path.
-      return self._select_binary_base_path(supportdir, version, name)
-    return os.path.join(supportdir, version, name)
+  def _make_deprecated_binary_request(self, supportdir, version, name):
+    return BinaryRequest(
+      supportdir=supportdir,
+      version=version,
+      name=name,
+      platform_dependent=True,
+      external_url_generator=None,
+      archiver=None)
 
   def select_binary(self, supportdir, version, name):
-    return self._select_file(
-      supportdir, version, name, platform_dependent=True)
+    binary_request = self._make_deprecated_binary_request(supportdir, version, name)
+    return self.select(binary_request)
+
+  def _make_deprecated_script_request(self, supportdir, version, name):
+    return BinaryRequest(
+      supportdir=supportdir,
+      version=version,
+      name=name,
+      platform_dependent=False,
+      external_url_generator=None,
+      archiver=None)
 
   def select_script(self, supportdir, version, name):
-    return self._select_file(
-      supportdir, version, name, platform_dependent=False)
-
-  @contextmanager
-  def _select_binary_stream(self, name, binary_path, fetcher=None):
-    """Select a binary located at a given path.
-
-    :param string binary_path: The path to the binary to fetch.
-    :param fetcher: Optional argument used only for testing, to 'pretend' to open urls.
-    :returns: a 'stream' to download it from a support directory. The returned 'stream' is actually
-      a lambda function which returns the files binary contents.
-    :raises: :class:`pants.binary_util.BinaryUtil.BinaryNotFound` if no binary of the given version
-      and name could be found for the current platform.
-    """
-
-    if not self._baseurls:
-      raise self.NoBaseUrlsError(
-          'No urls are defined for the --pants-support-baseurls option.')
-    downloaded_successfully = False
-    accumulated_errors = []
-    for baseurl in OrderedSet(self._baseurls):  # De-dup URLS: we only want to try each URL once.
-      url = posixpath.join(baseurl, binary_path)
-      logger.info('Attempting to fetch {name} binary from: {url} ...'.format(name=name, url=url))
-      try:
-        with temporary_file() as dest:
-          fetcher = fetcher or Fetcher(get_buildroot())
-          fetcher.download(url,
-                           listener=Fetcher.ProgressListener(),
-                           path_or_fd=dest,
-                           timeout_secs=self._timeout_secs)
-          logger.info('Fetched {name} binary from: {url} .'.format(name=name, url=url))
-          downloaded_successfully = True
-          dest.seek(0)
-          yield lambda: dest.read()
-          break
-      except (IOError, Fetcher.Error, ValueError) as e:
-        accumulated_errors.append('Failed to fetch binary from {url}: {error}'
-                                  .format(url=url, error=e))
-    if not downloaded_successfully:
-      raise self.BinaryNotFound(binary_path, accumulated_errors)
-
-  def _fetch_binary(self, name, binary_path):
-    bootstrap_dir = os.path.realpath(os.path.expanduser(self._pants_bootstrapdir))
-    bootstrapped_binary_path = os.path.join(bootstrap_dir, binary_path)
-    if not os.path.exists(bootstrapped_binary_path):
-      with safe_concurrent_creation(bootstrapped_binary_path) as downloadpath:
-        with self._select_binary_stream(name, binary_path) as stream:
-          with safe_open(downloadpath, 'wb') as bootstrapped_binary:
-            bootstrapped_binary.write(stream())
-          os.rename(downloadpath, bootstrapped_binary_path)
-          chmod_plus_x(bootstrapped_binary_path)
-
-    logger.debug('Selected {binary} binary bootstrapped to: {path}'
-                 .format(binary=name, path=bootstrapped_binary_path))
-    return bootstrapped_binary_path
+    binary_request = self._make_deprecated_script_request(supportdir, version, name)
+    return self.select(binary_request)
 
 
 class BinaryUtil(BinaryUtilPrivate):
diff --git a/src/python/pants/fs/BUILD b/src/python/pants/fs/BUILD
index 1c31218d1f7..c7d38d83728 100644
--- a/src/python/pants/fs/BUILD
+++ b/src/python/pants/fs/BUILD
@@ -3,7 +3,11 @@
 
 python_library(
   dependencies = [
+    'src/python/pants/base:deprecated',
     'src/python/pants/util:contextutil',
+    'src/python/pants/util:dirutil',
     'src/python/pants/util:meta',
+    'src/python/pants/util:process_handler',
+    'src/python/pants/util:strutil',
   ]
 )
diff --git a/src/python/pants/fs/archive.py b/src/python/pants/fs/archive.py
index e9501392e90..d5c3c5f442b 100644
--- a/src/python/pants/fs/archive.py
+++ b/src/python/pants/fs/archive.py
@@ -6,13 +6,18 @@
                         unicode_literals, with_statement)
 
 import os
+import sys
 from abc import abstractmethod
 from collections import OrderedDict
+from contextlib import contextmanager
 from zipfile import ZIP_DEFLATED
 
+from pants.base.deprecated import deprecated
 from pants.util.contextutil import open_tar, open_zip, temporary_dir
-from pants.util.dirutil import safe_concurrent_rename, safe_walk
+from pants.util.dirutil import (is_executable, safe_concurrent_rename, safe_walk,
+                                split_basename_and_dirname)
 from pants.util.meta import AbstractClass
+from pants.util.process_handler import subprocess
 from pants.util.strutil import ensure_text
 
 
@@ -21,16 +26,15 @@
 
 class Archiver(AbstractClass):
 
-  @classmethod
-  def extract(cls, path, outdir, filter_func=None, concurrency_safe=False):
+  def extract(self, path, outdir, concurrency_safe=False, **kwargs):
     """Extracts an archive's contents to the specified outdir with an optional filter.
 
+    Keyword arguments are forwarded to the instance's self._extract() method.
+
     :API: public
 
     :param string path: path to the zipfile to extract from
     :param string outdir: directory to extract files into
-    :param function filter_func: optional filter with the filename as the parameter.  Returns True
-      if the file should be extracted.  Note that filter_func is ignored for non-zip archives.
     :param bool concurrency_safe: True to use concurrency safe method.  Concurrency safe extraction
       will be performed on a temporary directory and the extacted directory will then be renamed
       atomically to the outdir.  As a side effect, concurrency safe extraction will not allow
@@ -38,14 +42,13 @@ def extract(cls, path, outdir, filter_func=None, concurrency_safe=False):
     """
     if concurrency_safe:
       with temporary_dir() as temp_dir:
-        cls._extract(path, temp_dir, filter_func=filter_func)
+        self._extract(path, temp_dir, **kwargs)
         safe_concurrent_rename(temp_dir, outdir)
     else:
       # Leave the existing default behavior unchanged and allows overlay of contents.
-      cls._extract(path, outdir, filter_func=filter_func)
+      self._extract(path, outdir, **kwargs)
 
-  @classmethod
-  def _extract(cls, path, outdir):
+  def _extract(self, path, outdir):
     raise NotImplementedError()
 
   @abstractmethod
@@ -65,9 +68,8 @@ class TarArchiver(Archiver):
   :API: public
   """
 
-  @classmethod
-  def _extract(cls, path, outdir, **kwargs):
-    with open_tar(path, errorlevel=1) as tar:
+  def _extract(self, path_or_file, outdir, **kwargs):
+    with open_tar(path_or_file, errorlevel=1, **kwargs) as tar:
       tar.extractall(outdir)
 
   def __init__(self, mode, extension):
@@ -90,15 +92,111 @@ def create(self, basedir, outdir, name, prefix=None, dereference=True):
     return tarpath
 
 
+class XZCompressedTarArchiver(TarArchiver):
+  """A workaround for the lack of xz support in Python 2.7.
+
+  Invokes an xz executable to decompress a .tar.xz into a tar stream, which is piped into the
+  extract() method.
+
+  NB: This class will raise an error if used to create an archive! This class can only currently be
+  used to extract from xz archives.
+  """
+
+  class XZArchiverError(Exception): pass
+
+  def __init__(self, xz_binary_path, xz_library_path):
+
+    # TODO(cosmicexplorer): test these exceptions somewhere!
+    if not is_executable(xz_binary_path):
+      raise self.XZArchiverError(
+        "The path {} does not name an existing executable file. An xz executable must be provided "
+        "to decompress xz archives."
+        .format(xz_binary_path))
+
+    self._xz_binary_path = xz_binary_path
+
+    if not os.path.isdir(xz_library_path):
+      raise self.XZArchiverError(
+        "The path {} does not name an existing directory. A directory containing liblzma.so must "
+        "be provided to decompress xz archives."
+        .format(xz_library_path))
+
+    lib_lzma_dylib = os.path.join(xz_library_path, 'liblzma.so')
+    if not os.path.isfile(lib_lzma_dylib):
+      raise self.XZArchiverError(
+        "The path {} names an existing directory, but it does not contain liblzma.so. A directory "
+        "containing liblzma.so must be provided to decompress xz archives."
+        .format(xz_library_path))
+
+    self._xz_library_path = xz_library_path
+
+    super(XZCompressedTarArchiver, self).__init__('r|', 'tar.xz')
+
+  @contextmanager
+  def _invoke_xz(self, xz_input_file):
+    """Run the xz command and yield a file object for its stdout.
+
+    This allows streaming the decompressed tar archive directly into a tar decompression stream,
+    which is significantly faster in practice than making a temporary file.
+    """
+    (xz_bin_dir, xz_filename) = split_basename_and_dirname(self._xz_binary_path)
+
+    # TODO(cosmicexplorer): --threads=0 is supposed to use "the number of processor cores on the
+    # machine", but I see no more than 100% cpu used at any point. This seems like it could be a
+    # bug? If performance is an issue, investigate further.
+    cmd = [xz_filename, '--decompress', '--stdout', '--keep', '--threads=0', xz_input_file]
+    env = {
+      # Isolate the path so we know we're using our provided version of xz.
+      'PATH': xz_bin_dir,
+      # Only allow our xz's lib directory to resolve the liblzma.so dependency at runtime.
+      'LD_LIBRARY_PATH': self._xz_library_path,
+    }
+    try:
+      # Pipe stderr to our own stderr, but leave stdout open so we can yield it.
+      process = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=sys.stderr,
+        env=env)
+    except OSError as e:
+      raise self.XZArchiverError(
+        "Error invoking xz with command {} and environment {} for input file {}: {}"
+        .format(cmd, env, xz_input_file, e),
+        e)
+
+    # This is a file object.
+    yield process.stdout
+
+    rc = process.wait()
+    if rc != 0:
+      raise self.XZArchiverError(
+        "Error decompressing xz input with command {} and environment {} for input file {}. "
+        "Exit code was: {}. "
+        .format(cmd, env, xz_input_file, rc))
+
+  def _extract(self, path, outdir):
+    with self._invoke_xz(path) as xz_decompressed_tar_stream:
+      return super(XZCompressedTarArchiver, self)._extract(
+        xz_decompressed_tar_stream, outdir, mode=self.mode)
+
+  def create(self, *args, **kwargs):
+    """
+    :raises: :class:`NotImplementedError`
+    """
+    raise NotImplementedError("XZCompressedTarArchiver can only extract, not create archives!")
+
+
 class ZipArchiver(Archiver):
   """An archiver that stores files in a zip file with optional compression.
 
   :API: public
   """
 
-  @classmethod
-  def _extract(cls, path, outdir, filter_func=None, **kwargs):
-    """Extract from a zip file, with an optional filter."""
+  def _extract(self, path, outdir, filter_func=None):
+    """Extract from a zip file, with an optional filter.
+
+    :param function filter_func: optional filter with the filename as the parameter.  Returns True
+                                 if the file should be extracted."""
     with open_zip(path) as archive_file:
       for name in archive_file.namelist():
         # While we're at it, we also perform this safety test.
@@ -134,23 +232,35 @@ def create(self, basedir, outdir, name, prefix=None):
           zip.write(full_path, relpath)
     return zippath
 
-archive_extensions = dict(tar='tar', tgz='tar.gz', tbz2='tar.bz2', zip='zip')
 
-TAR = TarArchiver('w:', archive_extensions['tar'])
-TGZ = TarArchiver('w:gz', archive_extensions['tgz'])
-TBZ2 = TarArchiver('w:bz2', archive_extensions['tbz2'])
-ZIP = ZipArchiver(ZIP_DEFLATED, archive_extensions['zip'])
+TAR = TarArchiver('w:', 'tar')
+TGZ = TarArchiver('w:gz', 'tar.gz')
+TBZ2 = TarArchiver('w:bz2', 'tar.bz2')
+ZIP = ZipArchiver(ZIP_DEFLATED, 'zip')
 
-_ARCHIVER_BY_TYPE = OrderedDict(tar=TAR, tgz=TGZ, tbz2=TBZ2, zip=ZIP)
+_ARCHIVER_BY_TYPE = OrderedDict(
+  tar=TAR,
+  tgz=TGZ,
+  tbz2=TBZ2,
+  zip=ZIP)
+
+archive_extensions = {
+  name:archiver.extension for name, archiver in _ARCHIVER_BY_TYPE.items()
+}
 
 TYPE_NAMES = frozenset(_ARCHIVER_BY_TYPE.keys())
 TYPE_NAMES_NO_PRESERVE_SYMLINKS = frozenset(['zip'])
 TYPE_NAMES_PRESERVE_SYMLINKS = TYPE_NAMES - TYPE_NAMES_NO_PRESERVE_SYMLINKS
 
 
-# TODO: Rename to `create_archiver`. Pretty much every caller of this method is going
-# to want to put the return value into a variable named `archiver`.
+# Pretty much every caller of this method is going to want to put the return value into a variable
+# named `archiver`.
+@deprecated(removal_version='1.8.0.dev0', hint_message='Use the create_archiver method instead.')
 def archiver(typename):
+  return create_archiver(typename)
+
+
+def create_archiver(typename):
   """Returns Archivers in common configurations.
 
   :API: public
@@ -190,4 +300,4 @@ def archiver_for_path(path_name):
       ext = ext[1:]  # Trim leading '.'.
     if not ext:
       raise ValueError('Could not determine archive type of path {}'.format(path_name))
-    return archiver(ext)
+    return create_archiver(ext)
diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py
index 5ef6ec4d364..2fa382de1b8 100644
--- a/src/python/pants/option/global_options.py
+++ b/src/python/pants/option/global_options.py
@@ -161,9 +161,14 @@ def register_bootstrap_options(cls, register):
              help='Timeout in seconds for URL reads when fetching binary tools from the '
                   'repos specified by --baseurls.')
     register('--binaries-path-by-id', type=dict, advanced=True,
-             help=('Maps output of uname for a machine to a binary search path. e.g. '
-                   '{("darwin", "15"): ["mac", "10.11"]), ("linux", "arm32"): ["linux"'
-                   ', "arm32"]}'))
+             help=("Maps output of uname for a machine to a binary search path: "
+                   "(sysname, id) -> (os, arch), e.g. {('darwin', '15'): ('mac', '10.11'), "
+                   "('linux', 'arm32'): ('linux', 'arm32')}."))
+    register('--allow-external-binary-tool-downloads', type=bool, default=True, advanced=True,
+             help="If False, require BinaryTool subclasses to download their contents from urls "
+                  "generated from --binaries-baseurls, even if the tool has an external url "
+                  "generator. This can be necessary if using Pants in an environment which cannot "
+                  "contact the wider Internet.")
 
     # Pants Daemon options.
     register('--pantsd-pailgun-host', advanced=True, default='127.0.0.1',
@@ -181,7 +186,7 @@ def register_bootstrap_options(cls, register):
     register('--watchman-version', advanced=True, default='4.9.0-pants1', help='Watchman version.')
     register('--watchman-supportdir', advanced=True, default='bin/watchman',
              help='Find watchman binaries under this dir. Used as part of the path to lookup '
-                  'the binary with --binary-util-baseurls and --pants-bootstrapdir.')
+                  'the binary with --binaries-baseurls and --pants-bootstrapdir.')
     register('--watchman-startup-timeout', type=float, advanced=True, default=30.0,
              help='The watchman socket timeout (in seconds) for the initial `watch-project` command. '
                   'This may need to be set higher for larger repos due to watchman startup cost.')
diff --git a/src/python/pants/pantsd/watchman_launcher.py b/src/python/pants/pantsd/watchman_launcher.py
index d367f2f62c2..d789cdc677f 100644
--- a/src/python/pants/pantsd/watchman_launcher.py
+++ b/src/python/pants/pantsd/watchman_launcher.py
@@ -7,7 +7,7 @@
 
 import logging
 
-from pants.binaries.binary_util import BinaryUtilPrivate
+from pants.binaries.binary_util import BinaryToolFetcher, BinaryUtilPrivate
 from pants.pantsd.watchman import Watchman
 from pants.util.memo import testable_memoized_property
 
@@ -20,12 +20,15 @@ def create(cls, bootstrap_options):
     """
     :param Options bootstrap_options: The bootstrap options bag.
     """
+    binary_tool_fetcher = BinaryToolFetcher(
+      bootstrap_dir=bootstrap_options.pants_bootstrapdir,
+      timeout_secs=bootstrap_options.binaries_fetch_timeout_secs)
     binary_util = BinaryUtilPrivate(
-      bootstrap_options.binaries_baseurls,
-      bootstrap_options.binaries_fetch_timeout_secs,
-      bootstrap_options.pants_bootstrapdir,
-      bootstrap_options.binaries_path_by_id
-    )
+      baseurls=bootstrap_options.binaries_baseurls,
+      binary_tool_fetcher=binary_tool_fetcher,
+      path_by_id=bootstrap_options.binaries_path_by_id,
+      # TODO(cosmicexplorer): do we need to test this?
+      allow_external_binary_tool_downloads=bootstrap_options.allow_external_binary_tool_downloads)
 
     return WatchmanLauncher(
       binary_util,
diff --git a/src/python/pants/util/dirutil.py b/src/python/pants/util/dirutil.py
index 1cfecf3bc6b..cfc494f3e8a 100644
--- a/src/python/pants/util/dirutil.py
+++ b/src/python/pants/util/dirutil.py
@@ -477,3 +477,9 @@ def rm_rf(name):
 def is_executable(path):
   """Returns whether a path names an existing executable file."""
   return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def split_basename_and_dirname(path):
+  if not os.path.isfile(path):
+    raise ValueError("{} does not exist or is not a regular file.".format(path))
+  return (os.path.dirname(path), os.path.basename(path))
diff --git a/src/python/pants/util/osutil.py b/src/python/pants/util/osutil.py
index 608942cd2eb..0f1847a3a48 100644
--- a/src/python/pants/util/osutil.py
+++ b/src/python/pants/util/osutil.py
@@ -12,38 +12,19 @@
 logger = logging.getLogger(__name__)
 
 
-_ID_BY_OS = {
-  'linux': lambda release, machine: ('linux', machine),
-  'darwin': lambda release, machine: ('darwin', release.split('.')[0]),
-}
-
-
 OS_ALIASES = {
   'darwin': {'macos', 'darwin', 'macosx', 'mac os x', 'mac'},
   'linux': {'linux', 'linux2'},
 }
 
 
-def get_os_name():
+def get_os_name(uname_result=None):
   """
   :API: public
   """
-  return os.uname()[0].lower()
-
-
-def get_os_id(uname_func=None):
-  """Return an OS identifier sensitive only to its major version.
-
-  :param uname_func: An `os.uname` compliant callable; intended for tests.
-  :returns: a tuple of (OS name, sub identifier) or `None` if the OS is not supported.
-  :rtype: tuple of string, string
-  """
-  uname_func = uname_func or os.uname
-  sysname, _, release, _, machine = uname_func()
-  os_id = _ID_BY_OS.get(sysname.lower())
-  if os_id:
-    return os_id(release, machine)
-  return None
+  if uname_result is None:
+    uname_result = os.uname()
+  return uname_result[0].lower()
 
 
 def normalize_os_name(os_name):
@@ -69,3 +50,22 @@ def all_normalized_os_names():
 
 def known_os_names():
   return reduce(set.union, OS_ALIASES.values())
+
+
+# TODO(cosmicexplorer): use this as the default value for the global --binaries-path-by-id option!
+# panstd testing fails saying no run trackers were created when I tried to do this.
+SUPPORTED_PLATFORM_NORMALIZED_NAMES = {
+  ('linux', 'x86_64'): ('linux', 'x86_64'),
+  ('linux', 'amd64'): ('linux', 'x86_64'),
+  ('linux', 'i386'): ('linux', 'i386'),
+  ('linux', 'i686'): ('linux', 'i386'),
+  ('darwin', '9'): ('mac', '10.5'),
+  ('darwin', '10'): ('mac', '10.6'),
+  ('darwin', '11'): ('mac', '10.7'),
+  ('darwin', '12'): ('mac', '10.8'),
+  ('darwin', '13'): ('mac', '10.9'),
+  ('darwin', '14'): ('mac', '10.10'),
+  ('darwin', '15'): ('mac', '10.11'),
+  ('darwin', '16'): ('mac', '10.12'),
+  ('darwin', '17'): ('mac', '10.13'),
+}
diff --git a/tests/python/pants_test/backend/codegen/protobuf/java/test_protobuf_gen.py b/tests/python/pants_test/backend/codegen/protobuf/java/test_protobuf_gen.py
index 2c6462cef57..7ac6df0648d 100644
--- a/tests/python/pants_test/backend/codegen/protobuf/java/test_protobuf_gen.py
+++ b/tests/python/pants_test/backend/codegen/protobuf/java/test_protobuf_gen.py
@@ -22,8 +22,8 @@ def setUp(self):
     super(ProtobufGenTest, self).setUp()
     self.set_options(pants_bootstrapdir='~/.cache/pants',
                      max_subprocess_args=100,
-                     pants_support_fetch_timeout_secs=1,
-                     pants_support_baseurls=['http://example.com/dummy_base_url'])
+                     binaries_fetch_timeout_secs=1,
+                     binaries_baseurls=['http://example.com/dummy_base_url'])
 
   @classmethod
   def task_type(cls):
diff --git a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_java_compile_integration.py b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_java_compile_integration.py
index 1274c72c2da..af6f58ff6e1 100644
--- a/tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_java_compile_integration.py
+++ b/tests/python/pants_test/backend/jvm/tasks/jvm_compile/java/test_java_compile_integration.py
@@ -7,7 +7,7 @@
 
 import os
 
-from pants.fs.archive import TarArchiver
+from pants.fs.archive import archiver_for_path
 from pants.util.contextutil import temporary_dir
 from pants.util.dirutil import safe_walk
 from pants_test.backend.jvm.tasks.jvm_compile.base_compile_integration_test import BaseCompileIT
@@ -114,9 +114,11 @@ def test_java_compile_reads_resource_mapping(self):
       self.assertTrue(os.path.exists(artifact_dir))
       artifacts = os.listdir(artifact_dir)
       self.assertEqual(len(artifacts), 1)
+      single_artifact = artifacts[0]
 
       with temporary_dir() as extract_dir:
-        TarArchiver.extract(os.path.join(artifact_dir, artifacts[0]), extract_dir)
+        artifact_path = os.path.join(artifact_dir, single_artifact)
+        archiver_for_path(artifact_path).extract(artifact_path, extract_dir)
         all_files = set()
         for dirpath, dirs, files in safe_walk(extract_dir):
           for name in files:
diff --git a/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py b/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py
index a5db427901a..75b176a2b3d 100644
--- a/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py
+++ b/tests/python/pants_test/backend/native/subsystems/test_native_toolchain.py
@@ -5,6 +5,8 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import unittest
+
 from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.util.contextutil import environment_as, get_joined_path
 from pants.util.process_handler import subprocess
@@ -17,6 +19,7 @@
 # FIXME(cosmicexplorer): We need to test gcc as well, but the gcc driver can't
 # find the right include directories for system headers in Travis. We need to
 # use the clang driver to find library paths, then use those when invoking gcc.
+@unittest.skip('Skipped because this entire backend is about to get redone, see #5780.')
 class TestNativeToolchain(BaseTest):
 
   def setUp(self):
diff --git a/tests/python/pants_test/base_test.py b/tests/python/pants_test/base_test.py
index 6fed162c55b..b8dcdf0cb3f 100644
--- a/tests/python/pants_test/base_test.py
+++ b/tests/python/pants_test/base_test.py
@@ -28,6 +28,7 @@
 from pants.build_graph.target import Target
 from pants.init.util import clean_global_runtime_state
 from pants.option.options_bootstrapper import OptionsBootstrapper
+from pants.option.scope import GLOBAL_SCOPE
 from pants.source.source_root import SourceRootConfig
 from pants.subsystem.subsystem import Subsystem
 from pants.task.goal_options_mixin import GoalOptionsMixin
@@ -228,7 +229,7 @@ def setUp(self):
     safe_mkdir(self.pants_workdir)
 
     self.options = defaultdict(dict)  # scope -> key-value mapping.
-    self.options[''] = {
+    self.options[GLOBAL_SCOPE] = {
       'pants_workdir': self.pants_workdir,
       'pants_supportdir': os.path.join(self.build_root, 'build-support'),
       'pants_distdir': os.path.join(self.build_root, 'dist'),
diff --git a/tests/python/pants_test/binaries/BUILD b/tests/python/pants_test/binaries/BUILD
index b21005bffb0..24740bcfde7 100644
--- a/tests/python/pants_test/binaries/BUILD
+++ b/tests/python/pants_test/binaries/BUILD
@@ -2,12 +2,11 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 python_tests(
-  name='binary_util',
-  sources=['test_binary_util.py'],
   dependencies=[
     '3rdparty/python:mock',
     'src/python/pants/binaries',
     'src/python/pants/net',
+    'src/python/pants/option',
     'src/python/pants/util:contextutil',
     'src/python/pants/util:dirutil',
     'tests/python/pants_test:base_test',
diff --git a/tests/python/pants_test/binaries/test_binary_tool.py b/tests/python/pants_test/binaries/test_binary_tool.py
new file mode 100644
index 00000000000..baa4678c610
--- /dev/null
+++ b/tests/python/pants_test/binaries/test_binary_tool.py
@@ -0,0 +1,146 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.binaries.binary_tool import BinaryToolBase
+from pants.binaries.binary_util import (BinaryToolFetcher, BinaryToolUrlGenerator,
+                                        BinaryUtilPrivate, HostPlatform)
+from pants.option.scope import GLOBAL_SCOPE
+from pants_test.base_test import BaseTest
+
+
+class DefaultVersion(BinaryToolBase):
+  options_scope = 'default-version-test'
+  name = 'default_version_test_tool'
+  default_version = 'XXX'
+
+
+class AnotherTool(BinaryToolBase):
+  options_scope = 'another-tool'
+  name = 'another_tool'
+  default_version = '0.0.1'
+
+
+class ReplacingLegacyOptionsTool(BinaryToolBase):
+  # TODO: check scope?
+  options_scope = 'replacing-legacy-options-tool'
+  name = 'replacing_legacy_options_tool'
+  default_version = 'a2f4ab23a4c'
+
+  replaces_scope = 'old_tool_scope'
+  replaces_name = 'old_tool_version'
+
+
+class BinaryUtilFakeUname(BinaryUtilPrivate):
+
+  def _host_platform(self):
+    return HostPlatform('xxx', 'yyy')
+
+
+class CustomUrlGenerator(BinaryToolUrlGenerator):
+
+  _DIST_URL_FMT = 'https://custom-url.example.org/files/custom_urls_tool-{version}-{system_id}'
+
+  _SYSTEM_ID = {
+    'xxx': 'zzz',
+  }
+
+  def generate_urls(self, version, host_platform):
+    base = self._DIST_URL_FMT.format(
+      version=version,
+      system_id=self._SYSTEM_ID[host_platform.os_name])
+    return [
+      base,
+      '{}-alternate'.format(base),
+    ]
+
+
+class CustomUrls(BinaryToolBase):
+  options_scope = 'custom-urls'
+  name = 'custom_urls_tool'
+  default_version = 'v2.1'
+
+  def get_external_url_generator(self):
+    return CustomUrlGenerator()
+
+  def _select_for_version(self, version):
+    binary_request = self._make_binary_request(version)
+    return BinaryUtilFakeUname.Factory._create_for_cls(BinaryUtilFakeUname).select(binary_request)
+
+
+# TODO(cosmicexplorer): these should have integration tests which use BinaryTool subclasses
+# overriding archive_type
+class BinaryToolBaseTest(BaseTest):
+
+  def setUp(self):
+    super(BinaryToolBaseTest, self).setUp()
+    self._context = self.context(
+      for_subsystems=[DefaultVersion, AnotherTool, ReplacingLegacyOptionsTool, CustomUrls],
+      options={
+        GLOBAL_SCOPE: {
+          'binaries_baseurls': ['https://binaries.example.org'],
+        },
+        'another-tool': {
+          'version': '0.0.2',
+        },
+        'default-version-test.another-tool': {
+          'version': 'YYY',
+        },
+        'custom-urls': {
+          'version': 'v2.3',
+        },
+        'old_tool_scope': {
+          'old_tool_version': '3',
+        },
+      })
+
+  def test_base_options(self):
+    # TODO: using extra_version_option_kwargs!
+    default_version_tool = DefaultVersion.global_instance()
+    self.assertEqual(default_version_tool.version(), 'XXX')
+
+    another_tool = AnotherTool.global_instance()
+    self.assertEqual(another_tool.version(), '0.0.2')
+
+    another_default_version_tool = DefaultVersion.scoped_instance(AnotherTool)
+    self.assertEqual(another_default_version_tool.version(), 'YYY')
+
+  def test_replacing_legacy_options(self):
+    replacing_legacy_options_tool = ReplacingLegacyOptionsTool.global_instance()
+    self.assertEqual(replacing_legacy_options_tool.version(), 'a2f4ab23a4c')
+    self.assertEqual(replacing_legacy_options_tool.version(self._context), '3')
+
+  def test_urls(self):
+    default_version_tool = DefaultVersion.global_instance()
+    self.assertIsNone(default_version_tool.get_external_url_generator())
+
+    with self.assertRaises(BinaryUtilPrivate.BinaryResolutionError) as cm:
+      default_version_tool.select()
+    err_msg = str(cm.exception)
+    self.assertIn(BinaryToolFetcher.BinaryNotFound.__name__, err_msg)
+    self.assertIn(
+      "Failed to fetch default_version_test_tool binary from any source:",
+      err_msg)
+    self.assertIn(
+      "Failed to fetch binary from https://binaries.example.org/bin/default_version_test_tool/XXX/default_version_test_tool:",
+      err_msg)
+
+    custom_urls_tool = CustomUrls.global_instance()
+    self.assertEqual(custom_urls_tool.version(), 'v2.3')
+
+    with self.assertRaises(BinaryUtilPrivate.BinaryResolutionError) as cm:
+      custom_urls_tool.select()
+    err_msg = str(cm.exception)
+    self.assertIn(BinaryToolFetcher.BinaryNotFound.__name__, err_msg)
+    self.assertIn(
+      "Failed to fetch custom_urls_tool binary from any source:",
+      err_msg)
+    self.assertIn(
+      "Failed to fetch binary from https://custom-url.example.org/files/custom_urls_tool-v2.3-zzz:",
+      err_msg)
+    self.assertIn(
+      "Failed to fetch binary from https://custom-url.example.org/files/custom_urls_tool-v2.3-zzz-alternate:",
+      err_msg)
diff --git a/tests/python/pants_test/binaries/test_binary_util.py b/tests/python/pants_test/binaries/test_binary_util.py
index 8bf55aaf41b..2aa4ee1849c 100644
--- a/tests/python/pants_test/binaries/test_binary_util.py
+++ b/tests/python/pants_test/binaries/test_binary_util.py
@@ -5,20 +5,35 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import logging
 import os
-import re
 
 import mock
 
-from pants.binaries.binary_util import BinaryUtilPrivate
+from pants.binaries.binary_util import (BinaryRequest, BinaryToolFetcher, BinaryToolUrlGenerator,
+                                        BinaryUtilPrivate)
 from pants.net.http.fetcher import Fetcher
 from pants.util.contextutil import temporary_dir
 from pants.util.dirutil import safe_open
 from pants_test.base_test import BaseTest
 
 
+logger = logging.getLogger(__name__)
+
+
+class ExternalUrlGenerator(BinaryToolUrlGenerator):
+
+  def generate_urls(self, version, host_platform):
+    return ['https://example.com/some-binary', 'https://example.com/same-binary']
+
+  # Make the __str__ deterministic, for testing exception messages.
+  def __str__(self):
+    return 'ExternalUrlGenerator(<example __str__()>)'
+
+
+# TODO(cosmicexplorer): test requests with an archiver!
 class BinaryUtilTest(BaseTest):
-  """Tests binary_util's pants_support_baseurls handling."""
+  """Tests binary_util's binaries_baseurls handling."""
 
   class MapFetcher(object):
     """Class which pretends to be a pants.net.http.Fetcher, but is actually a dictionary."""
@@ -49,39 +64,74 @@ def _fake_base(cls, name):
 
   @classmethod
   def _fake_url(cls, binaries, base, binary_key):
-    binary_util = BinaryUtilPrivate([], 0, '/tmp')
+    binary_util = cls._gen_binary_util()
     supportdir, version, name = binaries[binary_key]
-    binary = binary_util._select_binary_base_path(supportdir, version, binary_key)
-    return '{base}/{binary}'.format(base=base, binary=binary)
+    binary_request = binary_util._make_deprecated_binary_request(supportdir, version, name)
+
+    binary_path = binary_request.get_download_path(binary_util._host_platform())
+    return '{base}/{binary}'.format(base=base, binary=binary_path)
+
+  @classmethod
+  def _gen_binary_tool_fetcher(cls, bootstrap_dir='/tmp', timeout_secs=30, fetcher=None,
+                               ignore_cached_download=True):
+    return BinaryToolFetcher(
+      bootstrap_dir=bootstrap_dir,
+      timeout_secs=timeout_secs,
+      fetcher=fetcher,
+      ignore_cached_download=ignore_cached_download)
+
+  @classmethod
+  def _gen_binary_util(cls, baseurls=[], path_by_id=None, allow_external_binary_tool_downloads=True,
+                       uname_func=None, **kwargs):
+    return BinaryUtilPrivate(
+      baseurls=baseurls,
+      binary_tool_fetcher=cls._gen_binary_tool_fetcher(**kwargs),
+      path_by_id=path_by_id,
+      allow_external_binary_tool_downloads=allow_external_binary_tool_downloads,
+      uname_func=uname_func)
+
+  @classmethod
+  def _read_file(cls, file_path):
+    with open(file_path, 'rb') as result_file:
+      return result_file.read()
 
   def test_timeout(self):
     fetcher = mock.create_autospec(Fetcher, spec_set=True)
-    binary_util = BinaryUtilPrivate(baseurls=['http://binaries.example.com'],
-                                    timeout_secs=42,
-                                    bootstrapdir='/tmp')
+    timeout_value = 42
+    binary_util = self._gen_binary_util(baseurls=['http://binaries.example.com'],
+                                        timeout_secs=timeout_value,
+                                        fetcher=fetcher)
     self.assertFalse(fetcher.download.called)
 
-    with binary_util._select_binary_stream('a-binary', 'a-binary/v1.2/a-binary', fetcher=fetcher):
-      fetcher.download.assert_called_once_with('http://binaries.example.com/a-binary/v1.2/a-binary',
-                                               listener=mock.ANY,
-                                               path_or_fd=mock.ANY,
-                                               timeout_secs=42)
+    fetch_path = binary_util.select_script(supportdir='a-binary', version='v1.2', name='a-binary')
+    logger.debug("fetch_path: {}".format(fetch_path))
+    fetcher.download.assert_called_once_with('http://binaries.example.com/a-binary/v1.2/a-binary',
+                                             listener=mock.ANY,
+                                             path_or_fd=mock.ANY,
+                                             timeout_secs=timeout_value)
 
-  def test_nobases(self):
+  def test_no_base_urls_error(self):
     """Tests exception handling if build support urls are improperly specified."""
-    binary_util = BinaryUtilPrivate(baseurls=[], timeout_secs=30, bootstrapdir='/tmp')
-    with self.assertRaises(binary_util.NoBaseUrlsError):
-      binary_path = binary_util._select_binary_base_path(supportdir='bin/protobuf',
-                                                         version='2.4.1',
-                                                         name='protoc')
-      with binary_util._select_binary_stream(name='protoc', binary_path=binary_path):
-        self.fail('Expected acquisition of the stream to raise.')
+    binary_util = self._gen_binary_util()
+
+    with self.assertRaises(BinaryUtilPrivate.BinaryResolutionError) as cm:
+      binary_util.select_script("supportdir", "version", "name")
+    the_raised_exception_message = str(cm.exception)
+
+    self.assertIn(BinaryUtilPrivate.NoBaseUrlsError.__name__, the_raised_exception_message)
+    expected_msg = (
+      "Error resolving binary request BinaryRequest(supportdir=supportdir, version=version, "
+      "name=name, platform_dependent=False, external_url_generator=None, archiver=None): "
+      "--binaries-baseurls is empty.")
+    self.assertIn(expected_msg, the_raised_exception_message)
 
   def test_support_url_multi(self):
     """Tests to make sure existing base urls function as expected."""
 
+    bootstrap_dir = '/tmp'
+
     with temporary_dir() as invalid_local_files, temporary_dir() as valid_local_files:
-      binary_util = BinaryUtilPrivate(
+      binary_util = self._gen_binary_util(
         baseurls=[
           'BLATANTLY INVALID URL',
           'https://dl.bintray.com/pantsbuild/bin/reasonably-invalid-url',
@@ -89,18 +139,23 @@ def test_support_url_multi(self):
           valid_local_files,
           'https://dl.bintray.com/pantsbuild/bin/another-invalid-url',
         ],
-        timeout_secs=30,
-        bootstrapdir='/tmp')
+        bootstrap_dir=bootstrap_dir)
+
+      binary_request = binary_util._make_deprecated_binary_request(
+        supportdir='bin/protobuf',
+        version='2.4.1',
+        name='protoc')
 
-      binary_path = binary_util._select_binary_base_path(supportdir='bin/protobuf',
-                                                         version='2.4.1',
-                                                         name='protoc')
+      binary_path = binary_request.get_download_path(binary_util._host_platform())
       contents = b'proof'
       with safe_open(os.path.join(valid_local_files, binary_path), 'wb') as fp:
         fp.write(contents)
 
-      with binary_util._select_binary_stream(name='protoc', binary_path=binary_path) as stream:
-        self.assertEqual(contents, stream())
+      binary_path_abs = os.path.join(bootstrap_dir, binary_path)
+
+      self.assertEqual(binary_path_abs, binary_util.select(binary_request))
+
+      self.assertEqual(contents, self._read_file(binary_path_abs))
 
   def test_support_url_fallback(self):
     """Tests fallback behavior with multiple support baseurls.
@@ -110,7 +165,6 @@ def test_support_url_fallback(self):
     """
     fake_base, fake_url = self._fake_base, self._fake_url
     bases = [fake_base('apple'), fake_base('orange'), fake_base('banana')]
-    binary_util = BinaryUtilPrivate(bases, 30, '/tmp')
 
     binaries = {t[2]: t for t in (('bin/protobuf', '2.4.1', 'protoc'),
                                   ('bin/ivy', '4.3.7', 'ivy'),
@@ -124,68 +178,158 @@ def test_support_url_fallback(self):
       fake_url(binaries, bases[2], 'ivy'): 'UNSEEN IVY 2',
     })
 
+    binary_util = self._gen_binary_util(
+      baseurls=bases,
+      fetcher=fetcher)
+
     unseen = [item for item in fetcher.values() if item.startswith('SEEN ')]
     for supportdir, version, name in binaries.values():
-      binary_path = binary_util._select_binary_base_path(supportdir=supportdir,
-                                                         version=version,
-                                                         name=name)
-      with binary_util._select_binary_stream(name=name,
-                                             binary_path=binary_path,
-                                             fetcher=fetcher) as stream:
-        result = stream()
-        self.assertEqual(result, 'SEEN ' + name.upper())
-        unseen.remove(result)
+      binary_path_abs = binary_util.select_binary(
+        supportdir=supportdir,
+        version=version,
+        name=name)
+      expected_content = 'SEEN {}'.format(name.upper())
+      self.assertEqual(expected_content, self._read_file(binary_path_abs))
+      unseen.remove(expected_content)
     self.assertEqual(0, len(unseen))  # Make sure we've seen all the SEENs.
 
   def test_select_binary_base_path_linux(self):
-    binary_util = BinaryUtilPrivate([], 0, '/tmp')
-
     def uname_func():
       return "linux", "dontcare1", "dontcare2", "dontcare3", "amd64"
 
+    binary_util = self._gen_binary_util(uname_func=uname_func)
+
+    binary_request = binary_util._make_deprecated_binary_request("supportdir", "version", "name")
+
     self.assertEquals("supportdir/linux/x86_64/version/name",
-                      binary_util._select_binary_base_path("supportdir", "version", "name",
-                                                           uname_func=uname_func))
+                      binary_util._get_download_path(binary_request))
 
   def test_select_binary_base_path_darwin(self):
-    binary_util = BinaryUtilPrivate([], 0, '/tmp')
-
     def uname_func():
       return "darwin", "dontcare1", "14.9", "dontcare2", "dontcare3",
 
+    binary_util = self._gen_binary_util(uname_func=uname_func)
+
+    binary_request = binary_util._make_deprecated_binary_request("supportdir", "version", "name")
+
     self.assertEquals("supportdir/mac/10.10/version/name",
-                      binary_util._select_binary_base_path("supportdir", "version", "name",
-                                                           uname_func=uname_func))
+                      binary_util._get_download_path(binary_request))
 
   def test_select_binary_base_path_missing_os(self):
-    binary_util = BinaryUtilPrivate([], 0, '/tmp')
-
     def uname_func():
       return "vms", "dontcare1", "999.9", "dontcare2", "VAX9"
 
-    with self.assertRaisesRegexp(BinaryUtilPrivate.MissingMachineInfo,
-                                 r'Pants has no binaries for vms'):
-      binary_util._select_binary_base_path("supportdir", "version", "name", uname_func=uname_func)
+    binary_util = self._gen_binary_util(uname_func=uname_func)
+
+    with self.assertRaises(BinaryUtilPrivate.BinaryResolutionError) as cm:
+      binary_util.select_binary("supportdir", "version", "name")
+
+    the_raised_exception_message = str(cm.exception)
+
+    self.assertIn(BinaryUtilPrivate.MissingMachineInfo.__name__, the_raised_exception_message)
+    expected_msg = (
+      "Error resolving binary request BinaryRequest(supportdir=supportdir, version=version, "
+      "name=name, platform_dependent=True, external_url_generator=None, archiver=None): "
+      "Pants could not resolve binaries for the current host: platform 'vms' was not recognized. "
+      "Recognized platforms are: [u'darwin', u'linux'].")
+    self.assertIn(expected_msg, the_raised_exception_message)
 
   def test_select_binary_base_path_missing_version(self):
-    binary_util = BinaryUtilPrivate([], 0, '/tmp')
+    def uname_func():
+      return "darwin", "dontcare1", "999.9", "dontcare2", "x86_64"
+
+    binary_util = self._gen_binary_util(uname_func=uname_func)
+
+    with self.assertRaises(BinaryUtilPrivate.BinaryResolutionError) as cm:
+      binary_util.select_binary("mysupportdir", "myversion", "myname")
+    the_raised_exception_message = str(cm.exception)
+
+    self.assertIn(BinaryUtilPrivate.MissingMachineInfo.__name__, the_raised_exception_message)
+    expected_msg = (
+      "Error resolving binary request BinaryRequest(supportdir=mysupportdir, version=myversion, "
+      "name=myname, platform_dependent=True, external_url_generator=None, archiver=None): Pants could not "
+      "resolve binaries for the current host. Update --binaries-path-by-id to find binaries for "
+      "the current host platform (u\'darwin\', u\'999\').\\n--binaries-path-by-id was:")
+    self.assertIn(expected_msg, the_raised_exception_message)
 
+  def test_select_script_missing_version(self):
     def uname_func():
       return "darwin", "dontcare1", "999.9", "dontcare2", "x86_64"
 
-    os_id = ('darwin', '999')
-    with self.assertRaisesRegexp(BinaryUtilPrivate.MissingMachineInfo,
-                                 r'myname.*Update --binaries-path-by-id to find binaries for '
-                                 r'{}'.format(re.escape(repr(os_id)))):
-      binary_util._select_binary_base_path("supportdir", "myversion", "myname", uname_func=uname_func)
+    binary_util = self._gen_binary_util(uname_func=uname_func)
 
-  def test_select_binary_base_path_override(self):
-    binary_util = BinaryUtilPrivate([], 0, '/tmp',
-                                    {('darwin', '100'): ['skynet', '42']})
+    with self.assertRaises(BinaryUtilPrivate.BinaryResolutionError) as cm:
+      binary_util.select_script("mysupportdir", "myversion", "myname")
+    the_raised_exception_message = str(cm.exception)
+
+    self.assertIn(BinaryUtilPrivate.MissingMachineInfo.__name__, the_raised_exception_message)
+    expected_msg = (
+      "Error resolving binary request BinaryRequest(supportdir=mysupportdir, version=myversion, "
+      # platform_dependent=False when doing select_script()
+      "name=myname, platform_dependent=False, external_url_generator=None, archiver=None): Pants "
+      "could not resolve binaries for the current host. Update --binaries-path-by-id to find "
+      "binaries for the current host platform (u\'darwin\', u\'999\').\\n--binaries-path-by-id was:")
+    self.assertIn(expected_msg, the_raised_exception_message)
 
+  def test_select_binary_base_path_override(self):
     def uname_func():
       return "darwin", "dontcare1", "100.99", "dontcare2", "t1000"
 
+    binary_util = self._gen_binary_util(uname_func=uname_func,
+                                        path_by_id={('darwin', '100'): ['skynet', '42']})
+
+    binary_request = binary_util._make_deprecated_binary_request("supportdir", "version", "name")
+
     self.assertEquals("supportdir/skynet/42/version/name",
-                      binary_util._select_binary_base_path("supportdir", "version", "name",
-                                                           uname_func=uname_func))
+                      binary_util._get_download_path(binary_request))
+
+  def test_external_url_generator(self):
+    binary_util = self._gen_binary_util(baseurls=[])
+
+    binary_request = BinaryRequest(
+      supportdir='supportdir',
+      version='version',
+      name='name',
+      platform_dependent=False,
+      external_url_generator=ExternalUrlGenerator(),
+      # TODO: test archiver!
+      archiver=None)
+
+    with self.assertRaises(BinaryUtilPrivate.BinaryResolutionError) as cm:
+      binary_util.select(binary_request)
+    the_raised_exception_message = str(cm.exception)
+
+    self.assertIn(BinaryToolFetcher.BinaryNotFound.__name__, the_raised_exception_message)
+    expected_msg = (
+      "Error resolving binary request BinaryRequest(supportdir=supportdir, version=version, "
+      "name=name, platform_dependent=False, "
+      "external_url_generator=ExternalUrlGenerator(<example __str__()>), archiver=None): "
+      "Failed to fetch name binary from any source: (Failed to fetch binary from "
+      "https://example.com/some-binary: Fetch of https://example.com/some-binary failed with "
+      "status code 404, Failed to fetch binary from https://example.com/same-binary: Fetch of "
+      "https://example.com/same-binary failed with status code 404)'")
+    self.assertIn(expected_msg, the_raised_exception_message)
+
+  def test_disallowing_external_urls(self):
+    binary_util = self._gen_binary_util(baseurls=[], allow_external_binary_tool_downloads=False)
+
+    binary_request = binary_request = BinaryRequest(
+      supportdir='supportdir',
+      version='version',
+      name='name',
+      platform_dependent=False,
+      external_url_generator=ExternalUrlGenerator(),
+      # TODO: test archiver!
+      archiver=None)
+
+    with self.assertRaises(BinaryUtilPrivate.BinaryResolutionError) as cm:
+      binary_util.select(binary_request)
+    the_raised_exception_message = str(cm.exception)
+
+    self.assertIn(BinaryUtilPrivate.NoBaseUrlsError.__name__, the_raised_exception_message)
+    expected_msg = (
+      "Error resolving binary request BinaryRequest(supportdir=supportdir, version=version, "
+      "name=name, platform_dependent=False, "
+      "external_url_generator=ExternalUrlGenerator(<example __str__()>), archiver=None): "
+      "--binaries-baseurls is empty.")
+    self.assertIn(expected_msg, the_raised_exception_message)
diff --git a/tests/python/pants_test/fs/test_archive.py b/tests/python/pants_test/fs/test_archive.py
index 5d8ec667af3..388221c7b64 100644
--- a/tests/python/pants_test/fs/test_archive.py
+++ b/tests/python/pants_test/fs/test_archive.py
@@ -8,7 +8,7 @@
 import os
 import unittest
 
-from pants.fs.archive import archiver
+from pants.fs.archive import create_archiver
 from pants.util.contextutil import temporary_dir
 from pants.util.dirutil import relative_symlink, safe_mkdir, safe_walk, touch
 
@@ -53,12 +53,14 @@ def test_round_trip(prefix=None, concurrency_safe=False):
     test_round_trip(concurrency_safe=True)
 
   def test_tar(self):
-    self.round_trip(archiver('tar'), expected_ext='tar', empty_dirs=True)
-    self.round_trip(archiver('tgz'), expected_ext='tar.gz', empty_dirs=True)
-    self.round_trip(archiver('tbz2'), expected_ext='tar.bz2', empty_dirs=True)
+    # TODO: test XZCompressedTarArchiver? Needs an xz BinaryTool, so hard to see how to do in a
+    # unit test.
+    self.round_trip(create_archiver('tar'), expected_ext='tar', empty_dirs=True)
+    self.round_trip(create_archiver('tgz'), expected_ext='tar.gz', empty_dirs=True)
+    self.round_trip(create_archiver('tbz2'), expected_ext='tar.bz2', empty_dirs=True)
 
   def test_zip(self):
-    self.round_trip(archiver('zip'), expected_ext='zip', empty_dirs=False)
+    self.round_trip(create_archiver('zip'), expected_ext='zip', empty_dirs=False)
 
   def test_zip_filter(self):
     def do_filter(path):
@@ -69,9 +71,9 @@ def do_filter(path):
       touch(os.path.join(fromdir, 'disallowed.txt'))
 
       with temporary_dir() as archivedir:
-        archive = archiver('zip').create(fromdir, archivedir, 'archive')
+        archive = create_archiver('zip').create(fromdir, archivedir, 'archive')
         with temporary_dir() as todir:
-          archiver('zip').extract(archive, todir, filter_func=do_filter)
+          create_archiver('zip').extract(archive, todir, filter_func=do_filter)
           self.assertEquals(set(['allowed.txt']), self._listtree(todir, empty_dirs=False))
 
   def test_tar_dereference(self):
@@ -84,9 +86,9 @@ def check_archive_with_flags(archive_format, dereference):
         relative_symlink(filename, linkname)
 
         with temporary_dir() as archivedir:
-          archive = archiver(archive_format).create(fromdir, archivedir, 'archive', dereference=dereference)
+          archive = create_archiver(archive_format).create(fromdir, archivedir, 'archive', dereference=dereference)
           with temporary_dir() as todir:
-            archiver(archive_format).extract(archive, todir)
+            create_archiver(archive_format).extract(archive, todir)
             extracted_linkname = os.path.join(todir, 'link_to_a')
             assertion = self.assertFalse if dereference else self.assertTrue
             assertion(os.path.islink(extracted_linkname))