From 17a4826c9566cb944b36c88f6306729495e1d67c Mon Sep 17 00:00:00 2001
From: Chris Livingston <clivingston@twitter.com>
Date: Fri, 13 Jul 2018 17:15:40 -0700
Subject: [PATCH 1/5] Revert master merge

---
 .../hello/setup_requires/setup.cfg            |   2 +
 .../pants/backend/native/subsystems/conan.py  |  79 ++++++
 .../native/targets/external_native_library.py |  43 +++
 .../tasks/native_external_library_fetch.py    | 255 ++++++++++++++++++
 .../ctypes_with_third_party/BUILD             |  46 ++++
 .../ctypes_with_third_party/__init__.py       |   0
 .../ctypes_python_pkg/__init__.py             |   0
 .../ctypes_python_pkg/ctypes_wrapper.py       |  26 ++
 .../ctypes_with_third_party/main.py           |  13 +
 .../ctypes_with_third_party/setup.py          |  17 ++
 .../some_more_math.hpp                        |   8 +
 .../some_more_math_with_third_party.cpp       |  60 +++++
 .../test_native_external_library_fetch.py     |  74 +++++
 13 files changed, 623 insertions(+)
 create mode 100644 examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg
 create mode 100644 src/python/pants/backend/native/subsystems/conan.py
 create mode 100644 src/python/pants/backend/native/targets/external_native_library.py
 create mode 100644 src/python/pants/backend/native/tasks/native_external_library_fetch.py
 create mode 100644 testprojects/src/python/python_distribution/ctypes_with_third_party/BUILD
 create mode 100644 testprojects/src/python/python_distribution/ctypes_with_third_party/__init__.py
 create mode 100644 testprojects/src/python/python_distribution/ctypes_with_third_party/ctypes_python_pkg/__init__.py
 create mode 100644 testprojects/src/python/python_distribution/ctypes_with_third_party/ctypes_python_pkg/ctypes_wrapper.py
 create mode 100644 testprojects/src/python/python_distribution/ctypes_with_third_party/main.py
 create mode 100644 testprojects/src/python/python_distribution/ctypes_with_third_party/setup.py
 create mode 100644 testprojects/src/python/python_distribution/ctypes_with_third_party/some_more_math.hpp
 create mode 100644 testprojects/src/python/python_distribution/ctypes_with_third_party/some_more_math_with_third_party.cpp
 create mode 100644 tests/python/pants_test/backend/python/tasks/test_native_external_library_fetch.py

diff --git a/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg b/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg
new file mode 100644
index 00000000000..3480374bc2f
--- /dev/null
+++ b/examples/src/python/example/python_distribution/hello/setup_requires/setup.cfg
@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1
\ No newline at end of file
diff --git a/src/python/pants/backend/native/subsystems/conan.py b/src/python/pants/backend/native/subsystems/conan.py
new file mode 100644
index 00000000000..e4c177ab1c6
--- /dev/null
+++ b/src/python/pants/backend/native/subsystems/conan.py
@@ -0,0 +1,79 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+
+import logging
+import os
+
+from pex.interpreter import PythonInterpreter
+from pex.pex import PEX
+from pex.pex_builder import PEXBuilder
+from pex.pex_info import PexInfo
+
+from pants.backend.python.python_requirement import PythonRequirement
+from pants.backend.python.tasks.pex_build_util import dump_requirements
+from pants.backend.python.tasks.wrapped_pex import WrappedPEX
+from pants.base.build_environment import get_pants_cachedir
+from pants.subsystem.subsystem import Subsystem
+from pants.util.dirutil import safe_concurrent_creation
+from pants.util.objects import datatype
+
+
+logger = logging.getLogger(__name__)
+
+
+class Conan(Subsystem):
+  """Pex binary for the conan package manager."""
+  options_scope = 'conan'
+  default_conan_requirements = (
+    'conan==1.4.4',
+    'PyJWT>=1.4.0, <2.0.0',
+    'requests>=2.7.0, <3.0.0',
+    'colorama>=0.3.3, <0.4.0',
+    'PyYAML>=3.11, <3.13.0',
+    'patch==1.16',
+    'fasteners>=0.14.1',
+    'six>=1.10.0',
+    'node-semver==0.2.0',
+    'distro>=1.0.2, <1.2.0',
+    'pylint>=1.8.1, <1.9.0',
+    'future==0.16.0',
+    'pygments>=2.0, <3.0',
+    'astroid>=1.6, <1.7',
+    'deprecation>=2.0, <2.1'
+  )
+
+  @classmethod
+  def implementation_version(cls):
+    return super(Conan, cls).implementation_version() + [('Conan', 0)]
+
+  @classmethod
+  def register_options(cls, register):
+    super(Conan, cls).register_options(register)
+    register('--conan-requirements', type=list, default=cls.default_conan_requirements,
+             advanced=True, help='The requirements used to build the conan client pex.')
+
+  class ConanBinary(datatype(['pex'])):
+    """A `conan` PEX binary."""
+    pass
+
+  def bootstrap_conan(self):
+    pex_info = PexInfo.default()
+    pex_info.entry_point = 'conans.conan'
+    conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support')
+    conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary')
+    interpreter = PythonInterpreter.get()
+    if os.path.exists(conan_pex_path):
+      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter), interpreter)
+      return self.ConanBinary(pex=conan_binary)
+    else:  
+      with safe_concurrent_creation(conan_pex_path) as safe_path:
+        builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
+        reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements]
+        dump_requirements(builder, interpreter, reqs, logger)
+        builder.freeze()
+      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter), interpreter)
+      return self.ConanBinary(pex=conan_binary)
diff --git a/src/python/pants/backend/native/targets/external_native_library.py b/src/python/pants/backend/native/targets/external_native_library.py
new file mode 100644
index 00000000000..bd7247ace79
--- /dev/null
+++ b/src/python/pants/backend/native/targets/external_native_library.py
@@ -0,0 +1,43 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import re
+
+from pants.base.payload import Payload
+from pants.base.payload_field import PrimitiveField
+from pants.base.validation import assert_list
+from pants.build_graph.target import Target
+
+
+class ExternalNativeLibrary(Target):
+  """A set of Conan package strings to be passed to the Conan package manager."""
+
+  @classmethod
+  def alias(cls):
+    return 'external_native_library'
+
+  def __init__(self, payload=None, packages=None, **kwargs):
+    """
+    :param packages: a list of Conan-style package strings
+
+    Example:
+      lzo/2.10@twitter/stable
+    """
+    payload = payload or Payload()
+
+    assert_list(packages, key_arg='packages')
+    payload.add_fields({
+      'packages': PrimitiveField(packages),
+    })
+    super(ExternalNativeLibrary, self).__init__(payload=payload, **kwargs)
+
+  @property
+  def packages(self):
+    return self.payload.packages
+
+  @property
+  def lib_names(self):
+    return [re.match(r'^([^\/]+)\/', pkg_name).group(1) for pkg_name in self.payload.packages]
diff --git a/src/python/pants/backend/native/tasks/native_external_library_fetch.py b/src/python/pants/backend/native/tasks/native_external_library_fetch.py
new file mode 100644
index 00000000000..c9a6f7b6533
--- /dev/null
+++ b/src/python/pants/backend/native/tasks/native_external_library_fetch.py
@@ -0,0 +1,255 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import os
+import re
+from distutils.dir_util import copy_tree
+
+from pants.backend.native.config.environment import Platform
+from pants.backend.native.subsystems.conan import Conan
+from pants.backend.native.targets.external_native_library import ExternalNativeLibrary
+from pants.base.exceptions import TaskError
+from pants.invalidation.cache_manager import VersionedTargetSet
+from pants.task.task import Task
+from pants.util.contextutil import environment_as
+from pants.util.dirutil import safe_mkdir
+from pants.util.memo import memoized_property
+from pants.util.objects import Exactly, datatype
+from pants.util.process_handler import subprocess
+
+
+class ConanRequirement(datatype(['pkg_spec'])):
+  """A wrapper class to encapsulate a Conan package requirement."""
+
+  CONAN_OS_NAME = {
+    'darwin': lambda: 'Macos',
+    'linux': lambda: 'Linux',
+  }
+
+  def parse_conan_stdout_for_pkg_sha(self, stdout):
+    # TODO(cmlivingston): regex this
+    pkg_line = stdout.split('Packages')[1]
+    collected_matches = [line for line in pkg_line.split() if self.pkg_spec in line]
+    pkg_sha = collected_matches[0].split(':')[1]
+    return pkg_sha
+
+  @memoized_property
+  def directory_path(self):
+    return self.pkg_spec.replace('@', '/')
+
+  @memoized_property
+  def fetch_cmdline_args(self):
+    platform = Platform.create()
+    conan_os_name = platform.resolve_platform_specific(self.CONAN_OS_NAME)
+    args = ['install', self.pkg_spec, '-s', 'os={}'.format(conan_os_name)]
+    return args
+
+
+class NativeExternalLibraryFetch(Task):
+  options_scope = 'native-external-library-fetch'
+  native_library_constraint = Exactly(ExternalNativeLibrary)
+
+  class NativeExternalLibraryFetchError(TaskError):
+    pass
+
+  class NativeExternalLibraryFiles(object):
+    def __init__(self):
+      self.include_dir = None
+      self.lib_dir = None
+      self.lib_names = []
+
+    def add_lib_name(self, lib_name):
+      self.lib_names.append(lib_name)
+
+    def get_third_party_lib_args(self):
+      lib_args = []
+      if self.lib_names:
+        for lib_name in self.lib_names:
+          lib_args.append('-l{}'.format(lib_name))
+        lib_dir_arg = '-L{}'.format(self.lib_dir)
+        lib_args.append(lib_dir_arg)
+      return lib_args
+
+  @classmethod
+  def _parse_lib_name_from_library_filename(cls, filename):
+    match_group = re.match(r"^lib(.*)\.(a|so|dylib)$", filename)
+    if match_group:
+      return match_group.group(1)
+    return None
+
+  @classmethod
+  def register_options(cls, register):
+    super(NativeExternalLibraryFetch, cls).register_options(register)
+    register('--conan-remotes', type=list, default=['https://conan.bintray.com'], advanced=True,
+             fingerprint=True, help='The conan remote to download conan packages from.')
+
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(NativeExternalLibraryFetch, cls).subsystem_dependencies() + (Conan.scoped(cls),)
+
+  @classmethod
+  def product_types(cls):
+    return [cls.NativeExternalLibraryFiles]
+
+  @property
+  def cache_target_dirs(self):
+    return True
+
+  @memoized_property
+  def _conan_binary(self):
+    return Conan.scoped_instance(self).bootstrap_conan()
+
+  def execute(self):
+    task_product = self.context.products.get_data(self.NativeExternalLibraryFiles,
+                                                  self.NativeExternalLibraryFiles)
+
+    native_lib_tgts = self.context.targets(self.native_library_constraint.satisfied_by)
+    if native_lib_tgts:
+      with self.invalidated(native_lib_tgts,
+                            invalidate_dependents=True) as invalidation_check:
+        resolve_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
+        vts_results_dir = self._prepare_vts_results_dir(resolve_vts)
+        if invalidation_check.invalid_vts or not resolve_vts.valid:
+          for vt in invalidation_check.all_vts:
+            self._fetch_packages(vt, vts_results_dir)
+        self._populate_task_product(vts_results_dir, task_product)
+
+  def _prepare_vts_results_dir(self, vts):
+    """
+    Given a `VergetTargetSet`, prepare its results dir.
+    """
+    vt_set_results_dir = os.path.join(self.workdir, vts.cache_key.hash)
+    safe_mkdir(vt_set_results_dir)
+    return vt_set_results_dir
+
+  def _populate_task_product(self, results_dir, task_product):
+    """
+    Sets the relevant properties of the task product (`NativeExternalLibraryFiles`) object.
+    """
+    lib = os.path.join(results_dir, 'lib')
+    include = os.path.join(results_dir, 'include')
+
+    if os.path.exists(lib):
+      task_product.lib_dir = lib
+      for filename in os.listdir(lib):
+        lib_name = self._parse_lib_name_from_library_filename(filename)
+        if lib_name:
+          task_product.add_lib_name(lib_name)
+
+    if os.path.exists(include):
+      task_product.include_dir = include
+
+  def _get_conan_data_dir_path_for_package(self, pkg_dir_path, pkg_sha):
+    return os.path.join(self.workdir,
+                        '.conan',
+                        'data',
+                        pkg_dir_path,
+                        'package',
+                        pkg_sha)
+
+  def _remove_conan_center_remote_cmdline(self, conan_binary):
+    return conan_binary.pex.cmdline(['remote',
+                                     'remove',
+                                     'conan-center'])
+
+  def _add_pants_conan_remote_cmdline(self, conan_binary, remote_index_num, remote_url):
+    return conan_binary.pex.cmdline(['remote',
+                                      'add',
+                                      'pants-conan-remote-' + str(remote_index_num),
+                                      remote_url,
+                                      '--insert'])
+
+  def ensure_conan_remote_configuration(self, conan_binary):
+    """
+    Ensure that the conan registry.txt file is sanitized and loaded with
+    a pants-specific remote for package fetching.
+
+    :param conan_binary: The conan client pex to use for manipulating registry.txt.
+    """
+
+    # Conan will prepend the conan-center remote to the remote registry when
+    # bootstrapped for the first time, so we want to delete it from the registry
+    # and replace it with Pants-controlled remotes.
+    remove_conan_center_remote_cmdline = self._remove_conan_center_remote_cmdline(conan_binary)
+    try:
+      stdout = subprocess.check_output(remove_conan_center_remote_cmdline.split())
+      self.context.log.debug(stdout)
+    except subprocess.CalledProcessError as e:
+      if not "'conan-center' not found in remotes" in e.output:
+        raise TaskError('Error deleting conan-center from conan registry: {}'.format(e.output))
+
+    # Add the pants-specific conan remote.
+    index_num = 0
+    for remote_url in reversed(self.get_options().conan_remotes):
+      index_num += 1
+      # NB: --insert prepends a remote to conan's remote list. We reverse the options remote
+      # list to maintain a sensible default for conan emote search order.
+      add_pants_conan_remote_cmdline = self._add_pants_conan_remote_cmdline(conan_binary,
+                                                                            index_num,
+                                                                            remote_url)
+      try:
+        stdout = subprocess.check_output(add_pants_conan_remote_cmdline.split())
+        self.context.log.debug(stdout)
+      except subprocess.CalledProcessError as e:
+        if not "already exists in remotes" in e.output:
+          raise TaskError('Error adding pants-specific conan remote: {}'.format(e.output))
+
+  def _copy_package_contents_from_conan_dir(self, results_dir, conan_requirement, pkg_sha):
+    """
+    Copy the contents of the fetched package into the results directory of the versioned
+    target from the conan data directory.
+
+    :param results_dir: A results directory to copy conan package contents to.
+    :param conan_requirement: The `ConanRequirement` object that produced the package sha.
+    :param pkg_sha: The sha of the local conan package corresponding to the specification.
+    """
+    src = self._get_conan_data_dir_path_for_package(conan_requirement.directory_path, pkg_sha)
+    src_lib = os.path.join(src, 'lib')
+    src_include = os.path.join(src, 'include')
+    dest_lib = os.path.join(results_dir, 'lib')
+    dest_include = os.path.join(results_dir, 'include')
+    if os.path.exists(src_lib):
+      copy_tree(src_lib, dest_lib)
+    if os.path.exists(src_include):
+      copy_tree(src_include, dest_include)
+
+  def _fetch_packages(self, vt, vts_results_dir):
+    """
+    Invoke the conan pex to fetch conan packages specified by a
+    `ExternalLibLibrary` target.
+
+    :param vt: a versioned target containing conan package specifications.
+    :param vts_results_dir: the results directory of the VersionedTargetSet
+      for the purpose of aggregating package contents.
+    """
+
+    # NB: CONAN_USER_HOME specifies the directory to use for the .conan data directory.
+    # This will initially live under the workdir to provide easy debugging on the initial
+    # iteration of this system (a 'clean-all' will nuke the conan dir). In the future,
+    # it would be good to migrate this under ~/.cache/pants/conan for persistence.
+    with environment_as(CONAN_USER_HOME=self.workdir):
+      for pkg_spec in vt.target.packages:
+
+        conan_requirement = ConanRequirement(pkg_spec=pkg_spec)
+
+        # Prepare conan command line and ensure remote is configured properly.
+        self.ensure_conan_remote_configuration(self._conan_binary)
+        args = conan_requirement.fetch_cmdline_args
+        cmdline = self._conan_binary.pex.cmdline(args)
+
+        self.context.log.debug('Running conan.pex cmdline: {}'.format(cmdline))
+        self.context.log.debug('Conan remotes: {}'.format(self.get_options().conan_remotes))
+
+        # Invoke conan to pull package from remote.
+        try:
+          stdout = subprocess.check_output(cmdline.split())
+        except subprocess.CalledProcessError as e:
+          raise self.NativeExternalLibraryFetchError(
+            "Error invoking conan for fetch task: {}\n".format(e.output)
+          )
+
+        pkg_sha = conan_requirement.parse_conan_stdout_for_pkg_sha(stdout)
+        self._copy_package_contents_from_conan_dir(vts_results_dir, conan_requirement, pkg_sha)
diff --git a/testprojects/src/python/python_distribution/ctypes_with_third_party/BUILD b/testprojects/src/python/python_distribution/ctypes_with_third_party/BUILD
new file mode 100644
index 00000000000..fe38dd471e2
--- /dev/null
+++ b/testprojects/src/python/python_distribution/ctypes_with_third_party/BUILD
@@ -0,0 +1,46 @@
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+## Third party integration testing.
+
+ctypes_compatible_cpp_library(
+  name='cpp_library_with_third_party',
+  sources=['some_more_math.hpp', 'some_more_math_with_third_party.cpp'],
+  ctypes_native_library=native_artifact(lib_name='asdf-cpp-tp'),
+  dependencies=[':rang', ':cereal'],
+  fatal_warnings=False
+)
+
+python_dist(
+  name='python_dist_with_third_party_cpp',
+  sources=[
+    'setup.py',
+    'ctypes_python_pkg/__init__.py',
+    'ctypes_python_pkg/ctypes_wrapper.py',
+  ],
+  dependencies=[
+    ':cpp_library_with_third_party',
+  ],
+)
+
+python_binary(
+  name='bin_with_third_party',
+  source='main.py',
+  dependencies=[
+    ':python_dist_with_third_party_cpp',
+  ],
+)
+
+external_native_library(
+  name='rang',
+  packages=[
+     'rang/3.1.0@rang/stable',
+  ],
+)
+
+external_native_library(
+  name='cereal',
+  packages=[
+     'cereal/1.2.2@conan/stable',
+  ],
+)
diff --git a/testprojects/src/python/python_distribution/ctypes_with_third_party/__init__.py b/testprojects/src/python/python_distribution/ctypes_with_third_party/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/testprojects/src/python/python_distribution/ctypes_with_third_party/ctypes_python_pkg/__init__.py b/testprojects/src/python/python_distribution/ctypes_with_third_party/ctypes_python_pkg/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/testprojects/src/python/python_distribution/ctypes_with_third_party/ctypes_python_pkg/ctypes_wrapper.py b/testprojects/src/python/python_distribution/ctypes_with_third_party/ctypes_python_pkg/ctypes_wrapper.py
new file mode 100644
index 00000000000..675f4557cf8
--- /dev/null
+++ b/testprojects/src/python/python_distribution/ctypes_with_third_party/ctypes_python_pkg/ctypes_wrapper.py
@@ -0,0 +1,26 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import ctypes
+import os
+
+
+def get_generated_shared_lib(lib_name):
+  # These are the same filenames as in setup.py.
+  filename = 'lib{}.so'.format(lib_name)
+  # The data files are in the root directory, but we are in ctypes_python_pkg/.
+  rel_path = os.path.join(os.path.dirname(__file__), '..', filename)
+  return os.path.normpath(rel_path)
+
+
+asdf_cpp_lib_path = get_generated_shared_lib('asdf-cpp-tp')
+asdf_cpp_lib = ctypes.CDLL(asdf_cpp_lib_path)
+
+def f(x):
+  added = x + 3
+  multiplied = asdf_cpp_lib.multiply_by_three(added)
+  return multiplied
diff --git a/testprojects/src/python/python_distribution/ctypes_with_third_party/main.py b/testprojects/src/python/python_distribution/ctypes_with_third_party/main.py
new file mode 100644
index 00000000000..b0f77a64ce9
--- /dev/null
+++ b/testprojects/src/python/python_distribution/ctypes_with_third_party/main.py
@@ -0,0 +1,13 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from ctypes_python_pkg.ctypes_wrapper import f
+
+
+if __name__ == '__main__':
+  x = 3
+  print('x={}, f(x)={}'.format(x, f(x)))
diff --git a/testprojects/src/python/python_distribution/ctypes_with_third_party/setup.py b/testprojects/src/python/python_distribution/ctypes_with_third_party/setup.py
new file mode 100644
index 00000000000..c94bfa8c165
--- /dev/null
+++ b/testprojects/src/python/python_distribution/ctypes_with_third_party/setup.py
@@ -0,0 +1,17 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from setuptools import setup, find_packages
+
+
+setup(
+  name='ctypes_test',
+  version='0.0.1',
+  packages=find_packages(),
+  # Declare two files at the top-level directory (denoted by '').
+  data_files=[('', ['libasdf-cpp-tp.so'])],
+)
diff --git a/testprojects/src/python/python_distribution/ctypes_with_third_party/some_more_math.hpp b/testprojects/src/python/python_distribution/ctypes_with_third_party/some_more_math.hpp
new file mode 100644
index 00000000000..31953f8b9ba
--- /dev/null
+++ b/testprojects/src/python/python_distribution/ctypes_with_third_party/some_more_math.hpp
@@ -0,0 +1,8 @@
+#ifndef __SOME_MORE_MATH_HPP__
+#define __SOME_MORE_MATH_HPP__
+
+int mangled_function(int);
+
+extern "C" int multiply_by_three(int);
+
+#endif
diff --git a/testprojects/src/python/python_distribution/ctypes_with_third_party/some_more_math_with_third_party.cpp b/testprojects/src/python/python_distribution/ctypes_with_third_party/some_more_math_with_third_party.cpp
new file mode 100644
index 00000000000..62a35fdc136
--- /dev/null
+++ b/testprojects/src/python/python_distribution/ctypes_with_third_party/some_more_math_with_third_party.cpp
@@ -0,0 +1,60 @@
+#include "some_more_math.hpp"
+
+/* A simple C++11 header-only library for integration testing */
+#include "rang.hpp"
+
+/**
+A C++11 library for integration testing that contains a library archive (.dylib/.so)
+in addition to headers.
+
+This snippet is taken from the README at https://github.com/USCiLab/cereal.
+ */
+#include <cereal/types/unordered_map.hpp>
+#include <cereal/types/memory.hpp>
+#include <cereal/archives/binary.hpp>
+#include <fstream>
+
+struct MyRecord
+{
+  uint8_t x = 1;
+  uint8_t y = 2;
+  float z;
+
+  template <class Archive>
+  void serialize( Archive & ar )
+  {
+    ar( x, y, z );
+  }
+};
+
+struct SomeData
+{
+  int32_t id;
+  int data = 3;
+
+  template <class Archive>
+  void save( Archive & ar ) const
+  {
+    ar( data );
+  }
+
+  template <class Archive>
+  void load( Archive & ar )
+  {
+    static int32_t idGen = 0;
+    id = idGen++;
+    ar( data );
+  }
+};
+
+
+int mangled_function(int x) {
+
+	// cereal testing
+	MyRecord myRecord;
+	SomeData myData;
+
+	return x ^ 3;
+}
+
+extern "C" int multiply_by_three(int x) { return mangled_function(x * 3); }
diff --git a/tests/python/pants_test/backend/python/tasks/test_native_external_library_fetch.py b/tests/python/pants_test/backend/python/tasks/test_native_external_library_fetch.py
new file mode 100644
index 00000000000..941f8d3abc2
--- /dev/null
+++ b/tests/python/pants_test/backend/python/tasks/test_native_external_library_fetch.py
@@ -0,0 +1,74 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import textwrap
+import unittest
+
+from pants.backend.native.config.environment import Platform
+from pants.backend.native.tasks.native_external_library_fetch import (ConanRequirement,
+                                                                      NativeExternalLibraryFetch)
+
+
+class TestConanRequirement(unittest.TestCase):
+
+  CONAN_OS_NAME = {
+    'darwin': lambda: 'Macos',
+    'linux': lambda: 'Linux',
+  }
+
+  def test_parse_conan_stdout_for_pkg_hash(self):
+    tc_1 = textwrap.dedent("""
+      rang/3.1.0@rang/stable: Installing package\n
+      Requirements\n    rang/3.1.0@rang/stable from 'pants-conan-remote'
+      \nPackages\n    rang/3.1.0@rang/stable:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9\n\n
+      rang/3.1.0@rang/stable: Already installed!\n"
+    """
+    )
+    tc_2 = textwrap.dedent(
+      "rang/3.1.0@rang/stable: Not found, retrieving from server 'pants-conan-remote' \n"
+      "rang/3.1.0@rang/stable: Trying with 'pants-conan-remote'...\nDownloading conanmanifest.txt\n"
+      "\nDownloading conanfile.py\n\nrang/3.1.0@rang/stable: Installing package\nRequirements\n    "
+      "rang/3.1.0@rang/stable from 'pants-conan-remote'\nPackages\n    "
+      "rang/3.1.0@rang/stable:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9\n\nrang/3.1.0@rang/stable: "
+      "Retrieving package 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 from remote 'pants-conan-remote' \n"
+      "Downloading conanmanifest.txt\n\nDownloading conaninfo.txt\n\nDownloading conan_package.tgz\n\n"
+      "rang/3.1.0@rang/stable: Package installed 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9\n"
+    )
+    pkg_spec = 'rang/3.1.0@rang/stable'
+    expected_sha = '5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9'
+    cr = ConanRequirement(pkg_spec=pkg_spec)
+    sha1 = cr.parse_conan_stdout_for_pkg_sha(tc_1)
+    self.assertEqual(sha1, expected_sha)
+    sha2 = cr.parse_conan_stdout_for_pkg_sha(tc_2)
+    self.assertEqual(sha2, expected_sha)
+
+  def test_build_conan_cmdline_args(self):
+    pkg_spec = 'test/1.0.0@conan/stable'
+    cr = ConanRequirement(pkg_spec=pkg_spec)
+    platform = Platform.create()
+    conan_os_name = platform.resolve_platform_specific(self.CONAN_OS_NAME)
+    expected = ['install', 'test/1.0.0@conan/stable', '-s', 'os={}'.format(conan_os_name)]
+    self.assertEqual(cr.fetch_cmdline_args, expected)
+
+
+class TestNativeExternalLibraryFetch(unittest.TestCase):
+
+  def test_parse_lib_name_from_library_filename(self):
+    tc_1 = 'liblzo.a'
+    tc_2 = 'libtensorflow.so'
+    tc_3 = 'libz.dylib'
+    tc_4 = 'libbadextension.lol'
+    tc_5 = 'badfilename.so'
+    res = NativeExternalLibraryFetch._parse_lib_name_from_library_filename(tc_1)
+    self.assertEqual(res, 'lzo')
+    res = NativeExternalLibraryFetch._parse_lib_name_from_library_filename(tc_2)
+    self.assertEqual(res, 'tensorflow')
+    res = NativeExternalLibraryFetch._parse_lib_name_from_library_filename(tc_3)
+    self.assertEqual(res, 'z')
+    res = NativeExternalLibraryFetch._parse_lib_name_from_library_filename(tc_4)
+    self.assertEqual(res, None)
+    res = NativeExternalLibraryFetch._parse_lib_name_from_library_filename(tc_5)
+    self.assertEqual(res, None)

From 1018258de86790661911083fc78de84dd04ab9af Mon Sep 17 00:00:00 2001
From: Chris Livingston <clivingston@twitter.com>
Date: Mon, 16 Jul 2018 11:23:24 -0700
Subject: [PATCH 2/5] Add new objects to register.py

---
 src/python/pants/backend/native/register.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/python/pants/backend/native/register.py b/src/python/pants/backend/native/register.py
index 7447e8b36aa..a9a11e7d81b 100644
--- a/src/python/pants/backend/native/register.py
+++ b/src/python/pants/backend/native/register.py
@@ -10,11 +10,13 @@
 from pants.backend.native.subsystems.binaries.llvm import create_llvm_rules
 from pants.backend.native.subsystems.native_toolchain import create_native_toolchain_rules
 from pants.backend.native.subsystems.xcode_cli_tools import create_xcode_cli_tools_rules
+from pants.backend.native.targets.external_native_library import ExternalNativeLibrary
 from pants.backend.native.targets.native_artifact import NativeArtifact
 from pants.backend.native.targets.native_library import CLibrary, CppLibrary
 from pants.backend.native.tasks.c_compile import CCompile
 from pants.backend.native.tasks.cpp_compile import CppCompile
 from pants.backend.native.tasks.link_shared_libraries import LinkSharedLibraries
+from pants.backend.native.tasks.native_external_library_fetch import NativeExternalLibraryFetch
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.goal.task_registrar import TaskRegistrar as task
 
@@ -24,6 +26,7 @@ def build_file_aliases():
     targets={
       CLibrary.alias(): CLibrary,
       CppLibrary.alias(): CppLibrary,
+      ExternalNativeLibrary.alias(): ExternalNativeLibrary,
     },
     objects={
       NativeArtifact.alias(): NativeArtifact,
@@ -34,6 +37,7 @@ def build_file_aliases():
 def register_goals():
   # FIXME(#5962): register these under the 'compile' goal when we eliminate the product transitive
   # dependency from export -> compile.
+  task(name='native-third-party-fetch', action=NativeExternalLibraryFetch).install('native-compile')
   task(name='c-for-ctypes', action=CCompile).install('native-compile')
   task(name='cpp-for-ctypes', action=CppCompile).install('native-compile')
   task(name='shared-libraries', action=LinkSharedLibraries).install('link')

From c208cc4f90a38e66f555383f318b9cdb98240388 Mon Sep 17 00:00:00 2001
From: Chris Livingston <clivingston@twitter.com>
Date: Mon, 16 Jul 2018 14:07:42 -0700
Subject: [PATCH 3/5] Add cmdline splitting to support new WrappedPex
 functionality

---
 src/python/pants/backend/native/subsystems/conan.py         | 4 ++--
 src/python/pants/backend/native/tasks/native_compile.py     | 2 +-
 .../backend/native/tasks/native_external_library_fetch.py   | 6 +++---
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/python/pants/backend/native/subsystems/conan.py b/src/python/pants/backend/native/subsystems/conan.py
index e4c177ab1c6..1ddea0f53bd 100644
--- a/src/python/pants/backend/native/subsystems/conan.py
+++ b/src/python/pants/backend/native/subsystems/conan.py
@@ -67,7 +67,7 @@ def bootstrap_conan(self):
     conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary')
     interpreter = PythonInterpreter.get()
     if os.path.exists(conan_pex_path):
-      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter), interpreter)
+      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter))
       return self.ConanBinary(pex=conan_binary)
     else:  
       with safe_concurrent_creation(conan_pex_path) as safe_path:
@@ -75,5 +75,5 @@ def bootstrap_conan(self):
         reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements]
         dump_requirements(builder, interpreter, reqs, logger)
         builder.freeze()
-      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter), interpreter)
+      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter))
       return self.ConanBinary(pex=conan_binary)
diff --git a/src/python/pants/backend/native/tasks/native_compile.py b/src/python/pants/backend/native/tasks/native_compile.py
index 564c57cedb3..d1fa7f7e163 100644
--- a/src/python/pants/backend/native/tasks/native_compile.py
+++ b/src/python/pants/backend/native/tasks/native_compile.py
@@ -266,7 +266,7 @@ def _compile(self, compile_request):
         workunit.set_outcome(WorkUnit.FAILURE)
         raise self.NativeCompileError(
           "Error in '{section_name}' with command {cmd} for request {req}. Exit code was: {rc}."
-          .format(section_name=self.workunit_name, cmd=argv, req=compile_request, rc=rc))
+          .format(section_name=self.workunit_label, cmd=argv, req=compile_request, rc=rc))
 
   def collect_cached_objects(self, versioned_target):
     """Scan `versioned_target`'s results directory and return the output files from that directory.
diff --git a/src/python/pants/backend/native/tasks/native_external_library_fetch.py b/src/python/pants/backend/native/tasks/native_external_library_fetch.py
index c9a6f7b6533..94cb7550f59 100644
--- a/src/python/pants/backend/native/tasks/native_external_library_fetch.py
+++ b/src/python/pants/backend/native/tasks/native_external_library_fetch.py
@@ -175,7 +175,7 @@ def ensure_conan_remote_configuration(self, conan_binary):
     # and replace it with Pants-controlled remotes.
     remove_conan_center_remote_cmdline = self._remove_conan_center_remote_cmdline(conan_binary)
     try:
-      stdout = subprocess.check_output(remove_conan_center_remote_cmdline.split())
+      stdout = subprocess.check_output(remove_conan_center_remote_cmdline.split()[1:])
       self.context.log.debug(stdout)
     except subprocess.CalledProcessError as e:
       if not "'conan-center' not found in remotes" in e.output:
@@ -191,7 +191,7 @@ def ensure_conan_remote_configuration(self, conan_binary):
                                                                             index_num,
                                                                             remote_url)
       try:
-        stdout = subprocess.check_output(add_pants_conan_remote_cmdline.split())
+        stdout = subprocess.check_output(add_pants_conan_remote_cmdline.split()[1:])
         self.context.log.debug(stdout)
       except subprocess.CalledProcessError as e:
         if not "already exists in remotes" in e.output:
@@ -245,7 +245,7 @@ def _fetch_packages(self, vt, vts_results_dir):
 
         # Invoke conan to pull package from remote.
         try:
-          stdout = subprocess.check_output(cmdline.split())
+          stdout = subprocess.check_output(cmdline.split()[1:])
         except subprocess.CalledProcessError as e:
           raise self.NativeExternalLibraryFetchError(
             "Error invoking conan for fetch task: {}\n".format(e.output)

From 5dc841209415fdedefc07bfc3543ac413d3f69d9 Mon Sep 17 00:00:00 2001
From: Chris Livingston <clivingston@twitter.com>
Date: Mon, 16 Jul 2018 14:24:39 -0700
Subject: [PATCH 4/5] Integrate backend tasks with native external lib fetching

---
 .../native/tasks/link_shared_libraries.py     | 25 ++++++++++++++-----
 .../backend/native/tasks/native_compile.py    | 16 ++++++++++--
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/src/python/pants/backend/native/tasks/link_shared_libraries.py b/src/python/pants/backend/native/tasks/link_shared_libraries.py
index 57219b990e5..b76d2cf69bc 100644
--- a/src/python/pants/backend/native/tasks/link_shared_libraries.py
+++ b/src/python/pants/backend/native/tasks/link_shared_libraries.py
@@ -10,6 +10,7 @@
 from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.native.targets.native_library import NativeLibrary
 from pants.backend.native.tasks.native_compile import NativeTargetDependencies, ObjectFiles
+from pants.backend.native.tasks.native_external_library_fetch import NativeExternalLibraryFetch
 from pants.backend.native.tasks.native_task import NativeTask
 from pants.base.exceptions import TaskError
 from pants.base.workunit import WorkUnit, WorkUnitLabel
@@ -27,6 +28,7 @@ class LinkSharedLibraryRequest(datatype([
     'object_files',
     'native_artifact',
     'output_dir',
+    'external_libs_info'
 ])): pass
 
 
@@ -40,6 +42,7 @@ def product_types(cls):
   def prepare(cls, options, round_manager):
     round_manager.require(NativeTargetDependencies)
     round_manager.require(ObjectFiles)
+    round_manager.require(NativeExternalLibraryFetch.NativeExternalLibraryFiles)
 
   @property
   def cache_target_dirs(self):
@@ -76,17 +79,21 @@ def execute(self):
     native_target_deps_product = self.context.products.get(NativeTargetDependencies)
     compiled_objects_product = self.context.products.get(ObjectFiles)
     shared_libs_product = self.context.products.get(SharedLibrary)
+    external_libs_product = self.context.products.get_data(NativeExternalLibraryFetch.NativeExternalLibraryFiles)
 
     all_shared_libs_by_name = {}
 
+    # FIXME: convert this to a v2 engine dependency injection.
+    platform = Platform.create()
+
     with self.invalidated(targets_providing_artifacts,
                           invalidate_dependents=True) as invalidation_check:
       for vt in invalidation_check.all_vts:
         if vt.valid:
-          shared_library = self._retrieve_shared_lib_from_cache(vt)
+          shared_library = self._retrieve_shared_lib_from_cache(vt, platform)
         else:
           link_request = self._make_link_request(
-            vt, compiled_objects_product, native_target_deps_product)
+            vt, compiled_objects_product, native_target_deps_product, external_libs_product)
           shared_library = self._execute_link_request(link_request)
 
         same_name_shared_lib = all_shared_libs_by_name.get(shared_library.name, None)
@@ -102,16 +109,20 @@ def execute(self):
 
         shared_libs_product.add(vt.target, vt.target.target_base).append(shared_library)
 
-  def _retrieve_shared_lib_from_cache(self, vt):
+  def _retrieve_shared_lib_from_cache(self, vt, platform):
     native_artifact = vt.target.ctypes_native_library
     path_to_cached_lib = os.path.join(
-      vt.results_dir, native_artifact.as_shared_lib(self.linker.platform))
+      vt.results_dir, native_artifact.as_shared_lib(platform))
     if not os.path.isfile(path_to_cached_lib):
       raise self.LinkSharedLibrariesError("The shared library at {} does not exist!"
                                           .format(path_to_cached_lib))
     return SharedLibrary(name=native_artifact.lib_name, path=path_to_cached_lib)
 
-  def _make_link_request(self, vt, compiled_objects_product, native_target_deps_product):
+  def _make_link_request(self,
+                         vt,
+                         compiled_objects_product,
+                         native_target_deps_product,
+                         external_libs_product):
     self.context.log.debug("link target: {}".format(vt.target))
 
     deps = self._retrieve_single_product_at_target_base(native_target_deps_product, vt.target)
@@ -130,7 +141,8 @@ def _make_link_request(self, vt, compiled_objects_product, native_target_deps_pr
       linker=self.linker,
       object_files=all_compiled_object_files,
       native_artifact=vt.target.ctypes_native_library,
-      output_dir=vt.results_dir)
+      output_dir=vt.results_dir,
+      external_libs_info=external_libs_product)
 
   _SHARED_CMDLINE_ARGS = {
     'darwin': lambda: ['-mmacosx-version-min=10.11', '-Wl,-dylib'],
@@ -155,6 +167,7 @@ def _execute_link_request(self, link_request):
     # We are executing in the results_dir, so get absolute paths for everything.
     cmd = ([linker.exe_filename] +
            self._get_shared_lib_cmdline_args(platform) +
+           link_request.external_libs_info.get_third_party_lib_args() +
            ['-o', os.path.abspath(resulting_shared_lib_path)] +
            [os.path.abspath(obj) for obj in object_files])
 
diff --git a/src/python/pants/backend/native/tasks/native_compile.py b/src/python/pants/backend/native/tasks/native_compile.py
index d1fa7f7e163..6a1e7ee826e 100644
--- a/src/python/pants/backend/native/tasks/native_compile.py
+++ b/src/python/pants/backend/native/tasks/native_compile.py
@@ -11,6 +11,7 @@
 
 from pants.backend.native.config.environment import Executable, Platform
 from pants.backend.native.targets.native_library import NativeLibrary
+from pants.backend.native.tasks.native_external_library_fetch import NativeExternalLibraryFetch
 from pants.backend.native.tasks.native_task import NativeTask
 from pants.base.exceptions import TaskError
 from pants.base.workunit import WorkUnit, WorkUnitLabel
@@ -117,13 +118,16 @@ def _add_product_at_target_base(product_mapping, target, value):
   def execute(self):
     object_files_product = self.context.products.get(ObjectFiles)
     native_deps_product = self.context.products.get(NativeTargetDependencies)
+    external_libs_product = self.context.products.get_data(
+      NativeExternalLibraryFetch.NativeExternalLibraryFiles
+    )
     source_targets = self.context.targets(self.source_target_constraint.satisfied_by)
 
     with self.invalidated(source_targets, invalidate_dependents=True) as invalidation_check:
       for vt in invalidation_check.invalid_vts:
         deps = self.native_deps(vt.target)
         self._add_product_at_target_base(native_deps_product, vt.target, deps)
-        compile_request = self._make_compile_request(vt, deps)
+        compile_request = self._make_compile_request(vt, deps, external_libs_product)
         self.context.log.debug("compile_request: {}".format(compile_request))
         self._compile(compile_request)
 
@@ -184,10 +188,18 @@ def get_compiler(self):
   def _compiler(self):
     return self.get_compiler()
 
-  def _make_compile_request(self, versioned_target, dependencies):
+  def _get_third_party_include_dirs(self, external_libs_product):
+    directory = external_libs_product.include_dir
+    return [directory] if directory else []
+
+  def _make_compile_request(self, versioned_target, dependencies, external_libs_product):
     target = versioned_target.target
+
     include_dirs = [self._include_dirs_for_target(dep_tgt) for dep_tgt in dependencies]
+    include_dirs.extend(self._get_third_party_include_dirs(external_libs_product))
+
     sources_and_headers = self.get_sources_headers_for_target(target)
+
     return NativeCompileRequest(
       compiler=self._compiler,
       include_dirs=include_dirs,

From 77e326f0b9986d082eac29dea0d3696c8be29088 Mon Sep 17 00:00:00 2001
From: Chris Livingston <clivingston@twitter.com>
Date: Mon, 16 Jul 2018 14:31:27 -0700
Subject: [PATCH 5/5] Bulk add Danny's patch fixes

---
 .../native/subsystems/binaries/llvm.py        | 16 ++++-
 .../pants/backend/native/subsystems/conan.py  |  1 -
 .../backend/native/subsystems/libc_dev.py     | 40 +++++++++---
 .../native/subsystems/native_toolchain.py     | 61 ++++++++++++-------
 .../subsystems/utils/parse_search_dirs.py     |  4 +-
 .../pants/backend/native/tasks/cpp_compile.py | 10 +++
 6 files changed, 96 insertions(+), 36 deletions(-)

diff --git a/src/python/pants/backend/native/subsystems/binaries/llvm.py b/src/python/pants/backend/native/subsystems/binaries/llvm.py
index e7eff9ed8d0..fc6d3aaf4a4 100644
--- a/src/python/pants/backend/native/subsystems/binaries/llvm.py
+++ b/src/python/pants/backend/native/subsystems/binaries/llvm.py
@@ -13,7 +13,7 @@
 from pants.engine.rules import RootRule, rule
 from pants.engine.selectors import Select
 from pants.util.dirutil import is_readable_dir
-from pants.util.memo import memoized_method
+from pants.util.memo import memoized_method, memoized_property
 
 
 class LLVMReleaseUrlGenerator(BinaryToolUrlGenerator):
@@ -70,19 +70,29 @@ def linker(self, platform):
         self._PLATFORM_SPECIFIC_LINKER_NAME),
       library_dirs=[])
 
+  # FIXME: use ParseSearchDirs for this and other include directories -- we shouldn't be trying to
+  # guess the path here.
+  @memoized_property
+  def _common_include_dirs(self):
+    return [os.path.join(self.select(), 'lib/clang', self.version(), 'include')]
+
   def c_compiler(self):
     return CCompiler(
       path_entries=self.path_entries(),
       exe_filename='clang',
       library_dirs=[],
-      include_dirs=[])
+      include_dirs=self._common_include_dirs)
+
+  @memoized_property
+  def _cpp_include_dirs(self):
+    return [os.path.join(self.select(), 'include/c++/v1')]
 
   def cpp_compiler(self):
     return CppCompiler(
       path_entries=self.path_entries(),
       exe_filename='clang++',
       library_dirs=[],
-      include_dirs=[])
+      include_dirs=(self._cpp_include_dirs + self._common_include_dirs))
 
 
 # FIXME(#5663): use this over the XCode linker!
diff --git a/src/python/pants/backend/native/subsystems/conan.py b/src/python/pants/backend/native/subsystems/conan.py
index 1ddea0f53bd..e65ecd45f6b 100644
--- a/src/python/pants/backend/native/subsystems/conan.py
+++ b/src/python/pants/backend/native/subsystems/conan.py
@@ -4,7 +4,6 @@
 
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-
 import logging
 import os
 
diff --git a/src/python/pants/backend/native/subsystems/libc_dev.py b/src/python/pants/backend/native/subsystems/libc_dev.py
index 967d8709972..80599b80a3f 100644
--- a/src/python/pants/backend/native/subsystems/libc_dev.py
+++ b/src/python/pants/backend/native/subsystems/libc_dev.py
@@ -17,8 +17,15 @@
 class LibcDev(Subsystem):
   """Subsystem to detect and provide the host's installed version of a libc "dev" package.
 
-  This subsystem exists to give a useful error message if the package isn't
-  installed, and to allow a nonstandard install location.
+  A libc "dev" package is provided on most Linux systems by default, but may not be located at any
+  standardized path. We define a libc dev package as one which provides crti.o, an object file which
+  is part of any libc implementation and is required to create executables (more information
+  available at https://wiki.osdev.org/Creating_a_C_Library).
+
+  NB: This is currently unused except in CI, because we have no plans to support creating native
+  executables from C or C++ sources yet (PRs welcome!). It is used to provide an "end-to-end" test
+  of the compilation and linking toolchain in CI by creating and invoking a "hello world"
+  executable.
   """
 
   options_scope = 'libc'
@@ -37,7 +44,11 @@ def _parse_search_dirs(self):
   def register_options(cls, register):
     super(LibcDev, cls).register_options(register)
 
-    register('--libc-dir', type=dir_option, default='/usr/lib', fingerprint=True, advanced=True,
+    register('--enable-libc-search', type=bool, default=False, fingerprint=True, advanced=True,
+             help="Whether to search for the host's libc installation. Set to False if the host "
+                  "does not have a libc install with crti.o -- this file is necessary to create "
+                  "executables on Linux hosts.")
+    register('--libc-dir', type=dir_option, default=None, fingerprint=True, advanced=True,
              help='A directory containing a host-specific crti.o from libc.')
     register('--host-compiler', type=str, default='gcc', fingerprint=True, advanced=True,
              help='The host compiler to invoke with -print-search-dirs to find the host libc.')
@@ -74,12 +85,25 @@ def _get_host_libc_from_host_compiler(self):
                        fingerprint=hash_file(libc_crti_object_file))
 
   @memoized_property
-  def host_libc(self):
+  def _host_libc(self):
     """Use the --libc-dir option if provided, otherwise invoke a host compiler to find libc dev."""
     libc_dir_option = self.get_options().libc_dir
-    maybe_libc_crti = os.path.join(libc_dir_option, self._LIBC_INIT_OBJECT_FILE)
-    if os.path.isfile(maybe_libc_crti):
-      return HostLibcDev(crti_object=maybe_libc_crti,
-                         fingerprint=hash_file(maybe_libc_crti))
+    if libc_dir_option:
+      maybe_libc_crti = os.path.join(libc_dir_option, self._LIBC_INIT_OBJECT_FILE)
+      if os.path.isfile(maybe_libc_crti):
+        return HostLibcDev(crti_object=maybe_libc_crti,
+                           fingerprint=hash_file(maybe_libc_crti))
+      raise self.HostLibcDevResolutionError(
+        "Could not locate {} in directory {} provided by the --libc-dir option."
+        .format(self._LIBC_INIT_OBJECT_FILE, libc_dir_option))
 
     return self._get_host_libc_from_host_compiler()
+
+  def get_libc_dirs(self, platform):
+    if not self.get_options().enable_libc_search:
+      return []
+
+    return platform.resolve_platform_specific({
+      'darwin': lambda: [],
+      'linux': lambda: [self._host_libc.get_lib_dir()],
+    })
diff --git a/src/python/pants/backend/native/subsystems/native_toolchain.py b/src/python/pants/backend/native/subsystems/native_toolchain.py
index 3e98d92baf3..34dcf27b30b 100644
--- a/src/python/pants/backend/native/subsystems/native_toolchain.py
+++ b/src/python/pants/backend/native/subsystems/native_toolchain.py
@@ -71,23 +71,28 @@ def select_linker(platform, native_toolchain):
   #   'darwin': lambda: Get(Linker, XCodeCLITools, native_toolchain._xcode_cli_tools),
   #   'linux': lambda: Get(Linker, Binutils, native_toolchain._binutils),
   # })
+  #
+  # NB: We need to link through a provided compiler's frontend, and we need to know where all the
+  # compiler's libraries/etc are, so we set the executable name to the C++ compiler, which can find
+  # its own set of C++-specific files for the linker if necessary. Using e.g. 'g++' as the linker
+  # appears to produce byte-identical output when linking even C-only object files, and also
+  # happens to work when C++ is used.
+  # Currently, OSX links through the clang++ frontend, and Linux links through the g++ frontend.
   if platform.normalized_os_name == 'darwin':
     # TODO(#5663): turn this into LLVM when lld works.
     linker = yield Get(Linker, XCodeCLITools, native_toolchain._xcode_cli_tools)
-    libc_dirs = []
+    llvm_c_compiler = yield Get(LLVMCCompiler, NativeToolchain, native_toolchain)
+    c_compiler = llvm_c_compiler.c_compiler
+    llvm_cpp_compiler = yield Get(LLVMCppCompiler, NativeToolchain, native_toolchain)
+    cpp_compiler = llvm_cpp_compiler.cpp_compiler
   else:
     linker = yield Get(Linker, Binutils, native_toolchain._binutils)
-    libc_dirs = [native_toolchain._libc_dev.host_libc.get_lib_dir()]
+    gcc_c_compiler = yield Get(GCCCCompiler, NativeToolchain, native_toolchain)
+    c_compiler = gcc_c_compiler.c_compiler
+    gcc_cpp_compiler = yield Get(GCCCppCompiler, NativeToolchain, native_toolchain)
+    cpp_compiler = gcc_cpp_compiler.cpp_compiler
 
-  # NB: We need to link through a provided compiler's frontend, and we need to know where all the
-  # compiler's libraries/etc are, so we set the executable name to the C++ compiler, which can find
-  # its own set of C++-specific files for the linker if necessary. Using e.g. 'g++' as the linker
-  # appears to produce byte-identical output when linking even C-only object files, and also
-  # happens to work when C++ is used.
-  gcc_c_compiler = yield Get(GCCCCompiler, NativeToolchain, native_toolchain)
-  c_compiler = gcc_c_compiler.c_compiler
-  gcc_cpp_compiler = yield Get(GCCCppCompiler, NativeToolchain, native_toolchain)
-  cpp_compiler = gcc_cpp_compiler.cpp_compiler
+  libc_dirs = native_toolchain._libc_dev.get_libc_dirs(platform)
 
   # NB: If needing to create an environment for process invocation that could use either a compiler
   # or a linker (e.g. when we compile native code from `python_dist()`s), use the environment from
@@ -96,14 +101,14 @@ def select_linker(platform, native_toolchain):
   # FIXME(#5951): we need a way to compose executables more hygienically.
   linker = Linker(
     path_entries=(
-      c_compiler.path_entries +
       cpp_compiler.path_entries +
+      c_compiler.path_entries +
       linker.path_entries),
     exe_filename=cpp_compiler.exe_filename,
     library_dirs=(
       libc_dirs +
-      c_compiler.library_dirs +
       cpp_compiler.library_dirs +
+      c_compiler.library_dirs +
       linker.library_dirs))
 
   yield linker
@@ -147,7 +152,7 @@ def select_llvm_cpp_compiler(platform, native_toolchain):
       path_entries=(provided_clangpp.path_entries + xcode_clang.path_entries),
       exe_filename=provided_clangpp.exe_filename,
       library_dirs=(provided_clangpp.library_dirs + xcode_clang.library_dirs),
-      include_dirs=(xcode_clang.include_dirs + provided_clangpp.include_dirs))
+      include_dirs=(provided_clangpp.include_dirs + xcode_clang.include_dirs))
     final_llvm_cpp_compiler = LLVMCppCompiler(clang_with_xcode_paths)
   else:
     gcc_cpp_compiler = yield Get(GCCCppCompiler, GCC, native_toolchain._gcc)
@@ -231,16 +236,28 @@ def select_gcc_cpp_compiler(platform, native_toolchain):
   yield final_gcc_cpp_compiler
 
 
-@rule(CCompiler, [Select(NativeToolchain)])
-def select_c_compiler(native_toolchain):
-  llvm_c_compiler = yield Get(LLVMCCompiler, NativeToolchain, native_toolchain)
-  yield llvm_c_compiler.c_compiler
+@rule(CCompiler, [Select(NativeToolchain), Select(Platform)])
+def select_c_compiler(native_toolchain, platform):
+  if platform.normalized_os_name == 'darwin':
+    llvm_c_compiler = yield Get(LLVMCCompiler, NativeToolchain, native_toolchain)
+    c_compiler = llvm_c_compiler.c_compiler
+  else:
+    gcc_c_compiler = yield Get(GCCCCompiler, NativeToolchain, native_toolchain)
+    c_compiler = gcc_c_compiler.c_compiler
+
+  yield c_compiler
+
 
+@rule(CppCompiler, [Select(NativeToolchain), Select(Platform)])
+def select_cpp_compiler(native_toolchain, platform):
+  if platform.normalized_os_name == 'darwin':
+    llvm_cpp_compiler = yield Get(LLVMCppCompiler, NativeToolchain, native_toolchain)
+    cpp_compiler = llvm_cpp_compiler.cpp_compiler
+  else:
+    gcc_cpp_compiler = yield Get(GCCCppCompiler, NativeToolchain, native_toolchain)
+    cpp_compiler = gcc_cpp_compiler.cpp_compiler
 
-@rule(CppCompiler, [Select(NativeToolchain)])
-def select_cpp_compiler(native_toolchain):
-  llvm_cpp_compiler = yield Get(LLVMCppCompiler, NativeToolchain, native_toolchain)
-  yield llvm_cpp_compiler.cpp_compiler
+  yield cpp_compiler
 
 
 def create_native_toolchain_rules():
diff --git a/src/python/pants/backend/native/subsystems/utils/parse_search_dirs.py b/src/python/pants/backend/native/subsystems/utils/parse_search_dirs.py
index 128e62ef1bd..1129981ad5a 100644
--- a/src/python/pants/backend/native/subsystems/utils/parse_search_dirs.py
+++ b/src/python/pants/backend/native/subsystems/utils/parse_search_dirs.py
@@ -47,12 +47,12 @@ def _parse_libraries_from_compiler_search_dirs(self, compiler_exe, env):
     except OSError as e:
       # We use `safe_shlex_join` here to pretty-print the command.
       raise self.ParseSearchDirsError(
-        "Invocation of '{}' with argv {!r} failed."
+        "Invocation of '{}' with argv '{}' failed."
         .format(compiler_exe, safe_shlex_join(cmd)),
         e)
     except subprocess.CalledProcessError as e:
       raise self.ParseSearchDirsError(
-        "Invocation of '{}' with argv {!r} exited with non-zero code {}. output:\n{}"
+        "Invocation of '{}' with argv '{}' exited with non-zero code {}. output:\n{}"
         .format(compiler_exe, safe_shlex_join(cmd), e.returncode, e.output),
         e)
 
diff --git a/src/python/pants/backend/native/tasks/cpp_compile.py b/src/python/pants/backend/native/tasks/cpp_compile.py
index 7b2a6fbcc3f..6833b3eaa9d 100644
--- a/src/python/pants/backend/native/tasks/cpp_compile.py
+++ b/src/python/pants/backend/native/tasks/cpp_compile.py
@@ -41,6 +41,16 @@ def get_compile_settings(self):
   def get_compiler(self):
     return self._request_single(CppCompiler, self._toolchain)
 
+  def _make_compile_argv(self, compile_request):
+    # FIXME: this is a temporary fix, do not do any of this kind of introspection.
+    prev_argv = super(CppCompile, self)._make_compile_argv(compile_request)
+
+    if compile_request.compiler.exe_filename == 'clang++':
+      new_argv = [prev_argv[0], '-nobuiltininc', '-nostdinc++'] + prev_argv[1:]
+    else:
+      new_argv = prev_argv
+    return new_argv
+
   # FIXME(#5951): don't have any command-line args in the task or in the subsystem -- rather,
   # subsystem options should be used to populate an `Executable` which produces its own arguments.
   def extra_compile_args(self):