Conan (third party) support for ctypes native libraries (#5998)

### Problem

The new targets introduced in #5815 have no means of declaring, fetching, and using third party native dependencies.

### Solution

Integrate the Conan package manager to fetch packages from a remote package store via a new task, copy the package data to the results directory of `third_party_native_library` targets in play, and create a product that the compile and link tasks can consume. Plumb the directory paths provided by the product through to the command lines of the compile and link steps.

### Result

Users can now depend on third party native libraries in their `ctypes_compatible_cpp_library` targets from either conan-center or a remote URI that they specify via an option.

## Some notes
- The conan home directory is currently under .pants.d, instead of ~ or ~/.cache/pants. I did this for the short term to make debugging cache problems and other issues as simple as a ./pants clean-all. I don't think the perf loss will be too bad (1-2 seconds).
- Some string manipulation could be better done as regex, and I have TODOs for that
- I am going to follow up with upstream Conan about getting a flag for cleaner client output so the parse method does not need to be so ugly.

src/python/pants/backend/native/register.py

@@ -10,11 +10,13 @@
 from pants.backend.native.subsystems.binaries.llvm import create_llvm_rules
 from pants.backend.native.subsystems.native_toolchain import create_native_toolchain_rules
 from pants.backend.native.subsystems.xcode_cli_tools import create_xcode_cli_tools_rules
+from pants.backend.native.targets.external_native_library import ExternalNativeLibrary
 from pants.backend.native.targets.native_artifact import NativeArtifact
 from pants.backend.native.targets.native_library import CLibrary, CppLibrary
 from pants.backend.native.tasks.c_compile import CCompile
 from pants.backend.native.tasks.cpp_compile import CppCompile
 from pants.backend.native.tasks.link_shared_libraries import LinkSharedLibraries
+from pants.backend.native.tasks.native_external_library_fetch import NativeExternalLibraryFetch
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.goal.task_registrar import TaskRegistrar as task
 
@@ -24,6 +26,7 @@ def build_file_aliases():
     targets={
       CLibrary.alias(): CLibrary,
       CppLibrary.alias(): CppLibrary,
+      ExternalNativeLibrary.alias(): ExternalNativeLibrary,
     },
     objects={
       NativeArtifact.alias(): NativeArtifact,
@@ -34,6 +37,7 @@ def build_file_aliases():
 def register_goals():
   # FIXME(#5962): register these under the 'compile' goal when we eliminate the product transitive
   # dependency from export -> compile.
+  task(name='native-third-party-fetch', action=NativeExternalLibraryFetch).install('native-compile')
   task(name='c-for-ctypes', action=CCompile).install('native-compile')
   task(name='cpp-for-ctypes', action=CppCompile).install('native-compile')
   task(name='shared-libraries', action=LinkSharedLibraries).install('link')

src/python/pants/backend/native/subsystems/binaries/llvm.py

@@ -13,7 +13,7 @@
 from pants.engine.rules import RootRule, rule
 from pants.engine.selectors import Select
 from pants.util.dirutil import is_readable_dir
-from pants.util.memo import memoized_method
+from pants.util.memo import memoized_method, memoized_property
 
 
 class LLVMReleaseUrlGenerator(BinaryToolUrlGenerator):
@@ -70,19 +70,34 @@ def linker(self, platform):
         self._PLATFORM_SPECIFIC_LINKER_NAME),
       library_dirs=[])
 
+  # FIXME: use ParseSearchDirs for this and other include directories -- we shouldn't be trying to
+  # guess the path here.
+  # https://github.com/pantsbuild/pants/issues/6143
+  @memoized_property
+  def _common_include_dirs(self):
+    return [os.path.join(self.select(), 'lib/clang', self.version(), 'include')]
+
+  @memoized_property
+  def _common_lib_dirs(self):
+    return [os.path.join(self.select(), 'lib')]
+
   def c_compiler(self):
     return CCompiler(
       path_entries=self.path_entries(),
       exe_filename='clang',
-      library_dirs=[],
-      include_dirs=[])
+      library_dirs=self._common_lib_dirs,
+      include_dirs=self._common_include_dirs)
+
+  @memoized_property
+  def _cpp_include_dirs(self):
+    return [os.path.join(self.select(), 'include/c++/v1')]
 
   def cpp_compiler(self):
     return CppCompiler(
       path_entries=self.path_entries(),
       exe_filename='clang++',
-      library_dirs=[],
-      include_dirs=[])
+      library_dirs=self._common_lib_dirs,
+      include_dirs=(self._cpp_include_dirs + self._common_include_dirs))
 
 
 # FIXME(#5663): use this over the XCode linker!

src/python/pants/backend/native/subsystems/conan.py

@@ -0,0 +1,78 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import logging
+import os
+
+from pex.interpreter import PythonInterpreter
+from pex.pex import PEX
+from pex.pex_builder import PEXBuilder
+from pex.pex_info import PexInfo
+
+from pants.backend.python.python_requirement import PythonRequirement
+from pants.backend.python.tasks.pex_build_util import dump_requirements
+from pants.backend.python.tasks.wrapped_pex import WrappedPEX
+from pants.base.build_environment import get_pants_cachedir
+from pants.subsystem.subsystem import Subsystem
+from pants.util.dirutil import safe_concurrent_creation
+from pants.util.objects import datatype
+
+
+logger = logging.getLogger(__name__)
+
+
+class Conan(Subsystem):
+  """Pex binary for the conan package manager."""
+  options_scope = 'conan'
+  default_conan_requirements = (
+    'conan==1.4.4',
+    'PyJWT>=1.4.0, <2.0.0',
+    'requests>=2.7.0, <3.0.0',
+    'colorama>=0.3.3, <0.4.0',
+    'PyYAML>=3.11, <3.13.0',
+    'patch==1.16',
+    'fasteners>=0.14.1',
+    'six>=1.10.0',
+    'node-semver==0.2.0',
+    'distro>=1.0.2, <1.2.0',
+    'pylint>=1.8.1, <1.9.0',
+    'future==0.16.0',
+    'pygments>=2.0, <3.0',
+    'astroid>=1.6, <1.7',
+    'deprecation>=2.0, <2.1'
+  )
+
+  @classmethod
+  def implementation_version(cls):
+    return super(Conan, cls).implementation_version() + [('Conan', 0)]
+
+  @classmethod
+  def register_options(cls, register):
+    super(Conan, cls).register_options(register)
+    register('--conan-requirements', type=list, default=cls.default_conan_requirements,
+             advanced=True, help='The requirements used to build the conan client pex.')
+
+  class ConanBinary(datatype(['pex'])):
+    """A `conan` PEX binary."""
+    pass
+
+  def bootstrap_conan(self):
+    pex_info = PexInfo.default()
+    pex_info.entry_point = 'conans.conan'
+    conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support')
+    conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary')
+    interpreter = PythonInterpreter.get()
+    if os.path.exists(conan_pex_path):
+      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter))
+      return self.ConanBinary(pex=conan_binary)
+    else:
+      with safe_concurrent_creation(conan_pex_path) as safe_path:
+        builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
+        reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements]
+        dump_requirements(builder, interpreter, reqs, logger)
+        builder.freeze()
+      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter))
+      return self.ConanBinary(pex=conan_binary)

src/python/pants/backend/native/subsystems/native_toolchain.py

@@ -71,39 +71,44 @@ def select_linker(platform, native_toolchain):
   #   'darwin': lambda: Get(Linker, XCodeCLITools, native_toolchain._xcode_cli_tools),
   #   'linux': lambda: Get(Linker, Binutils, native_toolchain._binutils),
   # })
+  #
+  # NB: We need to link through a provided compiler's frontend, and we need to know where all the
+  # compiler's libraries/etc are, so we set the executable name to the C++ compiler, which can find
+  # its own set of C++-specific files for the linker if necessary. Using e.g. 'g++' as the linker
+  # appears to produce byte-identical output when linking even C-only object files, and also
+  # happens to work when C++ is used.
+  # Currently, OSX links through the clang++ frontend, and Linux links through the g++ frontend.
   if platform.normalized_os_name == 'darwin':
     # TODO(#5663): turn this into LLVM when lld works.
     linker = yield Get(Linker, XCodeCLITools, native_toolchain._xcode_cli_tools)
+    llvm_c_compiler = yield Get(LLVMCCompiler, NativeToolchain, native_toolchain)
+    c_compiler = llvm_c_compiler.c_compiler
+    llvm_cpp_compiler = yield Get(LLVMCppCompiler, NativeToolchain, native_toolchain)
+    cpp_compiler = llvm_cpp_compiler.cpp_compiler
   else:
     linker = yield Get(Linker, Binutils, native_toolchain._binutils)
+    gcc_c_compiler = yield Get(GCCCCompiler, NativeToolchain, native_toolchain)
+    c_compiler = gcc_c_compiler.c_compiler
+    gcc_cpp_compiler = yield Get(GCCCppCompiler, NativeToolchain, native_toolchain)
+    cpp_compiler = gcc_cpp_compiler.cpp_compiler
 
   libc_dirs = native_toolchain._libc_dev.get_libc_dirs(platform)
 
-  # NB: We need to link through a provided compiler's frontend, and we need to know where all the
-  # compiler's libraries/etc are, so we set the executable name to the C++ compiler, which can find
-  # its own set of C++-specific files for the linker if necessary. Using e.g. 'g++' as the linker
-  # appears to produce byte-identical output when linking even C-only object files, and also
-  # happens to work when C++ is used.
-  gcc_c_compiler = yield Get(GCCCCompiler, NativeToolchain, native_toolchain)
-  c_compiler = gcc_c_compiler.c_compiler
-  gcc_cpp_compiler = yield Get(GCCCppCompiler, NativeToolchain, native_toolchain)
-  cpp_compiler = gcc_cpp_compiler.cpp_compiler
-
   # NB: If needing to create an environment for process invocation that could use either a compiler
   # or a linker (e.g. when we compile native code from `python_dist()`s), use the environment from
   # the linker object (in addition to any further customizations), which has the paths from the C
   # and C++ compilers baked in.
   # FIXME(#5951): we need a way to compose executables more hygienically.
   linker = Linker(
     path_entries=(
-      c_compiler.path_entries +
       cpp_compiler.path_entries +
+      c_compiler.path_entries +
       linker.path_entries),
     exe_filename=cpp_compiler.exe_filename,
     library_dirs=(
       libc_dirs +
-      c_compiler.library_dirs +
       cpp_compiler.library_dirs +
+      c_compiler.library_dirs +
       linker.library_dirs))
 
   yield linker
@@ -147,7 +152,7 @@ def select_llvm_cpp_compiler(platform, native_toolchain):
       path_entries=(provided_clangpp.path_entries + xcode_clang.path_entries),
       exe_filename=provided_clangpp.exe_filename,
       library_dirs=(provided_clangpp.library_dirs + xcode_clang.library_dirs),
-      include_dirs=(xcode_clang.include_dirs + provided_clangpp.include_dirs))
+      include_dirs=(provided_clangpp.include_dirs + xcode_clang.include_dirs))
     final_llvm_cpp_compiler = LLVMCppCompiler(clang_with_xcode_paths)
   else:
     gcc_cpp_compiler = yield Get(GCCCppCompiler, GCC, native_toolchain._gcc)
@@ -231,16 +236,28 @@ def select_gcc_cpp_compiler(platform, native_toolchain):
   yield final_gcc_cpp_compiler
 
 
-@rule(CCompiler, [Select(NativeToolchain)])
-def select_c_compiler(native_toolchain):
-  llvm_c_compiler = yield Get(LLVMCCompiler, NativeToolchain, native_toolchain)
-  yield llvm_c_compiler.c_compiler
+@rule(CCompiler, [Select(NativeToolchain), Select(Platform)])
+def select_c_compiler(native_toolchain, platform):
+  if platform.normalized_os_name == 'darwin':
+    llvm_c_compiler = yield Get(LLVMCCompiler, NativeToolchain, native_toolchain)
+    c_compiler = llvm_c_compiler.c_compiler
+  else:
+    gcc_c_compiler = yield Get(GCCCCompiler, NativeToolchain, native_toolchain)
+    c_compiler = gcc_c_compiler.c_compiler
+
+  yield c_compiler
+
 
+@rule(CppCompiler, [Select(NativeToolchain), Select(Platform)])
+def select_cpp_compiler(native_toolchain, platform):
+  if platform.normalized_os_name == 'darwin':
+    llvm_cpp_compiler = yield Get(LLVMCppCompiler, NativeToolchain, native_toolchain)
+    cpp_compiler = llvm_cpp_compiler.cpp_compiler
+  else:
+    gcc_cpp_compiler = yield Get(GCCCppCompiler, NativeToolchain, native_toolchain)
+    cpp_compiler = gcc_cpp_compiler.cpp_compiler
 
-@rule(CppCompiler, [Select(NativeToolchain)])
-def select_cpp_compiler(native_toolchain):
-  llvm_cpp_compiler = yield Get(LLVMCppCompiler, NativeToolchain, native_toolchain)
-  yield llvm_cpp_compiler.cpp_compiler
+  yield cpp_compiler
 
 
 def create_native_toolchain_rules():

src/python/pants/backend/native/targets/external_native_library.py

@@ -0,0 +1,43 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import re
+
+from pants.base.payload import Payload
+from pants.base.payload_field import PrimitiveField
+from pants.base.validation import assert_list
+from pants.build_graph.target import Target
+
+
+class ExternalNativeLibrary(Target):
+  """A set of Conan package strings to be passed to the Conan package manager."""
+
+  @classmethod
+  def alias(cls):
+    return 'external_native_library'
+
+  def __init__(self, payload=None, packages=None, **kwargs):
+    """
+    :param packages: a list of Conan-style package strings
+
+    Example:
+      lzo/2.10@twitter/stable
+    """
+    payload = payload or Payload()
+
+    assert_list(packages, key_arg='packages')
+    payload.add_fields({
+      'packages': PrimitiveField(packages),
+    })
+    super(ExternalNativeLibrary, self).__init__(payload=payload, **kwargs)
+
+  @property
+  def packages(self):
+    return self.payload.packages
+
+  @property
+  def lib_names(self):
+    return [re.match(r'^([^\/]+)\/', pkg_name).group(1) for pkg_name in self.payload.packages]

src/python/pants/backend/native/tasks/cpp_compile.py

@@ -41,6 +41,17 @@ def get_compile_settings(self):
   def get_compiler(self):
     return self._request_single(CppCompiler, self._toolchain)
 
+  def _make_compile_argv(self, compile_request):
+    # FIXME: this is a temporary fix, do not do any of this kind of introspection.
+    # https://github.com/pantsbuild/pants/issues/5951
+    prev_argv = super(CppCompile, self)._make_compile_argv(compile_request)
+
+    if compile_request.compiler.exe_filename == 'clang++':
+      new_argv = [prev_argv[0], '-nobuiltininc', '-nostdinc++'] + prev_argv[1:]
+    else:
+      new_argv = prev_argv
+    return new_argv
+
   # FIXME(#5951): don't have any command-line args in the task or in the subsystem -- rather,
   # subsystem options should be used to populate an `Executable` which produces its own arguments.
   def extra_compile_args(self):