Conan (third party) support for ctypes native libraries

src/python/pants/backend/native/register.py

@@ -11,11 +11,13 @@
 from pants.backend.native.subsystems.binaries.llvm import create_llvm_rules
 from pants.backend.native.subsystems.native_toolchain import create_native_toolchain_rules
 from pants.backend.native.subsystems.xcode_cli_tools import create_xcode_cli_tools_rules
+from pants.backend.native.targets.external_native_library import ExternalNativeLibrary
 from pants.backend.native.targets.native_artifact import NativeArtifact
 from pants.backend.native.targets.native_library import CLibrary, CppLibrary
 from pants.backend.native.tasks.c_compile import CCompile
 from pants.backend.native.tasks.cpp_compile import CppCompile
 from pants.backend.native.tasks.link_shared_libraries import LinkSharedLibraries
+from pants.backend.native.tasks.native_external_library_fetch import NativeExternalLibraryFetch
 from pants.build_graph.build_file_aliases import BuildFileAliases
 from pants.goal.task_registrar import TaskRegistrar as task
 
@@ -25,6 +27,7 @@ def build_file_aliases():
     targets={
       CLibrary.alias(): CLibrary,
       CppLibrary.alias(): CppLibrary,
+      ExternalNativeLibrary.alias(): ExternalNativeLibrary
     },
     objects={
       NativeArtifact.alias(): NativeArtifact,
@@ -35,6 +38,7 @@ def build_file_aliases():
 def register_goals():
   # FIXME(#5962): register these under the 'compile' goal when we eliminate the product transitive
   # dependency from export -> compile.
+  task(name='native-third-party-fetch', action=NativeExternalLibraryFetch).install('native-compile')
   task(name='c-for-ctypes', action=CCompile).install('native-compile')
   task(name='cpp-for-ctypes', action=CppCompile).install('native-compile')
   task(name='shared-libraries', action=LinkSharedLibraries).install('link')

src/python/pants/backend/native/subsystems/conan.py

@@ -0,0 +1,91 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import logging
+import os
+
+from pex.interpreter import PythonInterpreter
+from pex.pex import PEX
+from pex.pex_builder import PEXBuilder
+from pex.pex_info import PexInfo
+
+from pants.backend.python.python_requirement import PythonRequirement
+from pants.backend.python.tasks.pex_build_util import dump_requirements
+from pants.backend.python.tasks.wrapped_pex import WrappedPEX
+from pants.base.build_environment import get_pants_cachedir
+from pants.subsystem.subsystem import Subsystem
+from pants.util.dirutil import safe_concurrent_creation
+
+
+log = logging.getLogger(__name__)
+
+
+class Conan(Subsystem):
+  """Pex binary for the conan package manager."""
+  options_scope = 'conan'
+
+  @staticmethod
+  def default_conan_requirements():
+    return (
+      'conan==1.4.4',
+      'PyJWT>=1.4.0, <2.0.0',
+      'requests>=2.7.0, <3.0.0',
+      'colorama>=0.3.3, <0.4.0',
+      'PyYAML>=3.11, <3.13.0',
+      'patch==1.16',
+      'fasteners>=0.14.1',
+      'six>=1.10.0',
+      'node-semver==0.2.0',
+      'distro>=1.0.2, <1.2.0',
+      'pylint>=1.8.1, <1.9.0',
+      'future==0.16.0',
+      'pygments>=2.0, <3.0',
+      'astroid>=1.6, <1.7',
+      'deprecation>=2.0, <2.1'
+    )
+
+  @classmethod
+  def register_options(cls, register):
+    super(Conan, cls).register_options(register)
+    register('--conan-requirements', type=list, default=cls.default_conan_requirements(),
+             advanced=True, help='The requirements used to build to conan client pex.')
+
+  class ConanBinary(object):
+    """A `conan` PEX binary."""
+
+    def __init__(self, pex):
+      self._pex = pex
+
+    @property
+    def pex(self):
+      """Return the conan binary PEX.
+
+      :rtype: :class:`pex.pex.PEX`
+      """
+      return self._pex
+
+  @classmethod
+  def implementation_version(cls):
+    return super(Conan, cls).implementation_version() + [('Conan', 1)]
+
+  def bootstrap_conan(self):
+    pex_info = PexInfo.default()
+    pex_info.entry_point = 'conans.conan'
+    conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support')
+    conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary')
+    interpreter = PythonInterpreter.get()
+    if os.path.exists(conan_pex_path):
+      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter), interpreter)
+      return self.ConanBinary(conan_binary)
+    else:  
+      with safe_concurrent_creation(conan_pex_path) as safe_path:
+        builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
+        reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements]
+        dump_requirements(builder, interpreter, reqs, log)
+        builder.freeze()
+      conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter), interpreter)
+      return self.ConanBinary(conan_binary)

src/python/pants/backend/native/targets/external_native_library.py

@@ -0,0 +1,42 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.base.payload import Payload
+from pants.base.payload_field import PrimitiveField
+from pants.base.validation import assert_list
+from pants.build_graph.target import Target
+
+
+class ExternalNativeLibrary(Target):
+  """A set of Conan package strings to be passed to the Conan package manager."""
+
+  @classmethod
+  def alias(cls):
+    return 'external_native_library'
+
+  def __init__(self, payload=None, packages=None, **kwargs):
+    """
+    :param packages: a list of Conan-style package strings
+
+    Example:
+      lzo/2.10@twitter/stable
+    """
+    payload = payload or Payload()
+
+    assert_list(packages, key_arg='packages')
+    payload.add_fields({
+      'packages': PrimitiveField(packages),
+    })
+    super(ExternalNativeLibrary, self).__init__(payload=payload, **kwargs)
+
+  @property
+  def packages(self):
+    return self.payload.packages
+
+  @property
+  def lib_names(self):
+    return [pkg_name.split('/')[0] for pkg_name in self.payload.packages]

src/python/pants/backend/native/tasks/link_shared_libraries.py

@@ -11,6 +11,7 @@
 from pants.backend.native.subsystems.native_toolchain import NativeToolchain
 from pants.backend.native.targets.native_library import NativeLibrary
 from pants.backend.native.tasks.native_compile import NativeTargetDependencies, ObjectFiles
+from pants.backend.native.tasks.native_external_library_fetch import NativeExternalLibraryFetch
 from pants.backend.native.tasks.native_task import NativeTask
 from pants.base.exceptions import TaskError
 from pants.base.workunit import WorkUnit, WorkUnitLabel
@@ -142,6 +143,16 @@ def _make_link_request(self, vt, compiled_objects_product, native_target_deps_pr
   def _get_shared_lib_cmdline_args(self):
     return self.linker.platform.resolve_platform_specific(self._SHARED_CMDLINE_ARGS)
 
+  def _get_third_party_lib_args(self):
+    lib_args = []
+    tp_files_product = self.context.products.get_data(NativeExternalLibraryFetch.NativeExternalLibraryFiles)
+    if tp_files_product.lib_names:
+      for lib_name in tp_files_product.lib_names:
+        lib_args.append('-l{}'.format(lib_name))
+      lib_dir_arg = '-L{}'.format(tp_files_product.lib)
+      lib_args.append(lib_dir_arg)
+    return lib_args
+
   def _execute_link_request(self, link_request):
     object_files = link_request.object_files
 
@@ -156,6 +167,8 @@ def _execute_link_request(self, link_request):
     # We are executing in the results_dir, so get absolute paths for everything.
     cmd = ([linker.exe_filename] +
            self._get_shared_lib_cmdline_args() +
+           self._get_third_party_lib_args() +
+           ['-lstdc++'] +
            ['-o', os.path.abspath(resulting_shared_lib_path)] +
            [os.path.abspath(obj) for obj in object_files])
 

src/python/pants/backend/native/tasks/native_compile.py

@@ -11,6 +11,7 @@
 
 from pants.backend.native.config.environment import Executable
 from pants.backend.native.targets.native_library import NativeLibrary
+from pants.backend.native.tasks.native_external_library_fetch import NativeExternalLibraryFetch
 from pants.backend.native.tasks.native_task import NativeTask
 from pants.base.exceptions import TaskError
 from pants.base.workunit import WorkUnit, WorkUnitLabel
@@ -55,6 +56,7 @@ class NativeCompile(NativeTask, AbstractClass):
   # `NativeCompile` will use `workunit_label` as the name of the workunit when executing the
   # compiler process. `workunit_label` must be set to a string.
   workunit_label = None
+  workunit_name = 'native-compile'
 
   @classmethod
   def product_types(cls):
@@ -185,10 +187,20 @@ def get_compiler(self):
   def _compiler(self):
     return self.get_compiler()
 
+  def get_third_party_include_dirs(self):
+    inc_dir = []
+    tp_files_product = self.context.products.get_data(NativeExternalLibraryFetch.NativeExternalLibraryFiles)
+    directory = tp_files_product.include
+    if directory:
+      inc_dir = [directory]
+    return inc_dir
+
   def _make_compile_request(self, versioned_target, dependencies):
     target = versioned_target.target
     include_dirs = [self._include_dirs_for_target(dep_tgt) for dep_tgt in dependencies]
+    include_dirs.extend(self.get_third_party_include_dirs())
     sources_and_headers = self.get_sources_headers_for_target(target)
+
     return NativeCompileRequest(
       compiler=self._compiler,
       include_dirs=include_dirs,