-
-
Notifications
You must be signed in to change notification settings - Fork 632
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Conan (third party) support for ctypes native libraries #5998
Changes from 44 commits
19b2824
91641b7
5898ce8
cb79337
41067f8
70cb412
3ea6586
afb45c8
8bc0e29
1dab9c0
8631b9c
6a95c66
d6dde33
932a8af
bc50b16
bab73bb
bd8f792
abaef25
47994c4
67eae3f
579e8dd
7439e18
4c41611
a9cc38f
ac050f3
9bc4c35
b1ecc81
611579f
05c2cf7
40b60bb
1bf69da
ee0fa06
0047140
7fc20ac
529f58c
0818064
5803352
5fcc69b
abdffa6
e0e772a
8d028c0
cf64653
51d4829
b7b5916
4a8bedb
ea4c701
5dbaea4
84be507
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
# coding=utf-8 | ||
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import absolute_import, division, print_function, unicode_literals | ||
|
||
import logging | ||
import os | ||
|
||
from pex.interpreter import PythonInterpreter | ||
from pex.pex import PEX | ||
from pex.pex_builder import PEXBuilder | ||
from pex.pex_info import PexInfo | ||
|
||
from pants.backend.python.python_requirement import PythonRequirement | ||
from pants.backend.python.tasks.pex_build_util import dump_requirements | ||
from pants.backend.python.tasks.wrapped_pex import WrappedPEX | ||
from pants.base.build_environment import get_pants_cachedir | ||
from pants.subsystem.subsystem import Subsystem | ||
from pants.util.dirutil import safe_concurrent_creation | ||
from pants.util.objects import datatype | ||
|
||
|
||
logger = logging.getLogger(__name__) | ||
|
||
|
||
class Conan(Subsystem): | ||
"""Pex binary for the conan package manager.""" | ||
options_scope = 'conan' | ||
default_conan_requirements = ( | ||
'conan==1.4.4', | ||
'PyJWT>=1.4.0, <2.0.0', | ||
'requests>=2.7.0, <3.0.0', | ||
'colorama>=0.3.3, <0.4.0', | ||
'PyYAML>=3.11, <3.13.0', | ||
'patch==1.16', | ||
'fasteners>=0.14.1', | ||
'six>=1.10.0', | ||
'node-semver==0.2.0', | ||
'distro>=1.0.2, <1.2.0', | ||
'pylint>=1.8.1, <1.9.0', | ||
'future==0.16.0', | ||
'pygments>=2.0, <3.0', | ||
'astroid>=1.6, <1.7', | ||
'deprecation>=2.0, <2.1' | ||
) | ||
|
||
@classmethod | ||
def implementation_version(cls): | ||
return super(Conan, cls).implementation_version() + [('Conan', 0)] | ||
|
||
@classmethod | ||
def register_options(cls, register): | ||
super(Conan, cls).register_options(register) | ||
register('--conan-requirements', type=list, default=cls.default_conan_requirements, | ||
advanced=True, help='The requirements used to build the conan client pex.') | ||
|
||
class ConanBinary(datatype(['pex'])): | ||
"""A `conan` PEX binary.""" | ||
pass | ||
|
||
def bootstrap_conan(self): | ||
pex_info = PexInfo.default() | ||
pex_info.entry_point = 'conans.conan' | ||
conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support') | ||
conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary') | ||
interpreter = PythonInterpreter.get() | ||
if os.path.exists(conan_pex_path): | ||
conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter)) | ||
return self.ConanBinary(pex=conan_binary) | ||
else: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This line has trailing whitespace. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ✅ |
||
with safe_concurrent_creation(conan_pex_path) as safe_path: | ||
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info) | ||
reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements] | ||
dump_requirements(builder, interpreter, reqs, logger) | ||
builder.freeze() | ||
conan_binary = WrappedPEX(PEX(conan_pex_path, interpreter)) | ||
return self.ConanBinary(pex=conan_binary) |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -71,39 +71,44 @@ def select_linker(platform, native_toolchain): | |
# 'darwin': lambda: Get(Linker, XCodeCLITools, native_toolchain._xcode_cli_tools), | ||
# 'linux': lambda: Get(Linker, Binutils, native_toolchain._binutils), | ||
# }) | ||
# | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This whole file needs much better documentation. I will do that as part of #5951. |
||
# NB: We need to link through a provided compiler's frontend, and we need to know where all the | ||
# compiler's libraries/etc are, so we set the executable name to the C++ compiler, which can find | ||
# its own set of C++-specific files for the linker if necessary. Using e.g. 'g++' as the linker | ||
# appears to produce byte-identical output when linking even C-only object files, and also | ||
# happens to work when C++ is used. | ||
# Currently, OSX links through the clang++ frontend, and Linux links through the g++ frontend. | ||
if platform.normalized_os_name == 'darwin': | ||
# TODO(#5663): turn this into LLVM when lld works. | ||
linker = yield Get(Linker, XCodeCLITools, native_toolchain._xcode_cli_tools) | ||
llvm_c_compiler = yield Get(LLVMCCompiler, NativeToolchain, native_toolchain) | ||
c_compiler = llvm_c_compiler.c_compiler | ||
llvm_cpp_compiler = yield Get(LLVMCppCompiler, NativeToolchain, native_toolchain) | ||
cpp_compiler = llvm_cpp_compiler.cpp_compiler | ||
else: | ||
linker = yield Get(Linker, Binutils, native_toolchain._binutils) | ||
gcc_c_compiler = yield Get(GCCCCompiler, NativeToolchain, native_toolchain) | ||
c_compiler = gcc_c_compiler.c_compiler | ||
gcc_cpp_compiler = yield Get(GCCCppCompiler, NativeToolchain, native_toolchain) | ||
cpp_compiler = gcc_cpp_compiler.cpp_compiler | ||
|
||
libc_dirs = native_toolchain._libc_dev.get_libc_dirs(platform) | ||
|
||
# NB: We need to link through a provided compiler's frontend, and we need to know where all the | ||
# compiler's libraries/etc are, so we set the executable name to the C++ compiler, which can find | ||
# its own set of C++-specific files for the linker if necessary. Using e.g. 'g++' as the linker | ||
# appears to produce byte-identical output when linking even C-only object files, and also | ||
# happens to work when C++ is used. | ||
gcc_c_compiler = yield Get(GCCCCompiler, NativeToolchain, native_toolchain) | ||
c_compiler = gcc_c_compiler.c_compiler | ||
gcc_cpp_compiler = yield Get(GCCCppCompiler, NativeToolchain, native_toolchain) | ||
cpp_compiler = gcc_cpp_compiler.cpp_compiler | ||
|
||
# NB: If needing to create an environment for process invocation that could use either a compiler | ||
# or a linker (e.g. when we compile native code from `python_dist()`s), use the environment from | ||
# the linker object (in addition to any further customizations), which has the paths from the C | ||
# and C++ compilers baked in. | ||
# FIXME(#5951): we need a way to compose executables more hygienically. | ||
linker = Linker( | ||
path_entries=( | ||
c_compiler.path_entries + | ||
cpp_compiler.path_entries + | ||
c_compiler.path_entries + | ||
linker.path_entries), | ||
exe_filename=cpp_compiler.exe_filename, | ||
library_dirs=( | ||
libc_dirs + | ||
c_compiler.library_dirs + | ||
cpp_compiler.library_dirs + | ||
c_compiler.library_dirs + | ||
linker.library_dirs)) | ||
|
||
yield linker | ||
|
@@ -147,7 +152,7 @@ def select_llvm_cpp_compiler(platform, native_toolchain): | |
path_entries=(provided_clangpp.path_entries + xcode_clang.path_entries), | ||
exe_filename=provided_clangpp.exe_filename, | ||
library_dirs=(provided_clangpp.library_dirs + xcode_clang.library_dirs), | ||
include_dirs=(xcode_clang.include_dirs + provided_clangpp.include_dirs)) | ||
include_dirs=(provided_clangpp.include_dirs + xcode_clang.include_dirs)) | ||
final_llvm_cpp_compiler = LLVMCppCompiler(clang_with_xcode_paths) | ||
else: | ||
gcc_cpp_compiler = yield Get(GCCCppCompiler, GCC, native_toolchain._gcc) | ||
|
@@ -231,16 +236,28 @@ def select_gcc_cpp_compiler(platform, native_toolchain): | |
yield final_gcc_cpp_compiler | ||
|
||
|
||
@rule(CCompiler, [Select(NativeToolchain)]) | ||
def select_c_compiler(native_toolchain): | ||
llvm_c_compiler = yield Get(LLVMCCompiler, NativeToolchain, native_toolchain) | ||
yield llvm_c_compiler.c_compiler | ||
@rule(CCompiler, [Select(NativeToolchain), Select(Platform)]) | ||
def select_c_compiler(native_toolchain, platform): | ||
if platform.normalized_os_name == 'darwin': | ||
llvm_c_compiler = yield Get(LLVMCCompiler, NativeToolchain, native_toolchain) | ||
c_compiler = llvm_c_compiler.c_compiler | ||
else: | ||
gcc_c_compiler = yield Get(GCCCCompiler, NativeToolchain, native_toolchain) | ||
c_compiler = gcc_c_compiler.c_compiler | ||
|
||
yield c_compiler | ||
|
||
|
||
@rule(CppCompiler, [Select(NativeToolchain), Select(Platform)]) | ||
def select_cpp_compiler(native_toolchain, platform): | ||
if platform.normalized_os_name == 'darwin': | ||
llvm_cpp_compiler = yield Get(LLVMCppCompiler, NativeToolchain, native_toolchain) | ||
cpp_compiler = llvm_cpp_compiler.cpp_compiler | ||
else: | ||
gcc_cpp_compiler = yield Get(GCCCppCompiler, NativeToolchain, native_toolchain) | ||
cpp_compiler = gcc_cpp_compiler.cpp_compiler | ||
|
||
@rule(CppCompiler, [Select(NativeToolchain)]) | ||
def select_cpp_compiler(native_toolchain): | ||
llvm_cpp_compiler = yield Get(LLVMCppCompiler, NativeToolchain, native_toolchain) | ||
yield llvm_cpp_compiler.cpp_compiler | ||
yield cpp_compiler | ||
|
||
|
||
def create_native_toolchain_rules(): | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
# coding=utf-8 | ||
# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import absolute_import, division, print_function, unicode_literals | ||
|
||
import re | ||
|
||
from pants.base.payload import Payload | ||
from pants.base.payload_field import PrimitiveField | ||
from pants.base.validation import assert_list | ||
from pants.build_graph.target import Target | ||
|
||
|
||
class ExternalNativeLibrary(Target): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a good name for this target at this point in time! It may be worth revisiting later, along with the C/C++ library target names. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ✅ |
||
"""A set of Conan package strings to be passed to the Conan package manager.""" | ||
|
||
@classmethod | ||
def alias(cls): | ||
return 'external_native_library' | ||
|
||
def __init__(self, payload=None, packages=None, **kwargs): | ||
""" | ||
:param packages: a list of Conan-style package strings | ||
|
||
Example: | ||
lzo/2.10@twitter/stable | ||
""" | ||
payload = payload or Payload() | ||
|
||
assert_list(packages, key_arg='packages') | ||
payload.add_fields({ | ||
'packages': PrimitiveField(packages), | ||
}) | ||
super(ExternalNativeLibrary, self).__init__(payload=payload, **kwargs) | ||
|
||
@property | ||
def packages(self): | ||
return self.payload.packages | ||
|
||
@property | ||
def lib_names(self): | ||
return [re.match(r'^([^\/]+)\/', pkg_name).group(1) for pkg_name in self.payload.packages] |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -41,6 +41,16 @@ def get_compile_settings(self): | |
def get_compiler(self): | ||
return self._request_single(CppCompiler, self._toolchain) | ||
|
||
def _make_compile_argv(self, compile_request): | ||
# FIXME: this is a temporary fix, do not do any of this kind of introspection. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could you please add a link to #5951 here? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ✅ |
||
prev_argv = super(CppCompile, self)._make_compile_argv(compile_request) | ||
|
||
if compile_request.compiler.exe_filename == 'clang++': | ||
new_argv = [prev_argv[0], '-nobuiltininc', '-nostdinc++'] + prev_argv[1:] | ||
else: | ||
new_argv = prev_argv | ||
return new_argv | ||
|
||
# FIXME(#5951): don't have any command-line args in the task or in the subsystem -- rather, | ||
# subsystem options should be used to populate an `Executable` which produces its own arguments. | ||
def extra_compile_args(self): | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is neat - one day we should generalize it in a manner similar to JVMTools.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
✅
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Agreed, this is really neat!