Skip to content

Commit

Permalink
(#23163) Arrow: Refactor auto options
Browse files Browse the repository at this point in the history 
* Arrow: Refactor auto options

* add todo

* make boost compulsory in msvc

* enable parquet and thrift by default

* bump boost version to avoid conflicts with thrift's

* try with re2

* bool compute

* back to previous default values

* avoid config

* fix

* Arrow: remove dead code (#6)

* remove dead code

* fix indentation

* Update recipes/arrow/all/conanfile.py

* remove warnings

---------

Co-authored-by: ericLemanissier <ericLemanissier@users.noreply.github.com>
  • Loading branch information
@danimtb @ericLemanissier
danimtb and ericLemanissier authored Apr 9, 2024
1 parent 9ac60ba commit aab7658
Showing 1 changed file with 43 additions and 172 deletions.
215 changes: 43 additions & 172 deletions recipes/arrow/all/conanfile.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from conan import ConanFile
from conan.errors import ConanInvalidConfiguration
from conan.errors import ConanInvalidConfiguration, ConanException
from conan.tools.build import check_min_cppstd, cross_building
from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout
from conan.tools.files import apply_conandata_patches, copy, export_conandata_patches, get, rmdir
Expand Down Expand Up @@ -71,13 +71,13 @@ class ArrowConan(ConanFile):
"shared": False,
"fPIC": True,
"gandiva": False,
"parquet": "auto",
"parquet": False,
"skyhook": False,
"substrait": False,
"acero": False,
"cli": False,
"compute": "auto",
"dataset_modules": "auto",
"compute": False,
"dataset_modules": False,
"deprecated": True,
"encryption": False,
"filesystem_layer": False,
Expand All @@ -86,29 +86,29 @@ class ArrowConan(ConanFile):
"simd_level": "default",
"runtime_simd_level": "max",
"with_backtrace": False,
"with_boost": "auto",
"with_boost": False,
"with_brotli": False,
"with_bz2": False,
"with_csv": False,
"with_cuda": False,
"with_flight_rpc": "auto",
"with_flight_rpc": False,
"with_flight_sql": False,
"with_gcs": False,
"with_gflags": "auto",
"with_jemalloc": "auto",
"with_gflags": False,
"with_jemalloc": False,
"with_mimalloc": False,
"with_glog": "auto",
"with_grpc": "auto",
"with_glog": False,
"with_grpc": False,
"with_json": False,
"with_thrift": "auto",
"with_llvm": "auto",
"with_openssl": "auto",
"with_thrift": False,
"with_llvm": False,
"with_openssl": False,
"with_opentelemetry": False,
"with_orc": False,
"with_protobuf": "auto",
"with_re2": "auto",
"with_protobuf": False,
"with_re2": False,
"with_s3": False,
"with_utf8proc": "auto",
"with_utf8proc": False,
"with_lz4": False,
"with_snappy": False,
"with_zlib": False,
Expand Down Expand Up @@ -144,35 +144,10 @@ def export_sources(self):
def config_options(self):
if self.settings.os == "Windows":
del self.options.fPIC
if Version(self.version) < "2.0.0":
del self.options.simd_level
del self.options.runtime_simd_level
elif Version(self.version) < "6.0.0":
self.options.simd_level = "sse4_2"
if Version(self.version) < "6.0.0":
del self.options.with_gcs
if Version(self.version) < "7.0.0":
del self.options.skyhook
del self.options.with_flight_sql
del self.options.with_opentelemetry
if Version(self.version) < "8.0.0":
del self.options.substrait

self.options.parquet = self._parquet()
self.options.compute = self._compute()
self.options.dataset_modules = self._dataset_modules()
self.options.with_boost = self._with_boost()
self.options.with_flight_rpc = self._with_flight_rpc()
self.options.with_gflags = self._with_gflags()
self.options.with_glog = self._with_glog()
self.options.with_grpc = self._with_grpc()
self.options.with_jemalloc = self._with_jemalloc()
self.options.with_thrift = self._with_thrift()
self.options.with_llvm = self._with_llvm()
self.options.with_openssl = self._with_openssl()
self.options.with_protobuf = self._with_protobuf()
self.options.with_re2 = self._with_re2()
self.options.with_utf8proc = self._with_utf8proc()
if is_msvc(self):
self.options.with_boost = True

def configure(self):
if self.options.shared:
Expand All @@ -181,115 +156,8 @@ def configure(self):
def layout(self):
cmake_layout(self, src_folder="src")

def _compute(self):
if self.options.compute == "auto":
return bool(self._parquet() or self._dataset_modules()) or bool(self.options.get_safe("substrait", False))
else:
return bool(self.options.compute)

def _parquet(self):
if self.options.parquet == "auto":
return bool(self.options.get_safe("substrait", False))
else:
return bool(self.options.parquet)

def _dataset_modules(self):
if self.options.dataset_modules == "auto":
return bool(self.options.get_safe("substrait", False))
else:
return bool(self.options.dataset_modules)

def _with_jemalloc(self):
if self.options.with_jemalloc == "auto":
return bool("BSD" in str(self.settings.os))
else:
return bool(self.options.with_jemalloc)

def _with_re2(self):
if self.options.with_re2 == "auto":
if self.options.gandiva or self.options.parquet:
return True
if Version(self) >= "7.0.0" and (self._compute() or self._dataset_modules()):
return True
return False
else:
return bool(self.options.with_re2)

def _with_protobuf(self):
if self.options.with_protobuf == "auto":
return bool(self.options.gandiva or self._with_flight_rpc() or self.options.with_orc or self.options.get_safe("substrait", False))
else:
return bool(self.options.with_protobuf)

def _with_flight_rpc(self):
if self.options.with_flight_rpc == "auto":
return bool(self.options.get_safe("with_flight_sql", False))
else:
return bool(self.options.with_flight_rpc)

def _with_gflags(self):
if self.options.with_gflags == "auto":
return bool(self._with_glog() or self._with_grpc())
else:
return bool(self.options.with_gflags)

def _with_glog(self):
if self.options.with_glog == "auto":
return False
else:
return bool(self.options.with_glog)

def _with_grpc(self):
if self.options.with_grpc == "auto":
return self._with_flight_rpc()
else:
return bool(self.options.with_grpc)

def _with_boost(self):
if self.options.with_boost == "auto":
if self.options.gandiva:
return True
version = Version(self.version)
if version.major == "1":
if self._parquet() and self.settings.compiler == "gcc" and self.settings.compiler.version < Version("4.9"):
return True
elif version.major >= "2":
if is_msvc(self):
return True
return False
else:
return bool(self.options.with_boost)

def _with_thrift(self):
if self.options.with_thrift == "auto":
return bool(self._parquet())
else:
return bool(self.options.with_thrift)

def _with_utf8proc(self):
if self.options.with_utf8proc == "auto":
return bool(self._compute() or self.options.gandiva)
else:
return bool(self.options.with_utf8proc)

def _with_llvm(self):
if self.options.with_llvm == "auto":
return bool(self.options.gandiva)
else:
return bool(self.options.with_llvm)

def _with_openssl(self):
if self.options.with_openssl == "auto":
return bool(self.options.encryption or self._with_flight_rpc() or self.options.with_s3)
else:
return bool(self.options.with_openssl)

def _requires_rapidjson(self):
if self.options.with_json:
return True
if Version(self.version) >= "7.0.0" and self.options.encryption:
return True
return False
return self.options.with_json or self.options.encryption

def requirements(self):
if self.options.with_thrift:
Expand All @@ -301,7 +169,7 @@ def requirements(self):
if self.options.with_mimalloc:
self.requires("mimalloc/1.7.6")
if self.options.with_boost:
self.requires("boost/1.83.0")
self.requires("boost/1.84.0")
if self.options.with_gflags:
self.requires("gflags/2.2.2")
if self.options.with_glog:
Expand Down Expand Up @@ -332,8 +200,7 @@ def requirements(self):
self.requires("lz4/1.9.4")
if self.options.with_snappy:
self.requires("snappy/1.1.9")
if Version(self.version) >= "6.0.0" and \
self.options.get_safe("simd_level") != None or \
if self.options.get_safe("simd_level") != None or \
self.options.get_safe("runtime_simd_level") != None:
self.requires("xsimd/9.0.1")
if self.options.with_zlib:
Expand All @@ -348,11 +215,21 @@ def requirements(self):
self.requires("libbacktrace/cci.20210118")

def validate(self):
# validate options with 'auto' as default value
auto_options = ["parquet", "compute", "dataset_modules", "with_boost", "with_flight_rpc", "with_gflags", "with_glog",
"with_grpc", "with_jemalloc", "with_thrift", "with_llvm", "with_openssl", "with_protobuf", "with_re2", "with_utf8proc"]
for option in auto_options:
assert "auto" not in str(self.options.get_safe(option)), f"Option '{option}' contains 'auto' value, wich is not allowed. Generally the final value should be True/False"
# Do not allow options with 'auto' value
# TODO: Remove "auto" from the possible values for these options
auto_options = [option for option, value in self.options.items() if value == "auto"]
if auto_options:
raise ConanException("Options with value 'auto' are deprecated. Please set them true/false or use its default value."
f" Please change the following options: {auto_options}")

# From https://github.com/conan-io/conan-center-index/pull/23163#issuecomment-2039808851
if self.options.gandiva:
if not self.options.with_re2:
raise ConanException("'with_re2' option should be True when'gandiva=True'")
if not self.options.with_boost:
raise ConanException("'with_boost' option should be True when'gandiva=True'")
if not self.options.with_utf8proc:
raise ConanException("'with_utf8proc' option should be True when'gandiva=True'")

if self.settings.compiler.get_safe("cppstd"):
check_min_cppstd(self, self._min_cppstd)
Expand All @@ -376,8 +253,6 @@ def validate(self):
if self.dependencies["jemalloc"].options.enable_cxx:
raise ConanInvalidConfiguration("jemmalloc.enable_cxx of a static jemalloc must be disabled")

if Version(self.version) < "6.0.0" and self.options.get_safe("simd_level") == "default":
raise ConanInvalidConfiguration(f"In {self.ref}, simd_level options is not supported `default` value.")

def build_requirements(self):
if Version(self.version) >= "13.0.0":
Expand Down Expand Up @@ -414,7 +289,7 @@ def generate(self):
tc.variables["ARROW_NO_DEPRECATED_API"] = not bool(self.options.deprecated)
tc.variables["ARROW_FLIGHT"] = self.options.with_flight_rpc
tc.variables["ARROW_FLIGHT_SQL"] = bool(self.options.get_safe("with_flight_sql", False))
tc.variables["ARROW_COMPUTE"] = self.options.compute
tc.variables["ARROW_COMPUTE"] = bool(self.options.compute)
tc.variables["ARROW_CSV"] = bool(self.options.with_csv)
tc.variables["ARROW_CUDA"] = bool(self.options.with_cuda)
tc.variables["ARROW_JEMALLOC"] = self.options.with_jemalloc
Expand Down Expand Up @@ -459,12 +334,9 @@ def generate(self):
tc.variables["ZLIB_SOURCE"] = "SYSTEM"
tc.variables["xsimd_SOURCE"] = "SYSTEM"
tc.variables["ARROW_WITH_ZSTD"] = bool(self.options.with_zstd)
if Version(self.version) >= "2.0":
tc.variables["zstd_SOURCE"] = "SYSTEM"
tc.variables["ARROW_SIMD_LEVEL"] = str(self.options.simd_level).upper()
tc.variables["ARROW_RUNTIME_SIMD_LEVEL"] = str(self.options.runtime_simd_level).upper()
else:
tc.variables["ZSTD_SOURCE"] = "SYSTEM"
tc.variables["zstd_SOURCE"] = "SYSTEM"
tc.variables["ARROW_SIMD_LEVEL"] = str(self.options.simd_level).upper()
tc.variables["ARROW_RUNTIME_SIMD_LEVEL"] = str(self.options.runtime_simd_level).upper()
if self.options.with_zstd:
tc.variables["ARROW_ZSTD_USE_SHARED"] = bool(self.dependencies["zstd"].options.shared)
tc.variables["ORC_SOURCE"] = "SYSTEM"
Expand Down Expand Up @@ -509,7 +381,7 @@ def generate(self):

def _patch_sources(self):
apply_conandata_patches(self)
if "7.0.0" <= Version(self.version) < "10.0.0":
if Version(self.version) < "10.0.0":
for filename in glob.glob(os.path.join(self.source_folder, "cpp", "cmake_modules", "Find*.cmake")):
if os.path.basename(filename) not in [
"FindArrow.cmake",
Expand Down Expand Up @@ -613,9 +485,8 @@ def package_info(self):
self.cpp_info.components["libgandiva"].requires.append("boost::boost")
if self.options.parquet and self.settings.compiler == "gcc" and self.settings.compiler.version < Version("4.9"):
self.cpp_info.components["libparquet"].requires.append("boost::boost")
if Version(self.version) >= "2.0":
# FIXME: only headers components is used
self.cpp_info.components["libarrow"].requires.append("boost::boost")
# FIXME: only headers components is used
self.cpp_info.components["libarrow"].requires.append("boost::boost")
if self.options.with_openssl:
self.cpp_info.components["libarrow"].requires.append("openssl::openssl")
if self.options.with_gflags:
Expand Down

0 comments on commit aab7658

Please sign in to comment.