Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NCBI C++ Toolkit v27.0.0 #16750

Closed
wants to merge 32 commits into from
Closed
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
274b1d8
Added v27.0.0, with Conan2 support
gouriano Mar 28, 2023
bcd0964
Modified per Conan center recommendations
gouriano Mar 28, 2023
223339a
Modified per Conan center recommendations
gouriano Mar 28, 2023
ba65e3a
Added verbose output into build
gouriano Mar 31, 2023
fd27356
Fixed compilation on GCC 7.2
gouriano Mar 31, 2023
eefdb78
Corrected compiler version check
gouriano Apr 3, 2023
592eeae
Fixed installation on Windows
gouriano Apr 4, 2023
8cc3d36
trigger rebuild
gouriano Apr 17, 2023
53dc735
Update recipes/ncbi-cxx-toolkit-public/all/dependencies/requirements-…
jcar87 Apr 17, 2023
99eda5a
Rely on GRPC to install PROTOBUF
gouriano Apr 17, 2023
4de55e9
Update recipes/ncbi-cxx-toolkit-public/all/test_package/CMakeLists.txt
jcar87 Apr 18, 2023
fb711b9
Update recipes/ncbi-cxx-toolkit-public/all/test_v1_package/CMakeLists…
jcar87 Apr 18, 2023
d262d42
Added patch type and description
gouriano Apr 18, 2023
ad7aacc
Change variable names to start with underscore
gouriano Apr 18, 2023
643e8fb
Added ApplicationServices framework on Mac
gouriano Apr 19, 2023
03b1276
Changed patch types
gouriano May 3, 2023
5b93304
Corrected requirements to fix conflicts.
gouriano May 4, 2023
5b424cf
Merge branch 'conan-io:master' into master
gouriano Jul 13, 2023
86eb8f3
Merge branch 'conan-io:master' into master
gouriano Sep 7, 2023
12883aa
Merge branch 'conan-io:master' into master
gouriano Sep 18, 2023
05ff3c1
Require zstd/1.5.5
gouriano Sep 18, 2023
0730040
Disable BerkeleyDB
gouriano Sep 19, 2023
ddb4291
Reenabled BerkeleyDB on Conan1
gouriano Sep 19, 2023
475bf6b
Merge branch 'conan-io:master' into master
gouriano Oct 31, 2023
6e81261
require xz_utils/5.4.4
gouriano Oct 31, 2023
b1a1b6c
Merge branch 'conan-io:master' into master
gouriano Nov 1, 2023
57f713b
Require libxml2/2.11.4
gouriano Nov 1, 2023
ccb7ba0
Removed print command.
gouriano Nov 6, 2023
d762520
Merge branch 'conan-io:master' into master
gouriano Feb 1, 2024
4237d6e
Updated conanfile and requirements
gouriano Feb 2, 2024
01c77e9
Merge branch 'conan-io:master' into master
gouriano Apr 4, 2024
7c14c2f
Merge branch 'conan-io:master' into master
gouriano Apr 5, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions recipes/ncbi-cxx-toolkit-public/all/conandata.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
sources:
"27.0.0":
url: "https://github.com/ncbi/ncbi-cxx-toolkit-public/archive/refs/tags/release-27.0.0.tar.gz"
sha256: "c8fb3f99c6fce4f170b381f3a7789c76a2ff1c23c094c9852e2e3de1fdc57277"

patches:
"27.0.0":
- patch_file: "patches/27.0.0-compiler.patch"
- patch_file: "patches/27.0.0-install.patch"
293 changes: 293 additions & 0 deletions recipes/ncbi-cxx-toolkit-public/all/conanfile.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,293 @@
from conan import ConanFile
from conan.errors import ConanInvalidConfiguration, ConanException
from conan.tools.microsoft import check_min_vs, is_msvc_static_runtime, is_msvc
from conan.tools.files import apply_conandata_patches, export_conandata_patches, get, copy
from conan.tools.build import check_min_cppstd, cross_building
from conan.tools.scm import Version
from conan.tools.cmake import CMakeDeps, CMakeToolchain, CMake, cmake_layout
import os
import yaml

required_conan_version = ">=1.53.0"


class NcbiCxxToolkit(ConanFile):
name = "ncbi-cxx-toolkit-public"
description = "NCBI C++ Toolkit -- a cross-platform application framework and a collection of libraries for working with biological data."
license = "CC0-1.0"
url = "https://github.com/conan-io/conan-center-index"
homepage = "https://ncbi.github.io/cxx-toolkit"
topics = ("ncbi", "biotechnology", "bioinformatics", "genbank", "gene",
"genome", "genetic", "sequence", "alignment", "blast",
"biological", "toolkit", "c++")
package_type = "library"
settings = "os", "arch", "compiler", "build_type"
options = {
"shared": [True, False],
"fPIC": [True, False],
"with_projects": ["ANY"],
"with_targets": ["ANY"],
"with_components": ["ANY"]
}
default_options = {
"shared": False,
"fPIC": True,
"with_projects": "",
"with_targets": "",
"with_components": ""
}
short_paths = True
tk_dependencies = None
tk_requirements = None
tk_componenttargets = set()
Copy link
Member

@AbrilRBS AbrilRBS Apr 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
tk_dependencies = None
tk_requirements = None
tk_componenttargets = set()
_tk_dependencies = None
_tk_requirements = None
_tk_componenttargets = set()

Custom user variables in recipes need to start with underscore, see https://docs.conan.io/2/knowledge/guidelines.html?highlight=public#forbidden-practices

This is why the recipe is failing the v2.0 checks (Conan expects to be able to serialize all "public" members, and the set() was breaking it)


@property
def _min_cppstd(self):
return 17

@property
def _compilers_minimum_version(self):
return {
"gcc": "7",
"clang": "7",
"apple-clang": "10",
}

@property
def _dependencies_folder(self):
return "dependencies"

@property
def _dependencies_filename(self):
return f"dependencies-{Version(self.version).major}.{Version(self.version).minor}.yml"

@property
def _requirements_filename(self):
return f"requirements-{Version(self.version).major}.{Version(self.version).minor}.yml"

@property
def _tk_dependencies(self):
if self.tk_dependencies is None:
dependencies_filepath = os.path.join(self.recipe_folder, self._dependencies_folder, self._dependencies_filename)
if not os.path.isfile(dependencies_filepath):
raise ConanException(f"Cannot find {dependencies_filepath}")
with open(dependencies_filepath, "r", encoding="utf-8") as f:
self.tk_dependencies = yaml.safe_load(f)
return self.tk_dependencies

@property
def _tk_requirements(self):
if self.tk_requirements is None:
requirements_filepath = os.path.join(self.recipe_folder, self._dependencies_folder, self._requirements_filename)
if not os.path.isfile(requirements_filepath):
raise ConanException(f"Cannot find {requirements_filepath}")
with open(requirements_filepath, "r", encoding="utf-8") as f:
self.tk_requirements = yaml.safe_load(f)
return self.tk_requirements

def _translate_req(self, key):
if "Boost" in key:
key = "Boost"
if key in self._tk_requirements["disabled"].keys():
if self.settings.os in self._tk_requirements["disabled"][key]:
return None
if key in self._tk_requirements["requirements"].keys():
return self._tk_requirements["requirements"][key]
return None

def _parse_option(self, data):
_res = set()
if data != "":
_data = str(data)
_data = _data.replace(",", ";")
_data = _data.replace(" ", ";")
_res.update(_data.split(";"))
if "" in _res:
_res.remove("")
return _res

def export(self):
copy(self, self._dependencies_filename,
os.path.join(self.recipe_folder, self._dependencies_folder),
os.path.join(self.export_folder, self._dependencies_folder))
copy(self, self._requirements_filename,
os.path.join(self.recipe_folder, self._dependencies_folder),
os.path.join(self.export_folder, self._dependencies_folder))

def export_sources(self):
export_conandata_patches(self)

def config_options(self):
if self.settings.os == "Windows":
del self.options.fPIC

def configure(self):
if self.options.shared:
self.options.rm_safe("fPIC")

def layout(self):
cmake_layout(self, src_folder="src")

def requirements(self):
_alltargets = self._parse_option(self.options.with_targets)
_required_components = set()
for _t in _alltargets:
for _component in self._tk_dependencies["components"]:
_libraries = self._tk_dependencies["libraries"][_component]
if _t in _libraries:
_required_components.add(_component)
break

_allcomponents = self._parse_option(self.options.with_components)
_required_components.update(_allcomponents)

if len(_required_components) > 0:
_todo = _required_components.copy()
_required_components.clear()
_next = set()
while len(_todo) > 0:
for _component in _todo:
if not _component in _required_components:
_required_components.add(_component)
if _component in self._tk_dependencies["dependencies"].keys():
for _n in self._tk_dependencies["dependencies"][_component]:
if not _n in _required_components:
_next.add(_n)
_todo = _next.copy()
_next.clear()

if len(_required_components) == 0:
_required_components.update( self._tk_dependencies["components"])
else:
for component in _required_components:
self.tk_componenttargets.update(self._tk_dependencies["libraries"][component])

requirements = set()
for component in _required_components:
libraries = self._tk_dependencies["libraries"][component]
for lib in libraries:
if lib in self._tk_dependencies["requirements"].keys():
requirements.update(self._tk_dependencies["requirements"][lib])

for req in requirements:
pkgs = self._translate_req(req)
if pkgs is not None:
for pkg in pkgs:
print("Package requires ", pkg)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
print("Package requires ", pkg)

Please, do not use print(), prefer self.output.info(). Requirements you can avoid, because is already listed by conan when solving the graph.

self.requires(pkg)

def validate(self):
if self.settings.compiler.cppstd:
check_min_cppstd(self, self._min_cppstd)
if self.settings.os not in ["Linux", "Macos", "Windows"]:
raise ConanInvalidConfiguration("This operating system is not supported")
if is_msvc(self):
check_min_vs(self, 192)
if self.options.shared and is_msvc_static_runtime(self):
raise ConanInvalidConfiguration("This configuration is not supported")
else:
minimum_version = self._compilers_minimum_version.get(str(self.settings.compiler), False)
if minimum_version and Version(self.settings.compiler.version) < minimum_version:
raise ConanInvalidConfiguration(f"This version of {self.settings.compiler} is not supported")
if cross_building(self):
raise ConanInvalidConfiguration("Cross compilation is not supported")

def source(self):
get(self, **self.conan_data["sources"][self.version], strip_root=True)
apply_conandata_patches(self)
root = os.path.join(os.getcwd(), "CMakeLists.txt")
with open(root, "w", encoding="utf-8") as f:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This could just be a file that is checked into git, you do not need to dynamically write this

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As a matter of fact, "root" CMakeLists.txt is located in src directory.
Why not in the tree root? There is no definite answer to this question. Tradition, maybe even convenience...
At present there is no consensus whether it makes sense to have it in the root.
It is required to create the Conan package, so here it is.

f.write("cmake_minimum_required(VERSION 3.15)\n")
f.write("project(ncbi-cpp)\n")
f.write("include(src/build-system/cmake/CMake.NCBItoolkit.cmake)\n")
f.write("add_subdirectory(src)\n")

def generate(self):
tc = CMakeToolchain(self)
tc.variables["NCBI_PTBCFG_PACKAGING"] = True
if self.options.shared:
tc.variables["NCBI_PTBCFG_ALLOW_COMPOSITE"] = True
tc.variables["NCBI_PTBCFG_PROJECT_LIST"] = str(self.options.with_projects) + ";-app/netcache"
if self.options.with_targets != "":
tc.variables["NCBI_PTBCFG_PROJECT_TARGETS"] = self.options.with_targets
if len(self.tk_componenttargets) != 0:
tc.variables["NCBI_PTBCFG_PROJECT_COMPONENTTARGETS"] = ";".join(self.tk_componenttargets)
if is_msvc(self):
tc.variables["NCBI_PTBCFG_CONFIGURATION_TYPES"] = self.settings.build_type
tc.generate()
cmdep = CMakeDeps(self)
cmdep.generate()

def build(self):
cmake = CMake(self)
cmake.configure()
# Visual Studio sometimes runs "out of heap space"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are there any links to this happening, either upstream of or on CCI so we can have a refernece to see if the problem is fixed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is no links.
We struggled with the problem several years ago, when adopting CMake.
The workaround that finally helped was using devenv.com (instead of devenv.exe) to build the solution.
It works fine in CI. When building it in Visual Studio IDE, we still have this problem.
It is less painfull though. People do not often build the whole toolkit. Also, running the build second time, usually succeeds.
With Conan package, we saw this problem again. We do not want to try our luck. Disabling parallel build is a safe choice.

if is_msvc(self):
cmake.parallel = False
cmake.build()

def package(self):
cmake = CMake(self)
cmake.install()

def package_info(self):
impfile = os.path.join(self.package_folder, "res", "ncbi-cpp-toolkit.imports")
with open(impfile, "r", encoding="utf-8") as f:
allexports = set(f.read().split())
absent = []
for component in self._tk_dependencies["components"]:
c_libs = []
libraries = self._tk_dependencies["libraries"][component]
for lib in libraries:
if lib in allexports:
c_libs.append(lib)
if len(c_libs) == 0 and len(libraries) != 0:
absent.append(component)
for component in self._tk_dependencies["components"]:
c_libs = []
c_reqs = []
n_reqs = set()
c_deps = self._tk_dependencies["dependencies"][component]
for c in c_deps:
if c in absent:
c_deps.remove(c)
c_reqs.extend(c_deps)
libraries = self._tk_dependencies["libraries"][component]
for lib in libraries:
if lib in allexports:
c_libs.append(lib)
if lib in self._tk_dependencies["requirements"].keys():
n_reqs.update(self._tk_dependencies["requirements"][lib])
for req in n_reqs:
pkgs = self._translate_req(req)
if pkgs is not None:
for pkg in pkgs:
pkg = pkg[:pkg.find("/")]
ref = pkg + "::" + pkg
c_reqs.append(ref)
if len(c_libs) != 0 or (len(libraries) == 0 and len(c_reqs) != 0):
self.cpp_info.components[component].libs = c_libs
self.cpp_info.components[component].requires = c_reqs

if self.settings.os == "Windows":
self.cpp_info.components["core"].defines.append("_UNICODE")
self.cpp_info.components["core"].defines.append("_CRT_SECURE_NO_WARNINGS=1")
else:
self.cpp_info.components["core"].defines.append("_MT")
self.cpp_info.components["core"].defines.append("_REENTRANT")
self.cpp_info.components["core"].defines.append("_THREAD_SAFE")
self.cpp_info.components["core"].defines.append("_FILE_OFFSET_BITS=64")
if self.options.shared:
self.cpp_info.components["core"].defines.append("NCBI_DLL_BUILD")
if self.settings.build_type == "Debug":
self.cpp_info.components["core"].defines.append("_DEBUG")
else:
self.cpp_info.components["core"].defines.append("NDEBUG")
if self.settings.os == "Windows":
self.cpp_info.components["core"].system_libs = ["ws2_32", "dbghelp"]
elif self.settings.os == "Linux":
self.cpp_info.components["core"].system_libs = ["dl", "rt", "m", "pthread", "resolv"]
elif self.settings.os == "Macos":
self.cpp_info.components["core"].system_libs = ["dl", "c", "m", "pthread", "resolv"]
self.cpp_info.components["core"].builddirs.append("res")
self.cpp_info.components["core"].build_modules = ["res/build-system/cmake/CMake.NCBIpkg.conan.cmake"]
Loading