diff --git a/conan/internal/deploy.py b/conan/internal/deploy.py index d5f04693d20..4f02f5c525e 100644 --- a/conan/internal/deploy.py +++ b/conan/internal/deploy.py @@ -1,4 +1,4 @@ -import filecmp +import glob import os import shutil @@ -33,7 +33,8 @@ def _load(path): return _load(cache_path) builtin_deploy = {"full_deploy.py": full_deploy, "direct_deploy.py": direct_deploy, - "runtime_deploy.py": runtime_deploy}.get(d) + "merged_deploy.py": merged_deploy, + "shared_deploy.py": shared_deploy}.get(d) if builtin_deploy is not None: return builtin_deploy raise ConanException(f"Cannot find deployer '{d}'") @@ -70,12 +71,15 @@ def do_deploys(conan_api, graph, deploy, deploy_package, deploy_folder): def full_deploy(graph, output_folder): """ - Deploys to output_folder + host/dep/0.1/Release/x86_64 subfolder + Deploys all dependencies into + /full_deploy///// + folder structure. """ # TODO: This deployer needs to be put somewhere else # TODO: Document that this will NOT work with editables conanfile = graph.root.conanfile - conanfile.output.info(f"Conan built-in full deployer to {output_folder}") + output = ConanOutput(scope="full_deploy") + output.info(f"Deploying to {output_folder}") for dep in conanfile.dependencies.values(): if dep.package_folder is None: continue @@ -86,104 +90,146 @@ def full_deploy(graph, output_folder): folder_name = os.path.join(folder_name, build_type) if arch: folder_name = os.path.join(folder_name, arch) - _deploy_single(dep, conanfile, output_folder, folder_name) + _deploy_package_folder(dep, conanfile, output_folder, folder_name, output) -def runtime_deploy(graph, output_folder): +def direct_deploy(graph, output_folder): + """ + Deploy all direct dependencies with /direct_deploy/ folder structure. + """ + # TODO: This deployer needs to be put somewhere else + # TODO: Document that this will NOT work with editables + output_folder = os.path.join(output_folder, "direct_deploy") + conanfile = graph.root.conanfile + output = ConanOutput(scope="full_deploy") + output.info(f"Deploying to {output_folder}") + # If the argument is --requires, the current conanfile is a virtual one with 1 single + # dependency, the "reference" package. If the argument is a local path, then all direct + # dependencies + for dep in conanfile.dependencies.filter({"direct": True}).values(): + _deploy_package_folder(dep, conanfile, output_folder, dep.ref.name, output) + + +def merged_deploy(graph, output_folder): """ - Deploy all the shared libraries and the executables of the dependencies in a flat directory. + Merge all host dependency package folders into a single /merged_deploy folder. + License files are copied as /merged_deploy/licenses/. + All non-license files must be unique across packages. """ conanfile = graph.root.conanfile - output = ConanOutput(scope="runtime_deploy") - output.info(f"Deploying dependencies runtime to folder: {output_folder}") - output.warning("This deployer is experimental and subject to change. " - "Please give feedback at https://github.com/conan-io/conan/issues") + output = ConanOutput(scope="merged_deploy") + output_folder = os.path.join(output_folder, "merged_deploy") + rmdir(output_folder) mkdir(output_folder) - symlinks = conanfile.conf.get("tools.deployer:symlinks", check_type=bool, default=True) + ignored = shutil.ignore_patterns("licenses", "conaninfo.txt", "conanmanifest.txt") for _, dep in conanfile.dependencies.host.items(): if dep.package_folder is None: - output.warning(f"{dep.ref} does not have any package folder, skipping binary") + output.error(f"{dep.ref} does not have a package folder, skipping") continue - count = 0 + _copytree(dep.package_folder, + output_folder, + conanfile, dep, output, dirs_exist_ok=True, ignore=ignored) + _copytree(os.path.join(dep.package_folder, "licenses"), + os.path.join(output_folder, "licenses", dep.ref.name), + conanfile, dep, output, dirs_exist_ok=True) + dep.set_deploy_folder(output_folder) + conanfile.output.success(f"Deployed dependencies to: {output_folder}") + + +def shared_deploy(graph, output_folder): + """ + Deploy all shared libraries from host dependencies into . + """ + conanfile = graph.root.conanfile + output = ConanOutput(scope="shared_deploy") + output.info(f"Deploying runtime dependencies to folder: {output_folder}") + mkdir(output_folder) + keep_symlinks = conanfile.conf.get("tools.deployer:symlinks", check_type=bool, default=True) + for _, dep in conanfile.dependencies.host.items(): + if dep.package_folder is None: + output.warning(f"{dep.ref} does not have a package folder, skipping") + continue + cpp_info = dep.cpp_info.aggregated_components() + copied_libs = set() + for bindir in cpp_info.bindirs: if not os.path.isdir(bindir): - output.warning(f"{dep.ref} {bindir} does not exist") continue - count += _flatten_directory(dep, bindir, output_folder, symlinks) + for lib in cpp_info.libs: + if _copy_pattern(f"{lib}.dll", bindir, output_folder, keep_symlinks): + copied_libs.add(lib) for libdir in cpp_info.libdirs: if not os.path.isdir(libdir): - output.warning(f"{dep.ref} {libdir} does not exist") - continue - count += _flatten_directory(dep, libdir, output_folder, symlinks, [".dylib", ".so"]) - - output.info(f"Copied {count} files from {dep.ref}") - conanfile.output.success(f"Runtime deployed to folder: {output_folder}") - - -def _flatten_directory(dep, src_dir, output_dir, symlinks, extension_filter=None): - """ - Copy all the files from the source directory in a flat output directory. - An optional string, named extension_filter, can be set to copy only the files with - the listed extensions. - """ - file_count = 0 - - output = ConanOutput(scope="runtime_deploy") - for src_dirpath, _, src_filenames in os.walk(src_dir, followlinks=symlinks): - for src_filename in src_filenames: - if extension_filter and not any(src_filename.endswith(ext) for ext in extension_filter): continue - - src_filepath = os.path.join(src_dirpath, src_filename) - dest_filepath = os.path.join(output_dir, src_filename) - if os.path.exists(dest_filepath): - if filecmp.cmp(src_filepath, dest_filepath): # Be efficient, do not copy - output.verbose(f"{dest_filepath} exists with same contents, skipping copy") - continue - else: - output.warning(f"{dest_filepath} exists and will be overwritten") - - try: - file_count += 1 - shutil.copy2(src_filepath, dest_filepath, follow_symlinks=symlinks) - output.verbose(f"Copied {src_filepath} into {output_dir}") - except Exception as e: - if "WinError 1314" in str(e): - ConanOutput().error("runtime_deploy: Windows symlinks require admin privileges " - "or 'Developer mode = ON'", error_type="exception") - raise ConanException(f"runtime_deploy: Copy of '{dep}' files failed: {e}.\nYou can " - f"use 'tools.deployer:symlinks' conf to disable symlinks") - return file_count - - -def _deploy_single(dep, conanfile, output_folder, folder_name): + for lib in cpp_info.libs: + if _copy_pattern(f"lib{lib}.so*", libdir, output_folder, keep_symlinks): + copied_libs.add(lib) + if _copy_pattern(f"lib{lib}.dylib", libdir, output_folder, keep_symlinks): + copied_libs.add(lib) + + output.info(f"Copied {len(copied_libs)} shared libraries from {dep.ref}: " + + ", ".join(sorted(copied_libs))) + not_found = copied_libs - set(cpp_info.libs) + if not_found: + output.error(f"Some {dep.ref} libraries were not found: " + + ", ".join(sorted(not_found))) + conanfile.output.success(f"Shared libraries deployed to folder: {output_folder}") + + +def _deploy_package_folder(dep, conanfile, output_folder, folder_name, output): new_folder = os.path.join(output_folder, folder_name) rmdir(new_folder) + _copytree(dep.package_folder, new_folder, conanfile, dep, output) + dep.set_deploy_folder(new_folder) + + +def _copytree(src, dst, conanfile, dep, output, **kwargs): symlinks = conanfile.conf.get("tools.deployer:symlinks", check_type=bool, default=True) try: - shutil.copytree(dep.package_folder, new_folder, symlinks=symlinks) + shutil.copytree(src, dst, symlinks=symlinks, **kwargs) except Exception as e: if "WinError 1314" in str(e): - ConanOutput().error("full_deploy: Symlinks in Windows require admin privileges " - "or 'Developer mode = ON'", error_type="exception") - raise ConanException(f"full_deploy: The copy of '{dep}' files failed: {e}.\nYou can " - f"use 'tools.deployer:symlinks' conf to disable symlinks") - dep.set_deploy_folder(new_folder) + output.error("Symlinks on Windows require admin privileges or 'Developer mode = ON'", + error_type="exception") + err = f"{output.scope}: Copying of '{dep}' files failed: {e}." + if symlinks: + err += "\nYou can use 'tools.deployer:symlinks' conf to disable symlinks" + raise ConanException(err) -def direct_deploy(graph, output_folder): +def _copy_pattern(pattern, src_dir, output_dir, keep_symlinks): """ - Deploys to output_folder a single package, + Copies all files matching the pattern from src_dir to output_dir. + Existing files are overwritten. """ - # TODO: This deployer needs to be put somewhere else - # TODO: Document that this will NOT work with editables - output_folder = os.path.join(output_folder, "direct_deploy") - conanfile = graph.root.conanfile - conanfile.output.info(f"Conan built-in pkg deployer to {output_folder}") - # If the argument is --requires, the current conanfile is a virtual one with 1 single - # dependency, the "reference" package. If the argument is a local path, then all direct - # dependencies - for dep in conanfile.dependencies.filter({"direct": True}).values(): - _deploy_single(dep, conanfile, output_folder, dep.ref.name) + file_count = 0 + output = ConanOutput(scope="deploy_shared") + for src in glob.glob(os.path.join(src_dir, pattern)): + dst = os.path.join(output_dir, os.path.basename(src)) + try: + if os.path.lexists(dst): + os.remove(dst) + shutil.copy2(src, dst, follow_symlinks=not keep_symlinks) + if os.path.islink(dst): + # Explicitly copy symlink targets since these don't always match the library name. + # E.g. libpng.so -> libpng16.so -> libpng16.so.16 -> libpng16.so.16.43.0 + _copy_symlinks(src, output_dir) + output.verbose(f"Copied {src}") + file_count += 1 + except Exception as e: + raise ConanException(f"{output.scope}: Copying of '{src}' to '{dst}' failed: {e}.") + return file_count + + +def _copy_symlinks(src, output_dir): + rel_link_target = os.readlink(src) + if os.path.normpath(rel_link_target) != os.path.basename(rel_link_target): + raise ConanException(f"Refusing to copy a symlink '{src}' targeting an external folder: {rel_link_target}") + link_target = os.path.join(os.path.dirname(src), rel_link_target) + link_dst = os.path.join(output_dir, rel_link_target) + if not os.path.lexists(link_dst): + shutil.copy2(link_target, link_dst, follow_symlinks=False) + if os.path.islink(link_dst): + _copy_symlinks(link_target, output_dir)