From 7073730b2fb767270b362073abd84e925396c0c5 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Wed, 28 Feb 2024 10:47:04 +0100 Subject: [PATCH 01/39] conan create remote WIP --- Dockerfile | 8 +++ conan/cli/commands/create.py | 90 +++++++++++++++++++++++++++++++++ conans/client/profile_loader.py | 16 +++++- conans/model/profile.py | 2 + 4 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..a3209e5acf0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu + +RUN apt update && apt upgrade -y +RUN apt install -y build-essential +RUN apt install -y python3-pip cmake +RUN pip3 install conan +ADD . /root/conan-io +RUN cd /root/conan-io && pip install -e . diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index d5b2e186eea..c3216832d3b 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -1,5 +1,7 @@ import os import shutil +import json +from io import BytesIO from conan.api.output import ConanOutput from conan.cli.args import add_lockfile_args, add_common_install_arguments @@ -11,6 +13,8 @@ from conan.errors import ConanException from conans.client.graph.graph import BINARY_BUILD from conans.util.files import mkdir +from conan.api.model import ListPattern +from conan.api.conan_api import ConfigAPI @conan_command(group="Creator", formatters={"json": format_graph_json}) @@ -33,6 +37,7 @@ def create(conan_api, parser, *args): parser.add_argument("-bt", "--build-test", action="append", help="Same as '--build' but only for the test_package requires. By default" " if not specified it will take the '--build' value if specified") + raw_args = args[0] args = parser.parse_args(*args) if args.test_missing and args.test_folder == "": @@ -62,6 +67,10 @@ def create(conan_api, parser, *args): lockfile = conan_api.lockfile.update_lockfile_export(lockfile, conanfile, ref, is_build) print_profiles(profile_host, profile_build) + if not os.environ.get("CONAN_REMOTE_ENVIRONMNET") and (profile_host.remote and profile_host.remote.get('remote') == 'docker'): + return _docker_runner(conan_api, profile_host, args, raw_args) + + if args.build is not None and args.build_test is None: args.build_test = args.build @@ -178,3 +187,84 @@ def _get_test_conanfile_path(tf, conanfile_path): return test_conanfile_path elif tf: raise ConanException(f"test folder '{tf}' not available, or it doesn't have a conanfile.py") + + +def _docker_runner(conan_api, profile, args, raw_args): + """ + run conan inside a Docker continer + """ + # import docker only if needed + import docker + docker_client = docker.from_env() + docker_api = docker.APIClient() + dockerfile = str(profile.remote.get('dockerfile', '')) + image = str(profile.remote.get('image', 'conanremote')) + + remote_home = os.path.join(args.path, '.conanremote') + tgz_path = os.path.join(remote_home, 'conan_cache_save.tgz') + volumes = { + args.path: {'bind': args.path, 'mode': 'rw'} + } + + environment = { + 'CONAN_REMOTE_WS': args.path, + 'CONAN_REMOTE_COMMAND': ' '.join(['conan create'] + raw_args), + 'CONAN_REMOTE_ENVIRONMNET': '1' + } + # https://docker-py.readthedocs.io/en/stable/api.html#module-docker.api.build + + ConanOutput().info(msg=f'\nBuilding the Docker image: {image}') + docker_build_logs = None + if dockerfile: + docker_build_logs = docker_api.build(path=dockerfile, tag=image) + else: + dockerfile = ''' +FROM ubuntu +RUN apt update && apt upgrade -y +RUN apt install -y build-essential +RUN apt install -y python3-pip cmake git +RUN cd /root && git clone https://github.com/davidsanfal/conan.git conan-io +RUN cd /root/conan-io && pip install -e . +''' + docker_build_logs = docker_api.build(fileobj=BytesIO(dockerfile.encode('utf-8')), tag=image) + for chunk in docker_build_logs: + for line in chunk.decode("utf-8").split('\r\n'): + if line: + stream = json.loads(line).get('stream') + if stream: + ConanOutput().info(stream.strip()) + + shutil.rmtree(remote_home, ignore_errors=True) + os.mkdir(remote_home) + conan_api.cache.save(conan_api.list.select(ListPattern("*")), tgz_path) + shutil.copytree(os.path.join(ConfigAPI(conan_api).home(), 'profiles'), os.path.join(remote_home, 'profiles')) + with open(os.path.join(remote_home, 'conan-remote-init.sh'), 'w+') as f: + f.writelines("""#!/bin/bash + +conan cache restore ${CONAN_REMOTE_WS}/.conanremote/conan_cache_save.tgz +mkdir ${HOME}/.conan2/profiles +cp -r ${CONAN_REMOTE_WS}/.conanremote/profiles/. -r ${HOME}/.conan2/profiles/. + +echo "Running: ${CONAN_REMOTE_COMMAND}" +eval "${CONAN_REMOTE_COMMAND}" + +conan cache save "*" --file ${CONAN_REMOTE_WS}/.conanremote/conan_cache_docker.tgz""") + + # Init docker python api + ConanOutput().info(msg=f'\Running the Docker container\n') + container = docker_client.containers.run(image, + f'/bin/bash {os.path.join(remote_home, "conan-remote-init.sh")}', + volumes=volumes, + environment=environment, + detach=True) + for line in container.attach(stdout=True, stream=True, logs=True): + ConanOutput().info(line.decode('utf-8', errors='ignore').strip()) + container.wait() + container.stop() + container.remove() + + tgz_path = os.path.join(remote_home, 'conan_cache_docker.tgz') + ConanOutput().info(f'New cache path: {tgz_path}') + package_list = conan_api.cache.restore(tgz_path) + return {"graph": {}, + "conan_api": conan_api} diff --git a/conans/client/profile_loader.py b/conans/client/profile_loader.py index fc83a734416..1d74cbb0fab 100644 --- a/conans/client/profile_loader.py +++ b/conans/client/profile_loader.py @@ -230,7 +230,8 @@ def get_profile(profile_text, base_profile=None): "platform_requires", "platform_tool_requires", "settings", "options", "conf", "buildenv", "runenv", - "replace_requires", "replace_tool_requires"]) + "replace_requires", "replace_tool_requires", + "remote"]) # Parse doc sections into Conan model, Settings, Options, etc settings, package_settings = _ProfileValueParser._parse_settings(doc) @@ -302,8 +303,21 @@ def load_replace(doc_replace_requires): base_profile.buildenv.update_profile_env(buildenv) if runenv is not None: base_profile.runenv.update_profile_env(runenv) + + + remote_info = _ProfileValueParser._parse_key_value(doc.remote) if doc.remote else {} + base_profile.remote.update(remote_info) return base_profile + @staticmethod + def _parse_key_value(raw_info): + result = OrderedDict() + for br_line in raw_info.splitlines(): + tokens = br_line.split("=", 1) + pattern, req_list = tokens + result[pattern] = req_list + return result + @staticmethod def _parse_tool_requires(doc): result = OrderedDict() diff --git a/conans/model/profile.py b/conans/model/profile.py index 7654ed9acf3..e319bea9eab 100644 --- a/conans/model/profile.py +++ b/conans/model/profile.py @@ -24,6 +24,7 @@ def __init__(self): self.conf = ConfDefinition() self.buildenv = ProfileEnvironment() self.runenv = ProfileEnvironment() + self.remote = {} # Cached processed values self.processed_settings = None # Settings with values, and smart completion @@ -124,6 +125,7 @@ def compose_profile(self, other): self.replace_requires.update(other.replace_requires) self.replace_tool_requires.update(other.replace_tool_requires) + self.remote.update(other.remote) current_platform_tool_requires = {r.name: r for r in self.platform_tool_requires} current_platform_tool_requires.update({r.name: r for r in other.platform_tool_requires}) From d852ca85d71cc7b7a5931e845f674ceafd64efcd Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Wed, 28 Feb 2024 16:27:51 +0100 Subject: [PATCH 02/39] runners added --- conan/cli/commands/create.py | 95 ++----------------------- conan/internal/runner/__init__.py | 0 conan/internal/runner/docker.py | 113 ++++++++++++++++++++++++++++++ conan/internal/runner/ssh.py | 3 + conans/client/profile_loader.py | 6 +- conans/model/profile.py | 4 +- 6 files changed, 128 insertions(+), 93 deletions(-) create mode 100644 conan/internal/runner/__init__.py create mode 100644 conan/internal/runner/docker.py create mode 100644 conan/internal/runner/ssh.py diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index c3216832d3b..9359425a295 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -1,7 +1,5 @@ import os import shutil -import json -from io import BytesIO from conan.api.output import ConanOutput from conan.cli.args import add_lockfile_args, add_common_install_arguments @@ -12,9 +10,9 @@ from conan.cli.printers.graph import print_graph_packages, print_graph_basic from conan.errors import ConanException from conans.client.graph.graph import BINARY_BUILD +from conan.internal.runner.docker import DockerRunner +from conan.internal.runner.ssh import SSHRunner from conans.util.files import mkdir -from conan.api.model import ListPattern -from conan.api.conan_api import ConfigAPI @conan_command(group="Creator", formatters={"json": format_graph_json}) @@ -67,9 +65,11 @@ def create(conan_api, parser, *args): lockfile = conan_api.lockfile.update_lockfile_export(lockfile, conanfile, ref, is_build) print_profiles(profile_host, profile_build) - if not os.environ.get("CONAN_REMOTE_ENVIRONMNET") and (profile_host.remote and profile_host.remote.get('remote') == 'docker'): - return _docker_runner(conan_api, profile_host, args, raw_args) - + if profile_host.runner and not os.environ.get("CONAN_REMOTE_ENVIRONMNET"): + return { + 'docker': DockerRunner, + 'ssh': SSHRunner + }[profile_host.runner.get('type')](conan_api, profile_host, args, raw_args).run() if args.build is not None and args.build_test is None: args.build_test = args.build @@ -187,84 +187,3 @@ def _get_test_conanfile_path(tf, conanfile_path): return test_conanfile_path elif tf: raise ConanException(f"test folder '{tf}' not available, or it doesn't have a conanfile.py") - - -def _docker_runner(conan_api, profile, args, raw_args): - """ - run conan inside a Docker continer - """ - # import docker only if needed - import docker - docker_client = docker.from_env() - docker_api = docker.APIClient() - dockerfile = str(profile.remote.get('dockerfile', '')) - image = str(profile.remote.get('image', 'conanremote')) - - remote_home = os.path.join(args.path, '.conanremote') - tgz_path = os.path.join(remote_home, 'conan_cache_save.tgz') - volumes = { - args.path: {'bind': args.path, 'mode': 'rw'} - } - - environment = { - 'CONAN_REMOTE_WS': args.path, - 'CONAN_REMOTE_COMMAND': ' '.join(['conan create'] + raw_args), - 'CONAN_REMOTE_ENVIRONMNET': '1' - } - # https://docker-py.readthedocs.io/en/stable/api.html#module-docker.api.build - - ConanOutput().info(msg=f'\nBuilding the Docker image: {image}') - docker_build_logs = None - if dockerfile: - docker_build_logs = docker_api.build(path=dockerfile, tag=image) - else: - dockerfile = ''' -FROM ubuntu -RUN apt update && apt upgrade -y -RUN apt install -y build-essential -RUN apt install -y python3-pip cmake git -RUN cd /root && git clone https://github.com/davidsanfal/conan.git conan-io -RUN cd /root/conan-io && pip install -e . -''' - docker_build_logs = docker_api.build(fileobj=BytesIO(dockerfile.encode('utf-8')), tag=image) - for chunk in docker_build_logs: - for line in chunk.decode("utf-8").split('\r\n'): - if line: - stream = json.loads(line).get('stream') - if stream: - ConanOutput().info(stream.strip()) - - shutil.rmtree(remote_home, ignore_errors=True) - os.mkdir(remote_home) - conan_api.cache.save(conan_api.list.select(ListPattern("*")), tgz_path) - shutil.copytree(os.path.join(ConfigAPI(conan_api).home(), 'profiles'), os.path.join(remote_home, 'profiles')) - with open(os.path.join(remote_home, 'conan-remote-init.sh'), 'w+') as f: - f.writelines("""#!/bin/bash - -conan cache restore ${CONAN_REMOTE_WS}/.conanremote/conan_cache_save.tgz -mkdir ${HOME}/.conan2/profiles -cp -r ${CONAN_REMOTE_WS}/.conanremote/profiles/. -r ${HOME}/.conan2/profiles/. - -echo "Running: ${CONAN_REMOTE_COMMAND}" -eval "${CONAN_REMOTE_COMMAND}" - -conan cache save "*" --file ${CONAN_REMOTE_WS}/.conanremote/conan_cache_docker.tgz""") - - # Init docker python api - ConanOutput().info(msg=f'\Running the Docker container\n') - container = docker_client.containers.run(image, - f'/bin/bash {os.path.join(remote_home, "conan-remote-init.sh")}', - volumes=volumes, - environment=environment, - detach=True) - for line in container.attach(stdout=True, stream=True, logs=True): - ConanOutput().info(line.decode('utf-8', errors='ignore').strip()) - container.wait() - container.stop() - container.remove() - - tgz_path = os.path.join(remote_home, 'conan_cache_docker.tgz') - ConanOutput().info(f'New cache path: {tgz_path}') - package_list = conan_api.cache.restore(tgz_path) - return {"graph": {}, - "conan_api": conan_api} diff --git a/conan/internal/runner/__init__.py b/conan/internal/runner/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py new file mode 100644 index 00000000000..20362dbb952 --- /dev/null +++ b/conan/internal/runner/docker.py @@ -0,0 +1,113 @@ +import os +import json +from io import BytesIO +import textwrap +import shutil + +from conan.api.model import ListPattern +from conan.api.output import ConanOutput +from conan.api.conan_api import ConfigAPI + + +class DockerRunner: + + def __init__(self, conan_api, profile, args, raw_args): + import docker + self.conan_api = conan_api + self.args = args + self.raw_args = raw_args + self.docker_client = docker.from_env() + self.docker_api = docker.APIClient() + self.dockerfile = str(profile.runner.get('dockerfile', '')) + self.image = str(profile.runner.get('image', 'conanrunner')) + self.cache = str(profile.runner.get('cache', 'copy')) + self.runner_home = os.path.join(args.path, '.conanrunner') + + def run(self): + """ + run conan inside a Docker continer + """ + ConanOutput().info(msg=f'\nBuilding the Docker image: {self.image}') + self.build_image() + ConanOutput().info(msg=f'\nInit container resources') + command, volumes, environment = self.create_runner_environment() + # Init docker python api + ConanOutput().info(msg=f'\nRunning the Docker container\n') + container = self.docker_client.containers.run(self.image, + command, + volumes=volumes, + environment=environment, + detach=True) + for line in container.attach(stdout=True, stream=True, logs=True): + ConanOutput().info(line.decode('utf-8', errors='ignore').strip()) + container.wait() + container.stop() + container.remove() + self.update_local_cache() + + def build_image(self): + docker_build_logs = None + if self.dockerfile: + docker_build_logs = self.docker_api.build(path=self.dockerfile, tag=self.image) + else: + dockerfile = textwrap.dedent(""" + FROM ubuntu + RUN apt update && apt upgrade -y + RUN apt install -y build-essential + RUN apt install -y python3-pip cmake git + RUN cd /root && git clone https://github.com/davidsanfal/conan.git conan-io + RUN cd /root/conan-io && pip install -e . + """) + + docker_build_logs = self.docker_api.build(fileobj=BytesIO(dockerfile.encode('utf-8')), + nocache=False, + tag=self.image) + for chunk in docker_build_logs: + for line in chunk.decode("utf-8").split('\r\n'): + if line: + stream = json.loads(line).get('stream') + if stream: + ConanOutput().info(stream.strip()) + + def create_runner_environment(self): + volumes = {self.args.path: {'bind': self.args.path, 'mode': 'rw'}} + environment = {'CONAN_REMOTE_ENVIRONMNET': '1'} + + if self.cache == 'shared': + volumes[ConfigAPI(self.conan_api).home()] = {'bind': '/root/.conan2', 'mode': 'rw'} + command = ' '.join(['conan create'] + self.raw_args) + + if self.cache in ['clean', 'copy']: + shutil.rmtree(self.runner_home, ignore_errors=True) + os.mkdir(self.runner_home) + shutil.copytree(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), os.path.join(self.runner_home, 'profiles')) + environment['CONAN_REMOTE_COMMAND'] = ' '.join(['conan create'] + self.raw_args) + environment['CONAN_REMOTE_WS'] = self.args.path + command = f'/bin/bash {os.path.join(self.runner_home, "conan-runner-init.sh")}' + conan_runner_init = textwrap.dedent(""" + mkdir -p ${HOME}/.conan2/profiles + echo "Updating profiles ..." + cp -r ${CONAN_REMOTE_WS}/.conanrunner/profiles/. -r ${HOME}/.conan2/profiles/. + echo "Running: ${CONAN_REMOTE_COMMAND}" + eval "${CONAN_REMOTE_COMMAND}" + """) + if self.cache == 'copy': + tgz_path = os.path.join(self.runner_home, 'conan_cache_save.tgz') + self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*")), tgz_path) + conan_runner_init = textwrap.dedent(""" + conan cache restore ${CONAN_REMOTE_WS}/.conanrunner/conan_cache_save.tgz + """) + conan_runner_init + textwrap.dedent(""" + conan cache save "*" --file ${CONAN_REMOTE_WS}/.conanrunner/conan_cache_docker.tgz + """) + with open(os.path.join(self.runner_home, 'conan-runner-init.sh'), 'w+') as f: + conan_runner_init = textwrap.dedent("""#!/bin/bash + """) + conan_runner_init + print(conan_runner_init) + f.writelines(conan_runner_init) + + return command, volumes, environment + + def update_local_cache(self): + if self.cache == 'copy': + tgz_path = os.path.join(self.runner_home, 'conan_cache_docker.tgz') + package_list = self.conan_api.cache.restore(tgz_path) diff --git a/conan/internal/runner/ssh.py b/conan/internal/runner/ssh.py new file mode 100644 index 00000000000..f52b4d54ed9 --- /dev/null +++ b/conan/internal/runner/ssh.py @@ -0,0 +1,3 @@ +class SSHRunner: + def __init__(self, conan_api, profile, args, raw_args): + pass diff --git a/conans/client/profile_loader.py b/conans/client/profile_loader.py index 1d74cbb0fab..3cac06f87f3 100644 --- a/conans/client/profile_loader.py +++ b/conans/client/profile_loader.py @@ -231,7 +231,7 @@ def get_profile(profile_text, base_profile=None): "platform_tool_requires", "settings", "options", "conf", "buildenv", "runenv", "replace_requires", "replace_tool_requires", - "remote"]) + "runner"]) # Parse doc sections into Conan model, Settings, Options, etc settings, package_settings = _ProfileValueParser._parse_settings(doc) @@ -305,8 +305,8 @@ def load_replace(doc_replace_requires): base_profile.runenv.update_profile_env(runenv) - remote_info = _ProfileValueParser._parse_key_value(doc.remote) if doc.remote else {} - base_profile.remote.update(remote_info) + runner = _ProfileValueParser._parse_key_value(doc.runner) if doc.runner else {} + base_profile.runner.update(runner) return base_profile @staticmethod diff --git a/conans/model/profile.py b/conans/model/profile.py index e319bea9eab..6160afd84d4 100644 --- a/conans/model/profile.py +++ b/conans/model/profile.py @@ -24,7 +24,7 @@ def __init__(self): self.conf = ConfDefinition() self.buildenv = ProfileEnvironment() self.runenv = ProfileEnvironment() - self.remote = {} + self.runner = {} # Cached processed values self.processed_settings = None # Settings with values, and smart completion @@ -125,7 +125,7 @@ def compose_profile(self, other): self.replace_requires.update(other.replace_requires) self.replace_tool_requires.update(other.replace_tool_requires) - self.remote.update(other.remote) + self.runner.update(other.runner) current_platform_tool_requires = {r.name: r for r in self.platform_tool_requires} current_platform_tool_requires.update({r.name: r for r in other.platform_tool_requires}) From acc000cd991700d242fb9c350851412e2923389c Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Mon, 4 Mar 2024 18:15:46 +0100 Subject: [PATCH 03/39] docker runner updated --- Dockerfile | 8 --- conan/cli/commands/create.py | 4 +- conan/internal/runner/docker.py | 122 ++++++++++++++++++-------------- conan/internal/runner/ssh.py | 10 ++- 4 files changed, 79 insertions(+), 65 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a3209e5acf0..00000000000 --- a/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM ubuntu - -RUN apt update && apt upgrade -y -RUN apt install -y build-essential -RUN apt install -y python3-pip cmake -RUN pip3 install conan -ADD . /root/conan-io -RUN cd /root/conan-io && pip install -e . diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 9359425a295..34fa20f3eec 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -65,11 +65,11 @@ def create(conan_api, parser, *args): lockfile = conan_api.lockfile.update_lockfile_export(lockfile, conanfile, ref, is_build) print_profiles(profile_host, profile_build) - if profile_host.runner and not os.environ.get("CONAN_REMOTE_ENVIRONMNET"): + if profile_build.runner and not os.environ.get("CONAN_REMOTE_ENVIRONMNET"): return { 'docker': DockerRunner, 'ssh': SSHRunner - }[profile_host.runner.get('type')](conan_api, profile_host, args, raw_args).run() + }[profile_build.runner.get('type')](conan_api, 'create', profile_host, args, raw_args).run() if args.build is not None and args.build_test is None: args.build_test = args.build diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 20362dbb952..f24fe0fb768 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -3,6 +3,7 @@ from io import BytesIO import textwrap import shutil +import docker from conan.api.model import ListPattern from conan.api.output import ConanOutput @@ -11,39 +12,55 @@ class DockerRunner: - def __init__(self, conan_api, profile, args, raw_args): - import docker + def __init__(self, conan_api, command, profile, args, raw_args): self.conan_api = conan_api self.args = args self.raw_args = raw_args self.docker_client = docker.from_env() self.docker_api = docker.APIClient() + self.command = command self.dockerfile = str(profile.runner.get('dockerfile', '')) self.image = str(profile.runner.get('image', 'conanrunner')) - self.cache = str(profile.runner.get('cache', 'copy')) + self.suffix = profile.runner.get('suffix', args.profile_host[0] if args.profile_host else 'docker') + self.remove = profile.runner.get('remove', True) + self.cache = str(profile.runner.get('cache', 'clean')) self.runner_home = os.path.join(args.path, '.conanrunner') + self.container = None - def run(self): + def run(self, use_cache=True): """ run conan inside a Docker continer """ ConanOutput().info(msg=f'\nBuilding the Docker image: {self.image}') self.build_image() - ConanOutput().info(msg=f'\nInit container resources') - command, volumes, environment = self.create_runner_environment() # Init docker python api - ConanOutput().info(msg=f'\nRunning the Docker container\n') - container = self.docker_client.containers.run(self.image, - command, - volumes=volumes, - environment=environment, - detach=True) - for line in container.attach(stdout=True, stream=True, logs=True): - ConanOutput().info(line.decode('utf-8', errors='ignore').strip()) - container.wait() - container.stop() - container.remove() - self.update_local_cache() + name = None if self.remove else f'conan-runner-{self.suffix}' + volumes, environment = self.create_runner_environment(use_cache) + if name: + ConanOutput().info(msg=f'\nRunning the Docker container: "{name}"\n') + try: + self.container = self.docker_client.containers.run(self.image, + "/bin/bash -c 'while true; do sleep 30; done;'", + name=name, + volumes=volumes, + environment=environment, + detach=True, + auto_remove=False) + self.init_container(use_cache) + except docker.errors.APIError as e: + if self.remove: + raise e + self.container = docker_client.containers.get(name) + self.container.start() + + # for line in container.attach(stdout=True, stream=True, logs=True): + # ConanOutput().info(line.decode('utf-8', errors='ignore').strip()) + self.run_command(' '.join([f'conan {self.command}'] + self.raw_args)) + # container.wait() + self.update_local_cache(use_cache) + self.container.stop() + if self.remove: + self.container.remove() def build_image(self): docker_build_logs = None @@ -69,45 +86,42 @@ def build_image(self): if stream: ConanOutput().info(stream.strip()) - def create_runner_environment(self): + def run_command(self, command): + try: + ConanOutput().info(msg=f'RUNNING: "/bin/bash -c \'{command}\'"') + exec_stream = self.container.exec_run(f'/bin/bash -c \'{command}\'', stream=True, tty=True) + while True: + print(next(exec_stream.output).decode('utf-8', errors='ignore').strip()) + except StopIteration: + pass + + def create_runner_environment(self, use_cache): volumes = {self.args.path: {'bind': self.args.path, 'mode': 'rw'}} environment = {'CONAN_REMOTE_ENVIRONMNET': '1'} + if use_cache: + if self.cache == 'shared': + volumes[ConfigAPI(self.conan_api).home()] = {'bind': '/root/.conan2', 'mode': 'rw'} + if self.cache in ['clean', 'copy']: + shutil.rmtree(self.runner_home, ignore_errors=True) + os.mkdir(self.runner_home) + shutil.copytree(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), os.path.join(self.runner_home, 'profiles')) + if self.cache == 'copy': + tgz_path = os.path.join(self.runner_home, 'conan_cache_save.tgz') + self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*")), tgz_path) + return volumes, environment - if self.cache == 'shared': - volumes[ConfigAPI(self.conan_api).home()] = {'bind': '/root/.conan2', 'mode': 'rw'} - command = ' '.join(['conan create'] + self.raw_args) - - if self.cache in ['clean', 'copy']: - shutil.rmtree(self.runner_home, ignore_errors=True) - os.mkdir(self.runner_home) - shutil.copytree(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), os.path.join(self.runner_home, 'profiles')) - environment['CONAN_REMOTE_COMMAND'] = ' '.join(['conan create'] + self.raw_args) - environment['CONAN_REMOTE_WS'] = self.args.path - command = f'/bin/bash {os.path.join(self.runner_home, "conan-runner-init.sh")}' - conan_runner_init = textwrap.dedent(""" - mkdir -p ${HOME}/.conan2/profiles - echo "Updating profiles ..." - cp -r ${CONAN_REMOTE_WS}/.conanrunner/profiles/. -r ${HOME}/.conan2/profiles/. - echo "Running: ${CONAN_REMOTE_COMMAND}" - eval "${CONAN_REMOTE_COMMAND}" - """) - if self.cache == 'copy': - tgz_path = os.path.join(self.runner_home, 'conan_cache_save.tgz') - self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*")), tgz_path) - conan_runner_init = textwrap.dedent(""" - conan cache restore ${CONAN_REMOTE_WS}/.conanrunner/conan_cache_save.tgz - """) + conan_runner_init + textwrap.dedent(""" - conan cache save "*" --file ${CONAN_REMOTE_WS}/.conanrunner/conan_cache_docker.tgz - """) - with open(os.path.join(self.runner_home, 'conan-runner-init.sh'), 'w+') as f: - conan_runner_init = textwrap.dedent("""#!/bin/bash - """) + conan_runner_init - print(conan_runner_init) - f.writelines(conan_runner_init) - - return command, volumes, environment + def init_container(self, use_cache): + if use_cache: + if self.cache in ['clean', 'copy']: + self.run_command('mkdir -p ${HOME}/.conan2/profiles') + self.run_command('cp -r '+self.args.path+'/.conanrunner/profiles/. ${HOME}/.conan2/profiles/.') + if self.cache == 'copy': + self.run_command('conan cache restore '+self.args.path+'/.conanrunner/conan_cache_save.tgz') + self.run_command('mkdir -p ${HOME}/.conan2/profiles') + self.run_command('cp -r '+self.args.path+'/.conanrunner/profiles/. -r ${HOME}/.conan2/profiles/.') - def update_local_cache(self): - if self.cache == 'copy': + def update_local_cache(self, use_cache): + if use_cache and self.cache in ['copy', 'clean']: + self.run_command('conan cache save "*" --file '+self.args.path+'/.conanrunner/conan_cache_docker.tgz') tgz_path = os.path.join(self.runner_home, 'conan_cache_docker.tgz') package_list = self.conan_api.cache.restore(tgz_path) diff --git a/conan/internal/runner/ssh.py b/conan/internal/runner/ssh.py index f52b4d54ed9..430ac259a5e 100644 --- a/conan/internal/runner/ssh.py +++ b/conan/internal/runner/ssh.py @@ -1,3 +1,11 @@ class SSHRunner: - def __init__(self, conan_api, profile, args, raw_args): + + def __init__(self, conan_api, command, profile, args, raw_args): + self.conan_api = conan_api + self.command = command + self.profile = profile + self.args = args + self.raw_args = raw_args + + def run(self, use_cache=True): pass From 284e62edfffc2ee8acb47f3d1981e8abd7304d47 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Tue, 5 Mar 2024 17:56:29 +0100 Subject: [PATCH 04/39] docker runner updated --- conan/cli/commands/create.py | 2 +- conan/internal/runner/docker.py | 111 +++++++++++++++----------------- 2 files changed, 53 insertions(+), 60 deletions(-) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 34fa20f3eec..8dae971e656 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -65,7 +65,7 @@ def create(conan_api, parser, *args): lockfile = conan_api.lockfile.update_lockfile_export(lockfile, conanfile, ref, is_build) print_profiles(profile_host, profile_build) - if profile_build.runner and not os.environ.get("CONAN_REMOTE_ENVIRONMNET"): + if profile_build.runner and not os.environ.get("CONAN_RUNNER_ENVIRONMENT"): return { 'docker': DockerRunner, 'ssh': SSHRunner diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index f24fe0fb768..631cfd6d24c 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -21,46 +21,41 @@ def __init__(self, conan_api, command, profile, args, raw_args): self.command = command self.dockerfile = str(profile.runner.get('dockerfile', '')) self.image = str(profile.runner.get('image', 'conanrunner')) - self.suffix = profile.runner.get('suffix', args.profile_host[0] if args.profile_host else 'docker') - self.remove = profile.runner.get('remove', True) + self.name = f'conan-runner-{profile.runner.get("suffix", "docker")}' + self.remove = int(profile.runner.get('remove', 1)) self.cache = str(profile.runner.get('cache', 'clean')) self.runner_home = os.path.join(args.path, '.conanrunner') self.container = None - def run(self, use_cache=True): + def run(self): """ run conan inside a Docker continer """ - ConanOutput().info(msg=f'\nBuilding the Docker image: {self.image}') + ConanOutput().title(msg=f'Building the Docker image: {self.image}') self.build_image() - # Init docker python api - name = None if self.remove else f'conan-runner-{self.suffix}' - volumes, environment = self.create_runner_environment(use_cache) - if name: - ConanOutput().info(msg=f'\nRunning the Docker container: "{name}"\n') + volumes, environment = self.create_runner_environment() try: - self.container = self.docker_client.containers.run(self.image, - "/bin/bash -c 'while true; do sleep 30; done;'", - name=name, - volumes=volumes, - environment=environment, - detach=True, - auto_remove=False) - self.init_container(use_cache) - except docker.errors.APIError as e: - if self.remove: - raise e - self.container = docker_client.containers.get(name) - self.container.start() - - # for line in container.attach(stdout=True, stream=True, logs=True): - # ConanOutput().info(line.decode('utf-8', errors='ignore').strip()) - self.run_command(' '.join([f'conan {self.command}'] + self.raw_args)) - # container.wait() - self.update_local_cache(use_cache) - self.container.stop() - if self.remove: - self.container.remove() + if self.docker_client.containers.list(all=True, filters={'name': self.name}): + self.container = self.docker_client.containers.get(self.name) + self.container.start() + else: + self.container = self.docker_client.containers.run(self.image, + "/bin/bash -c 'while true; do sleep 30; done;'", + name=self.name, + volumes=volumes, + environment=environment, + detach=True, + auto_remove=False) + self.init_container() + self.run_command(' '.join([f'conan {self.command}'] + self.raw_args)) + self.update_local_cache() + except: + pass + finally: + if self.container: + self.container.stop() + if self.remove: + self.container.remove() def build_image(self): docker_build_logs = None @@ -73,7 +68,7 @@ def build_image(self): RUN apt install -y build-essential RUN apt install -y python3-pip cmake git RUN cd /root && git clone https://github.com/davidsanfal/conan.git conan-io - RUN cd /root/conan-io && pip install -e . + RUN cd /root/conan-io && pip install docker && pip install -e . """) docker_build_logs = self.docker_api.build(fileobj=BytesIO(dockerfile.encode('utf-8')), @@ -86,42 +81,40 @@ def build_image(self): if stream: ConanOutput().info(stream.strip()) - def run_command(self, command): + def run_command(self, command, log=True): try: - ConanOutput().info(msg=f'RUNNING: "/bin/bash -c \'{command}\'"') exec_stream = self.container.exec_run(f'/bin/bash -c \'{command}\'', stream=True, tty=True) while True: - print(next(exec_stream.output).decode('utf-8', errors='ignore').strip()) + output = next(exec_stream.output) + if log: + ConanOutput().info(output.decode('utf-8', errors='ignore').strip()) except StopIteration: pass - def create_runner_environment(self, use_cache): + def create_runner_environment(self): volumes = {self.args.path: {'bind': self.args.path, 'mode': 'rw'}} - environment = {'CONAN_REMOTE_ENVIRONMNET': '1'} - if use_cache: - if self.cache == 'shared': - volumes[ConfigAPI(self.conan_api).home()] = {'bind': '/root/.conan2', 'mode': 'rw'} - if self.cache in ['clean', 'copy']: - shutil.rmtree(self.runner_home, ignore_errors=True) - os.mkdir(self.runner_home) - shutil.copytree(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), os.path.join(self.runner_home, 'profiles')) - if self.cache == 'copy': - tgz_path = os.path.join(self.runner_home, 'conan_cache_save.tgz') - self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*")), tgz_path) + environment = {'CONAN_RUNNER_ENVIRONMENT': '1'} + if self.cache == 'shared': + volumes[ConfigAPI(self.conan_api).home()] = {'bind': '/root/.conan2', 'mode': 'rw'} + if self.cache in ['clean', 'copy']: + shutil.rmtree(self.runner_home, ignore_errors=True) + os.mkdir(self.runner_home) + shutil.copytree(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), os.path.join(self.runner_home, 'profiles')) + if self.cache == 'copy': + tgz_path = os.path.join(self.runner_home, 'conan_cache_save.tgz') + self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path) return volumes, environment - def init_container(self, use_cache): - if use_cache: - if self.cache in ['clean', 'copy']: - self.run_command('mkdir -p ${HOME}/.conan2/profiles') - self.run_command('cp -r '+self.args.path+'/.conanrunner/profiles/. ${HOME}/.conan2/profiles/.') - if self.cache == 'copy': - self.run_command('conan cache restore '+self.args.path+'/.conanrunner/conan_cache_save.tgz') - self.run_command('mkdir -p ${HOME}/.conan2/profiles') - self.run_command('cp -r '+self.args.path+'/.conanrunner/profiles/. -r ${HOME}/.conan2/profiles/.') + def init_container(self): + if self.cache != 'shared': + self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) + self.run_command('cp -r '+self.args.path+'/.conanrunner/profiles/. ${HOME}/.conan2/profiles/.', log=False) + if self.cache in ['copy', 'clean']: + self.run_command('conan cache restore '+self.args.path+'/.conanrunner/conan_cache_save.tgz') - def update_local_cache(self, use_cache): - if use_cache and self.cache in ['copy', 'clean']: - self.run_command('conan cache save "*" --file '+self.args.path+'/.conanrunner/conan_cache_docker.tgz') + def update_local_cache(self): + if self.cache != 'shared': + self.run_command('conan cache save "*:*" --file '+self.args.path+'/.conanrunner/conan_cache_docker.tgz') tgz_path = os.path.join(self.runner_home, 'conan_cache_docker.tgz') + ConanOutput().subtitle(msg=f'Copy conan cache from runner') package_list = self.conan_api.cache.restore(tgz_path) From 9175aa66539a8ec607a549bfb91649951660eb36 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Wed, 6 Mar 2024 12:27:02 +0100 Subject: [PATCH 05/39] abs path added --- conan/internal/runner/docker.py | 46 +++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 631cfd6d24c..129bdcf6ebd 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -3,30 +3,35 @@ from io import BytesIO import textwrap import shutil -import docker - from conan.api.model import ListPattern from conan.api.output import ConanOutput from conan.api.conan_api import ConfigAPI +from conan.cli import make_abs_path -class DockerRunner: +def docker_info(msg): + ConanOutput().highlight('\n'+'-'*(2+len(msg))) + ConanOutput().highlight(f' {msg} ') + ConanOutput().highlight('-'*(2+len(msg))+'\n') + +class DockerRunner: def __init__(self, conan_api, command, profile, args, raw_args): + import docker self.conan_api = conan_api - self.args = args - self.raw_args = raw_args + self.abs_host_path = make_abs_path(args.path) + self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner') + self.abs_docker_path = os.path.join('/root/conanrunner', os.path.basename(self.abs_host_path)) self.docker_client = docker.from_env() self.docker_api = docker.APIClient() - self.command = command + raw_args[raw_args.index(args.path)] = self.abs_docker_path + self.command = ' '.join([f'conan {command}'] + raw_args) self.dockerfile = str(profile.runner.get('dockerfile', '')) self.image = str(profile.runner.get('image', 'conanrunner')) self.name = f'conan-runner-{profile.runner.get("suffix", "docker")}' self.remove = int(profile.runner.get('remove', 1)) self.cache = str(profile.runner.get('cache', 'clean')) - self.runner_home = os.path.join(args.path, '.conanrunner') self.container = None - def run(self): """ run conan inside a Docker continer @@ -47,7 +52,7 @@ def run(self): detach=True, auto_remove=False) self.init_container() - self.run_command(' '.join([f'conan {self.command}'] + self.raw_args)) + self.run_command(self.command) self.update_local_cache() except: pass @@ -83,6 +88,7 @@ def build_image(self): def run_command(self, command, log=True): try: + docker_info(f'Running inside container: "{command}"') exec_stream = self.container.exec_run(f'/bin/bash -c \'{command}\'', stream=True, tty=True) while True: output = next(exec_stream.output) @@ -92,29 +98,31 @@ def run_command(self, command, log=True): pass def create_runner_environment(self): - volumes = {self.args.path: {'bind': self.args.path, 'mode': 'rw'}} + volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}} environment = {'CONAN_RUNNER_ENVIRONMENT': '1'} if self.cache == 'shared': volumes[ConfigAPI(self.conan_api).home()] = {'bind': '/root/.conan2', 'mode': 'rw'} if self.cache in ['clean', 'copy']: - shutil.rmtree(self.runner_home, ignore_errors=True) - os.mkdir(self.runner_home) - shutil.copytree(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), os.path.join(self.runner_home, 'profiles')) + shutil.rmtree(self.abs_runner_home_path, ignore_errors=True) + os.mkdir(self.abs_runner_home_path) + shutil.copytree(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), + os.path.join(self.abs_runner_home_path, 'profiles')) if self.cache == 'copy': - tgz_path = os.path.join(self.runner_home, 'conan_cache_save.tgz') + tgz_path = os.path.join(self.abs_runner_home_path, 'conan_cache_save.tgz') + docker_info(f'Save host cache in: {tgz_path}') self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path) return volumes, environment def init_container(self): if self.cache != 'shared': self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) - self.run_command('cp -r '+self.args.path+'/.conanrunner/profiles/. ${HOME}/.conan2/profiles/.', log=False) + self.run_command('cp -r '+self.abs_docker_path+'/.conanrunner/profiles/. ${HOME}/.conan2/profiles/.', log=False) if self.cache in ['copy', 'clean']: - self.run_command('conan cache restore '+self.args.path+'/.conanrunner/conan_cache_save.tgz') + self.run_command('conan cache restore '+self.abs_docker_path+'/.conanrunner/conan_cache_save.tgz') def update_local_cache(self): if self.cache != 'shared': - self.run_command('conan cache save "*:*" --file '+self.args.path+'/.conanrunner/conan_cache_docker.tgz') - tgz_path = os.path.join(self.runner_home, 'conan_cache_docker.tgz') - ConanOutput().subtitle(msg=f'Copy conan cache from runner') + self.run_command('conan cache save "*:*" --file '+self.abs_docker_path+'/.conanrunner/conan_cache_docker.tgz') + tgz_path = os.path.join(self.abs_runner_home_path, 'conan_cache_docker.tgz') + docker_info(f'Restore host cache from: {tgz_path}') package_list = self.conan_api.cache.restore(tgz_path) From 03d53ca550430ef5b3c80cb256229b1557f105f6 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Wed, 6 Mar 2024 12:41:02 +0100 Subject: [PATCH 06/39] linux slashes fixed --- conan/internal/runner/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 129bdcf6ebd..a5ba2a72bb1 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -21,7 +21,7 @@ def __init__(self, conan_api, command, profile, args, raw_args): self.conan_api = conan_api self.abs_host_path = make_abs_path(args.path) self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner') - self.abs_docker_path = os.path.join('/root/conanrunner', os.path.basename(self.abs_host_path)) + self.abs_docker_path = os.path.join('/root/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/") self.docker_client = docker.from_env() self.docker_api = docker.APIClient() raw_args[raw_args.index(args.path)] = self.abs_docker_path From 559428bcd086d80f4e5e1a951f66942d1c1bb3c2 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Wed, 6 Mar 2024 13:10:49 +0100 Subject: [PATCH 07/39] info updated --- conan/internal/runner/docker.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index a5ba2a72bb1..5bcd22425fc 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -7,23 +7,27 @@ from conan.api.output import ConanOutput from conan.api.conan_api import ConfigAPI from conan.cli import make_abs_path +from conans.errors import ConanException def docker_info(msg): - ConanOutput().highlight('\n'+'-'*(2+len(msg))) - ConanOutput().highlight(f' {msg} ') - ConanOutput().highlight('-'*(2+len(msg))+'\n') + ConanOutput().highlight('\n┌'+'─'*(2+len(msg))+'┐') + ConanOutput().highlight(f'| {msg} |') + ConanOutput().highlight('└'+'─'*(2+len(msg))+'┘\n') class DockerRunner: def __init__(self, conan_api, command, profile, args, raw_args): import docker + try: + self.docker_client = docker.from_env() + self.docker_api = docker.APIClient() + except: + raise ConanException("Docker Client failed to initialize. Check if it is installed and running") self.conan_api = conan_api self.abs_host_path = make_abs_path(args.path) self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner') self.abs_docker_path = os.path.join('/root/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/") - self.docker_client = docker.from_env() - self.docker_api = docker.APIClient() raw_args[raw_args.index(args.path)] = self.abs_docker_path self.command = ' '.join([f'conan {command}'] + raw_args) self.dockerfile = str(profile.runner.get('dockerfile', '')) @@ -36,7 +40,7 @@ def run(self): """ run conan inside a Docker continer """ - ConanOutput().title(msg=f'Building the Docker image: {self.image}') + docker_info(f'Building the Docker image: {self.image}') self.build_image() volumes, environment = self.create_runner_environment() try: From 62de035df75466990369b8280e68a02a93b3c7fb Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Wed, 6 Mar 2024 18:33:06 +0100 Subject: [PATCH 08/39] restore only new packages --- conan/internal/runner/docker.py | 51 ++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 5bcd22425fc..062dcc7ef07 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -16,6 +16,15 @@ def docker_info(msg): ConanOutput().highlight('└'+'─'*(2+len(msg))+'┘\n') +def list_patterns(cache_info): + _pattern = [] + for reference, info in cache_info.items(): + for revisions in info.get('revisions', {}).values(): + for package in revisions.get('packages').keys(): + _pattern.append(f'{reference}:{package}') + return _pattern + + class DockerRunner: def __init__(self, conan_api, command, profile, args, raw_args): import docker @@ -58,8 +67,8 @@ def run(self): self.init_container() self.run_command(self.command) self.update_local_cache() - except: - pass + except Exception as e: + print(e) finally: if self.container: self.container.stop() @@ -90,16 +99,21 @@ def build_image(self): if stream: ConanOutput().info(stream.strip()) - def run_command(self, command, log=True): - try: - docker_info(f'Running inside container: "{command}"') - exec_stream = self.container.exec_run(f'/bin/bash -c \'{command}\'', stream=True, tty=True) - while True: - output = next(exec_stream.output) - if log: - ConanOutput().info(output.decode('utf-8', errors='ignore').strip()) - except StopIteration: - pass + def run_command(self, command, log=True, stdout=True, stderr=True, stream=True, tty=True): + + if log: + docker_info(f'Running in container: "{command}"') + exec_run = self.container.exec_run(f'/bin/bash -c \'{command}\'', stdout=stdout, stderr=stderr, stream=stream, tty=tty) + if stream: + try: + while True: + chunk = next(exec_run.output).decode('utf-8', errors='ignore').strip() + if log: + ConanOutput().info(chunk) + except StopIteration: + pass + else: + return exec_run.output.decode('utf-8', errors='ignore').strip() def create_runner_environment(self): volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}} @@ -126,7 +140,12 @@ def init_container(self): def update_local_cache(self): if self.cache != 'shared': - self.run_command('conan cache save "*:*" --file '+self.abs_docker_path+'/.conanrunner/conan_cache_docker.tgz') - tgz_path = os.path.join(self.abs_runner_home_path, 'conan_cache_docker.tgz') - docker_info(f'Restore host cache from: {tgz_path}') - package_list = self.conan_api.cache.restore(tgz_path) + package_list = self.run_command('conan list "*:*" --format=json 2>/dev/null', log=False, stream=False) + docker_cache_list = json.loads(package_list).get('Local Cache') + local_cache_list = self.conan_api.list.select(ListPattern('*:*', rrev=None, prev=None), None, remote=None, lru=None, profile=None) + diff_pattern = list(set(list_patterns(docker_cache_list)).difference(list_patterns(local_cache_list.recipes))) + for pattern in diff_pattern: + self.run_command(f'conan cache save "{pattern}" --file '+self.abs_docker_path+f'/.conanrunner/{pattern}.tgz', log=False, stream=False) + tgz_path = os.path.join(self.abs_runner_home_path, f'{pattern}.tgz') + docker_info(f'Restore host cache from: {tgz_path}') + package_list = self.conan_api.cache.restore(tgz_path) From c5653529b3524e879c56b270d21f3973dbb11389 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Wed, 6 Mar 2024 18:46:19 +0100 Subject: [PATCH 09/39] some info added --- conan/internal/runner/docker.py | 90 ++++++++++++++++----------------- conans/client/profile_loader.py | 2 +- 2 files changed, 45 insertions(+), 47 deletions(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 062dcc7ef07..0dac624dac8 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -35,16 +35,22 @@ def __init__(self, conan_api, command, profile, args, raw_args): raise ConanException("Docker Client failed to initialize. Check if it is installed and running") self.conan_api = conan_api self.abs_host_path = make_abs_path(args.path) + if args.format: + raise ConanException("format argument is forbidden if running in a docker runner") self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner') self.abs_docker_path = os.path.join('/root/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/") raw_args[raw_args.index(args.path)] = self.abs_docker_path - self.command = ' '.join([f'conan {command}'] + raw_args) - self.dockerfile = str(profile.runner.get('dockerfile', '')) - self.image = str(profile.runner.get('image', 'conanrunner')) + self.command = ' '.join([f'conan {command}'] + raw_args + ['-f json > create.json']) + self.dockerfile = profile.runner.get('dockerfile') + self.image = profile.runner.get('image') + if not (self.dockerfile or self.image): + raise ConanException("dockerfile path or docker image name is needed") + self.image = self.image or 'conan-runner-default' self.name = f'conan-runner-{profile.runner.get("suffix", "docker")}' self.remove = int(profile.runner.get('remove', 1)) self.cache = str(profile.runner.get('cache', 'clean')) self.container = None + def run(self): """ run conan inside a Docker continer @@ -52,58 +58,47 @@ def run(self): docker_info(f'Building the Docker image: {self.image}') self.build_image() volumes, environment = self.create_runner_environment() + docker_info('Creating the docker container') try: if self.docker_client.containers.list(all=True, filters={'name': self.name}): self.container = self.docker_client.containers.get(self.name) self.container.start() else: - self.container = self.docker_client.containers.run(self.image, - "/bin/bash -c 'while true; do sleep 30; done;'", - name=self.name, - volumes=volumes, - environment=environment, - detach=True, - auto_remove=False) + self.container = self.docker_client.containers.run( + self.image, + "/bin/bash -c 'while true; do sleep 30; done;'", + name=self.name, + volumes=volumes, + environment=environment, + detach=True, + auto_remove=False) self.init_container() self.run_command(self.command) self.update_local_cache() - except Exception as e: - print(e) + except: + ConanOutput().error(f'Something went wrong running "{self.command}" inside the container') finally: if self.container: + docker_info('Stopping container') self.container.stop() if self.remove: + docker_info('Removing container') self.container.remove() def build_image(self): - docker_build_logs = None if self.dockerfile: docker_build_logs = self.docker_api.build(path=self.dockerfile, tag=self.image) - else: - dockerfile = textwrap.dedent(""" - FROM ubuntu - RUN apt update && apt upgrade -y - RUN apt install -y build-essential - RUN apt install -y python3-pip cmake git - RUN cd /root && git clone https://github.com/davidsanfal/conan.git conan-io - RUN cd /root/conan-io && pip install docker && pip install -e . - """) - - docker_build_logs = self.docker_api.build(fileobj=BytesIO(dockerfile.encode('utf-8')), - nocache=False, - tag=self.image) - for chunk in docker_build_logs: - for line in chunk.decode("utf-8").split('\r\n'): - if line: - stream = json.loads(line).get('stream') - if stream: - ConanOutput().info(stream.strip()) - - def run_command(self, command, log=True, stdout=True, stderr=True, stream=True, tty=True): + for chunk in docker_build_logs: + for line in chunk.decode("utf-8").split('\r\n'): + if line: + stream = json.loads(line).get('stream') + if stream: + ConanOutput().info(stream.strip()) + def run_command(self, command, log=True, stream=True, tty=True): if log: docker_info(f'Running in container: "{command}"') - exec_run = self.container.exec_run(f'/bin/bash -c \'{command}\'', stdout=stdout, stderr=stderr, stream=stream, tty=tty) + exec_run = self.container.exec_run(f"/bin/bash -c '{command}'", stream=stream, tty=tty) if stream: try: while True: @@ -123,10 +118,14 @@ def create_runner_environment(self): if self.cache in ['clean', 'copy']: shutil.rmtree(self.abs_runner_home_path, ignore_errors=True) os.mkdir(self.abs_runner_home_path) + for file_name in ['global.conf', 'settings.yml', 'remotes.json']: + src_file = os.path.join(ConfigAPI(self.conan_api).home(), file_name) + if os.path.exists(src_file): + shutil.copy(src_file, os.path.join(self.abs_runner_home_path, file_name)) shutil.copytree(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), os.path.join(self.abs_runner_home_path, 'profiles')) if self.cache == 'copy': - tgz_path = os.path.join(self.abs_runner_home_path, 'conan_cache_save.tgz') + tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz') docker_info(f'Save host cache in: {tgz_path}') self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path) return volumes, environment @@ -135,17 +134,16 @@ def init_container(self): if self.cache != 'shared': self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) self.run_command('cp -r '+self.abs_docker_path+'/.conanrunner/profiles/. ${HOME}/.conan2/profiles/.', log=False) + for file_name in ['global.conf', 'settings.yml', 'remotes.json']: + if os.path.exists( os.path.join(self.abs_runner_home_path, file_name)): + self.run_command('cp '+self.abs_docker_path+f'/.conanrunner/{file_name} ${HOME}/.conan2/{file_name}', log=False) if self.cache in ['copy', 'clean']: - self.run_command('conan cache restore '+self.abs_docker_path+'/.conanrunner/conan_cache_save.tgz') + self.run_command('conan cache restore '+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz') def update_local_cache(self): if self.cache != 'shared': - package_list = self.run_command('conan list "*:*" --format=json 2>/dev/null', log=False, stream=False) - docker_cache_list = json.loads(package_list).get('Local Cache') - local_cache_list = self.conan_api.list.select(ListPattern('*:*', rrev=None, prev=None), None, remote=None, lru=None, profile=None) - diff_pattern = list(set(list_patterns(docker_cache_list)).difference(list_patterns(local_cache_list.recipes))) - for pattern in diff_pattern: - self.run_command(f'conan cache save "{pattern}" --file '+self.abs_docker_path+f'/.conanrunner/{pattern}.tgz', log=False, stream=False) - tgz_path = os.path.join(self.abs_runner_home_path, f'{pattern}.tgz') - docker_info(f'Restore host cache from: {tgz_path}') - package_list = self.conan_api.cache.restore(tgz_path) + self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json') + self.run_command('conan cache save --list=pkglist.json --file '+self.abs_docker_path+'/.conanrunner/docker_cache_save.tgz') + tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz') + docker_info(f'Restore host cache from: {tgz_path}') + package_list = self.conan_api.cache.restore(tgz_path) diff --git a/conans/client/profile_loader.py b/conans/client/profile_loader.py index 3cac06f87f3..6b1ba23e81a 100644 --- a/conans/client/profile_loader.py +++ b/conans/client/profile_loader.py @@ -315,7 +315,7 @@ def _parse_key_value(raw_info): for br_line in raw_info.splitlines(): tokens = br_line.split("=", 1) pattern, req_list = tokens - result[pattern] = req_list + result[pattern.strip()] = req_list.strip() return result @staticmethod From ea4f495bc84b1cd5cf5158dc2b153d1789c4a35b Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Wed, 6 Mar 2024 22:09:27 +0100 Subject: [PATCH 10/39] basic test added --- conan/internal/runner/docker.py | 24 ++++-- conans/requirements.txt | 1 + .../test/integration/command/runner_test.py | 75 +++++++++++++++++++ 3 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 conans/test/integration/command/runner_test.py diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 0dac624dac8..2435fd27c8a 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -7,6 +7,7 @@ from conan.api.output import ConanOutput from conan.api.conan_api import ConfigAPI from conan.cli import make_abs_path +from conans.client.profile_loader import ProfileLoader from conans.errors import ConanException @@ -34,12 +35,13 @@ def __init__(self, conan_api, command, profile, args, raw_args): except: raise ConanException("Docker Client failed to initialize. Check if it is installed and running") self.conan_api = conan_api + self.args = args self.abs_host_path = make_abs_path(args.path) if args.format: raise ConanException("format argument is forbidden if running in a docker runner") self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner') self.abs_docker_path = os.path.join('/root/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/") - raw_args[raw_args.index(args.path)] = self.abs_docker_path + raw_args[raw_args.index(args.path)] = f'"{self.abs_docker_path}"' self.command = ' '.join([f'conan {command}'] + raw_args + ['-f json > create.json']) self.dockerfile = profile.runner.get('dockerfile') self.image = profile.runner.get('image') @@ -118,12 +120,13 @@ def create_runner_environment(self): if self.cache in ['clean', 'copy']: shutil.rmtree(self.abs_runner_home_path, ignore_errors=True) os.mkdir(self.abs_runner_home_path) + os.mkdir(os.path.join(self.abs_runner_home_path, 'profiles')) for file_name in ['global.conf', 'settings.yml', 'remotes.json']: src_file = os.path.join(ConfigAPI(self.conan_api).home(), file_name) if os.path.exists(src_file): shutil.copy(src_file, os.path.join(self.abs_runner_home_path, file_name)) - shutil.copytree(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), - os.path.join(self.abs_runner_home_path, 'profiles')) + self._copy_profiles(self.args.profile_build) + self._copy_profiles(self.args.profile_host) if self.cache == 'copy': tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz') docker_info(f'Save host cache in: {tgz_path}') @@ -133,17 +136,24 @@ def create_runner_environment(self): def init_container(self): if self.cache != 'shared': self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) - self.run_command('cp -r '+self.abs_docker_path+'/.conanrunner/profiles/. ${HOME}/.conan2/profiles/.', log=False) + self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False) for file_name in ['global.conf', 'settings.yml', 'remotes.json']: if os.path.exists( os.path.join(self.abs_runner_home_path, file_name)): - self.run_command('cp '+self.abs_docker_path+f'/.conanrunner/{file_name} ${HOME}/.conan2/{file_name}', log=False) + self.run_command('cp "'+self.abs_docker_path+'/.conanrunner/'+file_name+'" ${HOME}/.conan2/'+file_name, log=False) if self.cache in ['copy', 'clean']: - self.run_command('conan cache restore '+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz') + self.run_command('conan cache restore "'+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz"') def update_local_cache(self): if self.cache != 'shared': self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json') - self.run_command('conan cache save --list=pkglist.json --file '+self.abs_docker_path+'/.conanrunner/docker_cache_save.tgz') + self.run_command('conan cache save --list=pkglist.json --file "'+self.abs_docker_path+'"/.conanrunner/docker_cache_save.tgz') tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz') docker_info(f'Restore host cache from: {tgz_path}') package_list = self.conan_api.cache.restore(tgz_path) + + def _copy_profiles(self, profiles): + cwd = os.getcwd() + if profiles: + for profile in profiles: + profile_path = ProfileLoader.get_profile_path(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), profile, cwd) + shutil.copy(profile_path, os.path.join(self.abs_runner_home_path, 'profiles', os.path.basename(profile_path))) \ No newline at end of file diff --git a/conans/requirements.txt b/conans/requirements.txt index 4555809e47e..f3c5092fd1f 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -7,3 +7,4 @@ fasteners>=0.15 distro>=1.4.0, <=1.8.0; sys_platform == 'linux' or sys_platform == 'linux2' Jinja2>=3.0, <4.0.0 python-dateutil>=2.8.0, <3 +docker>=7.0.0 diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py new file mode 100644 index 00000000000..b7c87d3d2f7 --- /dev/null +++ b/conans/test/integration/command/runner_test.py @@ -0,0 +1,75 @@ +import textwrap +import pytest +import docker +from conans.test.utils.tools import TestClient + + +def test_create_docker_runner(): + """ + Tests the ``conan create . `` + """ + try: + docker_client = docker.from_env() + docker_client.images.get('conan-runner-default') + except docker.errors.DockerException: + pytest.skip("docker client not found") + except docker.errors.ImageNotFound: + pytest.skip("docker 'conan-runner-default' image doesn't exist") + except docker.errors.APIError: + pytest.skip("docker server returns an error") + + client = TestClient() + profile_build = textwrap.dedent("""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + image=conan-runner-default + cache=copy + remove=1 + """) + profile_host = textwrap.dedent("""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + image=conan-runner-default + cache=copy + remove=1 + """) + conanfile = textwrap.dedent(""" + from conan import ConanFile + + class MyTest(ConanFile): + name = "pkg" + version = "0.2" + settings = "build_type", "compiler" + author = "John Doe" + license = "MIT" + url = "https://foo.bar.baz" + homepage = "https://foo.bar.site" + topics = "foo", "bar", "qux" + provides = "libjpeg", "libjpg" + deprecated = "other-pkg" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + """) + client.save({"conanfile.py": conanfile, "host": profile_host, "build": profile_build}) + client.run("create . -pr:h host -pr:b build") + + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out + assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out + assert "Removing container" in client.out From b7072b998c0d7cb0010bdc03a064f03319cfcb73 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Thu, 7 Mar 2024 18:03:15 +0100 Subject: [PATCH 11/39] new test added --- conan/internal/runner/docker.py | 73 +++---- .../command/dockerfiles/Dockerfile | 7 + .../command/dockerfiles/Dockerfile_ninja | 7 + .../command/dockerfiles/Dockerfile_test | 7 + .../test/integration/command/runner_test.py | 189 ++++++++++++++++-- 5 files changed, 234 insertions(+), 49 deletions(-) create mode 100644 conans/test/integration/command/dockerfiles/Dockerfile create mode 100644 conans/test/integration/command/dockerfiles/Dockerfile_ninja create mode 100644 conans/test/integration/command/dockerfiles/Dockerfile_test diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 2435fd27c8a..157dc67ce3d 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -1,7 +1,5 @@ import os import json -from io import BytesIO -import textwrap import shutil from conan.api.model import ListPattern from conan.api.output import ConanOutput @@ -29,9 +27,11 @@ def list_patterns(cache_info): class DockerRunner: def __init__(self, conan_api, command, profile, args, raw_args): import docker + import docker.api.build try: self.docker_client = docker.from_env() self.docker_api = docker.APIClient() + docker.api.build.process_dockerfile = lambda dockerfile, path: ('Dockerfile', dockerfile) except: raise ConanException("Docker Client failed to initialize. Check if it is installed and running") self.conan_api = conan_api @@ -44,12 +44,13 @@ def __init__(self, conan_api, command, profile, args, raw_args): raw_args[raw_args.index(args.path)] = f'"{self.abs_docker_path}"' self.command = ' '.join([f'conan {command}'] + raw_args + ['-f json > create.json']) self.dockerfile = profile.runner.get('dockerfile') + self.docker_build_path = profile.runner.get('docker_build_path') self.image = profile.runner.get('image') if not (self.dockerfile or self.image): - raise ConanException("dockerfile path or docker image name is needed") + raise ConanException("'dockerfile' or docker image name is needed") self.image = self.image or 'conan-runner-default' self.name = f'conan-runner-{profile.runner.get("suffix", "docker")}' - self.remove = int(profile.runner.get('remove', 1)) + self.remove = str(profile.runner.get('remove')).lower() == 'true' self.cache = str(profile.runner.get('cache', 'clean')) self.container = None @@ -58,23 +59,24 @@ def run(self): run conan inside a Docker continer """ docker_info(f'Building the Docker image: {self.image}') - self.build_image() + if self.dockerfile: + self.build_image() volumes, environment = self.create_runner_environment() docker_info('Creating the docker container') + if self.docker_client.containers.list(all=True, filters={'name': self.name}): + self.container = self.docker_client.containers.get(self.name) + self.container.start() + else: + self.container = self.docker_client.containers.run( + self.image, + "/bin/bash -c 'while true; do sleep 30; done;'", + name=self.name, + volumes=volumes, + environment=environment, + detach=True, + auto_remove=False) try: - if self.docker_client.containers.list(all=True, filters={'name': self.name}): - self.container = self.docker_client.containers.get(self.name) - self.container.start() - else: - self.container = self.docker_client.containers.run( - self.image, - "/bin/bash -c 'while true; do sleep 30; done;'", - name=self.name, - volumes=volumes, - environment=environment, - detach=True, - auto_remove=False) - self.init_container() + self.init_container() self.run_command(self.command) self.update_local_cache() except: @@ -88,29 +90,32 @@ def run(self): self.container.remove() def build_image(self): - if self.dockerfile: - docker_build_logs = self.docker_api.build(path=self.dockerfile, tag=self.image) - for chunk in docker_build_logs: + dockerfile_file_path = self.dockerfile + if os.path.isdir(self.dockerfile): + dockerfile_file_path = os.path.join(self.dockerfile, 'Dockerfile') + with open(dockerfile_file_path) as f: + build_path = self.docker_build_path or os.path.dirname(dockerfile_file_path) + ConanOutput().highlight(f"Dockerfile path: '{dockerfile_file_path}'") + ConanOutput().highlight(f"Docker build context: '{build_path}'\n") + docker_build_logs = self.docker_api.build(path=build_path, dockerfile=f.read(), tag=self.image) + for chunk in docker_build_logs: for line in chunk.decode("utf-8").split('\r\n'): if line: stream = json.loads(line).get('stream') if stream: ConanOutput().info(stream.strip()) - def run_command(self, command, log=True, stream=True, tty=True): + def run_command(self, command, log=True): if log: docker_info(f'Running in container: "{command}"') - exec_run = self.container.exec_run(f"/bin/bash -c '{command}'", stream=stream, tty=tty) - if stream: - try: - while True: - chunk = next(exec_run.output).decode('utf-8', errors='ignore').strip() - if log: - ConanOutput().info(chunk) - except StopIteration: - pass - else: - return exec_run.output.decode('utf-8', errors='ignore').strip() + exec_run = self.container.exec_run(f"/bin/bash -c '{command}'", stream=True, tty=True) + try: + while True: + chunk = next(exec_run.output).decode('utf-8', errors='ignore').strip() + if log: + ConanOutput().info(chunk) + except StopIteration: + pass def create_runner_environment(self): volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}} @@ -145,7 +150,7 @@ def init_container(self): def update_local_cache(self): if self.cache != 'shared': - self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json') + self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json', log=False) self.run_command('conan cache save --list=pkglist.json --file "'+self.abs_docker_path+'"/.conanrunner/docker_cache_save.tgz') tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz') docker_info(f'Restore host cache from: {tgz_path}') diff --git a/conans/test/integration/command/dockerfiles/Dockerfile b/conans/test/integration/command/dockerfiles/Dockerfile new file mode 100644 index 00000000000..87474dbe665 --- /dev/null +++ b/conans/test/integration/command/dockerfiles/Dockerfile @@ -0,0 +1,7 @@ +FROM ubuntu +RUN apt update && apt upgrade -y +RUN apt install -y build-essential +RUN apt install -y python3-pip cmake git build-essential +RUN pip install docker +COPY . /root/conan-io +RUN cd /root/conan-io && pip install -e . \ No newline at end of file diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_ninja b/conans/test/integration/command/dockerfiles/Dockerfile_ninja new file mode 100644 index 00000000000..42fca4df2a5 --- /dev/null +++ b/conans/test/integration/command/dockerfiles/Dockerfile_ninja @@ -0,0 +1,7 @@ +FROM ubuntu +RUN apt update && apt upgrade -y +RUN apt install -y build-essential +RUN apt install -y python3-pip cmake git build-essential ninja-build +RUN pip install docker +COPY . /root/conan-io +RUN cd /root/conan-io && pip install -e . \ No newline at end of file diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_test b/conans/test/integration/command/dockerfiles/Dockerfile_test new file mode 100644 index 00000000000..87474dbe665 --- /dev/null +++ b/conans/test/integration/command/dockerfiles/Dockerfile_test @@ -0,0 +1,7 @@ +FROM ubuntu +RUN apt update && apt upgrade -y +RUN apt install -y build-essential +RUN apt install -y python3-pip cmake git build-essential +RUN pip install docker +COPY . /root/conan-io +RUN cd /root/conan-io && pip install -e . \ No newline at end of file diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py index b7c87d3d2f7..69fd2b225ef 100644 --- a/conans/test/integration/command/runner_test.py +++ b/conans/test/integration/command/runner_test.py @@ -1,25 +1,113 @@ import textwrap +import os +from typing import Literal import pytest import docker from conans.test.utils.tools import TestClient +from conans.test.assets.cmake import gen_cmakelists +from conans.test.assets.sources import gen_function_h, gen_function_cpp -def test_create_docker_runner(): - """ - Tests the ``conan create . `` - """ + +def docker_skip(test_image=None): try: docker_client = docker.from_env() - docker_client.images.get('conan-runner-default') + if test_image: + docker_client.images.get(test_image) except docker.errors.DockerException: - pytest.skip("docker client not found") + return True except docker.errors.ImageNotFound: - pytest.skip("docker 'conan-runner-default' image doesn't exist") + return True except docker.errors.APIError: - pytest.skip("docker server returns an error") - + return True + return False + + +def conan_base_path(): + import conans + return os.path.dirname(os.path.dirname(conans.__file__)) + + +def dockerfile_path(name=None): + path = os.path.join(os.path.dirname(__file__), "dockerfiles") + if name: + path = os.path.join(path, name) + return path + + +@pytest.mark.skipif(docker_skip(), reason="Only docker running") +def test_create_docker_runner_dockerfile_folder_path(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + profile_build = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path()} + docker_build_path={conan_base_path()} + image=conan-runner-default-test + cache=copy + remove=True + """) + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path()} + docker_build_path={conan_base_path()} + image=conan-runner-default-test + cache=copy + remove=True + """) + conanfile = textwrap.dedent(""" + from conan import ConanFile + + class MyTest(ConanFile): + name = "pkg" + version = "0.2" + settings = "build_type", "compiler" + author = "John Doe" + license = "MIT" + url = "https://foo.bar.baz" + homepage = "https://foo.bar.site" + topics = "foo", "bar", "qux" + provides = "libjpeg", "libjpg" + deprecated = "other-pkg" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + """) + client.save({"conanfile.py": conanfile, "host": profile_host, "build": profile_build}) + client.run("create . -pr:h host -pr:b build") + + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out + assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out + assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip(), reason="Only docker running") +def test_create_docker_runner_dockerfile_file_path(): + """ + Tests the ``conan create . `` + """ client = TestClient() - profile_build = textwrap.dedent("""\ + profile_build = textwrap.dedent(f"""\ [settings] arch=x86_64 build_type=Release @@ -30,11 +118,13 @@ def test_create_docker_runner(): os=Linux [runner] type=docker - image=conan-runner-default + dockerfile={dockerfile_path("Dockerfile_test")} + docker_build_path={conan_base_path()} + image=conan-runner-default-test cache=copy - remove=1 + remove=True """) - profile_host = textwrap.dedent("""\ + profile_host = textwrap.dedent(f"""\ [settings] arch=x86_64 build_type=Release @@ -45,9 +135,11 @@ def test_create_docker_runner(): os=Linux [runner] type=docker - image=conan-runner-default + dockerfile={dockerfile_path("Dockerfile_test")} + docker_build_path={conan_base_path()} + image=conan-runner-default-test cache=copy - remove=1 + remove=True """) conanfile = textwrap.dedent(""" from conan import ConanFile @@ -73,3 +165,70 @@ class MyTest(ConanFile): assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip(), reason="Only docker running") +@pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) +@pytest.mark.tool("ninja") +def test_create_docker_runner_with_ninja(build_type: Literal['Release', 'Debug'], shared: bool): + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.cmake import CMake, CMakeToolchain + + class Library(ConanFile): + name = "hello" + version = "1.0" + settings = 'os', 'arch', 'compiler', 'build_type' + exports_sources = 'hello.h', '*.cpp', 'CMakeLists.txt' + options = {'shared': [True, False]} + default_options = {'shared': False} + + def generate(self): + tc = CMakeToolchain(self, generator="Ninja") + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + self.run(os.sep.join([".", "myapp"])) + + def package(self): + cmake = CMake(self) + cmake.install() + """) + + client = TestClient(path_with_spaces=False) + client.save({'conanfile.py': conanfile, + "CMakeLists.txt": gen_cmakelists(libsources=["hello.cpp"], + appsources=["main.cpp"], + install=True), + "hello.h": gen_function_h(name="hello"), + "hello.cpp": gen_function_cpp(name="hello", includes=["hello"]), + "main.cpp": gen_function_cpp(name="main", includes=["hello"], + calls=["hello"])}) + profile = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + image=conan-runner-ninja-test + dockerfile={dockerfile_path("Dockerfile_ninja")} + docker_build_path={conan_base_path()} + cache=copy + remove=True + """) + client.save({"profile": profile}) + settings = "-s os=Linux -s arch=x86_64 -s build_type={} -o hello/*:shared={}".format(build_type, shared) + # create should also work + client.run("create . --name=hello --version=1.0 {} -pr:h profile -pr:b profile".format(settings)) + print(client.out) + assert 'cmake -G "Ninja"' in client.out + assert "main: {}!".format(build_type) in client.out \ No newline at end of file From de279ec9a010d3abb4dff0fb72d09b99d54de591 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Mon, 11 Mar 2024 10:35:12 +0100 Subject: [PATCH 12/39] requiremnets fixed --- conans/requirements.txt | 2 +- conans/test/integration/command/runner_test.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/conans/requirements.txt b/conans/requirements.txt index f3c5092fd1f..82aa416dfa3 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -7,4 +7,4 @@ fasteners>=0.15 distro>=1.4.0, <=1.8.0; sys_platform == 'linux' or sys_platform == 'linux2' Jinja2>=3.0, <4.0.0 python-dateutil>=2.8.0, <3 -docker>=7.0.0 +docker>=5.0.0, <=5.0.3 diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py index 69fd2b225ef..ee98f712a33 100644 --- a/conans/test/integration/command/runner_test.py +++ b/conans/test/integration/command/runner_test.py @@ -1,6 +1,5 @@ import textwrap import os -from typing import Literal import pytest import docker from conans.test.utils.tools import TestClient @@ -36,6 +35,7 @@ def dockerfile_path(name=None): @pytest.mark.skipif(docker_skip(), reason="Only docker running") +@pytest.mark.xfail(reason="known docker ci issue") def test_create_docker_runner_dockerfile_folder_path(): """ Tests the ``conan create . `` @@ -102,6 +102,7 @@ class MyTest(ConanFile): @pytest.mark.skipif(docker_skip(), reason="Only docker running") +@pytest.mark.xfail(reason="known docker ci issue") def test_create_docker_runner_dockerfile_file_path(): """ Tests the ``conan create . `` @@ -168,9 +169,10 @@ class MyTest(ConanFile): @pytest.mark.skipif(docker_skip(), reason="Only docker running") +@pytest.mark.xfail(reason="known docker ci issue") @pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) @pytest.mark.tool("ninja") -def test_create_docker_runner_with_ninja(build_type: Literal['Release', 'Debug'], shared: bool): +def test_create_docker_runner_with_ninja(build_type, shared): conanfile = textwrap.dedent(""" import os from conan import ConanFile From f8c7cc142afbd2d0adcf075aa6a0093246a1ea07 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Mon, 11 Mar 2024 14:16:56 +0100 Subject: [PATCH 13/39] new docker exec added --- conan/cli/commands/create.py | 4 +- conan/internal/runner/__init__.py | 6 ++ conan/internal/runner/docker.py | 73 +++++++++++++------ conans/requirements.txt | 1 - .../test/integration/command/runner_test.py | 14 ---- 5 files changed, 57 insertions(+), 41 deletions(-) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 8dae971e656..7ea0a3f1de4 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -65,11 +65,11 @@ def create(conan_api, parser, *args): lockfile = conan_api.lockfile.update_lockfile_export(lockfile, conanfile, ref, is_build) print_profiles(profile_host, profile_build) - if profile_build.runner and not os.environ.get("CONAN_RUNNER_ENVIRONMENT"): + if profile_host.runner and not os.environ.get("CONAN_RUNNER_ENVIRONMENT"): return { 'docker': DockerRunner, 'ssh': SSHRunner - }[profile_build.runner.get('type')](conan_api, 'create', profile_host, args, raw_args).run() + }[profile_host.runner.get('type')](conan_api, 'create', profile_host, args, raw_args).run() if args.build is not None and args.build_test is None: args.build_test = args.build diff --git a/conan/internal/runner/__init__.py b/conan/internal/runner/__init__.py index e69de29bb2d..f3c3ba883bb 100644 --- a/conan/internal/runner/__init__.py +++ b/conan/internal/runner/__init__.py @@ -0,0 +1,6 @@ +class RunnerExection(Exception): + def __init__(self, *args, **kwargs): + self.command = kwargs.pop("command", None) + self.stdout_log = kwargs.pop("stdout_log", None) + self.stderr_log = kwargs.pop("stderr_log", None) + super(RunnerExection, self).__init__(*args, **kwargs) \ No newline at end of file diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 157dc67ce3d..572a7e5c94a 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -1,10 +1,12 @@ import os import json +import platform import shutil from conan.api.model import ListPattern from conan.api.output import ConanOutput from conan.api.conan_api import ConfigAPI from conan.cli import make_abs_path +from conan.internal.runner import RunnerExection from conans.client.profile_loader import ProfileLoader from conans.errors import ConanException @@ -33,7 +35,9 @@ def __init__(self, conan_api, command, profile, args, raw_args): self.docker_api = docker.APIClient() docker.api.build.process_dockerfile = lambda dockerfile, path: ('Dockerfile', dockerfile) except: - raise ConanException("Docker Client failed to initialize. Check if it is installed and running") + raise ConanException("Docker Client failed to initialize." + "\n - Check if docker is installed and running" + "\n - Run 'pip install docker>=5.0.0, <=5.0.3'") self.conan_api = conan_api self.args = args self.abs_host_path = make_abs_path(args.path) @@ -58,29 +62,35 @@ def run(self): """ run conan inside a Docker continer """ - docker_info(f'Building the Docker image: {self.image}') if self.dockerfile: + docker_info(f'Building the Docker image: {self.image}') self.build_image() volumes, environment = self.create_runner_environment() - docker_info('Creating the docker container') - if self.docker_client.containers.list(all=True, filters={'name': self.name}): - self.container = self.docker_client.containers.get(self.name) - self.container.start() - else: - self.container = self.docker_client.containers.run( - self.image, - "/bin/bash -c 'while true; do sleep 30; done;'", - name=self.name, - volumes=volumes, - environment=environment, - detach=True, - auto_remove=False) + try: + if self.docker_client.containers.list(all=True, filters={'name': self.name}): + docker_info('Starting the docker container') + self.container = self.docker_client.containers.get(self.name) + self.container.start() + else: + docker_info('Creating the docker container') + self.container = self.docker_client.containers.run( + self.image, + "/bin/bash -c 'while true; do sleep 30; done;'", + name=self.name, + volumes=volumes, + environment=environment, + detach=True, + auto_remove=False) + except Exception as e: + raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"' + f'\n\n{str(e)}') try: self.init_container() self.run_command(self.command) self.update_local_cache() - except: - ConanOutput().error(f'Something went wrong running "{self.command}" inside the container') + except RunnerExection as e: + raise ConanException(f'"{e.command}" inside docker fail' + f'\n\nLast command output: {str(e.stdout_log)}') finally: if self.container: docker_info('Stopping container') @@ -108,14 +118,29 @@ def build_image(self): def run_command(self, command, log=True): if log: docker_info(f'Running in container: "{command}"') - exec_run = self.container.exec_run(f"/bin/bash -c '{command}'", stream=True, tty=True) + exec_instance = self.docker_api.exec_create(self.container.id, f"/bin/bash -c '{command}'", tty=True) + exec_output = self.docker_api.exec_start(exec_instance['Id'], tty=True, stream=True, demux=True,) + stderr_log, stdout_log = '', '' try: - while True: - chunk = next(exec_run.output).decode('utf-8', errors='ignore').strip() - if log: - ConanOutput().info(chunk) - except StopIteration: - pass + for (stdout_out, stderr_out) in exec_output: + if stdout_out is not None: + stdout_log += stdout_out.decode('utf-8', errors='ignore').strip() + if log: + ConanOutput().info(stdout_out.decode('utf-8', errors='ignore').strip()) + if stderr_out is not None: + stderr_log += stderr_out.decode('utf-8', errors='ignore').strip() + if log: + ConanOutput().info(stderr_out.decode('utf-8', errors='ignore').strip()) + except Exception as e: + if platform.system() == 'Windows': + import pywintypes + if isinstance(e, pywintypes.error): + pass + else: + raise e + exit_metadata = self.docker_api.exec_inspect(exec_instance['Id']) + if exit_metadata['Running'] or exit_metadata['ExitCode'] > 0: + raise RunnerExection(command=command, stdout_log=stdout_log, stderr_log=stderr_log) def create_runner_environment(self): volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}} diff --git a/conans/requirements.txt b/conans/requirements.txt index 82aa416dfa3..4555809e47e 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -7,4 +7,3 @@ fasteners>=0.15 distro>=1.4.0, <=1.8.0; sys_platform == 'linux' or sys_platform == 'linux2' Jinja2>=3.0, <4.0.0 python-dateutil>=2.8.0, <3 -docker>=5.0.0, <=5.0.3 diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py index ee98f712a33..0e1a7288f69 100644 --- a/conans/test/integration/command/runner_test.py +++ b/conans/test/integration/command/runner_test.py @@ -50,13 +50,6 @@ def test_create_docker_runner_dockerfile_folder_path(): compiler.libcxx=libstdc++11 compiler.version=11 os=Linux - [runner] - type=docker - dockerfile={dockerfile_path()} - docker_build_path={conan_base_path()} - image=conan-runner-default-test - cache=copy - remove=True """) profile_host = textwrap.dedent(f"""\ [settings] @@ -117,13 +110,6 @@ def test_create_docker_runner_dockerfile_file_path(): compiler.libcxx=libstdc++11 compiler.version=11 os=Linux - [runner] - type=docker - dockerfile={dockerfile_path("Dockerfile_test")} - docker_build_path={conan_base_path()} - image=conan-runner-default-test - cache=copy - remove=True """) profile_host = textwrap.dedent(f"""\ [settings] From aa9dd9ab35f41b555c972d460beb5dec9f368ce1 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Tue, 12 Mar 2024 08:21:17 +0100 Subject: [PATCH 14/39] docker added to dev requirements --- conans/requirements_dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conans/requirements_dev.txt b/conans/requirements_dev.txt index decec3b2ef5..3792311e4ed 100644 --- a/conans/requirements_dev.txt +++ b/conans/requirements_dev.txt @@ -6,3 +6,5 @@ WebTest>=2.0.18, <2.1.0 bottle PyJWT pluginbase +docker>=5.0.0, <6.0.0 + From c9135702ddfcbd41b85f5287f168462dc1c31ffc Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Tue, 12 Mar 2024 21:35:38 +0100 Subject: [PATCH 15/39] docker conan version check added --- conan/internal/runner/__init__.py | 4 ++-- conan/internal/runner/docker.py | 15 ++++++++++++--- conans/requirements_dev.txt | 3 +-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/conan/internal/runner/__init__.py b/conan/internal/runner/__init__.py index f3c3ba883bb..8ee22e17661 100644 --- a/conan/internal/runner/__init__.py +++ b/conan/internal/runner/__init__.py @@ -1,6 +1,6 @@ -class RunnerExection(Exception): +class RunnerException(Exception): def __init__(self, *args, **kwargs): self.command = kwargs.pop("command", None) self.stdout_log = kwargs.pop("stdout_log", None) self.stderr_log = kwargs.pop("stderr_log", None) - super(RunnerExection, self).__init__(*args, **kwargs) \ No newline at end of file + super(RunnerException, self).__init__(*args, **kwargs) \ No newline at end of file diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 572a7e5c94a..a51f018a1d8 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -6,9 +6,10 @@ from conan.api.output import ConanOutput from conan.api.conan_api import ConfigAPI from conan.cli import make_abs_path -from conan.internal.runner import RunnerExection +from conan.internal.runner import RunnerException from conans.client.profile_loader import ProfileLoader from conans.errors import ConanException +from conans.model.version import Version def docker_info(msg): @@ -88,7 +89,9 @@ def run(self): self.init_container() self.run_command(self.command) self.update_local_cache() - except RunnerExection as e: + except ConanException as e: + raise e + except RunnerException as e: raise ConanException(f'"{e.command}" inside docker fail' f'\n\nLast command output: {str(e.stdout_log)}') finally: @@ -140,7 +143,8 @@ def run_command(self, command, log=True): raise e exit_metadata = self.docker_api.exec_inspect(exec_instance['Id']) if exit_metadata['Running'] or exit_metadata['ExitCode'] > 0: - raise RunnerExection(command=command, stdout_log=stdout_log, stderr_log=stderr_log) + raise RunnerException(command=command, stdout_log=stdout_log, stderr_log=stderr_log) + return stdout_log, stderr_log def create_runner_environment(self): volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}} @@ -164,6 +168,11 @@ def create_runner_environment(self): return volumes, environment def init_container(self): + min_conan_version = '2.1' + stdout, _ = self.run_command('conan --version', log=False) + docker_conan_version = str(stdout.replace('Conan version ', '').replace('\n', '').replace('\r', '')) # Remove all characters and color + if Version(docker_conan_version) <= Version(min_conan_version): + raise ConanException( f'conan version inside the container must be greater than {min_conan_version}') if self.cache != 'shared': self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False) diff --git a/conans/requirements_dev.txt b/conans/requirements_dev.txt index 3792311e4ed..9ffa1ea641b 100644 --- a/conans/requirements_dev.txt +++ b/conans/requirements_dev.txt @@ -6,5 +6,4 @@ WebTest>=2.0.18, <2.1.0 bottle PyJWT pluginbase -docker>=5.0.0, <6.0.0 - +docker From 9f5b600e1bddd7687acc66ca305fcd89cf6e7bd2 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Wed, 13 Mar 2024 10:32:41 +0100 Subject: [PATCH 16/39] more info added --- conan/internal/runner/docker.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index a51f018a1d8..f3bf96903ad 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -3,7 +3,7 @@ import platform import shutil from conan.api.model import ListPattern -from conan.api.output import ConanOutput +from conan.api.output import Color, ConanOutput from conan.api.conan_api import ConfigAPI from conan.cli import make_abs_path from conan.internal.runner import RunnerException @@ -12,10 +12,13 @@ from conans.model.version import Version -def docker_info(msg): - ConanOutput().highlight('\n┌'+'─'*(2+len(msg))+'┐') - ConanOutput().highlight(f'| {msg} |') - ConanOutput().highlight('└'+'─'*(2+len(msg))+'┘\n') +def docker_info(msg, error=False): + fg=Color.BRIGHT_MAGENTA + if error: + fg=Color.BRIGHT_RED + ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg) + ConanOutput().status(f'| {msg} |', fg=fg) + ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) def list_patterns(cache_info): @@ -67,6 +70,7 @@ def run(self): docker_info(f'Building the Docker image: {self.image}') self.build_image() volumes, environment = self.create_runner_environment() + error = False try: if self.docker_client.containers.list(all=True, filters={'name': self.name}): docker_info('Starting the docker container') @@ -82,6 +86,7 @@ def run(self): environment=environment, detach=True, auto_remove=False) + docker_info(f'Container {self.name} running') except Exception as e: raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"' f'\n\n{str(e)}') @@ -90,16 +95,19 @@ def run(self): self.run_command(self.command) self.update_local_cache() except ConanException as e: + error = True raise e except RunnerException as e: + error = True raise ConanException(f'"{e.command}" inside docker fail' f'\n\nLast command output: {str(e.stdout_log)}') finally: if self.container: - docker_info('Stopping container') + error_prefix = 'ERROR: ' if error else '' + docker_info(f'{error_prefix}Stopping container', error) self.container.stop() if self.remove: - docker_info('Removing container') + docker_info(f'{error_prefix}Removing container', error) self.container.remove() def build_image(self): @@ -116,7 +124,7 @@ def build_image(self): if line: stream = json.loads(line).get('stream') if stream: - ConanOutput().info(stream.strip()) + ConanOutput().status(stream.strip()) def run_command(self, command, log=True): if log: @@ -129,11 +137,11 @@ def run_command(self, command, log=True): if stdout_out is not None: stdout_log += stdout_out.decode('utf-8', errors='ignore').strip() if log: - ConanOutput().info(stdout_out.decode('utf-8', errors='ignore').strip()) + ConanOutput().status(stdout_out.decode('utf-8', errors='ignore').strip()) if stderr_out is not None: stderr_log += stderr_out.decode('utf-8', errors='ignore').strip() if log: - ConanOutput().info(stderr_out.decode('utf-8', errors='ignore').strip()) + ConanOutput().status(stderr_out.decode('utf-8', errors='ignore').strip()) except Exception as e: if platform.system() == 'Windows': import pywintypes @@ -169,9 +177,10 @@ def create_runner_environment(self): def init_container(self): min_conan_version = '2.1' - stdout, _ = self.run_command('conan --version', log=False) + stdout, _ = self.run_command('conan --version', log=True) docker_conan_version = str(stdout.replace('Conan version ', '').replace('\n', '').replace('\r', '')) # Remove all characters and color if Version(docker_conan_version) <= Version(min_conan_version): + ConanOutput().status(f'ERROR: conan version inside the container must be greater than {min_conan_version}', fg=Color.BRIGHT_RED) raise ConanException( f'conan version inside the container must be greater than {min_conan_version}') if self.cache != 'shared': self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) From 001014efd40d56cb170d04cc541aab0c9f68286b Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Fri, 15 Mar 2024 09:43:33 +0100 Subject: [PATCH 17/39] dockerfile interface updated --- conan/internal/runner/docker.py | 6 +++--- .../integration/command/dockerfiles/Dockerfile | 13 +++++++++---- .../command/dockerfiles/Dockerfile_ninja | 14 ++++++++++---- .../command/dockerfiles/Dockerfile_test | 13 +++++++++---- conans/test/integration/command/runner_test.py | 12 ++++++------ 5 files changed, 37 insertions(+), 21 deletions(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index f3bf96903ad..b62e592c73e 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -52,7 +52,7 @@ def __init__(self, conan_api, command, profile, args, raw_args): raw_args[raw_args.index(args.path)] = f'"{self.abs_docker_path}"' self.command = ' '.join([f'conan {command}'] + raw_args + ['-f json > create.json']) self.dockerfile = profile.runner.get('dockerfile') - self.docker_build_path = profile.runner.get('docker_build_path') + self.docker_build_context = profile.runner.get('docker_build_context') self.image = profile.runner.get('image') if not (self.dockerfile or self.image): raise ConanException("'dockerfile' or docker image name is needed") @@ -115,7 +115,7 @@ def build_image(self): if os.path.isdir(self.dockerfile): dockerfile_file_path = os.path.join(self.dockerfile, 'Dockerfile') with open(dockerfile_file_path) as f: - build_path = self.docker_build_path or os.path.dirname(dockerfile_file_path) + build_path = self.docker_build_context or os.path.dirname(dockerfile_file_path) ConanOutput().highlight(f"Dockerfile path: '{dockerfile_file_path}'") ConanOutput().highlight(f"Docker build context: '{build_path}'\n") docker_build_logs = self.docker_api.build(path=build_path, dockerfile=f.read(), tag=self.image) @@ -204,4 +204,4 @@ def _copy_profiles(self, profiles): if profiles: for profile in profiles: profile_path = ProfileLoader.get_profile_path(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), profile, cwd) - shutil.copy(profile_path, os.path.join(self.abs_runner_home_path, 'profiles', os.path.basename(profile_path))) \ No newline at end of file + shutil.copy(profile_path, os.path.join(self.abs_runner_home_path, 'profiles', os.path.basename(profile_path))) diff --git a/conans/test/integration/command/dockerfiles/Dockerfile b/conans/test/integration/command/dockerfiles/Dockerfile index 87474dbe665..2fc34704018 100644 --- a/conans/test/integration/command/dockerfiles/Dockerfile +++ b/conans/test/integration/command/dockerfiles/Dockerfile @@ -1,7 +1,12 @@ -FROM ubuntu -RUN apt update && apt upgrade -y -RUN apt install -y build-essential -RUN apt install -y python3-pip cmake git build-essential +FROM ubuntu:22.04 +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* RUN pip install docker COPY . /root/conan-io RUN cd /root/conan-io && pip install -e . \ No newline at end of file diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_ninja b/conans/test/integration/command/dockerfiles/Dockerfile_ninja index 42fca4df2a5..2c60e6918f1 100644 --- a/conans/test/integration/command/dockerfiles/Dockerfile_ninja +++ b/conans/test/integration/command/dockerfiles/Dockerfile_ninja @@ -1,7 +1,13 @@ -FROM ubuntu -RUN apt update && apt upgrade -y -RUN apt install -y build-essential -RUN apt install -y python3-pip cmake git build-essential ninja-build +FROM ubuntu:22.04 +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + ninja-build \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* RUN pip install docker COPY . /root/conan-io RUN cd /root/conan-io && pip install -e . \ No newline at end of file diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_test b/conans/test/integration/command/dockerfiles/Dockerfile_test index 87474dbe665..2fc34704018 100644 --- a/conans/test/integration/command/dockerfiles/Dockerfile_test +++ b/conans/test/integration/command/dockerfiles/Dockerfile_test @@ -1,7 +1,12 @@ -FROM ubuntu -RUN apt update && apt upgrade -y -RUN apt install -y build-essential -RUN apt install -y python3-pip cmake git build-essential +FROM ubuntu:22.04 +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* RUN pip install docker COPY . /root/conan-io RUN cd /root/conan-io && pip install -e . \ No newline at end of file diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py index 0e1a7288f69..ef695e297fb 100644 --- a/conans/test/integration/command/runner_test.py +++ b/conans/test/integration/command/runner_test.py @@ -35,7 +35,7 @@ def dockerfile_path(name=None): @pytest.mark.skipif(docker_skip(), reason="Only docker running") -@pytest.mark.xfail(reason="known docker ci issue") +@pytest.mark.xfail(reason="conan inside docker optional test") def test_create_docker_runner_dockerfile_folder_path(): """ Tests the ``conan create . `` @@ -63,7 +63,7 @@ def test_create_docker_runner_dockerfile_folder_path(): [runner] type=docker dockerfile={dockerfile_path()} - docker_build_path={conan_base_path()} + docker_build_context={conan_base_path()} image=conan-runner-default-test cache=copy remove=True @@ -95,7 +95,7 @@ class MyTest(ConanFile): @pytest.mark.skipif(docker_skip(), reason="Only docker running") -@pytest.mark.xfail(reason="known docker ci issue") +@pytest.mark.xfail(reason="conan inside docker optional test") def test_create_docker_runner_dockerfile_file_path(): """ Tests the ``conan create . `` @@ -123,7 +123,7 @@ def test_create_docker_runner_dockerfile_file_path(): [runner] type=docker dockerfile={dockerfile_path("Dockerfile_test")} - docker_build_path={conan_base_path()} + docker_build_context={conan_base_path()} image=conan-runner-default-test cache=copy remove=True @@ -155,7 +155,7 @@ class MyTest(ConanFile): @pytest.mark.skipif(docker_skip(), reason="Only docker running") -@pytest.mark.xfail(reason="known docker ci issue") +@pytest.mark.xfail(reason="conan inside docker optional test") @pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) @pytest.mark.tool("ninja") def test_create_docker_runner_with_ninja(build_type, shared): @@ -209,7 +209,7 @@ def package(self): type=docker image=conan-runner-ninja-test dockerfile={dockerfile_path("Dockerfile_ninja")} - docker_build_path={conan_base_path()} + docker_build_context={conan_base_path()} cache=copy remove=True """) From cbf0cf113a1e771b81c36973bb3df24bb202ed8f Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Mon, 18 Mar 2024 19:04:31 +0100 Subject: [PATCH 18/39] docker added as a extras_require --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1864af173a4..fc2bca6388c 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ def generate_long_description_file(): project_requirements = get_requires("conans/requirements.txt") dev_requirements = get_requires("conans/requirements_dev.txt") +docker_requirements = ['docker'] excluded_server_packages = ["conans.server*"] exclude = excluded_test_packages + excluded_server_packages @@ -112,10 +113,11 @@ def generate_long_description_file(): # List additional groups of dependencies here (e.g. development # dependencies). You can install these using the following syntax, # for example: - # $ pip install -e .[dev,test] + # $ pip install -e .[dev,test,docker] extras_require={ 'dev': dev_requirements, 'test': dev_requirements, + 'docker': docker_requirements }, # If there are data files included in your packages that need to be From e169fd9969372884d730480f376f09e4f2aa1cbd Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Tue, 19 Mar 2024 15:40:13 +0100 Subject: [PATCH 19/39] some fix --- conan/internal/runner/docker.py | 6 +++--- conans/test/integration/command/dockerfiles/Dockerfile | 1 - .../test/integration/command/dockerfiles/Dockerfile_ninja | 1 - conans/test/integration/command/dockerfiles/Dockerfile_test | 1 - 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index b62e592c73e..305089974cc 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -58,7 +58,7 @@ def __init__(self, conan_api, command, profile, args, raw_args): raise ConanException("'dockerfile' or docker image name is needed") self.image = self.image or 'conan-runner-default' self.name = f'conan-runner-{profile.runner.get("suffix", "docker")}' - self.remove = str(profile.runner.get('remove')).lower() == 'true' + self.remove = str(profile.runner.get('remove', 'false')).lower() == 'true' self.cache = str(profile.runner.get('cache', 'clean')) self.container = None @@ -182,13 +182,13 @@ def init_container(self): if Version(docker_conan_version) <= Version(min_conan_version): ConanOutput().status(f'ERROR: conan version inside the container must be greater than {min_conan_version}', fg=Color.BRIGHT_RED) raise ConanException( f'conan version inside the container must be greater than {min_conan_version}') - if self.cache != 'shared': + if self.cache == 'shared': self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False) for file_name in ['global.conf', 'settings.yml', 'remotes.json']: if os.path.exists( os.path.join(self.abs_runner_home_path, file_name)): self.run_command('cp "'+self.abs_docker_path+'/.conanrunner/'+file_name+'" ${HOME}/.conan2/'+file_name, log=False) - if self.cache in ['copy', 'clean']: + if self.cache in ['copy']: self.run_command('conan cache restore "'+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz"') def update_local_cache(self): diff --git a/conans/test/integration/command/dockerfiles/Dockerfile b/conans/test/integration/command/dockerfiles/Dockerfile index 2fc34704018..068de9a2efd 100644 --- a/conans/test/integration/command/dockerfiles/Dockerfile +++ b/conans/test/integration/command/dockerfiles/Dockerfile @@ -7,6 +7,5 @@ RUN apt-get update \ python3-pip \ python3-venv \ && rm -rf /var/lib/apt/lists/* -RUN pip install docker COPY . /root/conan-io RUN cd /root/conan-io && pip install -e . \ No newline at end of file diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_ninja b/conans/test/integration/command/dockerfiles/Dockerfile_ninja index 2c60e6918f1..acd643d2da6 100644 --- a/conans/test/integration/command/dockerfiles/Dockerfile_ninja +++ b/conans/test/integration/command/dockerfiles/Dockerfile_ninja @@ -8,6 +8,5 @@ RUN apt-get update \ python3-pip \ python3-venv \ && rm -rf /var/lib/apt/lists/* -RUN pip install docker COPY . /root/conan-io RUN cd /root/conan-io && pip install -e . \ No newline at end of file diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_test b/conans/test/integration/command/dockerfiles/Dockerfile_test index 2fc34704018..068de9a2efd 100644 --- a/conans/test/integration/command/dockerfiles/Dockerfile_test +++ b/conans/test/integration/command/dockerfiles/Dockerfile_test @@ -7,6 +7,5 @@ RUN apt-get update \ python3-pip \ python3-venv \ && rm -rf /var/lib/apt/lists/* -RUN pip install docker COPY . /root/conan-io RUN cd /root/conan-io && pip install -e . \ No newline at end of file From 7fa7de48dab281bb532111660602a4aad46ab27a Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 20 Mar 2024 17:18:49 +0000 Subject: [PATCH 20/39] ssh remote WIP --- conan/cli/commands/create.py | 2 +- conan/internal/runner/ssh.py | 217 ++++++++++++++++++++++++++++++++++- conans/requirements_dev.txt | 1 + 3 files changed, 216 insertions(+), 4 deletions(-) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 7ea0a3f1de4..60328ceb3d6 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -69,7 +69,7 @@ def create(conan_api, parser, *args): return { 'docker': DockerRunner, 'ssh': SSHRunner - }[profile_host.runner.get('type')](conan_api, 'create', profile_host, args, raw_args).run() + }[profile_host.runner.get('type')](conan_api, 'create', profile_host, profile_build, args, raw_args).run() if args.build is not None and args.build_test is None: args.build_test = args.build diff --git a/conan/internal/runner/ssh.py b/conan/internal/runner/ssh.py index 430ac259a5e..62703e48904 100644 --- a/conan/internal/runner/ssh.py +++ b/conan/internal/runner/ssh.py @@ -1,11 +1,222 @@ +from pathlib import Path +import pathlib +from conan.api.conan_api import ConfigAPI +from conan.api.output import Color, ConanOutput +from conans.errors import ConanException +from conans.client.profile_loader import ProfileLoader + +import os +from io import BytesIO +import sys + +def ssh_info(msg, error=False): + fg=Color.BRIGHT_MAGENTA + if error: + fg=Color.BRIGHT_RED + ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg) + ConanOutput().status(f'| {msg} |', fg=fg) + ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) + class SSHRunner: - def __init__(self, conan_api, command, profile, args, raw_args): + def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args): + from paramiko.config import SSHConfig + from paramiko.client import SSHClient self.conan_api = conan_api self.command = command - self.profile = profile + self.host_profile = host_profile + self.build_profile = build_profile + self.remote_host_profile = None + self.remote_build_profile = None + self.remote_python_command = None + self.remote_create_dir = None + self.remote_is_windows = None self.args = args self.raw_args = raw_args + self.ssh_config = None + self.remote_workspace = None + self.remote_conan = None + self.remote_conan_home = None + if host_profile.runner.get('use_ssh_config', False): + ssh_config_file = Path.home() / ".ssh" / "config" + ssh_config = SSHConfig.from_file(open(ssh_config_file)) + + hostname = host_profile.runner.get("host") # TODO: this one is required + if ssh_config and ssh_config.lookup(hostname): + hostname = ssh_config.lookup(hostname)['hostname'] + + self.client = SSHClient() + self.client.load_system_host_keys() + self.client.connect(hostname) + def run(self, use_cache=True): - pass + ssh_info('Got to SSHRunner.run(), doing nothing') + + self.ensure_runner_environment() + self.copy_working_conanfile_path() + + raw_args = self.raw_args + raw_args[raw_args.index(self.args.path)] = self.remote_create_dir + raw_args = " ".join(raw_args) + command = f"{self.remote_conan} create {raw_args}" + + # ssh_info(f"Remote command: {command}") + + _stdin, _stdout,_stderr = self.client.exec_command(command, get_pty=True) + + # TODO: achieve printing continuous output + + + self.client.close() + + def ensure_runner_environment(self): + has_python3_command = False + python_is_python3 = False + + _, _stdout, _stderr = self.client.exec_command("python3 --version") + has_python3_command = _stdout.channel.recv_exit_status() == 0 + if not has_python3_command: + _, _stdout, _stderr = self.client.exec_command("python --version") + if _stdout.channel.recv_exit_status() == 0 and "Python 3" in _stdout.read().decode(): + python_is_python3 = True + + python_command = "python" if python_is_python3 else "python3" + self.remote_python_command = python_command + + if not has_python3_command and not python_is_python3: + raise ConanException("Unable to locate working Python 3 executable in remote SSH environment") + + # Determine if remote host is Windows + _, _stdout, _ = self.client.exec_command(f'{python_command} -c "import os; print(os.name)"') + if _stdout.channel.recv_exit_status() != 0: + raise ConanException("Unable to determine remote OS type") + is_windows = _stdout.read().decode().strip() == "nt" + self.remote_is_windows = is_windows + + # Get remote user home folder + _, _stdout, _ = self.client.exec_command(f'{python_command} -c "from pathlib import Path; print(Path.home())"') + if _stdout.channel.recv_exit_status() != 0: + raise ConanException("Unable to determine remote home user folder") + home_folder = _stdout.read().decode().strip() + + # Expected remote paths + remote_folder = Path(home_folder) / ".conan2remote" + remote_folder = remote_folder.as_posix().replace("\\", "/") + self.remote_workspace = remote_folder + remote_conan_home = Path(home_folder) / ".conan2remote" / "conanhome" + remote_conan_home = remote_conan_home.as_posix().replace("\\", "/") + self.remote_conan_home = remote_conan_home + ssh_info(f"Remote workfolder: {remote_folder}") + + # Ensure remote folders exist + for folder in [remote_folder, remote_conan_home]: + _, _stdout, _stderr = self.client.exec_command(f"""{python_command} -c "import os; os.makedirs('{folder}', exist_ok=True)""") + if _stdout.channel.recv_exit_status() != 0: + ssh_info(f"Error creating remote folder: {_stderr.read().decode()}") + raise ConanException(f"Unable to create remote workfolder at {folder}") + + conan_venv = remote_folder + "/venv" + if is_windows: + conan_cmd = remote_folder + "/venv/Scripts/conan.exe" + else: + conan_cmd = remote_folder + "/venv/bin/conan" + + ssh_info(f"Expected remote conan home: {remote_conan_home}") + ssh_info(f"Expected remote conan command: {conan_cmd}") + + # Check if remote Conan executable exists, otherwise invoke pip inside venv + sftp = self.client.open_sftp() + try: + sftp.stat(conan_cmd) + has_remote_conan = True + except FileNotFoundError: + has_remote_conan = False + finally: + sftp.close() + + if not has_remote_conan: + _, _stdout, _stderr = self.client.exec_command(f"{python_command} -m venv {conan_venv}") + if _stdout.channel.recv_exit_status() != 0: + ssh_info(f"Unable to create remote venv: {_stderr.read().decode().strip()}") + + if is_windows: + python_command = remote_folder + "/venv" + "/Scripts" + "/python.exe" + else: + python_command = remote_folder + "/venv" + "/bin" + "/python" + + _, _stdout, _stderr = self.client.exec_command(f"{python_command} -m pip install git+https://github.com/conan-io/conan@feature/docker_wrapper") + if _stdout.channel.recv_exit_status() != 0: + # Note: this may fail on windows + ssh_info(f"Unable to install conan in venv: {_stderr.read().decode().strip()}") + + remote_env = { + 'CONAN_HOME': remote_conan_home, + 'CONAN_RUNNER_ENVIRONMENT': "1" + } + if is_windows: + # Wrapper script with environment variables preset + env_lines = "\n".join([f"set {k}={v}" for k,v in remote_env.items()]) + conan_bat_contents = f"""@echo off\n{env_lines}\n{conan_cmd} %*\n""" + conan_bat = remote_folder + "/conan.bat" + try: + sftp = self.client.open_sftp() + sftp.putfo(BytesIO(conan_bat_contents.encode()), conan_bat) + except: + raise ConanException("unable to set up Conan remote script") + finally: + sftp.close() + + self.remote_conan = conan_bat + _, _stdout, _stderr = self.client.exec_command(f"{self.remote_conan} config home") + ssh_info(f"Remote conan config home returned: {_stdout.read().decode().strip()}") + _, _stdout, _stderr = self.client.exec_command(f"{self.remote_conan} profile detect --force") + self._copy_profiles() + + + def _copy_profiles(self): + sftp = self.client.open_sftp() + + # TODO: very questionable choices here + try: + profiles = { + self.args.profile_host[0]: self.host_profile.dumps(), + self.args.profile_build[0]: self.build_profile.dumps() + } + + for name, contents in profiles.items(): + dest_filename = self.remote_conan_home + f"/profiles/{name}" + sftp.putfo(BytesIO(contents.encode()), dest_filename) + except: + raise ConanException("Unable to copy profiles to remote") + finally: + sftp.close() + + def copy_working_conanfile_path(self): + resolved_path = Path(self.args.path).resolve() + if resolved_path.is_file(): + resolved_path = resolved_path.parent + + if not resolved_path.is_dir(): + return ConanException("Error determining conanfile directory") + + # Create temporary destination directory + temp_dir_create_cmd = f"""{self.remote_python_command} -c "import tempfile; print(tempfile.mkdtemp(dir='{self.remote_workspace}'))""" + _, _stdout, _ = self.client.exec_command(temp_dir_create_cmd) + if _stdout.channel.recv_exit_status() != 0: + raise ConanException("Unable to create remote temporary directory") + self.remote_create_dir = _stdout.read().decode().strip().replace("\\", '/') + + # Copy current folder to destination using sftp + _Path = pathlib.PureWindowsPath if self.remote_is_windows else pathlib.PurePath + sftp = self.client.open_sftp() + for root, dirs, files in os.walk(resolved_path.as_posix()): + relative_root = Path(root).relative_to(resolved_path) + for dir in dirs: + dst = _Path(self.remote_create_dir).joinpath(relative_root).joinpath(dir).as_posix() + sftp.mkdir(dst) + for file in files: + orig = os.path.join(root, file) + dst = _Path(self.remote_create_dir).joinpath(relative_root).joinpath(file).as_posix() + sftp.put(orig, dst) + sftp.close() \ No newline at end of file diff --git a/conans/requirements_dev.txt b/conans/requirements_dev.txt index 9ffa1ea641b..0e1780844e8 100644 --- a/conans/requirements_dev.txt +++ b/conans/requirements_dev.txt @@ -7,3 +7,4 @@ bottle PyJWT pluginbase docker +paramiko From be0fcb01d78a0f29962602c8cbf6b255082862cd Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 20 Mar 2024 20:52:05 +0000 Subject: [PATCH 21/39] Run remote create command --- conan/internal/runner/ssh.py | 37 ++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/conan/internal/runner/ssh.py b/conan/internal/runner/ssh.py index 62703e48904..2e4b6848aa5 100644 --- a/conan/internal/runner/ssh.py +++ b/conan/internal/runner/ssh.py @@ -61,12 +61,19 @@ def run(self, use_cache=True): raw_args = " ".join(raw_args) command = f"{self.remote_conan} create {raw_args}" - # ssh_info(f"Remote command: {command}") + ssh_info(f"Remote command: {command}") - _stdin, _stdout,_stderr = self.client.exec_command(command, get_pty=True) - - # TODO: achieve printing continuous output - + stdout, _ = self._run_command(command) + first_line = True + while not stdout.channel.exit_status_ready(): + line = stdout.channel.recv(1024) + if first_line and self.remote_is_windows: + # Avoid clearing and moving the cursor when the remote server is Windows + # https://github.com/PowerShell/Win32-OpenSSH/issues/1738#issuecomment-789434169 + line = line.replace(b"\x1b[2J\x1b[m\x1b[H",b"") + sys.stdout.buffer.write(line) + sys.stdout.buffer.flush() + first_line = False self.client.close() @@ -219,4 +226,22 @@ def copy_working_conanfile_path(self): orig = os.path.join(root, file) dst = _Path(self.remote_create_dir).joinpath(relative_root).joinpath(file).as_posix() sftp.put(orig, dst) - sftp.close() \ No newline at end of file + sftp.close() + + def _run_command(self, command): + ''' Run a command in an SSH session. + When requesting a pseudo-terminal from the server, + ensure we pass width and height that matches the current + terminal + ''' + channel = self.client.get_transport().open_session() + if sys.stdout.isatty(): + width, height = os.get_terminal_size() + channel.get_pty(width=width, height=height) + + channel.exec_command(command) + + stdout = channel.makefile("r") + stderr = channel.makefile("r") + return stdout, stderr + \ No newline at end of file From f0f7d6fbb570bfa7b179f0ef6a281f54425d31b0 Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 20 Mar 2024 21:19:53 +0000 Subject: [PATCH 22/39] Restore remote cache --- conan/internal/runner/ssh.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/conan/internal/runner/ssh.py b/conan/internal/runner/ssh.py index 2e4b6848aa5..b47f0124739 100644 --- a/conan/internal/runner/ssh.py +++ b/conan/internal/runner/ssh.py @@ -1,5 +1,6 @@ from pathlib import Path import pathlib +import tempfile from conan.api.conan_api import ConfigAPI from conan.api.output import Color, ConanOutput from conans.errors import ConanException @@ -59,7 +60,10 @@ def run(self, use_cache=True): raw_args = self.raw_args raw_args[raw_args.index(self.args.path)] = self.remote_create_dir raw_args = " ".join(raw_args) - command = f"{self.remote_conan} create {raw_args}" + + _Path = pathlib.PureWindowsPath if self.remote_is_windows else pathlib.PurePath + remote_json_output = _Path(self.remote_create_dir).joinpath("conan_create.json").as_posix() + command = f"{self.remote_conan} create {raw_args} --format json > {remote_json_output}" ssh_info(f"Remote command: {command}") @@ -75,8 +79,10 @@ def run(self, use_cache=True): sys.stdout.buffer.flush() first_line = False - self.client.close() + if stdout.channel.recv_exit_status() == 0: + self.update_local_cache(remote_json_output) + # self.client.close() def ensure_runner_environment(self): has_python3_command = False python_is_python3 = False @@ -244,4 +250,24 @@ def _run_command(self, command): stdout = channel.makefile("r") stderr = channel.makefile("r") return stdout, stderr - \ No newline at end of file + + def update_local_cache(self, json_result): + # ('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json' + _Path = pathlib.PureWindowsPath if self.remote_is_windows else pathlib.PurePath + pkg_list_json = _Path(self.remote_create_dir).joinpath("pkg_list.json").as_posix() + pkg_list_command = f"{self.remote_conan} list --graph={json_result} --graph-binaries=build --format=json > {pkg_list_json}" + _, stdout, _ = self.client.exec_command(pkg_list_command) + if stdout.channel.recv_exit_status() != 0: + raise ConanException("Unable to generate remote package list") + + conan_cache_tgz = _Path(self.remote_create_dir).joinpath("cache.tgz").as_posix() + cache_save_command = f"{self.remote_conan} cache save --list {pkg_list_json} --file {conan_cache_tgz}" + _, stdout, _ = self.client.exec_command(cache_save_command) + if stdout.channel.recv_exit_status() != 0: + raise ConanException("Unable to save remote conan cache state") + + sftp = self.client.open_sftp() + with tempfile.TemporaryDirectory() as tmp: + local_cache_tgz = os.path.join(tmp, 'cache.tgz') + sftp.get(conan_cache_tgz, local_cache_tgz) + package_list = self.conan_api.cache.restore(local_cache_tgz) From 700176ec49f0cb9c167cf6ed6bdc43ed8c067157 Mon Sep 17 00:00:00 2001 From: Luis Caro <3535649+jcar87@users.noreply.github.com> Date: Tue, 26 Mar 2024 20:49:06 +0000 Subject: [PATCH 23/39] Add super rough implementation of wsl runner --- conan/cli/commands/create.py | 4 +- conan/internal/runner/wsl.py | 144 +++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 conan/internal/runner/wsl.py diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 60328ceb3d6..81ffa5801d2 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -12,6 +12,7 @@ from conans.client.graph.graph import BINARY_BUILD from conan.internal.runner.docker import DockerRunner from conan.internal.runner.ssh import SSHRunner +from conan.internal.runner.wsl import WSLRunner from conans.util.files import mkdir @@ -68,7 +69,8 @@ def create(conan_api, parser, *args): if profile_host.runner and not os.environ.get("CONAN_RUNNER_ENVIRONMENT"): return { 'docker': DockerRunner, - 'ssh': SSHRunner + 'ssh': SSHRunner, + 'wsl': WSLRunner, }[profile_host.runner.get('type')](conan_api, 'create', profile_host, profile_build, args, raw_args).run() if args.build is not None and args.build_test is None: diff --git a/conan/internal/runner/wsl.py b/conan/internal/runner/wsl.py new file mode 100644 index 00000000000..3e4744b6225 --- /dev/null +++ b/conan/internal/runner/wsl.py @@ -0,0 +1,144 @@ +from pathlib import PurePosixPath, PureWindowsPath, Path +from conan.api.output import Color, ConanOutput +from conans.errors import ConanException +from conans.util.runners import conan_run +from conans.client.subsystems import subsystem_path +from conan.tools.files import save +from io import StringIO +import tempfile +import os + +def wsl_info(msg, error=False): + fg=Color.BRIGHT_MAGENTA + if error: + fg=Color.BRIGHT_RED + ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg) + ConanOutput().status(f'| {msg} |', fg=fg) + ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) + + +class WSLRunner: + def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args): + self.conan_api = conan_api + self.command = command + self.host_profile = host_profile + self.build_profile = build_profile + self.remote_host_profile = None + self.remote_build_profile = None + self.remote_python_command = None + self.remote_conan = None + self.remote_conan_home = None + self.args = args + self.raw_args = raw_args + + # to pass to wsl.exe (optional, otherwise run with defaults) + distro = host_profile.runner.get("distribution", None) + user = host_profile.runner.get("user", None) + + self.shared_cache = host_profile.runner.get("shared_cache", False) + if self.shared_cache: + storage_path = Path(conan_api.config.home()) / 'p' # TODO: there's an API for this!! + self.remote_conan_cache = subsystem_path("wsl", storage_path.as_posix()) + + def run(self): + self.ensure_runner_environment() + + raw_args = self.raw_args + current_path = Path(self.args.path).resolve() + current_path_wsl = subsystem_path("wsl", current_path.as_posix()) + + raw_args[raw_args.index(self.args.path)] = current_path_wsl + raw_args = " ".join(raw_args) + + with tempfile.TemporaryDirectory() as tmp_dir: + if not self.shared_cache: + create_json = PureWindowsPath(tmp_dir).joinpath("create.json").as_posix() + raw_args += f" --format=json > {create_json}" + tmp_dir_wsl = subsystem_path("wsl", tmp_dir) + command = f"wsl.exe --cd {tmp_dir_wsl} -- CONAN_RUNNER_ENVIRONMENT=1 CONAN_HOME={self.remote_conan_home} {self.remote_conan} create {raw_args}" + rc = conan_run(command) + if rc == 0 and not self.shared_cache: + create_json_wsl = subsystem_path("wsl", create_json) + pkglist_json = PureWindowsPath(tmp_dir).joinpath("pkglist.json").as_posix() + pkglist_json_wsl = subsystem_path("wsl", pkglist_json) + + saved_cache = PureWindowsPath(tmp_dir).joinpath("saved_cache.tgz").as_posix() + saved_cache_wsl = subsystem_path("wsl", saved_cache) + conan_run(f"wsl.exe --cd {tmp_dir_wsl} -- CONAN_RUNNER_ENVIRONMENT=1 CONAN_HOME={self.remote_conan_home} {self.remote_conan} list --graph={create_json_wsl} --format=json > {pkglist_json}") + conan_run(f"wsl.exe --cd {tmp_dir_wsl} -- CONAN_RUNNER_ENVIRONMENT=1 CONAN_HOME={self.remote_conan_home} {self.remote_conan} cache save --list={pkglist_json_wsl} --file {saved_cache_wsl}") + self.conan_api.cache.restore(saved_cache) + else: + pass + #print(command) + + def ensure_runner_environment(self): + stdout = StringIO() + stderr = StringIO() + + ret = conan_run('wsl.exe echo $HOME', stdout=stdout) + if ret == 0: + remote_home = PurePosixPath(stdout.getvalue().strip()) + stdout = StringIO() + + remote_conan = remote_home / ".conan2remote" / "venv" / "bin" / "conan" + self.remote_conan = remote_conan.as_posix() + + wsl_info(self.remote_conan) + + conan_home = remote_home / ".conan2remote" / "conan_home" + self.remote_conan_home = conan_home + + has_conan = conan_run(f"wsl.exe CONAN_HOME={conan_home.as_posix()} {remote_conan} --version", stdout=stdout, stderr=stderr) == 0 + + if not has_conan: + wsl_info("Bootstrapping Conan in remote") + conan_run(f"wsl.exe mkdir -p {remote_home}/.conan2remote") + venv = remote_home / ".conan2remote"/ "venv" + python = venv / "bin" / "python" + self.remote_python_command = python + conan_run(f"wsl.exe python3 -m venv {venv.as_posix()}") + conan_run(f"wsl.exe {python} -m pip install pip wheel --upgrade") + conan_run(f"wsl.exe {python} -m pip install git+https://github.com/conan-io/conan@feature/docker_wrapper") + conan_run(f"wsl.exe CONAN_HOME={conan_home.as_posix()} {remote_conan} --version", stdout=stdout) + + remote_conan_version = stdout.getvalue().strip() + wsl_info(f"Remote conan version: {remote_conan_version}") + stdout = StringIO() + stderr = StringIO() + + # If this command succeeds, great - if not because it already exists, ignore + conan_run(f"wsl.exe CONAN_HOME={conan_home.as_posix()} {remote_conan} profile detect", stdout=stdout, stderr=stderr) + + + conf_content = f"core.cache:storage_path={self.remote_conan_cache}\n" if self.shared_cache else "" + with tempfile.TemporaryDirectory() as tmp: + global_conf = os.path.join(tmp, "global.conf") + save(None, path=global_conf, content=conf_content) + global_conf_wsl = subsystem_path("wsl", global_conf) + remote_global_conf = self.remote_conan_home.joinpath("global.conf") + conan_run(f"wsl.exe cp {global_conf_wsl} {remote_global_conf}") + + self._copy_profiles() + + def _copy_profiles(self): + # TODO: questionable choices, may fail + + # Note: see the use of \\wsl$\\, we could place the files + # directly. We would need to work out the exact distro name first + profiles = { + self.args.profile_host[0]: self.host_profile.dumps(), + self.args.profile_build[0]: self.build_profile.dumps() + } + + with tempfile.TemporaryDirectory() as tmp: + # path = os.path.join(tmp, 'something') + for name, contents in profiles.items(): + outfile = os.path.join(tmp, name) + save(None, path=outfile, content=contents) + outfile_wsl = subsystem_path("wsl", outfile) + remote_profile = self.remote_conan_home.joinpath("profiles").as_posix() + "/" + + # This works but copies the file with executable attribute + conan_run(f"wsl.exe cp {outfile_wsl} {remote_profile}") + + From 5e9d5335343a6297861ba470c551bd27de05b014 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Tue, 12 Mar 2024 08:21:17 +0100 Subject: [PATCH 24/39] docker added to dev requirements --- conan/internal/runner/docker copy.py | 295 +++++++++++++++++++++++++++ conans/requirements_dev.txt | 2 + 2 files changed, 297 insertions(+) create mode 100644 conan/internal/runner/docker copy.py diff --git a/conan/internal/runner/docker copy.py b/conan/internal/runner/docker copy.py new file mode 100644 index 00000000000..c49076685f6 --- /dev/null +++ b/conan/internal/runner/docker copy.py @@ -0,0 +1,295 @@ +from collections import namedtuple +import os +import json +import platform +import shutil +import yaml +from conan.api.model import ListPattern +from conan.api.output import Color, ConanOutput +from conan.api.conan_api import ConfigAPI +from conan.cli import make_abs_path +from conan.internal.runner import RunnerException +from conans.client.profile_loader import ProfileLoader +from conans.errors import ConanException +from conans.model.version import Version + + +def config_parser(file_path): + ''' + - image (str) + run: + - name -> name (str) + - containerEnv -> environment (dict) + - containerUser -> user (str or int) + - privileged -> privileged (boolean) + - capAdd -> cap_add (list) + - securityOpt -> security_opt (list) + - mounts -> volumes (dict) + + dockerfile: + - build.dockerfile -> dockerfile (str) -> dockerfile.read() + - build.context -> path (str) + - build.args -> buildargs (dict) + build.options -> X (python extra params [extra_hosts, network_mode, ...]) + - build.cacheFrom -> cache_from (list) + ''' + Build = namedtuple('Build', ['dockerfile', 'build_context', 'build_args', 'cache_from']) + Run = namedtuple('Run', ['name', 'environment', 'user', 'privileged', 'cap_add', 'security_opt', 'volumes']) + Conf = namedtuple('Conf', ['image', 'build', 'run']) + if file_path: + def _instans_or_error(value, obj): + if value and (not isinstance(value, obj)): + raise Exception(f"{value} must be a {obj}") + return value + with open(file_path, 'r') as f: + runnerfile = yaml.safe_load(f) + return Conf( + image=_instans_or_error(runnerfile.get('image'), str), + build=Build( + dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str), + build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str), + build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict), + cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list), + ), + run=Run( + name=_instans_or_error(runnerfile.get('run', {}).get('name'), str), + environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict), + user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str), + privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool), + cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list), + security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), dict), + volumes=_instans_or_error(runnerfile.get('run', {}).get('mounts'), dict), + ) + ) + else: + return Conf( + image=None, + build=Build(dockerfile=None, build_context=None, build_args=None, cache_from=None), + run=Run(name=None, environment=None, user=None, privileged=None, cap_add=None, security_opt=None, volumes=None) + ) + + +def docker_info(msg, error=False): + fg=Color.BRIGHT_MAGENTA + if error: + fg=Color.BRIGHT_RED + ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg) + ConanOutput().status(f'| {msg} |', fg=fg) + ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) + + +def list_patterns(cache_info): + _pattern = [] + for reference, info in cache_info.items(): + for revisions in info.get('revisions', {}).values(): + for package in revisions.get('packages').keys(): + _pattern.append(f'{reference}:{package}') + return _pattern + + +class DockerRunner: + def __init__(self, conan_api, command, profile, args, raw_args): + import docker + import docker.api.build + try: + self.docker_client = docker.from_env() + self.docker_api = docker.APIClient() + docker.api.build.process_dockerfile = lambda dockerfile, path: ('Dockerfile', dockerfile) + except: + raise ConanException("Docker Client failed to initialize." + "\n - Check if docker is installed and running" + "\n - Run 'pip install docker>=5.0.0, <=5.0.3'") + self.conan_api = conan_api + self.args = args + self.abs_host_path = make_abs_path(args.path) + if args.format: + raise ConanException("format argument is forbidden if running in a docker runner") + + # Runner config + self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner') + self.abs_docker_path = os.path.join('/root/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/") + + # Update conan command and some paths to run inside the container + raw_args[raw_args.index(args.path)] = self.abs_docker_path + self.profiles = [] + profile_list = set(self.args.profile_build + self.args.profile_host) + # Update the profile paths + for i, raw_arg in enumerate(raw_args): + for i, raw_profile in enumerate(profile_list): + _profile = ProfileLoader.get_profile_path(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), raw_profile, os.getcwd()) + _name = f'{os.path.basename(_profile)}_{i}' + if raw_profile in raw_arg: + raw_args[raw_args.index(raw_arg)] = raw_arg.replace(raw_profile, os.path.join(self.abs_docker_path, '.conanrunner/profiles', _name)) + self.profiles.append([_profile, os.path.join(self.abs_runner_home_path, 'profiles', _name)]) + self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) + + # Container config + # https://containers.dev/implementors/json_reference/ + self.configfile = config_parser(profile.runner.get('configfile')) + self.dockerfile = profile.runner.get('dockerfile') or self.configfile.build.dockerfile + self.docker_build_context = profile.runner.get('build_context') or self.configfile.build.build_context + self.image = profile.runner.get('image') or self.configfile.image + if not (self.dockerfile or self.image): + raise ConanException("'dockerfile' or docker image name is needed") + self.image = self.image or 'conan-runner-default' + self.name = self.configfile.image or f'conan-runner-{profile.runner.get("suffix", "docker")}' + self.remove = str(profile.runner.get('remove', 'false')).lower() == 'true' + self.cache = str(profile.runner.get('cache', 'clean')) + self.container = None + + def run(self): + """ + run conan inside a Docker continer + """ + if self.dockerfile: + docker_info(f'Building the Docker image: {self.image}') + self.build_image() + volumes, environment = self.create_runner_environment() + error = False + try: + if self.docker_client.containers.list(all=True, filters={'name': self.name}): + docker_info('Starting the docker container') + self.container = self.docker_client.containers.get(self.name) + self.container.start() + else: + if self.configfile.run.environment: + environment.update(self.configfile.run.environment) + if self.configfile.run.volumes: + volumes.update(self.configfile.run.volumes) + docker_info('Creating the docker container') + self.container = self.docker_client.containers.run( + self.image, + "/bin/bash -c 'while true; do sleep 30; done;'", + name=self.name, + volumes=volumes, + environment=environment, + user=self.configfile.run.user, + privileged=self.configfile.run.privileged, + cap_add=self.configfile.run.cap_add, + security_opt=self.configfile.run.security_opt, + detach=True, + auto_remove=False) + docker_info(f'Container {self.name} running') + except Exception as e: + raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"' + f'\n\n{str(e)}') + try: + self.init_container() + self.run_command(self.command) + self.update_local_cache() + except ConanException as e: + error = True + raise e + except RunnerException as e: + error = True + raise ConanException(f'"{e.command}" inside docker fail' + f'\n\nLast command output: {str(e.stdout_log)}') + finally: + if self.container: + error_prefix = 'ERROR: ' if error else '' + docker_info(f'{error_prefix}Stopping container', error) + self.container.stop() + if self.remove: + docker_info(f'{error_prefix}Removing container', error) + self.container.remove() + + def build_image(self): + dockerfile_file_path = self.dockerfile + if os.path.isdir(self.dockerfile): + dockerfile_file_path = os.path.join(self.dockerfile, 'Dockerfile') + with open(dockerfile_file_path) as f: + build_path = self.docker_build_context or os.path.dirname(dockerfile_file_path) + ConanOutput().highlight(f"Dockerfile path: '{dockerfile_file_path}'") + ConanOutput().highlight(f"Docker build context: '{build_path}'\n") + docker_build_logs = self.docker_api.build( + path=build_path, + dockerfile=f.read(), + tag=self.image, + buildargs=self.configfile.build.build_args, + cache_from=self.configfile.build.cache_from, + ) + for chunk in docker_build_logs: + for line in chunk.decode("utf-8").split('\r\n'): + if line: + stream = json.loads(line).get('stream') + if stream: + ConanOutput().status(stream.strip()) + + def run_command(self, command, log=True): + if log: + docker_info(f'Running in container: "{command}"') + exec_instance = self.docker_api.exec_create(self.container.id, f"/bin/bash -c '{command}'", tty=True) + exec_output = self.docker_api.exec_start(exec_instance['Id'], tty=True, stream=True, demux=True,) + stderr_log, stdout_log = '', '' + try: + for (stdout_out, stderr_out) in exec_output: + if stdout_out is not None: + stdout_log += stdout_out.decode('utf-8', errors='ignore').strip() + if log: + ConanOutput().status(stdout_out.decode('utf-8', errors='ignore').strip()) + if stderr_out is not None: + stderr_log += stderr_out.decode('utf-8', errors='ignore').strip() + if log: + ConanOutput().status(stderr_out.decode('utf-8', errors='ignore').strip()) + except Exception as e: + if platform.system() == 'Windows': + import pywintypes + if isinstance(e, pywintypes.error): + pass + else: + raise e + exit_metadata = self.docker_api.exec_inspect(exec_instance['Id']) + if exit_metadata['Running'] or exit_metadata['ExitCode'] > 0: + raise RunnerException(command=command, stdout_log=stdout_log, stderr_log=stderr_log) + return stdout_log, stderr_log + + def create_runner_environment(self): + shutil.rmtree(self.abs_runner_home_path, ignore_errors=True) + volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}} + environment = {'CONAN_RUNNER_ENVIRONMENT': '1'} + if self.cache == 'shared': + volumes[ConfigAPI(self.conan_api).home()] = {'bind': '/root/.conan2', 'mode': 'rw'} + if self.cache in ['clean', 'copy']: + os.mkdir(self.abs_runner_home_path) + os.mkdir(os.path.join(self.abs_runner_home_path, 'profiles')) + + # Copy all conan config files to docker workspace + for file_name in ['global.conf', 'settings.yml', 'remotes.json']: + src_file = os.path.join(ConfigAPI(self.conan_api).home(), file_name) + if os.path.exists(src_file): + shutil.copy(src_file, os.path.join(self.abs_runner_home_path, file_name)) + + # Copy all profiles to docker workspace + for current_path, new_path in self.profiles: + shutil.copy(current_path, new_path) + + if self.cache == 'copy': + tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz') + docker_info(f'Save host cache in: {tgz_path}') + self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path) + return volumes, environment + + def init_container(self): + min_conan_version = '2.1' + stdout, _ = self.run_command('conan --version', log=True) + docker_conan_version = str(stdout.split('Conan version ')[1].replace('\n', '').replace('\r', '')) # Remove all characters and color + print(docker_conan_version) + if Version(docker_conan_version) <= Version(min_conan_version): + ConanOutput().status(f'ERROR: conan version inside the container must be greater than {min_conan_version}', fg=Color.BRIGHT_RED) + raise ConanException( f'conan version inside the container must be greater than {min_conan_version}') + if self.cache != 'shared': + self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) + self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False) + for file_name in ['global.conf', 'settings.yml', 'remotes.json']: + if os.path.exists( os.path.join(self.abs_runner_home_path, file_name)): + self.run_command('cp "'+self.abs_docker_path+'/.conanrunner/'+file_name+'" ${HOME}/.conan2/'+file_name, log=False) + if self.cache in ['copy']: + self.run_command('conan cache restore "'+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz"') + + def update_local_cache(self): + if self.cache != 'shared': + self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json', log=False) + self.run_command('conan cache save --list=pkglist.json --file "'+self.abs_docker_path+'"/.conanrunner/docker_cache_save.tgz') + tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz') + docker_info(f'Restore host cache from: {tgz_path}') + package_list = self.conan_api.cache.restore(tgz_path) diff --git a/conans/requirements_dev.txt b/conans/requirements_dev.txt index 0e1780844e8..1a57f505dfd 100644 --- a/conans/requirements_dev.txt +++ b/conans/requirements_dev.txt @@ -8,3 +8,5 @@ PyJWT pluginbase docker paramiko +docker>=5.0.0, <6.0.0 + From 847c6637783ccd2b39b90a1449fd6b0e6cd0e7b2 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Tue, 12 Mar 2024 21:35:38 +0100 Subject: [PATCH 25/39] docker conan version check added --- conan/internal/runner/docker.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 305089974cc..d1e77231308 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -95,10 +95,8 @@ def run(self): self.run_command(self.command) self.update_local_cache() except ConanException as e: - error = True raise e except RunnerException as e: - error = True raise ConanException(f'"{e.command}" inside docker fail' f'\n\nLast command output: {str(e.stdout_log)}') finally: From 7fda3b5ebe3cc036fc36cb0f8aa2527c7c8d2628 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Tue, 19 Mar 2024 17:34:25 +0100 Subject: [PATCH 26/39] clean test added --- .../test/integration/command/runner_test.py | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py index ef695e297fb..9304143c93a 100644 --- a/conans/test/integration/command/runner_test.py +++ b/conans/test/integration/command/runner_test.py @@ -35,7 +35,7 @@ def dockerfile_path(name=None): @pytest.mark.skipif(docker_skip(), reason="Only docker running") -@pytest.mark.xfail(reason="conan inside docker optional test") +# @pytest.mark.xfail(reason="conan inside docker optional test") def test_create_docker_runner_dockerfile_folder_path(): """ Tests the ``conan create . `` @@ -51,7 +51,7 @@ def test_create_docker_runner_dockerfile_folder_path(): compiler.version=11 os=Linux """) - profile_host = textwrap.dedent(f"""\ + profile_host_copy = textwrap.dedent(f"""\ [settings] arch=x86_64 build_type=Release @@ -68,6 +68,25 @@ def test_create_docker_runner_dockerfile_folder_path(): cache=copy remove=True """) + + profile_host_clean = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path()} + docker_build_context={conan_base_path()} + image=conan-runner-default-test + cache=clean + remove=True + """) + conanfile = textwrap.dedent(""" from conan import ConanFile @@ -85,8 +104,15 @@ class MyTest(ConanFile): options = {"shared": [True, False], "fPIC": [True, False]} default_options = {"shared": False, "fPIC": True} """) - client.save({"conanfile.py": conanfile, "host": profile_host, "build": profile_build}) - client.run("create . -pr:h host -pr:b build") + client.save({"conanfile.py": conanfile, "host_copy": profile_host_copy, "host_clean": profile_host_clean, "build": profile_build}) + client.run("create . -pr:h host_copy -pr:b build") + + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out + assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out + assert "Removing container" in client.out + + client.run("create . -pr:h host_clean -pr:b build") assert "Restore: pkg/0.2" in client.out assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out @@ -95,7 +121,7 @@ class MyTest(ConanFile): @pytest.mark.skipif(docker_skip(), reason="Only docker running") -@pytest.mark.xfail(reason="conan inside docker optional test") +# @pytest.mark.xfail(reason="conan inside docker optional test") def test_create_docker_runner_dockerfile_file_path(): """ Tests the ``conan create . `` @@ -155,7 +181,7 @@ class MyTest(ConanFile): @pytest.mark.skipif(docker_skip(), reason="Only docker running") -@pytest.mark.xfail(reason="conan inside docker optional test") +# @pytest.mark.xfail(reason="conan inside docker optional test") @pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) @pytest.mark.tool("ninja") def test_create_docker_runner_with_ninja(build_type, shared): From 9a9e2981873455b11d42c765eac519ab26ac2394 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Wed, 20 Mar 2024 17:44:23 +0100 Subject: [PATCH 27/39] profile runner bug fixed --- conan/internal/runner/docker.py | 44 +++++++++---- .../test/integration/command/runner_test.py | 64 ++++++++++++++++++- 2 files changed, 91 insertions(+), 17 deletions(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index d1e77231308..c0b444edeb1 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -2,6 +2,7 @@ import json import platform import shutil +import itertools from conan.api.model import ListPattern from conan.api.output import Color, ConanOutput from conan.api.conan_api import ConfigAPI @@ -47,10 +48,26 @@ def __init__(self, conan_api, command, profile, args, raw_args): self.abs_host_path = make_abs_path(args.path) if args.format: raise ConanException("format argument is forbidden if running in a docker runner") + + # Runner config self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner') self.abs_docker_path = os.path.join('/root/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/") - raw_args[raw_args.index(args.path)] = f'"{self.abs_docker_path}"' - self.command = ' '.join([f'conan {command}'] + raw_args + ['-f json > create.json']) + + # Update conan command and some paths to run inside the container + raw_args[raw_args.index(args.path)] = self.abs_docker_path + self.profiles = [] + profile_list = set(self.args.profile_build + self.args.profile_host) + # Update the profile paths + for i, raw_arg in enumerate(raw_args): + for i, raw_profile in enumerate(profile_list): + _profile = ProfileLoader.get_profile_path(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), raw_profile, os.getcwd()) + _name = f'{os.path.basename(_profile)}_{i}' + if raw_profile in raw_arg: + raw_args[raw_args.index(raw_arg)] = raw_arg.replace(raw_profile, os.path.join(self.abs_docker_path, '.conanrunner/profiles', _name)) + self.profiles.append([_profile, os.path.join(self.abs_runner_home_path, 'profiles', _name)]) + self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) + + # Container config self.dockerfile = profile.runner.get('dockerfile') self.docker_build_context = profile.runner.get('docker_build_context') self.image = profile.runner.get('image') @@ -153,20 +170,25 @@ def run_command(self, command, log=True): return stdout_log, stderr_log def create_runner_environment(self): + shutil.rmtree(self.abs_runner_home_path, ignore_errors=True) volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}} environment = {'CONAN_RUNNER_ENVIRONMENT': '1'} if self.cache == 'shared': volumes[ConfigAPI(self.conan_api).home()] = {'bind': '/root/.conan2', 'mode': 'rw'} if self.cache in ['clean', 'copy']: - shutil.rmtree(self.abs_runner_home_path, ignore_errors=True) os.mkdir(self.abs_runner_home_path) os.mkdir(os.path.join(self.abs_runner_home_path, 'profiles')) + + # Copy all conan config files to docker workspace for file_name in ['global.conf', 'settings.yml', 'remotes.json']: src_file = os.path.join(ConfigAPI(self.conan_api).home(), file_name) if os.path.exists(src_file): shutil.copy(src_file, os.path.join(self.abs_runner_home_path, file_name)) - self._copy_profiles(self.args.profile_build) - self._copy_profiles(self.args.profile_host) + + # Copy all profiles to docker workspace + for current_path, new_path in self.profiles: + shutil.copy(current_path, new_path) + if self.cache == 'copy': tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz') docker_info(f'Save host cache in: {tgz_path}') @@ -176,11 +198,12 @@ def create_runner_environment(self): def init_container(self): min_conan_version = '2.1' stdout, _ = self.run_command('conan --version', log=True) - docker_conan_version = str(stdout.replace('Conan version ', '').replace('\n', '').replace('\r', '')) # Remove all characters and color + docker_conan_version = str(stdout.split('Conan version ')[1].replace('\n', '').replace('\r', '')) # Remove all characters and color + print(docker_conan_version) if Version(docker_conan_version) <= Version(min_conan_version): ConanOutput().status(f'ERROR: conan version inside the container must be greater than {min_conan_version}', fg=Color.BRIGHT_RED) raise ConanException( f'conan version inside the container must be greater than {min_conan_version}') - if self.cache == 'shared': + if self.cache != 'shared': self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False) for file_name in ['global.conf', 'settings.yml', 'remotes.json']: @@ -196,10 +219,3 @@ def update_local_cache(self): tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz') docker_info(f'Restore host cache from: {tgz_path}') package_list = self.conan_api.cache.restore(tgz_path) - - def _copy_profiles(self, profiles): - cwd = os.getcwd() - if profiles: - for profile in profiles: - profile_path = ProfileLoader.get_profile_path(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), profile, cwd) - shutil.copy(profile_path, os.path.join(self.abs_runner_home_path, 'profiles', os.path.basename(profile_path))) diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py index 9304143c93a..5d041442017 100644 --- a/conans/test/integration/command/runner_test.py +++ b/conans/test/integration/command/runner_test.py @@ -242,7 +242,65 @@ def package(self): client.save({"profile": profile}) settings = "-s os=Linux -s arch=x86_64 -s build_type={} -o hello/*:shared={}".format(build_type, shared) # create should also work - client.run("create . --name=hello --version=1.0 {} -pr:h profile -pr:b profile".format(settings)) - print(client.out) + client.run("create . --name=hello --version=1.0 {} -pr:h=profile -pr:b=profile".format(settings)) assert 'cmake -G "Ninja"' in client.out - assert "main: {}!".format(build_type) in client.out \ No newline at end of file + assert "main: {}!".format(build_type) in client.out + +@pytest.mark.skipif(docker_skip(), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_profile_abs_path(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + profile_build = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + """) + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path("Dockerfile_test")} + docker_build_context={conan_base_path()} + image=conan-runner-default-test + cache=copy + remove=True + """) + conanfile = textwrap.dedent(""" + from conan import ConanFile + + class MyTest(ConanFile): + name = "pkg" + version = "0.2" + settings = "build_type", "compiler" + author = "John Doe" + license = "MIT" + url = "https://foo.bar.baz" + homepage = "https://foo.bar.site" + topics = "foo", "bar", "qux" + provides = "libjpeg", "libjpg" + deprecated = "other-pkg" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + """) + client.save({"conanfile.py": conanfile, "host": profile_host, "build": profile_build}) + client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'") + + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out + assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out + assert "Removing container" in client.out \ No newline at end of file From 84f66030d74f97c8092feff5ed6067c617b86225 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Wed, 10 Apr 2024 18:36:19 +0200 Subject: [PATCH 28/39] docker configfile added --- conan/internal/runner/docker copy.py | 295 ------------------ conan/internal/runner/docker.py | 95 +++++- .../test/integration/command/runner_test.py | 77 ++++- 3 files changed, 157 insertions(+), 310 deletions(-) delete mode 100644 conan/internal/runner/docker copy.py diff --git a/conan/internal/runner/docker copy.py b/conan/internal/runner/docker copy.py deleted file mode 100644 index c49076685f6..00000000000 --- a/conan/internal/runner/docker copy.py +++ /dev/null @@ -1,295 +0,0 @@ -from collections import namedtuple -import os -import json -import platform -import shutil -import yaml -from conan.api.model import ListPattern -from conan.api.output import Color, ConanOutput -from conan.api.conan_api import ConfigAPI -from conan.cli import make_abs_path -from conan.internal.runner import RunnerException -from conans.client.profile_loader import ProfileLoader -from conans.errors import ConanException -from conans.model.version import Version - - -def config_parser(file_path): - ''' - - image (str) - run: - - name -> name (str) - - containerEnv -> environment (dict) - - containerUser -> user (str or int) - - privileged -> privileged (boolean) - - capAdd -> cap_add (list) - - securityOpt -> security_opt (list) - - mounts -> volumes (dict) - - dockerfile: - - build.dockerfile -> dockerfile (str) -> dockerfile.read() - - build.context -> path (str) - - build.args -> buildargs (dict) - build.options -> X (python extra params [extra_hosts, network_mode, ...]) - - build.cacheFrom -> cache_from (list) - ''' - Build = namedtuple('Build', ['dockerfile', 'build_context', 'build_args', 'cache_from']) - Run = namedtuple('Run', ['name', 'environment', 'user', 'privileged', 'cap_add', 'security_opt', 'volumes']) - Conf = namedtuple('Conf', ['image', 'build', 'run']) - if file_path: - def _instans_or_error(value, obj): - if value and (not isinstance(value, obj)): - raise Exception(f"{value} must be a {obj}") - return value - with open(file_path, 'r') as f: - runnerfile = yaml.safe_load(f) - return Conf( - image=_instans_or_error(runnerfile.get('image'), str), - build=Build( - dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str), - build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str), - build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict), - cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list), - ), - run=Run( - name=_instans_or_error(runnerfile.get('run', {}).get('name'), str), - environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict), - user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str), - privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool), - cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list), - security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), dict), - volumes=_instans_or_error(runnerfile.get('run', {}).get('mounts'), dict), - ) - ) - else: - return Conf( - image=None, - build=Build(dockerfile=None, build_context=None, build_args=None, cache_from=None), - run=Run(name=None, environment=None, user=None, privileged=None, cap_add=None, security_opt=None, volumes=None) - ) - - -def docker_info(msg, error=False): - fg=Color.BRIGHT_MAGENTA - if error: - fg=Color.BRIGHT_RED - ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg) - ConanOutput().status(f'| {msg} |', fg=fg) - ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) - - -def list_patterns(cache_info): - _pattern = [] - for reference, info in cache_info.items(): - for revisions in info.get('revisions', {}).values(): - for package in revisions.get('packages').keys(): - _pattern.append(f'{reference}:{package}') - return _pattern - - -class DockerRunner: - def __init__(self, conan_api, command, profile, args, raw_args): - import docker - import docker.api.build - try: - self.docker_client = docker.from_env() - self.docker_api = docker.APIClient() - docker.api.build.process_dockerfile = lambda dockerfile, path: ('Dockerfile', dockerfile) - except: - raise ConanException("Docker Client failed to initialize." - "\n - Check if docker is installed and running" - "\n - Run 'pip install docker>=5.0.0, <=5.0.3'") - self.conan_api = conan_api - self.args = args - self.abs_host_path = make_abs_path(args.path) - if args.format: - raise ConanException("format argument is forbidden if running in a docker runner") - - # Runner config - self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner') - self.abs_docker_path = os.path.join('/root/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/") - - # Update conan command and some paths to run inside the container - raw_args[raw_args.index(args.path)] = self.abs_docker_path - self.profiles = [] - profile_list = set(self.args.profile_build + self.args.profile_host) - # Update the profile paths - for i, raw_arg in enumerate(raw_args): - for i, raw_profile in enumerate(profile_list): - _profile = ProfileLoader.get_profile_path(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), raw_profile, os.getcwd()) - _name = f'{os.path.basename(_profile)}_{i}' - if raw_profile in raw_arg: - raw_args[raw_args.index(raw_arg)] = raw_arg.replace(raw_profile, os.path.join(self.abs_docker_path, '.conanrunner/profiles', _name)) - self.profiles.append([_profile, os.path.join(self.abs_runner_home_path, 'profiles', _name)]) - self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) - - # Container config - # https://containers.dev/implementors/json_reference/ - self.configfile = config_parser(profile.runner.get('configfile')) - self.dockerfile = profile.runner.get('dockerfile') or self.configfile.build.dockerfile - self.docker_build_context = profile.runner.get('build_context') or self.configfile.build.build_context - self.image = profile.runner.get('image') or self.configfile.image - if not (self.dockerfile or self.image): - raise ConanException("'dockerfile' or docker image name is needed") - self.image = self.image or 'conan-runner-default' - self.name = self.configfile.image or f'conan-runner-{profile.runner.get("suffix", "docker")}' - self.remove = str(profile.runner.get('remove', 'false')).lower() == 'true' - self.cache = str(profile.runner.get('cache', 'clean')) - self.container = None - - def run(self): - """ - run conan inside a Docker continer - """ - if self.dockerfile: - docker_info(f'Building the Docker image: {self.image}') - self.build_image() - volumes, environment = self.create_runner_environment() - error = False - try: - if self.docker_client.containers.list(all=True, filters={'name': self.name}): - docker_info('Starting the docker container') - self.container = self.docker_client.containers.get(self.name) - self.container.start() - else: - if self.configfile.run.environment: - environment.update(self.configfile.run.environment) - if self.configfile.run.volumes: - volumes.update(self.configfile.run.volumes) - docker_info('Creating the docker container') - self.container = self.docker_client.containers.run( - self.image, - "/bin/bash -c 'while true; do sleep 30; done;'", - name=self.name, - volumes=volumes, - environment=environment, - user=self.configfile.run.user, - privileged=self.configfile.run.privileged, - cap_add=self.configfile.run.cap_add, - security_opt=self.configfile.run.security_opt, - detach=True, - auto_remove=False) - docker_info(f'Container {self.name} running') - except Exception as e: - raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"' - f'\n\n{str(e)}') - try: - self.init_container() - self.run_command(self.command) - self.update_local_cache() - except ConanException as e: - error = True - raise e - except RunnerException as e: - error = True - raise ConanException(f'"{e.command}" inside docker fail' - f'\n\nLast command output: {str(e.stdout_log)}') - finally: - if self.container: - error_prefix = 'ERROR: ' if error else '' - docker_info(f'{error_prefix}Stopping container', error) - self.container.stop() - if self.remove: - docker_info(f'{error_prefix}Removing container', error) - self.container.remove() - - def build_image(self): - dockerfile_file_path = self.dockerfile - if os.path.isdir(self.dockerfile): - dockerfile_file_path = os.path.join(self.dockerfile, 'Dockerfile') - with open(dockerfile_file_path) as f: - build_path = self.docker_build_context or os.path.dirname(dockerfile_file_path) - ConanOutput().highlight(f"Dockerfile path: '{dockerfile_file_path}'") - ConanOutput().highlight(f"Docker build context: '{build_path}'\n") - docker_build_logs = self.docker_api.build( - path=build_path, - dockerfile=f.read(), - tag=self.image, - buildargs=self.configfile.build.build_args, - cache_from=self.configfile.build.cache_from, - ) - for chunk in docker_build_logs: - for line in chunk.decode("utf-8").split('\r\n'): - if line: - stream = json.loads(line).get('stream') - if stream: - ConanOutput().status(stream.strip()) - - def run_command(self, command, log=True): - if log: - docker_info(f'Running in container: "{command}"') - exec_instance = self.docker_api.exec_create(self.container.id, f"/bin/bash -c '{command}'", tty=True) - exec_output = self.docker_api.exec_start(exec_instance['Id'], tty=True, stream=True, demux=True,) - stderr_log, stdout_log = '', '' - try: - for (stdout_out, stderr_out) in exec_output: - if stdout_out is not None: - stdout_log += stdout_out.decode('utf-8', errors='ignore').strip() - if log: - ConanOutput().status(stdout_out.decode('utf-8', errors='ignore').strip()) - if stderr_out is not None: - stderr_log += stderr_out.decode('utf-8', errors='ignore').strip() - if log: - ConanOutput().status(stderr_out.decode('utf-8', errors='ignore').strip()) - except Exception as e: - if platform.system() == 'Windows': - import pywintypes - if isinstance(e, pywintypes.error): - pass - else: - raise e - exit_metadata = self.docker_api.exec_inspect(exec_instance['Id']) - if exit_metadata['Running'] or exit_metadata['ExitCode'] > 0: - raise RunnerException(command=command, stdout_log=stdout_log, stderr_log=stderr_log) - return stdout_log, stderr_log - - def create_runner_environment(self): - shutil.rmtree(self.abs_runner_home_path, ignore_errors=True) - volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}} - environment = {'CONAN_RUNNER_ENVIRONMENT': '1'} - if self.cache == 'shared': - volumes[ConfigAPI(self.conan_api).home()] = {'bind': '/root/.conan2', 'mode': 'rw'} - if self.cache in ['clean', 'copy']: - os.mkdir(self.abs_runner_home_path) - os.mkdir(os.path.join(self.abs_runner_home_path, 'profiles')) - - # Copy all conan config files to docker workspace - for file_name in ['global.conf', 'settings.yml', 'remotes.json']: - src_file = os.path.join(ConfigAPI(self.conan_api).home(), file_name) - if os.path.exists(src_file): - shutil.copy(src_file, os.path.join(self.abs_runner_home_path, file_name)) - - # Copy all profiles to docker workspace - for current_path, new_path in self.profiles: - shutil.copy(current_path, new_path) - - if self.cache == 'copy': - tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz') - docker_info(f'Save host cache in: {tgz_path}') - self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path) - return volumes, environment - - def init_container(self): - min_conan_version = '2.1' - stdout, _ = self.run_command('conan --version', log=True) - docker_conan_version = str(stdout.split('Conan version ')[1].replace('\n', '').replace('\r', '')) # Remove all characters and color - print(docker_conan_version) - if Version(docker_conan_version) <= Version(min_conan_version): - ConanOutput().status(f'ERROR: conan version inside the container must be greater than {min_conan_version}', fg=Color.BRIGHT_RED) - raise ConanException( f'conan version inside the container must be greater than {min_conan_version}') - if self.cache != 'shared': - self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) - self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False) - for file_name in ['global.conf', 'settings.yml', 'remotes.json']: - if os.path.exists( os.path.join(self.abs_runner_home_path, file_name)): - self.run_command('cp "'+self.abs_docker_path+'/.conanrunner/'+file_name+'" ${HOME}/.conan2/'+file_name, log=False) - if self.cache in ['copy']: - self.run_command('conan cache restore "'+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz"') - - def update_local_cache(self): - if self.cache != 'shared': - self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json', log=False) - self.run_command('conan cache save --list=pkglist.json --file "'+self.abs_docker_path+'"/.conanrunner/docker_cache_save.tgz') - tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz') - docker_info(f'Restore host cache from: {tgz_path}') - package_list = self.conan_api.cache.restore(tgz_path) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index c0b444edeb1..4e4135e1742 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -1,8 +1,9 @@ +from collections import namedtuple import os import json import platform import shutil -import itertools +import yaml from conan.api.model import ListPattern from conan.api.output import Color, ConanOutput from conan.api.conan_api import ConfigAPI @@ -13,6 +14,61 @@ from conans.model.version import Version +def config_parser(file_path): + ''' + - image (str) + run: + - name -> name (str) + - containerEnv -> environment (dict) + - containerUser -> user (str or int) + - privileged -> privileged (boolean) + - capAdd -> cap_add (list) + - securityOpt -> security_opt (list) + - mounts -> volumes (dict) + + dockerfile: + - build.dockerfile -> dockerfile (str) -> dockerfile.read() + - build.context -> path (str) + - build.args -> buildargs (dict) + build.options -> X (python extra params [extra_hosts, network_mode, ...]) + - build.cacheFrom -> cache_from (list) + ''' + Build = namedtuple('Build', ['dockerfile', 'build_context', 'build_args', 'cache_from']) + Run = namedtuple('Run', ['name', 'environment', 'user', 'privileged', 'cap_add', 'security_opt', 'volumes']) + Conf = namedtuple('Conf', ['image', 'build', 'run']) + if file_path: + def _instans_or_error(value, obj): + if value and (not isinstance(value, obj)): + raise Exception(f"{value} must be a {obj}") + return value + with open(file_path, 'r') as f: + runnerfile = yaml.safe_load(f) + return Conf( + image=_instans_or_error(runnerfile.get('image'), str), + build=Build( + dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str), + build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str), + build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict), + cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list), + ), + run=Run( + name=_instans_or_error(runnerfile.get('run', {}).get('name'), str), + environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict), + user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str), + privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool), + cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list), + security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), dict), + volumes=_instans_or_error(runnerfile.get('run', {}).get('mounts'), dict), + ) + ) + else: + return Conf( + image=None, + build=Build(dockerfile=None, build_context=None, build_args=None, cache_from=None), + run=Run(name=None, environment=None, user=None, privileged=None, cap_add=None, security_opt=None, volumes=None) + ) + + def docker_info(msg, error=False): fg=Color.BRIGHT_MAGENTA if error: @@ -32,7 +88,7 @@ def list_patterns(cache_info): class DockerRunner: - def __init__(self, conan_api, command, profile, args, raw_args): + def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args): import docker import docker.api.build try: @@ -44,6 +100,7 @@ def __init__(self, conan_api, command, profile, args, raw_args): "\n - Check if docker is installed and running" "\n - Run 'pip install docker>=5.0.0, <=5.0.3'") self.conan_api = conan_api + self.build_profile = build_profile self.args = args self.abs_host_path = make_abs_path(args.path) if args.format: @@ -68,15 +125,17 @@ def __init__(self, conan_api, command, profile, args, raw_args): self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) # Container config - self.dockerfile = profile.runner.get('dockerfile') - self.docker_build_context = profile.runner.get('docker_build_context') - self.image = profile.runner.get('image') + # https://containers.dev/implementors/json_reference/ + self.configfile = config_parser(host_profile.runner.get('configfile')) + self.dockerfile = host_profile.runner.get('dockerfile') or self.configfile.build.dockerfile + self.docker_build_context = host_profile.runner.get('build_context') or self.configfile.build.build_context + self.image = host_profile.runner.get('image') or self.configfile.image if not (self.dockerfile or self.image): raise ConanException("'dockerfile' or docker image name is needed") self.image = self.image or 'conan-runner-default' - self.name = f'conan-runner-{profile.runner.get("suffix", "docker")}' - self.remove = str(profile.runner.get('remove', 'false')).lower() == 'true' - self.cache = str(profile.runner.get('cache', 'clean')) + self.name = self.configfile.image or f'conan-runner-{host_profile.runner.get("suffix", "docker")}' + self.remove = str(host_profile.runner.get('remove', 'false')).lower() == 'true' + self.cache = str(host_profile.runner.get('cache', 'clean')) self.container = None def run(self): @@ -94,6 +153,10 @@ def run(self): self.container = self.docker_client.containers.get(self.name) self.container.start() else: + if self.configfile.run.environment: + environment.update(self.configfile.run.environment) + if self.configfile.run.volumes: + volumes.update(self.configfile.run.volumes) docker_info('Creating the docker container') self.container = self.docker_client.containers.run( self.image, @@ -101,6 +164,10 @@ def run(self): name=self.name, volumes=volumes, environment=environment, + user=self.configfile.run.user, + privileged=self.configfile.run.privileged, + cap_add=self.configfile.run.cap_add, + security_opt=self.configfile.run.security_opt, detach=True, auto_remove=False) docker_info(f'Container {self.name} running') @@ -112,8 +179,10 @@ def run(self): self.run_command(self.command) self.update_local_cache() except ConanException as e: + error = True raise e except RunnerException as e: + error = True raise ConanException(f'"{e.command}" inside docker fail' f'\n\nLast command output: {str(e.stdout_log)}') finally: @@ -128,12 +197,18 @@ def run(self): def build_image(self): dockerfile_file_path = self.dockerfile if os.path.isdir(self.dockerfile): - dockerfile_file_path = os.path.join(self.dockerfile, 'Dockerfile') + dockerfile_file_path = os.path.join(self.dockerfile, 'Dockerfile') with open(dockerfile_file_path) as f: build_path = self.docker_build_context or os.path.dirname(dockerfile_file_path) ConanOutput().highlight(f"Dockerfile path: '{dockerfile_file_path}'") ConanOutput().highlight(f"Docker build context: '{build_path}'\n") - docker_build_logs = self.docker_api.build(path=build_path, dockerfile=f.read(), tag=self.image) + docker_build_logs = self.docker_api.build( + path=build_path, + dockerfile=f.read(), + tag=self.image, + buildargs=self.configfile.build.build_args, + cache_from=self.configfile.build.cache_from, + ) for chunk in docker_build_logs: for line in chunk.decode("utf-8").split('\r\n'): if line: diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py index 5d041442017..aab2eeb0e30 100644 --- a/conans/test/integration/command/runner_test.py +++ b/conans/test/integration/command/runner_test.py @@ -63,7 +63,7 @@ def test_create_docker_runner_dockerfile_folder_path(): [runner] type=docker dockerfile={dockerfile_path()} - docker_build_context={conan_base_path()} + build_context={conan_base_path()} image=conan-runner-default-test cache=copy remove=True @@ -81,7 +81,7 @@ def test_create_docker_runner_dockerfile_folder_path(): [runner] type=docker dockerfile={dockerfile_path()} - docker_build_context={conan_base_path()} + build_context={conan_base_path()} image=conan-runner-default-test cache=clean remove=True @@ -149,7 +149,7 @@ def test_create_docker_runner_dockerfile_file_path(): [runner] type=docker dockerfile={dockerfile_path("Dockerfile_test")} - docker_build_context={conan_base_path()} + build_context={conan_base_path()} image=conan-runner-default-test cache=copy remove=True @@ -235,7 +235,7 @@ def package(self): type=docker image=conan-runner-ninja-test dockerfile={dockerfile_path("Dockerfile_ninja")} - docker_build_context={conan_base_path()} + build_context={conan_base_path()} cache=copy remove=True """) @@ -275,7 +275,7 @@ def test_create_docker_runner_profile_abs_path(): [runner] type=docker dockerfile={dockerfile_path("Dockerfile_test")} - docker_build_context={conan_base_path()} + build_context={conan_base_path()} image=conan-runner-default-test cache=copy remove=True @@ -300,6 +300,73 @@ class MyTest(ConanFile): client.save({"conanfile.py": conanfile, "host": profile_host, "build": profile_build}) client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'") + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out + assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out + assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip(), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_profile_abs_path_from_configfile(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + configfile = textwrap.dedent(f""" + image: conan-runner-default-test + build: + dockerfile: {dockerfile_path("Dockerfile_test")} + build_context: {conan_base_path()} + """) + client.save({"configfile.yaml": configfile}) + + + profile_build = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + """) + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + configfile={os.path.join(client.current_folder, 'configfile.yaml')} + cache=copy + remove=True + """) + conanfile = textwrap.dedent(""" + from conan import ConanFile + + class MyTest(ConanFile): + name = "pkg" + version = "0.2" + settings = "build_type", "compiler" + author = "John Doe" + license = "MIT" + url = "https://foo.bar.baz" + homepage = "https://foo.bar.site" + topics = "foo", "bar", "qux" + provides = "libjpeg", "libjpg" + deprecated = "other-pkg" + options = {"shared": [True, False], "fPIC": [True, False]} + default_options = {"shared": False, "fPIC": True} + """) + client.save({"conanfile.py": conanfile, "host": profile_host, "build": profile_build}) + client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'") + assert "Restore: pkg/0.2" in client.out assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out From 8fecd97a716c579ba24c5ded75ab0099cc700136 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Mon, 22 Apr 2024 09:02:49 +0200 Subject: [PATCH 29/39] conans/requirements_runner.txt added --- conans/requirements_dev.txt | 3 --- conans/requirements_runner.txt | 2 ++ setup.py | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 conans/requirements_runner.txt diff --git a/conans/requirements_dev.txt b/conans/requirements_dev.txt index 1a57f505dfd..dd09b1800eb 100644 --- a/conans/requirements_dev.txt +++ b/conans/requirements_dev.txt @@ -6,7 +6,4 @@ WebTest>=2.0.18, <2.1.0 bottle PyJWT pluginbase -docker -paramiko docker>=5.0.0, <6.0.0 - diff --git a/conans/requirements_runner.txt b/conans/requirements_runner.txt new file mode 100644 index 00000000000..ac5f6f5065e --- /dev/null +++ b/conans/requirements_runner.txt @@ -0,0 +1,2 @@ +paramiko +docker>=5.0.0, <6.0.0 diff --git a/setup.py b/setup.py index fc2bca6388c..f61b72ae21f 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def generate_long_description_file(): project_requirements = get_requires("conans/requirements.txt") dev_requirements = get_requires("conans/requirements_dev.txt") -docker_requirements = ['docker'] +runners_requirements = get_requires("conans/requirements_runner.txt") excluded_server_packages = ["conans.server*"] exclude = excluded_test_packages + excluded_server_packages @@ -113,11 +113,11 @@ def generate_long_description_file(): # List additional groups of dependencies here (e.g. development # dependencies). You can install these using the following syntax, # for example: - # $ pip install -e .[dev,test,docker] + # $ pip install -e .[dev,test,runners] extras_require={ 'dev': dev_requirements, 'test': dev_requirements, - 'docker': docker_requirements + 'runners': runners_requirements }, # If there are data files included in your packages that need to be From 07232cd4daae3ec90a3ddc2b28b7415ac549eaa0 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Tue, 23 Apr 2024 09:12:28 +0200 Subject: [PATCH 30/39] docker runner test skip fixed --- conans/requirements_dev.txt | 1 + conans/test/integration/command/runner_test.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/conans/requirements_dev.txt b/conans/requirements_dev.txt index dd09b1800eb..c81ee6c559c 100644 --- a/conans/requirements_dev.txt +++ b/conans/requirements_dev.txt @@ -7,3 +7,4 @@ bottle PyJWT pluginbase docker>=5.0.0, <6.0.0 +setuptools \ No newline at end of file diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py index aab2eeb0e30..510fa4218e4 100644 --- a/conans/test/integration/command/runner_test.py +++ b/conans/test/integration/command/runner_test.py @@ -12,7 +12,7 @@ def docker_skip(test_image=None): try: docker_client = docker.from_env() if test_image: - docker_client.images.get(test_image) + docker_client.images.pull(test_image) except docker.errors.DockerException: return True except docker.errors.ImageNotFound: @@ -34,7 +34,7 @@ def dockerfile_path(name=None): return path -@pytest.mark.skipif(docker_skip(), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") # @pytest.mark.xfail(reason="conan inside docker optional test") def test_create_docker_runner_dockerfile_folder_path(): """ @@ -120,7 +120,7 @@ class MyTest(ConanFile): assert "Removing container" in client.out -@pytest.mark.skipif(docker_skip(), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") # @pytest.mark.xfail(reason="conan inside docker optional test") def test_create_docker_runner_dockerfile_file_path(): """ @@ -180,7 +180,7 @@ class MyTest(ConanFile): assert "Removing container" in client.out -@pytest.mark.skipif(docker_skip(), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") # @pytest.mark.xfail(reason="conan inside docker optional test") @pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) @pytest.mark.tool("ninja") @@ -246,7 +246,7 @@ def package(self): assert 'cmake -G "Ninja"' in client.out assert "main: {}!".format(build_type) in client.out -@pytest.mark.skipif(docker_skip(), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") # @pytest.mark.xfail(reason="conan inside docker optional test") def test_create_docker_runner_profile_abs_path(): """ @@ -306,7 +306,7 @@ class MyTest(ConanFile): assert "Removing container" in client.out -@pytest.mark.skipif(docker_skip(), reason="Only docker running") +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") # @pytest.mark.xfail(reason="conan inside docker optional test") def test_create_docker_runner_profile_abs_path_from_configfile(): """ From 8056359bdf5ababda306ee282f16009a7770c04e Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Mon, 29 Apr 2024 18:28:36 +0200 Subject: [PATCH 31/39] configfile docker fixed --- conan/internal/runner/docker.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 4e4135e1742..333a774b5c6 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -15,24 +15,6 @@ def config_parser(file_path): - ''' - - image (str) - run: - - name -> name (str) - - containerEnv -> environment (dict) - - containerUser -> user (str or int) - - privileged -> privileged (boolean) - - capAdd -> cap_add (list) - - securityOpt -> security_opt (list) - - mounts -> volumes (dict) - - dockerfile: - - build.dockerfile -> dockerfile (str) -> dockerfile.read() - - build.context -> path (str) - - build.args -> buildargs (dict) - build.options -> X (python extra params [extra_hosts, network_mode, ...]) - - build.cacheFrom -> cache_from (list) - ''' Build = namedtuple('Build', ['dockerfile', 'build_context', 'build_args', 'cache_from']) Run = namedtuple('Run', ['name', 'environment', 'user', 'privileged', 'cap_add', 'security_opt', 'volumes']) Conf = namedtuple('Conf', ['image', 'build', 'run']) @@ -57,7 +39,7 @@ def _instans_or_error(value, obj): user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str), privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool), cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list), - security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), dict), + security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), list), volumes=_instans_or_error(runnerfile.get('run', {}).get('mounts'), dict), ) ) From 6a44fbf4fb533d603bfeb18868d0701ffa239726 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Tue, 30 Apr 2024 16:35:22 +0200 Subject: [PATCH 32/39] wip --- conan/cli/commands/create.py | 18 ++- conan/internal/runner/docker.py | 31 ++---- .../test/integration/command/runner_test.py | 104 ++++-------------- 3 files changed, 45 insertions(+), 108 deletions(-) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 81ffa5801d2..1cbf26bc350 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -67,11 +67,19 @@ def create(conan_api, parser, *args): print_profiles(profile_host, profile_build) if profile_host.runner and not os.environ.get("CONAN_RUNNER_ENVIRONMENT"): - return { - 'docker': DockerRunner, - 'ssh': SSHRunner, - 'wsl': WSLRunner, - }[profile_host.runner.get('type')](conan_api, 'create', profile_host, profile_build, args, raw_args).run() + try: + runner_type = profile_host.runner['type'] + except KeyError: + raise ConanException(f"Invalid runner configuration. 'type' must be defined") + try: + runner_instance = { + 'docker': DockerRunner, + 'ssh': SSHRunner, + 'wsl': WSLRunner, + }[runner_type] + except KeyError: + raise ConanException(f"Invalid runner type '{runner_type}'. Allowed values: 'docker' 'ssh' 'wsl'") + return runner_instance(conan_api, 'create', profile_host, profile_build, args, raw_args).run() if args.build is not None and args.build_test is None: args.build_test = args.build diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 333a774b5c6..c6db6b4b003 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -51,7 +51,7 @@ def _instans_or_error(value, obj): ) -def docker_info(msg, error=False): +def _docker_info(msg, error=False): fg=Color.BRIGHT_MAGENTA if error: fg=Color.BRIGHT_RED @@ -60,15 +60,6 @@ def docker_info(msg, error=False): ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) -def list_patterns(cache_info): - _pattern = [] - for reference, info in cache_info.items(): - for revisions in info.get('revisions', {}).values(): - for package in revisions.get('packages').keys(): - _pattern.append(f'{reference}:{package}') - return _pattern - - class DockerRunner: def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args): import docker @@ -80,7 +71,7 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar except: raise ConanException("Docker Client failed to initialize." "\n - Check if docker is installed and running" - "\n - Run 'pip install docker>=5.0.0, <=5.0.3'") + "\n - Run 'pip install pip install conan[runners]'") self.conan_api = conan_api self.build_profile = build_profile self.args = args @@ -125,13 +116,13 @@ def run(self): run conan inside a Docker continer """ if self.dockerfile: - docker_info(f'Building the Docker image: {self.image}') + _docker_info(f'Building the Docker image: {self.image}') self.build_image() volumes, environment = self.create_runner_environment() error = False try: if self.docker_client.containers.list(all=True, filters={'name': self.name}): - docker_info('Starting the docker container') + _docker_info('Starting the docker container') self.container = self.docker_client.containers.get(self.name) self.container.start() else: @@ -139,7 +130,7 @@ def run(self): environment.update(self.configfile.run.environment) if self.configfile.run.volumes: volumes.update(self.configfile.run.volumes) - docker_info('Creating the docker container') + _docker_info('Creating the docker container') self.container = self.docker_client.containers.run( self.image, "/bin/bash -c 'while true; do sleep 30; done;'", @@ -152,7 +143,7 @@ def run(self): security_opt=self.configfile.run.security_opt, detach=True, auto_remove=False) - docker_info(f'Container {self.name} running') + _docker_info(f'Container {self.name} running') except Exception as e: raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"' f'\n\n{str(e)}') @@ -170,10 +161,10 @@ def run(self): finally: if self.container: error_prefix = 'ERROR: ' if error else '' - docker_info(f'{error_prefix}Stopping container', error) + _docker_info(f'{error_prefix}Stopping container', error) self.container.stop() if self.remove: - docker_info(f'{error_prefix}Removing container', error) + _docker_info(f'{error_prefix}Removing container', error) self.container.remove() def build_image(self): @@ -200,7 +191,7 @@ def build_image(self): def run_command(self, command, log=True): if log: - docker_info(f'Running in container: "{command}"') + _docker_info(f'Running in container: "{command}"') exec_instance = self.docker_api.exec_create(self.container.id, f"/bin/bash -c '{command}'", tty=True) exec_output = self.docker_api.exec_start(exec_instance['Id'], tty=True, stream=True, demux=True,) stderr_log, stdout_log = '', '' @@ -248,7 +239,7 @@ def create_runner_environment(self): if self.cache == 'copy': tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz') - docker_info(f'Save host cache in: {tgz_path}') + _docker_info(f'Save host cache in: {tgz_path}') self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path) return volumes, environment @@ -274,5 +265,5 @@ def update_local_cache(self): self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json', log=False) self.run_command('conan cache save --list=pkglist.json --file "'+self.abs_docker_path+'"/.conanrunner/docker_cache_save.tgz') tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz') - docker_info(f'Restore host cache from: {tgz_path}') + _docker_info(f'Restore host cache from: {tgz_path}') package_list = self.conan_api.cache.restore(tgz_path) diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py index 510fa4218e4..3371e10be92 100644 --- a/conans/test/integration/command/runner_test.py +++ b/conans/test/integration/command/runner_test.py @@ -87,36 +87,20 @@ def test_create_docker_runner_dockerfile_folder_path(): remove=True """) - conanfile = textwrap.dedent(""" - from conan import ConanFile - - class MyTest(ConanFile): - name = "pkg" - version = "0.2" - settings = "build_type", "compiler" - author = "John Doe" - license = "MIT" - url = "https://foo.bar.baz" - homepage = "https://foo.bar.site" - topics = "foo", "bar", "qux" - provides = "libjpeg", "libjpg" - deprecated = "other-pkg" - options = {"shared": [True, False], "fPIC": [True, False]} - default_options = {"shared": False, "fPIC": True} - """) - client.save({"conanfile.py": conanfile, "host_copy": profile_host_copy, "host_clean": profile_host_clean, "build": profile_build}) + client.save({"host_copy": profile_host_copy, "host_clean": profile_host_clean, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=0.2") client.run("create . -pr:h host_copy -pr:b build") assert "Restore: pkg/0.2" in client.out - assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out - assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out assert "Removing container" in client.out client.run("create . -pr:h host_clean -pr:b build") assert "Restore: pkg/0.2" in client.out - assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out - assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out assert "Removing container" in client.out @@ -154,29 +138,13 @@ def test_create_docker_runner_dockerfile_file_path(): cache=copy remove=True """) - conanfile = textwrap.dedent(""" - from conan import ConanFile - - class MyTest(ConanFile): - name = "pkg" - version = "0.2" - settings = "build_type", "compiler" - author = "John Doe" - license = "MIT" - url = "https://foo.bar.baz" - homepage = "https://foo.bar.site" - topics = "foo", "bar", "qux" - provides = "libjpeg", "libjpg" - deprecated = "other-pkg" - options = {"shared": [True, False], "fPIC": [True, False]} - default_options = {"shared": False, "fPIC": True} - """) - client.save({"conanfile.py": conanfile, "host": profile_host, "build": profile_build}) + client.save({"host": profile_host, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=0.2") client.run("create . -pr:h host -pr:b build") - + print(client.out) assert "Restore: pkg/0.2" in client.out - assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out - assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out assert "Removing container" in client.out @@ -280,29 +248,14 @@ def test_create_docker_runner_profile_abs_path(): cache=copy remove=True """) - conanfile = textwrap.dedent(""" - from conan import ConanFile - - class MyTest(ConanFile): - name = "pkg" - version = "0.2" - settings = "build_type", "compiler" - author = "John Doe" - license = "MIT" - url = "https://foo.bar.baz" - homepage = "https://foo.bar.site" - topics = "foo", "bar", "qux" - provides = "libjpeg", "libjpg" - deprecated = "other-pkg" - options = {"shared": [True, False], "fPIC": [True, False]} - default_options = {"shared": False, "fPIC": True} - """) - client.save({"conanfile.py": conanfile, "host": profile_host, "build": profile_build}) + + client.save({"host": profile_host, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=0.2") client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'") assert "Restore: pkg/0.2" in client.out - assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out - assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out assert "Removing container" in client.out @@ -347,27 +300,12 @@ def test_create_docker_runner_profile_abs_path_from_configfile(): cache=copy remove=True """) - conanfile = textwrap.dedent(""" - from conan import ConanFile - - class MyTest(ConanFile): - name = "pkg" - version = "0.2" - settings = "build_type", "compiler" - author = "John Doe" - license = "MIT" - url = "https://foo.bar.baz" - homepage = "https://foo.bar.site" - topics = "foo", "bar", "qux" - provides = "libjpeg", "libjpg" - deprecated = "other-pkg" - options = {"shared": [True, False], "fPIC": [True, False]} - default_options = {"shared": False, "fPIC": True} - """) - client.save({"conanfile.py": conanfile, "host": profile_host, "build": profile_build}) + + client.save({"host": profile_host, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=0.2") client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'") assert "Restore: pkg/0.2" in client.out - assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307" in client.out - assert "Restore: pkg/0.2:a0826e5ee3b340fcc7a8ccde40224e3562316307 metadata" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out assert "Removing container" in client.out \ No newline at end of file From f151c3bf8768e00454098c3dcd92ed441fe801e0 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Fri, 3 May 2024 11:29:24 +0200 Subject: [PATCH 33/39] docker runner test added --- .../command/dockerfiles/Dockerfile_args | 12 ++++ .../test/integration/command/runner_test.py | 57 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 conans/test/integration/command/dockerfiles/Dockerfile_args diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_args b/conans/test/integration/command/dockerfiles/Dockerfile_args new file mode 100644 index 00000000000..67685e1f6fc --- /dev/null +++ b/conans/test/integration/command/dockerfiles/Dockerfile_args @@ -0,0 +1,12 @@ +ARG BASE_IMAGE +FROM $BASE_IMAGE +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* +COPY . /root/conan-io +RUN cd /root/conan-io && pip install -e . diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py index 3371e10be92..d6331da7140 100644 --- a/conans/test/integration/command/runner_test.py +++ b/conans/test/integration/command/runner_test.py @@ -305,6 +305,63 @@ def test_create_docker_runner_profile_abs_path_from_configfile(): client.run("new cmake_lib -d name=pkg -d version=0.2") client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'") + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_profile_abs_path_from_configfile_with_args(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + configfile = textwrap.dedent(f""" + image: conan-runner-default-test-with-args + build: + dockerfile: {dockerfile_path("Dockerfile_args")} + build_context: {conan_base_path()} + build_args: + BASE_IMAGE: ubuntu:22.04 + run: + name: my-conan-runner-container-with-args + """) + client.save({"configfile.yaml": configfile}) + + + profile_build = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + """) + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + configfile={os.path.join(client.current_folder, 'configfile.yaml')} + cache=copy + remove=True + """) + + client.save({"host": profile_host, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=0.2") + client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'") + print(client.out) + assert "test/integration/command/dockerfiles/Dockerfile_args" in client.out assert "Restore: pkg/0.2" in client.out assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out From a54eccacef382da43da727f8fa0e2e9feadc56df Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Fri, 3 May 2024 11:32:30 +0200 Subject: [PATCH 34/39] ssh and wsl disabled --- conan/cli/commands/create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 1cbf26bc350..1471c043302 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -74,8 +74,8 @@ def create(conan_api, parser, *args): try: runner_instance = { 'docker': DockerRunner, - 'ssh': SSHRunner, - 'wsl': WSLRunner, + # 'ssh': SSHRunner, + # 'wsl': WSLRunner, }[runner_type] except KeyError: raise ConanException(f"Invalid runner type '{runner_type}'. Allowed values: 'docker' 'ssh' 'wsl'") From 6e26f789c485dba9eba343aa609e8c7ad1193b41 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Fri, 3 May 2024 11:40:20 +0200 Subject: [PATCH 35/39] print remove --- conan/internal/runner/docker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index c6db6b4b003..6b598fc307f 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -247,7 +247,6 @@ def init_container(self): min_conan_version = '2.1' stdout, _ = self.run_command('conan --version', log=True) docker_conan_version = str(stdout.split('Conan version ')[1].replace('\n', '').replace('\r', '')) # Remove all characters and color - print(docker_conan_version) if Version(docker_conan_version) <= Version(min_conan_version): ConanOutput().status(f'ERROR: conan version inside the container must be greater than {min_conan_version}', fg=Color.BRIGHT_RED) raise ConanException( f'conan version inside the container must be greater than {min_conan_version}') From a6e17995abc395f39ad399e03ddd8c6671180689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20S=C3=A1nchez=20Falero?= Date: Mon, 6 May 2024 09:54:05 +0200 Subject: [PATCH 36/39] Update conan/cli/commands/create.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rubén Rincón Blanco --- conan/cli/commands/create.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 1471c043302..862c441e9ac 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -71,14 +71,15 @@ def create(conan_api, parser, *args): runner_type = profile_host.runner['type'] except KeyError: raise ConanException(f"Invalid runner configuration. 'type' must be defined") + runner_instances_map = { + 'docker': DockerRunner, + # 'ssh': SSHRunner, + # 'wsl': WSLRunner, + } try: - runner_instance = { - 'docker': DockerRunner, - # 'ssh': SSHRunner, - # 'wsl': WSLRunner, - }[runner_type] + runner_instance = runner_instances_map[runner_type] except KeyError: - raise ConanException(f"Invalid runner type '{runner_type}'. Allowed values: 'docker' 'ssh' 'wsl'") + raise ConanException(f"Invalid runner type '{runner_type}'. Allowed values: {', '.join(runner_instances_map.keys())}") return runner_instance(conan_api, 'create', profile_host, profile_build, args, raw_args).run() if args.build is not None and args.build_test is None: From fa8ab58fafe1baae11b676f35f7673d77e52cc76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20S=C3=A1nchez=20Falero?= Date: Mon, 6 May 2024 09:54:42 +0200 Subject: [PATCH 37/39] Update conan/cli/commands/create.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rubén Rincón Blanco --- conan/cli/commands/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 862c441e9ac..2478f587852 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -68,7 +68,7 @@ def create(conan_api, parser, *args): print_profiles(profile_host, profile_build) if profile_host.runner and not os.environ.get("CONAN_RUNNER_ENVIRONMENT"): try: - runner_type = profile_host.runner['type'] + runner_type = profile_host.runner['type'].lower() except KeyError: raise ConanException(f"Invalid runner configuration. 'type' must be defined") runner_instances_map = { From 2977c647624bd621d97c96f1281ce772995f808a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20S=C3=A1nchez=20Falero?= Date: Mon, 6 May 2024 09:55:57 +0200 Subject: [PATCH 38/39] Update conan/internal/runner/docker.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rubén Rincón Blanco --- conan/internal/runner/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 6b598fc307f..0544f3cfb01 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -21,7 +21,7 @@ def config_parser(file_path): if file_path: def _instans_or_error(value, obj): if value and (not isinstance(value, obj)): - raise Exception(f"{value} must be a {obj}") + raise ConanException(f"docker runner configfile syntax error: {value} must be a {obj.__name__}") return value with open(file_path, 'r') as f: runnerfile = yaml.safe_load(f) From fb6bab8b06264a4628e9e5389b9c06d38fcd8ec4 Mon Sep 17 00:00:00 2001 From: David Sanchez Date: Mon, 6 May 2024 11:56:56 +0200 Subject: [PATCH 39/39] docker runner tests with default profile added --- conan/cli/commands/create.py | 6 +- conan/internal/runner/docker.py | 7 +- .../dockerfiles/Dockerfile_profile_detect | 12 +++ .../test/integration/command/runner_test.py | 73 ++++++++++++++++++- 4 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 conans/test/integration/command/dockerfiles/Dockerfile_profile_detect diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 2478f587852..fe16deaf019 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -10,9 +10,6 @@ from conan.cli.printers.graph import print_graph_packages, print_graph_basic from conan.errors import ConanException from conans.client.graph.graph import BINARY_BUILD -from conan.internal.runner.docker import DockerRunner -from conan.internal.runner.ssh import SSHRunner -from conan.internal.runner.wsl import WSLRunner from conans.util.files import mkdir @@ -67,6 +64,9 @@ def create(conan_api, parser, *args): print_profiles(profile_host, profile_build) if profile_host.runner and not os.environ.get("CONAN_RUNNER_ENVIRONMENT"): + from conan.internal.runner.docker import DockerRunner + from conan.internal.runner.ssh import SSHRunner + from conan.internal.runner.wsl import WSLRunner try: runner_type = profile_host.runner['type'].lower() except KeyError: diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py index 0544f3cfb01..ebc6374e37d 100644 --- a/conan/internal/runner/docker.py +++ b/conan/internal/runner/docker.py @@ -86,7 +86,11 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar # Update conan command and some paths to run inside the container raw_args[raw_args.index(args.path)] = self.abs_docker_path self.profiles = [] - profile_list = set(self.args.profile_build + self.args.profile_host) + if self.args.profile_build and self.args.profile_host: + profile_list = set(self.args.profile_build + self.args.profile_host) + else: + profile_list = self.args.profile_host or self.args.profile_build + # Update the profile paths for i, raw_arg in enumerate(raw_args): for i, raw_profile in enumerate(profile_list): @@ -95,6 +99,7 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar if raw_profile in raw_arg: raw_args[raw_args.index(raw_arg)] = raw_arg.replace(raw_profile, os.path.join(self.abs_docker_path, '.conanrunner/profiles', _name)) self.profiles.append([_profile, os.path.join(self.abs_runner_home_path, 'profiles', _name)]) + self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) # Container config diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_profile_detect b/conans/test/integration/command/dockerfiles/Dockerfile_profile_detect new file mode 100644 index 00000000000..76f2b20a5de --- /dev/null +++ b/conans/test/integration/command/dockerfiles/Dockerfile_profile_detect @@ -0,0 +1,12 @@ +FROM ubuntu:22.04 +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* +COPY . /root/conan-io +RUN cd /root/conan-io && pip install -e . +RUN conan profile detect \ No newline at end of file diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py index d6331da7140..5341d73092a 100644 --- a/conans/test/integration/command/runner_test.py +++ b/conans/test/integration/command/runner_test.py @@ -7,7 +7,6 @@ from conans.test.assets.sources import gen_function_h, gen_function_cpp - def docker_skip(test_image=None): try: docker_client = docker.from_env() @@ -51,6 +50,7 @@ def test_create_docker_runner_dockerfile_folder_path(): compiler.version=11 os=Linux """) + profile_host_copy = textwrap.dedent(f"""\ [settings] arch=x86_64 @@ -214,6 +214,7 @@ def package(self): assert 'cmake -G "Ninja"' in client.out assert "main: {}!".format(build_type) in client.out + @pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") # @pytest.mark.xfail(reason="conan inside docker optional test") def test_create_docker_runner_profile_abs_path(): @@ -365,4 +366,72 @@ def test_create_docker_runner_profile_abs_path_from_configfile_with_args(): assert "Restore: pkg/0.2" in client.out assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out - assert "Removing container" in client.out \ No newline at end of file + assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_default_build_profile(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path("Dockerfile_profile_detect")} + build_context={conan_base_path()} + image=conan-runner-default-test + cache=clean + remove=True + """) + + client.save({"host_clean": profile_host}) + client.run("new cmake_lib -d name=pkg -d version=0.2") + client.run("create . -pr:h host_clean") + + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_default_build_profile_error(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path()} + build_context={conan_base_path()} + image=conan-runner-default-test + cache=clean + remove=True + """) + + client.save({"host_clean": profile_host}) + client.run("new cmake_lib -d name=pkg -d version=0.2") + with pytest.raises(Exception, match="ERROR: The default build profile '/root/.conan2/profiles/default' doesn't exist.") as exception: + client.run("create . -pr:h host_clean") \ No newline at end of file