Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

shared docker runner profiles fixed #16364

Merged
merged 2 commits into from
Jun 4, 2024
Merged

shared docker runner profiles fixed #16364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
96ce0fb
shared docker runner profiles fixed
davidsanfal May 29, 2024
2466afe
docker runner profiles only from profile folder or workdir folder
davidsanfal Jun 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 21 additions & 36 deletions conan/internal/runner/docker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from conan.api.conan_api import ConfigAPI
from conan.cli import make_abs_path
from conan.internal.runner import RunnerException
from conans.client.profile_loader import ProfileLoader
from conans.errors import ConanException
from conans.model.version import Version

Expand Down Expand Up @@ -79,29 +78,6 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar
if args.format:
raise ConanException("format argument is forbidden if running in a docker runner")

# Runner config
self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner')
self.abs_docker_path = os.path.join('/root/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/")

# Update conan command and some paths to run inside the container
raw_args[raw_args.index(args.path)] = self.abs_docker_path
self.profiles = []
if self.args.profile_build and self.args.profile_host:
profile_list = set(self.args.profile_build + self.args.profile_host)
else:
profile_list = self.args.profile_host or self.args.profile_build

# Update the profile paths
for i, raw_arg in enumerate(raw_args):
for i, raw_profile in enumerate(profile_list):
_profile = ProfileLoader.get_profile_path(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), raw_profile, os.getcwd())
_name = f'{os.path.basename(_profile)}_{i}'
if raw_profile in raw_arg:
raw_args[raw_args.index(raw_arg)] = raw_arg.replace(raw_profile, os.path.join(self.abs_docker_path, '.conanrunner/profiles', _name))
self.profiles.append([_profile, os.path.join(self.abs_runner_home_path, 'profiles', _name)])

self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json'])

# Container config
# https://containers.dev/implementors/json_reference/
self.configfile = config_parser(host_profile.runner.get('configfile'))
Expand All @@ -116,6 +92,15 @@ def __init__(self, conan_api, command, host_profile, build_profile, args, raw_ar
self.cache = str(host_profile.runner.get('cache', 'clean'))
self.container = None

# Runner config>
self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner')
self.docker_user_name = self.configfile.run.user or 'root'
self.abs_docker_path = os.path.join(f'/{self.docker_user_name}/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/")

# Update conan command and some paths to run inside the container
raw_args[raw_args.index(args.path)] = self.abs_docker_path
self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json'])

def run(self):
"""
run conan inside a Docker continer
Expand Down Expand Up @@ -194,10 +179,11 @@ def build_image(self):
if stream:
ConanOutput().status(stream.strip())

def run_command(self, command, log=True):
def run_command(self, command, workdir=None, log=True):
workdir = workdir or self.abs_docker_path
if log:
_docker_info(f'Running in container: "{command}"')
exec_instance = self.docker_api.exec_create(self.container.id, f"/bin/bash -c '{command}'", tty=True)
exec_instance = self.docker_api.exec_create(self.container.id, f"/bin/bash -c '{command}'", workdir=workdir, tty=True)
exec_output = self.docker_api.exec_start(exec_instance['Id'], tty=True, stream=True, demux=True,)
stderr_log, stdout_log = '', ''
try:
Expand Down Expand Up @@ -226,22 +212,22 @@ def create_runner_environment(self):
shutil.rmtree(self.abs_runner_home_path, ignore_errors=True)
volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}}
environment = {'CONAN_RUNNER_ENVIRONMENT': '1'}

if self.cache == 'shared':
volumes[ConfigAPI(self.conan_api).home()] = {'bind': '/root/.conan2', 'mode': 'rw'}
volumes[self.conan_api.home_folder] = {'bind': f'/{self.docker_user_name}/.conan2', 'mode': 'rw'}

if self.cache in ['clean', 'copy']:
# Copy all conan profiles and config files to docker workspace
os.mkdir(self.abs_runner_home_path)
os.mkdir(os.path.join(self.abs_runner_home_path, 'profiles'))

# Copy all conan config files to docker workspace
shutil.copytree(
os.path.join(self.conan_api.home_folder, 'profiles'),
os.path.join(self.abs_runner_home_path, 'profiles')
)
for file_name in ['global.conf', 'settings.yml', 'remotes.json']:
src_file = os.path.join(ConfigAPI(self.conan_api).home(), file_name)
src_file = os.path.join(self.conan_api.home_folder, file_name)
if os.path.exists(src_file):
shutil.copy(src_file, os.path.join(self.abs_runner_home_path, file_name))

# Copy all profiles to docker workspace
for current_path, new_path in self.profiles:
shutil.copy(current_path, new_path)

if self.cache == 'copy':
tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz')
_docker_info(f'Save host cache in: {tgz_path}')
Expand All @@ -257,7 +243,6 @@ def init_container(self):
raise ConanException( f'conan version inside the container must be greater than {min_conan_version}')
if self.cache != 'shared':
self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False)
self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False)
for file_name in ['global.conf', 'settings.yml', 'remotes.json']:
if os.path.exists( os.path.join(self.abs_runner_home_path, file_name)):
self.run_command('cp "'+self.abs_docker_path+'/.conanrunner/'+file_name+'" ${HOME}/.conan2/'+file_name, log=False)
Expand Down
190 changes: 133 additions & 57 deletions test/integration/command/runner_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,135 @@ def dockerfile_path(name=None):


@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running")
# @pytest.mark.xfail(reason="conan inside docker optional test")
def test_create_docker_runner_cache_shared():
czoido marked this conversation as resolved.
Show resolved Hide resolved
"""
Tests the ``conan create . ``
"""
client = TestClient()
profile_build = textwrap.dedent(f"""\
[settings]
arch=x86_64
build_type=Release
compiler=gcc
compiler.cppstd=gnu17
compiler.libcxx=libstdc++11
compiler.version=11
os=Linux
""")

profile_host = textwrap.dedent(f"""\
[settings]
arch=x86_64
build_type=Release
compiler=gcc
compiler.cppstd=gnu17
compiler.libcxx=libstdc++11
compiler.version=11
os=Linux
[runner]
type=docker
dockerfile={dockerfile_path()}
build_context={conan_base_path()}
image=conan-runner-default-test
cache=shared
remove=True
""")

client.save({"host": profile_host, "build": profile_build})
client.run("new cmake_lib -d name=pkg -d version=0.2")
client.run("create . -pr:h host -pr:b build")

assert "[100%] Built target example" in client.out
assert "Removing container" in client.out


@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running")
def test_create_docker_runner_cache_shared_profile_from_cache():
"""
Tests the ``conan create . ``
"""
client = TestClient()
profile_build = textwrap.dedent(f"""\
[settings]
arch=x86_64
build_type=Release
compiler=gcc
compiler.cppstd=gnu17
compiler.libcxx=libstdc++11
compiler.version=11
os=Linux
""")

profile_host = textwrap.dedent(f"""\
[settings]
arch=x86_64
build_type=Release
compiler=gcc
compiler.cppstd=gnu17
compiler.libcxx=libstdc++11
compiler.version=11
os=Linux
[runner]
type=docker
dockerfile={dockerfile_path()}
build_context={conan_base_path()}
image=conan-runner-default-test
cache=shared
remove=True
""")

client.save({"default_host": profile_host, "default_build": profile_build}, path=client.cache.profiles_path)
client.run("new cmake_lib -d name=pkg -d version=0.2")
client.run("create . -pr:h default_host -pr:b default_build")

assert "[100%] Built target example" in client.out
assert "Removing container" in client.out


@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running")
def test_create_docker_runner_cache_shared_profile_folder():
"""
Tests the ``conan create . ``
"""
client = TestClient()
profile_build = textwrap.dedent(f"""\
[settings]
arch=x86_64
build_type=Release
compiler=gcc
compiler.cppstd=gnu17
compiler.libcxx=libstdc++11
compiler.version=11
os=Linux
""")

profile_host = textwrap.dedent(f"""\
[settings]
arch=x86_64
build_type=Release
compiler=gcc
compiler.cppstd=gnu17
compiler.libcxx=libstdc++11
compiler.version=11
os=Linux
[runner]
type=docker
dockerfile={dockerfile_path()}
build_context={conan_base_path()}
image=conan-runner-default-test
cache=shared
remove=True
""")

client.save({"build": profile_build})
client.save({"docker_default": profile_host}, path = os.path.join(client.cache_folder, "profiles"))
client.run("new cmake_lib -d name=pkg -d version=0.2")
client.run("create . -pr:h docker_default -pr:b build")

assert "[100%] Built target example" in client.out
assert "Removing container" in client.out

@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running")
def test_create_docker_runner_dockerfile_folder_path():
"""
Tests the ``conan create . ``
Expand Down Expand Up @@ -105,7 +233,6 @@ def test_create_docker_runner_dockerfile_folder_path():


@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running")
# @pytest.mark.xfail(reason="conan inside docker optional test")
def test_create_docker_runner_dockerfile_file_path():
"""
Tests the ``conan create . ``
Expand Down Expand Up @@ -150,7 +277,6 @@ def test_create_docker_runner_dockerfile_file_path():


@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running")
# @pytest.mark.xfail(reason="conan inside docker optional test")
@pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)])
@pytest.mark.tool("ninja")
def test_create_docker_runner_with_ninja(build_type, shared):
Expand Down Expand Up @@ -215,55 +341,8 @@ def package(self):
assert 'cmake -G "Ninja"' in client.out
assert "main: {}!".format(build_type) in client.out


@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running")
# @pytest.mark.xfail(reason="conan inside docker optional test")
def test_create_docker_runner_profile_abs_path():
"""
Tests the ``conan create . ``
"""
client = TestClient()
profile_build = textwrap.dedent(f"""\
[settings]
arch=x86_64
build_type=Release
compiler=gcc
compiler.cppstd=gnu17
compiler.libcxx=libstdc++11
compiler.version=11
os=Linux
""")
profile_host = textwrap.dedent(f"""\
[settings]
arch=x86_64
build_type=Release
compiler=gcc
compiler.cppstd=gnu17
compiler.libcxx=libstdc++11
compiler.version=11
os=Linux
[runner]
type=docker
dockerfile={dockerfile_path("Dockerfile_test")}
build_context={conan_base_path()}
image=conan-runner-default-test
cache=copy
remove=True
""")

client.save({"host": profile_host, "build": profile_build})
client.run("new cmake_lib -d name=pkg -d version=0.2")
client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'")

assert "Restore: pkg/0.2" in client.out
assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out
assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out
assert "Removing container" in client.out


@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running")
# @pytest.mark.xfail(reason="conan inside docker optional test")
def test_create_docker_runner_profile_abs_path_from_configfile():
def test_create_docker_runner_from_configfile():
"""
Tests the ``conan create . ``
"""
Expand Down Expand Up @@ -307,7 +386,7 @@ def test_create_docker_runner_profile_abs_path_from_configfile():

client.save({"host": profile_host, "build": profile_build})
client.run("new cmake_lib -d name=pkg -d version=0.2")
client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'")
client.run("create . -pr:h 'host' -pr:b 'build'")

assert "Container my-custom-conan-runner-container running" in client.out
assert "Restore: pkg/0.2" in client.out
Expand All @@ -317,8 +396,7 @@ def test_create_docker_runner_profile_abs_path_from_configfile():


@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running")
# @pytest.mark.xfail(reason="conan inside docker optional test")
def test_create_docker_runner_profile_abs_path_from_configfile_with_args():
def test_create_docker_runner_from_configfile_with_args():
"""
Tests the ``conan create . ``
"""
Expand Down Expand Up @@ -364,7 +442,7 @@ def test_create_docker_runner_profile_abs_path_from_configfile_with_args():

client.save({"host": profile_host, "build": profile_build})
client.run("new cmake_lib -d name=pkg -d version=0.2")
client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'")
client.run("create . -pr:h 'host' -pr:b 'build'")

assert "test/integration/command/dockerfiles/Dockerfile_args" in client.out
assert "Restore: pkg/0.2" in client.out
Expand All @@ -374,7 +452,6 @@ def test_create_docker_runner_profile_abs_path_from_configfile_with_args():


@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running")
# @pytest.mark.xfail(reason="conan inside docker optional test")
def test_create_docker_runner_default_build_profile():
"""
Tests the ``conan create . ``
Expand Down Expand Up @@ -410,7 +487,6 @@ def test_create_docker_runner_default_build_profile():


@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running")
# @pytest.mark.xfail(reason="conan inside docker optional test")
def test_create_docker_runner_default_build_profile_error():
"""
Tests the ``conan create . ``
Expand Down