diff --git a/lib/jcsda-emc/spack-stack/stack/container_env.py b/lib/jcsda-emc/spack-stack/stack/container_env.py index 3a59caf5032f9a..e510846982b5a4 100644 --- a/lib/jcsda-emc/spack-stack/stack/container_env.py +++ b/lib/jcsda-emc/spack-stack/stack/container_env.py @@ -6,12 +6,11 @@ from spack.extensions.stack.stack_paths import ( common_path, container_path, - stack_path, template_path, ) -class StackContainer(): +class StackContainer: """Represents an abstract container. It takes in a container template (spack.yaml), the specs from an app, and its packages.yaml versions then writes out a merged file. @@ -21,7 +20,7 @@ def __init__(self, container, template, name, dir, base_packages) -> None: self.template = template self.container = container - test_path = os.path.join(container_path, container + '.yaml') + test_path = os.path.join(container_path, container + ".yaml") if os.path.exists(test_path): self.container_path = test_path elif os.path.isabs(container): @@ -34,54 +33,54 @@ def __init__(self, container, template, name, dir, base_packages) -> None: elif os.path.exists(os.path.join(template_path, self.template)): self.template_path = os.path.join(template_path, self.template) else: - raise Exception("Invalid app") + raise Exception("Invalid application template") - self.name = name if name else '{}'.format(container) + self.name = name if name else "{}".format(container) self.dir = dir self.env_dir = os.path.join(self.dir, self.name) if base_packages: self.base_packages = base_packages else: - self.base_packages = os.path.join(common_path, 'packages.yaml') + self.base_packages = os.path.join(common_path, "packages.yaml") def write(self): """Merge base packages and app's spack.yaml into output container file. """ - template_env = os.path.join(self.template_path, 'spack.yaml') - with open(template_env, 'r') as f: + template_env = os.path.join(self.template_path, "spack.yaml") + with open(template_env, "r") as f: # Spack uses :: to override settings. # but it's not understood when used in a spack.yaml filedata = f.read() - filedata.replace('::', ':') - filedata = filedata.replace('::', ':') + filedata = filedata.replace("::", ":") template_yaml = syaml.load_config(filedata) - with open(self.container_path, 'r') as f: + with open(self.container_path, "r") as f: container_yaml = syaml.load_config(f) # Create copy so we can modify it original_yaml = copy.deepcopy(container_yaml) - with open(self.base_packages, 'r') as f: + with open(self.base_packages, "r") as f: filedata = f.read() - filedata = filedata.replace('::', ':') + filedata = filedata.replace("::", ":") packages_yaml = syaml.load_config(filedata) - if 'packages' not in container_yaml['spack']: - container_yaml['spack']['packages'] = {} + if "packages" not in container_yaml["spack"]: + container_yaml["spack"]["packages"] = {} - container_yaml['spack']['packages'] = spack.config.merge_yaml( - container_yaml['spack']['packages'], packages_yaml['packages']) + container_yaml["spack"]["packages"] = spack.config.merge_yaml( + container_yaml["spack"]["packages"], packages_yaml["packages"] + ) container_yaml = spack.config.merge_yaml(container_yaml, template_yaml) # Merge the original back in so it takes precedence container_yaml = spack.config.merge_yaml(container_yaml, original_yaml) - container_yaml['spack']['container']['labels']['app'] = self.template + container_yaml["spack"]["container"]["labels"]["app"] = self.template os.makedirs(self.env_dir, exist_ok=True) - with open(os.path.join(self.env_dir, 'spack.yaml'), 'w') as f: + with open(os.path.join(self.env_dir, "spack.yaml"), "w") as f: syaml.dump_config(container_yaml, stream=f) diff --git a/lib/spack/spack/container/writers/__init__.py b/lib/spack/spack/container/writers/__init__.py index b8ac3dfbe36b9b..9ac1cfdab6ac31 100644 --- a/lib/spack/spack/container/writers/__init__.py +++ b/lib/spack/spack/container/writers/__init__.py @@ -244,10 +244,15 @@ def _package_info_from(self, package_list): @tengine.context_property def extra_instructions(self): - Extras = collections.namedtuple("Extra", ["build", "final"]) + Extras = collections.namedtuple("Extra", ["pre_build", "build", "pre_final", "final"]) extras = self.container_config.get("extra_instructions", {}) - build, final = extras.get("build", None), extras.get("final", None) - return Extras(build=build, final=final) + pre_build, build, pre_final, final = ( + extras.get("pre_build", None), + extras.get("build", None), + extras.get("pre_final", None), + extras.get("final", None), + ) + return Extras(pre_build=pre_build, build=build, pre_final=pre_final, final=final) @tengine.context_property def labels(self): diff --git a/lib/spack/spack/schema/container.py b/lib/spack/spack/schema/container.py index b05199f8bb5d25..f40bc5df33ed68 100644 --- a/lib/spack/spack/schema/container.py +++ b/lib/spack/spack/schema/container.py @@ -69,7 +69,12 @@ "extra_instructions": { "type": "object", "additionalProperties": False, - "properties": {"build": {"type": "string"}, "final": {"type": "string"}}, + "properties": { + "pre_build": {"type": "string"}, + "build": {"type": "string"}, + "pre_final": {"type": "string"}, + "final": {"type": "string"}, + }, }, # Reserved for properties that are specific to each format "singularity": { diff --git a/lib/spack/spack/test/container/docker.py b/lib/spack/spack/test/container/docker.py index f191cc4671cdf9..60fdbd55997b00 100644 --- a/lib/spack/spack/test/container/docker.py +++ b/lib/spack/spack/test/container/docker.py @@ -60,19 +60,29 @@ def test_strip_is_set_from_config(minimal_configuration): def test_extra_instructions_is_set_from_config(minimal_configuration): writer = writers.create(minimal_configuration) - assert writer.extra_instructions == (None, None) + assert writer.extra_instructions == (None, None, None, None) test_line = "RUN echo Hello world!" e = minimal_configuration["spack"]["container"] e["extra_instructions"] = {} + e["extra_instructions"]["pre_build"] = test_line + writer = writers.create(minimal_configuration) + assert writer.extra_instructions == (test_line, None, None, None) + e["extra_instructions"]["build"] = test_line + del e["extra_instructions"]["pre_build"] writer = writers.create(minimal_configuration) - assert writer.extra_instructions == (test_line, None) + assert writer.extra_instructions == (None, test_line, None, None) - e["extra_instructions"]["final"] = test_line + e["extra_instructions"]["pre_final"] = test_line del e["extra_instructions"]["build"] writer = writers.create(minimal_configuration) - assert writer.extra_instructions == (None, test_line) + assert writer.extra_instructions == (None, None, test_line, None) + + e["extra_instructions"]["final"] = test_line + del e["extra_instructions"]["pre_final"] + writer = writers.create(minimal_configuration) + assert writer.extra_instructions == (None, None, None, test_line) def test_custom_base_images(minimal_configuration): diff --git a/share/spack/templates/container/Dockerfile b/share/spack/templates/container/Dockerfile index b10dcf084a0258..11a72837939ff5 100644 --- a/share/spack/templates/container/Dockerfile +++ b/share/spack/templates/container/Dockerfile @@ -13,6 +13,10 @@ RUN {% if os_package_update %}{{ os_packages_build.update }} \ && {{ os_packages_build.clean }} {% endif %} +{% if extra_instructions.pre_build %} +{{ extra_instructions.pre_build }} +{% endif %} + # What we want to install and how we want to install it # is specified in a manifest file (spack.yaml) RUN mkdir {{ paths.environment }} \ @@ -48,6 +52,10 @@ COPY --from=builder {{ paths.hidden_view }} {{ paths.hidden_view }} COPY --from=builder {{ paths.view }} {{ paths.view }} COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh +{% if extra_instructions.pre_final %} +{{ extra_instructions.pre_final }} +{% endif %} + {% if os_packages_final %} RUN {% if os_package_update %}{{ os_packages_final.update }} \ && {% endif %}{{ os_packages_final.install }} {{ os_packages_final.list | join | replace('\n', ' ') }} \ diff --git a/share/spack/templates/container/singularity.def b/share/spack/templates/container/singularity.def index 684fe9f98818c3..2eba457a4d053a 100644 --- a/share/spack/templates/container/singularity.def +++ b/share/spack/templates/container/singularity.def @@ -12,6 +12,11 @@ Stage: build {{ os_packages_build.clean }} {% endif %} + + {% if extra_instructions.pre_build %} + {{ extra_instructions.pre_build }} + {% endif %} + # Create the manifest file for the installation in /opt/spack-environment mkdir {{ paths.environment }} && cd {{ paths.environment }} cat << EOF > spack.yaml @@ -61,6 +66,9 @@ Stage: final {{ paths.environment }}/environment_modifications.sh {{ paths.environment }}/environment_modifications.sh %post +{% if extra_instructions.pre_final %} + {{ extra_instructions.pre_final }} +{% endif %} {% if os_packages_final.list %} # Update, install and cleanup of system packages needed at run-time {% if os_package_update %}