Merge pull request spack#162 from climbfuji/feature/container_prebuil…

…d_docker_and_singularity

- Add extra instructions for container pre-build/pre-final phases for Docker and Singularity.
- black format lib/jcsda-emc/spack-stack/stack/container_env.py

lib/jcsda-emc/spack-stack/stack/container_env.py

@@ -6,12 +6,11 @@
 from spack.extensions.stack.stack_paths import (
     common_path,
     container_path,
-    stack_path,
     template_path,
 )
 
 
-class StackContainer():
+class StackContainer:
     """Represents an abstract container. It takes in a
     container template (spack.yaml), the specs from an app, and
     its packages.yaml versions then writes out a merged file.
@@ -21,7 +20,7 @@ def __init__(self, container, template, name, dir, base_packages) -> None:
         self.template = template
         self.container = container
 
-        test_path = os.path.join(container_path, container + '.yaml')
+        test_path = os.path.join(container_path, container + ".yaml")
         if os.path.exists(test_path):
             self.container_path = test_path
         elif os.path.isabs(container):
@@ -34,54 +33,54 @@ def __init__(self, container, template, name, dir, base_packages) -> None:
         elif os.path.exists(os.path.join(template_path, self.template)):
             self.template_path = os.path.join(template_path, self.template)
         else:
-            raise Exception("Invalid app")
+            raise Exception("Invalid application template")
 
-        self.name = name if name else '{}'.format(container)
+        self.name = name if name else "{}".format(container)
 
         self.dir = dir
         self.env_dir = os.path.join(self.dir, self.name)
         if base_packages:
             self.base_packages = base_packages
         else:
-            self.base_packages = os.path.join(common_path, 'packages.yaml')
+            self.base_packages = os.path.join(common_path, "packages.yaml")
 
     def write(self):
         """Merge base packages and app's spack.yaml into
         output container file.
         """
-        template_env = os.path.join(self.template_path, 'spack.yaml')
-        with open(template_env, 'r') as f:
+        template_env = os.path.join(self.template_path, "spack.yaml")
+        with open(template_env, "r") as f:
             # Spack uses :: to override settings.
             # but it's not understood when used in a spack.yaml
             filedata = f.read()
-            filedata.replace('::', ':')
-            filedata = filedata.replace('::', ':')
+            filedata = filedata.replace("::", ":")
             template_yaml = syaml.load_config(filedata)
 
-        with open(self.container_path, 'r') as f:
+        with open(self.container_path, "r") as f:
             container_yaml = syaml.load_config(f)
 
         # Create copy so we can modify it
         original_yaml = copy.deepcopy(container_yaml)
 
-        with open(self.base_packages, 'r') as f:
+        with open(self.base_packages, "r") as f:
             filedata = f.read()
-            filedata = filedata.replace('::', ':')
+            filedata = filedata.replace("::", ":")
             packages_yaml = syaml.load_config(filedata)
 
-        if 'packages' not in container_yaml['spack']:
-            container_yaml['spack']['packages'] = {}
+        if "packages" not in container_yaml["spack"]:
+            container_yaml["spack"]["packages"] = {}
 
-        container_yaml['spack']['packages'] = spack.config.merge_yaml(
-            container_yaml['spack']['packages'], packages_yaml['packages'])
+        container_yaml["spack"]["packages"] = spack.config.merge_yaml(
+            container_yaml["spack"]["packages"], packages_yaml["packages"]
+        )
 
         container_yaml = spack.config.merge_yaml(container_yaml, template_yaml)
         # Merge the original back in so it takes precedence
         container_yaml = spack.config.merge_yaml(container_yaml, original_yaml)
 
-        container_yaml['spack']['container']['labels']['app'] = self.template
+        container_yaml["spack"]["container"]["labels"]["app"] = self.template
 
         os.makedirs(self.env_dir, exist_ok=True)
 
-        with open(os.path.join(self.env_dir, 'spack.yaml'), 'w') as f:
+        with open(os.path.join(self.env_dir, "spack.yaml"), "w") as f:
             syaml.dump_config(container_yaml, stream=f)

lib/spack/spack/container/writers/__init__.py

@@ -244,10 +244,15 @@ def _package_info_from(self, package_list):
 
     @tengine.context_property
     def extra_instructions(self):
-        Extras = collections.namedtuple("Extra", ["build", "final"])
+        Extras = collections.namedtuple("Extra", ["pre_build", "build", "pre_final", "final"])
         extras = self.container_config.get("extra_instructions", {})
-        build, final = extras.get("build", None), extras.get("final", None)
-        return Extras(build=build, final=final)
+        pre_build, build, pre_final, final = (
+            extras.get("pre_build", None),
+            extras.get("build", None),
+            extras.get("pre_final", None),
+            extras.get("final", None),
+        )
+        return Extras(pre_build=pre_build, build=build, pre_final=pre_final, final=final)
 
     @tengine.context_property
     def labels(self):

lib/spack/spack/schema/container.py

@@ -69,7 +69,12 @@
         "extra_instructions": {
             "type": "object",
             "additionalProperties": False,
-            "properties": {"build": {"type": "string"}, "final": {"type": "string"}},
+            "properties": {
+                "pre_build": {"type": "string"},
+                "build": {"type": "string"},
+                "pre_final": {"type": "string"},
+                "final": {"type": "string"},
+            },
         },
         # Reserved for properties that are specific to each format
         "singularity": {

lib/spack/spack/test/container/docker.py

@@ -60,19 +60,29 @@ def test_strip_is_set_from_config(minimal_configuration):
 
 def test_extra_instructions_is_set_from_config(minimal_configuration):
     writer = writers.create(minimal_configuration)
-    assert writer.extra_instructions == (None, None)
+    assert writer.extra_instructions == (None, None, None, None)
 
     test_line = "RUN echo Hello world!"
     e = minimal_configuration["spack"]["container"]
     e["extra_instructions"] = {}
+    e["extra_instructions"]["pre_build"] = test_line
+    writer = writers.create(minimal_configuration)
+    assert writer.extra_instructions == (test_line, None, None, None)
+
     e["extra_instructions"]["build"] = test_line
+    del e["extra_instructions"]["pre_build"]
     writer = writers.create(minimal_configuration)
-    assert writer.extra_instructions == (test_line, None)
+    assert writer.extra_instructions == (None, test_line, None, None)
 
-    e["extra_instructions"]["final"] = test_line
+    e["extra_instructions"]["pre_final"] = test_line
     del e["extra_instructions"]["build"]
     writer = writers.create(minimal_configuration)
-    assert writer.extra_instructions == (None, test_line)
+    assert writer.extra_instructions == (None, None, test_line, None)
+
+    e["extra_instructions"]["final"] = test_line
+    del e["extra_instructions"]["pre_final"]
+    writer = writers.create(minimal_configuration)
+    assert writer.extra_instructions == (None, None, None, test_line)
 
 
 def test_custom_base_images(minimal_configuration):

share/spack/templates/container/Dockerfile

@@ -13,6 +13,10 @@ RUN {% if os_package_update %}{{ os_packages_build.update }} \
  && {{ os_packages_build.clean }}
 {% endif %}
 
+{% if extra_instructions.pre_build %}
+{{ extra_instructions.pre_build }}
+{% endif %}
+
 # What we want to install and how we want to install it
 # is specified in a manifest file (spack.yaml)
 RUN mkdir {{ paths.environment }} \
@@ -48,6 +52,10 @@ COPY --from=builder {{ paths.hidden_view }} {{ paths.hidden_view }}
 COPY --from=builder {{ paths.view }} {{ paths.view }}
 COPY --from=builder /etc/profile.d/z10_spack_environment.sh /etc/profile.d/z10_spack_environment.sh
 
+{% if extra_instructions.pre_final %}
+{{ extra_instructions.pre_final }}
+{% endif %}
+
 {% if os_packages_final %}
 RUN {% if os_package_update %}{{ os_packages_final.update }} \
  && {% endif %}{{ os_packages_final.install }} {{ os_packages_final.list | join | replace('\n', ' ') }} \

share/spack/templates/container/singularity.def

@@ -12,6 +12,11 @@ Stage: build
   {{ os_packages_build.clean }}
 
 {% endif %}
+
+  {% if extra_instructions.pre_build %}
+  {{ extra_instructions.pre_build }}
+  {% endif %}
+
   # Create the manifest file for the installation in /opt/spack-environment
   mkdir {{ paths.environment }} && cd {{ paths.environment }}
   cat << EOF > spack.yaml
@@ -61,6 +66,9 @@ Stage: final
   {{ paths.environment }}/environment_modifications.sh {{ paths.environment }}/environment_modifications.sh
 
 %post
+{% if extra_instructions.pre_final %}
+ {{ extra_instructions.pre_final }}
+{% endif %}
 {% if os_packages_final.list %}
   # Update, install and cleanup of system packages needed at run-time
   {% if os_package_update %}