Implementation mostly finished

frontend/azlinux/azlinux3.go

@@ -49,12 +49,12 @@ func (w azlinux3) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S
 
 	img := llb.Image(Azlinux3Ref, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...))
 	return img.Run(
-		w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, dalec.NoOption(), installWithConstraints(opts)),
+		w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, installWithConstraints(opts)),
 		dalec.WithConstraints(opts...),
 	).Root(), nil
 }
 
-func (w azlinux3) Install(pkgs []string, repoOpt llb.RunOption, opts ...installOpt) llb.RunOption {
+func (w azlinux3) Install(pkgs []string, opts ...installOpt) llb.RunOption {
 	var cfg installConfig
 	setInstallOptions(&cfg, opts)
 	return dalec.WithRunOptions(tdnfInstall(&cfg, "3.0", pkgs), w.tdnfCacheMount(cfg.root))

frontend/azlinux/handle_container.go

@@ -67,13 +67,31 @@ func handleContainer(w worker) gwclient.BuildFunc {
 				return nil, nil, err
 			}
 
+			testRepos := spec.GetTestRepos(targetKey)
+			withRepos, err := withRepoConfig(testRepos, sOpt, pg)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			withData, err := withRepoData(testRepos, sOpt, pg)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			keyMounts, keyPaths, err := withRepoKeys(testRepos, sOpt, pg)
+			if err != nil {
+				return nil, nil, err
+			}
+
 			withTestDeps := func(in llb.State) llb.State {
 				deps := spec.GetTestDeps(targetKey)
 				if len(deps) == 0 {
 					return in
 				}
+
+				testDeps := dalec.SortMapKeys(spec.GetTestDeps(targetKey))
 				return base.Run(
-					w.Install(spec.GetTestDeps(targetKey), dalec.NoOption(), atRoot("/tmp/rootfs")),
+					w.Install(testDeps, atRoot("/tmp/rootfs"), withMounts(withRepos, withData, keyMounts), importKeys(keyPaths)),
 					pg,
 					dalec.ProgressGroup("Install test dependencies"),
 				).AddMount("/tmp/rootfs", in)
@@ -149,6 +167,22 @@ func specToContainerLLB(w worker, spec *dalec.Spec, targetKey string, rpmDir llb
 		rootfs = llb.Image(ref, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...))
 	}
 
+	installTimeRepos := spec.GetInstallRepos(targetKey)
+	repoOpts, err := withRepoConfig(installTimeRepos, sOpt, opts...)
+	if err != nil {
+		return llb.Scratch(), err
+	}
+
+	repoDataOpts, err := withRepoData(installTimeRepos, sOpt, opts...)
+	if err != nil {
+		return llb.Scratch(), err
+	}
+
+	keyMounts, keyPaths, err := withRepoKeys(installTimeRepos, sOpt, opts...)
+	if err != nil {
+		return llb.Scratch(), err
+	}
+
 	if len(files) > 0 {
 		rpmMountDir := "/tmp/rpms"
 		updated := w.BasePackages()
@@ -157,7 +191,7 @@ func specToContainerLLB(w worker, spec *dalec.Spec, targetKey string, rpmDir llb
 		}
 
 		rootfs = builderImg.Run(
-			w.Install(updated, dalec.NoOption(), atRoot(workPath), noGPGCheck, withManifests, installWithConstraints(opts)),
+			w.Install(updated, atRoot(workPath), withMounts(repoOpts, repoDataOpts, keyMounts), importKeys(keyPaths), noGPGCheck, withManifests, installWithConstraints(opts)),
 			llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS")),
 			dalec.WithConstraints(opts...),
 		).AddMount(workPath, rootfs)

frontend/azlinux/handle_rpm.go

@@ -53,7 +53,7 @@ func handleRPM(w worker) gwclient.BuildFunc {
 }
 
 // Creates and installs an rpm meta-package that requires the passed in deps as runtime-dependencies
-func installBuildDepsPackage(target string, packageName string, w worker, deps map[string]dalec.PackageConstraints, repoOpts llb.RunOption, installOpts ...installOpt) installFunc {
+func installBuildDepsPackage(target string, packageName string, w worker, deps map[string]dalec.PackageConstraints, keyPaths []string, repoMounts llb.RunOption, installOpts ...installOpt) installFunc {
 	// depsOnly is a simple dalec spec that only includes build dependencies and their constraints
 	depsOnly := dalec.Spec{
 		Name:        fmt.Sprintf("%s-build-dependencies", packageName),
@@ -83,19 +83,21 @@ func installBuildDepsPackage(target string, packageName string, w worker, deps m
 		installOpts = append([]installOpt{
 			noGPGCheck,
 			withMounts(llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS"))),
+			withMounts(repoMounts),
+			importKeys(keyPaths),
 			installWithConstraints(opts),
 		}, installOpts...)
 
 		// install the built RPMs into the worker itself
-		return w.Install([]string{"/tmp/rpms/*/*.rpm"}, repoOpts, installOpts...), nil
+		return w.Install([]string{"/tmp/rpms/*/*.rpm"}, installOpts...), nil
 	}
 }
 
 // meant to return a run option for mounting all repo state
-func getRepoMounts(repos []dalec.PackageRepositoryConfig, sOpts dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+func withRepoData(repos []dalec.PackageRepositoryConfig, sOpts dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
 	var repoMountsOpts []llb.RunOption
 	for _, repo := range repos {
-		rs, err := mountsForRepo(repo, sOpts, opts...)
+		rs, err := repoDataAsMount(repo, sOpts, opts...)
 		if err != nil {
 			return nil, err
 		}
@@ -106,7 +108,7 @@ func getRepoMounts(repos []dalec.PackageRepositoryConfig, sOpts dalec.SourceOpts
 }
 
 // meant to return a run option for mounting state for a single repo
-func mountsForRepo(config dalec.PackageRepositoryConfig, sOpts dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+func repoDataAsMount(config dalec.PackageRepositoryConfig, sOpts dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
 	var mounts []llb.RunOption
 	for _, data := range config.Data {
 		repoState, err := data.Spec.AsMount(data.Dest, sOpts, opts...)
@@ -119,11 +121,11 @@ func mountsForRepo(config dalec.PackageRepositoryConfig, sOpts dalec.SourceOpts,
 	return dalec.WithRunOptions(mounts...), nil
 }
 
-// meant to return a run option for importing the config files for all repos (including keys)
-func importRepos(repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+// meant to return a run option for importing the config files for all repos
+func withRepoConfig(repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
 	configStates := []llb.RunOption{}
 	for _, repo := range repos {
-		mnts, err := importRepo(repo, sOpt, opts...)
+		mnts, err := repoConfigAsMount(repo, sOpt, opts...)
 		if err != nil {
 			return nil, err
 		}
@@ -134,14 +136,32 @@ func importRepos(repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, o
 	return dalec.WithRunOptions(configStates...), nil
 }
 
-func importRepo(config dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]llb.RunOption, error) {
+func withRepoKeys(configs []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, []string, error) {
+	keys := []llb.RunOption{}
+	names := []string{}
+	for _, config := range configs {
+		for name, repoKey := range config.Keys {
+			// each of these sources represent a gpg key file for a particular repo
+			gpgKey, err := repoKey.AsState(name, sOpt, dalec.ProgressGroup("Importing repo key: "+name))
+			if err != nil {
+				return nil, nil, err
+			}
+
+			keys = append(keys, llb.AddMount(filepath.Join("/etc/pki/rpm-gpg", name), gpgKey, llb.SourcePath(name)))
+			names = append(names, name)
+		}
+	}
+
+	return dalec.WithRunOptions(keys...), names, nil
+}
+
+func repoConfigAsMount(config dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]llb.RunOption, error) {
 	repoConfigs := []llb.RunOption{}
 	keys := []llb.RunOption{}
 
 	for name, repoConfig := range config.Config {
 		// each of these sources represent a repo config file
-		// TODO: need to handle file vs. dir case
-		repoConfigSt, err := repoConfig.AsState(name, sOpt, opts...)
+		repoConfigSt, err := repoConfig.AsState(name, sOpt, append(opts, dalec.ProgressGroup("Importing repo config: "+name))...)
 		if err != nil {
 			return nil, err
 		}
@@ -152,7 +172,7 @@ func importRepo(config dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opt
 
 	for name, repoKey := range config.Keys {
 		// each of these sources represent a gpg key file for a particular repo
-		gpgKey, err := repoKey.AsState(name, sOpt)
+		gpgKey, err := repoKey.AsState(name, sOpt, append(opts, dalec.ProgressGroup("Importing repo key: "+name))...)
 		if err != nil {
 			return nil, err
 		}
@@ -176,26 +196,31 @@ func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spe
 		return nil, err
 	}
 
-	repoOpts := append(opts, dalec.ProgressGroup("Install build repos"))
-	repoMounts, err := getRepoMounts(repos, sOpt, repoOpts...)
+	importRepo, err := withRepoConfig(repos, sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	repoData, err := withRepoData(repos, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}
 
-	importRepo, err := importRepos(repos, sOpt, repoOpts...)
+	repoKeys, keyNames, err := withRepoKeys(repos, sOpt)
 	if err != nil {
 		return nil, err
 	}
 
 	opts = append(opts, dalec.ProgressGroup("Install build deps"))
-	installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, deps, importRepo, installWithConstraints(opts))(ctx, client, sOpt)
+	installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, deps, keyNames, dalec.WithRunOptions(importRepo, repoData, repoKeys),
+		installWithConstraints(opts))(ctx, client, sOpt)
 	if err != nil {
 		return nil, err
 	}
 
 	return func(in llb.State) llb.State {
 		return in.
-			Run(importRepo, repoMounts, installOpt, dalec.ShArgs("echo 'OPT REPO:'; ls -lrt /opt/repo"), dalec.ShArgs("ls /etc/yum.repos.d/"), dalec.WithConstraints(opts...)).Root()
+			Run(installOpt, dalec.WithConstraints(opts...)).Root()
 	}, nil
 }
 

frontend/azlinux/handler.go

@@ -22,7 +22,7 @@ type installFunc func(context.Context, gwclient.Client, dalec.SourceOpts) (llb.R
 
 type worker interface {
 	Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error)
-	Install(pkgs []string, repoOpt llb.RunOption, opts ...installOpt) llb.RunOption
+	Install(pkgs []string, opts ...installOpt) llb.RunOption
 	DefaultImageConfig(context.Context, llb.ImageMetaResolver, *ocispecs.Platform) (*dalec.DockerImageSpec, error)
 	WorkerImageConfig(context.Context, llb.ImageMetaResolver, *ocispecs.Platform) (*dalec.DockerImageSpec, error)
 	BasePackages() []string

frontend/azlinux/install.go

@@ -16,6 +16,10 @@ type installConfig struct {
 	// this is needed when installing unsigned RPMs.
 	noGPGCheck bool
 
+	// path for gpg keys to import for using a repo. These keys
+	// must also be added as mounts
+	keys []string
+
 	// Sets the root path to install rpms too.
 	// this acts like installing to a chroot.
 	root string
@@ -32,6 +36,14 @@ func noGPGCheck(cfg *installConfig) {
 	cfg.noGPGCheck = true
 }
 
+func importKeys(keys []string) installOpt {
+	return func(cfg *installConfig) {
+		for _, key := range keys {
+			cfg.keys = append(cfg.keys, key)
+		}
+	}
+}
+
 func withMounts(opts ...llb.RunOption) installOpt {
 	return func(cfg *installConfig) {
 		cfg.mounts = append(cfg.mounts, opts...)
@@ -109,14 +121,33 @@ rm -rf `+rpmdbDir+`
 `)), opts...)
 }
 
+func importScript(keyPaths []string) string {
+	keyRoot := "/etc/pki/rpm-gpg"
+
+	var importScript string = "#!/usr/bin/env sh\nset -u\n"
+	for _, keyPath := range keyPaths {
+		keyName := filepath.Base(keyPath)
+		importScript += fmt.Sprintf("gpg --import %s\n", filepath.Join(keyRoot, keyName))
+	}
+
+	return importScript
+}
+
 const manifestSh = "manifest.sh"
 
 func tdnfInstall(cfg *installConfig, relVer string, pkgs []string) llb.RunOption {
 	cmdFlags := tdnfInstallFlags(cfg)
-	cmdArgs := fmt.Sprintf("set -ex; tdnf install -y --refresh --releasever=%s %s %s", relVer, cmdFlags, strings.Join(pkgs, " "))
+	cmdArgs := fmt.Sprintf("set -ex; tdnf makecache; tdnf repolist; tdnf install -y --refresh --releasever=%s %s %s", relVer, cmdFlags, strings.Join(pkgs, " "))
 
 	var runOpts []llb.RunOption
 
+	if len(cfg.keys) > 0 {
+		importScript := importScript(cfg.keys)
+		cmdArgs = "/tmp/import-keys.sh; " + cmdArgs
+		runOpts = append(runOpts, llb.AddMount("/tmp/import-keys.sh", llb.Scratch().File(llb.Mkfile("/import-keys.sh", 0755, []byte(importScript))),
+			llb.SourcePath("/import-keys.sh")))
+	}
+
 	if cfg.manifest {
 		mfstScript := manifestScript(cfg.root, cfg.constraints...)
 

frontend/azlinux/mariner2.go

@@ -46,15 +46,15 @@ func (w mariner2) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S
 	}
 
 	return base.Run(
-		w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, dalec.NoOption(), installWithConstraints(opts)),
+		w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, installWithConstraints(opts)),
 		dalec.WithConstraints(opts...),
 	).Root(), nil
 }
 
-func (w mariner2) Install(pkgs []string, repoOpts llb.RunOption, opts ...installOpt) llb.RunOption {
+func (w mariner2) Install(pkgs []string, opts ...installOpt) llb.RunOption {
 	var cfg installConfig
 	setInstallOptions(&cfg, opts)
-	return dalec.WithRunOptions(repoOpts, tdnfInstall(&cfg, "2.0", pkgs), w.tdnfCacheMount(cfg.root))
+	return dalec.WithRunOptions(tdnfInstall(&cfg, "2.0", pkgs), w.tdnfCacheMount(cfg.root))
 }
 
 func (w mariner2) BasePackages() []string {

frontend/jammy/handle_container.go

@@ -178,10 +178,11 @@ func installTestDeps(spec *dalec.Spec, targetKey string, opts ...llb.Constraints
 			return in
 		}
 
+		depsList := dalec.SortMapKeys(deps)
 		opts = append(opts, dalec.ProgressGroup("Install test dependencies"))
 
 		return in.Run(
-			dalec.ShArgs("apt-get update && apt-get install -y --no-install-recommends "+strings.Join(deps, " ")),
+			dalec.ShArgs("apt-get update && apt-get install -y --no-install-recommends "+strings.Join(depsList, " ")),
 			llb.AddEnv("DEBIAN_FRONTEND", "noninteractive"),
 			dalec.WithMountedAptCache(AptCachePrefix),
 		).Root()

helpers.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"path"
 	"path/filepath"
-	"slices"
 	"sort"
 	"sync/atomic"
 
@@ -114,6 +113,14 @@ func NoOption() llb.RunOption {
 	return NoRunOption(func(*llb.ExecInfo) {})
 }
 
+type MaybeRunOption func(*llb.ExecInfo)
+
+func (f MaybeRunOption) SetRunOption(ei *llb.ExecInfo) {
+	if f != nil {
+		f(ei)
+	}
+}
+
 func WithCopyAction(opts ...llb.CopyOption) llb.CopyOption {
 	return copyOptionFunc(func(fa *llb.CopyInfo) {
 		for _, opt := range opts {
@@ -340,7 +347,31 @@ func (s *Spec) GetBuildRepos(targetKey string) []PackageRepositoryConfig {
 	return deps.GetExtraRepos("build")
 }
 
-func (s *Spec) GetTestDeps(targetKey string) []string {
+func (s *Spec) GetInstallRepos(targetKey string) []PackageRepositoryConfig {
+	deps := s.GetPackageDeps(targetKey)
+	if deps == nil {
+		deps = s.Dependencies
+		if deps == nil {
+			return nil
+		}
+	}
+
+	return deps.GetExtraRepos("install")
+}
+
+func (s *Spec) GetTestRepos(targetKey string) []PackageRepositoryConfig {
+	deps := s.GetPackageDeps(targetKey)
+	if deps == nil {
+		deps = s.Dependencies
+		if deps == nil {
+			return nil
+		}
+	}
+
+	return deps.GetExtraRepos("test")
+}
+
+func (s *Spec) GetTestDeps(targetKey string) map[string]PackageConstraints {
 	var deps *PackageDependencies
 	if t, ok := s.Targets[targetKey]; ok {
 		deps = t.Dependencies
@@ -353,9 +384,7 @@ func (s *Spec) GetTestDeps(targetKey string) []string {
 		}
 	}
 
-	out := slices.Clone(deps.Test)
-	slices.Sort(out)
-	return out
+	return deps.Test
 }
 
 func (s *Spec) GetImagePost(target string) *PostInstall {

spec.go

@@ -306,7 +306,7 @@ type PackageDependencies struct {
 	// These packages are only installed for tests which have steps that require
 	// running a command in the built container.
 	// See [TestSpec] for more information.
-	Test []string `yaml:"test,omitempty" json:"test,omitempty"`
+	Test map[string]PackageConstraints `yaml:"test,omitempty" json:"test,omitempty"`
 
 	// ExtraRepos is used to inject extra package repositories that may be used to
 	// satisfy package dependencies in various stages.