Attempt to allow installing extra repos just for the build stage. Thi…

…ngs are broken right now, but the general idea is to create mounts for

1. The repo config files
2. Any pgp keys that need to be imported
3. Any repo data that needs to be accessible

and then to apply those mounts as RunOptions when we tdnf install the build dependencies package. This doesn't seem to be working properly and I'm
not confident RunOptions for the mounts are threaded through exactly right.

frontend/azlinux/azlinux3.go

@@ -49,12 +49,12 @@ func (w azlinux3) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S
 
 	img := llb.Image(Azlinux3Ref, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...))
 	return img.Run(
-		w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, installWithConstraints(opts)),
+		w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, dalec.NoOption(), installWithConstraints(opts)),
 		dalec.WithConstraints(opts...),
 	).Root(), nil
 }
 
-func (w azlinux3) Install(pkgs []string, opts ...installOpt) llb.RunOption {
+func (w azlinux3) Install(pkgs []string, repoOpt llb.RunOption, opts ...installOpt) llb.RunOption {
 	var cfg installConfig
 	setInstallOptions(&cfg, opts)
 	return dalec.WithRunOptions(tdnfInstall(&cfg, "3.0", pkgs), w.tdnfCacheMount(cfg.root))

frontend/azlinux/handle_container.go

@@ -73,7 +73,7 @@ func handleContainer(w worker) gwclient.BuildFunc {
 					return in
 				}
 				return base.Run(
-					w.Install(spec.GetTestDeps(targetKey), atRoot("/tmp/rootfs")),
+					w.Install(spec.GetTestDeps(targetKey), dalec.NoOption(), atRoot("/tmp/rootfs")),
 					pg,
 					dalec.ProgressGroup("Install test dependencies"),
 				).AddMount("/tmp/rootfs", in)
@@ -157,7 +157,7 @@ func specToContainerLLB(w worker, spec *dalec.Spec, targetKey string, rpmDir llb
 		}
 
 		rootfs = builderImg.Run(
-			w.Install(updated, atRoot(workPath), noGPGCheck, withManifests, installWithConstraints(opts)),
+			w.Install(updated, dalec.NoOption(), atRoot(workPath), noGPGCheck, withManifests, installWithConstraints(opts)),
 			llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS")),
 			dalec.WithConstraints(opts...),
 		).AddMount(workPath, rootfs)

frontend/azlinux/handle_rpm.go

@@ -53,7 +53,7 @@ func handleRPM(w worker) gwclient.BuildFunc {
 }
 
 // Creates and installs an rpm meta-package that requires the passed in deps as runtime-dependencies
-func installBuildDepsPackage(target string, packageName string, w worker, deps map[string]dalec.PackageConstraints, installOpts ...installOpt) installFunc {
+func installBuildDepsPackage(target string, packageName string, w worker, deps map[string]dalec.PackageConstraints, repoOpts llb.RunOption, installOpts ...installOpt) installFunc {
 	// depsOnly is a simple dalec spec that only includes build dependencies and their constraints
 	depsOnly := dalec.Spec{
 		Name:        fmt.Sprintf("%s-build-dependencies", packageName),
@@ -87,30 +87,115 @@ func installBuildDepsPackage(target string, packageName string, w worker, deps m
 		}, installOpts...)
 
 		// install the built RPMs into the worker itself
-		return w.Install([]string{"/tmp/rpms/*/*.rpm"}, installOpts...), nil
+		return w.Install([]string{"/tmp/rpms/*/*.rpm"}, repoOpts, installOpts...), nil
 	}
 }
 
+// meant to return a run option for mounting all repo state
+func getRepoMounts(repos []dalec.PackageRepositoryConfig, sOpts dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+	var repoMountsOpts []llb.RunOption
+	for _, repo := range repos {
+		rs, err := mountsForRepo(repo, sOpts, opts...)
+		if err != nil {
+			return nil, err
+		}
+		repoMountsOpts = append(repoMountsOpts, rs)
+	}
+
+	return dalec.WithRunOptions(repoMountsOpts...), nil
+}
+
+// meant to return a run option for mounting state for a single repo
+func mountsForRepo(config dalec.PackageRepositoryConfig, sOpts dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+	var mounts []llb.RunOption
+	for _, data := range config.Data {
+		repoState, err := data.Spec.AsMount(data.Dest, sOpts, opts...)
+		if err != nil {
+			return nil, err
+		}
+		mounts = append(mounts, llb.AddMount(data.Dest, repoState))
+	}
+
+	return dalec.WithRunOptions(mounts...), nil
+}
+
+// meant to return a run option for importing the config files for all repos (including keys)
+func importRepos(repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+	configStates := []llb.RunOption{}
+	for _, repo := range repos {
+		mnts, err := importRepo(repo, sOpt, opts...)
+		if err != nil {
+			return nil, err
+		}
+
+		configStates = append(configStates, mnts...)
+	}
+
+	return dalec.WithRunOptions(configStates...), nil
+}
+
+func importRepo(config dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]llb.RunOption, error) {
+	repoConfigs := []llb.RunOption{}
+	keys := []llb.RunOption{}
+
+	for name, repoConfig := range config.Config {
+		// each of these sources represent a repo config file
+		// TODO: need to handle file vs. dir case
+		repoConfigSt, err := repoConfig.AsState(name, sOpt, opts...)
+		if err != nil {
+			return nil, err
+		}
+
+		repoConfigs = append(repoConfigs,
+			llb.AddMount(filepath.Join("/etc/yum.repos.d", name), repoConfigSt, llb.SourcePath(name)))
+	}
+
+	for name, repoKey := range config.Keys {
+		// each of these sources represent a gpg key file for a particular repo
+		gpgKey, err := repoKey.AsState(name, sOpt)
+		if err != nil {
+			return nil, err
+		}
+
+		keys = append(keys, llb.AddMount(filepath.Join("/etc/pki/rpm-gpg", name), gpgKey, llb.SourcePath(name)))
+	}
+
+	return append(repoConfigs, keys...), nil
+}
+
 func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) {
 	deps := spec.GetBuildDeps(targetKey)
 	if len(deps) == 0 {
 		return func(in llb.State) llb.State { return in }, nil
 	}
 
+	repos := spec.GetBuildRepos(targetKey)
+
 	sOpt, err := frontend.SourceOptFromClient(ctx, client)
 	if err != nil {
 		return nil, err
 	}
 
-	opts = append(opts, dalec.ProgressGroup("Install build deps"))
+	repoOpts := append(opts, dalec.ProgressGroup("Install build repos"))
+	repoMounts, err := getRepoMounts(repos, sOpt, repoOpts...)
+	if err != nil {
+		return nil, err
+	}
 
-	installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, deps, installWithConstraints(opts))(ctx, client, sOpt)
+	importRepo, err := importRepos(repos, sOpt, repoOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	opts = append(opts, dalec.ProgressGroup("Install build deps"))
+	installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, deps, importRepo, installWithConstraints(opts))(ctx, client, sOpt)
 	if err != nil {
 		return nil, err
 	}
 
 	return func(in llb.State) llb.State {
-		return in.Run(installOpt, dalec.WithConstraints(opts...)).Root()
+		return in.
+			Run(importRepo, repoMounts, installOpt, dalec.ShArgs("echo 'OPT REPO:'; ls -lrt /opt/repo"), dalec.ShArgs("ls /etc/yum.repos.d/"), dalec.WithConstraints(opts...)).Root()
 	}, nil
 }
 

frontend/azlinux/handler.go

@@ -22,7 +22,7 @@ type installFunc func(context.Context, gwclient.Client, dalec.SourceOpts) (llb.R
 
 type worker interface {
 	Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, error)
-	Install(pkgs []string, opts ...installOpt) llb.RunOption
+	Install(pkgs []string, repoOpt llb.RunOption, opts ...installOpt) llb.RunOption
 	DefaultImageConfig(context.Context, llb.ImageMetaResolver, *ocispecs.Platform) (*dalec.DockerImageSpec, error)
 	WorkerImageConfig(context.Context, llb.ImageMetaResolver, *ocispecs.Platform) (*dalec.DockerImageSpec, error)
 	BasePackages() []string

frontend/azlinux/mariner2.go

@@ -46,15 +46,15 @@ func (w mariner2) Base(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.S
 	}
 
 	return base.Run(
-		w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, installWithConstraints(opts)),
+		w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential", "ca-certificates"}, dalec.NoOption(), installWithConstraints(opts)),
 		dalec.WithConstraints(opts...),
 	).Root(), nil
 }
 
-func (w mariner2) Install(pkgs []string, opts ...installOpt) llb.RunOption {
+func (w mariner2) Install(pkgs []string, repoOpts llb.RunOption, opts ...installOpt) llb.RunOption {
 	var cfg installConfig
 	setInstallOptions(&cfg, opts)
-	return dalec.WithRunOptions(tdnfInstall(&cfg, "2.0", pkgs), w.tdnfCacheMount(cfg.root))
+	return dalec.WithRunOptions(repoOpts, tdnfInstall(&cfg, "2.0", pkgs), w.tdnfCacheMount(cfg.root))
 }
 
 func (w mariner2) BasePackages() []string {

helpers.go

@@ -95,6 +95,33 @@ func WithRunOptions(opts ...llb.RunOption) llb.RunOption {
 	})
 }
 
+func WithFileActions(actions ...*llb.FileAction) llb.StateOption {
+	return func(s llb.State) llb.State {
+		for _, act := range actions {
+			s = s.File(act)
+		}
+
+		return s
+	}
+}
+
+type NoRunOption func(*llb.ExecInfo)
+
+func (f NoRunOption) SetRunOption(ei *llb.ExecInfo) {
+}
+
+func NoOption() llb.RunOption {
+	return NoRunOption(func(*llb.ExecInfo) {})
+}
+
+func WithCopyAction(opts ...llb.CopyOption) llb.CopyOption {
+	return copyOptionFunc(func(fa *llb.CopyInfo) {
+		for _, opt := range opts {
+			opt.SetCopyOption(fa)
+		}
+	})
+}
+
 type constraintsOptFunc func(*llb.Constraints)
 
 func (f constraintsOptFunc) SetConstraintsOption(c *llb.Constraints) {
@@ -301,6 +328,18 @@ func (s *Spec) GetBuildDeps(targetKey string) map[string]PackageConstraints {
 	return deps.Build
 }
 
+func (s *Spec) GetBuildRepos(targetKey string) []PackageRepositoryConfig {
+	deps := s.GetPackageDeps(targetKey)
+	if deps == nil {
+		deps = s.Dependencies
+		if deps == nil {
+			return nil
+		}
+	}
+
+	return deps.GetExtraRepos("build")
+}
+
 func (s *Spec) GetTestDeps(targetKey string) []string {
 	var deps *PackageDependencies
 	if t, ok := s.Targets[targetKey]; ok {
@@ -411,5 +450,6 @@ func (s *Spec) GetPackageDeps(target string) *PackageDependencies {
 	if deps := s.Targets[target]; deps.Dependencies != nil {
 		return deps.Dependencies
 	}
+
 	return s.Dependencies
 }

spec.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/fs"
 	"regexp"
+	"slices"
 	"strings"
 	"time"
 
@@ -312,14 +313,27 @@ type PackageDependencies struct {
 	ExtraRepos []PackageRepositoryConfig `yaml:"extra_repos,omitempty" json:"extra_repos,omitempty"`
 }
 
+func (p *PackageDependencies) GetExtraRepos(env string) []PackageRepositoryConfig {
+	var repos []PackageRepositoryConfig
+	for _, repo := range p.ExtraRepos {
+		if slices.Contains(repo.Envs, env) {
+			repos = append(repos, repo)
+		}
+	}
+
+	return repos
+}
+
 // PackageRepositoryConfig
 type PackageRepositoryConfig struct {
 	// Keys are the list of keys that need to be imported to use the configured
 	// repositories
-	Keys []Source `yaml:"keys,omitempty" json:"keys,omitempty"`
+	Keys map[string]Source `yaml:"keys,omitempty" json:"keys,omitempty"`
+
 	// Config list of repo configs to to add to the environment.  The format of
 	// these configs are distro specific (e.g. apt/yum configs).
-	Config []Source `yaml:"config" json:"config"`
+	Config map[string]Source `yaml:"config" json:"config"`
+
 	// Data lists all the extra data that needs to be made available for the
 	// provided repository config to work.
 	// As an example, if the provided config is referencing a file backed repository