Custom repo support for AzLinux (#366)

Adds the ability to specify extra repositories, along with optional key imports and data for creating the repo.

Signed-off-by: adamperlin <adamp@nanosoft.com>
Co-authored-by: Brian Goff <cpuguy83@gmail.com>

docs/spec.schema.json

@@ -548,12 +548,63 @@
 					},
 					"type": "array",
 					"description": "Test lists any extra packages required for running tests\nThese packages are only installed for tests which have steps that require\nrunning a command in the built container.\nSee [TestSpec] for more information."
+				},
+				"extra_repos": {
+					"items": {
+						"$ref": "#/$defs/PackageRepositoryConfig"
+					},
+					"type": "array",
+					"description": "ExtraRepos is used to inject extra package repositories that may be used to\nsatisfy package dependencies in various stages."
 				}
 			},
 			"additionalProperties": false,
 			"type": "object",
 			"description": "PackageDependencies is a list of dependencies for a package."
 		},
+		"PackageRepositoryConfig": {
+			"properties": {
+				"keys": {
+					"additionalProperties": {
+						"$ref": "#/$defs/Source"
+					},
+					"type": "object",
+					"description": "Keys are the list of keys that need to be imported to use the configured\nrepositories"
+				},
+				"config": {
+					"additionalProperties": {
+						"$ref": "#/$defs/Source"
+					},
+					"type": "object",
+					"description": "Config list of repo configs to to add to the environment.  The format of\nthese configs are distro specific (e.g. apt/yum configs)."
+				},
+				"data": {
+					"items": {
+						"$ref": "#/$defs/SourceMount"
+					},
+					"type": "array",
+					"description": "Data lists all the extra data that needs to be made available for the\nprovided repository config to work.\nAs an example, if the provided config is referencing a file backed repository\nthen data would include the file data, assuming its not already available\nin the environment."
+				},
+				"envs": {
+					"items": {
+						"type": "string",
+						"enum": [
+							"build",
+							"test",
+							"install"
+						]
+					},
+					"type": "array",
+					"description": "Envs specifies the list of environments to make the repositories available\nduring.\nAcceptable values are:\n - \"build\"   - Repositories are added prior to installing build dependencies\n - \"test\"    - Repositories are added prior to installing test dependencies\n - \"install\" - Repositories are added prior to installing the output\n               package in a container build target."
+				}
+			},
+			"additionalProperties": false,
+			"type": "object",
+			"required": [
+				"config",
+				"envs"
+			],
+			"description": "PackageRepositoryConfig"
+		},
 		"PackageSigner": {
 			"properties": {
 				"image": {
@@ -860,7 +911,7 @@
 				"dest",
 				"spec"
 			],
-			"description": "SourceMount is used to take a [Source] and mount it into a build step."
+			"description": "SourceMount wraps a [Source] with a target mount point."
 		},
 		"Spec": {
 			"properties": {

frontend/azlinux/handle_container.go

@@ -55,12 +55,22 @@ func specToContainerLLB(w worker, spec *dalec.Spec, targetKey string, rpmDir llb
 		rootfs = llb.Image(ref, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...))
 	}
 
+	installTimeRepos := spec.GetInstallRepos(targetKey)
+	importRepos, err := repoMountInstallOpts(installTimeRepos, sOpt, opts...)
+	if err != nil {
+		return llb.Scratch(), err
+	}
+
 	rpmMountDir := "/tmp/rpms"
 	pkgs := w.BasePackages()
 	pkgs = append(pkgs, filepath.Join(rpmMountDir, "**/*.rpm"))
 
+	installOpts := []installOpt{atRoot(workPath)}
+	installOpts = append(installOpts, importRepos...)
+	installOpts = append(installOpts, []installOpt{noGPGCheck, withManifests, installWithConstraints(opts)}...)
+
 	rootfs = builderImg.Run(
-		w.Install(pkgs, atRoot(workPath), noGPGCheck, withManifests, installWithConstraints(opts)),
+		w.Install(pkgs, installOpts...),
 		llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS")),
 		dalec.WithConstraints(opts...),
 	).AddMount(workPath, rootfs)

frontend/azlinux/handle_rpm.go

@@ -97,18 +97,52 @@ func runTests(ctx context.Context, client gwclient.Client, w worker, spec *dalec
 	return ref, errors.Wrap(err, "TESTS FAILED")
 }
 
+var azLinuxRepoConfig = dalec.RepoPlatformConfig{
+	ConfigRoot: "/etc/yum.repos.d",
+	GPGKeyRoot: "/etc/pki/rpm-gpg",
+}
+
+func repoMountInstallOpts(repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]installOpt, error) {
+	withRepos, err := dalec.WithRepoConfigs(repos, &azLinuxRepoConfig, sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	withData, err := dalec.WithRepoData(repos, sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	keyMounts, keyPaths, err := dalec.GetRepoKeys(repos, &azLinuxRepoConfig, sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	repoMounts := dalec.WithRunOptions(withRepos, withData, keyMounts)
+	return []installOpt{withMounts(repoMounts), importKeys(keyPaths)}, nil
+}
+
 func withTestDeps(w worker, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) {
 	base, err := w.Base(sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}
+
+	testRepos := spec.GetTestRepos(targetKey)
+	importRepos, err := repoMountInstallOpts(testRepos, sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
 	return func(in llb.State) llb.State {
 		deps := spec.GetTestDeps(targetKey)
 		if len(deps) == 0 {
 			return in
 		}
+
+		installOpts := []installOpt{atRoot("/tmp/rootfs")}
 		return base.Run(
-			w.Install(spec.GetTestDeps(targetKey), atRoot("/tmp/rootfs")),
+			w.Install(deps, append(installOpts, importRepos...)...),
 			dalec.WithConstraints(opts...),
 			dalec.ProgressGroup("Install test dependencies"),
 		).AddMount("/tmp/rootfs", in)
@@ -161,14 +195,21 @@ func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spe
 		return func(in llb.State) llb.State { return in }, nil
 	}
 
+	repos := spec.GetBuildRepos(targetKey)
+
 	sOpt, err := frontend.SourceOptFromClient(ctx, client)
 	if err != nil {
 		return nil, err
 	}
 
-	opts = append(opts, dalec.ProgressGroup("Install build deps"))
+	importRepos, err := repoMountInstallOpts(repos, sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
 
-	installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, deps, installWithConstraints(opts))(ctx, client, sOpt)
+	opts = append(opts, dalec.ProgressGroup("Install build deps"))
+	installOpt, err := installBuildDepsPackage(targetKey, spec.Name, w, deps,
+		append(importRepos, installWithConstraints(opts))...)(ctx, client, sOpt)
 	if err != nil {
 		return nil, err
 	}

frontend/azlinux/install.go

@@ -16,6 +16,10 @@ type installConfig struct {
 	// this is needed when installing unsigned RPMs.
 	noGPGCheck bool
 
+	// path for gpg keys to import for using a repo. These files for these keys
+	// must also be added as mounts
+	keys []string
+
 	// Sets the root path to install rpms too.
 	// this acts like installing to a chroot.
 	root string
@@ -32,6 +36,13 @@ func noGPGCheck(cfg *installConfig) {
 	cfg.noGPGCheck = true
 }
 
+// see comment in tdnfInstall for why this additional option is needed
+func importKeys(keys []string) installOpt {
+	return func(cfg *installConfig) {
+		cfg.keys = append(cfg.keys, keys...)
+	}
+}
+
 func withMounts(opts ...llb.RunOption) installOpt {
 	return func(cfg *installConfig) {
 		cfg.mounts = append(cfg.mounts, opts...)
@@ -109,14 +120,43 @@ rm -rf `+rpmdbDir+`
 `)), opts...)
 }
 
+func importGPGScript(keyPaths []string) string {
+	// all keys that are included should be mounted under this path
+	keyRoot := "/etc/pki/rpm-gpg"
+
+	var importScript string = "#!/usr/bin/env sh\nset -eux\n"
+	for _, keyPath := range keyPaths {
+		keyName := filepath.Base(keyPath)
+		importScript += fmt.Sprintf("gpg --import %s\n", filepath.Join(keyRoot, keyName))
+	}
+
+	return importScript
+}
+
 const manifestSh = "manifest.sh"
 
 func tdnfInstall(cfg *installConfig, relVer string, pkgs []string) llb.RunOption {
 	cmdFlags := tdnfInstallFlags(cfg)
-	cmdArgs := fmt.Sprintf("set -ex; tdnf install -y --refresh --releasever=%s %s %s", relVer, cmdFlags, strings.Join(pkgs, " "))
+	// tdnf makecache is needed to ensure that the package metadata is up to date if extra repo
+	// config files have been mounted
+	cmdArgs := fmt.Sprintf("set -ex; tdnf makecache; tdnf install -y --refresh --releasever=%s %s %s", relVer, cmdFlags, strings.Join(pkgs, " "))
 
 	var runOpts []llb.RunOption
 
+	// If we have keys to import in order to access a repo, we need to create a script to use `gpg` to import them
+	// This is an unfortunate consequence of a bug in tdnf (see https://github.com/vmware/tdnf/issues/471).
+	// The keys *should* be imported automatically by tdnf as long as the repo config references them correctly and
+	// we mount the key files themselves under the right path. However, tdnf does NOT do this
+	// currently if the keys are referenced via a `file:///` type url,
+	// and we must manually import the keys as well.
+	if len(cfg.keys) > 0 {
+		importScript := importGPGScript(cfg.keys)
+		cmdArgs = "/tmp/import-keys.sh; " + cmdArgs
+		runOpts = append(runOpts, llb.AddMount("/tmp/import-keys.sh",
+			llb.Scratch().File(llb.Mkfile("/import-keys.sh", 0755, []byte(importScript))),
+			llb.SourcePath("/import-keys.sh")))
+	}
+
 	if cfg.manifest {
 		mfstScript := manifestScript(cfg.root, cfg.constraints...)
 

helpers.go

@@ -301,6 +301,42 @@ func (s *Spec) GetBuildDeps(targetKey string) map[string]PackageConstraints {
 	return deps.Build
 }
 
+func (s *Spec) GetBuildRepos(targetKey string) []PackageRepositoryConfig {
+	deps := s.GetPackageDeps(targetKey)
+	if deps == nil {
+		deps = s.Dependencies
+		if deps == nil {
+			return nil
+		}
+	}
+
+	return deps.GetExtraRepos("build")
+}
+
+func (s *Spec) GetInstallRepos(targetKey string) []PackageRepositoryConfig {
+	deps := s.GetPackageDeps(targetKey)
+	if deps == nil {
+		deps = s.Dependencies
+		if deps == nil {
+			return nil
+		}
+	}
+
+	return deps.GetExtraRepos("install")
+}
+
+func (s *Spec) GetTestRepos(targetKey string) []PackageRepositoryConfig {
+	deps := s.GetPackageDeps(targetKey)
+	if deps == nil {
+		deps = s.Dependencies
+		if deps == nil {
+			return nil
+		}
+	}
+
+	return deps.GetExtraRepos("test")
+}
+
 func (s *Spec) GetTestDeps(targetKey string) []string {
 	var deps *PackageDependencies
 	if t, ok := s.Targets[targetKey]; ok {
@@ -412,6 +448,7 @@ func (s *Spec) GetPackageDeps(target string) *PackageDependencies {
 	if deps := s.Targets[target]; deps.Dependencies != nil {
 		return deps.Dependencies
 	}
+
 	return s.Dependencies
 }
 
@@ -420,3 +457,88 @@ type gitOptionFunc func(*llb.GitInfo)
 func (f gitOptionFunc) SetGitOption(gi *llb.GitInfo) {
 	f(gi)
 }
+
+type RepoPlatformConfig struct {
+	ConfigRoot string
+	GPGKeyRoot string
+}
+
+// Returns a run option which mounts the data dirs for all specified repos
+func WithRepoData(repos []PackageRepositoryConfig, sOpts SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+	var repoMountsOpts []llb.RunOption
+	for _, repo := range repos {
+		rs, err := repoDataAsMount(repo, sOpts, opts...)
+		if err != nil {
+			return nil, err
+		}
+		repoMountsOpts = append(repoMountsOpts, rs)
+	}
+
+	return WithRunOptions(repoMountsOpts...), nil
+}
+
+// Returns a run option for mounting the state (i.e., packages/metadata) for a single repo
+func repoDataAsMount(config PackageRepositoryConfig, sOpts SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+	var mounts []llb.RunOption
+	for _, data := range config.Data {
+		repoState, err := data.Spec.AsMount(data.Dest, sOpts, opts...)
+		if err != nil {
+			return nil, err
+		}
+		mounts = append(mounts, llb.AddMount(data.Dest, repoState))
+	}
+
+	return WithRunOptions(mounts...), nil
+}
+
+func repoConfigAsMount(config PackageRepositoryConfig, platformCfg *RepoPlatformConfig, sOpt SourceOpts, opts ...llb.ConstraintsOpt) ([]llb.RunOption, error) {
+	repoConfigs := []llb.RunOption{}
+
+	for name, repoConfig := range config.Config {
+		// each of these sources represent a repo config file
+		repoConfigSt, err := repoConfig.AsState(name, sOpt, append(opts, ProgressGroup("Importing repo config: "+name))...)
+		if err != nil {
+			return nil, err
+		}
+
+		repoConfigs = append(repoConfigs,
+			llb.AddMount(filepath.Join(platformCfg.ConfigRoot, name), repoConfigSt, llb.SourcePath(name)))
+	}
+
+	return repoConfigs, nil
+}
+
+// Returns a run option for importing the config files for all repos
+func WithRepoConfigs(repos []PackageRepositoryConfig, cfg *RepoPlatformConfig, sOpt SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+	configStates := []llb.RunOption{}
+	for _, repo := range repos {
+		mnts, err := repoConfigAsMount(repo, cfg, sOpt, opts...)
+		if err != nil {
+			return nil, err
+		}
+
+		configStates = append(configStates, mnts...)
+	}
+
+	return WithRunOptions(configStates...), nil
+}
+
+func GetRepoKeys(configs []PackageRepositoryConfig, cfg *RepoPlatformConfig, sOpt SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, []string, error) {
+	keys := []llb.RunOption{}
+	names := []string{}
+	for _, config := range configs {
+		for name, repoKey := range config.Keys {
+			// each of these sources represent a gpg key file for a particular repo
+			gpgKey, err := repoKey.AsState(name, sOpt, append(opts, ProgressGroup("Importing repo key: "+name))...)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			keys = append(keys,
+				llb.AddMount(filepath.Join(cfg.GPGKeyRoot, name), gpgKey, llb.SourcePath(name)))
+			names = append(names, name)
+		}
+	}
+
+	return WithRunOptions(keys...), names, nil
+}