Add Custom Repo Support For Jammy (#412)

frontend/azlinux/handle_container.go

@@ -56,7 +56,7 @@ func specToContainerLLB(w worker, spec *dalec.Spec, targetKey string, rpmDir llb
 	}
 
 	installTimeRepos := spec.GetInstallRepos(targetKey)
-	importRepos, err := repoMountInstallOpts(installTimeRepos, sOpt, opts...)
+	importRepos, err := repoMountInstallOpts(builderImg, installTimeRepos, sOpt, opts...)
 	if err != nil {
 		return llb.Scratch(), err
 	}

frontend/azlinux/handle_rpm.go

@@ -102,7 +102,7 @@ var azLinuxRepoConfig = dalec.RepoPlatformConfig{
 	GPGKeyRoot: "/etc/pki/rpm-gpg",
 }
 
-func repoMountInstallOpts(repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]installOpt, error) {
+func repoMountInstallOpts(worker llb.State, repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]installOpt, error) {
 	withRepos, err := dalec.WithRepoConfigs(repos, &azLinuxRepoConfig, sOpt, opts...)
 	if err != nil {
 		return nil, err
@@ -113,7 +113,7 @@ func repoMountInstallOpts(repos []dalec.PackageRepositoryConfig, sOpt dalec.Sour
 		return nil, err
 	}
 
-	keyMounts, keyPaths, err := dalec.GetRepoKeys(repos, &azLinuxRepoConfig, sOpt, opts...)
+	keyMounts, keyPaths, err := dalec.GetRepoKeys(worker, repos, &azLinuxRepoConfig, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -129,7 +129,7 @@ func withTestDeps(w worker, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey s
 	}
 
 	testRepos := spec.GetTestRepos(targetKey)
-	importRepos, err := repoMountInstallOpts(testRepos, sOpt, opts...)
+	importRepos, err := repoMountInstallOpts(base, testRepos, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -202,7 +202,12 @@ func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spe
 		return nil, err
 	}
 
-	importRepos, err := repoMountInstallOpts(repos, sOpt, opts...)
+	base, err := w.Base(sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	importRepos, err := repoMountInstallOpts(base, repos, sOpt, opts...)
 	if err != nil {
 		return nil, err
 	}

frontend/jammy/handle_container.go

@@ -17,7 +17,7 @@ import (
 const (
 	jammyRef = "mcr.microsoft.com/mirror/docker/library/ubuntu:jammy"
 
-	testRepoPath           = "/opt/testrepo"
+	testRepoPath           = "/opt/repo"
 	testRepoSourceListPath = "/etc/apt/sources.list.d/test-dalec-local-repo.list"
 )
 
@@ -71,7 +71,7 @@ func buildImageConfig(ctx context.Context, resolver llb.ImageMetaResolver, spec
 	return img, nil
 }
 
-func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, deb llb.State, targetKey string, includeTestRepo bool, opts ...llb.ConstraintsOpt) llb.State {
+func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, deb llb.State, targetKey string, includeTestRepo bool, opts ...llb.ConstraintsOpt) (llb.State, error) {
 	base := dalec.GetBaseOutputImage(spec, targetKey)
 
 	installSymlinks := func(in llb.State) llb.State {
@@ -90,6 +90,11 @@ func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts,
 			AddMount(workPath, in)
 	}
 
+	customRepoOpts, err := customRepoMounts(worker, spec.GetInstallRepos(targetKey), sOpt, opts...)
+	if err != nil {
+		return llb.Scratch(), err
+	}
+
 	if base == "" {
 		base = jammyRef
 	}
@@ -98,8 +103,10 @@ func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts,
 
 	debug := llb.Scratch().File(llb.Mkfile("debug", 0o644, []byte(`debug=2`)), opts...)
 	opts = append(opts, dalec.ProgressGroup("Install spec package"))
+
 	return baseImg.Run(
-		dalec.ShArgs("set -x; apt update && apt install -y /tmp/pkg/*.deb && exit 0; ls -lh /etc/apt/sources.list.d; ls -lh /etc/testrepo; mount; exit 42"),
+		dalec.ShArgs("set -x; apt update && apt install -y /tmp/pkg/*.deb"),
+		customRepoOpts,
 		llb.AddEnv("DEBIAN_FRONTEND", "noninteractive"),
 		llb.AddMount("/tmp/pkg", deb, llb.Readonly),
 		dalec.WithMountedAptCache(AptCachePrefix),
@@ -121,22 +128,27 @@ func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts,
 			llb.AddMount("/etc/dpkg/dpkg.cfg.d/excludes", tmp, llb.SourcePath("tmp")).SetRunOption(cfg)
 		}),
 	).Root().
-		With(installSymlinks)
+		With(installSymlinks), nil
 }
 
-func installTestDeps(spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) llb.StateOption {
-	return func(in llb.State) llb.State {
-		deps := spec.GetTestDeps(targetKey)
-		if len(deps) == 0 {
-			return in
-		}
+func installTestDeps(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) {
+	deps := spec.GetTestDeps(targetKey)
+	if len(deps) == 0 {
+		return func(s llb.State) llb.State { return s }, nil
+	}
 
-		opts = append(opts, dalec.ProgressGroup("Install test dependencies"))
+	extraRepoOpts, err := customRepoMounts(worker, spec.GetTestRepos(targetKey), sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
 
+	return func(in llb.State) llb.State {
+		opts = append(opts, dalec.ProgressGroup("Install test dependencies"))
 		return in.Run(
 			dalec.ShArgs("apt-get update && apt-get install -y --no-install-recommends "+strings.Join(deps, " ")),
 			llb.AddEnv("DEBIAN_FRONTEND", "noninteractive"),
+			extraRepoOpts,
 			dalec.WithMountedAptCache(AptCachePrefix),
 		).Root()
-	}
+	}, nil
 }

frontend/jammy/handle_deb.go

@@ -90,7 +90,10 @@ func runTests(ctx context.Context, client gwclient.Client, spec *dalec.Spec, sOp
 		frontend.Warn(ctx, client, worker, "Including test repo from worker image")
 	}
 
-	st := buildImageRootfs(worker, spec, sOpt, deb, targetKey, includeTestRepo, opts...)
+	st, err := buildImageRootfs(worker, spec, sOpt, deb, targetKey, includeTestRepo, opts...)
+	if err != nil {
+		return nil, err
+	}
 
 	def, err := st.Marshal(ctx, opts...)
 	if err != nil {
@@ -109,19 +112,49 @@ func runTests(ctx context.Context, client gwclient.Client, spec *dalec.Spec, sOp
 		return nil, err
 	}
 
-	err = frontend.RunTests(ctx, client, spec, ref, installTestDeps(spec, targetKey, opts...), targetKey)
+	withTestDeps, err := installTestDeps(worker, spec, sOpt, targetKey, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	err = frontend.RunTests(ctx, client, spec, ref, withTestDeps, targetKey)
 	return ref, err
 }
 
-func installPackages(ls ...string) llb.RunOption {
+var jammyRepoPlatformCfg = dalec.RepoPlatformConfig{
+	ConfigRoot: "/etc/apt/sources.list.d",
+	GPGKeyRoot: "/usr/share/keyrings",
+}
+
+func customRepoMounts(worker llb.State, repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) {
+	withRepos, err := dalec.WithRepoConfigs(repos, &jammyRepoPlatformCfg, sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	withData, err := dalec.WithRepoData(repos, sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	keyMounts, _, err := dalec.GetRepoKeys(worker, repos, &jammyRepoPlatformCfg, sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return dalec.WithRunOptions(withRepos, withData, keyMounts), nil
+}
+
+func installPackages(ls []string, rOpts ...llb.RunOption) llb.RunOption {
 	return dalec.RunOptFunc(func(ei *llb.ExecInfo) {
 		// This only runs apt-get update if the pkgcache is older than 10 minutes.
 		dalec.ShArgs(`set -ex; apt update; apt install -y ` + strings.Join(ls, " ")).SetRunOption(ei)
 		dalec.WithMountedAptCache(AptCachePrefix).SetRunOption(ei)
+		dalec.WithRunOptions(rOpts...).SetRunOption(ei)
 	})
 }
 
-func installWithConstraints(pkgPath string, pkgName string) llb.RunOption {
+func installWithConstraints(pkgPath string, pkgName string, rOpts ...llb.RunOption) llb.RunOption {
 	return dalec.RunOptFunc(func(ei *llb.ExecInfo) {
 		// The apt solver always tries to select the latest package version even when constraints specify that an older version should be installed and that older version is available in a repo.
 		// This leads the solver to simply refuse to install our target package if the latest version of ANY dependency package is incompatible with the constraints.
@@ -131,6 +164,7 @@ func installWithConstraints(pkgPath string, pkgName string) llb.RunOption {
 		dalec.ShArgs(`set -ex; dpkg -i --force-depends ` + pkgPath +
 			fmt.Sprintf(`; apt update; aptitude install -y -f -o "Aptitude::ProblemResolver::Hints::=reject %s :UNINST"`, pkgName)).SetRunOption(ei)
 		dalec.WithMountedAptCache(AptCachePrefix).SetRunOption(ei)
+		dalec.WithRunOptions(rOpts...).SetRunOption(ei)
 	})
 }
 
@@ -181,14 +215,23 @@ func workerBase(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, e
 		return *base, nil
 	}
 
-	return llb.Image(jammyRef, llb.WithMetaResolver(sOpt.Resolver)).With(basePackages(opts...)), nil
+	return llb.Image(jammyRef, llb.WithMetaResolver(sOpt.Resolver)).With(basePackages(opts...)).
+		// This file prevents installation of things like docs in ubuntu
+		// containers We don't want to exclude this because tests want to
+		// check things for docs in the build container. But we also don't
+		// want to remove this completely from the base worker image in the
+		// frontend because we usually don't want such things in the build
+		// environment. This is only needed because certain tests (which
+		// are using this customized builder image) are checking for files
+		// that are being excluded by this config file.
+		File(llb.Rm("/etc/dpkg/dpkg.cfg.d/excludes", llb.WithAllowNotFound(true))), nil
 }
 
 func basePackages(opts ...llb.ConstraintsOpt) llb.StateOption {
 	return func(in llb.State) llb.State {
 		opts = append(opts, dalec.ProgressGroup("Install base packages"))
 		return in.Run(
-			installPackages("aptitude", "dpkg-dev", "devscripts", "equivs", "fakeroot", "dh-make", "build-essential", "dh-apparmor", "dh-make", "dh-exec", "debhelper-compat="+deb.DebHelperCompat),
+			installPackages([]string{"aptitude", "dpkg-dev", "devscripts", "equivs", "fakeroot", "dh-make", "build-essential", "dh-apparmor", "dh-make", "dh-exec", "debhelper-compat=" + deb.DebHelperCompat}),
 			dalec.WithConstraints(opts...),
 		).Root()
 	}
@@ -231,13 +274,18 @@ func buildDepends(worker llb.State, sOpt dalec.SourceOpts, spec *dalec.Spec, tar
 		return nil, errors.Wrap(err, "error creating intermediate package for installing build dependencies")
 	}
 
+	customRepoOpts, err := customRepoMounts(worker, spec.GetBuildRepos(targetKey), sOpt, opts...)
+	if err != nil {
+		return nil, err
+	}
+
 	return func(in llb.State) llb.State {
 		const (
 			debPath = "/tmp/dalec/internal/build/deps"
 		)
 
 		return in.Run(
-			installWithConstraints(debPath+"/*.deb", depsSpec.Name),
+			installWithConstraints(debPath+"/*.deb", depsSpec.Name, customRepoOpts),
 			llb.AddMount(debPath, pkg, llb.Readonly),
 			dalec.WithConstraints(opts...),
 		).Root()

helpers.go

@@ -523,7 +523,7 @@ func WithRepoConfigs(repos []PackageRepositoryConfig, cfg *RepoPlatformConfig, s
 	return WithRunOptions(configStates...), nil
 }
 
-func GetRepoKeys(configs []PackageRepositoryConfig, cfg *RepoPlatformConfig, sOpt SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, []string, error) {
+func GetRepoKeys(worker llb.State, configs []PackageRepositoryConfig, cfg *RepoPlatformConfig, sOpt SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, []string, error) {
 	keys := []llb.RunOption{}
 	names := []string{}
 	for _, config := range configs {
@@ -534,8 +534,21 @@ func GetRepoKeys(configs []PackageRepositoryConfig, cfg *RepoPlatformConfig, sOp
 				return nil, nil, err
 			}
 
-			keys = append(keys,
-				llb.AddMount(filepath.Join(cfg.GPGKeyRoot, name), gpgKey, llb.SourcePath(name)))
+			mountPath := filepath.Join(cfg.GPGKeyRoot, name)
+
+			opt := runOptionFunc(func(ei *llb.ExecInfo) {
+				inPath := filepath.Join("/tmp/in", name)
+				outPath := filepath.Join("/tmp/out", name)
+				keySt := worker.Run(
+					// dearmor key if necessary
+					ShArgs(fmt.Sprintf("cat '%s' | gpg --dearmor --output '%s'", inPath, outPath)),
+					llb.AddMount(inPath, gpgKey, llb.SourcePath(name))).
+					AddMount("/tmp/out/", llb.Scratch())
+
+				llb.AddMount(mountPath, keySt, llb.SourcePath(name)).SetRunOption(ei)
+			})
+
+			keys = append(keys, opt)
 			names = append(names, name)
 		}
 	}