From ae1638f80f59d46d4881e5cca4c2753cd16e3bc2 Mon Sep 17 00:00:00 2001 From: adamperlin Date: Tue, 22 Oct 2024 16:35:06 -0700 Subject: [PATCH 1/2] Add jammy support --- frontend/azlinux/handle_container.go | 2 +- frontend/azlinux/handle_rpm.go | 17 ++++--- frontend/azlinux/handler.go | 2 +- frontend/jammy/handle_container.go | 36 +++++++++----- frontend/jammy/handle_deb.go | 62 +++++++++++++++++++++--- helpers.go | 20 ++++++-- test/azlinux_test.go | 72 +++++++++++++++------------- test/jammy_test.go | 67 +++++++++++++++++++------- 8 files changed, 196 insertions(+), 82 deletions(-) diff --git a/frontend/azlinux/handle_container.go b/frontend/azlinux/handle_container.go index 2ee8ff230..6b3ddb091 100644 --- a/frontend/azlinux/handle_container.go +++ b/frontend/azlinux/handle_container.go @@ -56,7 +56,7 @@ func specToContainerLLB(w worker, spec *dalec.Spec, targetKey string, rpmDir llb } installTimeRepos := spec.GetInstallRepos(targetKey) - importRepos, err := repoMountInstallOpts(installTimeRepos, sOpt, opts...) + importRepos, err := repoMountInstallOpts(builderImg, installTimeRepos, sOpt, opts...) if err != nil { return llb.Scratch(), err } diff --git a/frontend/azlinux/handle_rpm.go b/frontend/azlinux/handle_rpm.go index 1aa5e54d8..84a436d6b 100644 --- a/frontend/azlinux/handle_rpm.go +++ b/frontend/azlinux/handle_rpm.go @@ -102,7 +102,7 @@ var azLinuxRepoConfig = dalec.RepoPlatformConfig{ GPGKeyRoot: "/etc/pki/rpm-gpg", } -func repoMountInstallOpts(repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]installOpt, error) { +func repoMountInstallOpts(worker llb.State, repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) ([]installOpt, error) { withRepos, err := dalec.WithRepoConfigs(repos, &azLinuxRepoConfig, sOpt, opts...) if err != nil { return nil, err @@ -113,7 +113,7 @@ func repoMountInstallOpts(repos []dalec.PackageRepositoryConfig, sOpt dalec.Sour return nil, err } - keyMounts, keyPaths, err := dalec.GetRepoKeys(repos, &azLinuxRepoConfig, sOpt, opts...) + keyMounts, keyPaths, err := dalec.GetRepoKeys(worker, repos, &azLinuxRepoConfig, sOpt, opts...) if err != nil { return nil, err } @@ -129,7 +129,7 @@ func withTestDeps(w worker, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey s } testRepos := spec.GetTestRepos(targetKey) - importRepos, err := repoMountInstallOpts(testRepos, sOpt, opts...) + importRepos, err := repoMountInstallOpts(base, testRepos, sOpt, opts...) if err != nil { return nil, err } @@ -189,7 +189,7 @@ func installBuildDepsPackage(target string, packageName string, w worker, deps m } } -func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { +func installBuildDeps(ctx context.Context, w worker, sOpt dalec.SourceOpts, client gwclient.Client, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { deps := spec.GetBuildDeps(targetKey) if len(deps) == 0 { return func(in llb.State) llb.State { return in }, nil @@ -202,7 +202,12 @@ func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spe return nil, err } - importRepos, err := repoMountInstallOpts(repos, sOpt, opts...) + base, err := w.Base(sOpt, opts...) + if err != nil { + return nil, err + } + + importRepos, err := repoMountInstallOpts(base, repos, sOpt, opts...) if err != nil { return nil, err } @@ -225,7 +230,7 @@ func specToRpmLLB(ctx context.Context, w worker, client gwclient.Client, spec *d return llb.Scratch(), err } - installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...) + installOpt, err := installBuildDeps(ctx, w, sOpt, client, spec, targetKey, opts...) if err != nil { return llb.Scratch(), err } diff --git a/frontend/azlinux/handler.go b/frontend/azlinux/handler.go index dc2498f7e..fcd9c9f1f 100644 --- a/frontend/azlinux/handler.go +++ b/frontend/azlinux/handler.go @@ -83,7 +83,7 @@ func getSpecWorker(ctx context.Context, w worker, client gwclient.Client, sOpt d return llb.Scratch(), errors.New("spec contains go modules but does not have golang in build deps") } - installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...) + installOpt, err := installBuildDeps(ctx, w, sOpt, client, spec, targetKey, opts...) if err != nil { return llb.Scratch(), err } diff --git a/frontend/jammy/handle_container.go b/frontend/jammy/handle_container.go index 0ac003203..c957c4bfd 100644 --- a/frontend/jammy/handle_container.go +++ b/frontend/jammy/handle_container.go @@ -17,7 +17,7 @@ import ( const ( jammyRef = "mcr.microsoft.com/mirror/docker/library/ubuntu:jammy" - testRepoPath = "/opt/testrepo" + testRepoPath = "/opt/repo" testRepoSourceListPath = "/etc/apt/sources.list.d/test-dalec-local-repo.list" ) @@ -71,7 +71,7 @@ func buildImageConfig(ctx context.Context, resolver llb.ImageMetaResolver, spec return img, nil } -func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, deb llb.State, targetKey string, includeTestRepo bool, opts ...llb.ConstraintsOpt) llb.State { +func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, deb llb.State, targetKey string, includeTestRepo bool, opts ...llb.ConstraintsOpt) (llb.State, error) { base := dalec.GetBaseOutputImage(spec, targetKey) installSymlinks := func(in llb.State) llb.State { @@ -90,6 +90,11 @@ func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, AddMount(workPath, in) } + customRepoOpts, err := customRepoMounts(worker, spec.GetInstallRepos(targetKey), sOpt, opts...) + if err != nil { + return llb.Scratch(), err + } + if base == "" { base = jammyRef } @@ -98,8 +103,10 @@ func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, debug := llb.Scratch().File(llb.Mkfile("debug", 0o644, []byte(`debug=2`)), opts...) opts = append(opts, dalec.ProgressGroup("Install spec package")) + return baseImg.Run( - dalec.ShArgs("set -x; apt update && apt install -y /tmp/pkg/*.deb && exit 0; ls -lh /etc/apt/sources.list.d; ls -lh /etc/testrepo; mount; exit 42"), + dalec.ShArgs("set -x; apt update && apt install -y /tmp/pkg/*.deb"), + customRepoOpts, llb.AddEnv("DEBIAN_FRONTEND", "noninteractive"), llb.AddMount("/tmp/pkg", deb, llb.Readonly), dalec.WithMountedAptCache(AptCachePrefix), @@ -121,22 +128,27 @@ func buildImageRootfs(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, llb.AddMount("/etc/dpkg/dpkg.cfg.d/excludes", tmp, llb.SourcePath("tmp")).SetRunOption(cfg) }), ).Root(). - With(installSymlinks) + With(installSymlinks), nil } -func installTestDeps(spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) llb.StateOption { - return func(in llb.State) llb.State { - deps := spec.GetTestDeps(targetKey) - if len(deps) == 0 { - return in - } +func installTestDeps(worker llb.State, spec *dalec.Spec, sOpt dalec.SourceOpts, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { + deps := spec.GetTestDeps(targetKey) + if len(deps) == 0 { + return func(s llb.State) llb.State { return s }, nil + } - opts = append(opts, dalec.ProgressGroup("Install test dependencies")) + extraRepoOpts, err := customRepoMounts(worker, spec.GetTestRepos(targetKey), sOpt, opts...) + if err != nil { + return nil, err + } + return func(in llb.State) llb.State { + opts = append(opts, dalec.ProgressGroup("Install test dependencies")) return in.Run( dalec.ShArgs("apt-get update && apt-get install -y --no-install-recommends "+strings.Join(deps, " ")), llb.AddEnv("DEBIAN_FRONTEND", "noninteractive"), + extraRepoOpts, dalec.WithMountedAptCache(AptCachePrefix), ).Root() - } + }, nil } diff --git a/frontend/jammy/handle_deb.go b/frontend/jammy/handle_deb.go index aca64dfbe..c38c97ec8 100644 --- a/frontend/jammy/handle_deb.go +++ b/frontend/jammy/handle_deb.go @@ -90,7 +90,10 @@ func runTests(ctx context.Context, client gwclient.Client, spec *dalec.Spec, sOp frontend.Warn(ctx, client, worker, "Including test repo from worker image") } - st := buildImageRootfs(worker, spec, sOpt, deb, targetKey, includeTestRepo, opts...) + st, err := buildImageRootfs(worker, spec, sOpt, deb, targetKey, includeTestRepo, opts...) + if err != nil { + return nil, err + } def, err := st.Marshal(ctx, opts...) if err != nil { @@ -109,19 +112,49 @@ func runTests(ctx context.Context, client gwclient.Client, spec *dalec.Spec, sOp return nil, err } - err = frontend.RunTests(ctx, client, spec, ref, installTestDeps(spec, targetKey, opts...), targetKey) + withTestDeps, err := installTestDeps(worker, spec, sOpt, targetKey, opts...) + if err != nil { + return nil, err + } + + err = frontend.RunTests(ctx, client, spec, ref, withTestDeps, targetKey) return ref, err } -func installPackages(ls ...string) llb.RunOption { +var jammyRepoPlatformCfg = dalec.RepoPlatformConfig{ + ConfigRoot: "/etc/apt/sources.list.d", + GPGKeyRoot: "/usr/share/keyrings", +} + +func customRepoMounts(worker llb.State, repos []dalec.PackageRepositoryConfig, sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, error) { + withRepos, err := dalec.WithRepoConfigs(repos, &jammyRepoPlatformCfg, sOpt, opts...) + if err != nil { + return nil, err + } + + withData, err := dalec.WithRepoData(repos, sOpt, opts...) + if err != nil { + return nil, err + } + + keyMounts, _, err := dalec.GetRepoKeys(worker, repos, &jammyRepoPlatformCfg, sOpt, opts...) + if err != nil { + return nil, err + } + + return dalec.WithRunOptions(withRepos, withData, keyMounts), nil +} + +func installPackages(ls []string, rOpts ...llb.RunOption) llb.RunOption { return dalec.RunOptFunc(func(ei *llb.ExecInfo) { // This only runs apt-get update if the pkgcache is older than 10 minutes. dalec.ShArgs(`set -ex; apt update; apt install -y ` + strings.Join(ls, " ")).SetRunOption(ei) dalec.WithMountedAptCache(AptCachePrefix).SetRunOption(ei) + dalec.WithRunOptions(rOpts...).SetRunOption(ei) }) } -func installWithConstraints(pkgPath string, pkgName string) llb.RunOption { +func installWithConstraints(pkgPath string, pkgName string, rOpts ...llb.RunOption) llb.RunOption { return dalec.RunOptFunc(func(ei *llb.ExecInfo) { // The apt solver always tries to select the latest package version even when constraints specify that an older version should be installed and that older version is available in a repo. // This leads the solver to simply refuse to install our target package if the latest version of ANY dependency package is incompatible with the constraints. @@ -131,6 +164,7 @@ func installWithConstraints(pkgPath string, pkgName string) llb.RunOption { dalec.ShArgs(`set -ex; dpkg -i --force-depends ` + pkgPath + fmt.Sprintf(`; apt update; aptitude install -y -f -o "Aptitude::ProblemResolver::Hints::=reject %s :UNINST"`, pkgName)).SetRunOption(ei) dalec.WithMountedAptCache(AptCachePrefix).SetRunOption(ei) + dalec.WithRunOptions(rOpts...).SetRunOption(ei) }) } @@ -181,14 +215,23 @@ func workerBase(sOpt dalec.SourceOpts, opts ...llb.ConstraintsOpt) (llb.State, e return *base, nil } - return llb.Image(jammyRef, llb.WithMetaResolver(sOpt.Resolver)).With(basePackages(opts...)), nil + return llb.Image(jammyRef, llb.WithMetaResolver(sOpt.Resolver)).With(basePackages(opts...)). + // This file prevents installation of things like docs in ubuntu + // containers We don't want to exclude this because tests want to + // check things for docs in the build container. But we also don't + // want to remove this completely from the base worker image in the + // frontend because we usually don't want such things in the build + // environment. This is only needed because certain tests (which + // are using this customized builder image) are checking for files + // that are being excluded by this config file. + File(llb.Rm("/etc/dpkg/dpkg.cfg.d/excludes", llb.WithAllowNotFound(true))), nil } func basePackages(opts ...llb.ConstraintsOpt) llb.StateOption { return func(in llb.State) llb.State { opts = append(opts, dalec.ProgressGroup("Install base packages")) return in.Run( - installPackages("aptitude", "dpkg-dev", "devscripts", "equivs", "fakeroot", "dh-make", "build-essential", "dh-apparmor", "dh-make", "dh-exec", "debhelper-compat="+deb.DebHelperCompat), + installPackages([]string{"aptitude", "dpkg-dev", "devscripts", "equivs", "fakeroot", "dh-make", "build-essential", "dh-apparmor", "dh-make", "dh-exec", "debhelper-compat=" + deb.DebHelperCompat}), dalec.WithConstraints(opts...), ).Root() } @@ -231,13 +274,18 @@ func buildDepends(worker llb.State, sOpt dalec.SourceOpts, spec *dalec.Spec, tar return nil, errors.Wrap(err, "error creating intermediate package for installing build dependencies") } + customRepoOpts, err := customRepoMounts(worker, spec.GetBuildRepos(targetKey), sOpt, opts...) + if err != nil { + return nil, err + } + return func(in llb.State) llb.State { const ( debPath = "/tmp/dalec/internal/build/deps" ) return in.Run( - installWithConstraints(debPath+"/*.deb", depsSpec.Name), + installWithConstraints(debPath+"/*.deb", depsSpec.Name, customRepoOpts), llb.AddMount(debPath, pkg, llb.Readonly), dalec.WithConstraints(opts...), ).Root() diff --git a/helpers.go b/helpers.go index 272f025d0..02ba1c231 100644 --- a/helpers.go +++ b/helpers.go @@ -523,7 +523,7 @@ func WithRepoConfigs(repos []PackageRepositoryConfig, cfg *RepoPlatformConfig, s return WithRunOptions(configStates...), nil } -func GetRepoKeys(configs []PackageRepositoryConfig, cfg *RepoPlatformConfig, sOpt SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, []string, error) { +func GetRepoKeys(worker llb.State, configs []PackageRepositoryConfig, cfg *RepoPlatformConfig, sOpt SourceOpts, opts ...llb.ConstraintsOpt) (llb.RunOption, []string, error) { keys := []llb.RunOption{} names := []string{} for _, config := range configs { @@ -534,8 +534,22 @@ func GetRepoKeys(configs []PackageRepositoryConfig, cfg *RepoPlatformConfig, sOp return nil, nil, err } - keys = append(keys, - llb.AddMount(filepath.Join(cfg.GPGKeyRoot, name), gpgKey, llb.SourcePath(name))) + mountPath := filepath.Join(cfg.GPGKeyRoot, name) + + opt := runOptionFunc(func(ei *llb.ExecInfo) { + keySt := gpgKey + inPath := filepath.Join("/tmp/in", name) + outPath := filepath.Join("/tmp/out", name) + keySt = worker.Run( + // dearmor key if necessary + ShArgs(fmt.Sprintf("cat '%s' | gpg --dearmor --output '%s'", inPath, outPath)), + llb.AddMount(inPath, gpgKey, llb.SourcePath(name))). + AddMount("/tmp/out/", llb.Scratch()) + + llb.AddMount(mountPath, keySt, llb.SourcePath(name)).SetRunOption(ei) + }) + + keys = append(keys, opt) names = append(names, name) } } diff --git a/test/azlinux_test.go b/test/azlinux_test.go index ce17940cb..f095fb9ed 100644 --- a/test/azlinux_test.go +++ b/test/azlinux_test.go @@ -27,6 +27,23 @@ var azlinuxConstraints = constraintsSymbols{ LessThanOrEqual: "<=", } +var azlinuxTestRepoConfig = map[string]dalec.Source{ + "local.repo": { + Inline: &dalec.SourceInline{ + File: &dalec.SourceInlineFile{ + Contents: `[Local] +name=Local Repository +baseurl=file:///opt/repo +repo_gpgcheck=1 +priority=0 +enabled=1 +gpgkey=file:///etc/pki/rpm-gpg/public.key + `, + }, + }, + }, +} + func TestMariner2(t *testing.T) { t.Parallel() @@ -49,9 +66,11 @@ func TestMariner2(t *testing.T) { Targets: "/etc/systemd/system", }, Worker: workerConfig{ - ContextName: azlinux.Mariner2WorkerContextName, - CreateRepo: azlinuxWithRepo, - Constraints: azlinuxConstraints, + ContextName: azlinux.Mariner2WorkerContextName, + CreateRepo: azlinuxWithRepo, + SignRepo: signRepoAzLinux, + TestRepoConfig: azlinuxTestRepoConfig, + Constraints: azlinuxConstraints, }, Release: OSRelease{ ID: "mariner", @@ -79,9 +98,11 @@ func TestAzlinux3(t *testing.T) { Targets: "/etc/systemd/system", }, Worker: workerConfig{ - ContextName: azlinux.Azlinux3WorkerContextName, - CreateRepo: azlinuxWithRepo, - Constraints: azlinuxConstraints, + ContextName: azlinux.Azlinux3WorkerContextName, + CreateRepo: azlinuxWithRepo, + SignRepo: signRepoAzLinux, + TestRepoConfig: azlinuxTestRepoConfig, + Constraints: azlinuxConstraints, }, Release: OSRelease{ ID: "azurelinux", @@ -127,11 +148,12 @@ type workerConfig struct { // as well as optional state options for additional configuration. // the output [llb.StateOption] should install the repo into the worker image. CreateRepo func(llb.State, ...llb.StateOption) llb.StateOption + SignRepo func(llb.State) llb.StateOption // ContextName is the name of the worker context that the build target will use // to see if a custom worker is proivded in a context - ContextName string - - Constraints constraintsSymbols + ContextName string + TestRepoConfig map[string]dalec.Source + Constraints constraintsSymbols } type constraintsSymbols struct { @@ -1255,9 +1277,6 @@ Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/boot }) t.Run("custom repo", func(t *testing.T) { - if strings.HasPrefix(testConfig.Target.Package, "jammy") { - t.Skip("skipping custom repo test for jammy") - } t.Parallel() @@ -1379,7 +1398,7 @@ func testCustomLinuxWorker(ctx context.Context, t *testing.T, targetCfg targetCo }) } -func signRepo(gpgKey llb.State) llb.StateOption { +func signRepoAzLinux(gpgKey llb.State) llb.StateOption { // key should be a state that has a public key under /public.key return func(in llb.State) llb.State { return in.Run( @@ -1413,7 +1432,7 @@ Expire-Date: 0 %commit EOF `), pg). - Run(dalec.ShArgs("gpg --export --armor test@example.com > /tmp/gpg/PUBLIC-RPM-GPG-KEY; gpg --export-secret-keys --armor test@example.com > /tmp/gpg/private.key"), pg). + Run(dalec.ShArgs("gpg --export --armor test@example.com > /tmp/gpg/public.key; gpg --export-secret-keys --armor test@example.com > /tmp/gpg/private.key"), pg). AddMount("/tmp/gpg", llb.Scratch()) return st @@ -1466,22 +1485,7 @@ func testCustomRepo(ctx context.Context, t *testing.T, cfg testLinuxConfig) { ExtraRepos: []dalec.PackageRepositoryConfig{ { - Config: map[string]dalec.Source{ - "local.repo": { - Inline: &dalec.SourceInline{ - File: &dalec.SourceInlineFile{ - Contents: `[Local] -name=Local Repository -baseurl=file:///opt/repo -repo_gpgcheck=1 -priority=0 -enabled=1 -gpgkey=file:///etc/pki/rpm/PUBLIC-RPM-GPG-KEY - `, - }, - }, - }, - }, + Config: cfg.Worker.TestRepoConfig, Data: []dalec.SourceMount{ { Dest: "/opt/repo", @@ -1526,7 +1530,7 @@ gpgkey=file:///etc/pki/rpm/PUBLIC-RPM-GPG-KEY pkg := reqToState(ctx, client, sr, t) // create a repo using our existing worker - workerWithRepo := w.With(cfg.Worker.CreateRepo(pkg, signRepo(key))) + workerWithRepo := w.With(cfg.Worker.CreateRepo(pkg, cfg.Worker.SignRepo(key))) // copy out just the contents of the repo return llb.Scratch().File(llb.Copy(workerWithRepo, "/opt/repo", "/", &llb.CopyInfo{CopyDirContentsOnly: true})) @@ -1556,17 +1560,17 @@ gpgkey=file:///etc/pki/rpm/PUBLIC-RPM-GPG-KEY w := reqToState(ctx, gwc, sr, t) // generate a gpg key to sign the repo - // under /PUBLIC-RPM-GPG-KEY + // under /public.key gpgKey := generateGPGKey(w) repoState := getRepoState(ctx, gwc, w, gpgKey) spec := getSpec(map[string]dalec.Source{ // in the dalec spec, the public key will be passed in via build context - "PUBLIC-RPM-GPG-KEY": { + "public.key": { Context: &dalec.SourceContext{ Name: "repo-public-key", }, - Path: "PUBLIC-RPM-GPG-KEY", + Path: "public.key", }, }) diff --git a/test/jammy_test.go b/test/jammy_test.go index 6431b95bc..8bde7893d 100644 --- a/test/jammy_test.go +++ b/test/jammy_test.go @@ -8,6 +8,37 @@ import ( "github.com/moby/buildkit/client/llb" ) +func signRepoJammy(gpgKey llb.State) llb.StateOption { + // key should be a state that has a public key under /public.key + return func(in llb.State) llb.State { + // assuming in is the state that has the repo files under / including + // Release file + return in.Run( + dalec.ShArgs("gpg --import < /tmp/gpg/public.key"), + dalec.ShArgs("gpg --import < /tmp/gpg/private.key"), + llb.AddMount("/tmp/gpg", gpgKey, llb.Readonly), + dalec.ProgressGroup("Importing gpg key")). + Run( + dalec.ShArgs(`ID=$(gpg --list-keys --keyid-format LONG | grep -B 2 'test@example.com' | grep 'pub' | awk '{print $2}' | cut -d'/' -f2) && \ + gpg --list-keys --keyid-format LONG && \ + gpg --default-key $ID -abs -o /opt/repo/Release.gpg /opt/repo/Release && \ + gpg --default-key "$ID" --clearsign -o /opt/repo/InRelease /opt/repo/Release`), + llb.AddMount("/tmp/gpg", gpgKey, llb.Readonly), + dalec.ProgressGroup("signing repo"), + ).Root() + } +} + +var jammyTestRepoConfig = map[string]dalec.Source{ + "local.list": { + Inline: &dalec.SourceInline{ + File: &dalec.SourceInlineFile{ + Contents: `deb [signed-by=/usr/share/keyrings/public.key] copy:/opt/repo/ /`, + }, + }, + }, +} + func TestJammy(t *testing.T) { t.Parallel() @@ -31,35 +62,35 @@ func TestJammy(t *testing.T) { }, Worker: workerConfig{ ContextName: jammy.JammyWorkerContextName, + // /pkg1.deb ... CreateRepo: func(pkg llb.State, opts ...llb.StateOption) llb.StateOption { repoFile := []byte(` -deb [trusted=yes] copy:/opt/testrepo/ / +deb [trusted=yes] copy:/opt/repo/ / `) return func(in llb.State) llb.State { - repo := in.Run( - dalec.ShArgs("apt-get update && apt-get install -y apt-utils"), + withRepo := in.Run( + dalec.ShArgs("apt-get update && apt-get install -y apt-utils gnupg"), dalec.WithMountedAptCache(jammy.AptCachePrefix), - ). + ).File(llb.Copy(pkg, "/", "/opt/repo")). Run( llb.Dir("/opt/repo"), - dalec.ShArgs("apt-ftparchive packages . | gzip -1 > Packages.gz"), + dalec.ShArgs("apt-ftparchive packages . > Packages"), ). - AddMount("/opt/repo", pkg) + Run( + llb.Dir("/opt/repo"), + dalec.ShArgs("apt-ftparchive release . > Release"), + ).Root() + + for _, opt := range opts { + withRepo = opt(withRepo) + } - return in. - File(llb.Copy(repo, "/", "/opt/testrepo")). - File(llb.Mkfile("/etc/apt/sources.list.d/test-dalec-local-repo.list", 0o644, repoFile)). - // This file prevents installation of things like docs in ubuntu - // containers We don't want to exclude this because tests want to - // check things for docs in the build container. But we also don't - // want to remove this completely from the base worker image in the - // frontend because we usually don't want such things in the build - // environment. This is only needed because certain tests (which - // are using this customized builder image) are checking for files - // that are being excluded by this config file. - File(llb.Rm("/etc/dpkg/dpkg.cfg.d/excludes", llb.WithAllowNotFound(true))) + return withRepo. + File(llb.Mkfile("/etc/apt/sources.list.d/test-dalec-local-repo.list", 0o644, repoFile)) } }, + SignRepo: signRepoJammy, + TestRepoConfig: jammyTestRepoConfig, Constraints: constraintsSymbols{ Equal: "=", GreaterThan: ">>", From c4d58d0b7b95cd8cb0c1ca3bdb6da227a201c8a6 Mon Sep 17 00:00:00 2001 From: adamperlin Date: Tue, 22 Oct 2024 16:56:54 -0700 Subject: [PATCH 2/2] Fix linter errors --- frontend/azlinux/handle_rpm.go | 4 ++-- frontend/azlinux/handler.go | 2 +- helpers.go | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/frontend/azlinux/handle_rpm.go b/frontend/azlinux/handle_rpm.go index 84a436d6b..7d6da1d4d 100644 --- a/frontend/azlinux/handle_rpm.go +++ b/frontend/azlinux/handle_rpm.go @@ -189,7 +189,7 @@ func installBuildDepsPackage(target string, packageName string, w worker, deps m } } -func installBuildDeps(ctx context.Context, w worker, sOpt dalec.SourceOpts, client gwclient.Client, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { +func installBuildDeps(ctx context.Context, w worker, client gwclient.Client, spec *dalec.Spec, targetKey string, opts ...llb.ConstraintsOpt) (llb.StateOption, error) { deps := spec.GetBuildDeps(targetKey) if len(deps) == 0 { return func(in llb.State) llb.State { return in }, nil @@ -230,7 +230,7 @@ func specToRpmLLB(ctx context.Context, w worker, client gwclient.Client, spec *d return llb.Scratch(), err } - installOpt, err := installBuildDeps(ctx, w, sOpt, client, spec, targetKey, opts...) + installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...) if err != nil { return llb.Scratch(), err } diff --git a/frontend/azlinux/handler.go b/frontend/azlinux/handler.go index fcd9c9f1f..dc2498f7e 100644 --- a/frontend/azlinux/handler.go +++ b/frontend/azlinux/handler.go @@ -83,7 +83,7 @@ func getSpecWorker(ctx context.Context, w worker, client gwclient.Client, sOpt d return llb.Scratch(), errors.New("spec contains go modules but does not have golang in build deps") } - installOpt, err := installBuildDeps(ctx, w, sOpt, client, spec, targetKey, opts...) + installOpt, err := installBuildDeps(ctx, w, client, spec, targetKey, opts...) if err != nil { return llb.Scratch(), err } diff --git a/helpers.go b/helpers.go index 02ba1c231..d0c27386b 100644 --- a/helpers.go +++ b/helpers.go @@ -537,10 +537,9 @@ func GetRepoKeys(worker llb.State, configs []PackageRepositoryConfig, cfg *RepoP mountPath := filepath.Join(cfg.GPGKeyRoot, name) opt := runOptionFunc(func(ei *llb.ExecInfo) { - keySt := gpgKey inPath := filepath.Join("/tmp/in", name) outPath := filepath.Join("/tmp/out", name) - keySt = worker.Run( + keySt := worker.Run( // dearmor key if necessary ShArgs(fmt.Sprintf("cat '%s' | gpg --dearmor --output '%s'", inPath, outPath)), llb.AddMount(inPath, gpgKey, llb.SourcePath(name))).