Consider already selected packages during solve

.github/workflows/build-samples.yml

@@ -19,7 +19,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        arch: [x86_64, "386", armv7, aarch64, riscv64, s390x, ppc64le]
+        arch: [x86_64, aarch64]
 
     steps:
       - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
@@ -50,37 +50,6 @@ jobs:
             docker run --rm -v $(pwd)/$f:/sbom.json cgr.dev/chainguard/ntia-conformance-checker -v --file /sbom.json
           done
 
-  # Build a multi-arch nginx image for all archs.
-  build-nginx-multiarch:
-    name: build-nginx-multiarch
-    runs-on: ubuntu-latest
-
-    permissions:
-      contents: read
-
-    steps:
-      - uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
-        with:
-          egress-policy: audit
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v2.1.5
-        with:
-          go-version-file: 'go.mod'
-          check-latest: true
-      - run: |
-          make apko
-          ./apko build ./examples/nginx.yaml nginx:build /tmp/nginx.tar --arch x86_64,386,armv7,aarch64,s390x,ppc64le
-
-      - name: Check SBOM Conformance
-        run: |
-          set -euxo pipefail
-          for f in *.spdx.json; do
-            echo ::group::sbom.json
-            cat $f
-            echo ::endgroup::
-            docker run --rm -v $(pwd)/$f:/sbom.json cgr.dev/chainguard/ntia-conformance-checker -v --file /sbom.json
-          done
-
   build-all-examples-one-arch:
     name: build-all-examples-amd64
 
@@ -116,7 +85,7 @@ jobs:
             fi
           done
 
-  build-alpine-source-date-epoch:
+  build-wolfi-source-date-epoch:
     name: source-date-epoch
     runs-on: ubuntu-latest
 
@@ -141,11 +110,11 @@ jobs:
           SOURCE_DATE_EPOCH: "0"
         run: |
           make apko
-          FIRST=$(./apko publish ./examples/alpine-base.yaml localhost:5000/alpine --arch x86_64,386,armv7,aarch64,s390x,ppc64le 2> /dev/null)
+          FIRST=$(./apko publish ./examples/wolfi-base.yaml localhost:5000/wolfi --arch x86_64,aarch64 2> /dev/null)
 
           for idx in {2..10}
           do
-            NEXT=$(./apko publish ./examples/alpine-base.yaml localhost:5000/alpine --arch x86_64,386,armv7,aarch64,s390x,ppc64le 2> /dev/null)
+            NEXT=$(./apko publish ./examples/wolfi-base.yaml localhost:5000/wolfi --arch x86_64,aarch64 2> /dev/null)
 
             if [ "${FIRST}" = "${NEXT}" ]; then
               echo "Build ${idx} matches."
@@ -155,7 +124,7 @@ jobs:
             fi
           done
 
-  build-alpine-build-date-epoch:
+  build-wolfi-build-date-epoch:
     name: build-date-epoch
     runs-on: ubuntu-latest
 
@@ -180,11 +149,11 @@ jobs:
           make apko
           # Without SOURCE_DATE_EPOCH set, the timestamp of the image will be computed to be
           # the maximum build date of the resolved APKs.
-          FIRST=$(./apko publish ./examples/alpine-base.yaml localhost:5000/alpine --arch x86_64,386,armv7,aarch64,s390x,ppc64le 2> /dev/null)
+          FIRST=$(./apko publish ./examples/wolfi-base.yaml localhost:5000/wolfi --arch x86_64,aarch64 2> /dev/null)
 
           for idx in {2..10}
           do
-            NEXT=$(./apko publish ./examples/alpine-base.yaml localhost:5000/alpine --arch x86_64,386,armv7,aarch64,s390x,ppc64le 2> /dev/null)
+            NEXT=$(./apko publish ./examples/wolfi-base.yaml localhost:5000/wolfi --arch x86_64,aarch64 2> /dev/null)
 
             if [ "${FIRST}" = "${NEXT}" ]; then
               echo "Build ${idx} matches."
@@ -218,7 +187,7 @@ jobs:
           make apko
 
           # Build image with annotations.
-          ref=$(./apko publish ./examples/nginx.yaml localhost:5000/nginx --arch x86_64,386,armv7,aarch64,s390x,ppc64le)
+          ref=$(./apko publish ./examples/nginx.yaml localhost:5000/nginx --arch x86_64,aarch64)
 
           # Check index annotations.
           crane manifest $ref | jq -r '.annotations.foo' | grep bar

examples/abseil-regression.yaml

@@ -0,0 +1,13 @@
+# This was very slow once.
+contents:
+  keyring:
+    - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+  repositories:
+    - https://packages.wolfi.dev/os
+
+  packages:
+    - abseil-cpp-dev
+    - pkgconf
+
+archs:
+- arm64

examples/alias-only.yaml

@@ -2,4 +2,7 @@ contents:
   repositories:
     - https://dl-cdn.alpinelinux.org/alpine/edge/main
   packages:
-    - openssh-client
+    - openssh-client
+
+archs:
+- x86_64

examples/alpine-base-rootless.yaml

@@ -18,3 +18,6 @@ cmd: /bin/sh -l
 # optional environment configuration
 environment:
   PATH: /usr/sbin:/sbin:/usr/bin:/bin
+
+archs:
+  - amd64

examples/alpine-base.yaml

@@ -9,3 +9,6 @@ cmd: /bin/sh -l
 # optional environment configuration
 environment:
   PATH: /usr/sbin:/sbin:/usr/bin:/bin
+
+archs:
+  - amd64

examples/alpine-python3.yaml

@@ -10,3 +10,6 @@ cmd: /bin/sh -l
 # optional environment configuration
 environment:
   PATH: /usr/sbin:/sbin:/usr/bin:/bin
+
+archs:
+  - amd64

examples/alpine-slim.yaml

@@ -11,3 +11,6 @@ environment:
   PATH: /usr/sbin:/sbin:/usr/bin:/bin
 
 cmd: /bin/sh -l
+
+archs:
+  - amd64

examples/apko-devenv.yaml

@@ -32,3 +32,6 @@ contents:
     - docker-cli
 entrypoint:
   command: /bin/sh -l
+
+archs:
+  - amd64

examples/nginx-rootless.yaml

@@ -36,3 +36,6 @@ paths:
     uid: 10000
     gid: 10000
     permissions: 0o644
+
+archs:
+  - amd64

examples/nginx.yaml

@@ -1,8 +1,10 @@
 contents:
+  keyring:
+    - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
   repositories:
-    - https://dl-cdn.alpinelinux.org/alpine/edge/main
+    - https://packages.wolfi.dev/os
   packages:
-    - alpine-baselayout
+    - wolfi-baselayout
     - nginx
 
 entrypoint:
@@ -32,7 +34,7 @@ paths:
     permissions: 0o755
   - path: /etc/nginx/http.d/default.conf
     type: hardlink
-    source: /usr/share/nginx/http-default_server.conf
+    source: /etc/nginx/nginx.conf.default
     uid: 10000
     gid: 10000
     permissions: 0o644
@@ -41,3 +43,6 @@ work-dir: /usr/share/nginx
 
 annotations:
   foo: bar
+
+archs:
+  - amd64

hack/ci/00-build.sh

@@ -10,7 +10,7 @@ REF="apko.local/ci-testing:test"
 
 trap "rm -f ${OUTPUT_TAR}" EXIT
 
-for f in examples/alpine-base-rootless.yaml examples/wolfi-base.yaml; do
+for f in examples/wolfi-base.yaml; do
   echo "=== building $f"
 
   REF="apko.local/ci-testing:$(basename ${f})"

hack/ci/01-publish.sh

@@ -16,7 +16,7 @@ docker rm -f "${REGISTRY_CONTAINER_NAME}"
 docker run --name "${REGISTRY_CONTAINER_NAME}" \
   -d -p ${PORT}:5000 "${REGISTRY_BASE_IMAGE}"
 
-for f in examples/alpine-base-rootless.yaml examples/wolfi-base.yaml; do
+for f in examples/wolfi-base.yaml; do
   echo "=== building $f"
 
   REF="localhost:${PORT}/ci-testing:$(basename ${f})"

pkg/apk/apk/repo.go

@@ -203,6 +203,9 @@ type PkgResolver struct {
 	indexes      []NamedIndex
 	nameMap      map[string][]*repositoryPackage
 	installIfMap map[string][]*repositoryPackage // contains any package that should be installed if the named package is installed
+
+	// Short-circuit providers we have already selected.
+	selected map[string]*RepositoryPackage
 }
 
 // Clone returns a copy of PkgResolver.
@@ -211,6 +214,7 @@ func (p *PkgResolver) Clone() *PkgResolver {
 		indexes:      p.indexes,
 		nameMap:      maps.Clone(p.nameMap),
 		installIfMap: maps.Clone(p.installIfMap),
+		selected:     map[string]*RepositoryPackage{},
 	}
 }
 
@@ -234,7 +238,8 @@ func newPkgResolver(ctx context.Context, indexes []NamedIndex) *PkgResolver {
 		installIfMap = map[string][]*repositoryPackage{}
 	)
 	p := &PkgResolver{
-		indexes: indexes,
+		indexes:  indexes,
+		selected: map[string]*RepositoryPackage{},
 	}
 
 	// create a map of every package by name and version to its RepositoryPackage
@@ -383,6 +388,29 @@ func (p *PkgResolver) disqualifyConflicts(pkg *RepositoryPackage, dq map[*Reposi
 	}
 }
 
+func (p *PkgResolver) pick(pkg *RepositoryPackage) error {
+	if conflict, ok := p.selected[pkg.Name]; ok {
+		// Trying to re-select the same thing is fine actually.
+		if conflict == pkg {
+			return nil
+		}
+
+		return fmt.Errorf("selecting package %s conflicts with %s on %q", pkg.Filename(), conflict.Filename(), pkg.Name)
+	}
+
+	p.selected[pkg.Name] = pkg
+
+	for _, prov := range pkg.Provides {
+		constraint := cachedResolvePackageNameVersionPin(prov)
+		if conflict, ok := p.selected[constraint.name]; ok {
+			return fmt.Errorf("selecting package %s conflicts with %s on %q", pkg.Filename(), conflict.Filename(), constraint.name)
+		}
+		p.selected[constraint.name] = pkg
+	}
+
+	return nil
+}
+
 func (p *PkgResolver) disqualify(dq map[*RepositoryPackage]string, pkg *RepositoryPackage, reason string) {
 	dq[pkg] = reason
 
@@ -452,7 +480,7 @@ func (p *PkgResolver) constrain(constraints []string, dq map[*RepositoryPackage]
 // GetPackagesWithDependencies get all of the dependencies for the given packages based on the
 // indexes. Does not filter for installed already or not.
 func (p *PkgResolver) GetPackagesWithDependencies(ctx context.Context, packages []string, allArchs map[string][]NamedIndex) (toInstall []*RepositoryPackage, conflicts []string, err error) {
-	_, span := otel.Tracer("go-apk").Start(ctx, "GetPackageWithDependencies")
+	_, span := otel.Tracer("go-apk").Start(ctx, "GetPackagesWithDependencies")
 	defer span.End()
 
 	// Tracks all the packages we have disqualified and the reason we disqualified them.
@@ -494,10 +522,11 @@ func (p *PkgResolver) GetPackagesWithDependencies(ctx context.Context, packages
 
 	// now get the dependencies for each package
 	for _, pkgName := range packages {
-		pkg, deps, confs, err := p.GetPackageWithDependencies(pkgName, dependenciesMap, dq)
+		pkg, deps, confs, err := p.GetPackageWithDependencies(ctx, pkgName, dependenciesMap, dq)
 		if err != nil {
 			return toInstall, nil, &ConstraintError{pkgName, err}
 		}
+
 		for _, dep := range deps {
 			if _, ok := installTracked[dep.Name]; !ok {
 				toInstall = append(toInstall, dep)
@@ -527,7 +556,7 @@ func (p *PkgResolver) GetPackagesWithDependencies(ctx context.Context, packages
 // Requires the existing set because the logic for resolving dependencies between competing
 // options may depend on whether or not one already is installed.
 // Must not modify the existing map directly.
-func (p *PkgResolver) GetPackageWithDependencies(pkgName string, existing map[string]*RepositoryPackage, dq map[*RepositoryPackage]string) (*RepositoryPackage, []*RepositoryPackage, []string, error) {
+func (p *PkgResolver) GetPackageWithDependencies(ctx context.Context, pkgName string, existing map[string]*RepositoryPackage, dq map[*RepositoryPackage]string) (*RepositoryPackage, []*RepositoryPackage, []string, error) {
 	parents := make(map[string]bool)
 	localExisting := make(map[string]*RepositoryPackage, len(existing))
 	existingOrigins := map[string]bool{}
@@ -544,10 +573,11 @@ func (p *PkgResolver) GetPackageWithDependencies(pkgName string, existing map[st
 	}
 
 	pin := cachedResolvePackageNameVersionPin(pkgName).pin
-	deps, conflicts, err := p.getPackageDependencies(pkg, pin, true, parents, localExisting, existingOrigins, dq)
+	deps, conflicts, err := p.getPackageDependencies(ctx, pkg, pin, parents, localExisting, existingOrigins, dq)
 	if err != nil {
 		return nil, nil, nil, err
 	}
+
 	// eliminate duplication in dependencies
 	added := make(map[string]*RepositoryPackage, len(deps))
 	dependencies := make([]*RepositoryPackage, 0, len(deps))
@@ -672,7 +702,10 @@ func (p *PkgResolver) resolvePackage(pkgName string, dq map[*RepositoryPackage]s
 // It might change the order of install.
 // In other words, this _should_ be a DAG (acyclical), but because the packages
 // are just listing dependencies in text, it might be cyclical. We need to be careful of that.
-func (p *PkgResolver) getPackageDependencies(pkg *RepositoryPackage, allowPin string, allowSelfFulfill bool, parents map[string]bool, existing map[string]*RepositoryPackage, existingOrigins map[string]bool, dq map[*RepositoryPackage]string) (dependencies []*RepositoryPackage, conflicts []string, err error) {
+func (p *PkgResolver) getPackageDependencies(ctx context.Context, pkg *RepositoryPackage, allowPin string, parents map[string]bool, existing map[string]*RepositoryPackage, existingOrigins map[string]bool, dq map[*RepositoryPackage]string) (dependencies []*RepositoryPackage, conflicts []string, err error) {
+	if err := ctx.Err(); err != nil {
+		return nil, nil, context.Cause(ctx)
+	}
 	// check if the package we are checking is one of our parents, avoid cyclical graphs
 	if _, ok := parents[pkg.Name]; ok {
 		return nil, nil, nil
@@ -715,7 +748,7 @@ func (p *PkgResolver) getPackageDependencies(pkg *RepositoryPackage, allowPin st
 				continue
 			}
 
-			if allowSelfFulfill && pkg.Name == name {
+			if pkg.Name == name {
 				var (
 					actualVersion, requiredVersion Version
 					err1, err2                     error
@@ -733,6 +766,30 @@ func (p *PkgResolver) getPackageDependencies(pkg *RepositoryPackage, allowPin st
 				}
 			}
 
+			if picked, ok := p.selected[name]; ok {
+				if version == "" {
+					// If we don't care which version, and we've already selected something, fantastic.
+					continue
+				}
+
+				actualVersion, err := cachedParseVersion(picked.Version)
+				if err != nil {
+					return nil, nil, err
+				}
+				requiredVersion, err := cachedParseVersion(version)
+				if err != nil {
+					return nil, nil, err
+				}
+
+				// We do care which version and they match.
+				if compare.satisfies(actualVersion, requiredVersion) {
+					continue
+				}
+
+				// We already selected something to satisfy "name" and it does not match the "version" we need now.
+				return nil, nil, fmt.Errorf("we already selected %q=%q which conflicts with %q=%q", picked.Name, picked.Version, name, version)
+			}
+
 			// first see if it is a name of a package
 			depPkgWithVersions, ok := p.nameMap[name]
 			if !ok {
@@ -793,7 +850,12 @@ func (p *PkgResolver) getPackageDependencies(pkg *RepositoryPackage, allowPin st
 			childParents[k] = true
 		}
 		childParents[pkg.Name] = true
-		subDeps, confs, err := p.getPackageDependencies(depPkg, allowPin, true, childParents, existing, existingOrigins, dq)
+
+		if err := p.pick(pkg); err != nil {
+			return nil, nil, err
+		}
+
+		subDeps, confs, err := p.getPackageDependencies(ctx, depPkg, allowPin, childParents, existing, existingOrigins, dq)
 		if err != nil {
 			return nil, nil, &DepError{pkg, err}
 		}