diff --git a/go.mod b/go.mod
index 647e5b8e9..cbf2b0d4a 100644
--- a/go.mod
+++ b/go.mod
@@ -2,6 +2,8 @@ module chainguard.dev/apko
 
 go 1.20
 
+replace github.com/chainguard-dev/go-apk => ../go-apk
+
 require (
 	github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220920003936-cd2dbcbbab49
 	github.com/chainguard-dev/go-apk v0.0.0-20230830232041-24f27d33fced
diff --git a/go.sum b/go.sum
index 14410cd30..1df28e42c 100644
--- a/go.sum
+++ b/go.sum
@@ -98,8 +98,6 @@ github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220920003936-
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/chainguard-dev/go-apk v0.0.0-20230830232041-24f27d33fced h1:3FPKsvRFzRwDJ58Gbxt7Cjiz8eec4rtlf4uEsBqX/LA=
-github.com/chainguard-dev/go-apk v0.0.0-20230830232041-24f27d33fced/go.mod h1:I7uFl3LBMG1UQONJRQN+3lzZE4tw8myh87FzkDEztHg=
 github.com/chrismellard/docker-credential-acr-env v0.0.0-20220327082430-c57b701bfc08 h1:9Qh4lJ/KMr5iS1zfZ8I97+3MDpiKjl+0lZVUNBhdvRs=
 github.com/chrismellard/docker-credential-acr-env v0.0.0-20220327082430-c57b701bfc08/go.mod h1:MAuu1uDJNOS3T3ui0qmKdPUwm59+bO19BbTph2wZafE=
 github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
diff --git a/internal/cli/build.go b/internal/cli/build.go
index 595eb546d..899285950 100644
--- a/internal/cli/build.go
+++ b/internal/cli/build.go
@@ -24,7 +24,6 @@ import (
 	"sync"
 
 	"github.com/chainguard-dev/go-apk/pkg/apk"
-	apkfs "github.com/chainguard-dev/go-apk/pkg/fs"
 	coci "github.com/sigstore/cosign/v2/pkg/oci"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -33,6 +32,7 @@ import (
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sys/unix"
 
+	"chainguard.dev/apko/internal/tarfs"
 	"chainguard.dev/apko/pkg/build"
 	"chainguard.dev/apko/pkg/build/oci"
 	"chainguard.dev/apko/pkg/build/types"
@@ -178,7 +178,7 @@ func BuildCmd(ctx context.Context, imageRef, outputTarGZ string, archs []types.A
 
 // buildImage build all of the components of an image in a single working directory.
 // Each layer is a separate file, as are config, manifests, index and sbom.
-func buildImageComponents(ctx context.Context, wd string, archs []types.Architecture, opts ...build.Option) (idx coci.SignedImageIndex, sboms []types.SBOM, pkgs map[types.Architecture][]*apk.InstalledPackage, err error) {
+func buildImageComponents(ctx context.Context, workDir string, archs []types.Architecture, opts ...build.Option) (idx coci.SignedImageIndex, sboms []types.SBOM, pkgs map[types.Architecture][]*apk.InstalledPackage, err error) {
 	ctx, span := otel.Tracer("apko").Start(ctx, "buildImageComponents")
 	defer span.End()
 
@@ -215,7 +215,6 @@ func buildImageComponents(ctx context.Context, wd string, archs []types.Architec
 	o.Logger().Printf("building tags %v", o.Tags)
 
 	var errg errgroup.Group
-	workDir := wd
 	imageDir := filepath.Join(workDir, "image")
 	if err := os.MkdirAll(imageDir, 0755); err != nil {
 		return nil, nil, nil, fmt.Errorf("unable to create working image directory %s: %w", imageDir, err)
@@ -238,23 +237,17 @@ func buildImageComponents(ctx context.Context, wd string, archs []types.Architec
 
 	for _, arch := range archs {
 		arch := arch
-		// working directory for this architecture
-		wd := filepath.Join(workDir, arch.ToAPK())
 		bopts := slices.Clone(opts)
 		bopts = append(bopts,
 			build.WithArch(arch),
 			build.WithSBOM(imageDir),
 		)
 
-		fs := apkfs.DirFS(wd, apkfs.WithCreateDir())
-
-		bc, err := build.New(ctx, fs, bopts...)
+		bc, err := build.New(ctx, tarfs.New(), bopts...)
 		if err != nil {
 			return nil, nil, nil, err
 		}
 
-		bc.Logger().Infof("using working directory %s", wd)
-
 		// save the build context for later
 		contexts[arch] = bc
 
diff --git a/internal/tarfs/fs.go b/internal/tarfs/fs.go
index c69fc15cc..d784917a5 100644
--- a/internal/tarfs/fs.go
+++ b/internal/tarfs/fs.go
@@ -17,6 +17,7 @@ package tarfs
 import (
 	"archive/tar"
 	"bytes"
+	"crypto/sha1" //nolint:gosec // this is what apk tools is using
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
@@ -465,8 +466,22 @@ func (m *memFS) writeHeader(name string, te tarEntry) error {
 	want, got := te, existing.te
 
 	if got == nil {
-		// This shouldn't happen.
-		return fmt.Errorf("conflicting file for %q has no tar entry", name)
+		if existing.data == nil {
+			return fmt.Errorf("conflicting file for %q has no tar entry", name)
+		}
+
+		// This can happen when go-apk's InitKeyring conflicts with alpine-keys.
+		// Since those files will be in memory, quickly compute the checksum and
+		// ignore this file if they match.
+		h := sha1.New() //nolint:gosec // this is what apk tools is using
+		h.Write(existing.data)
+		checksum := h.Sum(nil)
+
+		if bytes.Equal(want.checksum, checksum) {
+			return nil
+		}
+
+		return fmt.Errorf("conflicting file for %q with checksum %x, existing has checksum %x", name, want.checksum, checksum)
 	}
 
 	// Files have the same checksum, that's fine.