first-level yum dependencies are now fetched

fixed rpm unpacking issues with symbolic/hard links

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/ulikunitz/xz v0.5.11
 	gitlab.alpinelinux.org/alpine/go v0.8.0
 	go.uber.org/zap v1.26.0
+	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
 	k8s.io/apimachinery v0.28.2
 	pault.ag/go/debian v0.15.0
 )

go.sum

@@ -642,6 +642,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ=
+golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

pkg/packages/rpm/rpm.go

@@ -12,8 +12,10 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/sassoftware/go-rpmutils/cpio"
 	"github.com/ulikunitz/xz"
+	"golang.org/x/exp/maps"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 )
@@ -67,64 +69,170 @@ func (p *PackageKeeper) Unpack(ctx context.Context, pkgFile string, rootfs fs.Fu
 		return fmt.Errorf("unsupported payload format: %s", format)
 	}
 
-	cpioReader := cpio.NewReader(xzReader)
+	return p.Extract(ctx, rootfs, xzReader)
+}
+
+// Extract the contents of a cpio stream from r to the destination directory dest
+func (p *PackageKeeper) Extract(ctx context.Context, rootfs fs.FullFS, rs io.Reader) error {
+	log := logr.FromContextOrDiscard(ctx)
+
+	linkMap := make(map[int][]string)
+
+	stream := cpio.NewReader(rs)
+
 	for {
-		hdr, err := cpioReader.Next()
+		entry, err := stream.Next()
 		if err == io.EOF {
 			break
 		}
 		if err != nil {
-			return fmt.Errorf("reading cpio: %w", err)
+			return fmt.Errorf("reading stream: %w", err)
+		}
+
+		// sanitize path
+		target := path.Clean(entry.Filename())
+		for strings.HasPrefix(target, "../") {
+			target = target[3:]
 		}
-		fileName := strings.TrimPrefix(hdr.Filename(), ".")
-		fileMode := os.FileMode(hdr.Mode()).Perm()
-		switch hdr.Mode() &^ 07777 {
+		target = filepath.Join("/", filepath.FromSlash(target))
+		if !strings.HasPrefix(target, string(filepath.Separator)) && "/" != target {
+			// this shouldn't happen due to the sanitization above but always check
+			return fmt.Errorf("invalid cpio path %q", entry.Filename())
+		}
+		// create the parent directory if it doesn't exist.
+		if dir := filepath.Dir(entry.Filename()); dir != "" {
+			log.V(2).Info("creating directory", "path", dir)
+			if err := rootfs.MkdirAll(dir, 0o755); err != nil {
+				return fmt.Errorf("creating directory: %w", err)
+			}
+		}
+		// FIXME: Need a makedev implementation in go.
+
+		switch entry.Mode() &^ 07777 {
+		case cpio.S_ISCHR:
+			// FIXME: skipping due to lack of makedev.
+			continue
+		case cpio.S_ISBLK:
+			// FIXME: skipping due to lack of makedev.
+			continue
+		case cpio.S_ISDIR:
+			log.V(2).Info("creating directory", "path", target)
+			m := os.FileMode(entry.Mode()).Perm()
+			if err := rootfs.Mkdir(target, m); err != nil && !os.IsExist(err) {
+				return fmt.Errorf("creating dir: %w", err)
+			}
+		case cpio.S_ISFIFO:
+			// skip
+			continue
+		case cpio.S_ISLNK:
+			buf := make([]byte, entry.Filesize())
+			if _, err := stream.Read(buf); err != nil {
+				return fmt.Errorf("reading symlink name: %w", err)
+			}
+			filename := string(buf)
+			log.V(2).Info("creating symlink", "path", target)
+			if err := rootfs.Symlink(filename, target); err != nil {
+				if os.IsExist(err) {
+					log.V(2).Info("skipping symlink since the target already exists", "path", target)
+					continue
+				}
+				return fmt.Errorf("creating symlink: %w", err)
+			}
 		case cpio.S_ISREG:
-			// create the target directory
-			if dir := filepath.Dir(fileName); dir != "" {
-				log.V(6).Info("creating directory", "dir", dir)
-				if err := rootfs.MkdirAll(dir, 0o755); err != nil {
-					return fmt.Errorf("creating directory: %w", err)
+			log.V(2).Info("creating file", "path", target)
+			// save hardlinks until after the target is written
+			if entry.Nlink() > 1 && entry.Filesize() == 0 {
+				l, ok := linkMap[entry.Ino()]
+				if !ok {
+					l = make([]string, 0)
 				}
+				l = append(l, target)
+				linkMap[entry.Ino()] = l
+				continue
 			}
-			log.V(5).Info("creating file", "file", fileName)
-			out, err := rootfs.Create(fileName)
+
+			f, err := rootfs.Create(target)
 			if err != nil {
-				return fmt.Errorf("creating file: %w", err)
+				return fmt.Errorf("creating file '%s': %w", target, err)
 			}
-			if _, err := io.Copy(out, cpioReader); err != nil {
-				_ = out.Close()
+			written, err := io.Copy(f, stream)
+			if err != nil {
 				return fmt.Errorf("copying file: %w", err)
 			}
-			_ = out.Close()
-			log.V(5).Info("updating file permissions", "file", fileName, "permissions", fileMode)
-			if err := rootfs.Chmod(fileName, fileMode); err != nil {
-				return fmt.Errorf("chmodding file %s: %w", fileName, err)
+			if written != int64(entry.Filesize()) {
+				return fmt.Errorf("short write")
+			}
+			if err := f.Close(); err != nil {
+				return err
+			}
+
+			// fix permissions
+			fileMode := os.FileMode(entry.Mode()).Perm()
+			log.V(5).Info("updating file permissions", "file", target, "permissions", fileMode)
+			if err := rootfs.Chmod(target, fileMode); err != nil {
+				return fmt.Errorf("chmodding file %s: %w", target, err)
+			}
+
+			// Create hardlinks after the file content is written.
+			if entry.Nlink() > 1 && entry.Filesize() > 0 {
+				l, ok := linkMap[entry.Ino()]
+				if !ok {
+					return fmt.Errorf("hardlinks missing")
+				}
+
+				for _, t := range l {
+					log.V(2).Info("creating hardlink", "target", target, "path", t)
+					if err := rootfs.Link(target, t); err != nil {
+						if os.IsExist(err) {
+							log.V(2).Info("skipping hardlink since the target already exists", "target", target, "path", t)
+							continue
+						}
+						return fmt.Errorf("creating hardlink: %w", err)
+					}
+				}
 			}
 		default:
-			log.V(4).Info("unknown header mode", "path", fileName, "mode", hdr.Mode())
+			return fmt.Errorf("unknown file mode 0%o for %s", entry.Mode(), entry.Filename())
 		}
 	}
 
 	return nil
 }
 
-func (p *PackageKeeper) Resolve(_ context.Context, pkg string) ([]lockfile.Package, error) {
+func (p *PackageKeeper) Resolve(ctx context.Context, pkg string) ([]lockfile.Package, error) {
+	log := logr.FromContextOrDiscard(ctx).WithValues("pkg", pkg)
+	// dedupe packages
+	packages := map[string]lockfile.Package{}
 	for _, idx := range p.indices {
-		for _, p := range idx.PackagesList {
+		for _, p := range idx.Package {
 			if p.Name == pkg {
-				return []lockfile.Package{
-					{
-						Name:      p.Name,
+				dependencies := idx.GetProviders(ctx, p.Format.Requires.Entry.GetValues())
+				for _, dep := range dependencies {
+					packages[fmt.Sprintf("%s-%s", dep.Name, dep.Version.Ver)] = lockfile.Package{
+						Name:      dep.Name,
 						Type:      v1.PackageRPM,
-						Version:   p.Version.Ver,
-						Resolved:  strings.TrimSuffix(idx.Source, "/") + "/" + strings.TrimPrefix(p.Location.Href, "/"),
-						Integrity: p.Checksum.Value,
-						Direct:    true,
-					},
-				}, nil
+						Version:   dep.Version.Ver,
+						Resolved:  strings.TrimSuffix(idx.Source, "/") + "/" + strings.TrimPrefix(dep.Location.Href, "/"),
+						Integrity: dep.Checksum.Text,
+						Direct:    false,
+					}
+					log.V(1).Info("collecting package", "name", dep.Name, "version", dep.Version.Ver)
+				}
+				packages[fmt.Sprintf("%s-%s", p.Name, p.Version.Ver)] = lockfile.Package{
+					Name:      p.Name,
+					Type:      v1.PackageRPM,
+					Version:   p.Version.Ver,
+					Resolved:  strings.TrimSuffix(idx.Source, "/") + "/" + strings.TrimPrefix(p.Location.Href, "/"),
+					Integrity: p.Checksum.Text,
+					Direct:    true,
+				}
+				log.V(1).Info("collecting package", "name", p.Name, "version", p.Version.Ver)
 			}
 		}
 	}
+	results := maps.Values(packages)
+	if len(results) > 0 {
+		return results, nil
+	}
 	return nil, fmt.Errorf("package could not be found in any index: %s", pkg)
 }

pkg/packages/rpm/rpm_test.go

@@ -22,19 +22,19 @@ func TestPackageKeeper_Unpack(t *testing.T) {
 	out := fs.DirFS(tempDir)
 
 	pkg := &PackageKeeper{}
-	err := pkg.Unpack(ctx, "./testdata/git-lfs-3.2.0-2.el8.x86_64.rpm", out)
+	err := pkg.Unpack(ctx, "./testdata/git-core-2.39.3-1.el8_8.x86_64.rpm", out)
 	assert.NoError(t, err)
 
 	assert.DirExists(t, filepath.Join(tempDir, "usr", "bin"))
-	assert.FileExists(t, filepath.Join(tempDir, "usr", "bin", "git-lfs"))
+	assert.FileExists(t, filepath.Join(tempDir, "usr", "bin", "git"))
 }
 
 func TestPackageKeeper_Resolve(t *testing.T) {
 	ctx := logr.NewContext(context.TODO(), testr.NewWithOptions(t, testr.Options{Verbosity: 10}))
 	pkg, err := NewPackageKeeper(ctx, []string{"https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi8/8/x86_64/appstream/os"})
 	require.NoError(t, err)
 
-	packageNames, err := pkg.Resolve(ctx, "git-lfs")
+	packageNames, err := pkg.Resolve(ctx, "git")
 	assert.NoError(t, err)
 	t.Logf("%+v", packageNames)
 }

pkg/yum/yumindex/requires.go

@@ -0,0 +1,33 @@
+package yumindex
+
+import (
+	"context"
+	"github.com/go-logr/logr"
+	"golang.org/x/exp/maps"
+	"slices"
+)
+
+func (m *Metadata) GetProviders(ctx context.Context, requires []string) []Package {
+	log := logr.FromContextOrDiscard(ctx)
+	log.V(1).Info("checking for packages", "requires", requires)
+
+	// collect a list of unique package matches
+	matches := map[string]Package{}
+
+	for _, pkg := range m.Package {
+		log.V(4).Info("checking package", "name", pkg.Name, "provides", pkg.Format.Provides)
+		for _, e := range pkg.Format.Provides.Entry {
+			// bunch of them are empty so just skip
+			// them
+			if e.Name == "" {
+				continue
+			}
+			if slices.Contains(requires, e.Name) {
+				log.V(2).Info("found matching package", "entry", e.Name)
+				matches[pkg.Name] = pkg
+			}
+		}
+	}
+
+	return maps.Values(matches)
+}

pkg/yum/yumindex/requires_test.go

@@ -0,0 +1,28 @@
+package yumindex
+
+import (
+	"context"
+	_ "embed"
+	"encoding/xml"
+	"github.com/go-logr/logr"
+	"github.com/go-logr/logr/testr"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+//go:embed testdata/aa4c51a3ffefb98ce7e412a98df164f40bfc07c4bc659aa855eb10f791d03c16-primary.xml
+var primaryDB string
+
+func TestMetadata_GetProviders(t *testing.T) {
+	ctx := logr.NewContext(context.TODO(), testr.NewWithOptions(t, testr.Options{Verbosity: 10}))
+
+	var metadata Metadata
+	require.NoError(t, xml.Unmarshal([]byte(primaryDB), &metadata))
+
+	matches := metadata.GetProviders(ctx, []string{"libacl.so.1()(64bit)", "libacl.so.1(ACL_1.0)(64bit)", "libc.so.6()(64bit)", "libc.so.6(GLIBC_2.11)(64bit)", "libc.so.6(GLIBC_2.14)(64bit)", "libc.so.6(GLIBC_2.15)(64bit)", "libc.so.6(GLIBC_2.2.5)(64bit)", "libc.so.6(GLIBC_2.28)(64bit)", "libc.so.6(GLIBC_2.3)(64bit)", "libc.so.6(GLIBC_2.3.4)(64bit)", "libc.so.6(GLIBC_2.4)(64bit)", "libselinux.so.1()(64bit)", "libtinfo.so.6()(64bit)", "rtld(GNU_HASH)"})
+	for _, m := range matches {
+		t.Logf("match: %s=%s", m.Name, m.Version.Ver)
+	}
+	assert.NotEmpty(t, matches)
+}