Merge pull request #1784 from imjasonh/rebuild

new command: rebuild

.gitignore

@@ -29,3 +29,7 @@ tags
 ctags
 
 .DS_Store
+
+
+x86_64/**
+aarch64/**

docs/md/melange.md

@@ -30,6 +30,7 @@ toc: true
 * [melange lint](/docs/md/melange_lint.md)	 - EXPERIMENTAL COMMAND - Lints an APK, checking for problems and errors
 * [melange package-version](/docs/md/melange_package-version.md)	 - Report the target package for a YAML configuration file
 * [melange query](/docs/md/melange_query.md)	 - Query a Melange YAML file for information
+* [melange rebuild](/docs/md/melange_rebuild.md)	 - Rebuild a melange package.
 * [melange scan](/docs/md/melange_scan.md)	 - Scan an existing APK to regenerate .PKGINFO
 * [melange sign](/docs/md/melange_sign.md)	 - Sign an APK package
 * [melange sign-index](/docs/md/melange_sign-index.md)	 - Sign an APK index

docs/md/melange_rebuild.md

@@ -0,0 +1,33 @@
+---
+title: "melange rebuild"
+slug: melange_rebuild
+url: /docs/md/melange_rebuild.md
+draft: false
+images: []
+type: "article"
+toc: true
+---
+## melange rebuild
+
+Rebuild a melange package.
+
+```
+melange rebuild [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for rebuild
+```
+
+### Options inherited from parent commands
+
+```
+      --log-level string   log level (e.g. debug, info, warn, error) (default "INFO")
+```
+
+### SEE ALSO
+
+* [melange](/docs/md/melange.md)	 - 
+

pkg/build/pipelines/README.md

@@ -56,6 +56,7 @@ Apply patches
 
 | Name | Required | Description | Default |
 | ---- | -------- | ----------- | ------- |
+| fuzz | false | Sets the maximum fuzz factor. This option only applies to context diffs, and causes patch to ignore up to that many lines in looking for places to install a hunk.  | 2 |
 | patches | false | A list of patches to apply, as a whitespace delimited string.  |  |
 | series | false | A quilt-style patch series file to apply.  |  |
 | strip-components | false | The number of path components to strip while extracting.  | 1 |

pkg/build/pipelines/cargo/README.md

@@ -16,7 +16,9 @@ Compile an auditable rust binary with Cargo
 | modroot | false | Top directory of the rust package, this is where the target package lives. Before building, the cargo pipeline wil cd into this directory. Defaults to current working directory  | . |
 | opts | false | Options to pass to cargo build. Defaults to release  | --release |
 | output | false | Filename to use when writing the binary. The final install location inside the apk will be in prefix / install-dir / output  |  |
+| output-dir | false | Directory where the binaris will be placed after building. Defaults to target/release  | target/release |
 | prefix | false | Installation prefix. Defaults to usr  | usr |
+| rustflags | false | Rustc flags to be passed to pass to all compiler invocations that Cargo performs. In contrast with cargo rustc, this is useful for passing a flag to all compiler instances. This string is split by whitespace.  |  |
 
 
 <!-- end:pipeline-reference-gen -->

pkg/cli/commands.go

@@ -67,6 +67,7 @@ func New() *cobra.Command {
 	cmd.AddCommand(test())
 	cmd.AddCommand(updateCache())
 	cmd.AddCommand(version.Version())
+	cmd.AddCommand(rebuild())
 	return cmd
 }
 

pkg/cli/rebuild.go

@@ -0,0 +1,134 @@
+package cli
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"time"
+
+	goapk "chainguard.dev/apko/pkg/apk/apk"
+	apko_types "chainguard.dev/apko/pkg/build/types"
+	"chainguard.dev/melange/pkg/build"
+	"chainguard.dev/melange/pkg/config"
+	"chainguard.dev/melange/pkg/container/docker"
+	"github.com/spf13/cobra"
+	"gopkg.in/ini.v1"
+	"gopkg.in/yaml.v3"
+)
+
+// TODO: Detect when the package is a subpackage (origin is different) and compare against the subpackage after building all packages.
+// TODO: Avoid rebuilding twice when rebuilding two subpackages of the same origin.
+// TODO: Add `--diff` flag to show the differences between the original and rebuilt packages, or document how to do it with shell commands.
+
+func rebuild() *cobra.Command {
+	return &cobra.Command{
+		Use:               "rebuild",
+		DisableAutoGenTag: true,
+		SilenceUsage:      true,
+		SilenceErrors:     true,
+		Short:             "Rebuild a melange package.",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+			r, err := docker.NewRunner(ctx)
+			if err != nil {
+				return fmt.Errorf("failed to create docker runner: %v", err)
+			}
+
+			for _, a := range args {
+				cfg, pkginfo, err := getConfig(a)
+				if err != nil {
+					return fmt.Errorf("failed to get config for %s: %v", a, err)
+				}
+
+				// TODO: This should not be necessary.
+				cfg.Environment.Contents.RuntimeRepositories = append(cfg.Environment.Contents.RuntimeRepositories, "https://packages.wolfi.dev/os")
+				cfg.Environment.Contents.Keyring = append(cfg.Environment.Contents.Keyring, "https://packages.wolfi.dev/os/wolfi-signing.rsa.pub")
+
+				f, err := os.CreateTemp("", "melange-rebuild-*.")
+				if err != nil {
+					return fmt.Errorf("failed to create temporary file: %v", err)
+				}
+				if err := yaml.NewEncoder(f).Encode(cfg); err != nil {
+					return fmt.Errorf("failed to encode stripped config: %v", err)
+				}
+				log.Println("wrote stripped config to", f.Name())
+
+				if err := BuildCmd(ctx,
+					[]apko_types.Architecture{apko_types.Architecture("amd64")},          // TODO configurable, or detect
+					build.WithConfigFileRepositoryURL("https://github.com/wolfi-dev/os"), // TODO get this from the package SBOM
+					build.WithConfigFileRepositoryCommit("TODO"),                         // TODO get this from the package SBOM
+					build.WithConfigFileLicense("Apache-2.0"),                            // TODO get this from the package SBOM
+					build.WithBuildDate(time.Unix(pkginfo.BuildDate, 0).Format(time.RFC3339)),
+					build.WithRunner(r), // TODO configurable
+					build.WithConfig(f.Name())); err != nil {
+					return fmt.Errorf("failed to rebuild %q: %v", a, err)
+				}
+			}
+
+			return nil
+		},
+	}
+}
+
+func getConfig(fn string) (*config.Configuration, *goapk.PackageInfo, error) {
+	f, err := os.Open(fn)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to open file %s: %v", fn, err)
+	}
+	defer f.Close()
+
+	var cfg *config.Configuration
+	var pkginfo *goapk.PackageInfo
+
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create gzip reader: %v", err)
+	}
+	defer gz.Close()
+	tr := tar.NewReader(gz)
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			if cfg == nil {
+				return nil, nil, fmt.Errorf("failed to find .melange.yaml in %s", fn)
+			}
+			if pkginfo == nil {
+				return nil, nil, fmt.Errorf("failed to find .PKGINFO in %s", fn)
+			}
+			return nil, nil, fmt.Errorf("failed to find necessary rebuild information in %s", fn)
+		} else if err != nil {
+			return nil, nil, fmt.Errorf("failed to read tar header: %v", err)
+		}
+
+		switch hdr.Name {
+		case ".melange.yaml":
+			cfg = new(config.Configuration)
+			if err := yaml.NewDecoder(io.LimitReader(tr, hdr.Size)).Decode(cfg); err != nil {
+				return nil, nil, fmt.Errorf("failed to decode .melange.yaml: %v", err)
+			}
+
+		case ".PKGINFO":
+			i, err := ini.ShadowLoad(tr)
+			if err != nil {
+				return nil, nil, fmt.Errorf("failed to load .PKGINFO: %v", err)
+			}
+			pkginfo = new(goapk.PackageInfo)
+			if err = i.MapTo(pkginfo); err != nil {
+				return nil, nil, fmt.Errorf("failed to map .PKGINFO: %v", err)
+			}
+
+		default:
+			// TODO: Get the SBOM, since we need some info from it too.
+			continue
+		}
+
+		if cfg != nil && pkginfo != nil {
+			return cfg, pkginfo, nil
+		}
+	}
+	// unreachable
+}

pkg/config/config.go

@@ -1267,7 +1267,7 @@ func (cfg *Configuration) propagatePipelines() {
 }
 
 // ParseConfiguration returns a decoded build Configuration using the parsing options provided.
-func ParseConfiguration(_ context.Context, configurationFilePath string, opts ...ConfigurationParsingOption) (*Configuration, error) {
+func ParseConfiguration(ctx context.Context, configurationFilePath string, opts ...ConfigurationParsingOption) (*Configuration, error) {
 	options := &configOptions{}
 	configurationDirPath := filepath.Dir(configurationFilePath)
 	options.include(opts...)
@@ -1474,7 +1474,7 @@ func ParseConfiguration(_ context.Context, configurationFilePath string, opts ..
 	}
 
 	// Finally, validate the configuration we ended up with before returning it for use downstream.
-	if err = cfg.validate(); err != nil {
+	if err = cfg.validate(ctx); err != nil {
 		return nil, fmt.Errorf("validating configuration %q: %w", cfg.Package.Name, err)
 	}
 
@@ -1499,7 +1499,7 @@ func (e ErrInvalidConfiguration) Unwrap() error {
 
 var packageNameRegex = regexp.MustCompile(`^[a-zA-Z\d][a-zA-Z\d+_.-]*$`)
 
-func (cfg Configuration) validate() error {
+func (cfg Configuration) validate(ctx context.Context) error {
 	if !packageNameRegex.MatchString(cfg.Package.Name) {
 		return ErrInvalidConfiguration{Problem: fmt.Errorf("package name must match regex %q", packageNameRegex)}
 	}
@@ -1513,7 +1513,7 @@ func (cfg Configuration) validate() error {
 	if err := validateDependenciesPriorities(cfg.Package.Dependencies); err != nil {
 		return ErrInvalidConfiguration{Problem: errors.New("priority must convert to integer")}
 	}
-	if err := validatePipelines(cfg.Pipeline); err != nil {
+	if err := validatePipelines(ctx, cfg.Pipeline); err != nil {
 		return ErrInvalidConfiguration{Problem: err}
 	}
 
@@ -1539,15 +1539,16 @@ func (cfg Configuration) validate() error {
 		if err := validateDependenciesPriorities(sp.Dependencies); err != nil {
 			return ErrInvalidConfiguration{Problem: errors.New("priority must convert to integer")}
 		}
-		if err := validatePipelines(sp.Pipeline); err != nil {
+		if err := validatePipelines(ctx, sp.Pipeline); err != nil {
 			return ErrInvalidConfiguration{Problem: err}
 		}
 	}
 
 	return nil
 }
 
-func validatePipelines(ps []Pipeline) error {
+func validatePipelines(ctx context.Context, ps []Pipeline) error {
+	log := clog.FromContext(ctx)
 	for _, p := range ps {
 		if p.With != nil && p.Uses == "" {
 			return fmt.Errorf("pipeline contains with but no uses")
@@ -1558,14 +1559,14 @@ func validatePipelines(ps []Pipeline) error {
 		}
 
 		if p.Uses != "" && len(p.Pipeline) > 0 {
-			return fmt.Errorf("pipeline cannot contain both uses %q and a pipeline", p.Uses)
+			log.Warnf("pipeline %q contains both uses and a pipeline", p.Name)
 		}
 
 		if len(p.With) > 0 && p.Runs != "" {
 			return fmt.Errorf("pipeline cannot contain both with and runs")
 		}
 
-		if err := validatePipelines(p.Pipeline); err != nil {
+		if err := validatePipelines(ctx, p.Pipeline); err != nil {
 			return err
 		}
 	}

pkg/config/config_test.go

@@ -416,13 +416,14 @@ func TestValidatePipelines(t *testing.T) {
 			p: []Pipeline{
 				{Uses: "deploy", Pipeline: []Pipeline{{Runs: "somescript.sh"}}},
 			},
-			wantErr: true,
+			wantErr: false, // only a warning.
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := validatePipelines(tt.p)
+			ctx := slogtest.Context(t)
+			err := validatePipelines(ctx, tt.p)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("validatePipelines() error = %v, wantErr %v", err, tt.wantErr)
 			}