From 71fc9bc643d8fd9daafe709347bdf1b4fa6fcabe Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Thu, 23 Feb 2023 11:11:29 -0500 Subject: [PATCH] cmd/release: deprecate Relui replaced release since Go 1.18 or so. We kept it around a bit longer while there were remaining Go+BoringCrypto releases that still used the release command. Those are no more, so to avoid confusion, remove cmd/release code given it's no longer maintained or supported. Updates golang/go#40279. Updates golang/go#58659. Fixes golang/go#45893. Change-Id: Id0d641bee49c9584c52e5616322f0656b89cd851 Reviewed-on: https://go-review.googlesource.com/c/build/+/470755 Reviewed-by: Dmitri Shuralyov Reviewed-by: Heschi Kreinick TryBot-Result: Gopher Robot Auto-Submit: Dmitri Shuralyov Reviewed-by: Carlos Amedee Run-TryBot: Dmitri Shuralyov --- cmd/release/README.md | 2 +- cmd/release/release.go | 312 +------------------------ cmd/release/upload.go | 302 ------------------------ cmd/release/upload_test.go | 26 --- doc/remote-buildlet.txt | 4 +- internal/task/releaselet/releaselet.go | 2 +- 6 files changed, 14 insertions(+), 634 deletions(-) delete mode 100644 cmd/release/upload.go delete mode 100644 cmd/release/upload_test.go diff --git a/cmd/release/README.md b/cmd/release/README.md index a2617914d5..0b44b8d984 100644 --- a/cmd/release/README.md +++ b/cmd/release/README.md @@ -4,4 +4,4 @@ # golang.org/x/build/cmd/release -Command release builds a Go release. +Command release was used to build Go releases before relui fully replaced its functionality. diff --git a/cmd/release/release.go b/cmd/release/release.go index a9888ff285..1983d292e0 100644 --- a/cmd/release/release.go +++ b/cmd/release/release.go @@ -2,317 +2,25 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Command release builds a Go release. +// Command release was used to build Go releases before relui fully replaced its functionality. +// +// Deprecated: Use relui instead. package main import ( - "bytes" - "context" "flag" "fmt" - "io" - "io/ioutil" - "log" - "math" - "math/rand" - "net/http" "os" - "path/filepath" - "regexp" - "runtime" - "strings" - "time" - - "golang.org/x/build/buildenv" - "golang.org/x/build/buildlet" - "golang.org/x/build/dashboard" - "golang.org/x/build/internal/gomote/protos" - "golang.org/x/build/internal/iapclient" - "golang.org/x/build/internal/releasetargets" - "golang.org/x/build/internal/task" - "golang.org/x/build/internal/workflow" - "golang.org/x/sync/errgroup" -) - -var ( - flagTarget = flag.String("target", "", "The specific target to build.") - flagWatch = flag.Bool("watch", false, "Watch the build.") - - flagStagingDir = flag.String("staging_dir", "", "If specified, use this as the staging directory for untested release artifacts. Default is the system temporary directory.") - - flagRevision = flag.String("rev", "", "Go revision to build") - flagVersion = flag.String("version", "", "Version string (go1.5.2)") - user = flag.String("user", username(), "coordinator username, appended to 'user-'") - flagSkipTests = flag.Bool("skip_tests", false, "skip all tests (only use if sufficient testing was done elsewhere)") - flagSkipLongTests = flag.Bool("skip_long_tests", false, "skip long tests (only use if sufficient testing was done elsewhere)") - - uploadMode = flag.Bool("upload", false, "Upload files (exclusive to all other flags)") -) - -var ( - coordClient *buildlet.GRPCCoordinatorClient - buildEnv *buildenv.Environment ) func main() { - flag.Parse() - rand.Seed(time.Now().UnixNano()) - - if *uploadMode { - buildenv.CheckUserCredentials() - userToken() // Call userToken for the side-effect of exiting if a gomote token doesn't exist. - if err := upload(flag.Args()); err != nil { - log.Fatal(err) - } - return - } - - ctx := &workflow.TaskContext{ - Context: context.TODO(), - Logger: &logger{*flagTarget}, - } - - if *flagRevision == "" { - log.Fatal("must specify -rev") - } - if *flagTarget == "" { - log.Fatal("must specify -target") - } - if *flagVersion == "" { - log.Fatal(`must specify -version flag (such as "go1.12" or "go1.13beta1")`) - } - stagingDir := *flagStagingDir - if stagingDir == "" { - var err error - stagingDir, err = ioutil.TempDir("", "go-release-staging_") - if err != nil { - log.Fatal(err) - } - } - if *flagTarget == "src" { - if err := writeSourceFile(ctx, *flagRevision, *flagVersion, *flagVersion+".src.tar.gz"); err != nil { - log.Fatalf("building source archive: %v", err) - } - return - } - - cc, err := iapclient.GRPCClient(ctx, "build.golang.org:443") - if err != nil { - log.Fatalf("Could not connect to coordinator: %v", err) - } - coordClient = &buildlet.GRPCCoordinatorClient{ - Client: protos.NewGomoteServiceClient(cc), - } - buildEnv = buildenv.Production - - targets, ok := releasetargets.TargetsForVersion(*flagVersion) - if !ok { - log.Fatalf("could not parse version %q", *flagVersion) - } - target, ok := targets[*flagTarget] - if !ok { - log.Fatalf("no such target %q in version %q", *flagTarget, *flagVersion) - } - if *flagSkipTests { - target.BuildOnly = true - } - if *flagSkipLongTests { - target.LongTestBuilder = "" - } - - ctx.Printf("Start.") - if err := doRelease(ctx, *flagRevision, *flagVersion, target, stagingDir, *flagWatch); err != nil { - ctx.Printf("Error: %v", err) - os.Exit(1) - } else { - ctx.Printf("Done.") - } -} - -const gerritURL = "https://go.googlesource.com/go" - -func doRelease(ctx *workflow.TaskContext, revision, version string, target *releasetargets.Target, stagingDir string, watch bool) error { - srcBuf := &bytes.Buffer{} - if err := task.WriteSourceArchive(ctx, http.DefaultClient, gerritURL, revision, version, srcBuf); err != nil { - return fmt.Errorf("Building source archive: %v", err) - } - - var stagingFiles []*os.File - stagingFile := func(ext string) (*os.File, error) { - f, err := ioutil.TempFile(stagingDir, fmt.Sprintf("%v.%v.%v.release-staging-*", version, target.Name, ext)) - stagingFiles = append(stagingFiles, f) - return f, err - } - // runWithBuildlet runs f with a newly-created builder. - runWithBuildlet := func(builder string, f func(*task.BuildletStep) error) error { - buildConfig, ok := dashboard.Builders[builder] - if !ok { - return fmt.Errorf("unknown builder: %v", buildConfig) - } - client, err := coordClient.CreateBuildlet(ctx, builder) - if err != nil { - return err - } - defer client.Close() - buildletStep := &task.BuildletStep{ - Target: target, - Buildlet: client, - BuildConfig: buildConfig, - LogWriter: os.Stdout, - } - if err := f(buildletStep); err != nil { - return err - } - return client.Close() + flag.Usage = func() { + fmt.Fprintln(os.Stderr, "Command golang.org/x/build/cmd/release is deprecated.") + fmt.Fprintln(os.Stderr, "Use golang.org/x/build/cmd/relui instead.") + flag.PrintDefaults() } - defer func() { - for _, f := range stagingFiles { - f.Close() - } - }() - - // Build the binary distribution. - binary, err := stagingFile("tar.gz") - if err != nil { - return err - } - if err := runWithBuildlet(target.Builder, func(step *task.BuildletStep) error { - return step.BuildBinary(ctx, srcBuf, binary) - }); err != nil { - return fmt.Errorf("Building binary archive: %v", err) - } - // Multiple tasks need to read the binary archive concurrently. Use a - // new SectionReader for each to keep them from conflicting. - binaryReader := func() io.Reader { return io.NewSectionReader(binary, 0, math.MaxInt64) } - - // Do everything else in parallel. - group, groupCtx := errgroup.WithContext(ctx) - - // If windows, produce the zip and MSI. - if target.GOOS == "windows" { - ctx := &workflow.TaskContext{Context: groupCtx, Logger: ctx.Logger} - msi, err := stagingFile("msi") - if err != nil { - return err - } - zip, err := stagingFile("zip") - if err != nil { - return err - } - group.Go(func() error { - if err := runWithBuildlet(target.Builder, func(step *task.BuildletStep) error { - return step.BuildWindowsMSI(ctx, binaryReader(), msi) - }); err != nil { - return fmt.Errorf("Building Windows artifacts: %v", err) - } - return nil - }) - group.Go(func() error { - return task.ConvertTGZToZIP(binaryReader(), zip) - }) - } - - // Run tests. - if !target.BuildOnly { - runTest := func(builder string) error { - ctx := &workflow.TaskContext{ - Context: groupCtx, - Logger: &logger{fmt.Sprintf("%v (tests on %v)", target.Name, builder)}, - } - if err := runWithBuildlet(builder, func(step *task.BuildletStep) error { - return step.TestTarget(ctx, binaryReader()) - }); err != nil { - return fmt.Errorf("Testing on %v: %v", builder, err) - } - return nil - } - group.Go(func() error { return runTest(target.Builder) }) - if target.LongTestBuilder != "" { - group.Go(func() error { return runTest(target.LongTestBuilder) }) - } - } - if err := group.Wait(); err != nil { - return err - } - - // If we get this far, the all.bash tests have passed (or been skipped). - // Move untested release files to their final locations. - stagingRe := regexp.MustCompile(`([^/]*)\.release-staging-.*`) - for _, f := range stagingFiles { - if err := f.Close(); err != nil { - return err - } - match := stagingRe.FindStringSubmatch(f.Name()) - if len(match) != 2 { - return fmt.Errorf("unexpected file name %q didn't match %v", f.Name(), stagingRe) - } - finalName := match[1] - ctx.Printf("Moving %q to %q.", f.Name(), finalName) - if err := os.Rename(f.Name(), finalName); err != nil { - return err - } - } - return nil -} - -type logger struct { - Name string -} - -func (l *logger) Printf(format string, args ...interface{}) { - format = fmt.Sprintf("%v: %s", l.Name, format) - log.Printf(format, args...) -} - -func writeSourceFile(ctx *workflow.TaskContext, revision, version, outPath string) error { - w, err := os.Create(outPath) - if err != nil { - return err - } - if err := task.WriteSourceArchive(ctx, http.DefaultClient, gerritURL, revision, version, w); err != nil { - return err - } - return w.Close() -} - -func homeDir() string { - if runtime.GOOS == "windows" { - return os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") - } - return os.Getenv("HOME") -} - -func configDir() string { - if runtime.GOOS == "windows" { - return filepath.Join(os.Getenv("APPDATA"), "Gomote") - } - if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" { - return filepath.Join(xdg, "gomote") - } - return filepath.Join(homeDir(), ".config", "gomote") -} - -func username() string { - if runtime.GOOS == "windows" { - return os.Getenv("USERNAME") - } - return os.Getenv("USER") -} + flag.Parse() -func userToken() string { - if *user == "" { - panic("userToken called with user flag empty") - } - keyDir := configDir() - baseFile := "user-" + *user + ".token" - tokenFile := filepath.Join(keyDir, baseFile) - slurp, err := ioutil.ReadFile(tokenFile) - if os.IsNotExist(err) { - log.Printf("Missing file %s for user %q. Change --user or obtain a token and place it there.", - tokenFile, *user) - } - if err != nil { - log.Fatal(err) - } - return strings.TrimSpace(string(slurp)) + flag.Usage() + os.Exit(1) } diff --git a/cmd/release/upload.go b/cmd/release/upload.go deleted file mode 100644 index c4e05be6ca..0000000000 --- a/cmd/release/upload.go +++ /dev/null @@ -1,302 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "math/rand" - "net/http" - "net/url" - "path/filepath" - "regexp" - "sort" - "strings" - "time" - - "cloud.google.com/go/storage" - "golang.org/x/sync/errgroup" - "google.golang.org/api/iterator" -) - -const ( - uploadURL = "https://go.dev/dl/upload" - storageBucket = "golang" -) - -// File represents a file on the go.dev downloads page. -// It should be kept in sync with the download code in x/website/internal/dl. -type File struct { - Filename string `json:"filename"` - OS string `json:"os"` - Arch string `json:"arch"` - Version string `json:"version"` - Checksum string `json:"-"` // SHA1; deprecated - ChecksumSHA256 string `json:"sha256"` - Size int64 `json:"size"` - Kind string `json:"kind"` // "archive", "installer", "source" -} - -// fileRe matches the files created by the release tool, such as: -// -// go1.5beta2.src.tar.gz -// go1.5.1.linux-386.tar.gz -// go1.5.windows-amd64.msi -var fileRe = regexp.MustCompile(`^(go[a-z0-9-.]+)\.(src|([a-z0-9]+)-([a-z0-9]+)(?:-([a-z0-9.]+))?)\.(tar\.gz|zip|pkg|msi)(.asc)?$`) - -func upload(files []string) error { - ctx := context.Background() - c, err := storage.NewClient(ctx) - if err != nil { - return err - } - defer c.Close() - - files, err = expandFiles(ctx, c, files) - if err != nil { - return err - } - - files = chooseBestFiles(files) - - var sitePayloads []*File - var uploaded []string - for _, name := range files { - base := filepath.Base(name) - log.Printf("Uploading %v to GCS ...", base) - m := fileRe.FindStringSubmatch(base) - if m == nil { - return fmt.Errorf("unrecognized file: %q", base) - } - - checksum, size, err := uploadArtifact(ctx, c, name) - if err != nil { - return fmt.Errorf("uploading %q: %v", name, err) - } - - uploaded = append(uploaded, base) - - if strings.HasSuffix(base, ".asc") { - // Don't add asc files to the download page, just upload it. - continue - } - - // Upload file.sha256. - fname := base + ".sha256" - if err := putObject(ctx, c, fname, []byte(checksum)); err != nil { - return fmt.Errorf("uploading %q: %v", base+".sha256", err) - } - uploaded = append(uploaded, fname) - - var kind string - switch { - case m[2] == "src": - kind = "source" - case strings.HasSuffix(base, ".tar.gz"), strings.HasSuffix(base, ".zip"): - kind = "archive" - case strings.HasSuffix(base, ".msi"), strings.HasSuffix(base, ".pkg"): - kind = "installer" - } - f := &File{ - Filename: base, - Version: m[1], - OS: m[3], - Arch: m[4], - ChecksumSHA256: checksum, - Size: size, - Kind: kind, - } - sitePayloads = append(sitePayloads, f) - } - - log.Println("Waiting for edge cache ...") - if err := waitForEdgeCache(uploaded); err != nil { - return fmt.Errorf("waitForEdgeCache(%+v): %v", uploaded, err) - } - - log.Println("Uploading payloads to go.dev ...") - for _, f := range sitePayloads { - if err := updateSite(f); err != nil { - return fmt.Errorf("updateSite(%+v): %v", f, err) - } - } - return nil -} - -func waitForEdgeCache(uploaded []string) error { - var g errgroup.Group - for _, u := range uploaded { - fname := u - g.Go(func() error { - // Add some jitter so that dozens of requests are not hitting the - // endpoint at once. - time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) - t := time.Tick(5 * time.Second) - var retries int - for { - url := "https://dl.google.com/go/" + fname - resp, err := http.Head(url) - if err != nil { - if retries < 3 { - retries++ - <-t - continue - } - return fmt.Errorf("http.Head(%q): %v", url, err) - } - retries = 0 - defer resp.Body.Close() - if resp.StatusCode == http.StatusOK { - log.Printf("%s is ready to go!", url) - break - } - <-t - } - return nil - }) - } - return g.Wait() -} - -func updateSite(f *File) error { - // Post file details to go.dev. - req, err := json.Marshal(f) - if err != nil { - return err - } - v := url.Values{"user": {*user}, "key": []string{userToken()}} - u := fmt.Sprintf("%s?%s", uploadURL, v.Encode()) - resp, err := http.Post(u, "application/json", bytes.NewReader(req)) - if err != nil { - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - b, _ := ioutil.ReadAll(resp.Body) - return fmt.Errorf("upload failed: %v\n%s", resp.Status, b) - } - return nil -} - -func putObject(ctx context.Context, c *storage.Client, name string, body []byte) error { - wr := c.Bucket(storageBucket).Object(name).NewWriter(ctx) - if _, err := wr.Write(body); err != nil { - return err - } - return wr.Close() -} - -// expandFiles expands any "/..." paths in GCS URIs to include files in its subtree. -func expandFiles(ctx context.Context, storageClient *storage.Client, files []string) ([]string, error) { - var expanded []string - for _, f := range files { - if !(strings.HasPrefix(f, "gs://") && strings.HasSuffix(f, "/...")) { - expanded = append(expanded, f) - continue - } - bucket, path := gcsParts(f) - - iter := storageClient.Bucket(bucket).Objects(ctx, &storage.Query{ - Prefix: strings.TrimSuffix(path, "..."), // Retain trailing "/" (if present). - }) - for { - attrs, err := iter.Next() - if err == iterator.Done { - break - } - if err != nil { - return nil, err - } - if filepath.Ext(attrs.Name) == ".sha256" { - // Ignore sha256 files. - continue - } - expanded = append(expanded, fmt.Sprintf("gs://%s/%s", attrs.Bucket, attrs.Name)) - } - } - return expanded, nil -} - -// gcsParts splits a GCS URI (e.g., "gs://bucket/path/to/object") into its bucket and path parts: -// ("bucket", "path/to/object") -// -// It assumes its input a well-formed GCS URI. -func gcsParts(uri string) (bucket, path string) { - parts := strings.SplitN(strings.TrimPrefix(uri, "gs://"), "/", 2) - return parts[0], parts[1] -} - -func chooseBestFiles(files []string) []string { - // map from basename to filepath/GCS URI. - best := make(map[string]string) - for _, f := range files { - base := filepath.Base(f) - if _, ok := best[base]; !ok { - best[base] = f - continue - } - - // Overwrite existing only if the new entry is signed. - if strings.HasPrefix(f, "gs://") && strings.Contains(f, "/signed/") { - best[base] = f - } - } - - var out []string - for _, path := range best { - out = append(out, path) - } - sort.Strings(out) // for prettier printing. - return out -} - -func uploadArtifact(ctx context.Context, storageClient *storage.Client, path string) (checksum string, size int64, err error) { - if strings.HasPrefix(path, "gs://") { - return uploadArtifactGCS(ctx, storageClient, path) - } - return uploadArtifactLocal(ctx, storageClient, path) -} - -func uploadArtifactGCS(ctx context.Context, storageClient *storage.Client, path string) (checksum string, size int64, err error) { - bucket, path := gcsParts(path) - base := filepath.Base(path) - src := storageClient.Bucket(bucket).Object(path) - dst := storageClient.Bucket(storageBucket).Object(base) - - r, err := storageClient.Bucket(bucket).Object(path + ".sha256").NewReader(ctx) - if err != nil { - return "", -1, fmt.Errorf("could not get sha256: %v", err) - } - checksumBytes, err := ioutil.ReadAll(r) - if err != nil { - return "", -1, fmt.Errorf("could not get sha256: %v", err) - } - copier := dst.CopierFrom(src) - attrs, err := copier.Run(ctx) - if err != nil { - return "", -1, err - } - return string(checksumBytes), attrs.Size, nil -} - -func uploadArtifactLocal(ctx context.Context, storageClient *storage.Client, path string) (checksum string, size int64, err error) { - base := filepath.Base(path) - - fileBytes, err := ioutil.ReadFile(path) - if err != nil { - return "", -1, fmt.Errorf("ioutil.ReadFile: %v", err) - } - // Upload file to Google Cloud Storage. - if err := putObject(ctx, storageClient, base, fileBytes); err != nil { - return "", -1, err - } - checksum = fmt.Sprintf("%x", sha256.Sum256(fileBytes)) - return checksum, int64(len(fileBytes)), nil -} diff --git a/cmd/release/upload_test.go b/cmd/release/upload_test.go deleted file mode 100644 index 85cd863c7b..0000000000 --- a/cmd/release/upload_test.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2022 Go Authors All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import "testing" - -func TestFileRe(t *testing.T) { - shouldMatch := []string{ - "go1.5beta2.src.tar.gz", - "go1.5.1.linux-386.tar.gz", - "go1.5.windows-amd64.msi", - - "go1.5beta2.src.tar.gz.asc", - "go1.5.1.linux-386.tar.gz.asc", - "go1.5.windows-amd64.msi.asc", - } - for _, fn := range shouldMatch { - t.Run(fn, func(t *testing.T) { - if !fileRe.MatchString(fn) { - t.Fatalf("want %q to match, didn't", fn) - } - }) - } -} diff --git a/doc/remote-buildlet.txt b/doc/remote-buildlet.txt index 0499e27965..6e20796643 100644 --- a/doc/remote-buildlet.txt +++ b/doc/remote-buildlet.txt @@ -1,6 +1,6 @@ A remote buildlet is a buildlet created or assigned by the coordinator, on request of a client (such as cmd/gomote or -cmd/release). The client must occasionally refresh the buildlet to +cmd/relui). The client must occasionally refresh the buildlet to keep it alive, and the coordinator proxies all requests to it. Currently, if the coordinator dies or restarts, all buildlets are lost. @@ -12,7 +12,7 @@ HTTP Basic authentication. Flow: -*) Client (e.g. pkg buildlet, via cmd/gomote or cmd/release) makes a +*) Client (e.g. pkg buildlet, via cmd/gomote or cmd/relui) makes a POST request over key-pinned TLS to the well-known TLS cert of farmer.golang.org diff --git a/internal/task/releaselet/releaselet.go b/internal/task/releaselet/releaselet.go index 43df2b0b05..92a067fd30 100644 --- a/internal/task/releaselet/releaselet.go +++ b/internal/task/releaselet/releaselet.go @@ -124,7 +124,7 @@ func windowsMSI() error { return err } - msi := filepath.Join(cwd, "msi") // known to cmd/release + msi := filepath.Join(cwd, "msi") // known to internal/task.BuildletStep.BuildWindowsMSI if err := os.Mkdir(msi, 0755); err != nil { return err }