[9.0](backport #42402) FIPS Build

dev-tools/mage/build.go

@@ -24,6 +24,7 @@
 	"log"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/josephspurrier/goversioninfo"
@@ -46,6 +47,39 @@
 	WinMetadata bool // Add resource metadata to Windows binaries (like add the version number to the .exe properties).
 }
 
+// buildTagRE is a regexp to match strings like "-tags=abcd"
+// but does not match "-tags= "
+var buildTagRE = regexp.MustCompile(`-tags=([\S]+)?`)
+
+// ParseBuildTags returns the ExtraFlags param where all flags that are go build tags are joined by a comma.
+//
+// For example if given -someflag=val1 -tags=buildtag1 -tags=buildtag2
+// It will return -someflag=val1 -tags=buildtag1,buildtag2
+func (b BuildArgs) ParseBuildTags() []string {
+	flags := make([]string, 0)
+	if len(b.ExtraFlags) == 0 {
+		return flags
+	}
+
+	buildTags := make([]string, 0)
+	for _, flag := range b.ExtraFlags {
+		if buildTagRE.MatchString(flag) {
+			arr := buildTagRE.FindStringSubmatch(flag)
+			if len(arr) != 2 || arr[1] == "" {
+				log.Printf("Parsing buildargs.ExtraFlags found strange flag %q ignoring value", flag)
+				continue
+			}
+			buildTags = append(buildTags, arr[1])
+		} else {
+			flags = append(flags, flag)
+		}
+	}
+	if len(buildTags) > 0 {
+		flags = append(flags, "-tags="+strings.Join(buildTags, ","))
+	}
+	return flags
+}
+
 // DefaultBuildArgs returns the default BuildArgs for use in builds.
 func DefaultBuildArgs() BuildArgs {
 	args := BuildArgs{
@@ -74,6 +108,10 @@
 		// Remove all file system paths from the compiled executable, to improve build reproducibility
 		args.ExtraFlags = append(args.ExtraFlags, "-trimpath")
 	}
+	if FIPSBuild {
+		args.ExtraFlags = append(args.ExtraFlags, "-tags=requirefips")
+		args.CGO = true
+	}
 
 	return args
 }
@@ -175,6 +213,10 @@
 	if params.CGO {
 		cgoEnabled = "1"
 	}
+	if FIPSBuild {
+		cgoEnabled = "1"
+		env["GOEXPERIMENT"] = "systemcrypto"
+	}
 	env["CGO_ENABLED"] = cgoEnabled
 
 	// Spec
@@ -186,7 +228,7 @@
 	if params.BuildMode != "" {
 		args = append(args, "-buildmode", params.BuildMode)
 	}
-	args = append(args, params.ExtraFlags...)
+	args = append(args, params.ParseBuildTags()...)
 
 	// ldflags
 	ldflags := params.LDFlags
@@ -248,7 +290,7 @@
 		},
 		StringFileInfo: goversioninfo.StringFileInfo{
 			CompanyName:      BeatVendor,
 			ProductName:      strings.Title(BeatName),
 			ProductVersion:   version,
 			FileVersion:      version,
 			FileDescription:  BeatDescription,

dev-tools/mage/build_test.go

@@ -0,0 +1,72 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_BuildArgs_ParseBuildTags(t *testing.T) {
+	tests := []struct {
+		name   string
+		input  []string
+		expect []string
+	}{{
+		name:   "no flags",
+		input:  nil,
+		expect: []string{},
+	}, {
+		name:   "multiple flags with no tags",
+		input:  []string{"-a", "-b", "-key=value"},
+		expect: []string{"-a", "-b", "-key=value"},
+	}, {
+		name:   "one build tag",
+		input:  []string{"-tags=example"},
+		expect: []string{"-tags=example"},
+	}, {
+		name:   "multiple build tags",
+		input:  []string{"-tags=example", "-tags=test"},
+		expect: []string{"-tags=example,test"},
+	}, {
+		name:   "joined build tags",
+		input:  []string{"-tags=example,test"},
+		expect: []string{"-tags=example,test"},
+	}, {
+		name:   "multiple build tags with other flags",
+		input:  []string{"-tags=example", "-tags=test", "-key=value", "-a"},
+		expect: []string{"-key=value", "-a", "-tags=example,test"},
+	}, {
+		name:   "incorrectly formatted tag",
+		input:  []string{"-tags= example"},
+		expect: []string{},
+	}, {
+		name:   "incorrectly formatted tag with valid tag",
+		input:  []string{"-tags= example", "-tags=test"},
+		expect: []string{"-tags=test"},
+	}}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			args := BuildArgs{ExtraFlags: tc.input}
+			flags := args.ParseBuildTags()
+			assert.EqualValues(t, tc.expect, flags)
+		})
+	}
+}

dev-tools/mage/common.go

@@ -511,6 +511,12 @@ func untar(sourceFile, destinationDir string) error {
 				return err
 			}
 		case tar.TypeReg:
+			// create containing folder if it doesn't exist yet
+			targetContainingDir := filepath.Dir(filepath.FromSlash(path))
+			if mkDirErr := os.MkdirAll(targetContainingDir, 0755); mkDirErr != nil {
+				return fmt.Errorf("creating container directory for file %s: %w", header.Name, mkDirErr)
+			}
+
 			writer, err := os.Create(path)
 			if err != nil {
 				return err

dev-tools/mage/crossbuild.go

@@ -248,6 +248,9 @@ func CrossBuildImage(platform string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	if FIPSBuild {
+		tagSuffix += "-fips"
+	}
 
 	return BeatsCrossBuildImage + ":" + goVersion + "-" + tagSuffix, nil
 }
@@ -331,6 +334,7 @@ func (b GolangCrossBuilder) Build() error {
 		"--env", "MAGEFILE_VERBOSE="+verbose,
 		"--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""),
 		"--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot),
+		"--env", fmt.Sprintf("FIPS=%v", FIPSBuild),
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 	)

dev-tools/mage/dockerbuilder.go

@@ -101,7 +101,7 @@
 }
 
 func (b *dockerBuilder) exposePorts() []string {
 	if ports, _ := b.ExtraVars["expose_ports"]; ports != "" {
 		return strings.Split(ports, ",")
 	}
 	return nil
@@ -194,7 +194,10 @@
 	if b.Snapshot {
 		tag = tag + "-SNAPSHOT"
 	}
+	if b.FIPS {
+		tag = tag + "-fips"
+	}
 	if repository, _ := b.ExtraVars["repository"]; repository != "" {
 		tag = fmt.Sprintf("%s/%s", repository, tag)
 	}
 	return tag, sh.Run("docker", "build", "-t", tag, b.buildDir)
@@ -204,7 +207,7 @@
 	if _, err := os.Stat(distributionsDir); os.IsNotExist(err) {
 		err := os.MkdirAll(distributionsDir, 0750)
 		if err != nil {
 			return fmt.Errorf("cannot create folder for docker artifacts: %+v", err)
 		}
 	}
 	// Save the container as artifact

dev-tools/mage/gotest.go

@@ -292,9 +292,9 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
 		}
 	}
 	if len(params.Tags) > 0 {
-		params := strings.Join(params.Tags, " ")
+		params := strings.Join(params.Tags, ",")
 		if params != "" {
-			testArgs = append(testArgs, "-tags", params)
+			testArgs = append(testArgs, "-tags="+params)
 		}
 	}
 	if params.CoverageProfileFile != "" {

dev-tools/mage/pkg.go

@@ -88,6 +88,7 @@ func Package() error {
 				spec.OS = target.GOOS()
 				spec.Arch = packageArch
 				spec.Snapshot = Snapshot
+				spec.FIPS = FIPSBuild
 				spec.evalContext = map[string]interface{}{
 					"GOOS":          target.GOOS(),
 					"GOARCH":        target.GOARCH(),
@@ -246,11 +247,11 @@ type packageBuilder struct {
 }
 
 func (b packageBuilder) Build() error {
-	fmt.Printf(">> package: Building %v type=%v for platform=%v\n", b.Spec.Name, b.Type, b.Platform.Name)
+	fmt.Printf(">> package: Building %v type=%v for platform=%v fips=%v\n", b.Spec.Name, b.Type, b.Platform.Name, b.Spec.FIPS)
 	log.Printf("Package spec: %+v", b.Spec)
 	if err := b.Type.Build(b.Spec); err != nil {
-		return fmt.Errorf("failed building %v type=%v for platform=%v: %w",
-			b.Spec.Name, b.Type, b.Platform.Name, err)
+		return fmt.Errorf("failed building %v type=%v for platform=%v fips=%v: %w",
+			b.Spec.Name, b.Type, b.Platform.Name, b.Spec.FIPS, err)
 	}
 	return nil
 }
@@ -344,6 +345,10 @@ func TestPackages(options ...TestPackagesOption) error {
 		args = append(args, "-root-owner")
 	}
 
+	if FIPSBuild {
+		args = append(args, "-fips")
+	}
+
 	args = append(args, "-files", MustExpand("{{.PWD}}/build/distributions/*"))
 
 	if out, err := goTest(args...); err != nil {

dev-tools/mage/pkgtypes.go

@@ -24,7 +24,7 @@
 	"compress/gzip"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -47,7 +47,7 @@
 	packageStagingDir = "build/package"
 
 	// defaultBinaryName specifies the output file for zip and tar.gz.
-	defaultBinaryName = "{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}"
+	defaultBinaryName = "{{.Name}}{{if .FIPS}}-fips{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}"
 )
 
 // PackageType defines the file format of the package (e.g. zip, rpm, etc).
@@ -79,6 +79,7 @@
 	Arch              string                 `yaml:"arch,omitempty"`
 	Vendor            string                 `yaml:"vendor,omitempty"`
 	Snapshot          bool                   `yaml:"snapshot"`
+	FIPS              bool                   `yaml:"fips"`
 	Version           string                 `yaml:"version,omitempty"`
 	License           string                 `yaml:"license,omitempty"`
 	URL               string                 `yaml:"url,omitempty"`
@@ -455,7 +456,7 @@
 // ImageName computes the image name from the spec. A template for the image
 // name can be configured by adding image_name to extra_vars.
 func (s PackageSpec) ImageName() (string, error) {
 	if name, _ := s.ExtraVars["image_name"]; name != "" {
 		imageName, err := s.Expand(name)
 		if err != nil {
 			return "", fmt.Errorf("failed to expand image_name: %w", err)
@@ -528,7 +529,7 @@
 
 	// NOTE: This uses .BeatName instead of .Name because we wanted the internal
 	// directory to not include "-oss".
-	return s.MustExpand("{{.BeatName}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}")
+	return s.MustExpand("{{.BeatName}}{{if .FIPS}}-fips{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}")
 }
 
 // PackageZip packages a zip file.
@@ -688,7 +689,7 @@
 	return nil
 }
 
 func replaceFileArch(filename string, pkgFile PackageFile, arch string) (string, PackageFile) {
 	filename = strings.ReplaceAll(filename, "universal", arch)
 	pkgFile.Source = strings.ReplaceAll(pkgFile.Source, "universal", arch)
 	pkgFile.Target = strings.ReplaceAll(pkgFile.Target, "universal", arch)
@@ -716,7 +717,7 @@
 	}
 
 	if err := HaveDocker(); err != nil {
 		return fmt.Errorf("packaging %v files requires docker: %v", fpmPackageType, err)
 	}
 
 	// Build a tar file as the input to FPM.
@@ -727,7 +728,7 @@
 	}
 	defer os.Remove(inputTar)
 
-	outputFile, err := spec.Expand("{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.Arch}}")
+	outputFile, err := spec.Expand("{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.Arch}}{{if .FIPS}}-fips{{end}}")
 	if err != nil {
 		return err
 	}
@@ -1015,7 +1016,7 @@
 // PackageDocker packages the Beat into a docker image.
 func PackageDocker(spec PackageSpec) error {
 	if err := HaveDocker(); err != nil {
 		return fmt.Errorf("docker daemon required to build images: %s", err)
 	}
 
 	b, err := newDockerBuilder(spec)

dev-tools/mage/settings.go

@@ -79,8 +79,9 @@ var (
 
 	BeatProjectType ProjectType
 
-	Snapshot bool
-	DevBuild bool
+	Snapshot  bool
+	DevBuild  bool
+	FIPSBuild bool
 
 	versionQualified bool
 	versionQualifier string
@@ -128,6 +129,11 @@ func init() {
 		panic(fmt.Errorf("failed to parse DEV env value: %w", err))
 	}
 
+	FIPSBuild, err = strconv.ParseBool(EnvOr("FIPS", "false"))
+	if err != nil {
+		panic(fmt.Errorf("failed to parse FIPS env value: %w", err))
+	}
+
 	versionQualifier, versionQualified = os.LookupEnv("VERSION_QUALIFIER")
 }
 
@@ -179,6 +185,7 @@ func varMap(args ...map[string]interface{}) map[string]interface{} {
 		"BeatUser":        BeatUser,
 		"Snapshot":        Snapshot,
 		"DEV":             DevBuild,
+		"FIPS":            FIPSBuild,
 		"Qualifier":       versionQualifier,
 		"CI":              CI,
 	}