X-Pack binaries

CHANGELOG-developer.asciidoc

@@ -44,3 +44,4 @@ The list below covers the major changes between 6.3.0 and master only.
   'go test'. This captures the log to a file, summarizes the result, produces a
   coverage profile (.cov), and produces an HTML coverage report. See
   `mage -h goTestUnit`. {pull}7766[7766]
+- Beats packaging now build non-oss binaries from code located in the x-pack folder. {issue}7783[7783]

Makefile

@@ -84,12 +84,14 @@ check: python-env
 .PHONY: check-headers
 check-headers:
 	@go get github.com/elastic/go-licenser
-	@go-licenser -d
+	@go-licenser -d -exclude x-pack
+	@go-licenser -d -license Elastic x-pack
 
 .PHONY: add-headers
 add-headers:
 	@go get github.com/elastic/go-licenser
-	@go-licenser
+	@go-licenser -exclude x-pack
+	@go-licenser -license Elastic x-pack
 
 # Corrects spelling errors
 .PHONY: misspell

auditbeat/magefile.go

@@ -57,6 +57,11 @@ func CrossBuild() error {
 	return mage.CrossBuild()
 }
 
+// CrossBuildXPack cross-builds the beat with XPack for all target platforms.
+func CrossBuildXPack() error {
+	return mage.CrossBuildXPack()
+}
+
 // CrossBuildGoDaemon cross-builds the go-daemon binary using Docker.
 func CrossBuildGoDaemon() error {
 	return mage.CrossBuildGoDaemon()
@@ -78,7 +83,7 @@ func Package() {
 	customizePackaging()
 
 	mg.Deps(Update)
-	mg.Deps(makeConfigTemplates, CrossBuild, CrossBuildGoDaemon)
+	mg.Deps(makeConfigTemplates, CrossBuild, CrossBuildXPack, CrossBuildGoDaemon)
 	mg.SerialDeps(mage.Package, TestPackages)
 }
 

dev-tools/mage/clean.go

@@ -35,6 +35,10 @@ var DefaultCleanPaths = []string{
 	"_meta/kibana.generated",
 	"_meta/kibana/5/index-pattern/{{.BeatName}}.json",
 	"_meta/kibana/6/index-pattern/{{.BeatName}}.json",
+
+	"../x-pack/{{.BeatName}}/build",
+	"../x-pack/{{.BeatName}}/{{.BeatName}}",
+	"../x-pack/{{.BeatName}}/{{.BeatName}}.exe",
 }
 
 // Clean clean generated build artifacts.

dev-tools/mage/crossbuild.go

@@ -63,6 +63,13 @@ func WithTarget(target string) func(params *crossBuildParams) {
 	}
 }
 
+// InDir specifies the base directory to use when cross-building.
+func InDir(path ...string) func(params *crossBuildParams) {
+	return func(params *crossBuildParams) {
+		params.InDir = filepath.Join(path...)
+	}
+}
+
 // Serially causes each cross-build target to be executed serially instead of
 // in parallel.
 func Serially() func(params *crossBuildParams) {
@@ -75,6 +82,7 @@ type crossBuildParams struct {
 	Platforms BuildPlatformList
 	Target    string
 	Serial    bool
+	InDir     string
 }
 
 // CrossBuild executes a given build target once for each target platform.
@@ -103,8 +111,7 @@ func CrossBuild(options ...CrossBuildOption) error {
 		if !buildPlatform.Flags.CanCrossBuild() {
 			return fmt.Errorf("unsupported cross build platform %v", buildPlatform.Name)
 		}
-
-		builder := GolangCrossBuilder{buildPlatform.Name, params.Target}
+		builder := GolangCrossBuilder{buildPlatform.Name, params.Target, params.InDir}
 		if params.Serial {
 			if err := builder.Build(); err != nil {
 				return errors.Wrapf(err, "failed cross-building target=%v for platform=%v",
@@ -120,6 +127,15 @@ func CrossBuild(options ...CrossBuildOption) error {
 	return nil
 }
 
+// CrossBuildXPack executes the 'golangCrossBuild' target in the Beat's
+// associated x-pack directory to produce a version of the Beat that contains
+// Elastic licensed content.
+func CrossBuildXPack(options ...CrossBuildOption) error {
+	o := []CrossBuildOption{InDir("x-pack", BeatName)}
+	o = append(o, options...)
+	return CrossBuild(o...)
+}
+
 // buildMage pre-compiles the magefile to a binary using the native GOOS/GOARCH
 // values for Docker. This is required to so that we can later pass GOOS and
 // GOARCH to mage for the cross-build. It has the benefit of speeding up the
@@ -166,6 +182,7 @@ func crossBuildImage(platform string) (string, error) {
 type GolangCrossBuilder struct {
 	Platform string
 	Target   string
+	InDir    string
 }
 
 // Build executes the build inside of Docker.
@@ -178,9 +195,16 @@ func (b GolangCrossBuilder) Build() error {
 	}
 
 	mountPoint := filepath.ToSlash(filepath.Join("/go", "src", repoInfo.RootImportPath))
-	workDir := mountPoint
-	if repoInfo.SubDir != "" {
-		workDir = filepath.ToSlash(filepath.Join(workDir, repoInfo.SubDir))
+	// use custom dir for build if given, subdir if not:
+	cwd := repoInfo.SubDir
+	if b.InDir != "" {
+		cwd = b.InDir
+	}
+	workDir := filepath.ToSlash(filepath.Join(mountPoint, cwd))
+
+	buildCmd, err := filepath.Rel(workDir, filepath.Join(mountPoint, repoInfo.SubDir, "build/mage-linux-amd64"))
+	if err != nil {
+		return errors.Wrap(err, "failed to determine mage-linux-amd64 relative path")
 	}
 
 	dockerRun := sh.RunCmd("docker", "run")
@@ -206,7 +230,7 @@ func (b GolangCrossBuilder) Build() error {
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 		image,
-		"--build-cmd", "build/mage-linux-amd64 "+b.Target,
+		"--build-cmd", buildCmd+" "+b.Target,
 		"-p", b.Platform,
 	)
 
@@ -215,25 +239,27 @@ func (b GolangCrossBuilder) Build() error {
 
 // DockerChown chowns files generated during build. EXEC_UID and EXEC_GID must
 // be set in the containers environment otherwise this is a noop.
-func DockerChown(file string) {
+func DockerChown(path string) {
 	// Chown files generated during build that are root owned.
 	uid, _ := strconv.Atoi(EnvOr("EXEC_UID", "-1"))
 	gid, _ := strconv.Atoi(EnvOr("EXEC_GID", "-1"))
 	if uid > 0 && gid > 0 {
-		if err := chownPaths(uid, gid, file); err != nil {
+		if err := chownPaths(uid, gid, path); err != nil {
 			log.Println(err)
 		}
 	}
 }
 
 // chownPaths will chown the file and all of the dirs specified in the path.
-func chownPaths(uid, gid int, file string) error {
-	pathParts := strings.Split(file, string(filepath.Separator))
-	for i := range pathParts {
-		chownDir := filepath.Join(pathParts[:i+1]...)
-		if err := os.Chown(chownDir, uid, gid); err != nil {
-			return errors.Wrapf(err, "failed to chown path=%v", chownDir)
+func chownPaths(uid, gid int, path string) error {
+	return filepath.Walk(path, func(name string, _ os.FileInfo, err error) error {
+		if err != nil {
+			return err
 		}
-	}
-	return nil
+		log.Printf("chown line: %s\n", name)
+		if err := os.Chown(name, uid, gid); err != nil {
+			return errors.Wrapf(err, "failed to chown path=%v", name)
+		}
+		return err
+	})
 }

dev-tools/mage/pkg.go

@@ -37,7 +37,7 @@ func Package() error {
 
 	if len(Packages) == 0 {
 		return errors.New("no package specs are registered. Call " +
-			"UseCommunityBeatPackaging or UseElasticBeatPackaging first.")
+			"UseCommunityBeatPackaging, UseElasticBeatPackaging or USeElasticBeatWithoutXPackPackaging first.")
 	}
 
 	var tasks []interface{}

dev-tools/mage/pkgspecs.go

@@ -62,6 +62,26 @@ func UseElasticBeatPackaging() {
 	}
 }
 
+// UseElasticBeatWithoutXPackPackaging configures the package target to build packages for
+// an Elastic Beat. This means it will generate two sets of packages -- one
+// that is purely OSS under Apache 2.0 and one that is licensed under the
+// Elastic License and may contain additional X-Pack features.
+//
+// NOTE: This method doesn't use binaries produced in the x-pack folder, this is
+// a temporary packaging target for projects that depends on beat but do have concrete x-pack
+// binaries.
+func UseElasticBeatWithoutXPackPackaging() {
+	beatsDir, err := ElasticBeatsDir()
+	if err != nil {
+		panic(err)
+	}
+
+	err = LoadNamedSpec("elastic_beat_without_xpack", filepath.Join(beatsDir, packageSpecFile))
+	if err != nil {
+		panic(err)
+	}
+}
+
 // LoadNamedSpec loads a packaging specification with the given name from the
 // specified YAML file. name should be a sub-key of 'specs'.
 func LoadNamedSpec(name, file string) error {

dev-tools/packaging/packages.yml

@@ -229,6 +229,78 @@ specs:
       spec:
         <<: *deb_rpm_spec
 
+  elastic_beat_without_xpack:
+    ###
+    # OSS Packages
+    ###
+    - os: windows
+      types: [zip]
+      spec:
+        <<: *windows_binary_spec
+        <<: *apache_license_for_binaries
+        name: '{{.BeatName}}-oss'
+
+    - os: darwin
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *apache_license_for_binaries
+        name: '{{.BeatName}}-oss'
+
+    - os: darwin
+      types: [dmg]
+      spec:
+        <<: *macos_beat_pkg_spec
+        <<: *apache_license_for_macos_pkg
+        name: '{{.BeatName}}-oss'
+
+    - os: linux
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *apache_license_for_binaries
+        name: '{{.BeatName}}-oss'
+
+    - os: linux
+      types: [deb, rpm]
+      spec:
+        <<: *deb_rpm_spec
+        <<: *apache_license_for_deb_rpm
+        name: '{{.BeatName}}-oss'
+
+    ###
+    # Elastic Licensed Packages
+    ###
+    - os: windows
+      types: [zip]
+      spec:
+        <<: *windows_binary_spec
+        <<: *elastic_license_for_binaries
+
+    - os: darwin
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *elastic_license_for_binaries
+
+    - os: darwin
+      types: [dmg]
+      spec:
+        <<: *macos_beat_pkg_spec
+        <<: *elastic_license_for_macos_pkg
+
+    - os: linux
+      types: [tgz]
+      spec:
+        <<: *binary_spec
+        <<: *elastic_license_for_binaries
+
+    - os: linux
+      types: [deb, rpm]
+      spec:
+        <<: *deb_rpm_spec
+        <<: *elastic_license_for_deb_rpm
+
   # Official Beats
   elastic_beat:
     ###
@@ -277,12 +349,17 @@ specs:
       spec:
         <<: *windows_binary_spec
         <<: *elastic_license_for_binaries
+        '{{.BeatName}}{{.BinaryExt}}':
+          source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
     - os: darwin
       types: [tgz]
       spec:
         <<: *binary_spec
         <<: *elastic_license_for_binaries
+        files:
+          /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
+            source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
     - os: darwin
       types: [dmg]
@@ -295,9 +372,15 @@ specs:
       spec:
         <<: *binary_spec
         <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
     - os: linux
       types: [deb, rpm]
       spec:
         <<: *deb_rpm_spec
         <<: *elastic_license_for_deb_rpm
+        files:
+          /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
+            source: ../x-pack/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

filebeat/magefile.go

@@ -57,6 +57,11 @@ func CrossBuild() error {
 	return mage.CrossBuild()
 }
 
+// CrossBuildXPack cross-builds the beat with XPack for all target platforms.
+func CrossBuildXPack() error {
+	return mage.CrossBuildXPack()
+}
+
 // CrossBuildGoDaemon cross-builds the go-daemon binary using Docker.
 func CrossBuildGoDaemon() error {
 	return mage.CrossBuildGoDaemon()
@@ -78,7 +83,7 @@ func Package() {
 	customizePackaging()
 
 	mg.Deps(Update, prepareModulePackaging)
-	mg.Deps(CrossBuild, CrossBuildGoDaemon)
+	mg.Deps(CrossBuild, CrossBuildXPack, CrossBuildGoDaemon)
 	mg.SerialDeps(mage.Package, TestPackages)
 }
 

heartbeat/magefile.go

@@ -57,6 +57,11 @@ func CrossBuild() error {
 	return mage.CrossBuild()
 }
 
+// CrossBuildXPack cross-builds the beat with XPack for all target platforms.
+func CrossBuildXPack() error {
+	return mage.CrossBuildXPack()
+}
+
 // CrossBuildGoDaemon cross-builds the go-daemon binary using Docker.
 func CrossBuildGoDaemon() error {
 	return mage.CrossBuildGoDaemon()
@@ -76,7 +81,7 @@ func Package() {
 
 	mage.UseElasticBeatPackaging()
 	mg.Deps(Update)
-	mg.Deps(CrossBuild, CrossBuildGoDaemon)
+	mg.Deps(CrossBuild, CrossBuildXPack, CrossBuildGoDaemon)
 	mg.SerialDeps(mage.Package, TestPackages)
 }
 

metricbeat/magefile.go

@@ -47,6 +47,11 @@ func GolangCrossBuild() error {
 	return mage.GolangCrossBuild(mage.DefaultGolangCrossBuildArgs())
 }
 
+// CrossBuildXPack cross-builds the beat with XPack for all target platforms.
+func CrossBuildXPack() error {
+	return mage.CrossBuildXPack()
+}
+
 // BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon).
 func BuildGoDaemon() error {
 	return mage.BuildGoDaemon()
@@ -78,7 +83,7 @@ func Package() {
 	customizePackaging()
 
 	mg.Deps(Update)
-	mg.Deps(CrossBuild, CrossBuildGoDaemon)
+	mg.Deps(CrossBuild, CrossBuildXPack, CrossBuildGoDaemon)
 	mg.SerialDeps(mage.Package, TestPackages)
 }