diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7c5a9ee1112..b62b9187856 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -357,6 +357,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix timeout option of GCP functions. {issue}16282[16282] {pull}16287[16287] - Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072] +- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070] ==== Added diff --git a/dev-tools/mage/gotool/go.go b/dev-tools/mage/gotool/go.go index 9022e0c0137..bb7066a4f3e 100644 --- a/dev-tools/mage/gotool/go.go +++ b/dev-tools/mage/gotool/go.go @@ -85,6 +85,25 @@ func ListDeps(pkg string) ([]string, error) { return getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg)) } +// ListDepsLocation calls `go list -dep` for every package spec given. +func ListDepsLocation(pkg string) (map[string]string, error) { + const tmpl = `{{if not .Standard}}{{.ImportPath}};{{.Dir}}{{end}}` + + lines, err := getLines(callGo(nil, "list", "-deps", "-f", tmpl, pkg)) + if err != nil { + return nil, err + } + deps := make(map[string]string, len(lines)) + for _, l := range lines { + parts := strings.Split(l, ";") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid number of parts") + } + deps[parts[0]] = parts[1] + } + return deps, nil +} + // ListTestFiles lists all go and cgo test files available in a package. func ListTestFiles(pkg string) ([]string, error) { const tmpl = `{{ range .TestGoFiles }}{{ printf "%s\n" . }}{{ end }}` + diff --git a/x-pack/functionbeat/dev-tools/packaging/packages.yml b/x-pack/functionbeat/dev-tools/packaging/packages.yml index 47379c8793e..e6c3346bf1a 100644 --- a/x-pack/functionbeat/dev-tools/packaging/packages.yml +++ b/x-pack/functionbeat/dev-tools/packaging/packages.yml @@ -65,6 +65,12 @@ shared: pkg/functionbeat-aws: source: 'provider/aws/build/golang-crossbuild/aws-linux-amd64' mode: 0755 + pkg/pubsub/vendor: + source: 'provider/gcp/build/pubsub/vendor' + mode: 0644 + pkg/storage/vendor: + source: 'provider/gcp/build/storage/vendor' + mode: 0644 pkg/pubsub/pubsub.go: source: 'provider/gcp/pubsub/pubsub.go' mode: 0655 diff --git a/x-pack/functionbeat/magefile.go b/x-pack/functionbeat/magefile.go index e1982b66400..a317bd0cb71 100644 --- a/x-pack/functionbeat/magefile.go +++ b/x-pack/functionbeat/magefile.go @@ -159,9 +159,11 @@ func BuildPkgForFunctions() error { err := os.RemoveAll("pkg") filesToCopy := map[string]string{ - filepath.Join("provider", "aws", "functionbeat-aws"): filepath.Join("pkg", "functionbeat-aws"), - filepath.Join("provider", "gcp", "pubsub", "pubsub.go"): filepath.Join("pkg", "pubsub", "pubsub.go"), - filepath.Join("provider", "gcp", "storage", "storage.go"): filepath.Join("pkg", "storage", "storage.go"), + filepath.Join("provider", "aws", "functionbeat-aws"): filepath.Join("pkg", "functionbeat-aws"), + filepath.Join("provider", "gcp", "pubsub", "pubsub.go"): filepath.Join("pkg", "pubsub", "pubsub.go"), + filepath.Join("provider", "gcp", "storage", "storage.go"): filepath.Join("pkg", "storage", "storage.go"), + filepath.Join("provider", "gcp", "build", "pubsub", "vendor"): filepath.Join("pkg", "pubsub", "vendor"), + filepath.Join("provider", "gcp", "build", "storage", "vendor"): filepath.Join("pkg", "storage", "vendor"), } for src, dest := range filesToCopy { c := &devtools.CopyTask{ diff --git a/x-pack/functionbeat/manager/gcp/template_builder.go b/x-pack/functionbeat/manager/gcp/template_builder.go index da468abee30..c6f1292ce15 100644 --- a/x-pack/functionbeat/manager/gcp/template_builder.go +++ b/x-pack/functionbeat/manager/gcp/template_builder.go @@ -190,5 +190,6 @@ func zipResources() map[string][]bundle.Resource { func zipResourcesOfFunc(typeName string) []bundle.Resource { root := filepath.Join("pkg", typeName) - return []bundle.Resource{&bundle.LocalFile{Path: filepath.Join(root, typeName+".go"), FileMode: 0755}} + vendor := bundle.Folder(filepath.Join("pkg", typeName, "vendor"), filepath.Join("pkg", typeName), 0644) + return append(vendor, &bundle.LocalFile{Path: filepath.Join(root, typeName+".go"), FileMode: 0755}) } diff --git a/x-pack/functionbeat/scripts/mage/update.go b/x-pack/functionbeat/scripts/mage/update.go index 468bdafbe0f..59b56cb6bed 100644 --- a/x-pack/functionbeat/scripts/mage/update.go +++ b/x-pack/functionbeat/scripts/mage/update.go @@ -5,9 +5,13 @@ package mage import ( + "os" + "path/filepath" + "github.com/magefile/mage/mg" devtools "github.com/elastic/beats/v7/dev-tools/mage" + "github.com/elastic/beats/v7/dev-tools/mage/gotool" ) // Update target namespace. @@ -20,7 +24,7 @@ var Aliases = map[string]interface{}{ // All updates all generated content. func (Update) All() { - mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs) + mg.Deps(Update.Fields, Update.IncludeFields, Update.Config, Update.FieldDocs, Update.VendorBeats) } // Config generates both the short and reference configs. @@ -46,3 +50,39 @@ func (Update) IncludeFields() error { return devtools.GenerateAllInOneFieldsGo() } + +// VendorBeats collects the vendor folder required to deploy the function for GCP. +func (Update) VendorBeats() error { + for _, f := range []string{"pubsub", "storage"} { + gcpVendorPath := filepath.Join("provider", "gcp", "build", f, "vendor") + err := os.RemoveAll(gcpVendorPath) + if err != nil { + return err + } + + deps, err := gotool.ListDepsLocation("github.com/elastic/beats/v7/x-pack/functionbeat/provider/gcp/" + f) + if err != nil { + return err + } + + for importPath, location := range deps { + cp := &devtools.CopyTask{ + Source: location, + Dest: filepath.Join(gcpVendorPath, importPath), + Mode: 0600, + DirMode: os.ModeDir | 0750, + Exclude: []string{ + ".*_test.go$", + ".*.yml", + }, + } + err = cp.Execute() + if err != nil { + return err + } + } + + } + + return nil +}