[7.x] Ensure integration package validity using elastic-package tool (#…

…4953) (#5017) * Ensure integration package validity using elastic-package tool (#4953) * Use elastic-package Add Make targets: - make check-package - make format-package (run by `make gen-package`) - make build-package (run by `make gen-package`) Some minor formatting was needed, as well as adding a link to the changelog entry. Later we should consider getting rid of our custom README.template rendering, and rely on `elastic-package build` to render a template located at `_dev/build/docs/README.template`, like in integrations. # Conflicts: # NOTICE.txt # go.sum * Update go.sum
elastic · Mar 29, 2021 · 231cd76 · 231cd76
1 parent d19d302
commit 231cd76
Show file tree

Hide file tree

Showing 23 changed files with 155 additions and 140 deletions.
diff --git a/Makefile b/Makefile
@@ -19,6 +19,7 @@ GOLINT=$(GOOSBUILD)/golint
 MAGE=$(GOOSBUILD)/mage
 REVIEWDOG=$(GOOSBUILD)/reviewdog
 STATICCHECK=$(GOOSBUILD)/staticcheck
+ELASTICPACKAGE=$(GOOSBUILD)/elastic-package
 
 PYTHON_ENV?=.
 PYTHON_BIN:=$(PYTHON_ENV)/build/ve/$(shell $(GO) env GOOS)/bin
@@ -78,13 +79,9 @@ check-approvals: $(APPROVALS)
 	@$(APPROVALS)
 
 .PHONY: check
-check: $(MAGE) check-fmt check-headers
+check: $(MAGE) check-fmt check-headers check-package
 	@$(MAGE) check
 
-.PHONY: gen-package
-gen-package: $(GENPACKAGE)
-	@$(GENPACKAGE)
-
 .PHONY: bench
 bench:
 	@$(GO) test -benchmem -run=XXX -benchtime=100ms -bench='.*' ./...
@@ -122,6 +119,11 @@ fields_sources=\
   $(shell find model -name fields.yml) \
   $(shell find x-pack/apm-server/fields -name fields.yml)
 
+.PHONY: gen-package gen-package-only
+gen-package: gen-package-only format-package build-package
+gen-package-only: $(GENPACKAGE)
+	@$(GENPACKAGE)
+
 fields: include/fields.go x-pack/apm-server/include/fields.go
 include/fields.go x-pack/apm-server/include/fields.go: $(MAGE) magefile.go $(fields_sources)
 	@$(MAGE) fields
@@ -231,6 +233,13 @@ ifndef CHECK_HEADERS_DISABLED
 	@$(GOLICENSER) -d -exclude build -license Elastic x-pack
 endif
 
+.PHONY: check-package format-package build-package
+check-package: $(ELASTICPACKAGE)
+	@for x in apmpackage/apm/*; do (cd $$x; echo "Checking $$x"; $(CURDIR)/$(ELASTICPACKAGE) check); done
+format-package: $(ELASTICPACKAGE)
+	@for x in apmpackage/apm/*; do (cd $$x; echo "Formatting $$x"; $(CURDIR)/$(ELASTICPACKAGE) format); done
+build-package: $(ELASTICPACKAGE)
+	@for x in apmpackage/apm/*; do (cd $$x; echo "Building $$x"; $(CURDIR)/$(ELASTICPACKAGE) build); done
 
 .PHONY: check-gofmt check-autopep8 gofmt autopep8
 check-fmt: check-gofmt check-autopep8
@@ -280,6 +289,9 @@ $(GOLICENSER): go.mod
 $(REVIEWDOG): go.mod
 	$(GO) build -o $@ github.com/reviewdog/reviewdog/cmd/reviewdog
 
+$(ELASTICPACKAGE): go.mod
+	$(GO) build -o $@ github.com/elastic/elastic-package
+
 $(PYTHON): $(PYTHON_BIN)
 $(PYTHON_BIN): $(PYTHON_BIN)/activate
 $(PYTHON_BIN)/activate: $(MAGE)

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -955,11 +955,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/gmux@v0.1.0/LIC
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-elasticsearch/v7
-Version: v7.5.1-0.20201007132508-ff965d99ba02
+Version: v7.11.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v7@v7.5.1-0.20201007132508-ff965d99ba02/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v7@v7.11.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -5811,14 +5811,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Masterminds/semver
-Version: v1.4.2
+Version: v1.5.0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!masterminds/semver@v1.4.2/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/!masterminds/semver@v1.5.0/LICENSE.txt:
 
-The Masterminds
-Copyright (C) 2014-2015, Matt Butcher and Matt Farina
+Copyright (C) 2014-2019, Matt Butcher and Matt Farina
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -13974,11 +13973,11 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
 
 --------------------------------------------------------------------------------
 Dependency : github.com/imdario/mergo
-Version: v0.3.6
+Version: v0.3.11
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/imdario/mergo@v0.3.6/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/imdario/mergo@v0.3.11/LICENSE:
 
 Copyright (c) 2013 Dario Castañé. All rights reserved.
 Copyright (c) 2012 The Go Authors. All rights reserved.

diff --git a/...ge/apm/0.1.0/_dev/docs/README.template.md → apmpackage/apm/0.1.0/README.template.md b/...ge/apm/0.1.0/_dev/docs/README.template.md → apmpackage/apm/0.1.0/README.template.md
diff --git a/apmpackage/apm/0.1.0/changelog.yml b/apmpackage/apm/0.1.0/changelog.yml
@@ -3,3 +3,4 @@
   changes:
     - description: initial release
       type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/apm-server/
diff --git a/apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ilm/default_policy.json
@@ -23,4 +23,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/app_metrics/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/app_metrics/fields/ecs.yml
@@ -71,10 +71,7 @@
 - name: destination.address
   level: extended
   type: keyword
-  description: Some event destination addresses are defined ambiguously. The event
-    will sometimes list an IP, a domain or a unix socket.  You should always store
-    the raw address in the `.address` field. Then it should be duplicated to `.ip`
-    or `.domain`, depending on which one it is.
+  description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
   ignore_above: 1024
 - name: destination.ip
   level: core
@@ -111,10 +108,10 @@
     A flat mapping of user-defined labels with string, boolean or number values.
   dynamic: true
   object_type_params:
-  - object_type: keyword
-  - object_type: boolean
-  - object_type: scaled_float
-    scaling_factor: 1000000
+    - object_type: keyword
+    - object_type: boolean
+    - object_type: scaled_float
+      scaling_factor: 1000000
 - name: observer.hostname
   type: keyword
   description: |
@@ -185,8 +182,8 @@
   description: |
     Unparsed version of the user_agent.
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: user_agent.os.family
   type: keyword
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ilm/default_policy.json
@@ -23,4 +23,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/error_logs/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/error_logs/fields/ecs.yml
@@ -71,10 +71,7 @@
 - name: destination.address
   level: extended
   type: keyword
-  description: Some event destination addresses are defined ambiguously. The event
-    will sometimes list an IP, a domain or a unix socket.  You should always store
-    the raw address in the `.address` field. Then it should be duplicated to `.ip`
-    or `.domain`, depending on which one it is.
+  description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
   ignore_above: 1024
 - name: destination.ip
   level: core
@@ -131,10 +128,10 @@
     A flat mapping of user-defined labels with string, boolean or number values.
   dynamic: true
   object_type_params:
-  - object_type: keyword
-  - object_type: boolean
-  - object_type: scaled_float
-    scaling_factor: 1000000
+    - object_type: keyword
+    - object_type: boolean
+    - object_type: scaled_float
+      scaling_factor: 1000000
 - name: observer.hostname
   type: keyword
   description: |
@@ -241,8 +238,8 @@
   description: |
     Unparsed version of the user_agent.
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: user_agent.os.family
   type: keyword
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/error_logs/fields/fields.yml b/apmpackage/apm/0.1.0/data_stream/error_logs/fields/fields.yml
@@ -130,8 +130,8 @@
   description: |
     Generic designation of a transaction in the scope of a single service (eg. 'GET /users/:id').
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: transaction.sampled
   type: boolean
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ilm/default_policy.json
@@ -23,4 +23,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/ecs.yml
@@ -71,10 +71,7 @@
 - name: destination.address
   level: extended
   type: keyword
-  description: Some event destination addresses are defined ambiguously. The event
-    will sometimes list an IP, a domain or a unix socket.  You should always store
-    the raw address in the `.address` field. Then it should be duplicated to `.ip`
-    or `.domain`, depending on which one it is.
+  description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
   ignore_above: 1024
 - name: destination.ip
   level: core
@@ -117,10 +114,10 @@
     A flat mapping of user-defined labels with string, boolean or number values.
   dynamic: true
   object_type_params:
-  - object_type: keyword
-  - object_type: boolean
-  - object_type: scaled_float
-    scaling_factor: 1000000
+    - object_type: keyword
+    - object_type: boolean
+    - object_type: scaled_float
+      scaling_factor: 1000000
 - name: observer.hostname
   type: keyword
   description: |
@@ -195,8 +192,8 @@
   description: |
     Unparsed version of the user_agent.
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: user_agent.os.family
   type: keyword
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/fields.yml b/apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/fields.yml
@@ -155,8 +155,8 @@
   description: |
     Generic designation of a transaction in the scope of a single service (eg. 'GET /users/:id').
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: transaction.root
   type: boolean
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ilm/default_policy.json
@@ -23,4 +23,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/profile_metrics/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/profile_metrics/fields/ecs.yml
@@ -71,10 +71,7 @@
 - name: destination.address
   level: extended
   type: keyword
-  description: Some event destination addresses are defined ambiguously. The event
-    will sometimes list an IP, a domain or a unix socket.  You should always store
-    the raw address in the `.address` field. Then it should be duplicated to `.ip`
-    or `.domain`, depending on which one it is.
+  description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
   ignore_above: 1024
 - name: destination.ip
   level: core
@@ -111,10 +108,10 @@
     A flat mapping of user-defined labels with string, boolean or number values.
   dynamic: true
   object_type_params:
-  - object_type: keyword
-  - object_type: boolean
-  - object_type: scaled_float
-    scaling_factor: 1000000
+    - object_type: keyword
+    - object_type: boolean
+    - object_type: scaled_float
+      scaling_factor: 1000000
 - name: observer.hostname
   type: keyword
   description: |
@@ -185,8 +182,8 @@
   description: |
     Unparsed version of the user_agent.
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: user_agent.os.family
   type: keyword
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ilm/default_policy.json
@@ -16,4 +16,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ilm/default_policy.json
@@ -23,4 +23,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/traces/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/traces/fields/ecs.yml
@@ -71,10 +71,7 @@
 - name: destination.address
   level: extended
   type: keyword
-  description: Some event destination addresses are defined ambiguously. The event
-    will sometimes list an IP, a domain or a unix socket.  You should always store
-    the raw address in the `.address` field. Then it should be duplicated to `.ip`
-    or `.domain`, depending on which one it is.
+  description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
   ignore_above: 1024
 - name: destination.ip
   level: core
@@ -133,10 +130,10 @@
     A flat mapping of user-defined labels with string, boolean or number values.
   dynamic: true
   object_type_params:
-  - object_type: keyword
-  - object_type: boolean
-  - object_type: scaled_float
-    scaling_factor: 1000000
+    - object_type: keyword
+    - object_type: boolean
+    - object_type: scaled_float
+      scaling_factor: 1000000
 - name: observer.hostname
   type: keyword
   description: |
@@ -247,8 +244,8 @@
   description: |
     Unparsed version of the user_agent.
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: user_agent.os.family
   type: keyword
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/traces/fields/fields.yml b/apmpackage/apm/0.1.0/data_stream/traces/fields/fields.yml
@@ -191,8 +191,8 @@
   description: |
     Generic designation of a transaction in the scope of a single service (eg. 'GET /users/:id').
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: transaction.result
   type: keyword
   description: |

diff --git a/apmpackage/cmd/gen-package/main.go b/apmpackage/cmd/gen-package/main.go
@@ -56,6 +56,7 @@ func main() {
 			log.Fatal(err)
 		}
 	}
+	// TODO(axw) rely on `elastic-package build` to build docs from a template, like in integrations.
 	generateDocs(inputFields, packageVersion)
 	log.Printf("Package fields and docs generated for version %s (stack %s)", packageVersion, stackVersion.String())
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -23,4 +23,4 @@ @@
                 }
             }
         }
-    }
+    }