Ensure integration package validity using elastic-package tool (#4953)

* Use elastic-package Add Make targets: - make check-package - make format-package (run by `make gen-package`) - make build-package (run by `make gen-package`) Some minor formatting was needed, as well as adding a link to the changelog entry. Later we should consider getting rid of our custom README.template rendering, and rely on `elastic-package build` to render a template located at `_dev/build/docs/README.template`, like in integrations.
elastic · Mar 12, 2021 · 49aa419 · 49aa419
1 parent 9e35907
commit 49aa419
Show file tree

Hide file tree

Showing 23 changed files with 231 additions and 331 deletions.
diff --git a/Makefile b/Makefile
@@ -19,6 +19,7 @@ GOLINT=$(GOOSBUILD)/golint
 MAGE=$(GOOSBUILD)/mage
 REVIEWDOG=$(GOOSBUILD)/reviewdog
 STATICCHECK=$(GOOSBUILD)/staticcheck
+ELASTICPACKAGE=$(GOOSBUILD)/elastic-package
 
 PYTHON_ENV?=.
 PYTHON_BIN:=$(PYTHON_ENV)/build/ve/$(shell $(GO) env GOOS)/bin
@@ -78,13 +79,9 @@ check-approvals: $(APPROVALS)
 	@$(APPROVALS)
 
 .PHONY: check
-check: $(MAGE) check-fmt check-headers
+check: $(MAGE) check-fmt check-headers check-package
 	@$(MAGE) check
 
-.PHONY: gen-package
-gen-package: $(GENPACKAGE)
-	@$(GENPACKAGE)
-
 .PHONY: bench
 bench:
 	@$(GO) test -benchmem -run=XXX -benchtime=100ms -bench='.*' ./...
@@ -122,6 +119,11 @@ fields_sources=\
   $(shell find model -name fields.yml) \
   $(shell find x-pack/apm-server/fields -name fields.yml)
 
+.PHONY: gen-package gen-package-only
+gen-package: gen-package-only format-package build-package
+gen-package-only: $(GENPACKAGE)
+	@$(GENPACKAGE)
+
 fields: include/fields.go x-pack/apm-server/include/fields.go
 include/fields.go x-pack/apm-server/include/fields.go: $(MAGE) magefile.go $(fields_sources)
 	@$(MAGE) fields
@@ -231,6 +233,13 @@ ifndef CHECK_HEADERS_DISABLED
 	@$(GOLICENSER) -d -exclude build -license Elastic x-pack
 endif
 
+.PHONY: check-package format-package build-package
+check-package: $(ELASTICPACKAGE)
+	@for x in apmpackage/apm/*; do (cd $$x; echo "Checking $$x"; $(CURDIR)/$(ELASTICPACKAGE) check); done
+format-package: $(ELASTICPACKAGE)
+	@for x in apmpackage/apm/*; do (cd $$x; echo "Formatting $$x"; $(CURDIR)/$(ELASTICPACKAGE) format); done
+build-package: $(ELASTICPACKAGE)
+	@for x in apmpackage/apm/*; do (cd $$x; echo "Building $$x"; $(CURDIR)/$(ELASTICPACKAGE) build); done
 
 .PHONY: check-gofmt check-autopep8 gofmt autopep8
 check-fmt: check-gofmt check-autopep8
@@ -280,6 +289,9 @@ $(GOLICENSER): go.mod
 $(REVIEWDOG): go.mod
 	$(GO) build -o $@ github.com/reviewdog/reviewdog/cmd/reviewdog
 
+$(ELASTICPACKAGE): go.mod
+	$(GO) build -o $@ github.com/elastic/elastic-package
+
 $(PYTHON): $(PYTHON_BIN)
 $(PYTHON_BIN): $(PYTHON_BIN)/activate
 $(PYTHON_BIN)/activate: $(MAGE)

diff --git a/NOTICE.txt b/NOTICE.txt
diff --git a/...ge/apm/0.1.0/_dev/docs/README.template.md → apmpackage/apm/0.1.0/README.template.md b/...ge/apm/0.1.0/_dev/docs/README.template.md → apmpackage/apm/0.1.0/README.template.md
diff --git a/apmpackage/apm/0.1.0/changelog.yml b/apmpackage/apm/0.1.0/changelog.yml
@@ -3,3 +3,4 @@
   changes:
     - description: initial release
       type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/apm-server/
diff --git a/apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ilm/default_policy.json
@@ -23,4 +23,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/app_metrics/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/app_metrics/fields/ecs.yml
@@ -71,10 +71,7 @@
 - name: destination.address
   level: extended
   type: keyword
-  description: Some event destination addresses are defined ambiguously. The event
-    will sometimes list an IP, a domain or a unix socket.  You should always store
-    the raw address in the `.address` field. Then it should be duplicated to `.ip`
-    or `.domain`, depending on which one it is.
+  description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
   ignore_above: 1024
 - name: destination.ip
   level: core
@@ -111,10 +108,10 @@
     A flat mapping of user-defined labels with string, boolean or number values.
   dynamic: true
   object_type_params:
-  - object_type: keyword
-  - object_type: boolean
-  - object_type: scaled_float
-    scaling_factor: 1000000
+    - object_type: keyword
+    - object_type: boolean
+    - object_type: scaled_float
+      scaling_factor: 1000000
 - name: observer.hostname
   type: keyword
   description: |
@@ -185,8 +182,8 @@
   description: |
     Unparsed version of the user_agent.
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: user_agent.os.family
   type: keyword
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ilm/default_policy.json
@@ -23,4 +23,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/error_logs/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/error_logs/fields/ecs.yml
@@ -71,10 +71,7 @@
 - name: destination.address
   level: extended
   type: keyword
-  description: Some event destination addresses are defined ambiguously. The event
-    will sometimes list an IP, a domain or a unix socket.  You should always store
-    the raw address in the `.address` field. Then it should be duplicated to `.ip`
-    or `.domain`, depending on which one it is.
+  description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
   ignore_above: 1024
 - name: destination.ip
   level: core
@@ -131,10 +128,10 @@
     A flat mapping of user-defined labels with string, boolean or number values.
   dynamic: true
   object_type_params:
-  - object_type: keyword
-  - object_type: boolean
-  - object_type: scaled_float
-    scaling_factor: 1000000
+    - object_type: keyword
+    - object_type: boolean
+    - object_type: scaled_float
+      scaling_factor: 1000000
 - name: observer.hostname
   type: keyword
   description: |
@@ -241,8 +238,8 @@
   description: |
     Unparsed version of the user_agent.
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: user_agent.os.family
   type: keyword
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/error_logs/fields/fields.yml b/apmpackage/apm/0.1.0/data_stream/error_logs/fields/fields.yml
@@ -130,8 +130,8 @@
   description: |
     Generic designation of a transaction in the scope of a single service (eg. 'GET /users/:id').
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: transaction.sampled
   type: boolean
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ilm/default_policy.json
@@ -23,4 +23,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/ecs.yml
@@ -71,10 +71,7 @@
 - name: destination.address
   level: extended
   type: keyword
-  description: Some event destination addresses are defined ambiguously. The event
-    will sometimes list an IP, a domain or a unix socket.  You should always store
-    the raw address in the `.address` field. Then it should be duplicated to `.ip`
-    or `.domain`, depending on which one it is.
+  description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
   ignore_above: 1024
 - name: destination.ip
   level: core
@@ -117,10 +114,10 @@
     A flat mapping of user-defined labels with string, boolean or number values.
   dynamic: true
   object_type_params:
-  - object_type: keyword
-  - object_type: boolean
-  - object_type: scaled_float
-    scaling_factor: 1000000
+    - object_type: keyword
+    - object_type: boolean
+    - object_type: scaled_float
+      scaling_factor: 1000000
 - name: observer.hostname
   type: keyword
   description: |
@@ -195,8 +192,8 @@
   description: |
     Unparsed version of the user_agent.
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: user_agent.os.family
   type: keyword
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/fields.yml b/apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/fields.yml
@@ -155,8 +155,8 @@
   description: |
     Generic designation of a transaction in the scope of a single service (eg. 'GET /users/:id').
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: transaction.root
   type: boolean
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ilm/default_policy.json
@@ -23,4 +23,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/profile_metrics/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/profile_metrics/fields/ecs.yml
@@ -71,10 +71,7 @@
 - name: destination.address
   level: extended
   type: keyword
-  description: Some event destination addresses are defined ambiguously. The event
-    will sometimes list an IP, a domain or a unix socket.  You should always store
-    the raw address in the `.address` field. Then it should be duplicated to `.ip`
-    or `.domain`, depending on which one it is.
+  description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
   ignore_above: 1024
 - name: destination.ip
   level: core
@@ -111,10 +108,10 @@
     A flat mapping of user-defined labels with string, boolean or number values.
   dynamic: true
   object_type_params:
-  - object_type: keyword
-  - object_type: boolean
-  - object_type: scaled_float
-    scaling_factor: 1000000
+    - object_type: keyword
+    - object_type: boolean
+    - object_type: scaled_float
+      scaling_factor: 1000000
 - name: observer.hostname
   type: keyword
   description: |
@@ -185,8 +182,8 @@
   description: |
     Unparsed version of the user_agent.
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: user_agent.os.family
   type: keyword
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ilm/default_policy.json
@@ -16,4 +16,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ilm/default_policy.json b/apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ilm/default_policy.json
@@ -23,4 +23,4 @@
             }
         }
     }
-}
+}
diff --git a/apmpackage/apm/0.1.0/data_stream/traces/fields/ecs.yml b/apmpackage/apm/0.1.0/data_stream/traces/fields/ecs.yml
@@ -71,10 +71,7 @@
 - name: destination.address
   level: extended
   type: keyword
-  description: Some event destination addresses are defined ambiguously. The event
-    will sometimes list an IP, a domain or a unix socket.  You should always store
-    the raw address in the `.address` field. Then it should be duplicated to `.ip`
-    or `.domain`, depending on which one it is.
+  description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
   ignore_above: 1024
 - name: destination.ip
   level: core
@@ -133,10 +130,10 @@
     A flat mapping of user-defined labels with string, boolean or number values.
   dynamic: true
   object_type_params:
-  - object_type: keyword
-  - object_type: boolean
-  - object_type: scaled_float
-    scaling_factor: 1000000
+    - object_type: keyword
+    - object_type: boolean
+    - object_type: scaled_float
+      scaling_factor: 1000000
 - name: observer.hostname
   type: keyword
   description: |
@@ -247,8 +244,8 @@
   description: |
     Unparsed version of the user_agent.
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: user_agent.os.family
   type: keyword
   description: |

diff --git a/apmpackage/apm/0.1.0/data_stream/traces/fields/fields.yml b/apmpackage/apm/0.1.0/data_stream/traces/fields/fields.yml
@@ -191,8 +191,8 @@
   description: |
     Generic designation of a transaction in the scope of a single service (eg. 'GET /users/:id').
   multi_fields:
-  - name: text
-    type: text
+    - name: text
+      type: text
 - name: transaction.result
   type: keyword
   description: |

diff --git a/apmpackage/cmd/gen-package/main.go b/apmpackage/cmd/gen-package/main.go
@@ -54,6 +54,7 @@ func main() {
 			log.Fatal(err)
 		}
 	}
+	// TODO(axw) rely on `elastic-package build` to build docs from a template, like in integrations.
 	generateDocs(inputFields, packageVersion)
 	log.Printf("Package fields and docs generated for version %s (stack %s)", packageVersion, stackVersion.String())
 }

diff --git a/apmpackage/cmd/gen-package/paths.go b/apmpackage/cmd/gen-package/paths.go
@@ -20,23 +20,23 @@ package main
 import "path/filepath"
 
 func docsTemplateFilePath(version string) string {
-	return filepath.Join("apmpackage/apm/", version, "/_dev/docs/README.template.md")
+	return filepath.Join("apmpackage", "apm", version, "README.template.md")
 }
 
 func docsFilePath(version string) string {
-	return filepath.Join("apmpackage/apm/", version, "/docs/README.md")
+	return filepath.Join("apmpackage", "apm", version, "docs/README.md")
 }
 
 func pipelinesPath(version, dataStream string) string {
-	return filepath.Join("apmpackage/apm/", version, "/data_stream/", dataStream, "/elasticsearch/ingest_pipeline/")
+	return filepath.Join("apmpackage", "apm", version, "data_stream", dataStream, "elasticsearch", "ingest_pipeline")
 }
 
 func dataStreamPath(version string) string {
-	return filepath.Join("apmpackage/apm/", version, "/data_stream/")
+	return filepath.Join("apmpackage", "apm", version, "data_stream")
 }
 
 func fieldsPath(version, dataStream string) string {
-	return filepath.Join(dataStreamPath(version), dataStream, "fields/")
+	return filepath.Join(dataStreamPath(version), dataStream, "fields")
 }
 
 func ecsFilePath(version, dataStream string) string {

diff --git a/go.mod b/go.mod
@@ -15,8 +15,9 @@ require (
 	github.com/elastic/apm-server/approvaltest v0.0.0-00010101000000-000000000000
 	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20210309205547-21d02a3ab3b8
 	github.com/elastic/ecs v1.8.0
+	github.com/elastic/elastic-package v0.0.0-20210310173719-3b8f76516ae3
 	github.com/elastic/gmux v0.1.0
-	github.com/elastic/go-elasticsearch/v7 v7.5.1-0.20201007132508-ff965d99ba02
+	github.com/elastic/go-elasticsearch/v7 v7.11.0
 	github.com/elastic/go-elasticsearch/v8 v8.0.0-20201007143536-4b4020669208
 	github.com/elastic/go-hdrhistogram v0.1.0
 	github.com/elastic/go-licenser v0.3.1
@@ -50,7 +51,7 @@ require (
 	github.com/tidwall/gjson v1.6.0
 	github.com/tidwall/sjson v1.1.1
 	github.com/urso/magetools v0.0.0-20200125210132-c2e338f92f3a // indirect
-	github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56
+	github.com/xeipuuv/gojsonschema v1.2.0
 	go.elastic.co/apm v1.11.0
 	go.elastic.co/apm/module/apmelasticsearch v1.7.2
 	go.elastic.co/apm/module/apmgrpc v1.7.0
-Original file line number
+Diff line change
@@ Expand Up / @@ -23,4 +23,4 @@ @@
                 }
             }
         }
-    }
+    }