Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ensure integration package validity using elastic-package tool #4953

Merged
merged 2 commits into from
Mar 12, 2021
Merged

Ensure integration package validity using elastic-package tool #4953

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6ef8efe
Use elastic-package
axw Mar 11, 2021
aa8b6f8
Update Makefile
axw Mar 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 17 additions & 5 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ GOLINT=$(GOOSBUILD)/golint
MAGE=$(GOOSBUILD)/mage
REVIEWDOG=$(GOOSBUILD)/reviewdog
STATICCHECK=$(GOOSBUILD)/staticcheck
ELASTICPACKAGE=$(GOOSBUILD)/elastic-package

PYTHON_ENV?=.
PYTHON_BIN:=$(PYTHON_ENV)/build/ve/$(shell $(GO) env GOOS)/bin
Expand Down Expand Up @@ -78,13 +79,9 @@ check-approvals: $(APPROVALS)
@$(APPROVALS)

.PHONY: check
check: $(MAGE) check-fmt check-headers
check: $(MAGE) check-fmt check-headers check-package
@$(MAGE) check

.PHONY: gen-package
gen-package: $(GENPACKAGE)
@$(GENPACKAGE)

.PHONY: bench
bench:
@$(GO) test -benchmem -run=XXX -benchtime=100ms -bench='.*' ./...
Expand Down Expand Up @@ -122,6 +119,11 @@ fields_sources=\
$(shell find model -name fields.yml) \
$(shell find x-pack/apm-server/fields -name fields.yml)

.PHONY: gen-package gen-package-only
gen-package: gen-package-only format-package build-package
gen-package-only: $(GENPACKAGE)
@$(GENPACKAGE)

fields: include/fields.go x-pack/apm-server/include/fields.go
include/fields.go x-pack/apm-server/include/fields.go: $(MAGE) magefile.go $(fields_sources)
@$(MAGE) fields
Expand Down Expand Up @@ -231,6 +233,13 @@ ifndef CHECK_HEADERS_DISABLED
@$(GOLICENSER) -d -exclude build -license Elastic x-pack
endif

.PHONY: check-package format-package build-package
check-package: $(ELASTICPACKAGE)
@for x in apmpackage/apm/*; do (cd $$x; echo "Checking $$x"; $(CURDIR)/$(ELASTICPACKAGE) check); done
format-package: $(ELASTICPACKAGE)
@for x in apmpackage/apm/*; do (cd $$x; echo "Formatting $$x"; $(CURDIR)/$(ELASTICPACKAGE) format); done
build-package: $(ELASTICPACKAGE)
@for x in apmpackage/apm/*; do (cd $$x; echo "Formatting $$x"; $(CURDIR)/$(ELASTICPACKAGE) build); done
axw marked this conversation as resolved.
Show resolved Hide resolved

.PHONY: check-gofmt check-autopep8 gofmt autopep8
check-fmt: check-gofmt check-autopep8
Expand Down Expand Up @@ -280,6 +289,9 @@ $(GOLICENSER): go.mod
$(REVIEWDOG): go.mod
$(GO) build -o $@ github.com/reviewdog/reviewdog/cmd/reviewdog

$(ELASTICPACKAGE): go.mod
$(GO) build -o $@ github.com/elastic/elastic-package

$(PYTHON): $(PYTHON_BIN)
$(PYTHON_BIN): $(PYTHON_BIN)/activate
$(PYTHON_BIN)/activate: $(MAGE)
Expand Down
328 changes: 73 additions & 255 deletions NOTICE.txt
View file
Open in desktop

Large diffs are not rendered by default.

0 ...ge/apm/0.1.0/_dev/docs/README.template.md → apmpackage/apm/0.1.0/README.template.md
View file
Open in desktop
File renamed without changes.
1 change: 1 addition & 0 deletions apmpackage/apm/0.1.0/changelog.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
changes:
- description: initial release
type: enhancement # can be one of: enhancement, bugfix, breaking-change
link: https://github.com/elastic/apm-server/
2 changes: 1 addition & 1 deletion apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ilm/default_policy.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@
}
}
}
}
}
17 changes: 7 additions & 10 deletions apmpackage/apm/0.1.0/data_stream/app_metrics/fields/ecs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,7 @@
- name: destination.address
level: extended
type: keyword
description: Some event destination addresses are defined ambiguously. The event
will sometimes list an IP, a domain or a unix socket. You should always store
the raw address in the `.address` field. Then it should be duplicated to `.ip`
or `.domain`, depending on which one it is.
description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
ignore_above: 1024
- name: destination.ip
level: core
Expand Down Expand Up @@ -111,10 +108,10 @@
A flat mapping of user-defined labels with string, boolean or number values.
dynamic: true
object_type_params:
- object_type: keyword
- object_type: boolean
- object_type: scaled_float
scaling_factor: 1000000
- object_type: keyword
- object_type: boolean
- object_type: scaled_float
scaling_factor: 1000000
- name: observer.hostname
type: keyword
description: |
Expand Down Expand Up @@ -185,8 +182,8 @@
description: |
Unparsed version of the user_agent.
multi_fields:
- name: text
type: text
- name: text
type: text
- name: user_agent.os.family
type: keyword
description: |
Expand Down
2 changes: 1 addition & 1 deletion apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ilm/default_policy.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@
}
}
}
}
}
17 changes: 7 additions & 10 deletions apmpackage/apm/0.1.0/data_stream/error_logs/fields/ecs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,7 @@
- name: destination.address
level: extended
type: keyword
description: Some event destination addresses are defined ambiguously. The event
will sometimes list an IP, a domain or a unix socket. You should always store
the raw address in the `.address` field. Then it should be duplicated to `.ip`
or `.domain`, depending on which one it is.
description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
ignore_above: 1024
- name: destination.ip
level: core
Expand Down Expand Up @@ -131,10 +128,10 @@
A flat mapping of user-defined labels with string, boolean or number values.
dynamic: true
object_type_params:
- object_type: keyword
- object_type: boolean
- object_type: scaled_float
scaling_factor: 1000000
- object_type: keyword
- object_type: boolean
- object_type: scaled_float
scaling_factor: 1000000
- name: observer.hostname
type: keyword
description: |
Expand Down Expand Up @@ -241,8 +238,8 @@
description: |
Unparsed version of the user_agent.
multi_fields:
- name: text
type: text
- name: text
type: text
- name: user_agent.os.family
type: keyword
description: |
Expand Down
4 changes: 2 additions & 2 deletions apmpackage/apm/0.1.0/data_stream/error_logs/fields/fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,8 @@
description: |
Generic designation of a transaction in the scope of a single service (eg. 'GET /users/:id').
multi_fields:
- name: text
type: text
- name: text
type: text
- name: transaction.sampled
type: boolean
description: |
Expand Down
2 changes: 1 addition & 1 deletion apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ilm/default_policy.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@
}
}
}
}
}
17 changes: 7 additions & 10 deletions apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/ecs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,7 @@
- name: destination.address
level: extended
type: keyword
description: Some event destination addresses are defined ambiguously. The event
will sometimes list an IP, a domain or a unix socket. You should always store
the raw address in the `.address` field. Then it should be duplicated to `.ip`
or `.domain`, depending on which one it is.
description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
ignore_above: 1024
- name: destination.ip
level: core
Expand Down Expand Up @@ -117,10 +114,10 @@
A flat mapping of user-defined labels with string, boolean or number values.
dynamic: true
object_type_params:
- object_type: keyword
- object_type: boolean
- object_type: scaled_float
scaling_factor: 1000000
- object_type: keyword
- object_type: boolean
- object_type: scaled_float
scaling_factor: 1000000
- name: observer.hostname
type: keyword
description: |
Expand Down Expand Up @@ -195,8 +192,8 @@
description: |
Unparsed version of the user_agent.
multi_fields:
- name: text
type: text
- name: text
type: text
- name: user_agent.os.family
type: keyword
description: |
Expand Down
4 changes: 2 additions & 2 deletions apmpackage/apm/0.1.0/data_stream/internal_metrics/fields/fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,8 @@
description: |
Generic designation of a transaction in the scope of a single service (eg. 'GET /users/:id').
multi_fields:
- name: text
type: text
- name: text
type: text
- name: transaction.root
type: boolean
description: |
Expand Down
2 changes: 1 addition & 1 deletion apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ilm/default_policy.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@
}
}
}
}
}
17 changes: 7 additions & 10 deletions apmpackage/apm/0.1.0/data_stream/profile_metrics/fields/ecs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,7 @@
- name: destination.address
level: extended
type: keyword
description: Some event destination addresses are defined ambiguously. The event
will sometimes list an IP, a domain or a unix socket. You should always store
the raw address in the `.address` field. Then it should be duplicated to `.ip`
or `.domain`, depending on which one it is.
description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
ignore_above: 1024
- name: destination.ip
level: core
Expand Down Expand Up @@ -111,10 +108,10 @@
A flat mapping of user-defined labels with string, boolean or number values.
dynamic: true
object_type_params:
- object_type: keyword
- object_type: boolean
- object_type: scaled_float
scaling_factor: 1000000
- object_type: keyword
- object_type: boolean
- object_type: scaled_float
scaling_factor: 1000000
- name: observer.hostname
type: keyword
description: |
Expand Down Expand Up @@ -185,8 +182,8 @@
description: |
Unparsed version of the user_agent.
multi_fields:
- name: text
type: text
- name: text
type: text
- name: user_agent.os.family
type: keyword
description: |
Expand Down
2 changes: 1 addition & 1 deletion apmpackage/apm/0.1.0/data_stream/sampled_traces/elasticsearch/ilm/default_policy.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@
}
}
}
}
}
2 changes: 1 addition & 1 deletion apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ilm/default_policy.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@
}
}
}
}
}
17 changes: 7 additions & 10 deletions apmpackage/apm/0.1.0/data_stream/traces/fields/ecs.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,7 @@
- name: destination.address
level: extended
type: keyword
description: Some event destination addresses are defined ambiguously. The event
will sometimes list an IP, a domain or a unix socket. You should always store
the raw address in the `.address` field. Then it should be duplicated to `.ip`
or `.domain`, depending on which one it is.
description: Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
ignore_above: 1024
- name: destination.ip
level: core
Expand Down Expand Up @@ -133,10 +130,10 @@
A flat mapping of user-defined labels with string, boolean or number values.
dynamic: true
object_type_params:
- object_type: keyword
- object_type: boolean
- object_type: scaled_float
scaling_factor: 1000000
- object_type: keyword
- object_type: boolean
- object_type: scaled_float
scaling_factor: 1000000
- name: observer.hostname
type: keyword
description: |
Expand Down Expand Up @@ -247,8 +244,8 @@
description: |
Unparsed version of the user_agent.
multi_fields:
- name: text
type: text
- name: text
type: text
- name: user_agent.os.family
type: keyword
description: |
Expand Down
4 changes: 2 additions & 2 deletions apmpackage/apm/0.1.0/data_stream/traces/fields/fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,8 @@
description: |
Generic designation of a transaction in the scope of a single service (eg. 'GET /users/:id').
multi_fields:
- name: text
type: text
- name: text
type: text
- name: transaction.result
type: keyword
description: |
Expand Down
1 change: 1 addition & 0 deletions apmpackage/cmd/gen-package/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ func main() {
log.Fatal(err)
}
}
// TODO(axw) rely on `elastic-package build` to build docs from a template, like in integrations.
generateDocs(inputFields, packageVersion)
log.Printf("Package fields and docs generated for version %s (stack %s)", packageVersion, stackVersion.String())
}
Expand Down
10 changes: 5 additions & 5 deletions apmpackage/cmd/gen-package/paths.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,23 +20,23 @@ package main
import "path/filepath"

func docsTemplateFilePath(version string) string {
return filepath.Join("apmpackage/apm/", version, "/_dev/docs/README.template.md")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmmm... I thought that _dev was a convention for things that don't actually belong to the published package.
Probably too far fetched, but I worry a bit about this change. Imagine eg. that instead of using README.md Kibana starts reading all *.md files and show only the first one, or concats them, etc.

return filepath.Join("apmpackage", "apm", version, "README.template.md")
}

func docsFilePath(version string) string {
return filepath.Join("apmpackage/apm/", version, "/docs/README.md")
return filepath.Join("apmpackage", "apm", version, "docs/README.md")
}

func pipelinesPath(version, dataStream string) string {
return filepath.Join("apmpackage/apm/", version, "/data_stream/", dataStream, "/elasticsearch/ingest_pipeline/")
return filepath.Join("apmpackage", "apm", version, "data_stream", dataStream, "elasticsearch", "ingest_pipeline")
}

func dataStreamPath(version string) string {
return filepath.Join("apmpackage/apm/", version, "/data_stream/")
return filepath.Join("apmpackage", "apm", version, "data_stream")
}

func fieldsPath(version, dataStream string) string {
return filepath.Join(dataStreamPath(version), dataStream, "fields/")
return filepath.Join(dataStreamPath(version), dataStream, "fields")
}

func ecsFilePath(version, dataStream string) string {
Expand Down
5 changes: 3 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ require (
github.com/elastic/apm-server/approvaltest v0.0.0-00010101000000-000000000000
github.com/elastic/beats/v7 v7.0.0-alpha2.0.20210309205547-21d02a3ab3b8
github.com/elastic/ecs v1.8.0
github.com/elastic/elastic-package v0.0.0-20210310173719-3b8f76516ae3
github.com/elastic/gmux v0.1.0
github.com/elastic/go-elasticsearch/v7 v7.5.1-0.20201007132508-ff965d99ba02
github.com/elastic/go-elasticsearch/v7 v7.11.0
github.com/elastic/go-elasticsearch/v8 v8.0.0-20201007143536-4b4020669208
github.com/elastic/go-hdrhistogram v0.1.0
github.com/elastic/go-licenser v0.3.1
Expand Down Expand Up @@ -50,7 +51,7 @@ require (
github.com/tidwall/gjson v1.6.0
github.com/tidwall/sjson v1.1.1
github.com/urso/magetools v0.0.0-20200125210132-c2e338f92f3a // indirect
github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56
github.com/xeipuuv/gojsonschema v1.2.0
go.elastic.co/apm v1.11.0
go.elastic.co/apm/module/apmelasticsearch v1.7.2
go.elastic.co/apm/module/apmgrpc v1.7.0
Expand Down
Loading