Merge branch 'master' into ecs-log-middleware

NOTICE.txt

@@ -639,7 +639,7 @@ Contents of "LICENSE":
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats/v7
 Version: v7.0.0
-Revision: 633285e75a03
+Revision: 0fda3061815d
 License type (autodetected): Apache-2.0
 
 --------------------------------------------------------------------
@@ -994,8 +994,7 @@ Contents of "NOTICE.txt":
 
 --------------------------------------------------------------------
 Dependency: github.com/elastic/gosigar
-Version: v0.10.6
-Revision: f115143bb233
+Version: v0.12.0
 License type (autodetected): Apache-2.0
 Contents of "NOTICE":
 
@@ -1011,7 +1010,7 @@ Contents of "NOTICE":
 
 --------------------------------------------------------------------
 Dependency: github.com/fatih/color
-Version: v1.9.0
+Version: v1.10.0
 License type (autodetected): MIT
 Contents of "LICENSE.md":
 
@@ -3573,8 +3572,7 @@ License type (autodetected): Apache-2.0
 
 --------------------------------------------------------------------
 Dependency: go.elastic.co/apm
-Version: v1.8.1
-Revision: 2aef45b9cf4b
+Version: v1.9.0
 License type (autodetected): Apache-2.0
 Contents of "NOTICE":
 
@@ -3905,7 +3903,7 @@ Contents of "LICENSE":
 
 --------------------------------------------------------------------
 Dependency: golang.org/x/net
-Revision: 942e2f445f3c
+Revision: 69a78807bb2b
 License type (autodetected): BSD-3-Clause
 Contents of "LICENSE":
 
@@ -4007,7 +4005,7 @@ Contents of "LICENSE":
 
 --------------------------------------------------------------------
 Dependency: golang.org/x/sys
-Revision: 201ba4db2418
+Revision: 5cba982894dd
 License type (autodetected): BSD-3-Clause
 Contents of "LICENSE":
 

_meta/beat.yml

@@ -27,6 +27,10 @@ apm-server:
   # Maximum number of new connections to accept simultaneously (0 means unlimited).
   #max_connections: 0
 
+  # Custom HTTP headers to add to all HTTP responses, e.g. for security policy compliance.
+  #response_headers:
+  #  X-My-Header: Contents of the header
+
   # If true (default), APM Server captures the IP of the instrumented service
   # or the IP and User Agent of the real user (RUM requests).
   #capture_personal_data: true

apm-server.docker.yml

@@ -27,6 +27,10 @@ apm-server:
   # Maximum number of new connections to accept simultaneously (0 means unlimited).
   #max_connections: 0
 
+  # Custom HTTP headers to add to all HTTP responses, e.g. for security policy compliance.
+  #response_headers:
+  #  X-My-Header: Contents of the header
+
   # If true (default), APM Server captures the IP of the instrumented service
   # or the IP and User Agent of the real user (RUM requests).
   #capture_personal_data: true

apm-server.yml

@@ -27,6 +27,10 @@ apm-server:
   # Maximum number of new connections to accept simultaneously (0 means unlimited).
   #max_connections: 0
 
+  # Custom HTTP headers to add to all HTTP responses, e.g. for security policy compliance.
+  #response_headers:
+  #  X-My-Header: Contents of the header
+
   # If true (default), APM Server captures the IP of the instrumented service
   # or the IP and User Agent of the real user (RUM requests).
   #capture_personal_data: true

apmpackage/README.md

@@ -0,0 +1,125 @@
+## Developer documentation
+
+### ~Requirements
+
+- Checkout `elastic/package-registry`, `elastic/package-storage` and `elastic/beats`
+- Have `elastic/package-spec` at hand
+
+### Guide
+
+#### Update / fix a package
+
+1. Actual work
+    - Make changes in `apmpackage/apm/<version>`, `apmpackage/docs/README.template.md` and/or code as needed
+    - Run `make update`. That will update fields, pipeline and doc files based on apm-server fields and pipeline defs.
+
+2. Run the registry
+    - Checkout a fresh master from the registry and run `mage build`
+    - Copy `apmpackage/apm` in apm-server to `build/package-storage/packages` in the registry
+    - `go run .`
+
+3. Run the stack
+    - Update Kibana settings with `xpack.fleet.registryUrl: http://localhost:8080`
+    - Start Kibana and Elasticsearch with X-Pack enabled. One way to do it is with a local Kibana:
+        - `yarn es snapshot --license trial --ssl -E xpack.security.authc.api_key.enabled=true`
+        - `yarn start --ssl`
+
+4. Test
+    - Go to the Fleet UI, install the integration and test what you need. You generally will want to have a look at the
+   installed assets (ie. templates and pipelines), and the generated `apm` input in the policy.
+    - If you need to change the package, you *must* remove the installed integration first. You can use the UI
+    or the API, eg: `curl -X DELETE -k -u elastic:changeme https://localhost:5601/abc/api/fleet/epm/packages/apm-0.1.0 -H 'kbn-xsrf: xyz'`
+    See [API docs](https://github.com/elastic/kibana/tree/master/x-pack/plugins/fleet/dev_docs/api) for details.
+    You normally don't need to restart the registry (an exception to this is eg. if you change a `hbs` template file).
+
+5. Upload to the snapshot registry
+    - When everything works and `apmpackage/apm/` changes have been merged to `master`, copy the new package to
+    `package-storage/packages` in the `package-storage` repo, `snapshot` branch.
+    Do *NOT* override any existing packages. Instead, bump the qualifier version. Eg: rename `apm/0.1.0-dev.1` to `apm/0.1.0-dev.2`
+    - Bump the qualifier in the contents too: `find . -type f  -print0 | xargs -0 sed -i ""  "s/0.1.0-dev.1/0.1.0-dev.2/g"`
+    This step can be done in a separate commit to facilitate review.
+
+
+#### Create a new package version
+
+Follow steps described above, except:
+
+1. New local version
+    - Copy `apmpackage/apm/<latest-version>` to `apmpackage/apm/<next-version>`.
+    - Add a new version mapping in the `versionMapping` variable at the top of `apmpackage/cmd/gen-package/main.go`.
+      Most likely this won't be needed in the future.
+    - Then do any changes in the new folder. The rest of the steps are the same.
+
+2. First dev version
+    - When copying to the `package-storage`, create the first version qualifier instead of bumping the last one.
+      Eg: `apm/0.2.0` -> `apm/0.2.0-dev.1`
+
+
+#### Run the Elastic Agent
+
+If you do code changes or a whole new version, you need to run the Elastic Agent locally.
+Most of the work here is done in `beats/x-pack/elastic-agent`
+
+0. Optional: Update the spec
+
+   The spec informs whether the Elastic Agent should or should not start apm-server based on the policy file,
+   and what settings to pass via GRPC call.
+    - Edit `spec/apm-server.yml`
+    - `mage update`
+
+1. Build / Package
+
+    *First time*
+    - `DEV=true PLATFORMS=darwin mage package` (replace platform as needed)
+    - Untar `build/distributions` contents
+
+    *Every time after*
+    - `DEV=true mage build`
+    - Copy `build/elastic-agent` to `build/distributions/elastic-agent-<version>-<platform>/data/elastic-agent-<hash>/`
+
+2. Override policy / apm-server
+    - Use the right `elastic-agent.yml` policy
+
+      It might be one you just generated with the UI, or one you have at hand with an apm input.
+      Copy to `build/distributions/elastic-agent-<version>-<platform>/elastic-agent.yml`
+
+    - Override apm-server in `install` and `downloads` folders. Approximately:
+      ```
+      # compile apm-server
+      cd ~/<path>/apm-server
+      make && make update
+
+      # tar and compress
+      cp build/fields/fields.yml .
+      tar cvf apm-server-<stack-version>-<platform>.tar apm-server LICENSE.txt NOTICE.txt README.md apm-server.yml ingest fields.yml
+      gzip apm-server-<stack-version>-<platform>.tar
+      sha512sum apm-server-<stack-version>-<platform>.tar.gz | tee apm-server-<stack-version>-<platform>.tar.gz.sha512
+
+      # delete old stuff
+      cd ~/<path>/beats/x-pack/elastic-agent/build/distributions/elastic-agent-<version>-<platform>/data/elastic-agent-<hash>/downloads
+      rm apm*
+      rm -rf ../install/apm*
+
+      # copy new files
+      mv <path>/apm-server-<stack-version>-<platform>.tar* .
+      mkdir -p ../install/apm-server-<stack-version>-<platform>
+      tar zxvf apm-server-<stack-version>-<platform> -C ../install/apm-server-<stack-version>-<platform>
+      ```
+3. Run the Elastic Agent
+    - `./build/distributions/<blablabla>/elastic-agent -e`
+    - Check apm-server logs at `build/distributions/<blablabla>/data/<blablabla>/logs/default`
+
+      (The last default in the path comes from the namespace in the policy)
+
+#### Promote a package
+
+Generally it should be done between FF and release.
+1. Remove the qualifier version from the package
+2. Push to the corresponding production branch(es)
+
+
+### Caveats
+
+Fleet is under active development and this guide might become obsolete quickly.
+
+Take everything with a grain of salt.

apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json

@@ -0,0 +1,25 @@
+{
+  "description": "Default enrichment for APM events",
+  "processors": [
+    {
+      "pipeline": {
+        "name": "metrics-apm.profiling-0.1.0-apm_user_agent"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.profiling-0.1.0-apm_user_geo"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.profiling-0.1.0-apm_ingest_timestamp"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.profiling-0.1.0-apm_remove_span_metadata"
+      }
+    }
+  ]
+}

apmpackage/apm/0.1.0/docs/README.md

@@ -1,25 +1,63 @@
 # APM Integration
 
-The APM integration installs templates and pipelines for APM data.
-If a policy contains an `apm` input, any Elastic Agent(s) set up with that policy will run an APM Server binary, and bind to `localhost:8200`.
+The APM integration installs Elasticsearch templates and Ingest Node pipelines for APM data.
+
+### How to use this integration
+
+When you add an APM integration to a policy, that policy will contain an `apm` input.
+If a policy contains an `apm` input, any Elastic Agent(s) set up with that policy will run locally an APM Server binary.
 You must configure your APM Agents to communicate with that APM Server.
 
 If you have RUM enabled, you must run APM Server centrally. Otherwise, you can run it at the edge machines.
 To do so, download and enroll an Elastic Agent in the same machines where your instrumented services run.
 
+If you want to change the default APM Server configuration, you need to edit the `elastic-agent.yml` policy file manually.
+Find the input with `type:apm` and add any settings under `apm-server`.
+For instance:
+
+```yaml
+inputs:
+  - id: ba928403-d7b8-4c09-adcb-d670c5eac89c
+    name: apm-1
+    revision: 1
+    type: apm
+    use_output: default
+    meta:
+      package:
+        name: apm
+        version: 0.1.0
+    data_stream:
+      namespace: default
+    apm-server:
+      rum:
+        enabled: true
+        event_rate.limit: 100
+      secret_token: changeme
+```
+
+Note that template, pipeline and ILM settings cannot be configured through this file - Templates and pipelines are installed by the integration,
+and ILM policies must be created externally. If you need additional pipelines, they must also be created externally.
+
+#### Namespace
+
+When you create a policy in the Fleet UI, under "Advanced Settings" you can choose a Namespace.
+In future versions, data streams created by the APM integration will include the service name,
+and you will be recommended to use the environment as namespace.
+
+This version doesn't automatically use the service name, so the recommendation instead is to use
+both the service name and the environment as the namespace.
 
 ### Compatibility and limitations
 
 The APM integration requires Kibana 7.11 and Elasticsearch with basic license.
 This version is experimental and has some limitations, listed bellow:
 
-- Elastic Cloud is not supported.
-- Standalone mode is not supported.
-- If you need to customize settings for APM Server, you need to update the agent policy manually.
-Look for `apm-server` in the `apm` input.
-- It is not possible to change APM Server settings dynamically.
-You must update the policy with any changes you need and stop the APM Server process.
+- It is not yet possible to change APM Server settings dynamically.
+You must update the policy with any changes you need and restart the APM Server process.
+- Sourcemap enrichment is not yet supported.
+- There is no default ILM policy for traces (spans and transactions).
 
+IMPORTANT: If you run APM Server with Elastic Agent manually in standalone mode, you must install the APM integration before ingestion starts.
 
 ### Configuration parameters
 
@@ -534,8 +572,8 @@ Metrics are written to `metrics-apm.*`, `metrics-apm.internal.*` and `metrics-ap
 
 ### Logs
 
-Logs are application log and error events.
-Logs are written to `logs-apm.*` and `logs-apm.error.*` indices.
+Logs are application error events.
+Logs are written to `logs-apm.error.*` indices.
 
 **Exported Fields**
 

apmpackage/apm/0.1.0/manifest.yml

@@ -4,9 +4,9 @@ title: Elastic APM
 version: 0.1.0
 license: basic
 description: Ingest APM data
-type: solution  # integration / solution
+type: integration
 categories:
-  - monitoring # TODO do we need a new category?
+  - monitoring
 release: experimental # experimental / beta / ga
 conditions:
   kibana.version: '^7.11.0'

apmpackage/cmd/gen-package/gendocs.go

@@ -29,8 +29,8 @@ import (
 func generateDocs(inputFields map[string][]field, version string) {
 	data := docsData{
 		Traces:             prepareFields(inputFields, version, "traces"),
-		Metrics:            prepareFields(inputFields, version, "metrics"),
-		Logs:               prepareFields(inputFields, version, "logs"),
+		Metrics:            prepareFields(inputFields, version, "app_metrics"),
+		Logs:               prepareFields(inputFields, version, "error_logs"),
 		TransactionExample: loadExample("transactions.json"),
 		SpanExample:        loadExample("spans.json"),
 		MetricsExample:     loadExample("metricsets.json"),
@@ -65,17 +65,17 @@ type docsData struct {
 	ErrorExample       string
 }
 
-func prepareFields(inputFields map[string][]field, version, streamType string) []field {
+func prepareFields(inputFields map[string][]field, version, stream string) []field {
 	extend := func(fs []field) []field {
 		var baseFields []field
-		for _, f := range loadFieldsFile(baseFieldsFilePath(version, streamType)) {
+		for _, f := range loadFieldsFile(baseFieldsFilePath(version, stream)) {
 			f.IsECS = true
 			baseFields = append(baseFields, f)
 		}
 		fs = append(baseFields, fs...)
 		return fs
 	}
-	return extend(inputFields[streamType])
+	return extend(inputFields[stream])
 }
 
 func loadExample(file string) string {