[ACM] Agent remote configuration management - Main functionality

_meta/beat.yml

@@ -183,6 +183,56 @@ apm-server:
   #ilm:
     #enabled: false
 
+  #kibana:
+    # For agent remote configuration, enabled must be true
+    #enabled: false
+
+    # Scheme and port can be left out and will be set to the default (http and 5601)
+    # In case you specify an additional path, the scheme is required: http://localhost:5601/path
+    # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+    #host: "localhost:5601"
+
+    # Optional protocol and basic auth credentials.
+    #protocol: "https"
+    #username: "elastic"
+    #password: "changeme"
+
+    # Optional HTTP path
+    #path: ""
+
+    # Enable custom SSL settings. Set to false to ignore custom SSL settings for secure communication.
+    #ssl.enabled: true
+
+    # Optional SSL configuration options. SSL is off by default, change the `protocol` option if you want to enable `https`.
+    # Configure SSL verification mode. If `none` is configured, all server hosts
+    # and certificates will be accepted. In this mode, SSL based connections are
+    # susceptible to man-in-the-middle attacks. Use only for testing. Default is
+    # `full`.
+    #ssl.verification_mode: full
+
+    # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
+    # 1.2 are enabled.
+    #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
+
+    # List of root certificates for HTTPS server verifications
+    #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+    # Certificate for SSL client authentication
+    #ssl.certificate: "/etc/pki/client/cert.pem"
+
+    # Client Certificate Key
+    #ssl.key: "/etc/pki/client/cert.key"
+
+    # Optional passphrase for decrypting the Certificate Key.
+    # It is recommended to use the provided keystore instead of entering the passphrase in plain text.
+    #ssl.key_passphrase: ''
+
+    # Configure cipher suites to be used for SSL connections
+    #ssl.cipher_suites: []
+
+    # Configure curve types for ECDHE based cipher suites
+    #ssl.curve_types: []
+
 #================================ General ======================================
 
 # Internal queue configuration for buffering events to be published.

agentcfg/fetch.go

@@ -0,0 +1,67 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package agentcfg
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/pkg/errors"
+
+	"github.com/elastic/beats/libbeat/common"
+
+	"github.com/elastic/apm-server/convert"
+	"github.com/elastic/beats/libbeat/kibana"
+)
+
+const endpoint = "/api/apm/settings/cm/search"
+
+var minVersion = common.Version{Major: 7, Minor: 3}
+
+// Fetch retrieves agent configuration from Kibana
+func Fetch(kbClient *kibana.Client, q Query, err error) (map[string]string, string, error) {
+	var doc Doc
+	resultBytes, err := request(kbClient, convert.ToReader(q), err)
+	err = convert.FromBytes(resultBytes, &doc, err)
+	return doc.Source.Settings, doc.ID, err
+}
+
+func request(kbClient *kibana.Client, r io.Reader, err error) ([]byte, error) {
+	if err != nil {
+		return nil, err
+	}
+	if kbClient == nil {
+		return nil, errors.New("No configured Kibana Client: provide apm-server.kibana.* settings")
+	}
+	if version := kbClient.GetVersion(); version.LessThan(&minVersion) {
+		return nil, errors.New(fmt.Sprintf("Needs Kibana version %s or higher", minVersion.String()))
+	}
+	resp, err := kbClient.Send(http.MethodPost, endpoint, nil, nil, r)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	result, err := ioutil.ReadAll(resp.Body)
+	if resp.StatusCode >= http.StatusMultipleChoices {
+		return nil, errors.New(string(result))
+	}
+	return result, err
+}

agentcfg/fetch_test.go

@@ -0,0 +1,71 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package agentcfg
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/apm-server/tests"
+	"github.com/elastic/beats/libbeat/kibana"
+)
+
+type m map[string]interface{}
+
+var q = Query{}
+
+func TestFetchNoClient(t *testing.T) {
+	kb, kerr := kibana.NewKibanaClient(nil)
+	_, _, ferr := Fetch(kb, q, kerr)
+	require.Error(t, ferr)
+	assert.Equal(t, ferr, kerr, kerr)
+}
+
+func TestFetchStringConversion(t *testing.T) {
+	kb := tests.MockKibana(http.StatusOK,
+		m{
+			"_id": "1",
+			"_source": m{
+				"settings": m{
+					"sampling_rate": 0.5,
+				},
+			},
+		})
+	result, etag, err := Fetch(kb, q, nil)
+	require.NoError(t, err)
+	assert.Equal(t, "1", etag, etag)
+	assert.Equal(t, map[string]string{"sampling_rate": "0.5"}, result, result)
+}
+
+func TestFetchVersionCheck(t *testing.T) {
+	kb := tests.MockKibana(http.StatusOK, m{})
+	kb.Connection.Version.Major = 6
+	_, _, err := Fetch(kb, q, nil)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "version")
+}
+
+func TestFetchError(t *testing.T) {
+	kb := tests.MockKibana(http.StatusNotFound, m{"error": "an error"})
+	_, _, err := Fetch(kb, q, nil)
+	require.Error(t, err)
+	assert.Equal(t, err.Error(), "{\"error\":\"an error\"}")
+}

agentcfg/model.go

@@ -0,0 +1,72 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package agentcfg
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+const (
+	// ServiceName keyword
+	ServiceName = "service.name"
+	// ServiceEnv keyword
+	ServiceEnv = "service.environment"
+)
+
+// Doc represents an elasticsearch document
+type Doc struct {
+	ID     string `json:"_id"`
+	Source Source `json:"_source"`
+}
+
+// Source represents the elasticsearch _source field of a document
+type Source struct {
+	Settings Settings `json:"settings"`
+}
+
+// Settings hold agent configuration
+type Settings map[string]string
+
+// UnmarshalJSON overrides default method to convert any JSON type to string
+func (s *Settings) UnmarshalJSON(b []byte) error {
+	in := make(map[string]interface{})
+	out := make(map[string]string)
+	err := json.Unmarshal(b, &in)
+	for k, v := range in {
+		out[k] = fmt.Sprintf("%v", v)
+	}
+	*s = out
+	return err
+}
+
+// NewQuery creates a Query struct
+func NewQuery(name, env string) Query {
+	return Query{Service{name, env}}
+}
+
+// Query represents an URL body or query params for agent configuration
+type Query struct {
+	Service Service `json:"service"`
+}
+
+// Service holds supported attributes for querying configuration
+type Service struct {
+	Name        string `json:"name"`
+	Environment string `json:"environment,omitempty"`
+}

apm-server.docker.yml

@@ -183,6 +183,56 @@ apm-server:
   #ilm:
     #enabled: false
 
+  #kibana:
+    # For agent remote configuration, enabled must be true
+    #enabled: false
+
+    # Scheme and port can be left out and will be set to the default (http and 5601)
+    # In case you specify an additional path, the scheme is required: http://localhost:5601/path
+    # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+    #host: "localhost:5601"
+
+    # Optional protocol and basic auth credentials.
+    #protocol: "https"
+    #username: "elastic"
+    #password: "changeme"
+
+    # Optional HTTP path
+    #path: ""
+
+    # Enable custom SSL settings. Set to false to ignore custom SSL settings for secure communication.
+    #ssl.enabled: true
+
+    # Optional SSL configuration options. SSL is off by default, change the `protocol` option if you want to enable `https`.
+    # Configure SSL verification mode. If `none` is configured, all server hosts
+    # and certificates will be accepted. In this mode, SSL based connections are
+    # susceptible to man-in-the-middle attacks. Use only for testing. Default is
+    # `full`.
+    #ssl.verification_mode: full
+
+    # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
+    # 1.2 are enabled.
+    #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
+
+    # List of root certificates for HTTPS server verifications
+    #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+    # Certificate for SSL client authentication
+    #ssl.certificate: "/etc/pki/client/cert.pem"
+
+    # Client Certificate Key
+    #ssl.key: "/etc/pki/client/cert.key"
+
+    # Optional passphrase for decrypting the Certificate Key.
+    # It is recommended to use the provided keystore instead of entering the passphrase in plain text.
+    #ssl.key_passphrase: ''
+
+    # Configure cipher suites to be used for SSL connections
+    #ssl.cipher_suites: []
+
+    # Configure curve types for ECDHE based cipher suites
+    #ssl.curve_types: []
+
 #================================ General ======================================
 
 # Internal queue configuration for buffering events to be published.

apm-server.yml

@@ -183,6 +183,56 @@ apm-server:
   #ilm:
     #enabled: false
 
+  #kibana:
+    # For agent remote configuration, enabled must be true
+    #enabled: false
+
+    # Scheme and port can be left out and will be set to the default (http and 5601)
+    # In case you specify an additional path, the scheme is required: http://localhost:5601/path
+    # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+    #host: "localhost:5601"
+
+    # Optional protocol and basic auth credentials.
+    #protocol: "https"
+    #username: "elastic"
+    #password: "changeme"
+
+    # Optional HTTP path
+    #path: ""
+
+    # Enable custom SSL settings. Set to false to ignore custom SSL settings for secure communication.
+    #ssl.enabled: true
+
+    # Optional SSL configuration options. SSL is off by default, change the `protocol` option if you want to enable `https`.
+    # Configure SSL verification mode. If `none` is configured, all server hosts
+    # and certificates will be accepted. In this mode, SSL based connections are
+    # susceptible to man-in-the-middle attacks. Use only for testing. Default is
+    # `full`.
+    #ssl.verification_mode: full
+
+    # List of supported/valid TLS versions. By default all TLS versions 1.0 up to
+    # 1.2 are enabled.
+    #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
+
+    # List of root certificates for HTTPS server verifications
+    #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+    # Certificate for SSL client authentication
+    #ssl.certificate: "/etc/pki/client/cert.pem"
+
+    # Client Certificate Key
+    #ssl.key: "/etc/pki/client/cert.key"
+
+    # Optional passphrase for decrypting the Certificate Key.
+    # It is recommended to use the provided keystore instead of entering the passphrase in plain text.
+    #ssl.key_passphrase: ''
+
+    # Configure cipher suites to be used for SSL connections
+    #ssl.cipher_suites: []
+
+    # Configure curve types for ECDHE based cipher suites
+    #ssl.curve_types: []
+
 #================================ General ======================================
 
 # Internal queue configuration for buffering events to be published.