Cloud Foundry metrics receiver #1 - config, factory, docs

.github/CODEOWNERS

@@ -69,6 +69,7 @@ receiver/awscontainerinsightreceiver/                @open-telemetry/collector-c
 receiver/awsecscontainermetricsreceiver/             @open-telemetry/collector-contrib-approvers @kbrockhoff @anuraaga
 receiver/awsxrayreceiver/                            @open-telemetry/collector-contrib-approvers @kbrockhoff @anuraaga
 receiver/carbonreceiver/                             @open-telemetry/collector-contrib-approvers @pjanotti
+receiver/cloudfoundryreceiver/                       @open-telemetry/collector-contrib-approvers @agoallikmaa @pellared
 receiver/collectdreceiver/                           @open-telemetry/collector-contrib-approvers @owais
 receiver/dockerstatsreceiver/                        @open-telemetry/collector-contrib-approvers @rmfitzpatrick
 receiver/dotnetdiagnosticsreceiver/                  @open-telemetry/collector-contrib-approvers @pmcollins @davmason

.github/dependabot.yml

@@ -401,6 +401,10 @@ updates:
     directory: "/receiver/carbonreceiver"
     schedule:
       interval: "weekly"
+  - package-ecosystem: "gomod"
+    directory: "/receiver/cloudfoundryreceiver"
+    schedule:
+      interval: "weekly"
   - package-ecosystem: "gomod"
     directory: "/receiver/collectdreceiver"
     schedule:

go.mod

@@ -500,6 +500,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxr
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/carbonreceiver => ./receiver/carbonreceiver
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/cloudfoundryreceiver => ./receiver/cloudfoundryreceiver
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/collectdreceiver => ./receiver/collectdreceiver
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dotnetdiagnosticsreceiver => ./receiver/dotnetdiagnosticsreceiver

receiver/cloudfoundryreceiver/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

receiver/cloudfoundryreceiver/README.md

@@ -0,0 +1,88 @@
+# Cloud Foundry Receiver
+
+The Cloud Foundry receiver connects to the RLP (Reverse Log Proxy) Gateway of the Cloud Foundry installation, typically
+available at the URL `https://log-stream.<cf-system-domain>`.
+
+## Authentication
+
+RLP Gateway authentication is performed by adding the Oauth2 token as the `Authorization` header. To obtain an OAuth2
+token to use for the RLP Gateway, a request is made to the UAA component which acts as the OAuth2 provider (URL
+specified by `uaa_url` configuration option, which typically is `https://uaa.<cf-system-domain>`). To authenticate with
+UAA, username and password/secret combination is used (`uaa_username` and `uaa_password` configuration options). This
+UAA user must have the `client_credentials` and `refresh_token` authorized grant types, and `logs.admin` authority.
+
+The following is an example sequence of commands to create the UAA user using the `uaac` command line utility:
+* `uaac target https://uaa.<cf-system-domain>`
+* `uaac token client get identity -s <identity-user-secret>`
+* `uaac client add <uaa_username> --name opentelemetry --secret <uaa_password> --authorized_grant_types client_credentials,refresh_token --authorities logs.admin`
+
+The `<uaa_username>` and `<uaa_password>` above can be set to anything as long as they match the values provided to the
+receiver configuration. The admin account (which is `identity` here) which has the permissions to create new clients may
+have a different name on different setups. The value of `--name` is not used for receiver configuration.
+
+## Configuration
+
+The receiver takes the following configuration options:
+* `rlp_gateway_url` - URL of the RLP gateway, required, typically `https://log-stream.<cf-system-domain>`
+* `rlp_gateway_skip_tls_verify` - whether to skip TLS verify for the RLP Gateway endpoint, default `false`
+* `rlp_gateway_shard_id` - metrics are load balanced among receivers that use the same shard ID, therefore this must 
+  only be set if there are multiple receivers which must both receive all the metrics instead of them being balanced
+  between them. Default value is `opentelemetry`
+* `uaa_url` - URL of the UAA provider, required, typically `https://uaa.<cf-system-domain>`
+* `uaa_skip_tls_verify` - whether to skip TLS verify for the UAA endpoint, default `false`
+* `uaa_username` - name of the UAA user (required grant types/authorities described above)
+* `uaa_username` - password of the UAA user
+* `http_timeout` - HTTP socket timeout used for RLP Gateway, default `10s`
+
+Example:
+
+```yaml
+receivers:
+  cloudfoundry:
+    rlp_gateway_url: "https://log-stream.sys.example.internal"
+    rlp_gateway_skip_tls_verify: false
+    rlp_gateway_shard_id: "opentelemetry"
+    uaa_url: "https://uaa.sys.example.internal"
+    uaa_skip_tls_verify: false
+    uaa_username: "otelclient"
+    uaa_password: "changeit"
+    http_timeout: "20s"
+```
+
+The full list of settings exposed for this receiver are documented [here](./config.go)
+with detailed sample configurations [here](./testdata/config.yaml).
+
+## Metrics
+
+Reported metrics are grouped under an instrumentation library named `otelcol/cloudfoundry`. Metric names are as
+specified by [Cloud Foundry metrics documentation](https://docs.cloudfoundry.org/running/all_metrics.html), but the
+origin name is prepended to the metric name with `.` separator. All metrics either of type `Gauge` or `Sum`.
+
+### Attributes
+
+All the metrics have the following attributes:
+* `origin` - origin name as documented by Cloud Foundry
+* `source` - for applications, the GUID of the application, otherwise equal to `origin`
+
+For CF/TAS deployed in BOSH, the following attributes are also present, using their canonical BOSH meanings:
+* `deployment` - BOSH deployment name
+* `index` - BOSH instance ID (GUID)
+* `ip` - BOSH instance IP
+* `job` - BOSH job name
+
+For metrics originating with `rep` origin name (specific to applications), the following metrics are present:
+* `instance_id` - numerical index of the application instance. However, also present for `bbs` origin, where it matches
+  the value of `index`
+* `process_id` - process ID (GUID). For a process of type "web" which is the main process of an application, this is
+  equal to `source_id` and `app_id`
+* `process_instance_id` - unique ID of a process instance, should be treated as an opaque string
+* `process_type` - process type. Each application has exactly one process of type `web`, but many have any number of
+  other processes
+
+On TAS/PCF versions 2.8.0+ and cf-deployment versions v11.1.0+, the following additional attributes are present for
+application metrics: `app_id`, `app_name`, `space_id`, `space_name`, `organization_id`, `organization_name` which
+provide the GUID and name of application, space and organization respectively.
+
+This might not be a comprehensive list of attributes, as the receiver passes on whatever attributes the gateway
+provides, which may include some that are specific to TAS and possibly new ones in future Cloud Foundry versions as
+well.

receiver/cloudfoundryreceiver/config.go

@@ -0,0 +1,68 @@
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudfoundryreceiver
+
+import (
+	"fmt"
+	"net/url"
+	"time"
+
+	"go.opentelemetry.io/collector/config"
+)
+
+// Config defines configuration for Collectd receiver.
+type Config struct {
+	config.ReceiverSettings `mapstructure:",squash"`
+
+	RLPGatewayURL           string        `mapstructure:"rlp_gateway_url"`
+	RLPGatewaySkipTLSVerify bool          `mapstructure:"rlp_gateway_skip_tls_verify"`
+	RLPGatewayShardID       string        `mapstructure:"rlp_gateway_shard_id"`
+	UAAUrl                  string        `mapstructure:"uaa_url"`
+	UAASkipTLSVerify        bool          `mapstructure:"uaa_skip_tls_verify"`
+	UAAUsername             string        `mapstructure:"uaa_username"`
+	UAAPassword             string        `mapstructure:"uaa_password"`
+	HTTPTimeout             time.Duration `mapstructure:"http_timeout"`
+}
+
+func (c *Config) Validate() error {
+	err := validateURLOption("rlp_gateway_url", c.RLPGatewayURL)
+	if err != nil {
+		return err
+	}
+
+	err = validateURLOption("uaa_url", c.UAAUrl)
+	if err != nil {
+		return err
+	}
+
+	if c.UAAUsername == "" {
+		return fmt.Errorf("username not specified")
+	}
+
+	return nil
+}
+
+func validateURLOption(name string, value string) error {
+	if value == "" {
+		return fmt.Errorf("%s not specified", name)
+	}
+
+	_, err := url.Parse(value)
+	if err != nil {
+		return fmt.Errorf("failed to parse %s value %s as url: %v", name, value, err)
+	}
+
+	return nil
+}

receiver/cloudfoundryreceiver/config_test.go

@@ -0,0 +1,58 @@
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudfoundryreceiver
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/config/configtest"
+)
+
+func TestLoadConfig(t *testing.T) {
+	factories, err := componenttest.NopFactories()
+	assert.Nil(t, err)
+
+	factory := NewFactory()
+	factories.Receivers[typeStr] = factory
+	cfg, err := configtest.LoadConfigAndValidate(path.Join(".", "testdata", "config.yaml"), factories)
+
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+
+	assert.Equal(t, len(cfg.Receivers), 2)
+
+	r0 := cfg.Receivers[config.NewID(typeStr)]
+	assert.Equal(t, r0, factory.CreateDefaultConfig())
+
+	r1 := cfg.Receivers[config.NewIDWithName(typeStr, "one")].(*Config)
+	assert.Equal(t, r1,
+		&Config{
+			ReceiverSettings:        config.NewReceiverSettings(config.NewIDWithName(typeStr, "one")),
+			RLPGatewayURL:           "https://log-stream.sys.example.internal",
+			RLPGatewaySkipTLSVerify: true,
+			RLPGatewayShardID:       "otel-test",
+			UAAUrl:                  "https://uaa.sys.example.internal",
+			UAASkipTLSVerify:        true,
+			UAAUsername:             "admin",
+			UAAPassword:             "test",
+			HTTPTimeout:             time.Second * 20,
+		})
+}

receiver/cloudfoundryreceiver/doc.go

@@ -0,0 +1,20 @@
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package cloudfoundryreceiver implements a receiver that can be used by the
+// Opentelemetry collector to receive Cloud Foundry metrics via its Reverse
+// Log Proxy (RLP) Gateway component. The protocol is handled by the
+// go-loggregator library, which uses HTTP to connect to the gateway and receive
+// JSON-protobuf encoded v2 Envelope messages as documented by loggregator-api.
+package cloudfoundryreceiver

receiver/cloudfoundryreceiver/factory.go

@@ -0,0 +1,64 @@
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudfoundryreceiver
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/consumer"
+	"go.opentelemetry.io/collector/receiver/receiverhelper"
+)
+
+// This file implements factory for Cloud Foundry receiver.
+
+const (
+	typeStr                  = "cloudfoundry"
+	defaultUAAUsername       = "admin"
+	defaultRLPGatewayShardID = "opentelemetry"
+	defaultURL               = "https://localhost"
+)
+
+// NewFactory creates a factory for collectd receiver.
+func NewFactory() component.ReceiverFactory {
+	return receiverhelper.NewFactory(
+		typeStr,
+		createDefaultConfig,
+		receiverhelper.WithMetrics(createMetricsReceiver))
+}
+
+func createDefaultConfig() config.Receiver {
+	return &Config{
+		ReceiverSettings:        config.NewReceiverSettings(config.NewID(typeStr)),
+		RLPGatewayURL:           defaultURL,
+		RLPGatewaySkipTLSVerify: false,
+		RLPGatewayShardID:       defaultRLPGatewayShardID,
+		UAAUsername:             defaultUAAUsername,
+		UAAPassword:             "",
+		UAAUrl:                  defaultURL,
+		UAASkipTLSVerify:        false,
+	}
+}
+
+func createMetricsReceiver(
+	_ context.Context,
+	params component.ReceiverCreateSettings,
+	cfg config.Receiver,
+	nextConsumer consumer.Metrics,
+) (component.MetricsReceiver, error) {
+	c := cfg.(*Config)
+	return newCloudFoundryReceiver(params.Logger, *c, nextConsumer)
+}

receiver/cloudfoundryreceiver/factory_test.go

@@ -0,0 +1,42 @@
+// Copyright 2019, OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudfoundryreceiver
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/config/configcheck"
+	"go.opentelemetry.io/collector/consumer/consumertest"
+)
+
+func TestCreateDefaultConfig(t *testing.T) {
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	assert.NotNil(t, cfg, "failed to create default config")
+	assert.NoError(t, configcheck.ValidateConfig(cfg))
+}
+
+func TestCreateReceiver(t *testing.T) {
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+
+	params := componenttest.NewNopReceiverCreateSettings()
+	tReceiver, err := factory.CreateMetricsReceiver(context.Background(), params, cfg, consumertest.NewNop())
+	assert.NoError(t, err)
+	assert.NotNil(t, tReceiver, "receiver creation failed")
+}