Splunkent update conf (open-telemetry#31183)

**Description:** Make changes to configuration of the application to
allow the user to specify endpoints corresponding to different Splunk
node types. Specifically, this update will allow users to define three
separate clients: indexer, cluster master, and search head. This change
will allow for the addition of metrics corresponding to these different
modes of operation within the Splunk enterprise deployment.

**Link to tracking Issue:**
[30254](open-telemetry#30254)

**Testing:** Unit tests were updated to run against new configuration
options.

**Documentation:** Updated README to reflect the new changes in
configuration.

.chloggen/splunkent-update-conf.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: 'enhancement'
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: splunkentreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Updated the config.go and propogated these changes to other receiver components. Change was necessary to differentiate different configurable endpoints."
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [30254]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

receiver/splunkenterprisereceiver/README.md

@@ -8,7 +8,9 @@ jobs.
 
 ## Configuration
 
-The following settings are required, omitting them will either cause your receiver to fail to compile or result in 4/5xx return codes during scraping.
+The following settings are required, omitting them will either cause your receiver to fail to compile or result in 4/5xx return codes during scraping. 
+
+**NOTE:** These must be set for each Splunk instance type (indexer, search head, or cluster master) from which you wish to pull metrics. At present, only one of each type is accepted, per configured receiver instance. This means, for example, that if you have three different "indexer" type instances that you would like to pull metrics from you will need to configure three different `splunkenterprise` receivers for each indexer node you wish to monitor.
 
 * `basicauth` (from [basicauthextension](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/basicauthextension)): A configured stanza for the basicauthextension.
 * `auth` (no default): String name referencing your auth extension.
@@ -23,16 +25,38 @@ Example:
 
 ```yaml
 extensions:
-    basicauth/client:
+    basicauth/indexer:
+        client_auth:
+            username: admin
+            password: securityFirst
+    basicauth/cluster_master:
         client_auth:
             username: admin
             password: securityFirst
 
 receivers:
     splunkenterprise:
-        auth: basicauth/client
-        endpoint: "https://localhost:8089"
-        timeout: 45s
+        indexer:
+            auth: 
+              authenticator: basicauth/indexer
+            endpoint: "https://localhost:8089"
+            timeout: 45s
+        cluster_master:
+            auth: 
+              authenticator: basicauth/cluster_master
+            endpoint: "https://localhost:8089"
+            timeout: 45s
+
+exporters:
+  logging:
+    loglevel: info
+
+service:
+  extensions: [basicauth/indexer, basicauth/cluster_master]
+  pipelines:
+    metrics:
+      receivers: [splunkenterprise]
+      exporters: [logging]
 ```
 
 For a full list of settings exposed by this receiver please look [here](./config.go) with a detailed configuration [here](./testdata/config.yaml).

receiver/splunkenterprisereceiver/client.go

@@ -5,6 +5,7 @@ package splunkenterprisereceiver // import "github.com/open-telemetry/openteleme
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -13,73 +14,172 @@ import (
 	"go.opentelemetry.io/collector/component"
 )
 
+// Indexer type "enum". Included in context sent from scraper functions
+const (
+	typeIdx = "IDX"
+	typeSh  = "SH"
+	typeCm  = "CM"
+)
+
+var (
+	errCtxMissingEndpointType = errors.New("context was passed without the endpoint type included")
+	errEndpointTypeNotFound   = errors.New("requested client is not configured and could not be found in splunkEntClient")
+	errNoClientFound          = errors.New("no client corresponding to the endpoint type was found")
+)
+
+// Type wrapper for accessing context value
+type endpointType string
+
+// Wrapper around splunkClientMap to avoid awkward reference/dereference stuff that arises when using maps in golang
 type splunkEntClient struct {
+	clients splunkClientMap
+}
+
+// The splunkEntClient is made up of a number of splunkClients defined for each configured endpoint
+type splunkClientMap map[any]splunkClient
+
+// The client does not carry the endpoint that is configured with it and golang does not support mixed
+// type arrays so this struct contains the pair: the client configured for the endpoint and the endpoint
+// itself
+type splunkClient struct {
 	client   *http.Client
 	endpoint *url.URL
 }
 
 func newSplunkEntClient(cfg *Config, h component.Host, s component.TelemetrySettings) (*splunkEntClient, error) {
-	client, err := cfg.ClientConfig.ToClient(h, s)
-	if err != nil {
-		return nil, err
+	var err error
+	var e *url.URL
+	var c *http.Client
+	clientMap := make(splunkClientMap)
+
+	// if the endpoint is defined, put it in the endpoints map for later use
+	// we already checked that url.Parse does not fail in cfg.Validate()
+	if cfg.IdxEndpoint.Endpoint != "" {
+		e, _ = url.Parse(cfg.IdxEndpoint.Endpoint)
+		c, err = cfg.IdxEndpoint.ToClient(h, s)
+		if err != nil {
+			return nil, err
+		}
+		clientMap[typeIdx] = splunkClient{
+			client:   c,
+			endpoint: e,
+		}
+	}
+	if cfg.SHEndpoint.Endpoint != "" {
+		e, _ = url.Parse(cfg.SHEndpoint.Endpoint)
+		c, err = cfg.SHEndpoint.ToClient(h, s)
+		if err != nil {
+			return nil, err
+		}
+		clientMap[typeSh] = splunkClient{
+			client:   c,
+			endpoint: e,
+		}
+	}
+	if cfg.CMEndpoint.Endpoint != "" {
+		e, _ = url.Parse(cfg.CMEndpoint.Endpoint)
+		c, err = cfg.CMEndpoint.ToClient(h, s)
+		if err != nil {
+			return nil, err
+		}
+		clientMap[typeCm] = splunkClient{
+			client:   c,
+			endpoint: e,
+		}
 	}
 
-	endpoint, _ := url.Parse(cfg.Endpoint)
-
-	return &splunkEntClient{
-		client:   client,
-		endpoint: endpoint,
-	}, nil
+	return &splunkEntClient{clients: clientMap}, nil
 }
 
 // For running ad hoc searches only
-func (c *splunkEntClient) createRequest(ctx context.Context, sr *searchResponse) (*http.Request, error) {
+func (c *splunkEntClient) createRequest(ctx context.Context, sr *searchResponse) (req *http.Request, err error) {
+	// get endpoint type from the context
+	eptType := ctx.Value(endpointType("type"))
+	if eptType == nil {
+		return nil, errCtxMissingEndpointType
+	}
+
 	// Running searches via Splunk's REST API is a two step process: First you submit the job to run
 	// this returns a jobid which is then used in the second part to retrieve the search results
 	if sr.Jobid == nil {
+		var u string
 		path := "/services/search/jobs/"
-		url, _ := url.JoinPath(c.endpoint.String(), path)
+
+		if e, ok := c.clients[eptType]; ok {
+			u, err = url.JoinPath(e.endpoint.String(), path)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, errNoClientFound
+		}
 
 		// reader for the response data
 		data := strings.NewReader(sr.search)
 
 		// return the build request, ready to be run by makeRequest
-		req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, data)
+		req, err = http.NewRequestWithContext(ctx, http.MethodPost, u, data)
 		if err != nil {
 			return nil, err
 		}
 
 		return req, nil
 	}
 	path := fmt.Sprintf("/services/search/jobs/%s/results", *sr.Jobid)
-	url, _ := url.JoinPath(c.endpoint.String(), path)
+	url, _ := url.JoinPath(c.clients[eptType].endpoint.String(), path)
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	req, err = http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return nil, err
 	}
 
 	return req, nil
 }
 
-func (c *splunkEntClient) createAPIRequest(ctx context.Context, apiEndpoint string) (*http.Request, error) {
-	url := c.endpoint.String() + apiEndpoint
+// forms an *http.Request for use with Splunk built-in API's (like introspection).
+func (c *splunkEntClient) createAPIRequest(ctx context.Context, apiEndpoint string) (req *http.Request, err error) {
+	var u string
+
+	// get endpoint type from the context
+	eptType := ctx.Value(endpointType("type"))
+	if eptType == nil {
+		return nil, errCtxMissingEndpointType
+	}
+
+	if e, ok := c.clients[eptType]; ok {
+		u = e.endpoint.String() + apiEndpoint
+	} else {
+		return nil, errNoClientFound
+	}
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	req, err = http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
 	if err != nil {
 		return nil, err
 	}
 
 	return req, nil
 }
 
-// Construct and perform a request to the API. Returns the searchResponse passed into the
-// function as state
+// Perform a request.
 func (c *splunkEntClient) makeRequest(req *http.Request) (*http.Response, error) {
-	res, err := c.client.Do(req)
-	if err != nil {
-		return nil, err
+	// get endpoint type from the context
+	eptType := req.Context().Value(endpointType("type"))
+	if eptType == nil {
+		return nil, errCtxMissingEndpointType
 	}
+	if sc, ok := c.clients[eptType]; ok {
+		res, err := sc.client.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		return res, nil
+	}
+	return nil, errEndpointTypeNotFound
+}
 
-	return res, nil
+// Check if the splunkEntClient contains a configured endpoint for the type of scraper
+// Returns true if an entry exists, false if not.
+func (c *splunkEntClient) isConfigured(v string) bool {
+	_, ok := c.clients[v]
+	return ok
 }

receiver/splunkenterprisereceiver/client_test.go

@@ -34,11 +34,9 @@ func (m *mockHost) GetExtensions() map[component.ID]component.Component {
 
 func TestClientCreation(t *testing.T) {
 	cfg := &Config{
-		ClientConfig: confighttp.ClientConfig{
+		IdxEndpoint: confighttp.ClientConfig{
 			Endpoint: "https://localhost:8089",
-			Auth: &configauth.Authentication{
-				AuthenticatorID: component.MustNewIDWithName("basicauth", "client"),
-			},
+			Auth:     &configauth.Authentication{AuthenticatorID: component.MustNewIDWithName("basicauth", "client")},
 		},
 		ScraperControllerSettings: scraperhelper.ScraperControllerSettings{
 			CollectionInterval: 10 * time.Second,
@@ -58,18 +56,16 @@ func TestClientCreation(t *testing.T) {
 
 	testEndpoint, _ := url.Parse("https://localhost:8089")
 
-	require.Equal(t, client.endpoint, testEndpoint)
+	require.Equal(t, testEndpoint, client.clients[typeIdx].endpoint)
 }
 
 // test functionality of createRequest which is used for building metrics out of
 // ad-hoc searches
 func TestClientCreateRequest(t *testing.T) {
 	cfg := &Config{
-		ClientConfig: confighttp.ClientConfig{
+		IdxEndpoint: confighttp.ClientConfig{
 			Endpoint: "https://localhost:8089",
-			Auth: &configauth.Authentication{
-				AuthenticatorID: component.MustNewIDWithName("basicauth", "client"),
-			},
+			Auth:     &configauth.Authentication{AuthenticatorID: component.MustNewIDWithName("basicauth", "client")},
 		},
 		ScraperControllerSettings: scraperhelper.ScraperControllerSettings{
 			CollectionInterval: 10 * time.Second,
@@ -131,6 +127,7 @@ func TestClientCreateRequest(t *testing.T) {
 	}
 
 	ctx := context.Background()
+	ctx = context.WithValue(ctx, endpointType("type"), typeIdx)
 	for _, test := range tests {
 		t.Run(test.desc, func(t *testing.T) {
 			req, err := test.client.createRequest(ctx, test.sr)
@@ -147,11 +144,9 @@ func TestClientCreateRequest(t *testing.T) {
 // createAPIRequest creates a request for api calls i.e. to introspection endpoint
 func TestAPIRequestCreate(t *testing.T) {
 	cfg := &Config{
-		ClientConfig: confighttp.ClientConfig{
+		IdxEndpoint: confighttp.ClientConfig{
 			Endpoint: "https://localhost:8089",
-			Auth: &configauth.Authentication{
-				AuthenticatorID: component.MustNewIDWithName("basicauth", "client"),
-			},
+			Auth:     &configauth.Authentication{AuthenticatorID: component.MustNewIDWithName("basicauth", "client")},
 		},
 		ScraperControllerSettings: scraperhelper.ScraperControllerSettings{
 			CollectionInterval: 10 * time.Second,
@@ -171,11 +166,12 @@ func TestAPIRequestCreate(t *testing.T) {
 	require.NoError(t, err)
 
 	ctx := context.Background()
+	ctx = context.WithValue(ctx, endpointType("type"), typeIdx)
 	req, err := client.createAPIRequest(ctx, "/test/endpoint")
 	require.NoError(t, err)
 
 	// build the expected request
-	expectedURL := client.endpoint.String() + "/test/endpoint"
+	expectedURL := client.clients[typeIdx].endpoint.String() + "/test/endpoint"
 	expected, _ := http.NewRequest(http.MethodGet, expectedURL, nil)
 
 	require.Equal(t, expected.URL, req.URL)