[Google Cloud Storage] Add support for alternative_host for system te…

…sts (#34413)

* [Google Cloud Storage] Add support for alternative_host to override HTTP client for system tests

* update changelog

(cherry picked from commit d1ef41c)

CHANGELOG.next.asciidoc

@@ -110,6 +110,34 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - httpjson input: Add request tracing logger. {issue}32402[32402] {pull}32412[32412]
 - Add cloudflare R2 to provider list in AWS S3 input. {pull}32620[32620]
 - Add support for single string containing multiple relation-types in getRFC5988Link. {pull}32811[32811]
+- Fix handling of invalid UserIP and LocalIP values. {pull}32896[32896]
+- Allow http_endpoint instances to share ports. {issue}32578[32578] {pull}33377[33377]
+- Improve httpjson documentation for split processor. {pull}33473[33473]
+- Added separation of transform context object inside httpjson. Introduced new clause `.parent_last_response.*` {pull}33499[33499]
+- Cloud Foundry input uses server-side filtering when retrieving logs. {pull}33456[33456]
+- Add `parse_aws_vpc_flow_log` processor. {pull}33656[33656]
+- Update `aws.vpcflow` dataset in AWS module have a configurable log `format` and to produce ECS 8.x fields. {pull}33699[33699]
+- Modified `aws-s3` input to reduce mutex contention when multiple SQS message are being processed concurrently. {pull}33658[33658]
+- Disable "event normalization" processing for the aws-s3 input to reduce allocations. {pull}33673[33673]
+- Add Common Expression Language input. {pull}31233[31233]
+- Add support for http+unix and http+npipe schemes in httpjson input. {issue}33571[33571] {pull}33610[33610]
+- Add support for http+unix and http+npipe schemes in cel input. {issue}33571[33571] {pull}33712[33712]
+- Add `decode_duration`, `move_fields` processors. {pull}31301[31301]
+- Add backup to bucket and delete functionality for the `aws-s3` input. {issue}30696[30696] {pull}33559[33559]
+- Add metrics for UDP packet processing. {pull}33870[33870]
+- Convert UDP input to v2 input. {pull}33930[33930]
+- Improve collection of risk information from Okta debug data. {issue}33677[33677] {pull}34030[34030]
+- Adding filename details from zip to response for httpjson {issue}33952[33952] {pull}34044[34044]
+- Allow user configuration of keep-alive behaviour for HTTPJSON and CEL inputs. {issue}33951[33951] {pull}34014[34014]
+- Add support for polling system UDP stats for UDP input metrics. {pull}34070[34070]
+- Add support for recognizing the log level in Elasticsearch JVM logs {pull}34159[34159]
+- Added metric `sqs_lag_time` for aws-s3 input. {pull}34306[34306]
+- Add metrics for TCP packet processing. {pull}34333[34333]
+- Add metrics for unix socket packet processing. {pull}34335[34335]
+- Add beta `take over` mode for `filestream` for simple migration from `log` inputs {pull}34292[34292]
+- Add pagination support for Salesforce module. {issue}34057[34057] {pull}34065[34065]
+- Allow users to redact sensitive data from CEL input debug logs. {pull}34302[34302]
+- Added support for HTTP destination override to Google Cloud Storage input. {pull}34413[34413]
 
 *Auditbeat*
 

x-pack/filebeat/input/gcs/client.go

@@ -7,6 +7,7 @@ package gcs
 import (
 	"context"
 	"errors"
+	"net/url"
 
 	"cloud.google.com/go/storage"
 	"google.golang.org/api/option"
@@ -15,6 +16,15 @@ import (
 )
 
 func fetchStorageClient(ctx context.Context, cfg config, log *logp.Logger) (*storage.Client, error) {
+	if cfg.AlternativeHost != "" {
+		var h *url.URL
+		h, err := url.Parse(cfg.AlternativeHost)
+		if err != nil {
+			return nil, err
+		}
+		h.Path = "storage/v1/"
+		return storage.NewClient(ctx, option.WithEndpoint(h.String()), option.WithoutAuthentication())
+	}
 	if cfg.Auth.CredentialsJSON != nil {
 		return storage.NewClient(ctx, option.WithCredentialsJSON([]byte(cfg.Auth.CredentialsJSON.AccountKey)))
 	} else if cfg.Auth.CredentialsFile != nil {

x-pack/filebeat/input/gcs/config.go

@@ -27,6 +27,8 @@ type config struct {
 	ParseJSON     *bool          `config:"parse_json,omitempty"`
 	BucketTimeOut *time.Duration `config:"bucket_timeout,omitempty"`
 	Buckets       []bucket       `config:"buckets" validate:"required"`
+	// This field is only used for system test purposes, to override the HTTP endpoint.
+	AlternativeHost string `config:"alternative_host,omitempty"`
 }
 
 // bucket contains the config for each specific object storage bucket in the root account