From 764791b134e62a8fba08a9fb5b8f37fa1fd1da68 Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Mon, 30 Jan 2023 12:22:46 +0100 Subject: [PATCH] [Google Cloud Storage] Add support for alternative_host for system tests (#34413) * [Google Cloud Storage] Add support for alternative_host to override HTTP client for system tests * update changelog (cherry picked from commit d1ef41c116c4370b245731eb40e4e894082b6214) --- CHANGELOG.next.asciidoc | 28 ++++++++++++++++++++++++++++ x-pack/filebeat/input/gcs/client.go | 10 ++++++++++ x-pack/filebeat/input/gcs/config.go | 2 ++ 3 files changed, 40 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a758bccf7044..86babe6fe170 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -110,6 +110,34 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff] - httpjson input: Add request tracing logger. {issue}32402[32402] {pull}32412[32412] - Add cloudflare R2 to provider list in AWS S3 input. {pull}32620[32620] - Add support for single string containing multiple relation-types in getRFC5988Link. {pull}32811[32811] +- Fix handling of invalid UserIP and LocalIP values. {pull}32896[32896] +- Allow http_endpoint instances to share ports. {issue}32578[32578] {pull}33377[33377] +- Improve httpjson documentation for split processor. {pull}33473[33473] +- Added separation of transform context object inside httpjson. Introduced new clause `.parent_last_response.*` {pull}33499[33499] +- Cloud Foundry input uses server-side filtering when retrieving logs. {pull}33456[33456] +- Add `parse_aws_vpc_flow_log` processor. {pull}33656[33656] +- Update `aws.vpcflow` dataset in AWS module have a configurable log `format` and to produce ECS 8.x fields. {pull}33699[33699] +- Modified `aws-s3` input to reduce mutex contention when multiple SQS message are being processed concurrently. {pull}33658[33658] +- Disable "event normalization" processing for the aws-s3 input to reduce allocations. {pull}33673[33673] +- Add Common Expression Language input. {pull}31233[31233] +- Add support for http+unix and http+npipe schemes in httpjson input. {issue}33571[33571] {pull}33610[33610] +- Add support for http+unix and http+npipe schemes in cel input. {issue}33571[33571] {pull}33712[33712] +- Add `decode_duration`, `move_fields` processors. {pull}31301[31301] +- Add backup to bucket and delete functionality for the `aws-s3` input. {issue}30696[30696] {pull}33559[33559] +- Add metrics for UDP packet processing. {pull}33870[33870] +- Convert UDP input to v2 input. {pull}33930[33930] +- Improve collection of risk information from Okta debug data. {issue}33677[33677] {pull}34030[34030] +- Adding filename details from zip to response for httpjson {issue}33952[33952] {pull}34044[34044] +- Allow user configuration of keep-alive behaviour for HTTPJSON and CEL inputs. {issue}33951[33951] {pull}34014[34014] +- Add support for polling system UDP stats for UDP input metrics. {pull}34070[34070] +- Add support for recognizing the log level in Elasticsearch JVM logs {pull}34159[34159] +- Added metric `sqs_lag_time` for aws-s3 input. {pull}34306[34306] +- Add metrics for TCP packet processing. {pull}34333[34333] +- Add metrics for unix socket packet processing. {pull}34335[34335] +- Add beta `take over` mode for `filestream` for simple migration from `log` inputs {pull}34292[34292] +- Add pagination support for Salesforce module. {issue}34057[34057] {pull}34065[34065] +- Allow users to redact sensitive data from CEL input debug logs. {pull}34302[34302] +- Added support for HTTP destination override to Google Cloud Storage input. {pull}34413[34413] *Auditbeat* diff --git a/x-pack/filebeat/input/gcs/client.go b/x-pack/filebeat/input/gcs/client.go index bb2b9135b5db..128ab753d926 100644 --- a/x-pack/filebeat/input/gcs/client.go +++ b/x-pack/filebeat/input/gcs/client.go @@ -7,6 +7,7 @@ package gcs import ( "context" "errors" + "net/url" "cloud.google.com/go/storage" "google.golang.org/api/option" @@ -15,6 +16,15 @@ import ( ) func fetchStorageClient(ctx context.Context, cfg config, log *logp.Logger) (*storage.Client, error) { + if cfg.AlternativeHost != "" { + var h *url.URL + h, err := url.Parse(cfg.AlternativeHost) + if err != nil { + return nil, err + } + h.Path = "storage/v1/" + return storage.NewClient(ctx, option.WithEndpoint(h.String()), option.WithoutAuthentication()) + } if cfg.Auth.CredentialsJSON != nil { return storage.NewClient(ctx, option.WithCredentialsJSON([]byte(cfg.Auth.CredentialsJSON.AccountKey))) } else if cfg.Auth.CredentialsFile != nil { diff --git a/x-pack/filebeat/input/gcs/config.go b/x-pack/filebeat/input/gcs/config.go index a322ecbdb9ea..a22ee5fde8b8 100644 --- a/x-pack/filebeat/input/gcs/config.go +++ b/x-pack/filebeat/input/gcs/config.go @@ -27,6 +27,8 @@ type config struct { ParseJSON *bool `config:"parse_json,omitempty"` BucketTimeOut *time.Duration `config:"bucket_timeout,omitempty"` Buckets []bucket `config:"buckets" validate:"required"` + // This field is only used for system test purposes, to override the HTTP endpoint. + AlternativeHost string `config:"alternative_host,omitempty"` } // bucket contains the config for each specific object storage bucket in the root account