Merge branch 'main' into filebeat_shipper_integration_test

.ci/bump-golang.yml

@@ -120,6 +120,15 @@ targets:
       content: ':go-version: {{ source "latestGoVersion" }}'
       file: libbeat/docs/version.asciidoc
       matchpattern: ':go-version: \d+.\d+.\d+'
+  update-devcontainer:
+    name: "Update .devcontainer/devcontainer.json"
+    sourceid: gomod
+    scmid: githubConfig
+    kind: file
+    spec:
+      content: 'go:1-{{ source "gomod" }}'
+      file: ./.devcontainer/devcontainer.json
+      matchpattern: 'go:1-\d+.\d+'
   update-auditbeat-dockerfile:
     name: "Update Auditbeat Dockerfile"
     sourceid: latestGoVersion

.devcontainer/devcontainer.json

@@ -0,0 +1,31 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/go
+// If you're wondering how to get SSH working for git pushes, make sure to ssh-add ~/.ssh/<YOURPRIVATEKEY> on your host system
+// to make sure the ssh agent has your key
+{
+	"name": "Beats Development Container",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/go:1-1.20-bullseye",
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/node:1": {},
+		"ghcr.io/devcontainers/features/python:1": {},
+		"ghcr.io/devcontainers/features/github-cli:1": {},
+		"ghcr.io/devcontainers/features/common-utils:2": {},
+		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// Mage is installed this way, and not via the feature plugin because that plugin was
+	// broken for me, and mage install is simple enough
+	"postCreateCommand": "cd /opt/; sudo mkdir mage; sudo chown $USER:$(id -g) mage; git clone --depth=1 https://github.com/magefile/mage && cd mage && go run bootstrap.go; npm i -g @elastic/synthetics; sudo env \"PATH=$PATH\" npx -yes playwright install-deps"
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.github/CODEOWNERS

@@ -66,6 +66,7 @@ CHANGELOG*
 /licenses/ @elastic/elastic-agent-data-plane
 /metricbeat/ @elastic/elastic-agent-data-plane
 /metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module.
+/metricbeat/helper/kubernetes @elastic/obs-cloudnative-monitoring
 /metricbeat/module/ @elastic/integrations
 /metricbeat/module/apache @elastic/obs-infraobs-integrations
 /metricbeat/module/beat/ @elastic/infra-monitoring-ui
@@ -115,6 +116,8 @@ CHANGELOG*
 /x-pack/filebeat/input/gcs/ @elastic/security-external-integrations
 /x-pack/filebeat/input/http_endpoint/ @elastic/security-external-integrations
 /x-pack/filebeat/input/httpjson/ @elastic/security-external-integrations
+/x-pack/filebeat/input/internal/httplog @elastic/security-external-integrations
+/x-pack/filebeat/input/internal/httpmon @elastic/security-external-integrations
 /x-pack/filebeat/input/lumberjack/ @elastic/security-external-integrations
 /x-pack/filebeat/input/netflow/ @elastic/security-external-integrations
 /x-pack/filebeat/input/o365audit/ @elastic/security-external-integrations

.golangci.yml

@@ -2,6 +2,9 @@
 run:
   # timeout for analysis, e.g. 30s, 5m, default is 1m
   timeout: 15m
+  build-tags: 
+    - synthetics
+    - integration
 
 issues:
   # Maximum count of issues with the same text.
@@ -76,6 +79,7 @@ linters-settings:
     exclude-functions:
       - (github.com/elastic/elastic-agent-libs/mapstr.M).Delete # Only returns ErrKeyNotFound, can safely be ignored.
       - (github.com/elastic/elastic-agent-libs/mapstr.M).Put # Can only fail on type conversions, usually safe to ignore.
+      - (github.com/elastic/elastic-agent-libs/logp).TestingSetup # Cannot return a non-nil error using the provided API.
 
   errorlint:
     # Check whether fmt.Errorf uses the %w verb for formatting errors. See the readme for caveats

CHANGELOG-developer.next.asciidoc

@@ -65,6 +65,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - The `queue.Batch` API now provides access to individual events instead of an array. {pull}31699[31699]
 - Rename `queue.Batch.ACK()` to `queue.Batch.Done()`. {pull}31903[31903]
 - `queue.ACKListener` has been removed. Queue configurations now accept an explicit callback function for ACK handling. {pull}35078[35078]
+- Split split httpmon out of x-pack/filebeat/input/internal/httplog. {pull}36385[36385]
 
 ==== Bugfixes
 
@@ -84,6 +85,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Skip HTTPJSON flakey test. {issue}34929[34929] {pull}35138[35138]
 - Fix ingest pipeline for panw module to parse url scheme correctly {pull}35757[35757]
 - Renamed an httpjson input metric to follow naming conventions. `httpjson_interval_pages_total` was renamed to `httpjson_interval_pages` because the `_total` suffix is reserved for counters. {issue}35933[35933] {pull}36169[36169]
+- Fixed some race conditions in tests {pull}36185[36185]
 
 ==== Added
 

CHANGELOG.asciidoc

@@ -3,6 +3,68 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.9.2]]
+=== Beats version 8.9.2
+https://github.com/elastic/beats/compare/v8.9.1\...v8.9.2[View commits]
+
+==== Bugfixes
+
+*Filebeat*
+
+- Fix panic when redact option is not provided to CEL input. {issue}36387[36387] {pull}36388[36388]
+- Update mito CEL extension library to v1.5.0. {pull}36146[36146]
+
+==== Added
+
+*Metricbeat*
+
+- Add Azure resource tags support to Azure Billing module {pull}36428[36428]
+
+
+[[release-notes-8.9.1]]
+=== Beats version 8.9.1
+https://github.com/elastic/beats/compare/v8.9.0\...v8.9.1[View commits]
+
+==== Bugfixes
+
+*Auditbeat*
+
+- auditd: Expanded the bitmask applied to ECS file.mode so that the SUID, SGID, and sticky bits can be represented. {pull}36294[36294]
+
+*Filebeat*
+
+- Fix panic when SQS input metrics getter is invoked. {pull}36101[36101] {issue}36077[36077]
+- Make CEL input's `now` global variable static for evaluation lifetime. {pull}36107[36107]
+- Fix handling of TCP/UDP address resolution during metric initialization. {issue}35064[35064] {pull}36287[36287]
+- Fix handling of Juniper SRX structured data when there is no leading Junos element. {issue}36270[36270] {pull}36308[36308]
+- Remove erroneous error log in GCPPubSub input. {pull}36296[36296]
+- Fix Filebeat Cisco module with missing escape character. {issue}36325[36325] {pull}36326[36326]
+
+*Heartbeat*
+
+- Enable heartbeat-wide publish timeout setting with run_once. {pull}35721[35721]
+- Added default timezone UTC to heartbeat docker images to fix synthetics journeys navigation errors. {pull}36193[36193]
+
+*Packetbeat*
+
+- Fix handling of Npcap installation options from Fleet. {pull}35541[35541] {pull}35935[35935]
+
+*Winlogbeat*
+
+- Fix powershell details regexp to prevent excessive backtracking when processing command invocations. {pull}36178[36178]
+
+==== Added
+
+*Affecting all Beats*
+
+- Upgrade Go to 1.19.12 {pull}36246[36246]
+- Add warning message to SysV init scripts for RPM-based systems that lack `/etc/rc.d/init.d/functions`. {issue}35708[35708] {pull}36188[36188]
+
+*Filebeat*
+
+- Add support for endpoint resolver in AWS config {pull}36208[36208]
+
+
 [[release-notes-8.9.0]]
 === Beats version 8.9.0
 https://github.com/elastic/beats/compare/v8.8.2\...v8.9.0[View commits]

CHANGELOG.next.asciidoc

@@ -62,35 +62,19 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Eliminate cloning of event in deepUpdate {pull}35945[35945]
 - Fix ndjson parser to store JSON fields correctly under `target` {issue}29395[29395]
 - Support build of projects outside of beats directory {pull}36126[36126]
-
+- Fix environment capture by `add_process_metadata` processor. {issue}36469[36469] {pull}36471[36471]
 
 
 *Auditbeat*
 
-- auditd: Expanded the bitmask applied to ECS file.mode so that the SUID, SGID, and sticky bits can be represented. {pull}36294[36294]
 
 *Filebeat*
 
 - [Gcs Input] - Added missing locks for safe concurrency {pull}34914[34914]
 - Fix the ignore_inactive option being ignored in Filebeat's filestream input {pull}34770[34770]
 - Fix TestMultiEventForEOFRetryHandlerInput unit test of CometD input {pull}34903[34903]
 - Add input instance id to request trace filename for httpjson and cel inputs {pull}35024[35024]
-- Fix panic in TCP and UDP inputs on Linux when collecting socket metrics from OS. {issue}35064[35064]
-- Correctly collect TCP and UDP metrics for unspecified address values. {pull}35111[35111]
-- Fix base for UDP and TCP queue metrics and UDP drops metric. {pull}35123[35123]
-- Sanitize filenames for request tracer in httpjson input. {pull}35143[35143]
-- decode_cef processor: Fix ECS output by making `observer.ip` into an array of strings instead of string. {issue}35140[35140] {pull}35149[35149]
-- Fix handling of MySQL audit logs with strict JSON parser. {issue}35158[35158] {pull}35160[35160]
-- Sanitize filenames for request tracer in cel input. {pull}35154[35154]
-- Fix accidental error overwrite in defer statement in entityanalytics Azure AD input. {issue}35153[35153] {pull}35169[35169]
-- Fixing the grok expression outputs of log files {pull}35221[35221]
 - Fixes "Can only start an input when all related states are finished" error when running under Elastic-Agent {pull}35250[35250] {issue}33653[33653]
-- Move repeated Windows event channel not found errors in winlog input to debug level.  {issue}35314[35314] {pull}35317[35317]
-- Fix crash when processing forwarded logs missing a message. {issue}34705[34705] {pull}34865[34865]
-- Fix crash when loading azurewebstorage cursor with no partially processed data. {pull}35433[35433]
-- Add support in s3 input for JSON with array of objects. {pull}35475[35475]
-- RFC5424 syslog timestamps with offset 'Z' will be treated as UTC rather than using the default timezone. {pull}35360[35360]
-- Fix syslog message parsing for fortinet.firewall to take into account quoted values. {pull}35522[35522]
 - [system] sync system/auth dataset with system integration 1.29.0. {pull}35581[35581]
 - [GCS Input] - Fixed an issue where bucket_timeout was being applied to the entire bucket poll interval and not individual bucket object read operations. Fixed a map write concurrency issue arising from data races when using a high number of workers. Fixed the flaky tests that were present in the GCS test suit. {pull}35605[35605]
 - Fix filestream false positive log error "filestream input with ID 'xyz' already exists" {issue}31767[31767]
@@ -116,34 +100,13 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix handling of Juniper SRX structured data when there is no leading junos element. {issue}36270[36270] {pull}36308[36308]
 - Remove erroneous error log in GCPPubSub input. {pull}36296[36296]
 - Fix Filebeat Cisco module with missing escape character {issue}36325[36325] {pull}36326[36326]
+- Fix panic when redact option is not provided to CEL input. {issue}36387[36387] {pull}36388[36388]
+- Remove 'onFilteredOut' and 'onDroppedOnPublish' callback logs {issue}36299[36299] {pull}36399[36399]
+- Added a fix for Crowdstrike pipeline handling process arrays {pull}36496[36496]
 
 *Heartbeat*
 
 - Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702]
-- Fix broken zip URL monitors. NOTE: Zip URL Monitors will be removed in version 8.7 and replaced with project monitors. {pull}33723[33723]
-- Fix integration hashing to prevent reloading all when updated. {pull}34697[34697]
-- Fix release of job limit semaphore when context is cancelled. {pull}34697[34697]
-- Fix bug where states.duration_ms was incorrect type. {pull}33563[33563]
-- Fix handling of long UDP messages in UDP input. {issue}33836[33836] {pull}33837[33837]
-- Fix browser monitor summary reporting as up when monitor is down. {issue}33374[33374] {pull}33819[33819]
-- Fix beat capabilities on Docker image. {pull}33584[33584]
-- Fix serialization of state duration to avoid scientific notation. {pull}34280[34280]
-- Enable nodejs engine strict validation when bundling synthetics. {pull}34470[34470]
-with the ecs field name `container`. {pull}34403[34403]
-automatic splitting at root level, if root level element is an array. {pull}34155[34155]
-- Fix broken mapping for state.ends field. {pull}34891[34891]
-- Fix issue using projects in airgapped environments by disabling npm audit. {pull}34936[34936]
-- Fix broken state ID location naming. {pull}35336[35336]
-- Fix project monitor temp directories permission to include group access. {pull}35398[35398]
-- Fix output pipeline exit on run_once. {pull}35376[35376]
-- Fix formatting issue with socket trace timeout. {pull}35434[35434]
-- Update gval version. {pull}35636[35636]
-- Fix serialization of processors when running diagnostics. {pull}35698[35698]
-- Filter dev flags for ui monitors inside synthetics_args. {pull}35788[35788]
-- Fix temp dir running out of space with project monitors. {issue}35843[35843]
-- Fixing the grok expression outputs of log files {pull}35221[35221]
-- Enable heartbeat-wide publish timeout setting with run_once. {pull}35721[35721]
-- Added default timezone UTC to heartbeat docker images to fix synthetics journeys navigation errors. {pull}36193[36193]
 
 *Metricbeat*
 
@@ -163,19 +126,19 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Resolve statsd module's prematurely halting of metrics parsing upon encountering an invalid packet. {pull}35075[35075]
 - Fix the gap in fetching forecast API metrics at the end of each month for Azure billing module  {pull}36142[36142]
 - Add option in SQL module to execute queries for all dbs. {pull}35688[35688]
+- Fix GCP compute metadata. {pull}36338[36338]
 - Add support for api_key authentication in elasticsearch module  {pull}36274[36274]
 - Add remaining dimensions for azure storage account to make them available for tsdb enablement. {pull}36331[36331]
+- Add missing 'TransactionType' dimension for Azure Storage Account. {pull}36413[36413]
 
 *Osquerybeat*
 
 
 *Packetbeat*
 
-- Fix handling of Npcap installation options from Fleet. {pull}35541[35541] {pull}35935[35935]
 
 *Winlogbeat*
 
-- Fix powershell details regexp to prevent excessive backtracking when processing command invocations. {pull}36178[36178]
 
 *Elastic Logging Plugin*
 
@@ -188,6 +151,8 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - When running under Elastic-Agent the status is now reported per Unit instead of the whole Beat {issue}35874[35874] {pull}36183[36183]
 - Add warning message to SysV init scripts for RPM-based systems that lack `/etc/rc.d/init.d/functions`. {issue}35708[35708] {pull}36188[36188]
 - Mark `translate_sid` processor is GA. {issue}36279[36279] {pull}36280[36280]
+- dns processor: Add support for forward lookups (`A`, `AAAA`, and `TXT`). {issue}11416[11416] {pull}36394[36394]
+- Mark `syslog` processor as GA, improve docs about how processor handles syslog messages. {issue}36416[36416] {pull}36417[36417]
 
 *Auditbeat*
 
@@ -242,6 +207,11 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}99999[99999]
 - Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}36286[36286]
 - [Azure] Add input metrics to the azure-eventhub input. {pull}35739[35739]
+- Reduce HTTPJSON metrics allocations. {pull}36282[36282]
+- Add support for a simplified input configuraton when running under Elastic-Agent {pull}36390[36390]
+- Make HTTPJSON response body decoding errors more informative. {pull}36481[36481]
+- Allow fine-grained control of entity analytics API requests for Okta provider. {issue}36440[36440] {pull}36492[36492]
+- Add support for expanding `journald.process.capabilities` into the human-readable effective capabilities in the ECS `process.thread.capabilities.effective` field. {issue}36454[36454] {pull}36470[36470]
 
 *Auditbeat*
 
@@ -263,6 +233,7 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 - Migrate Azure Billing, Monitor, and Storage metricsets to the newer SDK. {pull}33585[33585]
 - Add support for float64 values parsing for statsd metrics of counter type. {pull}35099[35099]
 - Add kubernetes.deployment.status.* fields for Kubernetes module {pull}35999[35999]
+- Add Azure resource tags support to Azure Billing module {pull}36428[36428]
 
 
 *Osquerybeat*
@@ -330,3 +301,9 @@ automatic splitting at root level, if root level element is an array. {pull}3415
 
 
 
+
+
+
+
+
+