From 4cd490001cfba3f0f1993c34681ed84d728c8e79 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 25 Jun 2024 14:32:56 +0200 Subject: [PATCH 01/12] Add basic samples --- dev-tools/packaging/packages.yml | 10 +++ .../samples/darwin/hostmetrics_resources.yml | 35 ++++++++++ .../pkg/otel/samples/darwin/platformlogs.yml | 26 +++++++ .../darwin/platformlogs_hostmetrics.yml | 38 ++++++++++ .../samples/linux/hostmetrics_resources.yml | 35 ++++++++++ .../otel/samples/linux/logs_hostmetrics.yml | 42 +++++++++++ .../samples/linux/logs_metrics_traces.yml | 70 +++++++++++++++++++ .../pkg/otel/samples/windows/hostmetrics.yml | 29 ++++++++ .../samples/windows/hostmetrics_resources.yml | 35 ++++++++++ 9 files changed, 320 insertions(+) create mode 100644 internal/pkg/otel/samples/darwin/hostmetrics_resources.yml create mode 100644 internal/pkg/otel/samples/darwin/platformlogs.yml create mode 100644 internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml create mode 100644 internal/pkg/otel/samples/linux/hostmetrics_resources.yml create mode 100644 internal/pkg/otel/samples/linux/logs_hostmetrics.yml create mode 100644 internal/pkg/otel/samples/linux/logs_metrics_traces.yml create mode 100644 internal/pkg/otel/samples/windows/hostmetrics.yml create mode 100644 internal/pkg/otel/samples/windows/hostmetrics_resources.yml diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 5e5156a0cfe..062e5f1164f 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -87,6 +87,9 @@ shared: 'data/{{.BeatName}}-{{ commit_short }}/otelcol': source: '{{ repo.RootDir }}/dev-tools/packaging/files/linux/otelcol.sh' mode: 0755 + 'otel_samples': + source: '{{ repo.RootDir }}/internal/pkg/otel/samples/linux/' + mode: 0755 - &agent_binary_common_files LICENSE.txt: @@ -160,6 +163,9 @@ shared: 'data/{{.BeatName}}-{{ commit_short }}/otelcol': source: '{{ repo.RootDir }}/dev-tools/packaging/files/darwin/otelcol.sh' mode: 0755 + 'otel_samples': + source: '{{ repo.RootDir }}/internal/pkg/otel/samples/darwin/' + mode: 0755 <<: *agent_darwin_app_bundle_files <<: *agent_binary_common_files @@ -201,6 +207,10 @@ shared: 'data/{{.BeatName}}-{{ commit_short }}/otelcol.ps1': source: '{{ repo.RootDir }}/dev-tools/packaging/files/windows/otelcol.ps1' mode: 0755 + # windows not supported yet + # 'otel_samples': + # source: '{{ repo.RootDir }}/internal/pkg/otel/samples/windows/' + # mode: 0755 - &agent_docker_spec <<: *agent_binary_spec diff --git a/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml b/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml new file mode 100644 index 00000000000..4c99dd591c0 --- /dev/null +++ b/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml @@ -0,0 +1,35 @@ +receivers: + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + +processors: + resourcedetection/system: + detectors: ["system"] + system: + hostname_sources: ["os"] + +exporters: + # Exporter to print the first 5 logs/metrics and then every 1000th + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 1000 + + # Exporter to send logs and metrics to Elasticsearch + elasticsearch/bulk: + endpoints: [http://localhost:9200] + user: otel + password: password + +service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [resourcedetection/system] + exporters: [debug, elasticsearch/bulk] \ No newline at end of file diff --git a/internal/pkg/otel/samples/darwin/platformlogs.yml b/internal/pkg/otel/samples/darwin/platformlogs.yml new file mode 100644 index 00000000000..3655f737649 --- /dev/null +++ b/internal/pkg/otel/samples/darwin/platformlogs.yml @@ -0,0 +1,26 @@ +receivers: + # Receiver for platform specific log files + filelog/platformlogs: + include: [ /var/log/*.log ] +# start_at: beginning + +exporters: + + # Exporter to print the first 5 logs/metrics and then every 1000th + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 1000 + + # Exporter to send logs and metrics to Elasticsearch + elasticsearch/bulk: + endpoints: [http://localhost:9200] + user: otel + password: password + +service: + pipelines: + logs/platformlogs: + receivers: [filelog/platformlogs] + processors: [] + exporters: [debug, elasticsearch/bulk] \ No newline at end of file diff --git a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml new file mode 100644 index 00000000000..c478802757c --- /dev/null +++ b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml @@ -0,0 +1,38 @@ +receivers: + # Receiver for platform specific log files + filelog/platformlogs: + include: [ /var/log/*.log ] +# start_at: beginning + + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + +exporters: + # Exporter to print the first 5 logs/metrics and then every 1000th + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 1000 + + # Exporter to send logs and metrics to Elasticsearch + elasticsearch/bulk: + endpoints: [http://localhost:9200] + user: otel + password: password + +service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [] + exporters: [debug, elasticsearch/bulk] + logs/platformlogs: + receivers: [filelog/platformlogs] + processors: [] + exporters: [debug, elasticsearch/bulk] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/hostmetrics_resources.yml b/internal/pkg/otel/samples/linux/hostmetrics_resources.yml new file mode 100644 index 00000000000..4c99dd591c0 --- /dev/null +++ b/internal/pkg/otel/samples/linux/hostmetrics_resources.yml @@ -0,0 +1,35 @@ +receivers: + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + +processors: + resourcedetection/system: + detectors: ["system"] + system: + hostname_sources: ["os"] + +exporters: + # Exporter to print the first 5 logs/metrics and then every 1000th + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 1000 + + # Exporter to send logs and metrics to Elasticsearch + elasticsearch/bulk: + endpoints: [http://localhost:9200] + user: otel + password: password + +service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [resourcedetection/system] + exporters: [debug, elasticsearch/bulk] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml new file mode 100644 index 00000000000..7c98358e852 --- /dev/null +++ b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml @@ -0,0 +1,42 @@ +receivers: + + # Receiver for platform specific log files + filelog/platformlogs: + include: [ /var/log/*.log ] +# start_at: beginning + + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + +exporters: + + # Exporter to print the first 5 logs/metrics and then every 1000th + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 1000 + + elasticsearch/bulk: + endpoints: [http://localhost:9200] + user: otel + password: password + logs_index: logs-generic-default + metrics_index: metrics-generic-default + +service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [] + exporters: [debug, elasticsearch/bulk] + + logs/platformlogs: + receivers: [filelog/platformlogs] + processors: [] + exporters: [debug, elasticsearch/bulk] diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml new file mode 100644 index 00000000000..5ea28f9d608 --- /dev/null +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml @@ -0,0 +1,70 @@ +receivers: + + # Receiver for platform specific log files + filelog/platformlogs: + include: [ /var/log/*.log ] +# start_at: beginning + + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + + # Receiver for logs, traces, and metrics from SDKs + otlp/fromsdk: + protocols: + grpc: + http: + +exporters: + + # Exporter to print the first 5 logs/metrics and then every 1000th + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 200 + + + otlp/apm: + endpoint: "http://localhost:8200" + headers: + # Elastic APM Server secret token or API key + Authorization: "Bearer <>" + + elasticsearch/bulk: + endpoints: [http://localhost:9200] + user: otel + password: password + logs_index: logs-generic-default + metrics_index: metrics-generic-default + +service: + pipelines: + traces/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [debug, otlp/apm] + + metrics/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [debug, otlp/apm] + + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [] + exporters: [debug, elasticsearch/bulk] + + logs/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [debug, otlp/apm] + + logs/platformlogs: + receivers: [filelog/platformlogs] + processors: [] + exporters: [debug, elasticsearch/bulk] \ No newline at end of file diff --git a/internal/pkg/otel/samples/windows/hostmetrics.yml b/internal/pkg/otel/samples/windows/hostmetrics.yml new file mode 100644 index 00000000000..2805aa1052e --- /dev/null +++ b/internal/pkg/otel/samples/windows/hostmetrics.yml @@ -0,0 +1,29 @@ +receivers: + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + +exporters: + # Exporter to print the first 5 logs/metrics and then every 1000th + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 1000 + + # Exporter to send logs and metrics to Elasticsearch + elasticsearch/bulk: + endpoints: [http://localhost:9200] + user: otel + password: password + +service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [] + exporters: [debug, elasticsearch/bulk] diff --git a/internal/pkg/otel/samples/windows/hostmetrics_resources.yml b/internal/pkg/otel/samples/windows/hostmetrics_resources.yml new file mode 100644 index 00000000000..4c99dd591c0 --- /dev/null +++ b/internal/pkg/otel/samples/windows/hostmetrics_resources.yml @@ -0,0 +1,35 @@ +receivers: + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + +processors: + resourcedetection/system: + detectors: ["system"] + system: + hostname_sources: ["os"] + +exporters: + # Exporter to print the first 5 logs/metrics and then every 1000th + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 1000 + + # Exporter to send logs and metrics to Elasticsearch + elasticsearch/bulk: + endpoints: [http://localhost:9200] + user: otel + password: password + +service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [resourcedetection/system] + exporters: [debug, elasticsearch/bulk] \ No newline at end of file From 853d93b5ede8b24d3c200f966c890778946220c0 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 25 Jun 2024 14:44:11 +0200 Subject: [PATCH 02/12] add commented out api key option --- internal/pkg/otel/samples/darwin/hostmetrics_resources.yml | 1 + internal/pkg/otel/samples/darwin/platformlogs.yml | 1 + internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml | 1 + internal/pkg/otel/samples/linux/hostmetrics_resources.yml | 1 + internal/pkg/otel/samples/linux/logs_hostmetrics.yml | 1 + internal/pkg/otel/samples/linux/logs_metrics_traces.yml | 1 + internal/pkg/otel/samples/windows/hostmetrics.yml | 1 + internal/pkg/otel/samples/windows/hostmetrics_resources.yml | 1 + 8 files changed, 8 insertions(+) diff --git a/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml b/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml index 4c99dd591c0..3e83059add1 100644 --- a/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml +++ b/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml @@ -26,6 +26,7 @@ exporters: endpoints: [http://localhost:9200] user: otel password: password + # api_key: <> # Elasticsearch API Key in "encoded" format. service: pipelines: diff --git a/internal/pkg/otel/samples/darwin/platformlogs.yml b/internal/pkg/otel/samples/darwin/platformlogs.yml index 3655f737649..dd8c750412c 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs.yml @@ -17,6 +17,7 @@ exporters: endpoints: [http://localhost:9200] user: otel password: password + # api_key: <> # Elasticsearch API Key in "encoded" format. service: pipelines: diff --git a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml index c478802757c..f89ae584673 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml @@ -25,6 +25,7 @@ exporters: endpoints: [http://localhost:9200] user: otel password: password + # api_key: <> # Elasticsearch API Key in "encoded" format. service: pipelines: diff --git a/internal/pkg/otel/samples/linux/hostmetrics_resources.yml b/internal/pkg/otel/samples/linux/hostmetrics_resources.yml index 4c99dd591c0..3e83059add1 100644 --- a/internal/pkg/otel/samples/linux/hostmetrics_resources.yml +++ b/internal/pkg/otel/samples/linux/hostmetrics_resources.yml @@ -26,6 +26,7 @@ exporters: endpoints: [http://localhost:9200] user: otel password: password + # api_key: <> # Elasticsearch API Key in "encoded" format. service: pipelines: diff --git a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml index 7c98358e852..cbca679f41c 100644 --- a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml @@ -28,6 +28,7 @@ exporters: password: password logs_index: logs-generic-default metrics_index: metrics-generic-default + # api_key: <> # Elasticsearch API Key in "encoded" format. service: pipelines: diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml index 5ea28f9d608..4e291af767c 100644 --- a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml @@ -39,6 +39,7 @@ exporters: endpoints: [http://localhost:9200] user: otel password: password + # api_key: <> # Elasticsearch API Key in "encoded" format. logs_index: logs-generic-default metrics_index: metrics-generic-default diff --git a/internal/pkg/otel/samples/windows/hostmetrics.yml b/internal/pkg/otel/samples/windows/hostmetrics.yml index 2805aa1052e..4872b658a07 100644 --- a/internal/pkg/otel/samples/windows/hostmetrics.yml +++ b/internal/pkg/otel/samples/windows/hostmetrics.yml @@ -19,6 +19,7 @@ exporters: elasticsearch/bulk: endpoints: [http://localhost:9200] user: otel + # api_key: <> # Elasticsearch API Key in "encoded" format. password: password service: diff --git a/internal/pkg/otel/samples/windows/hostmetrics_resources.yml b/internal/pkg/otel/samples/windows/hostmetrics_resources.yml index 4c99dd591c0..3e83059add1 100644 --- a/internal/pkg/otel/samples/windows/hostmetrics_resources.yml +++ b/internal/pkg/otel/samples/windows/hostmetrics_resources.yml @@ -26,6 +26,7 @@ exporters: endpoints: [http://localhost:9200] user: otel password: password + # api_key: <> # Elasticsearch API Key in "encoded" format. service: pipelines: From f6caafe597539828d7814459b7ef2818d0f2f8c3 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 25 Jun 2024 20:08:55 +0200 Subject: [PATCH 03/12] updated host metrics and host-logs-metrics-traces --- .../samples/darwin/hostmetrics_resources.yml | 6 +- .../otel/samples/darwin/logs_host_metrics.yml | 38 +++++++++++ .../samples/darwin/logs_metrics_traces.yml | 65 +++++++++++++++++++ .../pkg/otel/samples/darwin/platformlogs.yml | 6 +- .../darwin/platformlogs_hostmetrics.yml | 6 +- .../samples/linux/hostmetrics_resources.yml | 6 +- .../otel/samples/linux/logs_host_metrics.yml | 38 +++++++++++ .../otel/samples/linux/logs_hostmetrics.yml | 6 +- .../linux/logs_metrics_traces copy.yml | 65 +++++++++++++++++++ .../samples/linux/logs_metrics_traces.yml | 6 +- .../pkg/otel/samples/windows/hostmetrics.yml | 6 +- .../samples/windows/hostmetrics_resources.yml | 6 +- 12 files changed, 222 insertions(+), 32 deletions(-) create mode 100644 internal/pkg/otel/samples/darwin/logs_host_metrics.yml create mode 100644 internal/pkg/otel/samples/darwin/logs_metrics_traces.yml create mode 100644 internal/pkg/otel/samples/linux/logs_host_metrics.yml create mode 100644 internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml diff --git a/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml b/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml index 3e83059add1..a5b4c9bf567 100644 --- a/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml +++ b/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml @@ -23,10 +23,8 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch/bulk: - endpoints: [http://localhost:9200] - user: otel - password: password - # api_key: <> # Elasticsearch API Key in "encoded" format. + endpoints: [<>] + api_key: <> service: pipelines: diff --git a/internal/pkg/otel/samples/darwin/logs_host_metrics.yml b/internal/pkg/otel/samples/darwin/logs_host_metrics.yml new file mode 100644 index 00000000000..65b1fe1759a --- /dev/null +++ b/internal/pkg/otel/samples/darwin/logs_host_metrics.yml @@ -0,0 +1,38 @@ +receivers: + + # Receiver for platform specific log files + filelog/platformlogs: + include: [ /var/log/*.log ] +# start_at: beginning + + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + +processors: + elasticinframetrics: + +exporters: + + elasticsearch/bulk: + endpoints: [<>] + api_key: <> + logs_index: logs-generic-default + metrics_index: metrics-generic-default + +service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [elasticinframetrics] + exporters: [elasticsearch/bulk] + + logs/platformlogs: + receivers: [filelog/platformlogs] + processors: [] + exporters: [elasticsearch/bulk] diff --git a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml new file mode 100644 index 00000000000..b33b47fe3ab --- /dev/null +++ b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml @@ -0,0 +1,65 @@ +receivers: + + # Receiver for platform specific log files + filelog/platformlogs: + include: [ /var/log/*.log ] +# start_at: beginning + + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + + # Receiver for logs, traces, and metrics from SDKs + otlp/fromsdk: + protocols: + grpc: + http: + +processors: + elasticinframetrics: + +exporters: + + otlp/apm: + endpoint: <> + headers: + # Elastic APM Server secret token or API key + Authorization: "Bearer <>" + + elasticsearch/bulk: + endpoints: [<>] + api_key: <> + logs_index: logs-generic-default + metrics_index: metrics-generic-default + +service: + pipelines: + traces/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [otlp/apm] + + metrics/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [otlp/apm] + + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [elasticinframetrics] + exporters: [elasticsearch/bulk] + + logs/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [otlp/apm] + + logs/platformlogs: + receivers: [filelog/platformlogs] + processors: [] + exporters: [elasticsearch/bulk] \ No newline at end of file diff --git a/internal/pkg/otel/samples/darwin/platformlogs.yml b/internal/pkg/otel/samples/darwin/platformlogs.yml index dd8c750412c..aa499f5a0bb 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs.yml @@ -14,10 +14,8 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch/bulk: - endpoints: [http://localhost:9200] - user: otel - password: password - # api_key: <> # Elasticsearch API Key in "encoded" format. + endpoints: [<>] + api_key: <> service: pipelines: diff --git a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml index f89ae584673..2be13d7c3d9 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml @@ -22,10 +22,8 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch/bulk: - endpoints: [http://localhost:9200] - user: otel - password: password - # api_key: <> # Elasticsearch API Key in "encoded" format. + endpoints: [<>] + api_key: <> service: pipelines: diff --git a/internal/pkg/otel/samples/linux/hostmetrics_resources.yml b/internal/pkg/otel/samples/linux/hostmetrics_resources.yml index 3e83059add1..a5b4c9bf567 100644 --- a/internal/pkg/otel/samples/linux/hostmetrics_resources.yml +++ b/internal/pkg/otel/samples/linux/hostmetrics_resources.yml @@ -23,10 +23,8 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch/bulk: - endpoints: [http://localhost:9200] - user: otel - password: password - # api_key: <> # Elasticsearch API Key in "encoded" format. + endpoints: [<>] + api_key: <> service: pipelines: diff --git a/internal/pkg/otel/samples/linux/logs_host_metrics.yml b/internal/pkg/otel/samples/linux/logs_host_metrics.yml new file mode 100644 index 00000000000..65b1fe1759a --- /dev/null +++ b/internal/pkg/otel/samples/linux/logs_host_metrics.yml @@ -0,0 +1,38 @@ +receivers: + + # Receiver for platform specific log files + filelog/platformlogs: + include: [ /var/log/*.log ] +# start_at: beginning + + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + +processors: + elasticinframetrics: + +exporters: + + elasticsearch/bulk: + endpoints: [<>] + api_key: <> + logs_index: logs-generic-default + metrics_index: metrics-generic-default + +service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [elasticinframetrics] + exporters: [elasticsearch/bulk] + + logs/platformlogs: + receivers: [filelog/platformlogs] + processors: [] + exporters: [elasticsearch/bulk] diff --git a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml index cbca679f41c..7d1f5460a0b 100644 --- a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml @@ -23,12 +23,10 @@ exporters: sampling_thereafter: 1000 elasticsearch/bulk: - endpoints: [http://localhost:9200] - user: otel - password: password + endpoints: [<>] + api_key: <> logs_index: logs-generic-default metrics_index: metrics-generic-default - # api_key: <> # Elasticsearch API Key in "encoded" format. service: pipelines: diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml new file mode 100644 index 00000000000..b33b47fe3ab --- /dev/null +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml @@ -0,0 +1,65 @@ +receivers: + + # Receiver for platform specific log files + filelog/platformlogs: + include: [ /var/log/*.log ] +# start_at: beginning + + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + + # Receiver for logs, traces, and metrics from SDKs + otlp/fromsdk: + protocols: + grpc: + http: + +processors: + elasticinframetrics: + +exporters: + + otlp/apm: + endpoint: <> + headers: + # Elastic APM Server secret token or API key + Authorization: "Bearer <>" + + elasticsearch/bulk: + endpoints: [<>] + api_key: <> + logs_index: logs-generic-default + metrics_index: metrics-generic-default + +service: + pipelines: + traces/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [otlp/apm] + + metrics/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [otlp/apm] + + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [elasticinframetrics] + exporters: [elasticsearch/bulk] + + logs/fromsdk: + receivers: [otlp/fromsdk] + processors: [] + exporters: [otlp/apm] + + logs/platformlogs: + receivers: [filelog/platformlogs] + processors: [] + exporters: [elasticsearch/bulk] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml index 4e291af767c..512016d6824 100644 --- a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml @@ -36,10 +36,8 @@ exporters: Authorization: "Bearer <>" elasticsearch/bulk: - endpoints: [http://localhost:9200] - user: otel - password: password - # api_key: <> # Elasticsearch API Key in "encoded" format. + endpoints: [<>] + api_key: <> logs_index: logs-generic-default metrics_index: metrics-generic-default diff --git a/internal/pkg/otel/samples/windows/hostmetrics.yml b/internal/pkg/otel/samples/windows/hostmetrics.yml index 4872b658a07..fa6b41db4fe 100644 --- a/internal/pkg/otel/samples/windows/hostmetrics.yml +++ b/internal/pkg/otel/samples/windows/hostmetrics.yml @@ -17,10 +17,8 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch/bulk: - endpoints: [http://localhost:9200] - user: otel - # api_key: <> # Elasticsearch API Key in "encoded" format. - password: password + endpoints: [<>] + api_key: <> service: pipelines: diff --git a/internal/pkg/otel/samples/windows/hostmetrics_resources.yml b/internal/pkg/otel/samples/windows/hostmetrics_resources.yml index 3e83059add1..a5b4c9bf567 100644 --- a/internal/pkg/otel/samples/windows/hostmetrics_resources.yml +++ b/internal/pkg/otel/samples/windows/hostmetrics_resources.yml @@ -23,10 +23,8 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch/bulk: - endpoints: [http://localhost:9200] - user: otel - password: password - # api_key: <> # Elasticsearch API Key in "encoded" format. + endpoints: [<>] + api_key: <> service: pipelines: From ecfbdf3dbcdd654009847ba5e0e8c7b9205fe3df Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 26 Jun 2024 08:53:22 +0200 Subject: [PATCH 04/12] removed logs_index and metrics_index --- internal/pkg/otel/samples/darwin/logs_host_metrics.yml | 2 -- internal/pkg/otel/samples/darwin/logs_metrics_traces.yml | 2 -- internal/pkg/otel/samples/linux/logs_host_metrics.yml | 2 -- internal/pkg/otel/samples/linux/logs_hostmetrics.yml | 2 -- internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml | 2 -- internal/pkg/otel/samples/linux/logs_metrics_traces.yml | 2 -- 6 files changed, 12 deletions(-) diff --git a/internal/pkg/otel/samples/darwin/logs_host_metrics.yml b/internal/pkg/otel/samples/darwin/logs_host_metrics.yml index 65b1fe1759a..8a659c2dd0a 100644 --- a/internal/pkg/otel/samples/darwin/logs_host_metrics.yml +++ b/internal/pkg/otel/samples/darwin/logs_host_metrics.yml @@ -22,8 +22,6 @@ exporters: elasticsearch/bulk: endpoints: [<>] api_key: <> - logs_index: logs-generic-default - metrics_index: metrics-generic-default service: pipelines: diff --git a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml index b33b47fe3ab..f12fa8ba521 100644 --- a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml @@ -34,8 +34,6 @@ exporters: elasticsearch/bulk: endpoints: [<>] api_key: <> - logs_index: logs-generic-default - metrics_index: metrics-generic-default service: pipelines: diff --git a/internal/pkg/otel/samples/linux/logs_host_metrics.yml b/internal/pkg/otel/samples/linux/logs_host_metrics.yml index 65b1fe1759a..8a659c2dd0a 100644 --- a/internal/pkg/otel/samples/linux/logs_host_metrics.yml +++ b/internal/pkg/otel/samples/linux/logs_host_metrics.yml @@ -22,8 +22,6 @@ exporters: elasticsearch/bulk: endpoints: [<>] api_key: <> - logs_index: logs-generic-default - metrics_index: metrics-generic-default service: pipelines: diff --git a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml index 7d1f5460a0b..470c53b09f9 100644 --- a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml @@ -25,8 +25,6 @@ exporters: elasticsearch/bulk: endpoints: [<>] api_key: <> - logs_index: logs-generic-default - metrics_index: metrics-generic-default service: pipelines: diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml index b33b47fe3ab..f12fa8ba521 100644 --- a/internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml @@ -34,8 +34,6 @@ exporters: elasticsearch/bulk: endpoints: [<>] api_key: <> - logs_index: logs-generic-default - metrics_index: metrics-generic-default service: pipelines: diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml index 512016d6824..858b927d37d 100644 --- a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml @@ -38,8 +38,6 @@ exporters: elasticsearch/bulk: endpoints: [<>] api_key: <> - logs_index: logs-generic-default - metrics_index: metrics-generic-default service: pipelines: From 44c91884d13481936abb51d85f97ad2ecad569fa Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 27 Jun 2024 13:51:42 +0200 Subject: [PATCH 05/12] missing files --- .../logs_hostmetrics.yml} | 29 +-------------- .../otel/samples/linux/logs_hostmetrics.yml | 15 +++----- .../samples/linux/logs_metrics_traces.yml | 24 +++++------- .../pkg/otel/samples/linux/platformlogs.yml | 25 +++++++++++++ .../linux/platformlogs_hostmetrics.yml | 37 +++++++++++++++++++ 5 files changed, 79 insertions(+), 51 deletions(-) rename internal/pkg/otel/samples/{linux/logs_metrics_traces copy.yml => darwin/logs_hostmetrics.yml} (53%) create mode 100644 internal/pkg/otel/samples/linux/platformlogs.yml create mode 100644 internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml b/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml similarity index 53% rename from internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml rename to internal/pkg/otel/samples/darwin/logs_hostmetrics.yml index f12fa8ba521..8a659c2dd0a 100644 --- a/internal/pkg/otel/samples/linux/logs_metrics_traces copy.yml +++ b/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml @@ -14,50 +14,23 @@ receivers: cpu: memory: - # Receiver for logs, traces, and metrics from SDKs - otlp/fromsdk: - protocols: - grpc: - http: - processors: elasticinframetrics: exporters: - otlp/apm: - endpoint: <> - headers: - # Elastic APM Server secret token or API key - Authorization: "Bearer <>" - elasticsearch/bulk: endpoints: [<>] api_key: <> service: pipelines: - traces/fromsdk: - receivers: [otlp/fromsdk] - processors: [] - exporters: [otlp/apm] - - metrics/fromsdk: - receivers: [otlp/fromsdk] - processors: [] - exporters: [otlp/apm] - metrics/hostmetrics: receivers: [hostmetrics/system] processors: [elasticinframetrics] exporters: [elasticsearch/bulk] - logs/fromsdk: - receivers: [otlp/fromsdk] - processors: [] - exporters: [otlp/apm] - logs/platformlogs: receivers: [filelog/platformlogs] processors: [] - exporters: [elasticsearch/bulk] \ No newline at end of file + exporters: [elasticsearch/bulk] diff --git a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml index 470c53b09f9..8a659c2dd0a 100644 --- a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml @@ -14,13 +14,10 @@ receivers: cpu: memory: -exporters: +processors: + elasticinframetrics: - # Exporter to print the first 5 logs/metrics and then every 1000th - debug: - verbosity: detailed - sampling_initial: 5 - sampling_thereafter: 1000 +exporters: elasticsearch/bulk: endpoints: [<>] @@ -30,10 +27,10 @@ service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] - processors: [] - exporters: [debug, elasticsearch/bulk] + processors: [elasticinframetrics] + exporters: [elasticsearch/bulk] logs/platformlogs: receivers: [filelog/platformlogs] processors: [] - exporters: [debug, elasticsearch/bulk] + exporters: [elasticsearch/bulk] diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml index 858b927d37d..f12fa8ba521 100644 --- a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml @@ -20,17 +20,13 @@ receivers: grpc: http: -exporters: - - # Exporter to print the first 5 logs/metrics and then every 1000th - debug: - verbosity: detailed - sampling_initial: 5 - sampling_thereafter: 200 +processors: + elasticinframetrics: +exporters: otlp/apm: - endpoint: "http://localhost:8200" + endpoint: <> headers: # Elastic APM Server secret token or API key Authorization: "Bearer <>" @@ -44,24 +40,24 @@ service: traces/fromsdk: receivers: [otlp/fromsdk] processors: [] - exporters: [debug, otlp/apm] + exporters: [otlp/apm] metrics/fromsdk: receivers: [otlp/fromsdk] processors: [] - exporters: [debug, otlp/apm] + exporters: [otlp/apm] metrics/hostmetrics: receivers: [hostmetrics/system] - processors: [] - exporters: [debug, elasticsearch/bulk] + processors: [elasticinframetrics] + exporters: [elasticsearch/bulk] logs/fromsdk: receivers: [otlp/fromsdk] processors: [] - exporters: [debug, otlp/apm] + exporters: [otlp/apm] logs/platformlogs: receivers: [filelog/platformlogs] processors: [] - exporters: [debug, elasticsearch/bulk] \ No newline at end of file + exporters: [elasticsearch/bulk] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/platformlogs.yml b/internal/pkg/otel/samples/linux/platformlogs.yml new file mode 100644 index 00000000000..aa499f5a0bb --- /dev/null +++ b/internal/pkg/otel/samples/linux/platformlogs.yml @@ -0,0 +1,25 @@ +receivers: + # Receiver for platform specific log files + filelog/platformlogs: + include: [ /var/log/*.log ] +# start_at: beginning + +exporters: + + # Exporter to print the first 5 logs/metrics and then every 1000th + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 1000 + + # Exporter to send logs and metrics to Elasticsearch + elasticsearch/bulk: + endpoints: [<>] + api_key: <> + +service: + pipelines: + logs/platformlogs: + receivers: [filelog/platformlogs] + processors: [] + exporters: [debug, elasticsearch/bulk] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml new file mode 100644 index 00000000000..2be13d7c3d9 --- /dev/null +++ b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml @@ -0,0 +1,37 @@ +receivers: + # Receiver for platform specific log files + filelog/platformlogs: + include: [ /var/log/*.log ] +# start_at: beginning + + # Receiver for CPU, Disk, Memory, and Filesystem metrics + hostmetrics/system: + collection_interval: 30s + scrapers: + disk: + filesystem: + cpu: + memory: + +exporters: + # Exporter to print the first 5 logs/metrics and then every 1000th + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 1000 + + # Exporter to send logs and metrics to Elasticsearch + elasticsearch/bulk: + endpoints: [<>] + api_key: <> + +service: + pipelines: + metrics/hostmetrics: + receivers: [hostmetrics/system] + processors: [] + exporters: [debug, elasticsearch/bulk] + logs/platformlogs: + receivers: [filelog/platformlogs] + processors: [] + exporters: [debug, elasticsearch/bulk] \ No newline at end of file From c9ead48a785062547d7667f4c05c844a3b14f451 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 27 Jun 2024 18:45:50 +0200 Subject: [PATCH 06/12] Update internal/pkg/otel/samples/darwin/hostmetrics_resources.yml Co-authored-by: Andrzej Stencel --- internal/pkg/otel/samples/darwin/hostmetrics_resources.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml b/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml index a5b4c9bf567..1c057a52a8a 100644 --- a/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml +++ b/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml @@ -23,8 +23,8 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch/bulk: - endpoints: [<>] - api_key: <> + endpoints: [${env:ELASTIC_ENDPOINT}] + api_key: ${env:ELASTIC_API_KEY} service: pipelines: From 498fa2ec77fc7776b351e95b4846515968818512 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 27 Jun 2024 18:48:46 +0200 Subject: [PATCH 07/12] refactor config --- .../samples/darwin/hostmetrics_resources.yml | 34 ------------------ .../otel/samples/darwin/logs_host_metrics.yml | 36 ------------------- .../otel/samples/darwin/logs_hostmetrics.yml | 10 +++--- .../samples/darwin/logs_metrics_traces.yml | 10 +++--- .../pkg/otel/samples/darwin/platformlogs.yml | 8 ++--- .../darwin/platformlogs_hostmetrics.yml | 10 +++--- .../samples/linux/hostmetrics_resources.yml | 34 ------------------ .../otel/samples/linux/logs_host_metrics.yml | 36 ------------------- .../otel/samples/linux/logs_hostmetrics.yml | 10 +++--- .../samples/linux/logs_metrics_traces.yml | 10 +++--- .../pkg/otel/samples/linux/platformlogs.yml | 8 ++--- .../linux/platformlogs_hostmetrics.yml | 10 +++--- .../pkg/otel/samples/windows/hostmetrics.yml | 8 ++--- .../samples/windows/hostmetrics_resources.yml | 8 ++--- 14 files changed, 46 insertions(+), 186 deletions(-) delete mode 100644 internal/pkg/otel/samples/darwin/hostmetrics_resources.yml delete mode 100644 internal/pkg/otel/samples/darwin/logs_host_metrics.yml delete mode 100644 internal/pkg/otel/samples/linux/hostmetrics_resources.yml delete mode 100644 internal/pkg/otel/samples/linux/logs_host_metrics.yml diff --git a/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml b/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml deleted file mode 100644 index a5b4c9bf567..00000000000 --- a/internal/pkg/otel/samples/darwin/hostmetrics_resources.yml +++ /dev/null @@ -1,34 +0,0 @@ -receivers: - # Receiver for CPU, Disk, Memory, and Filesystem metrics - hostmetrics/system: - collection_interval: 30s - scrapers: - disk: - filesystem: - cpu: - memory: - -processors: - resourcedetection/system: - detectors: ["system"] - system: - hostname_sources: ["os"] - -exporters: - # Exporter to print the first 5 logs/metrics and then every 1000th - debug: - verbosity: detailed - sampling_initial: 5 - sampling_thereafter: 1000 - - # Exporter to send logs and metrics to Elasticsearch - elasticsearch/bulk: - endpoints: [<>] - api_key: <> - -service: - pipelines: - metrics/hostmetrics: - receivers: [hostmetrics/system] - processors: [resourcedetection/system] - exporters: [debug, elasticsearch/bulk] \ No newline at end of file diff --git a/internal/pkg/otel/samples/darwin/logs_host_metrics.yml b/internal/pkg/otel/samples/darwin/logs_host_metrics.yml deleted file mode 100644 index 8a659c2dd0a..00000000000 --- a/internal/pkg/otel/samples/darwin/logs_host_metrics.yml +++ /dev/null @@ -1,36 +0,0 @@ -receivers: - - # Receiver for platform specific log files - filelog/platformlogs: - include: [ /var/log/*.log ] -# start_at: beginning - - # Receiver for CPU, Disk, Memory, and Filesystem metrics - hostmetrics/system: - collection_interval: 30s - scrapers: - disk: - filesystem: - cpu: - memory: - -processors: - elasticinframetrics: - -exporters: - - elasticsearch/bulk: - endpoints: [<>] - api_key: <> - -service: - pipelines: - metrics/hostmetrics: - receivers: [hostmetrics/system] - processors: [elasticinframetrics] - exporters: [elasticsearch/bulk] - - logs/platformlogs: - receivers: [filelog/platformlogs] - processors: [] - exporters: [elasticsearch/bulk] diff --git a/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml index 8a659c2dd0a..b3b3871eba4 100644 --- a/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml @@ -19,18 +19,18 @@ processors: exporters: - elasticsearch/bulk: - endpoints: [<>] - api_key: <> + elasticsearch: + endpoints: [${env:ELASTIC_ENDPOINT}] + api_key: ${env:ELASTIC_API_KEY} service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] processors: [elasticinframetrics] - exporters: [elasticsearch/bulk] + exporters: [elasticsearch] logs/platformlogs: receivers: [filelog/platformlogs] processors: [] - exporters: [elasticsearch/bulk] + exporters: [elasticsearch] diff --git a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml index f12fa8ba521..a272feda5b8 100644 --- a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml @@ -31,9 +31,9 @@ exporters: # Elastic APM Server secret token or API key Authorization: "Bearer <>" - elasticsearch/bulk: - endpoints: [<>] - api_key: <> + elasticsearch: + endpoints: [${env:ELASTIC_ENDPOINT}] + api_key: ${env:ELASTIC_API_KEY} service: pipelines: @@ -50,7 +50,7 @@ service: metrics/hostmetrics: receivers: [hostmetrics/system] processors: [elasticinframetrics] - exporters: [elasticsearch/bulk] + exporters: [elasticsearch] logs/fromsdk: receivers: [otlp/fromsdk] @@ -60,4 +60,4 @@ service: logs/platformlogs: receivers: [filelog/platformlogs] processors: [] - exporters: [elasticsearch/bulk] \ No newline at end of file + exporters: [elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/darwin/platformlogs.yml b/internal/pkg/otel/samples/darwin/platformlogs.yml index aa499f5a0bb..554efd9edd7 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs.yml @@ -13,13 +13,13 @@ exporters: sampling_thereafter: 1000 # Exporter to send logs and metrics to Elasticsearch - elasticsearch/bulk: - endpoints: [<>] - api_key: <> + elasticsearch: + endpoints: [${env:ELASTIC_ENDPOINT}] + api_key: ${env:ELASTIC_API_KEY} service: pipelines: logs/platformlogs: receivers: [filelog/platformlogs] processors: [] - exporters: [debug, elasticsearch/bulk] \ No newline at end of file + exporters: [debug, elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml index 2be13d7c3d9..3545c474675 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml @@ -21,17 +21,17 @@ exporters: sampling_thereafter: 1000 # Exporter to send logs and metrics to Elasticsearch - elasticsearch/bulk: - endpoints: [<>] - api_key: <> + elasticsearch: + endpoints: [${env:ELASTIC_ENDPOINT}] + api_key: ${env:ELASTIC_API_KEY} service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] processors: [] - exporters: [debug, elasticsearch/bulk] + exporters: [debug, elasticsearch] logs/platformlogs: receivers: [filelog/platformlogs] processors: [] - exporters: [debug, elasticsearch/bulk] \ No newline at end of file + exporters: [debug, elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/hostmetrics_resources.yml b/internal/pkg/otel/samples/linux/hostmetrics_resources.yml deleted file mode 100644 index a5b4c9bf567..00000000000 --- a/internal/pkg/otel/samples/linux/hostmetrics_resources.yml +++ /dev/null @@ -1,34 +0,0 @@ -receivers: - # Receiver for CPU, Disk, Memory, and Filesystem metrics - hostmetrics/system: - collection_interval: 30s - scrapers: - disk: - filesystem: - cpu: - memory: - -processors: - resourcedetection/system: - detectors: ["system"] - system: - hostname_sources: ["os"] - -exporters: - # Exporter to print the first 5 logs/metrics and then every 1000th - debug: - verbosity: detailed - sampling_initial: 5 - sampling_thereafter: 1000 - - # Exporter to send logs and metrics to Elasticsearch - elasticsearch/bulk: - endpoints: [<>] - api_key: <> - -service: - pipelines: - metrics/hostmetrics: - receivers: [hostmetrics/system] - processors: [resourcedetection/system] - exporters: [debug, elasticsearch/bulk] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/logs_host_metrics.yml b/internal/pkg/otel/samples/linux/logs_host_metrics.yml deleted file mode 100644 index 8a659c2dd0a..00000000000 --- a/internal/pkg/otel/samples/linux/logs_host_metrics.yml +++ /dev/null @@ -1,36 +0,0 @@ -receivers: - - # Receiver for platform specific log files - filelog/platformlogs: - include: [ /var/log/*.log ] -# start_at: beginning - - # Receiver for CPU, Disk, Memory, and Filesystem metrics - hostmetrics/system: - collection_interval: 30s - scrapers: - disk: - filesystem: - cpu: - memory: - -processors: - elasticinframetrics: - -exporters: - - elasticsearch/bulk: - endpoints: [<>] - api_key: <> - -service: - pipelines: - metrics/hostmetrics: - receivers: [hostmetrics/system] - processors: [elasticinframetrics] - exporters: [elasticsearch/bulk] - - logs/platformlogs: - receivers: [filelog/platformlogs] - processors: [] - exporters: [elasticsearch/bulk] diff --git a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml index 8a659c2dd0a..b3b3871eba4 100644 --- a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml @@ -19,18 +19,18 @@ processors: exporters: - elasticsearch/bulk: - endpoints: [<>] - api_key: <> + elasticsearch: + endpoints: [${env:ELASTIC_ENDPOINT}] + api_key: ${env:ELASTIC_API_KEY} service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] processors: [elasticinframetrics] - exporters: [elasticsearch/bulk] + exporters: [elasticsearch] logs/platformlogs: receivers: [filelog/platformlogs] processors: [] - exporters: [elasticsearch/bulk] + exporters: [elasticsearch] diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml index f12fa8ba521..a272feda5b8 100644 --- a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml @@ -31,9 +31,9 @@ exporters: # Elastic APM Server secret token or API key Authorization: "Bearer <>" - elasticsearch/bulk: - endpoints: [<>] - api_key: <> + elasticsearch: + endpoints: [${env:ELASTIC_ENDPOINT}] + api_key: ${env:ELASTIC_API_KEY} service: pipelines: @@ -50,7 +50,7 @@ service: metrics/hostmetrics: receivers: [hostmetrics/system] processors: [elasticinframetrics] - exporters: [elasticsearch/bulk] + exporters: [elasticsearch] logs/fromsdk: receivers: [otlp/fromsdk] @@ -60,4 +60,4 @@ service: logs/platformlogs: receivers: [filelog/platformlogs] processors: [] - exporters: [elasticsearch/bulk] \ No newline at end of file + exporters: [elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/platformlogs.yml b/internal/pkg/otel/samples/linux/platformlogs.yml index aa499f5a0bb..554efd9edd7 100644 --- a/internal/pkg/otel/samples/linux/platformlogs.yml +++ b/internal/pkg/otel/samples/linux/platformlogs.yml @@ -13,13 +13,13 @@ exporters: sampling_thereafter: 1000 # Exporter to send logs and metrics to Elasticsearch - elasticsearch/bulk: - endpoints: [<>] - api_key: <> + elasticsearch: + endpoints: [${env:ELASTIC_ENDPOINT}] + api_key: ${env:ELASTIC_API_KEY} service: pipelines: logs/platformlogs: receivers: [filelog/platformlogs] processors: [] - exporters: [debug, elasticsearch/bulk] \ No newline at end of file + exporters: [debug, elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml index 2be13d7c3d9..3545c474675 100644 --- a/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml @@ -21,17 +21,17 @@ exporters: sampling_thereafter: 1000 # Exporter to send logs and metrics to Elasticsearch - elasticsearch/bulk: - endpoints: [<>] - api_key: <> + elasticsearch: + endpoints: [${env:ELASTIC_ENDPOINT}] + api_key: ${env:ELASTIC_API_KEY} service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] processors: [] - exporters: [debug, elasticsearch/bulk] + exporters: [debug, elasticsearch] logs/platformlogs: receivers: [filelog/platformlogs] processors: [] - exporters: [debug, elasticsearch/bulk] \ No newline at end of file + exporters: [debug, elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/windows/hostmetrics.yml b/internal/pkg/otel/samples/windows/hostmetrics.yml index fa6b41db4fe..f4845a2f848 100644 --- a/internal/pkg/otel/samples/windows/hostmetrics.yml +++ b/internal/pkg/otel/samples/windows/hostmetrics.yml @@ -16,13 +16,13 @@ exporters: sampling_thereafter: 1000 # Exporter to send logs and metrics to Elasticsearch - elasticsearch/bulk: - endpoints: [<>] - api_key: <> + elasticsearch: + endpoints: [${env:ELASTIC_ENDPOINT}] + api_key: ${env:ELASTIC_API_KEY} service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] processors: [] - exporters: [debug, elasticsearch/bulk] + exporters: [debug, elasticsearch] diff --git a/internal/pkg/otel/samples/windows/hostmetrics_resources.yml b/internal/pkg/otel/samples/windows/hostmetrics_resources.yml index a5b4c9bf567..b3844d3b890 100644 --- a/internal/pkg/otel/samples/windows/hostmetrics_resources.yml +++ b/internal/pkg/otel/samples/windows/hostmetrics_resources.yml @@ -22,13 +22,13 @@ exporters: sampling_thereafter: 1000 # Exporter to send logs and metrics to Elasticsearch - elasticsearch/bulk: - endpoints: [<>] - api_key: <> + elasticsearch: + endpoints: [${env:ELASTIC_ENDPOINT}] + api_key: ${env:ELASTIC_API_KEY} service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] processors: [resourcedetection/system] - exporters: [debug, elasticsearch/bulk] \ No newline at end of file + exporters: [debug, elasticsearch] \ No newline at end of file From dfc070f0bb233f398de6224fa762cd2a6c17018a Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 28 Jun 2024 11:21:51 +0200 Subject: [PATCH 08/12] updated configs with resrouces --- .../pkg/otel/samples/darwin/logs_hostmetrics.yml | 12 +++++++----- .../otel/samples/darwin/logs_metrics_traces.yml | 14 +++++++++----- internal/pkg/otel/samples/darwin/platformlogs.yml | 12 +++++++++--- .../samples/darwin/platformlogs_hostmetrics.yml | 12 +++++++++--- .../pkg/otel/samples/linux/logs_hostmetrics.yml | 12 +++++++----- .../pkg/otel/samples/linux/logs_metrics_traces.yml | 14 +++++++++----- internal/pkg/otel/samples/linux/platformlogs.yml | 12 +++++++++--- .../samples/linux/platformlogs_hostmetrics.yml | 12 +++++++++--- internal/pkg/otel/samples/windows/hostmetrics.yml | 2 +- .../otel/samples/windows/hostmetrics_resources.yml | 6 +++--- 10 files changed, 72 insertions(+), 36 deletions(-) diff --git a/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml index b3b3871eba4..1a25bcf894a 100644 --- a/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml @@ -1,5 +1,4 @@ receivers: - # Receiver for platform specific log files filelog/platformlogs: include: [ /var/log/*.log ] @@ -16,21 +15,24 @@ receivers: processors: elasticinframetrics: + resourcedetection: + detectors: ["system"] + system: + hostname_sources: ["os"] exporters: - elasticsearch: - endpoints: [${env:ELASTIC_ENDPOINT}] + endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] - processors: [elasticinframetrics] + processors: [elasticinframetrics, resourcedetection] exporters: [elasticsearch] logs/platformlogs: receivers: [filelog/platformlogs] - processors: [] + processors: [resourcedetection] exporters: [elasticsearch] diff --git a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml index a272feda5b8..d8e8956f509 100644 --- a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml @@ -22,17 +22,21 @@ receivers: processors: elasticinframetrics: + resourcedetection: + detectors: ["system"] + system: + hostname_sources: ["os"] exporters: otlp/apm: - endpoint: <> + endpoint: "${env:APM_ENDPOINT}" headers: # Elastic APM Server secret token or API key - Authorization: "Bearer <>" + Authorization: "Bearer ${env:APM_SECRET_KEY}" elasticsearch: - endpoints: [${env:ELASTIC_ENDPOINT}] + endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} service: @@ -49,7 +53,7 @@ service: metrics/hostmetrics: receivers: [hostmetrics/system] - processors: [elasticinframetrics] + processors: [elasticinframetrics, resourcedetection] exporters: [elasticsearch] logs/fromsdk: @@ -59,5 +63,5 @@ service: logs/platformlogs: receivers: [filelog/platformlogs] - processors: [] + processors: [resourcedetection] exporters: [elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/darwin/platformlogs.yml b/internal/pkg/otel/samples/darwin/platformlogs.yml index 554efd9edd7..1ea24e45895 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs.yml @@ -4,8 +4,14 @@ receivers: include: [ /var/log/*.log ] # start_at: beginning -exporters: +processors: + resourcedetection: + detectors: ["system"] + system: + hostname_sources: ["os"] + +exporters: # Exporter to print the first 5 logs/metrics and then every 1000th debug: verbosity: detailed @@ -14,12 +20,12 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch: - endpoints: [${env:ELASTIC_ENDPOINT}] + endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} service: pipelines: logs/platformlogs: receivers: [filelog/platformlogs] - processors: [] + processors: [resourcedetection] exporters: [debug, elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml index 3545c474675..17064590495 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml @@ -13,6 +13,12 @@ receivers: cpu: memory: +processors: + resourcedetection: + detectors: ["system"] + system: + hostname_sources: ["os"] + exporters: # Exporter to print the first 5 logs/metrics and then every 1000th debug: @@ -22,16 +28,16 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch: - endpoints: [${env:ELASTIC_ENDPOINT}] + endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] - processors: [] + processors: [resourcedetection] exporters: [debug, elasticsearch] logs/platformlogs: receivers: [filelog/platformlogs] - processors: [] + processors: [resourcedetection] exporters: [debug, elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml index b3b3871eba4..1a25bcf894a 100644 --- a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml @@ -1,5 +1,4 @@ receivers: - # Receiver for platform specific log files filelog/platformlogs: include: [ /var/log/*.log ] @@ -16,21 +15,24 @@ receivers: processors: elasticinframetrics: + resourcedetection: + detectors: ["system"] + system: + hostname_sources: ["os"] exporters: - elasticsearch: - endpoints: [${env:ELASTIC_ENDPOINT}] + endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] - processors: [elasticinframetrics] + processors: [elasticinframetrics, resourcedetection] exporters: [elasticsearch] logs/platformlogs: receivers: [filelog/platformlogs] - processors: [] + processors: [resourcedetection] exporters: [elasticsearch] diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml index a272feda5b8..d8e8956f509 100644 --- a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml @@ -22,17 +22,21 @@ receivers: processors: elasticinframetrics: + resourcedetection: + detectors: ["system"] + system: + hostname_sources: ["os"] exporters: otlp/apm: - endpoint: <> + endpoint: "${env:APM_ENDPOINT}" headers: # Elastic APM Server secret token or API key - Authorization: "Bearer <>" + Authorization: "Bearer ${env:APM_SECRET_KEY}" elasticsearch: - endpoints: [${env:ELASTIC_ENDPOINT}] + endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} service: @@ -49,7 +53,7 @@ service: metrics/hostmetrics: receivers: [hostmetrics/system] - processors: [elasticinframetrics] + processors: [elasticinframetrics, resourcedetection] exporters: [elasticsearch] logs/fromsdk: @@ -59,5 +63,5 @@ service: logs/platformlogs: receivers: [filelog/platformlogs] - processors: [] + processors: [resourcedetection] exporters: [elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/platformlogs.yml b/internal/pkg/otel/samples/linux/platformlogs.yml index 554efd9edd7..1ea24e45895 100644 --- a/internal/pkg/otel/samples/linux/platformlogs.yml +++ b/internal/pkg/otel/samples/linux/platformlogs.yml @@ -4,8 +4,14 @@ receivers: include: [ /var/log/*.log ] # start_at: beginning -exporters: +processors: + resourcedetection: + detectors: ["system"] + system: + hostname_sources: ["os"] + +exporters: # Exporter to print the first 5 logs/metrics and then every 1000th debug: verbosity: detailed @@ -14,12 +20,12 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch: - endpoints: [${env:ELASTIC_ENDPOINT}] + endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} service: pipelines: logs/platformlogs: receivers: [filelog/platformlogs] - processors: [] + processors: [resourcedetection] exporters: [debug, elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml index 3545c474675..17064590495 100644 --- a/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml @@ -13,6 +13,12 @@ receivers: cpu: memory: +processors: + resourcedetection: + detectors: ["system"] + system: + hostname_sources: ["os"] + exporters: # Exporter to print the first 5 logs/metrics and then every 1000th debug: @@ -22,16 +28,16 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch: - endpoints: [${env:ELASTIC_ENDPOINT}] + endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] - processors: [] + processors: [resourcedetection] exporters: [debug, elasticsearch] logs/platformlogs: receivers: [filelog/platformlogs] - processors: [] + processors: [resourcedetection] exporters: [debug, elasticsearch] \ No newline at end of file diff --git a/internal/pkg/otel/samples/windows/hostmetrics.yml b/internal/pkg/otel/samples/windows/hostmetrics.yml index f4845a2f848..e814ff4fc0f 100644 --- a/internal/pkg/otel/samples/windows/hostmetrics.yml +++ b/internal/pkg/otel/samples/windows/hostmetrics.yml @@ -17,7 +17,7 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch: - endpoints: [${env:ELASTIC_ENDPOINT}] + endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} service: diff --git a/internal/pkg/otel/samples/windows/hostmetrics_resources.yml b/internal/pkg/otel/samples/windows/hostmetrics_resources.yml index b3844d3b890..88fec1558f3 100644 --- a/internal/pkg/otel/samples/windows/hostmetrics_resources.yml +++ b/internal/pkg/otel/samples/windows/hostmetrics_resources.yml @@ -9,7 +9,7 @@ receivers: memory: processors: - resourcedetection/system: + resourcedetection: detectors: ["system"] system: hostname_sources: ["os"] @@ -23,12 +23,12 @@ exporters: # Exporter to send logs and metrics to Elasticsearch elasticsearch: - endpoints: [${env:ELASTIC_ENDPOINT}] + endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} service: pipelines: metrics/hostmetrics: receivers: [hostmetrics/system] - processors: [resourcedetection/system] + processors: [resourcedetection] exporters: [debug, elasticsearch] \ No newline at end of file From 9cbef7d7c8bff92e39fef7303f3267454cd2702f Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 28 Jun 2024 13:51:06 +0200 Subject: [PATCH 09/12] remove windows files --- .../pkg/otel/samples/windows/hostmetrics.yml | 28 --------------- .../samples/windows/hostmetrics_resources.yml | 34 ------------------- 2 files changed, 62 deletions(-) delete mode 100644 internal/pkg/otel/samples/windows/hostmetrics.yml delete mode 100644 internal/pkg/otel/samples/windows/hostmetrics_resources.yml diff --git a/internal/pkg/otel/samples/windows/hostmetrics.yml b/internal/pkg/otel/samples/windows/hostmetrics.yml deleted file mode 100644 index e814ff4fc0f..00000000000 --- a/internal/pkg/otel/samples/windows/hostmetrics.yml +++ /dev/null @@ -1,28 +0,0 @@ -receivers: - # Receiver for CPU, Disk, Memory, and Filesystem metrics - hostmetrics/system: - collection_interval: 30s - scrapers: - disk: - filesystem: - cpu: - memory: - -exporters: - # Exporter to print the first 5 logs/metrics and then every 1000th - debug: - verbosity: detailed - sampling_initial: 5 - sampling_thereafter: 1000 - - # Exporter to send logs and metrics to Elasticsearch - elasticsearch: - endpoints: ["${env:ELASTIC_ENDPOINT}"] - api_key: ${env:ELASTIC_API_KEY} - -service: - pipelines: - metrics/hostmetrics: - receivers: [hostmetrics/system] - processors: [] - exporters: [debug, elasticsearch] diff --git a/internal/pkg/otel/samples/windows/hostmetrics_resources.yml b/internal/pkg/otel/samples/windows/hostmetrics_resources.yml deleted file mode 100644 index 88fec1558f3..00000000000 --- a/internal/pkg/otel/samples/windows/hostmetrics_resources.yml +++ /dev/null @@ -1,34 +0,0 @@ -receivers: - # Receiver for CPU, Disk, Memory, and Filesystem metrics - hostmetrics/system: - collection_interval: 30s - scrapers: - disk: - filesystem: - cpu: - memory: - -processors: - resourcedetection: - detectors: ["system"] - system: - hostname_sources: ["os"] - -exporters: - # Exporter to print the first 5 logs/metrics and then every 1000th - debug: - verbosity: detailed - sampling_initial: 5 - sampling_thereafter: 1000 - - # Exporter to send logs and metrics to Elasticsearch - elasticsearch: - endpoints: ["${env:ELASTIC_ENDPOINT}"] - api_key: ${env:ELASTIC_API_KEY} - -service: - pipelines: - metrics/hostmetrics: - receivers: [hostmetrics/system] - processors: [resourcedetection] - exporters: [debug, elasticsearch] \ No newline at end of file From 51bb3ad69f1238afeb703a87659feb7215e45de4 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 28 Jun 2024 15:15:21 +0200 Subject: [PATCH 10/12] ecs mode enabled --- .../otel/samples/darwin/logs_hostmetrics.yml | 27 +++++++++++++++++ .../samples/darwin/logs_metrics_traces.yml | 29 +++++++++++++++++++ .../pkg/otel/samples/darwin/platformlogs.yml | 29 +++++++++++++++++++ .../darwin/platformlogs_hostmetrics.yml | 29 +++++++++++++++++++ .../otel/samples/linux/logs_hostmetrics.yml | 27 +++++++++++++++++ .../samples/linux/logs_metrics_traces.yml | 29 +++++++++++++++++++ .../pkg/otel/samples/linux/platformlogs.yml | 29 +++++++++++++++++++ .../linux/platformlogs_hostmetrics.yml | 29 +++++++++++++++++++ 8 files changed, 228 insertions(+) diff --git a/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml index 1a25bcf894a..434e5d08eae 100644 --- a/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml @@ -19,6 +19,33 @@ processors: detectors: ["system"] system: hostname_sources: ["os"] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true exporters: elasticsearch: diff --git a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml index d8e8956f509..c1f1265434f 100644 --- a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml @@ -26,6 +26,33 @@ processors: detectors: ["system"] system: hostname_sources: ["os"] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true exporters: @@ -38,6 +65,8 @@ exporters: elasticsearch: endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} + mapping: + mode: ecs service: pipelines: diff --git a/internal/pkg/otel/samples/darwin/platformlogs.yml b/internal/pkg/otel/samples/darwin/platformlogs.yml index 1ea24e45895..b70d2971bb8 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs.yml @@ -10,6 +10,33 @@ processors: detectors: ["system"] system: hostname_sources: ["os"] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true exporters: # Exporter to print the first 5 logs/metrics and then every 1000th @@ -22,6 +49,8 @@ exporters: elasticsearch: endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} + mapping: + mode: ecs service: pipelines: diff --git a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml index 17064590495..5ae0069ec55 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml @@ -18,6 +18,33 @@ processors: detectors: ["system"] system: hostname_sources: ["os"] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true exporters: # Exporter to print the first 5 logs/metrics and then every 1000th @@ -30,6 +57,8 @@ exporters: elasticsearch: endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} + mapping: + mode: ecs service: pipelines: diff --git a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml index 1a25bcf894a..434e5d08eae 100644 --- a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml @@ -19,6 +19,33 @@ processors: detectors: ["system"] system: hostname_sources: ["os"] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true exporters: elasticsearch: diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml index d8e8956f509..c1f1265434f 100644 --- a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml @@ -26,6 +26,33 @@ processors: detectors: ["system"] system: hostname_sources: ["os"] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true exporters: @@ -38,6 +65,8 @@ exporters: elasticsearch: endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} + mapping: + mode: ecs service: pipelines: diff --git a/internal/pkg/otel/samples/linux/platformlogs.yml b/internal/pkg/otel/samples/linux/platformlogs.yml index 1ea24e45895..b70d2971bb8 100644 --- a/internal/pkg/otel/samples/linux/platformlogs.yml +++ b/internal/pkg/otel/samples/linux/platformlogs.yml @@ -10,6 +10,33 @@ processors: detectors: ["system"] system: hostname_sources: ["os"] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true exporters: # Exporter to print the first 5 logs/metrics and then every 1000th @@ -22,6 +49,8 @@ exporters: elasticsearch: endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} + mapping: + mode: ecs service: pipelines: diff --git a/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml index 17064590495..5ae0069ec55 100644 --- a/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml @@ -18,6 +18,33 @@ processors: detectors: ["system"] system: hostname_sources: ["os"] + resource_attributes: + host.name: + enabled: true + host.id: + enabled: false + host.arch: + enabled: true + host.ip: + enabled: true + host.mac: + enabled: true + host.cpu.vendor.id: + enabled: true + host.cpu.family: + enabled: true + host.cpu.model.id: + enabled: true + host.cpu.model.name: + enabled: true + host.cpu.stepping: + enabled: true + host.cpu.cache.l2.size: + enabled: true + os.description: + enabled: true + os.type: + enabled: true exporters: # Exporter to print the first 5 logs/metrics and then every 1000th @@ -30,6 +57,8 @@ exporters: elasticsearch: endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} + mapping: + mode: ecs service: pipelines: From 1839dbf0bdad1eb87cfafdb379fd95de1041e833 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 28 Jun 2024 15:26:35 +0200 Subject: [PATCH 11/12] Update internal/pkg/otel/samples/darwin/logs_metrics_traces.yml Co-authored-by: Carson Ip --- internal/pkg/otel/samples/darwin/logs_metrics_traces.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml index c1f1265434f..f4127e2bbac 100644 --- a/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/darwin/logs_metrics_traces.yml @@ -67,6 +67,12 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: ecs + logs_dynamic_index: + enabled: true + metrics_dynamic_index: + enabled: true + traces_dynamic_index: + enabled: true service: pipelines: From 90376a8554694e0ed6f326bd98abc7a03b8fc44f Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Fri, 28 Jun 2024 15:28:36 +0200 Subject: [PATCH 12/12] ecs mode enabled --- internal/pkg/otel/samples/darwin/logs_hostmetrics.yml | 8 ++++++++ internal/pkg/otel/samples/darwin/platformlogs.yml | 6 ++++++ .../pkg/otel/samples/darwin/platformlogs_hostmetrics.yml | 6 ++++++ internal/pkg/otel/samples/linux/logs_hostmetrics.yml | 8 ++++++++ internal/pkg/otel/samples/linux/logs_metrics_traces.yml | 6 ++++++ internal/pkg/otel/samples/linux/platformlogs.yml | 6 ++++++ .../pkg/otel/samples/linux/platformlogs_hostmetrics.yml | 6 ++++++ 7 files changed, 46 insertions(+) diff --git a/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml index 434e5d08eae..01b0dcb9009 100644 --- a/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/logs_hostmetrics.yml @@ -51,6 +51,14 @@ exporters: elasticsearch: endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} + mapping: + mode: ecs + logs_dynamic_index: + enabled: true + metrics_dynamic_index: + enabled: true + traces_dynamic_index: + enabled: true service: pipelines: diff --git a/internal/pkg/otel/samples/darwin/platformlogs.yml b/internal/pkg/otel/samples/darwin/platformlogs.yml index b70d2971bb8..2455b1eb9a5 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs.yml @@ -51,6 +51,12 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: ecs + logs_dynamic_index: + enabled: true + metrics_dynamic_index: + enabled: true + traces_dynamic_index: + enabled: true service: pipelines: diff --git a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml index 5ae0069ec55..97bcd10facb 100644 --- a/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml @@ -59,6 +59,12 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: ecs + logs_dynamic_index: + enabled: true + metrics_dynamic_index: + enabled: true + traces_dynamic_index: + enabled: true service: pipelines: diff --git a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml index 434e5d08eae..01b0dcb9009 100644 --- a/internal/pkg/otel/samples/linux/logs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/logs_hostmetrics.yml @@ -51,6 +51,14 @@ exporters: elasticsearch: endpoints: ["${env:ELASTIC_ENDPOINT}"] api_key: ${env:ELASTIC_API_KEY} + mapping: + mode: ecs + logs_dynamic_index: + enabled: true + metrics_dynamic_index: + enabled: true + traces_dynamic_index: + enabled: true service: pipelines: diff --git a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml index c1f1265434f..f4127e2bbac 100644 --- a/internal/pkg/otel/samples/linux/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/linux/logs_metrics_traces.yml @@ -67,6 +67,12 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: ecs + logs_dynamic_index: + enabled: true + metrics_dynamic_index: + enabled: true + traces_dynamic_index: + enabled: true service: pipelines: diff --git a/internal/pkg/otel/samples/linux/platformlogs.yml b/internal/pkg/otel/samples/linux/platformlogs.yml index b70d2971bb8..2455b1eb9a5 100644 --- a/internal/pkg/otel/samples/linux/platformlogs.yml +++ b/internal/pkg/otel/samples/linux/platformlogs.yml @@ -51,6 +51,12 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: ecs + logs_dynamic_index: + enabled: true + metrics_dynamic_index: + enabled: true + traces_dynamic_index: + enabled: true service: pipelines: diff --git a/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml index 5ae0069ec55..97bcd10facb 100644 --- a/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/platformlogs_hostmetrics.yml @@ -59,6 +59,12 @@ exporters: api_key: ${env:ELASTIC_API_KEY} mapping: mode: ecs + logs_dynamic_index: + enabled: true + metrics_dynamic_index: + enabled: true + traces_dynamic_index: + enabled: true service: pipelines: