[GCP] Add cloudsql data streams (#4126)

* add cloudsql mysql data stream * add cloudsql postgresql data stream * add cloudsql sqlserver data stream * add cloudsql docs * add changelog and cloudsql policy template * bump manifest version * add mysql ingest pipeline * add postgresql ingest pipeline * add sql server ingest pipeline * add mysql datastream suffixes, units and metric_types add drop processor if database is not mysql * add postgres datastream suffixes, units and metric_types add drop processor if database is not postgres * add sqlserver datastream suffixes, units and metric_types add drop processor if database is not sqlserver * update cloudsql docs * bump package version * fix field types * update docs * remove duplicated fields from agent due to elastic-package check --------- Co-authored-by: Edoardo Tenani <526307+endorama@users.noreply.github.com>
elastic · Jul 20, 2023 · 27e77c6 · 27e77c6
1 parent aff8ec4
commit 27e77c6
Show file tree

Hide file tree

Showing 34 changed files with 2,823 additions and 4 deletions.
diff --git a/packages/gcp/_dev/build/docs/cloudsql.md b/packages/gcp/_dev/build/docs/cloudsql.md
@@ -0,0 +1,30 @@
+# CloudSQL
+
+The `cloudsql` dataset fetches metrics from [CloudSQL](https://cloud.google.com/sql) in Google Cloud Platform. It contains all metrics exported from the [GCP CloudSQL Monitoring API](https://cloud.google.com/monitoring/api/metrics_gcp#gcp-cloudsql).
+
+`gcp.labels.cloudsql.name` label is utilized to identify the type of Google Cloud SQL database that generated the metrics. In the pipelines, this label is crucial for distinguishing between various Cloud SQL database types and directing the metrics to their respective destinations. Current valid values are `mysql`, `postgres` and `sqlserver`. Other values will be dropped.
+
+## MySQL Metrics
+
+CloudSQL MySQL metrics.
+
+{{event "cloudsql_mysql"}}
+
+{{fields "cloudsql_mysql"}}
+
+## PostgreSQL Metrics
+
+CloudSQL PostgreSQL metrics.
+
+{{event "cloudsql_postgresql"}}
+
+{{fields "cloudsql_postgresql"}}
+
+
+## SQL Server Metrics
+
+CloudSQL SQL Server metrics.
+
+{{event "cloudsql_sqlserver"}}
+
+{{fields "cloudsql_sqlserver"}}
diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.24.0"
+  changes:
+    - description: Add GCP CloudSQL MySQL, Postgres, SQLServer data streams
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4126
 - version: "2.23.0"
   changes:
     - description: Convert security dashboards to lens.

diff --git a/packages/gcp/data_stream/cloudsql_mysql/agent/stream/stream.yml.hbs b/packages/gcp/data_stream/cloudsql_mysql/agent/stream/stream.yml.hbs
@@ -0,0 +1,64 @@
+metricsets: ["metrics"]
+period: {{period}}
+project_id: {{project_id}}
+{{#if credentials_file}}
+credentials_file_path: {{credentials_file}}
+{{/if}}
+{{#if credentials_json}}
+credentials_json: '{{credentials_json}}'
+{{/if}}
+{{#if region}}
+region: {{region}}
+{{/if}}
+exclude_labels: {{exclude_labels}}
+metrics:
+  - aligner: ALIGN_NONE
+    service: cloudsql
+    metric_types:
+      - "database/active_directory/domain_reachable"
+      - "database/active_directory/instance_available"
+      - "database/auto_failover_request_count"
+      - "database/available_for_failover"
+      - "database/cpu/reserved_cores"
+      - "database/cpu/usage_time"
+      - "database/cpu/utilization"
+      - "database/disk/bytes_used"
+      - "database/disk/bytes_used_by_data_type"
+      - "database/disk/quota"
+      - "database/disk/read_ops_count"
+      - "database/disk/utilization"
+      - "database/disk/write_ops_count"
+      - "database/instance_state"
+      - "database/memory/quota"
+      - "database/memory/total_usage"
+      - "database/memory/usage"
+      - "database/memory/utilization"
+      - "database/mysql/innodb_buffer_pool_pages_dirty"
+      - "database/mysql/innodb_buffer_pool_pages_free"
+      - "database/mysql/innodb_buffer_pool_pages_total"
+      - "database/mysql/innodb_data_fsyncs"
+      - "database/mysql/innodb_os_log_fsyncs"
+      - "database/mysql/innodb_pages_read"
+      - "database/mysql/innodb_pages_written"
+      - "database/mysql/queries"
+      - "database/mysql/questions"
+      - "database/mysql/received_bytes_count"
+      - "database/mysql/replication/last_io_errno"
+      - "database/mysql/replication/last_sql_errno"
+      - "database/mysql/replication/seconds_behind_master"
+      - "database/mysql/replication/slave_io_running"
+      - "database/mysql/replication/slave_io_running_state"
+      - "database/mysql/replication/slave_sql_running"
+      - "database/mysql/replication/slave_sql_running_state"
+      - "database/mysql/sent_bytes_count"
+      - "database/network/connections"
+      - "database/network/received_bytes_count"
+      - "database/network/sent_bytes_count"
+      - "database/replication/log_archive_failure_count"
+      - "database/replication/log_archive_success_count"
+      - "database/replication/network_lag"
+      - "database/replication/replica_lag"
+      - "database/replication/state"
+      - "database/state"
+      - "database/up"
+      - "database/uptime"
diff --git a/packages/gcp/data_stream/cloudsql_mysql/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/cloudsql_mysql/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,202 @@
+---
+description: Pipeline for parsing GCP CloudSQL MySQL metrics.
+processors:
+  - drop:
+      description: Drop if database is not MySQL.
+      if: "ctx?.gcp?.labels?.cloudsql?.name != 'mysql'"
+  - rename:
+      field: gcp.metrics.database.active_directory.domain_reachable.value
+      target_field: gcp.cloudsql_mysql.database.active_directory.domain_reachable
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.active_directory.instance_available.value
+      target_field: gcp.cloudsql_mysql.database.active_directory.instance_available
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.auto_failover_request_count.value
+      target_field: gcp.cloudsql_mysql.database.auto_failover_request.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.available_for_failover.value
+      target_field: gcp.cloudsql_mysql.database.available_for_failover
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.cpu.reserved_cores.value
+      target_field: gcp.cloudsql_mysql.database.cpu.reserved_cores.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.cpu.usage_time.value
+      target_field: gcp.cloudsql_mysql.database.cpu.usage_time.sec
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.cpu.utilization.value
+      target_field: gcp.cloudsql_mysql.database.cpu.utilization.pct
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.disk.bytes_used.value
+      target_field: gcp.cloudsql_mysql.database.disk.bytes_used.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.disk.bytes_used_by_data_type.value
+      target_field: gcp.cloudsql_mysql.database.disk.bytes_used_by_data_type.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.disk.quota.value
+      target_field: gcp.cloudsql_mysql.database.disk.quota.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.disk.read_ops_count.value
+      target_field: gcp.cloudsql_mysql.database.disk.read_ops.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.disk.utilization.value
+      target_field: gcp.cloudsql_mysql.database.disk.utilization.pct
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.disk.write_ops_count.value
+      target_field: gcp.cloudsql_mysql.database.disk.write_ops.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.instance_state.value
+      target_field: gcp.cloudsql_mysql.database.instance_state
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.memory.quota.value
+      target_field: gcp.cloudsql_mysql.database.memory.quota.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.memory.total_usage.value
+      target_field: gcp.cloudsql_mysql.database.memory.total_usage.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.memory.usage.value
+      target_field: gcp.cloudsql_mysql.database.memory.usage.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.memory.utilization.value
+      target_field: gcp.cloudsql_mysql.database.memory.utilization.pct
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.innodb_buffer_pool_pages_dirty.value
+      target_field: gcp.cloudsql_mysql.database.innodb_buffer_pool_pages_dirty.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.innodb_buffer_pool_pages_free.value
+      target_field: gcp.cloudsql_mysql.database.innodb_buffer_pool_pages_free.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.innodb_buffer_pool_pages_total.value
+      target_field: gcp.cloudsql_mysql.database.innodb_buffer_pool_pages_total.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.innodb_data_fsyncs.value
+      target_field: gcp.cloudsql_mysql.database.innodb_data_fsyncs.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.innodb_os_log_fsyncs.value
+      target_field: gcp.cloudsql_mysql.database.innodb_os_log_fsyncs.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.innodb_pages_read.value
+      target_field: gcp.cloudsql_mysql.database.innodb_pages_read.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.innodb_pages_written.value
+      target_field: gcp.cloudsql_mysql.database.innodb_pages_written.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.queries.value
+      target_field: gcp.cloudsql_mysql.database.queries.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.questions.value
+      target_field: gcp.cloudsql_mysql.database.questions.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.received_bytes_count.value
+      target_field: gcp.cloudsql_mysql.database.received_bytes.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.replication.last_io_errno.value
+      target_field: gcp.cloudsql_mysql.database.replication.last_io_errno
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.replication.last_sql_errno.value
+      target_field: gcp.cloudsql_mysql.database.replication.last_sql_errno
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.replication.seconds_behind_master.value
+      target_field: gcp.cloudsql_mysql.database.replication.seconds_behind_master.sec
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.replication.slave_io_running.value
+      target_field: gcp.cloudsql_mysql.database.replication.slave_io_running
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.replication.slave_io_running_state.value
+      target_field: gcp.cloudsql_mysql.database.replication.slave_io_running_state
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.replication.slave_sql_running.value
+      target_field: gcp.cloudsql_mysql.database.replication.slave_sql_running
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.replication.slave_sql_running_state.value
+      target_field: gcp.cloudsql_mysql.database.replication.slave_sql_running_state
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.mysql.sent_bytes_count.value
+      target_field: gcp.cloudsql_mysql.database.sent_bytes.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.network.connections.value
+      target_field: gcp.cloudsql_mysql.database.network.connections.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.network.received_bytes_count.value
+      target_field: gcp.cloudsql_mysql.database.network.received_bytes.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.network.sent_bytes_count.value
+      target_field: gcp.cloudsql_mysql.database.network.sent_bytes.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.replication.log_archive_failure_count.value
+      target_field: gcp.cloudsql_mysql.database.replication.log_archive_failure.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.replication.log_archive_success_count.value
+      target_field: gcp.cloudsql_mysql.database.replication.log_archive_success.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.replication.network_lag.value
+      target_field: gcp.cloudsql_mysql.database.replication.network_lag.sec
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.replication.replica_lag.value
+      target_field: gcp.cloudsql_mysql.database.replication.replica_lag.sec
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.replication.state.value
+      target_field: gcp.cloudsql_mysql.database.replication.state
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.state.value
+      target_field: gcp.cloudsql_mysql.database.state
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.up.value
+      target_field: gcp.cloudsql_mysql.database.up
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.database.uptime.value
+      target_field: gcp.cloudsql_mysql.database.uptime.sec
+      ignore_missing: true
+  - remove:
+      field:
+        - gcp.metrics
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'