From 411488d06d4ca8db597d2ebf149714d5d7bc6f38 Mon Sep 17 00:00:00 2001 From: Seth Cleveland Date: Mon, 15 Aug 2016 18:04:40 -0500 Subject: [PATCH 1/3] Support generic postgres custom metrics - Adding resource definition for postgres custom metrics (for validation) - Added custom metrics example and test specs --- manifests/integrations/postgres.pp | 23 ++++++++- .../integrations/postgres_custom_metric.pp | 27 +++++++++++ ...atadog_agent_integrations_postgres_spec.rb | 47 +++++++++++++++++++ templates/agent-conf.d/postgres.yaml.erb | 31 ++++++++++++ 4 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 manifests/integrations/postgres_custom_metric.pp diff --git a/manifests/integrations/postgres.pp b/manifests/integrations/postgres.pp index ae0b9583..07440d49 100644 --- a/manifests/integrations/postgres.pp +++ b/manifests/integrations/postgres.pp @@ -19,6 +19,12 @@ # Track per relation/table metrics. Array of strings. # Warning: this can collect lots of metrics per relation # (10 + 10 per index) +# $tags +# Optional array of tags +# $custom_metrics +# An array of custom metrics with the following hash keys - query, metrics, +# relation, descriptors. Refer to this guide for details on those fields: +# https://help.datadoghq.com/hc/en-us/articles/208385813-Postgres-custom-metric-collection-explained # # Sample Usage: # @@ -27,6 +33,18 @@ # dbname => 'postgres' # username => 'datadog', # password => 'some_pass', +# custom_metrics => { +# a_custom_query => { +# query => "select tag_column, %s from table", +# relation => false, +# metrics => { +# value_column => ["value_column.datadog.tag", "GAUGE"] +# }, +# descriptors => [ +# ["tag_column", "tag_column.datadog.tag"] +# ] +# } +# } # } # # @@ -37,7 +55,8 @@ $username = 'datadog', $password, $tags = [], - $tables = [] + $tables = [], + $custom_metrics = {}, ) inherits datadog_agent::params { include datadog_agent @@ -53,4 +72,6 @@ require => Package[$datadog_agent::params::package_name], notify => Service[$datadog_agent::params::service_name], } + + create_resources('datadog_agent::integrations::postgres_custom_metric', $custom_metrics) } diff --git a/manifests/integrations/postgres_custom_metric.pp b/manifests/integrations/postgres_custom_metric.pp new file mode 100644 index 00000000..9de36b25 --- /dev/null +++ b/manifests/integrations/postgres_custom_metric.pp @@ -0,0 +1,27 @@ +## +# The postgres_custom_metric defines a custom sql metric. +# https://help.datadoghq.com/hc/en-us/articles/208385813-Postgres-custom-metric-collection-explained +# +# $query: +# The custom metric SQL query. It must contain a '%s' for defining the metrics. +# +# $metrics: +# a hash of column name to metric definition. a metric definition is an array +# consisting of two columns -- the datadog metric name and the metric type. +# +# $relation: +# ? +# +# $descriptor: +# an array that maps an sql column's to a tag. Each descriptor consists of two +# fields -- column name, and datadog tag. +define datadog_agent::integrations::postgres_custom_metric( + $query, + $metrics, + $relation = false, + $descriptors = [], +) { + validate_re($query, '^.*%s.*$', 'custom_metrics require %s for metric substitution') + validate_hash($metrics) + validate_array($descriptors) +} diff --git a/spec/classes/datadog_agent_integrations_postgres_spec.rb b/spec/classes/datadog_agent_integrations_postgres_spec.rb index f7e514c9..8a1c13ad 100644 --- a/spec/classes/datadog_agent_integrations_postgres_spec.rb +++ b/spec/classes/datadog_agent_integrations_postgres_spec.rb @@ -55,6 +55,53 @@ it { should contain_file(conf_file).with_content(%r{username: monitoring}) } it { should contain_file(conf_file).with_content(%r{^[^#]*tags:\s+- foo\s+- bar\s+- baz}) } it { should contain_file(conf_file).with_content(%r{^[^#]*relations:\s+- furry\s+- fuzzy\s+- funky}) } + + context 'with custom metric query missing %s' do + let(:params) {{ + host: 'postgres1', + dbname: 'cats', + port: 4142, + username: 'monitoring', + password: 'abc123', + custom_metrics: { + 'query_is_missing_%s' => { + 'query' => 'select * from fuzz', + 'metrics' => { }, + } + } + }} + it do + expect { + is_expected.to compile + }.to raise_error(/custom_metrics require %s for metric substitution/) + end + end + + context 'with custom metric query' do + let(:params) {{ + host: 'postgres1', + dbname: 'cats', + port: 4142, + username: 'monitoring', + password: 'abc123', + custom_metrics: { + 'foo_gooo_bar_query' => { + 'query' => 'select foo, %s from bar', + 'metrics' => { + "gooo" => ["custom_metric.tag.gooo", "GAUGE"] + }, + 'descriptors' => [["foo", "custom_metric.tag.foo"]] + } + } + }} + it { is_expected.to compile } + it { should contain_file(conf_file).with_content(%r{^[^#]*custom_metrics:}) } + it { should contain_file(conf_file).with_content(%r{\s+query:\s*['"]?select foo, %s from bar['"]?}) } + it { should contain_file(conf_file).with_content(%r{\s+metrics:}) } + it { should contain_file(conf_file).with_content(%r{\s+"gooo":\s+\[custom_metric.tag.gooo, GAUGE\]}) } + it { should contain_file(conf_file).with_content(%r{\s+query.*\n\s+relation:\s*false}) } + it { should contain_file(conf_file).with_content(%r{\s+descriptors.*\n\s+-\s+\[foo, custom_metric.tag.foo\]}) } + end end end diff --git a/templates/agent-conf.d/postgres.yaml.erb b/templates/agent-conf.d/postgres.yaml.erb index 86608caf..81c800b5 100644 --- a/templates/agent-conf.d/postgres.yaml.erb +++ b/templates/agent-conf.d/postgres.yaml.erb @@ -41,3 +41,34 @@ instances: <%- end -%> <%- end -%> <% end -%> +# https://help.datadoghq.com/hc/en-us/articles/208385813-Postgres-custom-metric-collection-explained +# custom_metrics: +# - # Capture simple data +# query: SELECT name, address, %s from company where name = 'Paul'; # this query will be run and "%s" replaced with the parameters defined in the metrics section just below +# metrics: +# age: [postgresql.employee_age, GAUGE] # the value contained in column "age" will be captured and submitted as a gauge metric named "postgresql.employee.age" +# salary: [postgresql.employee_salary, GAUGE] +# relation: false # when relation is not an empty list, it gathers per-relation metrics on top of that. +# descriptors: +# - [name, name] # captures the content of the "name" column as a tag for the 2 metrics defined +# - [address, localisation] # captures the content of "address" column as a tag and renames this tag "localisation" +# +<% if @custom_metrics and ! @custom_metrics.empty? -%> + custom_metrics: + <%- Array(@custom_metrics).each do |n, custom_metric| -%> + - query: <%= custom_metric["query"] %> + relation: <%= custom_metric["relation"]? "true" : "false" %> + metrics: + <%- Array(custom_metric["metrics"]).each do |metric_name, metric_descriptor| -%> + "<%= metric_name %>": [<%= metric_descriptor[0] %>, <%= metric_descriptor[1] %>] + <%- end -%> + <%- if custom_metric["descriptors"] == nil || custom_metric["descriptors"].empty? -%> + descriptors: [] + <%- else -%> + descriptors: + <%- Array(custom_metric["descriptors"]).each do |descriptor| -%> + - [<%=descriptor[0]%>, <%=descriptor[1] %>] + <%- end -%> + <%- end -%> + <%- end -%> +<% end -%> From 5cd38a4f1cd306a12eed43aca20f8fe81f12b4c8 Mon Sep 17 00:00:00 2001 From: Seth Cleveland Date: Fri, 9 Sep 2016 12:11:43 -0500 Subject: [PATCH 2/3] swapped order of optional vs. non-optional class parameters to address failing builds --- manifests/integrations/mysql.pp | 6 +++--- manifests/integrations/pgbouncer.pp | 6 +++--- manifests/integrations/postgres.pp | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifests/integrations/mysql.pp b/manifests/integrations/mysql.pp index 5046ff01..3ea45fde 100644 --- a/manifests/integrations/mysql.pp +++ b/manifests/integrations/mysql.pp @@ -3,10 +3,10 @@ # This class will install the necessary configuration for the mysql integration # # Parameters: -# $host: -# The host mysql is running on # $password # The mysql password for the datadog user +# $host: +# The host mysql is running on # $user # The mysql user for the datadog user # $sock @@ -37,8 +37,8 @@ # # class datadog_agent::integrations::mysql( - $host = 'localhost', $password, + $host = 'localhost', $user = 'datadog', $sock = undef, $tags = [], diff --git a/manifests/integrations/pgbouncer.pp b/manifests/integrations/pgbouncer.pp index 75e95153..d9889df7 100644 --- a/manifests/integrations/pgbouncer.pp +++ b/manifests/integrations/pgbouncer.pp @@ -3,14 +3,14 @@ # This class will install the necessary configuration for the pgbouncer integration # # Parameters: +# $password +# The password for the datadog user # $host: # The host pgbouncer is listening on # $port # The pgbouncer port number # $username # The username for the datadog user -# $password -# The password for the datadog user # $tags # Optional array of tags # @@ -25,10 +25,10 @@ # # class datadog_agent::integrations::pgbouncer( + $password, $host = 'localhost', $port = '6432', $username = 'datadog', - $password, $tags = [], ) inherits datadog_agent::params { diff --git a/manifests/integrations/postgres.pp b/manifests/integrations/postgres.pp index 07440d49..592280a7 100644 --- a/manifests/integrations/postgres.pp +++ b/manifests/integrations/postgres.pp @@ -3,6 +3,8 @@ # This class will install the necessary configuration for the postgres integration # # Parameters: +# $password +# The password for the datadog user # $host: # The host postgres is running on # $dbname @@ -11,8 +13,6 @@ # The postgres port number # $username # The username for the datadog user -# $password -# The password for the datadog user # $tags # Optional array of tags # $tables @@ -49,11 +49,11 @@ # # class datadog_agent::integrations::postgres( + $password, $host = 'localhost', $dbname = 'postgres', $port = '5432', $username = 'datadog', - $password, $tags = [], $tables = [], $custom_metrics = {}, From efd2531d23efde8f6a841f8fedbcdf9c77f253c8 Mon Sep 17 00:00:00 2001 From: Seth Cleveland Date: Mon, 19 Sep 2016 10:17:23 -0500 Subject: [PATCH 3/3] correcting custom_metrics documentation --- manifests/integrations/postgres.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/integrations/postgres.pp b/manifests/integrations/postgres.pp index 592280a7..692bc772 100644 --- a/manifests/integrations/postgres.pp +++ b/manifests/integrations/postgres.pp @@ -22,7 +22,7 @@ # $tags # Optional array of tags # $custom_metrics -# An array of custom metrics with the following hash keys - query, metrics, +# A hash of custom metrics with the following keys - query, metrics, # relation, descriptors. Refer to this guide for details on those fields: # https://help.datadoghq.com/hc/en-us/articles/208385813-Postgres-custom-metric-collection-explained #