[agent6] adding support for the agent beta (DataDog#356)

* [agent6] add support for agent6 config locations.

* [agent6] adding agent6 repos.

* [agent6] adding datadog.yaml config template.

* [agent6] preparing main manifest for agent6 support.

fix lint issues

* [agent6][spec] fixes + spec test work

[spec] fix for older rubies.

[agent6] consistent YAML experience across puppets

[spec][agent6] cover old puppet ZAML regex

* [agent6][repo] remove beta if on stable and viceversa

* [agent6][spec] adding RHEL and Ubuntu repo tests

* [agent6][yum] no ensure on yumrepo -> use datadog repo definition for both beta and stable.

* [docker_daemon] agent6 goes back to docker.yaml for config

* [gemfile] pin jwt to 1.5.6

Gemfile

@@ -2,13 +2,15 @@ source "https://rubygems.org"
 
 group :test do
   gem "syck"
+  gem "public_suffix", "~> 2.0.5"
   gem "safe_yaml", "~> 1.0.4"
   gem "hiera-eyaml", "~> 2.1.0"
   gem "listen", "~> 3.0.0"
   gem "puppet", ENV['PUPPET_VERSION'] || '~> 4.2.0'
   gem "puppet-lint"
   gem "puppet-syntax"
   gem "puppetlabs_spec_helper"
+  gem "jwt", "~> 1.5.6"
   gem "rake"
   gem "rspec-puppet", '2.3.2'
 end

Rakefile

@@ -19,6 +19,7 @@ PuppetLint.configuration.fail_on_warnings = true
 PuppetLint.configuration.send('disable_class_parameter_defaults')
 # http://puppet-lint.com/checks/class_inherits_from_params_class/
 PuppetLint.configuration.send('disable_class_inherits_from_params_class')
+PuppetLint.configuration.send('disable_variable_scope')
 
 exclude_paths = [
   "pkg/**/*",

lib/puppet/reports/datadog_reports.rb

@@ -9,7 +9,7 @@
 
 Puppet::Reports.register_report(:datadog_reports) do
 
-  configfile = "/etc/dd-agent/datadog.yaml"
+  configfile = "/etc/datadog-agent/datadog-reports.yaml"
   raise(Puppet::ParseError, "Datadog report config file #{configfile} not readable") unless File.readable?(configfile)
   config = YAML.load_file(configfile)
   API_KEY = config[:datadog_api_key]

manifests/init.pp

@@ -102,6 +102,9 @@
 #   $use_dogstatsd
 #       Enables the dogstatsd server
 #       Boolean. Default: true
+#   $dogstatsd_socket
+#       Specifies the socket file to be used by dogstatsd. Must have use_dogstatsd set
+#       String. Default: empty
 #   $dogstatsd_port
 #       Specifies the port to be used by dogstatsd. Must have use_dogstatsd set
 #       String. Default: empty
@@ -139,6 +142,9 @@
 #   $custom_emitters
 #       Specifies a comma seperated list of non standard emitters to be used
 #       String. Default: empty
+#   $agent6_log_file
+#       Specifies the log file location for the agent6
+#       String. Default: empty
 #   $collector_log_file
 #       Specifies the log file location for the collector system
 #       String. Default: empty
@@ -199,6 +205,7 @@
   $manage_repo = true,
   $hostname_extraction_regex = nil,
   $dogstatsd_port = 8125,
+  $dogstatsd_socket = '',
   $statsd_forward_host = '',
   $statsd_forward_port = '',
   $statsd_histogram_percentiles = '0.95',
@@ -227,6 +234,7 @@
   $dogstatsd_normalize = true,
   $device_blacklist_re = '',
   $custom_emitters = '',
+  $agent6_log_file = '/var/log/datadog/agent.log',
   $collector_log_file = '',
   $forwarder_log_file = '',
   $dogstatsd_log_file = '',
@@ -240,7 +248,9 @@
   $sd_template_dir = '',
   $sd_jmx_enable = false,
   $consul_token = '',
+  $agent6_enable = $datadog_agent::params::agent6_enable,
   $conf_dir = $datadog_agent::params::conf_dir,
+  $conf6_dir = $datadog_agent::params::conf6_dir,
   $conf_dir_purge = $datadog_agent::params::conf_dir_purge,
   $service_name = $datadog_agent::params::service_name,
   $package_name = $datadog_agent::params::package_name,
@@ -306,6 +316,7 @@
   validate_string($statsd_forward_host)
   validate_string($device_blacklist_re)
   validate_string($custom_emitters)
+  validate_string($agent6_log_file)
   validate_string($collector_log_file)
   validate_string($forwarder_log_file)
   validate_string($dogstatsd_log_file)
@@ -320,6 +331,7 @@
   validate_bool($sd_jmx_enable)
   validate_string($consul_token)
   validate_bool($apm_enabled)
+  validate_bool($agent6_enable)
   validate_string($apm_env)
 
   if $hiera_tags {
@@ -352,74 +364,131 @@
   }
 
   case $::operatingsystem {
-    'Ubuntu','Debian' : { include datadog_agent::ubuntu }
+    'Ubuntu','Debian' : {
+      if !$agent6_enable {
+        include datadog_agent::ubuntu
+      } else {
+        include datadog_agent::ubuntu::agent6
+      }
+    }
     'RedHat','CentOS','Fedora','Amazon','Scientific' : {
-      class { 'datadog_agent::redhat':
-        manage_repo => $manage_repo,
+      if !$agent6_enable {
+        class { 'datadog_agent::redhat':
+          manage_repo => $manage_repo,
+        }
+      } else {
+        class { 'datadog_agent::redhat::agent6':
+          manage_repo => $manage_repo,
+        }
       }
     }
     default: { fail("Class[datadog_agent]: Unsupported operatingsystem: ${::operatingsystem}") }
   }
 
-  file { '/etc/dd-agent':
+  # required by reports even in agent5 scenario
+  file { '/etc/datadog-agent':
     ensure  => directory,
     owner   => $dd_user,
     group   => $dd_group,
     mode    => '0755',
     require => Package['datadog-agent'],
   }
 
-  file { $conf_dir:
-    ensure  => directory,
-    purge   => $conf_dir_purge,
-    recurse => true,
-    force   => $conf_dir_purge,
-    owner   => $dd_user,
-    group   => $dd_group,
-    notify  => Service['datadog-agent']
-  }
 
-  concat {'/etc/dd-agent/datadog.conf':
-    owner   => $datadog_agent::params::dd_user,
-    group   => $datadog_agent::params::dd_group,
-    mode    => '0640',
-    notify  => Service[$datadog_agent::params::service_name],
-    require => File['/etc/dd-agent'],
-  }
+  if !$agent6_enable {
+    file { '/etc/dd-agent':
+      ensure  => directory,
+      owner   => $dd_user,
+      group   => $dd_group,
+      mode    => '0755',
+      require => Package['datadog-agent'],
+    }
 
-  concat::fragment{ 'datadog header':
-    target  => '/etc/dd-agent/datadog.conf',
-    content => template('datadog_agent/datadog_header.conf.erb'),
-    order   => '01',
-  }
+    file { $conf_dir:
+      ensure  => directory,
+      purge   => $conf_dir_purge,
+      recurse => true,
+      force   => $conf_dir_purge,
+      owner   => $dd_user,
+      group   => $dd_group,
+      notify  => Service['datadog-agent']
+    }
 
-  concat::fragment{ 'datadog tags':
-    target  => '/etc/dd-agent/datadog.conf',
-    content => 'tags: ',
-    order   => '02',
-  }
+    concat {'/etc/dd-agent/datadog.conf':
+      owner   => $datadog_agent::params::dd_user,
+      group   => $datadog_agent::params::dd_group,
+      mode    => '0640',
+      notify  => Service[$datadog_agent::params::service_name],
+      require => File['/etc/dd-agent'],
+    }
 
-  concat::fragment{ 'datadog footer':
-    target  => '/etc/dd-agent/datadog.conf',
-    content => template('datadog_agent/datadog_footer.conf.erb'),
-    order   => '05',
-  }
+    concat::fragment{ 'datadog header':
+      target  => '/etc/dd-agent/datadog.conf',
+      content => template('datadog_agent/datadog_header.conf.erb'),
+      order   => '01',
+    }
+
+    concat::fragment{ 'datadog tags':
+      target  => '/etc/dd-agent/datadog.conf',
+      content => 'tags: ',
+      order   => '02',
+    }
+
+    concat::fragment{ 'datadog footer':
+      target  => '/etc/dd-agent/datadog.conf',
+      content => template('datadog_agent/datadog_footer.conf.erb'),
+      order   => '05',
+    }
+
+    if ($extra_template != '') {
+      concat::fragment{ 'datadog extra_template footer':
+        target  => '/etc/dd-agent/datadog.conf',
+        content => template($extra_template),
+        order   => '06',
+      }
+      $apm_footer_order = '07'
+    } else {
+      $apm_footer_order = '06'
+    }
 
-  if ($extra_template != '') {
-    concat::fragment{ 'datadog extra_template footer':
+    concat::fragment{ 'datadog apm footer':
       target  => '/etc/dd-agent/datadog.conf',
-      content => template($extra_template),
-      order   => '06',
+      content => template('datadog_agent/datadog_apm_footer.conf.erb'),
+      order   => $apm_footer_order,
     }
-    $apm_footer_order = '07'
   } else {
-    $apm_footer_order = '06'
-  }
 
-  concat::fragment{ 'datadog apm footer':
-    target  => '/etc/dd-agent/datadog.conf',
-    content => template('datadog_agent/datadog_apm_footer.conf.erb'),
-    order   => $apm_footer_order,
+    file { $conf6_dir:
+      ensure  => directory,
+      purge   => $conf_dir_purge,
+      recurse => true,
+      force   => $conf_dir_purge,
+      owner   => $dd_user,
+      group   => $dd_group,
+      notify  => Service['datadog-agent']
+    }
+
+    $agent_config = {
+      'api_key' => $api_key,
+      'dd_url' => $dd_url,
+      'cmd_port' => 5001,
+      'conf_path' => $datadog_agent::params::conf6_dir,
+      'enable_metadata_collection' => $collect_instance_metadata,
+      'dogstatsd_port' => $dogstatsd_port,
+      'dogstatsd_socket' => $dogstatsd_socket,
+      'dogstatsd_non_local_traffic' => $non_local_traffic,
+      'log_file' => $agent6_log_file,
+      'log_level' => $log_level,
+    }
+
+    file { '/etc/datadog-agent/datadog.yaml':
+      owner   => 'dd-agent',
+      group   => 'dd-agent',
+      mode    => '0640',
+      content => template('datadog_agent/datadog6.yaml.erb'),
+      notify  => Service[$datadog_agent::params::service_name],
+      require => File['/etc/datadog-agent'],
+    }
   }
 
 

manifests/integration.pp

@@ -12,7 +12,13 @@
   }
   validate_string($integration)
 
-  file { "${datadog_agent::conf_dir}/${integration}.yaml":
+  if $::datadog_agent::agent6_enable {
+    $dst = "${datadog_agent::conf6_dir}/${integration}.yaml"
+  } else {
+    $dst = "${datadog_agent::conf_dir}/${integration}.yaml"
+  }
+
+  file { $dst:
     ensure  => file,
     owner   => $datadog_agent::dd_user,
     group   => $datadog_agent::dd_group,

manifests/integrations/apache.pp

@@ -39,7 +39,13 @@
   validate_array($tags)
   validate_bool($disable_ssl_validation)
 
-  file { "${datadog_agent::params::conf_dir}/apache.yaml":
+  if $::datadog_agent::agent6_enable {
+    $dst = "${datadog_agent::conf6_dir}/apache.yaml"
+  } else {
+    $dst = "${datadog_agent::conf_dir}/apache.yaml"
+  }
+
+  file { $dst:
     ensure  => file,
     owner   => $datadog_agent::params::dd_user,
     group   => $datadog_agent::params::dd_group,

manifests/integrations/cacti.pp

@@ -20,7 +20,13 @@
 ) inherits datadog_agent::params {
   include datadog_agent
 
-  file { "${datadog_agent::params::conf_dir}/cacti.yaml":
+  if $::datadog_agent::agent6_enable {
+    $dst = "${datadog_agent::conf6_dir}/cacti.yaml"
+  } else {
+    $dst = "${datadog_agent::conf_dir}/cacti.yaml"
+  }
+
+  file { $dst:
     ensure  => file,
     owner   => $datadog_agent::params::dd_user,
     group   => $datadog_agent::params::dd_group,

manifests/integrations/cassandra.pp

@@ -35,7 +35,13 @@
   require ::datadog_agent
   validate_hash($tags)
 
-  file { "${datadog_agent::params::conf_dir}/cassandra.yaml":
+  if $::datadog_agent::agent6_enable {
+    $dst = "${datadog_agent::conf6_dir}/cassandra.yaml"
+  } else {
+    $dst = "${datadog_agent::conf_dir}/cassandra.yaml"
+  }
+
+  file { $dst:
     ensure  => file,
     owner   => $datadog_agent::params::dd_user,
     group   => $datadog_agent::params::dd_group,

manifests/integrations/ceph.pp

@@ -25,7 +25,13 @@
     content => "# This file is required for dd ceph \ndd-agent ALL=(ALL) NOPASSWD:/usr/bin/ceph\n"
   }
 
-  file { "${datadog_agent::params::conf_dir}/ceph.yaml":
+  if $::datadog_agent::agent6_enable {
+    $dst = "${datadog_agent::conf6_dir}/ceph.yaml"
+  } else {
+    $dst = "${datadog_agent::conf_dir}/ceph.yaml"
+  }
+
+  file { $dst:
     ensure  => file,
     owner   => $datadog_agent::params::dd_user,
     group   => $datadog_agent::params::dd_group,

manifests/integrations/consul.pp

@@ -39,7 +39,13 @@
   validate_bool($new_leader_checks)
   validate_array($service_whitelist)
 
-  file { "${datadog_agent::params::conf_dir}/consul.yaml":
+  if $::datadog_agent::agent6_enable {
+    $dst = "${datadog_agent::conf6_dir}/consul.yaml"
+  } else {
+    $dst = "${datadog_agent::conf_dir}/consul.yaml"
+  }
+
+  file { $dst:
     ensure  => file,
     owner   => $datadog_agent::params::dd_user,
     group   => $datadog_agent::params::dd_group,

manifests/integrations/directory.pp

@@ -31,7 +31,13 @@
     fail('you must specify a directory path within the datadog_agent::integrations::directory class')
   }
 
-  file { "${datadog_agent::params::conf_dir}/directory.yaml":
+  if $::datadog_agent::agent6_enable {
+    $dst = "${datadog_agent::conf6_dir}/directory.yaml"
+  } else {
+    $dst = "${datadog_agent::conf_dir}/directory.yaml"
+  }
+
+  file { $dst:
     ensure  => file,
     owner   => $datadog_agent::params::dd_user,
     group   => $datadog_agent::params::dd_group,

manifests/integrations/disk.pp

@@ -47,7 +47,13 @@
     validate_re($all_partitions, '^(no|yes)$', "all_partitions should be either 'yes' or 'no'")
   }
 
-  file { "${datadog_agent::params::conf_dir}/disk.yaml":
+  if $::datadog_agent::agent6_enable {
+    $dst = "${datadog_agent::conf6_dir}/disk.yaml"
+  } else {
+    $dst = "${datadog_agent::conf_dir}/disk.yaml"
+  }
+
+  file { $dst:
     ensure  => file,
     owner   => $datadog_agent::params::dd_user,
     group   => $datadog_agent::params::dd_group,