(SLV-672) Add system class to manage system metric

The new system class is very similar to the init.pp... this is to keep the two seperate during development.
They may get merged in teh future.  But the work flow is such that the master would have puppet metrics collector and sysystem included, while the compilers would only have system.  To merge them would require a clean way to turn off all pe metrics for the compilers.

Also added system_cpu and system_memory to use sar_metric to gather cpu and memory metrics.

README.md

@@ -37,6 +37,18 @@ node 'master.example.com' {
 }
 ```
 
+Optionally, you can also gather some basic system metrics.  Unlike the service metrics, this has to be enabled on each host you want metrics from, and the resulting data will be only on that host.  Do not include the top level puppet_metrics_collector on anything other than the master as it will collect the same data as the one on the master.
+
+```
+node 'master.example.com' {
+  include puppet_metrics_collector
+  include puppet_metrics_collector::system
+}
+node 'compilerA.example.com,compilerB.example.com,' {
+  include puppet_metrics_collector::system
+}
+``` 
+
 ### Configuration
 
 This module automatically configures the hosts it queries by querying PuppetDB for PE Infrastructure Hosts. If there is an error with automatic configuration of hosts, refer to [Manual Configuration of Hosts](#manual-configuration-of-hosts).

manifests/sar_metric.pp

@@ -0,0 +1,60 @@
+define puppet_metrics_collector::sar_metric (
+  String                    $output_dir,
+  String                    $scripts_dir,
+  Enum['absent', 'present'] $metric_ensure = 'present',
+  String                    $metrics_type = $title,
+  String                    $cron_minute = '*/5',
+  Integer                   $retention_days = 90,
+  Integer                   $polling_frequency_seconds = 1,
+  Integer                   $collection_frequency = 5, #minutes
+  String                    $metric_script_file = 'generate_system_metrics',
+) {
+
+  $metrics_output_dir = "${output_dir}/${metrics_type}"
+
+  $_metric_ensure = $metric_ensure ? {
+    'present' => directory,
+    'absent'  => absent,
+  }
+
+  file { $metrics_output_dir :
+    ensure => $_metric_ensure,
+  }
+
+  $script_file_name = "${scripts_dir}/${metric_script_file}"
+  $file_interval_seconds = $collection_frequency * 60
+
+  $metrics_command = join(["${script_file_name} --metric_type ${metrics_type}",
+                            " --file_interval ${file_interval_seconds}",
+                            " --polling_interval ${polling_frequency_seconds}",
+                            " --metrics_dir ${output_dir}"], '')
+
+  cron { "${metrics_type}_metrics_collection" :
+    ensure  => $metric_ensure,
+    command => $metrics_command,
+    user    => 'root',
+    minute  => $cron_minute,
+  }
+
+  $metrics_tidy_script_path = "${scripts_dir}/${metrics_type}_metrics_tidy"
+
+  file { $metrics_tidy_script_path :
+    ensure  => $metric_ensure,
+    mode    => '0744',
+    content => epp('puppet_metrics_collector/tidy_cron.epp', {
+      'metrics_output_dir' => $metrics_output_dir,
+      'metrics_type'       => $metrics_type,
+      'retention_days'     => $retention_days,
+    }),
+  }
+
+  # The hardcoded numbers with the fqdn call are to trigger the tidy to run at a randomly selected
+  # time between 12:00 AM and 3:00 AM
+  cron { "${metrics_type}_metrics_tidy" :
+    ensure  => $metric_ensure,
+    user    => 'root',
+    hour    => fqdn_rand(3, $metrics_type),
+    minute  => (5 * fqdn_rand(11, $metrics_type)),
+    command => $metrics_tidy_script_path
+  }
+}

manifests/system.pp

@@ -0,0 +1,48 @@
+class puppet_metrics_collector::system (
+  String        $output_dir                    = '/opt/puppetlabs/puppet-metrics-collector',
+  Integer       $collection_frequency          = 5, #minutes
+  Integer       $polling_frequency_seconds     = 1,
+  Integer       $retention_days                = 90,
+  String        $system_metrics_ensure         = present,
+  Boolean       $symlink_puppet_metrics_collector = true,
+) {
+  $scripts_dir = "${output_dir}/scripts"
+  $bin_dir     = "${output_dir}/bin"
+
+  #assume if output is defined, all of the rest will be too as the init.pp must be in use
+  #and thus we don't need to redefine these
+  if ! defined(File[$output_dir]) {
+    file { [ $output_dir, $scripts_dir, $bin_dir]:
+      ensure => directory,
+    }
+
+    file { "${scripts_dir}/generate_system_metrics":
+      ensure => present,
+      mode   => '0755',
+      source => 'puppet:///modules/puppet_metrics_collector/generate_system_metrics'
+    }
+
+    file { "${bin_dir}/puppet-metrics-collector":
+      ensure  => file,
+      owner   => 'root',
+      group   => 'root',
+      mode    => '0755',
+      content => epp('puppet_metrics_collector/puppet-metrics-collector.epp', {
+        'output_dir' => $output_dir,
+      }),
+    }
+
+    $symlink_ensure = $symlink_puppet_metrics_collector ? {
+      false => 'absent',
+      true  => 'symlink',
+    }
+
+    file { '/opt/puppetlabs/bin/puppet-metrics-collector':
+      ensure => $symlink_ensure,
+      target => "${bin_dir}/puppet-metrics-collector",
+    }
+
+  }
+  include puppet_metrics_collector::system_cpu
+  include puppet_metrics_collector::system_memory
+}

manifests/system_cpu.pp

@@ -0,0 +1,19 @@
+class puppet_metrics_collector::system_cpu (
+  Integer $collection_frequency      = $puppet_metrics_collector::system::collection_frequency,
+  Integer $polling_frequency_seconds = $puppet_metrics_collector::system::polling_frequency_seconds,
+  Integer $retention_days            = $puppet_metrics_collector::system::retention_days,
+  String  $metrics_ensure            = $puppet_metrics_collector::system::system_metrics_ensure,
+  ) {
+  Puppet_metrics_collector::Sar_metric {
+    output_dir                => $puppet_metrics_collector::system::output_dir,
+    scripts_dir               => $puppet_metrics_collector::system::scripts_dir,
+    cron_minute               => "*/${collection_frequency}",
+    collection_frequency      => $collection_frequency,
+    polling_frequency_seconds => $polling_frequency_seconds,
+    retention_days            => $retention_days,
+  }
+
+  puppet_metrics_collector::sar_metric { 'cpu' :
+    metric_ensure             => $metrics_ensure,
+  }
+}

manifests/system_memory.pp

@@ -0,0 +1,19 @@
+class puppet_metrics_collector::system_memory (
+  Integer $collection_frequency      = $puppet_metrics_collector::system::collection_frequency,
+  Integer $polling_frequency_seconds = $puppet_metrics_collector::system::polling_frequency_seconds,
+  Integer $retention_days            = $puppet_metrics_collector::system::retention_days,
+  String  $metrics_ensure            = $puppet_metrics_collector::system::system_metrics_ensure,
+  ) {
+  Puppet_metrics_collector::Sar_metric {
+    output_dir                => $puppet_metrics_collector::system::output_dir,
+    scripts_dir               => $puppet_metrics_collector::system::scripts_dir,
+    cron_minute               => "*/${collection_frequency}",
+    collection_frequency      => $collection_frequency,
+    polling_frequency_seconds => $polling_frequency_seconds,
+    retention_days            => $retention_days,
+  }
+
+  puppet_metrics_collector::sar_metric { 'memory' :
+    metric_ensure             => $metrics_ensure,
+  }
+}