(SLV-653) Add a script for generating system stats

The script is intended to be used as part of an optionial feature to gate system stats the same way other stats
are gathered and managed.
It uses sar which will poll every x seconds and generate an average, which can then be captured into a file at the
same interval as the puppet-metrics-collector polling interval.  This keeps the total data down while not missing
things that cause short duration spikes (like compiles that last 6 seconds).  Initial use will be in performace testing.

files/generate_system_metrics

@@ -0,0 +1,252 @@
+#!/opt/puppetlabs/puppet/bin/ruby
+# frozen_string_literal: true
+
+require "optparse"
+require "json"
+require "time"
+require "fileutils"
+
+# This script is intended to be run on a puppet infrastructure node as part of the
+# puppet_metrics_collector module.  It will generate system statistics in the file format that
+# other puppet_metrics_collector stats follow.  It will poll the system for data at a given
+# interval, and then output the average to a file once per given file interval.
+
+# General namespace for SystemMetrics module
+module SystemMetrics
+  # Main class for collecting system metrics into a json file
+  #
+  # @author Randell Pelak
+  #
+  # @attr [integer] polling_interval Time in seconds between calls to poll the system for data.
+  # @attr [integer] file_interval Time in seconds between the creation of each output file.
+  # @attr [string] metric_type cpu|memory
+  # @attr [string] metrics_dir The puppet_metrics_collector output directory.
+  # @attr [boolean] verbose Verbose output
+  # @attr [string] hostname Name of the host the metrics are from. In directory name and json file.
+  #
+  class GenerateSystemMetrics
+    #
+    # Initialize class
+    #
+    # @author Randell Pelak
+    #
+    # @param [integer] polling_interval Time in seconds between calls to poll the system for data.
+    # @param [integer] file_interval Tim in seconds between the creation of each output file.
+    # @param [string] metric_type cpu|memory
+    # @param [string] metrics_dir The puppet_metrics_collector output directory.
+    # @param [boolean] verbose Verbose output
+    #
+    # @return [void]
+    def initialize(polling_interval, file_interval, metric_type, metrics_dir, verbose = false)
+      @polling_interval = polling_interval
+      @file_interval = file_interval
+      @metric_type = metric_type
+      @metrics_dir = metrics_dir
+      @verbose = verbose
+
+      @hostname = %x[hostname].strip
+      puts "Hostname is: #{@hostname}" if @verbose
+      FileUtils.mkdir_p(metrics_dir) unless File.directory?(metrics_dir)
+    end
+
+    # Run sar to collect the raw system data
+    #
+    # @author Randell Pelak
+    #
+    # @return [string] raw output from sar
+    def run_sar
+      times_to_poll = ((@file_interval / @polling_interval) + 0.5).to_i
+      # sar inputs are polling interval and how many times to poll
+      comm_flags = " -r" if @metric_type =~ /memory/
+      comm = "/usr/bin/sar #{comm_flags} #{@polling_interval} #{times_to_poll}"
+      puts "sar command is: #{comm}" if @verbose
+      %x[#{comm}]
+    end
+
+    # Confirm the output from sar is valid
+    #
+    # @author Randell Pelak
+    #
+    # @param [string] sar_output
+    #
+    # @return [boolean]
+    def sar_output_valid?(sar_output)
+      last_line = sar_output.split("\n")[-1]
+      return false unless last_line =~ /^Average:/
+      header = get_header_fields(sar_output)
+      averages = get_averages_fields(sar_output)
+      return false unless header.count == averages.count
+      true
+    end
+
+    # Gets the header fields from sar output
+    #
+    # @author Randell Pelak
+    #
+    # @param [string] sar_output
+    #
+    # @return [array] header fields
+    def get_header_fields(sar_output)
+      arr = sar_output.split("\n")
+      arr[2].split(" ")[2..-1]
+    end
+
+    # Gets the averages fields from sar output
+    #
+    # @author Randell Pelak
+    #
+    # @param [string] sar_output
+    #
+    # @return [array] average fields
+    def get_averages_fields(sar_output)
+      arr = sar_output.split("\n")
+      arr[-1].split(" ")[1..-1]
+    end
+
+    # Convert the inputted sar output into json
+    #
+    # @author Randell Pelak
+    #
+    # @param [string] sar_output
+    # @param [obj] time_stamp_obj Time object to use for generating the filename
+    #
+    # @return [string] json of sar output
+    def convert_sar_output_to_json(sar_output, time_stamp_obj)
+      header = get_header_fields(sar_output)
+      averages = get_averages_fields(sar_output)
+      hostkey = @hostname.gsub('.', '-')
+
+      dataset = {'time_stamp_obj' => time_stamp_obj.utc.iso8601, 'servers' => {}}
+      #combine the arrays into a hash and convert into json
+      metrics_data = Hash[header.zip(averages)]
+      dataset['servers'][hostkey] = {@metric_type => metrics_data}
+      json_dataset = JSON.pretty_generate(dataset)
+      return json_dataset
+    end
+
+    # Create the file and put the json data in it
+    #
+    # @author Randell Pelak
+    #
+    # @param [string] json_dataset data in json format to put in file
+    # @param [obj] time_stamp_obj Time object to use for generating the filename
+    #
+    # @return [void]
+    def create_file(json_dataset, time_stamp_obj)
+      filename = time_stamp_obj.utc.strftime('%Y%m%dT%H%M%SZ') + '.json'
+      dirname = "#{@metrics_dir}/system_#{@metric_type}/#{@hostname}"
+      file_path = "#{dirname}/#{filename}"
+      FileUtils.mkdir_p(dirname) unless File.directory?(dirname)
+      puts "Creating json file: #{file_path}" if @verbose
+      File.open(file_path, 'w') do |file|
+        file.write(json_dataset)
+      end
+    end
+
+
+    # Get the data and create the file
+    #
+    # @author Randell Pelak
+    #
+    # @return [void]
+    def go
+      output = run_sar
+      raise(_("sar output invalid")) unless sar_output_valid?(output)
+      time_stamp_obj = Time.now
+      json_data = convert_sar_output_to_json(output, time_stamp_obj)
+      create_file(json_data,time_stamp_obj)
+    end
+
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+
+  VALID_METRIC_TYPES = %w[cpu memory]
+  FILE_INTERVAL_DEFAULT = 60 * 5
+  POLLING_INTERVAL_DEFAULT = 1
+  METRIC_TYPE_DEFAULT = "cpu"
+  METRICS_DIR_DEFAULT = "/opt/puppetlabs/puppet-metrics-collector"
+
+  DESCRIPTION = <<-DESCRIPTION
+    This script is intended to be run on a puppet infrastructure node as part of the
+    puppet_metrics_collector module.  It will generate system statistics in the file format that
+    other puppet_metrics_collector stats follow.  It will poll the system for data at a given
+    interval, and then output the average to a file once per given file interval.
+  DESCRIPTION
+
+  DEFAULTS = <<-DEFAULTS
+    The following defaults values are used if the options are not specified:
+      * polling_interval (-p, --polling_interval): #{POLLING_INTERVAL_DEFAULT}
+      * file_interval (-f, --file_interval): #{FILE_INTERVAL_DEFAULT}
+      * metric_type (-t, --metric_type): #{METRIC_TYPE_DEFAULT}
+      * metrics_dir (-m, --metrics_dir): #{METRICS_DIR_DEFAULT}
+      * verbose (-v, --verbose): False
+  DEFAULTS
+
+  options = {}
+
+  OptionParser.new do |opts|
+    opts.banner = "Usage: generate_system_stats.rb [options]"
+
+    opts.on("-h", "--help", "Display the help text") do
+      puts DESCRIPTION
+      puts opts
+      puts DEFAULTS
+      exit
+    end
+
+    opts.on("-p", "--polling_interval seconds", Integer,
+            _("Time in seconds between calls to poll the system for data.")) do |interval|
+      options[:polling_interval] = interval
+    end
+    opts.on("-f", "--file_interval seconds", Integer,
+            _("Time in seconds between the creation of each output file.")) do |interval|
+      options[:file_interval] = interval
+    end
+    opts.on("-t", "--metric_type type", String,
+            "One of: #{VALID_METRIC_TYPES.join(', ')}") do |type|
+      options[:metric_type] = type.downcase
+    end
+    opts.on("-m", "--metrics_dir dir_path", String,
+            "The puppet_metrics_collector output directory") do |metrics_dir|
+      options[:metrics_dir] = metrics_dir
+    end
+    opts.on("-v", "--verbose", String, "Enable Verbose output") { options[:verbose] = true }
+  end.parse!
+
+  if options[:metric_type]
+    unless VALID_METRIC_TYPES.include?(options[:metric_type])
+      options_error = "Invalid metric type #{options[:metric_type]}." +
+      " Must be one of: #{VALID_METRIC_TYPES.join(', ')}."
+      raise options_error
+    end
+  end
+
+  polling_interval = options[:polling_interval] || POLLING_INTERVAL_DEFAULT
+  file_interval = options[:file_interval] || FILE_INTERVAL_DEFAULT
+  metric_type= options[:metric_type] || METRIC_TYPE_DEFAULT
+  metrics_dir = options[:metrics_dir] || METRICS_DIR_DEFAULT
+  verbose = options[:verbose] || false
+
+  if options[:polling_interval] || options[:file_interval]
+    options_error = "Polling interval must be less than file interval"
+    raise options_error unless polling_interval < file_interval
+  end
+
+  if verbose
+    OPTION_SETTINGS = <<-SETTINGS
+      The following are the resulting options settings:
+        * polling_interval: #{polling_interval}
+        * file_interval: #{file_interval}
+        * metric_type: #{metric_type}
+        * metrics_dir: #{metrics_dir}
+        * verbose: #{verbose}
+    SETTINGS
+    puts OPTION_SETTINGS
+  end
+
+  obj = SystemMetrics::GenerateSystemMetrics.new(polling_interval, file_interval, metric_type,
+                                                 metrics_dir, verbose)
+  obj.go
+end