timeline.json.erb

<%
require 'json'
require 'ostruct'
require 'set'
services_blacklist_raw = (ENV['EXCLUDE_SERVICES'] || 'lbl7.*,netsvc-probe.*,consul-probed.*').split(',')
services_blacklist = services_blacklist_raw.map { |v| Regexp.new(v) }  # Compute the health of a Service
timeline_blacklist = ENV['CONSUL_TIMELINE_BLACKLIST'] || '(lbl7-.*|netsvc-.*)'
timeline_blacklist = Regexp.new(timeline_blacklist)

@current_time = Time.now.utc.iso8601
@last_service_info = {}
@newest_index = 0 unless @newest_index
cur_state = services.map do |service_name, _tags|
  next if services_blacklist.any? {|r| r.match(service_name)}
  next if timeline_blacklist.match(service_name)

  snodes = service(service_name)
  cur_stats = {
    'passing'  => 0,
    'warning'  => 0,
    'critical' => 0,
    'total'    => snodes.count
  }
  cur_index = snodes.endpoint.x_consul_index.to_i
  @newest_index = cur_index if @newest_index < cur_index
  @last_service_info[service_name] = { 'idx' => cur_index, 'stats' => cur_stats }
  snodes.each do |snode|
    case snode.status.downcase
    when 'passing'
      cur_stats['passing'] += 1
    when 'warning'
      cur_stats['warning'] += 1
    else
      cur_stats['critical'] += 1
    end
  end
  instances = snodes
              .sort { |a, b| a['Node']['Node'] <=> b['Node']['Node'] }
              .map do |instance|
    mod_index = instance['Service']['ModifyIndex'] || 0
    instance['Checks'].each do |chk|
      mindex = chk['ModifyIndex'] || 0
      mod_index = mindex if mod_index < mindex
    end
    node_name = instance['Node']['Node']
    ["#{node_name}:#{instance['Service']['ID']}",
     {
       'address' => instance.service_address,
       'node' => node_name,
       'fqdn' => instance.node_meta['fqdn'] || node_name,
       'port' => instance['Service']['Port'],
       'idx'  => mod_index || cur_index,
       'status' => instance.status,
       'stats' => cur_stats,
       'checks' => instance['Checks'].map { |check| [check['CheckID'], { 'name' => check['Name'], 'status' => check['Status'], 'output' => check['Output'] }] }.to_h
     }]
  end.to_h
  [service_name, instances]
end
                    .compact
                    .to_h

old_state = if @previous_state
              @previous_state
            else
              cur_state
            end

unless @events
  load File.expand_path(File.join(File.dirname(template_info['source']), 'ringbuffer.rb'))

  def diff(old, new_e)
    diff = OpenStruct.new
    diff.appeared = new_e - old
    diff.disappeared = old - new_e
    diff.stayed = new_e & old
    diff.all = new_e + old
    diff
  end

  @events = ::ConsulTimeline::SortedRingBuffer.new((ENV['CONSUL_TIMELINE_BUFFER'] || 10000).to_i, lambda {|a, b| a['idx'] <=> b['idx'] })
end
@new_events = []

def log_event(line)
  puts "#{Time.now.to_i} #{line}" if ENV['DEBUG_TIMELINE']
end

def store_event(service: nil, instance: nil, old_state: nil, new_state: nil, instance_info: nil, checks: [])
  @new_events << { 'service' => service, 'instance' => instance, 'old_state' => old_state, 'new_state' => new_state,
               'ts' => @current_time, 'instance_info' => instance_info }.tap do |ev|
    ev['checks'] = checks if checks
    ev['stats'] = instance_info['stats']
    ev['idx'] = instance_info['idx']
  end
end

def compute_checks(old_state, cur_state, service_name, instance_name)
  old_checks = old_state.dig(service_name, instance_name, 'checks') || {}
  new_checks = cur_state.dig(service_name, instance_name, 'checks') || {}
  old_index = old_state.dig(service_name, instance_name, 'idx')
  # In case of removal of a check (eg: maintenance), Idx might be decreased, ensure it is not
  if old_index
    new_index = cur_state.dig(service_name, instance_name, 'idx')
    cur_state[service_name][instance_name]['idx'] = @newest_index if new_index && new_index < old_index
  end
  all_checks = Set.new(old_checks.keys + new_checks.keys)
  checks = []
  all_checks.each do |check_id|
    old_status = old_state.dig(service_name, instance_name, 'checks', check_id, 'status')
    cur_status = cur_state.dig(service_name, instance_name, 'checks', check_id, 'status')

    next if old_status == cur_status
    check_name = cur_state.dig(service_name, instance_name, 'checks', check_id, 'name')
    check_name = old_state.dig(service_name, instance_name, 'checks', check_id, 'name') unless check_name
    check_name = check_id unless check_name
    checks << { 'id' => check_id,
                'old' => old_status,
                'new' => cur_status,
                'name' => check_name }.tap do |check|
                  check['output'] = (cur_state.dig(service_name, instance_name, 'checks', check_id, 'output') || '')[0..512]
                end
  end
  checks
end

service_diff = diff(old_state.keys, cur_state.keys)

service_diff.disappeared.each do |service_name|
  old_state[service_name].each do |instance_name, instance_info|
    checks = compute_checks(old_state, cur_state, service_name, instance_name)
    the_info = {}.merge(instance_info)
    the_info['idx'] = @newest_index
    the_info['stats'] = {
      'passing'  => 0,
      'warning'  => 0,
      'critical' => 0,
      'total'    => 0,
    }
    store_event(service: service_name,
                instance: instance_name,
                old_state: old_state[service_name][instance_name]['status'],
                new_state: nil,
                instance_info: the_info,
                checks: checks)
  end
end

def instances_are_equal(o_state, n_state)
  return true if o_state == n_state
  return false unless o_state.nil? || n_state.nil?
  %w[address node port status].each do |field|
    return false if o_state[field] != n_state[field]
  end
  true
end

(service_diff.stayed + service_diff.appeared).each do |service_name|
  instance_diff = diff((old_state[service_name] || {}).keys, cur_state[service_name].keys)

  instance_diff.disappeared.each do |instance_name|
    checks = compute_checks(old_state, cur_state, service_name, instance_name)
    the_info = {}.merge(old_state[service_name][instance_name])
    the_info['idx'] = @last_service_info[service_name]['idx']
    the_info['stats'] = @last_service_info[service_name]['stats']
    store_event(service: service_name,
                old_state: old_state[service_name][instance_name]['status'],
                new_state: nil,
                instance: instance_name,
                instance_info: the_info,
                checks: checks)
  end
  instance_diff.appeared.each do |instance_name|
    checks = compute_checks(old_state, cur_state, service_name, instance_name)
    store_event(service: service_name,
                old_state: nil,
                new_state: cur_state[service_name][instance_name]['status'],
                instance: instance_name,
                instance_info: cur_state[service_name][instance_name],
                checks: checks)
  end

  instance_diff.stayed.each do |instance_name, _instance_info|
    checks = compute_checks(old_state, cur_state, service_name, instance_name)
    o_state = old_state[service_name][instance_name]['status']
    n_state = cur_state[service_name][instance_name]['status']
    next if instances_are_equal(o_state, n_state) && checks.empty?
    store_event(service: service_name,
                old_state: o_state,
                new_state: n_state,
                instance: instance_name,
                instance_info: cur_state[service_name][instance_name],
                checks: checks)
  end
end
sorted_events = @new_events.sort do |a, b|
  res = 0
  %w[idx service instance].each do |f|
    res = a[f] <=> b[f]
    break if res != 0
  end
  res
end
sorted_events.each { |e| @events.push e }
@new_events.clear

# We save the previous state only when we have a complete state once
if template_info['was_rendered_once']
  warn "First full rendering completed at #{@current_time} !" unless @previous_state
  @previous_state = cur_state
end
%><%= JSON.generate(@events.to_a) %>