Skip to content
This repository has been archived by the owner on Jan 9, 2023. It is now read-only.

Commit

Permalink
Spin up actual elasticserach on the VMs
Browse files Browse the repository at this point in the history
  • Loading branch information
@simonswine
simonswine committed Feb 21, 2019
1 parent afac97e commit 90f25de
Showing 1 changed file with 94 additions and 159 deletions.
253 changes: 94 additions & 159 deletions puppet/modules/fluent_bit/spec/acceptance/fluentbit_spec.rb
View file
Original file line number Diff line number Diff line change
@@ -1,98 +1,51 @@
require 'spec_helper_acceptance'
require 'webrick'

class ESMock
attr_reader :port, :magic_logline, :magic_prefix, :semaphore
attr_accessor :magic_logline_appeared, :magic_prefix_appeared

def initialize(hosts)
reset
@semaphore = Mutex.new
@port = 9200
@http_server = WEBrick::HTTPServer.new(Port: @port)
@http_server.mount '/', ESMockHandle, self
@hosts = hosts
end

def start
# listen to local ES port
trap('INT') { @http_server.shutdown }
@http_server_thread = Thread.start { @http_server.start }
@ssh_connetions = []
module Helpers
def log_magic_line
hosts.each do |host|
# setup a remote tunnel to forward local es mock
conn = host.connection.connect
conn.forward.remote(@port, "127.0.0.1", @port) # port on host, host on host, remote port
@ssh_connections << conn
on host, "logger \"#{@magic_logline}\""
end
end

def stop
@ssh_connections.each do |conn|
conn.forward.cancel_remote(@port)
def find_magic_line
hosts.each do |host|
result = JSON.parse(host.execute("curl --retry 5 --fail -s -X GET 'http://localhost:9200/_all/_search?q=MESSAGE:\"#{@magic_logline}\"'"))
return nil unless result.key? 'hits'
return nil unless result['hits'].key? 'total'
total = result['hits']['total']
fail 'more than a single result found' if total > 1
return nil if total < 1
return result['hits']['hits'][0]
end
@http_server.shutdown
@http_server_thread.join
end

def reset
@magic_logline = "magic-logline-#{SecureRandom.uuid}"
@magic_prefix = "magic-prefix-#{SecureRandom.uuid}"
@magic_logline_appeared = nil
@magic_prefix_appeared = nil
end
end

class ESMockHandle < WEBrick::HTTPServlet::AbstractServlet
@server = nil

def initialize(i, server)
super i
@server = server
end

def do_POST(request, response)
key_message = 'MESSAGE'
key_index = 'index'
key__index = '_index'

index = nil
def retry_find_magic_line(tries=100)
count = 0
request.body.each_line do |line|
log_line = JSON.parse(line)

if log_line.key?(key_index) and log_line[key_index].key?(key__index)
index = log_line[key_index][key__index]
if index.include? @server.magic_prefix
$stderr.puts "magic prefix appeared: '#{log_line}'"
@server.semaphore.synchronize do
@server.magic_prefix_appeared = true
end
end
next
end
while true do
log_line = find_magic_line
return log_line unless log_line.nil?

count += 1

if log_line.key?(key_message) and log_line[key_message].include?(@server.magic_logline)
$stderr.puts "magic logline appeared: '#{log_line}'"
@server.semaphore.synchronize do
@server.magic_logline_appeared = true
end
end
fail "could not find magic logline after #{count} tries" if count > tries
sleep 1
end
return nil
end

$stderr.puts "received #{count} log lines for index '#{index}'"

response.status = 200
response.body = '{"took":30,"errors":false,"items":[]}'
def document_count
count = 0
result = JSON.parse(host.execute("curl --retry 5 --fail -s -X GET 'http://localhost:9200/_cat/indices?format=json"))
$stderr.puts "indices = #{result}"
# TODO: Count here
count
end
end

describe '::fluent_bit' do


RSpec.configure do |c|
c.include Helpers
end

describe '::fluent_bit' do
before(:all) do
hosts.each do |host|
# clear firewall
Expand All @@ -103,29 +56,37 @@ def do_POST(request, response)

# create some fake aws creds
on host, 'echo -e "AWS_ACCESS_KEY_ID=AKID1234567890\nAWS_SECRET_ACCESS_KEY=MY-SECRET-KEY" > /etc/sysconfig/aws-es-proxy-test'
end

@mock_es = ESMock.new hosts
@mock_es.start
# setup a local elasticsearch instance
on host, "cp #{$module_path}fluent_bit/spec/fixtures/elasticsearch.repo /etc/yum.repos.d"
on host, 'rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch'
on host, 'yum -y install java-1.8.0-openjdk elasticsearch wget'
on host, 'systemctl start elasticsearch.service'
on host, 'wget --retry-connrefused -T 60 http://localhost:9200/_cluster/health -O /dev/null 2> /dev/null'
end
end


after(:all) do
@mock_es.stop
let(:port) do
9200
end

context 'fluentbit with aws-es proxy' do
before(:all) do
@magic_logline = "magic-logline-#{SecureRandom.uuid}"
@magic_prefix = "magic-prefix-#{SecureRandom.uuid}"
end

let(:pp) do
pp = <<-EOS
host { 'fake.eu-west-1.es.amazonaws.com':
ensure => present,
ip => '127.0.0.1',
ensure => present,
ip => '127.0.0.1',
}
-> fluent_bit::output{"test":
config => {
"elasticsearch" => {
"host" => "fake.eu-west-1.es.amazonaws.com",
"port" => #{@mock_es.port},
"port" => #{port},
"tls" => false,
"amazonESProxy" => {
"port" => 9201
Expand All @@ -138,9 +99,6 @@ def do_POST(request, response)
pp
end

before(:all) do
@mock_es.reset
end

it 'should setup fluent bit without errors in first apply' do
hosts.each do |host|
Expand All @@ -152,52 +110,43 @@ def do_POST(request, response)
end

it 'should receive magic log line' do
hosts.each do |host|
on host, "logger \"#{@mock_es.magic_logline}\""
end

count = 0
while true
count += 1
fail "no log line received after 1000 tries" if count > 1000
break unless @mock_es.magic_logline_appeared.nil?
sleep(1)
end

expect(@mock_es.magic_logline_appeared).to equal(true)
expect(@mock_es.magic_prefix_appeared).to equal(nil)
log_magic_line
log_line = retry_find_magic_line
expect(log_line).not_to be_nil
expect(log_line['_index']).to start_with('logstash-')
end
end

context 'fluentbit with aws-es proxy using custom index' do
before(:all) do
@magic_logline = "magic-logline-#{SecureRandom.uuid}"
@magic_prefix = "magic-prefix-#{SecureRandom.uuid}"
end

let(:pp) do
pp = <<-EOS
host { 'fake.eu-west-1.es.amazonaws.com':
ensure => present,
ip => '127.0.0.1',
}
-> fluent_bit::output{"test":
config => {
"elasticsearch" => {
"host" => "fake.eu-west-1.es.amazonaws.com",
"port" => #{@mock_es.port},
"tls" => false,
"logstashPrefix" => "#{@mock_es.magic_prefix}",
"amazonESProxy" => {
"port" => 9201
},
},
"types" => ["all"],
},
}
host { 'fake.eu-west-1.es.amazonaws.com':
ensure => present,
ip => '127.0.0.1',
}
-> fluent_bit::output{"test":
config => {
"elasticsearch" => {
"host" => "fake.eu-west-1.es.amazonaws.com",
"port" => #{port},
"tls" => false,
"logstashPrefix" => "#{@magic_prefix}",
"amazonESProxy" => {
"port" => 9201
},
},
"types" => ["all"],
},
}
EOS
pp
end

before(:all) do
@mock_es.reset
end

it 'should setup fluent bit without errors in first apply' do
hosts.each do |host|
apply_manifest_on(host, pp, catch_failures: true)
Expand All @@ -208,43 +157,34 @@ def do_POST(request, response)
end

it 'should receive magic log line and prefix' do
hosts.each do |host|
on host, "logger \"#{@mock_es.magic_logline}\""
end

count = 0
while true
count += 1
fail "no log line received after 1000 tries" if count > 1000
break unless @mock_es.magic_logline_appeared.nil? or @mock_es.magic_logline_appeared.nil?
sleep(1)
end

expect(@mock_es.magic_logline_appeared).to equal(true)
expect(@mock_es.magic_prefix_appeared).to equal(true)
log_magic_line
log_line = retry_find_magic_line
expect(log_line).not_to be_nil
expect(log_line['_index']).to start_with(@magic_prefix)
end
end

context 'fluentbit without any output' do
before(:all) do
@magic_logline = "magic-logline-#{SecureRandom.uuid}"
@magic_prefix = "magic-prefix-#{SecureRandom.uuid}"
end

let(:pp) do
pp = <<-EOS
host { 'fake.eu-west-1.es.amazonaws.com':
ensure => present,
ip => '127.0.0.1',
}
-> fluent_bit::output{"test":
config => {
"types" => ["all"],
},
}
host { 'fake.eu-west-1.es.amazonaws.com':
ensure => present,
ip => '127.0.0.1',
}
-> fluent_bit::output{"test":
config => {
"types" => ["all"],
},
}
EOS
pp
end

before(:all) do
@mock_es.reset
end

it 'should setup fluent bit without errors in first apply' do
hosts.each do |host|
apply_manifest_on(host, pp, catch_failures: true)
Expand All @@ -255,14 +195,9 @@ def do_POST(request, response)
end

it 'should not receive magic log line and prefix' do
hosts.each do |host|
on host, "logger \"#{@mock_es.magic_logline}\""
end

sleep(10)
log_magic_line

expect(@mock_es.magic_logline_appeared).to equal(nil)
expect(@mock_es.magic_prefix_appeared).to equal(nil)
expect{retry_find_magic_line(tries=10)}.to raise_error(/could not find magic logline/)
end
end
end

0 comments on commit 90f25de

Please sign in to comment.