Docker integration tests stability improvements (elastic#13014)

* Docker integration tests stability improvements

This commit contains numerous fixes to improve the stability of the docker integration tests

* Patch Excon::UnixSocket

Socket.new running on arm64 on Ubuntu 18.04, causes an immediate SIGSEGV error and crash on
that OS, and, as far as I can tell, only that OS. `TCPSocket.new`,`UDPSocket.new` and
`UNIXSocket.new` do not. This commit patches the UnixSocket of the Excon library to
do the absolute simplest thing possible to avoid this error.

* Ensure that container is deleted even if #kill fails

* Add extra waits to handle the incremental way the payload returned by the monitoring
API increases as logstash starts up and pipelines load.

* Use pyenv to ensure the same version of python is used across different jenkins workers

* Add container logs to help diagnose failed test.

* Update the pipeline definition on multi-pipeline integration test

This was causing a pipeline to halt after startup causing intermittent test failures.

* Remove `;` to ensure failures are propagated appropriately

Co-authored-by: João Duarte <jsvd@users.noreply.github.com>

docker/Makefile

@@ -1,7 +1,7 @@
 SHELL=/bin/bash
 ELASTIC_REGISTRY ?= docker.elastic.co
-
-export PATH := ./bin:./venv/bin:$(PATH)
+PY_VERSION ?= 3.6.13
+export PATH := ./bin:$(HOME)/.pyenv/bin:$(HOME)/.pyenv/shims:./venv/bin:$(PATH)
 
 # Determine the version to build.
 ELASTIC_VERSION := $(shell ../vendor/jruby/bin/jruby bin/elastic-version)
@@ -142,20 +142,14 @@ push:
 
 # The tests are written in Python. Make a virtualenv to handle the dependencies.
 venv: requirements.txt
-	@if [ -z $$PYTHON3 ]; then\
-	    PY3_MINOR_VER=`python3 --version 2>&1 | cut -d " " -f 2 | cut -d "." -f 2`;\
-	    if (( $$PY3_MINOR_VER < 5 )); then\
-		echo "Couldn't find python3 in \$PATH that is >=3.5";\
-		echo "Please install python3.5 or later or explicity define the python3 executable name with \$PYTHON3";\
-		echo "Exiting here";\
-		exit 1;\
-	    else\
-		export PYTHON3="python3.$$PY3_MINOR_VER";\
-	   fi;\
-	fi;\
-	test -d venv || virtualenv --python=$$PYTHON3 venv;\
-	pip install -r requirements.txt;\
-	touch venv;\
+	LOCAL_PY_VER=`python3 --version 2>&1`&&\
+	echo "Was using $$LOCAL_PY_VER" &&\
+	eval "$$(pyenv init -)" && eval "$$(pyenv init --path)" &&\
+	pyenv install --skip-existing $(PY_VERSION) &&\
+	pyenv local $(PY_VERSION) &&\
+	python3 -mvenv venv && \
+	pip install -r requirements.txt &&\
+	touch venv
 
 # Make a Golang container that can compile our env2yaml tool.
 golang:

qa/docker/fixtures/multiple_pipelines/pipelines/basic2.cfg

@@ -1,7 +1,6 @@
 input {
-beats {
-id => 'multi_pipeline2'
-port => 5044
-}
+  stdin {
+    id => 'multi_pipeline2'
+  }
 }
 output { stdout {} }

qa/docker/patches/excon/unix_socket.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+module Excon
+  class UnixSocket < Excon::Socket
+    private
+    def connect
+      @socket = ::UNIXSocket.new(@data[:socket])
+    end
+  end
+end

qa/docker/shared_examples/container.rb

@@ -40,15 +40,15 @@
     end
 
     it 'should have the correct user' do
-      expect(exec_in_container(@container, 'whoami').chomp).to eql 'logstash'
+      expect(exec_in_container(@container, 'whoami')).to eql 'logstash'
     end
 
     it 'should have the correct home directory' do
-      expect(exec_in_container(@container, 'printenv HOME').chomp).to eql '/usr/share/logstash'
+      expect(exec_in_container(@container, 'printenv HOME')).to eql '/usr/share/logstash'
     end
 
     it 'should link /opt/logstash to /usr/share/logstash' do
-      expect(exec_in_container(@container, 'readlink /opt/logstash').chomp).to eql '/usr/share/logstash'
+      expect(exec_in_container(@container, 'readlink /opt/logstash')).to eql '/usr/share/logstash'
     end
 
     it 'should have all files owned by the logstash user' do
@@ -57,11 +57,11 @@
     end
 
     it 'should have a logstash user with uid 1000' do
-      expect(exec_in_container(@container, 'id -u logstash').chomp).to eql '1000'
+      expect(exec_in_container(@container, 'id -u logstash')).to eql '1000'
     end
 
     it 'should have a logstash user with gid 1000' do
-      expect(exec_in_container(@container, 'id -g logstash').chomp).to eql '1000'
+      expect(exec_in_container(@container, 'id -g logstash')).to eql '1000'
     end
 
     it 'should not have a RollingFile appender' do

qa/docker/shared_examples/container_config.rb

@@ -13,7 +13,8 @@
     let(:options) { {"HostConfig" => { "Binds" => ["#{FIXTURES_DIR}/simple_pipeline/:/usr/share/logstash/pipeline/"] } } }
 
     it 'should show the stats for that pipeline' do
-      expect(get_node_stats(@container)['pipelines']['main']['plugins']['inputs'][0]['id']).to eq 'simple_pipeline'
+      wait_for_pipeline(@container)
+      expect(get_plugin_info(@container, 'inputs', 'simple_pipeline')).not_to be nil
     end
   end
 
@@ -22,16 +23,19 @@
                                                    "#{FIXTURES_DIR}/multiple_pipelines/config/pipelines.yml:/usr/share/logstash/config/pipelines.yml"] } } }
 
     it "should show stats for both pipelines" do
-      expect(get_node_stats(@container)['pipelines']['pipeline_one']['plugins']['inputs'][0]['id']).to eq 'multi_pipeline1'
-      expect(get_node_stats(@container)['pipelines']['pipeline_two']['plugins']['inputs'][0]['id']).to eq 'multi_pipeline2'
+      wait_for_pipeline(@container, 'pipeline_one')
+      wait_for_pipeline(@container, 'pipeline_two')
+      expect(get_plugin_info(@container, 'inputs', 'multi_pipeline1', 'pipeline_one')).not_to be nil
+      expect(get_plugin_info(@container, 'inputs', 'multi_pipeline2', 'pipeline_two')).not_to be nil
     end
   end
 
   context 'when a custom `logstash.yml` is configured via volume bind' do
     let(:options) { {"HostConfig" => { "Binds" => ["#{FIXTURES_DIR}/custom_logstash_yml/logstash.yml:/usr/share/logstash/config/logstash.yml"] } } }
 
     it 'should change the value of pipeline.batch.size' do
-      expect(get_node_info(@container)['pipelines']['main']['batch_size']).to eq 200
+      wait_for_pipeline(@container)
+      expect(get_pipeline_setting(@container, 'batch_size')).to eq 200
     end
   end
 end

qa/docker/shared_examples/container_options.rb

@@ -3,6 +3,7 @@
   before do
     @image = find_image(flavor)
     @container = start_container(@image, options)
+    wait_for_pipeline(@container)
   end
 
   after do
@@ -13,31 +14,31 @@
     let(:options) { { 'ENV' => ['PIPELINE_WORKERS=32'] } }
 
     it "should correctly set the number of pipeline workers" do
-      expect(get_node_info(@container)['pipelines']['main']['workers']).to eql 32
+      expect(get_pipeline_setting(@container, 'workers')).to eql 32
     end
   end
 
   context 'when setting pipeline workers dot style' do
     let(:options) { { 'ENV' => ['pipeline.workers=64'] } }
 
     it "should correctly set the number of pipeline workers" do
-      expect(get_node_info(@container)['pipelines']['main']['workers']).to eql 64
+      expect(get_pipeline_setting(@container, 'workers')).to eql 64
     end
   end
 
   context 'when setting pipeline batch size' do
     let(:options) { { 'ENV' => ['pipeline.batch.size=123'] } }
 
     it "should correctly set the batch size" do
-      expect(get_node_info(@container)['pipelines']['main']['batch_size']).to eql 123
+      expect(get_pipeline_setting(@container, 'batch_size')).to eql 123
     end
   end
 
   context 'when setting pipeline batch delay' do
     let(:options) { { 'ENV' => ['pipeline.batch.delay=36'] } }
 
     it 'should correctly set batch delay' do
-      expect(get_node_info(@container)['pipelines']['main']['batch_delay']).to eql 36
+      expect(get_pipeline_setting(@container, 'batch_delay')).to eql 36
     end
   end
 

qa/docker/spec/spec_helper.rb

@@ -6,6 +6,7 @@
 require 'json'
 require 'stud/try'
 require 'docker-api'
+require_relative '../patches/excon/unix_socket'
 
 def version
   @version ||= LOGSTASH_VERSION
@@ -35,15 +36,25 @@ def start_container(image, options={})
 end
 
 def wait_for_logstash(container)
-  Stud.try(40.times, RSpec::Expectations::ExpectationNotMetError) do
-    expect(container.exec(['curl', '-s', 'http://localhost:9600/_node'])[0][0]).not_to be_empty
+  Stud.try(40.times, [NoMethodError, Docker::Error::ConflictError, RSpec::Expectations::ExpectationNotMetError, TypeError]) do
+    expect(logstash_available?(container)).to be true
+    expect(get_logstash_status(container)).to eql 'green'
+  end
+end
+
+def wait_for_pipeline(container, pipeline='main')
+  Stud.try(40.times, [NoMethodError, Docker::Error::ConflictError, RSpec::Expectations::ExpectationNotMetError, TypeError]) do
+    expect(pipeline_stats_available?(container, pipeline)).to be true
   end
 end
 
 def cleanup_container(container)
   unless container.nil?
-    container.kill
-    container.delete(:force=>true)
+    begin
+      container.stop
+    ensure
+      container.delete(:force=>true)
+    end
   end
 end
 
@@ -56,28 +67,77 @@ def license_agreement_for_flavor(flavor)
 end
 
 def get_logstash_status(container)
-  JSON.parse(container.exec(['curl', '-s', 'http://localhost:9600'])[0][0])['status']
+  make_request(container,'curl -s http://localhost:9600/')['status']
 end
 
-
 def get_node_info(container)
-  JSON.parse(container.exec(['curl', '-s', 'http://localhost:9600/_node'])[0][0])
+  make_request(container,'curl -s http://localhost:9600/_node/')
 end
 
 def get_node_stats(container)
-  JSON.parse(container.exec(['curl', '-s', 'http://localhost:9600/_node/stats'])[0][0])
+  make_request(container,'curl -s http://localhost:9600/_node/stats')
+end
+
+def get_pipeline_setting(container, property, pipeline='main')
+  make_request(container, "curl -s http://localhost:9600/_node/pipelines/#{pipeline}")
+          .dig('pipelines', pipeline, property)
+end
+
+def get_pipeline_stats(container, pipeline='main')
+  make_request(container, "curl -s http://localhost:9600/_node/stats/pipelines").dig('pipelines', pipeline)
+end
+
+def get_plugin_info(container, type, id, pipeline='main')
+  pipeline_info = make_request(container, "curl -s http://localhost:9600/_node/stats/pipelines")
+  all_plugins = pipeline_info.dig('pipelines', pipeline, 'plugins', type)
+  if all_plugins.nil?
+    # This shouldn't happen, so if it does, let's figure out why
+    puts container.logs(stdout: true)
+    puts "Unable to find plugins from #{pipeline_info}, when looking for #{type} plugins in #{pipeline}"
+    return nil
+  end
+  all_plugins.find{|plugin|plugin['id'] == id}
+end
+
+def logstash_available?(container)
+  response = exec_in_container_full(container, 'curl -s http://localhost:9600')
+  return false if response[:exitcode] != 0
+  !(response[:stdout].nil? || response[:stdout].empty?)
+end
+
+def pipeline_stats_available?(container, pipeline)
+  response = make_request(container, "curl -s http://localhost:9600/_node/stats/pipelines")
+  plugins = response.dig('pipelines', pipeline, 'plugins')
+  !(plugins.nil? || plugins.empty?)
+end
+
+def make_request(container, url)
+  JSON.parse(exec_in_container(container, url))
 end
 
 def get_settings(container)
   YAML.load(container.read_file('/usr/share/logstash/config/logstash.yml'))
 end
 
 def java_process(container, column)
-  exec_in_container(container, "ps -C java -o #{column}=").strip
+  exec_in_container(container, "ps -C java -o #{column}=")
+end
+
+# Runs the given command in the given container. This method returns
+# a hash including the `stdout` and `stderr` outputs and the exit code
+def exec_in_container_full(container, command)
+  response = container.exec(command.split)
+  {
+      :stdout => response[0],
+      :stderr => response[1],
+      :exitcode => response[2]
+  }
 end
 
+# Runs the given command in the given container. This method returns
+# only the stripped/chomped `stdout` output.
 def exec_in_container(container, command)
-  container.exec(command.split)[0].join
+  exec_in_container_full(container, command)[:stdout].join.chomp.strip
 end
 
 def running_architecture

qa/docker/spec/ubi8/container_spec.rb

@@ -21,7 +21,7 @@
     end
 
     it 'should be based on Red Hat Enterprise Linux' do
-      expect(exec_in_container(@container, 'cat /etc/redhat-release').chomp).to match /Red Hat Enterprise Linux/
+      expect(exec_in_container(@container, 'cat /etc/redhat-release')).to match /Red Hat Enterprise Linux/
     end
   end
 end