From 78206287d74a0b06dc91ced9cb8aa21303a5d7b7 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 09:33:50 +0100 Subject: [PATCH 01/13] CI: drop customized Dockerfile --- .ci/Dockerfile | 23 ----------------------- .ci/run.sh | 7 ++++--- 2 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 .ci/Dockerfile diff --git a/.ci/Dockerfile b/.ci/Dockerfile deleted file mode 100644 index db5c59e8..00000000 --- a/.ci/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -ARG ELASTIC_STACK_VERSION -FROM docker.elastic.co/logstash/logstash:$ELASTIC_STACK_VERSION -USER root -RUN yum install -y openssl -USER logstash -COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile -COPY --chown=logstash:logstash *.gemspec VERSION* version* /usr/share/plugins/plugin/ -RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml -ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin:/usr/share/logstash/jdk/bin" -ENV LOGSTASH_SOURCE="1" -ENV ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION -# DISTRIBUTION="default" (by default) or "oss" -ARG DISTRIBUTION -ENV DISTRIBUTION=$DISTRIBUTION -# INTEGRATION="true" while integration testing (false-y by default) -ARG INTEGRATION -ENV INTEGRATION=$INTEGRATION -RUN gem install bundler -v '< 2' -WORKDIR /usr/share/plugins/plugin -RUN bundle install --with test ci -COPY --chown=logstash:logstash . /usr/share/plugins/plugin -RUN bundle exec rake vendor -RUN .ci/setup.sh diff --git a/.ci/run.sh b/.ci/run.sh index 2d9dc683..ed8bd466 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -5,6 +5,7 @@ env set -ex -bundle exec rspec spec -bundle exec rake test:integration:setup -bundle exec rspec spec --tag integration -fd +jruby -rbundler/setup -S rspec -fd + +jruby -rbundler/setup -S rake test:integration:setup +jruby -rbundler/setup -S rspec spec --tag integration -fd From b9e38b448a3243214fbca05046a24101a185e821 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 10:23:57 +0100 Subject: [PATCH 02/13] bring back apt install openssl --- .ci/setup.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 .ci/setup.sh diff --git a/.ci/setup.sh b/.ci/setup.sh new file mode 100755 index 00000000..29b865c8 --- /dev/null +++ b/.ci/setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [ $(command -v apt) ]; then + sudo apt install -y openssl +else + sudo yum install -y openssl +fi \ No newline at end of file From 70de9ea51b738012d0b8342173d51ceded47218b Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 10:50:08 +0100 Subject: [PATCH 03/13] Test: fix failure when ECS mode is on (8.x) --- spec/inputs/beats_spec.rb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/spec/inputs/beats_spec.rb b/spec/inputs/beats_spec.rb index 301571ef..c3cb86aa 100644 --- a/spec/inputs/beats_spec.rb +++ b/spec/inputs/beats_spec.rb @@ -166,7 +166,14 @@ end context "tls meta-data" do - let(:config) { super().merge("host" => host, "ssl_peer_metadata" => true, "ssl_certificate_authorities" => [ certificate.ssl_cert ]) } + let(:config) do + super().merge( + "host" => host, + "ssl_peer_metadata" => true, + "ssl_certificate_authorities" => [ certificate.ssl_cert ], + "ecs_compatibility" => 'disabled' + ) + end let(:host) { "192.168.1.20" } let(:port) { 9002 } From 138400ac5174c063afd4fb67126117c835a74b16 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 11:47:09 +0100 Subject: [PATCH 04/13] Test: refactor for easier debugging --- spec/integration/filebeat_spec.rb | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/spec/integration/filebeat_spec.rb b/spec/integration/filebeat_spec.rb index 885ce45c..26572be4 100644 --- a/spec/integration/filebeat_spec.rb +++ b/spec/integration/filebeat_spec.rb @@ -182,22 +182,23 @@ let(:certificate_key_file_pkcs8) { ::File.join(temporary_directory, "certificate.pkcs8.key") } let(:certificate_file) { ::File.join(temporary_directory, "certificate.crt") } let(:passphrase) { "foobar" } - let(:beats) { - # Since we are using a shared context, this not obvious to make sure the openssl command - # is run before starting beats so we do it just before initializing it. + let(:input_config) { super().merge("ssl_key_passphrase" => passphrase, "ssl_key" => certificate_key_file_pkcs8) } + + let(:beats) { LogStash::Inputs::Beats.new(input_config) } + + before do FileUtils.mkdir_p(temporary_directory) - openssl_cmd = "openssl req -x509 -batch -newkey rsa:2048 -keyout #{temporary_directory}/certificate.key -out #{temporary_directory}/certificate.crt -subj /CN=localhost -passout pass:#{passphrase}" - system(openssl_cmd) - convert_key_cmd = "openssl pkcs8 -topk8 -in #{temporary_directory}/certificate.key -out #{certificate_key_file_pkcs8} -passin pass:#{passphrase} -passout pass:#{passphrase}" - system(convert_key_cmd) - LogStash::Inputs::Beats.new(input_config) - } - let(:input_config) { - super().merge({ - "ssl_key_passphrase" => passphrase, - "ssl_key" => certificate_key_file_pkcs8 - })} + cmd = "openssl req -x509 -batch -newkey rsa:2048 -keyout #{temporary_directory}/certificate.key -out #{temporary_directory}/certificate.crt -subj /CN=localhost -passout pass:#{passphrase}" + unless system(cmd) + fail "failed to run openssl command: #{$?} \n#{cmd}" + end + + cmd = "openssl pkcs8 -topk8 -in #{temporary_directory}/certificate.key -out #{certificate_key_file_pkcs8} -passin pass:#{passphrase} -passout pass:#{passphrase}" + unless system(cmd) + fail "failed to run openssl command: #{$?} \n#{cmd}" + end + end include_examples "send events" end From 77c6d464fff1ec12f3e64602c8ab13091780f42b Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 11:47:32 +0100 Subject: [PATCH 05/13] Temp: only run it spec - debug Ubuntu 8.x fail --- .ci/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/run.sh b/.ci/run.sh index ed8bd466..3cb92bb2 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -5,7 +5,7 @@ env set -ex -jruby -rbundler/setup -S rspec -fd +#jruby -rbundler/setup -S rspec -fd jruby -rbundler/setup -S rake test:integration:setup -jruby -rbundler/setup -S rspec spec --tag integration -fd +jruby -rbundler/setup -S rspec spec --tag integration -fd -e passphrase From 6d57bf9ab266ff29b734bfc9ce60c516be9d04c8 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 12:06:51 +0100 Subject: [PATCH 06/13] Refactor to using a before(:all) hook --- spec/integration/filebeat_spec.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/spec/integration/filebeat_spec.rb b/spec/integration/filebeat_spec.rb index 26572be4..5813cda0 100644 --- a/spec/integration/filebeat_spec.rb +++ b/spec/integration/filebeat_spec.rb @@ -177,29 +177,29 @@ # Refactor this to use Flores's PKI instead of openssl command line # see: https://github.com/jordansissel/ruby-flores/issues/7 context "with a passphrase" do - let!(:temporary_directory) { Stud::Temporary.pathname } - let(:certificate_key_file) { ::File.join(temporary_directory, "certificate.key") } - let(:certificate_key_file_pkcs8) { ::File.join(temporary_directory, "certificate.pkcs8.key") } - let(:certificate_file) { ::File.join(temporary_directory, "certificate.crt") } - let(:passphrase) { "foobar" } - let(:input_config) { super().merge("ssl_key_passphrase" => passphrase, "ssl_key" => certificate_key_file_pkcs8) } - let(:beats) { LogStash::Inputs::Beats.new(input_config) } + before(:all) do + @passphrase = "foobar".freeze - before do - FileUtils.mkdir_p(temporary_directory) + FileUtils.mkdir_p temporary_directory = Stud::Temporary.pathname - cmd = "openssl req -x509 -batch -newkey rsa:2048 -keyout #{temporary_directory}/certificate.key -out #{temporary_directory}/certificate.crt -subj /CN=localhost -passout pass:#{passphrase}" + cert_key = ::File.join(temporary_directory, "certificate.key") + cert_pub = ::File.join(temporary_directory, "certificate.crt") + @cert_key_pkcs8 = ::File.join(temporary_directory, "certificate.key.pkcs8") + + cmd = "openssl req -x509 -batch -newkey rsa:2048 -keyout #{cert_key} -out #{cert_pub} -passout pass:#{@passphrase} -subj /CN=localhost/" unless system(cmd) fail "failed to run openssl command: #{$?} \n#{cmd}" end - cmd = "openssl pkcs8 -topk8 -in #{temporary_directory}/certificate.key -out #{certificate_key_file_pkcs8} -passin pass:#{passphrase} -passout pass:#{passphrase}" + cmd = "openssl pkcs8 -topk8 -in #{cert_key} -out #{@cert_key_pkcs8} -passin pass:#{@passphrase} -passout pass:#{@passphrase}" unless system(cmd) fail "failed to run openssl command: #{$?} \n#{cmd}" end end + let(:input_config) { super().merge("ssl_key_passphrase" => @passphrase, "ssl_key" => @cert_key_pkcs8) } + include_examples "send events" end end From 46eebb1dab4ee266f834453647c207acd996bbf5 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 12:07:05 +0100 Subject: [PATCH 07/13] Test: print failures when retry-ing --- spec/support/integration_shared_context.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/support/integration_shared_context.rb b/spec/support/integration_shared_context.rb index acb27443..9bdb70d6 100644 --- a/spec/support/integration_shared_context.rb +++ b/spec/support/integration_shared_context.rb @@ -50,6 +50,7 @@ begin beats.run(queue) rescue => e + warn e.inspect if $VERBOSE retry unless beats.stop? end end From 66eb0eaa39a72699f609f5e4c5ebc1aec6813935 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 12:18:53 +0100 Subject: [PATCH 08/13] proper -subj (second try) --- spec/integration/filebeat_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/integration/filebeat_spec.rb b/spec/integration/filebeat_spec.rb index 5813cda0..5045556d 100644 --- a/spec/integration/filebeat_spec.rb +++ b/spec/integration/filebeat_spec.rb @@ -187,7 +187,7 @@ cert_pub = ::File.join(temporary_directory, "certificate.crt") @cert_key_pkcs8 = ::File.join(temporary_directory, "certificate.key.pkcs8") - cmd = "openssl req -x509 -batch -newkey rsa:2048 -keyout #{cert_key} -out #{cert_pub} -passout pass:#{@passphrase} -subj /CN=localhost/" + cmd = "openssl req -x509 -batch -newkey rsa:2048 -keyout #{cert_key} -out #{cert_pub} -passout pass:#{@passphrase} -subj \"/C=EU/O=Logstash/CN=localhost\"" unless system(cmd) fail "failed to run openssl command: #{$?} \n#{cmd}" end From 9bf7fea8910c4798aa022006824c53d9f6c121a6 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 12:42:36 +0100 Subject: [PATCH 09/13] Temp: debug plugin --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index a50fc739..cff604a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,2 +1,4 @@ +env: +- LOG_LEVEL=debug import: - logstash-plugins/.ci:travis/travis.yml@1.x \ No newline at end of file From 3a5e35115012f1183096a4a90473b374f3592d71 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 13:27:33 +0100 Subject: [PATCH 10/13] restore the it spec working again (on 7.x) --- spec/integration/filebeat_spec.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/spec/integration/filebeat_spec.rb b/spec/integration/filebeat_spec.rb index 5045556d..0d5a2572 100644 --- a/spec/integration/filebeat_spec.rb +++ b/spec/integration/filebeat_spec.rb @@ -184,10 +184,10 @@ FileUtils.mkdir_p temporary_directory = Stud::Temporary.pathname cert_key = ::File.join(temporary_directory, "certificate.key") - cert_pub = ::File.join(temporary_directory, "certificate.crt") + @cert_pub = ::File.join(temporary_directory, "certificate.crt") @cert_key_pkcs8 = ::File.join(temporary_directory, "certificate.key.pkcs8") - cmd = "openssl req -x509 -batch -newkey rsa:2048 -keyout #{cert_key} -out #{cert_pub} -passout pass:#{@passphrase} -subj \"/C=EU/O=Logstash/CN=localhost\"" + cmd = "openssl req -x509 -batch -newkey rsa:2048 -keyout #{cert_key} -out #{@cert_pub} -passout pass:#{@passphrase} -subj \"/C=EU/O=Logstash/CN=localhost\"" unless system(cmd) fail "failed to run openssl command: #{$?} \n#{cmd}" end @@ -198,7 +198,11 @@ end end - let(:input_config) { super().merge("ssl_key_passphrase" => @passphrase, "ssl_key" => @cert_key_pkcs8) } + let(:certificate_authorities) { [ @cert_pub ] } + + let(:input_config) do + super().merge("ssl_key_passphrase" => @passphrase, "ssl_key" => @cert_key_pkcs8, "ssl_certificate" => @cert_pub) + end include_examples "send events" end From 35f5c8374b2ab9a1a48dd22d29225447c5a7aded Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 13:27:56 +0100 Subject: [PATCH 11/13] revert temp LOG_LEVEL=debug --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cff604a8..a50fc739 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,2 @@ -env: -- LOG_LEVEL=debug import: - logstash-plugins/.ci:travis/travis.yml@1.x \ No newline at end of file From 8adfc3071360e032554bf2f64c8d0db0b3712a22 Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 14:55:23 +0100 Subject: [PATCH 12/13] will need to keep using -v1 algorithm --- spec/integration/filebeat_spec.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/integration/filebeat_spec.rb b/spec/integration/filebeat_spec.rb index 0d5a2572..e08a8a63 100644 --- a/spec/integration/filebeat_spec.rb +++ b/spec/integration/filebeat_spec.rb @@ -192,7 +192,9 @@ fail "failed to run openssl command: #{$?} \n#{cmd}" end - cmd = "openssl pkcs8 -topk8 -in #{cert_key} -out #{@cert_key_pkcs8} -passin pass:#{@passphrase} -passout pass:#{@passphrase}" + # NOTE: CentOS 7 base image (LS < 7.17) uses OpenSSL 1.0 while later is using Ubuntu 20.04 with OpenSSL 1.1.1 + # the default algorithm for `openssl pkcs8 -topk8` changed to -v2 which Java does not support (see GH-443) + cmd = "openssl pkcs8 -topk8 -in #{cert_key} -out #{@cert_key_pkcs8} -v1 PBE-SHA1-RC2-128 -passin pass:#{@passphrase} -passout pass:#{@passphrase}" unless system(cmd) fail "failed to run openssl command: #{$?} \n#{cmd}" end From a73f353dc18c4cb4f32b01107133f87cd0e4e14f Mon Sep 17 00:00:00 2001 From: kares Date: Thu, 30 Dec 2021 15:10:15 +0100 Subject: [PATCH 13/13] revert back to running all specs --- .ci/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/run.sh b/.ci/run.sh index 3cb92bb2..ed8bd466 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -5,7 +5,7 @@ env set -ex -#jruby -rbundler/setup -S rspec -fd +jruby -rbundler/setup -S rspec -fd jruby -rbundler/setup -S rake test:integration:setup -jruby -rbundler/setup -S rspec spec --tag integration -fd -e passphrase +jruby -rbundler/setup -S rspec spec --tag integration -fd