From 6186ad6fe2705a8f5510428c98dca322136cd968 Mon Sep 17 00:00:00 2001 From: Colin Surprenant Date: Thu, 4 Feb 2016 22:26:13 -0500 Subject: [PATCH 1/2] use Event#from_json when possible cosmetic reindent refactor for the array return value of from_json wrong paramter refactor to support testing from_json and legacy parser refactor to support testing from_json and legacy parser - take2 --- lib/logstash/codecs/json_lines.rb | 28 ++++++++++++------- spec/codecs/json_lines_spec.rb | 46 +++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/lib/logstash/codecs/json_lines.rb b/lib/logstash/codecs/json_lines.rb index 7fb865c..c917b93 100644 --- a/lib/logstash/codecs/json_lines.rb +++ b/lib/logstash/codecs/json_lines.rb @@ -36,26 +36,34 @@ def register @converter.logger = @logger end - def decode(data) + def decode(data, &block) @buffer.extract(data).each do |line| - yield guard(@converter.convert(line)) + parse(@converter.convert(line), &block) end - end # def decode + end def encode(event) # Tack on a @delimiter for now because previously most of logstash's JSON # outputs emitted one per line, and whitespace is OK in json. @on_event.call(event, "#{event.to_json}#{@delimiter}") - end # def encode + end private - def guard(data) - begin - LogStash::Event.new(LogStash::Json.load(data)) - rescue LogStash::Json::ParserError => e - LogStash::Event.new("message" => data, "tags" => ["_jsonparsefailure"]) - end + # from_json_parse uses the Event#from_json method to deserialize and directly produce events + def from_json_parse(json, &block) + LogStash::Event.from_json(json).each { |event| yield event } + rescue LogStash::Json::ParserError + yield LogStash::Event.new("message" => json, "tags" => ["_jsonparsefailure"]) end + # legacy_parse uses the LogStash::Json class to deserialize json + def legacy_parse(json, &block) + yield LogStash::Event.new(LogStash::Json.load(json)) + rescue LogStash::Json::ParserError + yield LogStash::Event.new("message" => json, "tags" => ["_jsonparsefailure"]) + end + + alias_method :parse, LogStash::Event.respond_to?(:from_json) ? :from_json_parse : :legacy_parse + end # class LogStash::Codecs::JSONLines diff --git a/spec/codecs/json_lines_spec.rb b/spec/codecs/json_lines_spec.rb index af933f5..ae2e186 100644 --- a/spec/codecs/json_lines_spec.rb +++ b/spec/codecs/json_lines_spec.rb @@ -6,9 +6,10 @@ require "insist" describe LogStash::Codecs::JSONLines do - subject do - next LogStash::Codecs::JSONLines.new - end + + let(:codec_options) { {} } + + shared_examples :codec do context "#decode" do it "should return an event from json data" do @@ -37,9 +38,7 @@ context "when using custom delimiter" do let(:delimiter) { "|" } let(:line) { "{\"hey\":1}|{\"hey\":2}|{\"hey\":3}|" } - subject do - next LogStash::Codecs::JSONLines.new("delimiter" => delimiter) - end + let(:codec_options) { { "delimiter" => delimiter } } it "should decode multiple lines separated by the delimiter" do result = [] @@ -122,9 +121,7 @@ context "when using custom delimiter" do let(:delimiter) { "|" } - subject do - next LogStash::Codecs::JSONLines.new("delimiter" => delimiter) - end + let(:codec_options) { { "delimiter" => delimiter } } it "should decode multiple lines separated by the delimiter" do subject.on_event do |e, d| @@ -170,4 +167,35 @@ expect(collector.last['field']).to eq('value2') end end + + end + + context "forcing legacy parsing" do + it_behaves_like :codec do + subject do + # register method is called in the constructor + LogStash::Codecs::JSONLines.new(codec_options) + end + + before(:each) do + # stub codec parse method to force use of the legacy parser. + # this is very implementation specific but I am not sure how + # this can be tested otherwise. + allow(subject).to receive(:parse) do |line, &block| + subject.send(:legacy_parse, line, &block) + end + end + end + end + + context "default parser choice" do + # here we cannot force the use of the Event#from_json since if this test is run in the + # legacy context (no Java Event) it will fail but if in the new context, it will be picked up. + it_behaves_like :codec do + subject do + # register method is called in the constructor + LogStash::Codecs::JSONLines.new(codec_options) + end + end + end end From 84c41b0fdd943f32def459148ce381f464021324 Mon Sep 17 00:00:00 2001 From: Colin Surprenant Date: Tue, 9 Feb 2016 11:12:47 -0500 Subject: [PATCH 2/2] bump version to 2.1.0 --- CHANGELOG.md | 3 +++ logstash-codec-json_lines.gemspec | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1405e5..b5f1722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 2.1.0 + - Backward compatible support for `Event#from_json` method https://github.com/logstash-plugins/logstash-codec-json_lines/pull/19 + ## 2.0.5 - Directly use buftok to avoid indirection through the line codec https://github.com/logstash-plugins/logstash-codec-json_lines/pull/18 diff --git a/logstash-codec-json_lines.gemspec b/logstash-codec-json_lines.gemspec index d83281f..b02a91a 100644 --- a/logstash-codec-json_lines.gemspec +++ b/logstash-codec-json_lines.gemspec @@ -1,14 +1,14 @@ Gem::Specification.new do |s| s.name = 'logstash-codec-json_lines' - s.version = '2.0.5' + s.version = '2.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "This codec will decode streamed JSON that is newline delimited." s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" s.authors = ["Elastic"] s.email = 'info@elastic.co' s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" - s.require_paths = ["lib"] + s.require_paths = ["lib"] # Files s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']