From 6186ad6fe2705a8f5510428c98dca322136cd968 Mon Sep 17 00:00:00 2001 From: Colin Surprenant Date: Thu, 4 Feb 2016 22:26:13 -0500 Subject: [PATCH] use Event#from_json when possible cosmetic reindent refactor for the array return value of from_json wrong paramter refactor to support testing from_json and legacy parser refactor to support testing from_json and legacy parser - take2 --- lib/logstash/codecs/json_lines.rb | 28 ++++++++++++------- spec/codecs/json_lines_spec.rb | 46 +++++++++++++++++++++++++------ 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/lib/logstash/codecs/json_lines.rb b/lib/logstash/codecs/json_lines.rb index 7fb865c..c917b93 100644 --- a/lib/logstash/codecs/json_lines.rb +++ b/lib/logstash/codecs/json_lines.rb @@ -36,26 +36,34 @@ def register @converter.logger = @logger end - def decode(data) + def decode(data, &block) @buffer.extract(data).each do |line| - yield guard(@converter.convert(line)) + parse(@converter.convert(line), &block) end - end # def decode + end def encode(event) # Tack on a @delimiter for now because previously most of logstash's JSON # outputs emitted one per line, and whitespace is OK in json. @on_event.call(event, "#{event.to_json}#{@delimiter}") - end # def encode + end private - def guard(data) - begin - LogStash::Event.new(LogStash::Json.load(data)) - rescue LogStash::Json::ParserError => e - LogStash::Event.new("message" => data, "tags" => ["_jsonparsefailure"]) - end + # from_json_parse uses the Event#from_json method to deserialize and directly produce events + def from_json_parse(json, &block) + LogStash::Event.from_json(json).each { |event| yield event } + rescue LogStash::Json::ParserError + yield LogStash::Event.new("message" => json, "tags" => ["_jsonparsefailure"]) end + # legacy_parse uses the LogStash::Json class to deserialize json + def legacy_parse(json, &block) + yield LogStash::Event.new(LogStash::Json.load(json)) + rescue LogStash::Json::ParserError + yield LogStash::Event.new("message" => json, "tags" => ["_jsonparsefailure"]) + end + + alias_method :parse, LogStash::Event.respond_to?(:from_json) ? :from_json_parse : :legacy_parse + end # class LogStash::Codecs::JSONLines diff --git a/spec/codecs/json_lines_spec.rb b/spec/codecs/json_lines_spec.rb index af933f5..ae2e186 100644 --- a/spec/codecs/json_lines_spec.rb +++ b/spec/codecs/json_lines_spec.rb @@ -6,9 +6,10 @@ require "insist" describe LogStash::Codecs::JSONLines do - subject do - next LogStash::Codecs::JSONLines.new - end + + let(:codec_options) { {} } + + shared_examples :codec do context "#decode" do it "should return an event from json data" do @@ -37,9 +38,7 @@ context "when using custom delimiter" do let(:delimiter) { "|" } let(:line) { "{\"hey\":1}|{\"hey\":2}|{\"hey\":3}|" } - subject do - next LogStash::Codecs::JSONLines.new("delimiter" => delimiter) - end + let(:codec_options) { { "delimiter" => delimiter } } it "should decode multiple lines separated by the delimiter" do result = [] @@ -122,9 +121,7 @@ context "when using custom delimiter" do let(:delimiter) { "|" } - subject do - next LogStash::Codecs::JSONLines.new("delimiter" => delimiter) - end + let(:codec_options) { { "delimiter" => delimiter } } it "should decode multiple lines separated by the delimiter" do subject.on_event do |e, d| @@ -170,4 +167,35 @@ expect(collector.last['field']).to eq('value2') end end + + end + + context "forcing legacy parsing" do + it_behaves_like :codec do + subject do + # register method is called in the constructor + LogStash::Codecs::JSONLines.new(codec_options) + end + + before(:each) do + # stub codec parse method to force use of the legacy parser. + # this is very implementation specific but I am not sure how + # this can be tested otherwise. + allow(subject).to receive(:parse) do |line, &block| + subject.send(:legacy_parse, line, &block) + end + end + end + end + + context "default parser choice" do + # here we cannot force the use of the Event#from_json since if this test is run in the + # legacy context (no Java Event) it will fail but if in the new context, it will be picked up. + it_behaves_like :codec do + subject do + # register method is called in the constructor + LogStash::Codecs::JSONLines.new(codec_options) + end + end + end end