From 2d094684c1583921a90f97ca6e638ce3f1b28526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Fri, 22 Dec 2017 23:23:44 +0100 Subject: [PATCH 01/22] Add test ArrayDestination --- test/support/test_array_destination.rb | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 test/support/test_array_destination.rb diff --git a/test/support/test_array_destination.rb b/test/support/test_array_destination.rb new file mode 100644 index 0000000..d4229c4 --- /dev/null +++ b/test/support/test_array_destination.rb @@ -0,0 +1,9 @@ +class TestArrayDestination + def initialize(array) + @array = array + end + + def write(row) + @array << row + end +end \ No newline at end of file From ca31e00cf0de30c4537243213827f66a62ea8925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Fri, 22 Dec 2017 23:34:56 +0100 Subject: [PATCH 02/22] Implement YieldingTransform tests This cherry-picks from #41 but improves the syntax a bit. --- test/support/test_yielding_transform.rb | 8 ++++++ test/test_yielding_runner.rb | 37 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 test/support/test_yielding_transform.rb create mode 100644 test/test_yielding_runner.rb diff --git a/test/support/test_yielding_transform.rb b/test/support/test_yielding_transform.rb new file mode 100644 index 0000000..04dad21 --- /dev/null +++ b/test/support/test_yielding_transform.rb @@ -0,0 +1,8 @@ +class TestYieldingTransform + def process(row) + row.fetch(:tags).each do |value| + yield({item: value}) + end + {item: "classic-return-value"} + end +end \ No newline at end of file diff --git a/test/test_yielding_runner.rb b/test/test_yielding_runner.rb new file mode 100644 index 0000000..eae2d01 --- /dev/null +++ b/test/test_yielding_runner.rb @@ -0,0 +1,37 @@ +require_relative 'helper' +require 'minitest/mock' +require_relative 'support/test_enumerable_source' +require_relative 'support/test_array_destination' +require_relative 'support/test_yielding_transform' +require_relative 'common/runner' +require 'kiba/yielding_runner' + +class TestYieldingRunner < Kiba::Test + include SharedRunnerTests + + def kiba_run(job) + runner = Object.new + runner.extend(Kiba::YieldingRunner) + runner.run(job) + end + + def test_yielding_class_transform + input_row = {tags: ["one", "two", "three"]} + destination_array = [] + + job = Kiba.parse do + source TestEnumerableSource, [input_row] + transform TestYieldingTransform + destination TestArrayDestination, destination_array + end + + kiba_run(job) + + assert_equal [ + {item: 'one'}, + {item: 'two'}, + {item: 'three'}, + {item: 'classic-return-value'} + ], destination_array + end +end From d30cc0bce0e3bc4a98f9a0f2c5be8b78f1f10fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Fri, 22 Dec 2017 23:35:22 +0100 Subject: [PATCH 03/22] Implement YieldingRunner This cherry-picks from #41 but with a more DRY code. --- lib/kiba/yielding_runner.rb | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 lib/kiba/yielding_runner.rb diff --git a/lib/kiba/yielding_runner.rb b/lib/kiba/yielding_runner.rb new file mode 100644 index 0000000..adef7aa --- /dev/null +++ b/lib/kiba/yielding_runner.rb @@ -0,0 +1,35 @@ +module Kiba + module YieldingRunner + include Runner + + def lazy_transform(from, t) + Enumerator::Lazy.new(from) do |yielder, input_row| + final_row = t.process(input_row) do |yielded_row| + yielder << yielded_row + end + yielder << final_row if final_row + end + end + + def process_rows(sources, transforms, destinations) + source_rows = Enumerator::Lazy.new(sources) do |yielder, source| + source.each do |row| + yielder << row + end + end + + rows = source_rows.lazy + + transforms.each do |transform| + rows = lazy_transform(rows, transform) + end + + rows.each do |row| + destinations.each do |destination| + destination.write(row) + end + end + destinations.find_all { |d| d.respond_to?(:close) }.each(&:close) + end + end +end \ No newline at end of file From e389a32406e05ae0460eccac482206749dec0340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Fri, 22 Dec 2017 23:54:01 +0100 Subject: [PATCH 04/22] Attempt to fix JRuby 1.7 build --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index aaee224..9ad565e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,9 @@ language: ruby before_install: - gem update bundler +env: + # required for the yielding runner + - JRUBY_OPTS=--2.0 rvm: - ruby-head - 2.4.3 From 55694c623751d7d146d197350e469b438983c7e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sat, 23 Dec 2017 00:21:43 +0100 Subject: [PATCH 05/22] Implement a mix-inspired config mechanism --- lib/kiba/context.rb | 4 ++++ lib/kiba/control.rb | 4 ++++ test/test_parser.rb | 23 +++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/lib/kiba/context.rb b/lib/kiba/context.rb index 2511739..4e634e9 100644 --- a/lib/kiba/context.rb +++ b/lib/kiba/context.rb @@ -3,6 +3,10 @@ class Context def initialize(control) @control = control end + + def config(context, context_config) + @control.config[context] = @control.config.fetch(context, {}).merge(context_config) + end def pre_process(&block) @control.pre_processes << { block: block } diff --git a/lib/kiba/control.rb b/lib/kiba/control.rb index 8abfd91..dc27eee 100644 --- a/lib/kiba/control.rb +++ b/lib/kiba/control.rb @@ -3,6 +3,10 @@ class Control def pre_processes @pre_processes ||= [] end + + def config + @config ||= {} + end def sources @sources ||= [] diff --git a/test/test_parser.rb b/test/test_parser.rb index b183ebb..6ce6758 100644 --- a/test/test_parser.rb +++ b/test/test_parser.rb @@ -100,4 +100,27 @@ def common_source_declaration ensure remove_files('test/tmp/etl-common.rb', 'test/tmp/etl-main.rb') end + + def test_config + control = Kiba.parse do + config :context, key: "value", other_key: "other_value" + end + + assert_equal({ context: { + key: "value", + other_key: "other_value" + }}, control.config) + end + + def test_config_override + control = Kiba.parse do + config :context, key: "value", other_key: "other_value" + config :context, key: "new_value" + end + + assert_equal({ context: { + key: "new_value", + other_key: "other_value" + }}, control.config) + end end From c79c326aeb00a2f2061d8572d75899ef8bbc5d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sat, 23 Dec 2017 00:27:05 +0100 Subject: [PATCH 06/22] Implement config-based runner choice A much better choice than what was originally implemented in #41, since: - it allows to decide which runner to pick on a per-ETL basis - it will work inside sidekiq (vs only on command line) --- lib/kiba.rb | 9 ++++++++- lib/kiba/runner.rb | 2 ++ lib/kiba/yielding_runner.rb | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/kiba.rb b/lib/kiba.rb index 898d36d..5d1a85c 100644 --- a/lib/kiba.rb +++ b/lib/kiba.rb @@ -7,4 +7,11 @@ require 'kiba/runner' Kiba.extend(Kiba::Parser) -Kiba.extend(Kiba::Runner) + +module Kiba + def self.run(job) + # NOTE: use Hash#dig when Ruby 2.2 reaches EOL + runner = job.config.fetch(:kiba, {}).fetch(:runner, Kiba::Runner) + runner.run(job) + end +end diff --git a/lib/kiba/runner.rb b/lib/kiba/runner.rb index 6ee8e80..777bd44 100644 --- a/lib/kiba/runner.rb +++ b/lib/kiba/runner.rb @@ -1,5 +1,7 @@ module Kiba module Runner + extend self + # allow to handle a block form just like a regular transform class AliasingProc < Proc alias_method :process, :call diff --git a/lib/kiba/yielding_runner.rb b/lib/kiba/yielding_runner.rb index adef7aa..1c836c1 100644 --- a/lib/kiba/yielding_runner.rb +++ b/lib/kiba/yielding_runner.rb @@ -1,6 +1,7 @@ module Kiba module YieldingRunner include Runner + extend self def lazy_transform(from, t) Enumerator::Lazy.new(from) do |yielder, input_row| From aff7a2984eb89eb954a63928fe27e1aacdbe5c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sat, 23 Dec 2017 00:28:20 +0100 Subject: [PATCH 07/22] Use config-based engine choice --- lib/kiba.rb | 1 + test/test_yielding_runner.rb | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/kiba.rb b/lib/kiba.rb index 5d1a85c..2eed5e1 100644 --- a/lib/kiba.rb +++ b/lib/kiba.rb @@ -5,6 +5,7 @@ require 'kiba/context' require 'kiba/parser' require 'kiba/runner' +require 'kiba/yielding_runner' Kiba.extend(Kiba::Parser) diff --git a/test/test_yielding_runner.rb b/test/test_yielding_runner.rb index eae2d01..c89dae2 100644 --- a/test/test_yielding_runner.rb +++ b/test/test_yielding_runner.rb @@ -10,9 +10,7 @@ class TestYieldingRunner < Kiba::Test include SharedRunnerTests def kiba_run(job) - runner = Object.new - runner.extend(Kiba::YieldingRunner) - runner.run(job) + Kiba.run(job) end def test_yielding_class_transform @@ -20,12 +18,14 @@ def test_yielding_class_transform destination_array = [] job = Kiba.parse do + config :kiba, runner: Kiba::YieldingRunner + source TestEnumerableSource, [input_row] transform TestYieldingTransform destination TestArrayDestination, destination_array end - kiba_run(job) + Kiba.run(job) assert_equal [ {item: 'one'}, From 9b3d10721b80a8b947d6e38c5d88fb032b435a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sat, 23 Dec 2017 00:32:38 +0100 Subject: [PATCH 08/22] Clean up tests --- test/common/runner.rb | 7 +++++++ test/test_runner.rb | 8 -------- test/test_yielding_runner.rb | 8 +------- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/test/common/runner.rb b/test/common/runner.rb index 5c35b13..2b2ec36 100644 --- a/test/common/runner.rb +++ b/test/common/runner.rb @@ -1,4 +1,11 @@ +require 'minitest/mock' +require_relative '../support/test_enumerable_source' + module SharedRunnerTests + def kiba_run(job) + Kiba.run(job) + end + def rows @rows ||= [ { identifier: 'first-row' }, diff --git a/test/test_runner.rb b/test/test_runner.rb index 70a2096..9fc0bbf 100644 --- a/test/test_runner.rb +++ b/test/test_runner.rb @@ -1,14 +1,6 @@ require_relative 'helper' -require 'minitest/mock' -require_relative 'support/test_enumerable_source' require_relative 'common/runner' class TestRunner < Kiba::Test include SharedRunnerTests - - def kiba_run(job) - runner = Object.new - runner.extend(Kiba::Runner) - runner.run(job) - end end diff --git a/test/test_yielding_runner.rb b/test/test_yielding_runner.rb index c89dae2..3403eb5 100644 --- a/test/test_yielding_runner.rb +++ b/test/test_yielding_runner.rb @@ -1,18 +1,12 @@ require_relative 'helper' -require 'minitest/mock' require_relative 'support/test_enumerable_source' require_relative 'support/test_array_destination' require_relative 'support/test_yielding_transform' require_relative 'common/runner' -require 'kiba/yielding_runner' class TestYieldingRunner < Kiba::Test include SharedRunnerTests - def kiba_run(job) - Kiba.run(job) - end - def test_yielding_class_transform input_row = {tags: ["one", "two", "three"]} destination_array = [] @@ -25,7 +19,7 @@ def test_yielding_class_transform destination TestArrayDestination, destination_array end - Kiba.run(job) + kiba_run(job) assert_equal [ {item: 'one'}, From 9f5e12671eb9a696ad5ca93c76bbb6e49b5a2096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sat, 23 Dec 2017 23:59:58 +0100 Subject: [PATCH 09/22] Remove close (already called by regular runner) --- lib/kiba/yielding_runner.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/kiba/yielding_runner.rb b/lib/kiba/yielding_runner.rb index 1c836c1..759c88f 100644 --- a/lib/kiba/yielding_runner.rb +++ b/lib/kiba/yielding_runner.rb @@ -30,7 +30,6 @@ def process_rows(sources, transforms, destinations) destination.write(row) end end - destinations.find_all { |d| d.respond_to?(:close) }.each(&:close) end end end \ No newline at end of file From 2694981432cd980e225153b9d88cfcb5e8af39f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 00:00:30 +0100 Subject: [PATCH 10/22] Extract method --- lib/kiba/yielding_runner.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/kiba/yielding_runner.rb b/lib/kiba/yielding_runner.rb index 759c88f..75a63ea 100644 --- a/lib/kiba/yielding_runner.rb +++ b/lib/kiba/yielding_runner.rb @@ -11,15 +11,17 @@ def lazy_transform(from, t) yielder << final_row if final_row end end - - def process_rows(sources, transforms, destinations) - source_rows = Enumerator::Lazy.new(sources) do |yielder, source| + + def lazy_source_rows(sources) + Enumerator::Lazy.new(sources) do |yielder, source| source.each do |row| yielder << row end - end + end.lazy + end - rows = source_rows.lazy + def process_rows(sources, transforms, destinations) + rows = lazy_source_rows(sources) transforms.each do |transform| rows = lazy_transform(rows, transform) From 1b209c7778f564be0519b0489e64089852938e75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 01:18:48 +0100 Subject: [PATCH 11/22] Add minitest-focus to development dependencies --- kiba.gemspec | 1 + test/helper.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/kiba.gemspec b/kiba.gemspec index c882c93..870dc37 100644 --- a/kiba.gemspec +++ b/kiba.gemspec @@ -17,4 +17,5 @@ Gem::Specification.new do |gem| gem.add_development_dependency 'rake' gem.add_development_dependency 'minitest', '~> 5.9' gem.add_development_dependency 'awesome_print' + gem.add_development_dependency 'minitest-focus' end diff --git a/test/helper.rb b/test/helper.rb index deb810c..fc1a78a 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -1,5 +1,6 @@ require 'minitest/autorun' require 'minitest/pride' +require 'minitest/focus' require 'kiba' class Kiba::Test < Minitest::Test From 97418ac5ae794b639c87c92f83589414f541651a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 01:27:00 +0100 Subject: [PATCH 12/22] Rename for clarity --- lib/kiba/yielding_runner.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/kiba/yielding_runner.rb b/lib/kiba/yielding_runner.rb index 75a63ea..76f971c 100644 --- a/lib/kiba/yielding_runner.rb +++ b/lib/kiba/yielding_runner.rb @@ -5,10 +5,10 @@ module YieldingRunner def lazy_transform(from, t) Enumerator::Lazy.new(from) do |yielder, input_row| - final_row = t.process(input_row) do |yielded_row| + returned_row = t.process(input_row) do |yielded_row| yielder << yielded_row end - yielder << final_row if final_row + yielder << returned_row if returned_row end end From 4e31d8878eacfe6d6b9fcc2888c43571b0fb16ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 01:29:41 +0100 Subject: [PATCH 13/22] Group code on one line --- lib/kiba/yielding_runner.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/kiba/yielding_runner.rb b/lib/kiba/yielding_runner.rb index 76f971c..428c4c4 100644 --- a/lib/kiba/yielding_runner.rb +++ b/lib/kiba/yielding_runner.rb @@ -14,9 +14,7 @@ def lazy_transform(from, t) def lazy_source_rows(sources) Enumerator::Lazy.new(sources) do |yielder, source| - source.each do |row| - yielder << row - end + source.each { |r| yielder << r } end.lazy end From 48b8cbda7123ccbd4d179d2e11382d9544612edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 01:30:06 +0100 Subject: [PATCH 14/22] Rewrite in a slightly more functional fashion --- lib/kiba/yielding_runner.rb | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/kiba/yielding_runner.rb b/lib/kiba/yielding_runner.rb index 428c4c4..acfc6b6 100644 --- a/lib/kiba/yielding_runner.rb +++ b/lib/kiba/yielding_runner.rb @@ -19,16 +19,10 @@ def lazy_source_rows(sources) end def process_rows(sources, transforms, destinations) - rows = lazy_source_rows(sources) - - transforms.each do |transform| - rows = lazy_transform(rows, transform) - end - - rows.each do |row| - destinations.each do |destination| - destination.write(row) - end + stream = lazy_source_rows(sources) + recurser = lambda { |stream,t| lazy_transform(stream, t) } + transforms.inject(stream, &recurser).each do |r| + destinations.each { |d| d.write(r) } end end end From 66a43716bfef75bfd1eaa8af0722fe7c6b765df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 01:30:34 +0100 Subject: [PATCH 15/22] Rename for clarity --- lib/kiba/yielding_runner.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/kiba/yielding_runner.rb b/lib/kiba/yielding_runner.rb index acfc6b6..10a93a1 100644 --- a/lib/kiba/yielding_runner.rb +++ b/lib/kiba/yielding_runner.rb @@ -3,7 +3,7 @@ module YieldingRunner include Runner extend self - def lazy_transform(from, t) + def transform_stream(from, t) Enumerator::Lazy.new(from) do |yielder, input_row| returned_row = t.process(input_row) do |yielded_row| yielder << yielded_row @@ -12,15 +12,15 @@ def lazy_transform(from, t) end end - def lazy_source_rows(sources) + def source_stream(sources) Enumerator::Lazy.new(sources) do |yielder, source| source.each { |r| yielder << r } end.lazy end def process_rows(sources, transforms, destinations) - stream = lazy_source_rows(sources) - recurser = lambda { |stream,t| lazy_transform(stream, t) } + stream = source_stream(sources) + recurser = lambda { |stream,t| transform_stream(stream, t) } transforms.inject(stream, &recurser).each do |r| destinations.each { |d| d.write(r) } end From 4939cab205eff45dd414b22c731f086bace8baf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 01:31:25 +0100 Subject: [PATCH 16/22] Remove duplicate lazy invokation --- lib/kiba/yielding_runner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kiba/yielding_runner.rb b/lib/kiba/yielding_runner.rb index 10a93a1..c4c8f9a 100644 --- a/lib/kiba/yielding_runner.rb +++ b/lib/kiba/yielding_runner.rb @@ -15,7 +15,7 @@ def transform_stream(from, t) def source_stream(sources) Enumerator::Lazy.new(sources) do |yielder, source| source.each { |r| yielder << r } - end.lazy + end end def process_rows(sources, transforms, destinations) From 1190565ebc7047eb3a949b2b080123df3f79c294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 10:39:19 +0100 Subject: [PATCH 17/22] Rename YieldingRunner to StreamingRunner --- lib/kiba.rb | 2 +- lib/kiba/{yielding_runner.rb => streaming_runner.rb} | 2 +- test/{test_yielding_runner.rb => test_streaming_runner.rb} | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename lib/kiba/{yielding_runner.rb => streaming_runner.rb} (96%) rename test/{test_yielding_runner.rb => test_streaming_runner.rb} (88%) diff --git a/lib/kiba.rb b/lib/kiba.rb index 2eed5e1..2c97c02 100644 --- a/lib/kiba.rb +++ b/lib/kiba.rb @@ -5,7 +5,7 @@ require 'kiba/context' require 'kiba/parser' require 'kiba/runner' -require 'kiba/yielding_runner' +require 'kiba/streaming_runner' Kiba.extend(Kiba::Parser) diff --git a/lib/kiba/yielding_runner.rb b/lib/kiba/streaming_runner.rb similarity index 96% rename from lib/kiba/yielding_runner.rb rename to lib/kiba/streaming_runner.rb index c4c8f9a..39f149e 100644 --- a/lib/kiba/yielding_runner.rb +++ b/lib/kiba/streaming_runner.rb @@ -1,5 +1,5 @@ module Kiba - module YieldingRunner + module StreamingRunner include Runner extend self diff --git a/test/test_yielding_runner.rb b/test/test_streaming_runner.rb similarity index 88% rename from test/test_yielding_runner.rb rename to test/test_streaming_runner.rb index 3403eb5..e918c08 100644 --- a/test/test_yielding_runner.rb +++ b/test/test_streaming_runner.rb @@ -4,7 +4,7 @@ require_relative 'support/test_yielding_transform' require_relative 'common/runner' -class TestYieldingRunner < Kiba::Test +class TestStreamingRunner < Kiba::Test include SharedRunnerTests def test_yielding_class_transform @@ -12,7 +12,7 @@ def test_yielding_class_transform destination_array = [] job = Kiba.parse do - config :kiba, runner: Kiba::YieldingRunner + config :kiba, runner: Kiba::StreamingRunner source TestEnumerableSource, [input_row] transform TestYieldingTransform From d2307206d82c47aa3cca38416133ec701656f48f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 10:49:55 +0100 Subject: [PATCH 18/22] Make config access opt-in This is a UX measure to make sure nobody will unknowingly shadow an existing "config" variable with the new system. Basically this avoids a breaking change in the behaviour. --- lib/kiba.rb | 1 + lib/kiba/context.rb | 4 ---- lib/kiba/dsl_extensions/config.rb | 9 +++++++++ test/test_parser.rb | 4 ++++ test/test_streaming_runner.rb | 2 ++ 5 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 lib/kiba/dsl_extensions/config.rb diff --git a/lib/kiba.rb b/lib/kiba.rb index 2c97c02..16c90a4 100644 --- a/lib/kiba.rb +++ b/lib/kiba.rb @@ -6,6 +6,7 @@ require 'kiba/parser' require 'kiba/runner' require 'kiba/streaming_runner' +require 'kiba/dsl_extensions/config' Kiba.extend(Kiba::Parser) diff --git a/lib/kiba/context.rb b/lib/kiba/context.rb index 4e634e9..ff66bd1 100644 --- a/lib/kiba/context.rb +++ b/lib/kiba/context.rb @@ -4,10 +4,6 @@ def initialize(control) @control = control end - def config(context, context_config) - @control.config[context] = @control.config.fetch(context, {}).merge(context_config) - end - def pre_process(&block) @control.pre_processes << { block: block } end diff --git a/lib/kiba/dsl_extensions/config.rb b/lib/kiba/dsl_extensions/config.rb new file mode 100644 index 0000000..397fe15 --- /dev/null +++ b/lib/kiba/dsl_extensions/config.rb @@ -0,0 +1,9 @@ +module Kiba + module DSLExtensions + module Config + def config(context, context_config) + @control.config[context] = @control.config.fetch(context, {}).merge(context_config) + end + end + end +end \ No newline at end of file diff --git a/test/test_parser.rb b/test/test_parser.rb index 6ce6758..a69da2f 100644 --- a/test/test_parser.rb +++ b/test/test_parser.rb @@ -103,6 +103,8 @@ def common_source_declaration def test_config control = Kiba.parse do + extend Kiba::DSLExtensions::Config + config :context, key: "value", other_key: "other_value" end @@ -114,6 +116,8 @@ def test_config def test_config_override control = Kiba.parse do + extend Kiba::DSLExtensions::Config + config :context, key: "value", other_key: "other_value" config :context, key: "new_value" end diff --git a/test/test_streaming_runner.rb b/test/test_streaming_runner.rb index e918c08..723616b 100644 --- a/test/test_streaming_runner.rb +++ b/test/test_streaming_runner.rb @@ -12,6 +12,8 @@ def test_yielding_class_transform destination_array = [] job = Kiba.parse do + extend Kiba::DSLExtensions::Config + config :kiba, runner: Kiba::StreamingRunner source TestEnumerableSource, [input_row] From 9435737815d7882089c27277fd6619b57772ebab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 11:06:06 +0100 Subject: [PATCH 19/22] Use merge! for more concise code --- lib/kiba/dsl_extensions/config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kiba/dsl_extensions/config.rb b/lib/kiba/dsl_extensions/config.rb index 397fe15..616dbc3 100644 --- a/lib/kiba/dsl_extensions/config.rb +++ b/lib/kiba/dsl_extensions/config.rb @@ -2,7 +2,7 @@ module Kiba module DSLExtensions module Config def config(context, context_config) - @control.config[context] = @control.config.fetch(context, {}).merge(context_config) + (@control.config[context] ||= {}).merge!(context_config) end end end From 8d6761c0737c45a5a173d3c4c63bd2a56a69ff62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 23:34:34 +0100 Subject: [PATCH 20/22] Use Enumerator instead of Enumerator::Lazy I've realized that lazy is not useful yet at this stage, and also brings an extra cost. I will switch back to lazy when I'll have an actual need (future work on parallelization etc). --- lib/kiba/streaming_runner.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/kiba/streaming_runner.rb b/lib/kiba/streaming_runner.rb index 39f149e..69eece8 100644 --- a/lib/kiba/streaming_runner.rb +++ b/lib/kiba/streaming_runner.rb @@ -3,18 +3,22 @@ module StreamingRunner include Runner extend self - def transform_stream(from, t) - Enumerator::Lazy.new(from) do |yielder, input_row| - returned_row = t.process(input_row) do |yielded_row| - yielder << yielded_row + def transform_stream(stream, t) + Enumerator.new do |y| + stream.each do |input_row| + returned_row = t.process(input_row) do |yielded_row| + y << yielded_row + end + y << returned_row if returned_row end - yielder << returned_row if returned_row end end def source_stream(sources) - Enumerator::Lazy.new(sources) do |yielder, source| - source.each { |r| yielder << r } + Enumerator.new do |y| + sources.each do |source| + source.each { |r| y << r } + end end end From 96839b461a5168d7a22a8f2b8d9db0f12f4c640a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Sun, 24 Dec 2017 23:35:16 +0100 Subject: [PATCH 21/22] Remove JRUBY_OPTS I think the previous commit should make the runner work on all supported platforms, unmodified. --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9ad565e..aaee224 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,6 @@ language: ruby before_install: - gem update bundler -env: - # required for the yielding runner - - JRUBY_OPTS=--2.0 rvm: - ruby-head - 2.4.3 From 627dbf2f3ee5da2dac5a9d998b7a69cbbd4e1e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barre=CC=80re?= Date: Mon, 25 Dec 2017 00:43:29 +0100 Subject: [PATCH 22/22] Remove whitespace --- lib/kiba/context.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kiba/context.rb b/lib/kiba/context.rb index ff66bd1..2511739 100644 --- a/lib/kiba/context.rb +++ b/lib/kiba/context.rb @@ -3,7 +3,7 @@ class Context def initialize(control) @control = control end - + def pre_process(&block) @control.pre_processes << { block: block } end