diff --git a/priv/bitcask.schema b/priv/bitcask.schema index 77c2ed92..99a7afd7 100644 --- a/priv/bitcask.schema +++ b/priv/bitcask.schema @@ -8,12 +8,6 @@ {datatype, directory} ]}. -%% @see bitcask.data_root -{mapping, "multi_backend.$name.bitcask.data_root", "riak_kv.multi_backend", [ - hidden, - {datatype, directory} -]}. - %% @doc Specifies the maximum time Bitcask will block on startup while %% attempting to create or open the data directory. You generally need %% not change this value. If for some reason the timeout is exceeded @@ -26,13 +20,6 @@ hidden ]}. -%% @see bitcask.open_timeout -{mapping, "multi_backend.$name.bitcask.open_timeout", "riak_kv.multi_backend", [ - {default, "4s"}, - {datatype, {duration, s}}, - hidden -]}. - %% @doc Changes the durability of writes by specifying when to %% synchronize data to disk. The default setting protects against data %% loss in the event of application failure (process death) but leaves @@ -65,25 +52,12 @@ hidden ]}. -%% @see bitcask.sync.strategy -{mapping, "multi_backend.$name.bitcask.sync.strategy", "riak_kv.multi_backend", [ - {default, none}, - {datatype, {enum, [none, o_sync, interval]}}, - hidden -]}. - %% @see bitcask.sync.strategy {mapping, "bitcask.sync.interval", "bitcask.sync_strategy", [ {datatype, {duration, s}}, hidden ]}. -%% @see bitcask.sync.strategy -{mapping, "multi_backend.$name.bitcask.sync.interval", "riak_kv.multi_backend", [ - {datatype, {duration, s}}, - hidden -]}. - {translation, "bitcask.sync_strategy", fun(Conf) -> @@ -92,8 +66,8 @@ none -> none; o_sync -> o_sync; interval -> - Interval = cuttlefish:conf_get("bitcask.sync.interval", Conf, undefined), - {seconds, Interval}; + Interval = cuttlefish:conf_get("bitcask.sync.interval", Conf, undefined), + {seconds, Interval}; _Default -> none end end}. @@ -108,13 +82,6 @@ hidden ]}. -%% @see bitcask.max_file_size -{mapping, "multi_backend.$name.bitcask.max_file_size", "riak_kv.multi_backend", [ - {default, "2GB"}, - {datatype, bytesize}, - hidden -]}. - %% @doc Lets you specify when during the day merge operations are %% allowed to be triggered. Valid options are: %% @@ -133,13 +100,6 @@ hidden ]}. -%% @see bitcask.merge.policy -{mapping, "multi_backend.$name.bitcask.merge.policy", "riak_kv.multi_backend", [ - {default, always}, - {datatype, {enum, [always, never, window]}}, - hidden -]}. - %% @see bitcask.merge.policy {mapping, "bitcask.merge.window.start", "bitcask.merge_window", [ {default, 0}, @@ -147,13 +107,6 @@ hidden ]}. -%% @see bitcask.merge.policy -{mapping, "multi_backend.$name.bitcask.merge.window.start", "riak_kv.multi_backend", [ - {default, 0}, - {datatype, integer}, - hidden -]}. - %% @see bitcask.merge.policy {mapping, "bitcask.merge.window.end", "bitcask.merge_window", [ {default, 23}, @@ -161,13 +114,6 @@ hidden ]}. -%% @see bitcask.merge.policy -{mapping, "multi_backend.$name.bitcask.merge.window.end", "riak_kv.multi_backend", [ - {default, 23}, - {datatype, integer}, - hidden -]}. - {translation, "bitcask.merge_window", fun(Conf) -> @@ -176,9 +122,9 @@ always -> always; never -> never; window -> - Start = cuttlefish:conf_get("bitcask.merge.window.start", Conf, undefined), - End = cuttlefish:conf_get("bitcask.merge.window.end", Conf, undefined), - {Start, End}; + Start = cuttlefish:conf_get("bitcask.merge.window.start", Conf, undefined), + End = cuttlefish:conf_get("bitcask.merge.window.end", Conf, undefined), + {Start, End}; _Default -> always end end}. @@ -201,19 +147,11 @@ {validators, ["is_percentage"]} ]}. -%% @see bitcask.merge.triggers.fragmentation -{mapping, "multi_backend.$name.bitcask.merge.triggers.fragmentation", "riak_kv.multi_backend", [ - {datatype, integer}, - hidden, - {default, 60}, - {validators, ["is_percentage"]} -]}. - {validator, "is_percentage", "must be a percentage", fun(Value) -> - Value >= 0 andalso Value =< 100 + Value >= 0 andalso Value =< 100 end}. %% @doc Describes how much data stored for dead keys in a single file @@ -227,20 +165,14 @@ %% directory, Bitcask will attempt to merge files. %% %% Default is: 512MB -{mapping, "bitcask.merge.triggers.dead_bytes", - "bitcask.dead_bytes_merge_trigger", - [ +{mapping, + "bitcask.merge.triggers.dead_bytes", + "bitcask.dead_bytes_merge_trigger", [ {datatype, bytesize}, hidden, {default, "512MB"} ]}. -%% @see bitcask.merge.triggers.dead_bytes -{mapping, "multi_backend.$name.bitcask.merge.triggers.dead_bytes", "riak_kv.multi_backend", [ - {datatype, bytesize}, - hidden, - {default, "512MB"} -]}. %% @doc Describes what ratio of dead keys to total keys in a file will %% cause it to be included in the merge. The value of this setting is @@ -250,17 +182,9 @@ %% merged, decreasing the value will cause more files to be merged. %% %% Default is: `40` -{mapping, "bitcask.merge.thresholds.fragmentation", - "bitcask.frag_threshold", -[ - {datatype, integer}, - hidden, - {default, 40}, - {validators, ["is_percentage"]} -]}. - -%% @see bitcask.merge.thresholds.fragmentation -{mapping, "multi_backend.$name.bitcask.thresholds.fragmentation", "riak_kv.multi_backend", [ +{mapping, + "bitcask.merge.thresholds.fragmentation", + "bitcask.frag_threshold", [ {datatype, integer}, hidden, {default, 40}, @@ -273,40 +197,28 @@ %% will cause more files to be merged. %% %% Default is: 128MB -{mapping, "bitcask.merge.thresholds.dead_bytes", +{mapping, + "bitcask.merge.thresholds.dead_bytes", "bitcask.dead_bytes_threshold", [ {datatype, bytesize}, hidden, {default, "128MB"} ]}. -%% @see bitcask.merge.thresholds.dead_bytes -{mapping, "multi_backend.$name.bitcask.thresholds.dead_bytes", "riak_kv.multi_backend", [ - {datatype, bytesize}, - hidden, - {default, "128MB"} -]}. - %% @doc Describes the minimum size a file must have to be _excluded_ %% from the merge. Files smaller than the threshold will be %% included. Increasing the value will cause _more_ files to be %% merged, decreasing the value will cause _fewer_ files to be merged. %% %% Default is: 10MB -{mapping, "bitcask.merge.thresholds.small_file", +{mapping, + "bitcask.merge.thresholds.small_file", "bitcask.small_file_threshold", [ {datatype, bytesize}, hidden, {default, "10MB"} ]}. -%% @see bitcask.merge.thresholds.small_file -{mapping, "multi_backend.$name.bitcask.thresholds.small_file", "riak_kv.multi_backend", [ - {datatype, bytesize}, - hidden, - {default, "10MB"} -]}. - %% @doc Fold keys thresholds will reuse the keydir if another fold was %% started less than `fold.max_age` ago and there were less than %% `fold.max_puts` updates. Otherwise it will wait until all current @@ -318,22 +230,15 @@ {default, unlimited} ]}. -%% @see bitcask.fold.max_age -{mapping, "multi_backend.$name.bitcask.fold.max_age", "riak_kv.multi_backend", [ - {datatype, [{atom, unlimited}, {duration, ms}]}, - hidden, - {default, unlimited} -]}. - {translation, "bitcask.max_fold_age", fun(Conf) -> - case cuttlefish:conf_get("bitcask.fold.max_age", Conf) of - unlimited -> -1; - I when is_integer(I) -> - %% app.config expects microseconds - I * 1000; - _ -> -1 %% The default, for safety - end + case cuttlefish:conf_get("bitcask.fold.max_age", Conf) of + unlimited -> -1; + I when is_integer(I) -> + %% app.config expects microseconds + I * 1000; + _ -> -1 %% The default, for safety + end end }. @@ -344,19 +249,12 @@ {default, 0} ]}. -%% @see bitcask.fold.max_age -{mapping, "multi_backend.$name.bitcask.fold.max_puts", "riak_kv.multi_backend", [ - {datatype, [integer, {atom, unlimited}]}, - hidden, - {default, 0} -]}. - {translation, "bitcask.max_fold_puts", fun(Conf) -> case cuttlefish:conf_get("bitcask.fold.max_puts", Conf) of - unlimited -> -1; - I when is_integer(I) -> I; - _ -> 0 %% default catch + unlimited -> -1; + I when is_integer(I) -> I; + _ -> 0 %% default catch end end }. @@ -373,13 +271,6 @@ {default, off} ]}. -%% @see bitcask.expiry -{mapping, "multi_backend.$name.bitcask.expiry", "riak_kv.multi_backend", [ - {datatype, [{atom, off}, {duration, s}]}, - hidden, - {default, off} -]}. - {translation, "bitcask.expiry_secs", fun(Conf) -> case cuttlefish:conf_get("bitcask.expiry", Conf) of @@ -400,19 +291,12 @@ hidden ]}. -%% @see bitcask.hintfile_checksums -{mapping, "multi_backend.$name.bitcask.hintfile_checksums", "riak_kv.multi_backend", [ - {default, strict}, - {datatype, {enum, [strict, allow_missing]}}, - hidden -]}. - {translation, "bitcask.require_hint_crc", fun(Conf) -> case cuttlefish:conf_get("bitcask.hintfile_checksums", Conf) of - strict -> true; - allow_missing -> false; - _ -> true + strict -> true; + allow_missing -> false; + _ -> true end end}. @@ -431,13 +315,6 @@ {default, 0} ]}. -%% @see bitcask.expiry.grace_time -{mapping, "multi_backend.$name.bitcask.expiry.grace_time", "riak_kv.multi_backend", [ - {datatype, {duration, s}}, - hidden, - {default, 0} -]}. - %% @doc Configure how Bitcask writes data to disk. %% erlang: Erlang's built-in file API %% nif: Direct calls to the POSIX C API @@ -450,10 +327,3 @@ {default, erlang}, {datatype, {enum, [erlang, nif]}} ]}. - -%% @see bitcask.io_mode -{mapping, "multi_backend.$name.bitcask.io_mode", "riak_kv.multi_backend", [ - {default, erlang}, - {datatype, {enum, [erlang, nif]}}, - hidden -]}. diff --git a/priv/bitcask_multi.schema b/priv/bitcask_multi.schema new file mode 100644 index 00000000..64bda74a --- /dev/null +++ b/priv/bitcask_multi.schema @@ -0,0 +1,137 @@ +%% -*- erlang -*- + +%%%% bitcask multi + +%% @see bitcask.data_root +{mapping, "multi_backend.$name.bitcask.data_root", "riak_kv.multi_backend", [ + hidden, + {datatype, directory} +]}. + +%% @see bitcask.open_timeout +{mapping, "multi_backend.$name.bitcask.open_timeout", "riak_kv.multi_backend", [ + {default, "4s"}, + {datatype, {duration, s}}, + hidden +]}. + +%% @see bitcask.sync.strategy +{mapping, "multi_backend.$name.bitcask.sync.strategy", "riak_kv.multi_backend", [ + {default, none}, + {datatype, {enum, [none, o_sync, interval]}}, + hidden +]}. + +%% @see bitcask.sync.strategy +{mapping, "multi_backend.$name.bitcask.sync.interval", "riak_kv.multi_backend", [ + {datatype, {duration, s}}, + hidden +]}. + +%% @see bitcask.max_file_size +{mapping, "multi_backend.$name.bitcask.max_file_size", "riak_kv.multi_backend", [ + {default, "2GB"}, + {datatype, bytesize}, + hidden +]}. + +%% @see bitcask.merge.policy +{mapping, "multi_backend.$name.bitcask.merge.policy", "riak_kv.multi_backend", [ + {default, always}, + {datatype, {enum, [always, never, window]}}, + hidden +]}. + +%% @see bitcask.merge.policy +{mapping, "multi_backend.$name.bitcask.merge.window.start", "riak_kv.multi_backend", [ + {default, 0}, + {datatype, integer}, + hidden +]}. + +%% @see bitcask.merge.policy +{mapping, "multi_backend.$name.bitcask.merge.window.end", "riak_kv.multi_backend", [ + {default, 23}, + {datatype, integer}, + hidden +]}. + +%% @see bitcask.merge.triggers.fragmentation +{mapping, "multi_backend.$name.bitcask.merge.triggers.fragmentation", "riak_kv.multi_backend", [ + {datatype, integer}, + hidden, + {default, 60}, + {validators, ["is_percentage"]} +]}. + +%% @see bitcask.merge.triggers.dead_bytes +{mapping, "multi_backend.$name.bitcask.merge.triggers.dead_bytes", "riak_kv.multi_backend", [ + {datatype, bytesize}, + hidden, + {default, "512MB"} +]}. + + +%% @see bitcask.merge.thresholds.fragmentation +{mapping, "multi_backend.$name.bitcask.merge.thresholds.fragmentation", "riak_kv.multi_backend", [ + {datatype, integer}, + hidden, + {default, 40}, + {validators, ["is_percentage"]} +]}. + +%% @see bitcask.merge.thresholds.dead_bytes +{mapping, "multi_backend.$name.bitcask.merge.thresholds.dead_bytes", "riak_kv.multi_backend", [ + {datatype, bytesize}, + hidden, + {default, "128MB"} +]}. + +%% @see bitcask.merge.thresholds.small_file +{mapping, "multi_backend.$name.bitcask.merge.thresholds.small_file", "riak_kv.multi_backend", [ + {datatype, bytesize}, + hidden, + {default, "10MB"} +]}. + +%% @see bitcask.fold.max_age +{mapping, "multi_backend.$name.bitcask.fold.max_age", "riak_kv.multi_backend", [ + {datatype, [{atom, unlimited}, {duration, ms}]}, + hidden, + {default, unlimited} +]}. + +%% @see bitcask.fold.max_age +{mapping, "multi_backend.$name.bitcask.fold.max_puts", "riak_kv.multi_backend", [ + {datatype, [integer, {atom, unlimited}]}, + hidden, + {default, 0} +]}. + +%% @see bitcask.expiry +{mapping, "multi_backend.$name.bitcask.expiry", "riak_kv.multi_backend", [ + {datatype, [{atom, off}, {duration, s}]}, + hidden, + {default, off} +]}. + +%% @see bitcask.hintfile_checksums +{mapping, "multi_backend.$name.bitcask.hintfile_checksums", "riak_kv.multi_backend", [ + {default, strict}, + {datatype, {enum, [strict, allow_missing]}}, + hidden +]}. + +%% @see bitcask.expiry.grace_time +{mapping, "multi_backend.$name.bitcask.expiry.grace_time", "riak_kv.multi_backend", [ + {datatype, {duration, s}}, + hidden, + {default, 0} +]}. + +%% @see bitcask.io_mode +{mapping, "multi_backend.$name.bitcask.io_mode", "riak_kv.multi_backend", [ + {default, erlang}, + {datatype, {enum, [erlang, nif]}}, + hidden +]}. diff --git a/test/bitcask_schema_tests.erl b/test/bitcask_schema_tests.erl index be5399c7..426b4a1b 100644 --- a/test/bitcask_schema_tests.erl +++ b/test/bitcask_schema_tests.erl @@ -113,10 +113,45 @@ override_schema_test() -> cuttlefish_unit:assert_not_configured(Config, "riak_kv.multi_backend"), ok. -%% this context() represents the substitution variables that rebar will use during the build process. -%% riak_core's schema file is written with some {{mustache_vars}} for substitution during packaging -%% cuttlefish doesn't have a great time parsing those, so we perform the substitutions first, because -%% that's how it would work in real life. +multi_backend_test() -> + Conf = [ + {["multi_backend", "default", "storage_backend"], bitcask}, + {["multi_backend", "default", "bitcask", "data_root"], "/data/default_bitcask"} + ], + %% The defaults are defined in ../priv/bitcask.schema. it is the file under test. + Config = cuttlefish_unit:generate_templated_config( + ["../priv/bitcask.schema", "../priv/bitcask_multi.schema", "../test/multi_backend.schema"], + Conf, context(), predefined_schema()), + %%io:format("Config: ~p~n", []), + + MultiBackendConfig = proplists:get_value(multi_backend, proplists:get_value(riak_kv, Config)), + + {<<"default">>, riak_kv_bitcask_backend, DefaultBackend} = lists:keyfind(<<"default">>, 1, MultiBackendConfig), + + cuttlefish_unit:assert_config(DefaultBackend, "data_root", "/data/default_bitcask"), + cuttlefish_unit:assert_config(DefaultBackend, "open_timeout", 4), + cuttlefish_unit:assert_config(DefaultBackend, "sync_strategy", none), + cuttlefish_unit:assert_config(DefaultBackend, "max_file_size", 2147483648), + cuttlefish_unit:assert_config(DefaultBackend, "merge_window", always), + cuttlefish_unit:assert_config(DefaultBackend, "frag_merge_trigger", 60), + cuttlefish_unit:assert_config(DefaultBackend, "dead_bytes_merge_trigger", 536870912), + cuttlefish_unit:assert_config(DefaultBackend, "frag_threshold", 40), + cuttlefish_unit:assert_config(DefaultBackend, "dead_bytes_threshold", 134217728), + cuttlefish_unit:assert_config(DefaultBackend, "small_file_threshold", 10485760), + cuttlefish_unit:assert_config(DefaultBackend, "max_fold_age", -1), + cuttlefish_unit:assert_config(DefaultBackend, "max_fold_puts", 0), + cuttlefish_unit:assert_config(DefaultBackend, "expiry_secs", -1), + cuttlefish_unit:assert_config(DefaultBackend, "require_hint_crc", true), + cuttlefish_unit:assert_config(DefaultBackend, "expiry_grace_time", 0), + cuttlefish_unit:assert_config(DefaultBackend, "io_mode", erlang), + ok. + +%% this context() represents the substitution variables that rebar +%% will use during the build process. riak_core's schema file is +%% written with some {{mustache_vars}} for substitution during +%% packaging cuttlefish doesn't have a great time parsing those, so we +%% perform the substitutions first, because that's how it would work +%% in real life. context() -> []. %% This predefined schema covers riak_kv's dependency on diff --git a/test/multi_backend.schema b/test/multi_backend.schema new file mode 100644 index 00000000..04257c9d --- /dev/null +++ b/test/multi_backend.schema @@ -0,0 +1,39 @@ +%% -*- erlang -*- + +%% Yo, this is just a test backend for bitcask's multi_backend schema. It's not real +%% If you really care about multi_backend, have a look at riak_kv/priv/multi_backend.schema + +%% @doc Storage_backend specifies the Erlang module defining the storage +%% mechanism that will be used on this node. +{mapping, "multi_backend.$name.storage_backend", "riak_kv.multi_backend", [ + {default, bitcask}, + {datatype, {enum, [bitcask, leveldb, memory]}}, + hidden +]}. + +{translation, "riak_kv.multi_backend", + fun(Conf, Schema) -> + %% group by $name into list, also cut the "multi_backend.$name" off every key + BackendNames = cuttlefish_variable:fuzzy_matches(["multi_backend","$name","storage_backend"], Conf), + %% for each in list, case statement on backend type + Backends = [ begin + BackendConfigName = ["multi_backend", Name], + {BackendModule, BackendConfig} = case cuttlefish:conf_get(BackendConfigName ++ ["storage_backend"], Conf) of + bitcask -> + BackendConfigPrefix = BackendConfigName ++ ["bitcask"], + SubConf = [ begin + {Key -- BackendConfigName, Value} + end || {Key, Value} <- cuttlefish_variable:filter_by_prefix(BackendConfigPrefix, Conf)], + + case cuttlefish_generator:map(Schema, SubConf) of + BackendProplist -> + {riak_kv_bitcask_backend, proplists:get_value(bitcask, BackendProplist)} + end + end, + {list_to_binary(Name), BackendModule, BackendConfig} + end || {"$name", Name} <- BackendNames], + case Backends of + [] -> throw(unset); + _ -> Backends + end + end}.