Skip to content

Commit

Permalink
Merge pull request #157 from basho/bugfix/jd/merge-small-file-multi-s…
Browse files Browse the repository at this point in the history 
…chema

Fixed an issue with some multi_backend versions not matching regular versions
  • Loading branch information
@joedevivo
joedevivo committed Mar 27, 2014
2 parents 755afed + 55fbfff commit bae0260
Show file tree
Hide file tree
Showing 4 changed files with 244 additions and 163 deletions.
188 changes: 29 additions & 159 deletions priv/bitcask.schema
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,6 @@
{datatype, directory}
]}.

%% @see bitcask.data_root
{mapping, "multi_backend.$name.bitcask.data_root", "riak_kv.multi_backend", [
hidden,
{datatype, directory}
]}.

%% @doc Specifies the maximum time Bitcask will block on startup while
%% attempting to create or open the data directory. You generally need
%% not change this value. If for some reason the timeout is exceeded
Expand All @@ -26,13 +20,6 @@
hidden
]}.

%% @see bitcask.open_timeout
{mapping, "multi_backend.$name.bitcask.open_timeout", "riak_kv.multi_backend", [
{default, "4s"},
{datatype, {duration, s}},
hidden
]}.

%% @doc Changes the durability of writes by specifying when to
%% synchronize data to disk. The default setting protects against data
%% loss in the event of application failure (process death) but leaves
Expand Down Expand Up @@ -65,25 +52,12 @@
hidden
]}.

%% @see bitcask.sync.strategy
{mapping, "multi_backend.$name.bitcask.sync.strategy", "riak_kv.multi_backend", [
{default, none},
{datatype, {enum, [none, o_sync, interval]}},
hidden
]}.

%% @see bitcask.sync.strategy
{mapping, "bitcask.sync.interval", "bitcask.sync_strategy", [
{datatype, {duration, s}},
hidden
]}.

%% @see bitcask.sync.strategy
{mapping, "multi_backend.$name.bitcask.sync.interval", "riak_kv.multi_backend", [
{datatype, {duration, s}},
hidden
]}.

{translation,
"bitcask.sync_strategy",
fun(Conf) ->
Expand All @@ -92,8 +66,8 @@
none -> none;
o_sync -> o_sync;
interval ->
Interval = cuttlefish:conf_get("bitcask.sync.interval", Conf, undefined),
{seconds, Interval};
Interval = cuttlefish:conf_get("bitcask.sync.interval", Conf, undefined),
{seconds, Interval};
_Default -> none
end
end}.
Expand All @@ -108,13 +82,6 @@
hidden
]}.

%% @see bitcask.max_file_size
{mapping, "multi_backend.$name.bitcask.max_file_size", "riak_kv.multi_backend", [
{default, "2GB"},
{datatype, bytesize},
hidden
]}.

%% @doc Lets you specify when during the day merge operations are
%% allowed to be triggered. Valid options are:
%%
Expand All @@ -133,41 +100,20 @@
hidden
]}.

%% @see bitcask.merge.policy
{mapping, "multi_backend.$name.bitcask.merge.policy", "riak_kv.multi_backend", [
{default, always},
{datatype, {enum, [always, never, window]}},
hidden
]}.

%% @see bitcask.merge.policy
{mapping, "bitcask.merge.window.start", "bitcask.merge_window", [
{default, 0},
{datatype, integer},
hidden
]}.

%% @see bitcask.merge.policy
{mapping, "multi_backend.$name.bitcask.merge.window.start", "riak_kv.multi_backend", [
{default, 0},
{datatype, integer},
hidden
]}.

%% @see bitcask.merge.policy
{mapping, "bitcask.merge.window.end", "bitcask.merge_window", [
{default, 23},
{datatype, integer},
hidden
]}.

%% @see bitcask.merge.policy
{mapping, "multi_backend.$name.bitcask.merge.window.end", "riak_kv.multi_backend", [
{default, 23},
{datatype, integer},
hidden
]}.

{translation,
"bitcask.merge_window",
fun(Conf) ->
Expand All @@ -176,9 +122,9 @@
always -> always;
never -> never;
window ->
Start = cuttlefish:conf_get("bitcask.merge.window.start", Conf, undefined),
End = cuttlefish:conf_get("bitcask.merge.window.end", Conf, undefined),
{Start, End};
Start = cuttlefish:conf_get("bitcask.merge.window.start", Conf, undefined),
End = cuttlefish:conf_get("bitcask.merge.window.end", Conf, undefined),
{Start, End};
_Default -> always
end
end}.
Expand All @@ -201,19 +147,11 @@
{validators, ["is_percentage"]}
]}.

%% @see bitcask.merge.triggers.fragmentation
{mapping, "multi_backend.$name.bitcask.merge.triggers.fragmentation", "riak_kv.multi_backend", [
{datatype, integer},
hidden,
{default, 60},
{validators, ["is_percentage"]}
]}.

{validator,
"is_percentage",
"must be a percentage",
fun(Value) ->
Value >= 0 andalso Value =< 100
Value >= 0 andalso Value =< 100
end}.

%% @doc Describes how much data stored for dead keys in a single file
Expand All @@ -227,20 +165,14 @@
%% directory, Bitcask will attempt to merge files.
%%
%% Default is: 512MB
{mapping, "bitcask.merge.triggers.dead_bytes",
"bitcask.dead_bytes_merge_trigger",
[
{mapping,
"bitcask.merge.triggers.dead_bytes",
"bitcask.dead_bytes_merge_trigger", [
{datatype, bytesize},
hidden,
{default, "512MB"}
]}.

%% @see bitcask.merge.triggers.dead_bytes
{mapping, "multi_backend.$name.bitcask.merge.triggers.dead_bytes", "riak_kv.multi_backend", [
{datatype, bytesize},
hidden,
{default, "512MB"}
]}.

%% @doc Describes what ratio of dead keys to total keys in a file will
%% cause it to be included in the merge. The value of this setting is
Expand All @@ -250,17 +182,9 @@
%% merged, decreasing the value will cause more files to be merged.
%%
%% Default is: `40`
{mapping, "bitcask.merge.thresholds.fragmentation",
"bitcask.frag_threshold",
[
{datatype, integer},
hidden,
{default, 40},
{validators, ["is_percentage"]}
]}.

%% @see bitcask.merge.thresholds.fragmentation
{mapping, "multi_backend.$name.bitcask.thresholds.fragmentation", "riak_kv.multi_backend", [
{mapping,
"bitcask.merge.thresholds.fragmentation",
"bitcask.frag_threshold", [
{datatype, integer},
hidden,
{default, 40},
Expand All @@ -273,40 +197,28 @@
%% will cause more files to be merged.
%%
%% Default is: 128MB
{mapping, "bitcask.merge.thresholds.dead_bytes",
{mapping,
"bitcask.merge.thresholds.dead_bytes",
"bitcask.dead_bytes_threshold", [
{datatype, bytesize},
hidden,
{default, "128MB"}
]}.

%% @see bitcask.merge.thresholds.dead_bytes
{mapping, "multi_backend.$name.bitcask.thresholds.dead_bytes", "riak_kv.multi_backend", [
{datatype, bytesize},
hidden,
{default, "128MB"}
]}.

%% @doc Describes the minimum size a file must have to be _excluded_
%% from the merge. Files smaller than the threshold will be
%% included. Increasing the value will cause _more_ files to be
%% merged, decreasing the value will cause _fewer_ files to be merged.
%%
%% Default is: 10MB
{mapping, "bitcask.merge.thresholds.small_file",
{mapping,
"bitcask.merge.thresholds.small_file",
"bitcask.small_file_threshold", [
{datatype, bytesize},
hidden,
{default, "10MB"}
]}.

%% @see bitcask.merge.thresholds.small_file
{mapping, "multi_backend.$name.bitcask.thresholds.small_file", "riak_kv.multi_backend", [
{datatype, bytesize},
hidden,
{default, "10MB"}
]}.

%% @doc Fold keys thresholds will reuse the keydir if another fold was
%% started less than `fold.max_age` ago and there were less than
%% `fold.max_puts` updates. Otherwise it will wait until all current
Expand All @@ -318,22 +230,15 @@
{default, unlimited}
]}.

%% @see bitcask.fold.max_age
{mapping, "multi_backend.$name.bitcask.fold.max_age", "riak_kv.multi_backend", [
{datatype, [{atom, unlimited}, {duration, ms}]},
hidden,
{default, unlimited}
]}.

{translation, "bitcask.max_fold_age",
fun(Conf) ->
case cuttlefish:conf_get("bitcask.fold.max_age", Conf) of
unlimited -> -1;
I when is_integer(I) ->
%% app.config expects microseconds
I * 1000;
_ -> -1 %% The default, for safety
end
case cuttlefish:conf_get("bitcask.fold.max_age", Conf) of
unlimited -> -1;
I when is_integer(I) ->
%% app.config expects microseconds
I * 1000;
_ -> -1 %% The default, for safety
end
end
}.

Expand All @@ -344,19 +249,12 @@
{default, 0}
]}.

%% @see bitcask.fold.max_age
{mapping, "multi_backend.$name.bitcask.fold.max_puts", "riak_kv.multi_backend", [
{datatype, [integer, {atom, unlimited}]},
hidden,
{default, 0}
]}.

{translation, "bitcask.max_fold_puts",
fun(Conf) ->
case cuttlefish:conf_get("bitcask.fold.max_puts", Conf) of
unlimited -> -1;
I when is_integer(I) -> I;
_ -> 0 %% default catch
unlimited -> -1;
I when is_integer(I) -> I;
_ -> 0 %% default catch
end
end
}.
Expand All @@ -373,13 +271,6 @@
{default, off}
]}.

%% @see bitcask.expiry
{mapping, "multi_backend.$name.bitcask.expiry", "riak_kv.multi_backend", [
{datatype, [{atom, off}, {duration, s}]},
hidden,
{default, off}
]}.

{translation, "bitcask.expiry_secs",
fun(Conf) ->
case cuttlefish:conf_get("bitcask.expiry", Conf) of
Expand All @@ -400,19 +291,12 @@
hidden
]}.

%% @see bitcask.hintfile_checksums
{mapping, "multi_backend.$name.bitcask.hintfile_checksums", "riak_kv.multi_backend", [
{default, strict},
{datatype, {enum, [strict, allow_missing]}},
hidden
]}.

{translation, "bitcask.require_hint_crc",
fun(Conf) ->
case cuttlefish:conf_get("bitcask.hintfile_checksums", Conf) of
strict -> true;
allow_missing -> false;
_ -> true
strict -> true;
allow_missing -> false;
_ -> true
end
end}.

Expand All @@ -431,13 +315,6 @@
{default, 0}
]}.

%% @see bitcask.expiry.grace_time
{mapping, "multi_backend.$name.bitcask.expiry.grace_time", "riak_kv.multi_backend", [
{datatype, {duration, s}},
hidden,
{default, 0}
]}.

%% @doc Configure how Bitcask writes data to disk.
%% erlang: Erlang's built-in file API
%% nif: Direct calls to the POSIX C API
Expand All @@ -450,10 +327,3 @@
{default, erlang},
{datatype, {enum, [erlang, nif]}}
]}.

%% @see bitcask.io_mode
{mapping, "multi_backend.$name.bitcask.io_mode", "riak_kv.multi_backend", [
{default, erlang},
{datatype, {enum, [erlang, nif]}},
hidden
]}.
Loading

0 comments on commit bae0260

Please sign in to comment.