diff --git a/src/lib/merkle_ledger/any_ledger.ml b/src/lib/merkle_ledger/any_ledger.ml index 147fa13e03a..e74ac391d90 100644 --- a/src/lib/merkle_ledger/any_ledger.ml +++ b/src/lib/merkle_ledger/any_ledger.ml @@ -13,55 +13,8 @@ * Props to @nholland for showing me this trick. * *) -open Core_kernel - -module type S = sig - type key - - type token_id - - type token_id_set - - type account_id - - type account_id_set - - type account - - type hash - - module Location : Location_intf.S - - (** The type of the witness for a base ledger exposed here so that it can - * be easily accessed from outside this module *) - type witness [@@deriving sexp_of] - - module type Base_intf = - Base_ledger_intf.S - with module Addr = Location.Addr - with module Location = Location - with type key := key - and type token_id := token_id - and type token_id_set := token_id_set - and type account_id := account_id - and type account_id_set := account_id_set - and type hash := hash - and type root_hash := hash - and type account := account - - val cast : (module Base_intf with type t = 'a) -> 'a -> witness - - module M : Base_intf with type t = witness -end - -module type Inputs_intf = sig - include Base_inputs_intf.S - - module Location : Location_intf.S -end - -module Make_base (Inputs : Inputs_intf) : - S +module Make_base (Inputs : Intf.Inputs.Intf) : + Intf.Ledger.ANY with module Location = Inputs.Location with type key := Inputs.Key.t and type token_id := Inputs.Token_id.t @@ -74,7 +27,7 @@ module Make_base (Inputs : Inputs_intf) : module Location = Location module type Base_intf = - Base_ledger_intf.S + Intf.Ledger.S with module Addr = Location.Addr with module Location = Location with type key := Inputs.Key.t diff --git a/src/lib/merkle_ledger/any_ledger.mli b/src/lib/merkle_ledger/any_ledger.mli new file mode 100644 index 00000000000..8a0c50a527a --- /dev/null +++ b/src/lib/merkle_ledger/any_ledger.mli @@ -0,0 +1,10 @@ +module Make_base (Inputs : Intf.Inputs.Intf) : + Intf.Ledger.ANY + with module Location = Inputs.Location + with type key := Inputs.Key.t + and type token_id := Inputs.Token_id.t + and type token_id_set := Inputs.Token_id.Set.t + and type account_id := Inputs.Account_id.t + and type hash := Inputs.Hash.t + and type account_id_set := Inputs.Account_id.Set.t + and type account := Inputs.Account.t diff --git a/src/lib/merkle_ledger/base_inputs_intf.ml b/src/lib/merkle_ledger/base_inputs_intf.ml deleted file mode 100644 index cda6d078c74..00000000000 --- a/src/lib/merkle_ledger/base_inputs_intf.ml +++ /dev/null @@ -1,18 +0,0 @@ -module type S = sig - module Key : Intf.Key - - module Token_id : Intf.Token_id - - module Account_id : - Intf.Account_id with type key := Key.t and type token_id := Token_id.t - - module Balance : Intf.Balance - - module Account : - Intf.Account - with type token_id := Token_id.t - and type account_id := Account_id.t - and type balance := Balance.t - - module Hash : Intf.Hash with type account := Account.t -end diff --git a/src/lib/merkle_ledger/base_ledger_intf.ml b/src/lib/merkle_ledger/base_ledger_intf.ml deleted file mode 100644 index 74c13db4376..00000000000 --- a/src/lib/merkle_ledger/base_ledger_intf.ml +++ /dev/null @@ -1,150 +0,0 @@ -open Core - -module type S = sig - (** a Merkle hash associated with the root node *) - type root_hash - - (** a Merkle hash associated any non-root node *) - type hash - - type account - - type key - - type token_id - - type token_id_set - - type account_id - - type account_id_set - - type index = int - - (** no deriving, purposely; signatures that include this one may add deriving *) - type t - - module Addr : module type of Merkle_address - - module Path : Merkle_path.S with type hash := hash - - module Location : sig - type t [@@deriving sexp, compare, hash] - - include Comparable.S with type t := t - end - - include - Syncable_intf.S - with type root_hash := root_hash - and type hash := hash - and type account := account - and type addr := Addr.t - and type path = Path.t - and type t := t - - (** list of accounts in the ledger *) - val to_list : t -> account list Async.Deferred.t - - (** list of accounts via slower sequential mechanism *) - val to_list_sequential : t -> account list - - (** iterate over all indexes and accounts *) - val iteri : t -> f:(index -> account -> unit) -> unit - - (** fold over accounts in the ledger, passing the Merkle address *) - val foldi : - t -> init:'accum -> f:(Addr.t -> 'accum -> account -> 'accum) -> 'accum - - (** the set of [account_id]s are ledger elements to skip during the fold, - because they're in a mask - *) - val foldi_with_ignored_accounts : - t - -> account_id_set - -> init:'accum - -> f:(Addr.t -> 'accum -> account -> 'accum) - -> 'accum - - (** fold over accounts until stop condition reached when calling [f]; calls [finish] for - result - *) - val fold_until : - t - -> init:'accum - -> f:('accum -> account -> ('accum, 'stop) Base.Continue_or_stop.t) - -> finish:('accum -> 'stop) - -> 'stop Async.Deferred.t - - (** set of account ids associated with accounts *) - val accounts : t -> account_id_set Async.Deferred.t - - (** Get the account id that owns a token. *) - val token_owner : t -> token_id -> account_id option - - (** Get the set of all accounts which own a token. *) - val token_owners : t -> account_id_set - - (** Get all of the tokens for which a public key has accounts. *) - val tokens : t -> key -> token_id_set - - val location_of_account : t -> account_id -> Location.t option - - val location_of_account_batch : - t -> account_id list -> (account_id * Location.t option) list - - (** This may return an error if the ledger is full. *) - val get_or_create_account : - t - -> account_id - -> account - -> ([ `Added | `Existed ] * Location.t) Or_error.t - - (** the ledger should not be used after calling [close] *) - val close : t -> unit - - (** for account locations in the ledger, the last (rightmost) filled location *) - val last_filled : t -> Location.t option - - val get_uuid : t -> Uuid.t - - (** return Some [directory] for ledgers that use a file system, else None *) - val get_directory : t -> string option - - val get : t -> Location.t -> account option - - val get_batch : t -> Location.t list -> (Location.t * account option) list - - val set : t -> Location.t -> account -> unit - - val set_batch : t -> (Location.t * account) list -> unit - - val get_at_index_exn : t -> int -> account - - val set_at_index_exn : t -> int -> account -> unit - - val index_of_account_exn : t -> account_id -> int - - (** meant to be a fast operation: the root hash is stored, rather - than calculated dynamically - *) - val merkle_root : t -> root_hash - - val merkle_path : t -> Location.t -> Path.t - - val merkle_path_at_index_exn : t -> int -> Path.t - - val merkle_path_batch : t -> Location.t list -> Path.t list - - val wide_merkle_path_batch : - t - -> Location.t list - -> [ `Left of hash * hash | `Right of hash * hash ] list list - - val get_hash_batch_exn : t -> Location.t list -> hash list - - (** Triggers when the ledger has been detached and should no longer be - accessed. - *) - val detached_signal : t -> unit Async_kernel.Deferred.t -end diff --git a/src/lib/merkle_ledger/database.ml b/src/lib/merkle_ledger/database.ml index 863fcbcdf54..7488725511c 100644 --- a/src/lib/merkle_ledger/database.ml +++ b/src/lib/merkle_ledger/database.ml @@ -1,33 +1,10 @@ -open Core - -module type Inputs_intf = sig - include Base_inputs_intf.S - - module Location : Location_intf.S - - module Location_binable : Hashable.S_binable with type t := Location.t - - module Kvdb : Intf.Key_value_database with type config := string - - module Storage_locations : Intf.Storage_locations -end - -module Make (Inputs : Inputs_intf) : - Database_intf.S - with module Location = Inputs.Location - and module Addr = Inputs.Location.Addr - and type key := Inputs.Key.t - and type token_id := Inputs.Token_id.t - and type token_id_set := Inputs.Token_id.Set.t - and type account := Inputs.Account.t - and type root_hash := Inputs.Hash.t - and type hash := Inputs.Hash.t - and type account_id := Inputs.Account_id.t - and type account_id_set := Inputs.Account_id.Set.t = struct +module Make (Inputs : Intf.Inputs.DATABASE) = struct (* The max depth of a merkle tree can never be greater than 253. *) open Inputs module Db_error = struct + [@@@warning "-4"] (* due to deriving sexp below *) + type t = Account_location_not_found | Out_of_leaves | Malformed_database [@@deriving sexp] end @@ -67,6 +44,8 @@ module Make (Inputs : Inputs_intf) : let depth t = t.depth let create ?directory_name ~depth () = + let open Core in + (* for ^/ and Unix below *) assert (depth < 0xfe) ; let uuid = Uuid_unix.create () in let directory = @@ -290,25 +269,13 @@ module Make (Inputs : Inputs_intf) : Result.map location_result ~f:(fun location -> set mdb key location ; location ) - let last_location_address mdb = - match - last_location_key () |> get_raw mdb |> Result.of_option ~error:() - |> Result.bind ~f:(Location.parse ~ledger_depth:mdb.depth) - with - | Error () -> - None - | Ok parsed_location -> - Some (Location.to_path_exn parsed_location) - let last_location mdb = - match - last_location_key () |> get_raw mdb |> Result.of_option ~error:() - |> Result.bind ~f:(Location.parse ~ledger_depth:mdb.depth) - with - | Error () -> - None - | Ok parsed_location -> - Some parsed_location + last_location_key () |> get_raw mdb + |> Option.bind ~f:(fun data -> + Location.parse ~ledger_depth:mdb.depth data |> Result.ok ) + + let last_location_address mdb = + Option.map (last_location mdb) ~f:Location.to_path_exn end let get_at_index_exn mdb index = @@ -590,7 +557,7 @@ module Make (Inputs : Inputs_intf) : let get_or_create_account mdb account_id account = match Account_location.get mdb account_id with - | Error Account_location_not_found -> ( + | Error Db_error.Account_location_not_found -> ( match Account_location.allocate mdb account_id with | Ok location -> set mdb location account ; @@ -599,7 +566,7 @@ module Make (Inputs : Inputs_intf) : | Error err -> Error (Error.create "get_or_create_account" err Db_error.sexp_of_t) ) - | Error err -> + | Error ((Db_error.Malformed_database | Db_error.Out_of_leaves) as err) -> Error (Error.create "get_or_create_account" err Db_error.sexp_of_t) | Ok location -> Ok (`Existed, location) diff --git a/src/lib/merkle_ledger/database.mli b/src/lib/merkle_ledger/database.mli new file mode 100644 index 00000000000..b399d3e09f6 --- /dev/null +++ b/src/lib/merkle_ledger/database.mli @@ -0,0 +1,12 @@ +module Make (Inputs : Intf.Inputs.DATABASE) : + Intf.Ledger.DATABASE + with module Location = Inputs.Location + and module Addr = Inputs.Location.Addr + and type key := Inputs.Key.t + and type token_id := Inputs.Token_id.t + and type token_id_set := Inputs.Token_id.Set.t + and type account := Inputs.Account.t + and type root_hash := Inputs.Hash.t + and type hash := Inputs.Hash.t + and type account_id := Inputs.Account_id.t + and type account_id_set := Inputs.Account_id.Set.t diff --git a/src/lib/merkle_ledger/database_intf.ml b/src/lib/merkle_ledger/database_intf.ml deleted file mode 100644 index 40f40195b40..00000000000 --- a/src/lib/merkle_ledger/database_intf.ml +++ /dev/null @@ -1,18 +0,0 @@ -module type S = sig - include Base_ledger_intf.S - - val create : ?directory_name:string -> depth:int -> unit -> t - - (** create_checkpoint would create the checkpoint and open a db connection to that checkpoint *) - val create_checkpoint : t -> directory_name:string -> unit -> t - - (** make_checkpoint would only create the checkpoint *) - val make_checkpoint : t -> directory_name:string -> unit - - val with_ledger : depth:int -> f:(t -> 'a) -> 'a - - module For_tests : sig - val gen_account_location : - ledger_depth:int -> Location.t Core.Quickcheck.Generator.t - end -end diff --git a/src/lib/merkle_ledger/dune b/src/lib/merkle_ledger/dune index 70339fc26c5..c85fcfa76c4 100644 --- a/src/lib/merkle_ledger/dune +++ b/src/lib/merkle_ledger/dune @@ -1,36 +1,49 @@ (library (name merkle_ledger) (public_name merkle_ledger) + (flags + ; Deactivated warnings + ; 40: name-out-scope (activate later) + ; + ; 41: ambiguous name (too many of them for now, activate later) + ; + ; 42: disambiguated-name (rely on type disambiguation ,not too bad but closer + ; module openings may both solve the warning *and* help the reader) + ; + ; 44: open-shadow-identifier (operation overloading is common in the codebase) + (:standard -w +a-40..42-44 -warn-error +a-70 -open Core_kernel)) (library_flags -linkall) + (modules_without_implementation + location_intf) (libraries ;; opam libraries - bitstring - async_kernel - core_kernel.uuid - bin_prot.shape - sexplib0 - integers - core.uuid - async - core - extlib - rocks - core_kernel - base.caml - base.base_internalhash_types - async_unix - ;; local libraries - merkle_address - immutable_array - direction - cache_dir - empty_hashes - key_value_database - mina_stdlib - visualization - ppx_version.runtime - bounded_types - ) + async + async_kernel + async_unix + base.base_internalhash_types + base.caml + bin_prot.shape + bitstring + core + core.uuid + core_kernel + core_kernel.uuid + extlib + integers + rocks + sexplib0 + ;; local libraries + bounded_types + cache_dir + direction + empty_hashes + immutable_array + key_value_database + merkle_address + mina_stdlib + ppx_version.runtime + visualization + ) (preprocess (pps ppx_mina ppx_version ppx_jane ppx_compare ppx_deriving.show ppx_deriving_yojson)) (instrumentation (backend bisect_ppx)) diff --git a/src/lib/merkle_ledger/graphviz.ml b/src/lib/merkle_ledger/graphviz.ml index 95204555772..894514cf361 100644 --- a/src/lib/merkle_ledger/graphviz.ml +++ b/src/lib/merkle_ledger/graphviz.ml @@ -1,57 +1,9 @@ -open Core open Async -(** Visualizable_ledger shows a subgraph of a merkle_ledger using Graphviz *) -module type S = sig - type addr - - type ledger - - type t - - (* Visualize will enumerate through all edges of a subtree with a - initial_address. It will then interpret all of the edges and nodes into an - intermediate form that will be easy to write into a dot file *) - val visualize : ledger -> initial_address:addr -> t - - (* Write will transform the intermediate form generate by visualize and save - the results into a dot file *) - val write : path:string -> name:string -> t -> unit Deferred.t -end - -module type Inputs_intf = sig - module Key : Intf.Key - - module Token_id : Intf.Token_id - - module Account_id : - Intf.Account_id with type key := Key.t and type token_id := Token_id.t - - module Balance : Intf.Balance - - module Account : - Intf.Account - with type account_id := Account_id.t - and type balance := Balance.t - - module Hash : Intf.Hash with type account := Account.t - - module Location : Location_intf.S - - module Ledger : - Base_ledger_intf.S - with module Addr = Location.Addr - and module Location = Location - and type account_id := Account_id.t - and type account_id_set := Account_id.Set.t - and type hash := Hash.t - and type root_hash := Hash.t - and type account := Account.t -end - -module Make (Inputs : Inputs_intf) : - S with type addr := Inputs.Location.Addr.t and type ledger := Inputs.Ledger.t = -struct +module Make (Inputs : Intf.Graphviz.I) : + Intf.Graphviz.S + with type addr := Inputs.Location.Addr.t + and type ledger := Inputs.Ledger.t = struct open Inputs module Account = struct diff --git a/src/lib/merkle_ledger/graphviz.mli b/src/lib/merkle_ledger/graphviz.mli new file mode 100644 index 00000000000..86a52fa54c5 --- /dev/null +++ b/src/lib/merkle_ledger/graphviz.mli @@ -0,0 +1,6 @@ +(** Visualizable_ledger shows a subgraph of a merkle_ledger using Graphviz *) + +module Make (Inputs : Intf.Graphviz.I) : + Intf.Graphviz.S + with type addr := Inputs.Location.Addr.t + and type ledger := Inputs.Ledger.t diff --git a/src/lib/merkle_ledger/intf.ml b/src/lib/merkle_ledger/intf.ml index df9c2769ff3..39de10c4c5e 100644 --- a/src/lib/merkle_ledger/intf.ml +++ b/src/lib/merkle_ledger/intf.ml @@ -1,4 +1,53 @@ -open Core +module type LOCATION = sig + module Addr : module type of Merkle_address + + module Prefix : sig + val generic : Unsigned.UInt8.t + + val account : Unsigned.UInt8.t + + val hash : ledger_depth:int -> int -> Unsigned.UInt8.t + end + + type t = Generic of Bigstring.t | Account of Addr.t | Hash of Addr.t + [@@deriving sexp, hash, compare] + + val is_generic : t -> bool + + val is_account : t -> bool + + val is_hash : t -> bool + + val height : ledger_depth:int -> t -> int + + val root_hash : t + + val last_direction : Addr.t -> Direction.t + + val build_generic : Bigstring.t -> t + + val parse : ledger_depth:int -> Bigstring.t -> (t, unit) Result.t + + val prefix_bigstring : Unsigned.UInt8.t -> Bigstring.t -> Bigstring.t + + val to_path_exn : t -> Addr.t + + val serialize : ledger_depth:int -> t -> Bigstring.t + + val parent : t -> t + + val next : t -> t Option.t + + val prev : t -> t Option.t + + val sibling : t -> t + + val order_siblings : t -> 'a -> 'a -> 'a * 'a + + val merkle_path_dependencies_exn : t -> (t * Direction.t) list + + include Comparable.S with type t := t +end module type Key = sig type t [@@deriving sexp] @@ -133,11 +182,341 @@ module type Key_value_database = sig -> key_data_pairs:(Bigstring.t * Bigstring.t) list -> unit + (** An association list, sorted by key *) val to_alist : t -> (Bigstring.t * Bigstring.t) list - (* an association list, sorted by key *) + val foldi : + t + -> init:'a + -> f:(int -> 'a -> key:Bigstring.t -> data:Bigstring.t -> 'a) + -> 'a + + val fold_until : + t + -> init:'a + -> f: + ( 'a + -> key:Bigstring.t + -> data:Bigstring.t + -> ('a, 'b) Continue_or_stop.t ) + -> finish:('a -> 'b) + -> 'b end module type Storage_locations = sig val key_value_db_dir : string end + +module type SYNCABLE = sig + type root_hash + + type hash + + type account + + type addr + + type t [@@deriving sexp] + + type path + + val depth : t -> int + + val num_accounts : t -> int + + val merkle_path_at_addr_exn : t -> addr -> path + + val get_inner_hash_at_addr_exn : t -> addr -> hash + + val set_inner_hash_at_addr_exn : t -> addr -> hash -> unit + + val set_all_accounts_rooted_at_exn : t -> addr -> account list -> unit + + val set_batch_accounts : t -> (addr * account) list -> unit + + (** Get all of the accounts that are in a subtree of the underlying Merkle + tree rooted at `address`. The accounts are ordered by their addresses. *) + val get_all_accounts_rooted_at_exn : t -> addr -> (addr * account) list + + val merkle_root : t -> root_hash +end + +module Inputs = struct + module type Intf = sig + module Key : Key + + module Token_id : Token_id + + module Account_id : + Account_id with type key := Key.t and type token_id := Token_id.t + + module Balance : Balance + + module Account : + Account + with type token_id := Token_id.t + and type account_id := Account_id.t + and type balance := Balance.t + + module Hash : Hash with type account := Account.t + + module Location : LOCATION + end + + module type DATABASE = sig + include Intf + + module Location_binable : Hashable.S_binable with type t := Location.t + + module Kvdb : Key_value_database with type config := string + + module Storage_locations : Storage_locations + end +end + +module Ledger = struct + module type S = sig + (** a Merkle hash associated with the root node *) + type root_hash + + (** a Merkle hash associated any non-root node *) + type hash + + type account + + type key + + type token_id + + type token_id_set + + type account_id + + type account_id_set + + type index = int + + (** no deriving, purposely; signatures that include this one may add deriving *) + type t + + module Addr : module type of Merkle_address + + module Path : Merkle_path.S with type hash := hash + + module Location : sig + type t [@@deriving sexp, compare, hash] + + include Comparable.S with type t := t + end + + include + SYNCABLE + with type root_hash := root_hash + and type hash := hash + and type account := account + and type addr := Addr.t + and type path = Path.t + and type t := t + + (** list of accounts in the ledger *) + val to_list : t -> account list Async.Deferred.t + + (** list of accounts via slower sequential mechanism *) + val to_list_sequential : t -> account list + + (** iterate over all indexes and accounts *) + val iteri : t -> f:(index -> account -> unit) -> unit + + (** fold over accounts in the ledger, passing the Merkle address *) + val foldi : + t -> init:'accum -> f:(Addr.t -> 'accum -> account -> 'accum) -> 'accum + + (** the set of [account_id]s are ledger elements to skip during the fold, + because they're in a mask + *) + val foldi_with_ignored_accounts : + t + -> account_id_set + -> init:'accum + -> f:(Addr.t -> 'accum -> account -> 'accum) + -> 'accum + + (** fold over accounts until stop condition reached when calling [f]; calls [finish] for + result + *) + val fold_until : + t + -> init:'accum + -> f:('accum -> account -> ('accum, 'stop) Base.Continue_or_stop.t) + -> finish:('accum -> 'stop) + -> 'stop Async.Deferred.t + + (** set of account ids associated with accounts *) + val accounts : t -> account_id_set Async.Deferred.t + + (** Get the account id that owns a token. *) + val token_owner : t -> token_id -> account_id option + + (** Get the set of all accounts which own a token. *) + val token_owners : t -> account_id_set + + (** Get all of the tokens for which a public key has accounts. *) + val tokens : t -> key -> token_id_set + + val location_of_account : t -> account_id -> Location.t option + + val location_of_account_batch : + t -> account_id list -> (account_id * Location.t option) list + + (** This may return an error if the ledger is full. *) + val get_or_create_account : + t + -> account_id + -> account + -> ([ `Added | `Existed ] * Location.t) Or_error.t + + (** the ledger should not be used after calling [close] *) + val close : t -> unit + + (** for account locations in the ledger, the last (rightmost) filled location *) + val last_filled : t -> Location.t option + + val get_uuid : t -> Uuid.t + + (** return Some [directory] for ledgers that use a file system, else None *) + val get_directory : t -> string option + + val get : t -> Location.t -> account option + + val get_batch : t -> Location.t list -> (Location.t * account option) list + + val set : t -> Location.t -> account -> unit + + val set_batch : t -> (Location.t * account) list -> unit + + val get_at_index_exn : t -> int -> account + + val set_at_index_exn : t -> int -> account -> unit + + val index_of_account_exn : t -> account_id -> int + + (** meant to be a fast operation: the root hash is stored, rather + than calculated dynamically + *) + val merkle_root : t -> root_hash + + val merkle_path : t -> Location.t -> Path.t + + val merkle_path_at_index_exn : t -> int -> Path.t + + val merkle_path_batch : t -> Location.t list -> Path.t list + + val wide_merkle_path_batch : + t + -> Location.t list + -> [ `Left of hash * hash | `Right of hash * hash ] list list + + val get_hash_batch_exn : t -> Location.t list -> hash list + + (** Triggers when the ledger has been detached and should no longer be + accessed. + *) + val detached_signal : t -> unit Async_kernel.Deferred.t + end + + module type NULL = sig + include S + + val create : depth:int -> unit -> t + end + + module type ANY = sig + type key + + type token_id + + type token_id_set + + type account_id + + type account_id_set + + type account + + type hash + + module Location : LOCATION + + (** The type of the witness for a base ledger exposed here so that it can + * be easily accessed from outside this module *) + type witness [@@deriving sexp_of] + + module type Base_intf = + S + with module Addr = Location.Addr + with module Location = Location + with type key := key + and type token_id := token_id + and type token_id_set := token_id_set + and type account_id := account_id + and type account_id_set := account_id_set + and type hash := hash + and type root_hash := hash + and type account := account + + val cast : (module Base_intf with type t = 'a) -> 'a -> witness + + module M : Base_intf with type t = witness + end + + module type DATABASE = sig + include S + + val create : ?directory_name:string -> depth:int -> unit -> t + + (** create_checkpoint would create the checkpoint and open a db connection to that checkpoint *) + val create_checkpoint : t -> directory_name:string -> unit -> t + + (** make_checkpoint would only create the checkpoint *) + val make_checkpoint : t -> directory_name:string -> unit + + val with_ledger : depth:int -> f:(t -> 'a) -> 'a + + module For_tests : sig + val gen_account_location : + ledger_depth:int -> Location.t Quickcheck.Generator.t + end + end +end + +module Graphviz = struct + module type S = sig + type addr + + type ledger + + type t + + (* Visualize will enumerate through all edges of a subtree with a + initial_address. It will then interpret all of the edges and nodes into an + intermediate form that will be easy to write into a dot file *) + val visualize : ledger -> initial_address:addr -> t + + (* Write will transform the intermediate form generate by visualize and save + the results into a dot file *) + val write : path:string -> name:string -> t -> unit Async.Deferred.t + end + + module type I = sig + include Inputs.Intf + + module Ledger : + Ledger.S + with module Addr = Location.Addr + and module Location = Location + and type account_id := Account_id.t + and type account_id_set := Account_id.Set.t + and type hash := Hash.t + and type account := Account.t + end +end diff --git a/src/lib/merkle_ledger/ledger_extras_intf.ml b/src/lib/merkle_ledger/ledger_extras_intf.ml deleted file mode 100644 index 9f2bb5bd930..00000000000 --- a/src/lib/merkle_ledger/ledger_extras_intf.ml +++ /dev/null @@ -1,15 +0,0 @@ -(* ledger_extras_intf.ml -- adds functionality to Base_ledger_intf.S *) - -module type S = sig - include Merkle_ledger_intf.S - - val with_ledger : f:(t -> 'a) -> 'a - - val set_at_addr_exn : t -> Addr.t -> account -> unit - - val account_id_of_index : t -> index -> account_id option - - val account_id_of_index_exn : t -> index -> account_id - - val recompute_tree : t -> unit -end diff --git a/src/lib/merkle_ledger/location.ml b/src/lib/merkle_ledger/location.ml index ee8e1c61e33..cee243a14f4 100644 --- a/src/lib/merkle_ledger/location.ml +++ b/src/lib/merkle_ledger/location.ml @@ -1,4 +1,3 @@ -open Core open Unsigned (* add functions to library module Bigstring so we can derive hash for the type t below *) @@ -6,12 +5,10 @@ module Bigstring = struct [%%versioned_binable module Stable = struct module V1 = struct - type t = Core_kernel.Bigstring.Stable.V1.t [@@deriving sexp, compare] + type t = Bigstring.Stable.V1.t [@@deriving sexp, compare] let to_latest = Fn.id - let equal = Bigstring.equal - let hash t = Bigstring.to_string t |> String.hash let hash_fold_t hash_state t = @@ -20,15 +17,14 @@ module Bigstring = struct include Bounded_types.String.Of_stringable (struct type nonrec t = t - let of_string s = Core_kernel.Bigstring.of_string s + let of_string s = Bigstring.of_string s - let to_string s = Core_kernel.Bigstring.to_string s + let to_string s = Bigstring.to_string s end) end end] - [%%define_locally - Bigstring.(get, length, equal, create, to_string, set, blit, sub)] + [%%define_locally Bigstring.(get, length, create, to_string, set, blit, sub)] include Hashable.Make (Stable.Latest) end @@ -53,14 +49,18 @@ module T = struct let hash ~ledger_depth depth = UInt8.of_int (ledger_depth - depth) end + [@@@warning "-4"] (* disabled because of deriving sexp *) + type t = Generic of Bigstring.t | Account of Addr.t | Hash of Addr.t [@@deriving hash, sexp, compare] - let is_generic = function Generic _ -> true | _ -> false + [@@@warning "+4"] + + let is_generic = function Generic _ -> true | Account _ | Hash _ -> false - let is_account = function Account _ -> true | _ -> false + let is_account = function Account _ -> true | Generic _ | Hash _ -> false - let is_hash = function Hash _ -> true | _ -> false + let is_hash = function Hash _ -> true | Account _ | Generic _ -> false let height ~ledger_depth : t -> int = function | Generic _ -> @@ -172,7 +172,7 @@ module T = struct match location with | Hash addr -> loop addr - | _ -> + | Account _ | Generic _ -> failwith "can only get merkle path dependencies of a hash location" type location = t [@@deriving sexp, compare] diff --git a/src/lib/merkle_ledger/location.mli b/src/lib/merkle_ledger/location.mli new file mode 100644 index 00000000000..7fc11b37d21 --- /dev/null +++ b/src/lib/merkle_ledger/location.mli @@ -0,0 +1,18 @@ +module Bigstring : sig + [%%versioned: + module Stable : sig + module V1 : sig + type t = Bigstring.Stable.V1.t [@@deriving sexp, compare] + + include Binable.S with type t := t + + val hash_fold_t : Hash.state -> t -> Hash.state + + val hash : t -> Hash.hash_value + end + end] + + include Hashable.S with type t := t +end + +module T : Location_intf.S diff --git a/src/lib/merkle_ledger/location_intf.ml b/src/lib/merkle_ledger/location_intf.mli similarity index 99% rename from src/lib/merkle_ledger/location_intf.ml rename to src/lib/merkle_ledger/location_intf.mli index 15cbe61d40e..19fb518e2e7 100644 --- a/src/lib/merkle_ledger/location_intf.ml +++ b/src/lib/merkle_ledger/location_intf.mli @@ -1,7 +1,5 @@ (* location_intf.ml -- interface file for Location *) -open Core - module type S = sig module Addr : module type of Merkle_address diff --git a/src/lib/merkle_ledger/merkle_ledger_intf.ml b/src/lib/merkle_ledger/merkle_ledger_intf.ml deleted file mode 100644 index 7b85045befb..00000000000 --- a/src/lib/merkle_ledger/merkle_ledger_intf.ml +++ /dev/null @@ -1,5 +0,0 @@ -module type S = sig - type t [@@deriving bin_io, sexp] - - include Base_ledger_intf.S with type t := t -end diff --git a/src/lib/merkle_ledger/merkle_path.ml b/src/lib/merkle_ledger/merkle_path.ml index a2a5acd294d..c837c8a9149 100644 --- a/src/lib/merkle_ledger/merkle_path.ml +++ b/src/lib/merkle_ledger/merkle_path.ml @@ -1,5 +1,3 @@ -open Core_kernel - module type S = sig type hash diff --git a/src/lib/merkle_ledger/merkle_path.mli b/src/lib/merkle_ledger/merkle_path.mli new file mode 100644 index 00000000000..158b5489511 --- /dev/null +++ b/src/lib/merkle_ledger/merkle_path.mli @@ -0,0 +1,23 @@ +module type S = sig + type hash + + type elem = [ `Left of hash | `Right of hash ] [@@deriving sexp, equal] + + val elem_hash : elem -> hash + + type t = elem list [@@deriving sexp, equal] + + val implied_root : t -> hash -> hash + + (** [check_path path leaf_hash root_hash] is used in tests to check that + [leaf_hash] along with [path] actually corresponds to [root_hash]. *) + val check_path : t -> hash -> hash -> bool +end + +module Make (Hash : sig + type t [@@deriving sexp, equal] + + val merge : height:int -> t -> t -> t + + val equal : t -> t -> bool +end) : S with type hash := Hash.t diff --git a/src/lib/merkle_ledger/merkle_path_intf.ml b/src/lib/merkle_ledger/merkle_path_intf.ml deleted file mode 100644 index 4627b1ddd3d..00000000000 --- a/src/lib/merkle_ledger/merkle_path_intf.ml +++ /dev/null @@ -1,11 +0,0 @@ -module type S = sig - type hash - - type elem = [ `Left of hash | `Right of hash ] [@@deriving sexp, equal] - - val elem_hash : elem -> hash - - type t = elem list [@@deriving sexp, equal] - - val implied_root : t -> hash -> hash -end diff --git a/src/lib/merkle_ledger/null_ledger.ml b/src/lib/merkle_ledger/null_ledger.ml index e9e511f15c2..157244e8c10 100644 --- a/src/lib/merkle_ledger/null_ledger.ml +++ b/src/lib/merkle_ledger/null_ledger.ml @@ -1,14 +1,6 @@ -open Core_kernel - -module type Inputs_intf = sig - include Base_inputs_intf.S - - module Location : Location_intf.S -end - -module Make (Inputs : Inputs_intf) : sig +module Make (Inputs : Intf.Inputs.Intf) : sig include - Base_ledger_intf.S + Intf.Ledger.NULL with module Addr = Inputs.Location.Addr with module Location = Inputs.Location with type key := Inputs.Key.t @@ -19,8 +11,6 @@ module Make (Inputs : Inputs_intf) : sig and type hash := Inputs.Hash.t and type root_hash := Inputs.Hash.t and type account := Inputs.Account.t - - val create : depth:int -> unit -> t end = struct open Inputs diff --git a/src/lib/merkle_ledger/null_ledger.mli b/src/lib/merkle_ledger/null_ledger.mli new file mode 100644 index 00000000000..e7a3b77c377 --- /dev/null +++ b/src/lib/merkle_ledger/null_ledger.mli @@ -0,0 +1,14 @@ +module Make (Inputs : Intf.Inputs.Intf) : sig + include + Intf.Ledger.NULL + with module Addr = Inputs.Location.Addr + with module Location = Inputs.Location + with type key := Inputs.Key.t + and type token_id := Inputs.Token_id.t + and type token_id_set := Inputs.Token_id.Set.t + and type account_id := Inputs.Account_id.t + and type account_id_set := Inputs.Account_id.Set.t + and type hash := Inputs.Hash.t + and type root_hash := Inputs.Hash.t + and type account := Inputs.Account.t +end diff --git a/src/lib/merkle_ledger/syncable_intf.ml b/src/lib/merkle_ledger/syncable_intf.ml deleted file mode 100644 index 7665106793d..00000000000 --- a/src/lib/merkle_ledger/syncable_intf.ml +++ /dev/null @@ -1,33 +0,0 @@ -module type S = sig - type root_hash - - type hash - - type account - - type addr - - type t [@@deriving sexp] - - type path - - val depth : t -> int - - val num_accounts : t -> int - - val merkle_path_at_addr_exn : t -> addr -> path - - val get_inner_hash_at_addr_exn : t -> addr -> hash - - val set_inner_hash_at_addr_exn : t -> addr -> hash -> unit - - val set_all_accounts_rooted_at_exn : t -> addr -> account list -> unit - - val set_batch_accounts : t -> (addr * account) list -> unit - - (** Get all of the accounts that are in a subtree of the underlying Merkle - tree rooted at `address`. The accounts are ordered by their addresses. *) - val get_all_accounts_rooted_at_exn : t -> addr -> (addr * account) list - - val merkle_root : t -> root_hash -end diff --git a/src/lib/merkle_ledger/util.ml b/src/lib/merkle_ledger/util.ml index f2061c85e73..af51910d7c5 100644 --- a/src/lib/merkle_ledger/util.ml +++ b/src/lib/merkle_ledger/util.ml @@ -1,5 +1,3 @@ -open Core_kernel - module type Inputs_intf = sig module Location : Location_intf.S diff --git a/src/lib/merkle_ledger/util.mli b/src/lib/merkle_ledger/util.mli new file mode 100644 index 00000000000..5473f664eb8 --- /dev/null +++ b/src/lib/merkle_ledger/util.mli @@ -0,0 +1,67 @@ +module type Inputs_intf = sig + module Location : Location_intf.S + + module Location_binable : Hashable.S_binable with type t := Location.t + + module Key : Intf.Key + + module Token_id : Intf.Token_id + + module Account_id : + Intf.Account_id with type key := Key.t and type token_id := Token_id.t + + module Balance : Intf.Balance + + module Account : + Intf.Account + with type balance := Balance.t + and type account_id := Account_id.t + and type token_id := Token_id.t + + module Hash : Intf.Hash with type account := Account.t + + module Base : sig + type t + + val get : t -> Location.t -> Account.t option + + val last_filled : t -> Location.t option + end + + val get_hash : Base.t -> Location.t -> Hash.t + + val location_of_account_addr : Location.Addr.t -> Location.t + + val location_of_hash_addr : Location.Addr.t -> Location.t + + val ledger_depth : Base.t -> int + + val set_raw_hash_batch : Base.t -> (Location.t * Hash.t) list -> unit + + val set_raw_account_batch : Base.t -> (Location.t * Account.t) list -> unit + + val set_location_batch : + last_location:Location.t + -> Base.t + -> (Account_id.t * Location.t) Mina_stdlib.Nonempty_list.t + -> unit +end + +module Make (Inputs : Inputs_intf) : sig + val get_all_accounts_rooted_at_exn : + Inputs.Base.t + -> Inputs.Location.Addr.t + -> (Inputs.Location.Addr.t * Inputs.Account.t) list + + val set_hash_batch : + Inputs.Base.t -> (Inputs.Location.t * Inputs.Hash.t) list -> unit + + val set_batch : + Inputs.Base.t -> (Inputs.Location.t * Inputs.Account.t) list -> unit + + val set_batch_accounts : + Inputs.Base.t -> (Inputs.Location.Addr.t * Inputs.Account.t) list -> unit + + val set_all_accounts_rooted_at_exn : + Inputs.Base.t -> Inputs.Location.Addr.t -> Inputs.Account.t list -> unit +end diff --git a/src/lib/merkle_ledger_tests/test_database.ml b/src/lib/merkle_ledger_tests/test_database.ml index 70c21a828d3..e6ac59a406f 100644 --- a/src/lib/merkle_ledger_tests/test_database.ml +++ b/src/lib/merkle_ledger_tests/test_database.ml @@ -7,7 +7,7 @@ let%test_module "test functor on in memory databases" = module Database = Merkle_ledger.Database module type DB = - Merkle_ledger.Database_intf.S + Intf.Ledger.DATABASE with type key := Key.t and type token_id := Token_id.t and type token_id_set := Token_id.Set.t diff --git a/src/lib/merkle_ledger_tests/test_mask.ml b/src/lib/merkle_ledger_tests/test_mask.ml index eea869a5753..a7455679115 100644 --- a/src/lib/merkle_ledger_tests/test_mask.ml +++ b/src/lib/merkle_ledger_tests/test_mask.ml @@ -667,7 +667,7 @@ module Make_maskable_and_mask_with_depth (Depth : Depth_S) = struct (* underlying Merkle tree *) module Base_db : - Merkle_ledger.Database_intf.S + Merkle_ledger.Intf.Ledger.DATABASE with module Location = Location and module Addr = Location.Addr and type account := Account.t diff --git a/src/lib/merkle_ledger_tests/test_stubs.ml b/src/lib/merkle_ledger_tests/test_stubs.ml index 3c47f97f083..61d26b19d0a 100644 --- a/src/lib/merkle_ledger_tests/test_stubs.ml +++ b/src/lib/merkle_ledger_tests/test_stubs.ml @@ -128,6 +128,17 @@ struct let remove t ~key = Bigstring_frozen.Table.remove t.table key let make_checkpoint _ _ = () + + let foldi t ~init ~f = + let i = ref (-1) in + let f ~key ~data accum = incr i ; f !i accum ~key ~data in + Bigstring_frozen.Table.fold t.table ~init ~f + + (* Relying on {!val:to_alist} is probably enough for testing purposes. *) + let fold_until t ~init ~f ~finish = + let f accum (key, data) = f accum ~key ~data in + let alist = to_alist t in + List.fold_until alist ~init ~f ~finish end module Storage_locations : Intf.Storage_locations = struct diff --git a/src/lib/merkle_mask/base_merkle_tree_intf.mli b/src/lib/merkle_mask/base_merkle_tree_intf.mli index 8c0266524ae..be6db18f7ea 100644 --- a/src/lib/merkle_mask/base_merkle_tree_intf.mli +++ b/src/lib/merkle_mask/base_merkle_tree_intf.mli @@ -1,4 +1,4 @@ (* base_merkle_tree_intf.ml *) (** base module type for masking and masked Merkle trees *) -module type S = Merkle_ledger.Base_ledger_intf.S +module type S = Merkle_ledger.Intf.Ledger.S diff --git a/src/lib/mina_ledger/ledger.ml b/src/lib/mina_ledger/ledger.ml index fe1ed65e6ec..470452519b0 100644 --- a/src/lib/mina_ledger/ledger.ml +++ b/src/lib/mina_ledger/ledger.ml @@ -106,7 +106,7 @@ module Ledger_inner = struct end module Db : - Merkle_ledger.Database_intf.S + Merkle_ledger.Intf.Ledger.DATABASE with module Location = Location_at_depth with module Addr = Location_at_depth.Addr with type root_hash := Ledger_hash.t @@ -122,7 +122,7 @@ module Ledger_inner = struct module Null = Null_ledger.Make (Inputs) module Any_ledger : - Merkle_ledger.Any_ledger.S + Merkle_ledger.Intf.Ledger.ANY with module Location = Location_at_depth with type account := Account.t and type key := Public_key.Compressed.t diff --git a/src/lib/mina_ledger/ledger.mli b/src/lib/mina_ledger/ledger.mli index adc8643d2cc..483b4222d81 100644 --- a/src/lib/mina_ledger/ledger.mli +++ b/src/lib/mina_ledger/ledger.mli @@ -5,7 +5,7 @@ open Mina_base module Location : Merkle_ledger.Location_intf.S module Db : - Merkle_ledger.Database_intf.S + Merkle_ledger.Intf.Ledger.DATABASE with module Location = Location with module Addr = Location.Addr with type root_hash := Ledger_hash.t @@ -18,7 +18,7 @@ module Db : and type account_id_set := Account_id.Set.t module Any_ledger : - Merkle_ledger.Any_ledger.S + Merkle_ledger.Intf.Ledger.ANY with module Location = Location with type account := Account.t and type key := Public_key.Compressed.t diff --git a/src/lib/mina_ledger/ledger_transfer.ml b/src/lib/mina_ledger/ledger_transfer.ml index 22efdb917a8..9efd4643d35 100644 --- a/src/lib/mina_ledger/ledger_transfer.ml +++ b/src/lib/mina_ledger/ledger_transfer.ml @@ -2,7 +2,7 @@ open Core_kernel open Mina_base module type Base_ledger_intf = - Merkle_ledger.Base_ledger_intf.S + Merkle_ledger.Intf.Ledger.S with type account := Account.t and type key := Signature_lib.Public_key.Compressed.t and type token_id := Token_id.t diff --git a/src/lib/syncable_ledger/syncable_ledger.ml b/src/lib/syncable_ledger/syncable_ledger.ml index a5db250cda5..44f1f972064 100644 --- a/src/lib/syncable_ledger/syncable_ledger.ml +++ b/src/lib/syncable_ledger/syncable_ledger.ml @@ -69,7 +69,7 @@ module type Inputs_intf = sig end module MT : - Merkle_ledger.Syncable_intf.S + Merkle_ledger.Intf.SYNCABLE with type hash := Hash.t and type root_hash := Root_hash.t and type addr := Addr.t diff --git a/src/lib/syncable_ledger/test.ml b/src/lib/syncable_ledger/test.ml index 4f32857daea..0d076706b12 100644 --- a/src/lib/syncable_ledger/test.ml +++ b/src/lib/syncable_ledger/test.ml @@ -4,7 +4,7 @@ open Pipe_lib open Network_peer module type Ledger_intf = sig - include Merkle_ledger.Syncable_intf.S + include Merkle_ledger.Intf.SYNCABLE type account_id