From 5efe58f3edec5dbdbc9e562d3886c2bd3a3596f6 Mon Sep 17 00:00:00 2001 From: "It's me, CI" Date: Fri, 6 Oct 2023 16:47:35 -0700 Subject: [PATCH 1/5] Internal docs for Mina_caqti --- src/lib/mina_caqti/README.md | 76 ++++++++++++++++++++++++++++++++ src/lib/mina_caqti/mina_caqti.ml | 8 ++++ 2 files changed, 84 insertions(+) create mode 100644 src/lib/mina_caqti/README.md diff --git a/src/lib/mina_caqti/README.md b/src/lib/mina_caqti/README.md new file mode 100644 index 00000000000..58423deadb5 --- /dev/null +++ b/src/lib/mina_caqti/README.md @@ -0,0 +1,76 @@ +Mina_caqti +========== + +This library is designed to assist in querying relational databases +using the Caqti library. It is used extensively for querying the +archive database in the modules `Processor` and `Load_data` modules in +`Archive_lib`. + +Constructing SQL queries +------------------------ + +Instead of writing out SQL queries as text, the +functions here can construct those queries from table information. + +For example, the `Token` module in the archive processor contains: +```ocaml + let table_name = "tokens" + + let find_by_id (module Conn : CONNECTION) id = + Conn.find + (Caqti_request.find Caqti_type.int typ + (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) + id +``` +The list `Fields.names` is generated from the `deriving fields` annotation on +the type `Token.t`. The call to `select_cols_fromid` constructs the query +``` +SELECT value,owner_public_key_id,owner_token_id FROM tokens WHERE id = ? +``` + +There are other SQL-building functions in the library, like +`select_cols`, `insert_into_cols`, and `select_insert_into_cols`, which +are documented in the source code. + +Custom array types +------------------ + +Another notable feature of the library are the custom array types, +which are used to provide a `Caqti.Type.t` for OCaml array types not +already built into Caqti. For example, `array_int_typ` is used to +give a type for the OCaml type `int array`. Such Caqti types can be +used for the input or result type of queries, or to provide type +annotations on columns in queries. In some cases, PostgreSQL may not +be able to decode data without such annotations. There's an example of +using an annotation in +`Archive_lib.Processor.Zkapp_field_array.add_if_doesn't_exist`. + +Encoding values as NULLs +------------------------ + +In the descriptions of the functions that follow, please note that the +values returned are in the `Deferred` monad, because they are the +result of database queries. For the `add...` functions, the result +actually has a `Deferred.Result.t` type, because queries can fail. For +the `get...` functions, a failure raises an exception. + +There are some zkApps-related functions that are useful for storing +`Set_or_keep.t` and `Or_ignore.t` values. The function +`add_if_zkapp_set` runs a query if the data is `Set`, returning its +result (if it succeeds), and if the data is `Keep`, returns `None`. +Similarly, `add_if_zkapp_check` runs a query if the data is `Check`, +returning its result (if it succeeds), and if the data is `Ignore`, +returns `None`. The functions `get_zkapp_set_or_keep` and +`get_zkapp_or_ignore` symmetrically, by converting a queried value to +a value construct with `Set` or `Check`, if not NULL, and converting a +NULL to `Keep` or `Ignore`. The use of NULL to encode these +zkApp-related values is mentioned as the `NULL convention` in the part +of the database schema in `zkapp_tables.sql`. + +The functions `add_if_some` and `get_opt_item` are similar to these +zkApps-related functions, except that the constructors involved are +`Some` and `None` for option types. Therefore, `add_if_some` runs its +query argument if the data has `Some` as its constructor, returning +the result, and otherwise returns `None`. The function `get_opt_item` +returns a `Some`-constructed value, if the item is not NULL in the +database, and `None` otherwise. diff --git a/src/lib/mina_caqti/mina_caqti.ml b/src/lib/mina_caqti/mina_caqti.ml index 8377968c98f..151b01e9056 100644 --- a/src/lib/mina_caqti/mina_caqti.ml +++ b/src/lib/mina_caqti/mina_caqti.ml @@ -5,6 +5,7 @@ open Core_kernel open Caqti_async open Mina_base +(* custom Caqti types for generating type annotations on queries *) type _ Caqti_type.field += | Array_nullable_int : int option array Caqti_type.field @@ -225,10 +226,12 @@ let add_if_some (f : 'arg -> ('res, 'err) Deferred.Result.t) : 'arg option -> ('res option, 'err) Deferred.Result.t = Fn.compose deferred_result_lift_opt @@ Option.map ~f +(* if zkApp-related item is Set, run `f` *) let add_if_zkapp_set (f : 'arg -> ('res, 'err) Deferred.Result.t) : 'arg Zkapp_basic.Set_or_keep.t -> ('res option, 'err) Deferred.Result.t = Fn.compose (add_if_some f) Zkapp_basic.Set_or_keep.to_option +(* if zkApp-related item is Check, run `f` *) let add_if_zkapp_check (f : 'arg -> ('res, 'err) Deferred.Result.t) : 'arg Zkapp_basic.Or_ignore.t -> ('res option, 'err) Deferred.Result.t = Fn.compose (add_if_some f) Zkapp_basic.Or_ignore.to_option @@ -278,6 +281,9 @@ let insert_into_cols ~(returning : string) ~(table_name : string) (String.concat ~sep:", " cols) values returning +(* run `select_cols` and return the result, if found + if not found, run `insert_into_cols` and return the result +*) let select_insert_into_cols ~(select : string * 'select Caqti_type.t) ~(table_name : string) ?tannot ~(cols : string list * 'cols Caqti_type.t) (module Conn : CONNECTION) (value : 'cols) = @@ -318,11 +324,13 @@ let make_get_opt ~of_option ~f item_opt = in of_option res_opt +(** convert options to Set or Keep for zkApps-related results *) let get_zkapp_set_or_keep (item_opt : 'arg option) ~(f : 'arg -> ('res, _) Deferred.Result.t) : 'res Zkapp_basic.Set_or_keep.t Deferred.t = make_get_opt ~of_option:Zkapp_basic.Set_or_keep.of_option ~f item_opt +(** convert options to Check or Ignore for zkApps-related results *) let get_zkapp_or_ignore (item_opt : 'arg option) ~(f : 'arg -> ('res, _) Deferred.Result.t) : 'res Zkapp_basic.Or_ignore.t Deferred.t = From 09d09a42a407252be7cae81e21a5331fe18e23b4 Mon Sep 17 00:00:00 2001 From: psteckler Date: Mon, 9 Oct 2023 11:16:05 -0700 Subject: [PATCH 2/5] Update src/lib/mina_caqti/README.md Co-authored-by: Barrie Byron --- src/lib/mina_caqti/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mina_caqti/README.md b/src/lib/mina_caqti/README.md index 58423deadb5..d874f8b379c 100644 --- a/src/lib/mina_caqti/README.md +++ b/src/lib/mina_caqti/README.md @@ -3,7 +3,7 @@ Mina_caqti This library is designed to assist in querying relational databases using the Caqti library. It is used extensively for querying the -archive database in the modules `Processor` and `Load_data` modules in +archive database in the `Processor` and `Load_data` modules in `Archive_lib`. Constructing SQL queries From 39efa6ea393bb6a03be980af0b0679af41f8d012 Mon Sep 17 00:00:00 2001 From: psteckler Date: Mon, 9 Oct 2023 11:18:47 -0700 Subject: [PATCH 3/5] Update src/lib/mina_caqti/README.md Co-authored-by: Barrie Byron --- src/lib/mina_caqti/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/mina_caqti/README.md b/src/lib/mina_caqti/README.md index d874f8b379c..c19fc34e4c4 100644 --- a/src/lib/mina_caqti/README.md +++ b/src/lib/mina_caqti/README.md @@ -35,8 +35,7 @@ are documented in the source code. Custom array types ------------------ -Another notable feature of the library are the custom array types, -which are used to provide a `Caqti.Type.t` for OCaml array types not +You can use custom array types to provide a `Caqti.Type.t` for OCaml array types not already built into Caqti. For example, `array_int_typ` is used to give a type for the OCaml type `int array`. Such Caqti types can be used for the input or result type of queries, or to provide type From c215aac731d54458fb9107d38c773b25429db616 Mon Sep 17 00:00:00 2001 From: psteckler Date: Mon, 9 Oct 2023 11:19:13 -0700 Subject: [PATCH 4/5] Update src/lib/mina_caqti/README.md Co-authored-by: Barrie Byron --- src/lib/mina_caqti/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/mina_caqti/README.md b/src/lib/mina_caqti/README.md index c19fc34e4c4..48710073273 100644 --- a/src/lib/mina_caqti/README.md +++ b/src/lib/mina_caqti/README.md @@ -49,8 +49,10 @@ Encoding values as NULLs In the descriptions of the functions that follow, please note that the values returned are in the `Deferred` monad, because they are the -result of database queries. For the `add...` functions, the result -actually has a `Deferred.Result.t` type, because queries can fail. For +result of database queries. +- For the `add...` functions, the result +actually has a `Deferred.Result.t` type because queries can fail. +- For the `get...` functions, a failure raises an exception. There are some zkApps-related functions that are useful for storing From ae7d17f13e2a6c8a2b5bd803819a40720a8a0dea Mon Sep 17 00:00:00 2001 From: "It's me, CI" Date: Mon, 9 Oct 2023 11:22:48 -0700 Subject: [PATCH 5/5] add missing word --- src/lib/mina_caqti/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/mina_caqti/README.md b/src/lib/mina_caqti/README.md index 48710073273..ed3beb29610 100644 --- a/src/lib/mina_caqti/README.md +++ b/src/lib/mina_caqti/README.md @@ -49,9 +49,9 @@ Encoding values as NULLs In the descriptions of the functions that follow, please note that the values returned are in the `Deferred` monad, because they are the -result of database queries. +result of database queries. - For the `add...` functions, the result -actually has a `Deferred.Result.t` type because queries can fail. +actually has a `Deferred.Result.t` type because queries can fail. - For the `get...` functions, a failure raises an exception. @@ -62,7 +62,7 @@ result (if it succeeds), and if the data is `Keep`, returns `None`. Similarly, `add_if_zkapp_check` runs a query if the data is `Check`, returning its result (if it succeeds), and if the data is `Ignore`, returns `None`. The functions `get_zkapp_set_or_keep` and -`get_zkapp_or_ignore` symmetrically, by converting a queried value to +`get_zkapp_or_ignore` operate symmetrically, by converting a queried value to a value construct with `Set` or `Check`, if not NULL, and converting a NULL to `Keep` or `Ignore`. The use of NULL to encode these zkApp-related values is mentioned as the `NULL convention` in the part