From 0bb72a39ddb2b3a54f247d0f25aaf3755a8d1bb4 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Fri, 5 Apr 2024 20:45:33 +0100 Subject: [PATCH 01/11] Attempt to dump db queries to stdout, assuming Caqti returns it --- src/app/berkeley_migration_verifier/sql.ml | 203 +++++++++++++-------- 1 file changed, 128 insertions(+), 75 deletions(-) diff --git a/src/app/berkeley_migration_verifier/sql.ml b/src/app/berkeley_migration_verifier/sql.ml index ff6c92ec9b3..5ecce4d32c8 100644 --- a/src/app/berkeley_migration_verifier/sql.ml +++ b/src/app/berkeley_migration_verifier/sql.ml @@ -1,11 +1,12 @@ +open Async open Caqti_async -let dump_sql_to_csv output_file ~sql = - Printf.sprintf "COPY ( %s ) TO '%s' DELIMITER ',' CSV HEADER " sql output_file +let dump_sql_to_csv_string ~sql = + Printf.sprintf "COPY ( %s ) TO STDOUT DELIMITER ',' CSV HEADER " sql module Mainnet = struct - let dump_accounts_created_to_csv_query ~output_file = - dump_sql_to_csv output_file + let dump_accounts_created_to_csv_query = + dump_sql_to_csv_string ~sql: {sql| ( SELECT height, value AS public_key, state_hash, receiver_account_creation_fee_paid AS creation_fee @@ -26,22 +27,24 @@ module Mainnet = struct WHERE receiver_account_creation_fee_paid IS NOT NULL ) ORDER BY height, public_key |sql} - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = - Conn.exec (dump_accounts_created_to_csv_query ~output_file) () + let open Deferred.Result.Let_syntax in + let%bind res = Conn.find (dump_accounts_created_to_csv_query) () in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) - let dump_state_and_ledger_hashes_to_csv_query ~output_file = + let dump_state_and_ledger_hashes_to_csv_query = (* Workaround for replacing output file as caqti has an issue with using ? in place of FILE argument*) - dump_sql_to_csv output_file + dump_sql_to_csv_string ~sql: " SELECT state_hash, ledger_hash FROM blocks\n\ \ WHERE chain_status = 'canonical'\n\ \ " - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string - let dump_block_hashes_till_height_query ~output_file ~height = - dump_sql_to_csv output_file + let dump_block_hashes_till_height_query ~height = + dump_sql_to_csv_string ~sql: (Printf.sprintf "\n\ @@ -49,27 +52,33 @@ module Mainnet = struct \ WHERE chain_status = 'canonical'\n\ \ AND height <= %d ORDER BY height\n\ \ " height ) - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_block_hashes_till_height (module Conn : CONNECTION) output_file height = - Conn.exec (dump_block_hashes_till_height_query ~output_file ~height) () - - let dump_block_hashes_query ~output_file = - dump_sql_to_csv output_file + let open Deferred.Result.Let_syntax in + let%bind res = + Conn.find (dump_block_hashes_till_height_query ~height) () + in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + + let dump_block_hashes_query = + dump_sql_to_csv_string ~sql: "\n\ \ SELECT state_hash, ledger_hash FROM blocks\n\ \ WHERE chain_status = 'canonical'\n\ \ ORDER BY height\n\ \ " - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_block_hashes (module Conn : CONNECTION) output_file = - Conn.exec (dump_block_hashes_query ~output_file) () + let open Deferred.Result.Let_syntax in + let%bind res = Conn.find (dump_block_hashes_query) () in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) - let dump_user_commands_till_height_query ~output_file ~height = - dump_sql_to_csv output_file + let dump_user_commands_till_height_query ~height = + dump_sql_to_csv_string ~sql: (Printf.sprintf "WITH user_command_ids AS\n\ @@ -87,14 +96,18 @@ module Mainnet = struct \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ fee_payer_keys.id ORDER BY height, sequence_no\n\ \ " height ) - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_user_commands_till_height (module Conn : CONNECTION) output_file height = - Conn.exec (dump_user_commands_till_height_query ~output_file ~height) () - - let dump_internal_commands_till_height_query ~output_file ~height = - dump_sql_to_csv output_file + let open Deferred.Result.Let_syntax in + let%bind res = + Conn.find (dump_user_commands_till_height_query ~height) () + in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + + let dump_internal_commands_till_height_query ~height = + dump_sql_to_csv_string ~sql: (Printf.sprintf "WITH internal_command_ids AS \n\ @@ -112,14 +125,20 @@ module Mainnet = struct \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ \ \n\ \ " height ) - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_internal_commands_till_height (module Conn : CONNECTION) output_file height = - Conn.exec (dump_internal_commands_till_height_query ~output_file ~height) () - - let dump_user_commands_query ~output_file = - dump_sql_to_csv output_file + let open Deferred.Result.Let_syntax in + let%bind res = + Conn.find + (dump_internal_commands_till_height_query ~height) + () + in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + + let dump_user_commands_query = + dump_sql_to_csv_string ~sql: "WITH user_command_ids AS\n\ \ ( SELECT height, sequence_no, user_command_id FROM \ @@ -135,13 +154,15 @@ module Mainnet = struct \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ fee_payer_keys.id ORDER BY height, sequence_no\n\ \ " - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_user_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_user_commands_query ~output_file) () + let open Deferred.Result.Let_syntax in + let%bind res = Conn.find (dump_user_commands_query) () in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) - let dump_internal_commands_query ~output_file = - dump_sql_to_csv output_file + let dump_internal_commands_query = + dump_sql_to_csv_string ~sql: (Printf.sprintf "WITH internal_command_ids AS \n\ @@ -158,10 +179,12 @@ module Mainnet = struct \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ \ \n\ \ " ) - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_internal_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_internal_commands_query ~output_file) () + let open Deferred.Result.Let_syntax in + let%bind res = Conn.find (dump_internal_commands_query) () in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) let mark_chain_till_fork_block_as_canonical_query = Caqti_request.exec Caqti_type.string @@ -192,8 +215,8 @@ module Mainnet = struct end module Berkeley = struct - let dump_accounts_created_to_csv_query ~output_file = - dump_sql_to_csv output_file + let dump_accounts_created_to_csv_query = + dump_sql_to_csv_string ~sql: {sql| SELECT height, value AS public_key, state_hash, creation_fee @@ -202,10 +225,12 @@ module Berkeley = struct JOIN account_identifiers ON account_identifier_id = account_identifiers.id JOIN public_keys ON public_key_id = public_keys.id ORDER BY height, public_key |sql} - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = - Conn.exec (dump_accounts_created_to_csv_query ~output_file) () + let open Deferred.Result.Let_syntax in + let%bind res = Conn.find (dump_accounts_created_to_csv_query) () in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) let height_query = Caqti_request.find Caqti_type.unit Caqti_type.int @@ -241,8 +266,8 @@ module Berkeley = struct let blocks_count (module Conn : CONNECTION) = Conn.find blocks_count_query () - let dump_user_commands_till_height_query ~output_file ~height = - dump_sql_to_csv output_file + let dump_user_commands_till_height_query ~height = + dump_sql_to_csv_string ~sql: (Printf.sprintf "WITH user_command_ids AS\n\ @@ -260,14 +285,18 @@ module Berkeley = struct \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ fee_payer_keys.id ORDER BY height, sequence_no\n\ \ " height ) - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_user_commands_till_height (module Conn : CONNECTION) output_file height = - Conn.exec (dump_user_commands_till_height_query ~output_file ~height) () - - let dump_internal_commands_till_height_query ~output_file ~height = - dump_sql_to_csv output_file + let open Deferred.Result.Let_syntax in + let%bind res = + Conn.find (dump_user_commands_till_height_query ~height) () + in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + + let dump_internal_commands_till_height_query ~height = + dump_sql_to_csv_string ~sql: (Printf.sprintf "WITH internal_command_ids AS \n\ @@ -285,14 +314,20 @@ module Berkeley = struct \ ORDER BY height, sequence_no, secondary_sequence_no, \ command_type \n\ \ " height ) - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_internal_commands_till_height (module Conn : CONNECTION) output_file height = - Conn.exec (dump_internal_commands_till_height_query ~output_file ~height) () - - let dump_user_commands_query ~output_file = - dump_sql_to_csv output_file + let open Deferred.Result.Let_syntax in + let%bind res = + Conn.find + (dump_internal_commands_till_height_query ~height) + () + in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + + let dump_user_commands_query = + dump_sql_to_csv_string ~sql: "WITH user_command_ids AS\n\ \ ( SELECT height, sequence_no, user_command_id FROM \ @@ -308,13 +343,15 @@ module Berkeley = struct \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ fee_payer_keys.id ORDER BY height, sequence_no\n\ \ " - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_user_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_user_commands_query ~output_file) () + let open Deferred.Result.Let_syntax in + let%bind res = Conn.find (dump_user_commands_query) () in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) - let dump_internal_commands_query ~output_file = - dump_sql_to_csv output_file + let dump_internal_commands_query = + dump_sql_to_csv_string ~sql: "WITH internal_command_ids AS \n\ \ ( SELECT internal_command_id, height, sequence_no, \ @@ -330,26 +367,30 @@ module Berkeley = struct \ ORDER BY height, sequence_no, secondary_sequence_no, \ command_type \n\ \ " - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_internal_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_internal_commands_query ~output_file) () + let open Deferred.Result.Let_syntax in + let%bind res = Conn.find (dump_internal_commands_query) () in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) - let dump_account_accessed_to_csv_query ~output_file = - dump_sql_to_csv output_file + let dump_account_accessed_to_csv_query = + dump_sql_to_csv_string ~sql: {sql| SELECT account_identifier_id AS id, block_id FROM accounts_accessed JOIN blocks ON block_id = blocks.id WHERE height <> 1 ORDER BY block_id, id |sql} - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_accounts_accessed_to_csv (module Conn : CONNECTION) output_file = - Conn.exec (dump_account_accessed_to_csv_query ~output_file) () + let open Deferred.Result.Let_syntax in + let%bind res = Conn.find (dump_account_accessed_to_csv_query) () in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) - let dump_block_hashes_till_height_query ~output_file ~height = - dump_sql_to_csv output_file + let dump_block_hashes_till_height_query ~height = + dump_sql_to_csv_string ~sql: (Printf.sprintf "SELECT state_hash, ledger_hash FROM blocks \n\ @@ -357,27 +398,33 @@ module Berkeley = struct \ AND height <= %d ORDER BY height\n\ \ \n\ \ " height ) - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_block_hashes_till_height (module Conn : CONNECTION) output_file height = - Conn.exec (dump_block_hashes_till_height_query ~output_file ~height) () - - let dump_block_hashes_query ~output_file = - dump_sql_to_csv output_file + let open Deferred.Result.Let_syntax in + let%bind res = + Conn.find (dump_block_hashes_till_height_query ~height) () + in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + + let dump_block_hashes_query = + dump_sql_to_csv_string ~sql: "\n\ \ SELECT state_hash, ledger_hash FROM blocks\n\ \ WHERE chain_status = 'canonical'\n\ \ ORDER BY height\n\ \ " - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_block_hashes (module Conn : CONNECTION) output_file = - Conn.exec (dump_block_hashes_query ~output_file) () + let open Deferred.Result.Let_syntax in + let%bind res = Conn.find (dump_block_hashes_query) () in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) - let dump_user_and_internal_command_info_to_csv_query ~output_file = - dump_sql_to_csv output_file + let dump_user_and_internal_command_info_to_csv_query = + dump_sql_to_csv_string ~sql: {sql| ( @@ -403,11 +450,17 @@ module Berkeley = struct INNER JOIN internal_commands ON id = internal_command_id INNER JOIN account_identifiers ON public_key_id = receiver_id ) ORDER BY block_id, id |sql} - |> Caqti_request.exec Caqti_type.unit + |> Caqti_request.find Caqti_type.unit Caqti_type.string let dump_user_and_internal_command_info_to_csv (module Conn : CONNECTION) output_file = - Conn.exec (dump_user_and_internal_command_info_to_csv_query ~output_file) () + let open Deferred.Result.Let_syntax in + let%bind res = + Conn.find + (dump_user_and_internal_command_info_to_csv_query) + () + in + Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) let get_account_accessed_count_query = Caqti_request.find Caqti_type.unit Caqti_type.int From dc963678c70e53daa3a8815df0522a5068a1d4b6 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Sat, 6 Apr 2024 04:11:53 +0800 Subject: [PATCH 02/11] collect the results --- src/app/berkeley_migration_verifier/sql.ml | 88 ++++++++++++---------- 1 file changed, 47 insertions(+), 41 deletions(-) diff --git a/src/app/berkeley_migration_verifier/sql.ml b/src/app/berkeley_migration_verifier/sql.ml index 5ecce4d32c8..5ed673fbaa9 100644 --- a/src/app/berkeley_migration_verifier/sql.ml +++ b/src/app/berkeley_migration_verifier/sql.ml @@ -27,12 +27,13 @@ module Mainnet = struct WHERE receiver_account_creation_fee_paid IS NOT NULL ) ORDER BY height, public_key |sql} - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find (dump_accounts_created_to_csv_query) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + let%bind res = Conn.collect_list dump_accounts_created_to_csv_query () in + Writer.with_file output_file ~f:(fun writer -> + return @@ List.iter (Writer.write writer) res ) let dump_state_and_ledger_hashes_to_csv_query = (* Workaround for replacing output file as caqti has an issue with using ? in place of FILE argument*) @@ -57,10 +58,9 @@ module Mainnet = struct let dump_block_hashes_till_height (module Conn : CONNECTION) output_file height = let open Deferred.Result.Let_syntax in - let%bind res = - Conn.find (dump_block_hashes_till_height_query ~height) () - in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + let%bind res = Conn.find (dump_block_hashes_till_height_query ~height) () in + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_block_hashes_query = dump_sql_to_csv_string @@ -74,8 +74,9 @@ module Mainnet = struct let dump_block_hashes (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find (dump_block_hashes_query) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + let%bind res = Conn.find dump_block_hashes_query () in + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_user_commands_till_height_query ~height = dump_sql_to_csv_string @@ -104,7 +105,8 @@ module Mainnet = struct let%bind res = Conn.find (dump_user_commands_till_height_query ~height) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_internal_commands_till_height_query ~height = dump_sql_to_csv_string @@ -131,11 +133,10 @@ module Mainnet = struct height = let open Deferred.Result.Let_syntax in let%bind res = - Conn.find - (dump_internal_commands_till_height_query ~height) - () + Conn.find (dump_internal_commands_till_height_query ~height) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_user_commands_query = dump_sql_to_csv_string @@ -158,8 +159,9 @@ module Mainnet = struct let dump_user_commands (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find (dump_user_commands_query) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + let%bind res = Conn.find dump_user_commands_query () in + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_internal_commands_query = dump_sql_to_csv_string @@ -183,8 +185,9 @@ module Mainnet = struct let dump_internal_commands (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find (dump_internal_commands_query) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + let%bind res = Conn.find dump_internal_commands_query () in + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let mark_chain_till_fork_block_as_canonical_query = Caqti_request.exec Caqti_type.string @@ -229,8 +232,9 @@ module Berkeley = struct let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find (dump_accounts_created_to_csv_query) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + let%bind res = Conn.find dump_accounts_created_to_csv_query () in + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let height_query = Caqti_request.find Caqti_type.unit Caqti_type.int @@ -293,7 +297,8 @@ module Berkeley = struct let%bind res = Conn.find (dump_user_commands_till_height_query ~height) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_internal_commands_till_height_query ~height = dump_sql_to_csv_string @@ -320,11 +325,10 @@ module Berkeley = struct height = let open Deferred.Result.Let_syntax in let%bind res = - Conn.find - (dump_internal_commands_till_height_query ~height) - () + Conn.find (dump_internal_commands_till_height_query ~height) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_user_commands_query = dump_sql_to_csv_string @@ -347,8 +351,9 @@ module Berkeley = struct let dump_user_commands (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find (dump_user_commands_query) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + let%bind res = Conn.find dump_user_commands_query () in + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_internal_commands_query = dump_sql_to_csv_string @@ -371,8 +376,9 @@ module Berkeley = struct let dump_internal_commands (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find (dump_internal_commands_query) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + let%bind res = Conn.find dump_internal_commands_query () in + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_account_accessed_to_csv_query = dump_sql_to_csv_string @@ -386,8 +392,9 @@ module Berkeley = struct let dump_accounts_accessed_to_csv (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find (dump_account_accessed_to_csv_query) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + let%bind res = Conn.find dump_account_accessed_to_csv_query () in + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_block_hashes_till_height_query ~height = dump_sql_to_csv_string @@ -403,10 +410,9 @@ module Berkeley = struct let dump_block_hashes_till_height (module Conn : CONNECTION) output_file height = let open Deferred.Result.Let_syntax in - let%bind res = - Conn.find (dump_block_hashes_till_height_query ~height) () - in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + let%bind res = Conn.find (dump_block_hashes_till_height_query ~height) () in + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_block_hashes_query = dump_sql_to_csv_string @@ -420,8 +426,9 @@ module Berkeley = struct let dump_block_hashes (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find (dump_block_hashes_query) () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + let%bind res = Conn.find dump_block_hashes_query () in + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let dump_user_and_internal_command_info_to_csv_query = dump_sql_to_csv_string @@ -456,11 +463,10 @@ module Berkeley = struct output_file = let open Deferred.Result.Let_syntax in let%bind res = - Conn.find - (dump_user_and_internal_command_info_to_csv_query) - () + Conn.find dump_user_and_internal_command_info_to_csv_query () in - Writer.with_file output_file ~f:(fun writer -> return @@ Writer.write writer res) + Writer.with_file output_file ~f:(fun writer -> + return @@ Writer.write writer res ) let get_account_accessed_count_query = Caqti_request.find Caqti_type.unit Caqti_type.int From cb2fdd6add4c381fca2b7fc66c590334a9de6dca Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Sat, 6 Apr 2024 04:23:17 +0800 Subject: [PATCH 03/11] more collect --- src/app/berkeley_migration_verifier/sql.ml | 94 +++++++++++----------- 1 file changed, 49 insertions(+), 45 deletions(-) diff --git a/src/app/berkeley_migration_verifier/sql.ml b/src/app/berkeley_migration_verifier/sql.ml index 5ed673fbaa9..d9b6aa73a6e 100644 --- a/src/app/berkeley_migration_verifier/sql.ml +++ b/src/app/berkeley_migration_verifier/sql.ml @@ -53,14 +53,16 @@ module Mainnet = struct \ WHERE chain_status = 'canonical'\n\ \ AND height <= %d ORDER BY height\n\ \ " height ) - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_block_hashes_till_height (module Conn : CONNECTION) output_file height = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find (dump_block_hashes_till_height_query ~height) () in + let%bind res = + Conn.collect_list (dump_block_hashes_till_height_query ~height) () + in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_block_hashes_query = dump_sql_to_csv_string @@ -70,13 +72,13 @@ module Mainnet = struct \ WHERE chain_status = 'canonical'\n\ \ ORDER BY height\n\ \ " - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_block_hashes (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find dump_block_hashes_query () in + let%bind res = Conn.collect_list dump_block_hashes_query () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_user_commands_till_height_query ~height = dump_sql_to_csv_string @@ -97,16 +99,16 @@ module Mainnet = struct \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ fee_payer_keys.id ORDER BY height, sequence_no\n\ \ " height ) - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_user_commands_till_height (module Conn : CONNECTION) output_file height = let open Deferred.Result.Let_syntax in let%bind res = - Conn.find (dump_user_commands_till_height_query ~height) () + Conn.collect_list (dump_user_commands_till_height_query ~height) () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_internal_commands_till_height_query ~height = dump_sql_to_csv_string @@ -127,16 +129,16 @@ module Mainnet = struct \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ \ \n\ \ " height ) - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_internal_commands_till_height (module Conn : CONNECTION) output_file height = let open Deferred.Result.Let_syntax in let%bind res = - Conn.find (dump_internal_commands_till_height_query ~height) () + Conn.collect_list (dump_internal_commands_till_height_query ~height) () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_user_commands_query = dump_sql_to_csv_string @@ -155,13 +157,13 @@ module Mainnet = struct \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ fee_payer_keys.id ORDER BY height, sequence_no\n\ \ " - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_user_commands (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find dump_user_commands_query () in + let%bind res = Conn.collect_list dump_user_commands_query () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_internal_commands_query = dump_sql_to_csv_string @@ -181,13 +183,13 @@ module Mainnet = struct \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ \ \n\ \ " ) - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_internal_commands (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find dump_internal_commands_query () in + let%bind res = Conn.collect_list dump_internal_commands_query () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let mark_chain_till_fork_block_as_canonical_query = Caqti_request.exec Caqti_type.string @@ -228,13 +230,13 @@ module Berkeley = struct JOIN account_identifiers ON account_identifier_id = account_identifiers.id JOIN public_keys ON public_key_id = public_keys.id ORDER BY height, public_key |sql} - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find dump_accounts_created_to_csv_query () in + let%bind res = Conn.collect_list dump_accounts_created_to_csv_query () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let height_query = Caqti_request.find Caqti_type.unit Caqti_type.int @@ -289,16 +291,16 @@ module Berkeley = struct \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ fee_payer_keys.id ORDER BY height, sequence_no\n\ \ " height ) - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_user_commands_till_height (module Conn : CONNECTION) output_file height = let open Deferred.Result.Let_syntax in let%bind res = - Conn.find (dump_user_commands_till_height_query ~height) () + Conn.collect_list (dump_user_commands_till_height_query ~height) () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_internal_commands_till_height_query ~height = dump_sql_to_csv_string @@ -319,16 +321,16 @@ module Berkeley = struct \ ORDER BY height, sequence_no, secondary_sequence_no, \ command_type \n\ \ " height ) - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_internal_commands_till_height (module Conn : CONNECTION) output_file height = let open Deferred.Result.Let_syntax in let%bind res = - Conn.find (dump_internal_commands_till_height_query ~height) () + Conn.collect_list (dump_internal_commands_till_height_query ~height) () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_user_commands_query = dump_sql_to_csv_string @@ -347,13 +349,13 @@ module Berkeley = struct \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ fee_payer_keys.id ORDER BY height, sequence_no\n\ \ " - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_user_commands (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find dump_user_commands_query () in + let%bind res = Conn.collect_list dump_user_commands_query () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_internal_commands_query = dump_sql_to_csv_string @@ -372,13 +374,13 @@ module Berkeley = struct \ ORDER BY height, sequence_no, secondary_sequence_no, \ command_type \n\ \ " - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_internal_commands (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find dump_internal_commands_query () in + let%bind res = Conn.collect_list dump_internal_commands_query () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_account_accessed_to_csv_query = dump_sql_to_csv_string @@ -388,13 +390,13 @@ module Berkeley = struct JOIN blocks ON block_id = blocks.id WHERE height <> 1 ORDER BY block_id, id |sql} - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_accounts_accessed_to_csv (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find dump_account_accessed_to_csv_query () in + let%bind res = Conn.collect_list dump_account_accessed_to_csv_query () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_block_hashes_till_height_query ~height = dump_sql_to_csv_string @@ -405,14 +407,16 @@ module Berkeley = struct \ AND height <= %d ORDER BY height\n\ \ \n\ \ " height ) - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_block_hashes_till_height (module Conn : CONNECTION) output_file height = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find (dump_block_hashes_till_height_query ~height) () in + let%bind res = + Conn.collect_list (dump_block_hashes_till_height_query ~height) () + in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_block_hashes_query = dump_sql_to_csv_string @@ -422,13 +426,13 @@ module Berkeley = struct \ WHERE chain_status = 'canonical'\n\ \ ORDER BY height\n\ \ " - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_block_hashes (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in - let%bind res = Conn.find dump_block_hashes_query () in + let%bind res = Conn.collect_list dump_block_hashes_query () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let dump_user_and_internal_command_info_to_csv_query = dump_sql_to_csv_string @@ -457,16 +461,16 @@ module Berkeley = struct INNER JOIN internal_commands ON id = internal_command_id INNER JOIN account_identifiers ON public_key_id = receiver_id ) ORDER BY block_id, id |sql} - |> Caqti_request.find Caqti_type.unit Caqti_type.string + |> Caqti_request.collect Caqti_type.unit Caqti_type.string let dump_user_and_internal_command_info_to_csv (module Conn : CONNECTION) output_file = let open Deferred.Result.Let_syntax in let%bind res = - Conn.find dump_user_and_internal_command_info_to_csv_query () + Conn.collect_list dump_user_and_internal_command_info_to_csv_query () in Writer.with_file output_file ~f:(fun writer -> - return @@ Writer.write writer res ) + return @@ List.iter (Writer.write writer) res ) let get_account_accessed_count_query = Caqti_request.find Caqti_type.unit Caqti_type.int From 896cbc5da46d5f84aa803653768b7519f571a249 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Sat, 6 Apr 2024 05:55:15 +0800 Subject: [PATCH 04/11] just do list comparison instead of diff --- .../berkeley_migration_verifier.ml | 219 ++----- src/app/berkeley_migration_verifier/sql.ml | 607 ++++++++---------- 2 files changed, 354 insertions(+), 472 deletions(-) diff --git a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml index 1e14a04088b..615a6b4aab8 100644 --- a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml +++ b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml @@ -11,6 +11,8 @@ module Check = struct let ok = Ok let err error = Error [ error ] + + let of_bool = function true -> Ok | false -> Error [ "failed" ] end let exit_code = ref 0 @@ -21,6 +23,9 @@ module Test = struct let of_check check ~name ~idx ~prefix test_count = { check; name = sprintf "[%d/%d] %s) %s " idx test_count prefix name } + let of_bool b ~name ~idx ~prefix test_count = + of_check (Check.of_bool b) ~name ~idx ~prefix test_count + let print_errors error_messages = if List.length error_messages > 10 then ( Async.printf " Details (truncated to only 10 errors out of %d ): \n" @@ -72,188 +77,112 @@ let no_pending_and_orphaned_blocks_in_migrated_db query_migrated_db ~height = blockchain height. However got %d vs %d" blocks_count height ) ) -let diff_files left right = - match%bind - Process.run ~prog:"diff" ~args:[ left; right ] ~accept_nonzero_exit:[ 1 ] () - with - | Ok output -> - if String.is_empty output then return Check.ok - else - return - (Check.err - (sprintf - "Discrepancies found between files %s and %s. To reproduce \ - please run `diff %s %s`" - left right left right ) ) - | Error error -> - return - (Check.err - (sprintf "Internal error when comparing files, due to %s" - (Error.to_string_hum error) ) ) - -let all_accounts_referred_in_commands_are_recorded migrated_pool ~work_dir = +let all_accounts_referred_in_commands_are_recorded migrated_pool = let query_migrated_db = Mina_caqti.query migrated_pool in - let user_and_internal_cmds = - Filename.concat work_dir "user_and_internal_cmds.csv" - in - let account_accessed = Filename.concat work_dir "account_accessed.csv" in - let open Deferred.Let_syntax in - let%bind () = + let%bind expected = query_migrated_db ~f:(fun db -> - Sql.Berkeley.dump_user_and_internal_command_info_to_csv db - user_and_internal_cmds ) + Sql.Berkeley.dump_user_and_internal_command_info db ) in - let%bind () = - query_migrated_db ~f:(fun db -> - Sql.Berkeley.dump_accounts_accessed_to_csv db account_accessed ) + let%map result = + query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_accounts_accessed db) in - diff_files user_and_internal_cmds account_accessed + List.equal Sql.Accounts_accessed.equal expected result -let accounts_created_table_is_correct migrated_pool mainnet_pool ~work_dir = +let accounts_created_table_is_correct migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in - let accounts_created_mainnet = - Filename.concat work_dir "accounts_created_mainnet.csv" - in - let accounts_created = Filename.concat work_dir "accounts_created.csv" in - let open Deferred.Let_syntax in - let%bind () = - query_mainnet_db ~f:(fun db -> - Sql.Mainnet.dump_accounts_created_to_csv db accounts_created_mainnet ) + let%bind expected = + query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_accounts_created db) in - let%bind () = - query_migrated_db ~f:(fun db -> - Sql.Berkeley.dump_accounts_created_to_csv db accounts_created ) + let%map result = + query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_accounts_created db) in - diff_files accounts_created_mainnet accounts_created + List.equal Sql.Accounts_created.equal expected result -let compare_hashes_till_height migrated_pool mainnet_pool ~height ~work_dir = +let compare_hashes_till_height migrated_pool mainnet_pool ~height = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in - let state_hashes_berk = Filename.concat work_dir "state_hashes_berk.csv" in - let state_hashes_main = Filename.concat work_dir "state_hashes_main.csv" in - let open Deferred.Let_syntax in - let%bind () = + let%bind expected = query_migrated_db ~f:(fun db -> - Sql.Berkeley.dump_block_hashes_till_height db state_hashes_berk height ) + Sql.Berkeley.dump_block_hashes_till_height db height ) in - let%bind () = + let%map result = query_mainnet_db ~f:(fun db -> - Sql.Mainnet.dump_block_hashes_till_height db state_hashes_main height ) + Sql.Mainnet.dump_block_hashes_till_height db height ) in + List.equal Sql.State_hash_and_ledger_hash.equal expected result - diff_files state_hashes_berk state_hashes_main - -let compare_hashes migrated_pool mainnet_pool ~work_dir = +let compare_hashes migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in - let state_hashes_berk = Filename.concat work_dir "state_hashes_berk.csv" in - let state_hashes_main = Filename.concat work_dir "state_hashes_main.csv" in - let open Deferred.Let_syntax in - let%bind () = - query_migrated_db ~f:(fun db -> - Sql.Berkeley.dump_block_hashes db state_hashes_berk ) + let%bind expected = + query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_block_hashes db) in - - let%bind () = - query_mainnet_db ~f:(fun db -> - Sql.Mainnet.dump_block_hashes db state_hashes_main ) + let%map result = + query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_block_hashes db) in + List.equal Sql.State_hash_and_ledger_hash.equal expected result - diff_files state_hashes_berk state_hashes_main - -let compare_user_commands_till_height migrated_pool mainnet_pool ~height - ~work_dir = +let compare_user_commands_till_height migrated_pool mainnet_pool ~height = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in - let user_commands_berk = Filename.concat work_dir "user_commands_berk.csv" in - let user_commands_main = Filename.concat work_dir "user_commands_main.csv" in - let open Deferred.Let_syntax in - let%bind () = + let%bind expected = query_migrated_db ~f:(fun db -> - Sql.Berkeley.dump_user_commands_till_height db user_commands_berk height ) + Sql.Berkeley.dump_user_commands_till_height db height ) in - let%bind () = + let%map result = query_mainnet_db ~f:(fun db -> - Sql.Mainnet.dump_user_commands_till_height db user_commands_main height ) + Sql.Mainnet.dump_user_commands_till_height db height ) in + List.equal Sql.User_command.equal expected result - diff_files user_commands_berk user_commands_main - -let compare_internal_commands_till_height migrated_pool mainnet_pool ~height - ~work_dir = +let compare_internal_commands_till_height migrated_pool mainnet_pool ~height = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in - let internal_commands_berk = - Filename.concat work_dir "internal_commands_berk.csv" - in - let internal_commands_main = - Filename.concat work_dir "internal_commands_main.csv" - in - let open Deferred.Let_syntax in - let%bind () = + let%bind expected = query_migrated_db ~f:(fun db -> - Sql.Berkeley.dump_internal_commands_till_height db - internal_commands_berk height ) + Sql.Berkeley.dump_internal_commands_till_height db height ) in - let%bind () = + let%map result = query_mainnet_db ~f:(fun db -> - Sql.Mainnet.dump_internal_commands_till_height db internal_commands_main - height ) + Sql.Mainnet.dump_internal_commands_till_height db height ) in + List.equal Sql.Internal_command.equal expected result - diff_files internal_commands_berk internal_commands_main - -let compare_user_commands migrated_pool mainnet_pool ~work_dir = +let compare_user_commands migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in - let user_commands_berk = Filename.concat work_dir "user_commands_berk.csv" in - let user_commands_main = Filename.concat work_dir "user_commands_main.csv" in - let open Deferred.Let_syntax in - let%bind () = - query_migrated_db ~f:(fun db -> - Sql.Berkeley.dump_user_commands db user_commands_berk ) + let%bind expected = + query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_user_commands db) in - let%bind () = - query_mainnet_db ~f:(fun db -> - Sql.Mainnet.dump_user_commands db user_commands_main ) + let%map result = + query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_user_commands db) in + List.equal Sql.User_command.equal expected result - diff_files user_commands_berk user_commands_main - -let compare_internal_commands migrated_pool mainnet_pool ~work_dir = +let compare_internal_commands migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in - let internal_commands_berk = - Filename.concat work_dir "internal_commands_berk.csv" - in - let internal_commands_main = - Filename.concat work_dir "internal_commands_main.csv" - in - let open Deferred.Let_syntax in - let%bind () = - query_migrated_db ~f:(fun db -> - Sql.Berkeley.dump_internal_commands db internal_commands_berk ) + let%bind expected = + query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_internal_commands db) in - - let%bind () = - query_mainnet_db ~f:(fun db -> - Sql.Mainnet.dump_internal_commands db internal_commands_main ) + let%map result = + query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_internal_commands db) in - diff_files internal_commands_berk internal_commands_main + List.equal Sql.Internal_command.equal expected result let compare_ledger_hash ~migrated_replayer_output ~fork_genesis_config_file = let checkpoint_ledger_hash = @@ -301,7 +230,6 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = in let test_count = 6 in - let work_dir = Filename.temp_dir_name in let%bind check = migrated_db_is_connected query_migrated_db ~height in Test.of_check check ~name:"Migrated blocks are connected" ~idx:1 ~prefix:"D3.1" test_count @@ -315,9 +243,9 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = |> Test.eval ; let%bind check = - all_accounts_referred_in_commands_are_recorded migrated_pool ~work_dir + all_accounts_referred_in_commands_are_recorded migrated_pool in - Test.of_check check + Test.of_bool check ~name: "All accounts referred in internal commands or transactions are \ recorded in the accounts_accessed table." @@ -325,26 +253,26 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = |> Test.eval ; let%bind check = - compare_hashes_till_height migrated_pool mainnet_pool ~height ~work_dir + compare_hashes_till_height migrated_pool mainnet_pool ~height in - Test.of_check check + Test.of_bool check ~name:"All block hashes (state_hash, ledger_hashes) are equal" ~idx:4 ~prefix:"D3.4" test_count |> Test.eval ; let%bind check = compare_user_commands_till_height migrated_pool mainnet_pool ~height - ~work_dir in - Test.of_check check ~name:"Verify user commands" ~idx:5 ~prefix:"D3.5" + + Test.of_bool check ~name:"Verify user commands" ~idx:5 ~prefix:"D3.5" test_count |> Test.eval ; let%bind check = compare_internal_commands_till_height migrated_pool mainnet_pool ~height - ~work_dir in - Test.of_check check ~name:"Verify internal commands" ~idx:6 ~prefix:"D3.6" + + Test.of_bool check ~name:"Verify internal commands" ~idx:6 ~prefix:"D3.6" test_count |> Test.eval ; @@ -391,7 +319,6 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri let query_migrated_db = Mina_caqti.query migrated_pool in let test_count = 8 in - let work_dir = Filename.temp_dir_name in let%bind check = migrated_db_is_connected query_migrated_db ~height:fork_height in @@ -408,9 +335,9 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri |> Test.eval ; let%bind check = - all_accounts_referred_in_commands_are_recorded migrated_pool ~work_dir + all_accounts_referred_in_commands_are_recorded migrated_pool in - Test.of_check check + Test.of_bool check ~name: "All accounts referred in internal commands or transactions are \ recorded in the accounts_accessed table." @@ -418,9 +345,9 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri |> Test.eval ; let%bind check = - accounts_created_table_is_correct migrated_pool mainnet_pool ~work_dir + accounts_created_table_is_correct migrated_pool mainnet_pool in - Test.of_check check + Test.of_bool check ~name: "The content of accounts_created table is correct (by checking \ against pre-migrated database)" @@ -432,23 +359,19 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri fork_state_hash ) in - let%bind check = compare_hashes migrated_pool mainnet_pool ~work_dir in - Test.of_check check + let%bind check = compare_hashes migrated_pool mainnet_pool in + Test.of_bool check ~name:"All block hashes (state_hash, ledger_hashes) are equal" ~idx:5 ~prefix:"D3.6" test_count |> Test.eval ; - let%bind check = - compare_user_commands migrated_pool mainnet_pool ~work_dir - in - Test.of_check check ~name:"Verify user commands" ~idx:6 ~prefix:"D3.7" + let%bind check = compare_user_commands migrated_pool mainnet_pool in + Test.of_bool check ~name:"Verify user commands" ~idx:6 ~prefix:"D3.7" test_count |> Test.eval ; - let%bind check = - compare_internal_commands migrated_pool mainnet_pool ~work_dir - in - Test.of_check check ~name:"Verify internal commands" ~idx:7 ~prefix:"D3.8" + let%bind check = compare_internal_commands migrated_pool mainnet_pool in + Test.of_bool check ~name:"Verify internal commands" ~idx:7 ~prefix:"D3.8" test_count |> Test.eval ; diff --git a/src/app/berkeley_migration_verifier/sql.ml b/src/app/berkeley_migration_verifier/sql.ml index d9b6aa73a6e..fd5407fcb6a 100644 --- a/src/app/berkeley_migration_verifier/sql.ml +++ b/src/app/berkeley_migration_verifier/sql.ml @@ -1,14 +1,75 @@ open Async open Caqti_async -let dump_sql_to_csv_string ~sql = - Printf.sprintf "COPY ( %s ) TO STDOUT DELIMITER ',' CSV HEADER " sql +module Accounts_created = struct + type t = + { height : String.t + ; public_key : String.t + ; state_hash : String.t + ; creation_fee : String.t + } + [@@deriving hlist, equal] + + let typ = + Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist + Caqti_type.[ string; string; string; string ] +end + +module State_hash_and_ledger_hash = struct + type t = { state_hash : String.t; ledger_hash : String.t } + [@@deriving hlist, equal] + + let typ = + Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist + Caqti_type.[ string; string ] +end + +module User_command = struct + type t = + { receiver : String.t + ; fee_payer : String.t + ; nonce : String.t + ; amount : String.t + ; fee : String.t + ; valid_until : String.t + ; memo : String.t + ; hash : String.t + } + [@@deriving hlist, equal] + + let typ = + Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist + Caqti_type. + [ string; string; string; string; string; string; string; string ] +end + +module Internal_command = struct + type t = + { receiver : String.t + ; fee : String.t + ; sequence_no : String.t + ; secondary_sequence_no : String.t + ; hash : String.t + } + [@@deriving hlist, equal] + + let typ = + Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist + Caqti_type.[ string; string; string; string; string ] +end + +module Accounts_accessed = struct + type t = { id : String.t; block_id : String.t } [@@deriving hlist, equal] + + let typ = + Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist + Caqti_type.[ string; string ] +end module Mainnet = struct - let dump_accounts_created_to_csv_query = - dump_sql_to_csv_string - ~sql: - {sql| + let dump_accounts_created_query = + Caqti_request.collect Caqti_type.unit Accounts_created.typ + {sql| ( SELECT height, value AS public_key, state_hash, receiver_account_creation_fee_paid AS creation_fee FROM blocks_user_commands JOIN blocks ON block_id = blocks.id AND chain_status = 'canonical' @@ -27,169 +88,124 @@ module Mainnet = struct WHERE receiver_account_creation_fee_paid IS NOT NULL ) ORDER BY height, public_key |sql} - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = - let open Deferred.Result.Let_syntax in - let%bind res = Conn.collect_list dump_accounts_created_to_csv_query () in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + let dump_accounts_created (module Conn : CONNECTION) = + Conn.collect_list dump_accounts_created_query () - let dump_state_and_ledger_hashes_to_csv_query = + let dump_state_and_ledger_hashes_query = (* Workaround for replacing output file as caqti has an issue with using ? in place of FILE argument*) - dump_sql_to_csv_string - ~sql: - " SELECT state_hash, ledger_hash FROM blocks\n\ - \ WHERE chain_status = 'canonical'\n\ - \ " - |> Caqti_request.find Caqti_type.unit Caqti_type.string + Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ + " SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ " let dump_block_hashes_till_height_query ~height = - dump_sql_to_csv_string - ~sql: - (Printf.sprintf - "\n\ - \ SELECT state_hash, ledger_hash FROM blocks\n\ - \ WHERE chain_status = 'canonical'\n\ - \ AND height <= %d ORDER BY height\n\ - \ " height ) - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_block_hashes_till_height (module Conn : CONNECTION) output_file - height = - let open Deferred.Result.Let_syntax in - let%bind res = - Conn.collect_list (dump_block_hashes_till_height_query ~height) () - in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ + (Printf.sprintf + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ AND height <= %d ORDER BY height\n\ + \ " height ) + + let dump_block_hashes_till_height (module Conn : CONNECTION) height = + Conn.collect_list (dump_block_hashes_till_height_query ~height) () let dump_block_hashes_query = - dump_sql_to_csv_string - ~sql: - "\n\ - \ SELECT state_hash, ledger_hash FROM blocks\n\ - \ WHERE chain_status = 'canonical'\n\ - \ ORDER BY height\n\ - \ " - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_block_hashes (module Conn : CONNECTION) output_file = - let open Deferred.Result.Let_syntax in - let%bind res = Conn.collect_list dump_block_hashes_query () in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ ORDER BY height\n\ + \ " + + let dump_block_hashes (module Conn : CONNECTION) = + Conn.collect_list dump_block_hashes_query () let dump_user_commands_till_height_query ~height = - dump_sql_to_csv_string - ~sql: - (Printf.sprintf - "WITH user_command_ids AS\n\ - \ ( SELECT height, sequence_no, user_command_id FROM \ - blocks_user_commands\n\ - \ INNER JOIN blocks ON blocks.id = block_id\n\ - \ WHERE chain_status = 'canonical'\n\ - \ AND height <= %d\n\ - \ )\n\ - \ SELECT receiver_keys.value, fee_payer_keys.value, nonce, \ - amount, fee, valid_until, memo, hash FROM user_commands\n\ - \ INNER JOIN user_command_ids ON user_command_id = id\n\ - \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ - receiver_keys.id\n\ - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " height ) - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_user_commands_till_height (module Conn : CONNECTION) output_file - height = - let open Deferred.Result.Let_syntax in - let%bind res = - Conn.collect_list (dump_user_commands_till_height_query ~height) () - in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + Caqti_request.collect Caqti_type.unit User_command.typ + (Printf.sprintf + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ + \ INNER JOIN blocks ON blocks.id = block_id\n\ + \ WHERE chain_status = 'canonical'\n\ + \ AND height <= %d\n\ + \ )\n\ + \ SELECT receiver_keys.value, fee_payer_keys.value, nonce, \ + amount, fee, valid_until, memo, hash FROM user_commands\n\ + \ INNER JOIN user_command_ids ON user_command_id = id\n\ + \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ + receiver_keys.id\n\ + \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " height ) + + let dump_user_commands_till_height (module Conn : CONNECTION) height = + Conn.collect_list (dump_user_commands_till_height_query ~height) () let dump_internal_commands_till_height_query ~height = - dump_sql_to_csv_string - ~sql: - (Printf.sprintf - "WITH internal_command_ids AS \n\ - \ ( SELECT internal_command_id, height, sequence_no, \ - secondary_sequence_no FROM blocks_internal_commands \n\ - \ INNER JOIN blocks ON blocks.id = block_id \n\ - \ WHERE chain_status = 'canonical'\n\ - \ AND height <= %d\n\ - \ ) \n\ - \ SELECT receiver_keys.value, fee, sequence_no, \ - secondary_sequence_no, hash FROM internal_commands \n\ - \ INNER JOIN internal_command_ids ON internal_command_id = id\n\ - \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ - receiver_keys.id\n\ - \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ - \ \n\ - \ " height ) - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_internal_commands_till_height (module Conn : CONNECTION) output_file - height = - let open Deferred.Result.Let_syntax in - let%bind res = - Conn.collect_list (dump_internal_commands_till_height_query ~height) () - in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + Caqti_request.collect Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ + secondary_sequence_no FROM blocks_internal_commands \n\ + \ INNER JOIN blocks ON blocks.id = block_id \n\ + \ WHERE chain_status = 'canonical'\n\ + \ AND height <= %d\n\ + \ ) \n\ + \ SELECT receiver_keys.value, fee, sequence_no, \ + secondary_sequence_no, hash FROM internal_commands \n\ + \ INNER JOIN internal_command_ids ON internal_command_id = id\n\ + \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ + receiver_keys.id\n\ + \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ + \ \n\ + \ " height ) + + let dump_internal_commands_till_height (module Conn : CONNECTION) height = + Conn.collect_list (dump_internal_commands_till_height_query ~height) () let dump_user_commands_query = - dump_sql_to_csv_string - ~sql: - "WITH user_command_ids AS\n\ - \ ( SELECT height, sequence_no, user_command_id FROM \ - blocks_user_commands\n\ - \ INNER JOIN blocks ON blocks.id = block_id\n\ - \ WHERE chain_status = 'canonical'\n\ - \ )\n\ - \ SELECT receiver_keys.value, fee_payer_keys.value, nonce, \ - amount, fee, valid_until, memo, hash FROM user_commands\n\ - \ INNER JOIN user_command_ids ON user_command_id = id\n\ - \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ - receiver_keys.id\n\ - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_user_commands (module Conn : CONNECTION) output_file = - let open Deferred.Result.Let_syntax in - let%bind res = Conn.collect_list dump_user_commands_query () in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + Caqti_request.collect Caqti_type.unit User_command.typ + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ + \ INNER JOIN blocks ON blocks.id = block_id\n\ + \ WHERE chain_status = 'canonical'\n\ + \ )\n\ + \ SELECT receiver_keys.value, fee_payer_keys.value, nonce, amount, \ + fee, valid_until, memo, hash FROM user_commands\n\ + \ INNER JOIN user_command_ids ON user_command_id = id\n\ + \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ + receiver_keys.id\n\ + \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " + + let dump_user_commands (module Conn : CONNECTION) = + Conn.collect_list dump_user_commands_query () let dump_internal_commands_query = - dump_sql_to_csv_string - ~sql: - (Printf.sprintf - "WITH internal_command_ids AS \n\ - \ ( SELECT internal_command_id, height, sequence_no, \ - secondary_sequence_no FROM blocks_internal_commands \n\ - \ INNER JOIN blocks ON blocks.id = block_id \n\ - \ WHERE chain_status = 'canonical'\n\ - \ ) \n\ - \ SELECT receiver_keys.value, fee, sequence_no, \ - secondary_sequence_no, hash FROM internal_commands \n\ - \ INNER JOIN internal_command_ids ON internal_command_id = id\n\ - \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ - receiver_keys.id\n\ - \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ - \ \n\ - \ " ) - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_internal_commands (module Conn : CONNECTION) output_file = - let open Deferred.Result.Let_syntax in - let%bind res = Conn.collect_list dump_internal_commands_query () in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + Caqti_request.collect Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ + secondary_sequence_no FROM blocks_internal_commands \n\ + \ INNER JOIN blocks ON blocks.id = block_id \n\ + \ WHERE chain_status = 'canonical'\n\ + \ ) \n\ + \ SELECT receiver_keys.value, fee, sequence_no, \ + secondary_sequence_no, hash FROM internal_commands \n\ + \ INNER JOIN internal_command_ids ON internal_command_id = id\n\ + \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ + receiver_keys.id\n\ + \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ + \ \n\ + \ " ) + + let dump_internal_commands (module Conn : CONNECTION) = + Conn.collect_list dump_internal_commands_query () let mark_chain_till_fork_block_as_canonical_query = Caqti_request.exec Caqti_type.string @@ -220,23 +236,18 @@ module Mainnet = struct end module Berkeley = struct - let dump_accounts_created_to_csv_query = - dump_sql_to_csv_string - ~sql: - {sql| + let dump_accounts_created_query = + Caqti_request.collect Caqti_type.unit Accounts_created.typ + {sql| SELECT height, value AS public_key, state_hash, creation_fee FROM accounts_created JOIN blocks ON block_id = blocks.id JOIN account_identifiers ON account_identifier_id = account_identifiers.id JOIN public_keys ON public_key_id = public_keys.id ORDER BY height, public_key |sql} - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = - let open Deferred.Result.Let_syntax in - let%bind res = Conn.collect_list dump_accounts_created_to_csv_query () in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + let dump_accounts_created (module Conn : CONNECTION) = + Conn.collect_list dump_accounts_created_query () let height_query = Caqti_request.find Caqti_type.unit Caqti_type.int @@ -273,171 +284,126 @@ module Berkeley = struct let blocks_count (module Conn : CONNECTION) = Conn.find blocks_count_query () let dump_user_commands_till_height_query ~height = - dump_sql_to_csv_string - ~sql: - (Printf.sprintf - "WITH user_command_ids AS\n\ - \ ( SELECT height, sequence_no, user_command_id FROM \ - blocks_user_commands\n\ - \ INNER JOIN blocks ON blocks.id = block_id\n\ - \ WHERE chain_status = 'canonical'\n\ - \ AND height <= %d\n\ - \ )\n\ - \ SELECT receiver_keys.value, fee_payer_keys.value, nonce, \ - amount, fee, valid_until, memo, hash FROM user_commands\n\ - \ INNER JOIN user_command_ids ON user_command_id = id\n\ - \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ - receiver_keys.id\n\ - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " height ) - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_user_commands_till_height (module Conn : CONNECTION) output_file - height = - let open Deferred.Result.Let_syntax in - let%bind res = - Conn.collect_list (dump_user_commands_till_height_query ~height) () - in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + Caqti_request.collect Caqti_type.unit User_command.typ + (Printf.sprintf + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ + \ INNER JOIN blocks ON blocks.id = block_id\n\ + \ WHERE chain_status = 'canonical'\n\ + \ AND height <= %d\n\ + \ )\n\ + \ SELECT receiver_keys.value, fee_payer_keys.value, nonce, \ + amount, fee, valid_until, memo, hash FROM user_commands\n\ + \ INNER JOIN user_command_ids ON user_command_id = id\n\ + \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ + receiver_keys.id\n\ + \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " height ) + + let dump_user_commands_till_height (module Conn : CONNECTION) height = + Conn.collect_list (dump_user_commands_till_height_query ~height) () let dump_internal_commands_till_height_query ~height = - dump_sql_to_csv_string - ~sql: - (Printf.sprintf - "WITH internal_command_ids AS \n\ - \ ( SELECT internal_command_id, height, sequence_no, \ - secondary_sequence_no FROM blocks_internal_commands \n\ - \ INNER JOIN blocks ON blocks.id = block_id \n\ - \ WHERE chain_status = 'canonical'\n\ - \ AND height <= %d\n\ - \ ) \n\ - \ SELECT receiver_keys.value, fee, sequence_no, \ - secondary_sequence_no, hash FROM internal_commands \n\ - \ INNER JOIN internal_command_ids ON internal_command_id = id\n\ - \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ - receiver_keys.id\n\ - \ ORDER BY height, sequence_no, secondary_sequence_no, \ - command_type \n\ - \ " height ) - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_internal_commands_till_height (module Conn : CONNECTION) output_file - height = - let open Deferred.Result.Let_syntax in - let%bind res = - Conn.collect_list (dump_internal_commands_till_height_query ~height) () - in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + Caqti_request.collect Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ + secondary_sequence_no FROM blocks_internal_commands \n\ + \ INNER JOIN blocks ON blocks.id = block_id \n\ + \ WHERE chain_status = 'canonical'\n\ + \ AND height <= %d\n\ + \ ) \n\ + \ SELECT receiver_keys.value, fee, sequence_no, \ + secondary_sequence_no, hash FROM internal_commands \n\ + \ INNER JOIN internal_command_ids ON internal_command_id = id\n\ + \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ + receiver_keys.id\n\ + \ ORDER BY height, sequence_no, secondary_sequence_no, \ + command_type \n\ + \ " height ) + + let dump_internal_commands_till_height (module Conn : CONNECTION) height = + Conn.collect_list (dump_internal_commands_till_height_query ~height) () let dump_user_commands_query = - dump_sql_to_csv_string - ~sql: - "WITH user_command_ids AS\n\ - \ ( SELECT height, sequence_no, user_command_id FROM \ - blocks_user_commands\n\ - \ INNER JOIN blocks ON blocks.id = block_id\n\ - \ WHERE chain_status = 'canonical'\n\ - \ )\n\ - \ SELECT receiver_keys.value, fee_payer_keys.value, nonce, \ - amount, fee, valid_until, memo, hash FROM user_commands\n\ - \ INNER JOIN user_command_ids ON user_command_id = id\n\ - \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ - receiver_keys.id\n\ - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_user_commands (module Conn : CONNECTION) output_file = - let open Deferred.Result.Let_syntax in - let%bind res = Conn.collect_list dump_user_commands_query () in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + Caqti_request.collect Caqti_type.unit User_command.typ + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ + \ INNER JOIN blocks ON blocks.id = block_id\n\ + \ WHERE chain_status = 'canonical'\n\ + \ )\n\ + \ SELECT receiver_keys.value, fee_payer_keys.value, nonce, amount, \ + fee, valid_until, memo, hash FROM user_commands\n\ + \ INNER JOIN user_command_ids ON user_command_id = id\n\ + \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ + receiver_keys.id\n\ + \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " + + let dump_user_commands (module Conn : CONNECTION) = + Conn.collect_list dump_user_commands_query () let dump_internal_commands_query = - dump_sql_to_csv_string - ~sql: - "WITH internal_command_ids AS \n\ - \ ( SELECT internal_command_id, height, sequence_no, \ - secondary_sequence_no FROM blocks_internal_commands \n\ - \ INNER JOIN blocks ON blocks.id = block_id \n\ - \ WHERE chain_status = 'canonical'\n\ - \ ) \n\ - \ SELECT receiver_keys.value, fee, sequence_no, \ - secondary_sequence_no, hash FROM internal_commands \n\ - \ INNER JOIN internal_command_ids ON internal_command_id = id\n\ - \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ - receiver_keys.id\n\ - \ ORDER BY height, sequence_no, secondary_sequence_no, \ - command_type \n\ - \ " - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_internal_commands (module Conn : CONNECTION) output_file = - let open Deferred.Result.Let_syntax in - let%bind res = Conn.collect_list dump_internal_commands_query () in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) - - let dump_account_accessed_to_csv_query = - dump_sql_to_csv_string - ~sql: - {sql| SELECT account_identifier_id AS id, block_id + Caqti_request.collect Caqti_type.unit Internal_command.typ + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ + secondary_sequence_no FROM blocks_internal_commands \n\ + \ INNER JOIN blocks ON blocks.id = block_id \n\ + \ WHERE chain_status = 'canonical'\n\ + \ ) \n\ + \ SELECT receiver_keys.value, fee, sequence_no, \ + secondary_sequence_no, hash FROM internal_commands \n\ + \ INNER JOIN internal_command_ids ON internal_command_id = id\n\ + \ INNER JOIN public_keys AS receiver_keys ON receiver_id = \ + receiver_keys.id\n\ + \ ORDER BY height, sequence_no, secondary_sequence_no, \ + command_type \n\ + \ " + + let dump_internal_commands (module Conn : CONNECTION) = + Conn.collect_list dump_internal_commands_query () + + let dump_accounts_accessed_query = + Caqti_request.collect Caqti_type.unit Accounts_accessed.typ + {sql| SELECT account_identifier_id AS id, block_id FROM accounts_accessed JOIN blocks ON block_id = blocks.id WHERE height <> 1 ORDER BY block_id, id |sql} - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - let dump_accounts_accessed_to_csv (module Conn : CONNECTION) output_file = - let open Deferred.Result.Let_syntax in - let%bind res = Conn.collect_list dump_account_accessed_to_csv_query () in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + let dump_accounts_accessed (module Conn : CONNECTION) = + Conn.collect_list dump_accounts_accessed_query () let dump_block_hashes_till_height_query ~height = - dump_sql_to_csv_string - ~sql: - (Printf.sprintf - "SELECT state_hash, ledger_hash FROM blocks \n\ - \ WHERE chain_status = 'canonical'\n\ - \ AND height <= %d ORDER BY height\n\ - \ \n\ - \ " height ) - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_block_hashes_till_height (module Conn : CONNECTION) output_file - height = - let open Deferred.Result.Let_syntax in - let%bind res = - Conn.collect_list (dump_block_hashes_till_height_query ~height) () - in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ + (Printf.sprintf + "SELECT state_hash, ledger_hash FROM blocks \n\ + \ WHERE chain_status = 'canonical'\n\ + \ AND height <= %d ORDER BY height\n\ + \ \n\ + \ " height ) + + let dump_block_hashes_till_height (module Conn : CONNECTION) height = + Conn.collect_list (dump_block_hashes_till_height_query ~height) () let dump_block_hashes_query = - dump_sql_to_csv_string - ~sql: - "\n\ - \ SELECT state_hash, ledger_hash FROM blocks\n\ - \ WHERE chain_status = 'canonical'\n\ - \ ORDER BY height\n\ - \ " - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_block_hashes (module Conn : CONNECTION) output_file = - let open Deferred.Result.Let_syntax in - let%bind res = Conn.collect_list dump_block_hashes_query () in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) - - let dump_user_and_internal_command_info_to_csv_query = - dump_sql_to_csv_string - ~sql: - {sql| + Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ ORDER BY height\n\ + \ " + + let dump_block_hashes (module Conn : CONNECTION) = + Conn.collect_list dump_block_hashes_query () + + let dump_user_and_internal_command_info_query = + Caqti_request.collect Caqti_type.unit Accounts_accessed.typ + {sql| ( WITH user_command_ids AS ( SELECT user_command_id, block_id, status FROM blocks_user_commands @@ -461,16 +427,9 @@ module Berkeley = struct INNER JOIN internal_commands ON id = internal_command_id INNER JOIN account_identifiers ON public_key_id = receiver_id ) ORDER BY block_id, id |sql} - |> Caqti_request.collect Caqti_type.unit Caqti_type.string - - let dump_user_and_internal_command_info_to_csv (module Conn : CONNECTION) - output_file = - let open Deferred.Result.Let_syntax in - let%bind res = - Conn.collect_list dump_user_and_internal_command_info_to_csv_query () - in - Writer.with_file output_file ~f:(fun writer -> - return @@ List.iter (Writer.write writer) res ) + + let dump_user_and_internal_command_info (module Conn : CONNECTION) = + Conn.collect_list dump_user_and_internal_command_info_query () let get_account_accessed_count_query = Caqti_request.find Caqti_type.unit Caqti_type.int From 08deff86539dcef12aea28d8a864cbd6303399d5 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Sun, 7 Apr 2024 05:20:59 +0800 Subject: [PATCH 05/11] upgrade patch --- buildkite/scripts/caqti-upgrade.patch | 467 +++++++++++++------------- 1 file changed, 235 insertions(+), 232 deletions(-) diff --git a/buildkite/scripts/caqti-upgrade.patch b/buildkite/scripts/caqti-upgrade.patch index a93e42fde32..9a5aeb9b5ee 100644 --- a/buildkite/scripts/caqti-upgrade.patch +++ b/buildkite/scripts/caqti-upgrade.patch @@ -63,7 +63,7 @@ index c69e1805d1..aac3552028 100644 SELECT COUNT( * ) FROM blocks WHERE parent_id IS NULL diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml -index 8b90db5952..939aaa8864 100644 +index d4f0e9c585..a954162791 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -29,6 +29,7 @@ open Mina_block @@ -284,7 +284,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -727,7 +728,7 @@ module Zkapp_permissions = struct +@@ -710,7 +711,7 @@ module Zkapp_permissions = struct let load (module Conn : CONNECTION) id = Conn.find @@ -293,7 +293,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -780,7 +781,7 @@ module Zkapp_timing_info = struct +@@ -763,7 +764,7 @@ module Zkapp_timing_info = struct let load (module Conn : CONNECTION) id = Conn.find @@ -302,7 +302,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -801,7 +802,7 @@ module Zkapp_uri = struct +@@ -784,7 +785,7 @@ module Zkapp_uri = struct let load (module Conn : CONNECTION) id = Conn.find @@ -311,7 +311,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "value" ]) ) id end -@@ -894,7 +895,7 @@ module Zkapp_updates = struct +@@ -877,7 +878,7 @@ module Zkapp_updates = struct let load (module Conn : CONNECTION) id = Conn.find @@ -320,7 +320,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -922,7 +923,7 @@ module Zkapp_balance_bounds = struct +@@ -905,7 +906,7 @@ module Zkapp_balance_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -329,7 +329,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -951,7 +952,7 @@ module Zkapp_nonce_bounds = struct +@@ -934,7 +935,7 @@ module Zkapp_nonce_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -338,7 +338,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1035,7 +1036,7 @@ module Zkapp_account_precondition = struct +@@ -1018,7 +1019,7 @@ module Zkapp_account_precondition = struct let load (module Conn : CONNECTION) id = Conn.find @@ -347,7 +347,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1063,7 +1064,7 @@ module Zkapp_token_id_bounds = struct +@@ -1046,7 +1047,7 @@ module Zkapp_token_id_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -356,7 +356,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1095,7 +1096,7 @@ module Zkapp_timestamp_bounds = struct +@@ -1078,7 +1079,7 @@ module Zkapp_timestamp_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -365,7 +365,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1123,7 +1124,7 @@ module Zkapp_length_bounds = struct +@@ -1106,7 +1107,7 @@ module Zkapp_length_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -374,7 +374,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1151,7 +1152,7 @@ module Zkapp_amount_bounds = struct +@@ -1134,7 +1135,7 @@ module Zkapp_amount_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -383,7 +383,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1186,7 +1187,7 @@ module Zkapp_global_slot_bounds = struct +@@ -1169,7 +1170,7 @@ module Zkapp_global_slot_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -392,7 +392,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1215,7 +1216,7 @@ module Timing_info = struct +@@ -1198,7 +1199,7 @@ module Timing_info = struct Account_identifiers.find (module Conn) account_id in Conn.find @@ -401,7 +401,7 @@ index 8b90db5952..939aaa8864 100644 {sql| SELECT account_identifier_id, initial_minimum_balance, cliff_time, cliff_amount, vesting_period, vesting_increment -@@ -1227,7 +1228,7 @@ module Timing_info = struct +@@ -1210,7 +1211,7 @@ module Timing_info = struct let find_by_account_identifier_id_opt (module Conn : CONNECTION) account_identifier_id = Conn.find_opt @@ -410,7 +410,7 @@ index 8b90db5952..939aaa8864 100644 {sql| SELECT account_identifier_id, initial_minimum_balance, cliff_time, cliff_amount, vesting_period, vesting_increment -@@ -1270,7 +1271,7 @@ module Timing_info = struct +@@ -1253,7 +1254,7 @@ module Timing_info = struct in match%bind Conn.find_opt @@ -419,7 +419,7 @@ index 8b90db5952..939aaa8864 100644 {sql| SELECT id FROM timing_info WHERE account_identifier_id = ? AND initial_minimum_balance = ? -@@ -1284,7 +1285,7 @@ module Timing_info = struct +@@ -1267,7 +1268,7 @@ module Timing_info = struct return id | None -> Conn.find @@ -428,7 +428,7 @@ index 8b90db5952..939aaa8864 100644 {sql| INSERT INTO timing_info (account_identifier_id,initial_minimum_balance, cliff_time, cliff_amount, vesting_period, vesting_increment) -@@ -1295,13 +1296,13 @@ module Timing_info = struct +@@ -1278,13 +1279,13 @@ module Timing_info = struct let load (module Conn : CONNECTION) id = Conn.find @@ -444,7 +444,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1310,13 +1311,13 @@ module Snarked_ledger_hash = struct +@@ -1293,13 +1294,13 @@ module Snarked_ledger_hash = struct let find (module Conn : CONNECTION) (t : Frozen_ledger_hash.t) = let hash = Frozen_ledger_hash.to_base58_check t in Conn.find @@ -460,7 +460,7 @@ index 8b90db5952..939aaa8864 100644 "SELECT value FROM snarked_ledger_hashes WHERE id = ?" ) id -@@ -1326,7 +1327,7 @@ module Snarked_ledger_hash = struct +@@ -1309,7 +1310,7 @@ module Snarked_ledger_hash = struct let hash = Frozen_ledger_hash.to_base58_check t in match%bind Conn.find_opt @@ -469,7 +469,7 @@ index 8b90db5952..939aaa8864 100644 "SELECT id FROM snarked_ledger_hashes WHERE value = ?" ) hash with -@@ -1334,13 +1335,13 @@ module Snarked_ledger_hash = struct +@@ -1317,13 +1318,13 @@ module Snarked_ledger_hash = struct return id | None -> Conn.find @@ -485,7 +485,7 @@ index 8b90db5952..939aaa8864 100644 "SELECT value FROM snarked_ledger_hashes WHERE id = ?" ) id end -@@ -1376,7 +1377,7 @@ module Zkapp_epoch_ledger = struct +@@ -1359,7 +1360,7 @@ module Zkapp_epoch_ledger = struct let load (module Conn : CONNECTION) id = Conn.find @@ -494,7 +494,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1436,7 +1437,7 @@ module Zkapp_epoch_data = struct +@@ -1419,7 +1420,7 @@ module Zkapp_epoch_data = struct let load (module Conn : CONNECTION) id = Conn.find @@ -503,7 +503,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1511,7 +1512,7 @@ module Zkapp_network_precondition = struct +@@ -1494,7 +1495,7 @@ module Zkapp_network_precondition = struct let load (module Conn : CONNECTION) id = Conn.find @@ -512,7 +512,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1603,7 +1604,7 @@ module Zkapp_events = struct +@@ -1586,7 +1587,7 @@ module Zkapp_events = struct let load (module Conn : CONNECTION) id = Conn.find @@ -521,7 +521,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "element_ids" ]) ) id end -@@ -1755,7 +1756,7 @@ module Zkapp_account_update_body = struct +@@ -1738,7 +1739,7 @@ module Zkapp_account_update_body = struct let load (module Conn : CONNECTION) id = Conn.find @@ -530,7 +530,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1786,7 +1787,7 @@ module Zkapp_account_update = struct +@@ -1769,7 +1770,7 @@ module Zkapp_account_update = struct let load (module Conn : CONNECTION) id = Conn.find @@ -539,7 +539,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1830,7 +1831,7 @@ module Zkapp_fee_payer_body = struct +@@ -1813,7 +1814,7 @@ module Zkapp_fee_payer_body = struct let load (module Conn : CONNECTION) id = Conn.find @@ -548,7 +548,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1887,7 +1888,7 @@ module Epoch_data = struct +@@ -1870,7 +1871,7 @@ module Epoch_data = struct let load (module Conn : CONNECTION) id = Conn.find @@ -557,7 +557,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1928,13 +1929,13 @@ module User_command = struct +@@ -1911,13 +1912,13 @@ module User_command = struct let find (module Conn : CONNECTION) ~(transaction_hash : Transaction_hash.t) ~v1_transaction_hash = Conn.find_opt @@ -573,7 +573,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id -@@ -1978,7 +1979,7 @@ module User_command = struct +@@ -1961,7 +1962,7 @@ module User_command = struct in (* TODO: Converting these uint64s to int64 can overflow; see #5419 *) Conn.find @@ -582,7 +582,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.insert_into_cols ~returning:"id" ~table_name ~tannot:(function | "command_type" -> Some "user_command_type" | _ -> None ) -@@ -2025,7 +2026,7 @@ module User_command = struct +@@ -2008,7 +2009,7 @@ module User_command = struct Public_key.add_if_doesn't_exist (module Conn) user_cmd.receiver in Conn.find @@ -591,7 +591,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.insert_into_cols ~returning:"id" ~table_name ~tannot:(function | "command_type" -> Some "user_command_type" | _ -> None ) -@@ -2066,14 +2067,14 @@ module User_command = struct +@@ -2049,14 +2050,14 @@ module User_command = struct let find_opt (module Conn : CONNECTION) ~(transaction_hash : Transaction_hash.t) = Conn.find_opt @@ -608,7 +608,7 @@ index 8b90db5952..939aaa8864 100644 @@ Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names ) id -@@ -2149,8 +2150,8 @@ module Internal_command = struct +@@ -2132,8 +2133,8 @@ module Internal_command = struct let find_opt (module Conn : CONNECTION) ~(v1_transaction_hash : bool) ~(transaction_hash : Transaction_hash.t) ~(command_type : string) = Conn.find_opt @@ -619,7 +619,7 @@ index 8b90db5952..939aaa8864 100644 Caqti_type.int (Mina_caqti.select_cols ~select:"id" ~table_name ~tannot:(function -@@ -2161,7 +2162,7 @@ module Internal_command = struct +@@ -2144,7 +2145,7 @@ module Internal_command = struct let load (module Conn : CONNECTION) ~(id : int) = Conn.find @@ -628,7 +628,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id -@@ -2182,7 +2183,7 @@ module Internal_command = struct +@@ -2165,7 +2166,7 @@ module Internal_command = struct Public_key.add_if_doesn't_exist (module Conn) internal_cmd.receiver in Conn.find @@ -637,7 +637,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.insert_into_cols ~returning:"id" ~table_name ~tannot:(function | "command_type" -> Some "internal_command_type" | _ -> None -@@ -2227,7 +2228,7 @@ module Fee_transfer = struct +@@ -2210,7 +2211,7 @@ module Fee_transfer = struct in Ok { kind; receiver_id; fee; hash } in @@ -646,7 +646,7 @@ index 8b90db5952..939aaa8864 100644 Caqti_type.custom ~encode ~decode rep let add_if_doesn't_exist (module Conn : CONNECTION) -@@ -2249,7 +2250,7 @@ module Fee_transfer = struct +@@ -2232,7 +2233,7 @@ module Fee_transfer = struct Public_key.add_if_doesn't_exist (module Conn) pk in Conn.find @@ -655,7 +655,7 @@ index 8b90db5952..939aaa8864 100644 {sql| INSERT INTO internal_commands (command_type, receiver_id, fee, hash) VALUES (?::internal_command_type, ?, ?, ?) -@@ -2277,7 +2278,7 @@ module Coinbase = struct +@@ -2260,7 +2261,7 @@ module Coinbase = struct let decode (_, receiver_id, amount, hash) = Ok { receiver_id; amount; hash } in @@ -664,7 +664,7 @@ index 8b90db5952..939aaa8864 100644 Caqti_type.custom ~encode ~decode rep let add_if_doesn't_exist (module Conn : CONNECTION) -@@ -2298,7 +2299,7 @@ module Coinbase = struct +@@ -2281,7 +2282,7 @@ module Coinbase = struct Public_key.add_if_doesn't_exist (module Conn) pk in Conn.find @@ -673,7 +673,7 @@ index 8b90db5952..939aaa8864 100644 {sql| INSERT INTO internal_commands (command_type, receiver_id, fee, hash) VALUES (?::internal_command_type, ?, ?, ?) -@@ -2337,7 +2338,7 @@ module Block_and_internal_command = struct +@@ -2320,7 +2321,7 @@ module Block_and_internal_command = struct Option.map ~f:Transaction_status.Failure.to_string failure_reason in Conn.exec @@ -682,7 +682,7 @@ index 8b90db5952..939aaa8864 100644 {sql| INSERT INTO blocks_internal_commands (block_id, internal_command_id, -@@ -2358,8 +2359,8 @@ module Block_and_internal_command = struct +@@ -2341,8 +2342,8 @@ module Block_and_internal_command = struct let find (module Conn : CONNECTION) ~block_id ~internal_command_id ~sequence_no ~secondary_sequence_no = Conn.find_opt @@ -693,7 +693,7 @@ index 8b90db5952..939aaa8864 100644 Caqti_type.string {sql| SELECT 'exists' FROM blocks_internal_commands WHERE block_id = $1 -@@ -2390,8 +2391,8 @@ module Block_and_internal_command = struct +@@ -2373,8 +2374,8 @@ module Block_and_internal_command = struct ~sequence_no ~secondary_sequence_no = let comma_cols = String.concat Fields.names ~sep:"," in Conn.find @@ -704,7 +704,7 @@ index 8b90db5952..939aaa8864 100644 typ (sprintf {sql| SELECT %s FROM blocks_internal_commands -@@ -2426,7 +2427,7 @@ module Block_and_signed_command = struct +@@ -2409,7 +2410,7 @@ module Block_and_signed_command = struct Option.map ~f:Transaction_status.Failure.to_string failure_reason in Conn.exec @@ -713,7 +713,7 @@ index 8b90db5952..939aaa8864 100644 {sql| INSERT INTO blocks_user_commands (block_id, user_command_id, -@@ -2456,8 +2457,8 @@ module Block_and_signed_command = struct +@@ -2439,8 +2440,8 @@ module Block_and_signed_command = struct let open Deferred.Result.Let_syntax in match%bind Conn.find_opt @@ -724,7 +724,7 @@ index 8b90db5952..939aaa8864 100644 Caqti_type.string {sql| SELECT 'exists' FROM blocks_user_commands WHERE block_id = $1 -@@ -2476,8 +2477,8 @@ module Block_and_signed_command = struct +@@ -2459,8 +2460,8 @@ module Block_and_signed_command = struct let load (module Conn : CONNECTION) ~block_id ~user_command_id ~sequence_no = let comma_cols = String.concat Fields.names ~sep:"," in Conn.find @@ -735,7 +735,7 @@ index 8b90db5952..939aaa8864 100644 typ (sprintf {sql| SELECT %s FROM blocks_user_commands -@@ -2511,7 +2512,7 @@ module Zkapp_account_update_failures = struct +@@ -2494,7 +2495,7 @@ module Zkapp_account_update_failures = struct let load (module Conn : CONNECTION) id = Conn.find @@ -744,7 +744,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "index"; "failures" ] ) ) id -@@ -2554,8 +2555,7 @@ module Block_and_zkapp_command = struct +@@ -2537,8 +2538,7 @@ module Block_and_zkapp_command = struct in Mina_caqti.select_insert_into_cols ~select: @@ -754,7 +754,7 @@ index 8b90db5952..939aaa8864 100644 ~table_name ~cols: ( [ "block_id" -@@ -2578,8 +2578,8 @@ module Block_and_zkapp_command = struct +@@ -2561,8 +2561,8 @@ module Block_and_zkapp_command = struct let load (module Conn : CONNECTION) ~block_id ~zkapp_command_id ~sequence_no = let comma_cols = String.concat Fields.names ~sep:"," in Conn.find @@ -765,7 +765,7 @@ index 8b90db5952..939aaa8864 100644 typ (Mina_caqti.select_cols ~table_name ~select:comma_cols ~cols:[ "block_id"; "zkapp_command_id"; "sequence_no" ] -@@ -2589,7 +2589,7 @@ module Block_and_zkapp_command = struct +@@ -2572,7 +2572,7 @@ module Block_and_zkapp_command = struct let all_from_block (module Conn : CONNECTION) ~block_id = let comma_cols = String.concat Fields.names ~sep:"," in Conn.collect_list @@ -774,7 +774,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols ~table_name ~select:comma_cols ~cols:[ "block_id" ] () ) ) block_id -@@ -2661,7 +2661,7 @@ module Zkapp_account = struct +@@ -2644,7 +2644,7 @@ module Zkapp_account = struct let load (module Conn : CONNECTION) id = Conn.find @@ -783,7 +783,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -2705,8 +2705,8 @@ module Accounts_accessed = struct +@@ -2688,8 +2688,8 @@ module Accounts_accessed = struct let find_opt (module Conn : CONNECTION) ~block_id ~account_identifier_id = let comma_cols = String.concat Fields.names ~sep:"," in Conn.find_opt @@ -794,7 +794,7 @@ index 8b90db5952..939aaa8864 100644 typ (sprintf {sql| SELECT %s -@@ -2777,7 +2777,7 @@ module Accounts_accessed = struct +@@ -2760,7 +2760,7 @@ module Accounts_accessed = struct } in Mina_caqti.select_insert_into_cols @@ -803,7 +803,7 @@ index 8b90db5952..939aaa8864 100644 ~table_name ~cols:(Fields.names, typ) (module Conn) account_accessed -@@ -2793,7 +2793,7 @@ module Accounts_accessed = struct +@@ -2776,7 +2776,7 @@ module Accounts_accessed = struct let all_from_block (module Conn : CONNECTION) block_id = let comma_cols = String.concat Fields.names ~sep:"," in Conn.collect_list @@ -812,7 +812,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols ~select:comma_cols ~table_name ~cols:[ "block_id" ] () ) ) block_id -@@ -2818,7 +2818,7 @@ module Accounts_created = struct +@@ -2801,7 +2801,7 @@ module Accounts_created = struct in let creation_fee = Currency.Fee.to_string creation_fee in Mina_caqti.select_insert_into_cols @@ -821,7 +821,7 @@ index 8b90db5952..939aaa8864 100644 ~table_name ~cols:(Fields.names, typ) (module Conn) { block_id; account_identifier_id; creation_fee } -@@ -2833,7 +2833,7 @@ module Accounts_created = struct +@@ -2816,7 +2816,7 @@ module Accounts_created = struct let all_from_block (module Conn : CONNECTION) block_id = Conn.collect_list @@ -830,7 +830,7 @@ index 8b90db5952..939aaa8864 100644 {sql| SELECT block_id, account_identifier_id, creation_fee FROM accounts_created WHERE block_id = ? -@@ -2899,14 +2899,14 @@ module Block = struct +@@ -2882,14 +2882,14 @@ module Block = struct "SELECT id FROM blocks WHERE state_hash = ?" ) (State_hash.to_base58_check state_hash) @@ -848,7 +848,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name:"blocks" ~cols:Fields.names) ) id -@@ -3011,7 +3011,7 @@ module Block = struct +@@ -2994,7 +2994,7 @@ module Block = struct let blockchain_state = Protocol_state.blockchain_state protocol_state in let%bind block_id = Conn.find @@ -857,7 +857,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.insert_into_cols ~returning:"id" ~table_name ~tannot:(function | "chain_status" -> -@@ -3393,9 +3393,7 @@ module Block = struct +@@ -3376,9 +3376,7 @@ module Block = struct in (* we don't need to specify all types here, just the ones that sql may infer incorrectly *) @@ -868,7 +868,7 @@ index 8b90db5952..939aaa8864 100644 | Bool -> Some "BOOL" | Int -> -@@ -3408,39 +3406,36 @@ module Block = struct +@@ -3391,39 +3389,36 @@ module Block = struct Some "BIGINT" | Float -> Some "FLOAT" @@ -927,7 +927,7 @@ index 8b90db5952..939aaa8864 100644 match typ with | Bool -> Bool.to_string value -@@ -3468,45 +3463,30 @@ module Block = struct +@@ -3451,45 +3446,30 @@ module Block = struct (* we are ignoring the enum annotation in this context because it's not always valid to apply *) (* NOTE: we assume enum values do not contain special characters (eg "'") *) "'" ^ value ^ "'" @@ -995,7 +995,7 @@ index 8b90db5952..939aaa8864 100644 in let render_row (type a) (typ : a Caqti_type.t) (value : a) : string = "(" ^ String.concat ~sep:"," (render_type typ value) ^ ")" -@@ -3557,8 +3537,8 @@ module Block = struct +@@ -3540,8 +3520,8 @@ module Block = struct in let%map entries = Conn.collect_list @@ -1006,7 +1006,7 @@ index 8b90db5952..939aaa8864 100644 query ) () in -@@ -3576,7 +3556,7 @@ module Block = struct +@@ -3559,7 +3539,7 @@ module Block = struct String.concat ~sep:"," @@ List.map ~f:(render_row typ) values in Conn.collect_list @@ -1015,7 +1015,7 @@ index 8b90db5952..939aaa8864 100644 (sprintf "INSERT INTO %s (%s) VALUES %s RETURNING id" table fields_sql values_sql ) ) () ) -@@ -3928,7 +3908,7 @@ module Block = struct +@@ -3911,7 +3891,7 @@ module Block = struct let ids_sql = String.concat ~sep:"," ids in let parent_ids_sql = String.concat ~sep:"," parent_ids in Conn.exec @@ -1024,7 +1024,7 @@ index 8b90db5952..939aaa8864 100644 (sprintf "UPDATE %s AS b SET parent_id = data.parent_id FROM (SELECT \ unnest(array[%s]) as id, unnest(array[%s]) as parent_id) AS \ -@@ -4138,7 +4118,7 @@ module Block = struct +@@ -4121,7 +4101,7 @@ module Block = struct Some id ) in Conn.find @@ -1033,7 +1033,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.insert_into_cols ~returning:"id" ~table_name ~tannot:(function | "sub_window_densities" -> -@@ -4299,8 +4279,8 @@ module Block = struct +@@ -4282,8 +4262,8 @@ module Block = struct let set_parent_id_if_null (module Conn : CONNECTION) ~parent_hash ~(parent_id : int) = Conn.exec @@ -1044,7 +1044,7 @@ index 8b90db5952..939aaa8864 100644 {sql| UPDATE blocks SET parent_id = ? WHERE parent_hash = ? AND parent_id IS NULL -@@ -4316,8 +4296,8 @@ module Block = struct +@@ -4299,8 +4279,8 @@ module Block = struct in let columns = concat Fields.names in Conn.collect_list @@ -1055,7 +1055,7 @@ index 8b90db5952..939aaa8864 100644 typ (sprintf {sql| WITH RECURSIVE chain AS ( -@@ -4343,37 +4323,37 @@ module Block = struct +@@ -4326,37 +4306,37 @@ module Block = struct let get_highest_canonical_block_opt (module Conn : CONNECTION) = Conn.find_opt @@ -1102,7 +1102,7 @@ index 8b90db5952..939aaa8864 100644 {sql| UPDATE blocks SET chain_status='orphaned' WHERE height = $2 AND state_hash <> $1 -@@ -4462,7 +4442,7 @@ module Block = struct +@@ -4445,7 +4425,7 @@ module Block = struct | None, Some num_blocks -> ( match%map Conn.find_opt @@ -1111,7 +1111,7 @@ index 8b90db5952..939aaa8864 100644 "SELECT MAX(height) FROM blocks" ) () with -@@ -4478,8 +4458,8 @@ module Block = struct +@@ -4461,8 +4441,8 @@ module Block = struct let%bind () = (* Delete user commands from old blocks. *) Conn.exec @@ -1122,7 +1122,7 @@ index 8b90db5952..939aaa8864 100644 "DELETE FROM user_commands\n\ WHERE id IN\n\ (SELECT user_command_id FROM blocks_user_commands\n\ -@@ -4490,8 +4470,8 @@ module Block = struct +@@ -4473,8 +4453,8 @@ module Block = struct let%bind () = (* Delete old blocks. *) Conn.exec @@ -1133,7 +1133,7 @@ index 8b90db5952..939aaa8864 100644 "DELETE FROM blocks WHERE blocks.height < ? OR blocks.timestamp < \ ?" ) (height, timestamp) -@@ -4499,7 +4479,7 @@ module Block = struct +@@ -4482,7 +4462,7 @@ module Block = struct let%bind () = (* Delete orphaned internal commands. *) Conn.exec @@ -1142,7 +1142,7 @@ index 8b90db5952..939aaa8864 100644 "DELETE FROM internal_commands\n\ WHERE id NOT IN\n\ (SELECT internal_commands.id FROM internal_commands\n\ -@@ -4510,7 +4490,7 @@ module Block = struct +@@ -4493,7 +4473,7 @@ module Block = struct let%bind () = (* Delete orphaned snarked ledger hashes. *) Conn.exec @@ -1151,7 +1151,7 @@ index 8b90db5952..939aaa8864 100644 "DELETE FROM snarked_ledger_hashes\n\ WHERE id NOT IN\n\ (SELECT snarked_ledger_hash_id FROM blocks)" ) -@@ -4519,7 +4499,7 @@ module Block = struct +@@ -4502,7 +4482,7 @@ module Block = struct let%bind () = (* Delete orphaned public keys. *) Conn.exec @@ -1160,7 +1160,7 @@ index 8b90db5952..939aaa8864 100644 "DELETE FROM public_keys\n\ WHERE id NOT IN (SELECT fee_payer_id FROM user_commands)\n\ AND id NOT IN (SELECT source_id FROM user_commands)\n\ -@@ -4944,7 +4924,13 @@ let setup_server ~metrics_server_port ~constraint_constants ~logger +@@ -4927,7 +4907,13 @@ let setup_server ~metrics_server_port ~constraint_constants ~logger Strict_pipe.Writer.write extensional_block_writer extensional_block ) ] in @@ -1455,10 +1455,10 @@ index 297081b7b3..0bdddcfdaa 100644 FROM accounts_accessed WHERE block_id = $1 diff --git a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml -index f377d693fc..d183565cbe 100644 +index 615a6b4aab..ae0d17a40a 100644 --- a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml +++ b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml -@@ -284,10 +284,18 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = +@@ -213,10 +213,18 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = let mainnet_archive_uri = Uri.of_string mainnet_archive_uri in let migrated_archive_uri = Uri.of_string migrated_archive_uri in let mainnet_pool = @@ -1479,7 +1479,7 @@ index f377d693fc..d183565cbe 100644 in match (mainnet_pool, migrated_pool) with -@@ -374,11 +382,15 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri +@@ -304,11 +312,15 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri let mainnet_archive_uri = Uri.of_string mainnet_archive_uri in let migrated_archive_uri = Uri.of_string migrated_archive_uri in @@ -1498,81 +1498,81 @@ index f377d693fc..d183565cbe 100644 match (mainnet_pool, migrated_pool) with diff --git a/src/app/berkeley_migration_verifier/sql.ml b/src/app/berkeley_migration_verifier/sql.ml -index ff6c92ec9b..36b4e32401 100644 +index fd5407fcb6..602eff73a9 100644 --- a/src/app/berkeley_migration_verifier/sql.ml +++ b/src/app/berkeley_migration_verifier/sql.ml -@@ -26,7 +26,7 @@ module Mainnet = struct - WHERE receiver_account_creation_fee_paid IS NOT NULL - ) - ORDER BY height, public_key |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = - Conn.exec (dump_accounts_created_to_csv_query ~output_file) () -@@ -38,7 +38,7 @@ module Mainnet = struct - " SELECT state_hash, ledger_hash FROM blocks\n\ - \ WHERE chain_status = 'canonical'\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes_till_height_query ~output_file ~height = - dump_sql_to_csv output_file -@@ -49,7 +49,7 @@ module Mainnet = struct - \ WHERE chain_status = 'canonical'\n\ - \ AND height <= %d ORDER BY height\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes_till_height (module Conn : CONNECTION) output_file - height = -@@ -63,7 +63,7 @@ module Mainnet = struct - \ WHERE chain_status = 'canonical'\n\ - \ ORDER BY height\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes (module Conn : CONNECTION) output_file = - Conn.exec (dump_block_hashes_query ~output_file) () -@@ -87,7 +87,7 @@ module Mainnet = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_user_commands_till_height (module Conn : CONNECTION) output_file - height = +@@ -68,7 +68,7 @@ end + + module Mainnet = struct + let dump_accounts_created_query = +- Caqti_request.collect Caqti_type.unit Accounts_created.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_created.typ + {sql| + ( SELECT height, value AS public_key, state_hash, receiver_account_creation_fee_paid AS creation_fee + FROM blocks_user_commands +@@ -94,13 +94,13 @@ module Mainnet = struct + + let dump_state_and_ledger_hashes_query = + (* Workaround for replacing output file as caqti has an issue with using ? in place of FILE argument*) +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + " SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ " + + let dump_block_hashes_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + (Printf.sprintf + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ @@ -112,7 +112,7 @@ module Mainnet = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ - \ \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_internal_commands_till_height (module Conn : CONNECTION) output_file - height = -@@ -135,7 +135,7 @@ module Mainnet = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_user_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_user_commands_query ~output_file) () -@@ -158,13 +158,13 @@ module Mainnet = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ - \ \n\ - \ " ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_internal_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_internal_commands_query ~output_file) () + Conn.collect_list (dump_block_hashes_till_height_query ~height) () + + let dump_block_hashes_query = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ +@@ -123,7 +123,7 @@ module Mainnet = struct + Conn.collect_list dump_block_hashes_query () + + let dump_user_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + (Printf.sprintf + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ +@@ -145,7 +145,7 @@ module Mainnet = struct + Conn.collect_list (dump_user_commands_till_height_query ~height) () + + let dump_internal_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -167,7 +167,7 @@ module Mainnet = struct + Conn.collect_list (dump_internal_commands_till_height_query ~height) () + + let dump_user_commands_query = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ +@@ -187,7 +187,7 @@ module Mainnet = struct + Conn.collect_list dump_user_commands_query () + + let dump_internal_commands_query = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -208,7 +208,7 @@ module Mainnet = struct + Conn.collect_list dump_internal_commands_query () let mark_chain_till_fork_block_as_canonical_query = - Caqti_request.exec Caqti_type.string @@ -1580,15 +1580,17 @@ index ff6c92ec9b..36b4e32401 100644 {sql| UPDATE blocks Set chain_status = 'canonical' -@@ -202,13 +202,13 @@ module Berkeley = struct - JOIN account_identifiers ON account_identifier_id = account_identifiers.id - JOIN public_keys ON public_key_id = public_keys.id - ORDER BY height, public_key |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit +@@ -237,7 +237,7 @@ end - let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = - Conn.exec (dump_accounts_created_to_csv_query ~output_file) () + module Berkeley = struct + let dump_accounts_created_query = +- Caqti_request.collect Caqti_type.unit Accounts_created.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_created.typ + {sql| + SELECT height, value AS public_key, state_hash, creation_fee + FROM accounts_created +@@ -250,7 +250,7 @@ module Berkeley = struct + Conn.collect_list dump_accounts_created_query () let height_query = - Caqti_request.find Caqti_type.unit Caqti_type.int @@ -1596,7 +1598,7 @@ index ff6c92ec9b..36b4e32401 100644 {sql| SELECT height from blocks order by height desc limit 1; |sql} -@@ -216,7 +216,7 @@ module Berkeley = struct +@@ -258,7 +258,7 @@ module Berkeley = struct let block_height (module Conn : CONNECTION) = Conn.find height_query () let canonical_blocks_count_till_height_query = @@ -1605,7 +1607,7 @@ index ff6c92ec9b..36b4e32401 100644 {sql| WITH RECURSIVE chain AS ( -@@ -234,7 +234,7 @@ module Berkeley = struct +@@ -276,7 +276,7 @@ module Berkeley = struct Conn.find canonical_blocks_count_till_height_query height let blocks_count_query = @@ -1614,79 +1616,80 @@ index ff6c92ec9b..36b4e32401 100644 {sql| SELECT count(*) FROM blocks ; |sql} -@@ -260,7 +260,7 @@ module Berkeley = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_user_commands_till_height (module Conn : CONNECTION) output_file - height = -@@ -285,7 +285,7 @@ module Berkeley = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, \ - command_type \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_internal_commands_till_height (module Conn : CONNECTION) output_file - height = -@@ -308,7 +308,7 @@ module Berkeley = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_user_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_user_commands_query ~output_file) () -@@ -330,7 +330,7 @@ module Berkeley = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, \ - command_type \n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_internal_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_internal_commands_query ~output_file) () -@@ -343,7 +343,7 @@ module Berkeley = struct +@@ -284,7 +284,7 @@ module Berkeley = struct + let blocks_count (module Conn : CONNECTION) = Conn.find blocks_count_query () + + let dump_user_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + (Printf.sprintf + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ +@@ -306,7 +306,7 @@ module Berkeley = struct + Conn.collect_list (dump_user_commands_till_height_query ~height) () + + let dump_internal_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -328,7 +328,7 @@ module Berkeley = struct + Conn.collect_list (dump_internal_commands_till_height_query ~height) () + + let dump_user_commands_query = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ +@@ -348,7 +348,7 @@ module Berkeley = struct + Conn.collect_list dump_user_commands_query () + + let dump_internal_commands_query = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ + secondary_sequence_no FROM blocks_internal_commands \n\ +@@ -368,7 +368,7 @@ module Berkeley = struct + Conn.collect_list dump_internal_commands_query () + + let dump_accounts_accessed_query = +- Caqti_request.collect Caqti_type.unit Accounts_accessed.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_accessed.typ + {sql| SELECT account_identifier_id AS id, block_id + FROM accounts_accessed JOIN blocks ON block_id = blocks.id - WHERE height <> 1 - ORDER BY block_id, id |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_accounts_accessed_to_csv (module Conn : CONNECTION) output_file = - Conn.exec (dump_account_accessed_to_csv_query ~output_file) () -@@ -357,7 +357,7 @@ module Berkeley = struct - \ AND height <= %d ORDER BY height\n\ - \ \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes_till_height (module Conn : CONNECTION) output_file - height = -@@ -371,7 +371,7 @@ module Berkeley = struct - \ WHERE chain_status = 'canonical'\n\ - \ ORDER BY height\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes (module Conn : CONNECTION) output_file = - Conn.exec (dump_block_hashes_query ~output_file) () -@@ -403,21 +403,21 @@ module Berkeley = struct - INNER JOIN internal_commands ON id = internal_command_id - INNER JOIN account_identifiers ON public_key_id = receiver_id - ) ORDER BY block_id, id |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_user_and_internal_command_info_to_csv (module Conn : CONNECTION) - output_file = - Conn.exec (dump_user_and_internal_command_info_to_csv_query ~output_file) () +@@ -379,7 +379,7 @@ module Berkeley = struct + Conn.collect_list dump_accounts_accessed_query () + + let dump_block_hashes_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + (Printf.sprintf + "SELECT state_hash, ledger_hash FROM blocks \n\ + \ WHERE chain_status = 'canonical'\n\ +@@ -391,7 +391,7 @@ module Berkeley = struct + Conn.collect_list (dump_block_hashes_till_height_query ~height) () + + let dump_block_hashes_query = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ +@@ -402,7 +402,7 @@ module Berkeley = struct + Conn.collect_list dump_block_hashes_query () + + let dump_user_and_internal_command_info_query = +- Caqti_request.collect Caqti_type.unit Accounts_accessed.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_accessed.typ + {sql| + ( + WITH user_command_ids AS +@@ -432,14 +432,14 @@ module Berkeley = struct + Conn.collect_list dump_user_and_internal_command_info_query () let get_account_accessed_count_query = - Caqti_request.find Caqti_type.unit Caqti_type.int From 87876a1328f9499bfd6a4332627b88d11f06121b Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Sun, 7 Apr 2024 06:09:21 +0800 Subject: [PATCH 06/11] upgrade caqti-upgrade-plus-archive-init-speedup.patch --- ...ti-upgrade-plus-archive-init-speedup.patch | 490 +++++++++--------- 1 file changed, 256 insertions(+), 234 deletions(-) diff --git a/buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch b/buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch index 698ad821538..3daf2f8546f 100644 --- a/buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch +++ b/buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch @@ -1,9 +1,3 @@ -From cb2463f19908e9c5531c7f80148e975060c954d9 Mon Sep 17 00:00:00 2001 -From: ember arlynx -Date: Sat, 30 Mar 2024 22:44:15 -0400 -Subject: [PATCH] applied caqti+archive - - diff --git a/opam.export b/opam.export index 655b623d35..130eb77140 100644 --- a/opam.export @@ -77,10 +71,10 @@ index 571327de81..f9dca92cd7 100644 + +DROP TABLE zkapp_verification_key_hashes; diff --git a/src/app/archive/lib/load_data.ml b/src/app/archive/lib/load_data.ml -index 1ca76d1a0b..4b4faf66dd 100644 +index 1ca76d1a0b..90cda002a4 100644 --- a/src/app/archive/lib/load_data.ml +++ b/src/app/archive/lib/load_data.ml -@@ -91,9 +91,9 @@ let update_of_id pool update_id = +@@ -91,9 +91,10 @@ let update_of_id pool update_id = let open Zkapp_basic in let query_db ~f = Mina_caqti.query ~f pool in let with_pool ~f arg = @@ -88,11 +82,12 @@ index 1ca76d1a0b..4b4faf66dd 100644 + let open Mina_caqti in Pool.use - (fun (module Conn : CONNECTION) -> f (module Conn : CONNECTION) arg) -+ (fun (module Conn : Mina_caqti.CONNECTION) -> f (module Conn : Mina_caqti.CONNECTION) arg) ++ (fun (module Conn : Mina_caqti.CONNECTION) -> ++ f (module Conn : Mina_caqti.CONNECTION) arg ) pool in let%bind { app_state_id -@@ -636,9 +636,9 @@ let get_account_accessed ~pool (account : Processor.Accounts_accessed.t) : +@@ -636,9 +637,10 @@ let get_account_accessed ~pool (account : Processor.Accounts_accessed.t) : (int * Account.t) Deferred.t = let query_db ~f = Mina_caqti.query ~f pool in let with_pool ~f arg = @@ -100,7 +95,8 @@ index 1ca76d1a0b..4b4faf66dd 100644 + let open Mina_caqti in Pool.use - (fun (module Conn : CONNECTION) -> f (module Conn : CONNECTION) arg) -+ (fun (module Conn : Mina_caqti.CONNECTION) -> f (module Conn : Mina_caqti.CONNECTION) arg) ++ (fun (module Conn : Mina_caqti.CONNECTION) -> ++ f (module Conn : Mina_caqti.CONNECTION) arg ) pool in let pk_of_id = pk_of_id pool in @@ -173,7 +169,7 @@ index c69e1805d1..bf15430c54 100644 pool with diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml -index 8b90db5952..313b424f55 100644 +index d4f0e9c585..274fb0839d 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -21,7 +21,6 @@ @@ -521,8 +517,7 @@ index 8b90db5952..313b424f55 100644 - (module Conn) - token_symbol + type local_copy = (string, int) Hashtbl.t - -- let load (module Conn : CONNECTION) id = ++ + let local_copies = Hashtbl.create (module String) + + let load_copy = @@ -534,7 +529,8 @@ index 8b90db5952..313b424f55 100644 + ~load_elt:(fun t_to_id (id, value) -> + Hashtbl.add_exn t_to_id ~key:value ~data:id ; + Deferred.unit ) -+ + +- let load (module Conn : CONNECTION) id = + let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) token_symbol = + let%bind t_to_id = load_copy (module Conn) in + let open Deferred.Result.Let_syntax in @@ -3009,7 +3005,7 @@ index 0669b3d4b2..dae5863bc1 100644 | Error (`Response_rejected _ as err) -> failwithf diff --git a/src/app/berkeley_migration/sql.ml b/src/app/berkeley_migration/sql.ml -index 297081b7b3..4b677e43f3 100644 +index 297081b7b3..68a5c27e67 100644 --- a/src/app/berkeley_migration/sql.ml +++ b/src/app/berkeley_migration/sql.ml @@ -1,7 +1,6 @@ @@ -3087,7 +3083,7 @@ index 297081b7b3..4b677e43f3 100644 {sql| SELECT id, state_hash, parent_id, parent_hash, creator_id, block_winner_id, snarked_ledger_hash_id, staking_epoch_data_id, next_epoch_data_id, ledger_hash, height, global_slot, -@@ -116,23 +115,23 @@ module Mainnet = struct +@@ -116,23 +115,24 @@ module Mainnet = struct ORDER BY height ASC |sql} ) @@ -3108,7 +3104,8 @@ index 297081b7b3..4b677e43f3 100644 height DESC LIMIT 1" ) - let get_subchain (module Conn : CONNECTION) ~start_block_id ~end_block_id = -+ let get_subchain (module Conn : Mina_caqti.CONNECTION) ~start_block_id ~end_block_id = ++ let get_subchain (module Conn : Mina_caqti.CONNECTION) ~start_block_id ++ ~end_block_id = (* derive query from type `t` *) Conn.collect_list - (Caqti_request.collect @@ -3118,7 +3115,7 @@ index 297081b7b3..4b677e43f3 100644 Caqti_type.int {sql| WITH RECURSIVE chain AS ( SELECT id, parent_id, height -@@ -194,9 +193,9 @@ module Mainnet = struct +@@ -194,9 +194,9 @@ module Mainnet = struct let decode t = Ok (of_hlist (tuple_to_hlist spec t)) in Caqti_type.custom ~encode ~decode (to_rep spec) @@ -3130,7 +3127,7 @@ index 297081b7b3..4b677e43f3 100644 {sql| SELECT block_id, user_command_id, sequence_no, status,failure_reason, -@@ -233,9 +232,9 @@ module Mainnet = struct +@@ -233,9 +233,9 @@ module Mainnet = struct let decode t = Ok (of_hlist (tuple_to_hlist spec t)) in Caqti_type.custom ~encode ~decode (to_rep spec) @@ -3142,7 +3139,7 @@ index 297081b7b3..4b677e43f3 100644 {sql| SELECT block_id, internal_command_id, sequence_no, secondary_sequence_no, receiver_account_creation_fee_paid, -@@ -266,12 +265,12 @@ module Mainnet = struct +@@ -266,12 +266,12 @@ module Mainnet = struct let decode ((typ, receiver_id, fee, token), hash) = Ok { typ; receiver_id; fee; token; hash } in @@ -3158,7 +3155,7 @@ index 297081b7b3..4b677e43f3 100644 {sql| SELECT type,receiver_id,fee,token,hash FROM internal_commands WHERE id = ? -@@ -336,9 +335,9 @@ module Mainnet = struct +@@ -336,9 +336,9 @@ module Mainnet = struct let decode t = Ok (of_hlist (tuple_to_hlist spec t)) in Caqti_type.custom ~encode ~decode (to_rep spec) @@ -3170,7 +3167,7 @@ index 297081b7b3..4b677e43f3 100644 {sql| SELECT type,fee_payer_id,source_id,receiver_id, fee_token,token, nonce,amount,fee,valid_until,memo,hash -@@ -350,16 +349,16 @@ end +@@ -350,16 +350,16 @@ end module Berkeley = struct module Block = struct @@ -3191,7 +3188,7 @@ index 297081b7b3..4b677e43f3 100644 {sql| SELECT height FROM blocks WHERE chain_status <> 'orphaned' -@@ -367,9 +366,9 @@ module Berkeley = struct +@@ -367,9 +367,9 @@ module Berkeley = struct LIMIT 1 |sql} ) @@ -3203,7 +3200,7 @@ index 297081b7b3..4b677e43f3 100644 {sql| SELECT id FROM blocks WHERE height = 1 -@@ -386,9 +385,9 @@ module Berkeley = struct +@@ -386,9 +386,9 @@ module Berkeley = struct let decode t = Ok (of_hlist (tuple_to_hlist spec t)) in Caqti_type.custom ~encode ~decode (to_rep spec) @@ -3215,7 +3212,7 @@ index 297081b7b3..4b677e43f3 100644 {sql| SELECT pk.value, t.value FROM account_identifiers ai INNER JOIN tokens t ON ai.token_id = t.id -@@ -398,9 +397,9 @@ module Berkeley = struct +@@ -398,9 +398,9 @@ module Berkeley = struct end module Accounts_accessed = struct @@ -3228,10 +3225,10 @@ index 297081b7b3..4b677e43f3 100644 FROM accounts_accessed WHERE block_id = $1 diff --git a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml -index f377d693fc..a97801e306 100644 +index 615a6b4aab..f624c19a5a 100644 --- a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml +++ b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml -@@ -284,10 +284,10 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = +@@ -213,10 +213,10 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = let mainnet_archive_uri = Uri.of_string mainnet_archive_uri in let migrated_archive_uri = Uri.of_string migrated_archive_uri in let mainnet_pool = @@ -3244,7 +3241,7 @@ index f377d693fc..a97801e306 100644 in match (mainnet_pool, migrated_pool) with -@@ -375,10 +375,10 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri +@@ -305,10 +305,10 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri let mainnet_archive_uri = Uri.of_string mainnet_archive_uri in let migrated_archive_uri = Uri.of_string migrated_archive_uri in let mainnet_pool = @@ -3258,112 +3255,125 @@ index f377d693fc..a97801e306 100644 match (mainnet_pool, migrated_pool) with diff --git a/src/app/berkeley_migration_verifier/sql.ml b/src/app/berkeley_migration_verifier/sql.ml -index ff6c92ec9b..c0e89348d2 100644 +index fd5407fcb6..ca655d4f45 100644 --- a/src/app/berkeley_migration_verifier/sql.ml +++ b/src/app/berkeley_migration_verifier/sql.ml -@@ -1,5 +1,3 @@ +@@ -1,5 +1,4 @@ + open Async -open Caqti_async -- - let dump_sql_to_csv output_file ~sql = - Printf.sprintf "COPY ( %s ) TO '%s' DELIMITER ',' CSV HEADER " sql output_file -@@ -26,9 +24,9 @@ module Mainnet = struct - WHERE receiver_account_creation_fee_paid IS NOT NULL + module Accounts_created = struct + type t = +@@ -68,7 +67,7 @@ end + + module Mainnet = struct + let dump_accounts_created_query = +- Caqti_request.collect Caqti_type.unit Accounts_created.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_created.typ + {sql| + ( SELECT height, value AS public_key, state_hash, receiver_account_creation_fee_paid AS creation_fee + FROM blocks_user_commands +@@ -89,18 +88,18 @@ module Mainnet = struct ) ORDER BY height, public_key |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = -+ let dump_accounts_created_to_csv (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_accounts_created_to_csv_query ~output_file) () - - let dump_state_and_ledger_hashes_to_csv_query ~output_file = -@@ -38,7 +36,7 @@ module Mainnet = struct - " SELECT state_hash, ledger_hash FROM blocks\n\ - \ WHERE chain_status = 'canonical'\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes_till_height_query ~output_file ~height = - dump_sql_to_csv output_file -@@ -49,10 +47,10 @@ module Mainnet = struct - \ WHERE chain_status = 'canonical'\n\ - \ AND height <= %d ORDER BY height\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_block_hashes_till_height (module Conn : CONNECTION) output_file -- height = -+ let dump_block_hashes_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_block_hashes_till_height_query ~output_file ~height) () - - let dump_block_hashes_query ~output_file = -@@ -63,9 +61,9 @@ module Mainnet = struct - \ WHERE chain_status = 'canonical'\n\ - \ ORDER BY height\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_block_hashes (module Conn : CONNECTION) output_file = -+ let dump_block_hashes (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_block_hashes_query ~output_file) () - - let dump_user_commands_till_height_query ~output_file ~height = -@@ -87,10 +85,10 @@ module Mainnet = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_user_commands_till_height (module Conn : CONNECTION) output_file -- height = + +- let dump_accounts_created (module Conn : CONNECTION) = ++ let dump_accounts_created (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_accounts_created_query () + + let dump_state_and_ledger_hashes_query = + (* Workaround for replacing output file as caqti has an issue with using ? in place of FILE argument*) +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + " SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ " + + let dump_block_hashes_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + (Printf.sprintf + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ +@@ -108,22 +107,23 @@ module Mainnet = struct + \ AND height <= %d ORDER BY height\n\ + \ " height ) + +- let dump_block_hashes_till_height (module Conn : CONNECTION) height = ++ let dump_block_hashes_till_height (module Conn : Mina_caqti.CONNECTION) height ++ = + Conn.collect_list (dump_block_hashes_till_height_query ~height) () + + let dump_block_hashes_query = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ ORDER BY height\n\ + \ " + +- let dump_block_hashes (module Conn : CONNECTION) = ++ let dump_block_hashes (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_block_hashes_query () + + let dump_user_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + (Printf.sprintf + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ +@@ -141,11 +141,12 @@ module Mainnet = struct + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " height ) + +- let dump_user_commands_till_height (module Conn : CONNECTION) height = + let dump_user_commands_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_user_commands_till_height_query ~output_file ~height) () - - let dump_internal_commands_till_height_query ~output_file ~height = -@@ -112,10 +110,10 @@ module Mainnet = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ - \ \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_internal_commands_till_height (module Conn : CONNECTION) output_file -- height = ++ height = + Conn.collect_list (dump_user_commands_till_height_query ~height) () + + let dump_internal_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -163,11 +164,12 @@ module Mainnet = struct + \ \n\ + \ " height ) + +- let dump_internal_commands_till_height (module Conn : CONNECTION) height = + let dump_internal_commands_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_internal_commands_till_height_query ~output_file ~height) () - - let dump_user_commands_query ~output_file = -@@ -135,9 +133,9 @@ module Mainnet = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_user_commands (module Conn : CONNECTION) output_file = -+ let dump_user_commands (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_user_commands_query ~output_file) () - - let dump_internal_commands_query ~output_file = -@@ -158,13 +156,13 @@ module Mainnet = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ - \ \n\ - \ " ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_internal_commands (module Conn : CONNECTION) output_file = -+ let dump_internal_commands (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_internal_commands_query ~output_file) () ++ height = + Conn.collect_list (dump_internal_commands_till_height_query ~height) () + + let dump_user_commands_query = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ +@@ -183,11 +185,11 @@ module Mainnet = struct + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " + +- let dump_user_commands (module Conn : CONNECTION) = ++ let dump_user_commands (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_user_commands_query () + + let dump_internal_commands_query = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -204,11 +206,11 @@ module Mainnet = struct + \ \n\ + \ " ) + +- let dump_internal_commands (module Conn : CONNECTION) = ++ let dump_internal_commands (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_internal_commands_query () let mark_chain_till_fork_block_as_canonical_query = - Caqti_request.exec Caqti_type.string @@ -3371,7 +3381,7 @@ index ff6c92ec9b..c0e89348d2 100644 {sql| UPDATE blocks Set chain_status = 'canonical' -@@ -186,8 +184,8 @@ module Mainnet = struct +@@ -230,14 +232,14 @@ module Mainnet = struct ) |sql} @@ -3382,16 +3392,20 @@ index ff6c92ec9b..c0e89348d2 100644 Conn.exec mark_chain_till_fork_block_as_canonical_query fork_state_hash end -@@ -202,21 +200,22 @@ module Berkeley = struct - JOIN account_identifiers ON account_identifier_id = account_identifiers.id + module Berkeley = struct + let dump_accounts_created_query = +- Caqti_request.collect Caqti_type.unit Accounts_created.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_created.typ + {sql| + SELECT height, value AS public_key, state_hash, creation_fee + FROM accounts_created +@@ -246,19 +248,20 @@ module Berkeley = struct JOIN public_keys ON public_key_id = public_keys.id ORDER BY height, public_key |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit -- let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = -+ let dump_accounts_created_to_csv (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_accounts_created_to_csv_query ~output_file) () +- let dump_accounts_created (module Conn : CONNECTION) = ++ let dump_accounts_created (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_accounts_created_query () let height_query = - Caqti_request.find Caqti_type.unit Caqti_type.int @@ -3410,7 +3424,7 @@ index ff6c92ec9b..c0e89348d2 100644 {sql| WITH RECURSIVE chain AS ( -@@ -230,16 +229,18 @@ module Berkeley = struct +@@ -272,19 +275,21 @@ module Berkeley = struct ) SELECT count(*) FROM chain where chain_status = 'canonical'; |sql} @@ -3430,111 +3444,119 @@ index ff6c92ec9b..c0e89348d2 100644 + let blocks_count (module Conn : Mina_caqti.CONNECTION) = + Conn.find blocks_count_query () - let dump_user_commands_till_height_query ~output_file ~height = - dump_sql_to_csv output_file -@@ -260,10 +261,10 @@ module Berkeley = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_user_commands_till_height (module Conn : CONNECTION) output_file -- height = + let dump_user_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + (Printf.sprintf + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ +@@ -302,11 +307,12 @@ module Berkeley = struct + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " height ) + +- let dump_user_commands_till_height (module Conn : CONNECTION) height = + let dump_user_commands_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_user_commands_till_height_query ~output_file ~height) () - - let dump_internal_commands_till_height_query ~output_file ~height = -@@ -285,10 +286,10 @@ module Berkeley = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, \ - command_type \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_internal_commands_till_height (module Conn : CONNECTION) output_file -- height = ++ height = + Conn.collect_list (dump_user_commands_till_height_query ~height) () + + let dump_internal_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -324,11 +330,12 @@ module Berkeley = struct + command_type \n\ + \ " height ) + +- let dump_internal_commands_till_height (module Conn : CONNECTION) height = + let dump_internal_commands_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_internal_commands_till_height_query ~output_file ~height) () - - let dump_user_commands_query ~output_file = -@@ -308,9 +309,9 @@ module Berkeley = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_user_commands (module Conn : CONNECTION) output_file = -+ let dump_user_commands (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_user_commands_query ~output_file) () - - let dump_internal_commands_query ~output_file = -@@ -330,9 +331,9 @@ module Berkeley = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, \ - command_type \n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_internal_commands (module Conn : CONNECTION) output_file = -+ let dump_internal_commands (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_internal_commands_query ~output_file) () - - let dump_account_accessed_to_csv_query ~output_file = -@@ -343,9 +344,10 @@ module Berkeley = struct ++ height = + Conn.collect_list (dump_internal_commands_till_height_query ~height) () + + let dump_user_commands_query = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ +@@ -344,11 +351,11 @@ module Berkeley = struct + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " + +- let dump_user_commands (module Conn : CONNECTION) = ++ let dump_user_commands (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_user_commands_query () + + let dump_internal_commands_query = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ + secondary_sequence_no FROM blocks_internal_commands \n\ +@@ -364,22 +371,22 @@ module Berkeley = struct + command_type \n\ + \ " + +- let dump_internal_commands (module Conn : CONNECTION) = ++ let dump_internal_commands (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_internal_commands_query () + + let dump_accounts_accessed_query = +- Caqti_request.collect Caqti_type.unit Accounts_accessed.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_accessed.typ + {sql| SELECT account_identifier_id AS id, block_id + FROM accounts_accessed JOIN blocks ON block_id = blocks.id WHERE height <> 1 ORDER BY block_id, id |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_accounts_accessed_to_csv (module Conn : CONNECTION) output_file = -+ let dump_accounts_accessed_to_csv (module Conn : Mina_caqti.CONNECTION) -+ output_file = - Conn.exec (dump_account_accessed_to_csv_query ~output_file) () - - let dump_block_hashes_till_height_query ~output_file ~height = -@@ -357,10 +359,10 @@ module Berkeley = struct - \ AND height <= %d ORDER BY height\n\ - \ \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_block_hashes_till_height (module Conn : CONNECTION) output_file -- height = -+ let dump_block_hashes_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_block_hashes_till_height_query ~output_file ~height) () - - let dump_block_hashes_query ~output_file = -@@ -371,9 +373,9 @@ module Berkeley = struct - \ WHERE chain_status = 'canonical'\n\ - \ ORDER BY height\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_block_hashes (module Conn : CONNECTION) output_file = -+ let dump_block_hashes (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_block_hashes_query ~output_file) () - - let dump_user_and_internal_command_info_to_csv_query ~output_file = -@@ -403,21 +405,21 @@ module Berkeley = struct - INNER JOIN internal_commands ON id = internal_command_id + +- let dump_accounts_accessed (module Conn : CONNECTION) = ++ let dump_accounts_accessed (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_accounts_accessed_query () + + let dump_block_hashes_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + (Printf.sprintf + "SELECT state_hash, ledger_hash FROM blocks \n\ + \ WHERE chain_status = 'canonical'\n\ +@@ -387,22 +394,23 @@ module Berkeley = struct + \ \n\ + \ " height ) + +- let dump_block_hashes_till_height (module Conn : CONNECTION) height = ++ let dump_block_hashes_till_height (module Conn : Mina_caqti.CONNECTION) height ++ = + Conn.collect_list (dump_block_hashes_till_height_query ~height) () + + let dump_block_hashes_query = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ ORDER BY height\n\ + \ " + +- let dump_block_hashes (module Conn : CONNECTION) = ++ let dump_block_hashes (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_block_hashes_query () + + let dump_user_and_internal_command_info_query = +- Caqti_request.collect Caqti_type.unit Accounts_accessed.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_accessed.typ + {sql| + ( + WITH user_command_ids AS +@@ -428,18 +436,19 @@ module Berkeley = struct INNER JOIN account_identifiers ON public_key_id = receiver_id ) ORDER BY block_id, id |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit -- let dump_user_and_internal_command_info_to_csv (module Conn : CONNECTION) -- output_file = -+ let dump_user_and_internal_command_info_to_csv -+ (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_user_and_internal_command_info_to_csv_query ~output_file) () +- let dump_user_and_internal_command_info (module Conn : CONNECTION) = ++ let dump_user_and_internal_command_info (module Conn : Mina_caqti.CONNECTION) ++ = + Conn.collect_list dump_user_and_internal_command_info_query () let get_account_accessed_count_query = - Caqti_request.find Caqti_type.unit Caqti_type.int @@ -3551,7 +3573,7 @@ index ff6c92ec9b..c0e89348d2 100644 {sql| select count(distinct ids.account_identifier_id) FROM -@@ -439,6 +441,7 @@ module Berkeley = struct +@@ -461,6 +470,7 @@ module Berkeley = struct |sql} From 00cf06dcf761a81e39f092f85b5b99b824872f06 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 8 Apr 2024 14:28:54 +0800 Subject: [PATCH 07/11] dump list to json if comparison failed --- .../berkeley_migration_verifier.ml | 170 ++++++++++++------ src/app/berkeley_migration_verifier/sql.ml | 57 +++--- 2 files changed, 149 insertions(+), 78 deletions(-) diff --git a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml index 615a6b4aab8..cacb9502b80 100644 --- a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml +++ b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml @@ -12,7 +12,22 @@ module Check = struct let err error = Error [ error ] - let of_bool = function true -> Ok | false -> Error [ "failed" ] + let comparison_failed ~left ~right ~left_content ~right_content = + let left_file = Filename.(concat temp_dir_name "left.json") + and right_file = Filename.(concat temp_dir_name "right.json") in + let%map () = + Deferred.all_unit + [ Writer.with_file left_file ~f:(fun w -> + return @@ Writer.write w @@ Yojson.Safe.pretty_to_string left ) + ; Writer.with_file right_file ~f:(fun w -> + return @@ Writer.write w @@ Yojson.Safe.pretty_to_string right ) + ] + in + err + @@ sprintf + "Discrepancies found between %s and %s. To reproduce please run `diff \ + %s %s`." + left_content right_content left_file right_file end let exit_code = ref 0 @@ -23,9 +38,6 @@ module Test = struct let of_check check ~name ~idx ~prefix test_count = { check; name = sprintf "[%d/%d] %s) %s " idx test_count prefix name } - let of_bool b ~name ~idx ~prefix test_count = - of_check (Check.of_bool b) ~name ~idx ~prefix test_count - let print_errors error_messages = if List.length error_messages > 10 then ( Async.printf " Details (truncated to only 10 errors out of %d ): \n" @@ -80,109 +92,157 @@ let no_pending_and_orphaned_blocks_in_migrated_db query_migrated_db ~height = let all_accounts_referred_in_commands_are_recorded migrated_pool = let query_migrated_db = Mina_caqti.query migrated_pool in let open Deferred.Let_syntax in - let%bind expected = + let%bind left = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_user_and_internal_command_info db ) in - let%map result = + let%bind right = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_accounts_accessed db) in - List.equal Sql.Accounts_accessed.equal expected result + if List.equal Sql.Accounts_accessed.equal left right then return Check.ok + else + Check.comparison_failed + ~left:(Sql.Accounts_accessed.list_to_yojson left) + ~right:(Sql.Accounts_accessed.list_to_yojson right) + ~left_content:"Berkeley.user_and_internal_command" + ~right_content:"Berkeley.accounts_accessed" let accounts_created_table_is_correct migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in let open Deferred.Let_syntax in - let%bind expected = + let%bind left = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_accounts_created db) in - let%map result = + let%bind right = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_accounts_created db) in - List.equal Sql.Accounts_created.equal expected result + if List.equal Sql.Accounts_created.equal left right then return Check.ok + else + Check.comparison_failed + ~left:(Sql.Accounts_created.list_to_yojson left) + ~right:(Sql.Accounts_created.list_to_yojson right) + ~left_content:"Mainnet.accounts_created" + ~right_content:"Berkeley.accounts_created" let compare_hashes_till_height migrated_pool mainnet_pool ~height = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in let open Deferred.Let_syntax in - let%bind expected = + let%bind left = + query_mainnet_db ~f:(fun db -> + Sql.Mainnet.dump_block_hashes_till_height db height ) + in + let%bind right = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_block_hashes_till_height db height ) in - let%map result = - query_mainnet_db ~f:(fun db -> - Sql.Mainnet.dump_block_hashes_till_height db height ) - in - List.equal Sql.State_hash_and_ledger_hash.equal expected result + if List.equal Sql.State_hash_and_ledger_hash.equal left right then + return Check.ok + else + Check.comparison_failed + ~left:(Sql.State_hash_and_ledger_hash.list_to_yojson left) + ~right:(Sql.State_hash_and_ledger_hash.list_to_yojson right) + ~left_content:"Mainnet.state_hashes_and_ledger_hashes" + ~right_content:"Berkeley.state_hashes_and_ledger_hashes" let compare_hashes migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in let open Deferred.Let_syntax in - let%bind expected = - query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_block_hashes db) - in - let%map result = + let%bind left = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_block_hashes db) in - List.equal Sql.State_hash_and_ledger_hash.equal expected result + let%bind right = + query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_block_hashes db) + in + + if List.equal Sql.State_hash_and_ledger_hash.equal left right then + return Check.ok + else + Check.comparison_failed + ~left:(Sql.State_hash_and_ledger_hash.list_to_yojson left) + ~right:(Sql.State_hash_and_ledger_hash.list_to_yojson right) + ~left_content:"Mainnet.state_hashes_and_ledger_hashes" + ~right_content:"Berkeley.state_hashes_and_ledger_hashes" let compare_user_commands_till_height migrated_pool mainnet_pool ~height = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in let open Deferred.Let_syntax in - let%bind expected = - query_migrated_db ~f:(fun db -> - Sql.Berkeley.dump_user_commands_till_height db height ) - in - - let%map result = + let%bind left = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_user_commands_till_height db height ) in - List.equal Sql.User_command.equal expected result + let%bind right = + query_migrated_db ~f:(fun db -> + Sql.Berkeley.dump_user_commands_till_height db height ) + in + if List.equal Sql.User_command.equal left right then return Check.ok + else + Check.comparison_failed + ~left:(Sql.User_command.list_to_yojson left) + ~right:(Sql.User_command.list_to_yojson right) + ~left_content:"Mainnet.user_command" + ~right_content:"Berkeley.user_command" let compare_internal_commands_till_height migrated_pool mainnet_pool ~height = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in let open Deferred.Let_syntax in - let%bind expected = - query_migrated_db ~f:(fun db -> - Sql.Berkeley.dump_internal_commands_till_height db height ) - in - - let%map result = + let%bind left = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_internal_commands_till_height db height ) in - List.equal Sql.Internal_command.equal expected result + let%bind right = + query_migrated_db ~f:(fun db -> + Sql.Berkeley.dump_internal_commands_till_height db height ) + in + if List.equal Sql.Internal_command.equal left right then return Check.ok + else + Check.comparison_failed + ~left:(Sql.Internal_command.list_to_yojson left) + ~right:(Sql.Internal_command.list_to_yojson right) + ~left_content:"Mainnet.internal_command" + ~right_content:"Berkeley.internal_command" let compare_user_commands migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in let open Deferred.Let_syntax in - let%bind expected = - query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_user_commands db) - in - - let%map result = + let%bind left = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_user_commands db) in - List.equal Sql.User_command.equal expected result + let%bind right = + query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_user_commands db) + in + if List.equal Sql.User_command.equal left right then return Check.ok + else + Check.comparison_failed + ~left:(Sql.User_command.list_to_yojson left) + ~right:(Sql.User_command.list_to_yojson right) + ~left_content:"Mainnet.user_command" + ~right_content:"Berkeley.user_command" let compare_internal_commands migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in let query_migrated_db = Mina_caqti.query migrated_pool in let open Deferred.Let_syntax in - let%bind expected = - query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_internal_commands db) - in - let%map result = + let%bind left = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_internal_commands db) in - List.equal Sql.Internal_command.equal expected result + let%bind right = + query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_internal_commands db) + in + if List.equal Sql.Internal_command.equal left right then return Check.ok + else + Check.comparison_failed + ~left:(Sql.Internal_command.list_to_yojson left) + ~right:(Sql.Internal_command.list_to_yojson right) + ~left_content:"Mainnet.internal_command" + ~right_content:"Berkeley.internal_command" let compare_ledger_hash ~migrated_replayer_output ~fork_genesis_config_file = let checkpoint_ledger_hash = @@ -245,7 +305,7 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = let%bind check = all_accounts_referred_in_commands_are_recorded migrated_pool in - Test.of_bool check + Test.of_check check ~name: "All accounts referred in internal commands or transactions are \ recorded in the accounts_accessed table." @@ -255,7 +315,7 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = let%bind check = compare_hashes_till_height migrated_pool mainnet_pool ~height in - Test.of_bool check + Test.of_check check ~name:"All block hashes (state_hash, ledger_hashes) are equal" ~idx:4 ~prefix:"D3.4" test_count |> Test.eval ; @@ -264,7 +324,7 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = compare_user_commands_till_height migrated_pool mainnet_pool ~height in - Test.of_bool check ~name:"Verify user commands" ~idx:5 ~prefix:"D3.5" + Test.of_check check ~name:"Verify user commands" ~idx:5 ~prefix:"D3.5" test_count |> Test.eval ; @@ -272,7 +332,7 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = compare_internal_commands_till_height migrated_pool mainnet_pool ~height in - Test.of_bool check ~name:"Verify internal commands" ~idx:6 ~prefix:"D3.6" + Test.of_check check ~name:"Verify internal commands" ~idx:6 ~prefix:"D3.6" test_count |> Test.eval ; @@ -337,7 +397,7 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri let%bind check = all_accounts_referred_in_commands_are_recorded migrated_pool in - Test.of_bool check + Test.of_check check ~name: "All accounts referred in internal commands or transactions are \ recorded in the accounts_accessed table." @@ -347,7 +407,7 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri let%bind check = accounts_created_table_is_correct migrated_pool mainnet_pool in - Test.of_bool check + Test.of_check check ~name: "The content of accounts_created table is correct (by checking \ against pre-migrated database)" @@ -360,18 +420,18 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri in let%bind check = compare_hashes migrated_pool mainnet_pool in - Test.of_bool check + Test.of_check check ~name:"All block hashes (state_hash, ledger_hashes) are equal" ~idx:5 ~prefix:"D3.6" test_count |> Test.eval ; let%bind check = compare_user_commands migrated_pool mainnet_pool in - Test.of_bool check ~name:"Verify user commands" ~idx:6 ~prefix:"D3.7" + Test.of_check check ~name:"Verify user commands" ~idx:6 ~prefix:"D3.7" test_count |> Test.eval ; let%bind check = compare_internal_commands migrated_pool mainnet_pool in - Test.of_bool check ~name:"Verify internal commands" ~idx:7 ~prefix:"D3.8" + Test.of_check check ~name:"Verify internal commands" ~idx:7 ~prefix:"D3.8" test_count |> Test.eval ; diff --git a/src/app/berkeley_migration_verifier/sql.ml b/src/app/berkeley_migration_verifier/sql.ml index fd5407fcb6a..40b6b3f114e 100644 --- a/src/app/berkeley_migration_verifier/sql.ml +++ b/src/app/berkeley_migration_verifier/sql.ml @@ -1,14 +1,17 @@ +open Core open Async open Caqti_async module Accounts_created = struct type t = - { height : String.t - ; public_key : String.t - ; state_hash : String.t - ; creation_fee : String.t + { height : string + ; public_key : string + ; state_hash : string + ; creation_fee : string } - [@@deriving hlist, equal] + [@@deriving hlist, equal, yojson] + + let list_to_yojson xs : Yojson.Safe.t = `List (List.map xs ~f:to_yojson) let typ = Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist @@ -16,8 +19,10 @@ module Accounts_created = struct end module State_hash_and_ledger_hash = struct - type t = { state_hash : String.t; ledger_hash : String.t } - [@@deriving hlist, equal] + type t = { state_hash : string; ledger_hash : string } + [@@deriving hlist, equal, yojson] + + let list_to_yojson xs : Yojson.Safe.t = `List (List.map xs ~f:to_yojson) let typ = Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist @@ -26,16 +31,18 @@ end module User_command = struct type t = - { receiver : String.t - ; fee_payer : String.t - ; nonce : String.t - ; amount : String.t - ; fee : String.t - ; valid_until : String.t - ; memo : String.t - ; hash : String.t + { receiver : string + ; fee_payer : string + ; nonce : string + ; amount : string + ; fee : string + ; valid_until : string + ; memo : string + ; hash : string } - [@@deriving hlist, equal] + [@@deriving hlist, equal, yojson] + + let list_to_yojson xs : Yojson.Safe.t = `List (List.map xs ~f:to_yojson) let typ = Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist @@ -45,13 +52,15 @@ end module Internal_command = struct type t = - { receiver : String.t - ; fee : String.t - ; sequence_no : String.t - ; secondary_sequence_no : String.t - ; hash : String.t + { receiver : string + ; fee : string + ; sequence_no : string + ; secondary_sequence_no : string + ; hash : string } - [@@deriving hlist, equal] + [@@deriving hlist, equal, yojson] + + let list_to_yojson xs : Yojson.Safe.t = `List (List.map xs ~f:to_yojson) let typ = Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist @@ -59,7 +68,9 @@ module Internal_command = struct end module Accounts_accessed = struct - type t = { id : String.t; block_id : String.t } [@@deriving hlist, equal] + type t = { id : string; block_id : string } [@@deriving hlist, equal, yojson] + + let list_to_yojson xs : Yojson.Safe.t = `List (List.map xs ~f:to_yojson) let typ = Mina_caqti.Type_spec.custom_type ~to_hlist ~of_hlist From c7ea25131c23fc5c46d3ae0c4b5d6a0a94e68d52 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 8 Apr 2024 14:45:24 +0800 Subject: [PATCH 08/11] refactor --- .../berkeley_migration_verifier.ml | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml index cacb9502b80..26e337e0bcb 100644 --- a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml +++ b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml @@ -18,16 +18,16 @@ module Check = struct let%map () = Deferred.all_unit [ Writer.with_file left_file ~f:(fun w -> - return @@ Writer.write w @@ Yojson.Safe.pretty_to_string left ) + return @@ Writer.write w @@ Yojson.Safe.pretty_to_string left_content ) ; Writer.with_file right_file ~f:(fun w -> - return @@ Writer.write w @@ Yojson.Safe.pretty_to_string right ) + return @@ Writer.write w @@ Yojson.Safe.pretty_to_string right_content ) ] in err @@ sprintf "Discrepancies found between %s and %s. To reproduce please run `diff \ %s %s`." - left_content right_content left_file right_file + left right left_file right_file end let exit_code = ref 0 @@ -103,10 +103,10 @@ let all_accounts_referred_in_commands_are_recorded migrated_pool = if List.equal Sql.Accounts_accessed.equal left right then return Check.ok else Check.comparison_failed - ~left:(Sql.Accounts_accessed.list_to_yojson left) - ~right:(Sql.Accounts_accessed.list_to_yojson right) - ~left_content:"Berkeley.user_and_internal_command" - ~right_content:"Berkeley.accounts_accessed" + ~left_content:(Sql.Accounts_accessed.list_to_yojson left) + ~right_content:(Sql.Accounts_accessed.list_to_yojson right) + ~left:"Berkeley.user_and_internal_command" + ~right:"Berkeley.accounts_accessed" let accounts_created_table_is_correct migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in @@ -121,10 +121,10 @@ let accounts_created_table_is_correct migrated_pool mainnet_pool = if List.equal Sql.Accounts_created.equal left right then return Check.ok else Check.comparison_failed - ~left:(Sql.Accounts_created.list_to_yojson left) - ~right:(Sql.Accounts_created.list_to_yojson right) - ~left_content:"Mainnet.accounts_created" - ~right_content:"Berkeley.accounts_created" + ~left_content:(Sql.Accounts_created.list_to_yojson left) + ~right_content:(Sql.Accounts_created.list_to_yojson right) + ~left:"Mainnet.accounts_created" + ~right:"Berkeley.accounts_created" let compare_hashes_till_height migrated_pool mainnet_pool ~height = let query_mainnet_db = Mina_caqti.query mainnet_pool in @@ -143,10 +143,10 @@ let compare_hashes_till_height migrated_pool mainnet_pool ~height = return Check.ok else Check.comparison_failed - ~left:(Sql.State_hash_and_ledger_hash.list_to_yojson left) - ~right:(Sql.State_hash_and_ledger_hash.list_to_yojson right) - ~left_content:"Mainnet.state_hashes_and_ledger_hashes" - ~right_content:"Berkeley.state_hashes_and_ledger_hashes" + ~left_content:(Sql.State_hash_and_ledger_hash.list_to_yojson left) + ~right_content:(Sql.State_hash_and_ledger_hash.list_to_yojson right) + ~left:"Mainnet.state_hashes_and_ledger_hashes" + ~right:"Berkeley.state_hashes_and_ledger_hashes" let compare_hashes migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in @@ -163,10 +163,10 @@ let compare_hashes migrated_pool mainnet_pool = return Check.ok else Check.comparison_failed - ~left:(Sql.State_hash_and_ledger_hash.list_to_yojson left) - ~right:(Sql.State_hash_and_ledger_hash.list_to_yojson right) - ~left_content:"Mainnet.state_hashes_and_ledger_hashes" - ~right_content:"Berkeley.state_hashes_and_ledger_hashes" + ~left_content:(Sql.State_hash_and_ledger_hash.list_to_yojson left) + ~right_content:(Sql.State_hash_and_ledger_hash.list_to_yojson right) + ~left:"Mainnet.state_hashes_and_ledger_hashes" + ~right:"Berkeley.state_hashes_and_ledger_hashes" let compare_user_commands_till_height migrated_pool mainnet_pool ~height = let query_mainnet_db = Mina_caqti.query mainnet_pool in @@ -183,10 +183,10 @@ let compare_user_commands_till_height migrated_pool mainnet_pool ~height = if List.equal Sql.User_command.equal left right then return Check.ok else Check.comparison_failed - ~left:(Sql.User_command.list_to_yojson left) - ~right:(Sql.User_command.list_to_yojson right) - ~left_content:"Mainnet.user_command" - ~right_content:"Berkeley.user_command" + ~left_content:(Sql.User_command.list_to_yojson left) + ~right_content:(Sql.User_command.list_to_yojson right) + ~left:"Mainnet.user_command" + ~right:"Berkeley.user_command" let compare_internal_commands_till_height migrated_pool mainnet_pool ~height = let query_mainnet_db = Mina_caqti.query mainnet_pool in @@ -203,10 +203,10 @@ let compare_internal_commands_till_height migrated_pool mainnet_pool ~height = if List.equal Sql.Internal_command.equal left right then return Check.ok else Check.comparison_failed - ~left:(Sql.Internal_command.list_to_yojson left) - ~right:(Sql.Internal_command.list_to_yojson right) - ~left_content:"Mainnet.internal_command" - ~right_content:"Berkeley.internal_command" + ~left_content:(Sql.Internal_command.list_to_yojson left) + ~right_content:(Sql.Internal_command.list_to_yojson right) + ~left:"Mainnet.internal_command" + ~right:"Berkeley.internal_command" let compare_user_commands migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in @@ -221,10 +221,10 @@ let compare_user_commands migrated_pool mainnet_pool = if List.equal Sql.User_command.equal left right then return Check.ok else Check.comparison_failed - ~left:(Sql.User_command.list_to_yojson left) - ~right:(Sql.User_command.list_to_yojson right) - ~left_content:"Mainnet.user_command" - ~right_content:"Berkeley.user_command" + ~left_content:(Sql.User_command.list_to_yojson left) + ~right_content:(Sql.User_command.list_to_yojson right) + ~left:"Mainnet.user_command" + ~right:"Berkeley.user_command" let compare_internal_commands migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in @@ -239,10 +239,10 @@ let compare_internal_commands migrated_pool mainnet_pool = if List.equal Sql.Internal_command.equal left right then return Check.ok else Check.comparison_failed - ~left:(Sql.Internal_command.list_to_yojson left) - ~right:(Sql.Internal_command.list_to_yojson right) - ~left_content:"Mainnet.internal_command" - ~right_content:"Berkeley.internal_command" + ~left_content:(Sql.Internal_command.list_to_yojson left) + ~right_content:(Sql.Internal_command.list_to_yojson right) + ~left:"Mainnet.internal_command" + ~right:"Berkeley.internal_command" let compare_ledger_hash ~migrated_replayer_output ~fork_genesis_config_file = let checkpoint_ledger_hash = From 39bf8ae9f38a1afefc40ffc4bc362c12f69a1f25 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 8 Apr 2024 15:00:23 +0800 Subject: [PATCH 09/11] update caqti-upgrade.patch --- buildkite/scripts/caqti-upgrade.patch | 469 +++++++++++++------------- 1 file changed, 236 insertions(+), 233 deletions(-) diff --git a/buildkite/scripts/caqti-upgrade.patch b/buildkite/scripts/caqti-upgrade.patch index a93e42fde32..202df08525e 100644 --- a/buildkite/scripts/caqti-upgrade.patch +++ b/buildkite/scripts/caqti-upgrade.patch @@ -63,7 +63,7 @@ index c69e1805d1..aac3552028 100644 SELECT COUNT( * ) FROM blocks WHERE parent_id IS NULL diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml -index 8b90db5952..939aaa8864 100644 +index d4f0e9c585..a954162791 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -29,6 +29,7 @@ open Mina_block @@ -284,7 +284,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -727,7 +728,7 @@ module Zkapp_permissions = struct +@@ -710,7 +711,7 @@ module Zkapp_permissions = struct let load (module Conn : CONNECTION) id = Conn.find @@ -293,7 +293,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -780,7 +781,7 @@ module Zkapp_timing_info = struct +@@ -763,7 +764,7 @@ module Zkapp_timing_info = struct let load (module Conn : CONNECTION) id = Conn.find @@ -302,7 +302,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -801,7 +802,7 @@ module Zkapp_uri = struct +@@ -784,7 +785,7 @@ module Zkapp_uri = struct let load (module Conn : CONNECTION) id = Conn.find @@ -311,7 +311,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "value" ]) ) id end -@@ -894,7 +895,7 @@ module Zkapp_updates = struct +@@ -877,7 +878,7 @@ module Zkapp_updates = struct let load (module Conn : CONNECTION) id = Conn.find @@ -320,7 +320,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -922,7 +923,7 @@ module Zkapp_balance_bounds = struct +@@ -905,7 +906,7 @@ module Zkapp_balance_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -329,7 +329,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -951,7 +952,7 @@ module Zkapp_nonce_bounds = struct +@@ -934,7 +935,7 @@ module Zkapp_nonce_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -338,7 +338,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1035,7 +1036,7 @@ module Zkapp_account_precondition = struct +@@ -1018,7 +1019,7 @@ module Zkapp_account_precondition = struct let load (module Conn : CONNECTION) id = Conn.find @@ -347,7 +347,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1063,7 +1064,7 @@ module Zkapp_token_id_bounds = struct +@@ -1046,7 +1047,7 @@ module Zkapp_token_id_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -356,7 +356,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1095,7 +1096,7 @@ module Zkapp_timestamp_bounds = struct +@@ -1078,7 +1079,7 @@ module Zkapp_timestamp_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -365,7 +365,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1123,7 +1124,7 @@ module Zkapp_length_bounds = struct +@@ -1106,7 +1107,7 @@ module Zkapp_length_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -374,7 +374,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1151,7 +1152,7 @@ module Zkapp_amount_bounds = struct +@@ -1134,7 +1135,7 @@ module Zkapp_amount_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -383,7 +383,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1186,7 +1187,7 @@ module Zkapp_global_slot_bounds = struct +@@ -1169,7 +1170,7 @@ module Zkapp_global_slot_bounds = struct let load (module Conn : CONNECTION) id = Conn.find @@ -392,7 +392,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1215,7 +1216,7 @@ module Timing_info = struct +@@ -1198,7 +1199,7 @@ module Timing_info = struct Account_identifiers.find (module Conn) account_id in Conn.find @@ -401,7 +401,7 @@ index 8b90db5952..939aaa8864 100644 {sql| SELECT account_identifier_id, initial_minimum_balance, cliff_time, cliff_amount, vesting_period, vesting_increment -@@ -1227,7 +1228,7 @@ module Timing_info = struct +@@ -1210,7 +1211,7 @@ module Timing_info = struct let find_by_account_identifier_id_opt (module Conn : CONNECTION) account_identifier_id = Conn.find_opt @@ -410,7 +410,7 @@ index 8b90db5952..939aaa8864 100644 {sql| SELECT account_identifier_id, initial_minimum_balance, cliff_time, cliff_amount, vesting_period, vesting_increment -@@ -1270,7 +1271,7 @@ module Timing_info = struct +@@ -1253,7 +1254,7 @@ module Timing_info = struct in match%bind Conn.find_opt @@ -419,7 +419,7 @@ index 8b90db5952..939aaa8864 100644 {sql| SELECT id FROM timing_info WHERE account_identifier_id = ? AND initial_minimum_balance = ? -@@ -1284,7 +1285,7 @@ module Timing_info = struct +@@ -1267,7 +1268,7 @@ module Timing_info = struct return id | None -> Conn.find @@ -428,7 +428,7 @@ index 8b90db5952..939aaa8864 100644 {sql| INSERT INTO timing_info (account_identifier_id,initial_minimum_balance, cliff_time, cliff_amount, vesting_period, vesting_increment) -@@ -1295,13 +1296,13 @@ module Timing_info = struct +@@ -1278,13 +1279,13 @@ module Timing_info = struct let load (module Conn : CONNECTION) id = Conn.find @@ -444,7 +444,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1310,13 +1311,13 @@ module Snarked_ledger_hash = struct +@@ -1293,13 +1294,13 @@ module Snarked_ledger_hash = struct let find (module Conn : CONNECTION) (t : Frozen_ledger_hash.t) = let hash = Frozen_ledger_hash.to_base58_check t in Conn.find @@ -460,7 +460,7 @@ index 8b90db5952..939aaa8864 100644 "SELECT value FROM snarked_ledger_hashes WHERE id = ?" ) id -@@ -1326,7 +1327,7 @@ module Snarked_ledger_hash = struct +@@ -1309,7 +1310,7 @@ module Snarked_ledger_hash = struct let hash = Frozen_ledger_hash.to_base58_check t in match%bind Conn.find_opt @@ -469,7 +469,7 @@ index 8b90db5952..939aaa8864 100644 "SELECT id FROM snarked_ledger_hashes WHERE value = ?" ) hash with -@@ -1334,13 +1335,13 @@ module Snarked_ledger_hash = struct +@@ -1317,13 +1318,13 @@ module Snarked_ledger_hash = struct return id | None -> Conn.find @@ -485,7 +485,7 @@ index 8b90db5952..939aaa8864 100644 "SELECT value FROM snarked_ledger_hashes WHERE id = ?" ) id end -@@ -1376,7 +1377,7 @@ module Zkapp_epoch_ledger = struct +@@ -1359,7 +1360,7 @@ module Zkapp_epoch_ledger = struct let load (module Conn : CONNECTION) id = Conn.find @@ -494,7 +494,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1436,7 +1437,7 @@ module Zkapp_epoch_data = struct +@@ -1419,7 +1420,7 @@ module Zkapp_epoch_data = struct let load (module Conn : CONNECTION) id = Conn.find @@ -503,7 +503,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1511,7 +1512,7 @@ module Zkapp_network_precondition = struct +@@ -1494,7 +1495,7 @@ module Zkapp_network_precondition = struct let load (module Conn : CONNECTION) id = Conn.find @@ -512,7 +512,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1603,7 +1604,7 @@ module Zkapp_events = struct +@@ -1586,7 +1587,7 @@ module Zkapp_events = struct let load (module Conn : CONNECTION) id = Conn.find @@ -521,7 +521,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "element_ids" ]) ) id end -@@ -1755,7 +1756,7 @@ module Zkapp_account_update_body = struct +@@ -1738,7 +1739,7 @@ module Zkapp_account_update_body = struct let load (module Conn : CONNECTION) id = Conn.find @@ -530,7 +530,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1786,7 +1787,7 @@ module Zkapp_account_update = struct +@@ -1769,7 +1770,7 @@ module Zkapp_account_update = struct let load (module Conn : CONNECTION) id = Conn.find @@ -539,7 +539,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1830,7 +1831,7 @@ module Zkapp_fee_payer_body = struct +@@ -1813,7 +1814,7 @@ module Zkapp_fee_payer_body = struct let load (module Conn : CONNECTION) id = Conn.find @@ -548,7 +548,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1887,7 +1888,7 @@ module Epoch_data = struct +@@ -1870,7 +1871,7 @@ module Epoch_data = struct let load (module Conn : CONNECTION) id = Conn.find @@ -557,7 +557,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -1928,13 +1929,13 @@ module User_command = struct +@@ -1911,13 +1912,13 @@ module User_command = struct let find (module Conn : CONNECTION) ~(transaction_hash : Transaction_hash.t) ~v1_transaction_hash = Conn.find_opt @@ -573,7 +573,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id -@@ -1978,7 +1979,7 @@ module User_command = struct +@@ -1961,7 +1962,7 @@ module User_command = struct in (* TODO: Converting these uint64s to int64 can overflow; see #5419 *) Conn.find @@ -582,7 +582,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.insert_into_cols ~returning:"id" ~table_name ~tannot:(function | "command_type" -> Some "user_command_type" | _ -> None ) -@@ -2025,7 +2026,7 @@ module User_command = struct +@@ -2008,7 +2009,7 @@ module User_command = struct Public_key.add_if_doesn't_exist (module Conn) user_cmd.receiver in Conn.find @@ -591,7 +591,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.insert_into_cols ~returning:"id" ~table_name ~tannot:(function | "command_type" -> Some "user_command_type" | _ -> None ) -@@ -2066,14 +2067,14 @@ module User_command = struct +@@ -2049,14 +2050,14 @@ module User_command = struct let find_opt (module Conn : CONNECTION) ~(transaction_hash : Transaction_hash.t) = Conn.find_opt @@ -608,7 +608,7 @@ index 8b90db5952..939aaa8864 100644 @@ Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names ) id -@@ -2149,8 +2150,8 @@ module Internal_command = struct +@@ -2132,8 +2133,8 @@ module Internal_command = struct let find_opt (module Conn : CONNECTION) ~(v1_transaction_hash : bool) ~(transaction_hash : Transaction_hash.t) ~(command_type : string) = Conn.find_opt @@ -619,7 +619,7 @@ index 8b90db5952..939aaa8864 100644 Caqti_type.int (Mina_caqti.select_cols ~select:"id" ~table_name ~tannot:(function -@@ -2161,7 +2162,7 @@ module Internal_command = struct +@@ -2144,7 +2145,7 @@ module Internal_command = struct let load (module Conn : CONNECTION) ~(id : int) = Conn.find @@ -628,7 +628,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id -@@ -2182,7 +2183,7 @@ module Internal_command = struct +@@ -2165,7 +2166,7 @@ module Internal_command = struct Public_key.add_if_doesn't_exist (module Conn) internal_cmd.receiver in Conn.find @@ -637,7 +637,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.insert_into_cols ~returning:"id" ~table_name ~tannot:(function | "command_type" -> Some "internal_command_type" | _ -> None -@@ -2227,7 +2228,7 @@ module Fee_transfer = struct +@@ -2210,7 +2211,7 @@ module Fee_transfer = struct in Ok { kind; receiver_id; fee; hash } in @@ -646,7 +646,7 @@ index 8b90db5952..939aaa8864 100644 Caqti_type.custom ~encode ~decode rep let add_if_doesn't_exist (module Conn : CONNECTION) -@@ -2249,7 +2250,7 @@ module Fee_transfer = struct +@@ -2232,7 +2233,7 @@ module Fee_transfer = struct Public_key.add_if_doesn't_exist (module Conn) pk in Conn.find @@ -655,7 +655,7 @@ index 8b90db5952..939aaa8864 100644 {sql| INSERT INTO internal_commands (command_type, receiver_id, fee, hash) VALUES (?::internal_command_type, ?, ?, ?) -@@ -2277,7 +2278,7 @@ module Coinbase = struct +@@ -2260,7 +2261,7 @@ module Coinbase = struct let decode (_, receiver_id, amount, hash) = Ok { receiver_id; amount; hash } in @@ -664,7 +664,7 @@ index 8b90db5952..939aaa8864 100644 Caqti_type.custom ~encode ~decode rep let add_if_doesn't_exist (module Conn : CONNECTION) -@@ -2298,7 +2299,7 @@ module Coinbase = struct +@@ -2281,7 +2282,7 @@ module Coinbase = struct Public_key.add_if_doesn't_exist (module Conn) pk in Conn.find @@ -673,7 +673,7 @@ index 8b90db5952..939aaa8864 100644 {sql| INSERT INTO internal_commands (command_type, receiver_id, fee, hash) VALUES (?::internal_command_type, ?, ?, ?) -@@ -2337,7 +2338,7 @@ module Block_and_internal_command = struct +@@ -2320,7 +2321,7 @@ module Block_and_internal_command = struct Option.map ~f:Transaction_status.Failure.to_string failure_reason in Conn.exec @@ -682,7 +682,7 @@ index 8b90db5952..939aaa8864 100644 {sql| INSERT INTO blocks_internal_commands (block_id, internal_command_id, -@@ -2358,8 +2359,8 @@ module Block_and_internal_command = struct +@@ -2341,8 +2342,8 @@ module Block_and_internal_command = struct let find (module Conn : CONNECTION) ~block_id ~internal_command_id ~sequence_no ~secondary_sequence_no = Conn.find_opt @@ -693,7 +693,7 @@ index 8b90db5952..939aaa8864 100644 Caqti_type.string {sql| SELECT 'exists' FROM blocks_internal_commands WHERE block_id = $1 -@@ -2390,8 +2391,8 @@ module Block_and_internal_command = struct +@@ -2373,8 +2374,8 @@ module Block_and_internal_command = struct ~sequence_no ~secondary_sequence_no = let comma_cols = String.concat Fields.names ~sep:"," in Conn.find @@ -704,7 +704,7 @@ index 8b90db5952..939aaa8864 100644 typ (sprintf {sql| SELECT %s FROM blocks_internal_commands -@@ -2426,7 +2427,7 @@ module Block_and_signed_command = struct +@@ -2409,7 +2410,7 @@ module Block_and_signed_command = struct Option.map ~f:Transaction_status.Failure.to_string failure_reason in Conn.exec @@ -713,7 +713,7 @@ index 8b90db5952..939aaa8864 100644 {sql| INSERT INTO blocks_user_commands (block_id, user_command_id, -@@ -2456,8 +2457,8 @@ module Block_and_signed_command = struct +@@ -2439,8 +2440,8 @@ module Block_and_signed_command = struct let open Deferred.Result.Let_syntax in match%bind Conn.find_opt @@ -724,7 +724,7 @@ index 8b90db5952..939aaa8864 100644 Caqti_type.string {sql| SELECT 'exists' FROM blocks_user_commands WHERE block_id = $1 -@@ -2476,8 +2477,8 @@ module Block_and_signed_command = struct +@@ -2459,8 +2460,8 @@ module Block_and_signed_command = struct let load (module Conn : CONNECTION) ~block_id ~user_command_id ~sequence_no = let comma_cols = String.concat Fields.names ~sep:"," in Conn.find @@ -735,7 +735,7 @@ index 8b90db5952..939aaa8864 100644 typ (sprintf {sql| SELECT %s FROM blocks_user_commands -@@ -2511,7 +2512,7 @@ module Zkapp_account_update_failures = struct +@@ -2494,7 +2495,7 @@ module Zkapp_account_update_failures = struct let load (module Conn : CONNECTION) id = Conn.find @@ -744,7 +744,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:[ "index"; "failures" ] ) ) id -@@ -2554,8 +2555,7 @@ module Block_and_zkapp_command = struct +@@ -2537,8 +2538,7 @@ module Block_and_zkapp_command = struct in Mina_caqti.select_insert_into_cols ~select: @@ -754,7 +754,7 @@ index 8b90db5952..939aaa8864 100644 ~table_name ~cols: ( [ "block_id" -@@ -2578,8 +2578,8 @@ module Block_and_zkapp_command = struct +@@ -2561,8 +2561,8 @@ module Block_and_zkapp_command = struct let load (module Conn : CONNECTION) ~block_id ~zkapp_command_id ~sequence_no = let comma_cols = String.concat Fields.names ~sep:"," in Conn.find @@ -765,7 +765,7 @@ index 8b90db5952..939aaa8864 100644 typ (Mina_caqti.select_cols ~table_name ~select:comma_cols ~cols:[ "block_id"; "zkapp_command_id"; "sequence_no" ] -@@ -2589,7 +2589,7 @@ module Block_and_zkapp_command = struct +@@ -2572,7 +2572,7 @@ module Block_and_zkapp_command = struct let all_from_block (module Conn : CONNECTION) ~block_id = let comma_cols = String.concat Fields.names ~sep:"," in Conn.collect_list @@ -774,7 +774,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols ~table_name ~select:comma_cols ~cols:[ "block_id" ] () ) ) block_id -@@ -2661,7 +2661,7 @@ module Zkapp_account = struct +@@ -2644,7 +2644,7 @@ module Zkapp_account = struct let load (module Conn : CONNECTION) id = Conn.find @@ -783,7 +783,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name ~cols:Fields.names) ) id end -@@ -2705,8 +2705,8 @@ module Accounts_accessed = struct +@@ -2688,8 +2688,8 @@ module Accounts_accessed = struct let find_opt (module Conn : CONNECTION) ~block_id ~account_identifier_id = let comma_cols = String.concat Fields.names ~sep:"," in Conn.find_opt @@ -794,7 +794,7 @@ index 8b90db5952..939aaa8864 100644 typ (sprintf {sql| SELECT %s -@@ -2777,7 +2777,7 @@ module Accounts_accessed = struct +@@ -2760,7 +2760,7 @@ module Accounts_accessed = struct } in Mina_caqti.select_insert_into_cols @@ -803,7 +803,7 @@ index 8b90db5952..939aaa8864 100644 ~table_name ~cols:(Fields.names, typ) (module Conn) account_accessed -@@ -2793,7 +2793,7 @@ module Accounts_accessed = struct +@@ -2776,7 +2776,7 @@ module Accounts_accessed = struct let all_from_block (module Conn : CONNECTION) block_id = let comma_cols = String.concat Fields.names ~sep:"," in Conn.collect_list @@ -812,7 +812,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols ~select:comma_cols ~table_name ~cols:[ "block_id" ] () ) ) block_id -@@ -2818,7 +2818,7 @@ module Accounts_created = struct +@@ -2801,7 +2801,7 @@ module Accounts_created = struct in let creation_fee = Currency.Fee.to_string creation_fee in Mina_caqti.select_insert_into_cols @@ -821,7 +821,7 @@ index 8b90db5952..939aaa8864 100644 ~table_name ~cols:(Fields.names, typ) (module Conn) { block_id; account_identifier_id; creation_fee } -@@ -2833,7 +2833,7 @@ module Accounts_created = struct +@@ -2816,7 +2816,7 @@ module Accounts_created = struct let all_from_block (module Conn : CONNECTION) block_id = Conn.collect_list @@ -830,7 +830,7 @@ index 8b90db5952..939aaa8864 100644 {sql| SELECT block_id, account_identifier_id, creation_fee FROM accounts_created WHERE block_id = ? -@@ -2899,14 +2899,14 @@ module Block = struct +@@ -2882,14 +2882,14 @@ module Block = struct "SELECT id FROM blocks WHERE state_hash = ?" ) (State_hash.to_base58_check state_hash) @@ -848,7 +848,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.select_cols_from_id ~table_name:"blocks" ~cols:Fields.names) ) id -@@ -3011,7 +3011,7 @@ module Block = struct +@@ -2994,7 +2994,7 @@ module Block = struct let blockchain_state = Protocol_state.blockchain_state protocol_state in let%bind block_id = Conn.find @@ -857,7 +857,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.insert_into_cols ~returning:"id" ~table_name ~tannot:(function | "chain_status" -> -@@ -3393,9 +3393,7 @@ module Block = struct +@@ -3376,9 +3376,7 @@ module Block = struct in (* we don't need to specify all types here, just the ones that sql may infer incorrectly *) @@ -868,7 +868,7 @@ index 8b90db5952..939aaa8864 100644 | Bool -> Some "BOOL" | Int -> -@@ -3408,39 +3406,36 @@ module Block = struct +@@ -3391,39 +3389,36 @@ module Block = struct Some "BIGINT" | Float -> Some "FLOAT" @@ -927,7 +927,7 @@ index 8b90db5952..939aaa8864 100644 match typ with | Bool -> Bool.to_string value -@@ -3468,45 +3463,30 @@ module Block = struct +@@ -3451,45 +3446,30 @@ module Block = struct (* we are ignoring the enum annotation in this context because it's not always valid to apply *) (* NOTE: we assume enum values do not contain special characters (eg "'") *) "'" ^ value ^ "'" @@ -995,7 +995,7 @@ index 8b90db5952..939aaa8864 100644 in let render_row (type a) (typ : a Caqti_type.t) (value : a) : string = "(" ^ String.concat ~sep:"," (render_type typ value) ^ ")" -@@ -3557,8 +3537,8 @@ module Block = struct +@@ -3540,8 +3520,8 @@ module Block = struct in let%map entries = Conn.collect_list @@ -1006,7 +1006,7 @@ index 8b90db5952..939aaa8864 100644 query ) () in -@@ -3576,7 +3556,7 @@ module Block = struct +@@ -3559,7 +3539,7 @@ module Block = struct String.concat ~sep:"," @@ List.map ~f:(render_row typ) values in Conn.collect_list @@ -1015,7 +1015,7 @@ index 8b90db5952..939aaa8864 100644 (sprintf "INSERT INTO %s (%s) VALUES %s RETURNING id" table fields_sql values_sql ) ) () ) -@@ -3928,7 +3908,7 @@ module Block = struct +@@ -3911,7 +3891,7 @@ module Block = struct let ids_sql = String.concat ~sep:"," ids in let parent_ids_sql = String.concat ~sep:"," parent_ids in Conn.exec @@ -1024,7 +1024,7 @@ index 8b90db5952..939aaa8864 100644 (sprintf "UPDATE %s AS b SET parent_id = data.parent_id FROM (SELECT \ unnest(array[%s]) as id, unnest(array[%s]) as parent_id) AS \ -@@ -4138,7 +4118,7 @@ module Block = struct +@@ -4121,7 +4101,7 @@ module Block = struct Some id ) in Conn.find @@ -1033,7 +1033,7 @@ index 8b90db5952..939aaa8864 100644 (Mina_caqti.insert_into_cols ~returning:"id" ~table_name ~tannot:(function | "sub_window_densities" -> -@@ -4299,8 +4279,8 @@ module Block = struct +@@ -4282,8 +4262,8 @@ module Block = struct let set_parent_id_if_null (module Conn : CONNECTION) ~parent_hash ~(parent_id : int) = Conn.exec @@ -1044,7 +1044,7 @@ index 8b90db5952..939aaa8864 100644 {sql| UPDATE blocks SET parent_id = ? WHERE parent_hash = ? AND parent_id IS NULL -@@ -4316,8 +4296,8 @@ module Block = struct +@@ -4299,8 +4279,8 @@ module Block = struct in let columns = concat Fields.names in Conn.collect_list @@ -1055,7 +1055,7 @@ index 8b90db5952..939aaa8864 100644 typ (sprintf {sql| WITH RECURSIVE chain AS ( -@@ -4343,37 +4323,37 @@ module Block = struct +@@ -4326,37 +4306,37 @@ module Block = struct let get_highest_canonical_block_opt (module Conn : CONNECTION) = Conn.find_opt @@ -1102,7 +1102,7 @@ index 8b90db5952..939aaa8864 100644 {sql| UPDATE blocks SET chain_status='orphaned' WHERE height = $2 AND state_hash <> $1 -@@ -4462,7 +4442,7 @@ module Block = struct +@@ -4445,7 +4425,7 @@ module Block = struct | None, Some num_blocks -> ( match%map Conn.find_opt @@ -1111,7 +1111,7 @@ index 8b90db5952..939aaa8864 100644 "SELECT MAX(height) FROM blocks" ) () with -@@ -4478,8 +4458,8 @@ module Block = struct +@@ -4461,8 +4441,8 @@ module Block = struct let%bind () = (* Delete user commands from old blocks. *) Conn.exec @@ -1122,7 +1122,7 @@ index 8b90db5952..939aaa8864 100644 "DELETE FROM user_commands\n\ WHERE id IN\n\ (SELECT user_command_id FROM blocks_user_commands\n\ -@@ -4490,8 +4470,8 @@ module Block = struct +@@ -4473,8 +4453,8 @@ module Block = struct let%bind () = (* Delete old blocks. *) Conn.exec @@ -1133,7 +1133,7 @@ index 8b90db5952..939aaa8864 100644 "DELETE FROM blocks WHERE blocks.height < ? OR blocks.timestamp < \ ?" ) (height, timestamp) -@@ -4499,7 +4479,7 @@ module Block = struct +@@ -4482,7 +4462,7 @@ module Block = struct let%bind () = (* Delete orphaned internal commands. *) Conn.exec @@ -1142,7 +1142,7 @@ index 8b90db5952..939aaa8864 100644 "DELETE FROM internal_commands\n\ WHERE id NOT IN\n\ (SELECT internal_commands.id FROM internal_commands\n\ -@@ -4510,7 +4490,7 @@ module Block = struct +@@ -4493,7 +4473,7 @@ module Block = struct let%bind () = (* Delete orphaned snarked ledger hashes. *) Conn.exec @@ -1151,7 +1151,7 @@ index 8b90db5952..939aaa8864 100644 "DELETE FROM snarked_ledger_hashes\n\ WHERE id NOT IN\n\ (SELECT snarked_ledger_hash_id FROM blocks)" ) -@@ -4519,7 +4499,7 @@ module Block = struct +@@ -4502,7 +4482,7 @@ module Block = struct let%bind () = (* Delete orphaned public keys. *) Conn.exec @@ -1160,7 +1160,7 @@ index 8b90db5952..939aaa8864 100644 "DELETE FROM public_keys\n\ WHERE id NOT IN (SELECT fee_payer_id FROM user_commands)\n\ AND id NOT IN (SELECT source_id FROM user_commands)\n\ -@@ -4944,7 +4924,13 @@ let setup_server ~metrics_server_port ~constraint_constants ~logger +@@ -4927,7 +4907,13 @@ let setup_server ~metrics_server_port ~constraint_constants ~logger Strict_pipe.Writer.write extensional_block_writer extensional_block ) ] in @@ -1455,10 +1455,10 @@ index 297081b7b3..0bdddcfdaa 100644 FROM accounts_accessed WHERE block_id = $1 diff --git a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml -index f377d693fc..d183565cbe 100644 +index 26e337e0bc..29c2059c57 100644 --- a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml +++ b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml -@@ -284,10 +284,18 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = +@@ -273,10 +273,18 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = let mainnet_archive_uri = Uri.of_string mainnet_archive_uri in let migrated_archive_uri = Uri.of_string migrated_archive_uri in let mainnet_pool = @@ -1479,7 +1479,7 @@ index f377d693fc..d183565cbe 100644 in match (mainnet_pool, migrated_pool) with -@@ -374,11 +382,15 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri +@@ -364,11 +372,15 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri let mainnet_archive_uri = Uri.of_string mainnet_archive_uri in let migrated_archive_uri = Uri.of_string migrated_archive_uri in @@ -1498,81 +1498,81 @@ index f377d693fc..d183565cbe 100644 match (mainnet_pool, migrated_pool) with diff --git a/src/app/berkeley_migration_verifier/sql.ml b/src/app/berkeley_migration_verifier/sql.ml -index ff6c92ec9b..36b4e32401 100644 +index 40b6b3f114..918d9377d1 100644 --- a/src/app/berkeley_migration_verifier/sql.ml +++ b/src/app/berkeley_migration_verifier/sql.ml -@@ -26,7 +26,7 @@ module Mainnet = struct - WHERE receiver_account_creation_fee_paid IS NOT NULL - ) - ORDER BY height, public_key |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = - Conn.exec (dump_accounts_created_to_csv_query ~output_file) () -@@ -38,7 +38,7 @@ module Mainnet = struct - " SELECT state_hash, ledger_hash FROM blocks\n\ - \ WHERE chain_status = 'canonical'\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes_till_height_query ~output_file ~height = - dump_sql_to_csv output_file -@@ -49,7 +49,7 @@ module Mainnet = struct - \ WHERE chain_status = 'canonical'\n\ - \ AND height <= %d ORDER BY height\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes_till_height (module Conn : CONNECTION) output_file - height = -@@ -63,7 +63,7 @@ module Mainnet = struct - \ WHERE chain_status = 'canonical'\n\ - \ ORDER BY height\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes (module Conn : CONNECTION) output_file = - Conn.exec (dump_block_hashes_query ~output_file) () -@@ -87,7 +87,7 @@ module Mainnet = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_user_commands_till_height (module Conn : CONNECTION) output_file - height = -@@ -112,7 +112,7 @@ module Mainnet = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ - \ \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_internal_commands_till_height (module Conn : CONNECTION) output_file - height = -@@ -135,7 +135,7 @@ module Mainnet = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_user_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_user_commands_query ~output_file) () -@@ -158,13 +158,13 @@ module Mainnet = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ - \ \n\ - \ " ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_internal_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_internal_commands_query ~output_file) () +@@ -79,7 +79,7 @@ end + + module Mainnet = struct + let dump_accounts_created_query = +- Caqti_request.collect Caqti_type.unit Accounts_created.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_created.typ + {sql| + ( SELECT height, value AS public_key, state_hash, receiver_account_creation_fee_paid AS creation_fee + FROM blocks_user_commands +@@ -105,13 +105,13 @@ module Mainnet = struct + + let dump_state_and_ledger_hashes_query = + (* Workaround for replacing output file as caqti has an issue with using ? in place of FILE argument*) +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + " SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ " + + let dump_block_hashes_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + (Printf.sprintf + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ +@@ -123,7 +123,7 @@ module Mainnet = struct + Conn.collect_list (dump_block_hashes_till_height_query ~height) () + + let dump_block_hashes_query = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ +@@ -134,7 +134,7 @@ module Mainnet = struct + Conn.collect_list dump_block_hashes_query () + + let dump_user_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + (Printf.sprintf + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ +@@ -156,7 +156,7 @@ module Mainnet = struct + Conn.collect_list (dump_user_commands_till_height_query ~height) () + + let dump_internal_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -178,7 +178,7 @@ module Mainnet = struct + Conn.collect_list (dump_internal_commands_till_height_query ~height) () + + let dump_user_commands_query = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ +@@ -198,7 +198,7 @@ module Mainnet = struct + Conn.collect_list dump_user_commands_query () + + let dump_internal_commands_query = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -219,7 +219,7 @@ module Mainnet = struct + Conn.collect_list dump_internal_commands_query () let mark_chain_till_fork_block_as_canonical_query = - Caqti_request.exec Caqti_type.string @@ -1580,15 +1580,17 @@ index ff6c92ec9b..36b4e32401 100644 {sql| UPDATE blocks Set chain_status = 'canonical' -@@ -202,13 +202,13 @@ module Berkeley = struct - JOIN account_identifiers ON account_identifier_id = account_identifiers.id - JOIN public_keys ON public_key_id = public_keys.id - ORDER BY height, public_key |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit +@@ -248,7 +248,7 @@ end - let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = - Conn.exec (dump_accounts_created_to_csv_query ~output_file) () + module Berkeley = struct + let dump_accounts_created_query = +- Caqti_request.collect Caqti_type.unit Accounts_created.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_created.typ + {sql| + SELECT height, value AS public_key, state_hash, creation_fee + FROM accounts_created +@@ -261,7 +261,7 @@ module Berkeley = struct + Conn.collect_list dump_accounts_created_query () let height_query = - Caqti_request.find Caqti_type.unit Caqti_type.int @@ -1596,7 +1598,7 @@ index ff6c92ec9b..36b4e32401 100644 {sql| SELECT height from blocks order by height desc limit 1; |sql} -@@ -216,7 +216,7 @@ module Berkeley = struct +@@ -269,7 +269,7 @@ module Berkeley = struct let block_height (module Conn : CONNECTION) = Conn.find height_query () let canonical_blocks_count_till_height_query = @@ -1605,7 +1607,7 @@ index ff6c92ec9b..36b4e32401 100644 {sql| WITH RECURSIVE chain AS ( -@@ -234,7 +234,7 @@ module Berkeley = struct +@@ -287,7 +287,7 @@ module Berkeley = struct Conn.find canonical_blocks_count_till_height_query height let blocks_count_query = @@ -1614,79 +1616,80 @@ index ff6c92ec9b..36b4e32401 100644 {sql| SELECT count(*) FROM blocks ; |sql} -@@ -260,7 +260,7 @@ module Berkeley = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_user_commands_till_height (module Conn : CONNECTION) output_file - height = -@@ -285,7 +285,7 @@ module Berkeley = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, \ - command_type \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_internal_commands_till_height (module Conn : CONNECTION) output_file - height = -@@ -308,7 +308,7 @@ module Berkeley = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_user_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_user_commands_query ~output_file) () -@@ -330,7 +330,7 @@ module Berkeley = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, \ - command_type \n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_internal_commands (module Conn : CONNECTION) output_file = - Conn.exec (dump_internal_commands_query ~output_file) () -@@ -343,7 +343,7 @@ module Berkeley = struct +@@ -295,7 +295,7 @@ module Berkeley = struct + let blocks_count (module Conn : CONNECTION) = Conn.find blocks_count_query () + + let dump_user_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + (Printf.sprintf + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ +@@ -317,7 +317,7 @@ module Berkeley = struct + Conn.collect_list (dump_user_commands_till_height_query ~height) () + + let dump_internal_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -339,7 +339,7 @@ module Berkeley = struct + Conn.collect_list (dump_internal_commands_till_height_query ~height) () + + let dump_user_commands_query = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ +@@ -359,7 +359,7 @@ module Berkeley = struct + Conn.collect_list dump_user_commands_query () + + let dump_internal_commands_query = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ + secondary_sequence_no FROM blocks_internal_commands \n\ +@@ -379,7 +379,7 @@ module Berkeley = struct + Conn.collect_list dump_internal_commands_query () + + let dump_accounts_accessed_query = +- Caqti_request.collect Caqti_type.unit Accounts_accessed.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_accessed.typ + {sql| SELECT account_identifier_id AS id, block_id + FROM accounts_accessed JOIN blocks ON block_id = blocks.id - WHERE height <> 1 - ORDER BY block_id, id |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_accounts_accessed_to_csv (module Conn : CONNECTION) output_file = - Conn.exec (dump_account_accessed_to_csv_query ~output_file) () -@@ -357,7 +357,7 @@ module Berkeley = struct - \ AND height <= %d ORDER BY height\n\ - \ \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes_till_height (module Conn : CONNECTION) output_file - height = -@@ -371,7 +371,7 @@ module Berkeley = struct - \ WHERE chain_status = 'canonical'\n\ - \ ORDER BY height\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes (module Conn : CONNECTION) output_file = - Conn.exec (dump_block_hashes_query ~output_file) () -@@ -403,21 +403,21 @@ module Berkeley = struct - INNER JOIN internal_commands ON id = internal_command_id - INNER JOIN account_identifiers ON public_key_id = receiver_id - ) ORDER BY block_id, id |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_user_and_internal_command_info_to_csv (module Conn : CONNECTION) - output_file = - Conn.exec (dump_user_and_internal_command_info_to_csv_query ~output_file) () +@@ -390,7 +390,7 @@ module Berkeley = struct + Conn.collect_list dump_accounts_accessed_query () + + let dump_block_hashes_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + (Printf.sprintf + "SELECT state_hash, ledger_hash FROM blocks \n\ + \ WHERE chain_status = 'canonical'\n\ +@@ -402,7 +402,7 @@ module Berkeley = struct + Conn.collect_list (dump_block_hashes_till_height_query ~height) () + + let dump_block_hashes_query = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ +@@ -413,7 +413,7 @@ module Berkeley = struct + Conn.collect_list dump_block_hashes_query () + + let dump_user_and_internal_command_info_query = +- Caqti_request.collect Caqti_type.unit Accounts_accessed.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_accessed.typ + {sql| + ( + WITH user_command_ids AS +@@ -443,14 +443,14 @@ module Berkeley = struct + Conn.collect_list dump_user_and_internal_command_info_query () let get_account_accessed_count_query = - Caqti_request.find Caqti_type.unit Caqti_type.int From 3127a903e5e11de96ff73632d58dc2312690e959 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 8 Apr 2024 15:09:41 +0800 Subject: [PATCH 10/11] update caqti-upgrade-plus-archive-initial-speedup.patch --- ...ti-upgrade-plus-archive-init-speedup.patch | 457 +++++++++--------- 1 file changed, 235 insertions(+), 222 deletions(-) diff --git a/buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch b/buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch index 698ad821538..944ee989886 100644 --- a/buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch +++ b/buildkite/scripts/caqti-upgrade-plus-archive-init-speedup.patch @@ -1,9 +1,3 @@ -From cb2463f19908e9c5531c7f80148e975060c954d9 Mon Sep 17 00:00:00 2001 -From: ember arlynx -Date: Sat, 30 Mar 2024 22:44:15 -0400 -Subject: [PATCH] applied caqti+archive - - diff --git a/opam.export b/opam.export index 655b623d35..130eb77140 100644 --- a/opam.export @@ -173,7 +167,7 @@ index c69e1805d1..bf15430c54 100644 pool with diff --git a/src/app/archive/lib/processor.ml b/src/app/archive/lib/processor.ml -index 8b90db5952..313b424f55 100644 +index d4f0e9c585..274fb0839d 100644 --- a/src/app/archive/lib/processor.ml +++ b/src/app/archive/lib/processor.ml @@ -21,7 +21,6 @@ @@ -521,8 +515,7 @@ index 8b90db5952..313b424f55 100644 - (module Conn) - token_symbol + type local_copy = (string, int) Hashtbl.t - -- let load (module Conn : CONNECTION) id = ++ + let local_copies = Hashtbl.create (module String) + + let load_copy = @@ -534,7 +527,8 @@ index 8b90db5952..313b424f55 100644 + ~load_elt:(fun t_to_id (id, value) -> + Hashtbl.add_exn t_to_id ~key:value ~data:id ; + Deferred.unit ) -+ + +- let load (module Conn : CONNECTION) id = + let add_if_doesn't_exist (module Conn : Mina_caqti.CONNECTION) token_symbol = + let%bind t_to_id = load_copy (module Conn) in + let open Deferred.Result.Let_syntax in @@ -3228,10 +3222,10 @@ index 297081b7b3..4b677e43f3 100644 FROM accounts_accessed WHERE block_id = $1 diff --git a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml -index f377d693fc..a97801e306 100644 +index 26e337e0bc..1360cda252 100644 --- a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml +++ b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml -@@ -284,10 +284,10 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = +@@ -273,10 +273,10 @@ let pre_fork_validations ~mainnet_archive_uri ~migrated_archive_uri () = let mainnet_archive_uri = Uri.of_string mainnet_archive_uri in let migrated_archive_uri = Uri.of_string migrated_archive_uri in let mainnet_pool = @@ -3244,7 +3238,7 @@ index f377d693fc..a97801e306 100644 in match (mainnet_pool, migrated_pool) with -@@ -375,10 +375,10 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri +@@ -365,10 +365,10 @@ let post_fork_validations ~mainnet_archive_uri ~migrated_archive_uri let mainnet_archive_uri = Uri.of_string mainnet_archive_uri in let migrated_archive_uri = Uri.of_string migrated_archive_uri in let mainnet_pool = @@ -3258,112 +3252,123 @@ index f377d693fc..a97801e306 100644 match (mainnet_pool, migrated_pool) with diff --git a/src/app/berkeley_migration_verifier/sql.ml b/src/app/berkeley_migration_verifier/sql.ml -index ff6c92ec9b..c0e89348d2 100644 +index 40b6b3f114..627350987a 100644 --- a/src/app/berkeley_migration_verifier/sql.ml +++ b/src/app/berkeley_migration_verifier/sql.ml -@@ -1,5 +1,3 @@ +@@ -1,6 +1,5 @@ + open Core + open Async -open Caqti_async -- - let dump_sql_to_csv output_file ~sql = - Printf.sprintf "COPY ( %s ) TO '%s' DELIMITER ',' CSV HEADER " sql output_file -@@ -26,9 +24,9 @@ module Mainnet = struct - WHERE receiver_account_creation_fee_paid IS NOT NULL + module Accounts_created = struct + type t = +@@ -79,7 +78,7 @@ end + + module Mainnet = struct + let dump_accounts_created_query = +- Caqti_request.collect Caqti_type.unit Accounts_created.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_created.typ + {sql| + ( SELECT height, value AS public_key, state_hash, receiver_account_creation_fee_paid AS creation_fee + FROM blocks_user_commands +@@ -100,18 +99,18 @@ module Mainnet = struct ) ORDER BY height, public_key |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = -+ let dump_accounts_created_to_csv (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_accounts_created_to_csv_query ~output_file) () - - let dump_state_and_ledger_hashes_to_csv_query ~output_file = -@@ -38,7 +36,7 @@ module Mainnet = struct - " SELECT state_hash, ledger_hash FROM blocks\n\ - \ WHERE chain_status = 'canonical'\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - - let dump_block_hashes_till_height_query ~output_file ~height = - dump_sql_to_csv output_file -@@ -49,10 +47,10 @@ module Mainnet = struct - \ WHERE chain_status = 'canonical'\n\ - \ AND height <= %d ORDER BY height\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_block_hashes_till_height (module Conn : CONNECTION) output_file -- height = -+ let dump_block_hashes_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_block_hashes_till_height_query ~output_file ~height) () - - let dump_block_hashes_query ~output_file = -@@ -63,9 +61,9 @@ module Mainnet = struct - \ WHERE chain_status = 'canonical'\n\ - \ ORDER BY height\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_block_hashes (module Conn : CONNECTION) output_file = -+ let dump_block_hashes (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_block_hashes_query ~output_file) () - - let dump_user_commands_till_height_query ~output_file ~height = -@@ -87,10 +85,10 @@ module Mainnet = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_user_commands_till_height (module Conn : CONNECTION) output_file -- height = -+ let dump_user_commands_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_user_commands_till_height_query ~output_file ~height) () - - let dump_internal_commands_till_height_query ~output_file ~height = -@@ -112,10 +110,10 @@ module Mainnet = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ - \ \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_internal_commands_till_height (module Conn : CONNECTION) output_file -- height = -+ let dump_internal_commands_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_internal_commands_till_height_query ~output_file ~height) () - - let dump_user_commands_query ~output_file = -@@ -135,9 +133,9 @@ module Mainnet = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_user_commands (module Conn : CONNECTION) output_file = -+ let dump_user_commands (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_user_commands_query ~output_file) () - - let dump_internal_commands_query ~output_file = -@@ -158,13 +156,13 @@ module Mainnet = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, type \n\ - \ \n\ - \ " ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_internal_commands (module Conn : CONNECTION) output_file = -+ let dump_internal_commands (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_internal_commands_query ~output_file) () + +- let dump_accounts_created (module Conn : CONNECTION) = ++ let dump_accounts_created (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_accounts_created_query () + + let dump_state_and_ledger_hashes_query = + (* Workaround for replacing output file as caqti has an issue with using ? in place of FILE argument*) +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + " SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ " + + let dump_block_hashes_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + (Printf.sprintf + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ +@@ -119,22 +118,22 @@ module Mainnet = struct + \ AND height <= %d ORDER BY height\n\ + \ " height ) + +- let dump_block_hashes_till_height (module Conn : CONNECTION) height = ++ let dump_block_hashes_till_height (module Conn : Mina_caqti.CONNECTION) height = + Conn.collect_list (dump_block_hashes_till_height_query ~height) () + + let dump_block_hashes_query = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ ORDER BY height\n\ + \ " + +- let dump_block_hashes (module Conn : CONNECTION) = ++ let dump_block_hashes (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_block_hashes_query () + + let dump_user_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + (Printf.sprintf + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ +@@ -152,11 +151,11 @@ module Mainnet = struct + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " height ) + +- let dump_user_commands_till_height (module Conn : CONNECTION) height = ++ let dump_user_commands_till_height (module Conn : Mina_caqti.CONNECTION) height = + Conn.collect_list (dump_user_commands_till_height_query ~height) () + + let dump_internal_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -174,11 +173,11 @@ module Mainnet = struct + \ \n\ + \ " height ) + +- let dump_internal_commands_till_height (module Conn : CONNECTION) height = ++ let dump_internal_commands_till_height (module Conn : Mina_caqti.CONNECTION) height = + Conn.collect_list (dump_internal_commands_till_height_query ~height) () + + let dump_user_commands_query = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ +@@ -194,11 +193,11 @@ module Mainnet = struct + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " + +- let dump_user_commands (module Conn : CONNECTION) = ++ let dump_user_commands (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_user_commands_query () + + let dump_internal_commands_query = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -215,11 +214,11 @@ module Mainnet = struct + \ \n\ + \ " ) + +- let dump_internal_commands (module Conn : CONNECTION) = ++ let dump_internal_commands (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_internal_commands_query () let mark_chain_till_fork_block_as_canonical_query = - Caqti_request.exec Caqti_type.string @@ -3371,7 +3376,7 @@ index ff6c92ec9b..c0e89348d2 100644 {sql| UPDATE blocks Set chain_status = 'canonical' -@@ -186,8 +184,8 @@ module Mainnet = struct +@@ -241,14 +240,14 @@ module Mainnet = struct ) |sql} @@ -3382,16 +3387,20 @@ index ff6c92ec9b..c0e89348d2 100644 Conn.exec mark_chain_till_fork_block_as_canonical_query fork_state_hash end -@@ -202,21 +200,22 @@ module Berkeley = struct - JOIN account_identifiers ON account_identifier_id = account_identifiers.id + module Berkeley = struct + let dump_accounts_created_query = +- Caqti_request.collect Caqti_type.unit Accounts_created.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_created.typ + {sql| + SELECT height, value AS public_key, state_hash, creation_fee + FROM accounts_created +@@ -257,19 +256,20 @@ module Berkeley = struct JOIN public_keys ON public_key_id = public_keys.id ORDER BY height, public_key |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit -- let dump_accounts_created_to_csv (module Conn : CONNECTION) output_file = -+ let dump_accounts_created_to_csv (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_accounts_created_to_csv_query ~output_file) () +- let dump_accounts_created (module Conn : CONNECTION) = ++ let dump_accounts_created (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_accounts_created_query () let height_query = - Caqti_request.find Caqti_type.unit Caqti_type.int @@ -3410,7 +3419,7 @@ index ff6c92ec9b..c0e89348d2 100644 {sql| WITH RECURSIVE chain AS ( -@@ -230,16 +229,18 @@ module Berkeley = struct +@@ -283,19 +283,21 @@ module Berkeley = struct ) SELECT count(*) FROM chain where chain_status = 'canonical'; |sql} @@ -3430,111 +3439,115 @@ index ff6c92ec9b..c0e89348d2 100644 + let blocks_count (module Conn : Mina_caqti.CONNECTION) = + Conn.find blocks_count_query () - let dump_user_commands_till_height_query ~output_file ~height = - dump_sql_to_csv output_file -@@ -260,10 +261,10 @@ module Berkeley = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_user_commands_till_height (module Conn : CONNECTION) output_file -- height = -+ let dump_user_commands_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_user_commands_till_height_query ~output_file ~height) () - - let dump_internal_commands_till_height_query ~output_file ~height = -@@ -285,10 +286,10 @@ module Berkeley = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, \ - command_type \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_internal_commands_till_height (module Conn : CONNECTION) output_file -- height = -+ let dump_internal_commands_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_internal_commands_till_height_query ~output_file ~height) () - - let dump_user_commands_query ~output_file = -@@ -308,9 +309,9 @@ module Berkeley = struct - \ INNER JOIN public_keys AS fee_payer_keys ON fee_payer_id = \ - fee_payer_keys.id ORDER BY height, sequence_no\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_user_commands (module Conn : CONNECTION) output_file = -+ let dump_user_commands (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_user_commands_query ~output_file) () - - let dump_internal_commands_query ~output_file = -@@ -330,9 +331,9 @@ module Berkeley = struct - \ ORDER BY height, sequence_no, secondary_sequence_no, \ - command_type \n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_internal_commands (module Conn : CONNECTION) output_file = -+ let dump_internal_commands (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_internal_commands_query ~output_file) () - - let dump_account_accessed_to_csv_query ~output_file = -@@ -343,9 +344,10 @@ module Berkeley = struct + let dump_user_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + (Printf.sprintf + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ +@@ -313,11 +315,11 @@ module Berkeley = struct + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " height ) + +- let dump_user_commands_till_height (module Conn : CONNECTION) height = ++ let dump_user_commands_till_height (module Conn : Mina_caqti.CONNECTION) height = + Conn.collect_list (dump_user_commands_till_height_query ~height) () + + let dump_internal_commands_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + (Printf.sprintf + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ +@@ -335,11 +337,11 @@ module Berkeley = struct + command_type \n\ + \ " height ) + +- let dump_internal_commands_till_height (module Conn : CONNECTION) height = ++ let dump_internal_commands_till_height (module Conn : Mina_caqti.CONNECTION) height = + Conn.collect_list (dump_internal_commands_till_height_query ~height) () + + let dump_user_commands_query = +- Caqti_request.collect Caqti_type.unit User_command.typ ++ Mina_caqti.collect_req Caqti_type.unit User_command.typ + "WITH user_command_ids AS\n\ + \ ( SELECT height, sequence_no, user_command_id FROM \ + blocks_user_commands\n\ +@@ -355,11 +357,11 @@ module Berkeley = struct + fee_payer_keys.id ORDER BY height, sequence_no\n\ + \ " + +- let dump_user_commands (module Conn : CONNECTION) = ++ let dump_user_commands (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_user_commands_query () + + let dump_internal_commands_query = +- Caqti_request.collect Caqti_type.unit Internal_command.typ ++ Mina_caqti.collect_req Caqti_type.unit Internal_command.typ + "WITH internal_command_ids AS \n\ + \ ( SELECT internal_command_id, height, sequence_no, \ + secondary_sequence_no FROM blocks_internal_commands \n\ +@@ -375,22 +377,22 @@ module Berkeley = struct + command_type \n\ + \ " + +- let dump_internal_commands (module Conn : CONNECTION) = ++ let dump_internal_commands (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_internal_commands_query () + + let dump_accounts_accessed_query = +- Caqti_request.collect Caqti_type.unit Accounts_accessed.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_accessed.typ + {sql| SELECT account_identifier_id AS id, block_id + FROM accounts_accessed JOIN blocks ON block_id = blocks.id WHERE height <> 1 ORDER BY block_id, id |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_accounts_accessed_to_csv (module Conn : CONNECTION) output_file = -+ let dump_accounts_accessed_to_csv (module Conn : Mina_caqti.CONNECTION) -+ output_file = - Conn.exec (dump_account_accessed_to_csv_query ~output_file) () - - let dump_block_hashes_till_height_query ~output_file ~height = -@@ -357,10 +359,10 @@ module Berkeley = struct - \ AND height <= %d ORDER BY height\n\ - \ \n\ - \ " height ) -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_block_hashes_till_height (module Conn : CONNECTION) output_file -- height = -+ let dump_block_hashes_till_height (module Conn : Mina_caqti.CONNECTION) -+ output_file height = - Conn.exec (dump_block_hashes_till_height_query ~output_file ~height) () - - let dump_block_hashes_query ~output_file = -@@ -371,9 +373,9 @@ module Berkeley = struct - \ WHERE chain_status = 'canonical'\n\ - \ ORDER BY height\n\ - \ " -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit - -- let dump_block_hashes (module Conn : CONNECTION) output_file = -+ let dump_block_hashes (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_block_hashes_query ~output_file) () - - let dump_user_and_internal_command_info_to_csv_query ~output_file = -@@ -403,21 +405,21 @@ module Berkeley = struct - INNER JOIN internal_commands ON id = internal_command_id + +- let dump_accounts_accessed (module Conn : CONNECTION) = ++ let dump_accounts_accessed (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_accounts_accessed_query () + + let dump_block_hashes_till_height_query ~height = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + (Printf.sprintf + "SELECT state_hash, ledger_hash FROM blocks \n\ + \ WHERE chain_status = 'canonical'\n\ +@@ -398,22 +400,22 @@ module Berkeley = struct + \ \n\ + \ " height ) + +- let dump_block_hashes_till_height (module Conn : CONNECTION) height = ++ let dump_block_hashes_till_height (module Conn : Mina_caqti.CONNECTION) height = + Conn.collect_list (dump_block_hashes_till_height_query ~height) () + + let dump_block_hashes_query = +- Caqti_request.collect Caqti_type.unit State_hash_and_ledger_hash.typ ++ Mina_caqti.collect_req Caqti_type.unit State_hash_and_ledger_hash.typ + "\n\ + \ SELECT state_hash, ledger_hash FROM blocks\n\ + \ WHERE chain_status = 'canonical'\n\ + \ ORDER BY height\n\ + \ " + +- let dump_block_hashes (module Conn : CONNECTION) = ++ let dump_block_hashes (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_block_hashes_query () + + let dump_user_and_internal_command_info_query = +- Caqti_request.collect Caqti_type.unit Accounts_accessed.typ ++ Mina_caqti.collect_req Caqti_type.unit Accounts_accessed.typ + {sql| + ( + WITH user_command_ids AS +@@ -439,18 +441,18 @@ module Berkeley = struct INNER JOIN account_identifiers ON public_key_id = receiver_id ) ORDER BY block_id, id |sql} -- |> Caqti_request.exec Caqti_type.unit -+ |> Mina_caqti.exec_req Caqti_type.unit -- let dump_user_and_internal_command_info_to_csv (module Conn : CONNECTION) -- output_file = -+ let dump_user_and_internal_command_info_to_csv -+ (module Conn : Mina_caqti.CONNECTION) output_file = - Conn.exec (dump_user_and_internal_command_info_to_csv_query ~output_file) () +- let dump_user_and_internal_command_info (module Conn : CONNECTION) = ++ let dump_user_and_internal_command_info (module Conn : Mina_caqti.CONNECTION) = + Conn.collect_list dump_user_and_internal_command_info_query () let get_account_accessed_count_query = - Caqti_request.find Caqti_type.unit Caqti_type.int @@ -3551,7 +3564,7 @@ index ff6c92ec9b..c0e89348d2 100644 {sql| select count(distinct ids.account_identifier_id) FROM -@@ -439,6 +441,7 @@ module Berkeley = struct +@@ -472,6 +474,7 @@ module Berkeley = struct |sql} From bde2b226cac4e5a128a78f619eedcad2f2e15618 Mon Sep 17 00:00:00 2001 From: Tang Jiawei Date: Mon, 8 Apr 2024 15:39:37 +0800 Subject: [PATCH 11/11] use yojson.safe.to_file instead --- .../berkeley_migration_verifier.ml | 64 ++++++++----------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml index 26e337e0bcb..25137a4c420 100644 --- a/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml +++ b/src/app/berkeley_migration_verifier/berkeley_migration_verifier.ml @@ -13,16 +13,10 @@ module Check = struct let err error = Error [ error ] let comparison_failed ~left ~right ~left_content ~right_content = - let left_file = Filename.(concat temp_dir_name "left.json") - and right_file = Filename.(concat temp_dir_name "right.json") in - let%map () = - Deferred.all_unit - [ Writer.with_file left_file ~f:(fun w -> - return @@ Writer.write w @@ Yojson.Safe.pretty_to_string left_content ) - ; Writer.with_file right_file ~f:(fun w -> - return @@ Writer.write w @@ Yojson.Safe.pretty_to_string right_content ) - ] - in + let left_file = Filename.(concat temp_dir_name @@ left ^ ".json") + and right_file = Filename.(concat temp_dir_name @@ right ^ ".json") in + Yojson.Safe.to_file left_file left_content ; + Yojson.Safe.to_file right_file right_content ; err @@ sprintf "Discrepancies found between %s and %s. To reproduce please run `diff \ @@ -97,10 +91,10 @@ let all_accounts_referred_in_commands_are_recorded migrated_pool = Sql.Berkeley.dump_user_and_internal_command_info db ) in - let%bind right = + let%map right = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_accounts_accessed db) in - if List.equal Sql.Accounts_accessed.equal left right then return Check.ok + if List.equal Sql.Accounts_accessed.equal left right then Check.ok else Check.comparison_failed ~left_content:(Sql.Accounts_accessed.list_to_yojson left) @@ -115,16 +109,15 @@ let accounts_created_table_is_correct migrated_pool mainnet_pool = let%bind left = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_accounts_created db) in - let%bind right = + let%map right = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_accounts_created db) in - if List.equal Sql.Accounts_created.equal left right then return Check.ok + if List.equal Sql.Accounts_created.equal left right then Check.ok else Check.comparison_failed ~left_content:(Sql.Accounts_created.list_to_yojson left) ~right_content:(Sql.Accounts_created.list_to_yojson right) - ~left:"Mainnet.accounts_created" - ~right:"Berkeley.accounts_created" + ~left:"Mainnet.accounts_created" ~right:"Berkeley.accounts_created" let compare_hashes_till_height migrated_pool mainnet_pool ~height = let query_mainnet_db = Mina_caqti.query mainnet_pool in @@ -134,13 +127,11 @@ let compare_hashes_till_height migrated_pool mainnet_pool ~height = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_block_hashes_till_height db height ) in - let%bind right = + let%map right = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_block_hashes_till_height db height ) in - - if List.equal Sql.State_hash_and_ledger_hash.equal left right then - return Check.ok + if List.equal Sql.State_hash_and_ledger_hash.equal left right then Check.ok else Check.comparison_failed ~left_content:(Sql.State_hash_and_ledger_hash.list_to_yojson left) @@ -155,12 +146,11 @@ let compare_hashes migrated_pool mainnet_pool = let%bind left = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_block_hashes db) in - let%bind right = + let%map right = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_block_hashes db) in - if List.equal Sql.State_hash_and_ledger_hash.equal left right then - return Check.ok + if List.equal Sql.State_hash_and_ledger_hash.equal left right then Check.ok else Check.comparison_failed ~left_content:(Sql.State_hash_and_ledger_hash.list_to_yojson left) @@ -176,17 +166,16 @@ let compare_user_commands_till_height migrated_pool mainnet_pool ~height = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_user_commands_till_height db height ) in - let%bind right = + let%map right = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_user_commands_till_height db height ) in - if List.equal Sql.User_command.equal left right then return Check.ok + if List.equal Sql.User_command.equal left right then Check.ok else Check.comparison_failed ~left_content:(Sql.User_command.list_to_yojson left) ~right_content:(Sql.User_command.list_to_yojson right) - ~left:"Mainnet.user_command" - ~right:"Berkeley.user_command" + ~left:"Mainnet.user_command" ~right:"Berkeley.user_command" let compare_internal_commands_till_height migrated_pool mainnet_pool ~height = let query_mainnet_db = Mina_caqti.query mainnet_pool in @@ -196,17 +185,16 @@ let compare_internal_commands_till_height migrated_pool mainnet_pool ~height = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_internal_commands_till_height db height ) in - let%bind right = + let%map right = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_internal_commands_till_height db height ) in - if List.equal Sql.Internal_command.equal left right then return Check.ok + if List.equal Sql.Internal_command.equal left right then Check.ok else Check.comparison_failed ~left_content:(Sql.Internal_command.list_to_yojson left) ~right_content:(Sql.Internal_command.list_to_yojson right) - ~left:"Mainnet.internal_command" - ~right:"Berkeley.internal_command" + ~left:"Mainnet.internal_command" ~right:"Berkeley.internal_command" let compare_user_commands migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in @@ -215,16 +203,15 @@ let compare_user_commands migrated_pool mainnet_pool = let%bind left = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_user_commands db) in - let%bind right = + let%map right = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_user_commands db) in - if List.equal Sql.User_command.equal left right then return Check.ok + if List.equal Sql.User_command.equal left right then Check.ok else Check.comparison_failed ~left_content:(Sql.User_command.list_to_yojson left) ~right_content:(Sql.User_command.list_to_yojson right) - ~left:"Mainnet.user_command" - ~right:"Berkeley.user_command" + ~left:"Mainnet.user_command" ~right:"Berkeley.user_command" let compare_internal_commands migrated_pool mainnet_pool = let query_mainnet_db = Mina_caqti.query mainnet_pool in @@ -233,16 +220,15 @@ let compare_internal_commands migrated_pool mainnet_pool = let%bind left = query_mainnet_db ~f:(fun db -> Sql.Mainnet.dump_internal_commands db) in - let%bind right = + let%map right = query_migrated_db ~f:(fun db -> Sql.Berkeley.dump_internal_commands db) in - if List.equal Sql.Internal_command.equal left right then return Check.ok + if List.equal Sql.Internal_command.equal left right then Check.ok else Check.comparison_failed ~left_content:(Sql.Internal_command.list_to_yojson left) ~right_content:(Sql.Internal_command.list_to_yojson right) - ~left:"Mainnet.internal_command" - ~right:"Berkeley.internal_command" + ~left:"Mainnet.internal_command" ~right:"Berkeley.internal_command" let compare_ledger_hash ~migrated_replayer_output ~fork_genesis_config_file = let checkpoint_ledger_hash =