Merge back to develop

buildkite/src/Jobs/Test/SingleNodeTest.dhall

@@ -32,7 +32,7 @@ let Docker = ../../Command/Docker/Type.dhall
 let Size = ../../Command/Size.dhall
 
 
-let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Profiles.Type.Standard
+let dependsOn = DebianVersions.dependsOn DebianVersions.DebVersion.Bullseye Profiles.Type.Lightnet
 
 
 let buildTestCmd : Size -> Command.Type = \(cmd_target : Size) ->

src/app/cli/src/init/client.ml

@@ -2232,6 +2232,23 @@ let thread_graph =
                   (humanize_graphql_error ~graphql_endpoint e) ) ;
              exit 1 ) )
 
+let signature_kind =
+  Command.basic
+    ~summary:"Print the signature kind that this binary is compiled with"
+    (let%map.Command () = Command.Param.return () in
+     fun () ->
+       let signature_kind_string =
+         match Mina_signature_kind.t with
+         | Mainnet ->
+             "mainnet"
+         | Testnet ->
+             "testnet"
+         | Other_network s ->
+             (* Prefix string to disambiguate *)
+             "other network: " ^ s
+       in
+       Core.print_endline signature_kind_string )
+
 let itn_create_accounts =
   Command.async ~summary:"Fund new accounts for incentivized testnet"
     (let open Command.Param in
@@ -2379,6 +2396,7 @@ let advanced =
     ; ("runtime-config", runtime_config)
     ; ("vrf", Cli_lib.Commands.Vrf.command_group)
     ; ("thread-graph", thread_graph)
+    ; ("print-signature-kind", signature_kind)
     ]
   in
   let cmds =

src/app/replayer/replayer.ml

@@ -640,7 +640,7 @@ let write_replayer_checkpoint ~logger ~ledger ~last_global_slot_since_genesis
 
 let main ~input_file ~output_file_opt ~migration_mode ~archive_uri
     ~continue_on_error ~checkpoint_interval ~checkpoint_output_folder_opt
-    ~checkpoint_file_prefix () =
+    ~checkpoint_file_prefix ~genesis_dir_opt () =
   let logger = Logger.create () in
   let json = Yojson.Safe.from_file input_file in
   let input =
@@ -665,22 +665,20 @@ let main ~input_file ~output_file_opt ~migration_mode ~archive_uri
          except that we don't consider loading from a tar file
       *)
       let query_db = Mina_caqti.query pool in
-      let%bind padded_accounts =
-        match
-          Genesis_ledger_helper.Ledger.padded_accounts_from_runtime_config_opt
-            ~logger ~proof_level input.genesis_ledger
-            ~ledger_name_prefix:"genesis_ledger"
+      let%bind packed_ledger =
+        match%bind
+          Genesis_ledger_helper.Ledger.load ~proof_level
+            ~genesis_dir:
+              (Option.value ~default:Cache_dir.autogen_path genesis_dir_opt)
+            ~logger ~constraint_constants input.genesis_ledger
         with
-        | None ->
+        | Error e ->
             [%log fatal]
-              "Could not load accounts from input runtime genesis ledger" ;
+              "Could not load accounts from input runtime genesis ledger %s"
+              (Error.to_string_hum e) ;
             exit 1
-        | Some accounts ->
-            return accounts
-      in
-      let packed_ledger =
-        Genesis_ledger_helper.Ledger.packed_genesis_ledger_of_accounts
-          ~depth:constraint_constants.ledger_depth padded_accounts
+        | Ok (packed_ledger, _, _) ->
+            return packed_ledger
       in
       let ledger = Lazy.force @@ Genesis_ledger.Packed.t packed_ledger in
       let epoch_ledgers_state_hash_opt =
@@ -1706,11 +1704,15 @@ let () =
            Param.flag "--checkpoint-output-folder"
              ~doc:"file Folder containing the resulting checkpoints"
              Param.(optional string)
+         and genesis_dir_opt =
+           Param.flag "--genesis-ledger-dir"
+             ~doc:"DIR Directory that contains the genesis ledger"
+             Param.(optional string)
          and checkpoint_file_prefix =
            Param.flag "--checkpoint-file-prefix"
              ~doc:"string Checkpoint file prefix (default: 'replayer')"
              Param.(optional_with_default "replayer" string)
          in
          main ~input_file ~output_file_opt ~migration_mode ~archive_uri
            ~checkpoint_interval ~continue_on_error ~checkpoint_output_folder_opt
-           ~checkpoint_file_prefix )))
+           ~checkpoint_file_prefix ~genesis_dir_opt )))

src/app/replayer/test/README.md

@@ -0,0 +1,83 @@
+## Replayer component test
+
+This folder holds static data which is used when testing replayer component. Name 'component' is used in context of test since it is not na unit tests but also not interfere with other mina components, so it cannot be called integration test. Basically test production version of replayer against manually prepared data and input config file using command:
+
+```
+mina-replayer --archive-uri {connection_string} --input-file input.json 
+```
+
+It expects success
+
+### Regenerate data
+
+Data generation is still manual process. Mina local network script (./script/mina-local-network/mina-local-network.sh) can be used to bootstrap small network and generate archive data. For example below command is usually used:
+
+```
+./scripts/mina-local-network/mina-local-network.sh -a -r -pu postgres -ppw postgres -zt -vt
+```
+
+where:
+- `-a` run archive (it will automatically create 'archive' schema)
+- `-r` removes any artifacts from previous run to have clear situation
+- `-pu -ppw` are database connection parameters
+- `-zt` ran zkapp transactions
+- `-vt` ran simple value transfer transactions
+
+> :warning: Prior to mina-local-network run you need to build mina, archive, zkapp_transaction, logproc apps
+
+It is important to mention that properly generated archive data should have at least 10 canonical blocks as we run replayer tests only on canonical blocks.
+
+After we are satisfied with data generation process. We can dump archive data and prepare input replayer file from genesis_ledger.json file from our local network
+
+#### Full script
+
+Disclaimer: I'm a nix user and has already setup nix on my machine 
+
+```
+nix develop mina
+
+dune build src/app/cli/src/mina.exe src/app/archive/archive.exe src/app/zkapp_test_transaction/zkapp_test_transaction.exe src/app/logproc/logproc.exe
+
+./scripts/mina-local-network/mina-local-network.sh -a -r -pu postgres -ppw postgres -zt -vt
+
+# archive_db.sql
+pg_dump -U postgres -d archive > archive_db.sql
+
+# input file 
+cp ~/.mina-network/mina-local-network-2-1-1/genesis_ledger.json
+cat genesis_ledger.json | jq '.ledger.accounts' > _tmp.json
+echo '{ "genesis_ledger": { "accounts": '$(cat _tmp.json)' } }' | jq > input.json
+
+
+```
+
+#### Alternatives
+
+As mentioned in previous section we need to have some canonical blocks in archive database. The more the better. However, with current value of K parameter (responsible for converting pending block into canonical) this process can take a lot of time (> 7hours). Fortunately there are alternative solutions  for this problem.
+
+a) We can alter input config and use `target_epoch_ledgers_state_hash` property in replayer input file to inform replayer that we want to replay also pending blocks. Example:
+
+```
+{ 
+    "target_epoch_ledgers_state_hash": "3NLbZ28M72eewCxYUCE3CwQo5c7wPzoiGcNC5Bbe8oEnrutXtZt9",
+    "genesis_ledger": {
+    "name": "release",
+    "num_accounts": 250,
+    "accounts": [
+     {
+      "pk": "B62qkamwHMkTvY3t9wu4Aw4LJTDJY4m6Sk48pJ2kSMtV1fxKP2SSzWq",
+   .....
+
+```
+
+b) Convert pending chain to canonical blocks using helper script:
+
+`./src/app/replayer/test/convert postgres://postgres:postgres@localhost:5432/archive '3NLbZ28M72eewCxYUCE3CwQo5c7wPzoiGcNC5Bbe8oEnrutXtZt9'`
+
+As a result archive database will now have blocks which are a part of chain from genesis block to target block converted to canonical. All blocks which are not a part of mentioned chain and have height smaller than target blocks will be orphaned. Rest will be left intact as pending. DO NOT USE on production.
+
+### Dependencies
+
+Replayer component tests uses postgres database only. It need to be accessible from host machine
+
+