feature: Genesis executes as a protocol update (#1008)

## Summary

Reworked the protocol update triggering, enactment and scenario
execution.
* Genesis is now a protocol update, and can be automatically resumed.
* Scenarios now execute in a separate batch group
* Protocol updates can now be triggered immediately after previous
updates - so e.g. we don't have to wait till epoch 5 to get cuttlefish
activated for testnets and local tests.

And now, post-merge-in of
#1009 we have a few minor
changes:
* Removes the name of genesis from `ProtocolConfig`, because it's relied
on by static logic, and I don't feel like this is something we should
allow people to change (this might break some of our tests/testnets, I'm
liaising with Les to discuss)
* Updates the testnet configuration to run Anemone/Bottlenose
back-to-back at genesis
* Changes the testing defaults to:
* Run all testnet-configured protocol updates on launch - this means
that, by default, tests are running against the latest engine. (This was
traditionally too slow to activate before, until we enabled back-to-back
configuration).
* But don't run scenarios (because this would slow down the tests too
much).
* Fixes various tests broken by running all protocol updates in our
tests

In next PRs:
* Other Cuttlefish implementation

## Testing

* Have tested that genesis can be resumed by stopping it mid-update
* I've tested that it can sync stokenet genesis locally

.dockerignore

@@ -4,6 +4,7 @@
 
 # Radix non-versioned files
 /**/RADIXDB/
+/**/RADIXDB_OLD/
 /**/NODEMOUNT/
 /**/RADIXDB_TEST/
 /**/logs/

.gitignore

@@ -48,6 +48,7 @@ api.users
 
 # radixdlt non-versioned files
 RADIXDB/
+RADIXDB_OLD/
 NODEMOUNT/
 RADIXDB_TEST/
 logs/

core/src/main/java/com/radixdlt/consensus/Blake2b256Hasher.java → common/src/main/java/com/radixdlt/crypto/Blake2b256Hasher.java

@@ -62,11 +62,9 @@
  * permissions under this License.
  */
 
-package com.radixdlt.consensus;
+package com.radixdlt.crypto;
 
 import com.google.common.hash.HashCode;
-import com.radixdlt.crypto.HashUtils;
-import com.radixdlt.crypto.Hasher;
 import com.radixdlt.serialization.DsonOutput;
 import com.radixdlt.serialization.Serialization;
 

common/src/main/java/com/radixdlt/monitoring/Metrics.java

@@ -287,8 +287,7 @@ public record UnfulfilledSyncRequest(UnfulfilledSyncRequestReason reason) {}
 
     public enum UnfulfilledSyncRequestReason {
       NOTHING_TO_SERVE_AT_THE_GIVEN_STATE_VERSION,
-      REFUSED_TO_SERVE_GENESIS,
-      REFUSED_TO_SERVE_PROTOCOL_UPDATE
+      REFUSED_TO_SERVE_PROTOCOL_UPDATE,
     }
   }
 

core-rust-bridge/src/main/java/com/radixdlt/environment/NodeRustEnvironment.java

@@ -65,6 +65,7 @@
 package com.radixdlt.environment;
 
 import com.google.common.reflect.TypeToken;
+import com.radixdlt.genesis.GenesisProvider;
 import com.radixdlt.mempool.MempoolRelayDispatcher;
 import com.radixdlt.sbor.NodeSborCodecs;
 import com.radixdlt.transactions.RawNotarizedTransaction;
@@ -84,13 +85,16 @@ public final class NodeRustEnvironment implements AutoCloseable {
   @SuppressWarnings("unused")
   private final long rustNodeRustEnvironmentPointer = 0;
 
+  private final GenesisProvider genesisProvider;
   private final MempoolRelayDispatcher<RawNotarizedTransaction> mempoolRelayDispatcher;
   private final FatalPanicHandler fatalPanicHandler;
 
   public NodeRustEnvironment(
+      GenesisProvider genesisProvider,
       MempoolRelayDispatcher<RawNotarizedTransaction> mempoolRelayDispatcher,
       FatalPanicHandler fatalPanicHandler,
       StateManagerConfig config) {
+    this.genesisProvider = genesisProvider;
     this.mempoolRelayDispatcher = mempoolRelayDispatcher;
     this.fatalPanicHandler = fatalPanicHandler;
     final var encodedConfig =
@@ -117,6 +121,18 @@ public void triggerMempoolRelay(byte[] notarizedTransactionPayload) {
         RawNotarizedTransaction.create(notarizedTransactionPayload));
   }
 
+  /** Lazily reads the genesis data. This method is called from Rust via JNI if required */
+  @SuppressWarnings("unused")
+  public byte[] readGenesisData() {
+    return this.genesisProvider.genesisData().value();
+  }
+
+  /** Lazily reads the genesis data hash. This method is called from Rust via JNI if required */
+  @SuppressWarnings("unused")
+  public byte[] readGenesisDataHash() {
+    return this.genesisProvider.genesisDataHash().asBytes();
+  }
+
   /**
    * Delegates the handling of a fatal Rust panic (e.g. happening while any write lock is held).
    * This method is called from Rust via JNI. In production, it is critical to shut down the Node's

core-rust-bridge/src/main/java/com/radixdlt/environment/ScenariosExecutionConfig.java

@@ -65,8 +65,7 @@
 package com.radixdlt.environment;
 
 import com.google.common.collect.ImmutableList;
-import com.radixdlt.networks.Network;
-import com.radixdlt.protocol.ProtocolUpdateTrigger;
+import com.google.common.collect.ImmutableMap;
 import com.radixdlt.sbor.codec.CodecMap;
 import com.radixdlt.sbor.codec.StructCodec;
 
@@ -79,31 +78,20 @@
  * them out is non-trivial (because of some persistence/hardcoding inherent to Genesis handling).
  */
 public record ScenariosExecutionConfig(
-    ImmutableList<ProtocolUpdateScenarios> afterProtocolUpdates) {
+    ImmutableMap<String, ImmutableList<String>> afterProtocolUpdates,
+    boolean runDefaultScenariosWhenUnspecified) {
 
   /** A default production-compatible configuration, which does not execute any Scenarios. */
   public static final ScenariosExecutionConfig NONE =
-      new ScenariosExecutionConfig(ImmutableList.of());
+      new ScenariosExecutionConfig(ImmutableMap.of(), false);
 
   /** A "full test" configuration, executing all Scenarios at appropriate Protocol Update states. */
-  public static final ScenariosExecutionConfig ALL =
-      new ScenariosExecutionConfig(
-          ImmutableList.of( // Note: no Anemone protocol scenarios existed.
-              new ProtocolUpdateScenarios(
-                  ProtocolUpdateTrigger.BOTTLENOSE,
-                  ImmutableList.of("account_locker", "maya_router", "access-controller-v2"))));
+  public static final ScenariosExecutionConfig ALL_FOR_NETWORK =
+      new ScenariosExecutionConfig(ImmutableMap.of(), true);
 
   public static void registerCodec(CodecMap codecMap) {
     codecMap.register(
         ScenariosExecutionConfig.class,
         codecs -> StructCodec.fromRecordComponents(ScenariosExecutionConfig.class, codecs));
   }
-
-  public static ScenariosExecutionConfig resolveForNetwork(Network network) {
-    if (Network.PRODUCTION_NETWORKS.contains(network)) {
-      return ScenariosExecutionConfig.NONE;
-    } else {
-      return ScenariosExecutionConfig.ALL;
-    }
-  }
 }

core/src/main/java/com/radixdlt/genesis/RawGenesisDataWithHash.java → core-rust-bridge/src/main/java/com/radixdlt/genesis/RawGenesisDataWithHash.java

@@ -66,7 +66,7 @@
 
 import com.google.common.hash.HashCode;
 import com.google.common.reflect.TypeToken;
-import com.radixdlt.consensus.Blake2b256Hasher;
+import com.radixdlt.crypto.Blake2b256Hasher;
 import com.radixdlt.crypto.Hasher;
 import com.radixdlt.sbor.NodeSborCodecs;
 import com.radixdlt.serialization.DefaultSerialization;

core-rust-bridge/src/main/java/com/radixdlt/protocol/ProtocolConfig.java

@@ -74,22 +74,19 @@
 import java.util.Map;
 
 public record ProtocolConfig(
-    String genesisProtocolVersion,
     ImmutableList<ProtocolUpdateTrigger> protocolUpdateTriggers,
     Map<String, byte[]> rawProtocolUpdateContentOverrides) {
 
   public static final String GENESIS_PROTOCOL_VERSION_NAME = "babylon-genesis";
+  public static final String ANEMONE_PROTOCOL_VERSION_NAME = "anemone";
+  public static final String BOTTLENOSE_PROTOCOL_VERSION_NAME = "bottlenose";
+
+  public static final String LATEST_PROTOCOL_VERSION_NAME = BOTTLENOSE_PROTOCOL_VERSION_NAME;
 
   public ProtocolConfig(ImmutableList<ProtocolUpdateTrigger> protocolUpdateTriggers) {
     this(protocolUpdateTriggers, Map.of());
   }
 
-  public ProtocolConfig(
-      ImmutableList<ProtocolUpdateTrigger> protocolUpdateTriggers,
-      Map<String, byte[]> rawProtocolUpdateContentOverrides) {
-    this(GENESIS_PROTOCOL_VERSION_NAME, protocolUpdateTriggers, rawProtocolUpdateContentOverrides);
-  }
-
   public static void registerCodec(CodecMap codecMap) {
     codecMap.register(
         ProtocolConfig.class,
@@ -104,10 +101,14 @@ public static ProtocolConfig sborDecode(byte[] encoded, String errorMessage) {
     }
   }
 
-  public static ProtocolConfig testingDefault() {
+  public static ProtocolConfig onlyGenesis() {
     return new ProtocolConfig(ImmutableList.of());
   }
 
+  public static ProtocolConfig testingDefault() {
+    return RustProtocolUpdate.resolveForNetwork(NetworkDefinition.INT_TEST_NET);
+  }
+
   public static ProtocolConfig resolveForNetwork(NetworkDefinition networkDefinition) {
     return RustProtocolUpdate.resolveForNetwork(networkDefinition);
   }

core-rust-bridge/src/main/java/com/radixdlt/protocol/ProtocolUpdateEnactmentCondition.java

@@ -129,4 +129,7 @@ public static void registerCodec(CodecMap codecMap) {
 
   record EnactAtStartOfEpochUnconditionally(UInt64 epoch)
       implements ProtocolUpdateEnactmentCondition {}
+
+  record EnactImmediatelyAfterEndOfProtocolUpdate(String protocolVersionName)
+      implements ProtocolUpdateEnactmentCondition {}
 }

core-rust-bridge/src/main/java/com/radixdlt/protocol/ProtocolUpdateTrigger.java

@@ -70,9 +70,6 @@
 public record ProtocolUpdateTrigger(
     String nextProtocolVersion, ProtocolUpdateEnactmentCondition enactmentCondition) {
 
-  public static final String ANEMONE = "anemone";
-  public static final String BOTTLENOSE = "bottlenose";
-
   public static void registerCodec(CodecMap codecMap) {
     codecMap.register(
         ProtocolUpdateTrigger.class,

core-rust-bridge/src/main/java/com/radixdlt/protocol/RustProtocolUpdate.java

@@ -66,6 +66,7 @@
 
 import com.google.common.reflect.TypeToken;
 import com.radixdlt.environment.NodeRustEnvironment;
+import com.radixdlt.lang.Tuple;
 import com.radixdlt.monitoring.Metrics;
 import com.radixdlt.rev2.NetworkDefinition;
 import com.radixdlt.sbor.Natives;
@@ -78,21 +79,24 @@ public final class RustProtocolUpdate {
 
   public RustProtocolUpdate(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) {
     final var timer = metrics.stateManager().nativeCall();
-    applyProtocolUpdateFunc =
-        Natives.builder(nodeRustEnvironment, RustProtocolUpdate::applyProtocolUpdate)
+    applyKnownPendingProtocolUpdateFunc =
+        Natives.builder(nodeRustEnvironment, RustProtocolUpdate::applyKnownPendingProtocolUpdate)
             .measure(
-                timer.label(new Metrics.MethodId(RustProtocolUpdate.class, "applyProtocolUpdate")))
+                timer.label(
+                    new Metrics.MethodId(
+                        RustProtocolUpdate.class, "applyKnownPendingProtocolUpdate")))
             .build(new TypeToken<>() {});
   }
 
-  public ProtocolUpdateResult applyProtocolUpdate(String protocolVersionName) {
-    return this.applyProtocolUpdateFunc.call(protocolVersionName);
+  public ProtocolUpdateResult applyKnownPendingProtocolUpdate() {
+    return this.applyKnownPendingProtocolUpdateFunc.call(Tuple.tuple());
   }
 
-  private static native byte[] applyProtocolUpdate(
+  private static native byte[] applyKnownPendingProtocolUpdate(
       NodeRustEnvironment nodeRustEnvironment, byte[] payload);
 
-  private final Natives.Call1<String, ProtocolUpdateResult> applyProtocolUpdateFunc;
+  private final Natives.Call1<Tuple.Tuple0, ProtocolUpdateResult>
+      applyKnownPendingProtocolUpdateFunc;
 
   public static String readinessSignalName(ProtocolUpdateTrigger protocolUpdateTrigger) {
     return readinessSignalNameFunc.call(protocolUpdateTrigger);

core-rust-bridge/src/main/java/com/radixdlt/sbor/NodeSborCodecs.java

@@ -130,7 +130,6 @@ public static void registerCodecsWithCodecMap(CodecMap codecMap) {
     LoggingConfig.registerCodec(codecMap);
     StateManagerConfig.registerCodec(codecMap);
     ScenariosExecutionConfig.registerCodec(codecMap);
-    ProtocolUpdateScenarios.registerCodec(codecMap);
     ProtocolConfig.registerCodec(codecMap);
     ProtocolUpdateTrigger.registerCodec(codecMap);
     ProtocolUpdateEnactmentCondition.registerCodec(codecMap);

core-rust-bridge/src/main/java/com/radixdlt/statecomputer/ProtocolState.java

@@ -78,7 +78,7 @@
 
 public record ProtocolState(
     ImmutableMap<UInt64, String> enactedProtocolUpdates,
-    ImmutableList<PendingProtocolUpdate> pendingProtocolUpdates) {
+    ImmutableMap<String, PendingProtocolUpdate> pendingProtocolUpdates) {
 
   public static void registerCodec(CodecMap codecMap) {
     codecMap.register(
@@ -109,7 +109,7 @@ public String currentProtocolVersion() {
   }
 
   public static ProtocolState testingEmpty() {
-    return new ProtocolState(ImmutableMap.of(), ImmutableList.of());
+    return new ProtocolState(ImmutableMap.of(), ImmutableMap.of());
   }
 
   public record PendingProtocolUpdate(

core-rust-bridge/src/main/java/com/radixdlt/statecomputer/RustStateComputer.java

@@ -83,10 +83,6 @@ public class RustStateComputer {
   public RustStateComputer(Metrics metrics, NodeRustEnvironment nodeRustEnvironment) {
     Objects.requireNonNull(nodeRustEnvironment);
     LabelledTimer<MethodId> timer = metrics.stateManager().nativeCall();
-    this.executeGenesisFunc =
-        Natives.builder(nodeRustEnvironment, RustStateComputer::executeGenesis)
-            .measure(timer.label(new MethodId(RustStateComputer.class, "executeGenesis")))
-            .build(new TypeToken<>() {});
     this.prepareFunc =
         Natives.builder(nodeRustEnvironment, RustStateComputer::prepare)
             .measure(timer.label(new MethodId(RustStateComputer.class, "prepare")))
@@ -105,15 +101,6 @@ public RustStateComputer(Metrics metrics, NodeRustEnvironment nodeRustEnvironmen
             .build(new TypeToken<>() {});
   }
 
-  public LedgerProof executeGenesis(byte[] rawGenesisData) {
-    return executeGenesisFunc.call(rawGenesisData);
-  }
-
-  private final Natives.Call1<byte[], LedgerProof> executeGenesisFunc;
-
-  private static native byte[] executeGenesis(
-      NodeRustEnvironment nodeRustEnvironment, byte[] payload);
-
   public PrepareResult prepare(PrepareRequest prepareRequest) {
     return prepareFunc.call(prepareRequest);
   }