Merge branch 'main' into gg/1076

README.md

@@ -12,10 +12,38 @@ Consisting of a data availability solution and a decentralized network of nodes
 rollups can leverage the Espresso Sequencer to give developers and end users fast confirmations, low (and fair) fees,
 and robust infrastructure.
 
-[Documentation](https://docs.espressosys.com/sequencer/espresso-sequencer-architecture/readme)
+[Official Documentation](https://docs.espressosys.com/sequencer/espresso-sequencer-architecture/readme)
+
+### Architecture
+
+This diagram below depicts a simplified view of the current architecture of the Espresso Sequencer. The diagram includes
+views of an Espresso Sequencer node, the Espresso Sequencer Network (nodes, CDN, builders, prover, state relay service),
+two rollups (one ZK rollup "Z", one optimistic rollup "O") that use the Espresso Sequencer for sequencing and some
+important L1 contracts.
+
+- Glossary
+  - Namespace: an identifier to distinguish rollups, akin to an Ethereum chain ID
+  - Rollup transaction: an transaction a user submits to a rollup, usually an EVM transaction
+  - Transaction: a transaction inside the Espresso Sequencer: a rollup transaction plus a namespace ID of the rollup
+  - Rollup block: a block in a rollup consisting only of transactions in this rollup
+  - Espresso block: a block produced by the Espresso sequencer containing transactions of multiple rollups
 
 ![Architecture diagram](./doc/architecture.svg)
 
+The sequence diagram below serves as a complement to the architecture diagram. The following interactions are depicted.
+
+1. Builders deposit funds into the fee contract on Ethereum Layer 1. These funds are later used to pay fees.
+2. Users submit transactions to the Submit APIs of sequencer nodes.
+3. The leader/proposer obtains a block from a builder.
+4. HotShot consensus creates new blocks containing sequenced rollup transactions.
+5. A rollup produces a rollup block with transactions sequenced by the Espresso sequencer.
+6. A proof for a HotShot state update is created and verified in the Light Client smart contract.
+7. A ZK rollup proves a correct state transaction by sending a proof to its rollup smart contract.
+8. A dispute is settled in an optimistic rollup. If necessary, the HotShot commitment is read from the Light Client
+   contract.
+
+![Sequence diagram](./doc/sequence-diagram.svg)
+
 # Running the demo
 
 Refer to [sequencer-example-l2](https://github.com/EspressoSystems/sequencer-example-l2) for instructions on how to run

doc/architecture.puml

@@ -1,82 +1,150 @@
 @startuml
-component "HotShot Sequencer" as Seq
-package "Layer 2s" as L2 {
-    component "Rollup 1" as L2_1 {
-        component "API (e.g. JSON-RPC)" as L2_1_RPC
-        component Executor as L2_1_Exec
-        component Prover as L2_1_Prover
-        component "State DB" as L2_1_State
+
+package "Espresso Sequencer Network" as Network #MistyRose {
+    component "Other\nSequencer\nNodes" as Nodes
+    package CDN {
+        component Broker
+        component Marshal
+        database KeyDB
     }
-    component "Rollup 2" as L2_2
-    component "Rollup N" as L2_N
-}
-package "Layer 1" as L1 {
-    component "Sequencer Contract" as Seq_L1
-    package "Rollup Contracts" as L2_L1 {
-        component "Rollup 1\nContract" as L2_1_L1
-        component "Rollup 2\nContract" as L2_2_L1
-        component "Rollup N\nContract" as L2_N_L1
+
+    'CDN Layout
+    Broker -> KeyDB: state
+    KeyDB -> Marshal: state
+
+    component "Sequencer Node" as Seq #Bisque {
+        database "Postgres" as DB
+        package APIs #LightCyan {
+            component "Submit" as Submit
+            component "Hotshot\nEvents\nStream\n" as EventsStream
+            component "Catchup" as Catchup
+            component "Status"
+            component "Query" as Query
+        }
+        package HotShot {
+            component "Networking"{
+                component Libp2p
+                component "CDN Client"
+            }
+            component "HotShot Events" as Events
+        }
+
+        package State as State {
+            component "State Signer" as Signer
+            component "ValidatedState" as Validated
+        }
     }
+    component "HotShot state\nprover service" as HSProver
+    component "HotShot state\nrelay service" as Relay
+    component "Builder" as Builder
 }
 
-Client -up-> L2_1_RPC : Transactions\nState queries
-note right of link
-1. Many clients submit transactions
-    to each L2 simultaneously (for
-    clarity only one is shown)
+' Network
+Nodes <-up- Catchup: "missing\nstate" " "
+Nodes <-> Libp2p: consensus\nmessages
+"CDN Client" <--> CDN: consensus\nmessages
+Nodes <--> CDN: consensus\nmessages
+note top of link
+    4. HotShot consensus
 end note
 
-L2_1_RPC -> Seq : Transaction\nRollup 1 ID
+' HotShot state
+Signer --> Relay: state, signature
 note top of link
-2. L2s forward transactions to sequencer.
+    6. HotShot proving
 end note
+Relay -> HSProver: state,\nsignature bundle
 
-Seq -down-> L2_1_Exec : Block
-Seq -down-> L2_2 : Block
-Seq -down-> L2_N : Block
-note right of link
-3. Sequencer produces blocks --
-    ordered lists of transactions.
-    L2s receive blocks and execute
-    transactions for their rollup.
+' Builder
+Builder --> HotShot: Espresso block
+note top of link
+    3. Leader
+    obtains
+    Espresso
+    block
 end note
+EventsStream -> Builder: transactions
 
-Seq -down-> Seq_L1 : Block Commitment\nQC
-note right of link
-4. Sequencer posts succinct commitment
-    to block to L1. Contract verifies proof
-    of sequencing (Quorum Certificate)
-    then stores the block commitment.
+' Events
+Networking <-up-> Events
+Events -up-> EventsStream: events
+Events -down-> Validated
+
+' Storage
+Query --> DB
+Validated -up-> DB
+DB --> Catchup: missing\nstate
+
+package "Layer 2s / Rollups" as L2s #LightYellow {
+    component "zk Rollup Z" as ZK #LightBlue {
+        component "API (e.g. JSON-RPC)" as ZK_RPC
+        component Executor as ZK_Exec
+        component Prover as ZK_Prover
+        ZK_RPC --[hidden]> ZK_Exec
+        ZK_RPC --[hidden]> ZK_Prover
+    }
+    component "Optimistic Rollup O" #LightGreen {
+        component "API (e.g. JSON-RPC)" as OR_RPC
+        component Proposer
+        component Challenger
+        OR_RPC --[hidden]> Proposer
+        OR_RPC --[hidden]> Challenger
+    }
+}
+package "Ethereum Layer 1" as L1 #LightGray {
+    package "Espresso Contracts" as Esp_L1 {
+        component "Light Client\nContract" as LC_L1
+        component "Fee\nContract" as Fee_L1
+    }
+    package "Rollup Contracts" {
+        component "zk Rollup Z\nContract" as ZK_L1
+        component "OR Rollup O\nContract" as OR_L1
+    }
+}
+
+' Transaction submission
+entity User
+User --> ZK_RPC : rollup transaction,\nstate queries
+note top of link
+    2. Transaction submission
 end note
+User --> OR_RPC
 
-L2_1_Prover -> L2_1_L1 : State\nProof
-L2_2 -down-> L2_2_L1 : State\nProof
-L2_N -down-> L2_N_L1 : State\nProof
-note right of link
-5. Rollups post updated state to L1.
-    ZK-rollups include a proof.
+ZK_RPC --> Submit : "transaction" " "
+Submit -> Events: transaction
+Query -up-> ZK_Exec : " " "Espresso\nheader,\nnamespace\ntransactions"
+note top of link
+    5. Rollup
+    block
+    production
 end note
+Query -up-> Proposer: " " "Espresso\nheader,\nnamespace\ntransactions"
 
-Seq_L1 -> L2_L1
-note bottom of link
-6. Rollup contracts read certified sequence
-    of block commitments from sequencer
-    contract. Verify state transition proofs
-    against this sequence (ZKR) or wait for
-    fraud proof against this sequence (ORU).
+
+' L1 details
+HSProver -up-> LC_L1 : HotShot\nstate\nproof
+LC_L1 -up-> ZK_L1 : " " "HotShot\ncommitment"
+LC_L1 -up-> OR_L1 : " " "HotShot\ncommitment"
+
+' ZK proof
+ZK_Prover --> ZK_L1 : rollup state proof
+note top of link
+    7. ZK Rollup
+    proving
+end note
+
+' OR dispute
+Proposer --> OR_L1: respond
+Challenger --> OR_L1: challenge
+note top of link
+    8. OR dispute
+end note
+
+Builder -> Fee_L1: deposit\nether
+note top of link
+    1. Fee
+    deposit
 end note
+Fee_L1 --> Validated: fee\ndeposits
 
-' L2 1 details
-L2_1_RPC <-down- L2_1_State : Read state
-L2_1_Exec -down-> L2_1_State : Write state
-L2_1_Prover <-down- L2_1_State : Read State
-
-' Layout
-L2_1_RPC -[hidden]r-> L2_1_Exec
-L2_1 -[hidden]r-> L2_2
-L2_2 -[hidden]r-> L2_N
-L2_1_L1 -[hidden]r-> L2_2_L1
-L2_2_L1 -[hidden]r-> L2_N_L1
-L2 -[hidden]d-> L1
-Seq -[hidden]d-> L2
 @enduml

doc/sequence-diagram.puml

@@ -0,0 +1,97 @@
+@startuml
+!pragma teoz true
+actor User
+
+box Layer 2s / Rollups #LightYellow
+    box "Optimistic Rollup O" #LightGreen
+        participant "API\n(RPC)" as orrpc
+        participant Proposer
+        participant Challenger
+    end box
+    box "ZK Rollup Z" #LightBlue
+        participant "API\n(RPC)" as zkrpc
+        participant "Prover" as rprover
+        participant "Executor" as executor
+    end box
+end box
+
+box "Ethereum L1" #LightGrey
+    participant "Rollup\nContract" as RC
+    participant "Light\nClient\nContract" as LC
+    participant "Fee\nContract" as fee
+end box
+
+box Espresso Sequencer Network #MistyRose
+    box Sequencer Node #Bisque
+        box APIs #LightCyan
+            participant "Submit\nAPI" as submit
+            participant "Query\nAPI" as query
+        end box
+        participant "HotShot\nEvents\nStream" as events
+        participant "Validated\nState" as ValidatedState
+        participant "State\nSigner" as signer
+    end box
+
+    participant Builder
+    participant "HotShot\nLeader" as leader
+    participant CDN
+    participant Libp2p
+    participant "HotShot\nreplicas" as replicas
+    participant "HotShot\nstate\nrelay\nservice" as relay
+    participant "HotShot\nstate\nprover\nservice" as prover
+end box
+
+group 1. Fee deposit
+    Builder -> fee: fee deposit
+end group
+
+group 2. Transaction submisssion
+    User -> orrpc: rollup O transaction
+    orrpc -> submit: transaction
+    User -> zkrpc: rollup Z transaction
+    zkrpc -> submit: transaction
+    events -> Builder: transactions
+end group
+
+group 3. Leader obtains Espresso block
+    Builder -> leader: available blocks
+    leader -> Builder: claim block
+end group
+
+group 4. HotShot consensus
+    leader <-> replicas: consensus messages
+    & leader <-> Libp2p:
+    & Libp2p <-> replicas:
+    & replicas -> CDN:
+    fee -> ValidatedState: credit fee deposit
+    ValidatedState -> ValidatedState: apply header\ncharge fee
+end group
+
+group 5. Rollup block production
+    query -> executor: Espresso header,\ntransactions in namespace
+    executor -> executor: verify\nnamespace\nproof
+    executor -> rprover: rollup block
+end group
+
+group 6. HotShot proving
+    signer -> relay: state, signature
+    relay -> prover: state,\nsignature bundle
+    prover -> "Light\nClient\nContract" as LC: proof
+    LC -> LC: verify
+end group
+
+group 7. ZK Rollup proving
+    rprover -> RC: rollup state proof
+    group verify
+        LC -> "Rollup\nContract" as RC: HotShot\ncommitment
+    end group
+end group
+
+group 8. Optimistic Rollup dispute
+    Challenger -> RC: challenge
+    loop until dispute resolved
+    Proposer -> RC: respond
+    LC --> RC: HotShot\ncommitment
+    Challenger -> RC: respond
+end group
+@enduml

flake.nix

@@ -124,8 +124,7 @@
             cargo-clippy = {
               enable = true;
               description = "Run clippy";
-              entry =
-                "cargo clippy --workspace --all-features --all-targets -- -D warnings";
+              entry = "just clippy";
               types_or = [ "rust" "toml" ];
               pass_filenames = false;
             };

justfile

@@ -30,6 +30,9 @@ test:
     cargo build --bin diff-test --release
     cargo test --release --all-features
 
+clippy:
+    cargo clippy --workspace --all-features --all-targets -- -D warnings
+
 # Helpful shortcuts for local development
 dev-orchestrator:
     target/release/orchestrator -p 8080 -n 1

utils/Cargo.toml

@@ -23,6 +23,3 @@ surf = "2.3.2"
 tempfile = "3.9.0"
 tracing = "0.1.37"
 url = "2.3.1"
-
-[dev-dependencies]
-bincode = { workspace = true }