docs: Add swarm mode testdrive example

testdrive/swarm/README.md

@@ -0,0 +1,25 @@
+# Swarm Mode Testdrive
+
+This testdrive example shows how to leverage docker-compose to run Scrolls in "swarm mode".
+
+## How does it work?
+
+The docker compose file will start several Scrolls instances, each instance as a different container. The config for each pipeline instructs Scrolls to crawl a different segment of the chain (by setting the intersect & finalize sections).
+
+> **Note**
+> The segments were constructed by hand. Since the chain-sync mini-protocol requires slot & hash to initiate, there's no _easy_ way to define the segments programatically. Automating this step is out of scope for this example. 
+
+Each Scrolls instance requires a different config file. To avoid duplicating of common values, we use a feature called 'layered config'. There's a `common.toml` file with the values shared across all instances. The instance-specific values are defined in `daemon-x.toml` files, Scrolls will _overlay_ these values on top of the ones defined in `common.toml`.
+
+> **Warning**
+> Swarm mode won't work correctly with reducers that require the 'enrich' stage. For the enrich stage to work correctly, it needs to crawl the chain from origin. Swarm mode relies on crawling the chain from different points. Attempting to run both features simultaneously will yield unknown results.
+
+The data from each Scrolls instance is sent to a single, shared Redis instance. The cursor for each pipeline is kept as a different Redis key to avoid collisions. It should be safe to stop / start the whole swarm, each pipeline will attempt to continue from where it was prior to the restart.
+
+## Getting started
+
+Run the following command from within the directory of this example:
+
+```sh
+docker-compose up
+```

testdrive/swarm/common.toml

@@ -0,0 +1,17 @@
+[source]
+type = "N2N"
+address = "relays-new.cardano-mainnet.iohk.io:3001"
+
+[[reducers]]
+type = "AddressByTxo"
+key_prefix = "c1"
+
+[storage]
+type = "Redis"
+connection_params = "redis://redis:6379"
+
+[chain]
+type = "Mainnet"
+
+[policy]
+missing_data = "Skip"

testdrive/swarm/daemon-a.toml

@@ -0,0 +1,8 @@
+[intersect]
+type = "Origin"
+
+[storage]
+cursor_key = "_cursor_a"
+
+[finalize]
+until_hash = "d74267944135fad75d90476630d032e7f11e460993a71ae5c024a00864da2b7b"

testdrive/swarm/daemon-b.toml

@@ -0,0 +1,12 @@
+[intersect]
+type = "Point"
+value = [
+    1329363,
+    "d74267944135fad75d90476630d032e7f11e460993a71ae5c024a00864da2b7b",
+]
+
+[storage]
+cursor_key = "_cursor_b"
+
+[finalize]
+until_hash = "16367ca4c0c6daed1f4c74d7f7af9c71f7e0a88dfb6289b34d33f8a9039eb3a2"

testdrive/swarm/daemon-c.toml

@@ -0,0 +1,12 @@
+[intersect]
+type = "Point"
+value = [
+    2659848,
+    "16367ca4c0c6daed1f4c74d7f7af9c71f7e0a88dfb6289b34d33f8a9039eb3a2",
+]
+
+[storage]
+cursor_key = "_cursor_c"
+
+[finalize]
+until_hash = "d30b6e06d72d479441d3bb832306926a81cd79c09e275741e9968ab7f1cc9a0a"

testdrive/swarm/daemon-d.toml

@@ -0,0 +1,12 @@
+[intersect]
+type = "Point"
+value = [
+    3989584,
+    "d30b6e06d72d479441d3bb832306926a81cd79c09e275741e9968ab7f1cc9a0a",
+]
+
+[storage]
+cursor_key = "_cursor_c"
+
+[finalize]
+until_hash = "4100ba6296f8e8437032c308ee24920027d96f00467a46ad36711bed23e10459"

testdrive/swarm/daemon-e.toml

@@ -0,0 +1,12 @@
+[intersect]
+type = "Point"
+value = [
+    21243299,
+    "4100ba6296f8e8437032c308ee24920027d96f00467a46ad36711bed23e10459",
+]
+
+[storage]
+cursor_key = "_cursor_c"
+
+[finalize]
+until_hash = "5f78088805e8ff56bdd2c11d13944d5d8ce99df041922711789dec346979256f"

testdrive/swarm/daemon-f.toml

@@ -0,0 +1,12 @@
+[intersect]
+type = "Point"
+value = [
+    48312866,
+    "5f78088805e8ff56bdd2c11d13944d5d8ce99df041922711789dec346979256f",
+]
+
+[storage]
+cursor_key = "_cursor_c"
+
+[finalize]
+until_hash = "8ae85d3f16843e8fbb572de471d5364e03b9a87c9097ce05690452e2e6afea64"

testdrive/swarm/data/.gitignore

@@ -0,0 +1 @@
+dump.rdb

testdrive/swarm/docker-compose.yml

@@ -0,0 +1,75 @@
+version: "3.7"
+
+services:
+  scrolls-a:
+    image: ghcr.io/txpipe/scrolls-testdrive:testdrive
+    command: [ "daemon" ]
+    environment:
+      - RUST_LOG=info
+    working_dir: "/"
+    volumes:
+      - ./daemon-a.toml:/scrolls.toml
+      - ./common.toml:/etc/scrolls/daemon.toml
+    links:
+      - redis
+  scrolls-b:
+    image: ghcr.io/txpipe/scrolls-testdrive:testdrive
+    command: [ "daemon" ]
+    environment:
+      - RUST_LOG=info
+    working_dir: "/"
+    volumes:
+      - ./daemon-b.toml:/scrolls.toml
+      - ./common.toml:/etc/scrolls/daemon.toml
+    links:
+      - redis
+  scrolls-c:
+    image: ghcr.io/txpipe/scrolls-testdrive:testdrive
+    command: [ "daemon" ]
+    environment:
+      - RUST_LOG=info
+    working_dir: "/"
+    volumes:
+      - ./daemon-c.toml:/scrolls.toml
+      - ./common.toml:/etc/scrolls/daemon.toml
+    links:
+      - redis
+  scrolls-d:
+    image: ghcr.io/txpipe/scrolls-testdrive:testdrive
+    command: [ "daemon" ]
+    environment:
+      - RUST_LOG=info
+    working_dir: "/"
+    volumes:
+      - ./daemon-d.toml:/scrolls.toml
+      - ./common.toml:/etc/scrolls/daemon.toml
+    links:
+      - redis
+  scrolls-e:
+    image: ghcr.io/txpipe/scrolls-testdrive:testdrive
+    command: [ "daemon" ]
+    environment:
+      - RUST_LOG=info
+    working_dir: "/"
+    volumes:
+      - ./daemon-e.toml:/scrolls.toml
+      - ./common.toml:/etc/scrolls/daemon.toml
+    links:
+      - redis
+  scrolls-f:
+    image: ghcr.io/txpipe/scrolls-testdrive:testdrive
+    command: [ "daemon" ]
+    environment:
+      - RUST_LOG=info
+    working_dir: "/"
+    volumes:
+      - ./daemon-f.toml:/scrolls.toml
+      - ./common.toml:/etc/scrolls/daemon.toml
+    links:
+      - redis
+  redis:
+    image: redis
+    volumes:
+      - ./data:/data
+    ports:
+      - "6379:6379"