refactor(store-indexer): add readme, refactor common env (#1533)

latticexyz · Sep 18, 2023 · b3c22a1 · b3c22a1
1 parent fa409e8
commit b3c22a1
Show file tree

Hide file tree

Showing 5 changed files with 100 additions and 46 deletions.
diff --git a/.changeset/thin-rice-trade.md b/.changeset/thin-rice-trade.md
@@ -0,0 +1,5 @@
+---
+"@latticexyz/store-indexer": patch
+---
+
+Added README and refactored handling of common environment variables
diff --git a/packages/store-indexer/README.md b/packages/store-indexer/README.md
@@ -0,0 +1,53 @@
+# store-indexer
+
+A minimal Typescript indexer for [MUD Store](https://mud.dev/store) events (built on [store-sync](https://npmjs.com/package/@latticexyz/store-sync))
+
+## Usage
+
+Install and run with:
+
+```sh
+npm install @latticexyz/store-indexer
+
+npm sqlite-indexer
+# or
+npm postgres-indexer
+```
+
+or execute the one of the package bins directly:
+
+```sh
+npx -p @latticexyz/store-indexer sqlite-indexer
+# or
+npx -p @latticexyz/store-indexer postgres-indexer
+```
+
+## Configuration
+
+Each indexer can be configured with environment variables.
+
+### Common environment variables
+
+| Variable           | Description                                                | Default   |
+| ------------------ | ---------------------------------------------------------- | --------- |
+| `HOST`             | Host that the indexer server listens on                    | `0.0.0.0` |
+| `PORT`             | Port that the indexer server listens on                    | `3001`    |
+| `RPC_HTTP_URL`     | HTTP URL for Ethereum RPC to fetch data from               |           |
+| `RPC_WS_URL`       | WebSocket URL for Ethereum RPC to fetch data from          |           |
+| `START_BLOCK`      | Block number to start indexing from                        | `0`       |
+| `MAX_BLOCK_RANGE`  | Maximum number of blocks to fetch from the RPC per request | `1000`    |
+| `POLLING_INTERVAL` | How often to poll for new blocks (in milliseconds)         | `1000`    |
+
+Note that you only need one of `RPC_HTTP_URL` or `RPC_WS_URL`, but we recommend both. The WebSocket URL will be prioritized and fall back to the HTTP URL if there are any connection issues.
+
+### Postgres indexer environment variables
+
+| Variable       | Description             | Default |
+| -------------- | ----------------------- | ------- |
+| `DATABASE_URL` | Postgres connection URL |         |
+
+### SQLite indexer environment variables
+
+| Variable          | Description              | Default      |
+| ----------------- | ------------------------ | ------------ |
+| `SQLITE_FILENAME` | SQLite database filename | `indexer.db` |
diff --git a/packages/store-indexer/bin/parseEnv.ts b/packages/store-indexer/bin/parseEnv.ts
@@ -0,0 +1,30 @@
+import { isDefined } from "@latticexyz/common/utils";
+import { z, ZodIntersection, ZodTypeAny } from "zod";
+
+const commonSchema = z.intersection(
+  z.object({
+    HOST: z.string().default("0.0.0.0"),
+    PORT: z.coerce.number().positive().default(3001),
+    START_BLOCK: z.coerce.bigint().nonnegative().default(0n),
+    MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),
+    POLLING_INTERVAL: z.coerce.number().positive().default(1000),
+  }),
+  z
+    .object({
+      RPC_HTTP_URL: z.string(),
+      RPC_WS_URL: z.string(),
+    })
+    .partial()
+    .refine((values) => Object.values(values).some(isDefined))
+);
+
+export function parseEnv<TSchema extends ZodTypeAny | undefined = undefined>(
+  schema?: TSchema
+): z.infer<TSchema extends ZodTypeAny ? ZodIntersection<typeof commonSchema, TSchema> : typeof commonSchema> {
+  const envSchema = schema !== undefined ? z.intersection(commonSchema, schema) : commonSchema;
+  return envSchema.parse(process.env, {
+    errorMap: (issue) => ({
+      message: `Missing or invalid environment variable: ${issue.path.join(".")}`,
+    }),
+  });
+}
diff --git a/packages/store-indexer/bin/postgres-indexer.ts b/packages/store-indexer/bin/postgres-indexer.ts
@@ -13,30 +13,13 @@ import { drizzle } from "drizzle-orm/postgres-js";
 import postgres from "postgres";
 import { cleanDatabase, postgresStorage, schemaVersion } from "@latticexyz/store-sync/postgres";
 import { createStoreSync } from "@latticexyz/store-sync";
+import { parseEnv } from "./parseEnv";
 
-const env = z
-  .intersection(
-    z.object({
-      HOST: z.string().default("0.0.0.0"),
-      PORT: z.coerce.number().positive().default(3001),
-      DATABASE_URL: z.string(),
-      START_BLOCK: z.coerce.bigint().nonnegative().default(0n),
-      MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),
-      POLLING_INTERVAL: z.coerce.number().positive().default(1000),
-    }),
-    z
-      .object({
-        RPC_HTTP_URL: z.string(),
-        RPC_WS_URL: z.string(),
-      })
-      .partial()
-      .refine((values) => Object.values(values).some(isDefined))
-  )
-  .parse(process.env, {
-    errorMap: (issue) => ({
-      message: `Missing or invalid environment variable: ${issue.path.join(".")}`,
-    }),
-  });
+const env = parseEnv(
+  z.object({
+    DATABASE_URL: z.string(),
+  })
+);
 
 const transports: Transport[] = [
   // prefer WS when specified

diff --git a/packages/store-indexer/bin/sqlite-indexer.ts b/packages/store-indexer/bin/sqlite-indexer.ts
@@ -13,30 +13,13 @@ import { chainState, schemaVersion, syncToSqlite } from "@latticexyz/store-sync/
 import { createQueryAdapter } from "../src/sqlite/createQueryAdapter";
 import { isDefined } from "@latticexyz/common/utils";
 import { combineLatest, filter, first } from "rxjs";
+import { parseEnv } from "./parseEnv";
 
-const env = z
-  .intersection(
-    z.object({
-      HOST: z.string().default("0.0.0.0"),
-      PORT: z.coerce.number().positive().default(3001),
-      SQLITE_FILENAME: z.string().default("indexer.db"),
-      START_BLOCK: z.coerce.bigint().nonnegative().default(0n),
-      MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),
-      POLLING_INTERVAL: z.coerce.number().positive().default(1000),
-    }),
-    z
-      .object({
-        RPC_HTTP_URL: z.string(),
-        RPC_WS_URL: z.string(),
-      })
-      .partial()
-      .refine((values) => Object.values(values).some(isDefined))
-  )
-  .parse(process.env, {
-    errorMap: (issue) => ({
-      message: `Missing or invalid environment variable: ${issue.path.join(".")}`,
-    }),
-  });
+const env = parseEnv(
+  z.object({
+    SQLITE_FILENAME: z.string().default("indexer.db"),
+  })
+);
 
 const transports: Transport[] = [
   // prefer WS when specified