feat(create-mud): use new sync packages (#1214)

Co-authored-by: alvrs <alvarius@lattice.xyz>

.changeset/tame-lemons-play.md

@@ -0,0 +1,194 @@
+---
+"@latticexyz/cli": patch
+"@latticexyz/common": major
+"@latticexyz/create-mud": major
+"@latticexyz/recs": patch
+"@latticexyz/store-indexer": patch
+---
+
+Templates and examples now use MUD's new sync packages, all built on top of [viem](https://viem.sh/). This greatly speeds up and stabilizes our networking code and improves types throughout.
+
+These new sync packages come with support for our `recs` package, including `encodeEntity` and `decodeEntity` utilities for composite keys.
+
+If you're using `store-cache` and `useRow`/`useRows`, you should wait to upgrade until we have a suitable replacement for those libraries. We're working on a [sql.js](https://github.com/sql-js/sql.js/)-powered sync module that will replace `store-cache`.
+
+**Migrate existing RECS apps to new sync packages**
+
+As you migrate, you may find some features replaced, removed, or not included by default. Please [open an issue](https://github.com/latticexyz/mud/issues/new) and let us know if we missed anything.
+
+1. Add `@latticexyz/store-sync` package to your app's `client` package and make sure `viem` is pinned to version `1.3.1` (otherwise you may get type errors)
+
+2. In your `supportedChains.ts`, replace `foundry` chain with our new `mudFoundry` chain.
+
+   ```diff
+   - import { foundry } from "viem/chains";
+   - import { MUDChain, latticeTestnet } from "@latticexyz/common/chains";
+   + import { MUDChain, latticeTestnet, mudFoundry } from "@latticexyz/common/chains";
+
+   - export const supportedChains: MUDChain[] = [foundry, latticeTestnet];
+   + export const supportedChains: MUDChain[] = [mudFoundry, latticeTestnet];
+   ```
+
+3. In `getNetworkConfig.ts`, remove the return type (to let TS infer it for now), remove now-unused config values, and add the viem `chain` object.
+
+   ```diff
+   - export async function getNetworkConfig(): Promise<NetworkConfig> {
+   + export async function getNetworkConfig() {
+   ```
+
+   ```diff
+     const initialBlockNumber = params.has("initialBlockNumber")
+       ? Number(params.get("initialBlockNumber"))
+   -   : world?.blockNumber ?? -1; // -1 will attempt to find the block number from RPC
+   +   : world?.blockNumber ?? 0n;
+   ```
+
+   ```diff
+   + return {
+   +   privateKey: getBurnerWallet().value,
+   +   chain,
+   +   worldAddress,
+   +   initialBlockNumber,
+   +   faucetServiceUrl: params.get("faucet") ?? chain.faucetUrl,
+   + };
+   ```
+
+4. In `setupNetwork.ts`, replace `setupMUDV2Network` with `syncToRecs`.
+
+   ```diff
+   - import { setupMUDV2Network } from "@latticexyz/std-client";
+   - import { createFastTxExecutor, createFaucetService, getSnapSyncRecords } from "@latticexyz/network";
+   + import { createFaucetService } from "@latticexyz/network";
+   + import { createPublicClient, fallback, webSocket, http, createWalletClient, getContract, Hex, parseEther, ClientConfig } from "viem";
+   + import { encodeEntity, syncToRecs } from "@latticexyz/store-sync/recs";
+   + import { createBurnerAccount, createContract, transportObserver } from "@latticexyz/common";
+   ```
+
+   ```diff
+   - const result = await setupMUDV2Network({
+   -   ...
+   - });
+
+   + const clientOptions = {
+   +   chain: networkConfig.chain,
+   +   transport: transportObserver(fallback([webSocket(), http()])),
+   +   pollingInterval: 1000,
+   + } as const satisfies ClientConfig;
+
+   + const publicClient = createPublicClient(clientOptions);
+
+   + const burnerAccount = createBurnerAccount(networkConfig.privateKey as Hex);
+   + const burnerWalletClient = createWalletClient({
+   +   ...clientOptions,
+   +   account: burnerAccount,
+   + });
+
+   + const { components, latestBlock$, blockStorageOperations$, waitForTransaction } = await syncToRecs({
+   +   world,
+   +   config: storeConfig,
+   +   address: networkConfig.worldAddress as Hex,
+   +   publicClient,
+   +   components: contractComponents,
+   +   startBlock: BigInt(networkConfig.initialBlockNumber),
+   +   indexerUrl: networkConfig.indexerUrl ?? undefined,
+   + });
+
+   + const worldContract = createContract({
+   +   address: networkConfig.worldAddress as Hex,
+   +   abi: IWorld__factory.abi,
+   +   publicClient,
+   +   walletClient: burnerWalletClient,
+   + });
+   ```
+
+   ```diff
+     // Request drip from faucet
+   - const signer = result.network.signer.get();
+   - if (networkConfig.faucetServiceUrl && signer) {
+   -   const address = await signer.getAddress();
+   + if (networkConfig.faucetServiceUrl) {
+   +   const address = burnerAccount.address;
+   ```
+
+   ```diff
+     const requestDrip = async () => {
+   -   const balance = await signer.getBalance();
+   +   const balance = await publicClient.getBalance({ address });
+       console.info(`[Dev Faucet]: Player balance -> ${balance}`);
+   -   const lowBalance = balance?.lte(utils.parseEther("1"));
+   +   const lowBalance = balance < parseEther("1");
+   ```
+
+   You can remove the previous ethers `worldContract`, snap sync code, and fast transaction executor.
+
+   The return of `setupNetwork` is a bit different than before, so you may have to do corresponding app changes.
+
+   ```diff
+   + return {
+   +   world,
+   +   components,
+   +   playerEntity: encodeEntity({ address: "address" }, { address: burnerWalletClient.account.address }),
+   +   publicClient,
+   +   walletClient: burnerWalletClient,
+   +   latestBlock$,
+   +   blockStorageOperations$,
+   +   waitForTransaction,
+   +   worldContract,
+   + };
+   ```
+
+5. Update `createSystemCalls` with the new return type of `setupNetwork`.
+
+   ```diff
+     export function createSystemCalls(
+   -   { worldSend, txReduced$, singletonEntity }: SetupNetworkResult,
+   +   { worldContract, waitForTransaction }: SetupNetworkResult,
+       { Counter }: ClientComponents
+     ) {
+        const increment = async () => {
+   -      const tx = await worldSend("increment", []);
+   -      await awaitStreamValue(txReduced$, (txHash) => txHash === tx.hash);
+   +      const tx = await worldContract.write.increment();
+   +      await waitForTransaction(tx);
+          return getComponentValue(Counter, singletonEntity);
+        };
+   ```
+
+6. (optional) If you still need a clock, you can create it with:
+
+   ```ts
+   import { map, filter } from "rxjs";
+   import { createClock } from "@latticexyz/network";
+
+   const clock = createClock({
+     period: 1000,
+     initialTime: 0,
+     syncInterval: 5000,
+   });
+
+   world.registerDisposer(() => clock.dispose());
+
+   latestBlock$
+     .pipe(
+       map((block) => Number(block.timestamp) * 1000), // Map to timestamp in ms
+       filter((blockTimestamp) => blockTimestamp !== clock.lastUpdateTime), // Ignore if the clock was already refreshed with this block
+       filter((blockTimestamp) => blockTimestamp !== clock.currentTime) // Ignore if the current local timestamp is correct
+     )
+     .subscribe(clock.update); // Update the local clock
+   ```
+
+If you're using the previous `LoadingState` component, you'll want to migrate to the new `SyncProgress`:
+
+```ts
+import { SyncStep, singletonEntity } from "@latticexyz/store-sync/recs";
+
+const syncProgress = useComponentValue(SyncProgress, singletonEntity, {
+  message: "Connecting",
+  percentage: 0,
+  step: SyncStep.INITIALIZE,
+});
+
+if (syncProgress.step === SyncStep.LIVE) {
+  // we're live!
+}
+```

e2e/.gitignore

@@ -1 +1,3 @@
-node_modules
+node_modules
+*.db
+*.db-journal

e2e/packages/client-vanilla/index.html

@@ -10,7 +10,7 @@
     </script>
     <div>
       <div id="block" title="block" data-testid="block">-1</div>
-      <div id="sync-state" title="sync-state" data-testid="sync-state"></div>
+      <div id="sync-step" title="sync-step" data-testid="sync-step"></div>
     </div>
   </body>
 </html>

e2e/packages/client-vanilla/package.json

@@ -19,6 +19,7 @@
     "@latticexyz/schema-type": "link:../../../packages/schema-type",
     "@latticexyz/services": "link:../../../packages/services",
     "@latticexyz/std-client": "link:../../../packages/std-client",
+    "@latticexyz/store-sync": "link:../../../packages/store-sync",
     "@latticexyz/utils": "link:../../../packages/utils",
     "@latticexyz/world": "link:../../../packages/world",
     "async-mutex": "^0.4.0",

e2e/packages/client-vanilla/src/index.ts

@@ -1,29 +1,26 @@
+import { Component, Entity, getComponentValue } from "@latticexyz/recs";
 import { setup } from "./mud/setup";
 
 const {
-  network: {
-    storeCache,
-    network: { blockNumber$ },
-    worldSend,
-    worldContract,
-    components: { LoadingState },
-  },
+  network: { components, latestBlock$, worldContract, waitForTransaction },
 } = await setup();
 
 const _window = window as any;
-_window.storeCache = storeCache;
 _window.worldContract = worldContract;
-_window.worldSend = worldSend;
+_window.waitForTransaction = waitForTransaction;
+
+_window.getComponentValue = (componentName: keyof typeof components, entity: Entity) =>
+  getComponentValue(components[componentName] as Component, entity);
 
 // Update block number in the UI
-blockNumber$.subscribe((blockNumber) => {
+latestBlock$.subscribe((block) => {
   const element = document.querySelector("#block");
-  if (element) element.innerHTML = String(blockNumber);
+  if (element) element.innerHTML = String(block.number);
 });
 
 // Update initial sync status in the UI
-LoadingState.update$.subscribe((value) => {
-  const syncState = value.value[0]?.state;
-  const element = document.querySelector("#sync-state");
-  if (element) element.innerHTML = String(syncState);
+components.SyncProgress.update$.subscribe(({ value }) => {
+  const syncStep = value[0]?.step;
+  const element = document.querySelector("#sync-step");
+  if (element) element.innerHTML = String(syncStep);
 });

e2e/packages/client-vanilla/src/mud/createSystemCalls.ts

@@ -1,18 +1,10 @@
-import { getComponentValue } from "@latticexyz/recs";
-import { awaitStreamValue } from "@latticexyz/utils";
 import { ClientComponents } from "./createClientComponents";
 import { SetupNetworkResult } from "./setupNetwork";
 
 export type SystemCalls = ReturnType<typeof createSystemCalls>;
 
 export function createSystemCalls(network: SetupNetworkResult, components: ClientComponents) {
-  // const increment = async () => {
-  //   const tx = await worldSend("increment", []);
-  //   await awaitStreamValue(txReduced$, (txHash) => txHash === tx.hash);
-  //   return getComponentValue(Counter, singletonEntity);
-  // };
-
   return {
-    // increment,
+    // TODO
   };
 }

e2e/packages/client-vanilla/src/mud/getNetworkConfig.ts

@@ -1,19 +1,12 @@
-import { SetupContractConfig, getBurnerWallet } from "@latticexyz/std-client";
+import { getBurnerWallet } from "@latticexyz/std-client";
 import worldsJson from "contracts/worlds.json";
 import { supportedChains } from "./supportedChains";
 
 const worlds = worldsJson as Partial<Record<string, { address: string; blockNumber?: number }>>;
 
-type NetworkConfig = SetupContractConfig & {
-  privateKey: string;
-  faucetServiceUrl?: string;
-  snapSync?: boolean;
-};
-
-export async function getNetworkConfig(): Promise<NetworkConfig> {
+export async function getNetworkConfig() {
   const params = new URLSearchParams(window.location.search);
-
-  const chainId = Number(params.get("chainId") || import.meta.env.VITE_CHAIN_ID || 31337);
+  const chainId = Number(params.get("chainId") || params.get("chainid") || import.meta.env.VITE_CHAIN_ID || 31337);
   const chainIndex = supportedChains.findIndex((c) => c.id === chainId);
   const chain = supportedChains[chainIndex];
   if (!chain) {
@@ -28,26 +21,16 @@ export async function getNetworkConfig(): Promise<NetworkConfig> {
 
   const initialBlockNumber = params.has("initialBlockNumber")
     ? Number(params.get("initialBlockNumber"))
-    : world?.blockNumber ?? -1; // -1 will attempt to find the block number from RPC
+    : world?.blockNumber ?? 0n;
 
   return {
-    clock: {
-      period: 1000,
-      initialTime: 0,
-      syncInterval: 5000,
-    },
-    provider: {
-      chainId,
-      jsonRpcUrl: params.get("rpc") ?? chain.rpcUrls.default.http[0],
-      wsRpcUrl: params.get("wsRpc") ?? chain.rpcUrls.default.webSocket?.[0],
-    },
     privateKey: params.get("privateKey") ?? getBurnerWallet().value,
     chainId,
-    modeUrl: params.get("mode") ?? chain.modeUrl,
+    chain,
     faucetServiceUrl: params.get("faucet") ?? chain.faucetUrl,
     worldAddress,
     initialBlockNumber,
-    snapSync: params.get("snapSync") === "true",
-    disableCache: params.get("cache") === "false",
+    indexerUrl: params.get("indexerUrl"),
+    rpcHttpUrl: params.get("rpcHttpUrl"),
   };
 }