feat(#376): Replace lazy transactions markers with a bitmap to reclai…

…m rent (#380)

* feat(#376): Replace lazy transactions markers with a bitmap to reclaim rent

* Switch to true bitmap to save rent over Vec<bool>

* -Appease clippy

packages/helium-admin-cli/src/close-lazy-transaction-markers.ts

@@ -0,0 +1,76 @@
+import * as anchor from "@coral-xyz/anchor";
+import { blockKey, init as initLazy, lazyTransactionsKey } from "@helium/lazy-transactions-sdk";
+import os from "os";
+import yargs from "yargs/yargs";
+import { bulkSendTransactions, chunks } from "@helium/spl-utils";
+import { Transaction } from "@solana/web3.js";
+
+export async function run(args: any = process.argv) {
+  const yarg = yargs(args).options({
+    wallet: {
+      alias: "k",
+      describe: "Anchor wallet keypair",
+      default: `${os.homedir()}/.config/solana/id.json`,
+    },
+    url: {
+      alias: "u",
+      default: "http://127.0.0.1:8899",
+      describe: "The solana url",
+    },
+    name: {
+      default: "nJWGUMOK",
+      describe: "The lazy transactions instance name"
+    }
+  });
+  const argv = await yarg.argv;
+  process.env.ANCHOR_WALLET = argv.wallet;
+  process.env.ANCHOR_PROVIDER_URL = argv.url;
+  anchor.setProvider(anchor.AnchorProvider.local(argv.url));
+  const provider = anchor.getProvider() as anchor.AnchorProvider;
+  const lazyProgram = await initLazy(provider);
+  const ltKey = lazyTransactionsKey(argv.name)[0];
+  const lt = await lazyProgram.account.lazyTransactionsV0.fetch(ltKey);
+
+  const blocks = await lazyProgram.account.block.all();
+  const blocksByKey = new Set(blocks.map((b) => b.publicKey.toString()));
+  const allIndices = new Array(1 << lt.maxDepth).fill(0).map((_, i) => i);
+  const blockIndices = allIndices.filter((bi) =>
+    blocksByKey.has(blockKey(ltKey, bi)[0].toBase58())
+  );
+  if (lt.executed.length !== 1 << lt.maxDepth) {
+    await lazyProgram.methods
+      .reinitializeExecutedTransactionsV0()
+      .accounts({
+        lazyTransactions: ltKey,
+      })
+      .rpc({ skipPreflight: true });
+  }
+  const instructions = await Promise.all(
+    blockIndices.map((bi) =>
+      lazyProgram.methods
+        .closeMarkerV0({
+          index: bi,
+        })
+        .accounts({
+          refund: provider.wallet.publicKey,
+          lazyTransactions: ltKey,
+          authority: provider.wallet.publicKey,
+        })
+        .instruction()
+    )
+  );
+
+  console.log(`${blocks.length} blocks to close`);
+  const txns = chunks(instructions, 10).map(chunk => {
+    const tx = new Transaction({
+      feePayer: provider.wallet.publicKey,
+    })
+    tx.add(...chunk)
+    return tx
+  })
+
+  await bulkSendTransactions(provider, txns, (status) => {
+    console.log(`Sending ${status.currentBatchProgress} / ${status.currentBatchSize} batch. ${status.totalProgress} / ${txns.length}`)
+  })
+  console.log("Done")
+}

packages/lazy-transactions-sdk/src/index.ts

@@ -323,3 +323,14 @@ export function compileNoMerkle(
 
   return compiledTransactions
 }
+
+export function isExecuted(
+  executed: Buffer,
+  index: number,
+): boolean {
+  const byteIndex = Math.floor(index / 8);
+  const bitIndex = index % 8;
+  const byte = executed[byteIndex];
+  const mask = 1 << bitIndex;
+  return (byte & mask) !== 0;
+}

packages/migration-service/src/index.ts

@@ -6,6 +6,7 @@ import { LazyTransactions } from "@helium/idls/lib/types/lazy_transactions";
 import {
   blockKey,
   init,
+  isExecuted,
   lazySignerKey,
   lazyTransactionsKey,
 } from "@helium/lazy-transactions-sdk";
@@ -120,18 +121,8 @@ async function getTransactions(
     .blockhash;
 
   console.log(`Found ${results.length} transactions to migrate}`);
-  const blocks = results.map((r) => blockKey(lazyTransactions, r.id)[0]);
-  const blocksExist = (
-    await Promise.all(
-      chunks(blocks, 100).map(
-        async (chunk) =>
-          await provider.connection.getMultipleAccountsInfo(
-            chunk as PublicKey[],
-            "confirmed"
-          )
-      )
-    )
-  ).flat();
+  const lt = await program.account.lazyTransactionsV0.fetch(lazyTransactions);
+  const executed = lt.executed;
   const lazyTxns = await program.account.lazyTransactionsV0.fetch(
     lazyTransactions
   );
@@ -156,7 +147,7 @@ async function getTransactions(
           },
           idx
         ) => {
-          const hasRun = blocksExist[idx];
+          const hasRun = isExecuted(executed, idx);
           const compiledTx = decompress(compiled);
           const block = blockKey(lazyTransactions, id)[0];
           const signers = decompressSigners(signersRaw);

packages/spl-utils/src/mplAssetAPI.ts

@@ -100,7 +100,7 @@ export async function getAssets(
     });
 
     const result = response.data
-      ? response.data.map((res) => res?.result || undefined)
+      ? response.data.map((res: any) => res?.result || undefined)
       : [];
 
     return [

programs/lazy-transactions/Cargo.toml

@@ -27,3 +27,4 @@ anchor-lang = { workspace = true }
 anchor-spl = { workspace = true }
 spl-concurrent-merkle-tree = "0.1.2"
 bytemuck = "1.13.0"
+shared-utils = { path = "../../utils/shared-utils" }

programs/lazy-transactions/src/error.rs

@@ -13,4 +13,7 @@ pub enum ErrorCode {
 
   #[msg("Invalid canopy length")]
   CanopyLengthMismatch,
+
+  #[msg("Transaction has already been executed")]
+  TransactionAlreadyExecuted,
 }

programs/lazy-transactions/src/instructions/close_marker_v0.rs

@@ -1,9 +1,9 @@
-use crate::state::*;
+use crate::{state::*, util::set_executed};
 use anchor_lang::prelude::*;
 
 #[derive(AnchorSerialize, AnchorDeserialize, Clone, Default)]
 pub struct CloseMarkerArgsV0 {
-  pub index: u64,
+  pub index: u32,
 }
 
 #[derive(Accounts)]
@@ -13,6 +13,7 @@ pub struct CloseMarkerV0<'info> {
   /// CHECK: Just receiving funds
   pub refund: UncheckedAccount<'info>,
   #[account(
+    mut,
     has_one = authority
   )]
   pub lazy_transactions: Account<'info, LazyTransactionsV0>,
@@ -26,6 +27,8 @@ pub struct CloseMarkerV0<'info> {
   pub block: Account<'info, Block>,
 }
 
-pub fn handler(_ctx: Context<CloseMarkerV0>, _args: CloseMarkerArgsV0) -> Result<()> {
+pub fn handler(ctx: Context<CloseMarkerV0>, args: CloseMarkerArgsV0) -> Result<()> {
+  set_executed(&mut ctx.accounts.lazy_transactions.executed, args.index);
+
   Ok(())
 }

programs/lazy-transactions/src/instructions/execute_transaction_v0.rs

@@ -1,5 +1,6 @@
 use crate::canopy::fill_in_proof_from_canopy;
 use crate::error::ErrorCode;
+use crate::util::{is_executed, set_executed};
 use crate::{merkle_proof::verify, state::*};
 use anchor_lang::{prelude::*, solana_program, solana_program::instruction::Instruction};
 
@@ -30,7 +31,9 @@ pub struct ExecuteTransactionV0<'info> {
   #[account(mut)]
   pub payer: Signer<'info>,
   #[account(
-    has_one = canopy
+    mut,
+    has_one = canopy,
+    constraint = !is_executed(&lazy_transactions.executed, args.index) @ ErrorCode::TransactionAlreadyExecuted,
   )]
   pub lazy_transactions: Account<'info, LazyTransactionsV0>,
   /// CHECK: Verified by has one
@@ -42,18 +45,20 @@ pub struct ExecuteTransactionV0<'info> {
   )]
   /// CHECK: You can throw things behind this signer and it will sign the tx via cpi
   pub lazy_signer: AccountInfo<'info>,
+  /// CHECK: Temporary. We can remove this once executed txns is fully populated
   #[account(
-    init,
-    payer = payer,
-    space = 8,
+    constraint = block.lamports() == 0,
+    constraint = block.data.borrow().len() == 0,
     seeds = ["block".as_bytes(), lazy_transactions.key().as_ref(), &args.index.to_le_bytes()],
     bump
   )]
-  pub block: Account<'info, Block>,
+  pub block: AccountInfo<'info>,
   pub system_program: Program<'info, System>,
 }
 
 pub fn handler(ctx: Context<ExecuteTransactionV0>, args: ExecuteTransactionArgsV0) -> Result<()> {
+  set_executed(&mut ctx.accounts.lazy_transactions.executed, args.index);
+
   let largest_acct_idx: usize = (*args
     .instructions
     .iter()

programs/lazy-transactions/src/instructions/initialize_lazy_transactions_v0.rs

@@ -1,4 +1,4 @@
-use crate::{canopy::check_canopy_bytes, id, state::*};
+use crate::{canopy::check_canopy_bytes, id, state::*, util::get_bitmap_len};
 use anchor_lang::prelude::*;
 
 #[derive(AnchorSerialize, AnchorDeserialize, Clone, Default)]
@@ -17,7 +17,7 @@ pub struct InitializeLazyTransactionsV0<'info> {
   #[account(
     init,
     payer = payer,
-    space = 8 + 60 + std::mem::size_of::<LazyTransactionsV0>(),
+    space = 8 + 60 + std::mem::size_of::<LazyTransactionsV0>() + (1 << args.max_depth),
     seeds = ["lazy_transactions".as_bytes(), args.name.as_bytes()],
     bump,
   )]
@@ -50,6 +50,7 @@ pub fn handler(
       canopy: ctx.accounts.canopy.key(),
       max_depth: args.max_depth,
       bump_seed: ctx.bumps["lazy_transactions"],
+      executed: vec![0; get_bitmap_len(args.max_depth)],
     });
 
   Ok(())

programs/lazy-transactions/src/instructions/mod.rs

@@ -2,12 +2,14 @@ pub mod close_canopy_v0;
 pub mod close_marker_v0;
 pub mod execute_transaction_v0;
 pub mod initialize_lazy_transactions_v0;
+pub mod reinitialize_executed_transactions_v0;
 pub mod set_canopy_v0;
 pub mod update_lazy_transactions_v0;
 
 pub use close_canopy_v0::*;
 pub use close_marker_v0::*;
 pub use execute_transaction_v0::*;
 pub use initialize_lazy_transactions_v0::*;
+pub use reinitialize_executed_transactions_v0::*;
 pub use set_canopy_v0::*;
 pub use update_lazy_transactions_v0::*;

programs/lazy-transactions/src/instructions/reinitialize_executed_transactions_v0.rs

@@ -0,0 +1,29 @@
+use crate::state::*;
+use crate::util::get_bitmap_len;
+use anchor_lang::prelude::*;
+use shared_utils::resize_to_fit;
+
+#[derive(Accounts)]
+pub struct ReinitializeExecutedTransactionsV0<'info> {
+  #[account(mut)]
+  pub payer: Signer<'info>,
+  pub authority: Signer<'info>,
+  #[account(
+    mut,
+    has_one = authority
+  )]
+  pub lazy_transactions: Box<Account<'info, LazyTransactionsV0>>,
+  pub system_program: Program<'info, System>,
+}
+
+pub fn handler(ctx: Context<ReinitializeExecutedTransactionsV0>) -> Result<()> {
+  ctx.accounts.lazy_transactions.executed =
+    vec![0; get_bitmap_len(ctx.accounts.lazy_transactions.max_depth)];
+
+  resize_to_fit(
+    &ctx.accounts.payer.to_account_info(),
+    &ctx.accounts.system_program.to_account_info(),
+    &ctx.accounts.lazy_transactions,
+  )?;
+  Ok(())
+}

programs/lazy-transactions/src/lib.rs

@@ -7,6 +7,7 @@ pub mod error;
 pub mod instructions;
 pub mod merkle_proof;
 pub mod state;
+pub mod util;
 
 pub use instructions::*;
 pub use state::*;
@@ -47,4 +48,10 @@ pub mod lazy_transactions {
   pub fn set_canopy_v0(ctx: Context<SetCanopyV0>, args: SetCanopyArgsV0) -> Result<()> {
     set_canopy_v0::handler(ctx, args)
   }
+
+  pub fn reinitialize_executed_transactions_v0(
+    ctx: Context<ReinitializeExecutedTransactionsV0>,
+  ) -> Result<()> {
+    reinitialize_executed_transactions_v0::handler(ctx)
+  }
 }

programs/lazy-transactions/src/state.rs

@@ -9,6 +9,8 @@ pub struct LazyTransactionsV0 {
   pub authority: Pubkey,
   pub canopy: Pubkey,
   pub bump_seed: u8,
+  // Bitmap of executed transactions
+  pub executed: Vec<u8>,
 }
 
 #[account]

programs/lazy-transactions/src/util.rs

@@ -0,0 +1,17 @@
+pub fn get_bitmap_len(max_depth: u32) -> usize {
+  let num_txns = 1 << max_depth;
+  // Div ceil, https://stackoverflow.com/questions/72442853/how-would-you-perform-ceiling-division
+  ((num_txns + 8 - 1) / 8) as usize
+}
+
+pub fn is_executed(bitmap: &[u8], index: u32) -> bool {
+  let byte = bitmap[index as usize / 8];
+  let bit = 1 << (index % 8);
+  byte & bit != 0
+}
+
+pub fn set_executed(bitmap: &mut [u8], index: u32) {
+  let byte = &mut bitmap[index as usize / 8];
+  let bit = 1 << (index % 8);
+  *byte |= bit;
+}

tests/lazy-transactions.ts

@@ -201,11 +201,12 @@ describe("lazy-transactions", () => {
         })
         .accounts({ lazyTransactions })
         .remainingAccounts(accounts)
-        .rpc({ skipPreflight: true });
+        .rpc();
 
       throw new Error("Should have failed");
     } catch (e: any) {
-      expect(e.toString()).to.not.include("Should have failed");
+      console.log(e.toString())
+      expect(e.toString()).to.include("Transaction has already been executed");
     }
 
     /// Attempt to close the canopy