From 470e081c9a55aed9b1079550f34e32a6a07afa72 Mon Sep 17 00:00:00 2001 From: Chewing Glass Date: Tue, 29 Aug 2023 13:24:55 -0500 Subject: [PATCH] Switch to true bitmap to save rent over Vec --- .../src/close-lazy-transaction-markers.ts | 76 +++++++++++++++++++ packages/lazy-transactions-sdk/src/index.ts | 11 +++ packages/migration-service/src/index.ts | 3 +- .../src/instructions/close_marker_v0.rs | 4 +- .../instructions/execute_transaction_v0.rs | 6 +- .../initialize_lazy_transactions_v0.rs | 4 +- .../reinitialize_executed_transactions_v0.rs | 29 +++++++ programs/lazy-transactions/src/lib.rs | 1 + programs/lazy-transactions/src/state.rs | 3 +- programs/lazy-transactions/src/util.rs | 17 +++++ tests/lazy-transactions.ts | 5 +- 11 files changed, 149 insertions(+), 10 deletions(-) create mode 100644 packages/helium-admin-cli/src/close-lazy-transaction-markers.ts create mode 100644 programs/lazy-transactions/src/instructions/reinitialize_executed_transactions_v0.rs create mode 100644 programs/lazy-transactions/src/util.rs diff --git a/packages/helium-admin-cli/src/close-lazy-transaction-markers.ts b/packages/helium-admin-cli/src/close-lazy-transaction-markers.ts new file mode 100644 index 000000000..4fef7d808 --- /dev/null +++ b/packages/helium-admin-cli/src/close-lazy-transaction-markers.ts @@ -0,0 +1,76 @@ +import * as anchor from "@coral-xyz/anchor"; +import { blockKey, init as initLazy, lazyTransactionsKey } from "@helium/lazy-transactions-sdk"; +import os from "os"; +import yargs from "yargs/yargs"; +import { bulkSendTransactions, chunks } from "@helium/spl-utils"; +import { Transaction } from "@solana/web3.js"; + +export async function run(args: any = process.argv) { + const yarg = yargs(args).options({ + wallet: { + alias: "k", + describe: "Anchor wallet keypair", + default: `${os.homedir()}/.config/solana/id.json`, + }, + url: { + alias: "u", + default: "http://127.0.0.1:8899", + describe: "The solana url", + }, + name: { + default: "nJWGUMOK", + describe: "The lazy transactions instance name" + } + }); + const argv = await yarg.argv; + process.env.ANCHOR_WALLET = argv.wallet; + process.env.ANCHOR_PROVIDER_URL = argv.url; + anchor.setProvider(anchor.AnchorProvider.local(argv.url)); + const provider = anchor.getProvider() as anchor.AnchorProvider; + const lazyProgram = await initLazy(provider); + const ltKey = lazyTransactionsKey(argv.name)[0]; + const lt = await lazyProgram.account.lazyTransactionsV0.fetch(ltKey); + + const blocks = await lazyProgram.account.block.all(); + const blocksByKey = new Set(blocks.map((b) => b.publicKey.toString())); + const allIndices = new Array(1 << lt.maxDepth).fill(0).map((_, i) => i); + const blockIndices = allIndices.filter((bi) => + blocksByKey.has(blockKey(ltKey, bi)[0].toBase58()) + ); + if (lt.executed.length !== 1 << lt.maxDepth) { + await lazyProgram.methods + .reinitializeExecutedTransactionsV0() + .accounts({ + lazyTransactions: ltKey, + }) + .rpc({ skipPreflight: true }); + } + const instructions = await Promise.all( + blockIndices.map((bi) => + lazyProgram.methods + .closeMarkerV0({ + index: bi, + }) + .accounts({ + refund: provider.wallet.publicKey, + lazyTransactions: ltKey, + authority: provider.wallet.publicKey, + }) + .instruction() + ) + ); + + console.log(`${blocks.length} blocks to close`); + const txns = chunks(instructions, 10).map(chunk => { + const tx = new Transaction({ + feePayer: provider.wallet.publicKey, + }) + tx.add(...chunk) + return tx + }) + + await bulkSendTransactions(provider, txns, (status) => { + console.log(`Sending ${status.currentBatchProgress} / ${status.currentBatchSize} batch. ${status.totalProgress} / ${txns.length}`) + }) + console.log("Done") +} diff --git a/packages/lazy-transactions-sdk/src/index.ts b/packages/lazy-transactions-sdk/src/index.ts index 5498af2de..6c5907ab7 100644 --- a/packages/lazy-transactions-sdk/src/index.ts +++ b/packages/lazy-transactions-sdk/src/index.ts @@ -323,3 +323,14 @@ export function compileNoMerkle( return compiledTransactions } + +export function isExecuted( + executed: Buffer, + index: number, +): boolean { + const byteIndex = Math.floor(index / 8); + const bitIndex = index % 8; + const byte = executed[byteIndex]; + const mask = 1 << bitIndex; + return (byte & mask) !== 0; +} diff --git a/packages/migration-service/src/index.ts b/packages/migration-service/src/index.ts index 512cbe887..bccec2e9b 100644 --- a/packages/migration-service/src/index.ts +++ b/packages/migration-service/src/index.ts @@ -6,6 +6,7 @@ import { LazyTransactions } from "@helium/idls/lib/types/lazy_transactions"; import { blockKey, init, + isExecuted, lazySignerKey, lazyTransactionsKey, } from "@helium/lazy-transactions-sdk"; @@ -146,7 +147,7 @@ async function getTransactions( }, idx ) => { - const hasRun = executed[idx]; + const hasRun = isExecuted(executed, idx); const compiledTx = decompress(compiled); const block = blockKey(lazyTransactions, id)[0]; const signers = decompressSigners(signersRaw); diff --git a/programs/lazy-transactions/src/instructions/close_marker_v0.rs b/programs/lazy-transactions/src/instructions/close_marker_v0.rs index 3b1c6aa42..b21ee215b 100644 --- a/programs/lazy-transactions/src/instructions/close_marker_v0.rs +++ b/programs/lazy-transactions/src/instructions/close_marker_v0.rs @@ -1,4 +1,4 @@ -use crate::state::*; +use crate::{state::*, util::set_executed}; use anchor_lang::prelude::*; #[derive(AnchorSerialize, AnchorDeserialize, Clone, Default)] @@ -28,7 +28,7 @@ pub struct CloseMarkerV0<'info> { } pub fn handler(ctx: Context, args: CloseMarkerArgsV0) -> Result<()> { - ctx.accounts.lazy_transactions.executed[args.index as usize] = true; + set_executed(&mut ctx.accounts.lazy_transactions.executed, args.index); Ok(()) } diff --git a/programs/lazy-transactions/src/instructions/execute_transaction_v0.rs b/programs/lazy-transactions/src/instructions/execute_transaction_v0.rs index 2ab7f800a..d95c28d6e 100644 --- a/programs/lazy-transactions/src/instructions/execute_transaction_v0.rs +++ b/programs/lazy-transactions/src/instructions/execute_transaction_v0.rs @@ -1,5 +1,6 @@ use crate::canopy::fill_in_proof_from_canopy; use crate::error::ErrorCode; +use crate::util::{is_executed, set_executed}; use crate::{merkle_proof::verify, state::*}; use anchor_lang::{prelude::*, solana_program, solana_program::instruction::Instruction}; @@ -30,8 +31,9 @@ pub struct ExecuteTransactionV0<'info> { #[account(mut)] pub payer: Signer<'info>, #[account( + mut, has_one = canopy, - constraint = lazy_transactions.executed[args.index as usize] == false @ ErrorCode::TransactionAlreadyExecuted, + constraint = !is_executed(&lazy_transactions.executed, args.index) @ ErrorCode::TransactionAlreadyExecuted, )] pub lazy_transactions: Account<'info, LazyTransactionsV0>, /// CHECK: Verified by has one @@ -55,7 +57,7 @@ pub struct ExecuteTransactionV0<'info> { } pub fn handler(ctx: Context, args: ExecuteTransactionArgsV0) -> Result<()> { - ctx.accounts.lazy_transactions.executed[args.index as usize] = true; + set_executed(&mut ctx.accounts.lazy_transactions.executed, args.index); let largest_acct_idx: usize = (*args .instructions diff --git a/programs/lazy-transactions/src/instructions/initialize_lazy_transactions_v0.rs b/programs/lazy-transactions/src/instructions/initialize_lazy_transactions_v0.rs index b9afc7b8e..30ae18882 100644 --- a/programs/lazy-transactions/src/instructions/initialize_lazy_transactions_v0.rs +++ b/programs/lazy-transactions/src/instructions/initialize_lazy_transactions_v0.rs @@ -1,4 +1,4 @@ -use crate::{canopy::check_canopy_bytes, id, state::*}; +use crate::{canopy::check_canopy_bytes, id, state::*, util::get_bitmap_len}; use anchor_lang::prelude::*; #[derive(AnchorSerialize, AnchorDeserialize, Clone, Default)] @@ -50,7 +50,7 @@ pub fn handler( canopy: ctx.accounts.canopy.key(), max_depth: args.max_depth, bump_seed: ctx.bumps["lazy_transactions"], - executed: vec![false; 1 << args.max_depth], + executed: vec![0; get_bitmap_len(args.max_depth)], }); Ok(()) diff --git a/programs/lazy-transactions/src/instructions/reinitialize_executed_transactions_v0.rs b/programs/lazy-transactions/src/instructions/reinitialize_executed_transactions_v0.rs new file mode 100644 index 000000000..138124028 --- /dev/null +++ b/programs/lazy-transactions/src/instructions/reinitialize_executed_transactions_v0.rs @@ -0,0 +1,29 @@ +use crate::state::*; +use crate::util::get_bitmap_len; +use anchor_lang::prelude::*; +use shared_utils::resize_to_fit; + +#[derive(Accounts)] +pub struct ReinitializeExecutedTransactionsV0<'info> { + #[account(mut)] + pub payer: Signer<'info>, + pub authority: Signer<'info>, + #[account( + mut, + has_one = authority + )] + pub lazy_transactions: Box>, + pub system_program: Program<'info, System>, +} + +pub fn handler(ctx: Context) -> Result<()> { + ctx.accounts.lazy_transactions.executed = + vec![0; get_bitmap_len(ctx.accounts.lazy_transactions.max_depth)]; + + resize_to_fit( + &ctx.accounts.payer.to_account_info(), + &ctx.accounts.system_program.to_account_info(), + &ctx.accounts.lazy_transactions, + )?; + Ok(()) +} diff --git a/programs/lazy-transactions/src/lib.rs b/programs/lazy-transactions/src/lib.rs index dde60f8c9..68244f28c 100644 --- a/programs/lazy-transactions/src/lib.rs +++ b/programs/lazy-transactions/src/lib.rs @@ -7,6 +7,7 @@ pub mod error; pub mod instructions; pub mod merkle_proof; pub mod state; +pub mod util; pub use instructions::*; pub use state::*; diff --git a/programs/lazy-transactions/src/state.rs b/programs/lazy-transactions/src/state.rs index a2438da44..b2f11535d 100644 --- a/programs/lazy-transactions/src/state.rs +++ b/programs/lazy-transactions/src/state.rs @@ -9,7 +9,8 @@ pub struct LazyTransactionsV0 { pub authority: Pubkey, pub canopy: Pubkey, pub bump_seed: u8, - pub executed: Vec, + // Bitmap of executed transactions + pub executed: Vec, } #[account] diff --git a/programs/lazy-transactions/src/util.rs b/programs/lazy-transactions/src/util.rs new file mode 100644 index 000000000..6a70f0979 --- /dev/null +++ b/programs/lazy-transactions/src/util.rs @@ -0,0 +1,17 @@ +pub fn get_bitmap_len(max_depth: u32) -> usize { + let num_txns = 1 << max_depth; + // Div ceil, https://stackoverflow.com/questions/72442853/how-would-you-perform-ceiling-division + ((num_txns + 8 - 1) / 8) as usize +} + +pub fn is_executed(bitmap: &Vec, index: u32) -> bool { + let byte = bitmap[index as usize / 8]; + let bit = 1 << (index % 8); + byte & bit != 0 +} + +pub fn set_executed(bitmap: &mut Vec, index: u32) { + let byte = &mut bitmap[index as usize / 8]; + let bit = 1 << (index % 8); + *byte |= bit; +} diff --git a/tests/lazy-transactions.ts b/tests/lazy-transactions.ts index ce4378387..61a99c084 100644 --- a/tests/lazy-transactions.ts +++ b/tests/lazy-transactions.ts @@ -201,11 +201,12 @@ describe("lazy-transactions", () => { }) .accounts({ lazyTransactions }) .remainingAccounts(accounts) - .rpc({ skipPreflight: true }); + .rpc(); throw new Error("Should have failed"); } catch (e: any) { - expect(e.toString()).to.not.include("Should have failed"); + console.log(e.toString()) + expect(e.toString()).to.include("Transaction has already been executed"); } /// Attempt to close the canopy