feat: ingest signer_bitvec (#1900)

* feat: ingest signer_bitvec

* test: add bitvec tests

migrations/1710856121027_stacks_block_signer-bitvec.js

@@ -0,0 +1,12 @@
+/* eslint-disable camelcase */
+
+/** @param { import("node-pg-migrate").MigrationBuilder } pgm */
+exports.up = pgm => {
+
+  pgm.addColumn('blocks', {
+    signer_bitvec: {
+      type: 'bit varying',
+    }
+  });
+
+};

src/datastore/common.ts

@@ -25,6 +25,7 @@ export interface DbBlock {
   execution_cost_write_length: number;
   tx_count: number;
   block_time: number;
+  signer_bitvec: string | null;
 }
 
 /** An interface representing the microblock data that can be constructed _only_ from the /new_microblocks payload */
@@ -844,6 +845,7 @@ export interface BlockQueryResult {
   execution_cost_write_count: string;
   execution_cost_write_length: string;
   tx_count: number;
+  signer_bitvec: string | null;
 }
 
 export interface MicroblockQueryResult {
@@ -1227,6 +1229,7 @@ export interface BlockInsertValues {
   execution_cost_write_count: number;
   execution_cost_write_length: number;
   tx_count: number;
+  signer_bitvec: string | null;
 }
 
 export interface MicroblockInsertValues {

src/datastore/helpers.ts

@@ -183,6 +183,7 @@ export const BLOCK_COLUMNS = [
   'execution_cost_write_count',
   'execution_cost_write_length',
   'tx_count',
+  'signer_bitvec',
 ];
 
 export const MICROBLOCK_COLUMNS = [
@@ -473,6 +474,7 @@ export function parseBlockQueryResult(row: BlockQueryResult): DbBlock {
     execution_cost_write_count: Number.parseInt(row.execution_cost_write_count),
     execution_cost_write_length: Number.parseInt(row.execution_cost_write_length),
     tx_count: row.tx_count,
+    signer_bitvec: row.signer_bitvec,
   };
   return block;
 }

src/datastore/pg-write-store.ts

@@ -465,6 +465,7 @@ export class PgWriteStore extends PgStore {
       execution_cost_write_count: block.execution_cost_write_count,
       execution_cost_write_length: block.execution_cost_write_length,
       tx_count: block.tx_count,
+      signer_bitvec: block.signer_bitvec,
     };
     const result = await sql`
       INSERT INTO blocks ${sql(values)}
@@ -3059,6 +3060,7 @@ export class PgWriteStore extends PgStore {
       execution_cost_write_count: block.execution_cost_write_count,
       execution_cost_write_length: block.execution_cost_write_length,
       tx_count: block.tx_count,
+      signer_bitvec: block.signer_bitvec,
     }));
     await sql`
       INSERT INTO blocks ${sql(values)}

src/event-stream/core-node-message.ts

@@ -271,6 +271,7 @@ export interface CoreNodeBlockMessage {
     };
   };
   block_time: number;
+  signer_bitvec?: string | null;
 }
 
 export interface CoreNodeParsedTxMessage {

src/event-stream/event-server.ts

@@ -6,7 +6,13 @@ import * as bodyParser from 'body-parser';
 import { asyncHandler } from '../api/async-handler';
 import PQueue from 'p-queue';
 import * as prom from 'prom-client';
-import { ChainID, assertNotNullish, getChainIDNetwork, getIbdBlockHeight } from '../helpers';
+import {
+  BitVec,
+  ChainID,
+  assertNotNullish,
+  getChainIDNetwork,
+  getIbdBlockHeight,
+} from '../helpers';
 import {
   CoreNodeBlockMessage,
   CoreNodeEventType,
@@ -291,6 +297,10 @@ async function handleBlockMessage(
     counts.events[event.type] += 1;
   }
 
+  const signerBitvec = msg.signer_bitvec
+    ? BitVec.consensusDeserializeToString(msg.signer_bitvec)
+    : null;
+
   const dbBlock: DbBlock = {
     canonical: true,
     block_hash: msg.block_hash,
@@ -311,6 +321,7 @@ async function handleBlockMessage(
     execution_cost_write_length: 0,
     tx_count: msg.transactions.length,
     block_time: msg.block_time,
+    signer_bitvec: signerBitvec,
   };
 
   logger.debug(`Received block ${msg.block_hash} (${msg.block_height}) from node`, dbBlock);
@@ -1148,6 +1159,7 @@ export function parseNewBlockMessage(chainId: ChainID, msg: CoreNodeBlockMessage
     execution_cost_write_count: totalCost.execution_cost_write_count,
     execution_cost_write_length: totalCost.execution_cost_write_length,
     tx_count: msg.transactions.length,
+    signer_bitvec: msg.signer_bitvec ?? null,
   };
 
   const dbMinerRewards: DbMinerReward[] = [];

src/helpers.ts

@@ -783,3 +783,45 @@ export function getUintEnvOrDefault(envName: string, defaultValue = 0) {
   }
   return Number(v);
 }
+
+export class BitVec {
+  bits: boolean[];
+  constructor(bits: boolean[]) {
+    this.bits = bits;
+  }
+
+  /**
+   * Deserialize a bit vector from a bytes in the consensus format:
+   *  - 2 bytes (u16): bit length (how many bits to read from the byte data)
+   *  - 4 bytes (u32): data length (how many remaining bytes to read)
+   */
+  static consensusDeserialize(serializedData: Uint8Array) {
+    const dataView = new DataView(serializedData.buffer, serializedData.byteOffset);
+    const bitLen = dataView.getUint16(0);
+    const dataLen = dataView.getUint32(2);
+    const bitVecBytes = serializedData.subarray(6, 6 + dataLen);
+    const bits = Array.from(
+      { length: bitLen },
+      (_, i) => !!(bitVecBytes[i >>> 3] & (128 >> i % 8))
+    );
+    return new BitVec(bits);
+  }
+
+  /** Return a base-2 string */
+  toString() {
+    return this.bits.map(b => (b ? '1' : '0')).join('');
+  }
+
+  /**
+   * Deserialize a bit vector from a bytes in the consensus format, and return as a base-2 string
+   */
+  static consensusDeserializeToString(serializedData: Uint8Array | string): string {
+    const data =
+      typeof serializedData === 'string'
+        ? Buffer.from(serializedData.replace(/^0x/, ''), 'hex')
+        : serializedData;
+    const bitVec = BitVec.consensusDeserialize(data);
+    const bitVecStr = bitVec.toString();
+    return bitVecStr;
+  }
+}

src/test-utils/test-builders.ts

@@ -125,6 +125,7 @@ function testBlock(args?: TestBlockArgs): DbBlock {
     execution_cost_write_count: 0,
     execution_cost_write_length: 0,
     tx_count: 1,
+    signer_bitvec: null,
   };
 }
 

src/tests/address-tests.ts

@@ -91,6 +91,7 @@ describe('address tests', () => {
       execution_cost_write_count: 0,
       execution_cost_write_length: 0,
       tx_count: 1,
+      signer_bitvec: null,
     };
     let indexIdIndex = 0;
     const createStxTx = (
@@ -884,6 +885,7 @@ describe('address tests', () => {
       execution_cost_write_count: 0,
       execution_cost_write_length: 0,
       tx_count: 1,
+      signer_bitvec: null,
     };
 
     let indexIdIndex = 0;
@@ -2078,6 +2080,7 @@ describe('address tests', () => {
       execution_cost_write_count: 0,
       execution_cost_write_length: 0,
       tx_count: 1,
+      signer_bitvec: null,
     };
     const txBuilder = await makeContractCall({
       contractAddress: 'ST11NJTTKGVT6D1HY4NJRVQWMQM7TVAR091EJ8P2Y',

src/tests/block-tests.ts

@@ -85,6 +85,7 @@ describe('block tests', () => {
       execution_cost_write_count: 0,
       execution_cost_write_length: 0,
       tx_count: 1,
+      signer_bitvec: null,
     };
     await db.updateBlock(client, block);
     const tx: DbTxRaw = {
@@ -532,6 +533,7 @@ describe('block tests', () => {
       execution_cost_write_count: 0,
       execution_cost_write_length: 0,
       tx_count: 1,
+      signer_bitvec: null,
     };
     const dbTx1: DbTxRaw = {
       ...dbBlock,

src/tests/cache-control-tests.ts

@@ -97,6 +97,7 @@ describe('cache-control tests', () => {
       execution_cost_write_count: 0,
       execution_cost_write_length: 0,
       tx_count: 1,
+      signer_bitvec: null,
     };
     const tx: DbTxRaw = {
       tx_id: '0x1234',