feat: ft holder indexing (#2030)

* feat: stx holder indexing

* feat: ft holder indexing

* feat: include total supply in response

* chore: cleanup debug code

* chore: update ft holders endpoint

docs/api/tokens/get-ft-holders.example.json

@@ -0,0 +1,20 @@
+{
+  "limit": 100,
+  "offset": 0,
+  "total": 3,
+  "total_supply": "11700",
+  "results": [
+    {
+      "address": "SP3Y2ZSH8P7D50B0VBTSX11S7XSG24M1VB9YFQA2K",
+      "balance": "10000"
+    },
+    {
+      "address": "SP3WJYXJZ4QK2V5V9VX2VXVZ6VXVZ6V2V5V2V2V2V",
+      "balance": "900"
+    },
+    {
+      "address": "SP3WJYXJZ4QK2V5V9VX2VXVZ6VXVZ6V2V5V2V2V2V",
+      "balance": "800"
+    }
+  ]
+}

docs/api/tokens/get-ft-holders.schema.json

@@ -0,0 +1,38 @@
+{
+  "description": "List of Fungible Token holders",
+  "title": "FungibleTokenHolderList",
+  "type": "object",
+  "required": [
+    "total_supply",
+    "results",
+    "limit",
+    "offset",
+    "total"
+  ],
+  "additionalProperties": false,
+  "properties": {
+    "limit": {
+      "type": "integer",
+      "maximum": 200,
+      "description": "The number of holders to return"
+    },
+    "offset": {
+      "type": "integer",
+      "description": "The number to holders to skip (starting at `0`)"
+    },
+    "total": {
+      "type": "integer",
+      "description": "The number of holders available"
+    },
+    "total_supply": {
+      "type": "string",
+      "description": "The total supply of the token (the sum of all balances)"
+    },
+    "results": {
+      "type": "array",
+      "items": {
+        "$ref": "../../entities/tokens/ft-holder-entry.schema.json"
+      }
+    }
+  }
+}

docs/entities/tokens/ft-holder-entry.schema.json

@@ -0,0 +1,14 @@
+{
+  "type": "object",
+  "title": "FtHolderEntry",
+  "required": ["address", "balance"],
+  "additionalProperties": false,
+  "properties": {
+    "address": {
+      "type": "string"
+    },
+    "balance": {
+      "type": "string"
+    }
+  }
+}

docs/generated.d.ts

@@ -103,6 +103,7 @@ export type SchemaMergeRootStub =
   | PoxCycleSignerStackersListResponse
   | PoxCycleSignersListResponse
   | PoxCycleListResponse
+  | FungibleTokenHolderList
   | {
       [k: string]: unknown | undefined;
     }
@@ -193,6 +194,7 @@ export type SchemaMergeRootStub =
   | PoxCycle
   | PoxSigner
   | PoxStacker
+  | FtHolderEntry
   | NonFungibleTokenHistoryEventWithTxId
   | NonFungibleTokenHistoryEventWithTxMetadata
   | NonFungibleTokenHistoryEvent
@@ -3437,6 +3439,32 @@ export interface PoxCycle {
   total_stacked_amount: string;
   total_signers: number;
 }
+/**
+ * List of Fungible Token holders
+ */
+export interface FungibleTokenHolderList {
+  /**
+   * The number of holders to return
+   */
+  limit: number;
+  /**
+   * The number to holders to skip (starting at `0`)
+   */
+  offset: number;
+  /**
+   * The number of holders available
+   */
+  total: number;
+  /**
+   * The total supply of the token (the sum of all balances)
+   */
+  total_supply: string;
+  results: FtHolderEntry[];
+}
+export interface FtHolderEntry {
+  address: string;
+  balance: string;
+}
 /**
  * List of Non-Fungible Token history events
  */

docs/openapi.yaml

@@ -3678,6 +3678,38 @@ paths:
                   value:
                     $ref: ./api/tokens/get-non-fungible-token-mints-tx-metadata.example.schema.json
 
+  /extended/v1/tokens/ft/{token}/holders:
+    get:
+      operationId: get_ft_holders
+      summary: Fungible token holders
+      description: |
+        Retrieves the list of Fungible Token holders for a given token ID. Specify `stx` for the `token` parameter to get the list of STX holders.
+      tags:
+        - Fungible Tokens
+      parameters:
+        - name: token
+          in: path
+          description: fungible token identifier
+          required: true
+          schema:
+            type: string
+          examples:
+            stx:
+              value: stx
+              summary: STX token
+            ft:
+              value: SP3Y2ZSH8P7D50B0VBTSX11S7XSG24M1VB9YFQA4K.token-aeusdc::aeUSDC
+              summary: fungible token
+      responses:
+        200:
+          description: Fungible Token holders
+          content:
+            application/json:
+              schema:
+                $ref: ./api/tokens/get-ft-holders.schema.json
+              example:
+                $ref: ./api/tokens/get-ft-holders.example.json
+
   /extended/v1/fee_rate:
     post:
       operationId: fetch_fee_rate

migrations/1720532894811_ft_balances.js

@@ -0,0 +1,125 @@
+/** @param { import("node-pg-migrate").MigrationBuilder } pgm */
+exports.up = pgm => {
+  pgm.createTable('ft_balances', {
+    id: {
+      type: 'bigserial',
+      primaryKey: true,
+    },
+    address: {
+      type: 'text',
+      notNull: true,
+    },
+    token: {
+      type: 'text',
+      notNull: true,
+    },
+    balance: {
+      type: 'numeric',
+      notNull: true,
+    }
+  });
+
+  pgm.addConstraint('ft_balances', 'unique_address_token', `UNIQUE(address, token)`);
+
+  // Speeds up "grab the addresses with the highest balance for a given token" queries
+  pgm.createIndex('ft_balances', [{ name: 'token' }, { name: 'balance', sort: 'DESC' }]);
+
+  // Speeds up "get the total supply of a given token" queries
+  pgm.createIndex('ft_balances', 'token');
+
+  // Populate the table with the current stx balances
+  pgm.sql(`
+    WITH all_balances AS (
+        SELECT sender AS address, -SUM(amount) AS balance_change
+        FROM stx_events
+        WHERE asset_event_type_id IN (1, 3) -- Transfers and Burns affect the sender's balance
+        AND canonical = true AND microblock_canonical = true
+        GROUP BY sender
+      UNION ALL
+        SELECT recipient AS address, SUM(amount) AS balance_change
+        FROM stx_events
+        WHERE asset_event_type_id IN (1, 2) -- Transfers and Mints affect the recipient's balance
+        AND canonical = true AND microblock_canonical = true
+        GROUP BY recipient
+    ),
+    net_balances AS (
+      SELECT address, SUM(balance_change) AS balance
+      FROM all_balances
+      GROUP BY address
+    ),
+    fees AS (
+      SELECT address, SUM(total_fees) AS total_fees
+      FROM (
+          SELECT sender_address AS address, SUM(fee_rate) AS total_fees
+          FROM txs
+          WHERE canonical = true AND microblock_canonical = true AND sponsored = false
+          GROUP BY sender_address
+        UNION ALL
+          SELECT sponsor_address AS address, SUM(fee_rate) AS total_fees
+          FROM txs
+          WHERE canonical = true AND microblock_canonical = true AND sponsored = true
+          GROUP BY sponsor_address
+      ) AS subquery
+      GROUP BY address
+    ),
+    rewards AS (
+      SELECT
+        recipient AS address,
+        SUM(
+          coinbase_amount + tx_fees_anchored + tx_fees_streamed_confirmed + tx_fees_streamed_produced
+        ) AS total_rewards
+      FROM miner_rewards
+      WHERE canonical = true
+      GROUP BY recipient
+    ),
+    all_addresses AS (
+      SELECT address FROM net_balances
+      UNION
+      SELECT address FROM fees
+      UNION
+      SELECT address FROM rewards
+    )
+    INSERT INTO ft_balances (address, balance, token)
+    SELECT
+      aa.address,
+      COALESCE(nb.balance, 0) - COALESCE(f.total_fees, 0) + COALESCE(r.total_rewards, 0) AS balance,
+      'stx' AS token
+    FROM all_addresses aa
+    LEFT JOIN net_balances nb ON aa.address = nb.address
+    LEFT JOIN fees f ON aa.address = f.address
+    LEFT JOIN rewards r ON aa.address = r.address
+  `);
+
+  // Populate the table with the current FT balances
+  pgm.sql(`
+    WITH all_balances AS (
+        SELECT sender AS address, asset_identifier, -SUM(amount) AS balance_change
+        FROM ft_events
+        WHERE asset_event_type_id IN (1, 3) -- Transfers and Burns affect the sender's balance
+          AND canonical = true 
+          AND microblock_canonical = true
+        GROUP BY sender, asset_identifier
+      UNION ALL
+        SELECT recipient AS address, asset_identifier, SUM(amount) AS balance_change
+        FROM ft_events
+        WHERE asset_event_type_id IN (1, 2) -- Transfers and Mints affect the recipient's balance
+          AND canonical = true 
+          AND microblock_canonical = true
+        GROUP BY recipient, asset_identifier
+    ),
+    net_balances AS (
+      SELECT address, asset_identifier, SUM(balance_change) AS balance
+      FROM all_balances
+      GROUP BY address, asset_identifier
+    )
+    INSERT INTO ft_balances (address, balance, token)
+    SELECT address, balance, asset_identifier AS token
+    FROM net_balances
+  `);
+
+};
+
+/** @param { import("node-pg-migrate").MigrationBuilder } pgm */
+exports.down = pgm => {
+  pgm.dropTable('ft_balances');
+};

src/api/pagination.ts

@@ -38,6 +38,7 @@ export enum ResourceType {
   BurnBlock,
   Signer,
   PoxCycle,
+  TokenHolders,
 }
 
 export const pagingQueryLimits: Record<ResourceType, { defaultLimit: number; maxLimit: number }> = {
@@ -89,6 +90,10 @@ export const pagingQueryLimits: Record<ResourceType, { defaultLimit: number; max
     defaultLimit: 20,
     maxLimit: 60,
   },
+  [ResourceType.TokenHolders]: {
+    defaultLimit: 100,
+    maxLimit: 200,
+  },
 };
 
 export function getPagingQueryLimit(

src/api/routes/tokens.ts

@@ -1,6 +1,7 @@
 import { asyncHandler } from '../async-handler';
 import * as express from 'express';
 import {
+  FungibleTokenHolderList,
   NonFungibleTokenHistoryEvent,
   NonFungibleTokenHistoryEventList,
   NonFungibleTokenHolding,
@@ -228,5 +229,25 @@ export function createTokenRouter(db: PgStore): express.Router {
     })
   );
 
+  router.get(
+    '/ft/:token/holders',
+    cacheHandler,
+    asyncHandler(async (req, res) => {
+      const token = req.params.token;
+      const limit = getPagingQueryLimit(ResourceType.TokenHolders, req.query.limit);
+      const offset = parsePagingQueryInput(req.query.offset ?? 0);
+      const { results, total, totalSupply } = await db.getTokenHolders({ token, limit, offset });
+      const response: FungibleTokenHolderList = {
+        limit: limit,
+        offset: offset,
+        total: total,
+        total_supply: totalSupply,
+        results: results,
+      };
+      setETagCacheHeaders(res);
+      res.status(200).json(response);
+    })
+  );
+
   return router;
 }

src/datastore/pg-store.ts

@@ -3288,6 +3288,39 @@ export class PgStore extends BasePgStore {
     return { found: true, result: queryResult };
   }
 
+  async getTokenHolders(args: { token: string; limit: number; offset: number }): Promise<{
+    results: { address: string; balance: string }[];
+    totalSupply: string;
+    total: number;
+  }> {
+    const holderResults = await this.sql<
+      { address: string; balance: string; count: number; total_supply: string }[]
+    >`
+      WITH totals AS (
+        SELECT 
+          SUM(balance) AS total,
+          COUNT(*)::int AS total_count
+        FROM ft_balances
+        WHERE token = ${args.token}
+      )
+      SELECT 
+        fb.address, 
+        fb.balance, 
+        ts.total AS total_supply,
+        ts.total_count AS count
+      FROM ft_balances fb
+      JOIN totals ts ON true
+      WHERE token = ${args.token}
+      ORDER BY balance DESC
+      LIMIT ${args.limit}
+      OFFSET ${args.offset}
+    `;
+    const results = holderResults.map(row => ({ address: row.address, balance: row.balance }));
+    const total = holderResults.length > 0 ? holderResults[0].count : 0;
+    const totalSupply = holderResults.length > 0 ? holderResults[0].total_supply : '0';
+    return { results, totalSupply, total };
+  }
+
   /**
    * Returns a list of NFTs owned by the given principal filtered by optional `asset_identifiers`,
    * including optional transaction metadata.