Merge pull request #304 from kodadot/main

🔖  Speck v14

README.md

@@ -162,6 +162,14 @@ CHAIN=polkadot # or kusama
 OFFER=<ID_OF_THE_COLLECTION>
 ```
 
+6. debugging the processor
+
+As the processor can run for a longer period of time it is useful to turn off the "features" that you do not need,
+Handlers that need to be always enabled are `createCollection` and `createItem` as they are the base for the rest of the processor.
+
+> [!NOTE]
+> If you do not wish to index `uniques` pallet you can turn it off by setting `UNIQUES_ENABLED=false` in `.env` file
+
 ### Note on Swaps
 
 1. Swaps can be overwritten at any time

db/migrations/1726485003107-Data.js

@@ -0,0 +1,13 @@
+module.exports = class Data1726485003107 {
+    name = 'Data1726485003107'
+
+    async up(db) {
+        await db.query(`ALTER TABLE "metadata_entity" ADD "kind" character varying(6)`)
+        await db.query(`ALTER TABLE "collection_entity" ADD "kind" character varying(6)`)
+    }
+
+    async down(db) {
+        await db.query(`ALTER TABLE "metadata_entity" DROP COLUMN "kind"`)
+        await db.query(`ALTER TABLE "collection_entity" DROP COLUMN "kind"`)
+    }
+}

package.json

@@ -16,13 +16,13 @@
     "query-node:start": "squid-graphql-server --subscriptions --max-response-size 10000 --dumb-cache in-memory --dumb-cache-ttl 1000 --dumb-cache-size 100 --dumb-cache-max-age 1000"
   },
   "dependencies": {
-    "@kodadot1/hyperdata": "^0.0.1-rc.4",
+    "@kodadot1/hyperdata": "^0.0.1-rc.5",
     "@kodadot1/metasquid": "^0.3.1-rc.0",
     "@kodadot1/minipfs": "^0.4.3-rc.2",
     "@subsquid/archive-registry": "3.3.2",
-    "@subsquid/graphql-server": "4.6.0",
+    "@subsquid/graphql-server": "4.7.0",
     "@subsquid/ss58": "2.0.2",
-    "@subsquid/substrate-processor": "8.4.1",
+    "@subsquid/substrate-processor": "8.5.1",
     "@subsquid/typeorm-migration": "1.3.0",
     "@subsquid/typeorm-store": "1.5.1",
     "dotenv": "^16.4.5",
@@ -33,9 +33,9 @@
     "typeorm": "0.3.17"
   },
   "devDependencies": {
-    "@subsquid/substrate-metadata-explorer": "3.1.2",
+    "@subsquid/substrate-metadata-explorer": "3.2.0",
     "@subsquid/substrate-typegen": "8.1.0",
-    "@subsquid/typeorm-codegen": "2.0.1",
+    "@subsquid/typeorm-codegen": "2.0.2",
     "@types/md5": "^2.3.5",
     "@types/node": "18.11.18",
     "@types/pg": "^8.11.4",

schema.graphql

@@ -16,6 +16,7 @@ type CollectionEntity @entity {
   id: ID!
   image: String
   issuer: String!
+  kind: Kind
   max: Int
   media: String
   meta: MetadataEntity
@@ -97,6 +98,7 @@ type MetadataEntity @entity {
   animationUrl: String
   type: String
   banner: String
+  kind: Kind
 }
 
 # Entity to represent an Attribute
@@ -259,6 +261,15 @@ enum TradeStatus {
   WITHDRAWN
 }
 
+enum Kind {
+  poap
+  pfp
+  genart
+  mixed
+  # audio
+  # video
+}
+
 # Entity to represent a Fungible Asset
 # defined on chain as  pub type Asset<T: Config<I>, I: 'static = ()>
 # https://github.com/paritytech/polkadot-sdk/blob/99234440f0f8b24f7e4d1d3a0102a9b19a408dd3/substrate/frame/assets/src/lib.rs#L325

speck.yaml

@@ -1,6 +1,6 @@
 manifestVersion: subsquid.io/v0.1
 name: speck
-version: 13
+version: 14
 description: 'SubSquid indexer for Uniques and Assets on Statemint'
 build: 
 deploy:
@@ -16,6 +16,7 @@ deploy:
     env:
       CHAIN: polkadot
       OFFER: 174
+      UNIQUES_ENABLED: true
   api:
     cmd:
       - npx

squid.yaml

@@ -16,6 +16,7 @@ deploy:
     env:
       CHAIN: kusama
       OFFER: 464
+      UNIQUES_ENABLED: true
   api:
     cmd:
       - npx

src/environment.ts

@@ -3,9 +3,10 @@ export type Chain = 'kusama' | 'rococo' | 'polkadot'
 export const CHAIN: Chain = process.env.CHAIN as Chain || 'kusama'
 export const COLLECTION_OFFER: string = process.env.OFFER || ''
 
+export const UNIQUES_ENABLED = process.env.UNIQUES === 'true'
 const UNIQUE_STARTING_BLOCK = 323_750 // 618838;
-// const _NFT_STARTING_BLOCK = 4_556_552
-export const STARTING_BLOCK = UNIQUE_STARTING_BLOCK
+const NFT_STARTING_BLOCK = 4_556_552
+export const STARTING_BLOCK = UNIQUES_ENABLED ? UNIQUE_STARTING_BLOCK : NFT_STARTING_BLOCK
 
 // Asset Hub
 const ARCHIVE_URL = `https://v2.archive.subsquid.io/network/asset-hub-${CHAIN}`
@@ -16,6 +17,7 @@ export const isProd = CHAIN !== 'rococo'
 console.table({
   CHAIN, ARCHIVE_URL, NODE_URL, STARTING_BLOCK,
   COLLECTION_OFFER,
+  UNIQUES_ENABLED,
   disabledRPC: false,
   environment: isProd ? 'production' : 'development',
 })

src/mappings/index.ts

@@ -8,8 +8,7 @@ import * as a from './assets'
 import * as n from './nfts'
 import * as u from './uniques'
 import { BatchContext, Context, SelectedEvent } from './utils/types'
-import { updateCache } from './utils/cache'
-import { logError } from './utils/logger'
+import { updateOfferCache } from './utils/cache'
 
 type HandlerFunction = <T extends SelectedEvent>(item: T, ctx: Context) => Promise<void>
 
@@ -234,105 +233,26 @@ export async function mainFrame(ctx: BatchContext<Store>): Promise<void> {
     }
   }
 
-  // const lastDate = new Date(ctx.blocks[ctx.blocks.length - 1].header.timestamp || start)
-  // await updateCache(lastDate, ctx.store)  
-
-  // const { contracts, tokens } = uniqueEntitySets(items)
-  // const collections = await finalizeCollections(contracts, ctx)
-  // const finish = await whatToDoWithTokens({ tokens, collections, items }, ctx)
-  // const complete = await completeTokens(ctx, finish)
-
-  // logger.info(`Batch completed, ${complete.length} tokens saved`)
+  if (ctx.isHead) {
+    const lastBlock = ctx.blocks[ctx.blocks.length - 1].header
+    const lastDate = new Date(lastBlock.timestamp || Date.now())
+    logger.info(`Found head block, updating cache`)
+    await updateOfferCache(lastDate, lastBlock.height, ctx.store)  
+  }
 }
 
-// function unwrapLog(log: Log, block: BlockData) {
-//   switch (log.topics[0]) {
-//     case ERC721_TRANSFER:
+// class Head {
+//   #height: number
 
-//       if (log.address !== Contracts.HueNft) {
-//         return null
-//       }
-//       return handle721Token(log, block)
-//     default:
-//       // console.log('unknown log', log.topics[0])
-//       return null
-//     // throw new Error('unknown log')
+//   constructor(height: number) {
+//     this.#height = height
 //   }
-// }
-
-type What = {
-  // tokens: Set<string>,
-  // collections: EnMap<CE>,
-  // items: ItemStateUpdate[],
-}
 
-export async function whatToDoWithTokens(x: What, ctx: Context) {
-  // // ctx.store.findBy(CE, {id: In([...collectionMap.keys()])})
-  // const knownTokens = await findByIdListAsMap(ctx.store, NE, tokens)
-  // const events: EventEntity[] = []
-  // for (const item of items) {
-  //   logger.debug(`APPLY ${item.interaction} on ${item.id}`)
-  //   let knownToken = knownTokens.get(item.id) ?? create(NE, item.id, {})
-  //   if (item.applyFrom) {
-  //     const collection = collections.get(item.contract)!
-  //     item.applyFrom(collection)
-  //   }
-  //   if (item.applyTo) {
-  //     knownToken = item.applyTo(knownToken)
-  //   }
-  //   events.push(item.event)
-  //   knownTokens.set(item.id, knownToken)
-  // }
-  // const values = [...knownTokens.values()]
-  // await ctx.store.upsert(values)
-  // await ctx.store.save(events)
-  // return knownTokens
-}
+//   get height() {
+//     return this.#height
+//   }
 
-type EnMap<T> = Map<string, T>
-// TODO: do only if event was mint.
-async function completeTokens(ctx: Context, tokenMap: EnMap<NE>) {
-  //   const collections = groupedItemsByCollection(tokenMap.keys())
-  //   const final: NE[] = []
-  //   const metadataFutures: Promise<Optional<MetadataEntity>>[] = []
-  //   for (const [contract, ids] of collections.entries()) {
-  //     const list = Array.from(ids)
-  //     const tokens = await multicallMetadataFetch(ctx, contract, list)
-  //     for (const [i, id] of list.entries()) {
-  //       const realId = createTokenId(contract, id)
-  //       const token = tokenMap.get(realId)!
-  //       if (!token.metadata) {
-  //         const metadata = tokens[i]
-  //         token.metadata = metadata
-  //         const getMeta = handleMetadata(metadata, ctx.store).then(m => {
-  //           if (m) {
-  //             token.meta = m
-  //             token.name = m.name
-  //             token.image = m.image
-  //             token.media = m.animationUrl
-  //           }
-  //           return m
-  //         })
-  //         metadataFutures.push(getMeta)
-  //         final.push(token)
-  //       }
-  //     }
-  //   }
-  //   const metaList = await Promise.all(metadataFutures)
-  //   const filtered = metaList.filter(m => m) as MetadataEntity[]
-  //   logger.debug(`Saving ${filtered.length} metadata`)
-  //   await ctx.store.save(filtered)
-  //   await ctx.store.save(final)
-  //   return final
-  // }
-  // async function multicallMetadataFetch(ctx: Context, collection: string, tokens: Array<string>): Promise<string[]> {
-  //   const tokenIds = tokens.map((id) => [BigInt(id)])
-  //   const contract = new Multicall(ctx, lastBatchBlock(ctx), MULTICALL_ADDRESS)
-  //   const metadata = await contract.aggregate(
-  //     erc721.functions.tokenURI,
-  //     collection,
-  //     tokenIds,
-  //     MULTICALL_BATCH_SIZE
-  //   )
-  //   return metadata
-}
+//   set height(height: number) {
+//     this.#height = height
+//   }
+// }

src/mappings/shared/metadata.ts

@@ -3,7 +3,7 @@ import { Optional } from '@kodadot1/metasquid/types'
 
 import type { Content } from '@kodadot1/hyperdata'
 import { logger } from '@kodadot1/metasquid/logger'
-import { MetadataEntity as Metadata } from '../../model/generated'
+import { Kind, MetadataEntity as Metadata } from '../../model/generated'
 import { isEmpty } from '../utils/helper'
 import { fetchMetadata } from '../utils/metadata'
 import { Store, attributeFrom } from '../utils/types'
@@ -31,6 +31,7 @@ export async function handleMetadata(id: string, store: Store): Promise<Optional
     name: metadata.name || '',
     type: metadata.type || '',
     banner: metadata.banner || '',
+    kind: metadata.kind as Kind || Kind.mixed,
   }
 
   const final = create<Metadata>(Metadata, id, partial)

src/mappings/utils/cache.ts

@@ -1,6 +1,6 @@
 import { type Content } from '@kodadot1/hyperdata'
 import { EntityWithId, create, emOf, getOrCreate } from '@kodadot1/metasquid/entity'
-import { CacheStatus, MetadataEntity } from '../../model'
+import { CacheStatus, Interaction, MetadataEntity } from '../../model'
 import logger, { logError, pending, success } from './logger'
 import { fetchAllMetadata } from './metadata'
 import { Store } from './types'
@@ -10,6 +10,8 @@ const STATUS_ID = '0'
 const METADATA_STATUS_ID = '1'
 const METADATA_DELAY_MIN = 15 // every 24 hours
 const TO_MINUTES = 60_000
+const OFFER_STATUS_ID = '2'
+// const OFFER_DELAY_MIN = 30 // every 15 minutes
 
 enum MetadataQuery {
   missing = `SELECT 
@@ -62,6 +64,15 @@ enum MetadataQuery {
   `,
 }
 
+enum OfferQuery {
+  expired = `UPDATE 
+    offer oe
+  SET status = 'EXPIRED'
+  WHERE status = 'ACTIVE'
+  AND expiration <= $1
+  RETURNING oe.id;`
+}
+
 const OPERATION = 'METADATA_CACHE' as any
 
 /**
@@ -103,6 +114,28 @@ export async function updateMetadataCache(timestamp: Date, store: Store): Promis
   }
 }
 
+/**
+ * Main entry point for updating the metadata cache
+ * @param timestamp - the timestamp of the block
+ * @param store - subsquid store to handle the cache
+**/
+export async function updateOfferCache(timestamp: Date, blockNumber: number, store: Store): Promise<void> {
+  const lastUpdate = await getOrCreate(store, CacheStatus, OFFER_STATUS_ID, { id: OFFER_STATUS_ID, lastBlockTimestamp: new Date(0) })
+  const passedMins = getPassedMinutes(timestamp, lastUpdate.lastBlockTimestamp)
+  pending(Interaction.OFFER, `${passedMins} MINS SINCE LAST UPDATE`)
+  if (passedMins >= DELAY_MIN) {
+    try {
+      await updateOfferAsExpired(store, blockNumber)
+      lastUpdate.lastBlockTimestamp = timestamp
+      await store.save(lastUpdate)
+      // success('[METADATA CACHE UPDATE]');
+    } catch (e) {
+      logError(e, (err) => logger.error(`[OFFER CACHE UPDATE] ${err.message}`))
+    }
+  }
+}
+
+
 /**
  * Main entry point for the cache update
  * @param timestamp - the timestamp of the block
@@ -144,4 +177,13 @@ async function updateMissingMetadata(store: Store) {
   // const nft = await emOf(store).query(MetadataQuery.nft);
   // const collection = await emOf(store).query(MetadataQuery.collection);
   // logger.info(`[CACHE UPDATE] MISSING METADATA - ${missing.length} NFTs, ${nft.length} NFTs, ${collection.length} Collections`);
+}
+
+export async function updateOfferAsExpired(store: Store, blockNumber: string | bigint | number): Promise<void> {
+  try {
+    const rows = await emOf(store).query(OfferQuery.expired, [blockNumber])
+    logger.info(`[OFFERS EXPIRATION POOLER] ${rows.length} Offers updated`)
+  } catch (e) {
+    logError(e, (err) => logger.error(`[OFFERS EXPIRATION POOLER] ${err.message}`))
+  }
 }

src/model/generated/_kind.ts

@@ -0,0 +1,6 @@
+export enum Kind {
+    poap = "poap",
+    pfp = "pfp",
+    genart = "genart",
+    mixed = "mixed",
+}

src/model/generated/collectionEntity.model.ts

@@ -2,6 +2,7 @@ import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, S
 import * as marshal from "./marshal"
 import {Attribute} from "./_attribute"
 import {CollectionEvent} from "./collectionEvent.model"
+import {Kind} from "./_kind"
 import {MetadataEntity} from "./metadataEntity.model"
 import {NFTEntity} from "./nftEntity.model"
 import {CollectionType} from "./_collectionType"
@@ -60,6 +61,9 @@ export class CollectionEntity {
     @StringColumn_({nullable: false})
     issuer!: string
 
+    @Column_("varchar", {length: 6, nullable: true})
+    kind!: Kind | undefined | null
+
     @IntColumn_({nullable: true})
     max!: number | undefined | null
 

src/model/generated/index.ts

@@ -1,5 +1,6 @@
 export * from "./collectionEntity.model"
 export * from "./_attribute"
+export * from "./_kind"
 export * from "./_collectionType"
 export * from "./_collectionSettings"
 export * from "./tokenEntity.model"