From ef2cff14c70df8ca33499a4cd485ccc13b8e6de1 Mon Sep 17 00:00:00 2001 From: crypticmeta Date: Wed, 9 Oct 2024 13:24:25 +0530 Subject: [PATCH] stop adding tokens to DB --- indexer/src/index.ts | 34 +- indexer/src/insertSkippedBlock.ts | 590 +++++++++++++++--------------- 2 files changed, 316 insertions(+), 308 deletions(-) diff --git a/indexer/src/index.ts b/indexer/src/index.ts index b13f2c66e2..f85b2df4f6 100644 --- a/indexer/src/index.ts +++ b/indexer/src/index.ts @@ -96,7 +96,7 @@ const INDEXER_VERSION = 1; const ORD_VERSION = "0.18.5"; // import mempoolJS from "cryptic-mempool"; -import { handlePreSaveLogic, InsertSkippedBlock } from './insertSkippedBlock'; +import { handlePreSaveLogic } from './insertSkippedBlock'; import axios from 'axios'; export function delay(sec: number): Promise { @@ -223,18 +223,18 @@ console.log(execSync("pwd", { stdio: 'inherit' })) console.log("Nothing new, waiting!!") // check latest mempool height and latest height in our db to find if we skipped some blocks - const {data: mempool_height} = await axios.get(`https://mempool.ordinalnovus.com/api/blocks/tip/height`); - if(mempool_height > current_height) - { - console.log({current_height}) - console.log(`We are ${mempool_height-current_height} Blocks Behind`); - for(let i = current_height + 1; i<=mempool_height; i++){ - await InsertSkippedBlock(i); + // const {data: mempool_height} = await axios.get(`https://mempool.ordinalnovus.com/api/blocks/tip/height`); + // if(mempool_height > current_height) + // { + // console.log({current_height}) + // console.log(`We are ${mempool_height-current_height} Blocks Behind`); + // for(let i = current_height + 1; i<=mempool_height; i++){ + // await InsertSkippedBlock(i); - } - await cleanup() + // } + // await cleanup() - } + // } return; } @@ -496,7 +496,8 @@ console.log(execSync("pwd", { stdio: 'inherit' })) const doc = processDoc(parts) - inscription_ops.push({ + if(doc){ + inscription_ops.push({ updateOne: { filter: { inscription_id: doc.inscription_id }, update: {$set: filterEmptyFields(doc)} @@ -507,6 +508,7 @@ console.log(execSync("pwd", { stdio: 'inherit' })) // console.log("--4", { doc, parts }) // inscription_ops.push(execute_on_db(sql_query_insert_content, [parts[4], content, parts[6], parts[7], block_height])) ord_sql_query_count += 1 + } } else { console.log({ insert_text_content: { @@ -525,6 +527,7 @@ console.log(execSync("pwd", { stdio: 'inherit' })) // parts, sql_query_insert_text_content, l // }}, '--6'); const doc = processDoc(parts) + if(doc){ inscription_ops.push({ updateOne: { @@ -535,6 +538,7 @@ console.log(execSync("pwd", { stdio: 'inherit' })) // inscription_ops.push(execute_on_db(sql_query_insert_text_content, [parts[4], content, parts[6], parts[7], block_height])) ord_sql_query_count += 1 + } } } } @@ -979,7 +983,7 @@ if(data.inscriptions.length){ async function forever_loop(){ while(true){ await main_index(); - await delay(60) + await delay(30) } } @@ -1316,6 +1320,10 @@ if (delegate_matches) { }), }; + + if(newItem.token){ + return null; + } return newItem } diff --git a/indexer/src/insertSkippedBlock.ts b/indexer/src/insertSkippedBlock.ts index 0c56cd929d..fd443828cb 100644 --- a/indexer/src/insertSkippedBlock.ts +++ b/indexer/src/insertSkippedBlock.ts @@ -1,290 +1,290 @@ -import dbConnect from "./lib/dbConnect" -import { BlockHashes, Inscription } from "./models"; -import axios from 'axios'; -import fetchContentFromProviders, { fetchInscriptionDetails } from "./utils"; -import crypto from "crypto" +// import dbConnect from "./lib/dbConnect" +// import { BlockHashes, Inscription } from "./models"; +// import axios from 'axios'; +// import fetchContentFromProviders, { fetchInscriptionDetails } from "./utils"; +// import crypto from "crypto" -export async function InsertSkippedBlock(block: number){ - try{ - await dbConnect(); - await checkPrevBlock(block); +// export async function InsertSkippedBlock(block: number){ +// try{ +// await dbConnect(); +// await checkPrevBlock(block); - console.log("checked prev block") +// console.log("checked prev block") - // previous block's inscriptions are present in DB - // Let's add this blocks inscriptions +// // previous block's inscriptions are present in DB +// // Let's add this blocks inscriptions - const url = `${process.env.PROVIDER}/block/${block}`; +// const url = `${process.env.PROVIDER}/block/${block}`; - console.log({ url }); +// console.log({ url }); - const result = await axios.get(url, { - headers: { Accept: "application/json" }, - }); +// const result = await axios.get(url, { +// headers: { Accept: "application/json" }, +// }); - const data = result.data; - // console.log({data}) +// const data = result.data; +// // console.log({data}) -if(data.inscriptions.length){ +// if(data.inscriptions.length){ - // Fetch existing inscriptions from the database - const existingInscriptions = await Inscription.find({ - inscription_id: { $in: data.inscriptions }, - }).select("inscription_id"); +// // Fetch existing inscriptions from the database +// const existingInscriptions = await Inscription.find({ +// inscription_id: { $in: data.inscriptions }, +// }).select("inscription_id"); - // Convert to a set for faster lookup - const existingSet = new Set( - existingInscriptions.map( - (inscription: { inscription_id: string }) => inscription.inscription_id - ) - ); +// // Convert to a set for faster lookup +// const existingSet = new Set( +// existingInscriptions.map( +// (inscription: { inscription_id: string }) => inscription.inscription_id +// ) +// ); - // Filter out inscriptions that already exist in the database - const filteredInscriptions = data.inscriptions.filter( - (inscription: string) => !existingSet.has(inscription) - ); +// // Filter out inscriptions that already exist in the database +// const filteredInscriptions = data.inscriptions.filter( +// (inscription: string) => !existingSet.has(inscription) +// ); - if (!filteredInscriptions.length) { - await BlockHashes.create({block_height: block, blockhash: data.hash}) - } +// if (!filteredInscriptions.length) { +// await BlockHashes.create({block_height: block, blockhash: data.hash}) +// } - const batchSize = 500; - let bulkOps:any = [] - for (let i = 0; i < filteredInscriptions.length; i += batchSize) { - // Extract a slice of 500 items at a time +// const batchSize = 500; +// let bulkOps:any = [] +// for (let i = 0; i < filteredInscriptions.length; i += batchSize) { +// // Extract a slice of 500 items at a time - // console.log(`BULKOPS items: ${bulkOps.length}`); - const selected = filteredInscriptions.slice(i, i + batchSize); +// // console.log(`BULKOPS items: ${bulkOps.length}`); +// const selected = filteredInscriptions.slice(i, i + batchSize); - // Call the function with the current batch - await processSingleBatch(selected, bulkOps); - // await wait(2); +// // Call the function with the current batch +// await processSingleBatch(selected, bulkOps); +// // await wait(2); - console.log("Processed: ", bulkOps.length); - } +// console.log("Processed: ", bulkOps.length); +// } - // Sort the bulkOps array based on the 'number' field in ascending order - bulkOps.sort((a: any, b: any) => { - return ( - a.updateOne.update.$set.inscription_number - - b.updateOne.update.$set.inscription_number - ); - }); +// // Sort the bulkOps array based on the 'number' field in ascending order +// bulkOps.sort((a: any, b: any) => { +// return ( +// a.updateOne.update.$set.inscription_number - +// b.updateOne.update.$set.inscription_number +// ); +// }); - // console.log("Starting transformation checks..."); - const transformedBulkDocs = await handlePreSaveLogic(bulkOps); +// // console.log("Starting transformation checks..."); +// const transformedBulkDocs = await handlePreSaveLogic(bulkOps); - // console.log(`After Transforming ${transformedBulkDocs.length} DAta`); - const bulkWriteOperations = transformedBulkDocs.map((transformedDoc) => { - return { - updateOne: { - filter: { inscription_id: transformedDoc.inscription_id }, // or whatever your unique identifier is - update: { $set: transformedDoc }, - upsert: true, - }, - }; - }); +// // console.log(`After Transforming ${transformedBulkDocs.length} DAta`); +// const bulkWriteOperations = transformedBulkDocs.map((transformedDoc) => { +// return { +// updateOne: { +// filter: { inscription_id: transformedDoc.inscription_id }, // or whatever your unique identifier is +// update: { $set: transformedDoc }, +// upsert: true, +// }, +// }; +// }); - if (bulkWriteOperations.length > 0) - { - await Inscription.bulkWrite(bulkWriteOperations); - // verify - await BlockHashes.create({block_height: block, blockhash: data.hash}) +// if (bulkWriteOperations.length > 0) +// { +// await Inscription.bulkWrite(bulkWriteOperations); +// // verify +// await BlockHashes.create({block_height: block, blockhash: data.hash}) - } +// } -}else{ - await BlockHashes.create({block_height: block, blockhash: data.hash}) -} +// }else{ +// await BlockHashes.create({block_height: block, blockhash: data.hash}) +// } -// await cleanup() +// // await cleanup() - } - catch(e: any){ - throw Error(e) - } -} - - -const processSingleBatch = async (inscriptions: string[], bulkOps: any) => { - // console.log(`started processing ${inscriptions.length} inscriptions`); - const batchPromises = inscriptions.map((inscription_id) => - processInscription(inscription_id, bulkOps) - ); +// } +// catch(e: any){ +// throw Error(e) +// } +// } - await Promise.allSettled(batchPromises); -}; -const processInscription = async (inscription_id: string, bulkOps: any) => { - if (!inscription_id) return; - let tags: string[] = []; - let content = null; - let sha; - let token = false; - let contentType = null; - let contentResponse = null; - try { - contentResponse = await fetchContentFromProviders(inscription_id); - contentType = contentResponse - ? contentResponse.headers["content-type"] - : null; - - if (contentResponse) { - if (/text|html|json|javascript/.test(contentType)) { - content = contentResponse.data; - - try { - - if (content.startsWith("cbrc-20:")) { - tags.push("cbrc"); - tags.push("token"); - token = true; - } - const parsedContent = JSON.parse(content.toString("utf-8")); - - if (parsedContent.p === "brc-20") { - tags.push("brc-20"); - tags.push("token"); - token = true; - } - else if ( - parsedContent.p === "sns" || - parsedContent.p.includes("sns") - ) { - tags.push("token"); - token = true; - } else if ( - parsedContent.p === "brc-21" || - parsedContent.p.includes("orc") - ) { - tags.push("token"); - token = true; - } else if ( - parsedContent.p && - parsedContent.tick && - parsedContent.amt - ) { - token = true; - tags.push("token"); - } else if ( - parsedContent.p && - parsedContent.op && - (parsedContent.dep || parsedContent.tick || parsedContent.amt) - ) { - token = true; - tags.push("token"); - tags.push("dmt"); - } - } catch (error) {} - - if (!token) { - // if content is not a token - if (typeof content === "object") { - content = contentResponse.data.toString("utf-8"); - } - // handle multiple tap mints like inscription 46225391 - if ( - content.includes(`"p":`) && - content.includes(`"op":`) && - (content.includes(`"tick":`) || content.includes(`"amt":`)) - ) { - if (!tags.includes("token")) tags.push("token"); - token = true; - } - - // if (!token) - sha = crypto - .createHash("sha256") - .update(content, "utf8") - .digest("hex"); - } - } else if (!token && !/video|audio/.test(contentType)) { - // if content is not a token or video/audio - sha = crypto - .createHash("sha256") - .update(contentResponse.data) - .digest("hex"); - } - } - } catch (e) {} +// const processSingleBatch = async (inscriptions: string[], bulkOps: any) => { +// // console.log(`started processing ${inscriptions.length} inscriptions`); +// const batchPromises = inscriptions.map((inscription_id) => +// processInscription(inscription_id, bulkOps) +// ); + +// await Promise.allSettled(batchPromises); +// }; + +// const processInscription = async (inscription_id: string, bulkOps: any) => { +// if (!inscription_id) return; +// let tags: string[] = []; +// let content = null; +// let sha; +// let token = false; +// let contentType = null; +// let contentResponse = null; +// try { +// contentResponse = await fetchContentFromProviders(inscription_id); +// contentType = contentResponse +// ? contentResponse.headers["content-type"] +// : null; + +// if (contentResponse) { +// if (/text|html|json|javascript/.test(contentType)) { +// content = contentResponse.data; + +// try { + +// if (content.startsWith("cbrc-20:")) { +// tags.push("cbrc"); +// tags.push("token"); +// token = true; +// } +// const parsedContent = JSON.parse(content.toString("utf-8")); + +// if (parsedContent.p === "brc-20") { +// tags.push("brc-20"); +// tags.push("token"); +// token = true; +// } +// else if ( +// parsedContent.p === "sns" || +// parsedContent.p.includes("sns") +// ) { +// tags.push("token"); +// token = true; +// } else if ( +// parsedContent.p === "brc-21" || +// parsedContent.p.includes("orc") +// ) { +// tags.push("token"); +// token = true; +// } else if ( +// parsedContent.p && +// parsedContent.tick && +// parsedContent.amt +// ) { +// token = true; +// tags.push("token"); +// } else if ( +// parsedContent.p && +// parsedContent.op && +// (parsedContent.dep || parsedContent.tick || parsedContent.amt) +// ) { +// token = true; +// tags.push("token"); +// tags.push("dmt"); +// } +// } catch (error) {} + +// if (!token) { +// // if content is not a token +// if (typeof content === "object") { +// content = contentResponse.data.toString("utf-8"); +// } +// // handle multiple tap mints like inscription 46225391 +// if ( +// content.includes(`"p":`) && +// content.includes(`"op":`) && +// (content.includes(`"tick":`) || content.includes(`"amt":`)) +// ) { +// if (!tags.includes("token")) tags.push("token"); +// token = true; +// } + +// // if (!token) +// sha = crypto +// .createHash("sha256") +// .update(content, "utf8") +// .digest("hex"); +// } +// } else if (!token && !/video|audio/.test(contentType)) { +// // if content is not a token or video/audio +// sha = crypto +// .createHash("sha256") +// .update(contentResponse.data) +// .digest("hex"); +// } +// } +// } catch (e) {} - let inscriptionDetails: any = {}; - inscriptionDetails = await fetchInscriptionDetails(inscription_id); +// let inscriptionDetails: any = {}; +// inscriptionDetails = await fetchInscriptionDetails(inscription_id); - if (!inscriptionDetails) { - throw Error("inscription details not found"); - } - tags = tags.filter((tag) => tag).map((tag) => tag.toLowerCase()); - - if (inscriptionDetails.metadata) - inscriptionDetails.metadata = new Map( - Object.entries(inscriptionDetails.metadata) - ); - - if ( - inscriptionDetails.metaprotocol && - inscriptionDetails.metaprotocol?.startsWith("cbrc-20") - ) { - token = true; - tags.push("token"); - tags.push("cbrc"); - } - if (inscriptionDetails.metaprotocol) - inscriptionDetails.parsed_metaprotocol = inscriptionDetails?.metaprotocol; - - const content_ = - (contentResponse && contentResponse.data.toString("utf-8")) || ""; - const truncatedContent = - content_.length > 15000 ? content_.substring(0, 15000) : content_; - - const inscriptionDoc = { - inscription_id, - inscription_number: inscriptionDetails.inscription_number, - address: inscriptionDetails.address, - content_type: contentType, - genesis_height: inscriptionDetails.genesis_height, - ...(inscriptionDetails.charms.length > 0 && { - charms_array: inscriptionDetails.charms, - }), - sat: inscriptionDetails.sat, - ...((token && !tags.includes("cbrc")) || - !contentResponse || - !sha || - /image|audio|zip|video/.test(contentType) - ? {} - : { content: truncatedContent }), - ...(!token && sha && { - sha, - }), - ...(token && { token }), - tags, - ...inscriptionDetails, - charms: 0, - }; - - if (!inscriptionDoc.genesis_height) { - console.dir(inscriptionDoc, { depth: null }); - throw Error("GENESIS HEIGHT MISSING"); - } - bulkOps.push({ - updateOne: { - filter: { - inscription_id, - }, - update: { $set: inscriptionDoc }, - upsert: true, - }, - }); -}; +// if (!inscriptionDetails) { +// throw Error("inscription details not found"); +// } +// tags = tags.filter((tag) => tag).map((tag) => tag.toLowerCase()); + +// if (inscriptionDetails.metadata) +// inscriptionDetails.metadata = new Map( +// Object.entries(inscriptionDetails.metadata) +// ); + +// if ( +// inscriptionDetails.metaprotocol && +// inscriptionDetails.metaprotocol?.startsWith("cbrc-20") +// ) { +// token = true; +// tags.push("token"); +// tags.push("cbrc"); +// } +// if (inscriptionDetails.metaprotocol) +// inscriptionDetails.parsed_metaprotocol = inscriptionDetails?.metaprotocol; + +// const content_ = +// (contentResponse && contentResponse.data.toString("utf-8")) || ""; +// const truncatedContent = +// content_.length > 15000 ? content_.substring(0, 15000) : content_; + +// const inscriptionDoc = { +// inscription_id, +// inscription_number: inscriptionDetails.inscription_number, +// address: inscriptionDetails.address, +// content_type: contentType, +// genesis_height: inscriptionDetails.genesis_height, +// ...(inscriptionDetails.charms.length > 0 && { +// charms_array: inscriptionDetails.charms, +// }), +// sat: inscriptionDetails.sat, +// ...((token && !tags.includes("cbrc")) || +// !contentResponse || +// !sha || +// /image|audio|zip|video/.test(contentType) +// ? {} +// : { content: truncatedContent }), +// ...(!token && sha && { +// sha, +// }), +// ...(token && { token }), +// tags, +// ...inscriptionDetails, +// charms: 0, +// }; + +// if (!inscriptionDoc.genesis_height) { +// console.dir(inscriptionDoc, { depth: null }); +// throw Error("GENESIS HEIGHT MISSING"); +// } +// bulkOps.push({ +// updateOne: { +// filter: { +// inscription_id, +// }, +// update: { $set: inscriptionDoc }, +// upsert: true, +// }, +// }); +// }; export const handlePreSaveLogic = async (bulkDocs: Array>) => { @@ -343,32 +343,32 @@ export const handlePreSaveLogic = async (bulkDocs: Array>) => { const doc = updateOne.update.$set; - if (i === 0 && doc.inscription_number > 0) { - const prevDocument = await Inscription.findOne({ - inscription_number: doc.inscription_number - 1, - }); - - if (!prevDocument || !prevDocument.inscription_id) { - await BlockHashes.deleteOne({block_height: doc.genesis_height - 1}); - await Inscription.deleteMany({genesis_height: doc.genesis_height - 1}) - await InsertSkippedBlock(doc.genesis_height - 1) - throw new Error( - `1) A document with number ${ - doc.inscription_number - 1 - } does not exist or inscriptionId is missing in it` - ); - } - } else if (i > 0) { - - const lastDoc = bulkDocs[i-1].updateOne.update.$set; - if (doc.inscription_number !== lastDoc.inscription_number + 1) { - throw new Error( - `2) The inscription_number for document at position ${i} is not consecutive. Expected ${ - lastDoc.inscription_number + 1 - }, but got ${doc.inscription_number}` - ); - } - } + // if (i === 0 && doc.inscription_number > 0) { + // const prevDocument = await Inscription.findOne({ + // inscription_number: doc.inscription_number - 1, + // }); + + // if (!prevDocument || !prevDocument.inscription_id) { + // await BlockHashes.deleteOne({block_height: doc.genesis_height - 1}); + // await Inscription.deleteMany({genesis_height: doc.genesis_height - 1}) + // await InsertSkippedBlock(doc.genesis_height - 1) + // throw new Error( + // `1) A document with number ${ + // doc.inscription_number - 1 + // } does not exist or inscriptionId is missing in it` + // ); + // } + // } else if (i > 0) { + + // const lastDoc = bulkDocs[i-1].updateOne.update.$set; + // if (doc.inscription_number !== lastDoc.inscription_number + 1) { + // throw new Error( + // `2) The inscription_number for document at position ${i} is not consecutive. Expected ${ + // lastDoc.inscription_number + 1 + // }, but got ${doc.inscription_number}` + // ); + // } + // } // Updated SHA version logic // if (doc.sha && !doc.token) { @@ -409,34 +409,34 @@ export const handlePreSaveLogic = async (bulkDocs: Array>) => { return transformedBulkOps; }; -async function checkPrevBlock(block: number){ +// async function checkPrevBlock(block: number){ - // make sure previous block data is complete; - const inscriptionsOfThisBlockInDB = await Inscription.countDocuments({genesis_height : block-1}); - const blockhash = await BlockHashes.findOne({block_height: block-1}) +// // make sure previous block data is complete; +// const inscriptionsOfThisBlockInDB = await Inscription.countDocuments({genesis_height : block-1}); +// const blockhash = await BlockHashes.findOne({block_height: block-1}) - const url = `${process.env.PROVIDER}/block/${block - 1}`; +// const url = `${process.env.PROVIDER}/block/${block - 1}`; -// console.log({ url }); +// // console.log({ url }); - const result = await axios.get(url, { - headers: { Accept: "application/json" }, - }); +// const result = await axios.get(url, { +// headers: { Accept: "application/json" }, +// }); - const data = result.data; +// const data = result.data; - if(inscriptionsOfThisBlockInDB != data.inscriptions.length) - // { await Inscription.deleteMany({genesis_height: block-1}) -// await InsertSkippedBlock(block-1) - throw Error("This BLOCK is messed up: "+block) +// if(inscriptionsOfThisBlockInDB != data.inscriptions.length) +// // { await Inscription.deleteMany({genesis_height: block-1}) +// // await InsertSkippedBlock(block-1) +// throw Error("This BLOCK is messed up: "+block) +// // } +// else if(!blockhash){ +// await BlockHashes.create({block_height: block - 1, blockhash: data.hash}) // } -else if(!blockhash){ - await BlockHashes.create({block_height: block - 1, blockhash: data.hash}) -} - return; -} \ No newline at end of file +// return; +// } \ No newline at end of file