From 8c26e303f468e405a0fd7572bd6a0e55853adaac Mon Sep 17 00:00:00 2001 From: hawyar Date: Tue, 20 Sep 2022 13:16:10 -0400 Subject: [PATCH 1/7] Add base provider class --- services/crawler/src/index.ts | 66 +++++++++++------ services/crawler/src/providers/Base.ts | 20 ++++++ services/crawler/src/providers/Ethereum.ts | 84 ++++++++++++++++++++++ services/crawler/src/providers/Iotex.ts | 11 +-- services/crawler/test/crawler.test.ts | 22 ++++-- 5 files changed, 167 insertions(+), 36 deletions(-) create mode 100644 services/crawler/src/providers/Base.ts create mode 100644 services/crawler/src/providers/Ethereum.ts diff --git a/services/crawler/src/index.ts b/services/crawler/src/index.ts index 0d9ffb948..9c1a088d3 100644 --- a/services/crawler/src/index.ts +++ b/services/crawler/src/index.ts @@ -1,7 +1,6 @@ import {GetObjectCommand, S3Client, S3ClientConfig,} from '@aws-sdk/client-s3' import { defaultProvider } from '@aws-sdk/credential-provider-node' -import { IotexService, newIotexService } from './providers/Iotex' import EventEmitter from 'events' import { PutObjectCommand } from '@aws-sdk/client-s3' import { EventTableColumn } from '@casimir/data' @@ -13,6 +12,9 @@ import { StartQueryExecutionCommand } from '@aws-sdk/client-athena' +import { IotexService, newIotexService } from './providers/Iotex' +import {EthereumService, newEthereumService} from './providers/Ethereum' +import { Chain } from './providers/Base' const defaultEventBucket = 'casimir-etl-event-bucket-dev' const queryOutputLocation = 's3://cms-lds-agg/cms_hcf_aggregates' @@ -21,10 +23,6 @@ const EE = new EventEmitter() let s3: S3Client | null = null let athena: AthenaClient | null = null -export enum Chain { - Iotex = 'iotex', -} - export interface CrawlerConfig { chain: Chain output?: `s3://${string}` @@ -33,7 +31,7 @@ export interface CrawlerConfig { class Crawler { config: CrawlerConfig - service: IotexService | null + service: IotexService | EthereumService | null EE: EventEmitter s3Client: S3Client | null athenaClient: AthenaClient | null @@ -78,6 +76,19 @@ class Crawler { this.EE.emit('init') return } + + if (this.config.chain === Chain.Ethereum) { + this.service = newEthereumService() + + if (this.config.verbose) { + this.EE.on('init', () => { + console.log(`Initialized crawler for: ${this.config.chain}`) + }) + } + + this.EE.emit('init') + return + } throw new Error('UnknownChain: chain is not supported') } async start (): Promise { @@ -89,7 +100,6 @@ class Crawler { if (this.service instanceof IotexService) { - const { chainMeta } = await this.service.getChainMetadata() const height = parseInt(chainMeta.height) const blocksPerRequest = 1000 @@ -128,6 +138,16 @@ class Crawler { } return } + + if (this.service instanceof EthereumService) { + const height = await this.service.currentBlock() + for (let i = 0; i < height; i++) { + const block = await this.service.client.getBlock(i) + // const key = `${block.hash}-events.json` + console.log(JSON.stringify(block, null, 2)) + } + return + } throw new Error('not implemented yet') } @@ -237,24 +257,28 @@ class Crawler { throw new Error('not implemented yet') } - on(event: 'block', cb: (b: IStreamBlocksResponse) => void): void { - if (event !== 'block') throw new Error('InvalidEvent: event is not supported') - - if (typeof cb !== 'function') throw new Error('InvalidCallback: callback is not a function') - + on(event: EventStreamType, cb: (b: IStreamBlocksResponse) => void): void { if (this.service === null) throw new Error('NullService: service is not initialized') - if (this.service instanceof IotexService) { - this.service.readableBlockStream().then((s: any) => { - s.on('data', (b: IStreamBlocksResponse) => { - cb(b) - }) + if (event == EventStreamType.IOTEX_BLOCK) { + if (this.service instanceof IotexService) { + this.service.readableBlockStream().then((s: any) => { + s.on('data', (b: IStreamBlocksResponse) => { + cb(b) + }) - s.on('error', (e: Error) => { - throw e + s.on('error', (e: Error) => { + throw e + }) }) - }) - return + return + } + } + + if (event === EventStreamType.ETH_BLOCK) { + if (this.service instanceof EthereumService) { + // stream it + } } throw new Error('not implemented yet') } diff --git a/services/crawler/src/providers/Base.ts b/services/crawler/src/providers/Base.ts new file mode 100644 index 000000000..d0d5a9821 --- /dev/null +++ b/services/crawler/src/providers/Base.ts @@ -0,0 +1,20 @@ +import { EventTableColumn } from '@casimir/data' +import {EventEmitter} from 'events' + +export enum Chain { + Iotex = 'iotex', + Ethereum = 'ethereum' +} + +export interface BaseService { + chain: Chain + provider: any + eventEmitter: EventEmitter | null + getChainMetadata(): Promise + getBlock(hash: string): Promise + getTransaction(tx: string): Promise + on(event: T, cb: () => void): void + getCurrentBlockHeight(): Promise + getLastProcessedBlockHeight(): Promise + convertToGlueSchema(obj: Record): EventTableColumn +} diff --git a/services/crawler/src/providers/Ethereum.ts b/services/crawler/src/providers/Ethereum.ts new file mode 100644 index 000000000..4abc9ab78 --- /dev/null +++ b/services/crawler/src/providers/Ethereum.ts @@ -0,0 +1,84 @@ +import { ethers } from 'ethers' +import { EventTableColumn } from '@casimir/data' +import { BaseService, Chain } from './Base' +import EventEmitter from 'events' + +export type EthereumServiceOptions = { + provider: string +} + +export class EthereumService implements BaseService { + chain: Chain + provider: ethers.providers.JsonRpcProvider + eventEmitter: EventEmitter | null = null + + constructor(opt: EthereumServiceOptions) { + this.chain = Chain.Ethereum + this.provider = new ethers.providers.JsonRpcProvider({ + url: opt.provider || 'http://localhost:8545', + }) + this.eventEmitter = null + } + + async getChainMetadata() { + const meta = await this.provider.getNetwork() + return meta + } + + async getBlock(hash: string): Promise { + const block = await this.provider.getBlock(hash) + return block + } + + async getCurrentBlock(): Promise { + const current = await this.provider.getBlockNumber() + return current + } + + // todo: get last proceesed block from athena + async getLastProcessedBlock(): Promise { + throw new Error('Method not implemented.') + } + + async getTransaction(tx: string): Promise { + const txData = await this.provider.getTransaction(tx) + return txData + } + + // todo: better interface and cleanup + on(event: T, cb: () => void): void { + this.provider.on(event, cb) + } + + getCurrentBlockHeight(): Promise { + return Promise.resolve(0) + } + + // todo: get last proceesed block from athena + getLastProcessedBlockHeight(): Promise { + return Promise.resolve(0) + } + + convertToGlueSchema(event: { type: string, block: ethers.providers.Block, tx: ethers.providers.TransactionResponse }): EventTableColumn { + const record: EventTableColumn = { + chain: Chain.Ethereum, + network: 'mainnet', + provider: 'casimir', + type: event.type, + created_at: new Date().toISOString().split('T')[0], + address: event.tx.from, + height: event.block.number, + to_address: event.tx.to || '', + candidate: '', + candidate_list: [], + amount: parseInt(event.tx.value.toString()), + duration: 0, + auto_stake: false, + } + return record + } +} + +export function newEthereumService (opt: EthereumServiceOptions): EthereumService { + return new EthereumService(opt) +} \ No newline at end of file diff --git a/services/crawler/src/providers/Iotex.ts b/services/crawler/src/providers/Iotex.ts index 4763bb3c8..042f0d7b4 100644 --- a/services/crawler/src/providers/Iotex.ts +++ b/services/crawler/src/providers/Iotex.ts @@ -12,9 +12,7 @@ import { import { from } from '@iotexproject/iotex-address-ts' import { Opts } from 'iotex-antenna/lib/antenna' import { EventTableColumn } from '@casimir/data' -import {Chain} from '../index' - -const IOTEX_CORE = 'http://localhost:14014' +import { Chain } from './Base' export type IotexOptions = Opts & { provider: string, @@ -43,7 +41,7 @@ export class IotexService { chainId: number client: Antenna constructor (opt: IotexOptions) { - this.provider = opt.provider || IOTEX_CORE + this.provider = opt.provider || 'http://localhost:14014' this.chainId = opt.chainId this.client = new Antenna(opt.provider, opt.chainId, { signer: opt.signer, @@ -57,11 +55,6 @@ export class IotexService { return meta } - async getServerMetadata (): Promise { - const meta = await this.client.iotx.getServerMeta({}) - return meta - } - async getBlocks(start: number, count: number): Promise { if (start < 0 || count < 0) { throw new Error('start and count must be greater than 0') diff --git a/services/crawler/test/crawler.test.ts b/services/crawler/test/crawler.test.ts index 980725544..c3b5f3962 100644 --- a/services/crawler/test/crawler.test.ts +++ b/services/crawler/test/crawler.test.ts @@ -2,15 +2,25 @@ import { crawler, Chain } from '../src/index' jest.setTimeout(20000) -test('get last block', async () => { - const supercrawler = await crawler({ - chain: Chain.Iotex, +// test('get last block', async () => { +// const supercrawler = await crawler({ +// chain: Chain.Iotex, +// verbose: true +// }) +// +// expect(supercrawler.service).not.toBe(null) +// const lastBlock = await supercrawler.retrieveLastBlock() +// expect(typeof lastBlock).toBe('number') +// }) + +test('init ethereum', async () => { + const ethCrawler = await crawler({ + chain: Chain.Ethereum, verbose: true }) - expect(supercrawler.service).not.toBe(null) - const lastBlock = await supercrawler.retrieveLastBlock() - expect(typeof lastBlock).toBe('number') + ethCrawler.start() + expect(ethCrawler.service).not.toBe(null) }) // test('stream', async () => { From 601dec66bbedddf1127c6f406e2e8896aab10d9a Mon Sep 17 00:00:00 2001 From: hawyar Date: Tue, 20 Sep 2022 18:47:09 -0400 Subject: [PATCH 2/7] Put aws helpers in @casimir/helpers --- common/helpers/src/index.ts | 156 +++++++++++++ services/crawler/src/index.ts | 245 ++++----------------- services/crawler/src/providers/Base.ts | 7 +- services/crawler/src/providers/Ethereum.ts | 136 +++++++++++- services/crawler/test/crawler.test.ts | 7 +- 5 files changed, 332 insertions(+), 219 deletions(-) diff --git a/common/helpers/src/index.ts b/common/helpers/src/index.ts index f205d68b7..c9ffb95c4 100644 --- a/common/helpers/src/index.ts +++ b/common/helpers/src/index.ts @@ -1,3 +1,9 @@ +import { S3Client, S3ClientConfig, PutObjectCommand, GetObjectCommand } from '@aws-sdk/client-s3' +import { AthenaClient, AthenaClientConfig } from '@aws-sdk/client-athena' +import { defaultProvider } from '@aws-sdk/credential-provider-node' +import {queryOutputLocation} from '@casimir/crawler/src' +import { StartQueryExecutionCommand, GetQueryExecutionCommand, StartQueryExecutionCommandInput } from '@aws-sdk/client-athena' + /** * Converts any string to PascalCase. * @@ -9,4 +15,154 @@ export function pascalCase(str: string): string { return str.replace(/\w+/g, (word) => { return word[0].toUpperCase() + word.slice(1).toLowerCase() }) +} + +let athena: AthenaClient | null = null +let s3: S3Client | null = null + +export async function newAthenaClient(opt?: AthenaClientConfig): Promise { + if (opt?.region === undefined) { + opt = { + region: 'us-east-2' + } + } + + if (opt.credentials === undefined) { + opt = { + credentials: defaultProvider() + } + } + const client = new AthenaClient(opt) + athena = client + + return client +} + +export async function newS3Client (opt?: S3ClientConfig): Promise { + if (s3) { + return s3 + } + + if (opt?.region === undefined) { + opt = { + region: 'us-east-2' + } + } + + if (opt.credentials === undefined) { + opt = { + credentials: defaultProvider() + } + } + + const client = new S3Client(opt) + s3 = client + + return client +} + +export async function uploadToS3( input: { bucket: string, key: string,data: string }): Promise { + if (!s3) { + s3 = await newS3Client() + } + + const upload = new PutObjectCommand({ + Bucket: input.bucket, + Key: input.key, + }) + const { $metadata } = await s3.send(upload) + if ($metadata.httpStatusCode !== 200) throw new Error('Error uploading to s3') +} + +export async function getFromS3(bucket: string, key: string): Promise { + if (!s3) { + s3 = await newS3Client() + } + + const { $metadata, Body } = await s3.send(new GetObjectCommand({ + Bucket: bucket, + Key: key + // Bucket: 'cms-lds-agg', + // Key: `cms_hcf_aggregates/${res.QueryExecutionId}.csv` + })) + + if ($metadata.httpStatusCode !== 200) throw new Error('FailedQuery: unable retrieve result from S3') + if (Body === undefined) throw new Error('InvalidQueryResult: query result is undefined') + + let chunk = '' + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + for await (const data of Body) { + chunk += data.toString() + } + return chunk +} + + +let retry = 0 +let backoff = 300 + +async function pollAthenaQueryOutput(queryId: string): Promise { + if (!athena) { + athena = await newAthenaClient() + } + + const getStateCmd = new GetQueryExecutionCommand({ + QueryExecutionId: queryId + }) + + const { $metadata, QueryExecution } = await athena.send(getStateCmd) + + if ($metadata.httpStatusCode !== 200) throw new Error('FailedQuery: unable to query Athena') + if (QueryExecution === undefined) throw new Error('InvalidQueryExecution: query execution is undefined') + if (QueryExecution.Status === undefined) throw new Error('InvalidQueryExecutionStatus: query execution status is undefined') + + if (QueryExecution.Status.State === 'QUEUED' || QueryExecution.Status.State === 'RUNNING') { + setTimeout(() => { + pollAthenaQueryOutput(queryId) + retry++ + backoff = backoff + 500 + }, backoff) + } + + if (QueryExecution.Status.State === 'FAILED') throw new Error('QueryFailed: query failed') + if (QueryExecution.Status.State === 'SUCCEEDED') + return +} + +export async function queryAthena(query: string): Promise { + + if (!athena) { + athena = await newAthenaClient() + } + + const execCmd = new StartQueryExecutionCommand({ + QueryString: query, + // QueryString: `SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" WHERE chain = '${chain}' ORDER BY height DESC LIMIT 1`, + WorkGroup: 'primary', + ResultConfiguration: { + OutputLocation: queryOutputLocation, + } + }) + + const { $metadata, QueryExecutionId } = await athena.send(execCmd) + + if ($metadata.httpStatusCode !== 200) { + throw new Error('FailedQuery: unable to query Athena') + } + + if (QueryExecutionId === undefined) { + throw new Error('InvalidQueryExecutionId: query execution id is undefined') + } + + await pollAthenaQueryOutput(QueryExecutionId) + + // wait for athena to finish writing to s3 + await new Promise(resolve => setTimeout(resolve, 2000)) + + const result = await getFromS3('cms-lds-agg', `cms_hcf_aggregates/${QueryExecutionId}.csv`) + + // const height = raw.split('\n').filter(l => l !== '')[1].replace(/"/g, '') + return result } \ No newline at end of file diff --git a/services/crawler/src/index.ts b/services/crawler/src/index.ts index 9c1a088d3..18dc5bb86 100644 --- a/services/crawler/src/index.ts +++ b/services/crawler/src/index.ts @@ -4,7 +4,6 @@ import { defaultProvider } from '@aws-sdk/credential-provider-node' import EventEmitter from 'events' import { PutObjectCommand } from '@aws-sdk/client-s3' import { EventTableColumn } from '@casimir/data' -import { IStreamBlocksResponse } from 'iotex-antenna/lib/rpc-method/types' import { AthenaClient, AthenaClientConfig, @@ -15,13 +14,14 @@ import { import { IotexService, newIotexService } from './providers/Iotex' import {EthereumService, newEthereumService} from './providers/Ethereum' import { Chain } from './providers/Base' -const defaultEventBucket = 'casimir-etl-event-bucket-dev' -const queryOutputLocation = 's3://cms-lds-agg/cms_hcf_aggregates' + +export const defaultEventBucket = 'casimir-etl-event-bucket-dev' +export const queryOutputLocation = 's3://cms-lds-agg/cms_hcf_aggregates' const EE = new EventEmitter() -let s3: S3Client | null = null -let athena: AthenaClient | null = null +export let s3: S3Client | null = null +export const athena: AthenaClient | null = null export interface CrawlerConfig { chain: Chain @@ -62,34 +62,16 @@ class Crawler { } async prepare (): Promise { - if (this.config.chain === Chain.Iotex) { - const service = await newIotexService() - - this.service = service - - if (this.config.verbose) { - this.EE.on('init', () => { - console.log(`Initialized crawler for: ${this.config.chain}`) - }) + switch (this.config.chain) { + case Chain.Iotex: + this.service = await newIotexService() + break + case Chain.Ethereum: + this.service = await newEthereumService() + break + default: + throw new Error('InvalidChain: chain is not supported') } - - this.EE.emit('init') - return - } - - if (this.config.chain === Chain.Ethereum) { - this.service = newEthereumService() - - if (this.config.verbose) { - this.EE.on('init', () => { - console.log(`Initialized crawler for: ${this.config.chain}`) - }) - } - - this.EE.emit('init') - return - } - throw new Error('UnknownChain: chain is not supported') } async start (): Promise { if (this.service == null) { @@ -140,9 +122,10 @@ class Crawler { } if (this.service instanceof EthereumService) { - const height = await this.service.currentBlock() + const height = await this.service.getCurrentBlockHeight() for (let i = 0; i < height; i++) { - const block = await this.service.client.getBlock(i) + const block = await this.service.getBlock('0') + console.log(block) // const key = `${block.hash}-events.json` console.log(JSON.stringify(block, null, 2)) } @@ -151,103 +134,6 @@ class Crawler { throw new Error('not implemented yet') } - async retrieveLastBlock(): Promise { - if (this.athenaClient === null) this.athenaClient = await newAthenaClient() - - const execCmd = new StartQueryExecutionCommand({ - QueryString: 'SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" ORDER BY height DESC LIMIT 1', - WorkGroup: 'primary', - ResultConfiguration: { - OutputLocation: queryOutputLocation, - } - }) - - const res = await this.athenaClient.send(execCmd) - - if (res.$metadata.httpStatusCode !== 200) { - throw new Error('FailedQuery: unable to query Athena') - } - - if (res.QueryExecutionId === undefined) { - throw new Error('InvalidQueryExecutionId: query execution id is undefined') - } - - if (s3 === null) s3 = await newS3Client() - - const getCmd = new GetQueryExecutionCommand({ - QueryExecutionId: res.QueryExecutionId, - }) - - const getRes = await this.athenaClient.send(getCmd) - - if (getRes.$metadata.httpStatusCode !== 200) { - throw new Error('FailedQuery: unable to query Athena') - } - - if (getRes.QueryExecution === undefined) { - throw new Error('InvalidQueryExecution: query execution is undefined') - } - - let retry = 0 - let backoff = 1000 - - const queryState = async (): Promise => { - const getStateCmd = new GetQueryExecutionCommand({ - QueryExecutionId: res.QueryExecutionId, - }) - - if (this.athenaClient === null) throw new Error('NullAthenaClient: athena client is not initialized') - - const getStateRes = await this.athenaClient.send(getStateCmd) - - if (getStateRes.$metadata.httpStatusCode !== 200) throw new Error('FailedQuery: unable to query Athena') - if (getStateRes.QueryExecution === undefined) throw new Error('InvalidQueryExecution: query execution is undefined') - if (getStateRes.QueryExecution.Status === undefined) throw new Error('InvalidQueryExecutionStatus: query execution status is undefined') - - if (getStateRes.QueryExecution.Status.State === 'QUEUED' || getStateRes.QueryExecution.Status.State === 'RUNNING') { - setTimeout(() => { - queryState() - retry++ - backoff = backoff + 500 - }, backoff) - } - if (getStateRes.QueryExecution.Status.State === 'FAILED') throw new Error('QueryFailed: query failed') - if (getStateRes.QueryExecution.Status.State === 'SUCCEEDED') return - } - - const getResultFromS3 = async (): Promise => { - if (s3 === null) throw new Error('NullS3Client: s3 client is not initialized') - - const {$metadata, Body} = await s3.send(new GetObjectCommand({ - Bucket: 'cms-lds-agg', - Key: `cms_hcf_aggregates/${res.QueryExecutionId}.csv` - })) - - if ($metadata.httpStatusCode !== 200) throw new Error('FailedQuery: unable retrieve result from S3') - if (Body === undefined) throw new Error('InvalidQueryResult: query result is undefined') - - let chunk = '' - - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - for await (const data of Body) { - chunk += data.toString() - } - return chunk - } - - await queryState() - - // wait for athena to write to s3 - await new Promise(resolve => setTimeout(resolve, 2000)) - - const raw = await getResultFromS3() - - const height = raw.split('\n').filter(l => l !== '')[1].replace(/"/g, '') - - return parseInt(height) - } - async stop(): Promise { if (this.service === null) throw new Error('NullService: service is not initialized') @@ -257,78 +143,31 @@ class Crawler { throw new Error('not implemented yet') } - on(event: EventStreamType, cb: (b: IStreamBlocksResponse) => void): void { - if (this.service === null) throw new Error('NullService: service is not initialized') - - if (event == EventStreamType.IOTEX_BLOCK) { - if (this.service instanceof IotexService) { - this.service.readableBlockStream().then((s: any) => { - s.on('data', (b: IStreamBlocksResponse) => { - cb(b) - }) - - s.on('error', (e: Error) => { - throw e - }) - }) - return - } - } - - if (event === EventStreamType.ETH_BLOCK) { - if (this.service instanceof EthereumService) { - // stream it - } - } - throw new Error('not implemented yet') - } -} - -async function newAthenaClient(opt?: AthenaClientConfig): Promise { - if (opt?.region === undefined) { - opt = { - region: 'us-east-2' - } - } - - if (opt.credentials === undefined) { - opt = { - credentials: defaultProvider() - } - } - - if (athena === null) { - athena = new AthenaClient(opt) - return athena - } - const client = new AthenaClient(opt) - athena = client - - return client -} - -async function newS3Client (opt?: S3ClientConfig): Promise { - if (opt?.region === undefined) { - opt = { - region: 'us-east-2' - } - } - - if (opt.credentials === undefined) { - opt = { - credentials: defaultProvider() - } - } - - if (s3 === null) { - s3 = new S3Client(opt) - return s3 - } - - const client = new S3Client(opt) - s3 = client - - return client + // on(event: any, cb: (b: IStreamBlocksResponse) => void): void { + // if (this.service === null) throw new Error('NullService: service is not initialized') + // + // if (event == EventStreamType.IOTEX_BLOCK) { + // if (this.service instanceof IotexService) { + // this.service.readableBlockStream().then((s: any) => { + // s.on('data', (b: IStreamBlocksResponse) => { + // cb(b) + // }) + // + // s.on('error', (e: Error) => { + // throw e + // }) + // }) + // return + // } + // } + // + // if (event === EventStreamType.ETH_BLOCK) { + // if (this.service instanceof EthereumService) { + // // stream it + // } + // } + // throw new Error('not implemented yet') + // } } export async function crawler (config: CrawlerConfig): Promise { diff --git a/services/crawler/src/providers/Base.ts b/services/crawler/src/providers/Base.ts index d0d5a9821..07d3b7053 100644 --- a/services/crawler/src/providers/Base.ts +++ b/services/crawler/src/providers/Base.ts @@ -1,5 +1,7 @@ import { EventTableColumn } from '@casimir/data' -import {EventEmitter} from 'events' +import { EventEmitter } from 'events' +import { AthenaClient } from '@aws-sdk/client-athena' +import { S3Client } from '@aws-sdk/client-s3' export enum Chain { Iotex = 'iotex', @@ -9,6 +11,8 @@ export enum Chain { export interface BaseService { chain: Chain provider: any + athenaClient: AthenaClient + s3Client: S3Client eventEmitter: EventEmitter | null getChainMetadata(): Promise getBlock(hash: string): Promise @@ -17,4 +21,5 @@ export interface BaseService { getCurrentBlockHeight(): Promise getLastProcessedBlockHeight(): Promise convertToGlueSchema(obj: Record): EventTableColumn + save(key: string, data: string): void } diff --git a/services/crawler/src/providers/Ethereum.ts b/services/crawler/src/providers/Ethereum.ts index 4abc9ab78..a0d8226ab 100644 --- a/services/crawler/src/providers/Ethereum.ts +++ b/services/crawler/src/providers/Ethereum.ts @@ -2,6 +2,10 @@ import { ethers } from 'ethers' import { EventTableColumn } from '@casimir/data' import { BaseService, Chain } from './Base' import EventEmitter from 'events' +import {GetObjectCommand, PutObjectCommand, S3Client} from '@aws-sdk/client-s3' +import {defaultEventBucket, newAthenaClient, newS3Client, queryOutputLocation, s3} from '../index' +import {AthenaClient, GetQueryExecutionCommand, StartQueryExecutionCommand} from '@aws-sdk/client-athena' +import {defaultProvider} from '@aws-sdk/credential-provider-node' export type EthereumServiceOptions = { provider: string @@ -9,6 +13,8 @@ export type EthereumServiceOptions = { export class EthereumService implements BaseService { chain: Chain + athenaClient: AthenaClient + s3Client: S3Client provider: ethers.providers.JsonRpcProvider eventEmitter: EventEmitter | null = null @@ -17,6 +23,14 @@ export class EthereumService implements BaseService { this.provider = new ethers.providers.JsonRpcProvider({ url: opt.provider || 'http://localhost:8545', }) + this.athenaClient = new AthenaClient({ + region: 'us-east-1', + credentials: defaultProvider() + }) + this.s3Client = new S3Client({ + region: 'us-east-2', + credentials: defaultProvider() + }) this.eventEmitter = null } @@ -30,17 +44,96 @@ export class EthereumService implements BaseService { return block } - async getCurrentBlock(): Promise { - const current = await this.provider.getBlockNumber() - return current - } + // todo: get last processed block from athena - // todo: get last proceesed block from athena - async getLastProcessedBlock(): Promise { - throw new Error('Method not implemented.') - } + async getLastProcessedBlock(chain: Chain): Promise { + const selectQuery= new StartQueryExecutionCommand({ + QueryString: `SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" WHERE chain = '${chain}' ORDER BY height DESC LIMIT 1`, + WorkGroup: 'primary', + ResultConfiguration: { + OutputLocation: queryOutputLocation, + } + }) + + const { $metadata, QueryExecutionId } = await this.athenaClient.send(selectQuery) + + if ($metadata.httpStatusCode !== 200) { + throw new Error('FailedQuery: unable to query Athena') + } + + if (QueryExecutionId === undefined) { + throw new Error('InvalidQueryExecutionId: query execution id is undefined') + } + + const getCmd = new GetQueryExecutionCommand({ QueryExecutionId }) + + const execRes = await this.athenaClient.send(getCmd) + + if (execRes.$metadata.httpStatusCode !== 200) { + throw new Error('FailedQuery: unable to query Athena') + } + + if (execRes.QueryExecution === undefined) { + throw new Error('InvalidQueryExecution: query execution is undefined') + } + + let retry = 0 + let backoff = 1000 + + const poll = async (): Promise => { + const getStateCmd = new GetQueryExecutionCommand({ QueryExecutionId }) + + const getStateRes = await this.athenaClient.send(getStateCmd) + + if (getStateRes.$metadata.httpStatusCode !== 200) throw new Error('FailedQuery: unable to query Athena') + if (getStateRes.QueryExecution === undefined) throw new Error('InvalidQueryExecution: query execution is undefined') + if (getStateRes.QueryExecution.Status === undefined) throw new Error('InvalidQueryExecutionStatus: query execution status is undefined') + + if (getStateRes.QueryExecution.Status.State === 'QUEUED' || getStateRes.QueryExecution.Status.State === 'RUNNING') { + setTimeout(() => { + poll() + retry++ + backoff = backoff + 500 + }, backoff) + } + if (getStateRes.QueryExecution.Status.State === 'FAILED') throw new Error('QueryFailed: query failed') + if (getStateRes.QueryExecution.Status.State === 'SUCCEEDED') return + } + + const getResult = async (): Promise => { + if (s3 === null) throw new Error('NullS3Client: s3 client is not initialized') + + const {$metadata, Body} = await s3.send(new GetObjectCommand({ + Bucket: 'cms-lds-agg', + Key: `cms_hcf_aggregates/${QueryExecutionId}.csv` + })) - async getTransaction(tx: string): Promise { + if ($metadata.httpStatusCode !== 200) throw new Error('FailedQuery: unable retrieve result from S3') + if (Body === undefined) throw new Error('InvalidQueryResult: query result is undefined') + + let chunk = '' + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + for await (const data of Body) { + chunk += data.toString() + } + return chunk + } + + await poll() + + // wait for athena writing to s3 + await new Promise(resolve => setTimeout(resolve, 2000)) + + const raw = await getResult() + + const height = raw.split('\n').filter(l => l !== '')[1].replace(/"/g, '') + return parseInt(height) + } + + + async getTransaction(tx: string): Promise { const txData = await this.provider.getTransaction(tx) return txData } @@ -50,11 +143,13 @@ export class EthereumService implements BaseService { this.provider.on(event, cb) } - getCurrentBlockHeight(): Promise { - return Promise.resolve(0) + async getCurrentBlockHeight(): Promise { + const h = await this.provider.getBlockNumber() + console.log('current block height', h) + return h } - // todo: get last proceesed block from athena + // todo: get last processed block from athena getLastProcessedBlockHeight(): Promise { return Promise.resolve(0) } @@ -77,6 +172,23 @@ export class EthereumService implements BaseService { } return record } + + async save(key: string, data: string): Promise { + if (key === undefined || key === null) throw new Error('InvalidKey: key is not defined') + if (data === undefined || data === null) throw new Error('InvalidData: data is not defined') + + const upload = new PutObjectCommand({ + Bucket: defaultEventBucket, + Key: key, + Body: data + }) + + const { $metadata } = await this.s3Client.send(upload).catch((e: Error) => { + throw e + }) + + if ($metadata.httpStatusCode !== 200) throw new Error('FailedUploadBlock: unable to upload block') + } } export function newEthereumService (opt: EthereumServiceOptions): EthereumService { diff --git a/services/crawler/test/crawler.test.ts b/services/crawler/test/crawler.test.ts index c3b5f3962..a04d596cf 100644 --- a/services/crawler/test/crawler.test.ts +++ b/services/crawler/test/crawler.test.ts @@ -1,4 +1,5 @@ -import { crawler, Chain } from '../src/index' +import { crawler } from '../src/index' +import { Chain } from '../src/providers/Base' jest.setTimeout(20000) @@ -13,13 +14,13 @@ jest.setTimeout(20000) // expect(typeof lastBlock).toBe('number') // }) -test('init ethereum', async () => { +test('init crawler for ethereum', async () => { const ethCrawler = await crawler({ chain: Chain.Ethereum, verbose: true }) - ethCrawler.start() + await ethCrawler.start() expect(ethCrawler.service).not.toBe(null) }) From 18a0edb65867a3b2bf8dafe1f753cfff4e80d3f1 Mon Sep 17 00:00:00 2001 From: hawyar Date: Wed, 21 Sep 2022 22:10:28 -0400 Subject: [PATCH 3/7] Add basic tests and more cleaning --- common/helpers/src/index.ts | 88 ++++++++- services/crawler/src/index.ts | 212 ++++++++++----------- services/crawler/src/providers/Base.ts | 19 +- services/crawler/src/providers/Ethereum.ts | 156 +++------------ services/crawler/src/providers/Iotex.ts | 175 ++++++++--------- services/crawler/test/crawler.test.ts | 43 ++++- 6 files changed, 328 insertions(+), 365 deletions(-) diff --git a/common/helpers/src/index.ts b/common/helpers/src/index.ts index c9ffb95c4..7ee5fc87b 100644 --- a/common/helpers/src/index.ts +++ b/common/helpers/src/index.ts @@ -1,8 +1,10 @@ import { S3Client, S3ClientConfig, PutObjectCommand, GetObjectCommand } from '@aws-sdk/client-s3' import { AthenaClient, AthenaClientConfig } from '@aws-sdk/client-athena' import { defaultProvider } from '@aws-sdk/credential-provider-node' -import {queryOutputLocation} from '@casimir/crawler/src' -import { StartQueryExecutionCommand, GetQueryExecutionCommand, StartQueryExecutionCommandInput } from '@aws-sdk/client-athena' +import { StartQueryExecutionCommand, GetQueryExecutionCommand } from '@aws-sdk/client-athena' +import { EventTableColumn } from '@casimir/data' +import {Chain} from '@casimir/crawler/src/providers/Base' +import {IotexNetworkType} from '@casimir/crawler/src/providers/Iotex' /** * Converts any string to PascalCase. @@ -17,9 +19,17 @@ export function pascalCase(str: string): string { }) } +// these are defined to reuse the clients let athena: AthenaClient | null = null let s3: S3Client | null = null +/** + * Creates a new Athena client + * + * @param opt - Athena client config + * @returns Athena client + * + */ export async function newAthenaClient(opt?: AthenaClientConfig): Promise { if (opt?.region === undefined) { opt = { @@ -38,6 +48,13 @@ export async function newAthenaClient(opt?: AthenaClientConfig): Promise { if (s3) { return s3 @@ -61,6 +78,14 @@ export async function newS3Client (opt?: S3ClientConfig): Promise { return client } +/** + * Uploads data to S3 + * + * @param input.bucket - Bucket destination + * @param input.key - Key destination + * @param input.data - Data to be uploaded + * + */ export async function uploadToS3( input: { bucket: string, key: string,data: string }): Promise { if (!s3) { s3 = await newS3Client() @@ -74,6 +99,15 @@ export async function uploadToS3( input: { bucket: string, key: string,data: str if ($metadata.httpStatusCode !== 200) throw new Error('Error uploading to s3') } +/** + * Get data from S3 + * + * @param input.bucket - Bucket destination + * @param input.key - Key destination + * @param input.data - Data to be uploaded + * @return data - Data from S3 + * + */ export async function getFromS3(bucket: string, key: string): Promise { if (!s3) { s3 = await newS3Client() @@ -99,10 +133,17 @@ export async function getFromS3(bucket: string, key: string): Promise { return chunk } - let retry = 0 let backoff = 300 +/** + * Poll for Athena query's result + * + * @param input.bucket - Bucket destination + * @param input.key - Key destination + * @param input.data - Data to be uploaded + * + */ async function pollAthenaQueryOutput(queryId: string): Promise { if (!athena) { athena = await newAthenaClient() @@ -131,7 +172,13 @@ async function pollAthenaQueryOutput(queryId: string): Promise { return } -export async function queryAthena(query: string): Promise { +/** + * Runs a SQL query on Athena table + * + * @param query - SQL query to run (make sure the correct permissions are set) + * @return string - Query result + */ +export async function queryAthena(query: string): Promise { if (!athena) { athena = await newAthenaClient() @@ -139,10 +186,9 @@ export async function queryAthena(query: string): Promise { const execCmd = new StartQueryExecutionCommand({ QueryString: query, - // QueryString: `SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" WHERE chain = '${chain}' ORDER BY height DESC LIMIT 1`, WorkGroup: 'primary', ResultConfiguration: { - OutputLocation: queryOutputLocation, + OutputLocation: 's3://cms-lds-agg/cms_hcf_aggregates/' } }) @@ -161,8 +207,32 @@ export async function queryAthena(query: string): Promise { // wait for athena to finish writing to s3 await new Promise(resolve => setTimeout(resolve, 2000)) - const result = await getFromS3('cms-lds-agg', `cms_hcf_aggregates/${QueryExecutionId}.csv`) + const raw = await getFromS3('cms-lds-agg', `cms_hcf_aggregates/${QueryExecutionId}.csv`) - // const height = raw.split('\n').filter(l => l !== '')[1].replace(/"/g, '') - return result + const rows = raw.split('\n').filter(r => r !== '') + + if (rows.length <= 1) { + throw new Error('InvalidQueryResult: query result is empty') + } + + const header = rows.splice(0, 1)[0].split(',').map((h: string) => h.trim().replace(/"/g, '')) + + const events: EventTableColumn[] = [] + + rows.forEach((curr, i) => { + const row = curr.split(',') + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const event: EventTableColumn = {} + row.forEach((r, i) => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + event[header[i]] = r.trim().replace(/"/g, '') + }) + + if (event) { + events.push(event) + } + }) + return events } \ No newline at end of file diff --git a/services/crawler/src/index.ts b/services/crawler/src/index.ts index 18dc5bb86..00b02756a 100644 --- a/services/crawler/src/index.ts +++ b/services/crawler/src/index.ts @@ -14,13 +14,15 @@ import { import { IotexService, newIotexService } from './providers/Iotex' import {EthereumService, newEthereumService} from './providers/Ethereum' import { Chain } from './providers/Base' +import {uploadToS3} from '@casimir/helpers' +import {ethers} from 'ethers' export const defaultEventBucket = 'casimir-etl-event-bucket-dev' export const queryOutputLocation = 's3://cms-lds-agg/cms_hcf_aggregates' const EE = new EventEmitter() -export let s3: S3Client | null = null +export const s3: S3Client | null = null export const athena: AthenaClient | null = null export interface CrawlerConfig { @@ -33,141 +35,133 @@ class Crawler { config: CrawlerConfig service: IotexService | EthereumService | null EE: EventEmitter - s3Client: S3Client | null - athenaClient: AthenaClient | null constructor (config: CrawlerConfig) { this.config = config this.service = null this.EE = EE - this.s3Client = null - this.athenaClient = null } - - async upload(key: string, data: string): Promise { - if (key === undefined || key === null) throw new Error('InvalidKey: key is not defined') - if (this.service === null) throw new Error('NullService: service is not initialized') - if (this.s3Client === null) this.s3Client = await newS3Client() - - const upload = new PutObjectCommand({ - Bucket: defaultEventBucket, - Key: key, - Body: data - }) - - const { $metadata } = await this.s3Client.send(upload).catch((e: Error) => { - throw e - }) - - if ($metadata.httpStatusCode !== 200) throw new Error('FailedUploadBlock: unable to upload block') - } - async prepare (): Promise { switch (this.config.chain) { case Chain.Iotex: - this.service = await newIotexService() + this.service = await newIotexService({ }) break case Chain.Ethereum: - this.service = await newEthereumService() + this.service = await newEthereumService({ url: 'http://localhost:8545' }) break default: throw new Error('InvalidChain: chain is not supported') } } async start (): Promise { - if (this.service == null) { - throw new Error('NullService: service is not initialized') - } - - if (s3 === null) s3 = await newS3Client() - - - if (this.service instanceof IotexService) { - const { chainMeta } = await this.service.getChainMetadata() - const height = parseInt(chainMeta.height) - const blocksPerRequest = 1000 - - const lastBlock = await this.retrieveLastBlock() - - const start = lastBlock === 0 ? 0 : lastBlock + 1 - const trips = Math.ceil(height / blocksPerRequest) - - for (let i = start; i < trips; i++) { - const { blkMetas: blocks } = await this.service.getBlocks(i, blocksPerRequest) - - if (blocks.length === 0) continue - - for await (const block of blocks) { - let events: EventTableColumn[] = [] - const actions = await this.service.getBlockActions(block.height, block.numActions) - - if (actions.length === 0 || actions[0].action.core === undefined) continue - for await (const action of actions) { - const core = action.action.core - if (core === undefined) continue - - const type = Object.keys(core).filter(k => k !== undefined)[Object.keys(core).length - 2] + if (this.service instanceof EthereumService) { + let events: EventTableColumn[] = [] + + const count = 0 + + const block = await this.service.getCurrentBlock() + for (let i = 0; i < block.number; i++) { + const blocksWithTransaction = await this.service.getBlock(i) + + const blockEvent: EventTableColumn = { + chain: Chain.Ethereum, + network: 'mainnet', + provider: 'casimir', + type: 'block', + created_at: new Date().toISOString(), + address: '', + height: blocksWithTransaction.number, + to_address: '', + candidate: '', + duration: 0, + candidate_list: [], + amount: 0, + auto_stake: false, + } - const event = this.service.convertToGlueSchema({ type, block, action}) - events.push(event) + const txEvents = blocksWithTransaction.transactions.map((tx: any): EventTableColumn => { + return { + chain: Chain.Ethereum, + network: 'mainnet', + provider: 'casimir', + type: 'transaction', + created_at: new Date().toISOString(), + address: tx.from, + height: tx.height, + to_address: tx.to || '', + candidate: '', + duration: 0, + candidate_list: [], + amount: parseInt(tx.value.toString()), + auto_stake: false, } - - const ndjson = events.map(a => JSON.stringify(a)).join('\n') - events.forEach(e => console.log(e.height + ' ' + e.address + ' ' + e.type)) - const key = `${block.hash}-events.json` - await this.upload(key, ndjson) - events = [] + }) + + events = [blockEvent, ...txEvents] + + if (i+1 % 1000 === 0) { + const key = `${block.hash}-events.json` + const ndjson = events.map((e: EventTableColumn) => JSON.stringify(e)).join('\n') + + // await uploadToS3({ + // bucket: 'some-bucket', + // key: `events/${key}`, + // data: ndjson + // }) + console.log(`uploading ${key} to s3 done`) + events = [] + continue } + console.log(`event: ${i}`) } return } - - if (this.service instanceof EthereumService) { - const height = await this.service.getCurrentBlockHeight() - for (let i = 0; i < height; i++) { - const block = await this.service.getBlock('0') - console.log(block) - // const key = `${block.hash}-events.json` - console.log(JSON.stringify(block, null, 2)) - } - return - } - throw new Error('not implemented yet') - } - - async stop(): Promise { - if (this.service === null) throw new Error('NullService: service is not initialized') - if (this.service instanceof IotexService) { + // const { chainMeta } = await this.service.getChainMetadata() + // const height = parseInt(chainMeta.height) + // const blocksPerRequest = 1000 + + // const lastBlock = await this.service.getLastProcessedBlockHeight() + + // const start = lastBlock === 0 ? 0 : lastBlock + 1 + // const trips = Math.ceil(height / blocksPerRequest) + + // for (let i = start; i < trips; i++) { + // const { blkMetas: blocks } = await this.service.getBlocks(i, blocksPerRequest) + // + // if (blocks.length === 0) continue + // + // for await (const block of blocks) { + // let events: EventTableColumn[] = [] + // const actions = await this.service.getBlockActions(block.height, block.numActions) + // + // if (actions.length === 0 || actions[0].action.core === undefined) continue + // for await (const action of actions) { + // const core = action.action.core + // if (core === undefined) continue + // + // const type = Object.keys(core).filter(k => k !== undefined)[Object.keys(core).length - 2] + // + // const event = this.service.convertToGlueSchema({ type, block, action}) + // events.push(event) + // } + // + // const ndjson = events.map(a => JSON.stringify(a)).join('\n') + // events.forEach(e => console.log(e.height + ' ' + e.address + ' ' + e.type)) + // const key = `${block.hash}-events.json` + // await uploadToS3({ + // bucket: defaultEventBucket, + // key, + // data: ndjson + // }) + // events = [] + // } + // } + const gensis = await this.service.getBlock(100000532) + console.log(gensis) return } throw new Error('not implemented yet') } - - // on(event: any, cb: (b: IStreamBlocksResponse) => void): void { - // if (this.service === null) throw new Error('NullService: service is not initialized') - // - // if (event == EventStreamType.IOTEX_BLOCK) { - // if (this.service instanceof IotexService) { - // this.service.readableBlockStream().then((s: any) => { - // s.on('data', (b: IStreamBlocksResponse) => { - // cb(b) - // }) - // - // s.on('error', (e: Error) => { - // throw e - // }) - // }) - // return - // } - // } - // - // if (event === EventStreamType.ETH_BLOCK) { - // if (this.service instanceof EthereumService) { - // // stream it - // } - // } - // throw new Error('not implemented yet') - // } } export async function crawler (config: CrawlerConfig): Promise { diff --git a/services/crawler/src/providers/Base.ts b/services/crawler/src/providers/Base.ts index 07d3b7053..93b339f39 100644 --- a/services/crawler/src/providers/Base.ts +++ b/services/crawler/src/providers/Base.ts @@ -1,7 +1,8 @@ import { EventTableColumn } from '@casimir/data' import { EventEmitter } from 'events' -import { AthenaClient } from '@aws-sdk/client-athena' -import { S3Client } from '@aws-sdk/client-s3' +import { ethers } from 'ethers' +import Antenna from 'iotex-antenna' +import {Block} from 'iotex-antenna/protogen/proto/types/blockchain_pb' export enum Chain { Iotex = 'iotex', @@ -10,16 +11,14 @@ export enum Chain { export interface BaseService { chain: Chain - provider: any - athenaClient: AthenaClient - s3Client: S3Client + provider: ethers.providers.JsonRpcProvider | Antenna eventEmitter: EventEmitter | null getChainMetadata(): Promise - getBlock(hash: string): Promise + getBlock(num: number): Promise getTransaction(tx: string): Promise - on(event: T, cb: () => void): void - getCurrentBlockHeight(): Promise - getLastProcessedBlockHeight(): Promise + getCurrentBlock(): Promise + getLastProcessedEvent(chain: Chain): Promise convertToGlueSchema(obj: Record): EventTableColumn - save(key: string, data: string): void + on(event: string, cb: (block: any) => void): void + save(key: string, data: string): Promise } diff --git a/services/crawler/src/providers/Ethereum.ts b/services/crawler/src/providers/Ethereum.ts index a0d8226ab..4fe8d75d7 100644 --- a/services/crawler/src/providers/Ethereum.ts +++ b/services/crawler/src/providers/Ethereum.ts @@ -1,20 +1,14 @@ +import EventEmitter from 'events' import { ethers } from 'ethers' import { EventTableColumn } from '@casimir/data' import { BaseService, Chain } from './Base' -import EventEmitter from 'events' -import {GetObjectCommand, PutObjectCommand, S3Client} from '@aws-sdk/client-s3' -import {defaultEventBucket, newAthenaClient, newS3Client, queryOutputLocation, s3} from '../index' -import {AthenaClient, GetQueryExecutionCommand, StartQueryExecutionCommand} from '@aws-sdk/client-athena' -import {defaultProvider} from '@aws-sdk/credential-provider-node' +import { queryAthena, uploadToS3 } from '@casimir/helpers' +import Antenna from 'iotex-antenna' -export type EthereumServiceOptions = { - provider: string -} +export type EthereumServiceOptions = any export class EthereumService implements BaseService { chain: Chain - athenaClient: AthenaClient - s3Client: S3Client provider: ethers.providers.JsonRpcProvider eventEmitter: EventEmitter | null = null @@ -23,14 +17,6 @@ export class EthereumService implements BaseService { this.provider = new ethers.providers.JsonRpcProvider({ url: opt.provider || 'http://localhost:8545', }) - this.athenaClient = new AthenaClient({ - region: 'us-east-1', - credentials: defaultProvider() - }) - this.s3Client = new S3Client({ - region: 'us-east-2', - credentials: defaultProvider() - }) this.eventEmitter = null } @@ -39,119 +25,36 @@ export class EthereumService implements BaseService { return meta } - async getBlock(hash: string): Promise { - const block = await this.provider.getBlock(hash) - return block + async getBlock(num: number): Promise { + return await this.provider.getBlockWithTransactions(num) } - // todo: get last processed block from athena - - async getLastProcessedBlock(chain: Chain): Promise { - const selectQuery= new StartQueryExecutionCommand({ - QueryString: `SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" WHERE chain = '${chain}' ORDER BY height DESC LIMIT 1`, - WorkGroup: 'primary', - ResultConfiguration: { - OutputLocation: queryOutputLocation, - } - }) - - const { $metadata, QueryExecutionId } = await this.athenaClient.send(selectQuery) - - if ($metadata.httpStatusCode !== 200) { - throw new Error('FailedQuery: unable to query Athena') - } - - if (QueryExecutionId === undefined) { - throw new Error('InvalidQueryExecutionId: query execution id is undefined') - } - - const getCmd = new GetQueryExecutionCommand({ QueryExecutionId }) - - const execRes = await this.athenaClient.send(getCmd) - - if (execRes.$metadata.httpStatusCode !== 200) { - throw new Error('FailedQuery: unable to query Athena') - } - - if (execRes.QueryExecution === undefined) { - throw new Error('InvalidQueryExecution: query execution is undefined') - } - - let retry = 0 - let backoff = 1000 - - const poll = async (): Promise => { - const getStateCmd = new GetQueryExecutionCommand({ QueryExecutionId }) - - const getStateRes = await this.athenaClient.send(getStateCmd) + async getLastProcessedEvent(chain: Chain): Promise { + const event = await queryAthena(`SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = '${Chain.Ethereum}' ORDER BY height DESC limit 1`) - if (getStateRes.$metadata.httpStatusCode !== 200) throw new Error('FailedQuery: unable to query Athena') - if (getStateRes.QueryExecution === undefined) throw new Error('InvalidQueryExecution: query execution is undefined') - if (getStateRes.QueryExecution.Status === undefined) throw new Error('InvalidQueryExecutionStatus: query execution status is undefined') - - if (getStateRes.QueryExecution.Status.State === 'QUEUED' || getStateRes.QueryExecution.Status.State === 'RUNNING') { - setTimeout(() => { - poll() - retry++ - backoff = backoff + 500 - }, backoff) - } - if (getStateRes.QueryExecution.Status.State === 'FAILED') throw new Error('QueryFailed: query failed') - if (getStateRes.QueryExecution.Status.State === 'SUCCEEDED') return + if (event.length === 1) { + return event[0] } - - const getResult = async (): Promise => { - if (s3 === null) throw new Error('NullS3Client: s3 client is not initialized') - - const {$metadata, Body} = await s3.send(new GetObjectCommand({ - Bucket: 'cms-lds-agg', - Key: `cms_hcf_aggregates/${QueryExecutionId}.csv` - })) - - if ($metadata.httpStatusCode !== 200) throw new Error('FailedQuery: unable retrieve result from S3') - if (Body === undefined) throw new Error('InvalidQueryResult: query result is undefined') - - let chunk = '' - - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - for await (const data of Body) { - chunk += data.toString() - } - return chunk - } - - await poll() - - // wait for athena writing to s3 - await new Promise(resolve => setTimeout(resolve, 2000)) - - const raw = await getResult() - - const height = raw.split('\n').filter(l => l !== '')[1].replace(/"/g, '') - return parseInt(height) + console.log('More than one event found') + return null } - - + async getTransaction(tx: string): Promise { const txData = await this.provider.getTransaction(tx) return txData } // todo: better interface and cleanup - on(event: T, cb: () => void): void { - this.provider.on(event, cb) + on(event:string, cb: (block: ethers.providers.Block) => void): void { + this.provider.on('block', async (blockNumber: number) => { + const block = await this.getBlock(blockNumber) + cb(block) + }) } - async getCurrentBlockHeight(): Promise { - const h = await this.provider.getBlockNumber() - console.log('current block height', h) - return h - } - - // todo: get last processed block from athena - getLastProcessedBlockHeight(): Promise { - return Promise.resolve(0) + async getCurrentBlock(): Promise { + const height = await this.provider.getBlockNumber() + return await this.provider.getBlock(height) } convertToGlueSchema(event: { type: string, block: ethers.providers.Block, tx: ethers.providers.TransactionResponse }): EventTableColumn { @@ -160,7 +63,7 @@ export class EthereumService implements BaseService { network: 'mainnet', provider: 'casimir', type: event.type, - created_at: new Date().toISOString().split('T')[0], + created_at: new Date().toISOString(), address: event.tx.from, height: event.block.number, to_address: event.tx.to || '', @@ -174,20 +77,7 @@ export class EthereumService implements BaseService { } async save(key: string, data: string): Promise { - if (key === undefined || key === null) throw new Error('InvalidKey: key is not defined') - if (data === undefined || data === null) throw new Error('InvalidData: data is not defined') - - const upload = new PutObjectCommand({ - Bucket: defaultEventBucket, - Key: key, - Body: data - }) - - const { $metadata } = await this.s3Client.send(upload).catch((e: Error) => { - throw e - }) - - if ($metadata.httpStatusCode !== 200) throw new Error('FailedUploadBlock: unable to upload block') + await uploadToS3({ bucket: 'b', key, data }).catch(e =>{ throw new Error(e) }) } } diff --git a/services/crawler/src/providers/Iotex.ts b/services/crawler/src/providers/Iotex.ts index 042f0d7b4..b2c00e252 100644 --- a/services/crawler/src/providers/Iotex.ts +++ b/services/crawler/src/providers/Iotex.ts @@ -12,16 +12,14 @@ import { import { from } from '@iotexproject/iotex-address-ts' import { Opts } from 'iotex-antenna/lib/antenna' import { EventTableColumn } from '@casimir/data' -import { Chain } from './Base' - -export type IotexOptions = Opts & { - provider: string, - chainId: 1 | 2 -} +import {BaseService, Chain} from './Base' +import EventEmitter from 'events' +import {queryAthena} from '@casimir/helpers' +import {ethers} from 'ethers' export enum IotexNetworkType { - Mainnet = 'mainnet', - Testnet = 'testnet' + Mainnet = '4689', + Testnet = '4690' } enum IotexActionType { @@ -36,103 +34,59 @@ enum IotexActionType { stakeRestake = 'stake_restake', } -export class IotexService { - provider: string - chainId: number - client: Antenna - constructor (opt: IotexOptions) { - this.provider = opt.provider || 'http://localhost:14014' - this.chainId = opt.chainId - this.client = new Antenna(opt.provider, opt.chainId, { - signer: opt.signer, - timeout: opt.timeout, - apiToken: opt.apiToken - }) - } - - async getChainMetadata (): Promise { - const meta = await this.client.iotx.getChainMeta({}) - return meta - } - - async getBlocks(start: number, count: number): Promise { - if (start < 0 || count < 0) { - throw new Error('start and count must be greater than 0') - } - - if (start === 0) { - start = 1 - } +export type IotexOptions = Opts - if (count === 0) { - count = 100 - } - - return await this.client.iotx.getBlockMetas({ byIndex: { start: start, count: count } }) - } +export class IotexService implements BaseService { + chain: Chain + // provider: Antenna + provider: ethers.providers.JsonRpcProvider + eventEmitter: EventEmitter | null = null - async getBlockMeta (blockHash: string): Promise { - const metas = await this.client.iotx.getBlockMetas({ - byHash: { - blkHash: blockHash - } + private readonly chainId: string + // todo: switch to ether and point to iotex rpc + constructor (opt: IotexOptions) { + this.chain = Chain.Iotex + this.provider = new ethers.providers.JsonRpcProvider({ + url: 'https://babel-api.mainnet.iotex.io', }) - return metas + // this.provider = new Antenna('https://iotexrpc.com', parseInt(IotexNetworkType.Mainnet), { + // signer: opt.signer, + // timeout: opt.timeout, + // apiToken: opt.apiToken + // }) + this.eventEmitter = null + this.chainId = IotexNetworkType.Mainnet } - convertEthToIotx (eth: string): string { - const add = from(eth) - return add.string() + async getChainMetadata (): Promise { + // return await this.provider.getNetwork() } - convertIotxToEth (iotx: string): string { - const add = from(iotx) - return add.stringEth() + async getBlock(num: number): Promise { + return await this.provider.getBlock(num) } - async getAccountActions (address: string, count: number): Promise { - const account = await this.client.iotx.getAccount({ - address - }) + async getLastProcessedEvent(chain: Chain): Promise { + const event = await queryAthena(`SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = '${Chain.Iotex}' ORDER BY height DESC limit 1`) - if (account.accountMeta === undefined) { - return [] + if (event.length === 1) { + return event[0] } - const actions = await this.client.iotx.getActions({ - byAddr: { - address: account.accountMeta.address, - start: 1, - count: count || 100 - } - }) - - return actions - } - - async readableBlockStream (): Promise> { - const stream = await this.client.iotx.streamBlocks({ - start: 1 - }) - return stream + console.log('More than one event found') + return null } - async getTxReceipt (actionHash: string): Promise { - const tx = await this.client.iotx.getReceiptByAction({ actionHash }) - return tx + // Note: Transactions in ethereum are called actions in Iotex + async getTransaction(tx: string): Promise { + return await this.provider.getTransaction(tx) } - async getBlockActions (index: number, count: number): Promise { - const actions = await this.client.iotx.getActions({ - byIndex: { - start: index, - count: count - } - }) - return actions.actionInfo + async getCurrentBlock(): Promise { + const current = await this.getChainMetadata() + return parseInt(current.chainMeta.height) } - convertToGlueSchema(obj: { type: string, block: IBlockMeta, action: IActionInfo}): EventTableColumn { const core = obj.action.action.core @@ -140,9 +94,10 @@ export class IotexService { const event: EventTableColumn = { chain: Chain.Iotex, - network: this.chainId === 1 ? IotexNetworkType.Mainnet : IotexNetworkType.Testnet, + // 'https://api.iotex.one:443' + network: IotexNetworkType.Mainnet, provider: 'casimir', - created_at: new Date(obj.block.timestamp.seconds * 1000).toISOString().split('T')[0], + created_at: new Date(obj.block.timestamp.seconds * 1000).toISOString(), type: '', address: obj.block.producerAddress, height: obj.block.height, @@ -152,7 +107,6 @@ export class IotexService { amount: 0, duration: 0, auto_stake: false, - // payload: {} } switch (obj.type) { case 'grantReward': @@ -183,7 +137,7 @@ export class IotexService { event.amount = parseInt(core.execution?.amount) } return event - case 'putPollResult': + case 'putPollResult': event.type = IotexActionType.putPollResult if (core.putPollResult?.candidates !== undefined) { event.candidate_list = core.putPollResult?.candidates.candidates.map(c => c.address) @@ -208,11 +162,40 @@ export class IotexService { throw new Error(`Action type ${obj.type} not supported`) } } + + on(event:string, cb: (block: IStreamBlocksResponse) => void): void { + // const stream = this.provider.iotx.streamBlocks({ start: 0 }) + // + // stream.on('data', (block: IStreamBlocksResponse) => { + // cb(block) + // }) + // + // stream.on('error', (err: any) => { + // throw new Error(err) + // }) + } + + async save(key: string, data: string): Promise { + return + } + + // private async getBlocks(start: number, count: number): Promise { + // if (start < 0 || count < 0) { + // throw new Error('start and count must be greater than 0') + // } + // + // if (start === 0) { + // start = 1 + // } + // + // if (count === 0) { + // count = 100 + // } + // + // return await this.provider.iotx.getBlockMetas({ byIndex: { start: start, count: count } }) + // } } -export async function newIotexService (opt?: IotexOptions): Promise { - return new IotexService({ - provider: opt?.provider ?? 'https://api.iotex.one:443', - chainId: opt?.chainId ?? 1 - }) +export async function newIotexService (opt: IotexOptions): Promise { + return new IotexService(opt) } \ No newline at end of file diff --git a/services/crawler/test/crawler.test.ts b/services/crawler/test/crawler.test.ts index a04d596cf..0d91a44f4 100644 --- a/services/crawler/test/crawler.test.ts +++ b/services/crawler/test/crawler.test.ts @@ -1,29 +1,56 @@ import { crawler } from '../src/index' import { Chain } from '../src/providers/Base' +import {queryAthena} from '@casimir/helpers' jest.setTimeout(20000) -// test('get last block', async () => { -// const supercrawler = await crawler({ +test('init crawler for iotex', async () => { + const iotex = await crawler({ + chain: Chain.Iotex, + verbose: true + }) + expect(iotex.service).not.toBe(null) +}) +// +// test('start crawler for iotex', async () => { +// const iotex = await crawler({ // chain: Chain.Iotex, // verbose: true // }) // -// expect(supercrawler.service).not.toBe(null) -// const lastBlock = await supercrawler.retrieveLastBlock() -// expect(typeof lastBlock).toBe('number') +// await iotex.start() +// // expect(iotex.service).not.toBe(null) // }) test('init crawler for ethereum', async () => { - const ethCrawler = await crawler({ + const eth = await crawler({ chain: Chain.Ethereum, verbose: true }) - await ethCrawler.start() - expect(ethCrawler.service).not.toBe(null) + expect(eth.service).not.toBe(null) }) + +test('init crawler for ethereum', async () => { + const queryResult = await queryAthena('SELECT * FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = \'iotex\' limit 10') + expect(queryResult.length).toBe(10) +}) + + + + +// test('get last block', async () => { +// const supercrawler = await crawler({ +// chain: Chain.Iotex, +// verbose: true +// }) +// +// expect(supercrawler.service).not.toBe(null) +// const lastBlock = await supercrawler.retrieveLastBlock() +// expect(typeof lastBlock).toBe('number') +// }) + // test('stream', async () => { // const supercrawler = await crawler({ // chain: Chain.Iotex, From d33f66ce29e2d0b577ffd1b372bf4f1de6d962ab Mon Sep 17 00:00:00 2001 From: hawyar Date: Fri, 23 Sep 2022 14:24:51 -0400 Subject: [PATCH 4/7] Clean providers and tests --- common/data/src/index.ts | 4 +- common/data/src/schemas/agg.schema.json | 2 +- common/data/src/schemas/event.schema.json | 2 +- common/helpers/src/index.ts | 9 +- services/crawler/README.md | 25 +-- services/crawler/src/index.ts | 224 +++++++++---------- services/crawler/src/providers/Base.ts | 13 +- services/crawler/src/providers/Ethereum.ts | 60 +++-- services/crawler/src/providers/Iotex.ts | 246 +++++++++------------ services/crawler/test/crawler.test.ts | 57 ++--- 10 files changed, 274 insertions(+), 368 deletions(-) diff --git a/common/data/src/index.ts b/common/data/src/index.ts index effb8aa96..a0d2bf0f8 100644 --- a/common/data/src/index.ts +++ b/common/data/src/index.ts @@ -19,7 +19,7 @@ export function schemaToGlueColumns(jsonSchema: JsonSchema): glue.Column[] { let type: glue.Type = glue.Schema[typeKey] - if (name.endsWith('at')) type = glue.Schema.DATE + if (name.endsWith('at')) type = glue.Schema.TIMESTAMP if (name === 'candidate_list') type = glue.Schema.array(glue.Schema.STRING) @@ -41,7 +41,7 @@ export type EventTableColumn = { to_address: string candidate: string candidate_list: string[] - amount: number + amount: string duration: number auto_stake: boolean // payload: Record diff --git a/common/data/src/schemas/agg.schema.json b/common/data/src/schemas/agg.schema.json index fffe31b71..37ba5dc9a 100644 --- a/common/data/src/schemas/agg.schema.json +++ b/common/data/src/schemas/agg.schema.json @@ -14,7 +14,7 @@ }, "first_staked_at": { "type": "string", - "description": "The first date (MM-DD-YYYY) that a wallet staked" + "description": "First staked at datestring" }, "total_staked_amount": { "type": "string", diff --git a/common/data/src/schemas/event.schema.json b/common/data/src/schemas/event.schema.json index f7a364fe8..6e96abd92 100644 --- a/common/data/src/schemas/event.schema.json +++ b/common/data/src/schemas/event.schema.json @@ -26,7 +26,7 @@ }, "created_at": { "type": "string", - "description": "The date (MM-DD-YYYY) of the event" + "description": "The datestring of the event" }, "address": { "type": "string", diff --git a/common/helpers/src/index.ts b/common/helpers/src/index.ts index 7ee5fc87b..31d83c0de 100644 --- a/common/helpers/src/index.ts +++ b/common/helpers/src/index.ts @@ -19,7 +19,6 @@ export function pascalCase(str: string): string { }) } -// these are defined to reuse the clients let athena: AthenaClient | null = null let s3: S3Client | null = null @@ -86,7 +85,7 @@ export async function newS3Client (opt?: S3ClientConfig): Promise { * @param input.data - Data to be uploaded * */ -export async function uploadToS3( input: { bucket: string, key: string,data: string }): Promise { +export async function uploadToS3( input: { bucket: string, key: string, data: string }): Promise { if (!s3) { s3 = await newS3Client() } @@ -94,7 +93,9 @@ export async function uploadToS3( input: { bucket: string, key: string,data: str const upload = new PutObjectCommand({ Bucket: input.bucket, Key: input.key, + Body: input.data }) + const { $metadata } = await s3.send(upload) if ($metadata.httpStatusCode !== 200) throw new Error('Error uploading to s3') } @@ -178,7 +179,7 @@ async function pollAthenaQueryOutput(queryId: string): Promise { * @param query - SQL query to run (make sure the correct permissions are set) * @return string - Query result */ -export async function queryAthena(query: string): Promise { +export async function queryAthena(query: string): Promise { if (!athena) { athena = await newAthenaClient() @@ -212,7 +213,7 @@ export async function queryAthena(query: string): Promise { const rows = raw.split('\n').filter(r => r !== '') if (rows.length <= 1) { - throw new Error('InvalidQueryResult: query result is empty') + return null } const header = rows.splice(0, 1)[0].split(',').map((h: string) => h.trim().replace(/"/g, '')) diff --git a/services/crawler/README.md b/services/crawler/README.md index 478c8cff5..9e30f7213 100644 --- a/services/crawler/README.md +++ b/services/crawler/README.md @@ -1,24 +1 @@ -## Crawler - -The crawler's purpose is to provide historical and real-time blockchain data to its consumers. - -### Pre-requisites -// todo - -## Usage -```js -const supercrawler = await crawler({ - chain: Chain.Iotex, - verbose: true -}) - -// start -await supercrawler.start() - -// subcribe to blocks -supercrawler.on('block', (block) => { - console.log(blocks) -}) - -// stop -supercrawler.stop() \ No newline at end of file +## Crawler \ No newline at end of file diff --git a/services/crawler/src/index.ts b/services/crawler/src/index.ts index 00b02756a..c9f00ab02 100644 --- a/services/crawler/src/index.ts +++ b/services/crawler/src/index.ts @@ -1,26 +1,18 @@ - -import {GetObjectCommand, S3Client, S3ClientConfig,} from '@aws-sdk/client-s3' -import { defaultProvider } from '@aws-sdk/credential-provider-node' -import EventEmitter from 'events' -import { PutObjectCommand } from '@aws-sdk/client-s3' +import { S3Client } from '@aws-sdk/client-s3' import { EventTableColumn } from '@casimir/data' -import { - AthenaClient, - AthenaClientConfig, - GetQueryExecutionCommand, - StartQueryExecutionCommand -} from '@aws-sdk/client-athena' - -import { IotexService, newIotexService } from './providers/Iotex' -import {EthereumService, newEthereumService} from './providers/Ethereum' +import { AthenaClient } from '@aws-sdk/client-athena' +import { IotexNetworkType, IotexService, newIotexService } from './providers/Iotex' +import { EthereumService, newEthereumService } from './providers/Ethereum' import { Chain } from './providers/Base' -import {uploadToS3} from '@casimir/helpers' -import {ethers} from 'ethers' +import { queryAthena, uploadToS3 } from '@casimir/helpers' +import { ethers } from 'ethers' export const defaultEventBucket = 'casimir-etl-event-bucket-dev' -export const queryOutputLocation = 's3://cms-lds-agg/cms_hcf_aggregates' -const EE = new EventEmitter() +enum CasimirEventType { + Block = 'block', + Transaction = 'transaction', +} export const s3: S3Client | null = null export const athena: AthenaClient | null = null @@ -34,16 +26,16 @@ export interface CrawlerConfig { class Crawler { config: CrawlerConfig service: IotexService | EthereumService | null - EE: EventEmitter constructor (config: CrawlerConfig) { this.config = config this.service = null - this.EE = EE } async prepare (): Promise { switch (this.config.chain) { case Chain.Iotex: - this.service = await newIotexService({ }) + this.service = newIotexService({ + network: IotexNetworkType.Mainnet, + }) break case Chain.Ethereum: this.service = await newEthereumService({ url: 'http://localhost:8545' }) @@ -52,115 +44,103 @@ class Crawler { throw new Error('InvalidChain: chain is not supported') } } + + async getLastProcessedEvent(): Promise { + const event = await queryAthena(`SELECT * FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = '${this.config.chain}' ORDER BY height DESC limit 1`) + + if (event !== null && event.length === 1) { + return event[0] + } + return null + } + + + async convertToGlueSchema(event: { block: ethers.providers.Block, tx: Record | null }): Promise { + const events: EventTableColumn[] = [] + + events.push({ + chain: this.config.chain, + network: this.service?.network || 'mainnet', + provider: 'casimir', + type: CasimirEventType.Block, + created_at: new Date(event.block.timestamp * 1000).toISOString(), + address: event.block.miner, + height: event.block.number, + to_address: '', + candidate: '', + duration: 0, + candidate_list: [], + amount: '0', + auto_stake: false + }) + + if (event.tx !== null) { + events.push({ + chain: this.config.chain, + network: this.service?.network || 'mainnet', + provider: 'casimir', + type: CasimirEventType.Transaction, + created_at: new Date(event.block.timestamp * 1000).toISOString(), + address: event.tx.from, + height: event.tx.blockNumber, + to_address: event.tx.to, + candidate: '', + duration: 0, + candidate_list: [], + amount: event.tx.value, + auto_stake: false, + }) + } + return events + } + async start (): Promise { - if (this.service instanceof EthereumService) { - let events: EventTableColumn[] = [] - - const count = 0 - - const block = await this.service.getCurrentBlock() - for (let i = 0; i < block.number; i++) { - const blocksWithTransaction = await this.service.getBlock(i) - - const blockEvent: EventTableColumn = { - chain: Chain.Ethereum, - network: 'mainnet', - provider: 'casimir', - type: 'block', - created_at: new Date().toISOString(), - address: '', - height: blocksWithTransaction.number, - to_address: '', - candidate: '', - duration: 0, - candidate_list: [], - amount: 0, - auto_stake: false, - } + if (this.service instanceof EthereumService) { + const lastEvent = await this.getLastProcessedEvent() + const current = await this.service.getCurrentBlock() - const txEvents = blocksWithTransaction.transactions.map((tx: any): EventTableColumn => { - return { - chain: Chain.Ethereum, - network: 'mainnet', - provider: 'casimir', - type: 'transaction', - created_at: new Date().toISOString(), - address: tx.from, - height: tx.height, - to_address: tx.to || '', - candidate: '', - duration: 0, - candidate_list: [], - amount: parseInt(tx.value.toString()), - auto_stake: false, - } - }) + const start = lastEvent !== null ? lastEvent.height + 1 : 0 + console.log(`starting from block ${start}`) - events = [blockEvent, ...txEvents] + const allEvents: EventTableColumn[] = [] - if (i+1 % 1000 === 0) { - const key = `${block.hash}-events.json` - const ndjson = events.map((e: EventTableColumn) => JSON.stringify(e)).join('\n') + for (let i = start; i < current.number; i++) { + const block = await this.service.getBlock(i) - // await uploadToS3({ - // bucket: 'some-bucket', - // key: `events/${key}`, - // data: ndjson - // }) - console.log(`uploading ${key} to s3 done`) - events = [] - continue - } - console.log(`event: ${i}`) + const ndjson = allEvents.map((e: EventTableColumn) => JSON.stringify(e)).join('\n') + } } + + if (this.service instanceof IotexService) { + const lastEvent = await this.getLastProcessedEvent() + + const current = await this.service.getCurrentBlock() + + const start = lastEvent !== null ? lastEvent.height + 1 : 0 + + console.log(`starting from block: ${start}`) + + const allEvents: EventTableColumn[] = [] + + for (let i = start; i < current.number; i++) { + const block = await this.service.getBlockWithTransactions(i) + + if (i % 3000 === 0 && i !== 0) { + const ndjson = allEvents.map((e: EventTableColumn) => JSON.stringify(e)).join('\n') + await uploadToS3({ + bucket: defaultEventBucket, + key: `${block.hash}-event.json`, + data: ndjson + }).finally(() => { + console.log('uploaded events batch to s3') + }) + allEvents.length = 0 + } + console.log(block) + } return } - if (this.service instanceof IotexService) { - // const { chainMeta } = await this.service.getChainMetadata() - // const height = parseInt(chainMeta.height) - // const blocksPerRequest = 1000 - - // const lastBlock = await this.service.getLastProcessedBlockHeight() - - // const start = lastBlock === 0 ? 0 : lastBlock + 1 - // const trips = Math.ceil(height / blocksPerRequest) - - // for (let i = start; i < trips; i++) { - // const { blkMetas: blocks } = await this.service.getBlocks(i, blocksPerRequest) - // - // if (blocks.length === 0) continue - // - // for await (const block of blocks) { - // let events: EventTableColumn[] = [] - // const actions = await this.service.getBlockActions(block.height, block.numActions) - // - // if (actions.length === 0 || actions[0].action.core === undefined) continue - // for await (const action of actions) { - // const core = action.action.core - // if (core === undefined) continue - // - // const type = Object.keys(core).filter(k => k !== undefined)[Object.keys(core).length - 2] - // - // const event = this.service.convertToGlueSchema({ type, block, action}) - // events.push(event) - // } - // - // const ndjson = events.map(a => JSON.stringify(a)).join('\n') - // events.forEach(e => console.log(e.height + ' ' + e.address + ' ' + e.type)) - // const key = `${block.hash}-events.json` - // await uploadToS3({ - // bucket: defaultEventBucket, - // key, - // data: ndjson - // }) - // events = [] - // } - // } - const gensis = await this.service.getBlock(100000532) - console.log(gensis) - return - } - throw new Error('not implemented yet') + throw new Error('ServiceNotSupported: service is not recognized') } } diff --git a/services/crawler/src/providers/Base.ts b/services/crawler/src/providers/Base.ts index 93b339f39..c067e50f1 100644 --- a/services/crawler/src/providers/Base.ts +++ b/services/crawler/src/providers/Base.ts @@ -1,8 +1,6 @@ import { EventTableColumn } from '@casimir/data' -import { EventEmitter } from 'events' import { ethers } from 'ethers' import Antenna from 'iotex-antenna' -import {Block} from 'iotex-antenna/protogen/proto/types/blockchain_pb' export enum Chain { Iotex = 'iotex', @@ -11,14 +9,13 @@ export enum Chain { export interface BaseService { chain: Chain + network: string provider: ethers.providers.JsonRpcProvider | Antenna - eventEmitter: EventEmitter | null getChainMetadata(): Promise - getBlock(num: number): Promise + getBlock(num: number): Promise + getCurrentBlock(): Promise getTransaction(tx: string): Promise - getCurrentBlock(): Promise - getLastProcessedEvent(chain: Chain): Promise - convertToGlueSchema(obj: Record): EventTableColumn + getLastProcessedEvent(): Promise + // convertToGlueSchema(obj: Record): EventTableColumn on(event: string, cb: (block: any) => void): void - save(key: string, data: string): Promise } diff --git a/services/crawler/src/providers/Ethereum.ts b/services/crawler/src/providers/Ethereum.ts index 4fe8d75d7..86c98e765 100644 --- a/services/crawler/src/providers/Ethereum.ts +++ b/services/crawler/src/providers/Ethereum.ts @@ -1,23 +1,26 @@ -import EventEmitter from 'events' import { ethers } from 'ethers' import { EventTableColumn } from '@casimir/data' import { BaseService, Chain } from './Base' import { queryAthena, uploadToS3 } from '@casimir/helpers' -import Antenna from 'iotex-antenna' -export type EthereumServiceOptions = any +export type EthereumServiceOptions = Record + +export enum EthereumNetworkType { + Mainnet = 'mainnet', + Testnet = 'testnet' +} export class EthereumService implements BaseService { chain: Chain + network: string provider: ethers.providers.JsonRpcProvider - eventEmitter: EventEmitter | null = null constructor(opt: EthereumServiceOptions) { this.chain = Chain.Ethereum + this.network = opt.network || EthereumNetworkType.Testnet this.provider = new ethers.providers.JsonRpcProvider({ - url: opt.provider || 'http://localhost:8545', + url: opt.url || 'http://localhost:8545', }) - this.eventEmitter = null } async getChainMetadata() { @@ -29,13 +32,12 @@ export class EthereumService implements BaseService { return await this.provider.getBlockWithTransactions(num) } - async getLastProcessedEvent(chain: Chain): Promise { + async getLastProcessedEvent(): Promise { const event = await queryAthena(`SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = '${Chain.Ethereum}' ORDER BY height DESC limit 1`) - if (event.length === 1) { + if (event !== null && event.length === 1) { return event[0] } - console.log('More than one event found') return null } @@ -57,28 +59,24 @@ export class EthereumService implements BaseService { return await this.provider.getBlock(height) } - convertToGlueSchema(event: { type: string, block: ethers.providers.Block, tx: ethers.providers.TransactionResponse }): EventTableColumn { - const record: EventTableColumn = { - chain: Chain.Ethereum, - network: 'mainnet', - provider: 'casimir', - type: event.type, - created_at: new Date().toISOString(), - address: event.tx.from, - height: event.block.number, - to_address: event.tx.to || '', - candidate: '', - candidate_list: [], - amount: parseInt(event.tx.value.toString()), - duration: 0, - auto_stake: false, - } - return record - } - - async save(key: string, data: string): Promise { - await uploadToS3({ bucket: 'b', key, data }).catch(e =>{ throw new Error(e) }) - } + // convertToGlueSchema(event: { type: string, block: ethers.providers.Block, tx: ethers.providers.TransactionResponse }): EventTableColumn { + // const record: EventTableColumn = { + // chain: Chain.Ethereum, + // network: this.network, + // provider: 'casimir', + // type: event.type, + // created_at: new Date(event.block.timestamp * 1000).toISOString(), + // address: event.tx.from, + // height: event.block.number, + // to_address: event.tx.to || '', + // candidate: '', + // candidate_list: [], + // amount: event.tx.value.toString(), + // duration: 0, + // auto_stake: false, + // } + // return record + // } } export function newEthereumService (opt: EthereumServiceOptions): EthereumService { diff --git a/services/crawler/src/providers/Iotex.ts b/services/crawler/src/providers/Iotex.ts index b2c00e252..da5e454f6 100644 --- a/services/crawler/src/providers/Iotex.ts +++ b/services/crawler/src/providers/Iotex.ts @@ -1,21 +1,9 @@ -import Antenna from 'iotex-antenna' -import { - ClientReadableStream, - IActionInfo, - IGetBlockMetasResponse, - IBlockMeta, - IGetChainMetaResponse, - IGetReceiptByActionResponse, - IGetServerMetaResponse, - IStreamBlocksResponse, -} from 'iotex-antenna/lib/rpc-method/types' -import { from } from '@iotexproject/iotex-address-ts' +import { IStreamBlocksResponse } from 'iotex-antenna/lib/rpc-method/types' import { Opts } from 'iotex-antenna/lib/antenna' import { EventTableColumn } from '@casimir/data' -import {BaseService, Chain} from './Base' -import EventEmitter from 'events' -import {queryAthena} from '@casimir/helpers' -import {ethers} from 'ethers' +import { BaseService, Chain } from './Base' +import { queryAthena } from '@casimir/helpers' +import { ethers } from 'ethers' export enum IotexNetworkType { Mainnet = '4689', @@ -34,168 +22,144 @@ enum IotexActionType { stakeRestake = 'stake_restake', } -export type IotexOptions = Opts +export type IotexOptions = Opts & { + network: IotexNetworkType +} export class IotexService implements BaseService { chain: Chain - // provider: Antenna + network: IotexNetworkType provider: ethers.providers.JsonRpcProvider - eventEmitter: EventEmitter | null = null private readonly chainId: string // todo: switch to ether and point to iotex rpc constructor (opt: IotexOptions) { this.chain = Chain.Iotex + this.network = opt.network || IotexNetworkType.Mainnet + this.chainId = IotexNetworkType.Mainnet this.provider = new ethers.providers.JsonRpcProvider({ - url: 'https://babel-api.mainnet.iotex.io', + url: this.network === IotexNetworkType.Mainnet ? 'https://babel-api.mainnet.iotex.io' : 'https://babel-api.testnet.iotex.io', }) - // this.provider = new Antenna('https://iotexrpc.com', parseInt(IotexNetworkType.Mainnet), { - // signer: opt.signer, - // timeout: opt.timeout, - // apiToken: opt.apiToken - // }) - this.eventEmitter = null - this.chainId = IotexNetworkType.Mainnet } async getChainMetadata (): Promise { - // return await this.provider.getNetwork() + return await this.provider.getNetwork() } - async getBlock(num: number): Promise { + async getBlock(num: number): Promise { return await this.provider.getBlock(num) } - async getLastProcessedEvent(chain: Chain): Promise { - const event = await queryAthena(`SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = '${Chain.Iotex}' ORDER BY height DESC limit 1`) + async getBlockWithTransactions(b: any): Promise { + const block = await this.getBlock(b) - if (event.length === 1) { - return event[0] + if (block.transactions.length !== 0) { + for (const tx of block.transactions) { + const receipt = await this.getTransaction(tx) + block.transactions.push(receipt) + } } - - console.log('More than one event found') - return null - } - - // Note: Transactions in ethereum are called actions in Iotex - async getTransaction(tx: string): Promise { - return await this.provider.getTransaction(tx) + return block } - async getCurrentBlock(): Promise { - const current = await this.getChainMetadata() - return parseInt(current.chainMeta.height) - } + async getLastProcessedEvent(): Promise { + const event = await queryAthena(`SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = '${Chain.Iotex}' ORDER BY height DESC limit 1`) - convertToGlueSchema(obj: { type: string, block: IBlockMeta, action: IActionInfo}): EventTableColumn { - const core = obj.action.action.core - - if (core === undefined) throw new Error('core is undefined') - - const event: EventTableColumn = { - chain: Chain.Iotex, - // 'https://api.iotex.one:443' - network: IotexNetworkType.Mainnet, - provider: 'casimir', - created_at: new Date(obj.block.timestamp.seconds * 1000).toISOString(), - type: '', - address: obj.block.producerAddress, - height: obj.block.height, - to_address: '', - candidate: '', - candidate_list: [], - amount: 0, - duration: 0, - auto_stake: false, - } - switch (obj.type) { - case 'grantReward': - event.type = IotexActionType.grantReward - return event - case 'transfer': - event.type = IotexActionType.transfer - - if (core.transfer?.amount !== undefined) { - event.amount = parseInt(core.transfer?.amount) - } - return event - case 'stakeCreate': - if (core.stakeCreate?.autoStake !== undefined) { - event.auto_stake = core.stakeCreate.autoStake - } - event.type = IotexActionType.createStake - return event - case 'stakeAddDeposit': - event.type = IotexActionType.stakeAddDeposit - if (core.stakeAddDeposit?.amount !== undefined) { - event.amount = parseInt(core.stakeAddDeposit?.amount) - } - return event - case 'execution': - event.type = IotexActionType.execution - if (core.execution?.amount !== undefined) { - event.amount = parseInt(core.execution?.amount) - } - return event - case 'putPollResult': - event.type = IotexActionType.putPollResult - if (core.putPollResult?.candidates !== undefined) { - event.candidate_list = core.putPollResult?.candidates.candidates.map(c => c.address) - } - return event - case 'stakeWithdraw': - event.type = IotexActionType.grantReward - return event - case 'stakeChangeCandidate': - event.type = IotexActionType.StakeChangeCandidate - return event - case 'stakeRestake': - event.type = IotexActionType.grantReward - if (core.stakeRestake?.autoStake !== undefined) { - event.auto_stake = core.stakeRestake.autoStake - } - if (core.stakeRestake?.stakedDuration !== undefined) { - event.duration = core.stakeRestake.stakedDuration - } - return event - default: - throw new Error(`Action type ${obj.type} not supported`) + if (event !== null && event.length === 1) { + return event[0] } + return null } - on(event:string, cb: (block: IStreamBlocksResponse) => void): void { - // const stream = this.provider.iotx.streamBlocks({ start: 0 }) - // - // stream.on('data', (block: IStreamBlocksResponse) => { - // cb(block) - // }) - // - // stream.on('error', (err: any) => { - // throw new Error(err) - // }) + async getTransaction(actionHash: string): Promise { + return await this.provider.getTransaction(actionHash) } - async save(key: string, data: string): Promise { - return + async getCurrentBlock(): Promise { + const height = await this.provider.getBlockNumber() + return await this.provider.getBlock(height) } - // private async getBlocks(start: number, count: number): Promise { - // if (start < 0 || count < 0) { - // throw new Error('start and count must be greater than 0') - // } + // convertToGlueSchema(obj: { type: string, block: ethers.providers.Block, tx: any}): EventTableColumn { + // const core = obj.action.action.core // - // if (start === 0) { - // start = 1 - // } + // if (core === undefined) throw new Error('core is undefined') // - // if (count === 0) { - // count = 100 + // const event: EventTableColumn = { + // chain: Chain.Iotex, + // network: IotexNetworkType.Mainnet, + // provider: 'casimir', + // created_at: new Date(obj.block.timestamp.seconds * 1000).toISOString(), + // type: obj.type, + // address: obj.block.producerAddress, + // height: obj.block.height, + // to_address: '', + // candidate: '', + // candidate_list: [], + // amount: '0', + // duration: 0, + // auto_stake: false, // } + // switch (obj.type) { + // case 'grantReward': + // event.type = IotexActionType.grantReward + // return event + // case 'transfer': + // event.type = IotexActionType.transfer // - // return await this.provider.iotx.getBlockMetas({ byIndex: { start: start, count: count } }) + // if (core.transfer?.amount !== undefined) { + // event.amount = core.transfer?.amount + // } + // return event + // case 'stakeCreate': + // if (core.stakeCreate?.autoStake !== undefined) { + // event.auto_stake = core.stakeCreate.autoStake + // } + // event.type = IotexActionType.createStake + // return event + // case 'stakeAddDeposit': + // event.type = IotexActionType.stakeAddDeposit + // if (core.stakeAddDeposit?.amount !== undefined) { + // event.amount = core.stakeAddDeposit?.amount + // } + // return event + // case 'execution': + // event.type = IotexActionType.execution + // if (core.execution?.amount !== undefined) { + // event.amount = core.execution?.amount + // } + // return event + // case 'putPollResult': + // event.type = IotexActionType.putPollResult + // if (core.putPollResult?.candidates !== undefined) { + // event.candidate_list = core.putPollResult?.candidates.candidates.map(c => c.address) + // } + // return event + // case 'stakeWithdraw': + // event.type = IotexActionType.grantReward + // return event + // case 'stakeChangeCandidate': + // event.type = IotexActionType.StakeChangeCandidate + // return event + // case 'stakeRestake': + // event.type = IotexActionType.grantReward + // if (core.stakeRestake?.autoStake !== undefined) { + // event.auto_stake = core.stakeRestake.autoStake + // } + // if (core.stakeRestake?.stakedDuration !== undefined) { + // event.duration = core.stakeRestake.stakedDuration + // } + // return event + // default: + // throw new Error(`Action type ${obj.type} not supported`) + // } // } + + on(event:string, cb: (block: IStreamBlocksResponse) => void): void { + + } } -export async function newIotexService (opt: IotexOptions): Promise { +export function newIotexService (opt: IotexOptions): IotexService { return new IotexService(opt) } \ No newline at end of file diff --git a/services/crawler/test/crawler.test.ts b/services/crawler/test/crawler.test.ts index 0d91a44f4..63a1c29a3 100644 --- a/services/crawler/test/crawler.test.ts +++ b/services/crawler/test/crawler.test.ts @@ -1,8 +1,7 @@ import { crawler } from '../src/index' import { Chain } from '../src/providers/Base' -import {queryAthena} from '@casimir/helpers' -jest.setTimeout(20000) +jest.setTimeout(1000000) test('init crawler for iotex', async () => { const iotex = await crawler({ @@ -11,55 +10,45 @@ test('init crawler for iotex', async () => { }) expect(iotex.service).not.toBe(null) }) -// -// test('start crawler for iotex', async () => { -// const iotex = await crawler({ -// chain: Chain.Iotex, -// verbose: true -// }) -// -// await iotex.start() -// // expect(iotex.service).not.toBe(null) -// }) test('init crawler for ethereum', async () => { const eth = await crawler({ chain: Chain.Ethereum, verbose: true }) - expect(eth.service).not.toBe(null) }) -test('init crawler for ethereum', async () => { - const queryResult = await queryAthena('SELECT * FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = \'iotex\' limit 10') - expect(queryResult.length).toBe(10) -}) - +test('get last processed event', async () => { + const supercrawler = await crawler({ + chain: Chain.Ethereum, + verbose: true + }) + const lastBlock = await supercrawler.getLastProcessedEvent() + if (!lastBlock) { + throw new Error('last block not found') + } + expect(lastBlock.chain).toBe('ethereum') + expect(lastBlock.height).not.toBe(null) +}) -// test('get last block', async () => { -// const supercrawler = await crawler({ -// chain: Chain.Iotex, +// test('start crawler for ethereum', async () => { +// const eth = await crawler({ +// chain: Chain.Ethereum, // verbose: true // }) -// -// expect(supercrawler.service).not.toBe(null) -// const lastBlock = await supercrawler.retrieveLastBlock() -// expect(typeof lastBlock).toBe('number') +// await eth.start() +// expect(eth.service).not.toBe(null) // }) -// test('stream', async () => { -// const supercrawler = await crawler({ +// test('start crawler for iotex', async () => { +// const iotex = await crawler({ // chain: Chain.Iotex, // verbose: true // }) -// -// expect(supercrawler).not.toBe(null) -// // supercrawler.on('block', (block) => { -// // console.log(block) -// // }) -// }) -// +// await iotex.start() +// expect(iotex.service).not.toBe(null) +// }) \ No newline at end of file From e0cbd5ab547dc9b57a37ca360af85b4fba8caade Mon Sep 17 00:00:00 2001 From: hawyar Date: Tue, 27 Sep 2022 14:35:23 -0400 Subject: [PATCH 5/7] Add rest of governance actions --- services/crawler/src/index.ts | 221 ++++++-------- services/crawler/src/providers/Base.ts | 21 -- services/crawler/src/providers/Ethereum.ts | 139 ++++++--- services/crawler/src/providers/Iotex.ts | 339 +++++++++++++-------- services/crawler/test/crawler.test.ts | 43 +-- 5 files changed, 418 insertions(+), 345 deletions(-) delete mode 100644 services/crawler/src/providers/Base.ts diff --git a/services/crawler/src/index.ts b/services/crawler/src/index.ts index c9f00ab02..bd128e149 100644 --- a/services/crawler/src/index.ts +++ b/services/crawler/src/index.ts @@ -1,156 +1,117 @@ -import { S3Client } from '@aws-sdk/client-s3' import { EventTableColumn } from '@casimir/data' -import { AthenaClient } from '@aws-sdk/client-athena' -import { IotexNetworkType, IotexService, newIotexService } from './providers/Iotex' +import {IotexNetworkType, IotexService, IotexActionType, newIotexService} from './providers/Iotex' import { EthereumService, newEthereumService } from './providers/Ethereum' -import { Chain } from './providers/Base' import { queryAthena, uploadToS3 } from '@casimir/helpers' -import { ethers } from 'ethers' -export const defaultEventBucket = 'casimir-etl-event-bucket-dev' +export enum Chain { + Iotex = 'iotex', + Ethereum = 'ethereum' +} -enum CasimirEventType { - Block = 'block', - Transaction = 'transaction', +export enum Provider { + Casimir = 'casimir', } -export const s3: S3Client | null = null -export const athena: AthenaClient | null = null +export const defaultEventBucket = 'casimir-etl-event-bucket-dev' export interface CrawlerConfig { - chain: Chain - output?: `s3://${string}` - verbose: boolean + chain: Chain + output?: `s3://${string}` + verbose?: boolean } class Crawler { - config: CrawlerConfig - service: IotexService | EthereumService | null - constructor (config: CrawlerConfig) { - this.config = config - this.service = null - } - async prepare (): Promise { - switch (this.config.chain) { - case Chain.Iotex: - this.service = newIotexService({ - network: IotexNetworkType.Mainnet, - }) - break - case Chain.Ethereum: - this.service = await newEthereumService({ url: 'http://localhost:8545' }) - break - default: + config: CrawlerConfig + service: EthereumService | IotexService | null + constructor(config: CrawlerConfig) { + this.config = config + this.service = null + } + + async setup(): Promise { + if (this.config.chain === Chain.Ethereum) { + this.service = await newEthereumService({ url: 'http://localhost:8545'}) + return + } + + if (this.config.chain === Chain.Iotex) { + this.service = await newIotexService({ url: 'https://api.iotex.one:443', network: IotexNetworkType.Mainnet }) + return + } throw new Error('InvalidChain: chain is not supported') - } - } + } - async getLastProcessedEvent(): Promise { - const event = await queryAthena(`SELECT * FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = '${this.config.chain}' ORDER BY height DESC limit 1`) + async getLastProcessedEvent(): Promise { + const event = await queryAthena(`SELECT * FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = '${this.config.chain}' ORDER BY height DESC limit 1`) - if (event !== null && event.length === 1) { - return event[0] + if (event !== null && event.length === 1) { + return event[0] + } + return null } - return null - } - - - async convertToGlueSchema(event: { block: ethers.providers.Block, tx: Record | null }): Promise { - const events: EventTableColumn[] = [] - - events.push({ - chain: this.config.chain, - network: this.service?.network || 'mainnet', - provider: 'casimir', - type: CasimirEventType.Block, - created_at: new Date(event.block.timestamp * 1000).toISOString(), - address: event.block.miner, - height: event.block.number, - to_address: '', - candidate: '', - duration: 0, - candidate_list: [], - amount: '0', - auto_stake: false - }) - - if (event.tx !== null) { - events.push({ - chain: this.config.chain, - network: this.service?.network || 'mainnet', - provider: 'casimir', - type: CasimirEventType.Transaction, - created_at: new Date(event.block.timestamp * 1000).toISOString(), - address: event.tx.from, - height: event.tx.blockNumber, - to_address: event.tx.to, - candidate: '', - duration: 0, - candidate_list: [], - amount: event.tx.value, - auto_stake: false, - }) - } - return events - } - - async start (): Promise { - if (this.service instanceof EthereumService) { - const lastEvent = await this.getLastProcessedEvent() - const current = await this.service.getCurrentBlock() - - const start = lastEvent !== null ? lastEvent.height + 1 : 0 - console.log(`starting from block ${start}`) - - const allEvents: EventTableColumn[] = [] - - for (let i = start; i < current.number; i++) { - const block = await this.service.getBlock(i) - - const ndjson = allEvents.map((e: EventTableColumn) => JSON.stringify(e)).join('\n') - } - } - - if (this.service instanceof IotexService) { - const lastEvent = await this.getLastProcessedEvent() - - const current = await this.service.getCurrentBlock() - - const start = lastEvent !== null ? lastEvent.height + 1 : 0 - - console.log(`starting from block: ${start}`) - - const allEvents: EventTableColumn[] = [] - - for (let i = start; i < current.number; i++) { - const block = await this.service.getBlockWithTransactions(i) - - if (i % 3000 === 0 && i !== 0) { - const ndjson = allEvents.map((e: EventTableColumn) => JSON.stringify(e)).join('\n') - await uploadToS3({ - bucket: defaultEventBucket, - key: `${block.hash}-event.json`, - data: ndjson - }).finally(() => { - console.log('uploaded events batch to s3') - }) - allEvents.length = 0 + + async start(): Promise { + if (this.service instanceof EthereumService) { + const lastEvent = await this.getLastProcessedEvent() + const current = await this.service.getCurrentBlock() + + const last = lastEvent !== null ? lastEvent.height : 0 + const start = parseInt(last.toString()+ 1) + + for (let i = start as number; i < current.number; i++) { + const { events, blockHash } = await this.service.getEvents(30021005 + i) + const ndjson = events.map((e: EventTableColumn) => JSON.stringify(e)).join('\n') + // await uploadToS3({ + // bucket: defaultEventBucket, + // key: `${blockHash}-event.json`, + // data: ndjson + // }).finally(() => { + // console.log(`block: ${i}, num of tx: ${events.length}`) + // }) + } + return + } + + if (this.service instanceof IotexService) { + const lastEvent = await this.getLastProcessedEvent() + const current = await this.service.getCurrentHeight() + + const start = lastEvent !== null ? lastEvent.height + 1 : 0 + console.log(`starting from block: ${start}`) + + for (let i = start; i < current; i++) { + const events = await this.service.getEvents(10000004 + i) + + console.log(events.length) + // await uploadToS3({ + // bucket: defaultEventBucket, + // key: `${block.hash}-event.json`, + // data: ndjson + // }).finally(() => { + // console.log('uploaded events batch to s3') + // }) } - console.log(block) + return } - return } - throw new Error('ServiceNotSupported: service is not recognized') - } } export async function crawler (config: CrawlerConfig): Promise { - const c = new Crawler({ - chain: config?.chain ?? Chain.Iotex, + const chainCrawler = new Crawler({ + chain: config.chain, output: config?.output ?? `s3://${defaultEventBucket}`, verbose: config?.verbose ?? false }) - await c.prepare() - return c -} \ No newline at end of file + await chainCrawler.setup() + return chainCrawler +} + +async function ee() { + const iotex = await crawler({ + chain: Chain.Iotex, + verbose: true, + }) + await iotex.start() +} +ee() \ No newline at end of file diff --git a/services/crawler/src/providers/Base.ts b/services/crawler/src/providers/Base.ts deleted file mode 100644 index c067e50f1..000000000 --- a/services/crawler/src/providers/Base.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { EventTableColumn } from '@casimir/data' -import { ethers } from 'ethers' -import Antenna from 'iotex-antenna' - -export enum Chain { - Iotex = 'iotex', - Ethereum = 'ethereum' -} - -export interface BaseService { - chain: Chain - network: string - provider: ethers.providers.JsonRpcProvider | Antenna - getChainMetadata(): Promise - getBlock(num: number): Promise - getCurrentBlock(): Promise - getTransaction(tx: string): Promise - getLastProcessedEvent(): Promise - // convertToGlueSchema(obj: Record): EventTableColumn - on(event: string, cb: (block: any) => void): void -} diff --git a/services/crawler/src/providers/Ethereum.ts b/services/crawler/src/providers/Ethereum.ts index 86c98e765..8163387fd 100644 --- a/services/crawler/src/providers/Ethereum.ts +++ b/services/crawler/src/providers/Ethereum.ts @@ -1,37 +1,83 @@ import { ethers } from 'ethers' import { EventTableColumn } from '@casimir/data' -import { BaseService, Chain } from './Base' -import { queryAthena, uploadToS3 } from '@casimir/helpers' +import {Chain, Provider} from '../index' +import { queryAthena } from '@casimir/helpers' -export type EthereumServiceOptions = Record - -export enum EthereumNetworkType { - Mainnet = 'mainnet', - Testnet = 'testnet' +export type EthereumServiceOptions = { + url: string + network?: string } -export class EthereumService implements BaseService { +export class EthereumService { chain: Chain network: string provider: ethers.providers.JsonRpcProvider - constructor(opt: EthereumServiceOptions) { this.chain = Chain.Ethereum - this.network = opt.network || EthereumNetworkType.Testnet + this.network = opt.network || 'mainnet' this.provider = new ethers.providers.JsonRpcProvider({ url: opt.url || 'http://localhost:8545', }) } - async getChainMetadata() { - const meta = await this.provider.getNetwork() - return meta + async getEvents(height: number): Promise<{ blockHash: string, events: EventTableColumn[] }> { + const events: EventTableColumn[] = [] + + const block = await this.provider.getBlockWithTransactions(height) + + events.push({ + chain: this.chain, + network: this.network, + provider: Provider.Casimir, + type: 'block', + created_at: new Date(block.timestamp * 1000).toISOString(), + address: block.miner, + height: block.number, + to_address: '', + candidate: '', + duration: 0, + candidate_list: [], + amount: '0', + auto_stake: false, + }) + + if (block.transactions.length > 0) { + for (const tx of block.transactions) { + events.push({ + chain: this.chain, + network: this.network, + provider: Provider.Casimir, + type: 'transaction', + created_at: new Date(block.timestamp * 1000).toISOString(), + address: tx.from, + height: block.number, + to_address: tx.to || '', + candidate: '', + candidate_list: [], + duration: 0, + amount: tx.value.toString(), + auto_stake: false, + }) + } + } + return { + blockHash: block.hash, + events, + } + } + async getCurrentBlock(): Promise { + const height = await this.provider.getBlockNumber() + return await this.provider.getBlock(height) } async getBlock(num: number): Promise { return await this.provider.getBlockWithTransactions(num) } + async getTransaction(tx: string): Promise { + return await this.provider.getTransaction(tx) + } + async getLastProcessedEvent(): Promise { const event = await queryAthena(`SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = '${Chain.Ethereum}' ORDER BY height DESC limit 1`) @@ -40,13 +86,7 @@ export class EthereumService implements BaseService { } return null } - - async getTransaction(tx: string): Promise { - const txData = await this.provider.getTransaction(tx) - return txData - } - // todo: better interface and cleanup on(event:string, cb: (block: ethers.providers.Block) => void): void { this.provider.on('block', async (blockNumber: number) => { const block = await this.getBlock(blockNumber) @@ -54,29 +94,46 @@ export class EthereumService implements BaseService { }) } - async getCurrentBlock(): Promise { - const height = await this.provider.getBlockNumber() - return await this.provider.getBlock(height) - } + convertToGlueSchema(raw: { block: any, tx: ethers.providers.TransactionResponse[] | null }): EventTableColumn[] { + const events: EventTableColumn[] = [] + + events.push({ + chain: this.chain, + network: this.network, + provider: 'casimir', + type: 'block', + created_at: new Date(raw.block.timestamp * 1000).toISOString(), + address: raw.block.miner, + height: raw.block.number, + to_address: '', + candidate: '', + duration: 0, + candidate_list: [], + amount: '0', + auto_stake: false + }) - // convertToGlueSchema(event: { type: string, block: ethers.providers.Block, tx: ethers.providers.TransactionResponse }): EventTableColumn { - // const record: EventTableColumn = { - // chain: Chain.Ethereum, - // network: this.network, - // provider: 'casimir', - // type: event.type, - // created_at: new Date(event.block.timestamp * 1000).toISOString(), - // address: event.tx.from, - // height: event.block.number, - // to_address: event.tx.to || '', - // candidate: '', - // candidate_list: [], - // amount: event.tx.value.toString(), - // duration: 0, - // auto_stake: false, - // } - // return record - // } + if (raw.tx !== null && raw.tx.length > 0) { + for (const tx of raw.tx) { + events.push({ + chain: this.chain, + network: this.network, + provider: 'casimir', + type: 'transaction', + created_at: new Date(raw.block.timestamp * 1000).toISOString(), + address: tx.from, + height: raw.block.number, + to_address: tx.to || '', + candidate: '', + candidate_list: [], + duration: 0, + amount: tx.value.toString(), + auto_stake: false + }) + } + } + return events + } } export function newEthereumService (opt: EthereumServiceOptions): EthereumService { diff --git a/services/crawler/src/providers/Iotex.ts b/services/crawler/src/providers/Iotex.ts index da5e454f6..0d7a60dbd 100644 --- a/services/crawler/src/providers/Iotex.ts +++ b/services/crawler/src/providers/Iotex.ts @@ -1,165 +1,262 @@ -import { IStreamBlocksResponse } from 'iotex-antenna/lib/rpc-method/types' + +import Antenna from 'iotex-antenna' +import { + ClientReadableStream, + IActionInfo, + IGetBlockMetasResponse, + IStreamBlocksResponse, +} from 'iotex-antenna/lib/rpc-method/types' import { Opts } from 'iotex-antenna/lib/antenna' import { EventTableColumn } from '@casimir/data' -import { BaseService, Chain } from './Base' -import { queryAthena } from '@casimir/helpers' -import { ethers } from 'ethers' +import {Chain, Provider} from '../index' export enum IotexNetworkType { - Mainnet = '4689', - Testnet = '4690' + Mainnet = 'mainnet', + Testnet = 'testnet', } -enum IotexActionType { +export enum IotexActionType { + grantReward = 'grantReward', + claimFromRewardingFund = 'claimFromRewardingFund', + depositToRewardingFund = 'depositToRewardingFund', + candidateRegister = 'candidateRegister', + candidateUpdate = 'candidateUpdate', + stakeCreate = 'stakeCreate', + stakeRestake = 'stakeRestake', + stakeAddDeposit = 'stakeAddDeposit', transfer = 'transfer', - grantReward = 'grant_reward', - createStake = 'create_stake', - stakeAddDeposit = 'stake_add_deposit', + stakeUnstake = 'stakeUnstake', + stakeWithdraw = 'stakeWithdraw', + + // non governance actions execution = 'execution', - putPollResult = 'put_poll_result', - stakeWithdraw = 'stake_withdraw', - StakeChangeCandidate = 'stake_change_candidate', - stakeRestake = 'stake_restake', + putPollResult = 'putPollResult', + StakeChangeCandidate = 'stakeChangeCandidate', } export type IotexOptions = Opts & { + url: string network: IotexNetworkType } -export class IotexService implements BaseService { +export class IotexService { chain: Chain network: IotexNetworkType - provider: ethers.providers.JsonRpcProvider - - private readonly chainId: string - // todo: switch to ether and point to iotex rpc + provider: Antenna + chainId: number constructor (opt: IotexOptions) { this.chain = Chain.Iotex this.network = opt.network || IotexNetworkType.Mainnet - this.chainId = IotexNetworkType.Mainnet - this.provider = new ethers.providers.JsonRpcProvider({ - url: this.network === IotexNetworkType.Mainnet ? 'https://babel-api.mainnet.iotex.io' : 'https://babel-api.testnet.iotex.io', + this.chainId = IotexNetworkType.Mainnet ? 4689 : 4690 + this.provider = new Antenna(opt.url, this.chainId, { + signer: opt.signer, + timeout: opt.timeout, + apiToken: opt.apiToken }) } - async getChainMetadata (): Promise { - return await this.provider.getNetwork() - } + deduceActionType (action: IActionInfo): IotexActionType | null { + const core = action.action.core + if (core === undefined) return null - async getBlock(num: number): Promise { - return await this.provider.getBlock(num) + const type = Object.keys(core).filter(k => k !== undefined)[Object.keys(core).length - 2] + return type as IotexActionType } - async getBlockWithTransactions(b: any): Promise { - const block = await this.getBlock(b) + async getEvents(height: number): Promise { + const events: EventTableColumn[] = [] + + const block = await this.provider.iotx.getBlockMetas({byIndex: {start: height, count: 1}}) + + events.push({ + chain: this.chain, + network: this.network, + provider: Provider.Casimir, + type: 'block', + created_at: block.blkMetas[0].timestamp.toString(), + address: block.blkMetas[0].producerAddress, + height: block.blkMetas[0].height, + to_address: '', + candidate: '', + duration: 0, + candidate_list: [], + amount: '0', + auto_stake: false, + }) + + const numOfAction = block.blkMetas[0].numActions + + console.log(block.blkMetas.length) + + if (numOfAction > 0) { + const actions = await this.getBlockActions(height, numOfAction) + + for (const action of actions) { + const actionCore = action.action.core + + if (actionCore === undefined) continue + + const actionEvent: Partial = {} + const actionType = this.deduceActionType(action) + + // console.log('numOfAction: ', numOfAction) + // console.log('type: ', actionType) + // console.log('-----') + + if (actionType === null) continue + actionEvent.type = actionType + + // if (actionType === IotexActionType.grantReward) {} + // if (actionType === IotexActionType.stakeUnstake) {} + // if (actionType === IotexActionType.stakeWithdraw) {} + + if (actionType === IotexActionType.transfer && actionCore.transfer) { + actionEvent.amount = actionCore.transfer.amount + actionEvent.to_address = actionCore.transfer.recipient + events.push(actionEvent as EventTableColumn) + return events + } + + if (actionType === IotexActionType.stakeCreate && actionCore.stakeCreate) { + actionEvent.amount = actionCore.stakeCreate.stakedAmount + actionEvent.candidate = actionCore.stakeCreate.candidateName + actionEvent.auto_stake = actionCore.stakeCreate.autoStake + actionEvent.duration = actionCore.stakeCreate.stakedDuration + events.push(actionEvent as EventTableColumn) + return events + } + + if (actionType === IotexActionType.stakeAddDeposit && actionCore.stakeAddDeposit) { + actionEvent.amount = actionCore.stakeAddDeposit.amount + events.push(actionEvent as EventTableColumn) + return events + } + + if (actionType === IotexActionType.execution && actionCore.execution) { + actionEvent.amount = actionCore.execution.amount + events.push(actionEvent as EventTableColumn) + return events + } + + if (actionType === IotexActionType.putPollResult && actionCore.putPollResult) { + if (actionCore.putPollResult.candidates) { + actionEvent.candidate_list = actionCore.putPollResult.candidates.candidates.map(c => c.address) + } + + if (actionCore.putPollResult.height) { + actionEvent.height = typeof actionCore.putPollResult.height === 'string' ? parseInt(actionCore.putPollResult.height) : actionCore.putPollResult.height + } + events.push(actionEvent as EventTableColumn) + return events + } + + if (actionType === IotexActionType.StakeChangeCandidate && actionCore.stakeChangeCandidate) { + actionEvent.candidate = actionCore.stakeChangeCandidate.candidateName + events.push(actionEvent as EventTableColumn) + return events + } + + if (actionType === IotexActionType.stakeRestake && actionCore.stakeRestake) { + actionEvent.duration = actionCore.stakeRestake.stakedDuration + actionEvent.auto_stake = actionCore.stakeRestake.autoStake + events.push(actionEvent as EventTableColumn) + return events + } - if (block.transactions.length !== 0) { - for (const tx of block.transactions) { - const receipt = await this.getTransaction(tx) - block.transactions.push(receipt) + if (actionType === IotexActionType.candidateRegister && actionCore.candidateRegister) { + actionEvent.amount = actionCore.candidateRegister.stakedAmount + actionEvent.duration = actionCore.candidateRegister.stakedDuration + actionEvent.auto_stake = actionCore.candidateRegister.autoStake + actionEvent.candidate = actionCore.candidateRegister.candidate.name + events.push(actionEvent as EventTableColumn) + return events + } + + if (actionType === IotexActionType.candidateUpdate && actionCore.candidateUpdate) { + actionEvent.candidate = actionCore.candidateUpdate.name + events.push(actionEvent as EventTableColumn) + return events + } + + if (actionType === IotexActionType.claimFromRewardingFund && actionCore.claimFromRewardingFund) { + actionEvent.amount = actionCore.claimFromRewardingFund.amount + return events + } + + if (actionType === IotexActionType.depositToRewardingFund && actionCore.depositToRewardingFund) { + actionEvent.amount = actionCore.depositToRewardingFund.amount + events.push(actionEvent as EventTableColumn) + return events + } } } - return block + return events } - async getLastProcessedEvent(): Promise { - const event = await queryAthena(`SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = '${Chain.Iotex}' ORDER BY height DESC limit 1`) + async getBlocks(start: number, count: number): Promise { + if (start < 0 || count < 0) { + throw new Error('start and count must be greater than 0') + } + + if (start === 0) { + start = 1 + } - if (event !== null && event.length === 1) { - return event[0] + if (count === 0) { + count = 100 } - return null + + const blocks = await this.provider.iotx.getBlockMetas({ byIndex: { start: start, count: count } }) + + return blocks } - async getTransaction(actionHash: string): Promise { - return await this.provider.getTransaction(actionHash) + async getTransaction(tx: string): Promise { + const receipt = await this.provider.iotx.getReceiptByAction({ actionHash: tx }) + return receipt } - async getCurrentBlock(): Promise { - const height = await this.provider.getBlockNumber() - return await this.provider.getBlock(height) + async getBlockActions (index: number, count: number): Promise { + const actions = await this.provider.iotx.getActions({ + byIndex: { + start: index, + count: count + } + }) + return actions.actionInfo } - // convertToGlueSchema(obj: { type: string, block: ethers.providers.Block, tx: any}): EventTableColumn { - // const core = obj.action.action.core - // - // if (core === undefined) throw new Error('core is undefined') - // - // const event: EventTableColumn = { - // chain: Chain.Iotex, - // network: IotexNetworkType.Mainnet, - // provider: 'casimir', - // created_at: new Date(obj.block.timestamp.seconds * 1000).toISOString(), - // type: obj.type, - // address: obj.block.producerAddress, - // height: obj.block.height, - // to_address: '', - // candidate: '', - // candidate_list: [], - // amount: '0', - // duration: 0, - // auto_stake: false, - // } - // switch (obj.type) { - // case 'grantReward': - // event.type = IotexActionType.grantReward - // return event - // case 'transfer': - // event.type = IotexActionType.transfer - // - // if (core.transfer?.amount !== undefined) { - // event.amount = core.transfer?.amount - // } - // return event - // case 'stakeCreate': - // if (core.stakeCreate?.autoStake !== undefined) { - // event.auto_stake = core.stakeCreate.autoStake - // } - // event.type = IotexActionType.createStake - // return event - // case 'stakeAddDeposit': - // event.type = IotexActionType.stakeAddDeposit - // if (core.stakeAddDeposit?.amount !== undefined) { - // event.amount = core.stakeAddDeposit?.amount - // } - // return event - // case 'execution': - // event.type = IotexActionType.execution - // if (core.execution?.amount !== undefined) { - // event.amount = core.execution?.amount - // } - // return event - // case 'putPollResult': - // event.type = IotexActionType.putPollResult - // if (core.putPollResult?.candidates !== undefined) { - // event.candidate_list = core.putPollResult?.candidates.candidates.map(c => c.address) - // } - // return event - // case 'stakeWithdraw': - // event.type = IotexActionType.grantReward - // return event - // case 'stakeChangeCandidate': - // event.type = IotexActionType.StakeChangeCandidate - // return event - // case 'stakeRestake': - // event.type = IotexActionType.grantReward - // if (core.stakeRestake?.autoStake !== undefined) { - // event.auto_stake = core.stakeRestake.autoStake - // } - // if (core.stakeRestake?.stakedDuration !== undefined) { - // event.duration = core.stakeRestake.stakedDuration - // } - // return event - // default: - // throw new Error(`Action type ${obj.type} not supported`) - // } - // } - - on(event:string, cb: (block: IStreamBlocksResponse) => void): void { + async getCurrentHeight(): Promise { + const { chainMeta } = await this.provider.iotx.getChainMeta({ + includePendingActions: false + }) + return parseInt(chainMeta.height) + } + + async getCurrentBlock(): Promise { + const { chainMeta } = await this.provider.iotx.getChainMeta({ + includePendingActions: false + }) + const block = await this.provider.iotx.getBlockMetas({ byIndex: { start: parseInt(chainMeta.height), count: 1 } }) + return block } -} + async readableBlockStream (): Promise> { + const stream = await this.provider.iotx.streamBlocks({ + start: 1 + }) + return stream + } + + on(event: string, callback: (data: IStreamBlocksResponse) => void): void { + this.provider.iotx.streamBlocks({ + start: 1 + }).on('data', (data: IStreamBlocksResponse) => { + callback(data) + }) + } +} export function newIotexService (opt: IotexOptions): IotexService { return new IotexService(opt) } \ No newline at end of file diff --git a/services/crawler/test/crawler.test.ts b/services/crawler/test/crawler.test.ts index 63a1c29a3..0e9731006 100644 --- a/services/crawler/test/crawler.test.ts +++ b/services/crawler/test/crawler.test.ts @@ -1,7 +1,5 @@ import { crawler } from '../src/index' -import { Chain } from '../src/providers/Base' - -jest.setTimeout(1000000) +import { Chain } from '../src/index' test('init crawler for iotex', async () => { const iotex = await crawler({ @@ -19,36 +17,17 @@ test('init crawler for ethereum', async () => { expect(eth.service).not.toBe(null) }) - -test('get last processed event', async () => { - const supercrawler = await crawler({ - chain: Chain.Ethereum, - verbose: true - }) - - const lastBlock = await supercrawler.getLastProcessedEvent() - - if (!lastBlock) { - throw new Error('last block not found') - } - expect(lastBlock.chain).toBe('ethereum') - expect(lastBlock.height).not.toBe(null) -}) - -// test('start crawler for ethereum', async () => { -// const eth = await crawler({ +// test('get last processed event', async () => { +// const supercrawler = await crawler({ // chain: Chain.Ethereum, // verbose: true // }) -// await eth.start() -// expect(eth.service).not.toBe(null) +// +// const lastBlock = await supercrawler.getLastProcessedEvent() +// +// if (!lastBlock) { +// throw new Error('last block not found') +// } +// expect(lastBlock.chain).toBe('ethereum') +// expect(lastBlock.height).not.toBe(null) // }) - -// test('start crawler for iotex', async () => { -// const iotex = await crawler({ -// chain: Chain.Iotex, -// verbose: true -// }) -// await iotex.start() -// expect(iotex.service).not.toBe(null) -// }) \ No newline at end of file From 3a9bef394f93737f0051055daebc50b11a0311b9 Mon Sep 17 00:00:00 2001 From: hawyar Date: Tue, 27 Sep 2022 21:38:18 -0400 Subject: [PATCH 6/7] Update schema description and remove unused funcs --- common/data/src/schemas/event.schema.json | 12 +-- common/helpers/src/index.ts | 13 +++- services/crawler/src/index.ts | 78 ++++++++++--------- services/crawler/src/providers/Ethereum.ts | 53 +------------ services/crawler/src/providers/Iotex.ts | 88 ++++++++++------------ services/crawler/test/crawler.test.ts | 32 ++++---- 6 files changed, 115 insertions(+), 161 deletions(-) diff --git a/common/data/src/schemas/event.schema.json b/common/data/src/schemas/event.schema.json index 6e96abd92..26095076c 100644 --- a/common/data/src/schemas/event.schema.json +++ b/common/data/src/schemas/event.schema.json @@ -22,19 +22,19 @@ }, "height": { "type": "integer", - "description": "The block height of the event" + "description": "The height of the event" }, "created_at": { "type": "string", - "description": "The datestring of the event" + "description": "The date and time of the event in ISO 8601 format e.g. 2015-03-04T22:44:30.652Z" }, "address": { "type": "string", - "description": "The address that initiated the event" + "description": "The address which initiated the event" }, "to_address": { "type": "string", - "description": "The address which received the action event" + "description": "The recipient's address" }, "candidate": { "type": "string", @@ -46,11 +46,11 @@ }, "amount": { "type": "string", - "description": "The amount of the currency in the event" + "description": "The amount of currency associated with the event" }, "duration":{ "type": "string", - "description": "The duration of the action" + "description": "The duration of the event" }, "auto_stake": { "type": "boolean", diff --git a/common/helpers/src/index.ts b/common/helpers/src/index.ts index 31d83c0de..ff3b351b7 100644 --- a/common/helpers/src/index.ts +++ b/common/helpers/src/index.ts @@ -3,8 +3,6 @@ import { AthenaClient, AthenaClientConfig } from '@aws-sdk/client-athena' import { defaultProvider } from '@aws-sdk/credential-provider-node' import { StartQueryExecutionCommand, GetQueryExecutionCommand } from '@aws-sdk/client-athena' import { EventTableColumn } from '@casimir/data' -import {Chain} from '@casimir/crawler/src/providers/Base' -import {IotexNetworkType} from '@casimir/crawler/src/providers/Iotex' /** * Converts any string to PascalCase. @@ -135,7 +133,7 @@ export async function getFromS3(bucket: string, key: string): Promise { } let retry = 0 -let backoff = 300 +let backoff = 500 /** * Poll for Athena query's result @@ -168,7 +166,14 @@ async function pollAthenaQueryOutput(queryId: string): Promise { }, backoff) } - if (QueryExecution.Status.State === 'FAILED') throw new Error('QueryFailed: query failed') + if (QueryExecution.Status.State === 'FAILED') { + const reason = QueryExecution.Status.StateChangeReason + if (reason && reason.includes('HIVE_BAD_DATA')) { + throw new Error('FailedQuery: Check the table for bad data') + } else { + throw new Error('QueryFailed: query failed') + } + } if (QueryExecution.Status.State === 'SUCCEEDED') return } diff --git a/services/crawler/src/index.ts b/services/crawler/src/index.ts index bd128e149..77a715215 100644 --- a/services/crawler/src/index.ts +++ b/services/crawler/src/index.ts @@ -1,5 +1,5 @@ import { EventTableColumn } from '@casimir/data' -import {IotexNetworkType, IotexService, IotexActionType, newIotexService} from './providers/Iotex' +import {IotexNetworkType, IotexService, newIotexService} from './providers/Iotex' import { EthereumService, newEthereumService } from './providers/Ethereum' import { queryAthena, uploadToS3 } from '@casimir/helpers' @@ -53,43 +53,58 @@ class Crawler { async start(): Promise { if (this.service instanceof EthereumService) { const lastEvent = await this.getLastProcessedEvent() - const current = await this.service.getCurrentBlock() const last = lastEvent !== null ? lastEvent.height : 0 - const start = parseInt(last.toString()+ 1) + const start = parseInt(last.toString()) + 1 + + if (this.config.verbose) { + console.log(`crawling ${this.config.chain} from block ${start}`) + } + + const current = await this.service.getCurrentBlock() for (let i = start as number; i < current.number; i++) { - const { events, blockHash } = await this.service.getEvents(30021005 + i) + const { events, blockHash } = await this.service.getEvents(i) const ndjson = events.map((e: EventTableColumn) => JSON.stringify(e)).join('\n') - // await uploadToS3({ - // bucket: defaultEventBucket, - // key: `${blockHash}-event.json`, - // data: ndjson - // }).finally(() => { - // console.log(`block: ${i}, num of tx: ${events.length}`) - // }) + await uploadToS3({ + bucket: defaultEventBucket, + key: `${blockHash}-event.json`, + data: ndjson + }).finally(() => { + if (this.config.verbose) { + console.log(`uploaded ${events.length} event at height ${i}`) + } + }) } return } if (this.service instanceof IotexService) { const lastEvent = await this.getLastProcessedEvent() - const current = await this.service.getCurrentHeight() - - const start = lastEvent !== null ? lastEvent.height + 1 : 0 - console.log(`starting from block: ${start}`) - - for (let i = start; i < current; i++) { - const events = await this.service.getEvents(10000004 + i) - - console.log(events.length) - // await uploadToS3({ - // bucket: defaultEventBucket, - // key: `${block.hash}-event.json`, - // data: ndjson - // }).finally(() => { - // console.log('uploaded events batch to s3') - // }) + + const currentBlock = await this.service.getCurrentBlock() + const currentHeight = currentBlock.blkMetas[0].height + + const last = lastEvent !== null ? lastEvent.height : 0 + const start = parseInt(last.toString()) + 1 + + if (this.config.verbose) { + console.log(`crawling ${this.config.chain} from block ${start}`) + } + + for (let i = start; i < currentHeight; i++) { + const { hash, events } = await this.service.getEvents(i) + const ndjson = events.map((e: EventTableColumn) => JSON.stringify(e)).join('\n') + + await uploadToS3({ + bucket: defaultEventBucket, + key: `${hash}-event.json`, + data: ndjson + }).finally(() => { + if (this.config.verbose) { + console.log(`uploaded ${events.length} event at height ${i}`) + } + }) } return } @@ -106,12 +121,3 @@ export async function crawler (config: CrawlerConfig): Promise { await chainCrawler.setup() return chainCrawler } - -async function ee() { - const iotex = await crawler({ - chain: Chain.Iotex, - verbose: true, - }) - await iotex.start() -} -ee() \ No newline at end of file diff --git a/services/crawler/src/providers/Ethereum.ts b/services/crawler/src/providers/Ethereum.ts index 8163387fd..c82e3541a 100644 --- a/services/crawler/src/providers/Ethereum.ts +++ b/services/crawler/src/providers/Ethereum.ts @@ -1,7 +1,6 @@ import { ethers } from 'ethers' import { EventTableColumn } from '@casimir/data' import {Chain, Provider} from '../index' -import { queryAthena } from '@casimir/helpers' export type EthereumServiceOptions = { url: string @@ -47,7 +46,7 @@ export class EthereumService { chain: this.chain, network: this.network, provider: Provider.Casimir, - type: 'transaction', + type: tx.type === 0 ? 'transfer' : 'contract', created_at: new Date(block.timestamp * 1000).toISOString(), address: tx.from, height: block.number, @@ -78,62 +77,12 @@ export class EthereumService { return await this.provider.getTransaction(tx) } - async getLastProcessedEvent(): Promise { - const event = await queryAthena(`SELECT height FROM "casimir_etl_database_dev"."casimir_etl_event_table_dev" where chain = '${Chain.Ethereum}' ORDER BY height DESC limit 1`) - - if (event !== null && event.length === 1) { - return event[0] - } - return null - } - on(event:string, cb: (block: ethers.providers.Block) => void): void { this.provider.on('block', async (blockNumber: number) => { const block = await this.getBlock(blockNumber) cb(block) }) } - - convertToGlueSchema(raw: { block: any, tx: ethers.providers.TransactionResponse[] | null }): EventTableColumn[] { - const events: EventTableColumn[] = [] - - events.push({ - chain: this.chain, - network: this.network, - provider: 'casimir', - type: 'block', - created_at: new Date(raw.block.timestamp * 1000).toISOString(), - address: raw.block.miner, - height: raw.block.number, - to_address: '', - candidate: '', - duration: 0, - candidate_list: [], - amount: '0', - auto_stake: false - }) - - if (raw.tx !== null && raw.tx.length > 0) { - for (const tx of raw.tx) { - events.push({ - chain: this.chain, - network: this.network, - provider: 'casimir', - type: 'transaction', - created_at: new Date(raw.block.timestamp * 1000).toISOString(), - address: tx.from, - height: raw.block.number, - to_address: tx.to || '', - candidate: '', - candidate_list: [], - duration: 0, - amount: tx.value.toString(), - auto_stake: false - }) - } - } - return events - } } export function newEthereumService (opt: EthereumServiceOptions): EthereumService { diff --git a/services/crawler/src/providers/Iotex.ts b/services/crawler/src/providers/Iotex.ts index 0d7a60dbd..70ef441e1 100644 --- a/services/crawler/src/providers/Iotex.ts +++ b/services/crawler/src/providers/Iotex.ts @@ -63,19 +63,21 @@ export class IotexService { return type as IotexActionType } - async getEvents(height: number): Promise { + async getEvents(height: number): Promise<{ hash: string, events: EventTableColumn[]}> { const events: EventTableColumn[] = [] const block = await this.provider.iotx.getBlockMetas({byIndex: {start: height, count: 1}}) + const blockMeta = block.blkMetas[0] + events.push({ chain: this.chain, network: this.network, provider: Provider.Casimir, type: 'block', - created_at: block.blkMetas[0].timestamp.toString(), - address: block.blkMetas[0].producerAddress, - height: block.blkMetas[0].height, + created_at: new Date(block.blkMetas[0].timestamp.seconds * 1000).toISOString(), + address: blockMeta.producerAddress, + height: blockMeta.height, to_address: '', candidate: '', duration: 0, @@ -84,37 +86,38 @@ export class IotexService { auto_stake: false, }) - const numOfAction = block.blkMetas[0].numActions - - console.log(block.blkMetas.length) + const numOfActions = block.blkMetas[0].numActions - if (numOfAction > 0) { - const actions = await this.getBlockActions(height, numOfAction) + if (numOfActions > 0) { + const actions = await this.getBlockActions(height, numOfActions) - for (const action of actions) { + const blockActions = actions.map((action) => { const actionCore = action.action.core + if (actionCore === undefined) return - if (actionCore === undefined) continue - - const actionEvent: Partial = {} const actionType = this.deduceActionType(action) - - // console.log('numOfAction: ', numOfAction) - // console.log('type: ', actionType) - // console.log('-----') - - if (actionType === null) continue - actionEvent.type = actionType - - // if (actionType === IotexActionType.grantReward) {} - // if (actionType === IotexActionType.stakeUnstake) {} - // if (actionType === IotexActionType.stakeWithdraw) {} + if (actionType === null) return + + const actionEvent: Partial = { + chain: this.chain, + network: this.network, + provider: Provider.Casimir, + type: actionType, + created_at: new Date(action.timestamp.seconds * 1000).toISOString(), + address: blockMeta.producerAddress, + height: blockMeta.height, + to_address: '', + candidate: '', + duration: 0, + candidate_list: [], + amount: '0', + auto_stake: false, + } if (actionType === IotexActionType.transfer && actionCore.transfer) { actionEvent.amount = actionCore.transfer.amount actionEvent.to_address = actionCore.transfer.recipient events.push(actionEvent as EventTableColumn) - return events } if (actionType === IotexActionType.stakeCreate && actionCore.stakeCreate) { @@ -123,19 +126,16 @@ export class IotexService { actionEvent.auto_stake = actionCore.stakeCreate.autoStake actionEvent.duration = actionCore.stakeCreate.stakedDuration events.push(actionEvent as EventTableColumn) - return events } if (actionType === IotexActionType.stakeAddDeposit && actionCore.stakeAddDeposit) { actionEvent.amount = actionCore.stakeAddDeposit.amount events.push(actionEvent as EventTableColumn) - return events } if (actionType === IotexActionType.execution && actionCore.execution) { actionEvent.amount = actionCore.execution.amount events.push(actionEvent as EventTableColumn) - return events } if (actionType === IotexActionType.putPollResult && actionCore.putPollResult) { @@ -147,20 +147,17 @@ export class IotexService { actionEvent.height = typeof actionCore.putPollResult.height === 'string' ? parseInt(actionCore.putPollResult.height) : actionCore.putPollResult.height } events.push(actionEvent as EventTableColumn) - return events } if (actionType === IotexActionType.StakeChangeCandidate && actionCore.stakeChangeCandidate) { actionEvent.candidate = actionCore.stakeChangeCandidate.candidateName events.push(actionEvent as EventTableColumn) - return events } if (actionType === IotexActionType.stakeRestake && actionCore.stakeRestake) { actionEvent.duration = actionCore.stakeRestake.stakedDuration actionEvent.auto_stake = actionCore.stakeRestake.autoStake events.push(actionEvent as EventTableColumn) - return events } if (actionType === IotexActionType.candidateRegister && actionCore.candidateRegister) { @@ -169,28 +166,33 @@ export class IotexService { actionEvent.auto_stake = actionCore.candidateRegister.autoStake actionEvent.candidate = actionCore.candidateRegister.candidate.name events.push(actionEvent as EventTableColumn) - return events } if (actionType === IotexActionType.candidateUpdate && actionCore.candidateUpdate) { actionEvent.candidate = actionCore.candidateUpdate.name events.push(actionEvent as EventTableColumn) - return events } if (actionType === IotexActionType.claimFromRewardingFund && actionCore.claimFromRewardingFund) { actionEvent.amount = actionCore.claimFromRewardingFund.amount - return events } if (actionType === IotexActionType.depositToRewardingFund && actionCore.depositToRewardingFund) { actionEvent.amount = actionCore.depositToRewardingFund.amount events.push(actionEvent as EventTableColumn) - return events } - } + + // if (actionType === IotexActionType.grantReward) {} + // if (actionType === IotexActionType.stakeUnstake) {} + // if (actionType === IotexActionType.stakeWithdraw) {} + return actionEvent + }) + events.push(...blockActions as EventTableColumn[]) + } + return { + hash: blockMeta.hash, + events } - return events } async getBlocks(start: number, count: number): Promise { @@ -211,11 +213,6 @@ export class IotexService { return blocks } - async getTransaction(tx: string): Promise { - const receipt = await this.provider.iotx.getReceiptByAction({ actionHash: tx }) - return receipt - } - async getBlockActions (index: number, count: number): Promise { const actions = await this.provider.iotx.getActions({ byIndex: { @@ -226,13 +223,6 @@ export class IotexService { return actions.actionInfo } - async getCurrentHeight(): Promise { - const { chainMeta } = await this.provider.iotx.getChainMeta({ - includePendingActions: false - }) - return parseInt(chainMeta.height) - } - async getCurrentBlock(): Promise { const { chainMeta } = await this.provider.iotx.getChainMeta({ includePendingActions: false diff --git a/services/crawler/test/crawler.test.ts b/services/crawler/test/crawler.test.ts index 0e9731006..2a9be356e 100644 --- a/services/crawler/test/crawler.test.ts +++ b/services/crawler/test/crawler.test.ts @@ -6,28 +6,32 @@ test('init crawler for iotex', async () => { chain: Chain.Iotex, verbose: true }) + // await iotex.start() expect(iotex.service).not.toBe(null) }) +// jest.setTimeout(1000000) + test('init crawler for ethereum', async () => { const eth = await crawler({ chain: Chain.Ethereum, verbose: true }) + // await eth.start() expect(eth.service).not.toBe(null) }) -// test('get last processed event', async () => { -// const supercrawler = await crawler({ -// chain: Chain.Ethereum, -// verbose: true -// }) -// -// const lastBlock = await supercrawler.getLastProcessedEvent() -// -// if (!lastBlock) { -// throw new Error('last block not found') -// } -// expect(lastBlock.chain).toBe('ethereum') -// expect(lastBlock.height).not.toBe(null) -// }) +test('query athena thru service', async () => { + const supercrawler = await crawler({ + chain: Chain.Ethereum, + verbose: true + }) + + const lastBlock = await supercrawler.getLastProcessedEvent() + + if (!lastBlock) { + throw new Error('last block not found') + } + expect(lastBlock.chain).toBe('ethereum') + expect(lastBlock.height).not.toBe(null) +}) From d06c08adac377e42707cdb0cdffff9e3e29bca8f Mon Sep 17 00:00:00 2001 From: Shane Earley Date: Wed, 28 Sep 2022 13:49:11 -0400 Subject: [PATCH 7/7] Fix esbuild mismatch --- common/data/package.json | 2 +- common/helpers/package.json | 2 +- package-lock.json | 1339 +++++++++++++++++++++++-- package.json | 2 +- services/crawler/package.json | 2 +- services/crawler/test/crawler.test.ts | 6 +- services/users/package.json | 2 +- 7 files changed, 1289 insertions(+), 66 deletions(-) diff --git a/common/data/package.json b/common/data/package.json index 46061c44a..febbd220d 100644 --- a/common/data/package.json +++ b/common/data/package.json @@ -10,7 +10,7 @@ }, "devDependencies": { "@types/node": "^17.0.38", - "esbuild": "^0.14.42", + "esbuild": "^0.15.9", "esno": "^0.16.3" }, "dependencies": { diff --git a/common/helpers/package.json b/common/helpers/package.json index ec7387576..be9bc8b8a 100644 --- a/common/helpers/package.json +++ b/common/helpers/package.json @@ -9,7 +9,7 @@ }, "devDependencies": { "@types/node": "^17.0.38", - "esbuild": "^0.14.42", + "esbuild": "^0.15.9", "esno": "^0.16.3" } } diff --git a/package-lock.json b/package-lock.json index 2ed038f6a..c6e044abf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -109,7 +109,7 @@ }, "devDependencies": { "@types/node": "^17.0.38", - "esbuild": "^0.14.42", + "esbuild": "^0.15.9", "esno": "^0.16.3" } }, @@ -118,11 +118,64 @@ "dev": true, "license": "MIT" }, + "common/data/node_modules/esbuild": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.9.tgz", + "integrity": "sha512-OnYr1rkMVxtmMHIAKZLMcEUlJmqcbxBz9QoBU8G9v455na0fuzlT/GLu6l+SRghrk0Mm2fSSciMmzV43Q8e0Gg==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.15.9", + "@esbuild/linux-loong64": "0.15.9", + "esbuild-android-64": "0.15.9", + "esbuild-android-arm64": "0.15.9", + "esbuild-darwin-64": "0.15.9", + "esbuild-darwin-arm64": "0.15.9", + "esbuild-freebsd-64": "0.15.9", + "esbuild-freebsd-arm64": "0.15.9", + "esbuild-linux-32": "0.15.9", + "esbuild-linux-64": "0.15.9", + "esbuild-linux-arm": "0.15.9", + "esbuild-linux-arm64": "0.15.9", + "esbuild-linux-mips64le": "0.15.9", + "esbuild-linux-ppc64le": "0.15.9", + "esbuild-linux-riscv64": "0.15.9", + "esbuild-linux-s390x": "0.15.9", + "esbuild-netbsd-64": "0.15.9", + "esbuild-openbsd-64": "0.15.9", + "esbuild-sunos-64": "0.15.9", + "esbuild-windows-32": "0.15.9", + "esbuild-windows-64": "0.15.9", + "esbuild-windows-arm64": "0.15.9" + } + }, + "common/data/node_modules/esbuild-darwin-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.9.tgz", + "integrity": "sha512-gI7dClcDN/HHVacZhTmGjl0/TWZcGuKJ0I7/xDGJwRQQn7aafZGtvagOFNmuOq+OBFPhlPv1T6JElOXb0unkSQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, "common/helpers": { "name": "@casimir/helpers", "devDependencies": { "@types/node": "^17.0.38", - "esbuild": "^0.14.42", + "esbuild": "^0.15.9", "esno": "^0.16.3" } }, @@ -131,6 +184,59 @@ "dev": true, "license": "MIT" }, + "common/helpers/node_modules/esbuild": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.9.tgz", + "integrity": "sha512-OnYr1rkMVxtmMHIAKZLMcEUlJmqcbxBz9QoBU8G9v455na0fuzlT/GLu6l+SRghrk0Mm2fSSciMmzV43Q8e0Gg==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.15.9", + "@esbuild/linux-loong64": "0.15.9", + "esbuild-android-64": "0.15.9", + "esbuild-android-arm64": "0.15.9", + "esbuild-darwin-64": "0.15.9", + "esbuild-darwin-arm64": "0.15.9", + "esbuild-freebsd-64": "0.15.9", + "esbuild-freebsd-arm64": "0.15.9", + "esbuild-linux-32": "0.15.9", + "esbuild-linux-64": "0.15.9", + "esbuild-linux-arm": "0.15.9", + "esbuild-linux-arm64": "0.15.9", + "esbuild-linux-mips64le": "0.15.9", + "esbuild-linux-ppc64le": "0.15.9", + "esbuild-linux-riscv64": "0.15.9", + "esbuild-linux-s390x": "0.15.9", + "esbuild-netbsd-64": "0.15.9", + "esbuild-openbsd-64": "0.15.9", + "esbuild-sunos-64": "0.15.9", + "esbuild-windows-32": "0.15.9", + "esbuild-windows-64": "0.15.9", + "esbuild-windows-arm64": "0.15.9" + } + }, + "common/helpers/node_modules/esbuild-darwin-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.9.tgz", + "integrity": "sha512-gI7dClcDN/HHVacZhTmGjl0/TWZcGuKJ0I7/xDGJwRQQn7aafZGtvagOFNmuOq+OBFPhlPv1T6JElOXb0unkSQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, "common/hw-transport-speculos": { "name": "@casimir/hw-transport-speculos", "version": "0.0.1", @@ -2965,6 +3071,38 @@ "esbuild": "*" } }, + "node_modules/@esbuild/android-arm": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.15.9.tgz", + "integrity": "sha512-VZPy/ETF3fBG5PiinIkA0W/tlsvlEgJccyN2DzWZEl0DlVKRbu91PvY2D6Lxgluj4w9QtYHjOWjAT44C+oQ+EQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.15.9.tgz", + "integrity": "sha512-O+NfmkfRrb3uSsTa4jE3WApidSe3N5++fyOVGP1SmMZi4A3BZELkhUUvj5hwmMuNdlpzAZ8iAPz2vmcR7DCFQA==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/@eslint/eslintrc": { "version": "1.3.0", "dev": true, @@ -11411,6 +11549,38 @@ "esbuild-windows-arm64": "0.14.54" } }, + "node_modules/esbuild-android-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.15.9.tgz", + "integrity": "sha512-HQCX7FJn9T4kxZQkhPjNZC7tBWZqJvhlLHPU2SFzrQB/7nDXjmTIFpFTjt7Bd1uFpeXmuwf5h5fZm+x/hLnhbw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-android-arm64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.15.9.tgz", + "integrity": "sha512-E6zbLfqbFVCNEKircSHnPiSTsm3fCRxeIMPfrkS33tFjIAoXtwegQfVZqMGR0FlsvVxp2NEDOUz+WW48COCjSg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/esbuild-darwin-64": { "version": "0.14.54", "cpu": [ @@ -11426,6 +11596,598 @@ "node": ">=12" } }, + "node_modules/esbuild-darwin-arm64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.15.9.tgz", + "integrity": "sha512-VZIMlcRN29yg/sv7DsDwN+OeufCcoTNaTl3Vnav7dL/nvsApD7uvhVRbgyMzv0zU/PP0xRhhIpTyc7lxEzHGSw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-freebsd-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.15.9.tgz", + "integrity": "sha512-uM4z5bTvuAXqPxrI204txhlsPIolQPWRMLenvGuCPZTnnGlCMF2QLs0Plcm26gcskhxewYo9LkkmYSS5Czrb5A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-freebsd-arm64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.15.9.tgz", + "integrity": "sha512-HHDjT3O5gWzicGdgJ5yokZVN9K9KG05SnERwl9nBYZaCjcCgj/sX8Ps1jvoFSfNCO04JSsHSOWo4qvxFuj8FoA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-32": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.15.9.tgz", + "integrity": "sha512-AQIdE8FugGt1DkcekKi5ycI46QZpGJ/wqcMr7w6YUmOmp2ohQ8eO4sKUsOxNOvYL7hGEVwkndSyszR6HpVHLFg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.15.9.tgz", + "integrity": "sha512-4RXjae7g6Qs7StZyiYyXTZXBlfODhb1aBVAjd+ANuPmMhWthQilWo7rFHwJwL7DQu1Fjej2sODAVwLbcIVsAYQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-arm": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.15.9.tgz", + "integrity": "sha512-3Zf2GVGUOI7XwChH3qrnTOSqfV1V4CAc/7zLVm4lO6JT6wbJrTgEYCCiNSzziSju+J9Jhf9YGWk/26quWPC6yQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-arm64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.15.9.tgz", + "integrity": "sha512-a+bTtxJmYmk9d+s2W4/R1SYKDDAldOKmWjWP0BnrWtDbvUBNOm++du0ysPju4mZVoEFgS1yLNW+VXnG/4FNwdQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-mips64le": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.15.9.tgz", + "integrity": "sha512-Zn9HSylDp89y+TRREMDoGrc3Z4Hs5u56ozZLQCiZAUx2+HdbbXbWdjmw3FdTJ/i7t5Cew6/Q+6kfO3KCcFGlyw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-ppc64le": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.15.9.tgz", + "integrity": "sha512-OEiOxNAMH9ENFYqRsWUj3CWyN3V8P3ZXyfNAtX5rlCEC/ERXrCEFCJji/1F6POzsXAzxvUJrTSTCy7G6BhA6Fw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-riscv64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.15.9.tgz", + "integrity": "sha512-ukm4KsC3QRausEFjzTsOZ/qqazw0YvJsKmfoZZm9QW27OHjk2XKSQGGvx8gIEswft/Sadp03/VZvAaqv5AIwNA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-linux-s390x": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.15.9.tgz", + "integrity": "sha512-uDOQEH55wQ6ahcIKzQr3VyjGc6Po/xblLGLoUk3fVL1qjlZAibtQr6XRfy5wPJLu/M2o0vQKLq4lyJ2r1tWKcw==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-netbsd-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.15.9.tgz", + "integrity": "sha512-yWgxaYTQz+TqX80wXRq6xAtb7GSBAp6gqLKfOdANg9qEmAI1Bxn04IrQr0Mzm4AhxvGKoHzjHjMgXbCCSSDxcw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-openbsd-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.15.9.tgz", + "integrity": "sha512-JmS18acQl4iSAjrEha1MfEmUMN4FcnnrtTaJ7Qg0tDCOcgpPPQRLGsZqhes0vmx8VA6IqRyScqXvaL7+Q0Uf3A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-sunos-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.15.9.tgz", + "integrity": "sha512-UKynGSWpzkPmXW3D2UMOD9BZPIuRaSqphxSCwScfEE05Be3KAmvjsBhht1fLzKpiFVJb0BYMd4jEbWMyJ/z1hQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-windows-32": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.15.9.tgz", + "integrity": "sha512-aqXvu4/W9XyTVqO/hw3rNxKE1TcZiEYHPsXM9LwYmKSX9/hjvfIJzXwQBlPcJ/QOxedfoMVH0YnhhQ9Ffb0RGA==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-windows-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.15.9.tgz", + "integrity": "sha512-zm7h91WUmlS4idMtjvCrEeNhlH7+TNOmqw5dJPJZrgFaxoFyqYG6CKDpdFCQXdyKpD5yvzaQBOMVTCBVKGZDEg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild-windows-arm64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.15.9.tgz", + "integrity": "sha512-yQEVIv27oauAtvtuhJVfSNMztJJX47ismRS6Sv2QMVV9RM+6xjbMWuuwM2nxr5A2/gj/mu2z9YlQxiwoFRCfZA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/linux-loong64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.14.54.tgz", + "integrity": "sha512-bZBrLAIX1kpWelV0XemxBZllyRmM6vgFQQG2GdNb+r3Fkp0FOh1NJSvekXDs7jq70k4euu1cryLMfU+mTXlEpw==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-android-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.14.54.tgz", + "integrity": "sha512-Tz2++Aqqz0rJ7kYBfz+iqyE3QMycD4vk7LBRyWaAVFgFtQ/O8EJOnVmTOiDWYZ/uYzB4kvP+bqejYdVKzE5lAQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-android-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.14.54.tgz", + "integrity": "sha512-F9E+/QDi9sSkLaClO8SOV6etqPd+5DgJje1F9lOWoNncDdOBL2YF59IhsWATSt0TLZbYCf3pNlTHvVV5VfHdvg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-darwin-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.14.54.tgz", + "integrity": "sha512-OPafJHD2oUPyvJMrsCvDGkRrVCar5aVyHfWGQzY1dWnzErjrDuSETxwA2HSsyg2jORLY8yBfzc1MIpUkXlctmw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-freebsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.14.54.tgz", + "integrity": "sha512-OKwd4gmwHqOTp4mOGZKe/XUlbDJ4Q9TjX0hMPIDBUWWu/kwhBAudJdBoxnjNf9ocIB6GN6CPowYpR/hRCbSYAg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-freebsd-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.14.54.tgz", + "integrity": "sha512-sFwueGr7OvIFiQT6WeG0jRLjkjdqWWSrfbVwZp8iMP+8UHEHRBvlaxL6IuKNDwAozNUmbb8nIMXa7oAOARGs1Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-linux-32": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.14.54.tgz", + "integrity": "sha512-1ZuY+JDI//WmklKlBgJnglpUL1owm2OX+8E1syCD6UAxcMM/XoWd76OHSjl/0MR0LisSAXDqgjT3uJqT67O3qw==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-linux-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.14.54.tgz", + "integrity": "sha512-EgjAgH5HwTbtNsTqQOXWApBaPVdDn7XcK+/PtJwZLT1UmpLoznPd8c5CxqsH2dQK3j05YsB3L17T8vE7cp4cCg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-linux-arm": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.14.54.tgz", + "integrity": "sha512-qqz/SjemQhVMTnvcLGoLOdFpCYbz4v4fUo+TfsWG+1aOu70/80RV6bgNpR2JCrppV2moUQkww+6bWxXRL9YMGw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-linux-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.14.54.tgz", + "integrity": "sha512-WL71L+0Rwv+Gv/HTmxTEmpv0UgmxYa5ftZILVi2QmZBgX3q7+tDeOQNqGtdXSdsL8TQi1vIaVFHUPDe0O0kdig==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-linux-mips64le": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.14.54.tgz", + "integrity": "sha512-qTHGQB8D1etd0u1+sB6p0ikLKRVuCWhYQhAHRPkO+OF3I/iSlTKNNS0Lh2Oc0g0UFGguaFZZiPJdJey3AGpAlw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-linux-ppc64le": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.14.54.tgz", + "integrity": "sha512-j3OMlzHiqwZBDPRCDFKcx595XVfOfOnv68Ax3U4UKZ3MTYQB5Yz3X1mn5GnodEVYzhtZgxEBidLWeIs8FDSfrQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-linux-riscv64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.14.54.tgz", + "integrity": "sha512-y7Vt7Wl9dkOGZjxQZnDAqqn+XOqFD7IMWiewY5SPlNlzMX39ocPQlOaoxvT4FllA5viyV26/QzHtvTjVNOxHZg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-linux-s390x": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.14.54.tgz", + "integrity": "sha512-zaHpW9dziAsi7lRcyV4r8dhfG1qBidQWUXweUjnw+lliChJqQr+6XD71K41oEIC3Mx1KStovEmlzm+MkGZHnHA==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-netbsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.14.54.tgz", + "integrity": "sha512-PR01lmIMnfJTgeU9VJTDY9ZerDWVFIUzAtJuDHwwceppW7cQWjBBqP48NdeRtoP04/AtO9a7w3viI+PIDr6d+w==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-openbsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.14.54.tgz", + "integrity": "sha512-Qyk7ikT2o7Wu76UsvvDS5q0amJvmRzDyVlL0qf5VLsLchjCa1+IAvd8kTBgUxD7VBUUVgItLkk609ZHUc1oCaw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-sunos-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.14.54.tgz", + "integrity": "sha512-28GZ24KmMSeKi5ueWzMcco6EBHStL3B6ubM7M51RmPwXQGLe0teBGJocmWhgwccA1GeFXqxzILIxXpHbl9Q/Kw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-windows-32": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.14.54.tgz", + "integrity": "sha512-T+rdZW19ql9MjS7pixmZYVObd9G7kcaZo+sETqNH4RCkuuYSuv9AGHUVnPoP9hhuE1WM1ZimHz1CIBHBboLU7w==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-windows-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.14.54.tgz", + "integrity": "sha512-AoHTRBUuYwXtZhjXZbA1pGfTo8cJo3vZIcWGLiUcTNgHpJJMC1rVA44ZereBHMJtotyN71S8Qw0npiCIkW96cQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/esbuild-windows-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.14.54.tgz", + "integrity": "sha512-M0kuUvXhot1zOISQGXwWn6YtS+Y/1RT9WrVIOywZnJHo3jCDyewAc79aKNQWFCQm+xNHVTq9h8dZKvygoXQQRg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/escalade": { "version": "3.1.1", "license": "MIT", @@ -37810,7 +38572,7 @@ "@types/jest": "^28.1.6", "@types/json-bigint": "^1.0.1", "@types/signal-exit": "^3.0.1", - "esbuild": "^0.15.6", + "esbuild": "^0.15.9", "jest": "^28.1.3", "ts-jest": "^28.0.7", "ts-node": "^10.9.1", @@ -38214,10 +38976,11 @@ "license": "MIT" }, "services/crawler/node_modules/esbuild": { - "version": "0.15.7", + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.9.tgz", + "integrity": "sha512-OnYr1rkMVxtmMHIAKZLMcEUlJmqcbxBz9QoBU8G9v455na0fuzlT/GLu6l+SRghrk0Mm2fSSciMmzV43Q8e0Gg==", "dev": true, "hasInstallScript": true, - "license": "MIT", "bin": { "esbuild": "bin/esbuild" }, @@ -38225,36 +38988,38 @@ "node": ">=12" }, "optionalDependencies": { - "@esbuild/linux-loong64": "0.15.7", - "esbuild-android-64": "0.15.7", - "esbuild-android-arm64": "0.15.7", - "esbuild-darwin-64": "0.15.7", - "esbuild-darwin-arm64": "0.15.7", - "esbuild-freebsd-64": "0.15.7", - "esbuild-freebsd-arm64": "0.15.7", - "esbuild-linux-32": "0.15.7", - "esbuild-linux-64": "0.15.7", - "esbuild-linux-arm": "0.15.7", - "esbuild-linux-arm64": "0.15.7", - "esbuild-linux-mips64le": "0.15.7", - "esbuild-linux-ppc64le": "0.15.7", - "esbuild-linux-riscv64": "0.15.7", - "esbuild-linux-s390x": "0.15.7", - "esbuild-netbsd-64": "0.15.7", - "esbuild-openbsd-64": "0.15.7", - "esbuild-sunos-64": "0.15.7", - "esbuild-windows-32": "0.15.7", - "esbuild-windows-64": "0.15.7", - "esbuild-windows-arm64": "0.15.7" + "@esbuild/android-arm": "0.15.9", + "@esbuild/linux-loong64": "0.15.9", + "esbuild-android-64": "0.15.9", + "esbuild-android-arm64": "0.15.9", + "esbuild-darwin-64": "0.15.9", + "esbuild-darwin-arm64": "0.15.9", + "esbuild-freebsd-64": "0.15.9", + "esbuild-freebsd-arm64": "0.15.9", + "esbuild-linux-32": "0.15.9", + "esbuild-linux-64": "0.15.9", + "esbuild-linux-arm": "0.15.9", + "esbuild-linux-arm64": "0.15.9", + "esbuild-linux-mips64le": "0.15.9", + "esbuild-linux-ppc64le": "0.15.9", + "esbuild-linux-riscv64": "0.15.9", + "esbuild-linux-s390x": "0.15.9", + "esbuild-netbsd-64": "0.15.9", + "esbuild-openbsd-64": "0.15.9", + "esbuild-sunos-64": "0.15.9", + "esbuild-windows-32": "0.15.9", + "esbuild-windows-64": "0.15.9", + "esbuild-windows-arm64": "0.15.9" } }, "services/crawler/node_modules/esbuild-darwin-64": { - "version": "0.15.7", + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.9.tgz", + "integrity": "sha512-gI7dClcDN/HHVacZhTmGjl0/TWZcGuKJ0I7/xDGJwRQQn7aafZGtvagOFNmuOq+OBFPhlPv1T6JElOXb0unkSQ==", "cpu": [ "x64" ], "dev": true, - "license": "MIT", "optional": true, "os": [ "darwin" @@ -39043,7 +39808,7 @@ "@types/cors": "^2.8.12", "@types/express": "^4.17.13", "@types/node": "^17.0.38", - "esbuild": "^0.14.42", + "esbuild": "^0.15.9", "esno": "^0.16.3" } }, @@ -39051,6 +39816,59 @@ "version": "17.0.45", "dev": true, "license": "MIT" + }, + "services/users/node_modules/esbuild": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.9.tgz", + "integrity": "sha512-OnYr1rkMVxtmMHIAKZLMcEUlJmqcbxBz9QoBU8G9v455na0fuzlT/GLu6l+SRghrk0Mm2fSSciMmzV43Q8e0Gg==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.15.9", + "@esbuild/linux-loong64": "0.15.9", + "esbuild-android-64": "0.15.9", + "esbuild-android-arm64": "0.15.9", + "esbuild-darwin-64": "0.15.9", + "esbuild-darwin-arm64": "0.15.9", + "esbuild-freebsd-64": "0.15.9", + "esbuild-freebsd-arm64": "0.15.9", + "esbuild-linux-32": "0.15.9", + "esbuild-linux-64": "0.15.9", + "esbuild-linux-arm": "0.15.9", + "esbuild-linux-arm64": "0.15.9", + "esbuild-linux-mips64le": "0.15.9", + "esbuild-linux-ppc64le": "0.15.9", + "esbuild-linux-riscv64": "0.15.9", + "esbuild-linux-s390x": "0.15.9", + "esbuild-netbsd-64": "0.15.9", + "esbuild-openbsd-64": "0.15.9", + "esbuild-sunos-64": "0.15.9", + "esbuild-windows-32": "0.15.9", + "esbuild-windows-64": "0.15.9", + "esbuild-windows-arm64": "0.15.9" + } + }, + "services/users/node_modules/esbuild-darwin-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.9.tgz", + "integrity": "sha512-gI7dClcDN/HHVacZhTmGjl0/TWZcGuKJ0I7/xDGJwRQQn7aafZGtvagOFNmuOq+OBFPhlPv1T6JElOXb0unkSQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } } }, "dependencies": { @@ -40755,7 +41573,7 @@ "@types/json-bigint": "^1.0.1", "@types/signal-exit": "^3.0.1", "arg": "^5.0.2", - "esbuild": "^0.15.6", + "esbuild": "^0.15.9", "iotex-antenna": "^0.31.3", "jest": "^28.1.3", "ts-jest": "^28.0.7", @@ -41039,34 +41857,39 @@ "dev": true }, "esbuild": { - "version": "0.15.7", - "dev": true, - "requires": { - "@esbuild/linux-loong64": "0.15.7", - "esbuild-android-64": "0.15.7", - "esbuild-android-arm64": "0.15.7", - "esbuild-darwin-64": "0.15.7", - "esbuild-darwin-arm64": "0.15.7", - "esbuild-freebsd-64": "0.15.7", - "esbuild-freebsd-arm64": "0.15.7", - "esbuild-linux-32": "0.15.7", - "esbuild-linux-64": "0.15.7", - "esbuild-linux-arm": "0.15.7", - "esbuild-linux-arm64": "0.15.7", - "esbuild-linux-mips64le": "0.15.7", - "esbuild-linux-ppc64le": "0.15.7", - "esbuild-linux-riscv64": "0.15.7", - "esbuild-linux-s390x": "0.15.7", - "esbuild-netbsd-64": "0.15.7", - "esbuild-openbsd-64": "0.15.7", - "esbuild-sunos-64": "0.15.7", - "esbuild-windows-32": "0.15.7", - "esbuild-windows-64": "0.15.7", - "esbuild-windows-arm64": "0.15.7" + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.9.tgz", + "integrity": "sha512-OnYr1rkMVxtmMHIAKZLMcEUlJmqcbxBz9QoBU8G9v455na0fuzlT/GLu6l+SRghrk0Mm2fSSciMmzV43Q8e0Gg==", + "dev": true, + "requires": { + "@esbuild/android-arm": "0.15.9", + "@esbuild/linux-loong64": "0.15.9", + "esbuild-android-64": "0.15.9", + "esbuild-android-arm64": "0.15.9", + "esbuild-darwin-64": "0.15.9", + "esbuild-darwin-arm64": "0.15.9", + "esbuild-freebsd-64": "0.15.9", + "esbuild-freebsd-arm64": "0.15.9", + "esbuild-linux-32": "0.15.9", + "esbuild-linux-64": "0.15.9", + "esbuild-linux-arm": "0.15.9", + "esbuild-linux-arm64": "0.15.9", + "esbuild-linux-mips64le": "0.15.9", + "esbuild-linux-ppc64le": "0.15.9", + "esbuild-linux-riscv64": "0.15.9", + "esbuild-linux-s390x": "0.15.9", + "esbuild-netbsd-64": "0.15.9", + "esbuild-openbsd-64": "0.15.9", + "esbuild-sunos-64": "0.15.9", + "esbuild-windows-32": "0.15.9", + "esbuild-windows-64": "0.15.9", + "esbuild-windows-arm64": "0.15.9" } }, "esbuild-darwin-64": { - "version": "0.15.7", + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.9.tgz", + "integrity": "sha512-gI7dClcDN/HHVacZhTmGjl0/TWZcGuKJ0I7/xDGJwRQQn7aafZGtvagOFNmuOq+OBFPhlPv1T6JElOXb0unkSQ==", "dev": true, "optional": true }, @@ -41587,13 +42410,50 @@ "requires": { "@aws-cdk/aws-glue-alpha": "^2.33.0-alpha.0", "@types/node": "^17.0.38", - "esbuild": "^0.14.42", + "esbuild": "^0.15.9", "esno": "^0.16.3" }, "dependencies": { "@types/node": { "version": "17.0.45", "dev": true + }, + "esbuild": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.9.tgz", + "integrity": "sha512-OnYr1rkMVxtmMHIAKZLMcEUlJmqcbxBz9QoBU8G9v455na0fuzlT/GLu6l+SRghrk0Mm2fSSciMmzV43Q8e0Gg==", + "dev": true, + "requires": { + "@esbuild/android-arm": "0.15.9", + "@esbuild/linux-loong64": "0.15.9", + "esbuild-android-64": "0.15.9", + "esbuild-android-arm64": "0.15.9", + "esbuild-darwin-64": "0.15.9", + "esbuild-darwin-arm64": "0.15.9", + "esbuild-freebsd-64": "0.15.9", + "esbuild-freebsd-arm64": "0.15.9", + "esbuild-linux-32": "0.15.9", + "esbuild-linux-64": "0.15.9", + "esbuild-linux-arm": "0.15.9", + "esbuild-linux-arm64": "0.15.9", + "esbuild-linux-mips64le": "0.15.9", + "esbuild-linux-ppc64le": "0.15.9", + "esbuild-linux-riscv64": "0.15.9", + "esbuild-linux-s390x": "0.15.9", + "esbuild-netbsd-64": "0.15.9", + "esbuild-openbsd-64": "0.15.9", + "esbuild-sunos-64": "0.15.9", + "esbuild-windows-32": "0.15.9", + "esbuild-windows-64": "0.15.9", + "esbuild-windows-arm64": "0.15.9" + } + }, + "esbuild-darwin-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.9.tgz", + "integrity": "sha512-gI7dClcDN/HHVacZhTmGjl0/TWZcGuKJ0I7/xDGJwRQQn7aafZGtvagOFNmuOq+OBFPhlPv1T6JElOXb0unkSQ==", + "dev": true, + "optional": true } } }, @@ -41626,13 +42486,50 @@ "version": "file:common/helpers", "requires": { "@types/node": "^17.0.38", - "esbuild": "^0.14.42", + "esbuild": "^0.15.9", "esno": "^0.16.3" }, "dependencies": { "@types/node": { "version": "17.0.45", "dev": true + }, + "esbuild": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.9.tgz", + "integrity": "sha512-OnYr1rkMVxtmMHIAKZLMcEUlJmqcbxBz9QoBU8G9v455na0fuzlT/GLu6l+SRghrk0Mm2fSSciMmzV43Q8e0Gg==", + "dev": true, + "requires": { + "@esbuild/android-arm": "0.15.9", + "@esbuild/linux-loong64": "0.15.9", + "esbuild-android-64": "0.15.9", + "esbuild-android-arm64": "0.15.9", + "esbuild-darwin-64": "0.15.9", + "esbuild-darwin-arm64": "0.15.9", + "esbuild-freebsd-64": "0.15.9", + "esbuild-freebsd-arm64": "0.15.9", + "esbuild-linux-32": "0.15.9", + "esbuild-linux-64": "0.15.9", + "esbuild-linux-arm": "0.15.9", + "esbuild-linux-arm64": "0.15.9", + "esbuild-linux-mips64le": "0.15.9", + "esbuild-linux-ppc64le": "0.15.9", + "esbuild-linux-riscv64": "0.15.9", + "esbuild-linux-s390x": "0.15.9", + "esbuild-netbsd-64": "0.15.9", + "esbuild-openbsd-64": "0.15.9", + "esbuild-sunos-64": "0.15.9", + "esbuild-windows-32": "0.15.9", + "esbuild-windows-64": "0.15.9", + "esbuild-windows-arm64": "0.15.9" + } + }, + "esbuild-darwin-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.9.tgz", + "integrity": "sha512-gI7dClcDN/HHVacZhTmGjl0/TWZcGuKJ0I7/xDGJwRQQn7aafZGtvagOFNmuOq+OBFPhlPv1T6JElOXb0unkSQ==", + "dev": true, + "optional": true } } }, @@ -41667,7 +42564,7 @@ "@types/express": "^4.17.13", "@types/node": "^17.0.38", "cors": "^2.8.5", - "esbuild": "^0.14.42", + "esbuild": "^0.15.9", "esno": "^0.16.3", "express": "^4.18.1", "serverless-http": "^3.0.1" @@ -41676,6 +42573,43 @@ "@types/node": { "version": "17.0.45", "dev": true + }, + "esbuild": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.15.9.tgz", + "integrity": "sha512-OnYr1rkMVxtmMHIAKZLMcEUlJmqcbxBz9QoBU8G9v455na0fuzlT/GLu6l+SRghrk0Mm2fSSciMmzV43Q8e0Gg==", + "dev": true, + "requires": { + "@esbuild/android-arm": "0.15.9", + "@esbuild/linux-loong64": "0.15.9", + "esbuild-android-64": "0.15.9", + "esbuild-android-arm64": "0.15.9", + "esbuild-darwin-64": "0.15.9", + "esbuild-darwin-arm64": "0.15.9", + "esbuild-freebsd-64": "0.15.9", + "esbuild-freebsd-arm64": "0.15.9", + "esbuild-linux-32": "0.15.9", + "esbuild-linux-64": "0.15.9", + "esbuild-linux-arm": "0.15.9", + "esbuild-linux-arm64": "0.15.9", + "esbuild-linux-mips64le": "0.15.9", + "esbuild-linux-ppc64le": "0.15.9", + "esbuild-linux-riscv64": "0.15.9", + "esbuild-linux-s390x": "0.15.9", + "esbuild-netbsd-64": "0.15.9", + "esbuild-openbsd-64": "0.15.9", + "esbuild-sunos-64": "0.15.9", + "esbuild-windows-32": "0.15.9", + "esbuild-windows-64": "0.15.9", + "esbuild-windows-arm64": "0.15.9" + } + }, + "esbuild-darwin-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-darwin-64/-/esbuild-darwin-64-0.15.9.tgz", + "integrity": "sha512-gI7dClcDN/HHVacZhTmGjl0/TWZcGuKJ0I7/xDGJwRQQn7aafZGtvagOFNmuOq+OBFPhlPv1T6JElOXb0unkSQ==", + "dev": true, + "optional": true } } }, @@ -42005,6 +42939,20 @@ "rollup-plugin-node-polyfills": "^0.2.1" } }, + "@esbuild/android-arm": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.15.9.tgz", + "integrity": "sha512-VZPy/ETF3fBG5PiinIkA0W/tlsvlEgJccyN2DzWZEl0DlVKRbu91PvY2D6Lxgluj4w9QtYHjOWjAT44C+oQ+EQ==", + "dev": true, + "optional": true + }, + "@esbuild/linux-loong64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.15.9.tgz", + "integrity": "sha512-O+NfmkfRrb3uSsTa4jE3WApidSe3N5++fyOVGP1SmMZi4A3BZELkhUUvj5hwmMuNdlpzAZ8iAPz2vmcR7DCFQA==", + "dev": true, + "optional": true + }, "@eslint/eslintrc": { "version": "1.3.0", "dev": true, @@ -47812,13 +48760,288 @@ "esbuild-windows-32": "0.14.54", "esbuild-windows-64": "0.14.54", "esbuild-windows-arm64": "0.14.54" + }, + "dependencies": { + "@esbuild/linux-loong64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.14.54.tgz", + "integrity": "sha512-bZBrLAIX1kpWelV0XemxBZllyRmM6vgFQQG2GdNb+r3Fkp0FOh1NJSvekXDs7jq70k4euu1cryLMfU+mTXlEpw==", + "dev": true, + "optional": true + }, + "esbuild-android-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.14.54.tgz", + "integrity": "sha512-Tz2++Aqqz0rJ7kYBfz+iqyE3QMycD4vk7LBRyWaAVFgFtQ/O8EJOnVmTOiDWYZ/uYzB4kvP+bqejYdVKzE5lAQ==", + "dev": true, + "optional": true + }, + "esbuild-android-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.14.54.tgz", + "integrity": "sha512-F9E+/QDi9sSkLaClO8SOV6etqPd+5DgJje1F9lOWoNncDdOBL2YF59IhsWATSt0TLZbYCf3pNlTHvVV5VfHdvg==", + "dev": true, + "optional": true + }, + "esbuild-darwin-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.14.54.tgz", + "integrity": "sha512-OPafJHD2oUPyvJMrsCvDGkRrVCar5aVyHfWGQzY1dWnzErjrDuSETxwA2HSsyg2jORLY8yBfzc1MIpUkXlctmw==", + "dev": true, + "optional": true + }, + "esbuild-freebsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.14.54.tgz", + "integrity": "sha512-OKwd4gmwHqOTp4mOGZKe/XUlbDJ4Q9TjX0hMPIDBUWWu/kwhBAudJdBoxnjNf9ocIB6GN6CPowYpR/hRCbSYAg==", + "dev": true, + "optional": true + }, + "esbuild-freebsd-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.14.54.tgz", + "integrity": "sha512-sFwueGr7OvIFiQT6WeG0jRLjkjdqWWSrfbVwZp8iMP+8UHEHRBvlaxL6IuKNDwAozNUmbb8nIMXa7oAOARGs1Q==", + "dev": true, + "optional": true + }, + "esbuild-linux-32": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.14.54.tgz", + "integrity": "sha512-1ZuY+JDI//WmklKlBgJnglpUL1owm2OX+8E1syCD6UAxcMM/XoWd76OHSjl/0MR0LisSAXDqgjT3uJqT67O3qw==", + "dev": true, + "optional": true + }, + "esbuild-linux-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.14.54.tgz", + "integrity": "sha512-EgjAgH5HwTbtNsTqQOXWApBaPVdDn7XcK+/PtJwZLT1UmpLoznPd8c5CxqsH2dQK3j05YsB3L17T8vE7cp4cCg==", + "dev": true, + "optional": true + }, + "esbuild-linux-arm": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.14.54.tgz", + "integrity": "sha512-qqz/SjemQhVMTnvcLGoLOdFpCYbz4v4fUo+TfsWG+1aOu70/80RV6bgNpR2JCrppV2moUQkww+6bWxXRL9YMGw==", + "dev": true, + "optional": true + }, + "esbuild-linux-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.14.54.tgz", + "integrity": "sha512-WL71L+0Rwv+Gv/HTmxTEmpv0UgmxYa5ftZILVi2QmZBgX3q7+tDeOQNqGtdXSdsL8TQi1vIaVFHUPDe0O0kdig==", + "dev": true, + "optional": true + }, + "esbuild-linux-mips64le": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.14.54.tgz", + "integrity": "sha512-qTHGQB8D1etd0u1+sB6p0ikLKRVuCWhYQhAHRPkO+OF3I/iSlTKNNS0Lh2Oc0g0UFGguaFZZiPJdJey3AGpAlw==", + "dev": true, + "optional": true + }, + "esbuild-linux-ppc64le": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.14.54.tgz", + "integrity": "sha512-j3OMlzHiqwZBDPRCDFKcx595XVfOfOnv68Ax3U4UKZ3MTYQB5Yz3X1mn5GnodEVYzhtZgxEBidLWeIs8FDSfrQ==", + "dev": true, + "optional": true + }, + "esbuild-linux-riscv64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.14.54.tgz", + "integrity": "sha512-y7Vt7Wl9dkOGZjxQZnDAqqn+XOqFD7IMWiewY5SPlNlzMX39ocPQlOaoxvT4FllA5viyV26/QzHtvTjVNOxHZg==", + "dev": true, + "optional": true + }, + "esbuild-linux-s390x": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.14.54.tgz", + "integrity": "sha512-zaHpW9dziAsi7lRcyV4r8dhfG1qBidQWUXweUjnw+lliChJqQr+6XD71K41oEIC3Mx1KStovEmlzm+MkGZHnHA==", + "dev": true, + "optional": true + }, + "esbuild-netbsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.14.54.tgz", + "integrity": "sha512-PR01lmIMnfJTgeU9VJTDY9ZerDWVFIUzAtJuDHwwceppW7cQWjBBqP48NdeRtoP04/AtO9a7w3viI+PIDr6d+w==", + "dev": true, + "optional": true + }, + "esbuild-openbsd-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.14.54.tgz", + "integrity": "sha512-Qyk7ikT2o7Wu76UsvvDS5q0amJvmRzDyVlL0qf5VLsLchjCa1+IAvd8kTBgUxD7VBUUVgItLkk609ZHUc1oCaw==", + "dev": true, + "optional": true + }, + "esbuild-sunos-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.14.54.tgz", + "integrity": "sha512-28GZ24KmMSeKi5ueWzMcco6EBHStL3B6ubM7M51RmPwXQGLe0teBGJocmWhgwccA1GeFXqxzILIxXpHbl9Q/Kw==", + "dev": true, + "optional": true + }, + "esbuild-windows-32": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.14.54.tgz", + "integrity": "sha512-T+rdZW19ql9MjS7pixmZYVObd9G7kcaZo+sETqNH4RCkuuYSuv9AGHUVnPoP9hhuE1WM1ZimHz1CIBHBboLU7w==", + "dev": true, + "optional": true + }, + "esbuild-windows-64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.14.54.tgz", + "integrity": "sha512-AoHTRBUuYwXtZhjXZbA1pGfTo8cJo3vZIcWGLiUcTNgHpJJMC1rVA44ZereBHMJtotyN71S8Qw0npiCIkW96cQ==", + "dev": true, + "optional": true + }, + "esbuild-windows-arm64": { + "version": "0.14.54", + "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.14.54.tgz", + "integrity": "sha512-M0kuUvXhot1zOISQGXwWn6YtS+Y/1RT9WrVIOywZnJHo3jCDyewAc79aKNQWFCQm+xNHVTq9h8dZKvygoXQQRg==", + "dev": true, + "optional": true + } } }, + "esbuild-android-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-android-64/-/esbuild-android-64-0.15.9.tgz", + "integrity": "sha512-HQCX7FJn9T4kxZQkhPjNZC7tBWZqJvhlLHPU2SFzrQB/7nDXjmTIFpFTjt7Bd1uFpeXmuwf5h5fZm+x/hLnhbw==", + "dev": true, + "optional": true + }, + "esbuild-android-arm64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-android-arm64/-/esbuild-android-arm64-0.15.9.tgz", + "integrity": "sha512-E6zbLfqbFVCNEKircSHnPiSTsm3fCRxeIMPfrkS33tFjIAoXtwegQfVZqMGR0FlsvVxp2NEDOUz+WW48COCjSg==", + "dev": true, + "optional": true + }, "esbuild-darwin-64": { "version": "0.14.54", "dev": true, "optional": true }, + "esbuild-darwin-arm64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-darwin-arm64/-/esbuild-darwin-arm64-0.15.9.tgz", + "integrity": "sha512-VZIMlcRN29yg/sv7DsDwN+OeufCcoTNaTl3Vnav7dL/nvsApD7uvhVRbgyMzv0zU/PP0xRhhIpTyc7lxEzHGSw==", + "dev": true, + "optional": true + }, + "esbuild-freebsd-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-64/-/esbuild-freebsd-64-0.15.9.tgz", + "integrity": "sha512-uM4z5bTvuAXqPxrI204txhlsPIolQPWRMLenvGuCPZTnnGlCMF2QLs0Plcm26gcskhxewYo9LkkmYSS5Czrb5A==", + "dev": true, + "optional": true + }, + "esbuild-freebsd-arm64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-freebsd-arm64/-/esbuild-freebsd-arm64-0.15.9.tgz", + "integrity": "sha512-HHDjT3O5gWzicGdgJ5yokZVN9K9KG05SnERwl9nBYZaCjcCgj/sX8Ps1jvoFSfNCO04JSsHSOWo4qvxFuj8FoA==", + "dev": true, + "optional": true + }, + "esbuild-linux-32": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-32/-/esbuild-linux-32-0.15.9.tgz", + "integrity": "sha512-AQIdE8FugGt1DkcekKi5ycI46QZpGJ/wqcMr7w6YUmOmp2ohQ8eO4sKUsOxNOvYL7hGEVwkndSyszR6HpVHLFg==", + "dev": true, + "optional": true + }, + "esbuild-linux-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-64/-/esbuild-linux-64-0.15.9.tgz", + "integrity": "sha512-4RXjae7g6Qs7StZyiYyXTZXBlfODhb1aBVAjd+ANuPmMhWthQilWo7rFHwJwL7DQu1Fjej2sODAVwLbcIVsAYQ==", + "dev": true, + "optional": true + }, + "esbuild-linux-arm": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm/-/esbuild-linux-arm-0.15.9.tgz", + "integrity": "sha512-3Zf2GVGUOI7XwChH3qrnTOSqfV1V4CAc/7zLVm4lO6JT6wbJrTgEYCCiNSzziSju+J9Jhf9YGWk/26quWPC6yQ==", + "dev": true, + "optional": true + }, + "esbuild-linux-arm64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-arm64/-/esbuild-linux-arm64-0.15.9.tgz", + "integrity": "sha512-a+bTtxJmYmk9d+s2W4/R1SYKDDAldOKmWjWP0BnrWtDbvUBNOm++du0ysPju4mZVoEFgS1yLNW+VXnG/4FNwdQ==", + "dev": true, + "optional": true + }, + "esbuild-linux-mips64le": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-mips64le/-/esbuild-linux-mips64le-0.15.9.tgz", + "integrity": "sha512-Zn9HSylDp89y+TRREMDoGrc3Z4Hs5u56ozZLQCiZAUx2+HdbbXbWdjmw3FdTJ/i7t5Cew6/Q+6kfO3KCcFGlyw==", + "dev": true, + "optional": true + }, + "esbuild-linux-ppc64le": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-ppc64le/-/esbuild-linux-ppc64le-0.15.9.tgz", + "integrity": "sha512-OEiOxNAMH9ENFYqRsWUj3CWyN3V8P3ZXyfNAtX5rlCEC/ERXrCEFCJji/1F6POzsXAzxvUJrTSTCy7G6BhA6Fw==", + "dev": true, + "optional": true + }, + "esbuild-linux-riscv64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-riscv64/-/esbuild-linux-riscv64-0.15.9.tgz", + "integrity": "sha512-ukm4KsC3QRausEFjzTsOZ/qqazw0YvJsKmfoZZm9QW27OHjk2XKSQGGvx8gIEswft/Sadp03/VZvAaqv5AIwNA==", + "dev": true, + "optional": true + }, + "esbuild-linux-s390x": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-linux-s390x/-/esbuild-linux-s390x-0.15.9.tgz", + "integrity": "sha512-uDOQEH55wQ6ahcIKzQr3VyjGc6Po/xblLGLoUk3fVL1qjlZAibtQr6XRfy5wPJLu/M2o0vQKLq4lyJ2r1tWKcw==", + "dev": true, + "optional": true + }, + "esbuild-netbsd-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-netbsd-64/-/esbuild-netbsd-64-0.15.9.tgz", + "integrity": "sha512-yWgxaYTQz+TqX80wXRq6xAtb7GSBAp6gqLKfOdANg9qEmAI1Bxn04IrQr0Mzm4AhxvGKoHzjHjMgXbCCSSDxcw==", + "dev": true, + "optional": true + }, + "esbuild-openbsd-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-openbsd-64/-/esbuild-openbsd-64-0.15.9.tgz", + "integrity": "sha512-JmS18acQl4iSAjrEha1MfEmUMN4FcnnrtTaJ7Qg0tDCOcgpPPQRLGsZqhes0vmx8VA6IqRyScqXvaL7+Q0Uf3A==", + "dev": true, + "optional": true + }, + "esbuild-sunos-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-sunos-64/-/esbuild-sunos-64-0.15.9.tgz", + "integrity": "sha512-UKynGSWpzkPmXW3D2UMOD9BZPIuRaSqphxSCwScfEE05Be3KAmvjsBhht1fLzKpiFVJb0BYMd4jEbWMyJ/z1hQ==", + "dev": true, + "optional": true + }, + "esbuild-windows-32": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-windows-32/-/esbuild-windows-32-0.15.9.tgz", + "integrity": "sha512-aqXvu4/W9XyTVqO/hw3rNxKE1TcZiEYHPsXM9LwYmKSX9/hjvfIJzXwQBlPcJ/QOxedfoMVH0YnhhQ9Ffb0RGA==", + "dev": true, + "optional": true + }, + "esbuild-windows-64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-windows-64/-/esbuild-windows-64-0.15.9.tgz", + "integrity": "sha512-zm7h91WUmlS4idMtjvCrEeNhlH7+TNOmqw5dJPJZrgFaxoFyqYG6CKDpdFCQXdyKpD5yvzaQBOMVTCBVKGZDEg==", + "dev": true, + "optional": true + }, + "esbuild-windows-arm64": { + "version": "0.15.9", + "resolved": "https://registry.npmjs.org/esbuild-windows-arm64/-/esbuild-windows-arm64-0.15.9.tgz", + "integrity": "sha512-yQEVIv27oauAtvtuhJVfSNMztJJX47ismRS6Sv2QMVV9RM+6xjbMWuuwM2nxr5A2/gj/mu2z9YlQxiwoFRCfZA==", + "dev": true, + "optional": true + }, "escalade": { "version": "3.1.1" }, diff --git a/package.json b/package.json index e64136aaa..855021b1a 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "services/*" ], "scripts": { - "clean": "npm exec --workspaces -- npx rimraf node_modules && npx rimraf node_modules", + "clean": "npm exec --workspaces -- npx rimraf node_modules && npx rimraf node_modules && npm i", "deploy": "scripts/cdk/deploy -d infrastructure/cdk", "deploy:templates": "scripts/pinpoint/deploy -d content/email/templates", "dev": "scripts/local/dev -m \"$npm_config_mock\" -s \"$npm_config_speculos\"", diff --git a/services/crawler/package.json b/services/crawler/package.json index 1486eb83a..87734f0de 100644 --- a/services/crawler/package.json +++ b/services/crawler/package.json @@ -23,7 +23,7 @@ "@types/jest": "^28.1.6", "@types/json-bigint": "^1.0.1", "@types/signal-exit": "^3.0.1", - "esbuild": "^0.15.6", + "esbuild": "^0.15.9", "jest": "^28.1.3", "ts-jest": "^28.0.7", "ts-node": "^10.9.1", diff --git a/services/crawler/test/crawler.test.ts b/services/crawler/test/crawler.test.ts index 2a9be356e..fddfd3497 100644 --- a/services/crawler/test/crawler.test.ts +++ b/services/crawler/test/crawler.test.ts @@ -6,18 +6,18 @@ test('init crawler for iotex', async () => { chain: Chain.Iotex, verbose: true }) - // await iotex.start() + await iotex.start() expect(iotex.service).not.toBe(null) }) -// jest.setTimeout(1000000) +jest.setTimeout(1000000) test('init crawler for ethereum', async () => { const eth = await crawler({ chain: Chain.Ethereum, verbose: true }) - // await eth.start() + await eth.start() expect(eth.service).not.toBe(null) }) diff --git a/services/users/package.json b/services/users/package.json index a0f2f34b5..a6a65f9b0 100644 --- a/services/users/package.json +++ b/services/users/package.json @@ -19,7 +19,7 @@ "@types/cors": "^2.8.12", "@types/express": "^4.17.13", "@types/node": "^17.0.38", - "esbuild": "^0.14.42", + "esbuild": "^0.15.9", "esno": "^0.16.3" } }