From 0e9ebdf917760dc50d4daee4daf4672133cb51aa Mon Sep 17 00:00:00 2001 From: hawyar Date: Wed, 19 Jul 2023 20:25:43 -0400 Subject: [PATCH 1/4] Add staking action schema --- common/data/src/schemas/actions.json | 61 +++++++++++++++++++ common/data/src/schemas/event.schema.json | 11 ++-- .../src/schemas/staking_action.schema.json | 32 ---------- common/data/src/schemas/wallet.schema.json | 40 ------------ 4 files changed, 67 insertions(+), 77 deletions(-) create mode 100644 common/data/src/schemas/actions.json delete mode 100644 common/data/src/schemas/staking_action.schema.json delete mode 100644 common/data/src/schemas/wallet.schema.json diff --git a/common/data/src/schemas/actions.json b/common/data/src/schemas/actions.json new file mode 100644 index 000000000..1ddc57c4f --- /dev/null +++ b/common/data/src/schemas/actions.json @@ -0,0 +1,61 @@ +{ + "$id": "https://casimir.co/agg.schema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "$comment": "analytics", + "title": "Staking actions events", + "type": "object", + "properties": { + "chain": { + "type": "string", + "description": "The chain which the event belongs to (e.g. iotex, ethereum)" + }, + "network": { + "type": "string", + "description": "Network type (e.g. mainnet, testnet)" + }, + "type": { + "type": "string", + "description": "Type of the event (e.g. wallet, stake)" + }, + "action": { + "type": "string", + "description": "For wallet (e.g. received or sent) and for stake (e.g. stake_deposited, stake_rebalanced, withdrawal_initiated, withdrawal_fulfilled)" + }, + "address": { + "type": "string", + "description": "Wallet address" + }, + "amount": { + "type": "string", + "description": "Recorded amount depending on the event type" + }, + "balance": { + "type": "string", + "description": "Wallet balance or the staked amount" + }, + "gas": { + "type": "integer", + "description": "Gas used for the transaction" + }, + "hash": { + "type": "string", + "description": "Transaction hash" + }, + "price": { + "type": "string", + "description": "The exchange price of the coin at the time of the event" + }, + "received_at": { + "type": "integer", + "description": "Timestamp of the event in unix format" + }, + "rewards_all_time": { + "type": "string", + "description": "Total rewards earned" + }, + "staking_fees": { + "type": "string", + "description": "Total staking fees" + } + } +} \ No newline at end of file diff --git a/common/data/src/schemas/event.schema.json b/common/data/src/schemas/event.schema.json index 8e44fa997..c8dff1567 100644 --- a/common/data/src/schemas/event.schema.json +++ b/common/data/src/schemas/event.schema.json @@ -1,7 +1,8 @@ { "$id": "https://casimir.co/event.schema.json", "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Event", + "$comment": "analytics", + "title": "Transaction events", "type": "object", "properties": { "chain": { @@ -10,7 +11,7 @@ }, "network": { "type": "string", - "description": "The network which the event was received on (e.g. mainnet, testnet)" + "description": "Network type (e.g. mainnet, testnet)" }, "provider": { "type": "string", @@ -33,8 +34,8 @@ "description": "The transaction hash" }, "received_at": { - "type": "string", - "description": "The timestamp of the event recieved by the blockchain (format: Modified ISO 8601 e.g. 2015-03-04 22:44:30.652)" + "type": "integer", + "description": "Timestamp of the event in unix format" }, "sender": { "type": "string", @@ -61,7 +62,7 @@ "description": "The exchange price of the coin at the time of the event" }, "gas_fee": { - "type": "string", + "type": "integer", "description": "The gas fee paid for the transaction" } } diff --git a/common/data/src/schemas/staking_action.schema.json b/common/data/src/schemas/staking_action.schema.json deleted file mode 100644 index fca3228e2..000000000 --- a/common/data/src/schemas/staking_action.schema.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "$id": "https://casimir.co/agg.schema.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Staking Action", - "type": "object", - "properties": { - "wallet_address": { - "type": "string", - "description": "The wallet address of the staker" - }, - "stake_deposit": { - "type": "integer", - "description": "The amount of stake deposited" - }, - "created_at": { - "type": "string", - "description": "The date and time of the staking action" - }, - "stake_rebalance": { - "type": "integer", - "description": "The amount of stake rebalanced" - }, - "withdrawal_amount": { - "type": "integer", - "description": "The amount of stake withdrawn" - }, - "distribute_reward": { - "type": "integer", - "description": "The amount of reward distributed" - } - } -} \ No newline at end of file diff --git a/common/data/src/schemas/wallet.schema.json b/common/data/src/schemas/wallet.schema.json deleted file mode 100644 index 8156fb22b..000000000 --- a/common/data/src/schemas/wallet.schema.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "$id": "https://casimir.co/agg.schema.json", - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Wallet", - "type": "object", - "properties": { - "wallet_address": { - "type": "string", - "description": "The wallet address" - }, - "wallet_balance": { - "type": "string", - "description": "The wallet balance" - }, - "tx_direction": { - "type": "string", - "description": "Ingoing or outgoing transaction" - }, - "tx_id": { - "type": "string", - "description": "The transaction id" - }, - "received_at": { - "type": "string", - "description": "The time the transaction was received" - }, - "amount": { - "type": "string", - "description": "The amount of the transaction" - }, - "price": { - "type": "string", - "description": "The exchnage price at the time of the transaction" - }, - "gas_fee": { - "type": "string", - "description": "The gas fee of the transaction" - } - } -} \ No newline at end of file From 634534367afb4b1951cb238a1f205c6cbdbeffbb Mon Sep 17 00:00:00 2001 From: hawyar Date: Wed, 19 Jul 2023 20:54:04 -0400 Subject: [PATCH 2/4] Update cdk resource for the new analytics schema --- common/data/src/index.ts | 6 ++-- .../{actions.json => action.schema.json} | 2 +- common/data/src/schemas/event.schema.json | 2 +- infrastructure/cdk/src/providers/analytics.ts | 29 +++++++------------ 4 files changed, 14 insertions(+), 25 deletions(-) rename common/data/src/schemas/{actions.json => action.schema.json} (98%) diff --git a/common/data/src/index.ts b/common/data/src/index.ts index d86044025..d5e3219ae 100644 --- a/common/data/src/index.ts +++ b/common/data/src/index.ts @@ -2,8 +2,7 @@ import accountSchema from './schemas/account.schema.json' import nonceSchema from './schemas/nonce.schema.json' import userSchema from './schemas/user.schema.json' import eventSchema from './schemas/event.schema.json' -import walletSchema from './schemas/wallet.schema.json' -import stakingActionSchema from './schemas/staking_action.schema.json' +import actionSchema from './schemas/action.schema.json' import userAccountSchema from './schemas/user_account.schema.json' import operatorStore from './mock/operator.store.json' import validatorStore from './mock/validator.store.json' @@ -19,8 +18,7 @@ export { userSchema, userAccountSchema, eventSchema, - walletSchema, - stakingActionSchema, + actionSchema, accountStore, userStore, operatorStore, diff --git a/common/data/src/schemas/actions.json b/common/data/src/schemas/action.schema.json similarity index 98% rename from common/data/src/schemas/actions.json rename to common/data/src/schemas/action.schema.json index 1ddc57c4f..98ee7aeae 100644 --- a/common/data/src/schemas/actions.json +++ b/common/data/src/schemas/action.schema.json @@ -2,7 +2,7 @@ "$id": "https://casimir.co/agg.schema.json", "$schema": "http://json-schema.org/draft-07/schema#", "$comment": "analytics", - "title": "Staking actions events", + "title": "Action", "type": "object", "properties": { "chain": { diff --git a/common/data/src/schemas/event.schema.json b/common/data/src/schemas/event.schema.json index c8dff1567..9a8705bae 100644 --- a/common/data/src/schemas/event.schema.json +++ b/common/data/src/schemas/event.schema.json @@ -2,7 +2,7 @@ "$id": "https://casimir.co/event.schema.json", "$schema": "http://json-schema.org/draft-07/schema#", "$comment": "analytics", - "title": "Transaction events", + "title": "Event", "type": "object", "properties": { "chain": { diff --git a/infrastructure/cdk/src/providers/analytics.ts b/infrastructure/cdk/src/providers/analytics.ts index 92bf38b7a..ab4f23f08 100644 --- a/infrastructure/cdk/src/providers/analytics.ts +++ b/infrastructure/cdk/src/providers/analytics.ts @@ -2,7 +2,7 @@ import { Construct } from 'constructs' import * as cdk from 'aws-cdk-lib' import * as s3 from 'aws-cdk-lib/aws-s3' import * as glue from '@aws-cdk/aws-glue-alpha' -import { Schema, eventSchema, walletSchema, stakingActionSchema } from '@casimir/data' +import { Schema, eventSchema, actionSchema } from '@casimir/data' import { kebabCase, pascalCase, snakeCase } from '@casimir/helpers' import { Config } from './config' import { AnalyticsStackProps } from '../interfaces/StackProps' @@ -21,8 +21,7 @@ export class AnalyticsStack extends cdk.Stack { /** Get Glue Columns from JSON Schema for each table */ const eventColumns = new Schema(eventSchema).getGlueColumns() - const walletColumns = new Schema(walletSchema).getGlueColumns() - const stakingActionColumns = new Schema(stakingActionSchema).getGlueColumns() + const actionColumns = new Schema(actionSchema).getGlueColumns() /** Create Glue DB */ const database = new glue.Database(this, config.getFullStackResourceName(this.name, 'database'), { @@ -33,12 +32,11 @@ export class AnalyticsStack extends cdk.Stack { const eventBucket = new s3.Bucket(this, config.getFullStackResourceName(this.name, 'event-bucket', config.dataVersion), { bucketName: kebabCase(config.getFullStackResourceName(this.name, 'event-bucket', config.dataVersion)), }) - const walletBucket = new s3.Bucket(this, config.getFullStackResourceName(this.name, 'wallet-bucket', config.dataVersion), { - bucketName: kebabCase(config.getFullStackResourceName(this.name, 'wallet-bucket', config.dataVersion)), - }) - const stakingActionBucket = new s3.Bucket(this, config.getFullStackResourceName(this.name, 'staking-action-bucket', config.dataVersion), { - bucketName: kebabCase(config.getFullStackResourceName(this.name, 'staking-action-bucket', config.dataVersion)), + + const actionBucket = new s3.Bucket(this, config.getFullStackResourceName(this.name, 'action-bucket', config.dataVersion), { + bucketName: kebabCase(config.getFullStackResourceName(this.name, 'action-bucket', config.dataVersion)), }) + new s3.Bucket(this, config.getFullStackResourceName(this.name, 'output-bucket', config.dataVersion)) /** Create Glue tables */ @@ -49,18 +47,11 @@ export class AnalyticsStack extends cdk.Stack { columns: eventColumns, dataFormat: glue.DataFormat.JSON, }) - new glue.Table(this, config.getFullStackResourceName(this.name, 'wallet-table', config.dataVersion), { - database: database, - tableName: snakeCase(config.getFullStackResourceName(this.name, 'wallet-table', config.dataVersion)), - bucket: walletBucket, - columns: walletColumns, - dataFormat: glue.DataFormat.JSON, - }) - new glue.Table(this, config.getFullStackResourceName(this.name, 'staking-action-table', config.dataVersion), { + new glue.Table(this, config.getFullStackResourceName(this.name, 'action-table', config.dataVersion), { database: database, - tableName: snakeCase(config.getFullStackResourceName(this.name, 'staking-action-table', config.dataVersion)), - bucket: stakingActionBucket, - columns: stakingActionColumns, + tableName: snakeCase(config.getFullStackResourceName(this.name, 'action-table', config.dataVersion)), + bucket: actionBucket, + columns: actionColumns, dataFormat: glue.DataFormat.JSON, }) } From 5c631aa3b94e4801383ce2fcde22809736d4e3cf Mon Sep 17 00:00:00 2001 From: hawyar Date: Wed, 19 Jul 2023 21:13:36 -0400 Subject: [PATCH 3/4] Fix analytics test --- infrastructure/cdk/test/analytics.test.ts | 27 ++++++----------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/infrastructure/cdk/test/analytics.test.ts b/infrastructure/cdk/test/analytics.test.ts index 8ed945dbe..5ac8025b6 100644 --- a/infrastructure/cdk/test/analytics.test.ts +++ b/infrastructure/cdk/test/analytics.test.ts @@ -2,7 +2,7 @@ import * as cdk from 'aws-cdk-lib' import * as assertions from 'aws-cdk-lib/assertions' import { Config } from '../src/providers/config' import { AnalyticsStack } from '../src/providers/analytics' -import { Schema, eventSchema, walletSchema, stakingActionSchema } from '@casimir/data' +import { Schema, eventSchema, actionSchema } from '@casimir/data' test('Analytics stack created', () => { const config = new Config() @@ -27,28 +27,15 @@ test('Analytics stack created', () => { expect(columnName).toEqual(name) } - const walletTable = Object.keys(resource).filter(key => key.includes('WalletTable'))[0] - const walletColumns = resource[walletTable].Properties.TableInput.StorageDescriptor.Columns - const walletGlueSchema = new Schema(walletSchema).getGlueColumns() + const actionTable = Object.keys(resource).filter(key => key.includes('WalletTable'))[0] + const actionColumns = resource[actionTable].Properties.TableInput.StorageDescriptor.Columns + const actionGlueSchema = new Schema(actionSchema).getGlueColumns() - for (const column of walletColumns) { + for (const column of actionColumns) { const { Name: name, Type: type } = column - const columnName = Object.keys(walletSchema.properties).filter(key => key === name)[0] - const columnType = walletGlueSchema.filter(key => key.name === name)[0].type.inputString - - expect(columnType).toEqual(type) - expect(columnName).toEqual(name) - } - - const stakingActionTable = Object.keys(resource).filter(key => key.includes('StakingActionTable'))[0] - const stakingActionColumns = resource[stakingActionTable].Properties.TableInput.StorageDescriptor.Columns - const stakingActionGlueSchema = new Schema(stakingActionSchema).getGlueColumns() - - for (const column of stakingActionColumns) { - const { Name: name, Type: type } = column - const columnName = Object.keys(stakingActionSchema.properties).filter(key => key === name)[0] - const columnType = stakingActionGlueSchema.filter(key => key.name === name)[0].type.inputString + const columnName = Object.keys(actionSchema.properties).filter(key => key === name)[0] + const columnType = actionGlueSchema.filter(key => key.name === name)[0].type.inputString expect(columnType).toEqual(type) expect(columnName).toEqual(name) From 1dfe6e338a91709a3df4c32677f64cc04cafb320 Mon Sep 17 00:00:00 2001 From: hawyar Date: Wed, 19 Jul 2023 23:09:09 -0400 Subject: [PATCH 4/4] Another fix --- infrastructure/cdk/test/analytics.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/cdk/test/analytics.test.ts b/infrastructure/cdk/test/analytics.test.ts index 5ac8025b6..8d9fa52c1 100644 --- a/infrastructure/cdk/test/analytics.test.ts +++ b/infrastructure/cdk/test/analytics.test.ts @@ -27,7 +27,7 @@ test('Analytics stack created', () => { expect(columnName).toEqual(name) } - const actionTable = Object.keys(resource).filter(key => key.includes('WalletTable'))[0] + const actionTable = Object.keys(resource).filter(key => key.includes('ActionTable'))[0] const actionColumns = resource[actionTable].Properties.TableInput.StorageDescriptor.Columns const actionGlueSchema = new Schema(actionSchema).getGlueColumns()