Merge pull request #116 from consensusnetworks/feature/iotex-actions

Feature/iotex actions

common/data/package.json

@@ -3,17 +3,17 @@
     "private": true,
     "main": "src/index.ts",
     "scripts": {
-      "configure:python": "poetry install && poetry run ipython kernel install --user --name=casimir-data",
-      "dev": "npx esno src/index.ts",
-      "build": "esbuild src/index.ts --bundle --minify --sourcemap --platform=node --target=es2020 --outfile=dist/index.js",
-      "test": "echo \"Error: no test specified\" && exit 1"
+        "configure:python": "poetry install && poetry run ipython kernel install --user --name=casimir-data",
+        "dev": "npx esno src/index.ts",
+        "build": "esbuild src/index.ts --bundle --minify --sourcemap --platform=node --target=es2020 --outfile=dist/index.js",
+        "test": "echo \"Error: no test specified\" && exit 1"
     },
     "devDependencies": {
-      "@types/node": "^17.0.38",
-      "esbuild": "^0.14.42",
-      "esno": "^0.16.3"
+        "@types/node": "^17.0.38",
+        "esbuild": "^0.14.42",
+        "esno": "^0.16.3"
     },
     "dependencies": {
         "@aws-cdk/aws-glue-alpha": "^2.33.0-alpha.0"
     }
-}
+}

common/data/src/index.ts

@@ -17,10 +17,34 @@ export function schemaToGlueColumns(jsonSchema: JsonSchema): glue.Column[] {
     // 'STRING' | 'INTEGER' | 'BOOLEAN' | 'DOUBLE' | 'DECIMAL' | 'BIGINT' | 'TIMESTAMP' | 'JSON' | 'DATE' 
     const typeKey = property.type.toUpperCase() as keyof glue.Schema
 
-    const type = glue.Schema[typeKey]
+    let type: glue.Type = glue.Schema[typeKey]
+
+    if (name.endsWith('_at')) type = glue.Schema.DATE
+    if (name === 'candidate_list') type = glue.Schema.array(glue.Schema.STRING)
+
     const comment = property.description
     return { name, type, comment }
   })
 }
 
+export type EventTableColumn = {
+  chain: string
+  network: string
+  provider: string
+  type: string
+  created_at: string
+  address: string
+  height: number
+  to_address: string
+  candidate: string
+  candidate_list: string[]
+  amount: number
+  duration: number
+  auto_stake: boolean
+  // payload: Record<string, unknown>
+}
+
+// export type EventTableColumn = {
+//   [key in keyof typeof eventSchema.properties]: // what goes here?
+
 export { eventSchema, aggSchema }

common/data/src/schemas/agg.schema.json

@@ -13,8 +13,8 @@
             "description": "The address of the aggregate"
         },
         "first_staked_at": {
-            "type": "date",
-            "description": "The first datestring (MM-DD-YYYY) that a wallet staked"
+            "type": "string",
+            "description": "The first date (MM-DD-YYYY) that a wallet staked"
         },
         "total_staked_amount": {
             "type": "string",

common/data/src/schemas/event.schema.json

@@ -4,33 +4,57 @@
 	"title": "Event",
 	"type": "object",
     "properties": {
+        "chain": {
+            "type": "string",
+            "description": "Name of the chain (e.g. iotex, ethereum)"
+        },
+        "network": {
+            "type": "string",
+            "description": "Name of the network (e.g. mainnet, testnet)"
+        },
+        "provider": {
+            "type": "string",
+            "description": "Name of the provider (e.g. casimir, infura, alchemy)"
+        },
         "type": {
             "type": "string",
-            "description": "The type of event"
+            "description": "Type of the event"
         },
-        "datestring": {
-            "type": "date",
-            "description": "The datestring (MM-DD-YYYY) of the event"
+        "height": {
+            "type": "integer",
+            "description": "The block height of the event"
+        },
+        "created_at": {
+            "type": "string",
+            "description": "The date (MM-DD-YYYY) of the event"
         },
         "address": {
             "type": "string",
             "description": "The address that initiated the event"
         },
-        "staked_candidate": {
+        "to_address": {
             "type": "string",
-            "description": "The name of the candidate that received the stake action event"
+            "description": "The address which received the action event"
         },
-        "staked_amount": {
+        "candidate": {
             "type": "string",
-            "description": "The amount staked or unstaked in the stake action event"
+            "description": "The name of the candidate associated the event"
         },
-        "staked_duration": {
+        "candidate_list": {
+            "type": "array",
+            "description": "The list of candidates in a stake action"
+        },
+        "amount": {
             "type": "integer",
-            "description": "The duration of the stake action event"
+            "description": "The amount of the currency in the event"
+        },
+        "duration":{
+            "type": "string",
+            "description": "The duration of the action"
         },
-        "auto_stake":{
+        "auto_stake": {
             "type": "boolean",
-            "description": "The compounding selection of the stake action event"
+            "description": "Is auto staking enabled"
         }
     }
 }

infrastructure/cdk/test/website.test.ts → infrastructure/cdk/test/all.test.ts

@@ -5,8 +5,7 @@ import { EtlStack } from '../lib/etl/etl-stack'
 import { UsersStack } from '../lib/users/users-stack'
 import { WebsiteStack } from '../lib/website/website-stack'
 
-// Todo actually test something
-test('Website Created', () => {
+test('All stacks created', () => {
 
   const defaultEnv = { account: '257202027633', region: 'us-east-2' }
   const project = 'Casimir'

infrastructure/cdk/test/etl.test.ts

@@ -0,0 +1,40 @@
+import * as cdk from 'aws-cdk-lib'
+import { Template } from 'aws-cdk-lib/assertions'
+import { EtlStack } from '../lib/etl/etl-stack'
+import { eventSchema, aggSchema } from '@casimir/data'
+
+test('ETL stack created', () => {
+
+  const defaultEnv = { account: '257202027633', region: 'us-east-2' }
+  const project = 'Casimir'
+  const stage = 'Test'
+
+  const app = new cdk.App()
+  const etlStack = new EtlStack(app, `${project}EtlStack${stage}`, { env: defaultEnv, project, stage })
+
+  const etlTemplate = Template.fromStack(etlStack)
+
+  const resource = etlTemplate.findResources('AWS::Glue::Table')
+
+  const eventTable = Object.keys(resource).filter(key => key.includes('EventTable'))
+  const eventColumns = resource[eventTable[0]].Properties.TableInput.StorageDescriptor.Columns
+
+  for (const column of eventColumns) {
+    const { Name: name } = column
+    const columnName = Object.keys(eventSchema.properties).filter(key => key === name)[0]
+    expect(columnName).toEqual(name)
+  }
+
+  const aggTable = Object.keys(resource).filter(key => key.includes('AggTable'))[0]
+  const aggColumns = resource[aggTable].Properties.TableInput.StorageDescriptor.Columns
+
+  for (const column of aggColumns) {
+    const { Name: name } = column
+    const columnName = Object.keys(aggSchema.properties).filter(key => key === name)[0]
+    expect(columnName).toEqual(name)
+  }
+
+  Object.keys(etlTemplate.findOutputs('*')).forEach(output => {
+    expect(output).toBeDefined()
+  })
+})