Merge pull request #167 from consensusnetworks/feature/clean-crawler

Feature/clean crawler

.eslintignore

@@ -1,2 +1,3 @@
+scripts/**/resources
 cdk.out
 *.js

common/data/src/index.ts

@@ -25,6 +25,14 @@ export function schemaToGlueColumns(jsonSchema: JsonSchema): glue.Column[] {
 
     if (name.endsWith('amount')) type = glue.Schema.BIG_INT
 
+    if (name === 'gas_used') type = glue.Schema.BIG_INT
+
+    if (name === 'gas_limit') type = glue.Schema.BIG_INT
+
+    if (name === 'base_fee') type = glue.Schema.BIG_INT
+
+    if (name === 'burnt_fee') type = glue.Schema.FLOAT
+
     const comment = property.description
     return { name, type, comment }
   })
@@ -54,13 +62,13 @@ export type EventTableSchema = {
   /** The amount value associated with the transaction */
   amount: string
   /** The total amount of gas used  */
-  gasUsed: number
+  gasUsed: string
   /** The gas limit provided by transactions in the block */
-  gasLimit: number
+  gasLimit: string
   /** Post-London upgrade this represents the minimum gasUsed multiplier required for a transaction to be included in a block */
-  baseFee: number
+  baseFee: string
   /** Post-London Upgrade, this represents the part of the tx fee that is burnt */
-  burntFee: number
+  burntFee: string
   /** The validator's address */
   validator: string
   /** The list of validators' addresses */

common/data/src/schemas/event.schema.json

@@ -49,19 +49,19 @@
             "description": "The amount of currency associated with the event"
         },
         "gas_used": {
-            "type": "big_int",
+            "type": "string",
             "description": "The total amount of gas used"
         },
         "gas_limit": {
-            "type": "big_int",
+            "type": "string",
             "description": "The gas limit provided by transactions in the block"
         },
         "base_fee": {
-            "type": "big_int",
+            "type": "string",
             "description": "Post-London upgrade this represents the minimum gasUsed multiplier required for a transaction to be included in a block"
         },
         "burnt_fee": {
-            "type": "float",
+            "type": "string",
             "description": "Post-London Upgrade, this represents the part of the tx fee that is burnt"
         },
         "validator": {

common/helpers/src/index.ts

@@ -187,7 +187,6 @@ async function pollAthenaQueryOutput(queryId: string): Promise<void> {
  * @return string - Query result
  */
 export async function queryAthena(query: string): Promise<EventTableSchema[] | null> {
-
   if (!athena) {
     athena = await newAthenaClient()
   }

package.json

@@ -13,6 +13,7 @@
     "clean": "rm -rf package-lock.json && npm exec --workspaces -- npx rimraf node_modules && npx rimraf node_modules && npm i",
     "deploy": "scripts/cdk/deploy -d infrastructure/cdk",
     "dev": "scripts/local/dev -f \"$npm_config_fork\" -m \"$npm_config_mock\" -n \"$npm_config_network\" -s \"$npm_config_speculos\" -t \"$npm_config_tunnel\"",
+    "dev:crawler": "scripts/crawler/dev -c \"$npm_config_chains\" -f \"$npm_config_fork\" -n \"$npm_config_network\" -u \"$npm_config_upload\"",
     "dev:ethereum": "scripts/ethereum/dev -f \"$npm_config_fork\"",
     "dev:landing": "npm run dev --workspace @casimir/landing",
     "docgen": "npm run docgen --workspace @casimir/ethereum",

scripts/crawler/dev

@@ -0,0 +1,90 @@
+#!/bin/bash
+# Run a Casimir crawler
+#
+# Example:
+#
+#    scripts/crawler/dev -c chains -f fork-name -n network-name -upload whether-to-upload (network overrides fork)
+#
+# Further information:
+# See https://docs.aws.amazon.com/cdk/api/v2/
+#
+
+# Configure and expose variables
+source scripts/aws/configure
+
+# Set RPC URL bases
+ethereum_mainnet=https://eth-mainnet.g.alchemy.com/v2 
+ethereum_testnet=https://eth-testnet.g.alchemy.com/v2
+
+# Set the stage
+export PUBLIC_STAGE=${STAGE}
+
+# Get args
+while getopts :c:f:n:u: flag
+do
+    case "${flag}" in
+        c) chains=${OPTARG};;
+        f) fork=${OPTARG};;
+        n) network=${OPTARG};;
+        u) upload=${OPTARG};;
+    esac
+done
+
+# Default to ethereum if chains is not set or set vaguely
+if [ -z "$chains" ] || [ "$chains" = true ]; then
+    chains="ethereum"
+fi
+
+# Default to mainnet if fork is set vaguely
+if [ "$fork" = true ]; then
+    fork="mainnet"
+fi
+
+# Default to mainnet if network is not set or set vaguely
+if [ -z "$network" ] || [ "$network" = true ]; then
+    network="mainnet"
+fi
+
+# Default to true if upload is not set
+if [ -z "$upload" ]; then
+    upload="enabled"
+fi
+
+
+export CHAINS=$chains
+export FORK=$fork
+export NETWORK=$network
+export UPLOAD=$upload
+
+commands=("npm run dev --workspace @casimir/crawler")
+
+# Loop over comma-separated string of chains
+IFS=',' read -r -a chain_list <<< "$chains"
+
+for chain in "${chain_list[@]}"
+do
+    # Expose RPC URL directly if network is set to mainnet or testnet
+    if [ -n "$network" ]; then
+        # Get the RPC API key from AWS
+        rpc_secret_id=consensus-networks-$chain-$network
+        rpc_key=$(aws secretsmanager get-secret-value \
+        --secret-id $rpc_secret_id \
+        --query SecretString \
+        --output text \
+        --profile $profile)
+
+        CHAIN=$(echo $chain | tr '[:lower:]' '[:upper:]')
+        rpc_base=${chain}_${network}
+        export "PUBLIC_${CHAIN}_RPC"="${!rpc_base}/$rpc_key"
+    else
+        # Pass fork or network to chain-specific script
+        commands+=("npm run dev:$chain --fork=$fork")
+    fi
+done
+
+for command in "${commands[@]}"
+do
+    $command &
+done
+
+wait && pkill -P $$

scripts/crawler/test

@@ -67,7 +67,7 @@ do
 
         CHAIN=$(echo $chain | tr '[:lower:]' '[:upper:]')
         rpc_base=${chain}_${network}
-        export "PUBLIC_${CHAIN}_RPC"="${!rpc_base}/$rpc_key"
+        export PUBLIC_${CHAIN}_RPC="${!rpc_base}/$rpc_key"
     else
         # Pass fork or network to chain-specific script
         commands+=("npm run dev:$chain --fork=$fork")

services/crawler/README.md

@@ -1 +1,34 @@
-## Crawler
+# @casimir/crawler
+
+Identify, structure and stream [@casimir/data events](common/data/src/schemas/event.schema.json) to s3.
+
+> 🚩 Run all commands from the monorepo root.
+
+## Development
+
+Run the crawler with default chains and mainnet networks.
+
+```zsh
+npm run dev:crawler # --upload=disabled disables s3 upload
+```
+
+**Available flags:**
+
+These flags modify the CHAINS, FORK, NETWORK, UPLOAD AND PUBLIC_${CHAIN}_RPC environment variables.
+
+| Name | Description | Default |
+| - | - | - |
+| --chains | Comma-separated list of chains | ethereum |
+| --fork | Network state to fork | mainnet |
+| --network | Network to query | mainnet |
+| --upload | Enable/disable upload to s3 | enabled |
+
+> Use an equals sign to set flags to variables, like `--upload=disabled`.
+
+## Testing
+
+Test the crawler with [./test](./test/) files.
+
+```zsh
+npm run test:crawler
+```