Add database schema docs

apps/web/src/composables/ledger.ts

@@ -1,5 +1,4 @@
-import EthersLedgerSigner from '@casimir/ethers-ledger-signer'
-import BitcoinLedgerSigner from '@casimir/bitcoin-ledger-signer'
+import { BitcoinLedgerSigner, EthersLedgerSigner } from '@casimir/wallets'
 import { ethers } from 'ethers'
 import { TransactionInit } from '@/interfaces/TransactionInit'
 import { MessageInit } from '@/interfaces/MessageInit'

apps/web/src/composables/trezor.ts

@@ -1,4 +1,4 @@
-import EthersTrezorSigner from '@casimir/ethers-trezor-signer'
+import { EthersTrezorSigner } from '@casimir/wallets'
 import useEthers from '@/composables/ethers'
 import useEnvironment from '@/composables/environment'
 import { ethers } from 'ethers'

common/data/README.md

@@ -2,15 +2,54 @@
 
 JSON Schemas and Jupyter Notebooks for Casimir data modeling, exploration, and analytics
 
-## JSON Schemas
+## Database Schemas
 
-Find the `event` table schema in [src/schemas/event.schema.json](src/schemas/event.schema.json) and the `agg` table schema in [src/schemas/agg.schema.json](src/schemas/agg.schema.json).
+Find the core JSON object schemas in [src/schemas/](src/schemas/). These are the source of truth for data modeling in Casimir. When we deploy our Glue and Postgres tables, we use the schemas to generate columns from the object properties. See the reference tables below (one for Glue, and one for Postgres) for descriptions and links to the current schemas.
 
-### Making Changes
+> 🚩 To make a schema change, create a branch from `develop`, edit the JSON, and then make a PR to `develop`.
 
-The JSON Schemas in [src/schemas](src/schemas/) are the source of truth for the data model. When we deploy our Glue tables, we use the JSON Schemas to generate the Glue columns. To make a schema change, create a branch from `develop`, edit the JSON, and then make a PR to `develop`.
+### Glue
 
-> 🚩 Schema versioning is coming soon.
+| Table | Schema | Description |
+| --- | --- | --- |
+| `events` | [event.schema.json](src/schemas/event.schema.json) | on or off-chain event |
+| `aggs` | [agg.schema.json](src/schemas/agg.schema.json) | aggregate of events |
+
+### Postgres
+
+| Table | Schema | Description |
+| --- | --- | --- |
+| `accounts` | [account.schema.json](src/schemas/account.schema.json) | wallet account |
+| `users` | [user.schema.json](src/schemas/user.schema.json) | user profile |
+
+Run a local Postgres instance with the schemas above.
+
+```zsh
+npm run dev:postgres --workspace @casimir/data
+```
+
+Run and watch a local Postgres instance.
+
+```zsh
+npm run watch:postgres --workspace @casimir/data
+```
+
+Query the local Postgres instance.
+
+```sql
+"Add a user"
+
+INSERT INTO users (address) VALUES ('0xd557a5745d4560B24D36A68b52351ffF9c86A212');
+
+"Add an account (with the same address as the user)"
+
+INSERT INTO accounts (address, owner_address) VALUES ('0xd557a5745d4560B24D36A68b52351ffF9c86A212', '0xd557a5745d4560B24D36A68b52351ffF9c86A212');
+
+"Query the user"
+
+SELECT u.*, json_agg(a.*) AS accounts FROM users u JOIN accounts a ON u.address = a.owner_address WHERE u.address = '0xd557a5745d4560B24D36A68b52351ffF9c86A212' GROUP BY u.address;
+
+```
 
 ## Notebooks
 

common/data/package.json

@@ -6,13 +6,13 @@
         "build": "echo '@casimir/data build not specified. Disregard this warning and any listed errors above if @casimir/types is not needed for the current project build.' && exit 0",
         "clean": "npx rimraf ./scripts/pgdata ./scripts/sql",
         "configure:python": "poetry install && poetry run ipython kernel install --user --name=casimir-data",
-        "dev": "ts-node --transpile-only scripts/db.ts --clean \"$npm_config_clean\" --tables \"$npm_config_tables\"",
-        "seed": "ts-node --transpile-only scripts/seed.ts",
-        "watch": "ts-node-dev --watch src --respawn --transpile-only scripts/db.ts --clean \"$npm_config_clean\" --tables \"$npm_config_tables\"",
+        "dev:postgres": "ts-node --transpile-only scripts/postgres.ts --clean \"$npm_config_clean\" --tables \"$npm_config_tables\"",
+        "watch:postgres": "ts-node-dev --watch src --respawn --transpile-only scripts/postgres.ts --clean \"$npm_config_clean\" --tables \"$npm_config_tables\"",
         "test": "echo \"Error: no test specified\" && exit 1"
     },
     "dependencies": {
-        "@aws-cdk/aws-glue-alpha": "^2.33.0-alpha.0"
+        "@aws-cdk/aws-glue-alpha": "^2.33.0-alpha.0",
+        "pg": "^8.10.0"
     },
     "devDependencies": {
         "@types/node": "^17.0.38",

common/data/scripts/db.ts → common/data/scripts/postgres.ts

@@ -11,11 +11,10 @@ const tableSchemas = {
 }
 
 /**
- * Run a local PG database with the given tables
+ * Run a local postgres database with the given tables
  * 
  * Arguments:
  *     --clean: delete existing pgdata before deploy (optional, i.e., --clear)
- *     --seed: seed database with test data (optional, i.e., --seed)
  *     --tables: tables to deploy (optional, i.e., --tables=accounts,users)
  */
 void async function () {
@@ -47,10 +46,4 @@ void async function () {
 
     /** Start local database */
     await spawnPromise('docker compose -f ./scripts/docker-compose.yaml up -d')
-
-    /** Default to no seed data */
-    const seed = argv.seed === true || argv.seed === 'true'
-    if (seed) {
-        await spawnPromise('npm run seed --workspace @casimir/data')
-    }
 }()

common/data/src/index.ts

@@ -5,8 +5,9 @@ import eventSchema from './schemas/event.schema.json'
 import aggSchema from './schemas/agg.schema.json'
 import operatorStore from './mock/operator.store.json'
 import validatorStore from './mock/validator.store.json'
-import { JsonSchema } from './interfaces/JsonSchema'
+import { Postgres } from './providers/postgres'
 import { JsonType, GlueType, PgType, Schema } from './providers/schema'
+import { JsonSchema } from './interfaces/JsonSchema'
 
 export {
   accountSchema,
@@ -16,6 +17,7 @@ export {
   aggSchema,
   operatorStore,
   validatorStore,
+  Postgres,
   Schema
 }
 

services/users/src/providers/postgres.ts → common/data/src/providers/postgres.ts

@@ -1,38 +1,41 @@
-import { Pool } from 'pg'
+import { Pool, PoolConfig } from 'pg'
 import { pascalCase } from '@casimir/helpers'
 
+/**
+ * Postgres database provider with pool client auto-connect-and-release
+ */
 export class Postgres {
     /** Postgres connection pool */
     private pool: Pool
 
-    constructor() {
-        this.pool = new Pool({ // These will become environment variables
-            host: '0.0.0.0',
-            port: 5432,
-            database: 'postgres',
-            user: 'postgres',
-            password: 'postgres'
-        })
+    /**
+     * Create a new Postgres database provider
+     * @param {PoolConfig} config - Postgres connection pool config
+     * @example
+     * ```ts
+     * const postgres = new Postgres()
+     * ```
+     */
+    constructor(config?: PoolConfig) {
+        this.pool = new Pool(config)
         process.on('exit', () => this.close())
     }
 
     /**
-     * Query the database (with pool client auto-connect-and-release)
+     * Query the database
      * @param text SQL query
      * @param params Query parameters
      * @returns Query result
      * 
      * @example
      * ```ts
-     * const pg = new Postgres()
-     * const res = await pg.query('SELECT $1::text as message', ['Hello world!'])
-     * console.log(res.rows[0].message) // Hello world!
+     * const { rows } = await postgres.query('SELECT * FROM messages WHERE text = $1', ['Hello world!'])
+     * if (rows.length) console.log(rows[0].text) // Hello world!
      * ```
      */
-    async query(text: string, params: any[] = []) { // Todo - use strict @casimir/types for params
+    async query(text: string, params: any[] = []) { // Todo - use union of stricter @casimir/types for params
         const client = await this.pool.connect()
         const res = await client.query(text, params)
-        console.log('Result:', res)
         const { rows } = res
 
         /** Convert snake_case to PascalCase */

common/data/src/providers/schema.ts

@@ -3,7 +3,7 @@ import { JsonSchema } from '../interfaces/JsonSchema'
 
 export type JsonType = 'string' | 'number' | 'integer' | 'boolean' | 'object' | 'array' | 'null'
 export type GlueType = glue.Type
-export type PgType = 'string' | 'integer' | 'boolean' | 'double' | 'decimal' | 'bigint' | 'timestamp' | 'json' | 'date'
+export type PgType = 'STRING' | 'INTEGER' | 'BOOLEAN' | 'DOUBLE' | 'DECIMAL' | 'BIGINT' | 'TIMESTAMP' | 'JSON' | 'DATE'
 
 export class Schema {
     /** Input JSON Schema object */
@@ -66,29 +66,29 @@ export class Schema {
         const columns = Object.keys(this.jsonSchema.properties).map((name: string) => {
             const property = this.jsonSchema.properties[name]
             let type = {
-                string: 'varchar',
-                number: 'double',
-                integer: 'integer',
-                boolean: 'boolean',
-                object: 'json',
-                array: 'json',
-                null: 'varchar'
+                string: 'VARCHAR',
+                number: 'DOUBLE',
+                integer: 'INTEGER',
+                boolean: 'BOOLEAN',
+                object: 'JSON',
+                array: 'JSON',
+                null: 'VARCHAR'
             }[property.type as JsonType] as PgType
 
-            if (name.endsWith('_at')) type = 'timestamp'
-            if (name.includes('balance')) type = 'bigint'
+            if (name.endsWith('_at')) type = 'TIMESTAMP'
+            if (name.includes('balance')) type = 'BIGINT'
 
             let column = `${name} ${type}`
 
             const comment = property.description
-            if (comment.includes('PK')) column += ' primary key'
+            if (comment.includes('PK')) column += ' PRIMARY KEY'
 
             return column
         })
 
         /** Make table name plural of schema objects (todo: check edge-cases) */
         const tableName = this.jsonSchema.title.toLowerCase() + 's'
 
-        return `create table ${tableName} (\n\t${columns.join(',\n\t')}\n);`
+        return `CREATE TABLE ${tableName} (\n\t${columns.join(',\n\t')}\n);`
     }
 }