feat(data-store): add migration of key stores

__tests__/migration.test.ts

@@ -0,0 +1,142 @@
+import { createAgent, TAgent, IDIDManager, IResolver, IKeyManager, IDataStore } from '@veramo/core'
+import { createConnection, Connection } from 'typeorm'
+import { DIDResolverPlugin } from '../packages/did-resolver'
+import { EthrDIDProvider } from '../packages/did-provider-ethr'
+import { WebDIDProvider } from '../packages/did-provider-web'
+import { KeyDIDProvider } from '../packages/did-provider-key'
+import { DIDComm, IDIDComm } from '../packages/did-comm'
+import { KeyManagementSystem, SecretBox } from '../packages/kms-local'
+import {
+  Entities,
+  IDataStoreORM,
+  DataStore,
+  DataStoreORM,
+  KeyStore,
+  DIDStore,
+  PrivateKeyStore,
+  migrations,
+} from '../packages/data-store'
+import { Resolver } from 'did-resolver'
+import { getResolver as ethrDidResolver } from 'ethr-did-resolver'
+import { getResolver as webDidResolver } from 'web-did-resolver'
+import { getDidKeyResolver } from '../packages/did-provider-key'
+import { KeyManager } from '../packages/key-manager'
+import { DIDManager } from '../packages/did-manager'
+import { FakeDidProvider, FakeDidResolver } from './utils/fake-did'
+import fs from 'fs'
+
+jest.setTimeout(30000)
+
+const databaseBeforeFile = __dirname + '/fixtures/local-database-before-3.0.sqlite'
+const databaseFile = __dirname + '/migrated.database.sqlite'
+const infuraProjectId = '5ffc47f65c4042ce847ef66a3fa70d4c'
+const secretKey = '29739248cad1bd1a0fc4d9b75cd4d2990de535baf5caadfdf8d8f86664aa830c'
+
+describe('database migration tests', () => {
+  describe('using pre-migration database fixture', () => {
+    type TestingAgentPlugins = IDIDManager & IKeyManager & IDataStore & IDataStoreORM & IResolver & IDIDComm
+    let agent: TAgent<TestingAgentPlugins>
+    let dbConnection: Promise<Connection>
+
+    beforeAll(async () => {
+      fs.copyFileSync(databaseBeforeFile, databaseFile)
+
+      dbConnection = createConnection({
+        name: 'test',
+        type: 'sqlite',
+        database: databaseFile,
+        synchronize: false,
+        migrations: migrations,
+        migrationsRun: true,
+        logging: false,
+        entities: Entities,
+      })
+
+      agent = createAgent<TestingAgentPlugins>({
+        context: {
+          // authenticatedDid: 'did:example:3456'
+        },
+        plugins: [
+          new KeyManager({
+            store: new KeyStore(dbConnection),
+            kms: {
+              local: new KeyManagementSystem({
+                keyStore: new PrivateKeyStore(dbConnection, new SecretBox(secretKey)),
+              }),
+            },
+          }),
+          new DIDManager({
+            store: new DIDStore(dbConnection),
+            defaultProvider: 'did:ethr:goerli',
+            providers: {
+              'did:ethr:goerli': new EthrDIDProvider({
+                defaultKms: 'local',
+                network: 'goerli',
+                rpcUrl: 'https://goerli.infura.io/v3/' + infuraProjectId,
+              }),
+              'did:web': new WebDIDProvider({
+                defaultKms: 'local',
+              }),
+              'did:key': new KeyDIDProvider({
+                defaultKms: 'local',
+              }),
+              'did:fake': new FakeDidProvider(),
+            },
+          }),
+          new DIDResolverPlugin({
+            resolver: new Resolver({
+              ...ethrDidResolver({ infuraProjectId }),
+              ...webDidResolver(),
+              ...getDidKeyResolver(),
+              ...new FakeDidResolver(() => agent).getDidFakeResolver(),
+            }),
+          }),
+          new DataStore(dbConnection),
+          new DataStoreORM(dbConnection),
+          new DIDComm(),
+        ],
+      })
+      return true
+    })
+    afterAll(async () => {
+      await (await dbConnection).close()
+      fs.unlinkSync(databaseFile)
+    })
+
+    it('loads a migrated key', async () => {
+      // output of agent.keyManagerGet() before migration
+      const key = {
+        kid: '04539ffde912c094bc48b64c9ee71b2baece24c710bcad2c7bacced615f60ae53949cdc95379eb50556d11cb0afab0e5a6ca8cb475d413b2f12307cc2d7f5438de',
+        kms: 'local',
+        type: 'Secp256k1',
+        publicKeyHex:
+          '04539ffde912c094bc48b64c9ee71b2baece24c710bcad2c7bacced615f60ae53949cdc95379eb50556d11cb0afab0e5a6ca8cb475d413b2f12307cc2d7f5438de',
+        privateKeyHex: 'a5e81a8cd50cf5c31d5b87db3e153e2817f86de350a60edc2335f76d5c3b4e0d',
+        meta: {
+          algorithms: ['ES256K', 'ES256K-R', 'eth_signTransaction', 'eth_signTypedData', 'eth_signMessage'],
+        },
+      }
+      const migratedKey = await agent.keyManagerGet({ kid: key.kid })
+      expect(migratedKey.kid).toEqual(key.kid)
+      expect(migratedKey).not.toHaveProperty('privateKeyHex')
+      const signedMessage = await agent.keyManagerSign({
+        data: 'hello world',
+        keyRef: migratedKey.kid,
+        algorithm: 'ES256K',
+        encoding: 'utf-8',
+      })
+      expect(signedMessage).toEqual(
+        'vzDocUViJh7ooOCZ-jBHKZddEsTa4yClHwhIL9SHJwjAv3bC6TZIcUnX36ZqNBWvLbnNAQvdtzqrVf3l0pv3QQ',
+      )
+    })
+
+    it('unpacks DIDComm message intended for migrated managed key', async () => {
+      const packed = {
+        message:
+          '{"protected":"eyJ0eXAiOiJhcHBsaWNhdGlvbi9kaWRjb21tLWVuY3J5cHRlZCtqc29uIiwiZW5jIjoiWEMyMFAifQ","iv":"mBAgYLce2JpmKtmlNQLG6w9lm6kqf4Ne","ciphertext":"D9_7Xxj51xn3T9yBU-rZmxSTrR82Pi4G7hWCDSxSpRUlmUh2uJoqeCHixSTFeZvFAfw2ryROjrxbpCh5Arg-wqrW3WwKGpVFHXO_r0jHso5lNMO-vGjxOULN","tag":"9Qs-esw1tcnM0jE_Q3LxIQ","recipients":[{"encrypted_key":"kGNaBfhPS2VETu-_iYaUwy13sC1ZVm3i_qYiYkuEleA","header":{"alg":"ECDH-ES+XC20PKW","iv":"1sK1pyOwy_hNY_WsJPGdoFqE8ken51IA","tag":"MplY66h-bHnuSdP1ZGLYyw","epk":{"kty":"OKP","crv":"X25519","x":"UZx8Uf3BJ-m3wm7sBjvCp1UXuHA9v0Qu5KvWfWyBNio"},"kid":"did:key:z6MkiPXoC2uAWPdQpotWxzNMJpaDbfPxaQWcbux5avNwEMfD#z6LSqL9zfeZa53RwpAkjxN7Gizvzv4rjAT7GwhLVYLPXK5dC"}}]}',
+      }
+      const msg = await agent.unpackDIDCommMessage(packed)
+      expect(msg.message.body).toEqual({ hello: 'world' })
+    })
+  })
+})

__tests__/shared/didManager.ts

@@ -298,7 +298,7 @@ export default (testContext: {
             id: `${did}#msg`,
             serviceEndpoint: 'https://example.org/messaging',
             type: 'Messaging',
-          } as Service,
+          },
         ],
       })
     })

packages/core/plugin.schema.json

@@ -655,7 +655,8 @@
               ],
               "description": "Optional. Key metadata. This should be used to determine which algorithms are supported."
             }
-          }
+          },
+          "description": "Represents an object type where a subset of keys are required and everything else is optional."
         },
         "IKeyManagerSharedSecretArgs": {
           "type": "object",
@@ -1271,7 +1272,8 @@
               ],
               "description": "Optional. Key metadata. This should be used to determine which algorithms are supported."
             }
-          }
+          },
+          "description": "Represents an object type where a subset of keys are required and everything else is optional."
         },
         "IDIDManagerRemoveKeyArgs": {
           "type": "object",

packages/data-store/src/entities/PreMigrationEntities.ts

@@ -0,0 +1,15 @@
+import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm'
+
+@Entity('key')
+export class PreMigrationKey extends BaseEntity {
+  @PrimaryColumn()
+  //@ts-ignore
+  kid: string
+
+  @Column()
+  //@ts-ignore
+  type: KeyType
+
+  @Column({ nullable: true })
+  privateKeyHex?: string
+}

packages/data-store/src/index.ts

@@ -34,8 +34,9 @@ import { Presentation } from './entities/presentation'
 import { Service } from './entities/service'
 import { Message, MetaData } from './entities/message'
 import { PrivateKey } from './entities/private-key'
-export const Entities = [Key, Identifier, Message, Claim, Credential, Presentation, Service, PrivateKey]
-export { KeyType, Key, Identifier, Message, Claim, Credential, Presentation, MetaData, Service, PrivateKey }
+import { PreMigrationKey } from './entities/PreMigrationEntities'
+export const Entities = [Key, Identifier, Message, Claim, Credential, Presentation, Service, PrivateKey, PreMigrationKey]
+export { KeyType, Key, Identifier, Message, Claim, Credential, Presentation, MetaData, Service, PrivateKey, PreMigrationKey }
 export { migrations } from './migrations'
 const schema = require('../plugin.schema.json')
 export { schema }

packages/data-store/src/migrations/createPrivateKeyStorage.ts

@@ -0,0 +1,83 @@
+import { MigrationInterface, QueryRunner, Table, TableColumn } from 'typeorm'
+import { PrivateKey } from '..'
+import { PreMigrationKey } from '../entities/PreMigrationEntities'
+import Debug from 'debug'
+const debug = Debug('veramo:data-store:key-migration')
+
+/**
+ * Migration of existing private keys from Veramo 2.x to Veramo 3.x
+ */
+export class CreatePrivateKeyStorage1629293428674 implements MigrationInterface {
+  async up(queryRunner: QueryRunner): Promise<void> {
+    // 1.create new table
+    debug(`creating new private-key table`)
+    await queryRunner.createTable(
+      new Table({
+        name: 'private-key',
+        columns: [
+          {
+            name: 'alias',
+            type: 'varchar',
+            isPrimary: true,
+          },
+          {
+            name: 'type',
+            type: 'varchar',
+          },
+          {
+            name: 'privateKeyHex',
+            type: 'varchar',
+          },
+        ],
+      }),
+      true,
+    )
+    // 2. copy key data
+    const keys: PreMigrationKey[] = await queryRunner.manager.find(PreMigrationKey)
+    debug(`got ${keys.length} potential keys to migrate`)
+    const privKeys = keys
+      .filter((key) => typeof key.privateKeyHex !== 'undefined' && key.privateKeyHex !== null)
+      .map((key) => ({
+        alias: key.kid,
+        type: key.type,
+        privateKeyHex: key.privateKeyHex,
+      }))
+    debug(`${privKeys.length} keys need to be migrated`)
+    await queryRunner.manager.createQueryBuilder().insert().into('private-key').values(privKeys).execute()
+    // 3. drop old column
+    debug(`dropping privKeyHex column from old key table`)
+    await queryRunner.dropColumn('key', 'privateKeyHex')
+    //4. done
+    debug(`migrated ${privKeys.length} keys to private key storage`)
+  }
+
+  async down(queryRunner: QueryRunner): Promise<void> {
+    // 1. add old column back
+    debug(`adding back privateKeyHex column to key table`)
+    await queryRunner.addColumn(
+      'key',
+      new TableColumn({
+        name: 'privateKeyHex',
+        type: 'varchar',
+        isNullable: true,
+      }),
+    )
+    // 2. copy key data
+    debug(`checking keys to be rolled back`)
+    const keys: PrivateKey[] = await queryRunner.manager.find(PrivateKey)
+    debug(`copying ${keys.length} keys`)
+    for (const key of keys) {
+      await queryRunner.manager
+      .createQueryBuilder()
+      .update(PreMigrationKey)
+      .set({ privateKeyHex: key.privateKeyHex })
+      .where('kid = :alias', { alias: key.alias })
+      .execute()
+    }
+    debug(`dropping private-key table`)
+    // 3. drop the new private key table
+    await queryRunner.dropTable('private-key')
+    // 4. done
+    debug(`rolled back ${keys.length} keys`)
+  }
+}

packages/data-store/src/migrations/index.ts

@@ -1 +1,2 @@
-export const migrations = []
+import { CreatePrivateKeyStorage1629293428674 } from './createPrivateKeyStorage'
+export const migrations = [CreatePrivateKeyStorage1629293428674]

packages/did-comm/src/utils.ts

@@ -196,8 +196,8 @@ export async function mapIdentifierKeysToDoc(
 export async function resolveDidOrThrow(didUrl: string, context: IAgentContext<IResolver>) {
   // TODO: add caching
   const docResult = await context.agent.resolveDid({ didUrl: didUrl })
-  const err = docResult.didResolutionMetadata.error
-  const msg = docResult.didResolutionMetadata.message
+  const err = docResult?.didResolutionMetadata?.error
+  const msg = docResult?.didResolutionMetadata?.message
   const didDocument = docResult.didDocument
   if (!isDefined(didDocument) || err) {
     throw new Error(`not_found: could not resolve DID document for '${didUrl}': ${err} ${msg}`)