From e379699486b621aa1e87b407f4d30fd95e510799 Mon Sep 17 00:00:00 2001 From: Benjamin Goering <171782+gobengo@users.noreply.github.com> Date: Wed, 22 Mar 2023 23:45:05 -0700 Subject: [PATCH] assert cid string kind in d1 cid column and r2 key --- .../migrations/0007_add_delegations_v3.sql | 1 + packages/access-api/src/models/delegations.js | 23 +++-- .../test/delegations-storage.test.js | 93 +++++++++++++++---- 3 files changed, 88 insertions(+), 29 deletions(-) diff --git a/packages/access-api/migrations/0007_add_delegations_v3.sql b/packages/access-api/migrations/0007_add_delegations_v3.sql index 6078d2347..4220ef3e0 100644 --- a/packages/access-api/migrations/0007_add_delegations_v3.sql +++ b/packages/access-api/migrations/0007_add_delegations_v3.sql @@ -9,6 +9,7 @@ context: we're going to start storing bytes outside of the database (e.g. in R2) CREATE TABLE IF NOT EXISTS delegations_v3 ( + /* cidv1 dag-ucan/dag-cbor sha2-256 */ cid TEXT NOT NULL PRIMARY KEY, audience TEXT NOT NULL, issuer TEXT NOT NULL, diff --git a/packages/access-api/src/models/delegations.js b/packages/access-api/src/models/delegations.js index 011667cf9..a4aa47263 100644 --- a/packages/access-api/src/models/delegations.js +++ b/packages/access-api/src/models/delegations.js @@ -3,6 +3,8 @@ import { delegationsToBytes, bytesToDelegations, } from '@web3-storage/access/encoding' +import { base32 } from 'multiformats/bases/base32' +import { CID } from 'multiformats' /** * @typedef {import('../types/access-api-cf-db').R2Bucket} R2Bucket @@ -34,7 +36,7 @@ export class UnexpectedDelegation extends Error { */ export function createDelegationRowUpdateV3(d) { return { - cid: d.cid.toV1().toString(), + cid: d.cid.toString(), audience: d.audience.did(), issuer: d.issuer.did(), } @@ -69,7 +71,6 @@ export class DbDelegationsStorageWithR2 { /** @type {DB} */ #db #delegationsTableName = delegationsV3Table - /* @type {(d: { cid: string }) => string} */ #getDagsKey = carFileKeyer /** @@ -146,14 +147,15 @@ export class DbDelegationsStorageWithR2 { /** * @param {Pick} row * @param {R2Bucket} dags - * @param {(d: { cid: string }) => string} keyer - builds k/v key strings for each delegation + * @param {(d: { cid: Ucanto.Delegation['cid'] }) => string} keyer - builds k/v key strings for each delegation * @returns {Promise} */ async #rowToDelegation(row, dags = this.#dags, keyer = this.#getDagsKey) { - const cidString = row.cid.toString() - const carBytesR2 = await dags.get(keyer({ cid: cidString })) + const cid = /** @type {Ucanto.UCANLink} */ (CID.parse(row.cid)) + const key = keyer({ cid }) + const carBytesR2 = await dags.get(key) if (!carBytesR2) { - throw new Error(`failed to read car bytes for cid ${cidString}`) + throw new Error(`failed to read car bytes for cid ${row.cid} key ${key}`) } const carBytes = new Uint8Array(await carBytesR2.arrayBuffer()) const delegations = bytesToDelegations(carBytes) @@ -187,22 +189,23 @@ async function count(db, delegationsTable) { } /** - * @param {{ cid: string }} ucan + * @param {{ cid: Ucanto.Delegation['cid'] }} ucan */ function carFileKeyer(ucan) { - return /** @type {const} */ (`${ucan.cid.toString()}.car`) + const key = /** @type {const} */ (`${ucan.cid.toString(base32)}.car`) + return key } /** * @param {R2Bucket} bucket * @param {Iterable} delegations - * @param {(d: { cid: string }) => string} keyer - builds k/v key strings for each delegation + * @param {(d: { cid: Ucanto.Delegation['cid'] }) => string} keyer - builds k/v key strings for each delegation */ async function writeDelegations(bucket, delegations, keyer) { return writeEntries( bucket, [...delegations].map((delegation) => { - const key = keyer({ cid: delegation.cid.toString() }) + const key = keyer(delegation) const carBytes = delegationsToBytes([delegation]) const value = carBytes return /** @type {[key: string, value: Uint8Array]} */ ([key, value]) diff --git a/packages/access-api/test/delegations-storage.test.js b/packages/access-api/test/delegations-storage.test.js index 271acfe66..d27114744 100644 --- a/packages/access-api/test/delegations-storage.test.js +++ b/packages/access-api/test/delegations-storage.test.js @@ -7,14 +7,12 @@ import * as principal from '@ucanto/principal' import * as Ucanto from '@ucanto/interface' import * as ucanto from '@ucanto/core' import { collect } from 'streaming-iterables' +import { CID } from 'multiformats' +import { base32 } from 'multiformats/bases/base32' describe('DelegationsStorage with sqlite+R2', () => { - testVariant( - () => createDbDelegationsStorageVariantWithR2().create(), - (name, doTest) => { - it(name, doTest) - } - ) + testVariant(createDbDelegationsStorageVariantWithR2, it) + testCloudflareVariant(createDbDelegationsStorageVariantWithR2, it) }) /** @@ -48,20 +46,17 @@ async function createDelegation(opts = {}) { * * @see https://github.com/web3-storage/w3protocol/issues/571 */ -function createDbDelegationsStorageVariantWithR2() { +async function createDbDelegationsStorageVariantWithR2() { + const { d1, mf } = await context() + const accessApiR2 = await mf.getR2Bucket('ACCESS_API_R2') + const delegationsStorage = new DbDelegationsStorageWithR2( + createD1Database(d1), + accessApiR2 + ) return { - /** - * @returns {Promise} - */ - create: async () => { - const { d1, mf } = await context() - const accessApiR2 = await mf.getR2Bucket('ACCESS_API_R2') - const delegationsStorage = new DbDelegationsStorageWithR2( - createD1Database(d1), - accessApiR2 - ) - return { delegations: delegationsStorage } - }, + d1, + delegations: delegationsStorage, + r2: accessApiR2, } } @@ -130,3 +125,63 @@ function testVariant(createVariant, test) { assert.deepEqual(carolDelegations.length, 0) }) } + +/** + * @param {() => Promise} createVariant - create a new test context + * @param {(name: string, test: () => Promise) => void} test - name a test + */ +function testCloudflareVariant(createVariant, test) { + test('puts into d1+r2', async () => { + const { d1, delegations, r2 } = await createVariant() + const multibasePrefixes = { base32: base32.prefix } + const expectMultibase = 'base32' + const multihashCodes = { + // https://github.com/multiformats/multicodec/blob/aa0c3a41473c0a3796cdf2175ac5552989b2a905/table.csv#L9 + 'sha2-256': 0x12, + } + const expectMultihash = multihashCodes['sha2-256'] + + const ucan1 = await createSampleDelegation() + await delegations.putMany(ucan1) + const listResult = await r2.list() + assert.deepEqual(listResult.objects.length, 1) + const r2KeyCidString = listResult.objects[0].key.split('.')[0] + + assert.deepEqual( + r2KeyCidString[0], + multibasePrefixes[expectMultibase], + `r2 key cid string uses multibase ${expectMultibase}` + ) + const r2KeyCid = CID.parse(r2KeyCidString) + + assert.deepEqual(r2KeyCid.version, 1) + assert.deepEqual(r2KeyCid.code, ucanto.UCAN.code) + assert.deepEqual( + r2KeyCid.multihash.code, + expectMultihash, + `keyCid multihash code is ${expectMultihash}` + ) + + // d1 cid column + const delegationsFromD1 = await d1 + .prepare(`select cid from delegations_v3`) + .all() + assert.equal(delegationsFromD1.results?.length, 1) + const d1CidString = /** @type {{cid:string}} */ ( + delegationsFromD1.results?.[0] + ).cid + assert.deepEqual( + d1CidString[0], + multibasePrefixes[expectMultibase], + `d1 cid column uses multibase ${expectMultibase}` + ) + const d1Cid = CID.parse(d1CidString) + assert.deepEqual(d1Cid.version, 1) + assert.deepEqual(d1Cid.code, ucanto.UCAN.code) + assert.deepEqual( + d1Cid.multihash.code, + expectMultihash, + `d1Cid multihash code is ${expectMultihash}` + ) + }) +}