diff --git a/lexicons/com/atproto/sync/getCheckout.json b/lexicons/com/atproto/sync/getCheckout.json new file mode 100644 index 00000000000..f71dd046846 --- /dev/null +++ b/lexicons/com/atproto/sync/getCheckout.json @@ -0,0 +1,21 @@ +{ + "lexicon": 1, + "id": "com.atproto.sync.getCheckout", + "defs": { + "main": { + "type": "query", + "description": "Gets the repo state.", + "parameters": { + "type": "params", + "required": ["did"], + "properties": { + "did": {"type": "string", "description": "The DID of the repo."}, + "commit": {"type": "string", "description": "The commit to get the checkout from. Defaults to current HEAD."} + } + }, + "output": { + "encoding": "application/vnd.ipld.car" + } + } + } +} \ No newline at end of file diff --git a/lexicons/com/atproto/sync/getCommitPath.json b/lexicons/com/atproto/sync/getCommitPath.json new file mode 100644 index 00000000000..adfb5b8eb1b --- /dev/null +++ b/lexicons/com/atproto/sync/getCommitPath.json @@ -0,0 +1,32 @@ +{ + "lexicon": 1, + "id": "com.atproto.sync.getCommitPath", + "defs": { + "main": { + "type": "query", + "description": "Gets the path of repo commits", + "parameters": { + "type": "params", + "required": ["did"], + "properties": { + "did": {"type": "string", "description": "The DID of the repo."}, + "latest": { "type": "string", "description": "The most recent commit"}, + "earliest": { "type": "string", "description": "The earliest commit to start from"} + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["commits"], + "properties": { + "commits": { + "type": "array", + "items": { "type": "string" } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/lexicons/com/atproto/sync/getRoot.json b/lexicons/com/atproto/sync/getHead.json similarity index 83% rename from lexicons/com/atproto/sync/getRoot.json rename to lexicons/com/atproto/sync/getHead.json index 93255fb2477..bcb713bcd86 100644 --- a/lexicons/com/atproto/sync/getRoot.json +++ b/lexicons/com/atproto/sync/getHead.json @@ -1,10 +1,10 @@ { "lexicon": 1, - "id": "com.atproto.sync.getRoot", + "id": "com.atproto.sync.getHead", "defs": { "main": { "type": "query", - "description": "Gets the current root CID of a repo.", + "description": "Gets the current HEAD CID of a repo.", "parameters": { "type": "params", "required": ["did"], diff --git a/lexicons/com/atproto/sync/getRecord.json b/lexicons/com/atproto/sync/getRecord.json new file mode 100644 index 00000000000..8ab77ceb34c --- /dev/null +++ b/lexicons/com/atproto/sync/getRecord.json @@ -0,0 +1,23 @@ +{ + "lexicon": 1, + "id": "com.atproto.sync.getRecord", + "defs": { + "main": { + "type": "query", + "description": "Gets blocks needed for existence or non-existence of record.", + "parameters": { + "type": "params", + "required": ["did", "collection", "rkey"], + "properties": { + "did": {"type": "string", "description": "The DID of the repo."}, + "collection": {"type": "string" }, + "rkey": {"type": "string" }, + "commit": {"type": "string", "description": "An optional past commit CID."} + } + }, + "output": { + "encoding": "application/vnd.ipld.car" + } + } + } +} \ No newline at end of file diff --git a/lexicons/com/atproto/sync/getRepo.json b/lexicons/com/atproto/sync/getRepo.json index 1956d312a7a..db8132a95d6 100644 --- a/lexicons/com/atproto/sync/getRepo.json +++ b/lexicons/com/atproto/sync/getRepo.json @@ -14,7 +14,7 @@ } }, "output": { - "encoding": "application/cbor" + "encoding": "application/vnd.ipld.car" } } } diff --git a/lexicons/com/atproto/sync/updateRepo.json b/lexicons/com/atproto/sync/updateRepo.json deleted file mode 100644 index e518cab81c1..00000000000 --- a/lexicons/com/atproto/sync/updateRepo.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "lexicon": 1, - "id": "com.atproto.sync.updateRepo", - "defs": { - "main": { - "type": "procedure", - "description": "Writes commits to a repo.", - "parameters": { - "type": "params", - "required": ["did"], - "properties": { - "did": {"type": "string", "description": "The DID of the repo."} - } - }, - "input": { - "encoding": "application/cbor" - } - } - } -} \ No newline at end of file diff --git a/packages/README.md b/packages/README.md index bb718fa5651..d7239708adc 100644 --- a/packages/README.md +++ b/packages/README.md @@ -9,7 +9,6 @@ ## Libraries - [API](./api): A library for communicating with ATP servers. -- [Auth](./auth): ATP's core permissioning library (based on UCANs). - [Common](./common): A library containing code which is shared between ATP packages. - [Crypto](./crypto): ATP's common cryptographic operations. - [DID Resolver](./did-resolver): A library for resolving ATP's Decentralized ID methods. diff --git a/packages/api/src/client/index.ts b/packages/api/src/client/index.ts index c1b61d10da2..0e6cf24cdbf 100644 --- a/packages/api/src/client/index.ts +++ b/packages/api/src/client/index.ts @@ -46,9 +46,11 @@ import * as ComAtprotoSessionCreate from './types/com/atproto/session/create' import * as ComAtprotoSessionDelete from './types/com/atproto/session/delete' import * as ComAtprotoSessionGet from './types/com/atproto/session/get' import * as ComAtprotoSessionRefresh from './types/com/atproto/session/refresh' +import * as ComAtprotoSyncGetCheckout from './types/com/atproto/sync/getCheckout' +import * as ComAtprotoSyncGetCommitPath from './types/com/atproto/sync/getCommitPath' +import * as ComAtprotoSyncGetHead from './types/com/atproto/sync/getHead' +import * as ComAtprotoSyncGetRecord from './types/com/atproto/sync/getRecord' import * as ComAtprotoSyncGetRepo from './types/com/atproto/sync/getRepo' -import * as ComAtprotoSyncGetRoot from './types/com/atproto/sync/getRoot' -import * as ComAtprotoSyncUpdateRepo from './types/com/atproto/sync/updateRepo' import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile' import * as AppBskyActorGetSuggestions from './types/app/bsky/actor/getSuggestions' import * as AppBskyActorProfile from './types/app/bsky/actor/profile' @@ -125,9 +127,11 @@ export * as ComAtprotoSessionCreate from './types/com/atproto/session/create' export * as ComAtprotoSessionDelete from './types/com/atproto/session/delete' export * as ComAtprotoSessionGet from './types/com/atproto/session/get' export * as ComAtprotoSessionRefresh from './types/com/atproto/session/refresh' +export * as ComAtprotoSyncGetCheckout from './types/com/atproto/sync/getCheckout' +export * as ComAtprotoSyncGetCommitPath from './types/com/atproto/sync/getCommitPath' +export * as ComAtprotoSyncGetHead from './types/com/atproto/sync/getHead' +export * as ComAtprotoSyncGetRecord from './types/com/atproto/sync/getRecord' export * as ComAtprotoSyncGetRepo from './types/com/atproto/sync/getRepo' -export * as ComAtprotoSyncGetRoot from './types/com/atproto/sync/getRoot' -export * as ComAtprotoSyncUpdateRepo from './types/com/atproto/sync/updateRepo' export * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile' export * as AppBskyActorGetSuggestions from './types/app/bsky/actor/getSuggestions' export * as AppBskyActorProfile from './types/app/bsky/actor/profile' @@ -662,36 +666,58 @@ export class SyncNS { this._service = service } - getRepo( - params?: ComAtprotoSyncGetRepo.QueryParams, - opts?: ComAtprotoSyncGetRepo.CallOptions, - ): Promise { + getCheckout( + params?: ComAtprotoSyncGetCheckout.QueryParams, + opts?: ComAtprotoSyncGetCheckout.CallOptions, + ): Promise { return this._service.xrpc - .call('com.atproto.sync.getRepo', params, undefined, opts) + .call('com.atproto.sync.getCheckout', params, undefined, opts) .catch((e) => { - throw ComAtprotoSyncGetRepo.toKnownErr(e) + throw ComAtprotoSyncGetCheckout.toKnownErr(e) }) } - getRoot( - params?: ComAtprotoSyncGetRoot.QueryParams, - opts?: ComAtprotoSyncGetRoot.CallOptions, - ): Promise { + getCommitPath( + params?: ComAtprotoSyncGetCommitPath.QueryParams, + opts?: ComAtprotoSyncGetCommitPath.CallOptions, + ): Promise { return this._service.xrpc - .call('com.atproto.sync.getRoot', params, undefined, opts) + .call('com.atproto.sync.getCommitPath', params, undefined, opts) .catch((e) => { - throw ComAtprotoSyncGetRoot.toKnownErr(e) + throw ComAtprotoSyncGetCommitPath.toKnownErr(e) }) } - updateRepo( - data?: ComAtprotoSyncUpdateRepo.InputSchema, - opts?: ComAtprotoSyncUpdateRepo.CallOptions, - ): Promise { + getHead( + params?: ComAtprotoSyncGetHead.QueryParams, + opts?: ComAtprotoSyncGetHead.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.sync.getHead', params, undefined, opts) + .catch((e) => { + throw ComAtprotoSyncGetHead.toKnownErr(e) + }) + } + + getRecord( + params?: ComAtprotoSyncGetRecord.QueryParams, + opts?: ComAtprotoSyncGetRecord.CallOptions, + ): Promise { return this._service.xrpc - .call('com.atproto.sync.updateRepo', opts?.qp, data, opts) + .call('com.atproto.sync.getRecord', params, undefined, opts) .catch((e) => { - throw ComAtprotoSyncUpdateRepo.toKnownErr(e) + throw ComAtprotoSyncGetRecord.toKnownErr(e) + }) + } + + getRepo( + params?: ComAtprotoSyncGetRepo.QueryParams, + opts?: ComAtprotoSyncGetRepo.CallOptions, + ): Promise { + return this._service.xrpc + .call('com.atproto.sync.getRepo', params, undefined, opts) + .catch((e) => { + throw ComAtprotoSyncGetRepo.toKnownErr(e) }) } } diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 256ad6c0847..93a18c7fc91 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -1823,9 +1823,9 @@ export const schemaDict = { }, }, }, - ComAtprotoSyncGetRepo: { + ComAtprotoSyncGetCheckout: { lexicon: 1, - id: 'com.atproto.sync.getRepo', + id: 'com.atproto.sync.getCheckout', defs: { main: { type: 'query', @@ -1838,25 +1838,69 @@ export const schemaDict = { type: 'string', description: 'The DID of the repo.', }, - from: { + commit: { type: 'string', - description: 'A past commit CID.', + description: + 'The commit to get the checkout from. Defaults to current HEAD.', }, }, }, output: { - encoding: 'application/cbor', + encoding: 'application/vnd.ipld.car', + }, + }, + }, + }, + ComAtprotoSyncGetCommitPath: { + lexicon: 1, + id: 'com.atproto.sync.getCommitPath', + defs: { + main: { + type: 'query', + description: 'Gets the path of repo commits', + parameters: { + type: 'params', + required: ['did'], + properties: { + did: { + type: 'string', + description: 'The DID of the repo.', + }, + latest: { + type: 'string', + description: 'The most recent commit', + }, + earliest: { + type: 'string', + description: 'The earliest commit to start from', + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['commits'], + properties: { + commits: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, }, }, }, }, - ComAtprotoSyncGetRoot: { + ComAtprotoSyncGetHead: { lexicon: 1, - id: 'com.atproto.sync.getRoot', + id: 'com.atproto.sync.getHead', defs: { main: { type: 'query', - description: 'Gets the current root CID of a repo.', + description: 'Gets the current HEAD CID of a repo.', parameters: { type: 'params', required: ['did'], @@ -1882,13 +1926,47 @@ export const schemaDict = { }, }, }, - ComAtprotoSyncUpdateRepo: { + ComAtprotoSyncGetRecord: { lexicon: 1, - id: 'com.atproto.sync.updateRepo', + id: 'com.atproto.sync.getRecord', defs: { main: { - type: 'procedure', - description: 'Writes commits to a repo.', + type: 'query', + description: + 'Gets blocks needed for existence or non-existence of record.', + parameters: { + type: 'params', + required: ['did', 'collection', 'rkey'], + properties: { + did: { + type: 'string', + description: 'The DID of the repo.', + }, + collection: { + type: 'string', + }, + rkey: { + type: 'string', + }, + commit: { + type: 'string', + description: 'An optional past commit CID.', + }, + }, + }, + output: { + encoding: 'application/vnd.ipld.car', + }, + }, + }, + }, + ComAtprotoSyncGetRepo: { + lexicon: 1, + id: 'com.atproto.sync.getRepo', + defs: { + main: { + type: 'query', + description: 'Gets the repo state.', parameters: { type: 'params', required: ['did'], @@ -1897,10 +1975,14 @@ export const schemaDict = { type: 'string', description: 'The DID of the repo.', }, + from: { + type: 'string', + description: 'A past commit CID.', + }, }, }, - input: { - encoding: 'application/cbor', + output: { + encoding: 'application/vnd.ipld.car', }, }, }, @@ -3753,9 +3835,11 @@ export const ids = { ComAtprotoSessionDelete: 'com.atproto.session.delete', ComAtprotoSessionGet: 'com.atproto.session.get', ComAtprotoSessionRefresh: 'com.atproto.session.refresh', + ComAtprotoSyncGetCheckout: 'com.atproto.sync.getCheckout', + ComAtprotoSyncGetCommitPath: 'com.atproto.sync.getCommitPath', + ComAtprotoSyncGetHead: 'com.atproto.sync.getHead', + ComAtprotoSyncGetRecord: 'com.atproto.sync.getRecord', ComAtprotoSyncGetRepo: 'com.atproto.sync.getRepo', - ComAtprotoSyncGetRoot: 'com.atproto.sync.getRoot', - ComAtprotoSyncUpdateRepo: 'com.atproto.sync.updateRepo', AppBskyActorGetProfile: 'app.bsky.actor.getProfile', AppBskyActorGetSuggestions: 'app.bsky.actor.getSuggestions', AppBskyActorProfile: 'app.bsky.actor.profile', diff --git a/packages/api/src/client/types/com/atproto/sync/updateRepo.ts b/packages/api/src/client/types/com/atproto/sync/getCheckout.ts similarity index 78% rename from packages/api/src/client/types/com/atproto/sync/updateRepo.ts rename to packages/api/src/client/types/com/atproto/sync/getCheckout.ts index 7ce40eeddbf..24373404bb4 100644 --- a/packages/api/src/client/types/com/atproto/sync/updateRepo.ts +++ b/packages/api/src/client/types/com/atproto/sync/getCheckout.ts @@ -9,19 +9,20 @@ import { lexicons } from '../../../../lexicons' export interface QueryParams { /** The DID of the repo. */ did: string + /** The commit to get the checkout from. Defaults to current HEAD. */ + commit?: string } -export type InputSchema = string | Uint8Array +export type InputSchema = undefined export interface CallOptions { headers?: Headers - qp?: QueryParams - encoding: 'application/cbor' } export interface Response { success: boolean headers: Headers + data: Uint8Array } export function toKnownErr(e: any) { diff --git a/packages/api/src/client/types/com/atproto/sync/getCommitPath.ts b/packages/api/src/client/types/com/atproto/sync/getCommitPath.ts new file mode 100644 index 00000000000..5d780d83923 --- /dev/null +++ b/packages/api/src/client/types/com/atproto/sync/getCommitPath.ts @@ -0,0 +1,39 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' + +export interface QueryParams { + /** The DID of the repo. */ + did: string + /** The most recent commit */ + latest?: string + /** The earliest commit to start from */ + earliest?: string +} + +export type InputSchema = undefined + +export interface OutputSchema { + commits: string[] + [k: string]: unknown +} + +export interface CallOptions { + headers?: Headers +} + +export interface Response { + success: boolean + headers: Headers + data: OutputSchema +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + } + return e +} diff --git a/packages/api/src/client/types/com/atproto/sync/getRoot.ts b/packages/api/src/client/types/com/atproto/sync/getHead.ts similarity index 100% rename from packages/api/src/client/types/com/atproto/sync/getRoot.ts rename to packages/api/src/client/types/com/atproto/sync/getHead.ts diff --git a/packages/api/src/client/types/com/atproto/sync/getRecord.ts b/packages/api/src/client/types/com/atproto/sync/getRecord.ts new file mode 100644 index 00000000000..debe13512eb --- /dev/null +++ b/packages/api/src/client/types/com/atproto/sync/getRecord.ts @@ -0,0 +1,34 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import { Headers, XRPCError } from '@atproto/xrpc' +import { ValidationResult } from '@atproto/lexicon' +import { isObj, hasProp } from '../../../../util' +import { lexicons } from '../../../../lexicons' + +export interface QueryParams { + /** The DID of the repo. */ + did: string + collection: string + rkey: string + /** An optional past commit CID. */ + commit?: string +} + +export type InputSchema = undefined + +export interface CallOptions { + headers?: Headers +} + +export interface Response { + success: boolean + headers: Headers + data: Uint8Array +} + +export function toKnownErr(e: any) { + if (e instanceof XRPCError) { + } + return e +} diff --git a/packages/auth/README.md b/packages/auth/README.md deleted file mode 100644 index 1fff67b6fc9..00000000000 --- a/packages/auth/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# ATP Auth Library - -ATP's core permissioning library (based on UCANs). \ No newline at end of file diff --git a/packages/auth/build.js b/packages/auth/build.js deleted file mode 100644 index 5628aa4f4eb..00000000000 --- a/packages/auth/build.js +++ /dev/null @@ -1,22 +0,0 @@ -const pkgJson = require('@npmcli/package-json') -const { nodeExternalsPlugin } = require('esbuild-node-externals') - -const buildShallow = - process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true' - -if (process.argv.includes('--update-main-to-dist')) { - return pkgJson - .load(__dirname) - .then((pkg) => pkg.update({ main: 'dist/index.js' })) - .then((pkg) => pkg.save()) -} - -require('esbuild').build({ - logLevel: 'info', - entryPoints: ['src/index.ts'], - bundle: true, - sourcemap: true, - outdir: 'dist', - platform: 'node', - plugins: buildShallow ? [nodeExternalsPlugin()] : [], -}) diff --git a/packages/auth/jest.config.js b/packages/auth/jest.config.js deleted file mode 100644 index 39049495ffc..00000000000 --- a/packages/auth/jest.config.js +++ /dev/null @@ -1,6 +0,0 @@ -const base = require('../../jest.config.base.js') - -module.exports = { - ...base, - displayName: 'Auth' -} diff --git a/packages/auth/package.json b/packages/auth/package.json deleted file mode 100644 index 7de17e56b59..00000000000 --- a/packages/auth/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "@atproto/auth", - "version": "0.0.1", - "main": "src/index.ts", - "license": "MIT", - "scripts": { - "test": "jest", - "prettier": "prettier --check src/", - "prettier:fix": "prettier --write src/", - "lint": "eslint . --ext .ts,.tsx", - "lint:fix": "yarn lint --fix", - "verify": "run-p prettier lint", - "verify:fix": "yarn prettier:fix && yarn lint:fix", - "build": "node ./build.js", - "postbuild": "tsc --build tsconfig.build.json", - "update-main-to-dist": "node ./update-pkg.js --update-main-to-dist", - "update-main-to-src": "node ./update-pkg.js --update-main-to-src", - "prepublish": "npm run update-main-to-dist", - "postpublish": "npm run update-main-to-src" - }, - "dependencies": { - "@atproto/crypto": "*", - "@atproto/did-resolver": "*", - "@ucans/core": "0.11.0", - "uint8arrays": "3.0.0" - } -} diff --git a/packages/auth/src/atp-capabilities.ts b/packages/auth/src/atp-capabilities.ts deleted file mode 100644 index 397a6bd924c..00000000000 --- a/packages/auth/src/atp-capabilities.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { atpCapability, parseAtpResource } from './atp-semantics' -import * as ucan from '@ucans/core' - -export const writeCap = ( - did: string, - collection?: string, - record?: string, -): ucan.Capability => { - let resource = did - if (collection) { - resource += '/' + collection - } - if (record) { - resource += '/' + record - } - return atpCapability(resource, 'WRITE') -} - -export const maintenanceCap = (did: string): ucan.Capability => { - return atpCapability(did, 'MAINTENANCE') -} - -export const vaguerCap = (cap: ucan.Capability): ucan.Capability | null => { - const rsc = parseAtpResource(cap.with) - if (rsc === null) return null - // can't go vaguer than every collection - if (rsc.collection === '*') return null - if (rsc.record === '*') return writeCap(rsc.did) - return writeCap(rsc.did, rsc.collection) -} diff --git a/packages/auth/src/atp-semantics.ts b/packages/auth/src/atp-semantics.ts deleted file mode 100644 index ac287feab5f..00000000000 --- a/packages/auth/src/atp-semantics.ts +++ /dev/null @@ -1,118 +0,0 @@ -import * as ucans from '@ucans/core' - -/* -ATP Ucans: - -Resource name: 'at' - -- Full permission for account: - at://did:example:userDid/* -- Permission to write to particular application collection: - at://did:example:userDid/com.foo.post/* -- Permission to create a single interaction on user's behalf: - at://did:example:userDid/com.foo.post/234567abcdefg - -Example: -{ - with: { scheme: "at", hierPart: "did:example:userDid/com.foo.post/*" }, - can: { namespace: "atp", segments: [ "WRITE" ] } -} - -At the moment, we support only two capability level: -- 'WRITE': this allows full create/update/delete permissions for the given resource -- 'MAINTENANCE': this does not allow updates to repo objects, but allows maintenance of the repo, such as repo creation -*/ - -export const ATP_ABILITY_LEVELS = { - SUPER_USER: 2, - WRITE: 1, - MAINTENANCE: 0, -} - -export const ATP_ABILITIES: string[] = Object.keys(ATP_ABILITY_LEVELS) - -export type AtpAbility = keyof typeof ATP_ABILITY_LEVELS - -export const isAtpCap = (cap: ucans.Capability): boolean => { - return cap.with.scheme === 'at' && isAtpAbility(cap.can) -} - -export const isAtpAbility = (ability: unknown): ability is AtpAbility => { - if (!ucans.ability.isAbility(ability)) return false - if (ability === ucans.ability.SUPERUSER) return true - const abilitySegment = ability.segments[0] - const isAtpAbilitySegment = - !!abilitySegment && ATP_ABILITIES.includes(abilitySegment) - return isAtpAbilitySegment && ability.namespace.toLowerCase() === 'atp' -} - -export const parseAtpAbility = ( - ability: ucans.ability.Ability, -): AtpAbility | null => { - if (ability === ucans.ability.SUPERUSER) return 'SUPER_USER' - if (isAtpAbility(ability)) return ability.segments[0] as AtpAbility - return null -} - -export const atpCapability = ( - resource: string, - ability: AtpAbility, -): ucans.Capability => { - return { - with: { scheme: 'at', hierPart: resource }, - can: { namespace: 'atp', segments: [ability] }, - } -} -export interface AtpResourcePointer { - did: string - collection: string - record: string -} - -// @TODO: ugly import on param -export const parseAtpResource = ( - pointer: ucans.capability.resourcePointer.ResourcePointer, -): AtpResourcePointer | null => { - if (pointer.scheme !== 'at') return null - - const parts = pointer.hierPart.split('/') - let [did, collection, record] = parts - if (!did) return null - if (!collection) collection = '*' - if (!record) record = '*' - return { - did, - collection, - record, - } -} - -export const atpSemantics: ucans.DelegationSemantics = { - canDelegateResource(parentResource, childResource) { - const parent = parseAtpResource(parentResource) - const child = parseAtpResource(childResource) - - if (parent == null || child == null) return false - if (parent.did !== child.did) return false - - if (parent.collection === '*') return true - if (parent.collection !== child.collection) return false - - if (parent.record === '*') return true - - return parent.record === child.record - }, - - canDelegateAbility(parentAbility, childAbility) { - const parent = parseAtpAbility(parentAbility) - const child = parseAtpAbility(childAbility) - - if (parent == null || child == null) return false - - if (ATP_ABILITY_LEVELS[child] > ATP_ABILITY_LEVELS[parent]) { - return false - } - - return true - }, -} diff --git a/packages/auth/src/auth-store.ts b/packages/auth/src/auth-store.ts deleted file mode 100644 index 66a5c284c95..00000000000 --- a/packages/auth/src/auth-store.ts +++ /dev/null @@ -1,181 +0,0 @@ -import * as ucan from '@ucans/core' -import { DidableKey } from '@ucans/core' - -import { atpSemantics, parseAtpResource } from './atp-semantics' -import { MONTH_IN_SEC, YEAR_IN_SEC } from './consts' -import { CapWithProof, Signer } from './types' -import { vaguerCap, writeCap } from './atp-capabilities' -import { PluginInjectedApi } from './plugins' - -export class AuthStore implements Signer { - protected keypair: DidableKey - protected ucanStore: ucan.StoreI | null = null - protected tokens: string[] - protected controlledDid: string | null - protected ucanApi: PluginInjectedApi - - constructor( - ucanApi: PluginInjectedApi, - keypair: DidableKey, - tokens: string[], - controlledDid?: string, - ) { - this.ucanApi = ucanApi - this.keypair = keypair - this.tokens = tokens - this.controlledDid = controlledDid || null - } - - // Update these for sub classes - // ---------------- - - protected async getKeypair(): Promise { - return this.keypair - } - - async addUcan(token: ucan.Ucan): Promise { - const ucanStore = await this.getUcanStore() - await ucanStore.add(token) - } - - async getUcanStore(): Promise { - if (!this.ucanStore) { - this.ucanStore = await this.ucanApi.Store.fromTokens( - atpSemantics, - this.tokens, - ) - } - return this.ucanStore as ucan.StoreI - } - - async clear(): Promise { - // noop - } - - async reset(): Promise { - // noop - } - - // ---------------- - - async keypairDid(): Promise { - const keypair = await this.getKeypair() - return keypair.did() - } - - async did(): Promise { - if (this.controlledDid) { - return this.controlledDid - } - return this.keypairDid() - } - - async canSignForDid(did: string): Promise { - if (did === this.controlledDid) return true - if (did === (await this.keypairDid())) return true - return false - } - - async sign(data: Uint8Array): Promise { - const keypair = await this.getKeypair() - return keypair.sign(data) - } - - async findProof(cap: ucan.Capability): Promise { - const ucanStore = await this.getUcanStore() - // we only handle atp caps right now - const resource = parseAtpResource(cap.with) - if (resource === null) return null - const res = await ucan.first( - ucanStore.findWithCapability(await this.did(), cap, resource.did), - ) - if (!res) return null - return res - } - - async findUcan(cap: ucan.Capability): Promise { - const chain = await this.findProof(cap) - if (chain === null) return null - return chain.ucan - } - - async hasUcan(cap: ucan.Capability): Promise { - const found = await this.findUcan(cap) - return found !== null - } - - async createUcan( - audience: string, - cap: ucan.Capability, - lifetime = MONTH_IN_SEC, - ): Promise { - const keypair = await this.getKeypair() - const ucanStore = await this.getUcanStore() - return this.ucanApi.Builder.create() - .issuedBy(keypair) - .toAudience(audience) - .withLifetimeInSeconds(lifetime) - .delegateCapability(cap, ucanStore) - .build() - } - - // Creates a UCAN that permissions all required caps - // We find the vaguest proof possible for each cap to avoid unnecessary duplication - async createUcanForCaps( - audience: string, - caps: ucan.Capability[], - lifetime = MONTH_IN_SEC, - ): Promise { - // @TODO make sure to dedupe proofs - const proofs: CapWithProof[] = [] - for (const cap of caps) { - const proof = await this.vaguestProofForCap(cap) - if (proof === null) { - throw new Error(`Could not find a ucan for capability: ${cap.with}`) - } - proofs.push(proof) - } - - const keypair = await this.getKeypair() - - let builder = this.ucanApi.Builder.create() - .issuedBy(keypair) - .toAudience(audience) - .withLifetimeInSeconds(lifetime) - - for (const prf of proofs) { - builder = builder.delegateCapability(prf.cap, prf.prf, atpSemantics) - } - - return builder.build() - } - - // Finds the most general proof for the given cap - // (And thus most likely to overlap with other proofs) - async vaguestProofForCap(cap: ucan.Capability): Promise { - const prf = await this.findProof(cap) - if (prf === null) return null - const vauger = vaguerCap(cap) - if (vauger === null) return { cap, prf } - const vaugerPrf = await this.vaguestProofForCap(vauger) - if (vaugerPrf === null) return { cap, prf } - return vaugerPrf - } - - // Claim a fully permissioned Ucan & add to store - // Mainly for dev purposes - async claimFull(): Promise { - const keypair = await this.getKeypair() - const ownDid = await this.did() - const token = await this.ucanApi.Builder.create() - .issuedBy(keypair) - .toAudience(ownDid) - .withLifetimeInSeconds(YEAR_IN_SEC) - .claimCapability(writeCap(ownDid)) - .build() - await this.addUcan(token) - return token - } -} - -export default AuthStore diff --git a/packages/auth/src/consts.ts b/packages/auth/src/consts.ts deleted file mode 100644 index f12d91badb9..00000000000 --- a/packages/auth/src/consts.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const MIN_IN_SEC = 60 -export const HOUR_IN_SEC = MIN_IN_SEC * 60 -export const DAY_IN_SEC = HOUR_IN_SEC * 24 -export const MONTH_IN_SEC = DAY_IN_SEC * 30 -export const YEAR_IN_SEC = DAY_IN_SEC * 365 diff --git a/packages/auth/src/index.ts b/packages/auth/src/index.ts deleted file mode 100644 index d28aca99bf1..00000000000 --- a/packages/auth/src/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -export * from './verifier' -export * from './atp-semantics' -export * from './atp-capabilities' -export * from './auth-store' -export * from './verify' -export * from './types' -export * from './signatures' -export * from './plugins' - -export { EcdsaKeypair } from '@atproto/crypto' - -export * as ucans from '@ucans/core' -export { encode as encodeUcan } from '@ucans/core' -export type { Ucan, DidableKey } from '@ucans/core' diff --git a/packages/auth/src/plugins.ts b/packages/auth/src/plugins.ts deleted file mode 100644 index 62f2dec1606..00000000000 --- a/packages/auth/src/plugins.ts +++ /dev/null @@ -1,61 +0,0 @@ -import * as ucans from '@ucans/core' - -// @TODO move to ucan package -export type BuildFn = (params: { - issuer: ucans.DidableKey - audience: string - capabilities?: Array - lifetimeInSeconds?: number - expiration?: number - notBefore?: number - facts?: Array - proofs?: Array - addNonce?: boolean -}) => Promise - -export type SignFn = ( - payload: ucans.UcanPayload, - jwtAlg: string, - signFn: (data: Uint8Array) => Promise, -) => Promise - -export type SignWithKeypairFn = ( - payload: ucans.UcanPayload, - keypair: ucans.Keypair, -) => Promise - -export type ValidateFn = ( - encodedUcan: string, - opts?: Partial, -) => Promise - -export type ValidateProofsFn = ( - ucan: ucans.Ucan, - opts?: Partial, -) => AsyncIterable - -export type VerifyFn = ( - ucan: string, - options: ucans.VerifyOptions, -) => Promise> - -export type DelegationChainsFn = ( - semantics: ucans.DelegationSemantics, - ucan: ucans.Ucan, - isRevoked?: (ucan: ucans.Ucan) => Promise, -) => AsyncIterable - -export type BuilderClass = any -export type StoreClass = any - -export type PluginInjectedApi = { - build: BuildFn - sign: SignFn - signWithKeypair: SignWithKeypairFn - validate: ValidateFn - validateProofs: ValidateProofsFn - verify: VerifyFn - Builder: BuilderClass - Store: StoreClass - delegationChains: DelegationChainsFn -} diff --git a/packages/auth/src/signatures.ts b/packages/auth/src/signatures.ts deleted file mode 100644 index e62eddfe995..00000000000 --- a/packages/auth/src/signatures.ts +++ /dev/null @@ -1,16 +0,0 @@ -import * as uint8arrays from 'uint8arrays' -import * as ucans from '@ucans/core' - -export const verifySignature = - (plugins: ucans.Plugins) => - async (did: string, data: Uint8Array, sig: Uint8Array): Promise => { - return plugins.verifySignature(did, data, sig) - } - -export const verifySignatureUtf8 = - (plugins: ucans.Plugins) => - async (did: string, data: string, sig: string): Promise => { - const dataBytes = uint8arrays.fromString(data, 'utf8') - const sigBytes = uint8arrays.fromString(sig, 'base64url') - return verifySignature(plugins)(did, dataBytes, sigBytes) - } diff --git a/packages/auth/src/types.ts b/packages/auth/src/types.ts deleted file mode 100644 index 29f7d5ad4a2..00000000000 --- a/packages/auth/src/types.ts +++ /dev/null @@ -1,10 +0,0 @@ -import * as ucan from '@ucans/core' - -export interface Signer { - sign: (data: Uint8Array) => Promise -} - -export type CapWithProof = { - cap: ucan.Capability - prf: ucan.DelegationChain -} diff --git a/packages/auth/src/verifier.ts b/packages/auth/src/verifier.ts deleted file mode 100644 index 91a34e2b5da..00000000000 --- a/packages/auth/src/verifier.ts +++ /dev/null @@ -1,123 +0,0 @@ -import * as ucans from '@ucans/core' -import { DidableKey, EcdsaKeypair, p256Plugin } from '@atproto/crypto' -import { PluginInjectedApi } from './plugins' -import { verifySignature, verifySignatureUtf8 } from './signatures' -import { verifyUcan, verifyAtpUcan, verifyFullWritePermission } from './verify' -import AuthStore from './auth-store' -import { DidResolver } from '@atproto/did-resolver' - -export const DID_KEY_PLUGINS = [p256Plugin] - -export type VerifierOpts = { - didResolver: DidResolver - plcUrl: string - resolutionTimeout: number - additionalDidMethods: Record - additionalDidKeys: [ucans.DidKeyPlugin] -} - -export class Verifier { - didResolver: DidResolver - plugins: ucans.Plugins - ucanApi: PluginInjectedApi - - constructor(opts: Partial = {}) { - const { - additionalDidKeys = [], - additionalDidMethods = {}, - plcUrl, - resolutionTimeout, - } = opts - - const resolver = - opts.didResolver ?? - new DidResolver({ - plcUrl, - timeout: resolutionTimeout, - }) - - // handles did:web & did:plc - const methodPlugins: ucans.DidMethodPlugin = { - checkJwtAlg: (_did, _jwtAlg) => { - return true - }, - verifySignature: async (did, data, sig) => { - const atpData = await resolver.resolveAtpData(did) - return this.verifySignature(atpData.signingKey, data, sig) - }, - } - - const plugins = new ucans.Plugins( - [...DID_KEY_PLUGINS, ...additionalDidKeys], - { - ...additionalDidMethods, - plc: methodPlugins, - web: methodPlugins, - }, - ) - - this.ucanApi = ucans.getPluginInjectedApi(plugins) - this.plugins = plugins - } - - loadAuthStore( - keypair: DidableKey, - tokens: string[], - controlledDid?: string, - ): AuthStore { - return new AuthStore(this.ucanApi, keypair, tokens, controlledDid) - } - - async createTempAuthStore(tokens: string[] = []): Promise { - const keypair = await EcdsaKeypair.create() - return this.loadAuthStore(keypair, tokens) - } - - async verifySignature( - did: string, - data: Uint8Array, - sig: Uint8Array, - ): Promise { - return verifySignature(this.plugins)(did, data, sig) - } - - async verifySignatureUtf8( - did: string, - data: string, - sig: string, - ): Promise { - return verifySignatureUtf8(this.plugins)(did, data, sig) - } - - async verifyUcan( - token: ucans.Ucan | string, - opts: ucans.VerifyOptions, - ): Promise { - return verifyUcan(this.ucanApi)(token, opts) - } - - async verifyAtpUcan( - token: ucans.Ucan | string, - audience: string, - cap: ucans.Capability, - ): Promise { - return verifyAtpUcan(this.ucanApi)(token, audience, cap) - } - - async verifyFullWritePermission( - token: ucans.Ucan | string, - audience: string, - repoDid: string, - ): Promise { - return verifyFullWritePermission(this.ucanApi)(token, audience, repoDid) - } - - async validateUcan( - encodedUcan: string, - opts?: Partial, - ): Promise { - return this.ucanApi.validate(encodedUcan, opts) - } -} - -export default Verifier diff --git a/packages/auth/src/verify.ts b/packages/auth/src/verify.ts deleted file mode 100644 index ac4a5748d61..00000000000 --- a/packages/auth/src/verify.ts +++ /dev/null @@ -1,53 +0,0 @@ -import * as ucans from '@ucans/core' -import { writeCap } from './atp-capabilities' -import { atpSemantics, parseAtpResource } from './atp-semantics' -import { PluginInjectedApi } from './plugins' - -export const verifyUcan = - (ucanApi: PluginInjectedApi) => - async ( - token: ucans.Ucan | string, - opts: ucans.VerifyOptions, - ): Promise => { - const encoded = typeof token === 'string' ? token : ucans.encode(token) - const res = await ucanApi.verify(encoded, { - ...opts, - semantics: opts.semantics || atpSemantics, - }) - if (!res.ok) { - if (res.error[0]) { - throw res.error[0] - } else { - throw new Error('Could not find requested capability') - } - } - return ucanApi.validate(encoded) - } - -export const verifyAtpUcan = - (ucanApi: PluginInjectedApi) => - async ( - token: ucans.Ucan | string, - audience: string, - cap: ucans.Capability, - ): Promise => { - const atpResource = parseAtpResource(cap.with) - if (atpResource === null) { - throw new Error(`Expected a valid atp resource: ${cap.with}`) - } - const repoDid = atpResource.did - return verifyUcan(ucanApi)(token, { - audience, - requiredCapabilities: [{ capability: cap, rootIssuer: repoDid }], - }) - } - -export const verifyFullWritePermission = - (ucanApi: PluginInjectedApi) => - async ( - token: ucans.Ucan | string, - audience: string, - repoDid: string, - ): Promise => { - return verifyAtpUcan(ucanApi)(token, audience, writeCap(repoDid)) - } diff --git a/packages/auth/tests/auth.test.ts b/packages/auth/tests/auth.test.ts deleted file mode 100644 index 346fbbb04d3..00000000000 --- a/packages/auth/tests/auth.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { writeCap } from '../src/atp-capabilities' -import { Verifier, AuthStore, Ucan, ucans } from '../src' - -describe('tokens for post', () => { - const collection = 'com.example.microblog' - const record = '3iwc-gvs-ehpk-2s' - const serverDid = 'did:example:fakeServerDid' - - const verifier = new Verifier() - - let authStore: AuthStore - let token: Ucan - let rootDid: string - let cap: ucans.Capability - let fullUcan: Ucan - - it('validates a fully claimed ucan from the root DID', async () => { - authStore = await verifier.createTempAuthStore() - fullUcan = await authStore.claimFull() - rootDid = await authStore.did() - - cap = writeCap(rootDid, collection, record) - - await verifier.verifyAtpUcan(fullUcan, fullUcan.payload.aud, cap) - }) - - it('creates a valid token for a post', async () => { - token = await authStore.createUcan(serverDid, cap, 30) - await verifier.verifyAtpUcan(token, serverDid, cap) - }) - - it('throws an error for the wrong collection', async () => { - const collectionCap = writeCap( - rootDid, - 'com.example.otherCollection', - record, - ) - try { - const res = await verifier.verifyAtpUcan(token, serverDid, collectionCap) - expect(res).toBe(null) - } catch (err) { - expect(err).toBeTruthy() - } - }) - - it('throws an error for the wrong record name', async () => { - const recordCap = writeCap(rootDid, collection, '3iwc-gvs-ehpk-2z') - try { - const res = await verifier.verifyAtpUcan(token, serverDid, recordCap) - expect(res).toBe(null) - } catch (err) { - expect(err).toBeTruthy() - } - }) -}) diff --git a/packages/auth/tsconfig.build.json b/packages/auth/tsconfig.build.json deleted file mode 100644 index 27df65b89e2..00000000000 --- a/packages/auth/tsconfig.build.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "./tsconfig.json", - "exclude": ["**/*.spec.ts", "**/*.test.ts"] -} \ No newline at end of file diff --git a/packages/auth/tsconfig.json b/packages/auth/tsconfig.json deleted file mode 100644 index c7c6a7eecd9..00000000000 --- a/packages/auth/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "rootDir": "./src", - "outDir": "./dist", // Your outDir, - "emitDeclarationOnly": true - }, - "include": ["./src","__tests__/**/**.ts"], - "references": [{ "path": "../crypto/tsconfig.build.json" }] -} \ No newline at end of file diff --git a/packages/auth/update-pkg.js b/packages/auth/update-pkg.js deleted file mode 100644 index 15d70bdab33..00000000000 --- a/packages/auth/update-pkg.js +++ /dev/null @@ -1,14 +0,0 @@ -const pkgJson = require('@npmcli/package-json') - -if (process.argv.includes('--update-main-to-dist')) { - return pkgJson - .load(__dirname) - .then((pkg) => pkg.update({ main: 'dist/index.js' })) - .then((pkg) => pkg.save()) -} -if (process.argv.includes('--update-main-to-src')) { - return pkgJson - .load(__dirname) - .then((pkg) => pkg.update({ main: 'src/index.ts' })) - .then((pkg) => pkg.save()) -} diff --git a/packages/aws/README.md b/packages/aws/README.md index eca629fc162..19e36786da6 100644 --- a/packages/aws/README.md +++ b/packages/aws/README.md @@ -1,3 +1,3 @@ # AWS KMS -A DidableKeypair-compatible wrapper for AWS KMS. \ No newline at end of file +A Keypair-compatible wrapper for AWS KMS. \ No newline at end of file diff --git a/packages/aws/src/kms.ts b/packages/aws/src/kms.ts index 02cc725738f..ac1e27c0d66 100644 --- a/packages/aws/src/kms.ts +++ b/packages/aws/src/kms.ts @@ -10,7 +10,7 @@ export type KmsConfig = { keyId: string } & Omit< 'apiVersion' > -export class KmsKeypair implements crypto.DidableKey { +export class KmsKeypair implements crypto.Keypair { jwtAlg = crypto.SECP256K1_JWT_ALG constructor( diff --git a/packages/common/src/blocks.ts b/packages/common/src/blocks.ts deleted file mode 100644 index 901ce19cb9f..00000000000 --- a/packages/common/src/blocks.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { CID } from 'multiformats/cid' -import * as Block from 'multiformats/block' -import * as rawCodec from 'multiformats/codecs/raw' -import { sha256 as blockHasher } from 'multiformats/hashes/sha2' -import * as mf from 'multiformats' -import * as blockCodec from '@ipld/dag-cbor' - -export const valueToIpldBlock = async (data: unknown) => { - return Block.encode({ - value: data, - codec: blockCodec, - hasher: blockHasher, - }) -} - -export const sha256RawToCid = (hash: Uint8Array): CID => { - const digest = mf.digest.create(blockHasher.code, hash) - return CID.createV1(rawCodec.code, digest) -} - -export const cidForData = async (data: unknown): Promise => { - const block = await valueToIpldBlock(data) - return block.cid -} - -export const valueToIpldBytes = (value: unknown): Uint8Array => { - return blockCodec.encode(value) -} - -export const ipldBytesToValue = (bytes: Uint8Array) => { - return blockCodec.decode(bytes) -} - -export const ipldBytesToRecord = (bytes: Uint8Array): object => { - const val = ipldBytesToValue(bytes) - if (typeof val !== 'object' || val === null) { - throw new Error(`Expected object, got: ${val}`) - } - return val -} diff --git a/packages/common/src/check.ts b/packages/common/src/check.ts index db1def6a629..47bdce95958 100644 --- a/packages/common/src/check.ts +++ b/packages/common/src/check.ts @@ -1,13 +1,22 @@ -export interface Def { +import { ZodError } from 'zod' + +export interface Checkable { parse: (obj: unknown) => T - safeParse: (obj: unknown) => { success: boolean } + safeParse: ( + obj: unknown, + ) => { success: true; data: T } | { success: false; error: ZodError } +} + +export interface Def { + name: string + schema: Checkable } -export const is = (obj: unknown, def: Def): obj is T => { +export const is = (obj: unknown, def: Checkable): obj is T => { return def.safeParse(obj).success } -export const assure = (def: Def, obj: unknown): T => { +export const assure = (def: Checkable, obj: unknown): T => { return def.parse(obj) } diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index a0031b26e14..0a16fa1d9d5 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -3,7 +3,7 @@ export * as util from './util' export * from './util' export * from './tid' -export * from './blocks' +export * from './ipld' export * from './logger' export * from './types' export * from './streams' diff --git a/packages/common/src/ipld.ts b/packages/common/src/ipld.ts new file mode 100644 index 00000000000..b235a6ca665 --- /dev/null +++ b/packages/common/src/ipld.ts @@ -0,0 +1,48 @@ +import { CID } from 'multiformats/cid' +import * as Block from 'multiformats/block' +import * as rawCodec from 'multiformats/codecs/raw' +import { sha256 } from 'multiformats/hashes/sha2' +import * as mf from 'multiformats' +import * as cborCodec from '@ipld/dag-cbor' +import { check, schema } from '.' + +export const dataToCborBlock = async (data: unknown) => { + return Block.encode({ + value: data, + codec: cborCodec, + hasher: sha256, + }) +} + +export const verifyCidForBytes = async (cid: CID, bytes: Uint8Array) => { + const digest = await sha256.digest(bytes) + const expected = CID.createV1(cid.code, digest) + if (!cid.equals(expected)) { + throw new Error( + `Not a valid CID for bytes. Expected: ${expected.toString()} Got: ${cid.toString()}`, + ) + } +} + +export const sha256RawToCid = (hash: Uint8Array): CID => { + const digest = mf.digest.create(sha256.code, hash) + return CID.createV1(rawCodec.code, digest) +} + +export const cidForCbor = async (data: unknown): Promise => { + const block = await dataToCborBlock(data) + return block.cid +} + +export const cborEncode = cborCodec.encode +export const cborDecode = cborCodec.decode + +export const cborBytesToRecord = ( + bytes: Uint8Array, +): Record => { + const val = cborDecode(bytes) + if (!check.is(val, schema.record)) { + throw new Error(`Expected object, got: ${val}`) + } + return val +} diff --git a/packages/common/src/types.ts b/packages/common/src/types.ts index 8710e43c230..b89a7c2a6b8 100644 --- a/packages/common/src/types.ts +++ b/packages/common/src/types.ts @@ -1,44 +1,41 @@ import * as mf from 'multiformats/cid' import { z } from 'zod' +import { Def } from './check' -const cid = z +const cidSchema = z .any() .refine((obj: unknown) => mf.CID.asCID(obj) !== null, { message: 'Not a CID', }) .transform((obj: unknown) => mf.CID.asCID(obj) as mf.CID) -export const isCid = (str: string): boolean => { - try { - mf.CID.parse(str) - return true - } catch (err) { - return false - } +export const schema = { + cid: cidSchema, + bytes: z.instanceof(Uint8Array), + string: z.string(), + record: z.record(z.string(), z.unknown()), + unknown: z.unknown(), } -const strToCid = z - .string() - .refine(isCid, { message: 'Not a valid CID' }) - .transform((str: string) => mf.CID.parse(str)) - -const bytes = z.instanceof(Uint8Array) -export type Bytes = z.infer - -const strToInt = z - .string() - .refine((str) => !isNaN(parseInt(str)), { - message: 'Cannot parse string to integer', - }) - .transform((str) => parseInt(str)) - -const strToBool = z.string().transform((str) => str === 'true' || str === 't') - export const def = { - string: z.string(), - cid, - strToCid, - bytes, - strToInt, - strToBool, + cid: { + name: 'cid', + schema: schema.cid, + } as Def, + bytes: { + name: 'bytes', + schema: schema.bytes, + } as Def, + string: { + name: 'string', + schema: schema.string, + } as Def, + record: { + name: 'record', + schema: schema.record, + } as Def>, + unknown: { + name: 'unknown', + schema: schema.unknown, + } as Def, } diff --git a/packages/common/src/util.ts b/packages/common/src/util.ts index 73c04047263..66c3a27b37e 100644 --- a/packages/common/src/util.ts +++ b/packages/common/src/util.ts @@ -67,9 +67,28 @@ export const asyncFilter = async ( export const isErrnoException = ( err: unknown, ): err is NodeJS.ErrnoException => { - return !!err && 'code' in err + return !!err && err['code'] } export const errHasMsg = (err: unknown, msg: string): boolean => { return !!err && typeof err === 'object' && err['message'] === msg } + +export const chunkArray = (arr: T[], chunkSize: number): T[][] => { + return arr.reduce((acc, cur, i) => { + const chunkI = Math.floor(i / chunkSize) + if (!acc[chunkI]) { + acc[chunkI] = [] + } + acc[chunkI].push(cur) + return acc + }, [] as T[][]) +} + +export const range = (num: number): number[] => { + const nums: number[] = [] + for (let i = 0; i < num; i++) { + nums.push(i) + } + return nums +} diff --git a/packages/crypto/package.json b/packages/crypto/package.json index f0322fc7184..5b947efaa7c 100644 --- a/packages/crypto/package.json +++ b/packages/crypto/package.json @@ -19,8 +19,8 @@ "postpublish": "npm run update-main-to-src" }, "dependencies": { + "@atproto/crypto": "*", "@noble/secp256k1": "^1.7.0", - "@ucans/core": "0.11.0", "big-integer": "^1.6.51", "multiformats": "^9.6.4", "one-webcrypto": "^1.0.3", diff --git a/packages/crypto/src/index.ts b/packages/crypto/src/index.ts index b51e9b1762d..487b2110697 100644 --- a/packages/crypto/src/index.ts +++ b/packages/crypto/src/index.ts @@ -4,6 +4,7 @@ export * from './did' export * from './multibase' export * from './random' export * from './sha' +export * from './types' export * from './verify' export * from './p256/keypair' @@ -11,5 +12,3 @@ export * from './p256/plugin' export * from './secp256k1/keypair' export * from './secp256k1/plugin' - -export type { DidableKey } from '@ucans/core' diff --git a/packages/crypto/src/p256/keypair.ts b/packages/crypto/src/p256/keypair.ts index 18c5136093f..3950b7669fb 100644 --- a/packages/crypto/src/p256/keypair.ts +++ b/packages/crypto/src/p256/keypair.ts @@ -1,17 +1,16 @@ import { webcrypto } from 'one-webcrypto' import * as uint8arrays from 'uint8arrays' - -import * as ucan from '@ucans/core' - +import { SupportedEncodings } from 'uint8arrays/util/bases' import * as did from '../did' import * as operations from './operations' import { P256_JWT_ALG } from '../const' +import { Keypair } from '../types' export type EcdsaKeypairOptions = { exportable: boolean } -export class EcdsaKeypair implements ucan.DidableKey { +export class EcdsaKeypair implements Keypair { jwtAlg = P256_JWT_ALG private publicKey: Uint8Array private keypair: CryptoKeyPair @@ -56,7 +55,7 @@ export class EcdsaKeypair implements ucan.DidableKey { return this.publicKey } - publicKeyStr(encoding: ucan.Encodings = 'base64pad'): string { + publicKeyStr(encoding: SupportedEncodings = 'base64pad'): string { return uint8arrays.toString(this.publicKey, encoding) } diff --git a/packages/crypto/src/p256/plugin.ts b/packages/crypto/src/p256/plugin.ts index a275736e53b..d2c304d0fde 100644 --- a/packages/crypto/src/p256/plugin.ts +++ b/packages/crypto/src/p256/plugin.ts @@ -1,8 +1,6 @@ -import { DidKeyPlugin } from '@ucans/core' - -import { P256_DID_PREFIX, P256_JWT_ALG } from '../const' - import * as operations from './operations' +import { DidKeyPlugin } from '../types' +import { P256_DID_PREFIX, P256_JWT_ALG } from '../const' export const p256Plugin: DidKeyPlugin = { prefix: P256_DID_PREFIX, diff --git a/packages/crypto/src/secp256k1/keypair.ts b/packages/crypto/src/secp256k1/keypair.ts index 37aaa5791fa..a06a7881a8e 100644 --- a/packages/crypto/src/secp256k1/keypair.ts +++ b/packages/crypto/src/secp256k1/keypair.ts @@ -1,14 +1,15 @@ -import * as ucan from '@ucans/core' import * as secp from '@noble/secp256k1' import * as uint8arrays from 'uint8arrays' +import { SupportedEncodings } from 'uint8arrays/util/bases' import * as did from '../did' import { SECP256K1_JWT_ALG } from '../const' +import { Keypair } from '../types' export type Secp256k1KeypairOptions = { exportable: boolean } -export class Secp256k1Keypair implements ucan.DidableKey { +export class Secp256k1Keypair implements Keypair { jwtAlg = SECP256K1_JWT_ALG private publicKey: Uint8Array @@ -40,7 +41,7 @@ export class Secp256k1Keypair implements ucan.DidableKey { return this.publicKey } - publicKeyStr(encoding: ucan.Encodings = 'base64pad'): string { + publicKeyStr(encoding: SupportedEncodings = 'base64pad'): string { return uint8arrays.toString(this.publicKey, encoding) } diff --git a/packages/crypto/src/secp256k1/plugin.ts b/packages/crypto/src/secp256k1/plugin.ts index 6db5bad3909..5184a3778fa 100644 --- a/packages/crypto/src/secp256k1/plugin.ts +++ b/packages/crypto/src/secp256k1/plugin.ts @@ -1,5 +1,5 @@ -import { DidKeyPlugin } from '@ucans/core' import * as operations from './operations' +import { DidKeyPlugin } from '../types' import { SECP256K1_DID_PREFIX, SECP256K1_JWT_ALG } from '../const' export const secp256k1Plugin: DidKeyPlugin = { diff --git a/packages/crypto/src/types.ts b/packages/crypto/src/types.ts new file mode 100644 index 00000000000..20c7a5ea78a --- /dev/null +++ b/packages/crypto/src/types.ts @@ -0,0 +1,19 @@ +export interface Signer { + sign(msg: Uint8Array): Promise +} + +export interface Didable { + did(): string +} + +export interface Keypair extends Signer, Didable {} + +export type DidKeyPlugin = { + prefix: Uint8Array + jwtAlg: string + verifySignature: ( + did: string, + msg: Uint8Array, + data: Uint8Array, + ) => Promise +} diff --git a/packages/crypto/src/verify.ts b/packages/crypto/src/verify.ts index 49a17e16f38..43b2670c7cd 100644 --- a/packages/crypto/src/verify.ts +++ b/packages/crypto/src/verify.ts @@ -1,15 +1,26 @@ +import * as uint8arrays from 'uint8arrays' import { parseDidKey } from './did' import plugins from './plugins' -export const verifyDidSig = ( - did: string, +export const verifySignature = ( + didKey: string, data: Uint8Array, sig: Uint8Array, ): Promise => { - const parsed = parseDidKey(did) + const parsed = parseDidKey(didKey) const plugin = plugins.find((p) => p.jwtAlg === parsed.jwtAlg) if (!plugin) { throw new Error(`Unsupported signature alg: :${parsed.jwtAlg}`) } - return plugin.verifySignature(did, data, sig) + return plugin.verifySignature(didKey, data, sig) +} + +export const verifySignatureUtf8 = async ( + didKey: string, + data: string, + sig: string, +): Promise => { + const dataBytes = uint8arrays.fromString(data, 'utf8') + const sigBytes = uint8arrays.fromString(sig, 'base64url') + return verifySignature(didKey, dataBytes, sigBytes) } diff --git a/packages/crypto/tsconfig.json b/packages/crypto/tsconfig.json index afddb2bd4e1..7584791c1c8 100644 --- a/packages/crypto/tsconfig.json +++ b/packages/crypto/tsconfig.json @@ -5,5 +5,8 @@ "outDir": "./dist", // Your outDir, "emitDeclarationOnly": true }, - "include": ["./src","__tests__/**/**.ts"] + "include": ["./src","__tests__/**/**.ts"], + "references": [ + { "path": "../crypto/tsconfig.build.json" } + ] } \ No newline at end of file diff --git a/packages/did-resolver/src/resolver.ts b/packages/did-resolver/src/resolver.ts index eaef35bf2b3..540b3cbbdcc 100644 --- a/packages/did-resolver/src/resolver.ts +++ b/packages/did-resolver/src/resolver.ts @@ -4,6 +4,7 @@ import { DIDResolutionOptions, DIDResolutionResult, } from 'did-resolver' +import * as crypto from '@atproto/crypto' import * as web from './web-resolver' import * as plc from './plc-resolver' import * as atpDid from './atp-did' @@ -55,6 +56,24 @@ export class DidResolver { const didDocument = await this.ensureResolveDid(did) return atpDid.ensureAtpDocument(didDocument) } + + async resolveSigningKey(did: string): Promise { + if (did.startsWith('did:key:')) { + return did + } else { + const data = await this.resolveAtpData(did) + return data.signingKey + } + } + + async verifySignature( + did: string, + data: Uint8Array, + sig: Uint8Array, + ): Promise { + const signingKey = await this.resolveSigningKey(did) + return crypto.verifySignature(signingKey, data, sig) + } } export const resolver = new DidResolver() diff --git a/packages/pds/package.json b/packages/pds/package.json index 0d671e32e0f..4792d97733d 100644 --- a/packages/pds/package.json +++ b/packages/pds/package.json @@ -25,7 +25,6 @@ "postpublish": "npm run update-main-to-src" }, "dependencies": { - "@atproto/auth": "*", "@atproto/common": "*", "@atproto/crypto": "*", "@atproto/did-resolver": "*", diff --git a/packages/pds/src/api/app/bsky/actor/updateProfile.ts b/packages/pds/src/api/app/bsky/actor/updateProfile.ts index da10c89bf08..d0f2a82b426 100644 --- a/packages/pds/src/api/app/bsky/actor/updateProfile.ts +++ b/packages/pds/src/api/app/bsky/actor/updateProfile.ts @@ -7,6 +7,7 @@ import * as Profile from '../../../../lexicon/types/app/bsky/actor/profile' import * as common from '@atproto/common' import * as repo from '../../../../repo' import AppContext from '../../../../context' +import { WriteOpAction } from '@atproto/repo' const profileNsid = lexicons.ids.AppBskyActorProfile @@ -15,7 +16,6 @@ export default function (server: Server, ctx: AppContext) { auth: ctx.accessVerifierCheckTakedown, handler: async ({ auth, input }) => { const did = auth.credentials.did - const authStore = await ctx.getAuthstore(did) const uri = new AtUri(`${did}/${profileNsid}/self`) const { profileCid, updated } = await ctx.db.transaction( @@ -71,11 +71,11 @@ export default function (server: Server, ctx: AppContext) { record: updated, }) - const commit = await repoTxn.writeToRepo(did, authStore, [write], now) + const commit = await repoTxn.writeToRepo(did, [write], now) await repoTxn.blobs.processWriteBlobs(did, commit, [write]) let profileCid: CID - if (write.action === 'update') { + if (write.action === WriteOpAction.Update) { profileCid = write.cid // Update profile record await dbTxn.db @@ -97,7 +97,7 @@ export default function (server: Server, ctx: AppContext) { }) .where('uri', '=', uri.toString()) .execute() - } else if (write.action === 'create') { + } else if (write.action === WriteOpAction.Create) { profileCid = write.cid await recordTxn.indexRecord(uri, profileCid, updated, now) } else { diff --git a/packages/pds/src/api/app/bsky/feed/setVote.ts b/packages/pds/src/api/app/bsky/feed/setVote.ts index 26d9b93331f..fc5102b856d 100644 --- a/packages/pds/src/api/app/bsky/feed/setVote.ts +++ b/packages/pds/src/api/app/bsky/feed/setVote.ts @@ -12,7 +12,6 @@ export default function (server: Server, ctx: AppContext) { const { subject, direction } = input.body const requester = auth.credentials.did - const authStore = await ctx.getAuthstore(requester) const now = new Date().toISOString() const exists = await ctx.services @@ -69,7 +68,7 @@ export default function (server: Server, ctx: AppContext) { } await Promise.all([ - await repoTxn.writeToRepo(requester, authStore, writes, now), + await repoTxn.writeToRepo(requester, writes, now), await repoTxn.indexWrites(writes, now), ]) diff --git a/packages/pds/src/api/app/bsky/notification/list.ts b/packages/pds/src/api/app/bsky/notification/list.ts index e7085bfcc06..76679a1d5fb 100644 --- a/packages/pds/src/api/app/bsky/notification/list.ts +++ b/packages/pds/src/api/app/bsky/notification/list.ts @@ -91,7 +91,7 @@ export default function (server: Server, ctx: AppContext) { }, reason: notif.reason, reasonSubject: notif.reasonSubject || undefined, - record: common.ipldBytesToRecord(notif.recordBytes), + record: common.cborBytesToRecord(notif.recordBytes), isRead: notif.indexedAt <= user.lastSeenNotifs, indexedAt: notif.indexedAt, })) diff --git a/packages/pds/src/api/com/atproto/account.ts b/packages/pds/src/api/com/atproto/account.ts index 27c0f4b49d9..2c0f73f7cab 100644 --- a/packages/pds/src/api/com/atproto/account.ts +++ b/packages/pds/src/api/com/atproto/account.ts @@ -1,7 +1,7 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import * as crypto from '@atproto/crypto' import * as handleLib from '@atproto/handle' -import { cidForData } from '@atproto/common' +import { cidForCbor } from '@atproto/common' import { Server, APP_BSKY_SYSTEM } from '../../../lexicon' import { countAll } from '../../../db/util' import * as lex from '../../../lexicon/lexicons' @@ -148,11 +148,10 @@ export default function (server: Server, ctx: AppContext) { }) // Setup repo root - const authStore = ctx.getAuthstore(did) - await repoTxn.createRepo(did, authStore, [write], now) + await repoTxn.createRepo(did, [write], now) await repoTxn.indexWrites([write], now) - const declarationCid = await cidForData(declaration) + const declarationCid = await cidForCbor(declaration) const access = ctx.auth.createAccessToken(did) const refresh = ctx.auth.createRefreshToken(did) await ctx.services.auth(dbTxn).grantRefreshToken(refresh.payload) diff --git a/packages/pds/src/api/com/atproto/repo.ts b/packages/pds/src/api/com/atproto/repo.ts index 52fe59c166b..3c31ead6d5d 100644 --- a/packages/pds/src/api/com/atproto/repo.ts +++ b/packages/pds/src/api/com/atproto/repo.ts @@ -9,6 +9,7 @@ import { PreparedWrite, } from '../../../repo' import AppContext from '../../../context' +import { WriteOpAction } from '@atproto/repo' export default function (server: Server, ctx: AppContext) { server.com.atproto.repo.describe(async ({ params }) => { @@ -112,8 +113,9 @@ export default function (server: Server, ctx: AppContext) { ) } - const authStore = ctx.getAuthstore(did) - const hasUpdate = tx.writes.some((write) => write.action === 'update') + const hasUpdate = tx.writes.some( + (write) => write.action === WriteOpAction.Update, + ) if (hasUpdate) { throw new InvalidRequestError(`Updates are not yet supported.`) } @@ -122,7 +124,7 @@ export default function (server: Server, ctx: AppContext) { try { writes = await Promise.all( tx.writes.map((write) => { - if (write.action === 'create') { + if (write.action === WriteOpAction.Create) { return repo.prepareCreate({ did, collection: write.collection, @@ -130,7 +132,7 @@ export default function (server: Server, ctx: AppContext) { rkey: write.rkey, validate, }) - } else if (write.action === 'delete') { + } else if (write.action === WriteOpAction.Delete) { return repo.prepareDelete({ did, collection: write.collection, @@ -153,7 +155,7 @@ export default function (server: Server, ctx: AppContext) { await ctx.db.transaction(async (dbTxn) => { const now = new Date().toISOString() const repoTxn = ctx.services.repo(dbTxn) - await repoTxn.processWrites(did, authStore, writes, now) + await repoTxn.processWrites(did, writes, now) }) }, }) @@ -168,7 +170,6 @@ export default function (server: Server, ctx: AppContext) { if (did !== requester) { throw new AuthRequiredError() } - const authStore = ctx.getAuthstore(did) if (validate === false) { throw new InvalidRequestError( 'Unvalidated writes are not yet supported.', @@ -197,7 +198,7 @@ export default function (server: Server, ctx: AppContext) { await ctx.db.transaction(async (dbTxn) => { const repoTxn = ctx.services.repo(dbTxn) - await repoTxn.processWrites(did, authStore, [write], now) + await repoTxn.processWrites(did, [write], now) }) return { @@ -220,15 +221,10 @@ export default function (server: Server, ctx: AppContext) { throw new AuthRequiredError() } - const authStore = ctx.getAuthstore(did) const now = new Date().toISOString() - const write = await repo.prepareDelete({ did, collection, rkey }) - await ctx.db.transaction(async (dbTxn) => { - await ctx.services - .repo(dbTxn) - .processWrites(did, authStore, [write], now) + await ctx.services.repo(dbTxn).processWrites(did, [write], now) }) }, }) diff --git a/packages/pds/src/api/com/atproto/sync.ts b/packages/pds/src/api/com/atproto/sync.ts index 45bbe07c7e6..bef883264b4 100644 --- a/packages/pds/src/api/com/atproto/sync.ts +++ b/packages/pds/src/api/com/atproto/sync.ts @@ -1,14 +1,15 @@ -import { Server } from '../../../lexicon' +import { CID } from 'multiformats/cid' +import * as repo from '@atproto/repo' import { InvalidRequestError } from '@atproto/xrpc-server' -import { def as common } from '@atproto/common' -import { Repo } from '@atproto/repo' -import SqlBlockstore from '../../../sql-blockstore' +import { Server } from '../../../lexicon' +import SqlRepoStorage from '../../../sql-repo-storage' import AppContext from '../../../context' export default function (server: Server, ctx: AppContext) { - server.com.atproto.sync.getRoot(async ({ params }) => { + server.com.atproto.sync.getHead(async ({ params }) => { const { did } = params - const root = await ctx.services.repo(ctx.db).getRepoRoot(did) + const storage = new SqlRepoStorage(ctx.db, did) + const root = await storage.getHead() if (root === null) { throw new InvalidRequestError(`Could not find root for DID: ${did}`) } @@ -18,23 +19,73 @@ export default function (server: Server, ctx: AppContext) { } }) + server.com.atproto.sync.getCommitPath(async ({ params }) => { + const { did } = params + const storage = new SqlRepoStorage(ctx.db, did) + const earliest = params.earliest ? CID.parse(params.earliest) : null + const latest = params.latest + ? CID.parse(params.latest) + : await storage.getHead() + if (latest === null) { + throw new InvalidRequestError(`Could not find root for DID: ${did}`) + } + const commitPath = await storage.getCommitPath(latest, earliest) + if (commitPath === null) { + throw new InvalidRequestError( + `Could not find a valid commit path from ${latest.toString()} to ${earliest?.toString()}`, + ) + } + const commits = commitPath.map((c) => c.toString()) + return { + encoding: 'application/json', + body: { commits }, + } + }) + server.com.atproto.sync.getRepo(async ({ params }) => { const { did, from = null } = params - const repoRoot = await ctx.services.repo(ctx.db).getRepoRoot(did) - if (repoRoot === null) { + const storage = new SqlRepoStorage(ctx.db, did) + const head = await storage.getHead() + if (head === null) { throw new InvalidRequestError(`Could not find repo for DID: ${did}`) } - const blockstore = new SqlBlockstore(ctx.db, did) - const repo = await Repo.load(blockstore, repoRoot) - const fromCid = from ? common.strToCid.parse(from) : null - const diff = await repo.getDiffCar(fromCid) + const fromCid = from ? CID.parse(from) : null + const diff = await repo.getDiff(storage, head, fromCid) return { - encoding: 'application/cbor', + encoding: 'application/vnd.ipld.car', body: Buffer.from(diff), } }) - server.com.atproto.sync.updateRepo(async () => { - throw new InvalidRequestError('Not implemented') + server.com.atproto.sync.getCheckout(async ({ params }) => { + const { did } = params + const storage = new SqlRepoStorage(ctx.db, did) + const commit = params.commit + ? CID.parse(params.commit) + : await storage.getHead() + if (!commit) { + throw new InvalidRequestError(`Could not find repo for DID: ${did}`) + } + const checkout = await repo.getCheckout(storage, commit) + return { + encoding: 'application/vnd.ipld.car', + body: Buffer.from(checkout), + } + }) + + server.com.atproto.sync.getRecord(async ({ params }) => { + const { did, collection, rkey } = params + const storage = new SqlRepoStorage(ctx.db, did) + const commit = params.commit + ? CID.parse(params.commit) + : await storage.getHead() + if (!commit) { + throw new InvalidRequestError(`Could not find repo for DID: ${did}`) + } + const proof = await repo.getRecords(storage, commit, [{ collection, rkey }]) + return { + encoding: 'application/vnd.ipld.car', + body: Buffer.from(proof), + } }) } diff --git a/packages/pds/src/auth.ts b/packages/pds/src/auth.ts index 2fadc81681e..b8679675c28 100644 --- a/packages/pds/src/auth.ts +++ b/packages/pds/src/auth.ts @@ -1,4 +1,3 @@ -import * as auth from '@atproto/auth' import * as crypto from '@atproto/crypto' import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import * as uint8arrays from 'uint8arrays' @@ -35,13 +34,11 @@ export class ServerAuth { private _secret: string private _adminPass: string didResolver: DidResolver - verifier: auth.Verifier constructor(opts: ServerAuthOpts) { this._secret = opts.jwtSecret this._adminPass = opts.adminPass this.didResolver = opts.didResolver - this.verifier = new auth.Verifier({ didResolver: opts.didResolver }) } createAccessToken(did: string, expiresIn?: string | number) { diff --git a/packages/pds/src/config.ts b/packages/pds/src/config.ts index 166deceae17..f14c52e5990 100644 --- a/packages/pds/src/config.ts +++ b/packages/pds/src/config.ts @@ -25,7 +25,6 @@ export interface ServerConfigValues { privacyPolicyUrl?: string termsOfServiceUrl?: string - blockstoreLocation?: string databaseLocation?: string availableUserDomains: string[] @@ -85,7 +84,6 @@ export class ServerConfig { const privacyPolicyUrl = process.env.PRIVACY_POLICY_URL const termsOfServiceUrl = process.env.TERMS_OF_SERVICE_URL - const blockstoreLocation = process.env.BLOCKSTORE_LOC const databaseLocation = process.env.DATABASE_LOC const blobstoreLocation = process.env.BLOBSTORE_LOC @@ -133,7 +131,6 @@ export class ServerConfig { inviteRequired, privacyPolicyUrl, termsOfServiceUrl, - blockstoreLocation, databaseLocation, availableUserDomains, imgUriSalt, @@ -245,14 +242,6 @@ export class ServerConfig { return this.cfg.termsOfServiceUrl } - get blockstoreLocation() { - return this.cfg.blockstoreLocation - } - - get useMemoryBlockstore() { - return !this.blockstoreLocation - } - get databaseLocation() { return this.cfg.databaseLocation } diff --git a/packages/pds/src/context.ts b/packages/pds/src/context.ts index 798f77dd847..744923844f8 100644 --- a/packages/pds/src/context.ts +++ b/packages/pds/src/context.ts @@ -1,5 +1,5 @@ -import { DidableKey } from '@atproto/auth' import * as plc from '@atproto/plc' +import * as crypto from '@atproto/crypto' import { Database } from './db' import { ServerConfig } from './config' import * as auth from './auth' @@ -14,7 +14,7 @@ export class AppContext { private opts: { db: Database blobstore: BlobStore - keypair: DidableKey + keypair: crypto.Keypair auth: auth.ServerAuth imgUriBuilder: ImageUriBuilder cfg: ServerConfig @@ -32,7 +32,7 @@ export class AppContext { return this.opts.blobstore } - get keypair(): DidableKey { + get keypair(): crypto.Keypair { return this.opts.keypair } @@ -79,10 +79,6 @@ export class AppContext { get plcClient(): plc.PlcClient { return new plc.PlcClient(this.cfg.didPlcUrl) } - - getAuthstore(did: string) { - return this.auth.verifier.loadAuthStore(this.keypair, [], did) - } } export default AppContext diff --git a/packages/pds/src/db/database-schema.ts b/packages/pds/src/db/database-schema.ts index 01f266db736..e2bf9d214b4 100644 --- a/packages/pds/src/db/database-schema.ts +++ b/packages/pds/src/db/database-schema.ts @@ -4,6 +4,8 @@ import * as didHandle from './tables/did-handle' import * as repoRoot from './tables/repo-root' import * as refreshToken from './tables/refresh-token' import * as record from './tables/record' +import * as repoCommitBlock from './tables/repo-commit-block' +import * as repoCommitHistory from './tables/repo-commit-history' import * as ipldBlock from './tables/ipld-block' import * as ipldBlockCreator from './tables/ipld-block-creator' import * as inviteCode from './tables/invite-code' @@ -30,6 +32,8 @@ export type DatabaseSchemaType = user.PartialDB & refreshToken.PartialDB & repoRoot.PartialDB & record.PartialDB & + repoCommitBlock.PartialDB & + repoCommitHistory.PartialDB & ipldBlock.PartialDB & ipldBlockCreator.PartialDB & inviteCode.PartialDB & diff --git a/packages/pds/src/db/migrations/20230118T223059239Z-repo-sync-data.ts b/packages/pds/src/db/migrations/20230118T223059239Z-repo-sync-data.ts new file mode 100644 index 00000000000..5e88ca67327 --- /dev/null +++ b/packages/pds/src/db/migrations/20230118T223059239Z-repo-sync-data.ts @@ -0,0 +1,104 @@ +import { chunkArray } from '@atproto/common' +import { MemoryBlockstore } from '@atproto/repo' +import { Kysely } from 'kysely' +import { CID } from 'multiformats/cid' +import { RepoCommitBlock } from '../tables/repo-commit-block' +import { RepoCommitHistory } from '../tables/repo-commit-history' + +const commitBlockTable = 'repo_commit_block' +const commitHistoryTable = 'repo_commit_history' + +export async function up(db: Kysely): Promise { + await db.schema + .createTable(commitBlockTable) + .addColumn('commit', 'varchar', (col) => col.notNull()) + .addColumn('block', 'varchar', (col) => col.notNull()) + .addColumn('creator', 'varchar', (col) => col.notNull()) + .addPrimaryKeyConstraint(`${commitBlockTable}_pkey`, [ + 'creator', + 'commit', + 'block', + ]) + .execute() + await db.schema + .createTable(commitHistoryTable) + .addColumn('commit', 'varchar', (col) => col.notNull()) + .addColumn('prev', 'varchar') + .addColumn('creator', 'varchar', (col) => col.notNull()) + .addPrimaryKeyConstraint(`${commitHistoryTable}_pkey`, [ + 'creator', + 'commit', + ]) + .execute() + + const migrateUser = async (did: string, root: CID) => { + const userBlocks = await db + .selectFrom('ipld_block') + .innerJoin( + 'ipld_block_creator as creator', + 'creator.cid', + 'ipld_block.cid', + ) + .where('creator.did', '=', did) + .select(['ipld_block.cid as cid', 'ipld_block.content as content']) + .execute() + const storage = new MemoryBlockstore() + userBlocks.forEach((row) => { + storage.putBlock(CID.parse(row.cid), row.content) + }) + + const commitData = await storage.getCommits(root, null) + if (!commitData) return + + const commitBlock: RepoCommitBlock[] = [] + const commitHistory: RepoCommitHistory[] = [] + + for (let i = 0; i < commitData.length; i++) { + const commit = commitData[i] + const prev = commitData[i - 1] + commit.blocks.forEach((_bytes, cid) => { + commitBlock.push({ + commit: commit.commit.toString(), + block: cid.toString(), + creator: did, + }) + }) + commitHistory.push({ + commit: commit.commit.toString(), + prev: prev ? prev.commit.toString() : null, + creator: did, + }) + } + const promises: Promise[] = [] + chunkArray(commitBlock, 500).forEach((batch) => { + promises.push( + db + .insertInto('repo_commit_block') + .values(batch) + .onConflict((oc) => oc.doNothing()) + .execute(), + ) + }) + chunkArray(commitHistory, 500).forEach((batch) => { + promises.push( + db + .insertInto('repo_commit_history') + .values(batch) + .onConflict((oc) => oc.doNothing()) + .execute(), + ) + }) + return Promise.all(promises) + } + + const userRoots = await db.selectFrom('repo_root').selectAll().execute() + + await Promise.all( + userRoots.map((row) => migrateUser(row.did, CID.parse(row.root))), + ) +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable(commitHistoryTable).execute() + await db.schema.dropTable(commitBlockTable).execute() +} diff --git a/packages/pds/src/db/migrations/index.ts b/packages/pds/src/db/migrations/index.ts index f23d6ab134f..fd1a66ec7c1 100644 --- a/packages/pds/src/db/migrations/index.ts +++ b/packages/pds/src/db/migrations/index.ts @@ -10,3 +10,4 @@ export * as _20221212T195416407Z from './20221212T195416407Z-post-media' export * as _20221215T220356370Z from './20221215T220356370Z-password-reset-otp' export * as _20221226T213635517Z from './20221226T213635517Z-mute-init' export * as _20221230T215012029Z from './20221230T215012029Z-moderation-init' +export * as _20230118T223059239Z from './20230118T223059239Z-repo-sync-data' diff --git a/packages/pds/src/db/tables/repo-commit-block.ts b/packages/pds/src/db/tables/repo-commit-block.ts new file mode 100644 index 00000000000..2493a4a93cd --- /dev/null +++ b/packages/pds/src/db/tables/repo-commit-block.ts @@ -0,0 +1,9 @@ +export interface RepoCommitBlock { + commit: string + block: string + creator: string +} + +export const tableName = 'repo_commit_block' + +export type PartialDB = { [tableName]: RepoCommitBlock } diff --git a/packages/pds/src/db/tables/repo-commit-history.ts b/packages/pds/src/db/tables/repo-commit-history.ts new file mode 100644 index 00000000000..d92cb21f3f1 --- /dev/null +++ b/packages/pds/src/db/tables/repo-commit-history.ts @@ -0,0 +1,9 @@ +export interface RepoCommitHistory { + commit: string + prev: string | null + creator: string +} + +export const tableName = 'repo_commit_history' + +export type PartialDB = { [tableName]: RepoCommitHistory } diff --git a/packages/pds/src/index.ts b/packages/pds/src/index.ts index ac7e53fead1..d13ffe5b76e 100644 --- a/packages/pds/src/index.ts +++ b/packages/pds/src/index.ts @@ -8,7 +8,7 @@ import express from 'express' import cors from 'cors' import http from 'http' import events from 'events' -import { DidableKey } from '@atproto/auth' +import * as crypto from '@atproto/crypto' import { BlobStore } from '@atproto/repo' import { DidResolver } from '@atproto/did-resolver' import API, { health } from './api' @@ -47,7 +47,7 @@ export class PDS { static create(opts: { db: Database blobstore: BlobStore - keypair: DidableKey + keypair: crypto.Keypair config: ServerConfig }): PDS { const { db, blobstore, keypair, config } = opts @@ -90,7 +90,12 @@ export class PDS { config.imgUriKey, ) - const services = createServices({ messageQueue, blobstore, imgUriBuilder }) + const services = createServices({ + keypair, + messageQueue, + blobstore, + imgUriBuilder, + }) const ctx = new AppContext({ db, diff --git a/packages/pds/src/lexicon/index.ts b/packages/pds/src/lexicon/index.ts index af0b8fc9a65..80419b5af66 100644 --- a/packages/pds/src/lexicon/index.ts +++ b/packages/pds/src/lexicon/index.ts @@ -39,9 +39,11 @@ import * as ComAtprotoSessionCreate from './types/com/atproto/session/create' import * as ComAtprotoSessionDelete from './types/com/atproto/session/delete' import * as ComAtprotoSessionGet from './types/com/atproto/session/get' import * as ComAtprotoSessionRefresh from './types/com/atproto/session/refresh' +import * as ComAtprotoSyncGetCheckout from './types/com/atproto/sync/getCheckout' +import * as ComAtprotoSyncGetCommitPath from './types/com/atproto/sync/getCommitPath' +import * as ComAtprotoSyncGetHead from './types/com/atproto/sync/getHead' +import * as ComAtprotoSyncGetRecord from './types/com/atproto/sync/getRecord' import * as ComAtprotoSyncGetRepo from './types/com/atproto/sync/getRepo' -import * as ComAtprotoSyncGetRoot from './types/com/atproto/sync/getRoot' -import * as ComAtprotoSyncUpdateRepo from './types/com/atproto/sync/updateRepo' import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile' import * as AppBskyActorGetSuggestions from './types/app/bsky/actor/getSuggestions' import * as AppBskyActorSearch from './types/app/bsky/actor/search' @@ -449,24 +451,38 @@ export class SyncNS { this._server = server } - getRepo( - cfg: ConfigOf>>, + getCheckout( + cfg: ConfigOf>>, ) { - const nsid = 'com.atproto.sync.getRepo' // @ts-ignore + const nsid = 'com.atproto.sync.getCheckout' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + + getCommitPath( + cfg: ConfigOf>>, + ) { + const nsid = 'com.atproto.sync.getCommitPath' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } - getRoot( - cfg: ConfigOf>>, + getHead( + cfg: ConfigOf>>, ) { - const nsid = 'com.atproto.sync.getRoot' // @ts-ignore + const nsid = 'com.atproto.sync.getHead' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } - updateRepo( - cfg: ConfigOf>>, + getRecord( + cfg: ConfigOf>>, ) { - const nsid = 'com.atproto.sync.updateRepo' // @ts-ignore + const nsid = 'com.atproto.sync.getRecord' // @ts-ignore + return this._server.xrpc.method(nsid, cfg) + } + + getRepo( + cfg: ConfigOf>>, + ) { + const nsid = 'com.atproto.sync.getRepo' // @ts-ignore return this._server.xrpc.method(nsid, cfg) } } diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 256ad6c0847..93a18c7fc91 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -1823,9 +1823,9 @@ export const schemaDict = { }, }, }, - ComAtprotoSyncGetRepo: { + ComAtprotoSyncGetCheckout: { lexicon: 1, - id: 'com.atproto.sync.getRepo', + id: 'com.atproto.sync.getCheckout', defs: { main: { type: 'query', @@ -1838,25 +1838,69 @@ export const schemaDict = { type: 'string', description: 'The DID of the repo.', }, - from: { + commit: { type: 'string', - description: 'A past commit CID.', + description: + 'The commit to get the checkout from. Defaults to current HEAD.', }, }, }, output: { - encoding: 'application/cbor', + encoding: 'application/vnd.ipld.car', + }, + }, + }, + }, + ComAtprotoSyncGetCommitPath: { + lexicon: 1, + id: 'com.atproto.sync.getCommitPath', + defs: { + main: { + type: 'query', + description: 'Gets the path of repo commits', + parameters: { + type: 'params', + required: ['did'], + properties: { + did: { + type: 'string', + description: 'The DID of the repo.', + }, + latest: { + type: 'string', + description: 'The most recent commit', + }, + earliest: { + type: 'string', + description: 'The earliest commit to start from', + }, + }, + }, + output: { + encoding: 'application/json', + schema: { + type: 'object', + required: ['commits'], + properties: { + commits: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, }, }, }, }, - ComAtprotoSyncGetRoot: { + ComAtprotoSyncGetHead: { lexicon: 1, - id: 'com.atproto.sync.getRoot', + id: 'com.atproto.sync.getHead', defs: { main: { type: 'query', - description: 'Gets the current root CID of a repo.', + description: 'Gets the current HEAD CID of a repo.', parameters: { type: 'params', required: ['did'], @@ -1882,13 +1926,47 @@ export const schemaDict = { }, }, }, - ComAtprotoSyncUpdateRepo: { + ComAtprotoSyncGetRecord: { lexicon: 1, - id: 'com.atproto.sync.updateRepo', + id: 'com.atproto.sync.getRecord', defs: { main: { - type: 'procedure', - description: 'Writes commits to a repo.', + type: 'query', + description: + 'Gets blocks needed for existence or non-existence of record.', + parameters: { + type: 'params', + required: ['did', 'collection', 'rkey'], + properties: { + did: { + type: 'string', + description: 'The DID of the repo.', + }, + collection: { + type: 'string', + }, + rkey: { + type: 'string', + }, + commit: { + type: 'string', + description: 'An optional past commit CID.', + }, + }, + }, + output: { + encoding: 'application/vnd.ipld.car', + }, + }, + }, + }, + ComAtprotoSyncGetRepo: { + lexicon: 1, + id: 'com.atproto.sync.getRepo', + defs: { + main: { + type: 'query', + description: 'Gets the repo state.', parameters: { type: 'params', required: ['did'], @@ -1897,10 +1975,14 @@ export const schemaDict = { type: 'string', description: 'The DID of the repo.', }, + from: { + type: 'string', + description: 'A past commit CID.', + }, }, }, - input: { - encoding: 'application/cbor', + output: { + encoding: 'application/vnd.ipld.car', }, }, }, @@ -3753,9 +3835,11 @@ export const ids = { ComAtprotoSessionDelete: 'com.atproto.session.delete', ComAtprotoSessionGet: 'com.atproto.session.get', ComAtprotoSessionRefresh: 'com.atproto.session.refresh', + ComAtprotoSyncGetCheckout: 'com.atproto.sync.getCheckout', + ComAtprotoSyncGetCommitPath: 'com.atproto.sync.getCommitPath', + ComAtprotoSyncGetHead: 'com.atproto.sync.getHead', + ComAtprotoSyncGetRecord: 'com.atproto.sync.getRecord', ComAtprotoSyncGetRepo: 'com.atproto.sync.getRepo', - ComAtprotoSyncGetRoot: 'com.atproto.sync.getRoot', - ComAtprotoSyncUpdateRepo: 'com.atproto.sync.updateRepo', AppBskyActorGetProfile: 'app.bsky.actor.getProfile', AppBskyActorGetSuggestions: 'app.bsky.actor.getSuggestions', AppBskyActorProfile: 'app.bsky.actor.profile', diff --git a/packages/pds/src/lexicon/types/com/atproto/sync/updateRepo.ts b/packages/pds/src/lexicon/types/com/atproto/sync/getCheckout.ts similarity index 66% rename from packages/pds/src/lexicon/types/com/atproto/sync/updateRepo.ts rename to packages/pds/src/lexicon/types/com/atproto/sync/getCheckout.ts index a80d15c53d2..969836384cd 100644 --- a/packages/pds/src/lexicon/types/com/atproto/sync/updateRepo.ts +++ b/packages/pds/src/lexicon/types/com/atproto/sync/getCheckout.ts @@ -11,13 +11,16 @@ import { HandlerAuth } from '@atproto/xrpc-server' export interface QueryParams { /** The DID of the repo. */ did: string + /** The commit to get the checkout from. Defaults to current HEAD. */ + commit?: string } -export type InputSchema = string | Uint8Array +export type InputSchema = undefined +export type HandlerInput = undefined -export interface HandlerInput { - encoding: 'application/cbor' - body: stream.Readable +export interface HandlerSuccess { + encoding: 'application/vnd.ipld.car' + body: Uint8Array | stream.Readable } export interface HandlerError { @@ -25,7 +28,7 @@ export interface HandlerError { message?: string } -export type HandlerOutput = HandlerError | void +export type HandlerOutput = HandlerError | HandlerSuccess export type Handler = (ctx: { auth: HA params: QueryParams diff --git a/packages/pds/src/lexicon/types/com/atproto/sync/getCommitPath.ts b/packages/pds/src/lexicon/types/com/atproto/sync/getCommitPath.ts new file mode 100644 index 00000000000..025e3819ac0 --- /dev/null +++ b/packages/pds/src/lexicon/types/com/atproto/sync/getCommitPath.ts @@ -0,0 +1,45 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import { ValidationResult } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams { + /** The DID of the repo. */ + did: string + /** The most recent commit */ + latest?: string + /** The earliest commit to start from */ + earliest?: string +} + +export type InputSchema = undefined + +export interface OutputSchema { + commits: string[] + [k: string]: unknown +} + +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/json' + body: OutputSchema +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type Handler = (ctx: { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +}) => Promise | HandlerOutput diff --git a/packages/pds/src/lexicon/types/com/atproto/sync/getRoot.ts b/packages/pds/src/lexicon/types/com/atproto/sync/getHead.ts similarity index 100% rename from packages/pds/src/lexicon/types/com/atproto/sync/getRoot.ts rename to packages/pds/src/lexicon/types/com/atproto/sync/getHead.ts diff --git a/packages/pds/src/lexicon/types/com/atproto/sync/getRecord.ts b/packages/pds/src/lexicon/types/com/atproto/sync/getRecord.ts new file mode 100644 index 00000000000..f10c0b580b6 --- /dev/null +++ b/packages/pds/src/lexicon/types/com/atproto/sync/getRecord.ts @@ -0,0 +1,40 @@ +/** + * GENERATED CODE - DO NOT MODIFY + */ +import express from 'express' +import stream from 'stream' +import { ValidationResult } from '@atproto/lexicon' +import { lexicons } from '../../../../lexicons' +import { isObj, hasProp } from '../../../../util' +import { HandlerAuth } from '@atproto/xrpc-server' + +export interface QueryParams { + /** The DID of the repo. */ + did: string + collection: string + rkey: string + /** An optional past commit CID. */ + commit?: string +} + +export type InputSchema = undefined +export type HandlerInput = undefined + +export interface HandlerSuccess { + encoding: 'application/vnd.ipld.car' + body: Uint8Array | stream.Readable +} + +export interface HandlerError { + status: number + message?: string +} + +export type HandlerOutput = HandlerError | HandlerSuccess +export type Handler = (ctx: { + auth: HA + params: QueryParams + input: HandlerInput + req: express.Request + res: express.Response +}) => Promise | HandlerOutput diff --git a/packages/pds/src/lexicon/types/com/atproto/sync/getRepo.ts b/packages/pds/src/lexicon/types/com/atproto/sync/getRepo.ts index 0291726e30f..7dad9e19089 100644 --- a/packages/pds/src/lexicon/types/com/atproto/sync/getRepo.ts +++ b/packages/pds/src/lexicon/types/com/atproto/sync/getRepo.ts @@ -19,7 +19,7 @@ export type InputSchema = undefined export type HandlerInput = undefined export interface HandlerSuccess { - encoding: 'application/cbor' + encoding: 'application/vnd.ipld.car' body: Uint8Array | stream.Readable } diff --git a/packages/pds/src/repo/prepare.ts b/packages/pds/src/repo/prepare.ts index fd16d26fbe7..a5efe55a51e 100644 --- a/packages/pds/src/repo/prepare.ts +++ b/packages/pds/src/repo/prepare.ts @@ -1,6 +1,6 @@ import { CID } from 'multiformats/cid' import { AtUri } from '@atproto/uri' -import { cidForData, TID } from '@atproto/common' +import { cidForCbor, TID } from '@atproto/common' import { PreparedCreate, PreparedUpdate, @@ -14,10 +14,11 @@ import { import * as lex from '../lexicon/lexicons' import { LexiconDefNotFoundError } from '@atproto/lexicon' import { - DeleteOp, + RecordDeleteOp, RecordCreateOp, RecordUpdateOp, RecordWriteOp, + WriteOpAction, } from '@atproto/repo' // @TODO do this dynamically off of schemas @@ -131,9 +132,9 @@ export const prepareCreate = async (opts: { } const rkey = opts.rkey || determineRkey(collection) return { - action: 'create', + action: WriteOpAction.Create, uri: AtUri.make(did, collection, rkey), - cid: await cidForData(record), + cid: await cidForCbor(record), record, blobs: blobsForWrite(record), } @@ -152,9 +153,9 @@ export const prepareUpdate = async (opts: { assertValidRecord(record) } return { - action: 'update', + action: WriteOpAction.Update, uri: AtUri.make(did, collection, rkey), - cid: await cidForData(record), + cid: await cidForCbor(record), record, blobs: blobsForWrite(record), } @@ -167,38 +168,38 @@ export const prepareDelete = (opts: { }): PreparedDelete => { const { did, collection, rkey } = opts return { - action: 'delete', + action: WriteOpAction.Delete, uri: AtUri.make(did, collection, rkey), } } export const createWriteToOp = (write: PreparedCreate): RecordCreateOp => ({ - action: 'create', + action: WriteOpAction.Create, collection: write.uri.collection, rkey: write.uri.rkey, - value: write.record, + record: write.record, }) export const updateWriteToOp = (write: PreparedUpdate): RecordUpdateOp => ({ - action: 'update', + action: WriteOpAction.Update, collection: write.uri.collection, rkey: write.uri.rkey, - value: write.record, + record: write.record, }) -export const deleteWriteToOp = (write: PreparedDelete): DeleteOp => ({ - action: 'delete', +export const deleteWriteToOp = (write: PreparedDelete): RecordDeleteOp => ({ + action: WriteOpAction.Delete, collection: write.uri.collection, rkey: write.uri.rkey, }) export const writeToOp = (write: PreparedWrite): RecordWriteOp => { switch (write.action) { - case 'create': + case WriteOpAction.Create: return createWriteToOp(write) - case 'update': + case WriteOpAction.Update: return updateWriteToOp(write) - case 'delete': + case WriteOpAction.Delete: return deleteWriteToOp(write) default: throw new Error(`Unrecognized action: ${write}`) diff --git a/packages/pds/src/repo/types.ts b/packages/pds/src/repo/types.ts index 242d7db8e2b..79763772c9f 100644 --- a/packages/pds/src/repo/types.ts +++ b/packages/pds/src/repo/types.ts @@ -1,5 +1,6 @@ import { CID } from 'multiformats/cid' import { AtUri } from '@atproto/uri' +import { WriteOpAction } from '@atproto/repo' export type ImageConstraint = { type: 'image' @@ -26,7 +27,7 @@ export type BlobRef = { } export type PreparedCreate = { - action: 'create' + action: WriteOpAction.Create uri: AtUri cid: CID record: Record @@ -34,7 +35,7 @@ export type PreparedCreate = { } export type PreparedUpdate = { - action: 'update' + action: WriteOpAction.Update uri: AtUri cid: CID record: Record @@ -42,7 +43,7 @@ export type PreparedUpdate = { } export type PreparedDelete = { - action: 'delete' + action: WriteOpAction.Delete uri: AtUri } diff --git a/packages/pds/src/services/actor.ts b/packages/pds/src/services/actor.ts index bac3274bd7d..c64e1755224 100644 --- a/packages/pds/src/services/actor.ts +++ b/packages/pds/src/services/actor.ts @@ -129,7 +129,7 @@ export class ActorService { ) { this.db.assertTransaction() log.debug({ handle, did }, 'registering did-handle') - const declarationCid = await common.cidForData(declaration) + const declarationCid = await common.cidForCbor(declaration) const updated = await this.db.db .updateTable('did_handle') .set({ diff --git a/packages/pds/src/services/feed/index.ts b/packages/pds/src/services/feed/index.ts index 6039b9ff840..a8fe6e68620 100644 --- a/packages/pds/src/services/feed/index.ts +++ b/packages/pds/src/services/feed/index.ts @@ -252,7 +252,7 @@ export class FeedService { uri: post.uri, cid: post.cid, author: author, - record: common.ipldBytesToRecord(post.recordBytes), + record: common.cborDecode(post.recordBytes), embed: embeds[uri], replyCount: post.replyCount, repostCount: post.repostCount, diff --git a/packages/pds/src/services/index.ts b/packages/pds/src/services/index.ts index 1f36425ca20..3228e115d3b 100644 --- a/packages/pds/src/services/index.ts +++ b/packages/pds/src/services/index.ts @@ -1,3 +1,4 @@ +import * as crypto from '@atproto/crypto' import { BlobStore } from '@atproto/repo' import Database from '../db' import { MessageQueue } from '../event-stream/types' @@ -10,17 +11,18 @@ import { RepoService } from './repo' import { ModerationService } from './moderation' export function createServices(resources: { + keypair: crypto.Keypair messageQueue: MessageQueue blobstore: BlobStore imgUriBuilder: ImageUriBuilder }): Services { - const { messageQueue, blobstore, imgUriBuilder } = resources + const { keypair, messageQueue, blobstore, imgUriBuilder } = resources return { actor: ActorService.creator(), auth: AuthService.creator(), feed: FeedService.creator(imgUriBuilder), record: RecordService.creator(messageQueue), - repo: RepoService.creator(messageQueue, blobstore), + repo: RepoService.creator(keypair, messageQueue, blobstore), moderation: ModerationService.creator(messageQueue, blobstore), } } diff --git a/packages/pds/src/services/moderation/index.ts b/packages/pds/src/services/moderation/index.ts index 01697511c98..91c7c27e41d 100644 --- a/packages/pds/src/services/moderation/index.ts +++ b/packages/pds/src/services/moderation/index.ts @@ -6,9 +6,9 @@ import { InvalidRequestError } from '@atproto/xrpc-server' import Database from '../../db' import { MessageQueue } from '../../event-stream/types' import { ModerationAction, ModerationReport } from '../../db/tables/moderation' -import { RepoService } from '../repo' import { RecordService } from '../record' import { ModerationViews } from './views' +import SqlRepoStorage from '../../sql-repo-storage' export class ModerationService { constructor( @@ -24,7 +24,6 @@ export class ModerationService { views = new ModerationViews(this.db, this.messageQueue) services = { - repo: RepoService.creator(this.messageQueue, this.blobstore), record: RecordService.creator(this.messageQueue), } @@ -139,7 +138,7 @@ export class ModerationService { // Resolve subject info let subjectInfo: SubjectInfo if ('did' in subject) { - const repo = await this.services.repo(this.db).getRepoRoot(subject.did) + const repo = await new SqlRepoStorage(this.db, subject.did).getHead() if (!repo) throw new InvalidRequestError('Repo not found') subjectInfo = { subjectType: 'com.atproto.repo.repoRef', @@ -302,7 +301,7 @@ export class ModerationService { // Resolve subject info let subjectInfo: SubjectInfo if ('did' in subject) { - const repo = await this.services.repo(this.db).getRepoRoot(subject.did) + const repo = await new SqlRepoStorage(this.db, subject.did).getHead() if (!repo) throw new InvalidRequestError('Repo not found') subjectInfo = { subjectType: 'com.atproto.repo.repoRef', diff --git a/packages/pds/src/services/moderation/views.ts b/packages/pds/src/services/moderation/views.ts index b7d3392e9f1..8fe83ab0816 100644 --- a/packages/pds/src/services/moderation/views.ts +++ b/packages/pds/src/services/moderation/views.ts @@ -1,5 +1,5 @@ import { Selectable } from 'kysely' -import { ipldBytesToRecord } from '@atproto/common' +import { cborBytesToRecord } from '@atproto/common' import { AtUri } from '@atproto/uri' import Database from '../../db' import { MessageQueue } from '../../event-stream/types' @@ -78,10 +78,10 @@ export class ModerationViews { const { email, declarationBytes, profileBytes } = infoByDid[r.did] ?? {} const relatedRecords: object[] = [] if (declarationBytes) { - relatedRecords.push(ipldBytesToRecord(declarationBytes)) + relatedRecords.push(cborBytesToRecord(declarationBytes)) } if (profileBytes) { - relatedRecords.push(ipldBytesToRecord(profileBytes)) + relatedRecords.push(cborBytesToRecord(profileBytes)) } return { did: r.did, diff --git a/packages/pds/src/services/record/index.ts b/packages/pds/src/services/record/index.ts index 4fcb9d2506e..f2af42ac3b3 100644 --- a/packages/pds/src/services/record/index.ts +++ b/packages/pds/src/services/record/index.ts @@ -147,7 +147,7 @@ export class RecordService { return { uri: row.uri, cid: row.cid, - value: common.ipldBytesToRecord(row.content), + value: common.cborDecode(row.content), } }) } @@ -180,7 +180,7 @@ export class RecordService { return { uri: record.uri, cid: record.cid, - value: common.ipldBytesToRecord(record.content), + value: common.cborDecode(record.content), indexedAt: record.indexedAt, takedownId: record.takedownId, } diff --git a/packages/pds/src/services/record/processor.ts b/packages/pds/src/services/record/processor.ts index 509a33572de..b1eff0f6913 100644 --- a/packages/pds/src/services/record/processor.ts +++ b/packages/pds/src/services/record/processor.ts @@ -104,7 +104,7 @@ export class RecordProcessor { if (!found) { return this.params.eventsForDelete(deleted, null) } - const record = common.ipldBytesToRecord(found.content) + const record = common.cborDecode(found.content) if (!this.matchesSchema(record)) { return this.params.eventsForDelete(deleted, null) } diff --git a/packages/pds/src/services/repo/blobs.ts b/packages/pds/src/services/repo/blobs.ts index ed18b2d4504..fb2ececd3ef 100644 --- a/packages/pds/src/services/repo/blobs.ts +++ b/packages/pds/src/services/repo/blobs.ts @@ -2,7 +2,7 @@ import stream from 'stream' import { CID } from 'multiformats/cid' import bytes from 'bytes' import { fromStream as fileTypeFromStream } from 'file-type' -import { BlobStore } from '@atproto/repo' +import { BlobStore, WriteOpAction } from '@atproto/repo' import { AtUri } from '@atproto/uri' import { sha256Stream } from '@atproto/crypto' import { cloneStream, sha256RawToCid, streamSize } from '@atproto/common' @@ -53,7 +53,10 @@ export class RepoBlobs { async processWriteBlobs(did: string, commit: CID, writes: PreparedWrite[]) { const blobPromises: Promise[] = [] for (const write of writes) { - if (write.action === 'create' || write.action === 'update') { + if ( + write.action === WriteOpAction.Create || + write.action === WriteOpAction.Update + ) { for (const blob of write.blobs) { blobPromises.push(this.verifyBlobAndMakePermanent(blob)) blobPromises.push(this.associateBlob(blob, write.uri, commit, did)) diff --git a/packages/pds/src/services/repo/index.ts b/packages/pds/src/services/repo/index.ts index 7806044c739..10a964520bc 100644 --- a/packages/pds/src/services/repo/index.ts +++ b/packages/pds/src/services/repo/index.ts @@ -1,15 +1,13 @@ import { CID } from 'multiformats/cid' -import * as auth from '@atproto/auth' -import { BlobStore, Repo } from '@atproto/repo' +import * as crypto from '@atproto/crypto' +import { BlobStore, Repo, WriteOpAction } from '@atproto/repo' import { InvalidRequestError } from '@atproto/xrpc-server' import Database from '../../db' -import { dbLogger as log } from '../../logger' import { MessageQueue } from '../../event-stream/types' -import SqlBlockstore from '../../sql-blockstore' +import SqlRepoStorage from '../../sql-repo-storage' import { PreparedCreate, PreparedWrite } from '../../repo/types' import { RepoBlobs } from './blobs' import { createWriteToOp, writeToOp } from '../../repo' -import { notSoftDeletedClause } from '../../db/util' import { RecordService } from '../record' export class RepoService { @@ -17,94 +15,38 @@ export class RepoService { constructor( public db: Database, + public keypair: crypto.Keypair, public messageQueue: MessageQueue, public blobstore: BlobStore, ) { this.blobs = new RepoBlobs(db, blobstore) } - static creator(messageQueue: MessageQueue, blobstore: BlobStore) { - return (db: Database) => new RepoService(db, messageQueue, blobstore) + static creator( + keypair: crypto.Keypair, + messageQueue: MessageQueue, + blobstore: BlobStore, + ) { + return (db: Database) => + new RepoService(db, keypair, messageQueue, blobstore) } services = { record: RecordService.creator(this.messageQueue), } - async getRepoRoot(did: string, forUpdate?: boolean): Promise { - let builder = this.db.db - .selectFrom('repo_root') - .selectAll() - .where('did', '=', did) - if (forUpdate) { - this.db.assertTransaction() - if (this.db.dialect !== 'sqlite') { - // SELECT FOR UPDATE is not supported by sqlite, but sqlite txs are SERIALIZABLE so we don't actually need it - builder = builder.forUpdate() - } - } - const found = await builder.executeTakeFirst() - return found ? CID.parse(found.root) : null - } - - async updateRepoRoot( - did: string, - root: CID, - prev: CID, - timestamp?: string, - ): Promise { - log.debug({ did, root: root.toString() }, 'updating repo root') - const res = await this.db.db - .updateTable('repo_root') - .set({ - root: root.toString(), - indexedAt: timestamp || new Date().toISOString(), - }) - .where('did', '=', did) - .where('root', '=', prev.toString()) - .executeTakeFirst() - if (res.numUpdatedRows > 0) { - log.info({ did, root: root.toString() }, 'updated repo root') - return true - } else { - log.info( - { did, root: root.toString() }, - 'failed to update repo root: misordered', - ) - return false - } - } - - async createRepo( - did: string, - authStore: auth.AuthStore, - writes: PreparedCreate[], - now: string, - ) { + async createRepo(did: string, writes: PreparedCreate[], now: string) { this.db.assertTransaction() - const blockstore = new SqlBlockstore(this.db, did, now) + const storage = new SqlRepoStorage(this.db, did, now) const writeOps = writes.map(createWriteToOp) - const repo = await Repo.create(blockstore, did, authStore, writeOps) - await this.db.db - .insertInto('repo_root') - .values({ - did: did, - root: repo.cid.toString(), - indexedAt: now, - }) - .execute() + await Repo.create(storage, did, this.keypair, writeOps) } - async processWrites( - did: string, - authStore: auth.AuthStore, - writes: PreparedWrite[], - now: string, - ) { + async processWrites(did: string, writes: PreparedWrite[], now: string) { // make structural write to repo & send to indexing // @TODO get commitCid first so we can do all db actions in tandem const [commit] = await Promise.all([ - this.writeToRepo(did, authStore, writes, now), + this.writeToRepo(did, writes, now), this.indexWrites(writes, now), ]) // make blobs permanent & associate w commit + recordUri in DB @@ -113,29 +55,20 @@ export class RepoService { async writeToRepo( did: string, - authStore: auth.AuthStore, writes: PreparedWrite[], now: string, ): Promise { this.db.assertTransaction() - const blockstore = new SqlBlockstore(this.db, did, now) - const currRoot = await this.getRepoRoot(did, true) + const storage = new SqlRepoStorage(this.db, did, now) + const currRoot = await storage.getHead(true) if (!currRoot) { throw new InvalidRequestError( `${did} is not a registered repo on this server`, ) } const writeOps = writes.map(writeToOp) - const repo = await Repo.load(blockstore, currRoot) - const updated = await repo - .stageUpdate(writeOps) - .createCommit(authStore, async (prev, curr) => { - const success = await this.updateRepoRoot(did, curr, prev, now) - if (!success) { - throw new Error('Repo root update failed, could not linearize') - } - return null - }) + const repo = await Repo.load(storage, currRoot) + const updated = await repo.applyCommit(writeOps, this.keypair) return updated.cid } @@ -144,9 +77,9 @@ export class RepoService { const recordTxn = this.services.record(this.db) await Promise.all( writes.map(async (write) => { - if (write.action === 'create') { + if (write.action === WriteOpAction.Create) { await recordTxn.indexRecord(write.uri, write.cid, write.record, now) - } else if (write.action === 'delete') { + } else if (write.action === WriteOpAction.Delete) { await recordTxn.deleteRecord(write.uri) } }), diff --git a/packages/pds/src/sql-blockstore.ts b/packages/pds/src/sql-blockstore.ts deleted file mode 100644 index d51df5674d2..00000000000 --- a/packages/pds/src/sql-blockstore.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { IpldStore } from '@atproto/repo' -import { CID } from 'multiformats/cid' -import Database from './db' -import { IpldBlock } from './db/tables/ipld-block' -import { IpldBlockCreator } from './db/tables/ipld-block-creator' - -export class SqlBlockstore extends IpldStore { - constructor( - public db: Database, - public did: string, - public timestamp?: string, - ) { - super() - } - - async getSavedBytes(cid: CID): Promise { - const found = await this.db.db - .selectFrom('ipld_block') - .where('cid', '=', cid.toString()) - .select('content') - .executeTakeFirst() - return found ? found.content : null - } - - async hasSavedBlock(cid: CID): Promise { - const found = await this.db.db - .selectFrom('ipld_block') - .where('cid', '=', cid.toString()) - .select('cid') - .executeTakeFirst() - return !!found - } - - async saveStaged(): Promise { - this.db.assertTransaction() - const blocks: IpldBlock[] = [] - const creators: IpldBlockCreator[] = [] - for (const staged of this.staged.entries()) { - const [cid, bytes] = staged - blocks.push({ - cid: cid.toString(), - size: bytes.length, - content: bytes, - indexedAt: this.timestamp || new Date().toISOString(), - }) - creators.push({ - cid: cid.toString(), - did: this.did, - }) - } - const insertBlocks = this.db.db - .insertInto('ipld_block') - .values(blocks) - .onConflict((oc) => oc.doNothing()) - .execute() - const insertCreators = this.db.db - .insertInto('ipld_block_creator') - .values(creators) - .onConflict((oc) => oc.doNothing()) - .execute() - await Promise.all([insertBlocks, insertCreators]) - this.clearStaged() - } - - async destroySaved(): Promise { - throw new Error('Destruction of SQL blockstore not allowed at runtime') - } -} - -export default SqlBlockstore diff --git a/packages/pds/src/sql-repo-storage.ts b/packages/pds/src/sql-repo-storage.ts new file mode 100644 index 00000000000..49411f2822c --- /dev/null +++ b/packages/pds/src/sql-repo-storage.ts @@ -0,0 +1,309 @@ +import { CommitData, RepoStorage, BlockMap, CidSet } from '@atproto/repo' +import { chunkArray } from '@atproto/common' +import { CID } from 'multiformats/cid' +import Database from './db' +import { IpldBlock } from './db/tables/ipld-block' +import { IpldBlockCreator } from './db/tables/ipld-block-creator' +import { RepoCommitBlock } from './db/tables/repo-commit-block' +import { RepoCommitHistory } from './db/tables/repo-commit-history' + +export class SqlRepoStorage extends RepoStorage { + cache: BlockMap = new BlockMap() + + constructor( + public db: Database, + public did: string, + public timestamp?: string, + ) { + super() + } + + async getHead(forUpdate?: boolean): Promise { + // if for update, we lock the row + let builder = this.db.db + .selectFrom('repo_root') + .selectAll() + .where('did', '=', this.did) + if (forUpdate && this.db.dialect !== 'sqlite') { + // SELECT FOR UPDATE is not supported by sqlite, but sqlite txs are SERIALIZABLE so we don't actually need it + builder = builder.forUpdate() + } + const found = await builder.executeTakeFirst() + if (!found) return null + + // if for update, we cache the blocks from last commit + // this must be split out into a separate query because of how pg handles SELECT FOR UPDATE with outer joins + if (forUpdate) { + const res = await this.db.db + .selectFrom('repo_commit_block') + .leftJoin('ipld_block', 'ipld_block.cid', 'repo_commit_block.block') + .where('repo_commit_block.commit', '=', found.root) + .where('repo_commit_block.creator', '=', this.did) + .select([ + 'ipld_block.cid as blockCid', + 'ipld_block.content as blockBytes', + ]) + .execute() + res.forEach((row) => { + if (row.blockCid && row.blockBytes) { + this.cache.set(CID.parse(row.blockCid), row.blockBytes) + } + }) + } + + return CID.parse(found.root) + } + + async getBytes(cid: CID): Promise { + const cached = this.cache.get(cid) + if (cached) return cached + const found = await this.db.db + .selectFrom('ipld_block') + .innerJoin( + 'ipld_block_creator as creator', + 'creator.cid', + 'ipld_block.cid', + ) + .where('creator.did', '=', this.did) + .where('ipld_block.cid', '=', cid.toString()) + .select('content') + .executeTakeFirst() + if (!found) return null + this.cache.set(cid, found.content) + return found.content + } + + async has(cid: CID): Promise { + const got = await this.getBytes(cid) + return !!got + } + + async getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> { + const cached = this.cache.getMany(cids) + if (cached.missing.length < 1) return cached + const missing = new CidSet(cached.missing) + const missingStr = cached.missing.map((c) => c.toString()) + const blocks = new BlockMap() + await Promise.all( + chunkArray(missingStr, 500).map(async (batch) => { + const res = await this.db.db + .selectFrom('ipld_block') + .innerJoin( + 'ipld_block_creator as creator', + 'creator.cid', + 'ipld_block.cid', + ) + .where('creator.did', '=', this.did) + .where('ipld_block.cid', 'in', batch) + .select(['ipld_block.cid as cid', 'ipld_block.content as content']) + .execute() + for (const row of res) { + const cid = CID.parse(row.cid) + blocks.set(cid, row.content) + missing.delete(cid) + } + }), + ) + this.cache.addMap(blocks) + blocks.addMap(cached.blocks) + return { blocks, missing: missing.toList() } + } + + async putBlock(cid: CID, block: Uint8Array): Promise { + this.db.assertTransaction() + const insertBlock = this.db.db + .insertInto('ipld_block') + .values({ + cid: cid.toString(), + size: block.length, + content: block, + indexedAt: this.timestamp || new Date().toISOString(), + }) + .onConflict((oc) => oc.doNothing()) + .execute() + const insertCreator = this.db.db + .insertInto('ipld_block_creator') + .values({ + cid: cid.toString(), + did: this.did, + }) + .onConflict((oc) => oc.doNothing()) + .execute() + await Promise.all([insertBlock, insertCreator]) + this.cache.set(cid, block) + } + + async putMany(toPut: BlockMap): Promise { + this.db.assertTransaction() + const blocks: IpldBlock[] = [] + const creators: IpldBlockCreator[] = [] + toPut.forEach((bytes, cid) => { + blocks.push({ + cid: cid.toString(), + size: bytes.length, + content: bytes, + indexedAt: this.timestamp || new Date().toISOString(), + }) + creators.push({ + cid: cid.toString(), + did: this.did, + }) + }) + const putBlocks = Promise.all( + chunkArray(blocks, 500).map((batch) => + this.db.db + .insertInto('ipld_block') + .values(batch) + .onConflict((oc) => oc.doNothing()) + .execute(), + ), + ) + const putCreators = Promise.all( + chunkArray(creators, 500).map((batch) => + this.db.db + .insertInto('ipld_block_creator') + .values(batch) + .onConflict((oc) => oc.doNothing()) + .execute(), + ), + ) + await Promise.all([putBlocks, putCreators]) + } + + async indexCommits(commits: CommitData[]): Promise { + this.db.assertTransaction() + const allBlocks = new BlockMap() + const commitBlocks: RepoCommitBlock[] = [] + const commitHistory: RepoCommitHistory[] = [] + for (const commit of commits) { + for (const block of commit.blocks.entries()) { + commitBlocks.push({ + commit: commit.commit.toString(), + block: block.cid.toString(), + creator: this.did, + }) + allBlocks.set(block.cid, block.bytes) + } + commitHistory.push({ + commit: commit.commit.toString(), + prev: commit.prev ? commit.prev.toString() : null, + creator: this.did, + }) + } + const insertCommitBlocks = Promise.all( + chunkArray(commitBlocks, 500).map((batch) => + this.db.db + .insertInto('repo_commit_block') + .values(batch) + .onConflict((oc) => oc.doNothing()) + .execute(), + ), + ) + const insertCommitHistory = Promise.all( + chunkArray(commitHistory, 500).map((batch) => + this.db.db + .insertInto('repo_commit_history') + .values(batch) + .onConflict((oc) => oc.doNothing()) + .execute(), + ), + ) + await Promise.all([ + this.putMany(allBlocks), + insertCommitBlocks, + insertCommitHistory, + ]) + } + + async updateHead(cid: CID, prev: CID | null): Promise { + if (prev === null) { + await this.db.db + .insertInto('repo_root') + .values({ + did: this.did, + root: cid.toString(), + indexedAt: this.getTimestamp(), + }) + .execute() + } else { + const res = await this.db.db + .updateTable('repo_root') + .set({ + root: cid.toString(), + indexedAt: this.getTimestamp(), + }) + .where('did', '=', this.did) + .where('root', '=', prev.toString()) + .executeTakeFirst() + if (res.numUpdatedRows < 1) { + throw new Error('failed to update repo root: misordered') + } + } + } + + private getTimestamp(): string { + return this.timestamp || new Date().toISOString() + } + + async getCommitPath( + latest: CID, + earliest: CID | null, + ): Promise { + const res = await this.db.db + .withRecursive('ancestor(commit, prev)', (cte) => + cte + .selectFrom('repo_commit_history as commit') + .select(['commit.commit as commit', 'commit.prev as prev']) + .where('commit', '=', latest.toString()) + .where('creator', '=', this.did) + .unionAll( + cte + .selectFrom('repo_commit_history as commit') + .select(['commit.commit as commit', 'commit.prev as prev']) + .innerJoin('ancestor', (join) => + join + .onRef('ancestor.prev', '=', 'commit.commit') + .on('commit.creator', '=', this.did), + ) + .if(earliest !== null, (qb) => + // @ts-ignore + qb.where('commit.commit', '!=', earliest?.toString() as string), + ), + ), + ) + .selectFrom('ancestor') + .select('commit') + .execute() + return res.map((row) => CID.parse(row.commit)).reverse() + } + async getBlocksForCommits( + commits: CID[], + ): Promise<{ [commit: string]: BlockMap }> { + if (commits.length === 0) return {} + const commitStrs = commits.map((commit) => commit.toString()) + const res = await this.db.db + .selectFrom('repo_commit_block') + .where('repo_commit_block.creator', '=', this.did) + .where('repo_commit_block.commit', 'in', commitStrs) + .innerJoin('ipld_block', 'ipld_block.cid', 'repo_commit_block.block') + .select([ + 'repo_commit_block.commit', + 'ipld_block.cid', + 'ipld_block.content', + ]) + .execute() + return res.reduce((acc, cur) => { + acc[cur.commit] ??= new BlockMap() + const cid = CID.parse(cur.cid) + acc[cur.commit].set(cid, cur.content) + this.cache.set(cid, cur.content) + return acc + }, {}) + } + + async destroy(): Promise { + throw new Error('Destruction of SQL repo storage not allowed at runtime') + } +} + +export default SqlRepoStorage diff --git a/packages/pds/tests/duplicate-records.test.ts b/packages/pds/tests/duplicate-records.test.ts index 928fc3de37c..d41b8a8c71b 100644 --- a/packages/pds/tests/duplicate-records.test.ts +++ b/packages/pds/tests/duplicate-records.test.ts @@ -2,7 +2,7 @@ import { AtUri } from '@atproto/uri' import { CloseFn, runTestServer } from './_util' import { Database } from '../src' import * as lex from '../src/lexicon/lexicons' -import { cidForData, TID, valueToIpldBytes } from '@atproto/common' +import { cidForCbor, TID, cborEncode } from '@atproto/common' import { CID } from 'multiformats/cid' import { APP_BSKY_GRAPH } from '../src/lexicon' import { Services } from '../src/services' @@ -37,8 +37,8 @@ describe('duplicate record', () => { } const putBlock = async (db: Database, data: object): Promise => { - const cid = await cidForData(data) - const bytes = await valueToIpldBytes(data) + const cid = await cidForCbor(data) + const bytes = await cborEncode(data) await db.db .insertInto('ipld_block') .values({ diff --git a/packages/pds/tests/event-stream/sync.test.ts b/packages/pds/tests/event-stream/sync.test.ts index 8683b81e1e1..4c85bcaa0bc 100644 --- a/packages/pds/tests/event-stream/sync.test.ts +++ b/packages/pds/tests/event-stream/sync.test.ts @@ -1,7 +1,7 @@ import { sql } from 'kysely' import AtpApi, { ServiceClient as AtpServiceClient } from '@atproto/api' -import { getWriteOpLog, RecordWriteOp } from '@atproto/repo' -import SqlBlockstore from '../../src/sql-blockstore' +import { getWriteLog, RecordWriteOp, WriteOpAction } from '@atproto/repo' +import SqlRepoStorage from '../../src/sql-repo-storage' import basicSeed from '../seeds/basic' import { SeedClient } from '../seeds/client' import { forSnapshot, runTestServer, TestServerInfo } from '../_util' @@ -78,11 +78,11 @@ describe('sync', () => { }) async function getOpLog(did: string) { - const { db, services } = ctx - const repoRoot = await services.repo(db).getRepoRoot(did) - if (!repoRoot) throw new Error('Missing repo root') - const blockstore = new SqlBlockstore(db, did) - return await getWriteOpLog(blockstore, null, repoRoot) + const { db } = ctx + const storage = new SqlRepoStorage(db, did) + const root = await storage.getHead() + if (!root) throw new Error('Missing repo root') + return await getWriteLog(storage, root, null) } function prepareWrites( @@ -92,21 +92,21 @@ describe('sync', () => { return Promise.all( ops.map((op) => { const { action } = op - if (action === 'create') { + if (action === WriteOpAction.Create) { return prepareCreate({ did, collection: op.collection, rkey: op.rkey, - record: op.value, + record: op.record, }) - } else if (action === 'update') { + } else if (action === WriteOpAction.Update) { return prepareUpdate({ did, collection: op.collection, rkey: op.rkey, - record: op.value, + record: op.record, }) - } else if (action === 'delete') { + } else if (action === WriteOpAction.Delete) { return prepareDelete({ did, collection: op.collection, diff --git a/packages/pds/tests/image/server.test.ts b/packages/pds/tests/image/server.test.ts index 93d57a17c55..c3c81a61473 100644 --- a/packages/pds/tests/image/server.test.ts +++ b/packages/pds/tests/image/server.test.ts @@ -7,7 +7,7 @@ import axios, { AxiosInstance } from 'axios' import { getInfo } from '../../src/image/sharp' import { BlobDiskCache, ImageProcessingServer } from '../../src/image/server' import { DiskBlobStore } from '../../src' -import { cidForData } from '@atproto/common' +import { cidForCbor } from '@atproto/common' import { CID } from 'multiformats/cid' describe('image processing server', () => { @@ -25,7 +25,7 @@ describe('image processing server', () => { path.join(os.tmpdir(), 'img-processing-tests'), ) // this CID isn't accurate for the data, but it works for the sake of the test - fileCid = await cidForData('key-landscape-small') + fileCid = await cidForCbor('key-landscape-small') await storage.putPermanent( fileCid, fs.createReadStream('tests/image/fixtures/key-landscape-small.jpg'), @@ -110,7 +110,7 @@ describe('image processing server', () => { }) it('errors on missing file.', async () => { - const missingCid = await cidForData('missing-file') + const missingCid = await cidForCbor('missing-file') const res = await client.get( server.uriBuilder.getSignedPath({ cid: missingCid, diff --git a/packages/pds/tests/image/uri.test.ts b/packages/pds/tests/image/uri.test.ts index dffdf336dc8..ddf5468fe51 100644 --- a/packages/pds/tests/image/uri.test.ts +++ b/packages/pds/tests/image/uri.test.ts @@ -1,4 +1,4 @@ -import { cidForData } from '@atproto/common' +import { cidForCbor } from '@atproto/common' import { CID } from 'multiformats/cid' import { ImageUriBuilder, BadPathError } from '../../src/image/uri' @@ -12,7 +12,7 @@ describe('image uri builder', () => { const key = 'f23ecd142835025f42c3db2cf25dd813956c178392760256211f9d315f8ab4d8' uriBuilder = new ImageUriBuilder(endpoint, salt, key) - cid = await cidForData('test cid') + cid = await cidForCbor('test cid') }) it('signs and verifies uri options.', () => { diff --git a/packages/pds/tests/migrations/duplicate-records.ts b/packages/pds/tests/migrations/duplicate-records.ts index 6a0e6caabbc..5f64073aa67 100644 --- a/packages/pds/tests/migrations/duplicate-records.ts +++ b/packages/pds/tests/migrations/duplicate-records.ts @@ -1,6 +1,7 @@ import { AtUri } from '@atproto/uri' +import * as crypto from '@atproto/crypto' +import { cidForCbor, TID } from '@atproto/common' import { Database, MemoryBlobStore } from '../../src' -import { cidForData, TID } from '@atproto/common' import * as lex from '../../src/lexicon/lexicons' import { APP_BSKY_GRAPH } from '../../src/lexicon' import SqlMessageQueue from '../../src/event-stream/message-queue' @@ -24,6 +25,7 @@ describe('duplicate record', () => { } messageQueue = new SqlMessageQueue('pds', db) services = createServices({ + keypair: await crypto.EcdsaKeypair.create(), messageQueue, blobstore: new MemoryBlobStore(), imgUriBuilder: new ImageUriBuilder('http://x', '00', '00'), @@ -40,7 +42,7 @@ describe('duplicate record', () => { const indexRecord = async (record: any, times: number): Promise => { const collection = record.$type - const cid = await cidForData(record) + const cid = await cidForCbor(record) await db.transaction(async (tx) => { const recordTx = services.record(tx) for (let i = 0; i < times; i++) { @@ -52,7 +54,7 @@ describe('duplicate record', () => { it('has duplicate records', async () => { const subjectUri = AtUri.make(did, lex.ids.AppBskyFeedPost, TID.nextStr()) - const subjectCid = await cidForData({ test: 'blah' }) + const subjectCid = await cidForCbor({ test: 'blah' }) const subjectDid = 'did:example:bob' const repost = { $type: lex.ids.AppBskyFeedRepost, diff --git a/packages/pds/tests/migrations/repo-sync-data.test.ts b/packages/pds/tests/migrations/repo-sync-data.test.ts new file mode 100644 index 00000000000..4fcb9e65de0 --- /dev/null +++ b/packages/pds/tests/migrations/repo-sync-data.test.ts @@ -0,0 +1,82 @@ +import AtpApi from '@atproto/api' +import { Database } from '../../src' +import { SeedClient } from '../seeds/client' +import { CloseFn, runTestServer } from '../_util' +import basicSeed from '../seeds/basic' +import { RepoCommitHistory } from '../../src/db/tables/repo-commit-history' +import { RepoCommitBlock } from '../../src/db/tables/repo-commit-block' + +describe('repo sync data migration', () => { + let close: CloseFn + let db: Database + + beforeAll(async () => { + const server = await runTestServer({ + dbPostgresSchema: 'migration_repo_sync', + }) + close = server.close + db = server.ctx.db + const client = AtpApi.service(server.url) + const sc = new SeedClient(client) + await basicSeed(sc, server.ctx.messageQueue) + }) + + afterAll(async () => { + await close() + }) + + const getHistory = async () => { + return await db.db + .selectFrom('repo_commit_history') + .selectAll() + .orderBy('commit') + .orderBy('prev') + .execute() + } + + const getBlocks = async () => { + return await db.db + .selectFrom('repo_commit_block') + .selectAll() + .orderBy('commit') + .orderBy('block') + .execute() + } + + let history: RepoCommitHistory[] + let blocks: RepoCommitBlock[] + + it('fetches the current state of the tables', async () => { + history = await getHistory() + blocks = await getBlocks() + }) + + it('migrates down', async () => { + const migration = await db.migrator.migrateTo('_20221230T215012029Z') + expect(migration.error).toBeUndefined() + // normal syntax for async exceptions as not catching the error for some reason + let err1 + try { + await getHistory() + } catch (e) { + err1 = e + } + expect(err1).toBeDefined() + let err2 + try { + await getHistory() + } catch (e) { + err2 = e + } + expect(err2).toBeDefined() + }) + + it('migrates up', async () => { + const migration = await db.migrator.migrateTo('_20230118T223059239Z') + expect(migration.error).toBeUndefined() + const migratedHistory = await getHistory() + const migratedBlocks = await getBlocks() + expect(migratedHistory).toEqual(history) + expect(migratedBlocks).toEqual(blocks) + }) +}) diff --git a/packages/pds/tests/sql-blockstore.test.ts b/packages/pds/tests/sql-blockstore.test.ts deleted file mode 100644 index 5987e0928bc..00000000000 --- a/packages/pds/tests/sql-blockstore.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { Database } from '../src' -import SqlBlockstore from '../src/sql-blockstore' -import { CloseFn, runTestServer } from './_util' - -describe('sql blockstore', () => { - let db: Database - let close: CloseFn - - beforeAll(async () => { - const server = await runTestServer({ - dbPostgresSchema: 'sql_blockstore', - }) - close = server.close - db = server.ctx.db - }) - - afterAll(async () => { - await close() - }) - - it('puts and gets blocks.', async () => { - const did = 'did:key:zQ3shokFTS3brHcDQrn82RUDfCZESWL1ZdCEJwekUDPQiYBme' - - const cid = await db.transaction(async (dbTxn) => { - const blockstore = new SqlBlockstore(dbTxn, did) - const cid = await blockstore.stage({ my: 'block' }) - await blockstore.saveStaged() - return cid - }) - - const blockstore = new SqlBlockstore(db, did) - const value = await blockstore.getUnchecked(cid) - - expect(value).toEqual({ my: 'block' }) - }) - - it('allows same content to be put multiple times by the same did.', async () => { - const did = 'did:key:zQ3shtxV1FrJfhqE1dvxYRcCknWNjHc3c5X1y3ZSoPDi2aur2' - - const cidA = await db.transaction(async (dbTxn) => { - const blockstore = new SqlBlockstore(dbTxn, did) - const cid = await blockstore.stage({ my: 'block' }) - await blockstore.saveStaged() - return cid - }) - - const cidB = await db.transaction(async (dbTxn) => { - const blockstore = new SqlBlockstore(dbTxn, did) - const cid = await blockstore.stage({ my: 'block' }) - await blockstore.saveStaged() - return cid - }) - - expect(cidA.equals(cidB)).toBe(true) - }) -}) diff --git a/packages/pds/tests/sql-repo-storage.test.ts b/packages/pds/tests/sql-repo-storage.test.ts new file mode 100644 index 00000000000..6f5069c615f --- /dev/null +++ b/packages/pds/tests/sql-repo-storage.test.ts @@ -0,0 +1,115 @@ +import { range, dataToCborBlock } from '@atproto/common' +import { def } from '@atproto/repo' +import BlockMap from '@atproto/repo/src/block-map' +import { Database } from '../src' +import SqlRepoStorage from '../src/sql-repo-storage' +import { CloseFn, runTestServer } from './_util' + +describe('sql repo storage', () => { + let db: Database + let close: CloseFn + + beforeAll(async () => { + const server = await runTestServer({ + dbPostgresSchema: 'sql_repo_storage', + }) + close = server.close + db = server.ctx.db + }) + + afterAll(async () => { + await close() + }) + + it('puts and gets blocks.', async () => { + const did = 'did:key:zQ3shokFTS3brHcDQrn82RUDfCZESWL1ZdCEJwekUDPQiYBme' + + const cid = await db.transaction(async (dbTxn) => { + const storage = new SqlRepoStorage(dbTxn, did) + const block = await dataToCborBlock({ my: 'block' }) + await storage.putBlock(block.cid, block.bytes) + return block.cid + }) + + const storage = new SqlRepoStorage(db, did) + const value = await storage.readObj(cid, def.unknown) + + expect(value).toEqual({ my: 'block' }) + }) + + it('allows same content to be put multiple times by the same did.', async () => { + const did = 'did:key:zQ3shtxV1FrJfhqE1dvxYRcCknWNjHc3c5X1y3ZSoPDi2aur2' + + const cidA = await db.transaction(async (dbTxn) => { + const storage = new SqlRepoStorage(dbTxn, did) + const block = await dataToCborBlock({ my: 'block' }) + await storage.putBlock(block.cid, block.bytes) + return block.cid + }) + + const cidB = await db.transaction(async (dbTxn) => { + const storage = new SqlRepoStorage(dbTxn, did) + const block = await dataToCborBlock({ my: 'block' }) + await storage.putBlock(block.cid, block.bytes) + return block.cid + }) + + expect(cidA.equals(cidB)).toBe(true) + }) + + it('applies commits', async () => { + const did = 'did:key:zQ3shtxV1FrJfhqE1dvxYRcCknWNjHc3c5X1y3ZSoPDi2aur3' + const blocks = await Promise.all( + range(10).map((num) => dataToCborBlock({ my: `block-${num}` })), + ) + const commits = await Promise.all( + range(2).map((num) => dataToCborBlock({ my: `commit-${num}` })), + ) + const blocks0 = new BlockMap() + blocks0.set(commits[0].cid, commits[0].bytes) + blocks.slice(0, 5).forEach((block) => { + blocks0.set(block.cid, block.bytes) + }) + const blocks1 = new BlockMap() + blocks1.set(commits[1].cid, commits[1].bytes) + blocks.slice(5, 10).forEach((block) => { + blocks1.set(block.cid, block.bytes) + }) + await db.transaction(async (dbTxn) => { + const storage = new SqlRepoStorage(dbTxn, did) + await storage.applyCommit({ + commit: commits[0].cid, + prev: null, + blocks: blocks0, + }) + await storage.applyCommit({ + commit: commits[1].cid, + prev: commits[0].cid, + blocks: blocks1, + }) + }) + + const storage = new SqlRepoStorage(db, did) + const head = await storage.getHead() + if (!head) { + throw new Error('could not get repo head') + } + expect(head.toString()).toEqual(commits[1].cid.toString()) + const commitPath = await storage.getCommitPath(head, null) + if (!commitPath) { + throw new Error('could not get commit path') + } + expect(commitPath.length).toBe(2) + expect(commitPath[0].equals(commits[0].cid)).toBeTruthy() + expect(commitPath[1].equals(commits[1].cid)).toBeTruthy() + const commitData = await storage.getCommits(head, null) + if (!commitData) { + throw new Error('could not get commit data') + } + expect(commitData.length).toBe(2) + expect(commitData[0].commit.equals(commits[0].cid)).toBeTruthy() + expect(commitData[0].blocks.equals(blocks0)).toBeTruthy() + expect(commitData[1].commit.equals(commits[1].cid)).toBeTruthy() + expect(commitData[1].blocks.equals(blocks1)).toBeTruthy() + }) +}) diff --git a/packages/pds/tests/sync.test.ts b/packages/pds/tests/sync.test.ts new file mode 100644 index 00000000000..e0d9c4e4dc4 --- /dev/null +++ b/packages/pds/tests/sync.test.ts @@ -0,0 +1,216 @@ +import AtpApi, { ServiceClient as AtpServiceClient } from '@atproto/api' +import { TID } from '@atproto/common' +import { randomStr } from '@atproto/crypto' +import { DidResolver } from '@atproto/did-resolver' +import * as repo from '@atproto/repo' +import { collapseWriteLog, MemoryBlockstore } from '@atproto/repo' +import { AtUri } from '@atproto/uri' +import { CID } from 'multiformats/cid' +import { CloseFn, runTestServer } from './_util' + +describe('repo sync', () => { + let client: AtpServiceClient + let did: string + + const repoData: repo.RepoContents = {} + const uris: AtUri[] = [] + const storage = new MemoryBlockstore() + let didResolver: DidResolver + let currRoot: CID | undefined + + let close: CloseFn + + beforeAll(async () => { + const server = await runTestServer({ + dbPostgresSchema: 'repo_sync', + }) + close = server.close + client = AtpApi.service(server.url) + const res = await client.com.atproto.account.create({ + email: 'alice@test.com', + handle: 'alice.test', + password: 'alice-pass', + }) + client.setHeader('authorization', `Bearer ${res.data.accessJwt}`) + did = res.data.did + didResolver = new DidResolver({ plcUrl: server.ctx.cfg.didPlcUrl }) + repoData['app.bsky.system.declaration'] = { + self: { + $type: 'app.bsky.system.declaration', + actorType: 'app.bsky.system.actorUser', + }, + } + }) + + afterAll(async () => { + await close() + }) + + it('creates and syncs some records', async () => { + const ADD_COUNT = 10 + for (let i = 0; i < ADD_COUNT; i++) { + const { obj, uri } = await makePost(client, did) + if (!repoData[uri.collection]) { + repoData[uri.collection] = {} + } + repoData[uri.collection][uri.rkey] = obj + uris.push(uri) + } + + const car = await client.com.atproto.sync.getRepo({ did }) + const synced = await repo.loadFullRepo( + storage, + new Uint8Array(car.data), + didResolver, + ) + expect(synced.writeLog.length).toBe(ADD_COUNT + 1) // +1 because of declaration + const ops = await collapseWriteLog(synced.writeLog) + expect(ops.length).toBe(ADD_COUNT + 1) + const loaded = await repo.Repo.load(storage, synced.root) + const contents = await loaded.getContents() + expect(contents).toEqual(repoData) + + currRoot = synced.root + }) + + it('syncs creates and deletes', async () => { + const ADD_COUNT = 10 + const DEL_COUNT = 4 + for (let i = 0; i < ADD_COUNT; i++) { + const { obj, uri } = await makePost(client, did) + if (!repoData[uri.collection]) { + repoData[uri.collection] = {} + } + repoData[uri.collection][uri.rkey] = obj + uris.push(uri) + } + // delete two that are already sync & two that have not been + for (let i = 0; i < DEL_COUNT; i++) { + const uri = uris[i * 5] + await client.app.bsky.feed.post.delete({ + did, + colleciton: uri.collection, + rkey: uri.rkey, + }) + delete repoData[uri.collection][uri.rkey] + } + + const car = await client.com.atproto.sync.getRepo({ + did, + from: currRoot?.toString(), + }) + const currRepo = await repo.Repo.load(storage, currRoot) + const synced = await repo.loadDiff( + currRepo, + new Uint8Array(car.data), + didResolver, + ) + expect(synced.writeLog.length).toBe(ADD_COUNT + DEL_COUNT) + const ops = await collapseWriteLog(synced.writeLog) + expect(ops.length).toBe(ADD_COUNT) // -2 because of dels of new records, +2 because of dels of old records + const loaded = await repo.Repo.load(storage, synced.root) + const contents = await loaded.getContents() + expect(contents).toEqual(repoData) + + currRoot = synced.root + }) + + it('syncs current root', async () => { + const root = await client.com.atproto.sync.getHead({ did }) + expect(root.data.root).toEqual(currRoot?.toString()) + }) + + it('syncs commit path', async () => { + const local = await storage.getCommitPath(currRoot as CID, null) + if (!local) { + throw new Error('Could not get local commit path') + } + const localStr = local.map((c) => c.toString()) + const commitPath = await client.com.atproto.sync.getCommitPath({ did }) + expect(commitPath.data.commits).toEqual(localStr) + + const partialCommitPath = await client.com.atproto.sync.getCommitPath({ + did, + earliest: localStr[2], + latest: localStr[15], + }) + expect(partialCommitPath.data.commits).toEqual(localStr.slice(3, 16)) + }) + + it('sync a repo checkout', async () => { + const car = await client.com.atproto.sync.getCheckout({ did }) + const checkoutStorage = new MemoryBlockstore() + const loaded = await repo.loadCheckout( + checkoutStorage, + new Uint8Array(car.data), + didResolver, + ) + expect(loaded.contents).toEqual(repoData) + const loadedRepo = await repo.Repo.load(checkoutStorage, loaded.root) + expect(await loadedRepo.getContents()).toEqual(repoData) + }) + + it('sync a record proof', async () => { + const collection = Object.keys(repoData)[0] + const rkey = Object.keys(repoData[collection])[0] + const car = await client.com.atproto.sync.getRecord({ + did, + collection, + rkey, + }) + const records = await repo.verifyRecords( + did, + new Uint8Array(car.data), + didResolver, + ) + const claim = { + collection, + rkey, + record: repoData[collection][rkey], + } + expect(records.length).toBe(1) + expect(records[0].record).toEqual(claim.record) + const result = await repo.verifyProofs( + did, + new Uint8Array(car.data), + [claim], + didResolver, + ) + expect(result.verified.length).toBe(1) + expect(result.unverified.length).toBe(0) + }) + + it('sync a proof of non-existence', async () => { + const collection = Object.keys(repoData)[0] + const rkey = TID.nextStr() // rkey that doesn't exist + const car = await client.com.atproto.sync.getRecord({ + did, + collection, + rkey, + }) + const claim = { + collection, + rkey, + record: null, + } + const result = await repo.verifyProofs( + did, + new Uint8Array(car.data), + [claim], + didResolver, + ) + expect(result.verified.length).toBe(1) + expect(result.unverified.length).toBe(0) + }) +}) + +const makePost = async (client: AtpServiceClient, did: string) => { + const obj = { + $type: 'app.bsky.feed.post', + text: randomStr(32, 'base32'), + createdAt: new Date().toISOString(), + } + const res = await client.app.bsky.feed.post.create({ did }, obj) + const uri = new AtUri(res.uri) + return { obj, uri } +} diff --git a/packages/pds/tests/views/votes.test.ts b/packages/pds/tests/views/votes.test.ts index 7a2d21328ab..58089f11881 100644 --- a/packages/pds/tests/views/votes.test.ts +++ b/packages/pds/tests/views/votes.test.ts @@ -161,7 +161,6 @@ describe('pds vote views', () => { } post = await getPost() - expect( (post.thread.post as AppBskyFeedGetPostThread.ThreadViewPost) .downvoteCount, diff --git a/packages/pds/tsconfig.json b/packages/pds/tsconfig.json index 9f2a85c4eb6..cf46760ee13 100644 --- a/packages/pds/tsconfig.json +++ b/packages/pds/tsconfig.json @@ -9,7 +9,6 @@ "include": ["./src","__tests__/**/**.ts"], "references": [ { "path": "../api/tsconfig.build.json" }, - { "path": "../auth/tsconfig.build.json" }, { "path": "../common/tsconfig.build.json" }, { "path": "../crypto/tsconfig.build.json" }, { "path": "../did-resolver/tsconfig.build.json" }, diff --git a/packages/plc/src/client/index.ts b/packages/plc/src/client/index.ts index eaa798edf92..97f48027881 100644 --- a/packages/plc/src/client/index.ts +++ b/packages/plc/src/client/index.ts @@ -1,7 +1,7 @@ import axios from 'axios' import { CID } from 'multiformats/cid' -import { DidableKey } from '@atproto/crypto' -import { check, cidForData } from '@atproto/common' +import { Keypair } from '@atproto/crypto' +import { check, cidForCbor } from '@atproto/common' import * as operations from '../lib/operations' import * as t from '../lib/types' @@ -28,7 +28,7 @@ export class PlcClient { } async createDid( - signingKey: DidableKey, + signingKey: Keypair, recoveryKey: string, handle: string, service: string, @@ -47,13 +47,13 @@ export class PlcClient { if (log.length === 0) { throw new Error(`Could not make update: DID does not exist: ${did}`) } - return cidForData(log[log.length - 1]) + return cidForCbor(log[log.length - 1]) } async rotateSigningKey( did: string, newKey: string, - signingKey: DidableKey, + signingKey: Keypair, prev?: CID, ) { prev = prev ? prev : await this.getPrev(did) @@ -68,7 +68,7 @@ export class PlcClient { async rotateRecoveryKey( did: string, newKey: string, - signingKey: DidableKey, + signingKey: Keypair, prev?: CID, ) { prev = prev ? prev : await this.getPrev(did) @@ -80,7 +80,7 @@ export class PlcClient { await axios.post(this.postOpUrl(did), op) } - async updateHandle(did: string, handle: string, signingKey: DidableKey) { + async updateHandle(did: string, handle: string, signingKey: Keypair) { const prev = await this.getPrev(did) const op = await operations.updateHandle( handle, @@ -90,7 +90,7 @@ export class PlcClient { await axios.post(this.postOpUrl(did), op) } - async updateAtpPds(did: string, service: string, signingKey: DidableKey) { + async updateAtpPds(did: string, service: string, signingKey: Keypair) { const prev = await this.getPrev(did) const op = await operations.updateAtpPds( service, diff --git a/packages/plc/src/lib/document.ts b/packages/plc/src/lib/document.ts index 5f49f6a28ca..347f0c5bd41 100644 --- a/packages/plc/src/lib/document.ts +++ b/packages/plc/src/lib/document.ts @@ -1,7 +1,7 @@ import { CID } from 'multiformats/cid' import * as uint8arrays from 'uint8arrays' import * as cbor from '@ipld/dag-cbor' -import { check, cidForData } from '@atproto/common' +import { check, cidForCbor } from '@atproto/common' import * as crypto from '@atproto/crypto' import * as t from './types' import { ServerError } from '../server/error' @@ -98,7 +98,7 @@ export const validateOperationLog = async ( handle: first.handle, atpPds: first.service, } - let prev = await cidForData(first) + let prev = await cidForCbor(first) for (const op of rest) { if (!op.prev || !CID.parse(op.prev).equals(prev)) { @@ -119,7 +119,7 @@ export const validateOperationLog = async ( } else { throw new ServerError(400, `Unknown operation: ${JSON.stringify(op)}`) } - prev = await cidForData(op) + prev = await cidForCbor(op) } return doc @@ -152,7 +152,7 @@ export const assureValidSig = async ( const dataBytes = new Uint8Array(cbor.encode(opData)) let isValid = true for (const did of allowedDids) { - isValid = await crypto.verifyDidSig(did, dataBytes, sigBytes) + isValid = await crypto.verifySignature(did, dataBytes, sigBytes) if (isValid) return } throw new ServerError(400, `Invalid signature on op: ${JSON.stringify(op)}`) diff --git a/packages/plc/src/lib/operations.ts b/packages/plc/src/lib/operations.ts index d3de446c70b..62317ec6d23 100644 --- a/packages/plc/src/lib/operations.ts +++ b/packages/plc/src/lib/operations.ts @@ -1,6 +1,6 @@ import * as cbor from '@ipld/dag-cbor' import * as uint8arrays from 'uint8arrays' -import { DidableKey, sha256 } from '@atproto/crypto' +import { Keypair, sha256 } from '@atproto/crypto' import * as t from './types' export const didForCreateOp = async (op: t.CreateOp, truncate = 24) => { @@ -12,7 +12,7 @@ export const didForCreateOp = async (op: t.CreateOp, truncate = 24) => { export const signOperation = async ( op: t.UnsignedOperation, - signingKey: DidableKey, + signingKey: Keypair, ): Promise => { const data = new Uint8Array(cbor.encode(op)) const sig = await signingKey.sign(data) @@ -23,7 +23,7 @@ export const signOperation = async ( } export const create = async ( - signingKey: DidableKey, + signingKey: Keypair, recoveryKey: string, handle: string, service: string, @@ -43,7 +43,7 @@ export const create = async ( export const rotateSigningKey = async ( newKey: string, prev: string, - signingKey: DidableKey, + signingKey: Keypair, ): Promise => { const op: t.UnsignedRotateSigningKeyOp = { type: 'rotate_signing_key', @@ -56,7 +56,7 @@ export const rotateSigningKey = async ( export const rotateRecoveryKey = async ( newKey: string, prev: string, - signingKey: DidableKey, + signingKey: Keypair, ): Promise => { const op: t.UnsignedRotateRecoveryKeyOp = { type: 'rotate_recovery_key', @@ -69,7 +69,7 @@ export const rotateRecoveryKey = async ( export const updateHandle = async ( handle: string, prev: string, - signingKey: DidableKey, + signingKey: Keypair, ): Promise => { const op: t.UnsignedUpdateHandleOp = { type: 'update_handle', @@ -82,7 +82,7 @@ export const updateHandle = async ( export const updateAtpPds = async ( service: string, prev: string, - signingKey: DidableKey, + signingKey: Keypair, ): Promise => { const op: t.UnsignedUpdateAtpPdsOp = { type: 'update_atp_pds', diff --git a/packages/plc/src/server/db.ts b/packages/plc/src/server/db.ts index 77109fef481..66bf50d7351 100644 --- a/packages/plc/src/server/db.ts +++ b/packages/plc/src/server/db.ts @@ -2,7 +2,7 @@ import { Kysely, Migrator, PostgresDialect, SqliteDialect } from 'kysely' import SqliteDB from 'better-sqlite3' import { Pool as PgPool, types as pgTypes } from 'pg' import { CID } from 'multiformats/cid' -import { cidForData } from '@atproto/common' +import { cidForCbor } from '@atproto/common' import * as document from '../lib/document' import * as t from '../lib/types' import { ServerError } from './error' @@ -92,7 +92,7 @@ export class Database { ops, proposed, ) - const cid = await cidForData(proposed) + const cid = await cidForCbor(proposed) await this.db .transaction() diff --git a/packages/plc/tests/document.test.ts b/packages/plc/tests/document.test.ts index c26b152e0ae..d57174699b2 100644 --- a/packages/plc/tests/document.test.ts +++ b/packages/plc/tests/document.test.ts @@ -1,4 +1,4 @@ -import { check, cidForData } from '@atproto/common' +import { check, cidForCbor } from '@atproto/common' import { EcdsaKeypair, parseDidKey, Secp256k1Keypair } from '@atproto/crypto' import * as uint8arrays from 'uint8arrays' import * as document from '../src/lib/document' @@ -47,7 +47,7 @@ describe('plc DID document', () => { it('allows for updating handle', async () => { handle = 'ali.example2.com' - const prev = await cidForData(ops[ops.length - 1]) + const prev = await cidForCbor(ops[ops.length - 1]) const op = await operations.updateHandle( handle, prev.toString(), @@ -65,7 +65,7 @@ describe('plc DID document', () => { it('allows for updating atpPds', async () => { atpPds = 'https://example2.com' - const prev = await cidForData(ops[ops.length - 1]) + const prev = await cidForCbor(ops[ops.length - 1]) const op = await operations.updateAtpPds( atpPds, prev.toString(), @@ -83,7 +83,7 @@ describe('plc DID document', () => { it('allows for rotating signingKey', async () => { const newSigningKey = await EcdsaKeypair.create() - const prev = await cidForData(ops[ops.length - 1]) + const prev = await cidForCbor(ops[ops.length - 1]) const op = await operations.rotateSigningKey( newSigningKey.did(), prev.toString(), @@ -102,7 +102,7 @@ describe('plc DID document', () => { }) it('no longer allows operations from old signing key', async () => { - const prev = await cidForData(ops[ops.length - 1]) + const prev = await cidForCbor(ops[ops.length - 1]) const op = await operations.updateHandle( 'bob', prev.toString(), @@ -113,7 +113,7 @@ describe('plc DID document', () => { it('allows for rotating recoveryKey', async () => { const newRecoveryKey = await Secp256k1Keypair.create() - const prev = await cidForData(ops[ops.length - 1]) + const prev = await cidForCbor(ops[ops.length - 1]) const op = await operations.rotateRecoveryKey( newRecoveryKey.did(), prev.toString(), @@ -132,7 +132,7 @@ describe('plc DID document', () => { }) it('no longer allows operations from old recovery key', async () => { - const prev = await cidForData(ops[ops.length - 1]) + const prev = await cidForCbor(ops[ops.length - 1]) const op = await operations.updateHandle( 'bob', prev.toString(), @@ -143,7 +143,7 @@ describe('plc DID document', () => { it('it allows recovery key to rotate signing key', async () => { const newKey = await EcdsaKeypair.create() - const prev = await cidForData(ops[ops.length - 1]) + const prev = await cidForCbor(ops[ops.length - 1]) const op = await operations.rotateSigningKey( newKey.did(), prev.toString(), @@ -157,7 +157,7 @@ describe('plc DID document', () => { it('it allows recovery key to rotate recovery key', async () => { const newKey = await Secp256k1Keypair.create() - const prev = await cidForData(ops[ops.length - 1]) + const prev = await cidForCbor(ops[ops.length - 1]) const op = await operations.rotateRecoveryKey( newKey.did(), prev.toString(), @@ -171,7 +171,7 @@ describe('plc DID document', () => { it('it allows recovery key to update handle', async () => { handle = 'ally.example3.com' - const prev = await cidForData(ops[ops.length - 1]) + const prev = await cidForCbor(ops[ops.length - 1]) const op = await operations.updateHandle( handle, prev.toString(), @@ -184,7 +184,7 @@ describe('plc DID document', () => { it('it allows recovery key to update atpPds', async () => { atpPds = 'https://example3.com' - const prev = await cidForData(ops[ops.length - 1]) + const prev = await cidForCbor(ops[ops.length - 1]) const op = await operations.updateAtpPds( atpPds, prev.toString(), @@ -196,7 +196,7 @@ describe('plc DID document', () => { }) it('requires operations to be in order', async () => { - const prev = await cidForData(ops[ops.length - 2]) + const prev = await cidForCbor(ops[ops.length - 2]) const op = await operations.updateAtpPds( 'foobar.com', prev.toString(), @@ -266,14 +266,14 @@ describe('plc DID document', () => { it('formats a valid DID document regardless of leading https://', async () => { handle = 'https://alice.example.com' - const prev = await cidForData(ops[ops.length - 1]) + const prev = await cidForCbor(ops[ops.length - 1]) const op1 = await operations.updateHandle( handle, prev.toString(), signingKey, ) atpPds = 'example.com' - const prev2 = await cidForData(op1) + const prev2 = await cidForCbor(op1) const op2 = await operations.updateAtpPds( atpPds, prev2.toString(), diff --git a/packages/plc/tests/server.test.ts b/packages/plc/tests/server.test.ts index bb43551c910..2190c5f8295 100644 --- a/packages/plc/tests/server.test.ts +++ b/packages/plc/tests/server.test.ts @@ -2,7 +2,7 @@ import { EcdsaKeypair } from '@atproto/crypto' import PlcClient from '../src/client' import * as document from '../src/lib/document' import { CloseFn, runTestServer } from './_util' -import { cidForData } from '@atproto/common' +import { cidForCbor } from '@atproto/common' import { AxiosError } from 'axios' import { Database } from '../src' @@ -104,7 +104,7 @@ describe('PLC server', () => { const newKey = await EcdsaKeypair.create() const ops = await client.getOperationLog(did) const forkPoint = ops[ops.length - 3] - const forkCid = await cidForData(forkPoint) + const forkCid = await cidForCbor(forkPoint) await client.rotateSigningKey(did, newKey.did(), recoveryKey, forkCid) signingKey = newKey diff --git a/packages/repo/bench/mst.bench.ts b/packages/repo/bench/mst.bench.ts index d250b6c1e7b..0cc2fe8e05b 100644 --- a/packages/repo/bench/mst.bench.ts +++ b/packages/repo/bench/mst.bench.ts @@ -31,7 +31,7 @@ describe('MST Benchmarks', () => { // const fanouts: Fanout[] = [8, 16, 32] const fanouts: Fanout[] = [16, 32] it('benchmarks various fanouts', async () => { - let benches: BenchmarkData[] = [] + const benches: BenchmarkData[] = [] for (const fanout of fanouts) { const blockstore = new MemoryBlockstore() let mst = await MST.create(blockstore, [], { fanout }) @@ -44,11 +44,11 @@ describe('MST Benchmarks', () => { const doneAdding = Date.now() - const root = await mst.save() + const root = await util.saveMst(blockstore, mst) const doneSaving = Date.now() - let reloaded = await MST.load(blockstore, root, { fanout }) + const reloaded = await MST.load(blockstore, root, { fanout }) const widthTracker = new NodeWidths() for await (const entry of reloaded.walk()) { await widthTracker.trackEntry(entry) @@ -64,6 +64,9 @@ describe('MST Benchmarks', () => { for (const entry of path) { if (entry.isTree()) { const bytes = await blockstore.getBytes(entry.pointer) + if (!bytes) { + throw new Error(`Bytes not found: ${entry.pointer}`) + } proofSize += bytes.byteLength } } diff --git a/packages/repo/bench/repo.bench.ts b/packages/repo/bench/repo.bench.ts index 986bd2ba120..a149337b95d 100644 --- a/packages/repo/bench/repo.bench.ts +++ b/packages/repo/bench/repo.bench.ts @@ -1,37 +1,46 @@ -import * as auth from '@atproto/auth' -import { MemoryBlockstore, Repo } from '../src' +import { TID } from '@atproto/common' +import * as crypto from '@atproto/crypto' +import { Secp256k1Keypair } from '@atproto/crypto' +import { MemoryBlockstore, Repo, WriteOpAction } from '../src' import * as util from '../tests/_util' describe('Repo Benchmarks', () => { - const verifier = new auth.Verifier() const size = 10000 let blockstore: MemoryBlockstore - let authStore: auth.AuthStore + let keypair: crypto.Keypair let repo: Repo beforeAll(async () => { blockstore = new MemoryBlockstore() - authStore = await verifier.createTempAuthStore() - await authStore.claimFull() - repo = await Repo.create(blockstore, await authStore.did(), authStore) + keypair = await Secp256k1Keypair.create() + repo = await Repo.create(blockstore, await keypair.did(), keypair) }) it('calculates size', async () => { - const posts = repo.getCollection('app.bsky.post') for (let i = 0; i < size; i++) { if (i % 500 === 0) { console.log(i) } - await posts.createRecord({ - $type: 'app.bsky.post', - text: util.randomStr(150), - reply: { - root: 'at://did:plc:1234abdefeoi23/app.bsky.post/12345678912345', - parent: 'at://did:plc:1234abdefeoi23/app.bsky.post/12345678912345', + + await repo.applyCommit( + { + action: WriteOpAction.Create, + collection: 'app.bsky.post', + rkey: TID.nextStr(), + value: { + $type: 'app.bsky.post', + text: util.randomStr(150), + reply: { + root: 'at://did:plc:1234abdefeoi23/app.bsky.post/12345678912345', + parent: + 'at://did:plc:1234abdefeoi23/app.bsky.post/12345678912345', + }, + createdAt: new Date().toISOString(), + }, }, - createdAt: new Date().toISOString(), - }) + keypair, + ) } console.log('SIZE: ', await blockstore.sizeInBytes()) diff --git a/packages/repo/package.json b/packages/repo/package.json index fb0b2f464cc..dda95a98e2b 100644 --- a/packages/repo/package.json +++ b/packages/repo/package.json @@ -22,8 +22,9 @@ "postpublish": "npm run update-main-to-src" }, "dependencies": { - "@atproto/auth": "*", "@atproto/common": "*", + "@atproto/crypto": "*", + "@atproto/did-resolver": "*", "@atproto/nsid": "*", "@ipld/car": "^3.2.3", "@ipld/dag-cbor": "^7.0.0", diff --git a/packages/repo/src/block-map.ts b/packages/repo/src/block-map.ts new file mode 100644 index 00000000000..3fc273e002a --- /dev/null +++ b/packages/repo/src/block-map.ts @@ -0,0 +1,86 @@ +import { dataToCborBlock } from '@atproto/common' +import { CID } from 'multiformats/cid' +import * as uint8arrays from 'uint8arrays' + +export class BlockMap { + private map: Map = new Map() + + async add(value: unknown): Promise { + const block = await dataToCborBlock(value) + this.set(block.cid, block.bytes) + return block.cid + } + + set(cid: CID, bytes: Uint8Array) { + this.map.set(cid.toString(), bytes) + } + + get(cid: CID): Uint8Array | undefined { + return this.map.get(cid.toString()) + } + + getMany(cids: CID[]): { blocks: BlockMap; missing: CID[] } { + const missing: CID[] = [] + const blocks = new BlockMap() + for (const cid of cids) { + const got = this.map.get(cid.toString()) + if (got) { + blocks.set(cid, got) + } else { + missing.push(cid) + } + } + return { blocks, missing } + } + + has(cid: CID): boolean { + return this.map.has(cid.toString()) + } + + clear(): void { + this.map.clear() + } + + forEach(cb: (bytes: Uint8Array, cid: CID) => void): void { + this.map.forEach((val, key) => cb(val, CID.parse(key))) + } + + entries(): Entry[] { + const entries: Entry[] = [] + this.forEach((bytes, cid) => { + entries.push({ cid, bytes }) + }) + return entries + } + + addMap(toAdd: BlockMap) { + toAdd.forEach((bytes, cid) => { + this.set(cid, bytes) + }) + } + + get size(): number { + return this.map.size + } + + equals(other: BlockMap): boolean { + if (this.size !== other.size) { + return false + } + for (const entry of this.entries()) { + const otherBytes = other.get(entry.cid) + if (!otherBytes) return false + if (!uint8arrays.equals(entry.bytes, otherBytes)) { + return false + } + } + return true + } +} + +type Entry = { + cid: CID + bytes: Uint8Array +} + +export default BlockMap diff --git a/packages/repo/src/blockstore/index.ts b/packages/repo/src/blockstore/index.ts deleted file mode 100644 index c8aab731483..00000000000 --- a/packages/repo/src/blockstore/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './ipld-store' -export * from './memory-blockstore' diff --git a/packages/repo/src/blockstore/ipld-store.ts b/packages/repo/src/blockstore/ipld-store.ts deleted file mode 100644 index de5eaff0eb3..00000000000 --- a/packages/repo/src/blockstore/ipld-store.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { CID } from 'multiformats/cid' -import { BlockWriter } from '@ipld/car/writer' - -import * as common from '@atproto/common' -import { check, util, valueToIpldBlock } from '@atproto/common' -import { BlockReader } from '@ipld/car/api' -import CidSet from '../cid-set' -import { CarReader } from '@ipld/car/reader' - -export abstract class IpldStore { - staged: Map - - constructor() { - this.staged = new Map() - } - - abstract getSavedBytes(cid: CID): Promise - abstract hasSavedBlock(cid: CID): Promise - abstract saveStaged(): Promise - abstract destroySaved(): Promise - - async stageBytes(k: CID, v: Uint8Array): Promise { - this.staged.set(k.toString(), v) - } - - async stage(value: unknown): Promise { - const block = await valueToIpldBlock(value) - await this.stageBytes(block.cid, block.bytes) - return block.cid - } - - async getBytes(cid: CID): Promise { - const fromStaged = this.staged.get(cid.toString()) - if (fromStaged) return fromStaged - const fromBlocks = await this.getSavedBytes(cid) - if (fromBlocks) return fromBlocks - throw new Error(`Not found: ${cid.toString()}`) - } - - async get(cid: CID, schema: check.Def): Promise { - const value = await this.getUnchecked(cid) - try { - return check.assure(schema, value) - } catch (err) { - throw new Error( - `Did not find expected object at ${cid.toString()}: ${err}`, - ) - } - } - - async getUnchecked(cid: CID): Promise { - const bytes = await this.getBytes(cid) - return common.ipldBytesToValue(bytes) - } - - async has(cid: CID): Promise { - return this.staged.has(cid.toString()) || this.hasSavedBlock(cid) - } - - async isMissing(cid: CID): Promise { - const has = await this.has(cid) - return !has - } - - async checkMissing(cids: CidSet): Promise { - const missing = await util.asyncFilter(cids.toList(), (c) => { - return this.isMissing(c) - }) - return new CidSet(missing) - } - - async clearStaged(): Promise { - this.staged.clear() - } - - async destroy(): Promise { - this.clearStaged() - await this.destroySaved() - } - - async addToCar(car: BlockWriter, cid: CID) { - car.put({ cid, bytes: await this.getBytes(cid) }) - } - - async stageCar(buf: Uint8Array): Promise { - const car = await CarReader.fromBytes(buf) - const roots = await car.getRoots() - if (roots.length !== 1) { - throw new Error(`Expected one root, got ${roots.length}`) - } - const rootCid = roots[0] - await this.stageCarBlocks(car) - return rootCid - } - - async stageCarBlocks(car: BlockReader): Promise { - for await (const block of car.blocks()) { - await this.stageBytes(block.cid, block.bytes) - } - } -} - -export default IpldStore diff --git a/packages/repo/src/blockstore/memory-blockstore.ts b/packages/repo/src/blockstore/memory-blockstore.ts deleted file mode 100644 index e9f49c9170b..00000000000 --- a/packages/repo/src/blockstore/memory-blockstore.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { CID } from 'multiformats/cid' -import IpldStore from './ipld-store' - -export class MemoryBlockstore extends IpldStore { - blocks: Map - - constructor() { - super() - this.blocks = new Map() - } - - async getSavedBytes(cid: CID): Promise { - return this.blocks.get(cid.toString()) || null - } - - async hasSavedBlock(cid: CID): Promise { - return this.blocks.has(cid.toString()) - } - - async saveStaged(): Promise { - this.staged.forEach((val, key) => { - this.blocks.set(key, val) - }) - this.clearStaged() - } - - async sizeInBytes(): Promise { - let total = 0 - for (const val of this.blocks.values()) { - total += val.byteLength - } - return total - } - - async destroySaved(): Promise { - this.blocks.clear() - } - - // Mainly for dev purposes - async getContents(): Promise> { - const contents: Record = {} - for (const key of this.blocks.keys()) { - contents[key] = await this.getUnchecked(CID.parse(key)) - } - return contents - } -} - -export default MemoryBlockstore diff --git a/packages/repo/src/cid-set.ts b/packages/repo/src/cid-set.ts index 2d1c7daac08..a639c967dd8 100644 --- a/packages/repo/src/cid-set.ts +++ b/packages/repo/src/cid-set.ts @@ -42,8 +42,7 @@ export class CidSet { } toList(): CID[] { - const arr = [...this.set] - return arr.map((c) => CID.parse(c)) + return [...this.set].map((c) => CID.parse(c)) } } diff --git a/packages/repo/src/data-diff.ts b/packages/repo/src/data-diff.ts new file mode 100644 index 00000000000..1e3e71230f8 --- /dev/null +++ b/packages/repo/src/data-diff.ts @@ -0,0 +1,108 @@ +import { CID } from 'multiformats' +import CidSet from './cid-set' +import { MST, mstDiff } from './mst' +import { DataStore } from './types' + +export class DataDiff { + adds: Record = {} + updates: Record = {} + deletes: Record = {} + + newCids: CidSet = new CidSet() + + static async of(curr: DataStore, prev: DataStore | null): Promise { + if (curr instanceof MST && (prev === null || prev instanceof MST)) { + return mstDiff(curr, prev) + } + throw new Error('Unsupported DataStore type for diff') + } + + recordAdd(key: string, cid: CID): void { + this.adds[key] = { key, cid } + this.newCids.add(cid) + } + + recordUpdate(key: string, prev: CID, cid: CID): void { + this.updates[key] = { key, prev, cid } + this.newCids.add(cid) + } + + recordDelete(key: string, cid: CID): void { + this.deletes[key] = { key, cid } + } + + recordNewCid(cid: CID): void { + this.newCids.add(cid) + } + + addDiff(diff: DataDiff) { + for (const add of diff.addList()) { + if (this.deletes[add.key]) { + const del = this.deletes[add.key] + if (del.cid !== add.cid) { + this.recordUpdate(add.key, del.cid, add.cid) + } + delete this.deletes[add.key] + } else { + this.recordAdd(add.key, add.cid) + } + } + for (const update of diff.updateList()) { + this.recordUpdate(update.key, update.prev, update.cid) + delete this.adds[update.key] + delete this.deletes[update.key] + } + for (const del of diff.deleteList()) { + if (this.adds[del.key]) { + delete this.adds[del.key] + } else { + delete this.updates[del.key] + this.recordDelete(del.key, del.cid) + } + } + this.newCids.addSet(diff.newCids) + } + + addList(): DataAdd[] { + return Object.values(this.adds) + } + + updateList(): DataUpdate[] { + return Object.values(this.updates) + } + + deleteList(): DataDelete[] { + return Object.values(this.deletes) + } + + newCidList(): CID[] { + return this.newCids.toList() + } + + updatedKeys(): string[] { + const keys = [ + ...Object.keys(this.adds), + ...Object.keys(this.updates), + ...Object.keys(this.deletes), + ] + return [...new Set(keys)] + } +} + +export type DataAdd = { + key: string + cid: CID +} + +export type DataUpdate = { + key: string + prev: CID + cid: CID +} + +export type DataDelete = { + key: string + cid: CID +} + +export default DataDiff diff --git a/packages/repo/src/error.ts b/packages/repo/src/error.ts new file mode 100644 index 00000000000..ec2293b2a85 --- /dev/null +++ b/packages/repo/src/error.ts @@ -0,0 +1,32 @@ +import { Def } from '@atproto/common/src/check' +import { CID } from 'multiformats/cid' + +export class MissingBlockError extends Error { + constructor(public cid: CID, def?: Def) { + let msg = `block not found: ${cid.toString()}` + if (def) { + msg += `, expected type: ${def.name}` + } + super(msg) + } +} + +export class MissingBlocksError extends Error { + constructor(public context: string, public cids: CID[]) { + const cidStr = cids.map((c) => c.toString()) + super(`missing ${context} blocks: ${cidStr}`) + } +} + +export class MissingCommitBlocksError extends Error { + constructor(public commit: CID, public cids: CID[]) { + const cidStr = cids.map((c) => c.toString()) + super(`missing blocks for commit ${commit.toString()}: ${cidStr}`) + } +} + +export class UnexpectedObjectError extends Error { + constructor(public cid: CID, public def: Def) { + super(`unexpected object at ${cid.toString()}, expected: ${def.name}`) + } +} diff --git a/packages/repo/src/index.ts b/packages/repo/src/index.ts index 89272e74db3..db914ccd148 100644 --- a/packages/repo/src/index.ts +++ b/packages/repo/src/index.ts @@ -1,7 +1,9 @@ -export * from './blockstore' +export * from './block-map' +export * from './cid-set' export * from './repo' export * from './mst' export * from './storage' +export * from './sync' export * from './types' export * from './verify' export * from './util' diff --git a/packages/repo/src/mst/diff.ts b/packages/repo/src/mst/diff.ts index a4ca15af653..37ca66a2267 100644 --- a/packages/repo/src/mst/diff.ts +++ b/packages/repo/src/mst/diff.ts @@ -1,106 +1,129 @@ -import * as auth from '@atproto/auth' -import { CID } from 'multiformats' -import CidSet from '../cid-set' -import { parseRecordKey } from '../util' - -export class DataDiff { - adds: Record = {} - updates: Record = {} - deletes: Record = {} - - newCids: CidSet = new CidSet() - - recordAdd(key: string, cid: CID): void { - this.adds[key] = { key, cid } - this.newCids.add(cid) - } - - recordUpdate(key: string, prev: CID, cid: CID): void { - this.updates[key] = { key, prev, cid } - this.newCids.add(cid) - } - - recordDelete(key: string, cid: CID): void { - this.deletes[key] = { key, cid } +import { DataDiff } from '../data-diff' +import MST from './mst' +import MstWalker from './walker' + +export const nullDiff = async (tree: MST): Promise => { + const diff = new DataDiff() + for await (const entry of tree.walk()) { + if (entry.isLeaf()) { + diff.recordAdd(entry.key, entry.value) + } else { + diff.recordNewCid(entry.pointer) + } } + return diff +} - recordNewCid(cid: CID): void { - this.newCids.add(cid) +export const mstDiff = async ( + curr: MST, + prev: MST | null, +): Promise => { + await curr.getPointer() + if (prev === null) { + return nullDiff(curr) } - addDiff(diff: DataDiff) { - for (const add of diff.addList()) { - if (this.deletes[add.key]) { - const del = this.deletes[add.key] - if (del.cid !== add.cid) { - this.recordUpdate(add.key, del.cid, add.cid) - } - delete this.deletes[add.key] + await prev.getPointer() + const diff = new DataDiff() + + const leftWalker = new MstWalker(prev) + const rightWalker = new MstWalker(curr) + while (!leftWalker.status.done || !rightWalker.status.done) { + // if one walker is finished, continue walking the other & logging all nodes + if (leftWalker.status.done && !rightWalker.status.done) { + const node = rightWalker.status.curr + if (node.isLeaf()) { + diff.recordAdd(node.key, node.value) } else { - this.recordAdd(add.key, add.cid) + diff.recordNewCid(node.pointer) } + await rightWalker.advance() + continue + } else if (!leftWalker.status.done && rightWalker.status.done) { + const node = leftWalker.status.curr + if (node.isLeaf()) { + diff.recordDelete(node.key, node.value) + } + await leftWalker.advance() + continue } - for (const update of diff.updateList()) { - this.recordUpdate(update.key, update.prev, update.cid) - delete this.adds[update.key] - delete this.deletes[update.key] - } - for (const del of diff.deleteList()) { - if (this.adds[del.key]) { - delete this.adds[del.key] + if (leftWalker.status.done || rightWalker.status.done) break + const left = leftWalker.status.curr + const right = rightWalker.status.curr + if (left === null || right === null) break + + // if both pointers are leaves, record an update & advance both or record the lowest key and advance that pointer + if (left.isLeaf() && right.isLeaf()) { + if (left.key === right.key) { + if (!left.value.equals(right.value)) { + diff.recordUpdate(left.key, left.value, right.value) + } + await leftWalker.advance() + await rightWalker.advance() + } else if (left.key < right.key) { + diff.recordDelete(left.key, left.value) + await leftWalker.advance() } else { - delete this.updates[del.key] - this.recordDelete(del.key, del.cid) + diff.recordAdd(right.key, right.value) + await rightWalker.advance() } + continue } - this.newCids.addSet(diff.newCids) - } - addList(): DataAdd[] { - return Object.values(this.adds) - } - - updateList(): DataUpdate[] { - return Object.values(this.updates) - } - - deleteList(): DataDelete[] { - return Object.values(this.deletes) - } + // next, ensure that we're on the same layer + // if one walker is at a higher layer than the other, we need to do one of two things + // if the higher walker is pointed at a tree, step into that tree to try to catch up with the lower + // if the higher walker is pointed at a leaf, then advance the lower walker to try to catch up the higher + if (leftWalker.layer() > rightWalker.layer()) { + if (left.isLeaf()) { + if (right.isLeaf()) { + diff.recordAdd(right.key, right.value) + } else { + diff.recordNewCid(right.pointer) + } + await rightWalker.advance() + } else { + await leftWalker.stepInto() + } + continue + } else if (leftWalker.layer() < rightWalker.layer()) { + if (right.isLeaf()) { + if (left.isLeaf()) { + diff.recordDelete(left.key, left.value) + } + await leftWalker.advance() + } else { + diff.recordNewCid(right.pointer) + await rightWalker.stepInto() + } + continue + } - newCidList(): CID[] { - return this.newCids.toList() - } + // if we're on the same level, and both pointers are trees, do a comparison + // if they're the same, step over. if they're different, step in to find the subdiff + if (left.isTree() && right.isTree()) { + if (left.pointer.equals(right.pointer)) { + await leftWalker.stepOver() + await rightWalker.stepOver() + } else { + diff.recordNewCid(right.pointer) + await leftWalker.stepInto() + await rightWalker.stepInto() + } + continue + } - updatedKeys(): string[] { - const keys = [ - ...Object.keys(this.adds), - ...Object.keys(this.updates), - ...Object.keys(this.deletes), - ] - return [...new Set(keys)] - } + // finally, if one pointer is a tree and the other is a leaf, simply step into the tree + if (left.isLeaf() && right.isTree()) { + await diff.recordNewCid(right.pointer) + await rightWalker.stepInto() + continue + } else if (left.isTree() && right.isLeaf()) { + await leftWalker.stepInto() + continue + } - neededCapabilities(rootDid: string): auth.ucans.Capability[] { - return this.updatedKeys().map((key) => { - const { collection, rkey } = parseRecordKey(key) - return auth.writeCap(rootDid, collection, rkey) - }) + throw new Error('Unidentifiable case in diff walk') } -} - -export type DataAdd = { - key: string - cid: CID -} - -export type DataUpdate = { - key: string - prev: CID - cid: CID -} - -export type DataDelete = { - key: string - cid: CID + return diff } diff --git a/packages/repo/src/mst/mst.ts b/packages/repo/src/mst/mst.ts index b45da78ce1c..b00f851ce50 100644 --- a/packages/repo/src/mst/mst.ts +++ b/packages/repo/src/mst/mst.ts @@ -1,13 +1,15 @@ import z from 'zod' import { CID } from 'multiformats' -import IpldStore from '../blockstore/ipld-store' -import { def, cidForData } from '@atproto/common' -import { DataDiff } from './diff' +import { ReadableBlockstore } from '../storage' +import { schema as common, cidForCbor } from '@atproto/common' import { DataStore } from '../types' import { BlockWriter } from '@ipld/car/api' import * as util from './util' -import MstWalker from './walker' +import BlockMap from '../block-map' +import CidSet from '../cid-set' +import { MissingBlockError, MissingBlocksError } from '../error' +import * as parse from '../parse' /** * This is an implementation of a Merkle Search Tree (MST) @@ -39,18 +41,23 @@ import MstWalker from './walker' * Then the first will be described as `prefix: 0, key: 'bsky/posts/abcdefg'`, * and the second will be described as `prefix: 16, key: 'hi'.` */ -const subTreePointer = z.nullable(def.cid) +const subTreePointer = z.nullable(common.cid) const treeEntry = z.object({ p: z.number(), // prefix count of utf-8 chars that this key shares with the prev key k: z.string(), // the rest of the key outside the shared prefix - v: def.cid, // value + v: common.cid, // value t: subTreePointer, // next subtree (to the right of leaf) }) -export const nodeDataDef = z.object({ +const nodeData = z.object({ l: subTreePointer, // left-most subtree e: z.array(treeEntry), //entries }) -export type NodeData = z.infer +export type NodeData = z.infer + +export const nodeDataDef = { + name: 'mst node', + schema: nodeData, +} export type NodeEntry = MST | Leaf @@ -62,7 +69,7 @@ export type MstOpts = { } export class MST implements DataStore { - blockstore: IpldStore + storage: ReadableBlockstore fanout: Fanout entries: NodeEntry[] | null layer: number | null @@ -70,13 +77,13 @@ export class MST implements DataStore { outdatedPointer = false constructor( - blockstore: IpldStore, + storage: ReadableBlockstore, fanout: Fanout, pointer: CID, entries: NodeEntry[] | null, layer: number | null, ) { - this.blockstore = blockstore + this.storage = storage this.fanout = fanout this.entries = entries this.layer = layer @@ -84,29 +91,33 @@ export class MST implements DataStore { } static async create( - blockstore: IpldStore, + storage: ReadableBlockstore, entries: NodeEntry[] = [], opts?: Partial, ): Promise { const pointer = await util.cidForEntries(entries) const { layer = 0, fanout = DEFAULT_MST_FANOUT } = opts || {} - return new MST(blockstore, fanout, pointer, entries, layer) + return new MST(storage, fanout, pointer, entries, layer) } static async fromData( - blockstore: IpldStore, + storage: ReadableBlockstore, data: NodeData, opts?: Partial, ): Promise { const { layer = null, fanout = DEFAULT_MST_FANOUT } = opts || {} - const entries = await util.deserializeNodeData(blockstore, data, opts) - const pointer = await cidForData(data) - return new MST(blockstore, fanout, pointer, entries, layer) + const entries = await util.deserializeNodeData(storage, data, opts) + const pointer = await cidForCbor(data) + return new MST(storage, fanout, pointer, entries, layer) } - static load(blockstore: IpldStore, cid: CID, opts?: Partial): MST { + static load( + storage: ReadableBlockstore, + cid: CID, + opts?: Partial, + ): MST { const { layer = null, fanout = DEFAULT_MST_FANOUT } = opts || {} - return new MST(blockstore, fanout, cid, null, layer) + return new MST(storage, fanout, cid, null, layer) } // Immutability @@ -115,7 +126,7 @@ export class MST implements DataStore { // We never mutate an MST, we just return a new MST with updated values async newTree(entries: NodeEntry[]): Promise { const mst = new MST( - this.blockstore, + this.storage, this.fanout, this.pointer, entries, @@ -132,13 +143,13 @@ export class MST implements DataStore { async getEntries(): Promise { if (this.entries) return [...this.entries] if (this.pointer) { - const data = await this.blockstore.get(this.pointer, nodeDataDef) + const data = await this.storage.readObj(this.pointer, nodeDataDef) const firstLeaf = data.e[0] const layer = firstLeaf !== undefined ? await util.leadingZerosOnHash(firstLeaf.k, this.fanout) : undefined - this.entries = await util.deserializeNodeData(this.blockstore, data, { + this.entries = await util.deserializeNodeData(this.storage, data, { layer, fanout: this.fanout, }) @@ -197,32 +208,22 @@ export class MST implements DataStore { // Core functionality // ------------------- - // Persist the MST to the blockstore - // If the topmost tree only has one entry and it's a subtree, we can eliminate the topmost tree - // However, lower trees with only one entry must be preserved - async stage(): Promise { - return this.stageRecurse(true) - } - - async stageRecurse(trimTop = false): Promise { + // Return the necessary blocks to persist the MST to repo storage + async getUnstoredBlocks(): Promise<{ root: CID; blocks: BlockMap }> { + const blocks = new BlockMap() const pointer = await this.getPointer() - const alreadyHas = await this.blockstore.has(pointer) - if (alreadyHas) return pointer + const alreadyHas = await this.storage.has(pointer) + if (alreadyHas) return { root: pointer, blocks } const entries = await this.getEntries() - if (entries.length === 1 && trimTop) { - const node = entries[0] - if (node.isTree()) { - return node.stageRecurse(true) - } - } const data = util.serializeNodeData(entries) - await this.blockstore.stage(data) + await blocks.add(data) for (const entry of entries) { if (entry.isTree()) { - await entry.stageRecurse(false) + const subtree = await entry.getUnstoredBlocks() + blocks.addMap(subtree.blocks) } } - return pointer + return { root: pointer, blocks: blocks } } // Adds a new leaf for the given key/value pair @@ -288,7 +289,7 @@ export class MST implements DataStore { if (left) updated.push(left) updated.push(new Leaf(key, value)) if (right) updated.push(right) - const newRoot = await MST.create(this.blockstore, updated, { + const newRoot = await MST.create(this.storage, updated, { layer: keyZeros, fanout: this.fanout, }) @@ -329,6 +330,11 @@ export class MST implements DataStore { // Deletes the value at the given key async delete(key: string): Promise { + const altered = await this.deleteRecurse(key) + return altered.trimTop() + } + + async deleteRecurse(key: string): Promise { const index = await this.findGtOrEqualLeafIndex(key) const found = await this.atIndex(index) // if found, remove it on this level @@ -349,7 +355,7 @@ export class MST implements DataStore { // else recurse down to find it const prev = await this.atIndex(index - 1) if (prev?.isTree()) { - const subtree = await prev.delete(key) + const subtree = await prev.deleteRecurse(key) const subTreeEntries = await subtree.getEntries() if (subTreeEntries.length === 0) { return this.removeEntry(index - 1) @@ -361,114 +367,6 @@ export class MST implements DataStore { } } - // Walk two MSTs to find the semantic changes - async diff(other: MST): Promise { - await this.getPointer() - await other.getPointer() - const diff = new DataDiff() - - const leftWalker = new MstWalker(this) - const rightWalker = new MstWalker(other) - while (!leftWalker.status.done || !rightWalker.status.done) { - // if one walker is finished, continue walking the other & logging all nodes - if (leftWalker.status.done && !rightWalker.status.done) { - const node = rightWalker.status.curr - if (node.isLeaf()) { - diff.recordAdd(node.key, node.value) - } else { - diff.recordNewCid(node.pointer) - } - await rightWalker.advance() - continue - } else if (!leftWalker.status.done && rightWalker.status.done) { - const node = leftWalker.status.curr - if (node.isLeaf()) { - diff.recordDelete(node.key, node.value) - } - await leftWalker.advance() - continue - } - if (leftWalker.status.done || rightWalker.status.done) break - const left = leftWalker.status.curr - const right = rightWalker.status.curr - if (left === null || right === null) break - - // if both pointers are leaves, record an update & advance both or record the lowest key and advance that pointer - if (left.isLeaf() && right.isLeaf()) { - if (left.key === right.key) { - if (!left.value.equals(right.value)) { - diff.recordUpdate(left.key, left.value, right.value) - } - await leftWalker.advance() - await rightWalker.advance() - } else if (left.key < right.key) { - diff.recordDelete(left.key, left.value) - await leftWalker.advance() - } else { - diff.recordAdd(right.key, right.value) - await rightWalker.advance() - } - continue - } - - // next, ensure that we're on the same layer - // if one walker is at a higher layer than the other, we need to do one of two things - // if the higher walker is pointed at a tree, step into that tree to try to catch up with the lower - // if the higher walker is pointed at a leaf, then advance the lower walker to try to catch up the higher - if (leftWalker.layer() > rightWalker.layer()) { - if (left.isLeaf()) { - if (right.isLeaf()) { - diff.recordAdd(right.key, right.value) - } else { - diff.recordNewCid(right.pointer) - } - await rightWalker.advance() - } else { - await leftWalker.stepInto() - } - continue - } else if (leftWalker.layer() < rightWalker.layer()) { - if (right.isLeaf()) { - if (left.isLeaf()) { - diff.recordDelete(left.key, left.value) - } - await leftWalker.advance() - } else { - diff.recordNewCid(right.pointer) - await rightWalker.stepInto() - } - continue - } - - // if we're on the same level, and both pointers are trees, do a comparison - // if they're the same, step over. if they're different, step in to find the subdiff - if (left.isTree() && right.isTree()) { - if (left.pointer.equals(right.pointer)) { - await leftWalker.stepOver() - await rightWalker.stepOver() - } else { - diff.recordNewCid(right.pointer) - await leftWalker.stepInto() - await rightWalker.stepInto() - } - continue - } - - // finally, if one pointer is a tree and the other is a leaf, simply step into the tree - if (left.isLeaf() && right.isTree()) { - await diff.recordNewCid(right.pointer) - await rightWalker.stepInto() - continue - } else if (left.isTree() && right.isLeaf()) { - await leftWalker.stepInto() - continue - } - - throw new Error('Unidentifiable case in diff walk') - } - return diff - } - // Simple Operations // ------------------- @@ -543,6 +441,16 @@ export class MST implements DataStore { return this.newTree(update) } + // if the topmost node in the tree only points to another tree, trim the top and return the subtree + async trimTop(): Promise { + const entries = await this.getEntries() + if (entries.length === 1 && entries[0].isTree()) { + return entries[0].trimTop() + } else { + return this + } + } + // Subtree & Splits // ------------------- @@ -604,7 +512,7 @@ export class MST implements DataStore { async createChild(): Promise { const layer = await this.getLayer() - return MST.create(this.blockstore, [], { + return MST.create(this.storage, [], { layer: layer - 1, fanout: this.fanout, }) @@ -612,7 +520,7 @@ export class MST implements DataStore { async createParent(): Promise { const layer = await this.getLayer() - const parent = await MST.create(this.blockstore, [this], { + const parent = await MST.create(this.storage, [this], { layer: layer + 1, fanout: this.fanout, }) @@ -660,7 +568,11 @@ export class MST implements DataStore { } } - async list(count: number, after?: string, before?: string): Promise { + async list( + count = Number.MAX_SAFE_INTEGER, + after?: string, + before?: string, + ): Promise { const vals: Leaf[] = [] for await (const leaf of this.walkLeavesFrom(after || '')) { if (leaf.key === after) continue @@ -726,6 +638,22 @@ export class MST implements DataStore { return nodes } + // Walks tree & returns all cids + async allCids(): Promise { + const cids = new CidSet() + const entries = await this.getEntries() + for (const entry of entries) { + if (entry.isLeaf()) { + cids.add(entry.value) + } else { + const subtreeCids = await entry.allCids() + cids.addSet(subtreeCids) + } + } + cids.add(await this.getPointer()) + return cids + } + // Walks tree & returns all leaves async leaves() { const leaves: Leaf[] = [] @@ -741,17 +669,97 @@ export class MST implements DataStore { return leaves.length } + // Reachable tree traversal + // ------------------- + + // Walk reachable branches of tree & emit nodes, consumer can bail at any point by returning false + async *walkReachable(): AsyncIterable { + yield this + const entries = await this.getEntries() + for (const entry of entries) { + if (entry.isTree()) { + try { + for await (const e of entry.walk()) { + yield e + } + } catch (err) { + if (err instanceof MissingBlockError) { + continue + } else { + throw err + } + } + } else { + yield entry + } + } + } + + async reachableLeaves(): Promise { + const leaves: Leaf[] = [] + for await (const entry of this.walkReachable()) { + if (entry.isLeaf()) leaves.push(entry) + } + return leaves + } + // Sync Protocol async writeToCarStream(car: BlockWriter): Promise { - for await (const entry of this.walk()) { - if (entry.isTree()) { - const pointer = await entry.getPointer() - await this.blockstore.addToCar(car, pointer) + const entries = await this.getEntries() + const leaves = new CidSet() + let toFetch = new CidSet() + toFetch.add(await this.getPointer()) + for (const entry of entries) { + if (entry.isLeaf()) { + leaves.add(entry.value) } else { - await this.blockstore.addToCar(car, entry.value) + toFetch.add(await entry.getPointer()) + } + } + while (toFetch.size() > 0) { + const nextLayer = new CidSet() + const fetched = await this.storage.getBlocks(toFetch.toList()) + if (fetched.missing.length > 0) { + throw new MissingBlocksError('mst node', fetched.missing) + } + for (const cid of toFetch.toList()) { + const found = await parse.getAndParse(fetched.blocks, cid, nodeDataDef) + await car.put({ cid, bytes: found.bytes }) + const entries = await util.deserializeNodeData(this.storage, found.obj) + + for (const entry of entries) { + if (entry.isLeaf()) { + leaves.add(entry.value) + } else { + nextLayer.add(await entry.getPointer()) + } + } } + toFetch = nextLayer + } + const leafData = await this.storage.getBlocks(leaves.toList()) + if (leafData.missing.length > 0) { + throw new MissingBlocksError('mst leaf', leafData.missing) + } + + for (const leaf of leafData.blocks.entries()) { + await car.put(leaf) + } + } + + async cidsForPath(key: string): Promise { + const cids: CID[] = [await this.getPointer()] + const index = await this.findGtOrEqualLeafIndex(key) + const found = await this.atIndex(index) + if (found && found.isLeaf() && found.key === key) { + return [...cids, found.value] + } + const prev = await this.atIndex(index - 1) + if (prev && prev.isTree()) { + return [...cids, ...(await prev.cidsForPath(key))] } + return cids } // Matching Leaf interface diff --git a/packages/repo/src/mst/util.ts b/packages/repo/src/mst/util.ts index 6e9cff24185..0865ce27a39 100644 --- a/packages/repo/src/mst/util.ts +++ b/packages/repo/src/mst/util.ts @@ -1,9 +1,9 @@ import { CID } from 'multiformats' import * as uint8arrays from 'uint8arrays' -import IpldStore from '../blockstore/ipld-store' +import { ReadableBlockstore } from '../storage' import { sha256 } from '@atproto/crypto' import { MST, Leaf, NodeEntry, NodeData, MstOpts, Fanout } from './mst' -import { cidForData } from '@atproto/common' +import { cidForCbor } from '@atproto/common' type SupportedBases = 'base2' | 'base8' | 'base16' | 'base32' | 'base64' @@ -39,7 +39,7 @@ export const layerForEntries = async ( } export const deserializeNodeData = async ( - blockstore: IpldStore, + storage: ReadableBlockstore, data: NodeData, opts?: Partial, ): Promise => { @@ -47,7 +47,7 @@ export const deserializeNodeData = async ( const entries: NodeEntry[] = [] if (data.l !== null) { entries.push( - await MST.load(blockstore, data.l, { + await MST.load(storage, data.l, { layer: layer ? layer - 1 : undefined, fanout, }), @@ -60,7 +60,7 @@ export const deserializeNodeData = async ( lastKey = key if (entry.t !== null) { entries.push( - await MST.load(blockstore, entry.t, { + await MST.load(storage, entry.t, { layer: layer ? layer - 1 : undefined, fanout, }), @@ -118,5 +118,5 @@ export const countPrefixLen = (a: string, b: string): number => { export const cidForEntries = async (entries: NodeEntry[]): Promise => { const data = serializeNodeData(entries) - return cidForData(data) + return cidForCbor(data) } diff --git a/packages/repo/src/parse.ts b/packages/repo/src/parse.ts new file mode 100644 index 00000000000..646cecd3fef --- /dev/null +++ b/packages/repo/src/parse.ts @@ -0,0 +1,30 @@ +import { check, cborDecode } from '@atproto/common' +import { CID } from 'multiformats/cid' +import BlockMap from './block-map' +import { MissingBlockError, UnexpectedObjectError } from './error' + +export const getAndParse = async ( + blocks: BlockMap, + cid: CID, + def: check.Def, +): Promise<{ obj: T; bytes: Uint8Array }> => { + const bytes = blocks.get(cid) + if (!bytes) { + throw new MissingBlockError(cid, def) + } + return parseObj(bytes, cid, def) +} + +export const parseObj = ( + bytes: Uint8Array, + cid: CID, + def: check.Def, +): { obj: T; bytes: Uint8Array } => { + const obj = cborDecode(bytes) + const res = def.schema.safeParse(obj) + if (res.success) { + return { obj: res.data, bytes } + } else { + throw new UnexpectedObjectError(cid, def) + } +} diff --git a/packages/repo/src/readable-repo.ts b/packages/repo/src/readable-repo.ts new file mode 100644 index 00000000000..d2b4bb19922 --- /dev/null +++ b/packages/repo/src/readable-repo.ts @@ -0,0 +1,88 @@ +import { CID } from 'multiformats/cid' +import { + RepoRoot, + Commit, + def, + DataStore, + RepoMeta, + RepoContents, +} from './types' +import { ReadableBlockstore } from './storage' +import { MST } from './mst' +import log from './logger' +import * as util from './util' +import * as parse from './parse' +import { MissingBlocksError } from './error' + +type Params = { + storage: ReadableBlockstore + data: DataStore + commit: Commit + root: RepoRoot + meta: RepoMeta + cid: CID +} + +export class ReadableRepo { + storage: ReadableBlockstore + data: DataStore + commit: Commit + root: RepoRoot + meta: RepoMeta + cid: CID + + constructor(params: Params) { + this.storage = params.storage + this.data = params.data + this.commit = params.commit + this.root = params.root + this.meta = params.meta + this.cid = params.cid + } + + static async load(storage: ReadableBlockstore, commitCid: CID) { + const commit = await storage.readObj(commitCid, def.commit) + const root = await storage.readObj(commit.root, def.repoRoot) + const meta = await storage.readObj(root.meta, def.repoMeta) + const data = await MST.load(storage, root.data) + log.info({ did: meta.did }, 'loaded repo for') + return new ReadableRepo({ + storage, + data, + commit, + root, + meta, + cid: commitCid, + }) + } + + get did(): string { + return this.meta.did + } + + async getRecord(collection: string, rkey: string): Promise { + const dataKey = collection + '/' + rkey + const cid = await this.data.get(dataKey) + if (!cid) return null + return this.storage.readObj(cid, def.unknown) + } + + async getContents(): Promise { + const entries = await this.data.list() + const cids = entries.map((e) => e.value) + const { blocks, missing } = await this.storage.getBlocks(cids) + if (missing.length > 0) { + throw new MissingBlocksError('getContents record', missing) + } + const contents: RepoContents = {} + for (const entry of entries) { + const { collection, rkey } = util.parseDataKey(entry.key) + contents[collection] ??= {} + const parsed = await parse.getAndParse(blocks, entry.value, def.record) + contents[collection][rkey] = parsed.obj + } + return contents + } +} + +export default ReadableRepo diff --git a/packages/repo/src/repo.ts b/packages/repo/src/repo.ts index 2ef7a421633..79632f80958 100644 --- a/packages/repo/src/repo.ts +++ b/packages/repo/src/repo.ts @@ -1,6 +1,5 @@ import { CID } from 'multiformats/cid' -import { CarWriter } from '@ipld/car' -import { BlockWriter } from '@ipld/car/writer' +import * as crypto from '@atproto/crypto' import { RepoRoot, Commit, @@ -9,303 +8,180 @@ import { RepoMeta, RecordCreateOp, RecordWriteOp, + CommitData, + WriteOpAction, } from './types' -import { streamToArray } from '@atproto/common' -import IpldStore from './blockstore/ipld-store' -import * as auth from '@atproto/auth' +import { RepoStorage } from './storage' import { MST } from './mst' +import DataDiff from './data-diff' import log from './logger' -import * as util from './util' +import BlockMap from './block-map' +import { ReadableRepo } from './readable-repo' type Params = { - blockstore: IpldStore + storage: RepoStorage data: DataStore commit: Commit root: RepoRoot meta: RepoMeta cid: CID - stagedWrites: RecordWriteOp[] } -export class Repo { - blockstore: IpldStore - data: DataStore - commit: Commit - root: RepoRoot - meta: RepoMeta - cid: CID - stagedWrites: RecordWriteOp[] +export class Repo extends ReadableRepo { + storage: RepoStorage constructor(params: Params) { - this.blockstore = params.blockstore - this.data = params.data - this.commit = params.commit - this.root = params.root - this.meta = params.meta - this.cid = params.cid - this.stagedWrites = params.stagedWrites + super(params) } - static async create( - blockstore: IpldStore, + static async formatInitCommit( + storage: RepoStorage, did: string, - authStore: auth.AuthStore, + keypair: crypto.Keypair, initialRecords: RecordCreateOp[] = [], - ): Promise { - let tokenCid: CID | null = null - if (!(await authStore.canSignForDid(did))) { - const foundUcan = await authStore.findUcan(auth.maintenanceCap(did)) - if (foundUcan === null) { - throw new Error(`No valid Ucan for creating repo`) - } - tokenCid = await blockstore.stage(auth.encodeUcan(foundUcan)) - } + ): Promise { + const newBlocks = new BlockMap() - let data = await MST.create(blockstore) + let data = await MST.create(storage) for (const write of initialRecords) { - const cid = await blockstore.stage(write.value) + const cid = await newBlocks.add(write.record) const dataKey = write.collection + '/' + write.rkey data = await data.add(dataKey, cid) } - const dataCid = await data.stage() + const unstoredData = await data.getUnstoredBlocks() + newBlocks.addMap(unstoredData.blocks) const meta: RepoMeta = { did, version: 1, datastore: 'mst', } - const metaCid = await blockstore.stage(meta) + const metaCid = await newBlocks.add(meta) const root: RepoRoot = { meta: metaCid, prev: null, - auth_token: tokenCid, - data: dataCid, + data: unstoredData.root, } + const rootCid = await newBlocks.add(root) - const rootCid = await blockstore.stage(root) const commit: Commit = { root: rootCid, - sig: await authStore.sign(rootCid.bytes), + sig: await keypair.sign(rootCid.bytes), } + const commitCid = await newBlocks.add(commit) - const cid = await blockstore.stage(commit) - - await blockstore.saveStaged() + return { + commit: commitCid, + prev: null, + blocks: newBlocks, + } + } + static async create( + storage: RepoStorage, + did: string, + keypair: crypto.Keypair, + initialRecords: RecordCreateOp[] = [], + ): Promise { + const commit = await Repo.formatInitCommit( + storage, + did, + keypair, + initialRecords, + ) + await storage.applyCommit(commit) log.info({ did }, `created repo`) - return new Repo({ - blockstore, - data, - commit, - root, - meta, - cid, - stagedWrites: [], - }) + return Repo.load(storage, commit.commit) } - static async load(blockstore: IpldStore, cid: CID) { - const commit = await blockstore.get(cid, def.commit) - const root = await blockstore.get(commit.root, def.repoRoot) - const meta = await blockstore.get(root.meta, def.repoMeta) - const data = await MST.load(blockstore, root.data) + static async load(storage: RepoStorage, cid?: CID) { + const commitCid = cid || (await storage.getHead()) + if (!commitCid) { + throw new Error('No cid provided and none in storage') + } + const commit = await storage.readObj(commitCid, def.commit) + const root = await storage.readObj(commit.root, def.repoRoot) + const meta = await storage.readObj(root.meta, def.repoMeta) + const data = await MST.load(storage, root.data) log.info({ did: meta.did }, 'loaded repo for') return new Repo({ - blockstore, + storage, data, commit, root, meta, - cid, - stagedWrites: [], - }) - } - - private updateRepo(params: Partial): Repo { - return new Repo({ - blockstore: params.blockstore || this.blockstore, - data: params.data || this.data, - commit: params.commit || this.commit, - root: params.root || this.root, - meta: params.meta || this.meta, - cid: params.cid || this.cid, - stagedWrites: params.stagedWrites || this.stagedWrites, - }) - } - - get did(): string { - return this.meta.did - } - - async getRecord(collection: string, rkey: string): Promise { - const dataKey = collection + '/' + rkey - const cid = await this.data.get(dataKey) - if (!cid) return null - return this.blockstore.getUnchecked(cid) - } - - stageUpdate(write: RecordWriteOp | RecordWriteOp[]): Repo { - const writeArr = Array.isArray(write) ? write : [write] - return this.updateRepo({ - stagedWrites: [...this.stagedWrites, ...writeArr], + cid: commitCid, }) } async createCommit( - authStore: auth.AuthStore, - performUpdate?: (prev: CID, curr: CID) => Promise, - ): Promise { + toWrite: RecordWriteOp | RecordWriteOp[], + keypair: crypto.Keypair, + ): Promise { + const writes = Array.isArray(toWrite) ? toWrite : [toWrite] + const newBlocks = new BlockMap() + let data = this.data - for (const write of this.stagedWrites) { - if (write.action === 'create') { - const cid = await this.blockstore.stage(write.value) + for (const write of writes) { + if (write.action === WriteOpAction.Create) { + const cid = await newBlocks.add(write.record) const dataKey = write.collection + '/' + write.rkey data = await data.add(dataKey, cid) - } else if (write.action === 'update') { - const cid = await this.blockstore.stage(write.value) + } else if (write.action === WriteOpAction.Update) { + const cid = await newBlocks.add(write.record) const dataKey = write.collection + '/' + write.rkey data = await data.update(dataKey, cid) - } else if (write.action === 'delete') { + } else if (write.action === WriteOpAction.Delete) { const dataKey = write.collection + '/' + write.rkey data = await data.delete(dataKey) } } - const token = (await authStore.canSignForDid(this.did)) - ? null - : await util.ucanForOperation(this.data, data, this.did, authStore) - const tokenCid = token ? await this.blockstore.stage(token) : null - const dataCid = await data.stage() + const unstoredData = await data.getUnstoredBlocks() + newBlocks.addMap(unstoredData.blocks) + + // ensure we're not missing any blocks that were removed and then readded in this commit + const diff = await DataDiff.of(data, this.data) + const found = newBlocks.getMany(diff.newCidList()) + if (found.missing.length > 0) { + const fromStorage = await this.storage.getBlocks(found.missing) + if (fromStorage.missing.length > 0) { + // this shouldn't ever happen + throw new Error( + 'Could not find block for commit in Datastore or storage', + ) + } + newBlocks.addMap(fromStorage.blocks) + } + const root: RepoRoot = { meta: this.root.meta, prev: this.cid, - auth_token: tokenCid, - data: dataCid, + data: unstoredData.root, } - const rootCid = await this.blockstore.stage(root) + const rootCid = await newBlocks.add(root) + const commit: Commit = { root: rootCid, - sig: await authStore.sign(rootCid.bytes), - } - const commitCid = await this.blockstore.stage(commit) - - if (performUpdate) { - const rebaseOn = await performUpdate(this.cid, commitCid) - if (rebaseOn) { - await this.blockstore.clearStaged() - const rebaseRepo = await Repo.load(this.blockstore, rebaseOn) - return rebaseRepo.createCommit(authStore, performUpdate) - } else { - await this.blockstore.saveStaged() - } - } else { - await this.blockstore.saveStaged() - } - - return this.updateRepo({ - cid: commitCid, - root, - commit, - data, - stagedWrites: [], - }) - } - - async revert(count: number): Promise { - let revertTo = this.cid - for (let i = 0; i < count; i++) { - const commit = await this.blockstore.get(revertTo, def.commit) - const root = await this.blockstore.get(commit.root, def.repoRoot) - if (root.prev === null) { - throw new Error(`Could not revert ${count} commits`) - } - revertTo = root.prev + sig: await keypair.sign(rootCid.bytes), } - return Repo.load(this.blockstore, revertTo) - } - - // CAR FILES - // ----------- + const commitCid = await newBlocks.add(commit) - async getCarNoHistory(): Promise { - return this.openCar((car: BlockWriter) => { - return this.writeCheckoutToCarStream(car) - }) - } - - async getDiffCar(to: CID | null): Promise { - return this.openCar((car: BlockWriter) => { - return this.writeCommitsToCarStream(car, to, this.cid) - }) - } - - async getFullHistory(): Promise { - return this.getDiffCar(null) - } - - private async openCar( - fn: (car: BlockWriter) => Promise, - ): Promise { - const { writer, out } = CarWriter.create([this.cid]) - try { - await fn(writer) - } finally { - writer.close() - } - return streamToArray(out) - } - - async writeCheckoutToCarStream(car: BlockWriter): Promise { - const commit = await this.blockstore.get(this.cid, def.commit) - const root = await this.blockstore.get(commit.root, def.repoRoot) - await this.blockstore.addToCar(car, this.cid) - await this.blockstore.addToCar(car, commit.root) - await this.blockstore.addToCar(car, root.meta) - if (root.auth_token) { - await this.blockstore.addToCar(car, root.auth_token) + return { + commit: commitCid, + prev: this.cid, + blocks: newBlocks, } - await this.data.writeToCarStream(car) } - async writeCommitsToCarStream( - car: BlockWriter, - oldestCommit: CID | null, - recentCommit: CID, - ): Promise { - const commitPath = await util.getCommitPath( - this.blockstore, - oldestCommit, - recentCommit, - ) - if (commitPath === null) { - throw new Error('Could not find shared history') - } - if (commitPath.length === 0) return - const firstHeadInPath = await Repo.load(this.blockstore, commitPath[0]) - // handle the first commit - let prevHead: Repo | null = - firstHeadInPath.root.prev !== null - ? await Repo.load(this.blockstore, firstHeadInPath.root.prev) - : null - for (const commit of commitPath) { - const nextHead = await Repo.load(this.blockstore, commit) - await this.blockstore.addToCar(car, nextHead.cid) - await this.blockstore.addToCar(car, nextHead.commit.root) - await this.blockstore.addToCar(car, nextHead.root.meta) - if (nextHead.root.auth_token) { - await this.blockstore.addToCar(car, nextHead.root.auth_token) - } - if (prevHead === null) { - await nextHead.data.writeToCarStream(car) - } else { - const diff = await prevHead.data.diff(nextHead.data) - await Promise.all( - diff.newCidList().map((cid) => this.blockstore.addToCar(car, cid)), - ) - } - prevHead = nextHead - } + async applyCommit( + toWrite: RecordWriteOp | RecordWriteOp[], + keypair: crypto.Keypair, + ): Promise { + const commit = await this.createCommit(toWrite, keypair) + await this.storage.applyCommit(commit) + return Repo.load(this.storage, commit.commit) } } diff --git a/packages/repo/src/storage/index.ts b/packages/repo/src/storage/index.ts index c9f6f047dc0..c5eb715d59a 100644 --- a/packages/repo/src/storage/index.ts +++ b/packages/repo/src/storage/index.ts @@ -1 +1,5 @@ +export * from './readable-blockstore' +export * from './repo-storage' +export * from './memory-blockstore' +export * from './sync-storage' export * from './types' diff --git a/packages/repo/src/storage/memory-blockstore.ts b/packages/repo/src/storage/memory-blockstore.ts new file mode 100644 index 00000000000..dc35dae667a --- /dev/null +++ b/packages/repo/src/storage/memory-blockstore.ts @@ -0,0 +1,121 @@ +import { CID } from 'multiformats/cid' +import { CommitData, def } from '../types' +import BlockMap from '../block-map' +import { MST } from '../mst' +import DataDiff from '../data-diff' +import { MissingCommitBlocksError } from '../error' +import RepoStorage from './repo-storage' + +export class MemoryBlockstore extends RepoStorage { + blocks: BlockMap + head: CID | null = null + + constructor(blocks?: BlockMap) { + super() + this.blocks = new BlockMap() + if (blocks) { + this.blocks.addMap(blocks) + } + } + + async getHead(): Promise { + return this.head + } + + async getBytes(cid: CID): Promise { + return this.blocks.get(cid) || null + } + + async has(cid: CID): Promise { + return this.blocks.has(cid) + } + + async getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> { + return this.blocks.getMany(cids) + } + + async putBlock(cid: CID, block: Uint8Array): Promise { + this.blocks.set(cid, block) + } + + async putMany(blocks: BlockMap): Promise { + this.blocks.addMap(blocks) + } + + async indexCommits(commits: CommitData[]): Promise { + commits.forEach((commit) => { + this.blocks.addMap(commit.blocks) + }) + } + + async updateHead(cid: CID, _prev: CID | null): Promise { + this.head = cid + } + + async applyCommit(commit: CommitData): Promise { + this.blocks.addMap(commit.blocks) + this.head = commit.commit + } + + async getCommitPath( + latest: CID, + earliest: CID | null, + ): Promise { + let curr: CID | null = latest + const path: CID[] = [] + while (curr !== null) { + path.push(curr) + const commit = await this.readObj(curr, def.commit) + const root = await this.readObj(commit.root, def.repoRoot) + if (!earliest && root.prev === null) { + return path.reverse() + } else if (earliest && root.prev.equals(earliest)) { + return path.reverse() + } + curr = root.prev + } + return null + } + + async getBlocksForCommits( + commits: CID[], + ): Promise<{ [commit: string]: BlockMap }> { + const commitData: { [commit: string]: BlockMap } = {} + let prevData: MST | null = null + for (const commitCid of commits) { + const commit = await this.readObj(commitCid, def.commit) + const root = await this.readObj(commit.root, def.repoRoot) + const data = await MST.load(this, root.data) + const diff = await DataDiff.of(data, prevData) + const { blocks, missing } = await this.getBlocks([ + commitCid, + commit.root, + ...diff.newCidList(), + ]) + if (missing.length > 0) { + throw new MissingCommitBlocksError(commitCid, missing) + } + if (!root.prev) { + const meta = await this.readObjAndBytes(root.meta, def.repoMeta) + blocks.set(root.meta, meta.bytes) + } + commitData[commitCid.toString()] = blocks + prevData = data + } + return commitData + } + + async sizeInBytes(): Promise { + let total = 0 + this.blocks.forEach((bytes) => { + total += bytes.byteLength + }) + return total + } + + async destroy(): Promise { + this.blocks.clear() + } +} + +export default MemoryBlockstore diff --git a/packages/repo/src/storage/readable-blockstore.ts b/packages/repo/src/storage/readable-blockstore.ts new file mode 100644 index 00000000000..bd06dd9ab08 --- /dev/null +++ b/packages/repo/src/storage/readable-blockstore.ts @@ -0,0 +1,38 @@ +import { check } from '@atproto/common' +import { CID } from 'multiformats/cid' +import BlockMap from '../block-map' +import { MissingBlockError } from '../error' +import * as parse from '../parse' + +export abstract class ReadableBlockstore { + abstract getBytes(cid: CID): Promise + abstract has(cid: CID): Promise + abstract getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> + + async attemptRead( + cid: CID, + def: check.Def, + ): Promise<{ obj: T; bytes: Uint8Array } | null> { + const bytes = await this.getBytes(cid) + if (!bytes) return null + return parse.parseObj(bytes, cid, def) + } + + async readObjAndBytes( + cid: CID, + def: check.Def, + ): Promise<{ obj: T; bytes: Uint8Array }> { + const read = await this.attemptRead(cid, def) + if (!read) { + throw new MissingBlockError(cid, def) + } + return read + } + + async readObj(cid: CID, def: check.Def): Promise { + const obj = await this.readObjAndBytes(cid, def) + return obj.obj + } +} + +export default ReadableBlockstore diff --git a/packages/repo/src/storage/repo-storage.ts b/packages/repo/src/storage/repo-storage.ts new file mode 100644 index 00000000000..7383545cdff --- /dev/null +++ b/packages/repo/src/storage/repo-storage.ts @@ -0,0 +1,42 @@ +import { CID } from 'multiformats/cid' +import BlockMap from '../block-map' +import { CommitBlockData, CommitData } from '../types' +import ReadableBlockstore from './readable-blockstore' + +export abstract class RepoStorage extends ReadableBlockstore { + abstract getHead(forUpdate?: boolean): Promise + abstract getCommitPath( + latest: CID, + earliest: CID | null, + ): Promise + abstract getBlocksForCommits( + commits: CID[], + ): Promise<{ [commit: string]: BlockMap }> + + abstract putBlock(cid: CID, block: Uint8Array): Promise + abstract putMany(blocks: BlockMap): Promise + abstract updateHead(cid: CID, prev: CID | null): Promise + abstract indexCommits(commit: CommitData[]): Promise + + async applyCommit(commit: CommitData): Promise { + await Promise.all([ + this.indexCommits([commit]), + this.updateHead(commit.commit, commit.prev), + ]) + } + + async getCommits( + latest: CID, + earliest: CID | null, + ): Promise { + const commitPath = await this.getCommitPath(latest, earliest) + if (!commitPath) return null + const blocksByCommit = await this.getBlocksForCommits(commitPath) + return commitPath.map((commit) => ({ + commit, + blocks: blocksByCommit[commit.toString()] || new BlockMap(), + })) + } +} + +export default RepoStorage diff --git a/packages/repo/src/storage/sync-storage.ts b/packages/repo/src/storage/sync-storage.ts new file mode 100644 index 00000000000..02a039ca57b --- /dev/null +++ b/packages/repo/src/storage/sync-storage.ts @@ -0,0 +1,35 @@ +import { CID } from 'multiformats/cid' +import BlockMap from '../block-map' +import ReadableBlockstore from './readable-blockstore' + +export class SyncStorage extends ReadableBlockstore { + constructor( + public staged: ReadableBlockstore, + public saved: ReadableBlockstore, + ) { + super() + } + + async getBytes(cid: CID): Promise { + const got = await this.staged.getBytes(cid) + if (got) return got + return this.saved.getBytes(cid) + } + + async getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }> { + const fromStaged = await this.staged.getBlocks(cids) + const fromSaved = await this.saved.getBlocks(fromStaged.missing) + const blocks = fromStaged.blocks + blocks.add(fromSaved.blocks) + return { + blocks, + missing: fromSaved.missing, + } + } + + async has(cid: CID): Promise { + return (await this.staged.has(cid)) || (await this.saved.has(cid)) + } +} + +export default SyncStorage diff --git a/packages/repo/src/sync.ts b/packages/repo/src/sync.ts deleted file mode 100644 index fd809b1eb71..00000000000 --- a/packages/repo/src/sync.ts +++ /dev/null @@ -1,38 +0,0 @@ -import * as auth from '@atproto/auth' -import { IpldStore } from './blockstore' -import { DataDiff } from './mst' -import Repo from './repo' -import * as verify from './verify' - -export const loadRepoFromCar = async ( - carBytes: Uint8Array, - blockstore: IpldStore, - verifier: auth.Verifier, -): Promise => { - const root = await blockstore.stageCar(carBytes) - const repo = await Repo.load(blockstore, root) - await verify.verifyUpdates(blockstore, null, repo.cid, verifier) - await blockstore.saveStaged() - return repo -} - -export const loadDiff = async ( - repo: Repo, - diffCar: Uint8Array, - verifier: auth.Verifier, -): Promise<{ repo: Repo; diff: DataDiff }> => { - const blockstore = repo.blockstore - const root = await blockstore.stageCar(diffCar) - const diff = await verify.verifyUpdates( - repo.blockstore, - repo.cid, - root, - verifier, - ) - const updatedRepo = await Repo.load(blockstore, root) - await blockstore.saveStaged() - return { - repo: updatedRepo, - diff, - } -} diff --git a/packages/repo/src/sync/consumer.ts b/packages/repo/src/sync/consumer.ts new file mode 100644 index 00000000000..b66bc477bee --- /dev/null +++ b/packages/repo/src/sync/consumer.ts @@ -0,0 +1,128 @@ +import { CID } from 'multiformats/cid' +import { DidResolver } from '@atproto/did-resolver' +import { MemoryBlockstore, RepoStorage } from '../storage' +import Repo from '../repo' +import * as verify from '../verify' +import * as util from '../util' +import { CommitData, RepoContents, WriteLog } from '../types' +import CidSet from '../cid-set' +import { MissingBlocksError } from '../error' + +// Checkouts +// ------------- + +export const loadCheckout = async ( + storage: RepoStorage, + repoCar: Uint8Array, + didResolver: DidResolver, +): Promise<{ root: CID; contents: RepoContents }> => { + const { root, blocks } = await util.readCar(repoCar) + const updateStorage = new MemoryBlockstore(blocks) + const checkout = await verify.verifyCheckout(updateStorage, root, didResolver) + + const checkoutBlocks = await updateStorage.getBlocks( + checkout.newCids.toList(), + ) + if (checkoutBlocks.missing.length > 0) { + throw new MissingBlocksError('sync', checkoutBlocks.missing) + } + await Promise.all([ + storage.putMany(checkoutBlocks.blocks), + storage.updateHead(root, null), + ]) + + return { + root, + contents: checkout.contents, + } +} + +// Diffs +// ------------- + +export const loadFullRepo = async ( + storage: RepoStorage, + repoCar: Uint8Array, + didResolver: DidResolver, +): Promise<{ root: CID; writeLog: WriteLog }> => { + const { root, blocks } = await util.readCar(repoCar) + const updateStorage = new MemoryBlockstore(blocks) + const updates = await verify.verifyFullHistory( + updateStorage, + root, + didResolver, + ) + + const [writeLog] = await Promise.all([ + persistUpdates(storage, updateStorage, updates), + storage.updateHead(root, null), + ]) + + return { + root, + writeLog, + } +} + +export const loadDiff = async ( + repo: Repo, + diffCar: Uint8Array, + didResolver: DidResolver, +): Promise<{ root: CID; writeLog: WriteLog }> => { + const { root, blocks } = await util.readCar(diffCar) + const updateStorage = new MemoryBlockstore(blocks) + const updates = await verify.verifyUpdates( + repo, + updateStorage, + root, + didResolver, + ) + + const [writeLog] = await Promise.all([ + persistUpdates(repo.storage, updateStorage, updates), + repo.storage.updateHead(root, repo.cid), + ]) + + return { + root, + writeLog, + } +} + +// Helpers +// ------------- + +export const persistUpdates = async ( + storage: RepoStorage, + updateStorage: RepoStorage, + updates: verify.VerifiedUpdate[], +): Promise => { + const newCids = new CidSet() + for (const update of updates) { + newCids.addSet(update.newCids) + } + + const diffBlocks = await updateStorage.getBlocks(newCids.toList()) + if (diffBlocks.missing.length > 0) { + throw new MissingBlocksError('sync', diffBlocks.missing) + } + const commits: CommitData[] = updates.map((update) => { + const forCommit = diffBlocks.blocks.getMany(update.newCids.toList()) + if (forCommit.missing.length > 0) { + throw new MissingBlocksError('sync', forCommit.missing) + } + return { + commit: update.commit, + prev: update.prev, + blocks: forCommit.blocks, + } + }) + + await storage.indexCommits(commits) + + return Promise.all( + updates.map((upd) => + util.diffToWriteDescripts(upd.diff, diffBlocks.blocks), + ), + ) +} diff --git a/packages/repo/src/sync/index.ts b/packages/repo/src/sync/index.ts new file mode 100644 index 00000000000..b00ebdaf008 --- /dev/null +++ b/packages/repo/src/sync/index.ts @@ -0,0 +1,2 @@ +export * from './consumer' +export * from './provider' diff --git a/packages/repo/src/sync/provider.ts b/packages/repo/src/sync/provider.ts new file mode 100644 index 00000000000..fa5aeb7712d --- /dev/null +++ b/packages/repo/src/sync/provider.ts @@ -0,0 +1,107 @@ +import { def, RecordPath } from '../types' +import { BlockWriter } from '@ipld/car/writer' +import { CID } from 'multiformats/cid' +import CidSet from '../cid-set' +import { MissingBlocksError } from '../error' +import { RepoStorage } from '../storage' +import { RepoRoot } from '../types' +import * as util from '../util' +import { MST } from '../mst' + +// Checkouts +// ------------- + +export const getCheckout = async ( + storage: RepoStorage, + cid: CID, +): Promise => { + return util.writeCar(cid, async (car: BlockWriter) => { + const root = await writeCommitAndRootToCar(storage, car, cid) + const meta = await storage.readObjAndBytes(root.meta, def.repoMeta) + await car.put({ cid: root.meta, bytes: meta.bytes }) + const mst = MST.load(storage, root.data) + await mst.writeToCarStream(car) + }) +} + +// Diffs +// ------------- + +export const getDiff = async ( + storage: RepoStorage, + latest: CID, + earliest: CID | null, +): Promise => { + return util.writeCar(latest, (car: BlockWriter) => { + return writeCommitsToCarStream(storage, car, latest, earliest) + }) +} + +export const getFullRepo = async ( + storage: RepoStorage, + cid: CID, +): Promise => { + return getDiff(storage, cid, null) +} + +export const writeCommitsToCarStream = async ( + storage: RepoStorage, + car: BlockWriter, + latest: CID, + earliest: CID | null, +): Promise => { + const commits = await storage.getCommits(latest, earliest) + if (commits === null) { + throw new Error('Could not find shared history') + } + if (commits.length === 0) return + for (const commit of commits) { + for (const entry of commit.blocks.entries()) { + await car.put(entry) + } + } +} + +// Narrow slices +// ------------- + +export const getRecords = async ( + storage: RepoStorage, + commit: CID, + paths: RecordPath[], +): Promise => { + return util.writeCar(commit, async (car: BlockWriter) => { + const root = await writeCommitAndRootToCar(storage, car, commit) + const mst = MST.load(storage, root.data) + const cidsForPaths = await Promise.all( + paths.map((p) => + mst.cidsForPath(util.formatDataKey(p.collection, p.rkey)), + ), + ) + const allCids = cidsForPaths.reduce((acc, cur) => { + return acc.addSet(new CidSet(cur)) + }, new CidSet()) + const found = await storage.getBlocks(allCids.toList()) + if (found.missing.length > 0) { + throw new MissingBlocksError('writeRecordsToCarStream', found.missing) + } + for (const block of found.blocks.entries()) { + await car.put(block) + } + }) +} + +// Helpers +// ------------- + +export const writeCommitAndRootToCar = async ( + storage: RepoStorage, + car: BlockWriter, + cid: CID, +): Promise => { + const commit = await storage.readObjAndBytes(cid, def.commit) + await car.put({ cid: cid, bytes: commit.bytes }) + const root = await storage.readObjAndBytes(commit.obj.root, def.repoRoot) + await car.put({ cid: commit.obj.root, bytes: root.bytes }) + return root.obj +} diff --git a/packages/repo/src/types.ts b/packages/repo/src/types.ts index 82ded11e279..340717b1343 100644 --- a/packages/repo/src/types.ts +++ b/packages/repo/src/types.ts @@ -1,8 +1,11 @@ import { z } from 'zod' import { BlockWriter } from '@ipld/car/writer' -import { def as common } from '@atproto/common' +import { schema as common, def as commonDef } from '@atproto/common' import { CID } from 'multiformats' -import { DataDiff } from './mst' +import BlockMap from './block-map' + +// Repo nodes +// --------------- const repoMeta = z.object({ did: z.string(), @@ -14,7 +17,7 @@ export type RepoMeta = z.infer const repoRoot = z.object({ meta: common.cid, prev: common.cid.nullable(), - auth_token: common.cid.nullable(), + auth_token: common.cid.nullable().optional(), data: common.cid, }) export type RepoRoot = z.infer @@ -25,64 +28,114 @@ const commit = z.object({ }) export type Commit = z.infer -export const cidCreateOp = z.object({ - action: z.literal('create'), - collection: z.string(), - rkey: z.string(), - cid: common.cid, -}) -export type CidCreateOp = z.infer +export const schema = { + ...common, + repoMeta, + repoRoot, + commit, +} -export const cidUpdateOp = z.object({ - action: z.literal('update'), - collection: z.string(), - rkey: z.string(), - cid: common.cid, -}) -export type CidUpdateOp = z.infer +export const def = { + ...commonDef, + repoMeta: { + name: 'repo meta', + schema: schema.repoMeta, + }, + repoRoot: { + name: 'repo root', + schema: schema.repoRoot, + }, + commit: { + name: 'commit', + schema: schema.commit, + }, +} -export const deleteOp = z.object({ - action: z.literal('delete'), - collection: z.string(), - rkey: z.string(), -}) -export type DeleteOp = z.infer +// Repo Operations +// --------------- -export const cidWriteOp = z.union([cidCreateOp, cidUpdateOp, deleteOp]) -export type CidWriteOp = z.infer +export enum WriteOpAction { + Create = 'create', + Update = 'update', + Delete = 'delete', +} -export const recordCreateOp = z.object({ - action: z.literal('create'), - collection: z.string(), - rkey: z.string(), - value: z.any(), -}) -export type RecordCreateOp = z.infer +export type RecordCreateOp = { + action: WriteOpAction.Create + collection: string + rkey: string + record: Record +} -export const recordUpdateOp = z.object({ - action: z.literal('update'), - collection: z.string(), - rkey: z.string(), - value: z.any(), -}) -export type RecordUpdateOp = z.infer +export type RecordUpdateOp = { + action: WriteOpAction.Update + collection: string + rkey: string + record: Record +} -export const recordWriteOp = z.union([recordCreateOp, recordUpdateOp, deleteOp]) -export type RecordWriteOp = z.infer +export type RecordDeleteOp = { + action: WriteOpAction.Delete + collection: string + rkey: string +} -export const def = { - ...common, - repoMeta, - repoRoot, - commit, - cidWriteOp, - recordWriteOp, +export type RecordWriteOp = RecordCreateOp | RecordUpdateOp | RecordDeleteOp + +export type RecordCreateDescript = RecordCreateOp & { + cid: CID } -export interface CarStreamable { - writeToCarStream(car: BlockWriter): Promise +export type RecordUpdateDescript = RecordUpdateOp & { + prev: CID + cid: CID +} + +export type RecordDeleteDescript = RecordDeleteOp & { + cid: CID +} + +export type RecordWriteDescript = + | RecordCreateDescript + | RecordUpdateDescript + | RecordDeleteDescript + +export type WriteLog = RecordWriteDescript[][] + +// Updates/Commits +// --------------- + +export type CommitBlockData = { + commit: CID + blocks: BlockMap } +export type CommitData = CommitBlockData & { + prev: CID | null +} + +export type RepoUpdate = CommitData & { + ops: RecordWriteOp[] +} + +export type RepoRecord = Record +export type CollectionContents = Record +export type RepoContents = Record + +export type RecordPath = { + collection: string + rkey: string +} + +export type RecordClaim = { + collection: string + rkey: string + record: RepoRecord | null +} + +// DataStores +// --------------- + export type DataValue = { key: string value: CID @@ -93,9 +146,9 @@ export interface DataStore { update(key: string, value: CID): Promise delete(key: string): Promise get(key: string): Promise - list(count: number, after?: string, before?: string): Promise + list(count?: number, after?: string, before?: string): Promise listWithPrefix(prefix: string, count?: number): Promise - diff(other: DataStore): Promise - stage(): Promise + getUnstoredBlocks(): Promise<{ root: CID; blocks: BlockMap }> writeToCarStream(car: BlockWriter): Promise + cidsForPath(key: string): Promise } diff --git a/packages/repo/src/util.ts b/packages/repo/src/util.ts index 2a01c88ecc0..3eff1027ea8 100644 --- a/packages/repo/src/util.ts +++ b/packages/repo/src/util.ts @@ -1,88 +1,192 @@ import { CID } from 'multiformats/cid' -import * as auth from '@atproto/auth' +import { CarReader } from '@ipld/car/reader' +import { BlockWriter, CarWriter } from '@ipld/car/writer' +import { Block as CarBlock } from '@ipld/car/api' +import { def, streamToArray, verifyCidForBytes } from '@atproto/common' import Repo from './repo' -import { DataDiff, MST } from './mst' -import { IpldStore } from './blockstore' -import { DataStore, RecordWriteOp, def } from './types' +import { MST } from './mst' +import DataDiff from './data-diff' +import { RepoStorage } from './storage' +import { + DataStore, + RecordCreateDescript, + RecordDeleteDescript, + RecordPath, + RecordUpdateDescript, + RecordWriteDescript, + WriteLog, + WriteOpAction, +} from './types' +import BlockMap from './block-map' +import { MissingBlocksError } from './error' +import * as parse from './parse' -export const ucanForOperation = async ( - prevData: DataStore, - newData: DataStore, - rootDid: string, - authStore: auth.AuthStore, -): Promise => { - const diff = await prevData.diff(newData) - const neededCaps = diff.neededCapabilities(rootDid) - const ucanForOp = await authStore.createUcanForCaps(rootDid, neededCaps, 30) - return auth.encodeUcan(ucanForOp) +export async function* verifyIncomingCarBlocks( + car: AsyncIterable, +): AsyncIterable { + for await (const block of car) { + await verifyCidForBytes(block.cid, block.bytes) + yield block + } } -export const getCommitPath = async ( - blockstore: IpldStore, - earliest: CID | null, - latest: CID, -): Promise => { - let curr: CID | null = latest - const path: CID[] = [] - while (curr !== null) { - path.push(curr) - const commit = await blockstore.get(curr, def.commit) - if (earliest && curr.equals(earliest)) { - return path.reverse() - } - const root = await blockstore.get(commit.root, def.repoRoot) - if (!earliest && root.prev === null) { - return path.reverse() - } - curr = root.prev +export const writeCar = async ( + root: CID, + fn: (car: BlockWriter) => Promise, +): Promise => { + const { writer, out } = CarWriter.create(root) + const bytes = streamToArray(out) + try { + await fn(writer) + } finally { + writer.close() } - return null + return bytes } -export const getWriteOpLog = async ( - blockstore: IpldStore, - earliest: CID | null, +export const readCar = async ( + bytes: Uint8Array, +): Promise<{ root: CID; blocks: BlockMap }> => { + const car = await CarReader.fromBytes(bytes) + const roots = await car.getRoots() + if (roots.length !== 1) { + throw new Error(`Expected one root, got ${roots.length}`) + } + const root = roots[0] + const blocks = new BlockMap() + for await (const block of verifyIncomingCarBlocks(car.blocks())) { + await blocks.set(block.cid, block.bytes) + } + return { + root, + blocks, + } +} + +export const getWriteLog = async ( + storage: RepoStorage, latest: CID, -): Promise => { - const commits = await getCommitPath(blockstore, earliest, latest) + earliest: CID | null, +): Promise => { + const commits = await storage.getCommitPath(latest, earliest) if (!commits) throw new Error('Could not find shared history') - const heads = await Promise.all(commits.map((c) => Repo.load(blockstore, c))) + const heads = await Promise.all(commits.map((c) => Repo.load(storage, c))) // Turn commit path into list of diffs - let prev: DataStore = await MST.create(blockstore) // Empty + let prev: DataStore = await MST.create(storage) // Empty const msts = heads.map((h) => h.data) const diffs: DataDiff[] = [] for (const mst of msts) { - diffs.push(await prev.diff(mst)) + diffs.push(await DataDiff.of(mst, prev)) prev = mst } + const fullDiff = collapseDiffs(diffs) + const diffBlocks = await storage.getBlocks(fullDiff.newCidList()) + if (diffBlocks.missing.length > 0) { + throw new MissingBlocksError('write op log', diffBlocks.missing) + } // Map MST diffs to write ops - return Promise.all(diffs.map((diff) => diffToWriteOps(blockstore, diff))) + return Promise.all( + diffs.map((diff) => diffToWriteDescripts(diff, diffBlocks.blocks)), + ) } -export const diffToWriteOps = ( - blockstore: IpldStore, +export const diffToWriteDescripts = ( diff: DataDiff, -): Promise => { + blocks: BlockMap, +): Promise => { return Promise.all([ ...diff.addList().map(async (add) => { - const { collection, rkey } = parseRecordKey(add.key) - const value = await blockstore.getUnchecked(add.cid) - return { action: 'create' as const, collection, rkey, value } + const { collection, rkey } = parseDataKey(add.key) + const value = await parse.getAndParse(blocks, add.cid, def.record) + return { + action: WriteOpAction.Create, + collection, + rkey, + cid: add.cid, + record: value.obj, + } as RecordCreateDescript }), ...diff.updateList().map(async (upd) => { - const { collection, rkey } = parseRecordKey(upd.key) - const value = await blockstore.getUnchecked(upd.cid) - return { action: 'update' as const, collection, rkey, value } + const { collection, rkey } = parseDataKey(upd.key) + const value = await parse.getAndParse(blocks, upd.cid, def.record) + return { + action: WriteOpAction.Update, + collection, + rkey, + cid: upd.cid, + prev: upd.prev, + record: value.obj, + } as RecordUpdateDescript }), ...diff.deleteList().map((del) => { - const { collection, rkey } = parseRecordKey(del.key) - return { action: 'delete' as const, collection, rkey } + const { collection, rkey } = parseDataKey(del.key) + return { + action: WriteOpAction.Delete, + collection, + rkey, + cid: del.cid, + } as RecordDeleteDescript }), ]) } -export const parseRecordKey = (key: string) => { +export const collapseWriteLog = (log: WriteLog): RecordWriteDescript[] => { + const creates: Record = {} + const updates: Record = {} + const deletes: Record = {} + for (const commit of log) { + for (const op of commit) { + const key = op.collection + '/' + op.rkey + if (op.action === WriteOpAction.Create) { + const del = deletes[key] + if (del) { + if (del.cid !== op.cid) { + updates[key] = { + ...op, + action: WriteOpAction.Update, + prev: del.cid, + } + } + delete deletes[key] + } else { + creates[key] = op + } + } else if (op.action === WriteOpAction.Update) { + updates[key] = op + delete creates[key] + delete deletes[key] + } else if (op.action === WriteOpAction.Delete) { + if (creates[key]) { + delete creates[key] + } else { + delete updates[key] + deletes[key] = op + } + } else { + throw new Error(`unknown action: ${op}`) + } + } + } + return [ + ...Object.values(creates), + ...Object.values(updates), + ...Object.values(deletes), + ] +} + +export const collapseDiffs = (diffs: DataDiff[]): DataDiff => { + return diffs.reduce((acc, cur) => { + acc.addDiff(cur) + return acc + }, new DataDiff()) +} + +export const parseDataKey = (key: string): RecordPath => { const parts = key.split('/') if (parts.length !== 2) throw new Error(`Invalid record key: ${key}`) return { collection: parts[0], rkey: parts[1] } } + +export const formatDataKey = (collection: string, rkey: string): string => { + return collection + '/' + rkey +} diff --git a/packages/repo/src/verify.ts b/packages/repo/src/verify.ts index 8939ac1c1cf..12f67f5d0db 100644 --- a/packages/repo/src/verify.ts +++ b/packages/repo/src/verify.ts @@ -1,62 +1,241 @@ import { CID } from 'multiformats/cid' -import * as auth from '@atproto/auth' -import { IpldStore } from './blockstore' +import { DidResolver } from '@atproto/did-resolver' +import * as crypto from '@atproto/crypto' +import { MemoryBlockstore, ReadableBlockstore, RepoStorage } from './storage' +import DataDiff from './data-diff' +import SyncStorage from './storage/sync-storage' +import ReadableRepo from './readable-repo' import Repo from './repo' -import { DataDiff } from './mst' +import CidSet from './cid-set' import * as util from './util' +import { RecordClaim, RepoContents } from './types' import { def } from './types' +import { MST } from './mst' +import { cidForCbor } from '@atproto/common' + +export type VerifiedCheckout = { + contents: RepoContents + newCids: CidSet +} + +export const verifyCheckout = async ( + storage: ReadableBlockstore, + head: CID, + didResolver: DidResolver, +): Promise => { + const repo = await ReadableRepo.load(storage, head) + const validSig = await didResolver.verifySignature( + repo.did, + repo.commit.root.bytes, + repo.commit.sig, + ) + if (!validSig) { + throw new RepoVerificationError( + `Invalid signature on commit: ${repo.cid.toString()}`, + ) + } + const diff = await DataDiff.of(repo.data, null) + const newCids = new CidSet([ + repo.cid, + repo.commit.root, + repo.root.meta, + ]).addSet(diff.newCids) + + const contents: RepoContents = {} + for (const add of diff.addList()) { + const { collection, rkey } = util.parseDataKey(add.key) + if (!contents[collection]) { + contents[collection] = {} + } + contents[collection][rkey] = await storage.readObj(add.cid, def.record) + } + + return { + contents, + newCids, + } +} + +export type VerifiedUpdate = { + commit: CID + prev: CID | null + diff: DataDiff + newCids: CidSet +} + +export const verifyFullHistory = async ( + storage: RepoStorage, + head: CID, + didResolver: DidResolver, +): Promise => { + const commitPath = await storage.getCommitPath(head, null) + if (commitPath === null) { + throw new RepoVerificationError('Could not find shared history') + } else if (commitPath.length < 1) { + throw new RepoVerificationError('Expected at least one commit') + } + const baseRepo = await Repo.load(storage, commitPath[0]) + const baseDiff = await DataDiff.of(baseRepo.data, null) + const baseRepoCids = new CidSet([ + baseRepo.cid, + baseRepo.commit.root, + baseRepo.root.meta, + ]).addSet(baseDiff.newCids) + const init: VerifiedUpdate = { + commit: baseRepo.cid, + prev: null, + diff: baseDiff, + newCids: baseRepoCids, + } + const updates = await verifyCommitPath( + baseRepo, + storage, + commitPath.slice(1), + didResolver, + ) + return [init, ...updates] +} export const verifyUpdates = async ( - blockstore: IpldStore, - earliest: CID | null, - latest: CID, - verifier: auth.Verifier, -): Promise => { - const commitPath = await util.getCommitPath(blockstore, earliest, latest) + repo: ReadableRepo, + updateStorage: RepoStorage, + updateRoot: CID, + didResolver: DidResolver, +): Promise => { + const commitPath = await updateStorage.getCommitPath(updateRoot, repo.cid) if (commitPath === null) { - throw new Error('Could not find shared history') + throw new RepoVerificationError('Could not find shared history') } - const fullDiff = new DataDiff() - if (commitPath.length === 0) return fullDiff - let prevRepo = await Repo.load(blockstore, commitPath[0]) - for (const commit of commitPath.slice(1)) { - const nextRepo = await Repo.load(blockstore, commit) - const diff = await prevRepo.data.diff(nextRepo.data) + const syncStorage = new SyncStorage(updateStorage, repo.storage) + return verifyCommitPath(repo, syncStorage, commitPath, didResolver) +} - if (!nextRepo.root.meta.equals(prevRepo.root.meta)) { - throw new Error('Not supported: repo metadata updated') - } +export const verifyCommitPath = async ( + baseRepo: ReadableRepo, + storage: ReadableBlockstore, + commitPath: CID[], + didResolver: DidResolver, +): Promise => { + const signingKey = await didResolver.resolveSigningKey(baseRepo.did) + const updates: VerifiedUpdate[] = [] + if (commitPath.length === 0) return updates + let prevRepo = baseRepo + for (const commit of commitPath) { + const nextRepo = await ReadableRepo.load(storage, commit) + const diff = await DataDiff.of(nextRepo.data, prevRepo.data) - let didForSignature: string - if (nextRepo.root.auth_token) { - // verify auth token covers all necessary writes - const encodedToken = await blockstore.get( - nextRepo.root.auth_token, - def.string, - ) - const token = await verifier.validateUcan(encodedToken) - const neededCaps = diff.neededCapabilities(prevRepo.did) - for (const cap of neededCaps) { - await verifier.verifyAtpUcan(token, prevRepo.did, cap) - } - didForSignature = token.payload.iss - } else { - didForSignature = prevRepo.did + if (!nextRepo.root.meta.equals(prevRepo.root.meta)) { + throw new RepoVerificationError('Not supported: repo metadata updated') } // verify signature matches repo root + auth token - // const commit = await toRepo.getCommit() - const validSig = await verifier.verifySignature( - didForSignature, + const validSig = await crypto.verifySignature( + signingKey, nextRepo.commit.root.bytes, nextRepo.commit.sig, ) if (!validSig) { - throw new Error(`Invalid signature on commit: ${nextRepo.cid.toString()}`) + throw new RepoVerificationError( + `Invalid signature on commit: ${nextRepo.cid.toString()}`, + ) } - fullDiff.addDiff(diff) + const newCids = new CidSet([nextRepo.cid, nextRepo.commit.root]).addSet( + diff.newCids, + ) + + updates.push({ + commit: nextRepo.cid, + prev: prevRepo.cid, + diff, + newCids, + }) prevRepo = nextRepo } - return fullDiff + return updates } + +export const verifyProofs = async ( + did: string, + proofs: Uint8Array, + claims: RecordClaim[], + didResolver: DidResolver, +): Promise<{ verified: RecordClaim[]; unverified: RecordClaim[] }> => { + const car = await util.readCar(proofs) + const blockstore = new MemoryBlockstore(car.blocks) + const commit = await blockstore.readObj(car.root, def.commit) + const validSig = await didResolver.verifySignature( + did, + commit.root.bytes, + commit.sig, + ) + if (!validSig) { + throw new RepoVerificationError( + `Invalid signature on commit: ${car.root.toString()}`, + ) + } + const root = await blockstore.readObj(commit.root, def.repoRoot) + const mst = MST.load(blockstore, root.data) + const verified: RecordClaim[] = [] + const unverified: RecordClaim[] = [] + for (const claim of claims) { + const found = await mst.get( + util.formatDataKey(claim.collection, claim.rkey), + ) + const record = found ? await blockstore.readObj(found, def.record) : null + if (claim.record === null) { + if (record === null) { + verified.push(claim) + } else { + unverified.push(claim) + } + } else { + const expected = await cidForCbor(claim.record) + if (expected.equals(found)) { + verified.push(claim) + } else { + unverified.push(claim) + } + } + } + return { verified, unverified } +} + +export const verifyRecords = async ( + did: string, + proofs: Uint8Array, + didResolver: DidResolver, +): Promise => { + const car = await util.readCar(proofs) + const blockstore = new MemoryBlockstore(car.blocks) + const commit = await blockstore.readObj(car.root, def.commit) + const validSig = await didResolver.verifySignature( + did, + commit.root.bytes, + commit.sig, + ) + if (!validSig) { + throw new RepoVerificationError( + `Invalid signature on commit: ${car.root.toString()}`, + ) + } + const root = await blockstore.readObj(commit.root, def.repoRoot) + const mst = MST.load(blockstore, root.data) + + const records: RecordClaim[] = [] + const leaves = await mst.reachableLeaves() + for (const leaf of leaves) { + const { collection, rkey } = util.parseDataKey(leaf.key) + const record = await blockstore.attemptRead(leaf.value, def.record) + if (record) { + records.push({ + collection, + rkey, + record: record.obj, + }) + } + } + return records +} + +export class RepoVerificationError extends Error {} diff --git a/packages/repo/tests/_util.ts b/packages/repo/tests/_util.ts index 349e9500b93..efb8a142184 100644 --- a/packages/repo/tests/_util.ts +++ b/packages/repo/tests/_util.ts @@ -1,20 +1,32 @@ +import fs from 'fs' import { CID } from 'multiformats' -import { TID } from '@atproto/common' -import * as auth from '@atproto/auth' -import IpldStore from '../src/blockstore/ipld-store' +import { TID, dataToCborBlock } from '@atproto/common' +import * as crypto from '@atproto/crypto' import { Repo } from '../src/repo' -import { MemoryBlockstore } from '../src/blockstore' -import { DataDiff, MST } from '../src/mst' -import fs from 'fs' -import { RecordWriteOp } from '../src' +import { RepoStorage } from '../src/storage' +import { MST } from '../src/mst' +import { + BlockMap, + collapseWriteLog, + CollectionContents, + RecordWriteOp, + RepoContents, + RecordPath, + RepoRoot, + WriteLog, + WriteOpAction, + RecordClaim, +} from '../src' +import { Keypair } from '@atproto/crypto' type IdMapping = Record -const fakeStore = new MemoryBlockstore() - -export const randomCid = async (store: IpldStore = fakeStore): Promise => { - const str = randomStr(50) - return store.stage({ test: str }) +export const randomCid = async (storage?: RepoStorage): Promise => { + const block = await dataToCborBlock({ test: randomStr(50) }) + if (storage) { + await storage.putBlock(block.cid, block.bytes) + } + return block.cid } export const generateBulkTids = (count: number): TID[] => { @@ -27,7 +39,7 @@ export const generateBulkTids = (count: number): TID[] => { export const generateBulkTidMapping = async ( count: number, - blockstore: IpldStore = fakeStore, + blockstore?: RepoStorage, ): Promise => { const ids = generateBulkTids(count) const obj: IdMapping = {} @@ -76,32 +88,29 @@ export const generateObject = (): Record => { export const testCollections = ['com.example.posts', 'com.example.likes'] -export type CollectionData = Record -export type RepoData = Record - export const fillRepo = async ( repo: Repo, - authStore: auth.AuthStore, + keypair: crypto.Keypair, itemsPerCollection: number, -): Promise<{ repo: Repo; data: RepoData }> => { - const repoData: RepoData = {} +): Promise<{ repo: Repo; data: RepoContents }> => { + const repoData: RepoContents = {} const writes: RecordWriteOp[] = [] for (const collName of testCollections) { - const collData: CollectionData = {} + const collData: CollectionContents = {} for (let i = 0; i < itemsPerCollection; i++) { const object = generateObject() const rkey = TID.nextStr() collData[rkey] = object writes.push({ - action: 'create', + action: WriteOpAction.Create, collection: collName, rkey, - value: object, + record: object, }) } repoData[collName] = collData } - const updated = await repo.stageUpdate(writes).createCommit(authStore) + const updated = await repo.applyCommit(writes, keypair) return { repo: updated, data: repoData, @@ -110,17 +119,16 @@ export const fillRepo = async ( export const editRepo = async ( repo: Repo, - prevData: RepoData, - authStore: auth.AuthStore, + prevData: RepoContents, + keypair: crypto.Keypair, params: { adds?: number updates?: number deletes?: number }, -): Promise<{ repo: Repo; data: RepoData }> => { +): Promise<{ repo: Repo; data: RepoContents }> => { const { adds = 0, updates = 0, deletes = 0 } = params - const repoData: RepoData = {} - const writes: RecordWriteOp[] = [] + const repoData: RepoContents = {} for (const collName of testCollections) { const collData = prevData[collName] const shuffled = shuffle(Object.entries(collData)) @@ -129,94 +137,133 @@ export const editRepo = async ( const object = generateObject() const rkey = TID.nextStr() collData[rkey] = object - writes.push({ - action: 'create', - collection: collName, - rkey, - value: object, - }) + repo = await repo.applyCommit( + { + action: WriteOpAction.Create, + collection: collName, + rkey, + record: object, + }, + keypair, + ) } const toUpdate = shuffled.slice(0, updates) for (let i = 0; i < toUpdate.length; i++) { const object = generateObject() const rkey = toUpdate[i][0] - writes.push({ - action: 'update', - collection: collName, - rkey, - value: object, - }) + repo = await repo.applyCommit( + { + action: WriteOpAction.Update, + collection: collName, + rkey, + record: object, + }, + keypair, + ) collData[rkey] = object } const toDelete = shuffled.slice(updates, deletes) for (let i = 0; i < toDelete.length; i++) { const rkey = toDelete[i][0] - writes.push({ - action: 'delete', - collection: collName, - rkey, - }) + repo = await repo.applyCommit( + { + action: WriteOpAction.Delete, + collection: collName, + rkey, + }, + keypair, + ) delete collData[rkey] } repoData[collName] = collData } - const updated = await repo.stageUpdate(writes).createCommit(authStore) return { - repo: updated, + repo, data: repoData, } } -export const checkRepo = async (repo: Repo, data: RepoData): Promise => { - for (const collName of Object.keys(data)) { - const collData = data[collName] - for (const rkey of Object.keys(collData)) { - const record = await repo.getRecord(collName, rkey) - expect(record).toEqual(collData[rkey]) +export const verifyRepoDiff = async ( + writeLog: WriteLog, + before: RepoContents, + after: RepoContents, +): Promise => { + const getVal = (op: RecordWriteOp, data: RepoContents) => { + return (data[op.collection] || {})[op.rkey] + } + const ops = await collapseWriteLog(writeLog) + + for (const op of ops) { + if (op.action === WriteOpAction.Create) { + expect(getVal(op, before)).toBeUndefined() + expect(getVal(op, after)).toEqual(op.record) + } else if (op.action === WriteOpAction.Update) { + expect(getVal(op, before)).toBeDefined() + expect(getVal(op, after)).toEqual(op.record) + } else if (op.action === WriteOpAction.Delete) { + expect(getVal(op, before)).toBeDefined() + expect(getVal(op, after)).toBeUndefined() + } else { + throw new Error('unexpected op type') } } } -export const checkRepoDiff = async ( - diff: DataDiff, - before: RepoData, - after: RepoData, -): Promise => { - const getObjectCid = async ( - key: string, - data: RepoData, - ): Promise => { - const parts = key.split('/') - const collection = parts[0] - const obj = (data[collection] || {})[parts[1]] - return obj === undefined ? undefined : fakeStore.stage(obj) +export const contentsToClaims = (contents: RepoContents): RecordClaim[] => { + const claims: RecordClaim[] = [] + for (const coll of Object.keys(contents)) { + for (const rkey of Object.keys(contents[coll])) { + claims.push({ + collection: coll, + rkey: rkey, + record: contents[coll][rkey], + }) + } } + return claims +} - for (const add of diff.addList()) { - const beforeCid = await getObjectCid(add.key, before) - const afterCid = await getObjectCid(add.key, after) - - expect(beforeCid).toBeUndefined() - expect(afterCid).toEqual(add.cid) - } +export const pathsForOps = (ops: RecordWriteOp[]): RecordPath[] => + ops.map((op) => ({ collection: op.collection, rkey: op.rkey })) - for (const update of diff.updateList()) { - const beforeCid = await getObjectCid(update.key, before) - const afterCid = await getObjectCid(update.key, after) +export const saveMst = async (storage: RepoStorage, mst: MST): Promise => { + const diff = await mst.getUnstoredBlocks() + await storage.putMany(diff.blocks) + return diff.root +} - expect(beforeCid).toEqual(update.prev) - expect(afterCid).toEqual(update.cid) +// Creating repo +// ------------------- +export const addBadCommit = async ( + repo: Repo, + keypair: Keypair, +): Promise => { + const obj = generateObject() + const blocks = new BlockMap() + const cid = await blocks.add(obj) + const updatedData = await repo.data.add(`com.example.test/${TID.next()}`, cid) + const unstoredData = await updatedData.getUnstoredBlocks() + blocks.addMap(unstoredData.blocks) + const root: RepoRoot = { + meta: repo.root.meta, + prev: repo.cid, + data: unstoredData.root, } - - for (const del of diff.deleteList()) { - const beforeCid = await getObjectCid(del.key, before) - const afterCid = await getObjectCid(del.key, after) - - expect(beforeCid).toEqual(del.cid) - expect(afterCid).toBeUndefined() + const rootCid = await blocks.add(root) + // we generate a bad sig by signing the data cid instead of root cid + const commit = { + root: rootCid, + sig: await keypair.sign(unstoredData.root.bytes), } + const commitCid = await blocks.add(commit) + await repo.storage.applyCommit({ + commit: commitCid, + prev: repo.cid, + blocks: blocks, + }) + return await Repo.load(repo.storage, commitCid) } // Logging diff --git a/packages/repo/tests/mst.test.ts b/packages/repo/tests/mst.test.ts index b459f22a5c6..346861f8ce0 100644 --- a/packages/repo/tests/mst.test.ts +++ b/packages/repo/tests/mst.test.ts @@ -1,7 +1,8 @@ -import { MST, DataAdd, DataUpdate, DataDelete } from '../src/mst' +import { MST } from '../src/mst' +import DataDiff, { DataAdd, DataUpdate, DataDelete } from '../src/data-diff' import { countPrefixLen } from '../src/mst/util' -import { MemoryBlockstore } from '../src/blockstore' +import { MemoryBlockstore } from '../src/storage' import * as util from './_util' import { CID } from 'multiformats' @@ -90,8 +91,8 @@ describe('Merkle Search Tree', () => { }) it('saves and loads from blockstore', async () => { - const cid = await mst.stage() - const loaded = await MST.load(blockstore, cid) + const root = await util.saveMst(blockstore, mst) + const loaded = await MST.load(blockstore, root) const origNodes = await mst.allNodes() const loadedNodes = await loaded.allNodes() expect(origNodes.length).toBe(loadedNodes.length) @@ -131,7 +132,7 @@ describe('Merkle Search Tree', () => { expectedDels[entry[0]] = { key: entry[0], cid: entry[1] } } - const diff = await mst.diff(toDiff) + const diff = await DataDiff.of(toDiff, mst) expect(diff.addList().length).toBe(100) expect(diff.updateList().length).toBe(100) @@ -175,7 +176,7 @@ describe('Merkle Search Tree', () => { const layer = await mst.getLayer() expect(layer).toBe(1) mst = await mst.delete(layer1) - const root = await mst.stage() + const root = await util.saveMst(blockstore, mst) const loaded = MST.load(blockstore, root) const loadedLayer = await loaded.getLayer() expect(loadedLayer).toBe(0) @@ -224,7 +225,7 @@ describe('Merkle Search Tree', () => { const layer = await mst.getLayer() expect(layer).toBe(2) - const root = await mst.stage() + const root = await util.saveMst(blockstore, mst) mst = MST.load(blockstore, root, { fanout: 32 }) const allTids = [...layer0, ...layer1, layer2] @@ -256,7 +257,7 @@ describe('Merkle Search Tree', () => { } mst = await mst.add(layer2, cid) - const root = await mst.stage() + const root = await util.saveMst(blockstore, mst) mst = MST.load(blockstore, root, { fanout: 32 }) const layer = await mst.getLayer() diff --git a/packages/repo/tests/repo.test.ts b/packages/repo/tests/repo.test.ts index 896c9585c65..631c180b94d 100644 --- a/packages/repo/tests/repo.test.ts +++ b/packages/repo/tests/repo.test.ts @@ -1,28 +1,27 @@ -import * as auth from '@atproto/auth' - +import * as crypto from '@atproto/crypto' import { Repo } from '../src/repo' -import { MemoryBlockstore } from '../src/blockstore' +import { MemoryBlockstore } from '../src/storage' import * as util from './_util' import { TID } from '@atproto/common' +import { RepoContents, WriteOpAction } from '../src' +import { Secp256k1Keypair } from '@atproto/crypto' describe('Repo', () => { - const verifier = new auth.Verifier() const collName = 'com.example.posts' - let blockstore: MemoryBlockstore - let authStore: auth.AuthStore + let storage: MemoryBlockstore + let keypair: crypto.Keypair let repo: Repo - let repoData: util.RepoData + let repoData: RepoContents it('creates repo', async () => { - blockstore = new MemoryBlockstore() - authStore = await verifier.createTempAuthStore() - await authStore.claimFull() - repo = await Repo.create(blockstore, await authStore.did(), authStore) + storage = new MemoryBlockstore() + keypair = await Secp256k1Keypair.create() + repo = await Repo.create(storage, keypair.did(), keypair) }) it('has proper metadata', async () => { - expect(repo.meta.did).toEqual(await authStore.did()) + expect(repo.meta.did).toEqual(keypair.did()) expect(repo.meta.version).toBe(1) expect(repo.meta.datastore).toBe('mst') }) @@ -30,61 +29,66 @@ describe('Repo', () => { it('does basic operations', async () => { const rkey = TID.nextStr() const record = util.generateObject() - repo = await repo - .stageUpdate({ - action: 'create', + repo = await repo.applyCommit( + { + action: WriteOpAction.Create, collection: collName, - rkey: rkey, - value: record, - }) - .createCommit(authStore) + rkey, + record, + }, + keypair, + ) let got = await repo.getRecord(collName, rkey) expect(got).toEqual(record) const updatedRecord = util.generateObject() - repo = await repo - .stageUpdate({ - action: 'update', + repo = await repo.applyCommit( + { + action: WriteOpAction.Update, collection: collName, - rkey: rkey, - value: updatedRecord, - }) - .createCommit(authStore) + rkey, + record: updatedRecord, + }, + keypair, + ) got = await repo.getRecord(collName, rkey) expect(got).toEqual(updatedRecord) - repo = await repo - .stageUpdate({ - action: 'delete', + repo = await repo.applyCommit( + { + action: WriteOpAction.Delete, collection: collName, rkey: rkey, - }) - .createCommit(authStore) + }, + keypair, + ) got = await repo.getRecord(collName, rkey) expect(got).toBeNull() }) it('adds content collections', async () => { - const filled = await util.fillRepo(repo, authStore, 100) + const filled = await util.fillRepo(repo, keypair, 100) repo = filled.repo repoData = filled.data - await util.checkRepo(repo, repoData) + const contents = await repo.getContents() + expect(contents).toEqual(repoData) }) it('edits and deletes content', async () => { - const edited = await util.editRepo(repo, repoData, authStore, { + const edited = await util.editRepo(repo, repoData, keypair, { adds: 20, updates: 20, deletes: 20, }) repo = edited.repo - await util.checkRepo(repo, repoData) + const contents = await repo.getContents() + expect(contents).toEqual(repoData) }) it('adds a valid signature to commit', async () => { const commit = await repo.commit - const verified = await verifier.verifySignature( + const verified = await crypto.verifySignature( repo.did, commit.root.bytes, commit.sig, @@ -93,14 +97,15 @@ describe('Repo', () => { }) it('sets correct DID', async () => { - expect(repo.did).toEqual(await authStore.did()) + expect(repo.did).toEqual(await keypair.did()) }) it('loads from blockstore', async () => { - const reloadedRepo = await Repo.load(blockstore, repo.cid) + const reloadedRepo = await Repo.load(storage, repo.cid) - await util.checkRepo(reloadedRepo, repoData) - expect(repo.meta.did).toEqual(await authStore.did()) + const contents = await reloadedRepo.getContents() + expect(contents).toEqual(repoData) + expect(repo.meta.did).toEqual(keypair.did()) expect(repo.meta.version).toBe(1) expect(repo.meta.datastore).toBe('mst') }) diff --git a/packages/repo/tests/sync.test.ts b/packages/repo/tests/sync.test.ts deleted file mode 100644 index b67954f0d67..00000000000 --- a/packages/repo/tests/sync.test.ts +++ /dev/null @@ -1,129 +0,0 @@ -import * as auth from '@atproto/auth' -import { TID } from '@atproto/common' -import { Repo, RepoRoot, verifyUpdates, ucanForOperation } from '../src' -import { MemoryBlockstore } from '../src/blockstore' -import * as sync from '../src/sync' - -import * as util from './_util' - -describe('Sync', () => { - const verifier = new auth.Verifier() - - let aliceBlockstore: MemoryBlockstore, bobBlockstore: MemoryBlockstore - let aliceRepo: Repo - let aliceAuth: auth.AuthStore - let repoData: util.RepoData - - beforeAll(async () => { - aliceBlockstore = new MemoryBlockstore() - aliceAuth = await verifier.createTempAuthStore() - await aliceAuth.claimFull() - aliceRepo = await Repo.create( - aliceBlockstore, - await aliceAuth.did(), - aliceAuth, - ) - bobBlockstore = new MemoryBlockstore() - }) - - it('syncs an empty repo', async () => { - const car = await aliceRepo.getFullHistory() - const repoBob = await sync.loadRepoFromCar(car, bobBlockstore, verifier) - const data = await repoBob.data.list(10) - expect(data.length).toBe(0) - }) - - let bobRepo: Repo - - it('syncs a repo that is starting from scratch', async () => { - const filled = await util.fillRepo(aliceRepo, aliceAuth, 100) - aliceRepo = filled.repo - repoData = filled.data - await aliceRepo.getFullHistory() - - const car = await aliceRepo.getFullHistory() - bobRepo = await sync.loadRepoFromCar(car, bobBlockstore, verifier) - const diff = await verifyUpdates(bobBlockstore, null, bobRepo.cid, verifier) - await util.checkRepo(bobRepo, repoData) - await util.checkRepoDiff(diff, {}, repoData) - }) - - it('syncs a repo that is behind', async () => { - // add more to alice's repo & have bob catch up - const beforeData = JSON.parse(JSON.stringify(repoData)) - const edited = await util.editRepo(aliceRepo, repoData, aliceAuth, { - adds: 20, - updates: 20, - deletes: 20, - }) - aliceRepo = edited.repo - repoData = edited.data - const diffCar = await aliceRepo.getDiffCar(bobRepo.cid) - const loaded = await sync.loadDiff(bobRepo, diffCar, verifier) - await util.checkRepo(loaded.repo, repoData) - await util.checkRepoDiff(loaded.diff, beforeData, repoData) - }) - - it('throws an error on invalid UCANs', async () => { - const obj = util.generateObject() - const cid = await aliceBlockstore.stage(obj) - const updatedData = await aliceRepo.data.add( - `com.example.test/${TID.next()}`, - cid, - ) - // we create an unrelated token for bob & try to permission alice's repo commit with it - const bobAuth = await verifier.createTempAuthStore() - const badUcan = await bobAuth.claimFull() - const auth_token = await aliceBlockstore.stage(auth.encodeUcan(badUcan)) - const dataCid = await updatedData.stage() - const root: RepoRoot = { - meta: aliceRepo.root.meta, - prev: aliceRepo.cid, - auth_token, - data: dataCid, - } - const rootCid = await aliceBlockstore.stage(root) - const commit = { - root: rootCid, - sig: await aliceAuth.sign(rootCid.bytes), - } - const commitCid = await aliceBlockstore.stage(commit) - const badAliceRepo = await Repo.load(aliceBlockstore, commitCid) - const diffCar = await badAliceRepo.getDiffCar(bobRepo.cid) - await expect(sync.loadDiff(bobRepo, diffCar, verifier)).rejects.toThrow() - // await aliceBlockstore.clearStaged() - }) - - it('throws on a bad signature', async () => { - const obj = util.generateObject() - const cid = await aliceBlockstore.stage(obj) - const updatedData = await aliceRepo.data.add( - `com.example.test/${TID.next()}`, - cid, - ) - const authToken = await ucanForOperation( - aliceRepo.data, - updatedData, - aliceRepo.did, - aliceAuth, - ) - const authCid = await aliceBlockstore.stage(authToken) - const dataCid = await updatedData.stage() - const root: RepoRoot = { - meta: aliceRepo.root.meta, - prev: aliceRepo.cid, - auth_token: authCid, - data: dataCid, - } - const rootCid = await aliceBlockstore.stage(root) - // we generated a bad sig by signing the data cid instead of root cid - const commit = { - root: rootCid, - sig: await aliceAuth.sign(dataCid.bytes), - } - const commitCid = await aliceBlockstore.stage(commit) - const badAliceRepo = await Repo.load(aliceBlockstore, commitCid) - const diffCar = await badAliceRepo.getDiffCar(bobRepo.cid) - await expect(sync.loadDiff(bobRepo, diffCar, verifier)).rejects.toThrow() - }) -}) diff --git a/packages/repo/tests/sync/checkout.test.ts b/packages/repo/tests/sync/checkout.test.ts new file mode 100644 index 00000000000..edd911770a1 --- /dev/null +++ b/packages/repo/tests/sync/checkout.test.ts @@ -0,0 +1,56 @@ +import * as crypto from '@atproto/crypto' +import { DidResolver } from '@atproto/did-resolver' +import { Repo, RepoContents, RepoVerificationError } from '../../src' +import { MemoryBlockstore } from '../../src/storage' +import * as sync from '../../src/sync' + +import * as util from '../_util' + +describe('Checkout Sync', () => { + let storage: MemoryBlockstore + let syncStorage: MemoryBlockstore + let repo: Repo + let keypair: crypto.Keypair + let repoData: RepoContents + const didResolver = new DidResolver() + + beforeAll(async () => { + storage = new MemoryBlockstore() + keypair = await crypto.Secp256k1Keypair.create() + repo = await Repo.create(storage, keypair.did(), keypair) + syncStorage = new MemoryBlockstore() + const filled = await util.fillRepo(repo, keypair, 20) + repo = filled.repo + repoData = filled.data + }) + + it('sync a non-historical repo checkout', async () => { + const checkoutCar = await sync.getCheckout(storage, repo.cid) + const checkout = await sync.loadCheckout( + syncStorage, + checkoutCar, + didResolver, + ) + const checkoutRepo = await Repo.load(syncStorage, checkout.root) + const contents = await checkoutRepo.getContents() + expect(contents).toEqual(repoData) + expect(checkout.contents).toEqual(repoData) + }) + + it('does not sync unneeded blocks during checkout', async () => { + const commitPath = await storage.getCommitPath(repo.cid, null) + if (!commitPath) { + throw new Error('Could not get commitPath') + } + const hasGenesisCommit = await syncStorage.has(commitPath[0]) + expect(hasGenesisCommit).toBeFalsy() + }) + + it('throws on a bad signature', async () => { + const badRepo = await util.addBadCommit(repo, keypair) + const checkoutCar = await sync.getCheckout(storage, badRepo.cid) + await expect( + sync.loadCheckout(syncStorage, checkoutCar, didResolver), + ).rejects.toThrow(RepoVerificationError) + }) +}) diff --git a/packages/repo/tests/sync/diff.test.ts b/packages/repo/tests/sync/diff.test.ts new file mode 100644 index 00000000000..4a467fe0436 --- /dev/null +++ b/packages/repo/tests/sync/diff.test.ts @@ -0,0 +1,72 @@ +import * as crypto from '@atproto/crypto' +import { DidResolver } from '@atproto/did-resolver' +import { Repo, RepoContents } from '../../src' +import { MemoryBlockstore } from '../../src/storage' +import * as sync from '../../src/sync' + +import * as util from '../_util' + +describe('Diff Sync', () => { + let storage: MemoryBlockstore + let syncStorage: MemoryBlockstore + let repo: Repo + let keypair: crypto.Keypair + let repoData: RepoContents + const didResolver = new DidResolver() + + beforeAll(async () => { + storage = new MemoryBlockstore() + keypair = await crypto.Secp256k1Keypair.create() + repo = await Repo.create(storage, keypair.did(), keypair) + syncStorage = new MemoryBlockstore() + }) + + let syncRepo: Repo + + it('syncs an empty repo', async () => { + const car = await sync.getFullRepo(storage, repo.cid) + const loaded = await sync.loadFullRepo(syncStorage, car, didResolver) + syncRepo = await Repo.load(syncStorage, loaded.root) + const data = await syncRepo.data.list(10) + expect(data.length).toBe(0) + }) + + it('syncs a repo that is starting from scratch', async () => { + const filled = await util.fillRepo(repo, keypair, 100) + repo = filled.repo + repoData = filled.data + + const car = await sync.getFullRepo(storage, repo.cid) + const loaded = await sync.loadFullRepo(syncStorage, car, didResolver) + syncRepo = await Repo.load(syncStorage, loaded.root) + const contents = await syncRepo.getContents() + expect(contents).toEqual(repoData) + await util.verifyRepoDiff(loaded.writeLog, {}, repoData) + }) + + it('syncs a repo that is behind', async () => { + // add more to providers's repo & have consumer catch up + const beforeData = structuredClone(repoData) + const edited = await util.editRepo(repo, repoData, keypair, { + adds: 20, + updates: 20, + deletes: 20, + }) + repo = edited.repo + repoData = edited.data + const diffCar = await sync.getDiff(storage, repo.cid, syncRepo.cid) + const loaded = await sync.loadDiff(syncRepo, diffCar, didResolver) + syncRepo = await Repo.load(syncStorage, loaded.root) + const contents = await syncRepo.getContents() + expect(contents).toEqual(repoData) + await util.verifyRepoDiff(loaded.writeLog, beforeData, repoData) + }) + + it('throws on a bad signature', async () => { + const badRepo = await util.addBadCommit(repo, keypair) + const diffCar = await sync.getDiff(storage, badRepo.cid, syncRepo.cid) + await expect(sync.loadDiff(syncRepo, diffCar, didResolver)).rejects.toThrow( + 'Invalid signature on commit', + ) + }) +}) diff --git a/packages/repo/tests/sync/narrow.test.ts b/packages/repo/tests/sync/narrow.test.ts new file mode 100644 index 00000000000..50272359b6f --- /dev/null +++ b/packages/repo/tests/sync/narrow.test.ts @@ -0,0 +1,145 @@ +import { TID } from '@atproto/common' +import * as crypto from '@atproto/crypto' +import { DidResolver } from '@atproto/did-resolver' +import { RecordClaim, Repo, RepoContents } from '../../src' +import { MemoryBlockstore } from '../../src/storage' +import * as verify from '../../src/verify' +import * as sync from '../../src/sync' + +import * as util from '../_util' + +describe('Narrow Sync', () => { + let storage: MemoryBlockstore + let repo: Repo + let keypair: crypto.Keypair + let repoData: RepoContents + const didResolver = new DidResolver() + + beforeAll(async () => { + storage = new MemoryBlockstore() + keypair = await crypto.Secp256k1Keypair.create() + repo = await Repo.create(storage, keypair.did(), keypair) + const filled = await util.fillRepo(repo, keypair, 5) + repo = filled.repo + repoData = filled.data + }) + + const getProofs = async (claims: RecordClaim[]) => { + return sync.getRecords(storage, repo.cid, claims) + } + + const doVerify = (proofs: Uint8Array, claims: RecordClaim[]) => { + return verify.verifyProofs(repo.did, proofs, claims, didResolver) + } + + it('verifies valid records', async () => { + const claims = util.contentsToClaims(repoData) + const proofs = await getProofs(claims) + const results = await doVerify(proofs, claims) + expect(results.verified.length).toBeGreaterThan(0) + expect(results.verified).toEqual(claims) + expect(results.unverified.length).toBe(0) + }) + + it('verifies record nonexistence', async () => { + const claims: RecordClaim[] = [ + { + collection: util.testCollections[0], + rkey: TID.nextStr(), // does not exist + record: null, + }, + ] + const proofs = await getProofs(claims) + const results = await doVerify(proofs, claims) + expect(results.verified.length).toBeGreaterThan(0) + expect(results.verified).toEqual(claims) + expect(results.unverified.length).toBe(0) + }) + + it('does not verify a record that doesnt exist', async () => { + const realClaims = util.contentsToClaims(repoData) + const claims: RecordClaim[] = [ + { + ...realClaims[0], + rkey: TID.nextStr(), + }, + ] + const proofs = await getProofs(claims) + const results = await doVerify(proofs, claims) + expect(results.verified.length).toBe(0) + expect(results.unverified.length).toBeGreaterThan(0) + expect(results.unverified).toEqual(claims) + }) + + it('does not verify an invalid record at a real path', async () => { + const realClaims = util.contentsToClaims(repoData) + const claims: RecordClaim[] = [ + { + ...realClaims[0], + record: util.generateObject(), + }, + ] + const proofs = await getProofs(claims) + const results = await doVerify(proofs, claims) + expect(results.verified.length).toBe(0) + expect(results.unverified.length).toBeGreaterThan(0) + expect(results.unverified).toEqual(claims) + }) + + it('does not verify a delete where the record does exist', async () => { + const realClaims = util.contentsToClaims(repoData) + const claims: RecordClaim[] = [ + { + collection: realClaims[0].collection, + rkey: realClaims[0].rkey, + record: null, + }, + ] + const proofs = await getProofs(claims) + const results = await doVerify(proofs, claims) + expect(results.verified.length).toBe(0) + expect(results.unverified.length).toBeGreaterThan(0) + expect(results.unverified).toEqual(claims) + }) + + it('can determine record proofs from car file', async () => { + const possible = util.contentsToClaims(repoData) + const claims = [ + //random sampling of records + possible[0], + possible[4], + possible[5], + possible[8], + ] + const proofs = await getProofs(claims) + const records = await verify.verifyRecords(repo.did, proofs, didResolver) + for (const record of records) { + const foundClaim = claims.find( + (claim) => + claim.collection === record.collection && claim.rkey === record.rkey, + ) + if (!foundClaim) { + throw new Error('Could not find record for claim') + } + expect(foundClaim.record).toEqual( + repoData[record.collection][record.rkey], + ) + } + }) + + it('verifyRecords throws on a bad signature', async () => { + const badRepo = await util.addBadCommit(repo, keypair) + const claims = util.contentsToClaims(repoData) + const proofs = await sync.getRecords(storage, badRepo.cid, claims) + const fn = verify.verifyRecords(repo.did, proofs, didResolver) + await expect(fn).rejects.toThrow(verify.RepoVerificationError) + }) + + it('verifyProofs throws on a bad signature', async () => { + const badRepo = await util.addBadCommit(repo, keypair) + const claims = util.contentsToClaims(repoData) + const proofs = await sync.getRecords(storage, badRepo.cid, claims) + const fn = verify.verifyProofs(repo.did, proofs, claims, didResolver) + await expect(fn).rejects.toThrow(verify.RepoVerificationError) + }) +}) diff --git a/packages/repo/tsconfig.json b/packages/repo/tsconfig.json index 1eefc7bc8f3..c2095dfbba5 100644 --- a/packages/repo/tsconfig.json +++ b/packages/repo/tsconfig.json @@ -7,8 +7,9 @@ }, "include": ["./src","__tests__/**/**.ts"], "references": [ - { "path": "../auth/tsconfig.build.json" }, { "path": "../common/tsconfig.build.json" }, + { "path": "../crypto/tsconfig.build.json" }, + { "path": "../did-resolver/tsconfig.build.json" }, { "path": "../nsid/tsconfig.build.json" }, ] } \ No newline at end of file diff --git a/packages/xrpc-server/tests/bodies.test.ts b/packages/xrpc-server/tests/bodies.test.ts index 2bb41589030..26550c64710 100644 --- a/packages/xrpc-server/tests/bodies.test.ts +++ b/packages/xrpc-server/tests/bodies.test.ts @@ -2,7 +2,7 @@ import * as http from 'http' import { Readable } from 'stream' import { gzipSync } from 'zlib' import xrpc from '@atproto/xrpc' -import { bytesToStream, cidForData } from '@atproto/common' +import { bytesToStream, cidForCbor } from '@atproto/common' import { randomBytes } from '@atproto/crypto' import { createServer, closeServer } from './_util' import * as xrpcServer from '../src' @@ -113,7 +113,7 @@ describe('Bodies', () => { for await (const data of ctx.input.body) { buffers.push(data) } - const cid = await cidForData(Buffer.concat(buffers)) + const cid = await cidForCbor(Buffer.concat(buffers)) return { encoding: 'json', body: { cid: cid.toString() }, @@ -169,7 +169,7 @@ describe('Bodies', () => { it('supports blobs and compression', async () => { const bytes = randomBytes(1024) - const expectedCid = await cidForData(bytes) + const expectedCid = await cidForCbor(bytes) const { data: uncompressed } = await client.call( 'io.example.blobTest', diff --git a/tsconfig.json b/tsconfig.json index d705e5ccc11..2fc569e99c1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -27,7 +27,6 @@ { "path": "./packages/plc/tsconfig.build.json" }, { "path": "./packages/pds/tsconfig.build.json" }, { "path": "./packages/api/tsconfig.build.json" }, - { "path": "./packages/auth/tsconfig.build.json" }, { "path": "./packages/aws/tsconfig.build.json" }, { "path": "./packages/common/tsconfig.build.json" }, { "path": "./packages/crypto/tsconfig.build.json" }, diff --git a/yarn.lock b/yarn.lock index 7ef232ddee7..19106623ce2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4155,13 +4155,6 @@ "@typescript-eslint/types" "5.38.1" eslint-visitor-keys "^3.3.0" -"@ucans/core@0.11.0": - version "0.11.0" - resolved "https://registry.npmjs.org/@ucans/core/-/core-0.11.0.tgz" - integrity sha512-SHX67e313kKBaur5Cp+6WFeOLC7aBhkf1i1jIFpFb9f0f1cvM/lC3mjzOyUBeDg3QwmcN5QSZzaogVFvuVvzvg== - dependencies: - uint8arrays "3.0.0" - JSONStream@^1.0.4: version "1.3.5" resolved "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz"