From 237b0c6cda70ae3e156bac8a011a2739b346ae4b Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 7 Jun 2024 10:45:45 +0100 Subject: [PATCH] feat: publish index claim (#1487) This PR updates the `index/add` handler to publish an [index claim](https://github.com/w3s-project/content-claims?tab=readme-ov-file#index-claim) for the added index. Reading an index from the index claim allows freeway and hoverboard to serve blocks from all of the shards without having to request a location claim for each block. refs https://github.com/w3s-project/blob-fetcher/pull/10 --- packages/capabilities/src/index/index.js | 2 +- packages/upload-api/package.json | 2 +- packages/upload-api/src/index/add.js | 35 ++++- packages/upload-api/src/types.ts | 20 +++ .../upload-api/src/types/content-claims.ts | 31 +++++ packages/upload-api/src/types/index.ts | 5 +- .../test/external-service/content-claims.js | 131 ++++++++++++++++++ .../upload-api/test/external-service/index.js | 10 +- .../{ipni-service.js => ipni.js} | 0 packages/upload-api/test/handlers/index.js | 74 ++++++++++ packages/upload-api/test/helpers/context.js | 22 ++- pnpm-lock.yaml | 15 +- 12 files changed, 322 insertions(+), 25 deletions(-) create mode 100644 packages/upload-api/src/types/content-claims.ts create mode 100644 packages/upload-api/test/external-service/content-claims.js rename packages/upload-api/test/external-service/{ipni-service.js => ipni.js} (100%) diff --git a/packages/capabilities/src/index/index.js b/packages/capabilities/src/index/index.js index 44df4f10b..d11a81352 100644 --- a/packages/capabilities/src/index/index.js +++ b/packages/capabilities/src/index/index.js @@ -39,7 +39,7 @@ export const add = capability({ with: SpaceDID, nb: Schema.struct({ /** Content Archive (CAR) containing the `Index`. */ - index: Schema.link({ code: CAR.code }), + index: Schema.link({ code: CAR.code, version: 1 }), }), derives: (claimed, delegated) => and(equalWith(claimed, delegated)) || diff --git a/packages/upload-api/package.json b/packages/upload-api/package.json index 116445efe..387a75ecf 100644 --- a/packages/upload-api/package.json +++ b/packages/upload-api/package.json @@ -201,7 +201,7 @@ "@web3-storage/access": "workspace:^", "@web3-storage/blob-index": "workspace:^", "@web3-storage/capabilities": "workspace:^", - "@web3-storage/content-claims": "^5.0.0", + "@web3-storage/content-claims": "^5.1.0", "@web3-storage/did-mailto": "workspace:^", "@web3-storage/filecoin-api": "workspace:^", "multiformats": "^12.1.2", diff --git a/packages/upload-api/src/index/add.js b/packages/upload-api/src/index/add.js index 9c0945fbd..f6be18860 100644 --- a/packages/upload-api/src/index/add.js +++ b/packages/upload-api/src/index/add.js @@ -2,6 +2,7 @@ import * as Server from '@ucanto/server' import { ok, error } from '@ucanto/server' import * as Index from '@web3-storage/capabilities/index' import { ShardedDAGIndex } from '@web3-storage/blob-index' +import { Assert } from '@web3-storage/content-claims/capability' import { concat } from 'uint8arrays' import * as API from '../types.js' @@ -61,13 +62,21 @@ const add = async ({ capability }, context) => { shardDigests.map((s) => assertAllocated(context, space, s, 'ShardNotFound')) ) for (const res of shardAllocRes) { - if (!res.ok) return res + if (res.error) return res } // TODO: randomly validate slices in the index correspond to slices in the blob - // publish the index data to IPNI - return context.ipniService.publish(idxRes.ok) + const publishRes = await Promise.all([ + // publish the index data to IPNI + context.ipniService.publish(idxRes.ok), + // publish a content claim for the index + publishIndexClaim(context, { content: idxRes.ok.content, index: idxLink }), + ]) + for (const res of publishRes) { + if (res.error) return res + } + return ok({}) } /** @@ -87,3 +96,23 @@ const assertAllocated = async (context, space, digest, errorName) => { ) return ok({}) } + +/** + * @param {API.ClaimsClientContext} ctx + * @param {{ content: API.UnknownLink, index: API.CARLink }} params + */ +const publishIndexClaim = async (ctx, { content, index }) => { + const { invocationConfig, connection } = ctx.claimsService + const { issuer, audience, with: resource, proofs } = invocationConfig + const res = await Assert.index + .invoke({ + issuer, + audience, + with: resource, + nb: { content, index }, + expiration: Infinity, + proofs, + }) + .execute(connection) + return res.out +} diff --git a/packages/upload-api/src/types.ts b/packages/upload-api/src/types.ts index f24419b25..9329503ae 100644 --- a/packages/upload-api/src/types.ts +++ b/packages/upload-api/src/types.ts @@ -204,6 +204,8 @@ import { StorageGetError } from './types/storage.js' import { AllocationsStorage, BlobsStorage, BlobAddInput } from './types/blob.js' export type { AllocationsStorage, BlobsStorage, BlobAddInput } import { IPNIService, IndexServiceContext } from './types/index.js' +import { ClaimsClientConfig } from './types/content-claims.js' +import { Claim } from '@web3-storage/content-claims/client/api' export type { IndexServiceContext, IPNIService, @@ -211,6 +213,12 @@ export type { BlobNotFound, ShardedDAGIndex, } from './types/index.js' +export type { + ClaimsInvocationConfig, + ClaimsClientConfig, + ClaimsClientContext, + Service as ClaimsService, +} from './types/content-claims.js' export interface Service extends StorefrontService, W3sService { store: { @@ -590,6 +598,18 @@ export interface UcantoServerTestContext ipniService: IPNIService & { query(digest: MultihashDigest): Promise> } + + carStoreBucket: CarStoreBucket & Deactivator + blobsStorage: BlobsStorage & Deactivator + claimsService: ClaimsClientConfig & ClaimReader & Deactivator +} + +export interface ClaimReader { + read(digest: MultihashDigest): Promise> +} + +export interface Deactivator { + deactivate: () => Promise } export interface StoreTestContext {} diff --git a/packages/upload-api/src/types/content-claims.ts b/packages/upload-api/src/types/content-claims.ts new file mode 100644 index 000000000..cb18c8756 --- /dev/null +++ b/packages/upload-api/src/types/content-claims.ts @@ -0,0 +1,31 @@ +import { + ConnectionView, + DID, + Principal, + Proof, + Signer, +} from '@ucanto/interface' +import { Service } from '@web3-storage/content-claims/server/service/api' + +export type { ConnectionView, DID, Principal, Proof, Signer } +export type { Service } + +export interface ClaimsInvocationConfig { + /** Signing authority issuing the UCAN invocation(s). */ + issuer: Signer + /** The principal delegated to in the current UCAN. */ + audience: Principal + /** The resource the invocation applies to. */ + with: DID + /** Proof(s) the issuer has the capability to perform the action. */ + proofs?: Proof[] +} + +export interface ClaimsClientConfig { + invocationConfig: ClaimsInvocationConfig + connection: ConnectionView +} + +export interface ClaimsClientContext { + claimsService: ClaimsClientConfig +} diff --git a/packages/upload-api/src/types/index.ts b/packages/upload-api/src/types/index.ts index cd963971e..2aaf4c89a 100644 --- a/packages/upload-api/src/types/index.ts +++ b/packages/upload-api/src/types/index.ts @@ -2,8 +2,9 @@ import { MultihashDigest } from 'multiformats' import { Failure, Result, Unit } from '@ucanto/interface' import { ShardedDAGIndex } from '@web3-storage/blob-index/types' import { AllocationsStorage } from './blob.js' +import { ClaimsClientContext } from './content-claims.js' -export type { ShardedDAGIndex } +export type { ShardedDAGIndex, ClaimsClientContext } /** * Service that allows publishing a set of multihashes to IPNI for a @@ -26,7 +27,7 @@ export interface BlobRetriever { ): Promise, BlobNotFound>> } -export interface IndexServiceContext { +export interface IndexServiceContext extends ClaimsClientContext { allocationsStorage: AllocationsStorage blobRetriever: BlobRetriever ipniService: IPNIService diff --git a/packages/upload-api/test/external-service/content-claims.js b/packages/upload-api/test/external-service/content-claims.js new file mode 100644 index 000000000..5677a46f5 --- /dev/null +++ b/packages/upload-api/test/external-service/content-claims.js @@ -0,0 +1,131 @@ +import * as API from '../../src/types.js' +import { connect } from '@ucanto/client' +import { ed25519 } from '@ucanto/principal' +import { CAR, HTTP } from '@ucanto/transport' +import { Assert } from '@web3-storage/content-claims/capability' +import * as Client from '@web3-storage/content-claims/client' +import * as Server from '@web3-storage/content-claims/server' +import { DigestMap } from '@web3-storage/blob-index' + +/** + * @param {object} params + * @param {API.Signer} params.serviceSigner + * @param {API.Transport.Channel} params.channel + * @returns {Promise} + */ +export const create = async ({ serviceSigner, channel }) => { + const agent = await ed25519.generate() + const proofs = [ + await Assert.assert.delegate({ + issuer: serviceSigner, + with: serviceSigner.did(), + audience: agent, + }), + ] + return { + invocationConfig: { + issuer: agent, + with: serviceSigner.did(), + audience: serviceSigner, + proofs, + }, + connection: connect({ + id: serviceSigner, + codec: CAR.outbound, + channel, + }), + } +} + +/** + * @param {{ http?: import('node:http') }} [options] + * @returns {Promise} + */ +export const activate = async ({ http } = {}) => { + const serviceSigner = await ed25519.generate() + + const claimStore = new ClaimStorage() + /** @param {API.MultihashDigest} content */ + const read = async (content) => { + /** @type {import('@web3-storage/content-claims/client/api').Claim[]} */ + const claims = [] + await Server.walkClaims( + { claimFetcher: claimStore }, + content, + new Set() + ).pipeTo( + new WritableStream({ + async write(block) { + const claim = await Client.decode(block.bytes) + claims.push(claim) + }, + }) + ) + return { ok: claims } + } + + const server = Server.createServer({ + id: serviceSigner, + codec: CAR.inbound, + claimStore, + validateAuthorization: () => ({ ok: {} }), + }) + + if (!http) { + const conf = await create({ serviceSigner, channel: server }) + return Object.assign(conf, { read, deactivate: async () => {} }) + } + + const httpServer = http.createServer(async (req, res) => { + const chunks = [] + for await (const chunk of req) { + chunks.push(chunk) + } + + const { headers, body } = await server.request({ + // @ts-expect-error + headers: req.headers, + body: new Uint8Array(await new Blob(chunks).arrayBuffer()), + }) + + res.writeHead(200, headers) + res.write(body) + res.end() + }) + await new Promise((resolve) => httpServer.listen(resolve)) + // @ts-expect-error + const { port } = httpServer.address() + const serviceURL = new URL(`http://127.0.0.1:${port}`) + + const channel = HTTP.open({ url: serviceURL, method: 'POST' }) + const conf = await create({ serviceSigner, channel }) + return Object.assign(conf, { + read, + deactivate: () => + new Promise((resolve, reject) => { + httpServer.closeAllConnections() + httpServer.close((err) => { + err ? reject(err) : resolve(undefined) + }) + }), + }) +} + +class ClaimStorage { + constructor() { + /** @type {Map} */ + this.data = new DigestMap() + } + + /** @param {import('@web3-storage/content-claims/server/api').Claim} claim */ + async put(claim) { + const claims = this.data.get(claim.content) ?? [] + claims.push(claim) + this.data.set(claim.content, claims) + } + + /** @param {API.MultihashDigest} content */ + async get(content) { + return this.data.get(content) ?? [] + } +} diff --git a/packages/upload-api/test/external-service/index.js b/packages/upload-api/test/external-service/index.js index fe65f50b5..3dc000b45 100644 --- a/packages/upload-api/test/external-service/index.js +++ b/packages/upload-api/test/external-service/index.js @@ -1,5 +1,11 @@ -import { IPNIService } from './ipni-service.js' +import { IPNIService } from './ipni.js' +import * as ClaimsService from './content-claims.js' -export const getExternalServiceImplementations = async () => ({ +/** + * @param {object} [options] + * @param {import('node:http')} [options.http] + */ +export const getExternalServiceImplementations = async (options) => ({ ipniService: new IPNIService(), + claimsService: await ClaimsService.activate(options), }) diff --git a/packages/upload-api/test/external-service/ipni-service.js b/packages/upload-api/test/external-service/ipni.js similarity index 100% rename from packages/upload-api/test/external-service/ipni-service.js rename to packages/upload-api/test/external-service/ipni.js diff --git a/packages/upload-api/test/handlers/index.js b/packages/upload-api/test/handlers/index.js index 95e600053..d74ff04d7 100644 --- a/packages/upload-api/test/handlers/index.js +++ b/packages/upload-api/test/handlers/index.js @@ -171,4 +171,78 @@ export const test = { assert.ok(receipt.out.error) assert.equal(receipt.out.error?.name, 'ShardNotFound') }, + 'index/add should publish index claim': async (assert, context) => { + const { proof, spaceDid } = await registerSpace(alice, context) + const contentCAR = await randomCAR(32) + const contentCARBytes = new Uint8Array(await contentCAR.arrayBuffer()) + + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // upload the content CAR to the space + await uploadBlob( + context, + { + connection, + issuer: alice, + audience: context.id, + with: spaceDid, + proofs: [proof], + }, + { + cid: contentCAR.cid, + bytes: contentCARBytes, + } + ) + + const index = await fromShardArchives(contentCAR.roots[0], [ + contentCARBytes, + ]) + const indexCAR = Result.unwrap(await index.archive()) + const indexLink = await CAR.link(indexCAR) + + // upload the index CAR to the space + await uploadBlob( + context, + { + connection, + issuer: alice, + audience: context.id, + with: spaceDid, + proofs: [proof], + }, + { + cid: indexLink, + bytes: indexCAR, + } + ) + + const indexAdd = IndexCapabilities.add.invoke({ + issuer: alice, + audience: context.id, + with: spaceDid, + nb: { index: indexLink }, + proofs: [proof], + }) + const receipt = await indexAdd.execute(connection) + Result.try(receipt.out) + + // ensure an index claim exists for the content root + const claims = Result.unwrap( + await context.claimsService.read(contentCAR.roots[0].multihash) + ) + + let found = false + for (const c of claims) { + if ( + c.type === 'assert/index' && + c.index.toString() === indexLink.toString() + ) { + found = true + } + } + assert.ok(found, 'did not found index claim') + }, } diff --git a/packages/upload-api/test/helpers/context.js b/packages/upload-api/test/helpers/context.js index cbe8edddb..a6f58a3ac 100644 --- a/packages/upload-api/test/helpers/context.js +++ b/packages/upload-api/test/helpers/context.js @@ -5,8 +5,6 @@ import { getStoreImplementations as getFilecoinStoreImplementations, getQueueImplementations as getFilecoinQueueImplementations, } from '@web3-storage/filecoin-api/test/context/service' -import { BlobsStorage } from '../storage/blobs-storage.js' -import { CarStoreBucket } from '../storage/car-store-bucket.js' import * as Email from '../../src/utils/email.js' import { create as createRevocationChecker } from '../../src/utils/revocation.js' import { createServer, connect } from '../../src/lib.js' @@ -51,7 +49,7 @@ export const createContext = async ( } = getFilecoinStoreImplementations() const email = Email.debug() - const externalServices = await getExternalServiceImplementations() + const externalServices = await getExternalServiceImplementations(options) /** @type { import('../../src/types.js').UcantoServerContext } */ const serviceContext = { @@ -100,6 +98,7 @@ export const createContext = async ( return { ...serviceContext, + ...serviceStores, ...externalServices, mail: /** @type {TestTypes.DebugEmail} */ (serviceContext.email), service: /** @type {TestTypes.ServiceSigner} */ (serviceContext.id), @@ -113,14 +112,9 @@ export const createContext = async ( * * @param {Types.UcantoServerTestContext} context */ -export const cleanupContext = async (context) => { - /** @type {CarStoreBucket & { deactivate: () => Promise }}} */ - // @ts-ignore type misses S3 bucket properties like accessKey - const carStoreBucket = context.carStoreBucket - await carStoreBucket.deactivate() - - /** @type {BlobsStorage & { deactivate: () => Promise }}} */ - // @ts-ignore type misses S3 bucket properties like accessKey - const blobsStorage = context.blobsStorage - await blobsStorage.deactivate() -} +export const cleanupContext = (context) => + Promise.all([ + context.carStoreBucket.deactivate(), + context.blobsStorage.deactivate(), + context.claimsService.deactivate(), + ]) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a834dfa27..25497f4e6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -447,8 +447,8 @@ importers: specifier: workspace:^ version: link:../capabilities '@web3-storage/content-claims': - specifier: ^5.0.0 - version: 5.0.0 + specifier: ^5.1.0 + version: 5.1.0 '@web3-storage/did-mailto': specifier: workspace:^ version: link:../did-mailto @@ -4467,6 +4467,17 @@ packages: carstream: 2.1.0 multiformats: 13.1.0 + /@web3-storage/content-claims@5.1.0: + resolution: {integrity: sha512-3VStFKoeieRpRU7brFjKTsAuAffQzYDIZ8F3Gh0+niw+MgzBK72osW+fftdquT8neWir34Ndu3mBUKKJ3ck1RQ==} + dependencies: + '@ucanto/client': 9.0.1 + '@ucanto/interface': 10.0.1 + '@ucanto/server': 10.0.0 + '@ucanto/transport': 9.1.1 + carstream: 2.1.0 + multiformats: 13.1.0 + dev: false + /@web3-storage/data-segment@3.2.0: resolution: {integrity: sha512-SM6eNumXzrXiQE2/J59+eEgCRZNYPxKhRoHX2QvV3/scD4qgcf4g+paWBc3UriLEY1rCboygGoPsnqYJNyZyfA==} dependencies: