From 363759ad47c43e873954f0645bd6afbce46a0ca3 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 7 Nov 2023 20:05:59 +0100 Subject: [PATCH 1/2] feat: filecoin info --- packages/capabilities/src/filecoin/index.js | 1 + packages/capabilities/src/filecoin/lib.js | 6 +- .../capabilities/src/filecoin/storefront.js | 29 ++ packages/capabilities/src/index.js | 1 + packages/capabilities/src/types.ts | 19 + packages/filecoin-api/src/errors.js | 24 ++ packages/filecoin-api/src/storefront/api.ts | 5 + .../filecoin-api/src/storefront/service.js | 111 +++++- .../filecoin-api/test/context/receipts.js | 94 +++++ .../filecoin-api/test/events/storefront.js | 1 + .../filecoin-api/test/services/storefront.js | 371 +++++++++++++++++- packages/filecoin-api/test/storefront.spec.js | 14 + packages/filecoin-client/src/storefront.js | 34 ++ packages/filecoin-client/src/types.ts | 4 + .../filecoin-client/test/helpers/mocks.js | 1 + .../filecoin-client/test/storefront.test.js | 54 +++ packages/upload-api/test/helpers/context.js | 17 + packages/w3up-client/README.md | 24 ++ packages/w3up-client/package.json | 4 +- .../w3up-client/src/capability/filecoin.js | 34 ++ packages/w3up-client/src/client.js | 2 + packages/w3up-client/src/service.js | 13 + packages/w3up-client/src/types.ts | 2 + .../test/capability/filecoin.test.js | 165 ++++++++ packages/w3up-client/test/helpers/car.js | 2 +- packages/w3up-client/test/helpers/mocks.js | 7 +- packages/w3up-client/test/helpers/random.js | 40 ++ packages/w3up-client/test/test.js | 1 + pnpm-lock.yaml | 16 +- 29 files changed, 1082 insertions(+), 14 deletions(-) create mode 100644 packages/w3up-client/src/capability/filecoin.js create mode 100644 packages/w3up-client/test/capability/filecoin.test.js diff --git a/packages/capabilities/src/filecoin/index.js b/packages/capabilities/src/filecoin/index.js index d7675e0fd..aee774398 100644 --- a/packages/capabilities/src/filecoin/index.js +++ b/packages/capabilities/src/filecoin/index.js @@ -16,4 +16,5 @@ export { filecoinOffer as offer, filecoinSubmit as submit, filecoinAccept as accept, + filecoinInfo as info, } from './storefront.js' diff --git a/packages/capabilities/src/filecoin/lib.js b/packages/capabilities/src/filecoin/lib.js index fd4ae8cee..76bc77271 100644 --- a/packages/capabilities/src/filecoin/lib.js +++ b/packages/capabilities/src/filecoin/lib.js @@ -3,11 +3,13 @@ import { Schema } from '@ucanto/validator' /** * @see https://github.com/filecoin-project/FIPs/pull/758/files */ -const FR32_SHA2_256_TRUNC254_PADDED_BINARY_TREE = /** @type {const} */ (0x1011) +export const FR32_SHA2_256_TRUNC254_PADDED_BINARY_TREE = /** @type {const} */ ( + 0x1011 +) /** * @see https://github.com/filecoin-project/FIPs/pull/758/files */ -const RAW_CODE = /** @type {const} */ (0x55) +export const RAW_CODE = /** @type {const} */ (0x55) export const PieceLink = /** @type {import('../types.js').PieceLinkSchema} */ ( Schema.link({ diff --git a/packages/capabilities/src/filecoin/storefront.js b/packages/capabilities/src/filecoin/storefront.js index 88b103690..0303a6692 100644 --- a/packages/capabilities/src/filecoin/storefront.js +++ b/packages/capabilities/src/filecoin/storefront.js @@ -106,3 +106,32 @@ export const filecoinAccept = capability({ ) }, }) + +/** + * Capability allowing an agent to _request_ info about a content piece in + * Filecoin deals. + */ +export const filecoinInfo = capability({ + can: 'filecoin/info', + /** + * DID of the space the content is stored in. + */ + with: Schema.did(), + nb: Schema.struct({ + /** + * CID of the content that resulted in Filecoin piece. + */ + content: Schema.link(), + /** + * CID of the piece. + */ + piece: PieceLink.optional(), + }), + derives: (claim, from) => { + return ( + and(equalWith(claim, from)) || + and(checkLink(claim.nb.content, from.nb.content, 'nb.content')) || + ok({}) + ) + }, +}) diff --git a/packages/capabilities/src/index.js b/packages/capabilities/src/index.js index 7c44a620c..5d814cda3 100644 --- a/packages/capabilities/src/index.js +++ b/packages/capabilities/src/index.js @@ -73,6 +73,7 @@ export const abilitiesAsStrings = [ Storefront.filecoinOffer.can, Storefront.filecoinSubmit.can, Storefront.filecoinAccept.can, + Storefront.filecoinInfo.can, Aggregator.pieceOffer.can, Aggregator.pieceAccept.can, Dealer.aggregateOffer.can, diff --git a/packages/capabilities/src/types.ts b/packages/capabilities/src/types.ts index c72d98830..1038fed0b 100644 --- a/packages/capabilities/src/types.ts +++ b/packages/capabilities/src/types.ts @@ -305,6 +305,21 @@ export interface ProofNotFound extends Ucanto.Failure { name: 'ProofNotFound' } +export interface FilecoinInfoSuccess { + piece: PieceLink + deals: FilecoinInfoAcceptedDeal[] +} +export interface FilecoinInfoAcceptedDeal + extends DataAggregationProof, + DealDetails { + aggregate: PieceLink +} + +export type FilecoinInfoFailure = + | ContentNotFound + | InvalidContentPiece + | Ucanto.Failure + // filecoin aggregator export interface PieceOfferSuccess { /** @@ -549,6 +564,9 @@ export type FilecoinSubmit = InferInvokedCapability< export type FilecoinAccept = InferInvokedCapability< typeof StorefrontCaps.filecoinAccept > +export type FilecoinInfo = InferInvokedCapability< + typeof StorefrontCaps.filecoinInfo +> export type PieceOffer = InferInvokedCapability< typeof AggregatorCaps.pieceOffer > @@ -610,6 +628,7 @@ export type AbilitiesArray = [ FilecoinOffer['can'], FilecoinSubmit['can'], FilecoinAccept['can'], + FilecoinInfo['can'], PieceOffer['can'], PieceAccept['can'], AggregateOffer['can'], diff --git a/packages/filecoin-api/src/errors.js b/packages/filecoin-api/src/errors.js index 94f27277e..df3a58db4 100644 --- a/packages/filecoin-api/src/errors.js +++ b/packages/filecoin-api/src/errors.js @@ -48,6 +48,30 @@ export class RecordNotFound extends Server.Failure { } } +export const ContentNotFoundErrorName = /** @type {const} */ ('ContentNotFound') +export class ContentNotFound extends Server.Failure { + get reason() { + return this.message + } + + get name() { + return ContentNotFoundErrorName + } +} + +export const InvalidContentPieceErrorName = /** @type {const} */ ( + 'InvalidContentPiece' +) +export class InvalidContentPiece extends Server.Failure { + get reason() { + return this.message + } + + get name() { + return InvalidContentPieceErrorName + } +} + export const EncodeRecordErrorName = /** @type {const} */ ('EncodeRecordFailed') export class EncodeRecordFailed extends Server.Failure { get reason() { diff --git a/packages/filecoin-api/src/storefront/api.ts b/packages/filecoin-api/src/storefront/api.ts index 3ffd7da56..11d876b0d 100644 --- a/packages/filecoin-api/src/storefront/api.ts +++ b/packages/filecoin-api/src/storefront/api.ts @@ -10,6 +10,7 @@ import { PieceLink } from '@web3-storage/data-segment' import { AggregatorService, StorefrontService, + DealTrackerService, } from '@web3-storage/filecoin-client/types' import { Store, @@ -64,6 +65,10 @@ export interface ServiceContext { * Stores receipts for tasks. */ receiptStore: ReceiptStore + /** + * Deal tracker connection to find out available deals for an aggregate. + */ + dealTrackerService: ServiceConfig /** * Service options. */ diff --git a/packages/filecoin-api/src/storefront/service.js b/packages/filecoin-api/src/storefront/service.js index 8d756c7ab..0a2c27ba4 100644 --- a/packages/filecoin-api/src/storefront/service.js +++ b/packages/filecoin-api/src/storefront/service.js @@ -3,9 +3,15 @@ import * as Client from '@ucanto/client' import * as CAR from '@ucanto/transport/car' import * as StorefrontCaps from '@web3-storage/capabilities/filecoin/storefront' import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' +import { DealTracker } from '@web3-storage/filecoin-client' // eslint-disable-next-line no-unused-vars import * as API from '../types.js' -import { QueueOperationFailed, StoreOperationFailed } from '../errors.js' +import { + QueueOperationFailed, + StoreOperationFailed, + ContentNotFound, + InvalidContentPiece, +} from '../errors.js' /** * @param {API.Input} input @@ -226,6 +232,105 @@ async function findDataAggregationProof({ taskStore, receiptStore }, task) { } } +/** + * @param {API.Input} input + * @param {import('./api.js').ServiceContext} context + * @returns {Promise | API.UcantoInterface.JoinBuilder>} + */ +export const filecoinInfo = async ({ capability }, context) => { + const { piece, content } = capability.nb + + const queryRecords = await context.pieceStore.query({ content }) + if (queryRecords.error) { + return { error: new StoreOperationFailed(queryRecords.error.message) } + } else if (!queryRecords.ok.length) { + return { + error: new ContentNotFound( + `no piece record was previously stored for content ${content.toString()}` + ), + } + } + if (Boolean(piece) && !queryRecords.ok[0].piece.equals(piece)) { + return { + error: new InvalidContentPiece( + `received piece ${piece?.toString()} is not the same as previously computed ${ + queryRecords.ok[0].piece + } for content ${content.toString()}` + ), + } + } + + // Check if `piece/accept` receipt exists to get to know aggregate where it is included on a deal + const pieceAcceptInvocation = await StorefrontCaps.filecoinAccept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: { + piece: queryRecords.ok[0].piece, + content, + }, + expiration: Infinity, + }) + .delegate() + + const pieceAcceptReceiptGet = await context.receiptStore.get( + pieceAcceptInvocation.link() + ) + if (pieceAcceptReceiptGet.error) { + // TODO: see receipt chain to report processing + /** @type {API.UcantoInterface.OkBuilder} */ + const processingResult = Server.ok({ + piece: queryRecords.ok[0].piece, + deals: [], + }) + return processingResult + } + + const pieceAcceptOut = /** @type {API.FilecoinAcceptSuccess} */ ( + pieceAcceptReceiptGet.ok?.out.ok + ) + + // Query current info of aggregate from deal tracker + const info = await DealTracker.dealInfo( + context.dealTrackerService.invocationConfig, + pieceAcceptOut.aggregate, + { connection: context.dealTrackerService.connection } + ) + + if (info.out.error) { + return { + error: info.out.error, + } + } + const deals = Object.entries(info.out.ok.deals || {}) + if (!deals.length) { + // Should not happen if there is `piece/accept` receipt + return { + error: new Server.Failure( + `no deals were obtained for aggregate ${pieceAcceptOut.aggregate} where piece ${queryRecords.ok[0].piece} is included` + ), + } + } + + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ + piece: queryRecords.ok[0].piece, + deals: deals.map(([dealId, dealDetails]) => ({ + aggregate: pieceAcceptOut.aggregate, + provider: dealDetails.provider, + inclusion: pieceAcceptOut.inclusion, + aux: { + dataType: 0n, + dataSource: { + dealID: BigInt(dealId), + }, + }, + })), + }) + return result +} + export const ProofNotFoundName = /** @type {const} */ ('ProofNotFound') export class ProofNotFound extends Server.Failure { get reason() { @@ -255,6 +360,10 @@ export function createService(context) { capability: StorefrontCaps.filecoinAccept, handler: (input) => filecoinAccept(input, context), }), + info: Server.provideAdvanced({ + capability: StorefrontCaps.filecoinInfo, + handler: (input) => filecoinInfo(input, context), + }), }, } } diff --git a/packages/filecoin-api/test/context/receipts.js b/packages/filecoin-api/test/context/receipts.js index 6f5f859ed..1d3176562 100644 --- a/packages/filecoin-api/test/context/receipts.js +++ b/packages/filecoin-api/test/context/receipts.js @@ -1,4 +1,5 @@ import { Receipt } from '@ucanto/core' +import * as StorefrontCaps from '@web3-storage/capabilities/filecoin/storefront' import * as AggregatorCaps from '@web3-storage/capabilities/filecoin/aggregator' import * as DealerCaps from '@web3-storage/capabilities/filecoin/dealer' @@ -12,6 +13,7 @@ import * as API from '../../src/types.js' * @param {API.PieceLink} context.aggregate * @param {string} context.group * @param {API.PieceLink} context.piece + * @param {API.CARLink} context.content * @param {import('@ucanto/interface').Block} context.piecesBlock * @param {API.InclusionProof} context.inclusionProof * @param {API.AggregateAcceptSuccess} context.aggregateAcceptStatus @@ -23,10 +25,47 @@ export async function createInvocationsAndReceiptsForDealDataProofChain({ aggregate, group, piece, + content, piecesBlock, inclusionProof, aggregateAcceptStatus, }) { + const filecoinOfferInvocation = await StorefrontCaps.filecoinOffer + .invoke({ + issuer: storefront, + audience: storefront, + with: storefront.did(), + nb: { + piece, + content, + }, + expiration: Infinity, + }) + .delegate() + const filecoinSubmitInvocation = await StorefrontCaps.filecoinSubmit + .invoke({ + issuer: storefront, + audience: storefront, + with: storefront.did(), + nb: { + piece, + content, + }, + expiration: Infinity, + }) + .delegate() + const filecoinAcceptInvocation = await StorefrontCaps.filecoinAccept + .invoke({ + issuer: storefront, + audience: storefront, + with: storefront.did(), + nb: { + piece, + content, + }, + expiration: Infinity, + }) + .delegate() const pieceOfferInvocation = await AggregatorCaps.pieceOffer .invoke({ issuer: storefront, @@ -76,6 +115,55 @@ export async function createInvocationsAndReceiptsForDealDataProofChain({ expiration: Infinity, }) .delegate() + + // Receipts + const filecoinOfferReceipt = await Receipt.issue({ + issuer: storefront, + ran: filecoinOfferInvocation.cid, + result: { + ok: /** @type {API.FilecoinOfferSuccess} */ ({ + piece, + }), + }, + fx: { + join: filecoinAcceptInvocation.cid, + fork: [filecoinSubmitInvocation.cid], + }, + }) + + const filecoinSubmitReceipt = await Receipt.issue({ + issuer: storefront, + ran: filecoinSubmitInvocation.cid, + result: { + ok: /** @type {API.FilecoinSubmitSuccess} */ ({ + piece, + }), + }, + fx: { + join: pieceOfferInvocation.cid, + fork: [], + }, + }) + + const filecoinAcceptReceipt = await Receipt.issue({ + issuer: storefront, + ran: filecoinAcceptInvocation.cid, + result: { + ok: /** @type {API.FilecoinAcceptSuccess} */ ({ + piece, + aggregate, + inclusion: inclusionProof, + aux: { + ...aggregateAcceptStatus, + }, + }), + }, + fx: { + join: undefined, + fork: [], + }, + }) + const pieceOfferReceipt = await Receipt.issue({ issuer: aggregator, ran: pieceOfferInvocation.cid, @@ -130,12 +218,18 @@ export async function createInvocationsAndReceiptsForDealDataProofChain({ return { invocations: { + filecoinOfferInvocation, + filecoinSubmitInvocation, + filecoinAcceptInvocation, pieceOfferInvocation, pieceAcceptInvocation, aggregateOfferInvocation, aggregateAcceptInvocation, }, receipts: { + filecoinOfferReceipt, + filecoinSubmitReceipt, + filecoinAcceptReceipt, pieceOfferReceipt, pieceAcceptReceipt, aggregateOfferReceipt, diff --git a/packages/filecoin-api/test/events/storefront.js b/packages/filecoin-api/test/events/storefront.js index 9de711d01..ceadee9c8 100644 --- a/packages/filecoin-api/test/events/storefront.js +++ b/packages/filecoin-api/test/events/storefront.js @@ -365,6 +365,7 @@ export const test = { aggregate: aggregate.link, group, piece: piece.link, + content: piece.content, piecesBlock, inclusionProof: { subtree: inclusionProof.ok[0], diff --git a/packages/filecoin-api/test/services/storefront.js b/packages/filecoin-api/test/services/storefront.js index 928deac62..347c2a711 100644 --- a/packages/filecoin-api/test/services/storefront.js +++ b/packages/filecoin-api/test/services/storefront.js @@ -1,6 +1,8 @@ import { Filecoin, Aggregator } from '@web3-storage/capabilities' +import * as Server from '@ucanto/server' import { CBOR } from '@ucanto/core' import * as Signer from '@ucanto/principal/ed25519' +import * as DealTrackerCaps from '@web3-storage/capabilities/filecoin/deal-tracker' import pWaitFor from 'p-wait-for' import * as API from '../../src/types.js' @@ -8,6 +10,8 @@ import * as StorefrontApi from '../../src/storefront/api.js' import { createServer, connect } from '../../src/storefront/service.js' import { + ContentNotFoundErrorName, + InvalidContentPieceErrorName, QueueOperationErrorName, StoreOperationErrorName, } from '../../src/errors.js' @@ -16,6 +20,8 @@ import { createInvocationsAndReceiptsForDealDataProofChain } from '../context/re import { getStoreImplementations } from '../context/store-implementations.js' import { FailingStore } from '../context/store.js' import { FailingQueue } from '../context/queue.js' +import { mockService } from '../context/mocks.js' +import { getConnection } from '../context/service.js' /** * @typedef {import('../../src/storefront/api.js').PieceRecord} PieceRecord @@ -200,7 +206,7 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, QueueOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, filecoinSubmitQueue: new FailingQueue(), }) @@ -232,7 +238,7 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, StoreOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, pieceStore: getStoreImplementations(FailingStore).storefront.pieceStore, }) @@ -306,7 +312,7 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, QueueOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, pieceOfferQueue: new FailingQueue(), }) @@ -363,6 +369,7 @@ export const test = { aggregate: aggregate.link, group, piece: piece.link, + content: piece.content, piecesBlock, inclusionProof: { subtree: inclusionProof.ok[0], @@ -447,11 +454,361 @@ export const test = { assert.ok(response.out.error) assert.equal(response.out.error?.name, StoreOperationErrorName) }, - (context) => ({ + async (context) => ({ ...context, pieceStore: getStoreImplementations(FailingStore).storefront.pieceStore, }) ), + 'filecoin/info gets aggregate where piece was included together with deals and inclusion proof': + wichMockableContext( + async (assert, context) => { + const { agent, aggregator, dealer } = await getServiceContext() + const group = context.id.did() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Create piece and aggregate for test + const { aggregate, pieces } = await randomAggregate(10, 128) + const piece = pieces[0] + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Store piece into store + const putRes = await context.pieceStore.put({ + piece: piece.link.link(), + content: piece.content.link(), + group: context.id.did(), + status: 'submitted', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + assert.ok(putRes.ok) + + // Create inclusion proof for test + const inclusionProof = aggregate.resolveProof(piece.link) + if (inclusionProof.error) { + throw new Error('could not compute inclusion proof') + } + + // Create invocations and receipts for chain into DealDataProof + const dealMetadata = { + dataType: 0n, + dataSource: { + dealID: 111n, + }, + } + const { invocations, receipts } = + await createInvocationsAndReceiptsForDealDataProofChain({ + storefront: context.id, + aggregator, + dealer, + aggregate: aggregate.link, + group, + piece: piece.link, + content: piece.content, + piecesBlock, + inclusionProof: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + aggregateAcceptStatus: { + ...dealMetadata, + aggregate: aggregate.link, + }, + }) + + const storedInvocationsAndReceiptsRes = + await storeInvocationsAndReceipts({ + invocations, + receipts, + taskStore: context.taskStore, + receiptStore: context.receiptStore, + }) + assert.ok(storedInvocationsAndReceiptsRes.ok) + + // agent invocation + const filecoinInfoInv = Filecoin.info.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: piece.link.link(), + content: piece.content.link(), + }, + }) + + const response = await filecoinInfoInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.ok(response.out.ok.piece.equals(piece.link.link())) + assert.equal(response.out.ok.deals.length, 1) + assert.ok(response.out.ok.deals[0].aggregate.equals(aggregate.link)) + assert.deepEqual( + BigInt(response.out.ok.deals[0].aux.dataType), + dealMetadata.dataType + ) + assert.deepEqual( + BigInt(response.out.ok.deals[0].aux.dataSource.dealID), + dealMetadata.dataSource.dealID + ) + assert.ok(response.out.ok.deals[0].inclusion.index) + assert.ok(response.out.ok.deals[0].inclusion.subtree) + }, + async (context) => { + /** + * Mock deal tracker to return deals + */ + const dealTrackerSigner = await Signer.generate() + const service = mockService({ + deal: { + info: Server.provideAdvanced({ + capability: DealTrackerCaps.dealInfo, + handler: async () => { + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ + deals: { + 111: { + provider: 'f11111', + }, + }, + }) + + return result + }, + }), + }, + }) + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection + + return { + ...context, + service, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: context.id, + with: context.id.did(), + audience: dealTrackerSigner, + }, + }, + } + } + ), + 'filecoin/info gets aggregate where piece was included together with deals and inclusion proof by giving content': + wichMockableContext( + async (assert, context) => { + const { agent, aggregator, dealer } = await getServiceContext() + const group = context.id.did() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Create piece and aggregate for test + const { aggregate, pieces } = await randomAggregate(10, 128) + const piece = pieces[0] + const offer = pieces.map((p) => p.link) + const piecesBlock = await CBOR.write(offer) + + // Store piece into store + const putRes = await context.pieceStore.put({ + piece: piece.link.link(), + content: piece.content.link(), + group: context.id.did(), + status: 'submitted', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + assert.ok(putRes.ok) + + // Create inclusion proof for test + const inclusionProof = aggregate.resolveProof(piece.link) + if (inclusionProof.error) { + throw new Error('could not compute inclusion proof') + } + + // Create invocations and receipts for chain into DealDataProof + const dealMetadata = { + dataType: 0n, + dataSource: { + dealID: 111n, + }, + } + const { invocations, receipts } = + await createInvocationsAndReceiptsForDealDataProofChain({ + storefront: context.id, + aggregator, + dealer, + aggregate: aggregate.link, + group, + piece: piece.link, + content: piece.content, + piecesBlock, + inclusionProof: { + subtree: inclusionProof.ok[0], + index: inclusionProof.ok[1], + }, + aggregateAcceptStatus: { + ...dealMetadata, + aggregate: aggregate.link, + }, + }) + + const storedInvocationsAndReceiptsRes = + await storeInvocationsAndReceipts({ + invocations, + receipts, + taskStore: context.taskStore, + receiptStore: context.receiptStore, + }) + assert.ok(storedInvocationsAndReceiptsRes.ok) + + // agent invocation + const filecoinInfoInv = Filecoin.info.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + // Piece was previously set + piece: undefined, + content: piece.content.link(), + }, + }) + + const response = await filecoinInfoInv.execute(connection) + if (response.out.error) { + throw new Error('invocation failed', { cause: response.out.error }) + } + assert.ok(response.out.ok) + assert.ok(response.out.ok.piece.equals(piece.link.link())) + assert.equal(response.out.ok.deals.length, 1) + assert.ok(response.out.ok.deals[0].aggregate.equals(aggregate.link)) + assert.deepEqual( + BigInt(response.out.ok.deals[0].aux.dataType), + dealMetadata.dataType + ) + assert.deepEqual( + BigInt(response.out.ok.deals[0].aux.dataSource.dealID), + dealMetadata.dataSource.dealID + ) + assert.ok(response.out.ok.deals[0].inclusion.index) + assert.ok(response.out.ok.deals[0].inclusion.subtree) + }, + async (context) => { + /** + * Mock deal tracker to return deals + */ + const dealTrackerSigner = await Signer.generate() + const service = mockService({ + deal: { + info: Server.provideAdvanced({ + capability: DealTrackerCaps.dealInfo, + handler: async ({ invocation, context }) => { + /** @type {API.UcantoInterface.OkBuilder} */ + const result = Server.ok({ + deals: { + 111: { + provider: 'f11111', + }, + }, + }) + + return result + }, + }), + }, + }) + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection + + return { + ...context, + service, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: context.id, + with: context.id.did(), + audience: dealTrackerSigner, + }, + }, + } + } + ), + 'filecoin/info fails if content is not known': async (assert, context) => { + const { agent } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Create piece and aggregate for test + const { pieces } = await randomAggregate(10, 128) + const piece = pieces[0] + + // agent invocation + const filecoinInfoInv = Filecoin.info.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + piece: piece.link, + content: piece.content.link(), + }, + }) + + const response = await filecoinInfoInv.execute(connection) + assert.ok(response.out.error) + assert.equal(response.out.error?.name, ContentNotFoundErrorName) + }, + 'filecoin/info fails if piece provided for content is different from the previously computed': + async (assert, context) => { + const { agent } = await getServiceContext() + const connection = connect({ + id: context.id, + channel: createServer(context), + }) + + // Create piece and aggregate for test + const { pieces } = await randomAggregate(10, 128) + const piece = pieces[0] + + // Store piece into store + const putRes = await context.pieceStore.put({ + piece: piece.link.link(), + content: piece.content.link(), + group: context.id.did(), + status: 'submitted', + insertedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }) + assert.ok(putRes.ok) + + // agent invocation + const filecoinInfoInv = Filecoin.info.invoke({ + issuer: agent, + audience: connection.id, + with: agent.did(), + nb: { + // give wrong piece for content + piece: pieces[1].link, + content: piece.content.link(), + }, + }) + + const response = await filecoinInfoInv.execute(connection) + assert.ok(response.out.error) + assert.equal(response.out.error?.name, InvalidContentPieceErrorName) + }, } /** @@ -501,12 +858,12 @@ async function getServiceContext() { /** * @param {API.Test} testFn - * @param {(context: StorefrontApi.ServiceContext) => StorefrontApi.ServiceContext} mockContextFunction + * @param {(context: StorefrontApi.ServiceContext) => Promise} mockContextFunction */ function wichMockableContext(testFn, mockContextFunction) { // @ts-ignore - return function (...args) { - const modifiedArgs = [args[0], mockContextFunction(args[1])] + return async function (...args) { + const modifiedArgs = [args[0], await mockContextFunction(args[1])] // @ts-ignore return testFn(...modifiedArgs) } diff --git a/packages/filecoin-api/test/storefront.spec.js b/packages/filecoin-api/test/storefront.spec.js index 8bd534a00..677afe425 100644 --- a/packages/filecoin-api/test/storefront.spec.js +++ b/packages/filecoin-api/test/storefront.spec.js @@ -24,6 +24,7 @@ describe('storefront', () => { define(name, async () => { const storefrontSigner = await Signer.generate() const aggregatorSigner = await Signer.generate() + const dealTrackerSigner = await Signer.generate() // resources /** @type {Map} */ @@ -34,6 +35,11 @@ describe('storefront', () => { const { storefront: { pieceStore, receiptStore, taskStore }, } = getStoreImplementations() + const service = getMockService() + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection await test( { @@ -54,6 +60,14 @@ describe('storefront', () => { pieceOfferQueue, taskStore, receiptStore, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: storefrontSigner, + with: storefrontSigner.did(), + audience: dealTrackerSigner, + }, + }, queuedMessages, validateAuthorization, } diff --git a/packages/filecoin-client/src/storefront.js b/packages/filecoin-client/src/storefront.js index 917f11b2a..27be0f795 100644 --- a/packages/filecoin-client/src/storefront.js +++ b/packages/filecoin-client/src/storefront.js @@ -147,3 +147,37 @@ export async function filecoinAccept( return await invocation.execute(conn) } + +/** + * The `filecoin/info` task can be executed to request info about a content piece + * in Filecoin. It issues a signed receipt of the execution result. + * + * @param {import('./types.js').InvocationConfig} conf - Configuration + * @param {import('multiformats').UnknownLink} content + * @param {import('@web3-storage/data-segment').PieceLink} [piece] + * @param {import('./types.js').RequestOptions} [options] + */ +export async function filecoinInfo( + { issuer, with: resource, proofs, audience }, + content, + piece, + options = {} +) { + /* c8 ignore next */ + const conn = options.connection ?? connection + + const invocation = Storefront.filecoinInfo.invoke({ + issuer, + /* c8 ignore next */ + audience: audience ?? services.STOREFRONT.principal, + with: resource, + nb: { + content, + piece, + }, + proofs, + expiration: Infinity, + }) + + return await invocation.execute(conn) +} diff --git a/packages/filecoin-client/src/types.ts b/packages/filecoin-client/src/types.ts index 762c522ea..752ae4711 100644 --- a/packages/filecoin-client/src/types.ts +++ b/packages/filecoin-client/src/types.ts @@ -16,6 +16,9 @@ import { FilecoinAccept, FilecoinAcceptSuccess, FilecoinAcceptFailure, + FilecoinInfo, + FilecoinInfoSuccess, + FilecoinInfoFailure, PieceOffer, PieceOfferSuccess, PieceOfferFailure, @@ -75,6 +78,7 @@ export interface StorefrontService { FilecoinAcceptSuccess, FilecoinAcceptFailure > + info: ServiceMethod } } diff --git a/packages/filecoin-client/test/helpers/mocks.js b/packages/filecoin-client/test/helpers/mocks.js index 3b4fb52c4..ebe42207c 100644 --- a/packages/filecoin-client/test/helpers/mocks.js +++ b/packages/filecoin-client/test/helpers/mocks.js @@ -18,6 +18,7 @@ export function mockService(impl) { offer: withCallCount(impl.filecoin?.offer ?? notImplemented), submit: withCallCount(impl.filecoin?.submit ?? notImplemented), accept: withCallCount(impl.filecoin?.accept ?? notImplemented), + info: withCallCount(impl.filecoin?.info ?? notImplemented), }, piece: { offer: withCallCount(impl.piece?.offer ?? notImplemented), diff --git a/packages/filecoin-client/test/storefront.test.js b/packages/filecoin-client/test/storefront.test.js index acb00d235..e490b49cd 100644 --- a/packages/filecoin-client/test/storefront.test.js +++ b/packages/filecoin-client/test/storefront.test.js @@ -9,6 +9,7 @@ import { filecoinOffer, filecoinSubmit, filecoinAccept, + filecoinInfo, } from '../src/storefront.js' import { randomAggregate, randomCargo } from './helpers/random.js' import { mockService } from './helpers/mocks.js' @@ -261,6 +262,59 @@ describe('storefront', () => { // does not include effect fx in receipt assert.ok(!res.fx.join) }) + + it('agent asks info of a filecoin piece', async () => { + const { agent } = await getContext() + const [cargo] = await randomCargo(1, 100) + + /** @type {import('@web3-storage/capabilities/types').FilecoinOfferSuccess} */ + const filecoinOfferResponse = { + piece: cargo.link, + } + + // Create Ucanto service + const service = mockService({ + filecoin: { + info: Server.provideAdvanced({ + capability: StorefrontCaps.filecoinInfo, + handler: async ({ invocation }) => { + assert.strictEqual(invocation.issuer.did(), agent.did()) + assert.strictEqual(invocation.capabilities.length, 1) + const invCap = invocation.capabilities[0] + assert.strictEqual(invCap.can, StorefrontCaps.filecoinInfo.can) + assert.equal(invCap.with, invocation.issuer.did()) + assert.ok(invCap.nb) + const { content, piece } = invCap.nb + // piece link + assert.ok(piece) + assert.ok(piece?.equals(cargo.link.link())) + // content + assert.ok(content.equals(cargo.content.link())) + + return Server.ok({ + piece, + deals: [], + }) + }, + }), + }, + }) + + const res = await filecoinInfo( + { + issuer: agent, + with: agent.did(), + audience: storefrontService, + }, + cargo.content, + cargo.link, + { connection: getConnection(service).connection } + ) + + assert.ok(res.out.ok) + assert.ok(res.out.ok.piece.equals(filecoinOfferResponse.piece)) + assert.deepEqual(res.out.ok.deals, []) + }) }) async function getContext() { diff --git a/packages/upload-api/test/helpers/context.js b/packages/upload-api/test/helpers/context.js index acefa960b..313f4584b 100644 --- a/packages/upload-api/test/helpers/context.js +++ b/packages/upload-api/test/helpers/context.js @@ -1,5 +1,7 @@ import * as Signer from '@ucanto/principal/ed25519' import { + getConnection, + getMockService, getStoreImplementations, getQueueImplementations, } from '@web3-storage/filecoin-api/test/context/service' @@ -41,8 +43,15 @@ export const createContext = async ( const usageStorage = new UsageStorage(storeTable) const signer = await Signer.generate() const aggregatorSigner = await Signer.generate() + const dealTrackerSigner = await Signer.generate() const id = signer.withDID('did:web:test.web3.storage') + const service = getMockService() + const dealTrackerConnection = getConnection( + dealTrackerSigner, + service + ).connection + /** @type {Map} */ const queuedMessages = new Map() const { @@ -86,6 +95,14 @@ export const createContext = async ( receiptStore, taskStore, requirePaymentPlan, + dealTrackerService: { + connection: dealTrackerConnection, + invocationConfig: { + issuer: id, + with: id.did(), + audience: dealTrackerSigner, + }, + }, ...createRevocationChecker({ revocationsStorage }), } diff --git a/packages/w3up-client/README.md b/packages/w3up-client/README.md index d823a3c98..86e51fcd3 100644 --- a/packages/w3up-client/README.md +++ b/packages/w3up-client/README.md @@ -406,6 +406,8 @@ sequenceDiagram - [`capability.upload.add`](#capabilityuploadadd) - [`capability.upload.list`](#capabilityuploadlist) - [`capability.upload.remove`](#capabilityuploadremove) + - [`capability.filecoin.offer`](#capabilityfilecoinoffer) + - [`capability.filecoin.info`](#capabilityfilecoininfo) - [Types](#types) - [`Capability`](#capability) - [`CARMetadata`](#carmetadata) @@ -688,6 +690,28 @@ function remove( Remove a upload by root data CID. +### `capability.filecoin.offer` + +```ts +function offer ( + content: CID, + piece: PieceLink, +): Promise +``` + +Offer a Filecoin "piece" to be added to an aggregate that will be offered for Filecoin deal(s). + +### `capability.filecoin.info` + +```ts +function info ( + content: CID, + piece: PieceLink +): Promise +``` + +Get know deals and aggregate info of a Filecoin "piece" previously offered. + ## Types ### `Capability` diff --git a/packages/w3up-client/package.json b/packages/w3up-client/package.json index 616ab88e9..705045e15 100644 --- a/packages/w3up-client/package.json +++ b/packages/w3up-client/package.json @@ -80,9 +80,10 @@ "@ucanto/interface": "^9.0.0", "@ucanto/principal": "^9.0.0", "@ucanto/transport": "^9.0.0", - "@web3-storage/did-mailto": "workspace:^", "@web3-storage/access": "workspace:^", "@web3-storage/capabilities": "workspace:^", + "@web3-storage/did-mailto": "workspace:^", + "@web3-storage/filecoin-client": "workspace:^", "@web3-storage/upload-client": "workspace:^" }, "devDependencies": { @@ -93,6 +94,7 @@ "@types/mocha": "^10.0.1", "@types/node": "^20.8.4", "@ucanto/server": "^9.0.1", + "@web3-storage/data-segment": "^5.0.0", "@web3-storage/eslint-config-w3up": "workspace:^", "assert": "^2.0.0", "c8": "^7.13.0", diff --git a/packages/w3up-client/src/capability/filecoin.js b/packages/w3up-client/src/capability/filecoin.js new file mode 100644 index 000000000..b7ab2b474 --- /dev/null +++ b/packages/w3up-client/src/capability/filecoin.js @@ -0,0 +1,34 @@ +import { Storefront } from '@web3-storage/filecoin-client' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' +import { Base } from '../base.js' + +/** + * Client for interacting with the `filecoin/*` capabilities. + */ +export class FilecoinClient extends Base { + /** + * Offer a Filecoin "piece" to the resource. + * + * @param {import('multiformats').UnknownLink} content + * @param {import('@web3-storage/capabilities/types').PieceLink} piece + */ + async offer(content, piece) { + const conf = await this._invocationConfig([FilecoinCapabilities.offer.can]) + return Storefront.filecoinOffer(conf, content, piece, { + connection: this._serviceConf.filecoin, + }) + } + + /** + * Request info about a content piece in Filecoin deals + * + * @param {import('multiformats').UnknownLink} content + * @param {import('@web3-storage/capabilities/types').PieceLink} [piece] + */ + async info(content, piece) { + const conf = await this._invocationConfig([FilecoinCapabilities.info.can]) + return Storefront.filecoinInfo(conf, content, piece, { + connection: this._serviceConf.filecoin, + }) + } +} diff --git a/packages/w3up-client/src/client.js b/packages/w3up-client/src/client.js index 016719d76..f2662dd59 100644 --- a/packages/w3up-client/src/client.js +++ b/packages/w3up-client/src/client.js @@ -14,6 +14,7 @@ import { StoreClient } from './capability/store.js' import { UploadClient } from './capability/upload.js' import { SpaceClient } from './capability/space.js' import { AccessClient } from './capability/access.js' +import { FilecoinClient } from './capability/filecoin.js' export * as Access from './capability/access.js' export { StoreClient, UploadClient, SpaceClient, AccessClient } @@ -31,6 +32,7 @@ export class Client extends Base { store: new StoreClient(agentData, options), upload: new UploadClient(agentData, options), space: new SpaceClient(agentData, options), + filecoin: new FilecoinClient(agentData, options), } } diff --git a/packages/w3up-client/src/service.js b/packages/w3up-client/src/service.js index 26d00daba..20bb09365 100644 --- a/packages/w3up-client/src/service.js +++ b/packages/w3up-client/src/service.js @@ -26,8 +26,21 @@ export const uploadServiceConnection = connect({ }), }) +export const filecoinServiceURL = new URL('https://up.web3.storage') +export const filecoinServicePrincipal = DID.parse('did:web:web3.storage') + +export const filecoinServiceConnection = connect({ + id: filecoinServicePrincipal, + codec: CAR.outbound, + channel: HTTP.open({ + url: filecoinServiceURL, + method: 'POST', + }), +}) + /** @type {import('./types.js').ServiceConf} */ export const serviceConf = { access: accessServiceConnection, upload: uploadServiceConnection, + filecoin: filecoinServiceConnection, } diff --git a/packages/w3up-client/src/types.ts b/packages/w3up-client/src/types.ts index 5201c7109..c2e8bbe28 100644 --- a/packages/w3up-client/src/types.ts +++ b/packages/w3up-client/src/types.ts @@ -13,6 +13,7 @@ import type { Unit, } from '@ucanto/interface' import { type Client } from './client.js' +import { StorefrontService } from '@web3-storage/filecoin-client/storefront' export * from '@ucanto/interface' export * from '@web3-storage/did-mailto' export type { Agent, CapabilityQuery } from '@web3-storage/access/agent' @@ -28,6 +29,7 @@ export type ProofQuery = Record> export interface ServiceConf { access: ConnectionView upload: ConnectionView + filecoin: ConnectionView } export interface ClientFactoryOptions { diff --git a/packages/w3up-client/test/capability/filecoin.test.js b/packages/w3up-client/test/capability/filecoin.test.js new file mode 100644 index 000000000..1cf0d14ea --- /dev/null +++ b/packages/w3up-client/test/capability/filecoin.test.js @@ -0,0 +1,165 @@ +import assert from 'assert' +import { + create as createServer, + provide, + provideAdvanced, + ok, +} from '@ucanto/server' +import * as CAR from '@ucanto/transport/car' +import * as Signer from '@ucanto/principal/ed25519' +import { Filecoin as FilecoinCapabilities } from '@web3-storage/capabilities' +import { AgentData } from '@web3-storage/access/agent' + +import { randomAggregate, randomCargo } from '../helpers/random.js' +import { mockService, mockServiceConf } from '../helpers/mocks.js' +import { Client } from '../../src/client.js' +import { validateAuthorization } from '../helpers/utils.js' + +describe('FilecoinClient', () => { + describe('offer', () => { + it('should send an offer', async () => { + const service = mockService({ + filecoin: { + offer: provideAdvanced({ + capability: FilecoinCapabilities.offer, + handler: async ({ invocation, context }) => { + const invCap = invocation.capabilities[0] + assert.ok(invCap.nb) + + // Create effect for receipt with self signed queued operation + const submitfx = await FilecoinCapabilities.submit + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + expiration: Infinity, + }) + .delegate() + + const acceptfx = await FilecoinCapabilities.accept + .invoke({ + issuer: context.id, + audience: context.id, + with: context.id.did(), + nb: invCap.nb, + expiration: Infinity, + }) + .delegate() + + return ok({ + piece: invCap.nb.piece, + }) + .fork(submitfx.link()) + .join(acceptfx.link()) + }, + }), + }, + }) + const server = createServer({ + id: await Signer.generate(), + service, + codec: CAR.inbound, + validateAuthorization, + }) + + const alice = new Client(await AgentData.create(), { + // @ts-ignore + serviceConf: await mockServiceConf(server), + }) + + const space = await alice.createSpace('test') + const auth = await space.createAuthorization(alice) + alice.addSpace(auth) + await alice.setCurrentSpace(space.did()) + + const [cargo] = await randomCargo(1, 100) + const res = await alice.capability.filecoin.offer( + cargo.content, + cargo.link + ) + + assert(service.filecoin.offer.called) + assert.equal(service.filecoin.offer.callCount, 1) + assert(res.out.ok) + assert(res.out.ok.piece.equals(cargo.link)) + assert(res.fx.join) + assert(res.fx.fork.length) + }) + }) + describe('info', () => { + it('should get piece info', async () => { + const { pieces, aggregate } = await randomAggregate(10, 100) + const cargo = pieces[0] + // compute proof for piece in aggregate + const proof = aggregate.resolveProof(cargo.link) + if (proof.error) { + throw new Error('could not compute proof') + } + /** @type {import('@web3-storage/capabilities/types').FilecoinInfoSuccess} */ + const filecoinAcceptResponse = { + piece: cargo.link, + deals: [ + { + aggregate: aggregate.link, + provider: 'f1111', + inclusion: { + subtree: proof.ok[0], + index: proof.ok[1], + }, + aux: { + dataType: 0n, + dataSource: { + dealID: 1138n, + }, + }, + }, + ], + } + const service = mockService({ + filecoin: { + info: provide(FilecoinCapabilities.info, ({ invocation }) => { + const invCap = invocation.capabilities[0] + assert.ok(invCap.nb) + + return ok(filecoinAcceptResponse) + }), + }, + }) + const server = createServer({ + id: await Signer.generate(), + service, + codec: CAR.inbound, + validateAuthorization, + }) + + const alice = new Client(await AgentData.create(), { + // @ts-ignore + serviceConf: await mockServiceConf(server), + }) + + const space = await alice.createSpace('test') + const auth = await space.createAuthorization(alice) + alice.addSpace(auth) + await alice.setCurrentSpace(space.did()) + + const res = await alice.capability.filecoin.info( + cargo.content, + cargo.link + ) + + assert(service.filecoin.info.called) + assert.equal(service.filecoin.info.callCount, 1) + assert(res.out.ok) + assert(res.out.ok.piece.equals(cargo.link)) + assert.equal(res.out.ok.deals.length, 1) + assert(res.out.ok.deals[0].aggregate.equals(aggregate.link)) + assert(res.out.ok.deals[0].aux.dataSource.dealID) + assert(res.out.ok.deals[0].provider) + assert.deepEqual(res.out.ok.deals[0].inclusion, { + subtree: proof.ok[0], + index: proof.ok[1], + }) + }) + }) +}) diff --git a/packages/w3up-client/test/helpers/car.js b/packages/w3up-client/test/helpers/car.js index 031b85f10..07aa974db 100644 --- a/packages/w3up-client/test/helpers/car.js +++ b/packages/w3up-client/test/helpers/car.js @@ -22,5 +22,5 @@ export async function toCAR(bytes) { const blob = new Blob(chunks) const cid = await CAR.codec.link(new Uint8Array(await blob.arrayBuffer())) - return Object.assign(blob, { cid, roots: [root] }) + return Object.assign(blob, { cid, roots: [root], bytes }) } diff --git a/packages/w3up-client/test/helpers/mocks.js b/packages/w3up-client/test/helpers/mocks.js index 7f98a3544..89fcbed1d 100644 --- a/packages/w3up-client/test/helpers/mocks.js +++ b/packages/w3up-client/test/helpers/mocks.js @@ -14,6 +14,7 @@ const notImplemented = () => { * upload: Partial * space: Partial * ucan: Partial + * filecoin: Partial * }>} impl */ export function mockService(impl) { @@ -42,6 +43,10 @@ export function mockService(impl) { ucan: { revoke: withCallCount(impl.ucan?.revoke ?? notImplemented), }, + filecoin: { + offer: withCallCount(impl.filecoin?.offer ?? notImplemented), + info: withCallCount(impl.filecoin?.info ?? notImplemented), + }, } } @@ -72,5 +77,5 @@ export async function mockServiceConf(server) { codec: CAR.outbound, channel: server, }) - return { access: connection, upload: connection } + return { access: connection, upload: connection, filecoin: connection } } diff --git a/packages/w3up-client/test/helpers/random.js b/packages/w3up-client/test/helpers/random.js index b7e8aedb4..b1f512707 100644 --- a/packages/w3up-client/test/helpers/random.js +++ b/packages/w3up-client/test/helpers/random.js @@ -1,3 +1,4 @@ +import { Aggregate, Piece } from '@web3-storage/data-segment' import { toCAR } from './car.js' /** @param {number} size */ @@ -29,3 +30,42 @@ export async function randomCAR(size) { const bytes = await randomBytes(size) return toCAR(bytes) } + +/** + * @param {number} length + * @param {number} size + */ +export async function randomCargo(length, size) { + const cars = await Promise.all( + Array.from({ length }).map(() => randomCAR(size)) + ) + + return cars.map((car) => { + const piece = Piece.fromPayload(car.bytes) + + return { + link: piece.link, + height: piece.height, + root: piece.root, + padding: piece.padding, + content: car.cid, + } + }) +} + +/** + * @param {number} length + * @param {number} size + */ +export async function randomAggregate(length, size) { + const pieces = await randomCargo(length, size) + + const aggregateBuild = Aggregate.build({ + pieces, + }) + + return { + pieces, + aggregate: aggregateBuild, + } +} diff --git a/packages/w3up-client/test/test.js b/packages/w3up-client/test/test.js index 5ffe569ae..7bf1f9105 100644 --- a/packages/w3up-client/test/test.js +++ b/packages/w3up-client/test/test.js @@ -42,6 +42,7 @@ export const setup = async () => { serviceConf: { access: context.connection, upload: context.connection, + filecoin: context.connection, }, }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 577ff38e5..f6f92e6dc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,4 +1,4 @@ -lockfileVersion: '6.0' +lockfileVersion: '6.1' settings: autoInstallPeers: true @@ -552,6 +552,9 @@ importers: '@web3-storage/did-mailto': specifier: workspace:^ version: link:../did-mailto + '@web3-storage/filecoin-client': + specifier: workspace:^ + version: link:../filecoin-client '@web3-storage/upload-client': specifier: workspace:^ version: link:../upload-client @@ -574,6 +577,9 @@ importers: '@ucanto/server': specifier: ^9.0.1 version: 9.0.1 + '@web3-storage/data-segment': + specifier: ^5.0.0 + version: 5.0.0 '@web3-storage/eslint-config-w3up': specifier: workspace:^ version: link:../eslint-config-w3up @@ -3567,6 +3573,14 @@ packages: multiformats: 11.0.2 sync-multihash-sha2: 1.0.0 + /@web3-storage/data-segment@5.0.0: + resolution: {integrity: sha512-5CbElsxec2DsKhEHEh3XRGISAyna+bCjKjjvFrLcYyXLCaiSt/nF3ypcllxwjpE4newMUArymGKGzzZnRWL2kg==} + dependencies: + '@ipld/dag-cbor': 9.0.6 + multiformats: 11.0.2 + sync-multihash-sha2: 1.0.0 + dev: true + /@web3-storage/sigv4@1.0.2: resolution: {integrity: sha512-ZUXKK10NmuQgPkqByhb1H3OQxkIM0CIn2BMPhGQw7vQw8WIzrBkk9IJiAVfJ/UVBFrf6uzPbx2lEBLt4diCMnQ==} dependencies: From 9f33fc424594f2c67bf11d821ebeb3514ad996a9 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 8 Nov 2023 16:47:59 +0100 Subject: [PATCH 2/2] chore: address review comments --- packages/capabilities/src/filecoin/lib.js | 6 +- .../capabilities/src/filecoin/storefront.js | 10 +- packages/filecoin-api/src/errors.js | 24 --- .../filecoin-api/src/storefront/service.js | 38 ++-- packages/filecoin-api/test/context/store.js | 8 +- .../filecoin-api/test/services/storefront.js | 187 ------------------ packages/filecoin-client/src/storefront.js | 6 +- .../filecoin-client/test/storefront.test.js | 8 +- packages/w3up-client/README.md | 1 - .../w3up-client/src/capability/filecoin.js | 7 +- .../test/capability/filecoin.test.js | 5 +- 11 files changed, 30 insertions(+), 270 deletions(-) diff --git a/packages/capabilities/src/filecoin/lib.js b/packages/capabilities/src/filecoin/lib.js index 76bc77271..fd4ae8cee 100644 --- a/packages/capabilities/src/filecoin/lib.js +++ b/packages/capabilities/src/filecoin/lib.js @@ -3,13 +3,11 @@ import { Schema } from '@ucanto/validator' /** * @see https://github.com/filecoin-project/FIPs/pull/758/files */ -export const FR32_SHA2_256_TRUNC254_PADDED_BINARY_TREE = /** @type {const} */ ( - 0x1011 -) +const FR32_SHA2_256_TRUNC254_PADDED_BINARY_TREE = /** @type {const} */ (0x1011) /** * @see https://github.com/filecoin-project/FIPs/pull/758/files */ -export const RAW_CODE = /** @type {const} */ (0x55) +const RAW_CODE = /** @type {const} */ (0x55) export const PieceLink = /** @type {import('../types.js').PieceLinkSchema} */ ( Schema.link({ diff --git a/packages/capabilities/src/filecoin/storefront.js b/packages/capabilities/src/filecoin/storefront.js index 0303a6692..53f0cc4b3 100644 --- a/packages/capabilities/src/filecoin/storefront.js +++ b/packages/capabilities/src/filecoin/storefront.js @@ -118,19 +118,17 @@ export const filecoinInfo = capability({ */ with: Schema.did(), nb: Schema.struct({ - /** - * CID of the content that resulted in Filecoin piece. - */ - content: Schema.link(), /** * CID of the piece. + * + * @see https://github.com/filecoin-project/FIPs/pull/758/files */ - piece: PieceLink.optional(), + piece: PieceLink, }), derives: (claim, from) => { return ( and(equalWith(claim, from)) || - and(checkLink(claim.nb.content, from.nb.content, 'nb.content')) || + and(checkLink(claim.nb.piece, from.nb.piece, 'nb.piece')) || ok({}) ) }, diff --git a/packages/filecoin-api/src/errors.js b/packages/filecoin-api/src/errors.js index df3a58db4..94f27277e 100644 --- a/packages/filecoin-api/src/errors.js +++ b/packages/filecoin-api/src/errors.js @@ -48,30 +48,6 @@ export class RecordNotFound extends Server.Failure { } } -export const ContentNotFoundErrorName = /** @type {const} */ ('ContentNotFound') -export class ContentNotFound extends Server.Failure { - get reason() { - return this.message - } - - get name() { - return ContentNotFoundErrorName - } -} - -export const InvalidContentPieceErrorName = /** @type {const} */ ( - 'InvalidContentPiece' -) -export class InvalidContentPiece extends Server.Failure { - get reason() { - return this.message - } - - get name() { - return InvalidContentPieceErrorName - } -} - export const EncodeRecordErrorName = /** @type {const} */ ('EncodeRecordFailed') export class EncodeRecordFailed extends Server.Failure { get reason() { diff --git a/packages/filecoin-api/src/storefront/service.js b/packages/filecoin-api/src/storefront/service.js index 0a2c27ba4..5f93a6177 100644 --- a/packages/filecoin-api/src/storefront/service.js +++ b/packages/filecoin-api/src/storefront/service.js @@ -9,8 +9,7 @@ import * as API from '../types.js' import { QueueOperationFailed, StoreOperationFailed, - ContentNotFound, - InvalidContentPiece, + RecordNotFoundErrorName, } from '../errors.js' /** @@ -238,26 +237,16 @@ async function findDataAggregationProof({ taskStore, receiptStore }, task) { * @returns {Promise | API.UcantoInterface.JoinBuilder>} */ export const filecoinInfo = async ({ capability }, context) => { - const { piece, content } = capability.nb + const { piece } = capability.nb - const queryRecords = await context.pieceStore.query({ content }) - if (queryRecords.error) { - return { error: new StoreOperationFailed(queryRecords.error.message) } - } else if (!queryRecords.ok.length) { + // Get piece in store + const getPiece = await context.pieceStore.get({ piece }) + if (getPiece.error && getPiece.error.name === RecordNotFoundErrorName) { return { - error: new ContentNotFound( - `no piece record was previously stored for content ${content.toString()}` - ), - } - } - if (Boolean(piece) && !queryRecords.ok[0].piece.equals(piece)) { - return { - error: new InvalidContentPiece( - `received piece ${piece?.toString()} is not the same as previously computed ${ - queryRecords.ok[0].piece - } for content ${content.toString()}` - ), + error: getPiece.error, } + } else if (getPiece.error) { + return { error: new StoreOperationFailed(getPiece.error.message) } } // Check if `piece/accept` receipt exists to get to know aggregate where it is included on a deal @@ -267,8 +256,8 @@ export const filecoinInfo = async ({ capability }, context) => { audience: context.id, with: context.id.did(), nb: { - piece: queryRecords.ok[0].piece, - content, + piece, + content: getPiece.ok.content, }, expiration: Infinity, }) @@ -278,10 +267,9 @@ export const filecoinInfo = async ({ capability }, context) => { pieceAcceptInvocation.link() ) if (pieceAcceptReceiptGet.error) { - // TODO: see receipt chain to report processing /** @type {API.UcantoInterface.OkBuilder} */ const processingResult = Server.ok({ - piece: queryRecords.ok[0].piece, + piece, deals: [], }) return processingResult @@ -308,14 +296,14 @@ export const filecoinInfo = async ({ capability }, context) => { // Should not happen if there is `piece/accept` receipt return { error: new Server.Failure( - `no deals were obtained for aggregate ${pieceAcceptOut.aggregate} where piece ${queryRecords.ok[0].piece} is included` + `no deals were obtained for aggregate ${pieceAcceptOut.aggregate} where piece ${piece} is included` ), } } /** @type {API.UcantoInterface.OkBuilder} */ const result = Server.ok({ - piece: queryRecords.ok[0].piece, + piece, deals: deals.map(([dealId, dealDetails]) => ({ aggregate: pieceAcceptOut.aggregate, provider: dealDetails.provider, diff --git a/packages/filecoin-api/test/context/store.js b/packages/filecoin-api/test/context/store.js index 51d19b1fa..02f98e61e 100644 --- a/packages/filecoin-api/test/context/store.js +++ b/packages/filecoin-api/test/context/store.js @@ -1,5 +1,5 @@ import * as API from '../../src/types.js' -import { RecordNotFound, StoreOperationFailed } from '../../src/errors.js' +import { StoreOperationFailed, RecordNotFound } from '../../src/errors.js' /** * @typedef {import('../../src/types.js').StorePutError} StorePutError @@ -47,7 +47,7 @@ export class Store { const t = this.getFn(this.items, item) if (!t) { return { - error: new RecordNotFound(), + error: new RecordNotFound('not found'), } } return { @@ -85,7 +85,7 @@ export class Store { const t = this.queryFn(this.items, search) if (!t) { return { - error: new RecordNotFound(), + error: new RecordNotFound('not found'), } } return { @@ -123,7 +123,7 @@ export class UpdatableStore extends Store { const t = this.updateFn(this.items, key, item) if (!t) { return { - error: new RecordNotFound(), + error: new RecordNotFound('not found'), } } return { diff --git a/packages/filecoin-api/test/services/storefront.js b/packages/filecoin-api/test/services/storefront.js index 347c2a711..fdc9c7c24 100644 --- a/packages/filecoin-api/test/services/storefront.js +++ b/packages/filecoin-api/test/services/storefront.js @@ -10,8 +10,6 @@ import * as StorefrontApi from '../../src/storefront/api.js' import { createServer, connect } from '../../src/storefront/service.js' import { - ContentNotFoundErrorName, - InvalidContentPieceErrorName, QueueOperationErrorName, StoreOperationErrorName, } from '../../src/errors.js' @@ -535,7 +533,6 @@ export const test = { with: agent.did(), nb: { piece: piece.link.link(), - content: piece.content.link(), }, }) @@ -587,149 +584,6 @@ export const test = { service ).connection - return { - ...context, - service, - dealTrackerService: { - connection: dealTrackerConnection, - invocationConfig: { - issuer: context.id, - with: context.id.did(), - audience: dealTrackerSigner, - }, - }, - } - } - ), - 'filecoin/info gets aggregate where piece was included together with deals and inclusion proof by giving content': - wichMockableContext( - async (assert, context) => { - const { agent, aggregator, dealer } = await getServiceContext() - const group = context.id.did() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Create piece and aggregate for test - const { aggregate, pieces } = await randomAggregate(10, 128) - const piece = pieces[0] - const offer = pieces.map((p) => p.link) - const piecesBlock = await CBOR.write(offer) - - // Store piece into store - const putRes = await context.pieceStore.put({ - piece: piece.link.link(), - content: piece.content.link(), - group: context.id.did(), - status: 'submitted', - insertedAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - }) - assert.ok(putRes.ok) - - // Create inclusion proof for test - const inclusionProof = aggregate.resolveProof(piece.link) - if (inclusionProof.error) { - throw new Error('could not compute inclusion proof') - } - - // Create invocations and receipts for chain into DealDataProof - const dealMetadata = { - dataType: 0n, - dataSource: { - dealID: 111n, - }, - } - const { invocations, receipts } = - await createInvocationsAndReceiptsForDealDataProofChain({ - storefront: context.id, - aggregator, - dealer, - aggregate: aggregate.link, - group, - piece: piece.link, - content: piece.content, - piecesBlock, - inclusionProof: { - subtree: inclusionProof.ok[0], - index: inclusionProof.ok[1], - }, - aggregateAcceptStatus: { - ...dealMetadata, - aggregate: aggregate.link, - }, - }) - - const storedInvocationsAndReceiptsRes = - await storeInvocationsAndReceipts({ - invocations, - receipts, - taskStore: context.taskStore, - receiptStore: context.receiptStore, - }) - assert.ok(storedInvocationsAndReceiptsRes.ok) - - // agent invocation - const filecoinInfoInv = Filecoin.info.invoke({ - issuer: agent, - audience: connection.id, - with: agent.did(), - nb: { - // Piece was previously set - piece: undefined, - content: piece.content.link(), - }, - }) - - const response = await filecoinInfoInv.execute(connection) - if (response.out.error) { - throw new Error('invocation failed', { cause: response.out.error }) - } - assert.ok(response.out.ok) - assert.ok(response.out.ok.piece.equals(piece.link.link())) - assert.equal(response.out.ok.deals.length, 1) - assert.ok(response.out.ok.deals[0].aggregate.equals(aggregate.link)) - assert.deepEqual( - BigInt(response.out.ok.deals[0].aux.dataType), - dealMetadata.dataType - ) - assert.deepEqual( - BigInt(response.out.ok.deals[0].aux.dataSource.dealID), - dealMetadata.dataSource.dealID - ) - assert.ok(response.out.ok.deals[0].inclusion.index) - assert.ok(response.out.ok.deals[0].inclusion.subtree) - }, - async (context) => { - /** - * Mock deal tracker to return deals - */ - const dealTrackerSigner = await Signer.generate() - const service = mockService({ - deal: { - info: Server.provideAdvanced({ - capability: DealTrackerCaps.dealInfo, - handler: async ({ invocation, context }) => { - /** @type {API.UcantoInterface.OkBuilder} */ - const result = Server.ok({ - deals: { - 111: { - provider: 'f11111', - }, - }, - }) - - return result - }, - }), - }, - }) - const dealTrackerConnection = getConnection( - dealTrackerSigner, - service - ).connection - return { ...context, service, @@ -762,53 +616,12 @@ export const test = { with: agent.did(), nb: { piece: piece.link, - content: piece.content.link(), }, }) const response = await filecoinInfoInv.execute(connection) assert.ok(response.out.error) - assert.equal(response.out.error?.name, ContentNotFoundErrorName) }, - 'filecoin/info fails if piece provided for content is different from the previously computed': - async (assert, context) => { - const { agent } = await getServiceContext() - const connection = connect({ - id: context.id, - channel: createServer(context), - }) - - // Create piece and aggregate for test - const { pieces } = await randomAggregate(10, 128) - const piece = pieces[0] - - // Store piece into store - const putRes = await context.pieceStore.put({ - piece: piece.link.link(), - content: piece.content.link(), - group: context.id.did(), - status: 'submitted', - insertedAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - }) - assert.ok(putRes.ok) - - // agent invocation - const filecoinInfoInv = Filecoin.info.invoke({ - issuer: agent, - audience: connection.id, - with: agent.did(), - nb: { - // give wrong piece for content - piece: pieces[1].link, - content: piece.content.link(), - }, - }) - - const response = await filecoinInfoInv.execute(connection) - assert.ok(response.out.error) - assert.equal(response.out.error?.name, InvalidContentPieceErrorName) - }, } /** diff --git a/packages/filecoin-client/src/storefront.js b/packages/filecoin-client/src/storefront.js index 27be0f795..06cd36555 100644 --- a/packages/filecoin-client/src/storefront.js +++ b/packages/filecoin-client/src/storefront.js @@ -153,13 +153,11 @@ export async function filecoinAccept( * in Filecoin. It issues a signed receipt of the execution result. * * @param {import('./types.js').InvocationConfig} conf - Configuration - * @param {import('multiformats').UnknownLink} content - * @param {import('@web3-storage/data-segment').PieceLink} [piece] + * @param {import('@web3-storage/data-segment').PieceLink} piece * @param {import('./types.js').RequestOptions} [options] */ export async function filecoinInfo( { issuer, with: resource, proofs, audience }, - content, piece, options = {} ) { @@ -172,11 +170,9 @@ export async function filecoinInfo( audience: audience ?? services.STOREFRONT.principal, with: resource, nb: { - content, piece, }, proofs, - expiration: Infinity, }) return await invocation.execute(conn) diff --git a/packages/filecoin-client/test/storefront.test.js b/packages/filecoin-client/test/storefront.test.js index e490b49cd..557ed0507 100644 --- a/packages/filecoin-client/test/storefront.test.js +++ b/packages/filecoin-client/test/storefront.test.js @@ -284,12 +284,9 @@ describe('storefront', () => { assert.strictEqual(invCap.can, StorefrontCaps.filecoinInfo.can) assert.equal(invCap.with, invocation.issuer.did()) assert.ok(invCap.nb) - const { content, piece } = invCap.nb + const { piece } = invCap.nb // piece link - assert.ok(piece) - assert.ok(piece?.equals(cargo.link.link())) - // content - assert.ok(content.equals(cargo.content.link())) + assert.ok(piece.equals(cargo.link.link())) return Server.ok({ piece, @@ -306,7 +303,6 @@ describe('storefront', () => { with: agent.did(), audience: storefrontService, }, - cargo.content, cargo.link, { connection: getConnection(service).connection } ) diff --git a/packages/w3up-client/README.md b/packages/w3up-client/README.md index 86e51fcd3..699e60f8f 100644 --- a/packages/w3up-client/README.md +++ b/packages/w3up-client/README.md @@ -705,7 +705,6 @@ Offer a Filecoin "piece" to be added to an aggregate that will be offered for Fi ```ts function info ( - content: CID, piece: PieceLink ): Promise ``` diff --git a/packages/w3up-client/src/capability/filecoin.js b/packages/w3up-client/src/capability/filecoin.js index b7ab2b474..9a5c8e8d1 100644 --- a/packages/w3up-client/src/capability/filecoin.js +++ b/packages/w3up-client/src/capability/filecoin.js @@ -22,12 +22,11 @@ export class FilecoinClient extends Base { /** * Request info about a content piece in Filecoin deals * - * @param {import('multiformats').UnknownLink} content - * @param {import('@web3-storage/capabilities/types').PieceLink} [piece] + * @param {import('@web3-storage/capabilities/types').PieceLink} piece */ - async info(content, piece) { + async info(piece) { const conf = await this._invocationConfig([FilecoinCapabilities.info.can]) - return Storefront.filecoinInfo(conf, content, piece, { + return Storefront.filecoinInfo(conf, piece, { connection: this._serviceConf.filecoin, }) } diff --git a/packages/w3up-client/test/capability/filecoin.test.js b/packages/w3up-client/test/capability/filecoin.test.js index 1cf0d14ea..53427c4fc 100644 --- a/packages/w3up-client/test/capability/filecoin.test.js +++ b/packages/w3up-client/test/capability/filecoin.test.js @@ -143,10 +143,7 @@ describe('FilecoinClient', () => { alice.addSpace(auth) await alice.setCurrentSpace(space.did()) - const res = await alice.capability.filecoin.info( - cargo.content, - cargo.link - ) + const res = await alice.capability.filecoin.info(cargo.link) assert(service.filecoin.info.called) assert.equal(service.filecoin.info.callCount, 1)