diff --git a/packages/interop/src/fixtures/load-fixtures.ts b/packages/interop/src/fixtures/load-fixtures.ts index 5416149..9fa5c29 100644 --- a/packages/interop/src/fixtures/load-fixtures.ts +++ b/packages/interop/src/fixtures/load-fixtures.ts @@ -8,18 +8,8 @@ import { path as kuboPath } from 'kubo' */ export async function loadFixtures (IPFS_PATH = undefined): Promise { const kuboBinary = process.env.KUBO_BINARY ?? kuboPath() - /** - * fast-glob does not like windows paths, see https://github.com/mrmlnc/fast-glob/issues/237 - * fast-glob performs search from process.cwd() by default, which will be: - * 1. the root of the monorepo when running tests in CI - * 2. the package root when running tests in the package directory - */ - let globRoot = process.cwd().replace(/\\/g, '/') - if (!globRoot.includes('packages/interop')) { - // we only want car files from the interop package - globRoot = [...globRoot.split('/'), 'packages/interop'].join('/') - } - for (const carFile of await fg.glob('src/fixtures/data/*.car', { cwd: globRoot })) { + + for (const carFile of await fg.glob('**/fixtures/data/*.car')) { await $({ env: { IPFS_PATH } })`${kuboBinary} dag import --pin-roots=false --offline ${carFile}` } } diff --git a/packages/interop/src/json.spec.ts b/packages/interop/src/json.spec.ts index ce1f519..97f14b1 100644 --- a/packages/interop/src/json.spec.ts +++ b/packages/interop/src/json.spec.ts @@ -14,7 +14,9 @@ describe('@helia/verified-fetch - json', () => { // child2: QmWNBJX6fZyNTLWNYBHxAHpBctCP43R2zeqV2G8uavqFZn // partial JSON verifiedFetch = await createVerifiedFetch({ gateways: ['http://127.0.0.1:8180'], - routers: [] + routers: ['http://127.0.0.1:8180'], + allowInsecure: true, + allowLocal: true }) }) @@ -23,7 +25,10 @@ describe('@helia/verified-fetch - json', () => { }) it('handles UnixFS-chunked JSON file', async () => { - const resp = await verifiedFetch(CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')) + const resp = await verifiedFetch(CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), { + allowLocal: true, + allowInsecure: true + }) expect(resp).to.be.ok() const jsonObj = await resp.json() expect(jsonObj).to.be.ok() diff --git a/packages/interop/src/unixfs-dir.spec.ts b/packages/interop/src/unixfs-dir.spec.ts index 013609d..12e104c 100644 --- a/packages/interop/src/unixfs-dir.spec.ts +++ b/packages/interop/src/unixfs-dir.spec.ts @@ -10,9 +10,10 @@ describe('@helia/verified-fetch - unixfs directory', () => { before(async () => { verifiedFetch = await createVerifiedFetch({ gateways: ['http://127.0.0.1:8180'], - routers: [] + routers: [], + allowInsecure: true, + allowLocal: true }) - verifiedFetch = await createVerifiedFetch() }) after(async () => { @@ -26,7 +27,11 @@ describe('@helia/verified-fetch - unixfs directory', () => { 'http://example.com/ipfs/bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q' ].forEach((url: string) => { it(`request to unixfs directory with ${url} should return a 301 with a trailing slash`, async () => { - const response = await verifiedFetch(url, { redirect: 'manual' }) + const response = await verifiedFetch(url, { + redirect: 'manual', + allowLocal: true, + allowInsecure: true + }) expect(response).to.be.ok() expect(response.status).to.equal(301) expect(response.headers.get('location')).to.equal(`${url}/`) @@ -38,20 +43,29 @@ describe('@helia/verified-fetch - unixfs directory', () => { describe('XKCD Barrel Part 1', () => { it('fails to load when passed the root', async () => { // The spec says we should generate HTML with directory listings, but we don't do that yet, so expect a failure - const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR') + const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR', { + allowLocal: true, + allowInsecure: true + }) expect(resp).to.be.ok() expect(resp.status).to.equal(501) // TODO: we should do a directory listing instead }) it('can return a string for unixfs pathed data', async () => { - const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1 - alt.txt') + const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1 - alt.txt', { + allowLocal: true, + allowInsecure: true + }) expect(resp).to.be.ok() const text = await resp.text() expect(text).to.equal('Don\'t we all.') }) it('can return an image for unixfs pathed data', async () => { - const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1.png') + const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1.png', { + allowLocal: true, + allowInsecure: true + }) expect(resp).to.be.ok() const imgData = await resp.blob() expect(imgData).to.be.ok() @@ -64,7 +78,9 @@ describe('@helia/verified-fetch - unixfs directory', () => { await verifiedFetch.stop() verifiedFetch = await createVerifiedFetch({ gateways: ['http://127.0.0.1:8180'], - routers: [] + routers: ['http://127.0.0.1:8180'], + allowInsecure: true, + allowLocal: true }, { contentTypeParser: (bytes) => { return filetypemime(bytes)?.[0] @@ -73,7 +89,10 @@ describe('@helia/verified-fetch - unixfs directory', () => { }) it('can return an image content-type for unixfs pathed data', async () => { - const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1.png') + const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1.png', { + allowLocal: true, + allowInsecure: true + }) // tediously this is actually a jpeg file with a .png extension expect(resp.headers.get('content-type')).to.equal('image/jpeg') }) @@ -82,7 +101,10 @@ describe('@helia/verified-fetch - unixfs directory', () => { // from https://github.com/ipfs/gateway-conformance/blob/193833b91f2e9b17daf45c84afaeeae61d9d7c7e/fixtures/trustless_gateway_car/single-layer-hamt-with-multi-block-files.car describe('HAMT-sharded directory', () => { it('loads path /ipfs/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt', async () => { - const resp = await verifiedFetch('ipfs://bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt') + const resp = await verifiedFetch('ipfs://bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt', { + allowLocal: true, + allowInsecure: true + }) expect(resp).to.be.ok() const text = await resp.text() // npx kubo@0.25.0 cat '/ipfs/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt' diff --git a/packages/interop/src/websites.spec.ts b/packages/interop/src/websites.spec.ts index 9ca6bad..ff2036d 100644 --- a/packages/interop/src/websites.spec.ts +++ b/packages/interop/src/websites.spec.ts @@ -10,7 +10,9 @@ describe('@helia/verified-fetch - websites', () => { // 2024-01-22 CID for _dnslink.helia-identify.on.fleek.co verifiedFetch = await createVerifiedFetch({ gateways: ['http://127.0.0.1:8180'], - routers: [] + routers: ['http://127.0.0.1:8180'], + allowInsecure: true, + allowLocal: true }) }) @@ -19,7 +21,10 @@ describe('@helia/verified-fetch - websites', () => { }) it('loads index.html when passed helia-identify.on.fleek.co root CID', async () => { - const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv') + const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv', { + allowLocal: true, + allowInsecure: true + }) expect(resp).to.be.ok() const html = await resp.text() expect(html).to.be.ok() @@ -27,7 +32,10 @@ describe('@helia/verified-fetch - websites', () => { }) it('loads helia-identify.on.fleek.co index.html directly ', async () => { - const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv/index.html') + const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv/index.html', { + allowLocal: true, + allowInsecure: true + }) expect(resp).to.be.ok() const html = await resp.text() expect(html).to.be.ok() @@ -52,7 +60,9 @@ describe('@helia/verified-fetch - websites', () => { before(async () => { verifiedFetch = await createVerifiedFetch({ gateways: ['http://127.0.0.1:8180'], - routers: [] + routers: ['http://127.0.0.1:8180'], + allowInsecure: true, + allowLocal: true }) }) @@ -61,7 +71,10 @@ describe('@helia/verified-fetch - websites', () => { }) it('loads index.html when passed fake-blog.libp2p.io root CID', async () => { - const resp = await verifiedFetch('ipfs://QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw') + const resp = await verifiedFetch('ipfs://QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw', { + allowLocal: true, + allowInsecure: true + }) expect(resp).to.be.ok() const html = await resp.text() expect(html).to.be.ok() diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index 2830a83..9e4c048 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -122,17 +122,20 @@ You can see variations of Helia and js-libp2p configuration options at delegatedHTTPRouting(routerUrl)) + ] }) ) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index e7238ce..b8e1419 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -57,12 +57,12 @@ "release": "aegir release" }, "dependencies": { - "@helia/block-brokers": "^2.1.0", - "@helia/car": "^3.1.3", - "@helia/http": "^1.0.4", - "@helia/interface": "^4.2.0", - "@helia/ipns": "^7.2.1", - "@helia/routers": "^1.0.3", + "@helia/block-brokers": "^3.0.0", + "@helia/car": "^3.1.5", + "@helia/http": "^1.0.7", + "@helia/interface": "^4.3.0", + "@helia/ipns": "^7.2.2", + "@helia/routers": "^1.1.0", "@ipld/dag-cbor": "^9.2.0", "@ipld/dag-json": "^10.2.0", "@ipld/dag-pb": "^4.1.0", @@ -79,17 +79,17 @@ "it-pipe": "^3.0.1", "it-tar": "^6.0.5", "it-to-browser-readablestream": "^2.0.6", + "lru-cache": "^10.2.0", "multiformats": "^13.1.0", "progress-events": "^1.0.0", "uint8arrays": "^5.0.3" }, "devDependencies": { - "@helia/car": "^3.1.3", - "@helia/dag-cbor": "^3.0.3", - "@helia/dag-json": "^3.0.3", - "@helia/json": "^3.0.3", - "@helia/unixfs": "^3.0.4", - "@helia/utils": "^0.2.0", + "@helia/dag-cbor": "^3.0.4", + "@helia/dag-json": "^3.0.4", + "@helia/json": "^3.0.4", + "@helia/unixfs": "^3.0.6", + "@helia/utils": "^0.3.0", "@ipld/car": "^5.3.0", "@libp2p/interface-compliance-tests": "^5.3.4", "@libp2p/logger": "^4.0.9", @@ -100,7 +100,7 @@ "blockstore-core": "^4.4.1", "browser-readablestream-to-it": "^2.0.5", "datastore-core": "^9.2.9", - "helia": "^4.1.1", + "helia": "^4.2.1", "ipfs-unixfs-importer": "^15.2.5", "ipns": "^9.1.0", "it-all": "^3.0.4", diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index d68d3d1..25c8277 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -90,22 +90,25 @@ * * The [helia](https://www.npmjs.com/package/helia) module is configured with a libp2p node that is suited for decentralized applications, alternatively [@helia/http](https://www.npmjs.com/package/@helia/http) is available which uses HTTP gateways for all network operations. * - * See variations of [Helia and js-libp2p configuration options](https://helia.io/interfaces/helia.HeliaInit.html) + * You can see variations of Helia and js-libp2p configuration options at . * * ```typescript * import { trustlessGateway } from '@helia/block-brokers' * import { createHeliaHTTP } from '@helia/http' - * import { delegatedHTTPRouting } from '@helia/routers' + * import { delegatedHTTPRouting, httpGatewayRouting } from '@helia/routers' * import { createVerifiedFetch } from '@helia/verified-fetch' * * const fetch = await createVerifiedFetch( * await createHeliaHTTP({ * blockBrokers: [ - * trustlessGateway({ + * trustlessGateway() + * ], + * routers: [ + * delegatedHTTPRouting('http://delegated-ipfs.dev'), + * httpGatewayRouting({ * gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] * }) - * ], - * routers: ['http://delegated-ipfs.dev'].map((routerUrl) => delegatedHTTPRouting(routerUrl)) + * ] * }) * ) * @@ -588,12 +591,12 @@ * 1. `TypeError` - If the resource argument is not a string, CID, or CID string. * 2. `TypeError` - If the options argument is passed and not an object. * 3. `TypeError` - If the options argument is passed and is malformed. - * 4. `AbortError` - If the content request is aborted due to user aborting provided AbortSignal. + * 4. `AbortError` - If the content request is aborted due to user aborting provided AbortSignal. Note that this is a `AbortError` from `@libp2p/interface` and not the standard `AbortError` from the Fetch API. */ import { trustlessGateway } from '@helia/block-brokers' import { createHeliaHTTP } from '@helia/http' -import { delegatedHTTPRouting } from '@helia/routers' +import { delegatedHTTPRouting, httpGatewayRouting } from '@helia/routers' import { dns } from '@multiformats/dns' import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' import type { GetBlockProgressEvents, Helia } from '@helia/interface' @@ -647,6 +650,31 @@ export interface CreateVerifiedFetchInit { * @default [dnsJsonOverHttps('https://cloudflare-dns.com/dns-query'),dnsJsonOverHttps('https://dns.google/resolve')] */ dnsResolvers?: DNSResolver[] | DNSResolvers + + /** + * By default we will not connect to any HTTP Gateways providers over local or + * loopback addresses, this is because they are typically running on remote + * peers that have published private addresses by mistate. + * + * Pass `true` here to connect to local Gateways as well, this may be useful + * in testing environments. + * + * @default false + */ + allowLocal?: boolean + + /** + * By default we will not connect to any gateways over HTTP addresses, + * requring HTTPS connections instead. This is because it will cause + * "mixed-content" errors to appear in the console when running in secure + * browser contexts. + * + * Pass `true` here to connect to insecure Gateways as well, this may be + * useful in testing environments. + * + * @default false + */ + allowInsecure?: boolean } export interface CreateVerifiedFetchOptions { @@ -659,6 +687,22 @@ export interface CreateVerifiedFetchOptions { * @default undefined */ contentTypeParser?: ContentTypeParser + + /** + * Blockstore sessions are cached for reuse with requests with the same + * base URL or CID. This parameter controls how many to cache. Once this limit + * is reached older/less used sessions will be evicted from the cache. + * + * @default 100 + */ + sessionCacheSize?: number + + /** + * How long each blockstore session should stay in the cache for. + * + * @default 60000 + */ + sessionTTLms?: number } /** @@ -698,6 +742,46 @@ export type VerifiedFetchProgressEvents = * progress events. */ export interface VerifiedFetchInit extends RequestInit, ProgressOptions { + /** + * If true, try to create a blockstore session - this can reduce overall + * network traffic by first querying for a set of peers that have the data we + * wish to retrieve. Subsequent requests for data using the session will only + * be sent to those peers, unless they don't have the data, in which case + * further peers will be added to the session. + * + * Sessions are cached based on the CID/IPNS name they attempt to access. That + * is, requests for `https://qmfoo.ipfs.localhost/bar.txt` and + * `https://qmfoo.ipfs.localhost/baz.txt` would use the same session, if this + * argument is true for both fetch requests. + * + * @default true + */ + session?: boolean + + /** + * By default we will not connect to any HTTP Gateways providers over local or + * loopback addresses, this is because they are typically running on remote + * peers that have published private addresses by mistate. + * + * Pass `true` here to connect to local Gateways as well, this may be useful + * in testing environments. + * + * @default false + */ + allowLocal?: boolean + + /** + * By default we will not connect to any gateways over HTTP addresses, + * requring HTTPS connections instead. This is because it will cause + * "mixed-content" errors to appear in the console when running in secure + * browser contexts. + * + * Pass `true` here to connect to insecure Gateways as well, this may be + * useful in testing environments. + * + * @default false + */ + allowInsecure?: boolean } /** @@ -708,10 +792,16 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni init = await createHeliaHTTP({ blockBrokers: [ trustlessGateway({ - gateways: init?.gateways + allowInsecure: init?.allowInsecure, + allowLocal: init?.allowLocal + }) + ], + routers: [ + ...(init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl)), + httpGatewayRouting({ + gateways: init?.gateways ?? [] }) ], - routers: (init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl)), dns: createDns(init?.dnsResolvers) }) } diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts index ef39bb4..63e761d 100644 --- a/packages/verified-fetch/src/utils/parse-url-string.ts +++ b/packages/verified-fetch/src/utils/parse-url-string.ts @@ -71,7 +71,7 @@ function matchUrlGroupsGuard (groups?: null | { [key in string]: string; } | Mat (queryString == null || typeof queryString === 'string') } -function matchURLString (urlString: string): MatchUrlGroups { +export function matchURLString (urlString: string): MatchUrlGroups { for (const pattern of [URL_REGEX, PATH_REGEX, PATH_GATEWAY_REGEX, SUBDOMAIN_GATEWAY_REGEX]) { const match = urlString.match(pattern) diff --git a/packages/verified-fetch/src/utils/resource-to-cache-key.ts b/packages/verified-fetch/src/utils/resource-to-cache-key.ts new file mode 100644 index 0000000..3a8f9be --- /dev/null +++ b/packages/verified-fetch/src/utils/resource-to-cache-key.ts @@ -0,0 +1,30 @@ +import { CID } from 'multiformats/cid' +import { matchURLString } from './parse-url-string.js' + +/** + * Takes a resource and returns a session cache key as an IPFS or IPNS path with + * any trailing segments removed. + * + * E.g. + * + * - Qmfoo -> /ipfs/Qmfoo + * - https://Qmfoo.ipfs.gateway.org -> /ipfs/Qmfoo + * - https://gateway.org/ipfs/Qmfoo -> /ipfs/Qmfoo + * - https://gateway.org/ipfs/Qmfoo/bar.txt -> /ipfs/Qmfoo + * - etc + */ +export function resourceToSessionCacheKey (url: string | CID): string { + const cid = CID.asCID(url) + + if (cid != null) { + return `ipfs://${cid}` + } + + try { + return `ipfs://${CID.parse(url.toString())}` + } catch {} + + const { protocol, cidOrPeerIdOrDnsLink } = matchURLString(url.toString()) + + return `${protocol}://${cidOrPeerIdOrDnsLink}` +} diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index fe7600e..e5c5364 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -9,6 +9,7 @@ import { peerIdFromString } from '@libp2p/peer-id' import { Key } from 'interface-datastore' import { exporter } from 'ipfs-unixfs-exporter' import toBrowserReadableStream from 'it-to-browser-readablestream' +import { LRUCache } from 'lru-cache' import { code as jsonCode } from 'multiformats/codecs/json' import { code as rawCode } from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' @@ -24,34 +25,43 @@ import { getResolvedAcceptHeader } from './utils/get-resolved-accept-header.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { tarStream } from './utils/get-tar-stream.js' import { parseResource } from './utils/parse-resource.js' +import { resourceToSessionCacheKey } from './utils/resource-to-cache-key.js' import { setCacheControlHeader } from './utils/response-headers.js' import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse, badRangeResponse, okRangeResponse, badGatewayResponse, notFoundResponse } from './utils/responses.js' import { selectOutputType } from './utils/select-output-type.js' import { isObjectNode, walkPath } from './utils/walk-path.js' -import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' +import type { CIDDetail, ContentTypeParser, CreateVerifiedFetchOptions, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' import type { RequestFormatShorthand } from './types.js' import type { ParsedUrlStringResults } from './utils/parse-url-string' -import type { Helia } from '@helia/interface' -import type { DNSResolver } from '@multiformats/dns/resolvers' +import type { Helia, SessionBlockstore } from '@helia/interface' +import type { Blockstore } from 'interface-blockstore' import type { ObjectNode, UnixFSEntry } from 'ipfs-unixfs-exporter' import type { CID } from 'multiformats/cid' +const SESSION_CACHE_MAX_SIZE = 100 +const SESSION_CACHE_TTL_MS = 60 * 1000 + interface VerifiedFetchComponents { helia: Helia ipns?: IPNS } -/** - * Potential future options for the VerifiedFetch constructor. - */ -interface VerifiedFetchInit { - contentTypeParser?: ContentTypeParser - dnsResolvers?: DNSResolver[] -} - interface FetchHandlerFunctionArg { cid: CID path: string + + /** + * A key for use with the blockstore session cache + */ + cacheKey: string + + /** + * Whether to use a session during fetch operations + * + * @default true + */ + session: boolean + options?: Omit & AbortOptions /** @@ -129,15 +139,38 @@ export class VerifiedFetch { private readonly ipns: IPNS private readonly log: Logger private readonly contentTypeParser: ContentTypeParser | undefined + private readonly blockstoreSessions: LRUCache - constructor ({ helia, ipns }: VerifiedFetchComponents, init?: VerifiedFetchInit) { + constructor ({ helia, ipns }: VerifiedFetchComponents, init?: CreateVerifiedFetchOptions) { this.helia = helia this.log = helia.logger.forComponent('helia:verified-fetch') this.ipns = ipns ?? heliaIpns(helia) this.contentTypeParser = init?.contentTypeParser + this.blockstoreSessions = new LRUCache({ + max: init?.sessionCacheSize ?? SESSION_CACHE_MAX_SIZE, + ttl: init?.sessionTTLms ?? SESSION_CACHE_TTL_MS, + dispose: (store) => { + store.close() + } + }) this.log.trace('created VerifiedFetch instance') } + private getBlockstore (root: CID, key: string, useSession: boolean, options?: AbortOptions): Blockstore { + if (!useSession) { + return this.helia.blockstore + } + + let session = this.blockstoreSessions.get(key) + + if (session == null) { + session = this.helia.blockstore.createSession(root, options) + this.blockstoreSessions.set(key, session) + } + + return session + } + /** * Accepts an `ipns://...` URL as a string and returns a `Response` containing * a raw IPNS record. @@ -178,8 +211,9 @@ export class VerifiedFetch { * Accepts a `CID` and returns a `Response` with a body stream that is a CAR * of the `DAG` referenced by the `CID`. */ - private async handleCar ({ resource, cid, options }: FetchHandlerFunctionArg): Promise { - const c = car(this.helia) + private async handleCar ({ resource, cid, session, cacheKey, options }: FetchHandlerFunctionArg): Promise { + const blockstore = this.getBlockstore(cid, cacheKey, session, options) + const c = car({ blockstore, dagWalkers: this.helia.dagWalkers }) const stream = toBrowserReadableStream(c.stream(cid, options)) const response = okResponse(resource, stream) @@ -192,12 +226,13 @@ export class VerifiedFetch { * Accepts a UnixFS `CID` and returns a `.tar` file containing the file or * directory structure referenced by the `CID`. */ - private async handleTar ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise { + private async handleTar ({ resource, cid, path, session, cacheKey, options }: FetchHandlerFunctionArg): Promise { if (cid.code !== dagPbCode && cid.code !== rawCode) { return notAcceptableResponse('only UnixFS data can be returned in a TAR file') } - const stream = toBrowserReadableStream(tarStream(`/ipfs/${cid}/${path}`, this.helia.blockstore, options)) + const blockstore = this.getBlockstore(cid, cacheKey, session, options) + const stream = toBrowserReadableStream(tarStream(`/ipfs/${cid}/${path}`, blockstore, options)) const response = okResponse(resource, stream) response.headers.set('content-type', 'application/x-tar') @@ -205,9 +240,10 @@ export class VerifiedFetch { return response } - private async handleJson ({ resource, cid, path, accept, options }: FetchHandlerFunctionArg): Promise { + private async handleJson ({ resource, cid, path, accept, session, cacheKey, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) - const block = await this.helia.blockstore.get(cid, options) + const blockstore = this.getBlockstore(cid, cacheKey, session, options) + const block = await blockstore.get(cid, options) let body: string | Uint8Array if (accept === 'application/vnd.ipld.dag-cbor' || accept === 'application/cbor') { @@ -231,14 +267,15 @@ export class VerifiedFetch { return response } - private async handleDagCbor ({ resource, cid, path, accept, options }: FetchHandlerFunctionArg): Promise { + private async handleDagCbor ({ resource, cid, path, accept, session, cacheKey, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) let terminalElement: ObjectNode | undefined let ipfsRoots: CID[] | undefined + const blockstore = this.getBlockstore(cid, cacheKey, session, options) // need to walk path, if it exists, to get the terminal element try { - const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options) + const pathDetails = await walkPath(blockstore, `${cid.toString()}/${path}`, options) ipfsRoots = pathDetails.ipfsRoots const potentialTerminalElement = pathDetails.terminalElement if (potentialTerminalElement == null) { @@ -256,7 +293,7 @@ export class VerifiedFetch { this.log.error('error walking path %s', path, err) return badGatewayResponse(resource, 'Error walking path') } - const block = terminalElement?.node ?? await this.helia.blockstore.get(cid, options) + const block = terminalElement?.node ?? await blockstore.get(cid, options) let body: string | Uint8Array @@ -304,14 +341,15 @@ export class VerifiedFetch { return response } - private async handleDagPb ({ cid, path, resource, options }: FetchHandlerFunctionArg): Promise { + private async handleDagPb ({ cid, path, resource, cacheKey, session, options }: FetchHandlerFunctionArg): Promise { let terminalElement: UnixFSEntry | undefined let ipfsRoots: CID[] | undefined let redirected = false const byteRangeContext = new ByteRangeContext(this.helia.logger, options?.headers) + const blockstore = this.getBlockstore(cid, cacheKey, session, options) try { - const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options) + const pathDetails = await walkPath(blockstore, `${cid.toString()}/${path}`, options) ipfsRoots = pathDetails.ipfsRoots terminalElement = pathDetails.terminalElement } catch (err: any) { @@ -415,9 +453,10 @@ export class VerifiedFetch { } } - private async handleRaw ({ resource, cid, path, options, accept }: FetchHandlerFunctionArg): Promise { + private async handleRaw ({ resource, cid, path, session, cacheKey, options, accept }: FetchHandlerFunctionArg): Promise { const byteRangeContext = new ByteRangeContext(this.helia.logger, options?.headers) - const result = await this.helia.blockstore.get(cid, options) + const blockstore = this.getBlockstore(cid, cacheKey, session, options) + const result = await blockstore.get(cid, options) byteRangeContext.setBody(result) const response = okRangeResponse(resource, byteRangeContext.getBody(), { byteRangeContext, log: this.log }, { redirected: false @@ -520,7 +559,8 @@ export class VerifiedFetch { let response: Response let reqFormat: RequestFormatShorthand | undefined - const handlerArgs: FetchHandlerFunctionArg = { resource: resource.toString(), cid, path, accept, options } + const cacheKey = resourceToSessionCacheKey(resource) + const handlerArgs: FetchHandlerFunctionArg = { resource: resource.toString(), cid, path, accept, cacheKey, session: options?.session ?? true, options } if (accept === 'application/vnd.ipfs.ipns-record') { // the user requested a raw IPNS record diff --git a/packages/verified-fetch/test/abort-handling.spec.ts b/packages/verified-fetch/test/abort-handling.spec.ts index cacbed1..972ba71 100644 --- a/packages/verified-fetch/test/abort-handling.spec.ts +++ b/packages/verified-fetch/test/abort-handling.spec.ts @@ -60,11 +60,14 @@ describe('abort-handling', function () { peerIdResolverCalled.resolve() return getAbortablePromise(options.signal) }) - blockRetriever = stubInterface>>({ + blockRetriever = stubInterface>>({ retrieve: sandbox.stub().callsFake(async (cid, options) => { blockBrokerRetrieveCalled.resolve() return getAbortablePromise(options.signal) - }) + }), + createSession: () => { + return blockRetriever + } }) logger = prefixLogger('test:abort-handling') diff --git a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts index 9d330f8..f8ff641 100644 --- a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts +++ b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts @@ -21,7 +21,7 @@ describe('custom dns-resolvers', () => { it('is used when passed to createVerifiedFetch', async () => { const customDnsResolver = Sinon.stub().withArgs('_dnslink.some-non-cached-domain.com').resolves({ Answer: [{ - data: 'dnslink=/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg' + data: 'dnslink=/ipfs/bafkqac3imvwgy3zao5xxe3de' }] }) @@ -30,8 +30,9 @@ describe('custom dns-resolvers', () => { dnsResolvers: [customDnsResolver] }) const response = await fetch('ipns://some-non-cached-domain.com') - expect(response.status).to.equal(502) - expect(response.statusText).to.equal('Bad Gateway') + expect(response.status).to.equal(200) + expect(response.statusText).to.equal('OK') + await expect(response.text()).to.eventually.equal('hello world') expect(customDnsResolver.callCount).to.equal(1) expect(customDnsResolver.getCall(0).args).to.deep.equal(['_dnslink.some-non-cached-domain.com', { @@ -44,7 +45,7 @@ describe('custom dns-resolvers', () => { it('is used when passed to VerifiedFetch', async () => { const customDnsResolver = Sinon.stub().withArgs('_dnslink.some-non-cached-domain2.com').resolves({ Answer: [{ - data: 'dnslink=/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg' + data: 'dnslink=/ipfs/bafkqac3imvwgy3zao5xxe3de' }] }) @@ -62,8 +63,9 @@ describe('custom dns-resolvers', () => { }) const response = await verifiedFetch.fetch('ipns://some-non-cached-domain2.com') - expect(response.status).to.equal(502) - expect(response.statusText).to.equal('Bad Gateway') + expect(response.status).to.equal(200) + expect(response.statusText).to.equal('OK') + await expect(response.text()).to.eventually.equal('hello world') expect(customDnsResolver.callCount).to.equal(1) expect(customDnsResolver.getCall(0).args).to.deep.equal(['_dnslink.some-non-cached-domain2.com', { diff --git a/packages/verified-fetch/test/utils/resource-to-cache-key.spec.ts b/packages/verified-fetch/test/utils/resource-to-cache-key.spec.ts new file mode 100644 index 0000000..ea4da76 --- /dev/null +++ b/packages/verified-fetch/test/utils/resource-to-cache-key.spec.ts @@ -0,0 +1,55 @@ +import { expect } from 'aegir/chai' +import { CID } from 'multiformats/cid' +import { resourceToSessionCacheKey } from '../../src/utils/resource-to-cache-key.js' + +describe('resource-to-cache-key', () => { + it('converts url with IPFS path', () => { + expect(resourceToSessionCacheKey('https://localhost:8080/ipfs/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA')) + .to.equal('ipfs://QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA') + }) + + it('converts url with IPFS path and resource path', () => { + expect(resourceToSessionCacheKey('https://localhost:8080/ipfs/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA/foo/bar/baz.txt')) + .to.equal('ipfs://QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA') + }) + + it('converts url with IPNS path', () => { + expect(resourceToSessionCacheKey('https://localhost:8080/ipns/ipfs.io')) + .to.equal('ipns://ipfs.io') + }) + + it('converts url with IPNS path and resource path', () => { + expect(resourceToSessionCacheKey('https://localhost:8080/ipns/ipfs.io/foo/bar/baz.txt')) + .to.equal('ipns://ipfs.io') + }) + + it('converts IPFS subdomain', () => { + expect(resourceToSessionCacheKey('https://QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA.ipfs.localhost:8080')) + .to.equal('ipfs://QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA') + }) + + it('converts IPFS subdomain with path', () => { + expect(resourceToSessionCacheKey('https://QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA.ipfs.localhost:8080/foo/bar/baz.txt')) + .to.equal('ipfs://QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA') + }) + + it('converts IPNS subdomain', () => { + expect(resourceToSessionCacheKey('https://ipfs.io.ipns.localhost:8080')) + .to.equal('ipns://ipfs.io') + }) + + it('converts IPNS subdomain with resource path', () => { + expect(resourceToSessionCacheKey('https://ipfs.io.ipns.localhost:8080/foo/bar/baz.txt')) + .to.equal('ipns://ipfs.io') + }) + + it('converts CID', () => { + expect(resourceToSessionCacheKey(CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA'))) + .to.equal('ipfs://QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA') + }) + + it('converts CID string', () => { + expect(resourceToSessionCacheKey('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA')) + .to.equal('ipfs://QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA') + }) +}) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 7338a31..93bf802 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -14,6 +14,7 @@ import * as ipldJson from 'multiformats/codecs/json' import * as raw from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' import { sha256 } from 'multiformats/hashes/sha2' +import pDefer from 'p-defer' import Sinon from 'sinon' import { stubInterface } from 'sinon-ts' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' @@ -857,4 +858,69 @@ describe('@helia/verifed-fetch', () => { expect(resp.status).to.equal(404) }) }) + + describe('sessions', () => { + let helia: Helia + let verifiedFetch: VerifiedFetch + + beforeEach(async () => { + helia = await createHelia() + verifiedFetch = new VerifiedFetch({ + helia + }) + }) + + afterEach(async () => { + await stop(helia, verifiedFetch) + }) + + it('should use sessions', async () => { + const getSpy = Sinon.spy(helia.blockstore, 'get') + const deferred = pDefer() + const controller = new AbortController() + const originalCreateSession = helia.blockstore.createSession.bind(helia.blockstore) + + // blockstore.createSession is called, blockstore.get is not + helia.blockstore.createSession = Sinon.stub().callsFake((root, options) => { + deferred.resolve() + return originalCreateSession(root, options) + }) + + const p = verifiedFetch.fetch('http://example.com/ipfs/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJA', { + signal: controller.signal + }) + + await deferred.promise + + expect(getSpy.called).to.be.false() + + controller.abort() + await expect(p).to.eventually.be.rejected() + }) + + it('should not use sessions when session option is false', async () => { + const sessionSpy = Sinon.spy(helia.blockstore, 'createSession') + const deferred = pDefer() + const controller = new AbortController() + const originalGet = helia.blockstore.get.bind(helia.blockstore) + + // blockstore.get is called, blockstore.createSession is not + helia.blockstore.get = Sinon.stub().callsFake(async (cid, options) => { + deferred.resolve() + return originalGet(cid, options) + }) + + const p = verifiedFetch.fetch('http://example.com/ipfs/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN/foo/i-do-not-exist', { + signal: controller.signal, + session: false + }) + + await deferred.promise + + expect(sessionSpy.called).to.be.false() + + controller.abort() + await expect(p).to.eventually.be.rejected() + }) + }) }) diff --git a/typedoc.json b/typedoc.json index 8a5f92a..4e23d34 100644 --- a/typedoc.json +++ b/typedoc.json @@ -1,6 +1,6 @@ { "$schema": "https://typedoc.org/schema.json", - "name": "Helia Routing V1 HTTP API", + "name": "Helia Verified Fetch", "exclude": [ "packages/interop" ]