From 94fb322c7b56677ce3e1fe50e7e4b0a73297fbab Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 29 Feb 2024 17:04:00 +0000 Subject: [PATCH 1/2] feat: support redirects for UnixFS directories Adds support for simulating redirects for UnixFS directories. We're somewhat in uncharted water here because window.fetch does this transparently unless you specify a `redirect` option, none of which actually allow you to follow a redirect. The states we can be in are: 1. URL: `ipfs://QmFoo/dir/` - Happy path - 200 response - `response.redirected = false` - `response.url = 'ipfs://QmFoo/dir'` 2: URL: `ipfs://QmFoo/dir`, `redirect: 'follow'` - The default value - Simulates automatically following a redirect - 200 response - `response.redirected = true` - `response.url = 'ipfs://QmFoo/dir/'` 3: URL: `ipfs://QmFoo/dir`, `redirect: 'error'` - Return an error if a redirect would take place - Throws `TypeError('Failed to Fetch')` same as `window.fetch` 4: URL: `ipfs://QmFoo/dir`, `redirect: 'manual'` - Allows a caller to take action on the redirect - 301 response - `response.redirected = false` - `response.url = 'ipfs://QmFoo/dir` - `response.headers.get('location') = 'ipfs://QmFoo/dir/'` Number 4 is the furthest from [the fetch spec](https://fetch.spec.whatwg.org/#concept-request-redirect-mode) but to follow the spec would make it impossible to actually follow a redirect. --- .../verified-fetch/src/utils/responses.ts | 83 +++++++++++++++++-- packages/verified-fetch/src/verified-fetch.ts | 55 +++++++----- .../test/verified-fetch.spec.ts | 78 +++++++++++++++++ 3 files changed, 190 insertions(+), 26 deletions(-) diff --git a/packages/verified-fetch/src/utils/responses.ts b/packages/verified-fetch/src/utils/responses.ts index 220a191..a596370 100644 --- a/packages/verified-fetch/src/utils/responses.ts +++ b/packages/verified-fetch/src/utils/responses.ts @@ -1,29 +1,98 @@ -export function okResponse (body?: BodyInit | null): Response { - return new Response(body, { +function setField (response: Response, name: string, value: string | boolean): void { + Object.defineProperty(response, name, { + enumerable: true, + configurable: false, + set: () => {}, + get: () => value + }) +} + +function setType (response: Response, value: 'basic' | 'cors' | 'error' | 'opaque' | 'opaqueredirect'): void { + setField(response, 'type', value) +} + +function setUrl (response: Response, value: string): void { + setField(response, 'url', value) +} + +function setRedirected (response: Response): void { + setField(response, 'redirected', true) +} + +export interface ResponseOptions extends ResponseInit { + redirected?: boolean +} + +export function okResponse (url: string, body?: BodyInit | null, init?: ResponseOptions): Response { + const response = new Response(body, { + ...(init ?? {}), status: 200, statusText: 'OK' }) + + if (init?.redirected === true) { + setRedirected(response) + } + + setType(response, 'basic') + setUrl(response, url) + + return response } -export function notSupportedResponse (body?: BodyInit | null): Response { +export function notSupportedResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response { const response = new Response(body, { + ...(init ?? {}), status: 501, statusText: 'Not Implemented' }) response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header + + setType(response, 'basic') + setUrl(response, url) + return response } -export function notAcceptableResponse (body?: BodyInit | null): Response { - return new Response(body, { +export function notAcceptableResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response { + const response = new Response(body, { + ...(init ?? {}), status: 406, statusText: 'Not Acceptable' }) + + setType(response, 'basic') + setUrl(response, url) + + return response } -export function badRequestResponse (body?: BodyInit | null): Response { - return new Response(body, { +export function badRequestResponse (url: string, body?: BodyInit | null, init?: ResponseInit): Response { + const response = new Response(body, { + ...(init ?? {}), status: 400, statusText: 'Bad Request' }) + + setType(response, 'basic') + setUrl(response, url) + + return response +} + +export function movedPermanentlyResponse (url: string, location: string, init?: ResponseInit): Response { + const response = new Response(null, { + ...(init ?? {}), + status: 301, + statusText: 'Moved Permanently', + headers: { + ...(init?.headers ?? {}), + location + } + }) + + setType(response, 'basic') + setUrl(response, url) + + return response } diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 8935ad1..ca8c6db 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -22,7 +22,7 @@ import { getETag } from './utils/get-e-tag.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { tarStream } from './utils/get-tar-stream.js' import { parseResource } from './utils/parse-resource.js' -import { badRequestResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js' +import { badRequestResponse, movedPermanentlyResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js' import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js' import { walkPath } from './utils/walk-path.js' import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' @@ -167,7 +167,7 @@ export class VerifiedFetch { const buf = await this.helia.datastore.get(datastoreKey, options) const record = DHTRecord.deserialize(buf) - const response = okResponse(record.value) + const response = okResponse(resource, record.value) response.headers.set('content-type', 'application/vnd.ipfs.ipns-record') return response @@ -177,11 +177,11 @@ export class VerifiedFetch { * Accepts a `CID` and returns a `Response` with a body stream that is a CAR * of the `DAG` referenced by the `CID`. */ - private async handleCar ({ cid, options }: FetchHandlerFunctionArg): Promise { + private async handleCar ({ resource, cid, options }: FetchHandlerFunctionArg): Promise { const c = car(this.helia) const stream = toBrowserReadableStream(c.stream(cid, options)) - const response = okResponse(stream) + const response = okResponse(resource, stream) response.headers.set('content-type', 'application/vnd.ipld.car; version=1') return response @@ -191,20 +191,20 @@ export class VerifiedFetch { * Accepts a UnixFS `CID` and returns a `.tar` file containing the file or * directory structure referenced by the `CID`. */ - private async handleTar ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + private async handleTar ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise { if (cid.code !== dagPbCode && cid.code !== rawCode) { return notAcceptableResponse('only UnixFS data can be returned in a TAR file') } const stream = toBrowserReadableStream(tarStream(`/ipfs/${cid}/${path}`, this.helia.blockstore, options)) - const response = okResponse(stream) + const response = okResponse(resource, stream) response.headers.set('content-type', 'application/x-tar') return response } - private async handleJson ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise { + private async handleJson ({ resource, cid, path, accept, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) const block = await this.helia.blockstore.get(cid, options) let body: string | Uint8Array @@ -218,19 +218,19 @@ export class VerifiedFetch { body = ipldDagCbor.encode(obj) } catch (err) { this.log.error('could not transform %c to application/vnd.ipld.dag-cbor', err) - return notAcceptableResponse() + return notAcceptableResponse(resource) } } else { // skip decoding body = block } - const response = okResponse(body) + const response = okResponse(resource, body) response.headers.set('content-type', accept ?? 'application/json') return response } - private async handleDagCbor ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise { + private async handleDagCbor ({ resource, cid, path, accept, options }: FetchHandlerFunctionArg): Promise { this.log.trace('fetching %c/%s', cid, path) const block = await this.helia.blockstore.get(cid, options) @@ -248,7 +248,7 @@ export class VerifiedFetch { body = ipldDagJson.encode(obj) } catch (err) { this.log.error('could not transform %c to application/vnd.ipld.dag-json', err) - return notAcceptableResponse() + return notAcceptableResponse(resource) } } else { try { @@ -257,7 +257,7 @@ export class VerifiedFetch { if (accept === 'application/json') { this.log('could not decode DAG-CBOR as JSON-safe, but the client sent "Accept: application/json"', err) - return notAcceptableResponse() + return notAcceptableResponse(resource) } this.log('could not decode DAG-CBOR as JSON-safe, falling back to `application/octet-stream`', err) @@ -265,7 +265,7 @@ export class VerifiedFetch { } } - const response = okResponse(body) + const response = okResponse(resource, body) if (accept == null) { accept = body instanceof Uint8Array ? 'application/octet-stream' : 'application/json' @@ -276,9 +276,10 @@ export class VerifiedFetch { return response } - private async handleDagPb ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + private async handleDagPb ({ cid, path, resource, options }: FetchHandlerFunctionArg): Promise { let terminalElement: UnixFSEntry | undefined let ipfsRoots: CID[] | undefined + let redirected = false try { const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options) @@ -293,6 +294,21 @@ export class VerifiedFetch { if (terminalElement?.type === 'directory') { const dirCid = terminalElement.cid + // https://specs.ipfs.tech/http-gateways/path-gateway/#use-in-directory-url-normalization + if (path !== '' && !path.endsWith('/')) { + if (options?.redirect === 'error') { + this.log('could not redirect to %s/ as redirect option was set to "error"', resource) + throw new TypeError('Failed to fetch') + } else if (options?.redirect === 'manual') { + this.log('returning 301 permanent redirect to %s/', resource) + return movedPermanentlyResponse(resource, `${resource}/`) + } + + // fall-through simulates following the redirect? + resource = `${resource}/` + redirected = true + } + const rootFilePath = 'index.html' try { this.log.trace('found directory at %c/%s, looking for index.html', cid, path) @@ -304,7 +320,6 @@ export class VerifiedFetch { this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, stat.cid) path = rootFilePath resolvedCID = stat.cid - // terminalElement = stat } catch (err: any) { this.log('error loading path %c/%s', dirCid, rootFilePath, err) return notSupportedResponse('Unable to find index.html for directory at given path. Support for directories with implicit root is not implemented') @@ -322,7 +337,9 @@ export class VerifiedFetch { const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, { onProgress: options?.onProgress }) - const response = okResponse(stream) + const response = okResponse(resource, stream, { + redirected + }) await this.setContentType(firstChunk, path, response) if (ipfsRoots != null) { @@ -332,9 +349,9 @@ export class VerifiedFetch { return response } - private async handleRaw ({ cid, path, options }: FetchHandlerFunctionArg): Promise { + private async handleRaw ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise { const result = await this.helia.blockstore.get(cid, options) - const response = okResponse(result) + const response = okResponse(resource, result) // if the user has specified an `Accept` header that corresponds to a raw // type, honour that header, so for example they don't request @@ -418,7 +435,7 @@ export class VerifiedFetch { this.log('output type %s', accept) if (acceptHeader != null && accept == null) { - return notAcceptableResponse() + return notAcceptableResponse(resource.toString()) } let response: Response diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 9dc49c7..227fc5e 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -133,6 +133,84 @@ describe('@helia/verifed-fetch', () => { expect(new Uint8Array(data)).to.equalBytes(finalRootFileContent) }) + it('should return a 301 with a trailing slash when a directory is requested without a trailing slash', async () => { + const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) + + const fs = unixfs(helia) + const res = await last(fs.addAll([{ + path: 'foo/index.html', + content: finalRootFileContent + }], { + wrapWithDirectory: true + })) + + if (res == null) { + throw new Error('Import failed') + } + + const stat = await fs.stat(res.cid) + expect(stat.type).to.equal('directory') + + const ipfsResponse = await verifiedFetch.fetch(`ipfs://${res.cid}/foo`, { + redirect: 'manual' + }) + expect(ipfsResponse).to.be.ok() + expect(ipfsResponse.status).to.equal(301) + expect(ipfsResponse.headers.get('location')).to.equal(`ipfs://${res.cid}/foo/`) + expect(ipfsResponse.url).to.equal(`ipfs://${res.cid}/foo`) + }) + + it('should simulate following a redirect to a path with a slash when a directory is requested without a trailing slash', async () => { + const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) + + const fs = unixfs(helia) + const res = await last(fs.addAll([{ + path: 'foo/index.html', + content: finalRootFileContent + }], { + wrapWithDirectory: true + })) + + if (res == null) { + throw new Error('Import failed') + } + + const stat = await fs.stat(res.cid) + expect(stat.type).to.equal('directory') + + const ipfsResponse = await verifiedFetch.fetch(`ipfs://${res.cid}/foo`) + expect(ipfsResponse).to.be.ok() + expect(ipfsResponse.type).to.equal('basic') + expect(ipfsResponse.status).to.equal(200) + expect(ipfsResponse.redirected).to.be.true() + expect(ipfsResponse.url).to.equal(`ipfs://${res.cid}/foo/`) + }) + + it('should not redirect when a directory is requested with a trailing slash', async () => { + const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) + + const fs = unixfs(helia) + const res = await last(fs.addAll([{ + path: 'foo/index.html', + content: finalRootFileContent + }], { + wrapWithDirectory: true + })) + + if (res == null) { + throw new Error('Import failed') + } + + const stat = await fs.stat(res.cid) + expect(stat.type).to.equal('directory') + + const ipfsResponse = await verifiedFetch.fetch(`ipfs://${res.cid}/foo/`) + expect(ipfsResponse).to.be.ok() + expect(ipfsResponse.status).to.equal(200) + expect(ipfsResponse.redirected).to.be.false() + expect(ipfsResponse.url).to.equal(`ipfs://${res.cid}/foo/`) + }) + it('should allow use as a stream', async () => { const content = new Uint8Array([0x01, 0x02, 0x03]) From af5cee9edb8e23c86a2854423d5cb651dcdb0e90 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 1 Mar 2024 08:56:50 +0000 Subject: [PATCH 2/2] chore: document redirect option --- packages/verified-fetch/README.md | 73 ++++++++++++++++++++++++++++ packages/verified-fetch/src/index.ts | 73 ++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index 50631d5..ea8bf4e 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -408,6 +408,79 @@ console.info(res.headers.get('accept')) // application/octet-stream const buf = await res.arrayBuffer() // raw bytes, not processed as JSON ``` +## Redirects + +If a requested URL contains a path component, that path component resolves to +a UnixFS directory, but the URL does not have a trailing slash, one will be +added to form a canonical URL for that resource, otherwise the request will +be resolved as normal. + +```typescript +import { verifiedFetch } from '@helia/verified-fetch' + +const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir') + +console.info(res.url) // ipfs://bafyfoo/path/to/dir/ +``` + +It's possible to prevent this behaviour and/or handle a redirect manually +through use of the [redirect](https://developer.mozilla.org/en-US/docs/Web/API/fetch#redirect) +option. + +## Example - Redirect: follow + +This is the default value and is what happens if no value is specified. + +```typescript +import { verifiedFetch } from '@helia/verified-fetch' + +const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir', { + redirect: 'follow' +}) + +console.info(res.status) // 200 +console.info(res.url) // ipfs://bafyfoo/path/to/dir/ +console.info(res.redirected) // true +``` + +## Example - Redirect: error + +This causes a `TypeError` to be thrown if a URL would cause a redirect. + +```typescript + +import { verifiedFetch } from '@helia/verified-fetch' + +const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir', { + redirect: 'error' +}) +// throw TypeError('Failed to fetch') +``` + +## Example - Redirect: manual + +Manual redirects allow the user to process the redirect. A [301](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/301) +is returned, and the location to redirect to is available as the "location" +response header. + +This differs slightly from HTTP fetch which returns an opaque response as the +browser itself is expected to process the redirect and hide all details from +the user. + +```typescript + +import { verifiedFetch } from '@helia/verified-fetch' + +const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir', { + redirect: 'manual' +}) + +console.info(res.status) // 301 +console.info(res.url) // ipfs://bafyfoo/path/to/dir +console.info(res.redirected) // false +console.info(res.headers.get('location')) // ipfs://bafyfoo/path/to/dir/ +``` + ## Comparison to fetch This module attempts to act as similarly to the `fetch()` API as possible. diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 564d5cf..aebbb44 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -379,6 +379,79 @@ * const buf = await res.arrayBuffer() // raw bytes, not processed as JSON * ``` * + * ## Redirects + * + * If a requested URL contains a path component, that path component resolves to + * a UnixFS directory, but the URL does not have a trailing slash, one will be + * added to form a canonical URL for that resource, otherwise the request will + * be resolved as normal. + * + * ```typescript + * import { verifiedFetch } from '@helia/verified-fetch' + * + * const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir') + * + * console.info(res.url) // ipfs://bafyfoo/path/to/dir/ + * ``` + * + * It's possible to prevent this behaviour and/or handle a redirect manually + * through use of the [redirect](https://developer.mozilla.org/en-US/docs/Web/API/fetch#redirect) + * option. + * + * @example Redirect: follow + * + * This is the default value and is what happens if no value is specified. + * + * ```typescript + * import { verifiedFetch } from '@helia/verified-fetch' + * + * const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir', { + * redirect: 'follow' + * }) + * + * console.info(res.status) // 200 + * console.info(res.url) // ipfs://bafyfoo/path/to/dir/ + * console.info(res.redirected) // true + * ``` + * + * @example Redirect: error + * + * This causes a `TypeError` to be thrown if a URL would cause a redirect. + * + * ```typescript + * + * import { verifiedFetch } from '@helia/verified-fetch' + * + * const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir', { + * redirect: 'error' + * }) + * // throw TypeError('Failed to fetch') + * ``` + * + * @example Redirect: manual + * + * Manual redirects allow the user to process the redirect. A [301](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/301) + * is returned, and the location to redirect to is available as the "location" + * response header. + * + * This differs slightly from HTTP fetch which returns an opaque response as the + * browser itself is expected to process the redirect and hide all details from + * the user. + * + * ```typescript + * + * import { verifiedFetch } from '@helia/verified-fetch' + * + * const res = await verifiedFetch('ipfs://bafyfoo/path/to/dir', { + * redirect: 'manual' + * }) + * + * console.info(res.status) // 301 + * console.info(res.url) // ipfs://bafyfoo/path/to/dir + * console.info(res.redirected) // false + * console.info(res.headers.get('location')) // ipfs://bafyfoo/path/to/dir/ + * ``` + * * ## Comparison to fetch * * This module attempts to act as similarly to the `fetch()` API as possible.