feat(upload-client): Remove ipfs-utils

packages/upload-client/package.json

@@ -92,7 +92,6 @@
     "@web3-storage/capabilities": "workspace:^",
     "@web3-storage/data-segment": "^5.1.0",
     "@web3-storage/filecoin-client": "workspace:^",
-    "ipfs-utils": "^9.0.14",
     "multiformats": "^13.3.2",
     "p-retry": "^6.2.1",
     "varint": "^6.0.0"

packages/upload-client/src/fetch-with-upload-progress.js

@@ -1,8 +1,167 @@
-import ipfsUtilsFetch from 'ipfs-utils/src/http/fetch.js'
+// HTTP/2 or Node.js/undici
+
+/**
+ *
+ * @param {AsyncIterable<Uint8Array>} iterable
+ * @returns {ReadableStream}
+ */
+function iterableToStream(iterable) {
+  const iterator = iterable[Symbol.asyncIterator]()
+  return new ReadableStream({
+    async pull(controller) {
+      const { value, done } = await iterator.next()
+      if (value) {
+        controller.enqueue(value)
+      }
+      if (done) {
+        controller.close()
+      }
+    },
+  })
+}
+
+/**
+ * Takes body from fetch response as body and `onUploadProgress` handler
+ * and returns async iterable that emits body chunks and emits
+ * `onUploadProgress`.
+ *
+ * @param {ReadableStream} body
+ * @param {import('./types.js').ProgressFn} onUploadProgress
+ * @returns {AsyncIterable<Uint8Array>}
+ */
+const iterateBodyWithProgress = async function* (body, onUploadProgress) {
+  if (body instanceof ReadableStream) {
+    const reader = body.getReader()
+    const total = 0 // If the total size is unknown
+    const lengthComputable = false
+    let loaded = 0
+
+    try {
+      while (true) {
+        const { done, value } = await reader.read()
+        if (done) break
+
+        loaded += value.byteLength
+        yield value // Yield the chunk
+        onUploadProgress({ total, loaded, lengthComputable })
+      }
+      // eslint-disable-next-line no-useless-catch
+    } catch (e) {
+      throw e
+    } finally {
+      reader.releaseLock() // Ensure the reader lock is released
+    }
+  } else {
+    throw new Error('Body is not a ReadableStream')
+  }
+}
+
+/**
+ * Takes fetch options and wraps request body to track upload progress if
+ * `onUploadProgress` is supplied. Otherwise returns options as is.
+ *
+ * @param {import('./types.js').FetchOptions} options
+ * @returns {import('./types.js').FetchOptions}
+ */
+const withUploadProgress = (options) => {
+  const { onUploadProgress, body } = options
+
+  if (!onUploadProgress) {
+    return options
+  }
+
+  const rsp = new Response(body)
+  // @ts-expect-error web streams from node and web have different types
+  const source = iterateBodyWithProgress(rsp.body, onUploadProgress)
+  const stream = iterableToStream(source)
+  return {
+    ...options,
+    body: stream,
+  }
+}
+
+// HTTP/1.1 and browsers
+
+/* c8 ignore start */
+/**
+ *
+ * @param {string | URL} url
+ * @param {import('./types.js').FetchOptions} init
+ */
+const fetchXhr = (url, { onUploadProgress, ...init }) => {
+  if (!onUploadProgress) {
+    return fetch(url, init)
+  }
+  return /** @type {Promise<Response>} */ (
+    new Promise((resolve, reject) => {
+      const xhr = new XMLHttpRequest()
+      xhr.open(init.method || 'GET', url, true)
+      xhr.upload.addEventListener('progress', (e) =>
+        onUploadProgress({
+          total: e.total,
+          loaded: e.loaded,
+          lengthComputable: e.lengthComputable,
+          url,
+        })
+      )
+      xhr.upload.addEventListener('loadend', (e) =>
+        onUploadProgress({
+          total: e.total,
+          loaded: e.loaded,
+          lengthComputable: e.lengthComputable,
+          url,
+        })
+      )
+      xhr.onreadystatechange = () => {
+        if (xhr.readyState === 4) {
+          // @ts-expect-error doesn't have to match Response 100%
+          resolve({
+            status: xhr.status,
+            statusText: xhr.statusText,
+            body: xhr.response,
+            ok: ((xhr.status / 100) | 0) == 2,
+          })
+        }
+      }
+      xhr.onerror = (err) => reject(err)
+      Object.entries(init.headers || {}).forEach(([key, value]) =>
+        xhr.setRequestHeader(key, value)
+      )
+      /**
+       * @type {XMLHttpRequestBodyInit}
+       */
+      // @ts-expect-error ReadableStream as body is not supported by XHR
+      const body = init.body
+      xhr.send(body)
+    })
+  )
+}
+
+/* c8 ignore stop */
 
 /**
  * @type {import('./types.js').FetchWithUploadProgress}
  */
-export const fetchWithUploadProgress = (url, init) => {
-  return ipfsUtilsFetch.fetch(url, init)
+export const fetchWithUploadProgress = (url, init = {}) => {
+  /**
+   * @type {"http/0.9"| "http/1.0" | "http/1.1" | "http/2" | "h2" | "h2c" | "h3" | undefined}
+   * @see https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming/nextHopProtocol
+   */
+  const protocol =
+    // @ts-expect-error nextHopProtocol is missing from types but is widely available
+    performance.getEntriesByType('resource')[0]?.nextHopProtocol
+  const preH2 =
+    protocol === 'http/0.9' ||
+    protocol === 'http/1.0' ||
+    protocol === 'http/1.1'
+
+  /* c8 ignore next 3 */
+  if (
+    typeof globalThis.XMLHttpRequest !== 'undefined' &&
+    preH2 &&
+    Boolean(init.onUploadProgress)
+  ) {
+    return fetchXhr(url, init)
+  }
+  return fetch(url, withUploadProgress(init))
 }

packages/upload-client/src/types.ts

@@ -1,7 +1,3 @@
-import type {
-  FetchOptions as IpfsUtilsFetchOptions,
-  ProgressStatus as XHRProgressStatus,
-} from 'ipfs-utils/src/types.js'
 import { Link, UnknownLink, Version, MultihashHasher } from 'multiformats'
 import { Block, EncoderSettings } from '@ipld/unixfs'
 import {
@@ -84,15 +80,13 @@ import {
   Position,
 } from '@web3-storage/blob-index/types'
 
-type Override<T, R> = Omit<T, keyof R> & R
-
-type FetchOptions = Override<
-  IpfsUtilsFetchOptions,
-  {
-    // `fetch` is a browser API and browsers don't have `Readable`
-    body: Exclude<IpfsUtilsFetchOptions['body'], import('node:stream').Readable>
-  }
->
+type FetchOptions = RequestInit & {
+  /**
+   * Can be passed to track upload progress.
+   * Note that if this option in passed underlying request will be performed using `XMLHttpRequest` and response will not be streamed.
+   */
+  onUploadProgress?: ProgressFn
+}
 
 export type {
   FetchOptions,
@@ -152,11 +146,16 @@ export type {
   Position,
 }
 
-export interface ProgressStatus extends XHRProgressStatus {
-  url?: string
+export interface ProgressStatus {
+  total: number
+  loaded: number
+  lengthComputable: boolean
+  url?: string | URL
 }
 
-export type ProgressFn = (status: ProgressStatus) => void
+export interface ProgressFn {
+  (status: ProgressStatus): void
+}
 
 export interface Service extends StorefrontService {
   ucan: {
@@ -308,7 +307,7 @@ export interface Connectable {
 }
 
 export type FetchWithUploadProgress = (
-  url: string,
+  url: string | URL,
   init?: FetchOptions
 ) => Promise<Response>
 

packages/upload-client/test/blob.test.js

@@ -93,7 +93,6 @@ describe('Blob.add', () => {
         receiptsEndpoint,
       }
     )
-
     assert(site)
     assert.equal(site.capabilities[0].can, Assert.location.can)
     // we're not verifying this as it's a mocked value

packages/upload-client/test/fetch-with-upload-progress.test.js

@@ -0,0 +1,42 @@
+import { createServer } from 'node:http'
+import { fetchWithUploadProgress } from '../src/fetch-with-upload-progress.js'
+import assert from 'node:assert'
+
+const encoder = new TextEncoder()
+const body = encoder.encode('Hello World'.repeat(1024))
+
+describe('fetchWithUploadProgress', () => {
+  it('should stream with Node.js and Bun HTTP/1.1', async () => {
+    const server = createServer((req, res) => {
+      res.end(`Hello from ${req.httpVersion}`)
+    })
+
+    const listener = server.listen()
+
+    /**
+     * @type {import('node:net').AddressInfo}
+     */
+    // @ts-expect-error - it's always AddressInfo
+    const address = listener.address()
+
+    let total = 0
+
+    await fetchWithUploadProgress(new URL(`http://localhost:${address.port}`), {
+      method: 'POST',
+      body,
+      // @ts-expect-error - this is needed by recent versions of node - see https://github.com/bluesky-social/atproto/pull/470 for more info
+      duplex: 'half',
+      onUploadProgress: (payload) => {
+        total += payload.loaded
+        assert.equal(payload.lengthComputable, false)
+      },
+    })
+      .then((res) => res.text())
+      .then((text) => {
+        assert.equal(text, 'Hello from 1.1')
+        assert.equal(total, body.byteLength)
+      }).finally(() => {
+        server.close()
+      })
+  })
+})

packages/w3up-client/package.json

@@ -127,12 +127,12 @@
   "scripts": {
     "attw": "attw --pack .",
     "lint": "tsc --build && eslint '**/*.{js,ts}' && prettier --check '**/*.{js,ts,yml,json}' --ignore-path ../../.gitignore",
-    "build": "npm run build:tsc && npm run build:bundle:browser",
+    "build": "pnpm build:tsc && pnpm build:bundle:browser",
     "build:tsc": "tsc --build",
     "build:bundle:browser": "esbuild src/index.js --bundle --minify --target=chrome130 --format=iife --global-name=w3up --outfile=browser.min.js",
     "dev": "tsc --build --watch",
     "check": "tsc --build",
-    "prepare": "npm run build",
+    "prepare": "pnpm build",
     "test": "npm-run-all -p -r mock:* test:all",
     "test:all": "run-s test:node test:browser",
     "test:node": "hundreds -r html -r text mocha 'test/**/!(*.browser).test.js' -n experimental-vm-modules -n no-warnings -n stack-trace-limit=1000 -t 10000",
@@ -143,7 +143,7 @@
     "mock:gateway-server": "PORT=5001 node test/helpers/gateway-server.js",
     "coverage": "c8 report -r html && open coverage/index.html",
     "rc": "npm version prerelease --preid rc",
-    "docs": "npm run build && typedoc --out docs-generated"
+    "docs": "pnpm build && typedoc --out docs-generated"
   },
   "dependencies": {
     "@ipld/dag-ucan": "^3.4.0",