feat: various fixes and improvements

README.md

@@ -36,6 +36,7 @@ $ docker run -it -p 8080:8080 -e DEBUG="helia-http-gateway" helia
 | Variable | Description | Default |
 | --- | --- | --- |
 | `DEBUG` | Debug level | `''`|
+| `FASTIFY_DEBUG` | Debug level for fastify's logger | `''`|
 | `PORT` | Port to listen on | `8080` |
 | `HOST` | Host to listen on | `0.0.0.0` |
 | `USE_SUBDOMAINS` | Whether to use [origin isolation](https://docs.ipfs.tech/how-to/gateway-best-practices/#use-subdomain-gateway-resolution-for-origin-isolation) | `false` |

debugging/README.md

@@ -0,0 +1,50 @@
+This file documents some methods used for debugging and testing of helia-http-gateway.
+
+Any files in this directory should be considered temporary and may be deleted at any time.
+
+You should also run any of these scripts from the repository root.
+
+# Scripts
+
+## test-gateways.sh
+This script is used to test the gateways. It assumes you have booted up the `helia-http-gateway` via docker or otherwise and will query the gateway for the same websites listed at https://probelab.io/websites/#time-to-first-byte-using-kubo, outputting HTTP status codes and response times.
+
+*Example*
+```sh
+./debugging/test-gateways.sh
+```
+
+## until-death.sh
+This script will start up the gateway and run the `test-gateway.sh` script until the gateway dies. This is useful for load-testing helia-http-gateway in a similar manner to how it will be used by https://github.com/plprobelab/tiros
+
+*Example*
+```sh
+./debugging/until-death.sh
+```
+
+# Profiling in chrome/chromium devtools
+
+## Setup
+
+1. Start the process
+
+```sh
+# in terminal 1
+npm run start:inspect
+```
+
+2. Open `chrome://inspect` and click the 'inspect' link for the process you just started
+  * it should say something like `dist/src/index.js file:///Users/sgtpooki/code/work/protocol.ai/ipfs/helia-http-gateway/dist/src/index.js` with an 'inspect' link below it.
+
+3. In the inspector, click the `performance` or `memory` tab and start recording.
+
+## Execute the operations that will be profiled:
+
+1. In another terminal, run
+
+```sh
+# in terminal 2
+npm run debug:test-gateways # or npm run debug:until-death
+```
+
+2. Stop recording in the inspector and analyze the results.

debugging/test-gateways.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+# Query all endpoints until failure
+# This script is intended to be run from the root of the helia-http-gateway repository
+
+PORT=${PORT:-8080}
+# If localhost:$PORT is not listening, then exit with non-zero error code
+if ! nc -z localhost $PORT; then
+  echo "localhost:$PORT is not listening"
+  exit 1
+fi
+
+ensure_gateway_running() {
+  npx wait-on "tcp:$PORT" -t 1000 || exit 1
+}
+
+max_timeout=60
+test_website() {
+  ensure_gateway_running
+  local website=$1
+  echo "Requesting $website"
+  curl -m $max_timeout -s --no-progress-meter -o /dev/null -w "%{url}: HTTP_%{http_code} in %{time_total} seconds (TTFB: %{time_starttransfer}, rediect: %{time_redirect})\n" -L $website
+  echo "running GC"
+  curl -X POST -m $max_timeout -s --no-progress-meter -o /dev/null -w "%{url}: HTTP_%{http_code} in %{time_total} seconds\n" http://localhost:$PORT/api/v0/repo/gc
+}
+
+test_website http://localhost:$PORT/ipns/blog.ipfs.tech
+
+test_website  http://localhost:$PORT/ipns/blog.libp2p.io
+
+test_website  http://localhost:$PORT/ipns/consensuslab.world
+
+test_website  http://localhost:$PORT/ipns/docs.ipfs.tech
+
+test_website  http://localhost:$PORT/ipns/docs.libp2p.io
+
+test_website  http://localhost:$PORT/ipns/drand.love
+
+test_website  http://localhost:$PORT/ipns/fil.org
+
+test_website  http://localhost:$PORT/ipns/filecoin.io
+
+test_website  http://localhost:$PORT/ipns/green.filecoin.io
+
+test_website http://localhost:$PORT/ipns/ipfs.tech
+
+test_website  http://localhost:$PORT/ipns/ipld.io
+
+test_website  http://localhost:$PORT/ipns/libp2p.io
+
+test_website  http://localhost:$PORT/ipns/n0.computer
+
+test_website  http://localhost:$PORT/ipns/probelab.io
+
+test_website http://localhost:$PORT/ipns/protocol.ai
+
+test_website http://localhost:$PORT/ipns/research.protocol.ai
+
+test_website http://localhost:$PORT/ipns/singularity.storage
+
+test_website http://localhost:$PORT/ipns/specs.ipfs.tech
+
+# test_website http://localhost:$PORT/ipns/strn.network
+test_website http://localhost:$PORT/ipns/saturn.tech
+
+test_website http://localhost:$PORT/ipns/web3.storage

debugging/until-death.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# If this is not executed from the root of the helia-http-gateway repository, then exit with non-zero error code
+if [ ! -f "package.json" ]; then
+  echo "This script must be executed from the root of the helia-http-gateway repository"
+  exit 1
+fi
+
+# You have to pass `DEBUG=" " to disable debugging when using this script`
+export DEBUG=${DEBUG:-"helia-http-gateway,helia-http-gateway:server,helia-http-gateway:*:helia-fetch"}
+export PORT=${PORT:-8080}
+
+gateway_already_running=false
+if nc -z localhost $PORT; then
+  echo "gateway is already running"
+  gateway_already_running=true
+fi
+
+start_gateway() {
+  if [ "$gateway_already_running" = true ]; then
+    echo "gateway is already running"
+    return
+  fi
+  npm run build
+
+  # npx clinic doctor --open=false -- node dist/src/index.js &
+  node dist/src/index.js &
+  # echo "process id: $!"
+}
+start_gateway & process_pid=$!
+
+ensure_gateway_running() {
+  npx wait-on "tcp:$PORT" -t 1000 || exit 1
+}
+
+
+cleanup_called=false
+cleanup() {
+  if [ "$cleanup_called" = true ]; then
+    echo "cleanup already called"
+    return
+  fi
+  # kill $process_pid
+  # when we're done, ensure the process is killed by sending a SIGTEM
+  # kill -s SIGTERM $process_pid
+  # kill any process listening on $PORT
+  # fuser -k $PORT/tcp
+  # kill any process listening on $PORT with SIGTERM
+
+  if [ "$gateway_already_running" = true ]; then
+    echo "gateway was already running"
+    return
+  fi
+
+  kill -s SIGINT $(lsof -i :$PORT -t)
+  # exit 1
+}
+
+trap cleanup SIGINT
+trap cleanup SIGTERM
+
+# if we get a non-zero exit code, we know the server is no longer listening
+# we should also exit early after 4 loops
+# iterations=0
+# max_loops=1
+while [ $? -ne 1 ]; do
+#   # iterations=$((iterations+1))
+#   if [ $iterations -gt $max_loops ]; then
+#     echo "exiting after $max_loops loops"
+#     break
+#   fi
+  ensure_gateway_running
+  ./debugging/test-gateways.sh 2>&1 | tee -a debugging/test-gateways.log
+done
+
+cleanup

e2e-tests/smoketest.spec.ts

@@ -0,0 +1,59 @@
+import { test, expect } from '@playwright/test'
+import { PORT } from '../src/constants.js'
+
+// test all the same pages listed at https://probelab.io/websites/
+const pages = [
+  // 'blog.ipfs.tech', // timing out
+  'blog.libp2p.io',
+  'consensuslab.world',
+  'docs.ipfs.tech',
+  'docs.libp2p.io',
+  'drand.love',
+  'fil.org',
+  // 'filecoin.io', // timing out
+  // 'green.filecoin.io', // timing out
+  'ipfs.tech',
+  'ipld.io',
+  'libp2p.io',
+  'n0.computer',
+  'probelab.io',
+  'protocol.ai',
+  'research.protocol.ai',
+  'singularity.storage',
+  'specs.ipfs.tech',
+  // 'strn.network' // redirects to saturn.tech
+  'saturn.tech'
+  // 'web3.storage' // timing out
+]
+
+// increase default test timeout to 2 minutes
+test.setTimeout(120000)
+
+// now for each page, make sure we can request the website, the content is not empty, and status code is 200
+test.beforeEach(async ({ context }) => {
+  // Block any asset requests for tests in this file.
+  await context.route(/.(css|js|svg|png|jpg|woff2|otf|webp|ttf|json)(?:\?.*)?$/, async route => route.abort())
+})
+
+test.afterEach(async ({ page }) => {
+  test.setTimeout(30000)
+  const result = await page.request.post(`http://localhost:${PORT}/api/v0/repo/gc`)
+  expect(result?.status()).toBe(200)
+
+  const maybeContent = await result?.text()
+  expect(maybeContent).toEqual('OK')
+})
+
+pages.forEach((pagePath) => {
+  const url = `http://${pagePath}.ipns.localhost:${PORT}`
+  test(`helia-http-gateway can load path '${url}'`, async ({ page }) => {
+    // only wait for 'commit' because we don't want to wait for all the assets to load, we just want to make sure that they *would* load (e.g. the html is valid)
+    const heliaGatewayResponse = await page.goto(`${url}`, { waitUntil: 'commit' })
+    expect(heliaGatewayResponse?.status()).toBe(200)
+    // await page.waitForSelector('body')
+    expect(await heliaGatewayResponse?.text()).not.toEqual('')
+    const headers = heliaGatewayResponse?.headers()
+    expect(headers).not.toBeNull()
+    expect(headers?.['content-type']).toContain('text/html')
+  })
+})