Use a hardened, HTTP2 compatible, client to perform proxied requests

.changeset/healthy-cats-unite.md

@@ -0,0 +1,5 @@
+---
+"@atproto/pds": patch
+---
+
+Use a hardened, HTTP2 compatible, client to perform proxied requests

packages/pds/package.json

@@ -66,6 +66,7 @@
     "sharp": "^0.32.6",
     "typed-emitter": "^2.1.0",
     "uint8arrays": "3.0.0",
+    "undici": "^6.19.8",
     "zod": "^3.23.8"
   },
   "devDependencies": {

packages/pds/src/config/config.ts

@@ -236,7 +236,11 @@ export const envToCfg = (env: ServerEnvironment): ServerConfig => {
   const crawlersCfg: ServerConfig['crawlers'] = env.crawlers ?? []
 
   const fetchCfg: ServerConfig['fetch'] = {
+    allowHTTP2: env.fetchAllowHTTP2 ?? false,
     disableSsrfProtection: env.fetchDisableSsrfProtection ?? false,
+    headersTimeout: env.fetchHeadersTimeout ?? 10e3,
+    bodyTimeout: env.fetchBodyTimeout ?? 30e3,
+    maxResponseSize: env.fetchMaxResponseSize ?? 512 * 1024, // 512kb
   }
 
   const oauthCfg: ServerConfig['oauth'] = entrywayCfg
@@ -392,7 +396,11 @@ export type EntrywayConfig = {
 }
 
 export type FetchConfig = {
+  allowHTTP2: boolean
   disableSsrfProtection: boolean
+  headersTimeout: number
+  bodyTimeout: number
+  maxResponseSize: number
 }
 
 export type OAuthConfig = {

packages/pds/src/config/env.ts

@@ -118,7 +118,11 @@ export const readEnv = (): ServerEnvironment => {
     ),
 
     // fetch
+    fetchAllowHTTP2: envBool('PDS_FETCH_ALLOW_HTTP2'),
     fetchDisableSsrfProtection: envBool('PDS_DISABLE_SSRF_PROTECTION'),
+    fetchHeadersTimeout: envInt('PDS_FETCH_HEADERS_TIMEOUT'),
+    fetchBodyTimeout: envInt('PDS_FETCH_BODY_TIMEOUT'),
+    fetchMaxResponseSize: envInt('PDS_FETCH_MAX_RESPONSE_SIZE'),
   }
 }
 
@@ -234,5 +238,9 @@ export type ServerEnvironment = {
   plcRotationKeyK256PrivateKeyHex?: string
 
   // fetch
+  fetchAllowHTTP2?: boolean
   fetchDisableSsrfProtection?: boolean
+  fetchHeadersTimeout?: number
+  fetchBodyTimeout?: number
+  fetchMaxResponseSize?: number
 }

packages/pds/src/context.ts

@@ -1,4 +1,5 @@
 import assert from 'node:assert'
+import * as undici from 'undici'
 import * as nodemailer from 'nodemailer'
 import { Redis } from 'ioredis'
 import * as plc from '@did-plc/lib'
@@ -13,10 +14,12 @@ import {
   createServiceAuthHeaders,
 } from '@atproto/xrpc-server'
 import {
-  JoseKey,
   Fetch,
-  safeFetchWrap,
+  JoseKey,
   OAuthVerifier,
+  isUnicastIp,
+  safeFetchWrap,
+  unicastLookup,
 } from '@atproto/oauth-provider'
 
 import { ServerConfig, ServerSecrets } from './config'
@@ -59,6 +62,7 @@ export type AppContextOptions = {
   moderationAgent?: AtpAgent
   reportingAgent?: AtpAgent
   entrywayAgent?: AtpAgent
+  safeAgent: undici.Agent
   safeFetch: Fetch
   authProvider?: PdsOAuthProvider
   authVerifier: AuthVerifier
@@ -85,6 +89,7 @@ export class AppContext {
   public moderationAgent: AtpAgent | undefined
   public reportingAgent: AtpAgent | undefined
   public entrywayAgent: AtpAgent | undefined
+  public safeAgent: undici.Agent
   public safeFetch: Fetch
   public authVerifier: AuthVerifier
   public authProvider?: PdsOAuthProvider
@@ -110,6 +115,7 @@ export class AppContext {
     this.moderationAgent = opts.moderationAgent
     this.reportingAgent = opts.reportingAgent
     this.entrywayAgent = opts.entrywayAgent
+    this.safeAgent = opts.safeAgent
     this.safeFetch = opts.safeFetch
     this.authVerifier = opts.authVerifier
     this.authProvider = opts.authProvider
@@ -256,12 +262,36 @@ export class AppContext {
       appviewCdnUrlPattern: cfg.bskyAppView?.cdnUrlPattern,
     })
 
+    const safeAgent = new undici.Agent({
+      allowH2: cfg.fetch.allowHTTP2, // This is experimental
+      headersTimeout: cfg.fetch.headersTimeout,
+      maxResponseSize: cfg.fetch.maxResponseSize,
+      bodyTimeout: cfg.fetch.bodyTimeout,
+      factory:
+        cfg.fetch.disableSsrfProtection || cfg.service.devMode
+          ? undefined
+          : (origin, opts) => {
+              const { protocol, hostname } =
+                origin instanceof URL ? origin : new URL(origin)
+              if (protocol !== 'https:') {
+                throw new Error(`Forbidden protocol "${protocol}"`)
+              }
+              if (isUnicastIp(hostname) === false) {
+                throw new Error('Hostname resolved to non-unicast address')
+              }
+              return new undici.Pool(origin, opts)
+            },
+      connect: {
+        lookup: cfg.fetch.disableSsrfProtection ? undefined : unicastLookup,
+      },
+    })
+
     // A fetch() function that protects against SSRF attacks, large responses &
     // known bad domains. This function can safely be used to fetch user
     // provided URLs (unless "disableSsrfProtection" is true, of course).
     const safeFetch = safeFetchWrap({
       allowHttp: cfg.fetch.disableSsrfProtection,
-      responseMaxSize: 512 * 1024, // 512kB
+      responseMaxSize: cfg.fetch.maxResponseSize,
       ssrfProtection: !cfg.fetch.disableSsrfProtection,
       fetch: async (input, init) => {
         const request = input instanceof Request ? input : null
@@ -333,6 +363,7 @@ export class AppContext {
       moderationAgent,
       reportingAgent,
       entrywayAgent,
+      safeAgent,
       safeFetch,
       authVerifier,
       authProvider,