feat: Capture response body

src/Recording.ts

@@ -101,6 +101,7 @@ export default class Recording {
     elapsed: number,
     status: number,
     headers?: Record<string, string>,
+    returnValue?: AppMap.Parameter,
   ): AppMap.HttpClientResponseEvent {
     assert(this.stream);
 
@@ -109,6 +110,7 @@ export default class Recording {
       http_client_response: compactObject({
         status_code: status,
         headers,
+        return_value: returnValue,
       }),
       id: this.nextId++,
       thread_id: 0,
@@ -159,6 +161,7 @@ export default class Recording {
     elapsed: number,
     status: number,
     headers?: Record<string, string>,
+    returnValue?: AppMap.Parameter,
   ): AppMap.HttpServerResponseEvent {
     assert(this.stream);
 
@@ -167,6 +170,7 @@ export default class Recording {
       http_server_response: compactObject({
         status_code: status,
         headers,
+        return_value: returnValue,
       }),
       id: this.nextId++,
       thread_id: 0,

src/config.ts

@@ -6,18 +6,23 @@ import { cwd } from "node:process";
 import { PackageJson } from "type-fest";
 import YAML from "yaml";
 
+import { warn } from "./message";
 import PackageMatcher, { Package, parsePackages } from "./PackageMatcher";
 import locateFileUp from "./util/findFileUp";
 import lazyOpt from "./util/lazyOpt";
 import tryOr from "./util/tryOr";
 
+const responseBodyMaxLengthDefault = 10000;
+const kResponseBodyMaxLengthEnvar = "APPMAP_RESPONSE_BODY_MAX_LENGTH";
+
 export class Config {
   public readonly relativeAppmapDir: string;
   public readonly appName: string;
   public readonly root: string;
   public readonly configPath: string;
   public readonly default: boolean;
   public readonly packages: PackageMatcher;
+  public readonly responseBodyMaxLength: number;
 
   constructor(pwd = cwd()) {
     const configDir = locateFileUp("appmap.yml", process.env.APPMAP_ROOT ?? pwd);
@@ -45,6 +50,17 @@ export class Config {
         },
       ],
     );
+
+    let envResponseBodyMaxLength: number | undefined;
+    if (process.env[kResponseBodyMaxLengthEnvar] != undefined) {
+      const value = parseInt(process.env[kResponseBodyMaxLengthEnvar]);
+      envResponseBodyMaxLength = value >= 0 ? value : undefined;
+      if (envResponseBodyMaxLength == undefined)
+        warn(`Environment variable ${kResponseBodyMaxLengthEnvar} must be a non-negative integer.`);
+    }
+
+    this.responseBodyMaxLength =
+      envResponseBodyMaxLength ?? config?.response_body_max_length ?? responseBodyMaxLengthDefault;
   }
 
   private absoluteAppmapDir?: string;
@@ -77,6 +93,7 @@ interface ConfigFile {
   appmap_dir?: string;
   name?: string;
   packages?: Package[];
+  response_body_max_length?: number;
 }
 
 function readConfigFile(path: string | undefined): ConfigFile | undefined {
@@ -89,6 +106,10 @@ function readConfigFile(path: string | undefined): ConfigFile | undefined {
   if ("name" in config) result.name = String(config.name);
   if ("appmap_dir" in config) result.appmap_dir = String(config.appmap_dir);
   if ("packages" in config) result.packages = parsePackages(config.packages);
+  if ("response_body_max_length" in config) {
+    const value = parseInt(String(config.response_body_max_length));
+    result.response_body_max_length = value >= 0 ? value : undefined;
+  }
 
   return result;
 }

src/hooks/http.ts

@@ -6,6 +6,7 @@ import type https from "node:https";
 import { URL } from "node:url";
 
 import type AppMap from "../AppMap";
+import config from "../config";
 import Recording from "../Recording";
 import { info, warn } from "../message";
 import { parameter } from "../parameter";
@@ -107,13 +108,17 @@ function handleClientRequest(request: http.ClientRequest) {
     );
 
     request.on("response", (response) => {
+      const capture = new BodyCapture();
+      response.on("data", (chunk: Chunk) => {
+        capture.push(chunk);
+      });
+
       response.once("end", () => {
         if (!recording.running) {
           warnRecordingNotRunning(url);
           return;
         }
-
-        handleClientResponse(clientRequestEvent, startTime, response);
+        handleClientResponse(clientRequestEvent, startTime, response, capture);
       });
     });
   });
@@ -134,13 +139,17 @@ function handleClientResponse(
   requestEvent: AppMap.HttpClientRequestEvent,
   startTime: number,
   response: http.IncomingMessage,
+  capture: BodyCapture,
 ): void {
   assert(response.statusCode != undefined);
   recording.httpClientResponse(
     requestEvent.id,
     getTime() - startTime,
     response.statusCode,
     normalizeHeaders(response.headers),
+    capture.createReturnValue(
+      response.headers["content-type"]?.startsWith("application/json") ?? false,
+    ),
   );
 }
 
@@ -155,6 +164,92 @@ function shouldIgnoreRequest(request: http.IncomingMessage) {
   return false;
 }
 
+type Chunk = string | Buffer | Uint8Array;
+class BodyCapture {
+  private chunks: Buffer[] = [];
+  private currentLength = 0;
+  private totalBodyLength = 0;
+
+  push(chunk: Chunk, encoding?: BufferEncoding) {
+    if (!chunk) return;
+    this.totalBodyLength += chunk.length;
+    if (config.responseBodyMaxLength <= this.currentLength) return;
+
+    if (typeof chunk === "string") chunk = Buffer.from(chunk, encoding);
+    this.chunks.push(Buffer.from(chunk));
+    this.currentLength += chunk.length;
+  }
+
+  createReturnValue(isJson: boolean) {
+    let returnValue: AppMap.Parameter | undefined;
+    let caputuredBodyString: string = Buffer.concat(this.chunks).toString("utf8");
+    if (caputuredBodyString.length > config.responseBodyMaxLength)
+      caputuredBodyString = caputuredBodyString.substring(0, config.responseBodyMaxLength);
+
+    if (caputuredBodyString.length > 0) {
+      // If it's truncated add the rider
+      if (this.totalBodyLength > caputuredBodyString.length) {
+        const truncatedLength = this.totalBodyLength - caputuredBodyString.length;
+        caputuredBodyString += `... (${truncatedLength} more characters)`;
+      } else if (isJson) {
+        // Not truncated, try to parse JSON and make it a Parameter
+        try {
+          const obj: unknown = JSON.parse(caputuredBodyString);
+          returnValue = parameter(obj);
+        } catch {
+          /* Cannot be parsed */
+        }
+      }
+      if (returnValue == undefined)
+        returnValue = { class: "[ResponseBody]", value: caputuredBodyString };
+    }
+    return returnValue;
+  }
+}
+
+function captureResponseBody(response: http.ServerResponse, capture: BodyCapture) {
+  const originalWrite = response.write.bind(response);
+  const originalEnd = response.end.bind(response);
+
+  type WriteCallback = (error: Error | null | undefined) => void;
+
+  response.write = function (
+    chunk: Chunk,
+    encoding?: BufferEncoding | WriteCallback,
+    callback?: WriteCallback,
+  ) {
+    if (typeof encoding === "function") {
+      callback = encoding;
+      encoding = undefined;
+    }
+
+    capture.push(chunk, encoding);
+
+    if (encoding != null) return originalWrite(chunk, encoding, callback);
+    return originalWrite(chunk, callback);
+  };
+
+  type EndCallback = () => void;
+
+  response.end = function (
+    chunk?: Chunk | EndCallback,
+    encoding?: BufferEncoding | EndCallback,
+    callback?: EndCallback,
+  ) {
+    if (!chunk || typeof chunk === "function") {
+      return originalEnd(chunk);
+    } else if (typeof encoding === "function") {
+      callback = encoding;
+      encoding = undefined;
+    }
+
+    capture.push(chunk, encoding);
+
+    if (encoding != null) return originalEnd(chunk, encoding, callback);
+    return originalEnd(chunk, callback);
+  };
+}
+
 function handleRequest(request: http.IncomingMessage, response: http.ServerResponse) {
   if (!(request.method && request.url)) return;
 
@@ -169,9 +264,13 @@ function handleRequest(request: http.IncomingMessage, response: http.ServerRespo
     normalizeHeaders(request.headers),
     url.searchParams,
   );
+
+  const capture = new BodyCapture();
+  captureResponseBody(response, capture);
+
   const startTime = getTime();
   response.once("finish", () => {
-    handleResponse(request, requestEvent, startTime, timestamp, response);
+    handleResponse(request, requestEvent, startTime, timestamp, response, capture);
   });
 }
 
@@ -238,13 +337,19 @@ function handleResponse(
   startTime: number,
   timestamp: string | undefined,
   response: http.ServerResponse<http.IncomingMessage>,
+  capture: BodyCapture,
 ): void {
   if (fixupEvent(request, requestEvent)) recording.fixup(requestEvent);
+
+  const contentType = response.getHeader("Content-Type");
+  const isJson = typeof contentType == "string" && contentType.startsWith("application/json");
+
   recording.httpResponse(
     requestEvent.id,
     getTime() - startTime,
     response.statusCode,
     normalizeHeaders(response.getHeaders()),
+    capture.createReturnValue(isJson),
   );
   if (remoteRunning) return;
   const { request_method, path_info } = requestEvent.http_server_request;