Track and properly expose encoding errors within editable bodies

src/model/http/editable-body.ts

@@ -20,8 +20,8 @@ export class EditableBody {
 
     /**
      * We effectively track three levels of encoded result in these states:
-     * - The last successful result: a fully completed result, useful to access
-     *   a previous result synchronously even if a new result is pending.
+     * - The last successful & overall results: useful to access to read
+     *   a previous result synchronously even while a new result is pending.
      * - A current maybe-WIP encoding promise: this is always the most recently
      *   started encoding promise, but due to throttling it may not return the
      *   mostly recently provided decoded data. This is always set (it is not
@@ -34,7 +34,10 @@ export class EditableBody {
      */
 
     @observable.ref
-    private _encodedBody: Buffer | undefined;
+    private _lastEncodedBody: Buffer | undefined;
+
+    @observable.ref
+    private _lastEncodingResult: Buffer | Error | undefined;
 
     @observable.ref
     private _encodingPromise: ObservablePromise<Buffer>;
@@ -53,10 +56,10 @@ export class EditableBody {
         this._decodedBody = initialDecodedBody;
 
         if (initialEncodedBody) {
-            this._encodedBody = initialEncodedBody;
+            this._lastEncodedBody = this._lastEncodingResult = initialEncodedBody;
             this._encodingPromise = observablePromise(Promise.resolve(initialEncodedBody));
         } else {
-            this._encodedBody = undefined;
+            this._lastEncodedBody = this._lastEncodingResult = undefined;
             this._encodingPromise = this.updateEncodedBody();
         }
 
@@ -80,56 +83,94 @@ export class EditableBody {
 
         const encodeBodyDeferred = this._throttledEncodingDeferred;
 
-        this._encodingPromise = encodeBodyDeferred.promise;
-        this._throttledEncodingDeferred = undefined;
+        runInAction(() => {
+            this._encodingPromise = encodeBodyDeferred.promise;
+            this._throttledEncodingDeferred = undefined;
+        });
 
         const encodings = this.contentEncodings;
 
-        const encodedBody = await encodeBody(this._decodedBody, encodings)
-            .catch((e) => {
-                logError(e, { encodings });
-                return this._decodedBody; // If encoding fails, we send raw data instead
+        try {
+            const encodedBody = await encodeBody(this._decodedBody, encodings);
+
+            runInAction(() => {
+                // Update the latest results, if we're still the latest encoding request
+                if (this._encodingPromise === encodeBodyDeferred.promise) {
+                    this._lastEncodedBody = this._lastEncodingResult = encodedBody;
+                }
             });
 
-        runInAction(() => {
-            // Update the encoded body, if we're the latest encoding request
-            if (this._encodingPromise === encodeBodyDeferred.promise) {
-                this._encodedBody = encodedBody;
-            }
-        });
+            encodeBodyDeferred.resolve(encodedBody);
+        } catch (e: any) {
+            runInAction(() => {
+                // Update the latest results, if we're still the latest encoding request
+                if (this._encodingPromise === encodeBodyDeferred.promise) {
+                    this._lastEncodingResult = e;
+                }
+            });
 
-        encodeBodyDeferred.resolve(encodedBody);
+            encodeBodyDeferred.reject(e);
+        }
     }, this.options.throttleDuration ?? 500, { leading: true, trailing: true });
 
-    @action
-    updateDecodedBody(newBody: Buffer) {
-        this._decodedBody = newBody;
-    }
-
     @computed.struct
     private get contentEncodings() {
         return asHeaderArray(getHeaderValues(this.getHeaders(), 'content-encoding'));
     }
 
+    @action
+    updateDecodedBody(newBody: Buffer) {
+        this._decodedBody = newBody;
+    }
+
     /**
      * A synchronous value, providing the length of the latest encoded body value. This is initially
      * undefined, and then always set after the first successful encoding, but may be outdated
      * compared to the real decoded data.
      */
     @computed
     get latestEncodedLength() {
-        return this._encodedBody?.byteLength;
+        return this._lastEncodedBody?.byteLength;
+    }
+
+    /**
+     * A synchronous value, providing the raw data of the latest encoding result. This is initially
+     * undefined, and then always set after the first successful encoding to either a decoded Buffer
+     * or some kind of error.
+     */
+    @computed
+    get latestEncodingResult() {
+        if (Buffer.isBuffer(this._lastEncodingResult)) {
+            return { state: 'fulfilled', value: this._lastEncodingResult };
+        } else if (this._lastEncodingResult === undefined) {
+            return { state: 'pending' };
+        } else {
+            return { state: 'rejected', value: this._lastEncodingResult }
+        }
     }
 
     /**
      * Always a promise (although it may already be resolved) representing the encoded result of
-     * the current decoded body. Waiting on this will always return a value (in error cases,
-     * the decoded value is returned directly, but encodingError is set).
+     * the current decoded body.
      */
-    get encoded() {
+    get encodingPromise() {
         return this._throttledEncodingDeferred?.promise ?? this._encodingPromise;
     }
 
+    /**
+     * Equivalent to getEncodingPromise, but returning the decoded body as a fallback value if any
+     * errors occurred during encoding.
+     *
+     * Always a promise (although it may already be resolved) representing the encoded result of
+     * the current decoded body.
+     */
+    get encodingBestEffortPromise() {
+        return this.encodingPromise.catch(() => this._decodedBody);
+    }
+
+    /**
+     * The decoded body data itself - always updated and exposed synchronously.
+     */
     get decoded() {
         return this._decodedBody;
     }

src/model/http/exchange-breakpoint.ts

@@ -198,9 +198,9 @@ export abstract class Breakpoint<T extends BreakpointInProgress> {
 
             ...(versionSatisfies(await serverVersion, RAW_BODY_SUPPORTED)
                 // Mockttp v3+ skips auto-encoding only if you use rawBody:
-                ? { rawBody: await this.editableBody.encoded }
+                ? { rawBody: await this.editableBody.encodingBestEffortPromise }
                 // Old Mockttp doesn't support rawBody, never auto-encodes:
-                : { body: await this.editableBody.encoded }
+                : { body: await this.editableBody.encodingBestEffortPromise }
             ),
 
             // Psuedo-headers those will be generated automatically from the other,

src/model/send/send-store.ts

@@ -88,11 +88,13 @@ export class SendStore {
             lookupOptions: passthroughOptions.lookupOptions
         };
 
+        const encodedBody = await requestInput.rawBody.encodingBestEffortPromise;
+
         const responseStream = await ServerApi.sendRequest({
             url: requestInput.url,
             method: requestInput.method,
             headers: requestInput.headers,
-            rawBody: await requestInput.rawBody.encoded
+            rawBody: encodedBody
         }, requestOptions);
 
         const exchange = this.eventStore.recordSentRequest({
@@ -105,7 +107,7 @@ export class SendStore {
             hostname: url.hostname,
             headers: rawHeadersToHeaders(requestInput.headers),
             rawHeaders: requestInput.headers,
-            body: { buffer: await requestInput.rawBody.encoded },
+            body: { buffer: encodedBody },
             timingEvents: {} as TimingEvents,
             tags: ['httptoolkit:manually-sent-request']
         });

test/unit/model/http/editable-body.spec.ts

@@ -1,5 +1,7 @@
 import * as zlib from 'zlib';
 
+import { observable } from 'mobx';
+
 import { expect } from '../../../test-setup';
 
 import { EditableBody } from '../../../../src/model/http/editable-body';
@@ -15,14 +17,14 @@ describe("Editable bodies", () => {
         );
 
         // Check the encoded result synchronously
-        const initialEncodedState = body.encoded;
+        const initialEncodedState = body.encodingPromise;
 
         await delay(0); // Let the encoding promise resolve but little more
 
-        expect(body.encoded).to.equal(initialEncodedState);
+        expect(body.encodingPromise).to.equal(initialEncodedState);
 
-        expect(body.encoded.state).to.equal('fulfilled');
-        const encodedBody = body.encoded.value as Buffer;
+        expect(body.encodingPromise.state).to.equal('fulfilled');
+        const encodedBody = body.encodingPromise.value as Buffer;
         expect(encodedBody.toString('utf8')).to.equal('hello')
     });
 
@@ -34,15 +36,15 @@ describe("Editable bodies", () => {
             { throttleDuration: 0 }
         );
 
-        const initialEncodedState = body.encoded;
+        const initialEncodedState = body.encodingPromise;
 
         await delay(10); // Wait, in case some other encoding kicks in
 
-        expect(body.encoded).to.equal(initialEncodedState);
+        expect(body.encodingPromise).to.equal(initialEncodedState);
         expect(body.latestEncodedLength).to.equal(2);
 
-        expect(body.encoded.state).to.equal('fulfilled');
-        const encodedBody = body.encoded.value as Buffer;
+        expect(body.encodingPromise.state).to.equal('fulfilled');
+        const encodedBody = body.encodingPromise.value as Buffer;
         expect(encodedBody.toString('utf8')).to.equal('hi')
     });
 
@@ -55,7 +57,7 @@ describe("Editable bodies", () => {
 
         expect(body.latestEncodedLength).to.equal(undefined);
 
-        const encodedBody = await body.encoded;
+        const encodedBody = await body.encodingPromise;
         expect(zlib.gunzipSync(encodedBody).toString('utf8')).to.equal('hello');
         expect(body.latestEncodedLength).to.equal(encodedBody.length);
     });
@@ -71,7 +73,7 @@ describe("Editable bodies", () => {
         await delay(0);
         body.updateDecodedBody(Buffer.from('updated'));
 
-        const encodedBody = await body.encoded;
+        const encodedBody = await body.encodingPromise;
         expect((await encodedBody).toString('utf8')).to.equal('updated')
     });
 
@@ -86,7 +88,7 @@ describe("Editable bodies", () => {
         await delay(0);
         body.updateDecodedBody(Buffer.from('updated'));
 
-        const encodedBody = await body.encoded;
+        const encodedBody = await body.encodingPromise;
         expect(encodedBody.toString('utf8')).to.equal('updated');
     });
 
@@ -105,19 +107,61 @@ describe("Editable bodies", () => {
         body.updateDecodedBody(Buffer.from('updated'));
         expect(body.latestEncodedLength).to.equal(5); // Still shows old value during encoding
 
-        await body.encoded;
+        await body.encodingPromise;
         expect(body.latestEncodedLength).to.equal(7); // Correct new value after encoding
     });
 
-    it("should return the decoded raw body if encoding fails", async () => {
+    it("should throw encoding errors by default if encoding fails", async () => {
+        const body = new EditableBody(
+            Buffer.from('hello'),
+            undefined,
+            () => [['content-encoding', 'invalid-unknown-encoding']], // <-- this will fail
+            { throttleDuration: 0 }
+        );
+
+        try {
+            const result = await body.encodingPromise;
+            expect.fail(`Encoding should have thrown an error but returned ${result}`);
+        } catch (e: any) {
+            expect(e.message).to.equal('Unsupported encoding: invalid-unknown-encoding');
+        }
+    });
+
+    it("should still expose the previous errors during encoding", async () => {
+        let headers = observable.box([['content-encoding', 'invalid-unknown-encoding']] as Array<[string, string]>);
+        const body = new EditableBody(
+            Buffer.from('hello'),
+            undefined,
+            () => headers.get(), // <-- these headesr will fail to encode initially
+            { throttleDuration: 0 }
+        );
+
+        expect(body.latestEncodingResult).to.deep.equal({ state: 'pending' }); // Initial pre-encoding value
+        await delay(0);
+
+        // Initial failure:
+        const secondResult = body.latestEncodingResult;
+        expect(secondResult.state).to.equal('rejected');
+        expect((secondResult.value as any).message).to.equal('Unsupported encoding: invalid-unknown-encoding');
+
+        headers.set([]);
+        body.updateDecodedBody(Buffer.from('updated'));
+
+        expect(body.latestEncodingResult).to.deep.equal(secondResult); // Still shows initial failure during encoding
+
+        await body.encodingPromise;
+        expect(body.latestEncodingResult).to.deep.equal({ state: 'fulfilled', value: Buffer.from('updated') }); // Correct new value after encoding
+    });
+
+    it("should return the decoded raw body as a best effort if encoding fails", async () => {
         const body = new EditableBody(
             Buffer.from('hello'),
             undefined,
             () => [['content-encoding', 'invalid-unknown-encoding']], // <-- this will fail
             { throttleDuration: 0 }
         );
 
-        const encodedBody = await body.encoded;
+        const encodedBody = await body.encodingBestEffortPromise;
         expect(encodedBody.toString('utf8')).to.equal('hello');
     });
 
@@ -135,7 +179,7 @@ describe("Editable bodies", () => {
         await delay(1);
         body.updateDecodedBody(Buffer.from('third update'));
 
-        const updatedEncodedState = body.encoded;
+        const updatedEncodedState = body.encodingPromise;
         expect((await updatedEncodedState).toString('utf8')).to.equal('third update')
     });