worker: do not crash when JSTransferable lists untransferable value

src/node_messaging.cc

@@ -343,18 +343,20 @@ class SerializerDelegate : public ValueSerializer::Delegate {
 
  private:
   Maybe<bool> WriteHostObject(BaseObjectPtr<BaseObject> host_object) {
+    BaseObject::TransferMode mode = host_object->GetTransferMode();
+    if (mode == BaseObject::TransferMode::kUntransferable) {
+      ThrowDataCloneError(env_->clone_unsupported_type_str());
+      return Nothing<bool>();
+    }
+
     for (uint32_t i = 0; i < host_objects_.size(); i++) {
       if (host_objects_[i] == host_object) {
         serializer->WriteUint32(i);
         return Just(true);
       }
     }
 
-    BaseObject::TransferMode mode = host_object->GetTransferMode();
-    if (mode == BaseObject::TransferMode::kUntransferable) {
-      ThrowDataCloneError(env_->clone_unsupported_type_str());
-      return Nothing<bool>();
-    } else if (mode == BaseObject::TransferMode::kTransferable) {
+    if (mode == BaseObject::TransferMode::kTransferable) {
       THROW_ERR_MISSING_TRANSFERABLE_IN_TRANSFER_LIST(env_);
       return Nothing<bool>();
     }

test/parallel/test-worker-message-port-jstransferable-nested-untransferable.js

@@ -0,0 +1,38 @@
+// Flags: --expose-internals --no-warnings
+'use strict';
+const common = require('../common');
+const assert = require('assert');
+const {
+  JSTransferable, kTransfer, kTransferList
+} = require('internal/worker/js_transferable');
+const { MessageChannel } = require('worker_threads');
+
+// Transferring a JSTransferable that refers to another, untransferable, value
+// in its transfer list should not crash hard.
+
+class OuterTransferable extends JSTransferable {
+  constructor() {
+    super();
+    // Create a detached MessagePort at this.inner
+    const c = new MessageChannel();
+    this.inner = c.port1;
+    c.port2.postMessage(this.inner, [ this.inner ]);
+  }
+
+  [kTransferList] = common.mustCall(() => {
+    return [ this.inner ];
+  });
+
+  [kTransfer] = common.mustCall(() => {
+    return {
+      data: { inner: this.inner },
+      deserializeInfo: 'does-not:matter'
+    };
+  });
+}
+
+const { port1 } = new MessageChannel();
+const ot = new OuterTransferable();
+assert.throws(() => {
+  port1.postMessage(ot, [ot]);
+}, { name: 'DataCloneError' });