crypto: add cipher update/final methods encoding validation

Refs #45189

PR-URL: #45990
Refs: #45189
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Filip Skokan <panva.ip@gmail.com>

Author: vitpavlenko

lib/internal/crypto/cipher.js

@@ -27,6 +27,7 @@ const {
     ERR_CRYPTO_INVALID_STATE,
     ERR_INVALID_ARG_TYPE,
     ERR_INVALID_ARG_VALUE,
+    ERR_UNKNOWN_ENCODING,
   }
 } = require('internal/errors');
 
@@ -91,9 +92,14 @@ const privateDecrypt = rsaFunctionFor(_privateDecrypt, RSA_PKCS1_OAEP_PADDING,
                                       'private');
 
 function getDecoder(decoder, encoding) {
-  encoding = normalizeEncoding(encoding);
+  const normalizedEncoding = normalizeEncoding(encoding);
   decoder = decoder || new StringDecoder(encoding);
-  assert(decoder.encoding === encoding, 'Cannot change encoding');
+  if (decoder.encoding !== normalizedEncoding) {
+    if (normalizedEncoding === undefined) {
+      throw new ERR_UNKNOWN_ENCODING(encoding);
+    }
+    assert(false, 'Cannot change encoding');
+  }
   return decoder;
 }
 

test/parallel/test-crypto-encoding-validation-error.js

@@ -0,0 +1,52 @@
+'use strict';
+const common = require('../common');
+if (!common.hasCrypto)
+  common.skip('missing crypto');
+
+// This test checks if error is thrown in case of wrong encoding provided into cipher.
+
+const assert = require('assert');
+const { createCipheriv, randomBytes } = require('crypto');
+
+const createCipher = () => {
+  return createCipheriv('aes-256-cbc', randomBytes(32), randomBytes(16));
+};
+
+{
+  const cipher = createCipher();
+  cipher.update('test', 'utf-8', 'utf-8');
+
+  assert.throws(
+    () => cipher.update('666f6f', 'hex', 'hex'),
+    { message: /Cannot change encoding/ }
+  );
+}
+
+{
+  const cipher = createCipher();
+  cipher.update('test', 'utf-8', 'utf-8');
+
+  assert.throws(
+    () => cipher.final('hex'),
+    { message: /Cannot change encoding/ }
+  );
+}
+
+{
+  const cipher = createCipher();
+  cipher.update('test', 'utf-8', 'utf-8');
+
+  assert.throws(
+    () => cipher.final('bad2'),
+    { message: /^Unknown encoding: bad2$/, code: 'ERR_UNKNOWN_ENCODING' }
+  );
+}
+
+{
+  const cipher = createCipher();
+
+  assert.throws(
+    () => cipher.update('test', 'utf-8', 'bad3'),
+    { message: /^Unknown encoding: bad3$/, code: 'ERR_UNKNOWN_ENCODING' }
+  );
+}