Use the ENGINE API for overriding libcrypto rand

The RAND_set_rand_method has been deprecated in OpenSSL and replaced
by the ENGINE functionality. This change uses the ENGINE API to
over-ride RAND.

tests/unit/s2n_override_openssl_random_test.c

@@ -19,6 +19,9 @@
 
 #include "utils/s2n_random.h"
 #include "utils/s2n_blob.h"
+
+#include <openssl/engine.h>
+#include <openssl/dh.h>
 #include <s2n.h>
 
 #if !defined(OPENSSL_IS_BORINGSSL)
@@ -33,7 +36,7 @@ static uint8_t dhparams[] =
 
 static int mock_called = 0;
 
-int mock_openssl_compat_rand(unsigned char *buf, int num)
+static int mock_openssl_compat_rand(unsigned char *buf, int num)
 {
     int r = s2n_get_random_data(buf, num);
     if (r < 0) {
@@ -45,31 +48,21 @@ int mock_openssl_compat_rand(unsigned char *buf, int num)
     return 1;
 }
 
-void mock_openssl_compat_seed(const void *buf, int num)
-{
-
-}
-
-int mock_openssl_compat_status()
+static int mock_openssl_compat_status()
 {
     return 1;
 }
 
-void mock_openssl_compat_cleanup()
-{
-
-}
-
-void mock_openssl_compat_add(const void *buf, int num, double entropy)
+static int mock_openssl_compat_init(ENGINE *e)
 {
-
+    return 1;
 }
 
 RAND_METHOD mock_openssl_rand_method = {
-    .seed = mock_openssl_compat_seed,
+    .seed = NULL,
     .bytes = mock_openssl_compat_rand,
-    .cleanup = mock_openssl_compat_cleanup,
-    .add = mock_openssl_compat_add,
+    .cleanup = NULL,
+    .add = NULL,
     .pseudorand = mock_openssl_compat_rand,
     .status = mock_openssl_compat_status
 };
@@ -85,9 +78,21 @@ int main(int argc, char **argv)
 
     EXPECT_SUCCESS(s2n_init());
 
-    /* Over-ride OpenSSL's PRNG */
-    RAND_set_rand_method(&mock_openssl_rand_method);
-
+    ENGINE *e = ENGINE_new();
+    EXPECT_NOT_NULL(e);
+
+    EXPECT_TRUE(ENGINE_set_id(e, "s2n_test") == 1);
+    EXPECT_TRUE(ENGINE_set_name(e, "s2n_test entropy generator") == 1);
+    EXPECT_TRUE(ENGINE_set_flags(e, ENGINE_FLAGS_NO_REGISTER_ALL) == 1);
+    EXPECT_TRUE(ENGINE_set_init_function(e, mock_openssl_compat_init) == 1);
+    EXPECT_TRUE(ENGINE_set_RAND(e, &mock_openssl_rand_method) == 1);
+    EXPECT_TRUE(ENGINE_add(e) == 1);
+    EXPECT_TRUE(ENGINE_free(e) == 1);
+
+    EXPECT_NOT_NULL(e = ENGINE_by_id("s2n_test"));
+    EXPECT_TRUE(ENGINE_init(e) == 1);
+    EXPECT_TRUE(ENGINE_set_default(e, ENGINE_METHOD_RAND) == 1);
+
     /* Parse the DH params */
     b.data = dhparams;
     b.size = sizeof(dhparams);
@@ -101,7 +106,7 @@ int main(int argc, char **argv)
 
     EXPECT_EQUAL(mock_called, 0);
 
-    EXPECT_SUCCESS(s2n_dh_generate_ephemeral_key(&dh_params));
+    EXPECT_TRUE(DH_generate_key(dh_params.dh) == 1);
 
     /* Verify that our mock random is called and that over-riding works */
     EXPECT_EQUAL(mock_called, 1);

utils/s2n_random.c

@@ -13,6 +13,8 @@
  * permissions and limitations under the License.
  */
 
+#include <openssl/engine.h>
+
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -106,9 +108,7 @@ int s2n_random(int max)
 }
 
 #ifndef OPENSSL_IS_BORINGSSL
-static const RAND_METHOD *original_rand_method;
-
-int openssl_compat_rand(unsigned char *buf, int num)
+int s2n_openssl_compat_rand(unsigned char *buf, int num)
 {
     int r = s2n_get_random_data(buf, num);
     if (r < 0) {
@@ -117,33 +117,23 @@ int openssl_compat_rand(unsigned char *buf, int num)
     return 1;
 }
 
-void openssl_compat_seed(const void *buf, int num)
-{
-
-}
-
-int openssl_compat_status()
+int s2n_openssl_compat_status()
 {
     return 1;
 }
 
-void openssl_compat_cleanup()
-{
-
-}
-
-void openssl_compat_add(const void *buf, int num, double entropy)
+int s2n_openssl_compat_init()
 {
-
+    return 1;
 }
 
 RAND_METHOD s2n_openssl_rand_method = {
-    .seed = openssl_compat_seed,
-    .bytes = openssl_compat_rand,
-    .cleanup = openssl_compat_cleanup,
-    .add = openssl_compat_add,
-    .pseudorand = openssl_compat_rand,
-    .status = openssl_compat_status
+    .seed = NULL,
+    .bytes = s2n_openssl_compat_rand,
+    .cleanup = NULL,
+    .add = NULL,
+    .pseudorand = s2n_openssl_compat_rand,
+    .status = s2n_openssl_compat_status
 };
 #endif
 
@@ -158,10 +148,27 @@ int s2n_init()
     GUARD(s2n_cbc_masks_init());
 
 #ifndef OPENSSL_IS_BORINGSSL
-    original_rand_method = RAND_get_rand_method();
 
-    /* Over-ride OpenSSL's PRNG. NOTE: there is a unit test to validate that this works */
-    RAND_set_rand_method(&s2n_openssl_rand_method);
+    /* Create an engine */
+    ENGINE *e = ENGINE_new();
+    if (e == NULL ||
+        ENGINE_set_id(e, "s2n") != 1 ||
+        ENGINE_set_name(e, "s2n entropy generator") != 1 ||
+        ENGINE_set_flags(e, ENGINE_FLAGS_NO_REGISTER_ALL) != 1 ||
+        ENGINE_set_init_function(e, s2n_openssl_compat_init) != 1 ||
+        ENGINE_set_RAND(e, &s2n_openssl_rand_method) != 1 ||
+        ENGINE_add(e) != 1 ||
+        ENGINE_free(e) != 1) {
+        S2N_ERROR(S2N_ERR_OPEN_RANDOM);
+    }
+
+    /* Use that engine for rand() */
+    e = ENGINE_by_id("s2n");
+    if (e == NULL ||
+        ENGINE_init(e) != 1 ||
+        ENGINE_set_default(e, ENGINE_METHOD_RAND) != 1) {
+        S2N_ERROR(S2N_ERR_OPEN_RANDOM);
+    }
 #endif
 
     return 0;
@@ -175,10 +182,5 @@ int s2n_cleanup()
 
     GUARD(close(entropy_fd));
 
-#ifndef OPENSSL_IS_BORINGSSL
-    /* Restore OpenSSL's original random methods */
-    RAND_set_rand_method(original_rand_method);
-#endif
-
     return 0;
 }