From 85edb09c595336388737b3beed24f82cb39aaa57 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 10 Sep 2024 17:54:11 +0200
Subject: [PATCH] crypto: Workaround fips bug causing RAND_bytes to fail

Symtom:
RAND_bytes fails on Ubuntu pro with fips enabled.

Problem:
init_curve_types() switched FIPS mode off and on again (and generates
random keys) which seem to cause RAND_bytes to fail accessing its
fips provider.

Solution:
Remove init_curve_types() call at init as this check will be done
by get_curve_cnt() anyway first time crypto:supports(curves) is called.
---
 lib/crypto/c_src/algorithms.c | 42 +++++------------------------------
 1 file changed, 5 insertions(+), 37 deletions(-)

diff --git a/lib/crypto/c_src/algorithms.c b/lib/crypto/c_src/algorithms.c
index c7e1fdea0436..578f79e4f1f1 100644
--- a/lib/crypto/c_src/algorithms.c
+++ b/lib/crypto/c_src/algorithms.c
@@ -39,8 +39,7 @@ void init_pubkey_types(ErlNifEnv* env);
 
 static ERL_NIF_TERM algo_curve[2][89]; /* increase when extending the list */
 static ErlNifMutex* mtx_init_curve_types;
-void init_curve_types(ErlNifEnv* env);
-int get_curve_cnt(ErlNifEnv* env, int fips);
+static int get_curve_cnt(ErlNifEnv* env, int fips);
 
 static unsigned int algo_rsa_opts_cnt, algo_rsa_opts_fips_cnt;
 static ERL_NIF_TERM algo_rsa_opts[11]; /* increase when extending the list */
@@ -56,7 +55,6 @@ void init_algorithms_types(ErlNifEnv* env)
     init_hash_types(env);
 #endif
     init_pubkey_types(env);
-    init_curve_types(env);
     init_rsa_opts_types(env);
     /* ciphers and macs are initiated statically */
 }
@@ -230,9 +228,9 @@ ERL_NIF_TERM curve_algorithms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[
     return enif_make_list_from_array(env, algo_curve[fips_mode], algo_curve_cnt);
 }
 
-int init_curves(ErlNifEnv* env, int fips);
+static int init_curves(ErlNifEnv* env, int fips);
 #if defined(HAVE_EC)
-int valid_curve(int nid);
+static int valid_curve(int nid);
 #endif
 
 int get_curve_cnt(ErlNifEnv* env, int fips) {
@@ -266,38 +264,6 @@ int get_curve_cnt(ErlNifEnv* env, int fips) {
     return cnt;
 }
 
-void init_curve_types(ErlNifEnv* env) {
-    /* Initialize the curve counters and curve's lists
-       by calling get_curve_cnt
-    */
-#ifdef FIPS_SUPPORT
-    if (FIPS_MODE()) {
-        // FIPS enabled
-        get_curve_cnt(env, 1);
-        FIPS_mode_set(0); // disable
-        get_curve_cnt(env, 0);
-        FIPS_mode_set(1); // re-enable
-    } else {
-        // FIPS disabled but available
-        get_curve_cnt(env, 0);
-        FIPS_mode_set(1); // enable
-        get_curve_cnt(env, 1);
-        FIPS_mode_set(0); // re-disable
-    }
-#else
-    // FIPS mode is not available
-    get_curve_cnt(env, 0);
-#endif
-
-# ifdef DEBUG
-    {
-        int curve_cnt = get_curve_cnt(env, 0);
-        ASSERT(curve_cnt <= sizeof(algo_curve[0])/sizeof(ERL_NIF_TERM));
-    }
-# endif 
-}
-
-
 int init_curves(ErlNifEnv* env, int fips) {
 #if defined(HAVE_EC)
     int cnt = 0;
@@ -647,6 +613,8 @@ int init_curves(ErlNifEnv* env, int fips) {
 #endif
     }
 
+    ASSERT(cnt <= sizeof(algo_curve[0])/sizeof(ERL_NIF_TERM));
+
     return cnt;
 #else /* if not HAVE_EC */
     return 0;