From 69cf179746c30396a7a9c8ce0fe3c9194bfb52a3 Mon Sep 17 00:00:00 2001
From: Mark Kogan <mkogan@redhat.com>
Date: Wed, 21 Dec 2022 16:37:09 +0000
Subject: [PATCH] rgw: under fips & openssl 3.x allow md5 iusage in select rgw
 ops

openssl 3.x (ex:RHEL9) requires a different override mechanism for MD5 usage under FIPS
for non-cryptographic putposes than openssl 1.x (RHEL8)

fixes: https://tracker.ceph.com/issues/58332

Signed-off-by: Mark Kogan <mkogan@redhat.com>
---
 src/common/ceph_crypto.cc | 19 +++++++++++++++++--
 src/common/ceph_crypto.h  |  1 +
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/common/ceph_crypto.cc b/src/common/ceph_crypto.cc
index d658edd6af1..18e655b937a 100644
--- a/src/common/ceph_crypto.cc
+++ b/src/common/ceph_crypto.cc
@@ -196,14 +196,29 @@ ssl::OpenSSLDigest::OpenSSLDigest(const EVP_MD * _type)
 
 ssl::OpenSSLDigest::~OpenSSLDigest() {
   EVP_MD_CTX_destroy(mpContext);
+  if (mpType_FIPS) {
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    EVP_MD_free(mpType_FIPS);
+#endif  // OPENSSL_VERSION_NUMBER >= 0x30000000L
+  }
 }
 
 void ssl::OpenSSLDigest::Restart() {
-  EVP_DigestInit_ex(mpContext, mpType, NULL);
+  if (mpType_FIPS) {
+    EVP_DigestInit_ex(mpContext, mpType_FIPS, NULL);
+  } else {
+    EVP_DigestInit_ex(mpContext, mpType, NULL);
+  }
 }
 
 void ssl::OpenSSLDigest::SetFlags(int flags) {
-  EVP_MD_CTX_set_flags(mpContext, flags);
+  if (flags == EVP_MD_CTX_FLAG_NON_FIPS_ALLOW && OpenSSL_version_num() >= 0x30000000L && mpType == EVP_md5() && !mpType_FIPS) {
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    mpType_FIPS = EVP_MD_fetch(NULL, "MD5", "fips=no");
+#endif  // OPENSSL_VERSION_NUMBER >= 0x30000000L
+  } else {
+    EVP_MD_CTX_set_flags(mpContext, flags);
+  }
   this->Restart();
 }
 
diff --git a/src/common/ceph_crypto.h b/src/common/ceph_crypto.h
index dd1b14ffab6..bcdc0044cbd 100644
--- a/src/common/ceph_crypto.h
+++ b/src/common/ceph_crypto.h
@@ -54,6 +54,7 @@ namespace TOPNSPC::crypto {
       private:
 	EVP_MD_CTX *mpContext;
 	const EVP_MD *mpType;
+        EVP_MD *mpType_FIPS = nullptr;
       public:
 	OpenSSLDigest (const EVP_MD *_type);
 	~OpenSSLDigest ();