Make crypto.c compile/link with OpenSSL 3

- Initialize libssl using non-deprecated APIs

OpenSSL 3 deprecated `SSL_library_init` and `SSL_load_error_strings` in
favor of `OPENSSL_init_ssl`. Use `OPENSSL_init_ssl` when dealing with
OpenSSL 1.1 and newer to unbreak the build with OpenSSL 3.

- Move to EVP_MD* APIs

OpenSSL 3 deprecated all of the `MD5_`* APIs. Move to the equivalent
`EVP_MD`* APIs so the code doesn't need to be pinned down to 1.1
compatible APIs and uplifted later.

Some work will likely be required to deal with other OpenSSL offshoots,
e.g., BoringSSL, LibreSSL, etc.

Co-authored-by: Pierre Pronchery <pierre@freebsdfoundation.org>
Co-authored-by: Ed Maste <emaste@FreeBSD.org>
Signed-off-by: Enji Cooper <yaneurabeya@gmail.com>
Sponsored by:	The FreeBSD Foundation

Author: 3 people

crypto.c

@@ -33,13 +33,19 @@
  * SUCH DAMAGE.
  */
 
+#include <openssl/opensslv.h>
+#if (OPENSSL_VERSION_NUMBER >= 0x300000L)
+#define	IS_OPENSSL3	1
+#endif
+
 #include <openssl/x509.h>
 #include <openssl/md5.h>
 #include <openssl/ssl.h>
 #include <openssl/err.h>
 #include <openssl/pem.h>
 #include <openssl/rand.h>
 
+#include <assert.h>
 #include <strings.h>
 #include <string.h>
 #include <syslog.h>
@@ -115,8 +121,16 @@ smtp_init_crypto(int fd, int feature, struct smtp_features* features)
 
 	/* XXX clean up on error/close */
 	/* Init SSL library */
+#if (OPENSSL_VERSION_NUMBER >= 0x10100000L)
+	error = OPENSSL_init_ssl(0, NULL);
+	if (error != 1) {
+		syslog(LOG_WARNING, "remote delivery deferred: SSL init failed: %s", ssl_errstr());
+		return (1);
+	}
+#else
 	SSL_library_init();
 	SSL_load_error_strings();
+#endif
 
 	// Allow any possible version
 #if (OPENSSL_VERSION_NUMBER >= 0x10100000L)
@@ -225,7 +239,12 @@ void
 hmac_md5(unsigned char *text, int text_len, unsigned char *key, int key_len,
     unsigned char* digest)
 {
-        MD5_CTX context;
+#ifdef	IS_OPENSSL3
+	EVP_MD		*md;
+	EVP_MD_CTX	*context;
+#else
+	MD5_CTX		context;
+#endif
         unsigned char k_ipad[65];    /* inner padding -
                                       * key XORd with ipad
                                       */
@@ -234,15 +253,23 @@ hmac_md5(unsigned char *text, int text_len, unsigned char *key, int key_len,
                                       */
         unsigned char tk[16];
         int i;
-        /* if key is longer than 64 bytes reset it to key=MD5(key) */
-        if (key_len > 64) {
-
-                MD5_CTX      tctx;
 
-                MD5_Init(&tctx);
-                MD5_Update(&tctx, key, key_len);
-                MD5_Final(tk, &tctx);
+#ifdef	IS_OPENSSL3
+	context = EVP_MD_CTX_new();
+	assert(context != NULL);
+#endif
 
+        /* if key is longer than 64 bytes reset it to key=MD5(key) */
+        if (key_len > 64) {
+#ifdef	IS_OPENSSL3
+		EVP_DigestInit_ex(context, md, NULL);
+		EVP_DigestUpdate(context, key, key_len);
+		EVP_DigestFinal_ex(context, tk, NULL);
+#else
+                MD5_Init(&context);
+                MD5_Update(&context, key, key_len);
+                MD5_Final(tk, &context);
+#endif
                 key = tk;
                 key_len = 16;
         }
@@ -270,14 +297,43 @@ hmac_md5(unsigned char *text, int text_len, unsigned char *key, int key_len,
                 k_ipad[i] ^= 0x36;
                 k_opad[i] ^= 0x5c;
         }
+
+#ifdef	IS_OPENSSL3
+        /**
+         * Perform inner MD5.
+         */
+
+	/* Init context for first pass. */
+	EVP_DigestInit_ex(context, md, NULL);
+	/* Start with inner pad. */
+	EVP_DigestUpdate(context, k_ipad, 64);
+	/* Update with text of datagram. */
+	EVP_DigestUpdate(context, text, text_len);
+	/* Finish up first pass. */
+	EVP_DigestFinal_ex(context, digest, NULL);
+
+	/**
+         * Perform outer MD5.
+         */
+
+	/* Re-init context for second pass. */
+	EVP_DigestInit_ex(context, md, NULL);
+	/* Start with outer pad. */
+	EVP_DigestUpdate(context, k_opad, 64);
+	/* Update with results of first hash. */
+	EVP_DigestUpdate(context, digest, 16);
+	/* Finish up second pass. */
+	EVP_DigestFinal_ex(context, digest, NULL);
+#else
         /*
          * perform inner MD5
          */
         MD5_Init(&context);                   /* init context for 1st
                                               * pass */
         MD5_Update(&context, k_ipad, 64);     /* start with inner pad */
         MD5_Update(&context, text, text_len); /* then text of datagram */
-        MD5_Final(digest, &context);          /* finish up 1st pass */
+
+	MD5_Final(digest, &context);          /* finish up 1st pass */
         /*
          * perform outer MD5
          */
@@ -287,6 +343,7 @@ hmac_md5(unsigned char *text, int text_len, unsigned char *key, int key_len,
         MD5_Update(&context, digest, 16);     /* then results of 1st
                                               * hash */
         MD5_Final(digest, &context);          /* finish up 2nd pass */
+#endif
 }
 
 /*