Update for openssl 1.1

OpenSSL 1.1 has made significant non-backwards compatible changes to its
API as outlined in:
https://wiki.openssl.org/index.php/OpenSSL_1.1.0_Changes

BRANCH=none
BUG=chromium:738114
TEST=cros_workon --host start vboot_reference
TEST=w/ openssl-1.0.2k: sudo emerge vboot_reference
TEST=w/ openssl-1.1.0e: sudo emerge vboot_reference
 => both build ok
 $ futility version
  => command runs without error
TEST=cros_workon --board=soraka start vboot_reference coreboot
TEST=w/ openssl-1.0.2k: emerge-soraka vboot_reference coreboot
TEST=w/ openssl-1.1.0e: emerge-soraka vboot_reference coreboot
 => All build ok

Change-Id: I37cfc8cbb04a092eab7b0b3224f475b82609447c
Reviewed-on: https://chromium-review.googlesource.com/557739
Commit-Ready: Daniel Kurtz <djkurtz@chromium.org>
Tested-by: Daniel Kurtz <djkurtz@chromium.org>
Reviewed-by: Randall Spangler <rspangler@chromium.org>
Reviewed-by: Mike Frysinger <vapier@chromium.org>

3 files changed, 38 insertions, 4 deletions

diff --git a/host/include/openssl_compat.h b/host/include/openssl_compat.h
new file mode 100644
index 00000000..7771f32a
--- /dev/null
+++ b/host/include/openssl_compat.h
@@ -0,0 +1,26 @@
+/* Copyright 2017 The Chromium OS Authors. All rights reserved.
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#ifndef VBOOT_REFERENCE_OPENSSL_COMPAT_H_
+#define VBOOT_REFERENCE_OPENSSL_COMPAT_H_
+
+#include <openssl/rsa.h>
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+
+static inline void RSA_get0_key(const RSA *rsa, const BIGNUM **n,
+				const BIGNUM **e, const BIGNUM **d)
+{
+	if (n != NULL)
+		*n = rsa->n;
+	if (e != NULL)
+		*e = rsa->e;
+	if (d != NULL)
+		*d = rsa->d;
+}
+
+#endif  /* OPENSSL_VERSION_NUMBER < 0x10100000L */
+
+#endif  /* VBOOT_REFERENCE_OPENSSL_COMPAT_H_ */
diff --git a/host/lib/util_misc.c b/host/lib/util_misc.c
index 95acecb9..d2e694d0 100644
--- a/host/lib/util_misc.c
+++ b/host/lib/util_misc.c
@@ -18,6 +18,7 @@
 #include "2common.h"
 #include "2sha.h"
 #include "host_common.h"
+#include "openssl_compat.h"
 #include "util_misc.h"
 #include "vb2_common.h"
 #include "host_key2.h"
@@ -73,6 +74,7 @@ int vb_keyb_from_rsa(struct rsa_st *rsa_private_key,
 	BIGNUM *N0inv = NULL, *R = NULL, *RR = NULL;
 	BIGNUM *RRTemp = NULL, *NnumBits = NULL;
 	BIGNUM *n = NULL, *rr = NULL;
+	const BIGNUM *rsa_private_key_n;
 	BN_CTX *bn_ctx = BN_CTX_new();
 	uint32_t n0invout;
 	uint32_t bufsize;
@@ -80,7 +82,7 @@ int vb_keyb_from_rsa(struct rsa_st *rsa_private_key,
 	int retval = 1;
 
 	/* Size of RSA key in 32-bit words */
-	nwords = BN_num_bits(rsa_private_key->n) / 32;
+	nwords = RSA_size(rsa_private_key) / 4;
 
 	bufsize = (2 + nwords + nwords) * sizeof(uint32_t);
 	outbuf = malloc(bufsize);
@@ -109,7 +111,8 @@ int vb_keyb_from_rsa(struct rsa_st *rsa_private_key,
 	NEW_BIGNUM(B);
 #undef NEW_BIGNUM
 
-	BN_copy(N, rsa_private_key->n);
+	RSA_get0_key(rsa_private_key, &rsa_private_key_n, NULL, NULL);
+	BN_copy(N, rsa_private_key_n);
 	BN_set_word(Big1, 1L);
 	BN_set_word(Big2, 2L);
 	BN_set_word(Big32, 32L);
diff --git a/host/lib21/host_key.c b/host/lib21/host_key.c
index 4f82d10c..c0235b3b 100644
--- a/host/lib21/host_key.c
+++ b/host/lib21/host_key.c
@@ -17,6 +17,7 @@
 #include "host_common.h"
 #include "host_key2.h"
 #include "host_misc.h"
+#include "openssl_compat.h"
 
 const struct vb2_text_vs_enum vb2_text_vs_sig[] = {
 	{"RSA1024", VB2_SIG_RSA1024},
@@ -565,8 +566,12 @@ int vb2_public_key_hash(struct vb2_public_key *key,
 
 enum vb2_signature_algorithm vb2_rsa_sig_alg(struct rsa_st *rsa)
 {
-	int exp = BN_get_word(rsa->e);
-	int bits = BN_num_bits(rsa->n);
+	const BIGNUM *e, *n;
+	int exp, bits;
+
+	RSA_get0_key(rsa, &n, &e, NULL);
+	exp = BN_get_word(e);
+	bits = BN_num_bits(n);
 
 	switch (exp) {
 	case RSA_3: