Update RSA key generation to work with OpenSSL 1.1.0

OpenSSL no longer allows you to mess around with rsa->e, etc., so we
have to use RSA_get0_key() and similar functions to access the struct
internals.

eschalot.c

@@ -242,7 +242,7 @@ void *
 worker(void *arg)
 {
 	SHA_CTX		hash, copy;
-	RSA		*rsa;
+	RSA		*rsa = NULL;
 	uint8_t		*tmp, *der,
 			buf[SHA_DIGEST_LENGTH],
 			onion[ONION_LENP1],
@@ -251,14 +251,15 @@ worker(void *arg)
 	uint64_t	*counter;
 	/* Public exponent and the "big-endian" version of it */
 	unsigned int	e, e_be;
+	BIGNUM *big_e = BN_new();
+	BN_set_word(big_e, (unsigned long) RSA_E_START);
 
 	counter = (uint64_t *)arg;
 
 	while (!done) {
 		/* Generate a new RSA key every time e reaches RSA_E_LIMIT */
-		rsa = RSA_generate_key(RSA_KEYS_BITLEN, RSA_E_START,
-		    NULL, NULL);
-		if (!rsa)
+		rsa = RSA_new();
+		if (!RSA_generate_key_ex(rsa, RSA_KEYS_BITLEN, big_e, NULL))
 			error("RSA Key Generation failed!\n");
 		
 		/* Too chatty - disable. */
@@ -277,6 +278,7 @@ worker(void *arg)
 		SHA1_Update(&hash, der, derlen - SIZE_OF_E);
 		free(der);
 		e = RSA_E_START - 2; /* public exponent */
+		BN_set_word(big_e, (unsigned long) e);
 
 		/* Main loop */
 		while  ((e < RSA_E_LIMIT) && !done) {
@@ -306,7 +308,11 @@ worker(void *arg)
 			if (search(buf, onion)) {
 				/* Found a possible key,
 				 * from here on down performance is not critical. */
-				if (!BN_bin2bn((uint8_t *)&e_be, SIZE_OF_E, rsa->e))
+				BIGNUM *new_e;
+				new_e = BN_bin2bn((uint8_t *)&e_be, SIZE_OF_E, NULL);
+				if (new_e == NULL)
+					error("Failed to convert e to BIGNUM!\n");
+				if(!RSA_set0_key(rsa, NULL, new_e, NULL))
 					error("Failed to set e in RSA key!\n");
 				if (!validkey(rsa))
 					error("A bad key was found!\n");
@@ -420,16 +426,36 @@ validkey(RSA *rsa)
 		*gcd = BN_CTX_get(ctx),		/* GCD(p - 1, q - 1) */
 		*lambda = BN_CTX_get(ctx),	/* LCM(p - 1, q - 1) */
 		*tmp = BN_CTX_get(ctx);		/* temporary storage */
-
-	BN_sub(p1, rsa->p, BN_value_one());	/* p - 1 */
-	BN_sub(q1, rsa->q, BN_value_one());	/* q - 1 */
+	BIGNUM *n = BN_CTX_get(ctx),
+		   *e = BN_CTX_get(ctx),
+		   *d = BN_CTX_get(ctx);
+	BIGNUM *p = BN_CTX_get(ctx),
+		   *q = BN_CTX_get(ctx);
+	BIGNUM *dmp1 = BN_CTX_get(ctx),
+		   *dmq1 = BN_CTX_get(ctx),
+		   *iqmp = BN_CTX_get(ctx);
+
+	RSA_get0_key(rsa, (const BIGNUM **)&n, (const BIGNUM **)&e, (const BIGNUM **)&d);
+	if (e == NULL)
+		error("RSA_get0_key() failed!\n");
+
+	RSA_get0_factors(rsa, (const BIGNUM **)&p, (const BIGNUM **)&q);
+	if (p == NULL || q == NULL)
+		error("RSA_get0_factors() failed!\n");
+
+	RSA_get0_crt_params(rsa, (const BIGNUM **)&dmp1, (const BIGNUM **)&dmq1, (const BIGNUM **)&iqmp);
+	if (dmp1 == NULL || dmq1 == NULL)
+		error("RSA_get0_crt_params() failed!\n");
+
+	BN_sub(p1, p, BN_value_one());	/* p - 1 */
+	BN_sub(q1, q, BN_value_one());	/* q - 1 */
 	BN_gcd(gcd, p1, q1, ctx);	   	/* gcd(p - 1, q - 1) */
 
 	BN_div(tmp, NULL, p1, gcd, ctx);
 	BN_mul(lambda, q1, tmp, ctx);		/* lambda(n) */
 
 	/* Check if e is coprime to lambda(n). */
-	BN_gcd(tmp, lambda, rsa->e, ctx);
+	BN_gcd(tmp, lambda, e, ctx);
 	if (!BN_is_one(tmp)) {
 		verbose("WARNING: Key check failed - e is coprime to lambda!\n");
 		return 0;
@@ -437,16 +463,16 @@ validkey(RSA *rsa)
 
 	/* Check if public exponent e is less than n - 1. */
 	/* Subtract n from e to avoid checking BN_is_zero. */
-	BN_sub(tmp, rsa->e, rsa->n);
-	if (!tmp->neg) {
+	BN_sub(tmp, n, BN_value_one());
+	if (BN_cmp(e, tmp) >= 0) {
 		verbose("WARNING: Key check failed - e is less than (n - 1)!\n");
 		return 0;
 	}
 
-	BN_mod_inverse(rsa->d, rsa->e, lambda, ctx);	/* d */
-	BN_mod(rsa->dmp1, rsa->d, p1, ctx);		/* d mod(p - 1) */
-	BN_mod(rsa->dmq1, rsa->d, q1, ctx);		/* d mod(q - 1) */
-	BN_mod_inverse(rsa->iqmp, rsa->q, rsa->p, ctx);	/* q ^ -1 mod p */
+	BN_mod_inverse(d, e, lambda, ctx);	/* d */
+	BN_mod(dmp1, d, p1, ctx);		/* d mod(p - 1) */
+	BN_mod(dmq1, d, q1, ctx);		/* d mod(q - 1) */
+	BN_mod_inverse(iqmp, q, p, ctx);	/* q ^ -1 mod p */
 	BN_CTX_end(ctx);
 	BN_CTX_free(ctx);