openssl EVP fails randomly in decryption - c++

Question

I have a problem in using higher level envelope decryption fron openssl/evp.h
EVP_OpenFinal and EVP_OpenInit fail randomly when I try to run the same program multiple times with the same values. I am working on it for last 5 days but no luck. I am writing my complete code here. Please help me out. It would be very nice if someone can try to reproduce the errors, and let me know where I am wrong.
Error message:
This application has requested the Runtime to terminate it in an unusual way.
Please contact the application's support team for more information.
4944:error:0606506D:lib(6):func(101):reason(109):.\crypto\evp\evp_enc.c:460:
And here is the code
#include <iostream>
#include <openssl/pem.h>
#include <openssl/ssl.h>
#include <openssl/rsa.h>
#include <openssl/evp.h>
#include <openssl/bio.h>
#include <openssl/err.h>
#include <stdio.h>
void handleErrors(void);
int envelope_seal(EVP_PKEY **pub_key, unsigned char *plaintext, int plaintext_len,
unsigned char **encrypted_key, int *encrypted_key_len, unsigned char *iv,
unsigned char *ciphertext);
int envelope_open(EVP_PKEY *priv_key, unsigned char *ciphertext, int ciphertext_len,
unsigned char *encrypted_key, int encrypted_key_len, unsigned char *iv,
unsigned char *plaintext);
using namespace std;
int main()
{
RSA *rsa_pubkey= RSA_new();
RSA *rsa_prikey= RSA_new();
EVP_PKEY *evp_pubkey = EVP_PKEY_new();
EVP_PKEY *evp_prikey = EVP_PKEY_new();
FILE *rsa_prikey_file = NULL;
FILE *rsa_pubkey_file = NULL;
rsa_pubkey_file = fopen("pubkey.pem", "r");
if (!rsa_pubkey_file)
{
fprintf(stderr, "Error loading PEM RSA Public Key File.\n");
exit(2);
}
PEM_read_RSA_PUBKEY(rsa_pubkey_file, &rsa_pubkey,NULL, NULL);
EVP_PKEY_assign_RSA(evp_pubkey,rsa_pubkey);
rsa_prikey_file = fopen("key.pem", "r");
if (!rsa_prikey_file)
{
fprintf(stderr, "Error loading PEM RSA private Key File.\n");
exit(2);
}
PEM_read_RSAPrivateKey(rsa_prikey_file, &rsa_prikey,NULL, NULL);
EVP_PKEY_assign_RSA(evp_prikey,rsa_prikey);
unsigned char *plaintext = (unsigned char*) "The quick brown fox jumps over thes lazy dog";
unsigned char ciphertext[256] = {};
unsigned char plaintextt[256] = {};
int ciphertextlength;
unsigned char *encKey = (unsigned char*)malloc(RSA_size(rsa_pubkey));
unsigned char iv[16] = {};
envelope_seal(&evp_pubkey, plaintext,strlen((const char*)plaintext),&encKey, &ciphertextlength,iv,ciphertext);
envelope_open(evp_prikey, ciphertext, strlen((const char*)ciphertext), encKey, strlen((const char*)encKey),iv,plaintextt);
std::cout <<"Result: "<<plaintextt<<std::endl;
EVP_PKEY_free(evp_pubkey);
EVP_PKEY_free(evp_prikey);
free(ciphertext);
free(encKey);
}
int envelope_seal(EVP_PKEY **pub_key, unsigned char *plaintext, int plaintext_len,
unsigned char **encrypted_key, int *encrypted_key_len, unsigned char *iv,
unsigned char *ciphertext)
{
EVP_CIPHER_CTX *ctx;
int ciphertext_len;
int len;
if(!(ctx = EVP_CIPHER_CTX_new())) handleErrors();
if(1 != EVP_SealInit(ctx, EVP_aes_128_cbc(),
encrypted_key,
encrypted_key_len,
iv,
pub_key, 1))
{
handleErrors();
}
if(1 != EVP_SealUpdate(ctx, ciphertext, &len, plaintext, plaintext_len))
{
handleErrors();
}
ciphertext_len = len;
if(1 != EVP_SealFinal(ctx, ciphertext + len, &len))
{
handleErrors();
}
ciphertext_len += len;
EVP_CIPHER_CTX_free(ctx);
return ciphertext_len;
}
int envelope_open(EVP_PKEY *priv_key, unsigned char *ciphertext, int ciphertext_len,
unsigned char *encrypted_key, int encrypted_key_len, unsigned char *iv,
unsigned char *plaintext)
{
EVP_CIPHER_CTX *ctx;
int len;
int plaintext_len;
if(!(ctx = EVP_CIPHER_CTX_new()))
{
handleErrors();
}
if(1 != EVP_OpenInit(ctx, EVP_aes_128_cbc(), encrypted_key,
encrypted_key_len, iv, priv_key))
{
handleErrors();
}
if(1 != EVP_OpenUpdate(ctx, plaintext, &len, ciphertext, ciphertext_len))
{
handleErrors();
}
plaintext_len = len;
if(1 != EVP_OpenFinal(ctx, plaintext + len, &len))
{
handleErrors();
}
plaintext_len += len;
EVP_CIPHER_CTX_free(ctx);
plaintext[plaintext_len] = '\0';
return plaintext_len;
}
void handleErrors(void)
{
perror("Error: ");
ERR_print_errors_fp(stderr);
abort();
}

Answer 1

envelope_seal() gives back the length of the encrypted string, I have to put this length in envelope_open()
int **length** = envelope_seal(&evp_pubkey, plaintext,strlen((const char*)plaintext),&encKey, &ciphertextlength,iv,ciphertext);
envelope_open(evp_prikey, ciphertext, **length**, encKey, strlen((const char*)encKey),iv,plaintextt);

Answer 2

The random crashing is caused by your length calculations before envelope_seal and envelope_open. The lengths should look like:
unsigned char *plaintext = (unsigned char*) "The quick brown fox jumps over thes lazy dog";
int ciphertextlength;
unsigned int enclen = EVP_PKEY_size(evp_prikey);
unsigned char *encKey = (unsigned char*)malloc(enclen);
unsigned char *ciphertext = (unsigned char*)malloc(enclen);
unsigned char plaintextt[enclen] = {};
unsigned char iv[16] = {};
unsigned int length = envelope_seal(&evp_pubkey, plaintext,strlen((const char*)plaintext),&encKey, &ciphertextlength,iv,ciphertext);
envelope_open(evp_prikey, ciphertext, length, encKey, enclen,iv,plaintextt);
As your answer states envelope_seal returns the size. You cannot get the size of a char* as your crashing always occured when the size of your plaintext was less than 48.