OpenSSL RSA Public/Private Encryption/Decryption with pkcs12 file - c++

Question

I am porting a service from C# to C++ - which involves RSA encryption. Using OpenSSL 1.0.2. (I wrote my first line of C++ two weeks ago - so please bear with me.)
I am trying to encrypt/decrypt data - using RSA. Encryption uses a public key which is retrieved by invoking a server via HTTPS (The public key of the SSL certificate is used).
Decryption happens on the server - which loads the private key from a PFX file (PKCS12).
I am having problems with decryption - error:0407109F:rsa routines:RSA_padding_check_PKCS1_type_2:pkcs decoding error
What am I doing wrong?
Encryption - getting public key from server (HTTPS)
int MyUtil::EncryptWithPublicKeyRetrievedFromServer(int length, unsigned char* bytesToEncrypt, unsigned char* output){
//Get Certificate from server
boost::asio::ssl::context ctx(boost::asio::ssl::context::sslv23);
boost::asio::io_service io_service;
ctx.set_default_verify_paths();
client myClient = client(io_service, ctx);
auto cert = myClient.GetX509Certificate("www.myHttpsDomain.com");
//Get public key
auto key = X509_get_pubkey(cert);
auto rsa = EVP_PKEY_get1_RSA(key);
//Encrypt
return RSA_public_encrypt(length, bytesToEncrypt, output, rsa, RSA_PKCS1_PADDING);
}
Decryption - using private key
int MyUtil::DecryptWithPrivateKey(int length, unsigned char* bytesToDecrypt, unsigned char *output){
auto filePath = "C:\\myCertificateWithPrivateAndPublicKey.pfx";
auto pass = "MySecretPassword";
FILE *fp;
EVP_PKEY *pkey;
X509 *cert;
STACK_OF(X509) *ca = NULL;
PKCS12 *p12;
OpenSSL_add_all_algorithms();
ERR_load_crypto_strings();
#pragma warning (disable : 4996) //MS complains about fopen
if (!(fp = fopen(filePath, "rb"))) {
fprintf(stderr, "Error opening file %s\n", filePath);
exit(1);
}
p12 = d2i_PKCS12_fp(fp, NULL);
fclose(fp);
if (!p12) {
fprintf(stderr, "Error reading PKCS#12 file\n");
ERR_print_errors_fp(stderr);
exit(1);
}
//Parse PKCS12 (PFX) file in order to get private key
if (!PKCS12_parse(p12, pass, &pkey, &cert, &ca)) {
fprintf(stderr, "Error parsing PKCS#12 file\n");
ERR_print_errors_fp(stderr);
exit(1);
}
PKCS12_free(p12);
//Decrypt
auto rsa = EVP_PKEY_get1_RSA(pkey);
int sizeOfDecryptedData;
if (sizeOfDecryptedData = RSA_private_decrypt(length, bytesToDecrypt, output, rsa, RSA_PKCS1_PADDING) == -1)
{
auto err = new char;
ERR_load_crypto_strings();
ERR_error_string(ERR_get_error(), err);
fprintf(stderr, "Error encrypting message: %s\n", err);
//--> error:0407109F:rsa routines:RSA_padding_check_PKCS1_type_2:pkcs decoding error
delete err;
}
sk_X509_pop_free(ca, X509_free);
X509_free(cert);
EVP_PKEY_free(pkey);
fclose(fp);
return sizeOfDecryptedData;
}
I also tried loading the private key through the PEM format and then decrypt
int MyUtil::DecryptWithPrivateKey(int length, unsigned char* bytesToDecrypt, unsigned char *output){
char *private_key_file_name = "C:\\privatekey.cer";
#pragma warning (disable : 4996) //MS complains about fopen
FILE *fp = fopen(private_key_file_name, "r");
RSA *rsa = PEM_read_RSAPrivateKey(fp, NULL, NULL, NULL);
fclose(fp);
if (sizeOfDecryptedData = RSA_private_decrypt(length, bytesToDecrypt, output, rsa, RSA_PKCS1_PADDING) == -1)
{
auto err = new char;
ERR_load_crypto_strings();
ERR_error_string(ERR_get_error(), err);
fprintf(stderr, "Error encrypting message: %s\n", err);
//--> error:0407109F:rsa routines:RSA_padding_check_PKCS1_type_2:pkcs decoding error
delete err;
}
return decryptedBytes;
}