Decryption failure using openssl EVP_CipherFinal_ex - c++

Question

I am trying to encrypt and decrypt a buffer using openssl.
I took some example code from here (see the 2nd do_crypt function towards the end of the page) and modified it to take the message to encrypt/decrypt from a vector instead of an input file.
Unfortunately, during the decryption stage, the EVP_CipherFinal_ex funcition fails, and I don't know why.
I have tried to make a compilable example from my code (but i haven't been able to test it), that will show you what i've done so far :
#include <iostream>
#include <vector>
#include <openssl/evp.h>
using namespace std;
typedef vector<uint8_t> Vect;
Vect _cipherMessage(const Vect& ai_in, const Vect& ai_key, const Vect& ai_iv, int ai_encode_decode)
{
cout << ((ai_encode_decode==1)?"ENCODE":"DECODE") << endl;
Vect w_ret;
uint8_t outbuf[1024 + EVP_MAX_BLOCK_LENGTH];
int outlen;
EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
EVP_CipherInit_ex(ctx, EVP_aes_128_cbc(), NULL, ai_key.data(), ai_iv.data(), ai_encode_decode);
if (!EVP_CipherUpdate(ctx, outbuf, &outlen, ai_in.data(), ai_in.size()))
{
// Error
cout << "EVP_CipherUpdate" << endl;
EVP_CIPHER_CTX_free(ctx);
return w_ret;
}
if (!EVP_CipherFinal_ex(ctx, outbuf + outlen, &outlen))
{
// Error
cout << "EVP_CipherFinal_ex" << endl;
EVP_CIPHER_CTX_free(ctx);
return w_ret;
}
EVP_CIPHER_CTX_free(ctx);
for (uint32_t i=0 ; i<outlen ; i++)
{
w_ret.push_back(outbuf[i]);
}
return w_ret;
}
int main()
{
Vect w_inputFrame { 0x53, 0x69, 0x78, 0x74, 0x65, 0x65, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4c, 0x6f, 0x6e, 0x67 }; // "SixteenBytesLong" in hex
Vect w_key { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f };
Vect w_iv { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f };
Vect w_encryptedFrame = _cipherMessage(w_inputFrame, w_key, w_iv, 1);
Vect w_decryptedFrame = _cipherMessage(w_encryptedFrame, w_key, w_iv, 0);
cout << "Encryption / Decryption : " << ((w_inputFrame == w_decryptedFrame)?"SUCCESS":"FAILURE") << endl;
return 0;
}
Please could someone tell me what I am doing wrong ?
NOTE :
I have tested the openssl command on my device and was able to successfully encrypt / decrypt the message :
echo -n SixteenBytesLong | openssl enc -e -aes-128-cbc -K 0102030405060708090a0b0c0d0e0f -iv 0102030405060708090a0b0c0d0e0f -nosalt -nopad -base64 -p
key=0102030405060708090A0B0C0D0E0F00
iv =0102030405060708090A0B0C0D0E0F00
Ow74k7Pm+g8KQNiKZQptgw==
echo Ow74k7Pm+g8KQNiKZQptgw== | openssl enc -d -aes-128-cbc -K 0102030405060708090a0b0c0d0e0f -iv 0102030405060708090a0b0c0d0e0f -nosalt -nopad -base64 -p
key=0102030405060708090A0B0C0D0E0F00
iv =0102030405060708090A0B0C0D0E0F00
SixteenBytesLong

Answer 1

The main problem is that outlen is being overwritten the call to EVP_CipherFinal_ex. So your total outlen is incorrect when you get to the end of the funciton.
You could do something like:
Vect _cipherMessage(const Vect& ai_in, const Vect& ai_key, const Vect& ai_iv, int ai_encode_decode)
{
cout << ((ai_encode_decode==1)?"ENCODE":"DECODE") << endl;
Vect w_ret;
uint8_t outbuf[1024 + EVP_MAX_BLOCK_LENGTH];
int outlen;
int totalOutlen;
EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
EVP_CipherInit_ex(ctx, EVP_aes_128_cbc(), NULL, ai_key.data(), ai_iv.data(), ai_encode_decode);
if (!EVP_CipherUpdate(ctx, outbuf, &totalOutlen, ai_in.data(), (int)ai_in.size()))
{
// Error
cout << "EVP_CipherUpdate: " << ERR_error_string(ERR_get_error(), nullptr) << endl;
EVP_CIPHER_CTX_free(ctx);
return w_ret;
}
if (!EVP_CipherFinal_ex(ctx, outbuf + totalOutlen, &outlen))
{
// Error
cout << "EVP_CipherFinal_ex : " << ERR_error_string(ERR_get_error(), nullptr) << endl;
EVP_CIPHER_CTX_free(ctx);
return w_ret;
}
totalOutlen += outlen;
EVP_CIPHER_CTX_free(ctx);
for (int i=0 ; i<totalOutlen ; i++)
{
w_ret.push_back(outbuf[i]);
}
return w_ret;
}
Another problem is that the if the ai_in is larger than (1024 + EVP_MAX_BLOCK_LENGTH) you are going overwrite and corrupt your stack.
You could do something like this:
namespace
{
template<typename T, typename D>
std::unique_ptr<T, D> make_handle(T* handle, D deleter)
{
return std::unique_ptr<T, D>{handle, deleter};
}
}
Vect _cipherMessage(const Vect& ai_in, const Vect& ai_key, const Vect& ai_iv, int ai_encode_decode)
{
cout << ((ai_encode_decode==1)?"ENCODE":"DECODE") << endl;
Vect w_ret;
int outlen;
int totalOutlen;
w_ret.resize(ai_in.size() + EVP_MAX_BLOCK_LENGTH);
auto ctx = make_handle(EVP_CIPHER_CTX_new(), EVP_CIPHER_CTX_free);
if(!ctx)
{
// Error
cout << "EVP_CIPHER_CTX_new: " << ERR_error_string(ERR_get_error(), nullptr) << endl;
return w_ret;
}
if(!EVP_CipherInit_ex(ctx.get(), EVP_aes_128_cbc(), nullptr, ai_key.data(), ai_iv.data(), ai_encode_decode))
{
// Error
cout << "EVP_CipherInit_ex: " << ERR_error_string(ERR_get_error(), nullptr) << endl;
return w_ret;
}
if (!EVP_CipherUpdate(ctx.get(), w_ret.data(), &totalOutlen, ai_in.data(), (int)ai_in.size()))
{
// Error
cout << "EVP_CipherUpdate: " << ERR_error_string(ERR_get_error(), nullptr) << endl;
return w_ret;
}
if (!EVP_CipherFinal_ex(ctx.get(), w_ret.data() + totalOutlen, &outlen))
{
// Error
cout << "EVP_CipherFinal_ex : " << ERR_error_string(ERR_get_error(), nullptr) << endl;
return w_ret;
}
totalOutlen += outlen;
w_ret.resize(totalOutlen);
return w_ret;
}