Mupdf Encryption Decryption Issue - c++
I am facing some issues while creating a build for Mupdf to read encrypted pdf files. I am seeing the file, pdf_crypt.c to understand how this can be done. But i cannot understand how to do it.
If i put it in plain terms,
I will be encrypting my pdf with a AES 256 bit key.
Where should i put the AES key in pdf_crypt.c, so that mupdf can read the file by decrypting it at runtime ?
I am posting the pdf_crypt.c file for reference..
#include "mupdf/pdf.h"
enum
{
PDF_CRYPT_NONE,
PDF_CRYPT_RC4,
PDF_CRYPT_AESV2,
PDF_CRYPT_AESV3,
PDF_CRYPT_UNKNOWN,
};
enum
{
PDF_PERM_PRINT = 1 << 2,
PDF_PERM_CHANGE = 1 << 3,
PDF_PERM_COPY = 1 << 4,
PDF_PERM_NOTES = 1 << 5,
PDF_PERM_FILL_FORM = 1 << 8,
PDF_PERM_ACCESSIBILITY = 1 << 9,
PDF_PERM_ASSEMBLE = 1 << 10,
PDF_PERM_HIGH_RES_PRINT = 1 << 11,
PDF_DEFAULT_PERM_FLAGS = 0xfffc
};
typedef struct pdf_crypt_filter_s pdf_crypt_filter;
struct pdf_crypt_filter_s
{
int method;
int length;
};
struct pdf_crypt_s
{
pdf_obj *id;
int v;
int length;
pdf_obj *cf;
pdf_crypt_filter stmf;
pdf_crypt_filter strf;
int r;
unsigned char o[48];
unsigned char u[48];
unsigned char oe[32];
unsigned char ue[32];
int p;
int encrypt_metadata;
unsigned char key[32]; /* decryption key generated from password */
};
static void pdf_parse_crypt_filter(fz_context *ctx, pdf_crypt_filter *cf, pdf_crypt *crypt, pdf_obj *name);
/*
* Create crypt object for decrypting strings and streams
* given the Encryption and ID objects.
*/
pdf_crypt *
pdf_new_crypt(fz_context *ctx, pdf_obj *dict, pdf_obj *id)
{
pdf_crypt *crypt;
pdf_obj *obj;
crypt = fz_malloc_struct(ctx, pdf_crypt);
/* Common to all security handlers (PDF 1.7 table 3.18) */
obj = pdf_dict_get(ctx, dict, PDF_NAME_Filter);
if (!pdf_is_name(ctx, obj))
{
pdf_drop_crypt(ctx, crypt);
fz_throw(ctx, FZ_ERROR_GENERIC, "unspecified encryption handler");
}
if (!pdf_name_eq(ctx, PDF_NAME_Standard, obj) != 0)
{
pdf_drop_crypt(ctx, crypt);
fz_throw(ctx, FZ_ERROR_GENERIC, "unknown encryption handler: '%s'", pdf_to_name(ctx, obj));
}
crypt->v = 0;
obj = pdf_dict_get(ctx, dict, PDF_NAME_V);
if (pdf_is_int(ctx, obj))
crypt->v = pdf_to_int(ctx, obj);
if (crypt->v != 1 && crypt->v != 2 && crypt->v != 4 && crypt->v != 5)
{
pdf_drop_crypt(ctx, crypt);
fz_throw(ctx, FZ_ERROR_GENERIC, "unknown encryption version");
}
/* Standard security handler (PDF 1.7 table 3.19) */
obj = pdf_dict_get(ctx, dict, PDF_NAME_R);
if (pdf_is_int(ctx, obj))
crypt->r = pdf_to_int(ctx, obj);
else if (crypt->v <= 4)
{
fz_warn(ctx, "encryption dictionary missing revision value, guessing...");
if (crypt->v < 2)
crypt->r = 2;
else if (crypt->v == 2)
crypt->r = 3;
else if (crypt->v == 4)
crypt->r = 4;
}
else
{
pdf_drop_crypt(ctx, crypt);
fz_throw(ctx, FZ_ERROR_GENERIC, "encryption dictionary missing version and revision value");
}
if (crypt->r < 1 || crypt->r > 6)
{
int r = crypt->r;
pdf_drop_crypt(ctx, crypt);
fz_throw(ctx, FZ_ERROR_GENERIC, "unknown crypt revision %d", r);
}
obj = pdf_dict_get(ctx, dict, PDF_NAME_O);
if (pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) == 32)
memcpy(crypt->o, pdf_to_str_buf(ctx, obj), 32);
/* /O and /U are supposed to be 48 bytes long for revision 5 and 6, they're often longer, though */
else if (crypt->r >= 5 && pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) >= 48)
memcpy(crypt->o, pdf_to_str_buf(ctx, obj), 48);
else
{
pdf_drop_crypt(ctx, crypt);
fz_throw(ctx, FZ_ERROR_GENERIC, "encryption dictionary missing owner password");
}
obj = pdf_dict_get(ctx, dict, PDF_NAME_U);
if (pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) == 32)
memcpy(crypt->u, pdf_to_str_buf(ctx, obj), 32);
/* /O and /U are supposed to be 48 bytes long for revision 5 and 6, they're often longer, though */
else if (crypt->r >= 5 && pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) >= 48)
memcpy(crypt->u, pdf_to_str_buf(ctx, obj), 48);
else if (pdf_is_string(ctx, obj) && pdf_to_str_len(ctx, obj) < 32)
{
fz_warn(ctx, "encryption password key too short (%d)", pdf_to_str_len(ctx, obj));
memcpy(crypt->u, pdf_to_str_buf(ctx, obj), pdf_to_str_len(ctx, obj));
}
else
{
pdf_drop_crypt(ctx, crypt);
fz_throw(ctx, FZ_ERROR_GENERIC, "encryption dictionary missing user password");
}
obj = pdf_dict_get(ctx, dict, PDF_NAME_P);
if (pdf_is_int(ctx, obj))
crypt->p = pdf_to_int(ctx, obj);
else
{
fz_warn(ctx, "encryption dictionary missing permissions");
crypt->p = 0xfffffffc;
}
if (crypt->r == 5 || crypt->r == 6)
{
obj = pdf_dict_get(ctx, dict, PDF_NAME_OE);
if (!pdf_is_string(ctx, obj) || pdf_to_str_len(ctx, obj) != 32)
{
pdf_drop_crypt(ctx, crypt);
fz_throw(ctx, FZ_ERROR_GENERIC, "encryption dictionary missing owner encryption key");
}
memcpy(crypt->oe, pdf_to_str_buf(ctx, obj), 32);
obj = pdf_dict_get(ctx, dict, PDF_NAME_UE);
if (!pdf_is_string(ctx, obj) || pdf_to_str_len(ctx, obj) != 32)
{
pdf_drop_crypt(ctx, crypt);
fz_throw(ctx, FZ_ERROR_GENERIC, "encryption dictionary missing user encryption key");
}
memcpy(crypt->ue, pdf_to_str_buf(ctx, obj), 32);
}
crypt->encrypt_metadata = 1;
obj = pdf_dict_get(ctx, dict, PDF_NAME_EncryptMetadata);
if (pdf_is_bool(ctx, obj))
crypt->encrypt_metadata = pdf_to_bool(ctx, obj);
/* Extract file identifier string */
if (pdf_is_array(ctx, id) && pdf_array_len(ctx, id) == 2)
{
obj = pdf_array_get(ctx, id, 0);
if (pdf_is_string(ctx, obj))
crypt->id = pdf_keep_obj(ctx, obj);
}
else
fz_warn(ctx, "missing file identifier, may not be able to do decryption");
/* Determine encryption key length */
crypt->length = 40;
if (crypt->v == 2 || crypt->v == 4)
{
obj = pdf_dict_get(ctx, dict, PDF_NAME_Length);
if (pdf_is_int(ctx, obj))
crypt->length = pdf_to_int(ctx, obj);
/* work-around for pdf generators that assume length is in bytes */
if (crypt->length < 40)
crypt->length = crypt->length * 8;
if (crypt->length % 8 != 0)
{
pdf_drop_crypt(ctx, crypt);
fz_throw(ctx, FZ_ERROR_GENERIC, "invalid encryption key length");
}
if (crypt->length < 40 || crypt->length > 128)
{
pdf_drop_crypt(ctx, crypt);
fz_throw(ctx, FZ_ERROR_GENERIC, "invalid encryption key length");
}
}
if (crypt->v == 5)
crypt->length = 256;
if (crypt->v == 1 || crypt->v == 2)
{
crypt->stmf.method = PDF_CRYPT_RC4;
crypt->stmf.length = crypt->length;
crypt->strf.method = PDF_CRYPT_RC4;
crypt->strf.length = crypt->length;
}
if (crypt->v == 4 || crypt->v == 5)
{
crypt->stmf.method = PDF_CRYPT_NONE;
crypt->stmf.length = crypt->length;
crypt->strf.method = PDF_CRYPT_NONE;
crypt->strf.length = crypt->length;
obj = pdf_dict_get(ctx, dict, PDF_NAME_CF);
if (pdf_is_dict(ctx, obj))
{
crypt->cf = pdf_keep_obj(ctx, obj);
}
else
{
crypt->cf = NULL;
}
fz_try(ctx)
{
obj = pdf_dict_get(ctx, dict, PDF_NAME_StmF);
if (pdf_is_name(ctx, obj))
pdf_parse_crypt_filter(ctx, &crypt->stmf, crypt, obj);
obj = pdf_dict_get(ctx, dict, PDF_NAME_StrF);
if (pdf_is_name(ctx, obj))
pdf_parse_crypt_filter(ctx, &crypt->strf, crypt, obj);
}
fz_catch(ctx)
{
pdf_drop_crypt(ctx, crypt);
fz_rethrow_message(ctx, "cannot parse string crypt filter (%d %d R)", pdf_to_num(ctx, obj), pdf_to_gen(ctx, obj));
}
/* in crypt revision 4, the crypt filter determines the key length */
if (crypt->strf.method != PDF_CRYPT_NONE)
crypt->length = crypt->stmf.length;
}
return crypt;
}
void
pdf_drop_crypt(fz_context *ctx, pdf_crypt *crypt)
{
pdf_drop_obj(ctx, crypt->id);
pdf_drop_obj(ctx, crypt->cf);
fz_free(ctx, crypt);
}
/*
* Parse a CF dictionary entry (PDF 1.7 table 3.22)
*/
static void
pdf_parse_crypt_filter(fz_context *ctx, pdf_crypt_filter *cf, pdf_crypt *crypt, pdf_obj *name)
{
pdf_obj *obj;
pdf_obj *dict;
int is_identity = (pdf_name_eq(ctx, name, PDF_NAME_Identity));
int is_stdcf = (!is_identity && pdf_name_eq(ctx, name, PDF_NAME_StdCF));
if (!is_identity && !is_stdcf)
fz_throw(ctx, FZ_ERROR_GENERIC, "Crypt Filter not Identity or StdCF (%d %d R)", pdf_to_num(ctx, crypt->cf), pdf_to_gen(ctx, crypt->cf));
cf->method = PDF_CRYPT_NONE;
cf->length = crypt->length;
if (!crypt->cf)
{
cf->method = (is_identity ? PDF_CRYPT_NONE : PDF_CRYPT_RC4);
return;
}
dict = pdf_dict_get(ctx, crypt->cf, name);
if (!pdf_is_dict(ctx, dict))
fz_throw(ctx, FZ_ERROR_GENERIC, "cannot parse crypt filter (%d %d R)", pdf_to_num(ctx, crypt->cf), pdf_to_gen(ctx, crypt->cf));
obj = pdf_dict_get(ctx, dict, PDF_NAME_CFM);
if (pdf_is_name(ctx, obj))
{
if (pdf_name_eq(ctx, PDF_NAME_None, obj))
cf->method = PDF_CRYPT_NONE;
else if (pdf_name_eq(ctx, PDF_NAME_V2, obj))
cf->method = PDF_CRYPT_RC4;
else if (pdf_name_eq(ctx, PDF_NAME_AESV2, obj))
cf->method = PDF_CRYPT_AESV2;
else if (pdf_name_eq(ctx, PDF_NAME_AESV3, obj))
cf->method = PDF_CRYPT_AESV3;
else
fz_warn(ctx, "unknown encryption method: %s", pdf_to_name(ctx, obj));
}
obj = pdf_dict_get(ctx, dict, PDF_NAME_Length);
if (pdf_is_int(ctx, obj))
cf->length = pdf_to_int(ctx, obj);
/* the length for crypt filters is supposed to be in bytes not bits */
if (cf->length < 40)
cf->length = cf->length * 8;
if ((cf->length % 8) != 0)
fz_throw(ctx, FZ_ERROR_GENERIC, "invalid key length: %d", cf->length);
if ((crypt->r == 1 || crypt->r == 2 || crypt->r == 3 || crypt->r == 4) &&
(cf->length < 0 || cf->length > 128))
fz_throw(ctx, FZ_ERROR_GENERIC, "invalid key length: %d", cf->length);
if ((crypt->r == 5 || crypt->r == 6) && cf->length != 256)
fz_throw(ctx, FZ_ERROR_GENERIC, "invalid key length: %d", cf->length);
}
/*
* Compute an encryption key (PDF 1.7 algorithm 3.2)
*/
static const unsigned char padding[32] =
{
0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41,
0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08,
0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80,
0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a
};
static void
pdf_compute_encryption_key(fz_context *ctx, pdf_crypt *crypt, unsigned char *password, int pwlen, unsigned char *key)
{
unsigned char buf[32];
unsigned int p;
int i, n;
fz_md5 md5;
n = crypt->length / 8;
/* Step 1 - copy and pad password string */
if (pwlen > 32)
pwlen = 32;
memcpy(buf, password, pwlen);
memcpy(buf + pwlen, padding, 32 - pwlen);
/* Step 2 - init md5 and pass value of step 1 */
fz_md5_init(&md5);
fz_md5_update(&md5, buf, 32);
/* Step 3 - pass O value */
fz_md5_update(&md5, crypt->o, 32);
/* Step 4 - pass P value as unsigned int, low-order byte first */
p = (unsigned int) crypt->p;
buf[0] = (p) & 0xFF;
buf[1] = (p >> 8) & 0xFF;
buf[2] = (p >> 16) & 0xFF;
buf[3] = (p >> 24) & 0xFF;
fz_md5_update(&md5, buf, 4);
/* Step 5 - pass first element of ID array */
fz_md5_update(&md5, (unsigned char *)pdf_to_str_buf(ctx, crypt->id), pdf_to_str_len(ctx, crypt->id));
/* Step 6 (revision 4 or greater) - if metadata is not encrypted pass 0xFFFFFFFF */
if (crypt->r >= 4)
{
if (!crypt->encrypt_metadata)
{
buf[0] = 0xFF;
buf[1] = 0xFF;
buf[2] = 0xFF;
buf[3] = 0xFF;
fz_md5_update(&md5, buf, 4);
}
}
/* Step 7 - finish the hash */
fz_md5_final(&md5, buf);
/* Step 8 (revision 3 or greater) - do some voodoo 50 times */
if (crypt->r >= 3)
{
for (i = 0; i < 50; i++)
{
fz_md5_init(&md5);
fz_md5_update(&md5, buf, n);
fz_md5_final(&md5, buf);
}
}
/* Step 9 - the key is the first 'n' bytes of the result */
memcpy(key, buf, n);
}
/*
* Compute an encryption key (PDF 1.7 ExtensionLevel 3 algorithm 3.2a)
*/
static void
pdf_compute_encryption_key_r5(fz_context *ctx, pdf_crypt *crypt, unsigned char *password, int pwlen, int ownerkey, unsigned char *validationkey)
{
unsigned char buffer[128 + 8 + 48];
fz_sha256 sha256;
fz_aes aes;
/* Step 2 - truncate UTF-8 password to 127 characters */
if (pwlen > 127)
pwlen = 127;
/* Step 3/4 - test password against owner/user key and compute encryption key */
memcpy(buffer, password, pwlen);
if (ownerkey)
{
memcpy(buffer + pwlen, crypt->o + 32, 8);
memcpy(buffer + pwlen + 8, crypt->u, 48);
}
else
memcpy(buffer + pwlen, crypt->u + 32, 8);
fz_sha256_init(&sha256);
fz_sha256_update(&sha256, buffer, pwlen + 8 + (ownerkey ? 48 : 0));
fz_sha256_final(&sha256, validationkey);
/* Step 3.5/4.5 - compute file encryption key from OE/UE */
memcpy(buffer + pwlen, crypt->u + 40, 8);
fz_sha256_init(&sha256);
fz_sha256_update(&sha256, buffer, pwlen + 8);
fz_sha256_final(&sha256, buffer);
/* clear password buffer and use it as iv */
memset(buffer + 32, 0, sizeof(buffer) - 32);
if (aes_setkey_dec(&aes, buffer, crypt->length))
fz_throw(ctx, FZ_ERROR_GENERIC, "AES key init failed (keylen=%d)", crypt->length);
aes_crypt_cbc(&aes, AES_DECRYPT, 32, buffer + 32, ownerkey ? crypt->oe : crypt->ue, crypt->key);
}
/*
* Compute an encryption key (PDF 1.7 ExtensionLevel 8 algorithm)
*
* Adobe has not yet released the details, so the algorithm reference is:
* http://esec-lab.sogeti.com/post/The-undocumented-password-validation-algorithm-of-Adobe-Reader-X
*/
static void
pdf_compute_hardened_hash_r6(fz_context *ctx, unsigned char *password, int pwlen, unsigned char salt[16], unsigned char *ownerkey, unsigned char hash[32])
{
unsigned char data[(128 + 64 + 48) * 64];
unsigned char block[64];
int block_size = 32;
int data_len = 0;
int i, j, sum;
fz_sha256 sha256;
fz_sha384 sha384;
fz_sha512 sha512;
fz_aes aes;
/* Step 1: calculate initial data block */
fz_sha256_init(&sha256);
fz_sha256_update(&sha256, password, pwlen);
fz_sha256_update(&sha256, salt, 8);
if (ownerkey)
fz_sha256_update(&sha256, ownerkey, 48);
fz_sha256_final(&sha256, block);
for (i = 0; i < 64 || i < data[data_len * 64 - 1] + 32; i++)
{
/* Step 2: repeat password and data block 64 times */
memcpy(data, password, pwlen);
memcpy(data + pwlen, block, block_size);
if (ownerkey)
memcpy(data + pwlen + block_size, ownerkey, 48);
data_len = pwlen + block_size + (ownerkey ? 48 : 0);
for (j = 1; j < 64; j++)
memcpy(data + j * data_len, data, data_len);
/* Step 3: encrypt data using data block as key and iv */
if (aes_setkey_enc(&aes, block, 128))
fz_throw(ctx, FZ_ERROR_GENERIC, "AES key init failed (keylen=%d)", 128);
aes_crypt_cbc(&aes, AES_ENCRYPT, data_len * 64, block + 16, data, data);
/* Step 4: determine SHA-2 hash size for this round */
for (j = 0, sum = 0; j < 16; j++)
sum += data[j];
/* Step 5: calculate data block for next round */
block_size = 32 + (sum % 3) * 16;
switch (block_size)
{
case 32:
fz_sha256_init(&sha256);
fz_sha256_update(&sha256, data, data_len * 64);
fz_sha256_final(&sha256, block);
break;
case 48:
fz_sha384_init(&sha384);
fz_sha384_update(&sha384, data, data_len * 64);
fz_sha384_final(&sha384, block);
break;
case 64:
fz_sha512_init(&sha512);
fz_sha512_update(&sha512, data, data_len * 64);
fz_sha512_final(&sha512, block);
break;
}
}
memset(data, 0, sizeof(data));
memcpy(hash, block, 32);
}
static void
pdf_compute_encryption_key_r6(fz_context *ctx, pdf_crypt *crypt, unsigned char *password, int pwlen, int ownerkey, unsigned char *validationkey)
{
unsigned char hash[32];
unsigned char iv[16];
fz_aes aes;
if (pwlen > 127)
pwlen = 127;
pdf_compute_hardened_hash_r6(ctx, password, pwlen,
(ownerkey ? crypt->o : crypt->u) + 32,
ownerkey ? crypt->u : NULL, validationkey);
pdf_compute_hardened_hash_r6(ctx, password, pwlen,
crypt->u + 40, NULL, hash);
memset(iv, 0, sizeof(iv));
if (aes_setkey_dec(&aes, hash, 256))
fz_throw(ctx, FZ_ERROR_GENERIC, "AES key init failed (keylen=256)");
aes_crypt_cbc(&aes, AES_DECRYPT, 32, iv,
ownerkey ? crypt->oe : crypt->ue, crypt->key);
}
/*
* Computing the user password (PDF 1.7 algorithm 3.4 and 3.5)
* Also save the generated key for decrypting objects and streams in crypt->key.
*/
static void
pdf_compute_user_password(fz_context *ctx, pdf_crypt *crypt, unsigned char *password, int pwlen, unsigned char *output)
{
if (crypt->r == 2)
{
fz_arc4 arc4;
pdf_compute_encryption_key(ctx, crypt, password, pwlen, crypt->key);
fz_arc4_init(&arc4, crypt->key, crypt->length / 8);
fz_arc4_encrypt(&arc4, output, padding, 32);
}
if (crypt->r == 3 || crypt->r == 4)
{
unsigned char xor[32];
unsigned char digest[16];
fz_md5 md5;
fz_arc4 arc4;
int i, x, n;
n = crypt->length / 8;
pdf_compute_encryption_key(ctx, crypt, password, pwlen, crypt->key);
fz_md5_init(&md5);
fz_md5_update(&md5, padding, 32);
fz_md5_update(&md5, (unsigned char*)pdf_to_str_buf(ctx, crypt->id), pdf_to_str_len(ctx, crypt->id));
fz_md5_final(&md5, digest);
fz_arc4_init(&arc4, crypt->key, n);
fz_arc4_encrypt(&arc4, output, digest, 16);
for (x = 1; x <= 19; x++)
{
for (i = 0; i < n; i++)
xor[i] = crypt->key[i] ^ x;
fz_arc4_init(&arc4, xor, n);
fz_arc4_encrypt(&arc4, output, output, 16);
}
memcpy(output + 16, padding, 16);
}
if (crypt->r == 5)
{
pdf_compute_encryption_key_r5(ctx, crypt, password, pwlen, 0, output);
}
if (crypt->r == 6)
{
pdf_compute_encryption_key_r6(ctx, crypt, password, pwlen, 0, output);
}
}
/*
* Authenticating the user password (PDF 1.7 algorithm 3.6
* and ExtensionLevel 3 algorithm 3.11)
* This also has the side effect of saving a key generated
* from the password for decrypting objects and streams.
*/
static int
pdf_authenticate_user_password(fz_context *ctx, pdf_crypt *crypt, unsigned char *password, int pwlen)
{
unsigned char output[32];
pdf_compute_user_password(ctx, crypt, password, pwlen, output);
if (crypt->r == 2 || crypt->r == 5 || crypt->r == 6)
return memcmp(output, crypt->u, 32) == 0;
if (crypt->r == 3 || crypt->r == 4)
return memcmp(output, crypt->u, 16) == 0;
return 0;
}
/*
* Authenticating the owner password (PDF 1.7 algorithm 3.7
* and ExtensionLevel 3 algorithm 3.12)
* Generates the user password from the owner password
* and calls pdf_authenticate_user_password.
*/
static int
pdf_authenticate_owner_password(fz_context *ctx, pdf_crypt *crypt, unsigned char *ownerpass, int pwlen)
{
unsigned char pwbuf[32];
unsigned char key[32];
unsigned char xor[32];
unsigned char userpass[32];
int i, n, x;
fz_md5 md5;
fz_arc4 arc4;
if (crypt->r == 5)
{
/* PDF 1.7 ExtensionLevel 3 algorithm 3.12 */
pdf_compute_encryption_key_r5(ctx, crypt, ownerpass, pwlen, 1, key);
return !memcmp(key, crypt->o, 32);
}
else if (crypt->r == 6)
{
/* PDF 1.7 ExtensionLevel 8 algorithm */
pdf_compute_encryption_key_r6(ctx, crypt, ownerpass, pwlen, 1, key);
return !memcmp(key, crypt->o, 32);
}
n = crypt->length / 8;
/* Step 1 -- steps 1 to 4 of PDF 1.7 algorithm 3.3 */
/* copy and pad password string */
if (pwlen > 32)
pwlen = 32;
memcpy(pwbuf, ownerpass, pwlen);
memcpy(pwbuf + pwlen, padding, 32 - pwlen);
/* take md5 hash of padded password */
fz_md5_init(&md5);
fz_md5_update(&md5, pwbuf, 32);
fz_md5_final(&md5, key);
/* do some voodoo 50 times (Revision 3 or greater) */
if (crypt->r >= 3)
{
for (i = 0; i < 50; i++)
{
fz_md5_init(&md5);
fz_md5_update(&md5, key, 16);
fz_md5_final(&md5, key);
}
}
/* Step 2 (Revision 2) */
if (crypt->r == 2)
{
fz_arc4_init(&arc4, key, n);
fz_arc4_encrypt(&arc4, userpass, crypt->o, 32);
}
/* Step 2 (Revision 3 or greater) */
if (crypt->r >= 3)
{
memcpy(userpass, crypt->o, 32);
for (x = 0; x < 20; x++)
{
for (i = 0; i < n; i++)
xor[i] = key[i] ^ (19 - x);
fz_arc4_init(&arc4, xor, n);
fz_arc4_encrypt(&arc4, userpass, userpass, 32);
}
}
return pdf_authenticate_user_password(ctx, crypt, userpass, 32);
}
static void pdf_docenc_from_utf8(char *password, const char *utf8, int n)
{
int i = 0, k, c;
while (*utf8 && i + 1 < n)
{
utf8 += fz_chartorune(&c, utf8);
for (k = 0; k < 256; k++)
{
if (c == pdf_doc_encoding[k])
{
password[i++] = k;
break;
}
}
/* FIXME: drop characters that can't be encoded or return an error? */
}
password[i] = 0;
}
static void pdf_saslprep_from_utf8(char *password, const char *utf8, int n)
{
/* TODO: stringprep with SALSprep profile */
fz_strlcpy(password, utf8, n);
}
int
pdf_authenticate_password(fz_context *ctx, pdf_document *doc, const char *pwd_utf8)
{
char password[2048];
if (doc->crypt)
{
password[0] = 0;
if (pwd_utf8)
{
if (doc->crypt->r <= 4)
pdf_docenc_from_utf8(password, pwd_utf8, sizeof password);
else
pdf_saslprep_from_utf8(password, pwd_utf8, sizeof password);
}
if (pdf_authenticate_user_password(ctx, doc->crypt, (unsigned char *)password, strlen(password)))
return 1;
if (pdf_authenticate_owner_password(ctx, doc->crypt, (unsigned char *)password, strlen(password)))
return 1;
return 0;
}
return 1;
}
int
pdf_needs_password(fz_context *ctx, pdf_document *doc)
{
if (!doc->crypt)
return 0;
if (pdf_authenticate_password(ctx, doc, ""))
return 0;
return 1;
}
int
pdf_has_permission(fz_context *ctx, pdf_document *doc, fz_permission p)
{
if (!doc->crypt)
return 1;
switch (p)
{
case FZ_PERMISSION_PRINT: return doc->crypt->p & PDF_PERM_PRINT;
case FZ_PERMISSION_COPY: return doc->crypt->p & PDF_PERM_COPY;
case FZ_PERMISSION_EDIT: return doc->crypt->p & PDF_PERM_CHANGE;
case FZ_PERMISSION_ANNOTATE: return doc->crypt->p & PDF_PERM_NOTES;
}
return 1;
}
unsigned char *
pdf_crypt_key(fz_context *ctx, pdf_document *doc)
{
if (doc->crypt)
return doc->crypt->key;
return NULL;
}
int
pdf_crypt_version(fz_context *ctx, pdf_document *doc)
{
if (doc->crypt)
return doc->crypt->v;
return 0;
}
int pdf_crypt_revision(fz_context *ctx, pdf_document *doc)
{
if (doc->crypt)
return doc->crypt->r;
return 0;
}
char *
pdf_crypt_method(fz_context *ctx, pdf_document *doc)
{
if (doc->crypt)
{
switch (doc->crypt->strf.method)
{
case PDF_CRYPT_NONE: return "None";
case PDF_CRYPT_RC4: return "RC4";
case PDF_CRYPT_AESV2: return "AES";
case PDF_CRYPT_AESV3: return "AES";
case PDF_CRYPT_UNKNOWN: return "Unknown";
}
}
return "None";
}
int
pdf_crypt_length(fz_context *ctx, pdf_document *doc)
{
if (doc->crypt)
return doc->crypt->length;
return 0;
}
/*
* PDF 1.7 algorithm 3.1 and ExtensionLevel 3 algorithm 3.1a
*
* Using the global encryption key that was generated from the
* password, create a new key that is used to decrypt individual
* objects and streams. This key is based on the object and
* generation numbers.
*/
static int
pdf_compute_object_key(pdf_crypt *crypt, pdf_crypt_filter *cf, int num, int gen, unsigned char *key, int max_len)
{
fz_md5 md5;
unsigned char message[5];
int key_len = crypt->length / 8;
if (key_len > max_len)
key_len = max_len;
if (cf->method == PDF_CRYPT_AESV3)
{
memcpy(key, crypt->key, key_len);
return key_len;
}
fz_md5_init(&md5);
fz_md5_update(&md5, crypt->key, key_len);
message[0] = (num) & 0xFF;
message[1] = (num >> 8) & 0xFF;
message[2] = (num >> 16) & 0xFF;
message[3] = (gen) & 0xFF;
message[4] = (gen >> 8) & 0xFF;
fz_md5_update(&md5, message, 5);
if (cf->method == PDF_CRYPT_AESV2)
fz_md5_update(&md5, (unsigned char *)"sAlT", 4);
fz_md5_final(&md5, key);
if (key_len + 5 > 16)
return 16;
return key_len + 5;
}
/*
* PDF 1.7 algorithm 3.1 and ExtensionLevel 3 algorithm 3.1a
*
* Decrypt all strings in obj modifying the data in-place.
* Recurse through arrays and dictionaries, but do not follow
* indirect references.
*/
static void
pdf_crypt_obj_imp(fz_context *ctx, pdf_crypt *crypt, pdf_obj *obj, unsigned char *key, int keylen)
{
unsigned char *s;
int i, n;
if (pdf_is_indirect(ctx, obj))
return;
if (pdf_is_string(ctx, obj))
{
s = (unsigned char *)pdf_to_str_buf(ctx, obj);
n = pdf_to_str_len(ctx, obj);
if (crypt->strf.method == PDF_CRYPT_RC4)
{
fz_arc4 arc4;
fz_arc4_init(&arc4, key, keylen);
fz_arc4_encrypt(&arc4, s, s, n);
}
if (crypt->strf.method == PDF_CRYPT_AESV2 || crypt->strf.method == PDF_CRYPT_AESV3)
{
if (n == 0)
{
/* Empty strings are permissible */
}
else if (n & 15 || n < 32)
fz_warn(ctx, "invalid string length for aes encryption");
else
{
unsigned char iv[16];
fz_aes aes;
memcpy(iv, s, 16);
if (aes_setkey_dec(&aes, key, keylen * 8))
fz_throw(ctx, FZ_ERROR_GENERIC, "AES key init failed (keylen=%d)", keylen * 8);
aes_crypt_cbc(&aes, AES_DECRYPT, n - 16, iv, s + 16, s);
/* delete space used for iv and padding bytes at end */
if (s[n - 17] < 1 || s[n - 17] > 16)
fz_warn(ctx, "aes padding out of range");
else
pdf_set_str_len(ctx, obj, n - 16 - s[n - 17]);
}
}
}
else if (pdf_is_array(ctx, obj))
{
n = pdf_array_len(ctx, obj);
for (i = 0; i < n; i++)
{
pdf_crypt_obj_imp(ctx, crypt, pdf_array_get(ctx, obj, i), key, keylen);
}
}
else if (pdf_is_dict(ctx, obj))
{
n = pdf_dict_len(ctx, obj);
for (i = 0; i < n; i++)
{
pdf_crypt_obj_imp(ctx, crypt, pdf_dict_get_val(ctx, obj, i), key, keylen);
}
}
}
void
pdf_crypt_obj(fz_context *ctx, pdf_crypt *crypt, pdf_obj *obj, int num, int gen)
{
unsigned char key[32];
int len;
len = pdf_compute_object_key(crypt, &crypt->strf, num, gen, key, 32);
pdf_crypt_obj_imp(ctx, crypt, obj, key, len);
}
Firstly, I should say a few words about licensing. MuPDF is licensed under 2 different licences.
The first is the GNU AGPL. This is a very strict license that places lots of requirements upon you for you to be able to use the code. The biggest requirement is that anyone that gets a copy of your app has the rights to demand the full source code for your entire app. This would clearly include any decryption key, meaning your DRM would be useless. You should read the license carefully to make sure it's suitable for what you want to do before continuing.
If you use MuPDF under the GNU AGPL license, we cannot guarantee you any support.
If you can't abide by the terms of the GNU AGPL, then you can obtain a commercial license from Artifex (mail sales#artifex.com with as many details of your project as possible and they'll talk to you about a customised licensing proposal). This frees you from all the onerous terms of the GNU AGPL.
If you are unable to comply with the GNU AGPL, or you are unwilling to pay for a commercial license, then you cannot use MuPDF.
Now, onto the actual question...
pdf_crypt.c implements the standard decryption handlers used in PDF. It sounds to me like you want to do something non-standard. As such, pdf_crypt.c will require some changes.
One technique would be to create a perfectly normal encrypted PDF file using a password. Your app can provide the password to MuPDF as it opens the file, and decryption will work seamlessly for you. The user need not even know that there is a password involved. In terms of coding this is the simplest possible way to go.
Another technique would be to do block based encryption of the file, and to decrypt blocks of the file as you load it - we have customers who do DRM in their products like this.
It's impossible to give you more information without knowing more about what you want to do.
If you wish to discuss this problem more, your best bet is probably to come to the #ghostscript irc channel (see mupdf.com for a link that will open this in your web browser).
Related
How to read sei unregistered user data from ffmpeg in H264?
First, I set the user_data to video by bsf. For example, ffmpeg -i dump.flv -c:v copy -bsf:v h264_metadata=sei_user_data='086f3693-b7b3-4f2c-9653-21492feee5b8+hello' dump_sei.flv Ant then I use the api to read sei unregistered_user_data by ffmpeg after h264 encode, but I didn't get it. // avcodec_send_packet // avcodec_receive_frame // ... H264Context *h = m_pVidDecodeCtx->priv_data; H264SEIContext sei = h->sei; H264SEIUnregistered unregistered = sei.unregistered; if (unregistered.buf_ref != NULL) { AVBufferRef *buf = *(unregistered.buf_ref); if (buf != NULL && buf->buffer != NULL) { printf("sei data: %s\n", buf->buffer); } } The version of ffmpeg is v4.4.
A great treasure where you can find your answers to all your questions about SEI is h264_sei.c file . (that exist in libavcodec folder) static int decode_registered_user_data_afd(H264SEIAFD *h, GetBitContext *gb, int size) { int flag; if (size-- < 1) return AVERROR_INVALIDDATA; skip_bits(gb, 1); // 0 flag = get_bits(gb, 1); // active_format_flag skip_bits(gb, 6); // reserved if (flag) { if (size-- < 1) return AVERROR_INVALIDDATA; skip_bits(gb, 4); // reserved h->active_format_description = get_bits(gb, 4); h->present = 1; } return 0; } static int decode_registered_user_data_closed_caption(H264SEIA53Caption *h, GetBitContext *gb, void *logctx, int size) { if (size < 3) return AVERROR(EINVAL); return ff_parse_a53_cc(&h->buf_ref, gb->buffer + get_bits_count(gb) / 8, size); } static int decode_registered_user_data(H264SEIContext *h, GetBitContext *gb, void *logctx, int size) { int country_code, provider_code; if (size < 3) return AVERROR_INVALIDDATA; size -= 3; country_code = get_bits(gb, 8); // itu_t_t35_country_code if (country_code == 0xFF) { if (size < 1) return AVERROR_INVALIDDATA; skip_bits(gb, 8); // itu_t_t35_country_code_extension_byte size--; } if (country_code != 0xB5) { // usa_country_code av_log(logctx, AV_LOG_VERBOSE, "Unsupported User Data Registered ITU-T T35 SEI message (country_code = %d)\n", country_code); return 0; } /* itu_t_t35_payload_byte follows */ provider_code = get_bits(gb, 16); switch (provider_code) { case 0x31: { // atsc_provider_code uint32_t user_identifier; if (size < 4) return AVERROR_INVALIDDATA; size -= 4; user_identifier = get_bits_long(gb, 32); switch (user_identifier) { case MKBETAG('D', 'T', 'G', '1'): // afd_data return decode_registered_user_data_afd(&h->afd, gb, size); case MKBETAG('G', 'A', '9', '4'): // closed captions return decode_registered_user_data_closed_caption(&h->a53_caption, gb, logctx, size); default: av_log(logctx, AV_LOG_VERBOSE, "Unsupported User Data Registered ITU-T T35 SEI message (atsc user_identifier = 0x%04x)\n", user_identifier); break; } break; } default: av_log(logctx, AV_LOG_VERBOSE, "Unsupported User Data Registered ITU-T T35 SEI message (provider_code = %d)\n", provider_code); break; } return 0; } static int decode_unregistered_user_data(H264SEIUnregistered *h, GetBitContext *gb, void *logctx, int size) { uint8_t *user_data; int e, build, i; AVBufferRef *buf_ref, **tmp; if (size < 16 || size >= INT_MAX - 1) return AVERROR_INVALIDDATA; tmp = av_realloc_array(h->buf_ref, h->nb_buf_ref + 1, sizeof(*h->buf_ref)); if (!tmp) return AVERROR(ENOMEM); h->buf_ref = tmp; buf_ref = av_buffer_alloc(size + 1); if (!buf_ref) return AVERROR(ENOMEM); user_data = buf_ref->data; for (i = 0; i < size; i++) user_data[i] = get_bits(gb, 8); user_data[i] = 0; buf_ref->size = size; h->buf_ref[h->nb_buf_ref++] = buf_ref; e = sscanf(user_data + 16, "x264 - core %d", &build); if (e == 1 && build > 0) h->x264_build = build; if (e == 1 && build == 1 && !strncmp(user_data+16, "x264 - core 0000", 16)) h->x264_build = 67; return 0; }
Data transfer over sockets[TCP] how to pack multiple integer in c/c++ and transfer the data with send() recv()?
I'm making a small client/server based game, on linux in c/c++ and I need to send the player turn to the server. Here is my problem. I want to send two integers to the server and sometimes it works perfectly, but sometimes the server receives both integer in the first recv() and its stuck. I know that the best way is to package the messages. The problem is I don't know how the syntax should look like. In theory--> the player input would be like an int column = 4 and a second int row = 1 and I package the message as 4|1 or something like this. Then I send from client to server and encode it on the server. An example would be great or maybe some advice how stuff like this is handled probably. I'm still very new to socket programming. Here is how my function looks like: Client: #define BUFFER 512 void send_turn_to_server(int sock, int row, int column) { // sends row to server from player turn char char_row[BUFFER]; sprintf(char_row, "%d", row); char *message_from_client = char_row; int len, bytes_sent_row; len = strlen(message_from_client); if (sendall(sock, message_from_client, &len) == -1) { perror("sendall"); printf("We only sent %d bytes because of the error!\n", len); } char char_column[BUFFER]; int bytes_sent_column; //sends column from player turn //sprintf converts the int to char sprintf(char_column, "%d", column); char *column_from_client = char_column; len = strlen(column_from_client); if (sendall(sock, column_from_client, &len) == -1) { perror("sendall"); printf("We only sent %d bytes because of the error!\n", len); } cout << "send_turn_to_server_complete" << endl; } Here I use a function from Beej's Guide to Network Programming, so I can be sure the whole buffer is sent. Client: int sendall(int s, char *buf, int *len) { int total = 0; // how many bytes we've sent int bytesleft = *len; // how many we have left to send int n; while (total < *len) { n = send(s, buf + total, bytesleft, 0); if (n == -1) { break; } total += n; bytesleft -= n; } *len = total; // return number actually sent here return n == -1 ? -1 : 0; // return -1 on failure, 0 on success } Server: int receive_player_turn(int sock, int &int_row, int &int_column) { int byte_count; char buf[BUFFER]; byte_count = recv(sock, buf, sizeof buf, 0); cout << "The row from player: " << buf << endl; //The C library function int atoi(const char *str) converts the string argument str to an integer (type int). int_row = atoi(buf); //cleans the buffer bzero(buf, sizeof(buf)); byte_count = recv(sock, buf, sizeof buf, 0); cout << buf << endl; cout << "The column from player: " << buf << endl; //converts the char string to an int int_column = atoi(buf); cout << endl << "receive player turn worked" << endl << "players turn was in the row " << int_row << " and in the column " << int_column + 1 << endl; return int_row, int_column; } output correct from server: Player connected: SchleichsSalaticus The row from player: 7 4 The column from player: 4 receive player turn worked players turn was in the row 7 and in the column 5 7 4 output wrong from server: Player connected: SchleichsSalaticus The row from player: 74
The issue is that TCP is a continuous stream, with no concept of the start or end of a ”message” because it is not message-based. Most times, people use a very simple ”framing protocol” whereby you always send a 4-byte header on every transfer which tells the recipient how many bytes to read, then you send that many bytes as your message. Use htonl() to send the 4-byte header in network byte order then you will be interoperable. There is a very similar example here.
One possible solution could be defining a format for the message that client send to server. For example you could define a protocol as follow: [4 bytes length of your message][2 bytes for first player][2 bytes for second one] and in server side you should at first in rcv function get 4 bytes and extract the length of the arrived message and based on the receiving length(L) call again the rcv function with size L after that you should parse received messaged and extract the turn of each players.
If all your messages are expected to be of same length, then you do not need a message header. Something like that given below should work fine. In general you should be prepared to receive less or more than your expected message, as well as for one message to be split across many receives. Also, I would recommend one function that receives bytes making no assumption about what they mean, and another that interprets them. Then the first one can be applied more broadly. Treat the following only as pseudo code. not tested. // use a buffer length double of MESSAGE_LENGTH. static int offset = 0; // not thread safe. // loop to receive a message. while(offset < MESSAGE_LENGTH) { byte_count = recv(sock, &buf[offset], (sizeof(buf)-offset), 0); if(byte_count > 0) { offset += byte_count; } else { // add error handling here. close socket. break out of loop } } // process buf here, but do not clear it. // received message always starts at buf[0]. if(no receive error above) { process_received_message(buf); // } // move part of next message (if any) to start of buffer. if(offset > MESSAGE_LENGTH) { // copy the start of next message to start of buffer. // and remember the new offset to avoid overwriting them. char* pSrc = &buf[MESSAGE_LENGTH]; char* pSrcEnd = &buf[offset]; char* pDest = buf; while(pSrc < pSrcEnd){ *pDest++ = *pSrc++; } //or memcpy. offset -= MESSAGE_LENGTH; } else { offset = 0; }
On many hardware architectures, integers and other types have alignment requirements. The compiler normally takes care of this, but when in a buffer, unaligned accesses can be an issue. Furthermore, the server and the client might not use the same byte order. Here is a set of inline helper functions you can use to pack and unpack integer types to/from a buffer: /* SPDX-License-Identifier: CC0-1.0 */ #ifndef PACKING_H #define PACKING_H #include <stdint.h> /* Packing and unpacking unsigned and signed integers in little-endian byte order. Works on all architectures and OSes when compiled using a standards-conforming C implementation, C99 or later. */ static inline void pack_u8(unsigned char *dst, uint8_t val) { dst[0] = val & 255; } static inline void pack_u16(unsigned char *dst, uint16_t val) { dst[0] = val & 255; dst[1] = (val >> 8) & 255; } static inline void pack_u24(unsigned char *dst, uint32_t val) { dst[0] = val & 255; dst[1] = (val >> 8) & 255; dst[2] = (val >> 16) & 255; } static inline void pack_u32(unsigned char *dst, uint32_t val) { dst[0] = val & 255; dst[1] = (val >> 8) & 255; dst[2] = (val >> 16) & 255; dst[3] = (val >> 24) & 255; } static inline void pack_u40(unsigned char *dst, uint64_t val) { dst[0] = val & 255; dst[1] = (val >> 8) & 255; dst[2] = (val >> 16) & 255; dst[3] = (val >> 24) & 255; dst[4] = (val >> 32) & 255; } static inline void pack_u48(unsigned char *dst, uint64_t val) { dst[0] = val & 255; dst[1] = (val >> 8) & 255; dst[2] = (val >> 16) & 255; dst[3] = (val >> 24) & 255; dst[4] = (val >> 32) & 255; dst[5] = (val >> 40) & 255; } static inline void pack_u56(unsigned char *dst, uint64_t val) { dst[0] = val & 255; dst[1] = (val >> 8) & 255; dst[2] = (val >> 16) & 255; dst[3] = (val >> 24) & 255; dst[4] = (val >> 32) & 255; dst[5] = (val >> 40) & 255; dst[6] = (val >> 48) & 255; } static inline void pack_u64(unsigned char *dst, uint64_t val) { dst[0] = val & 255; dst[1] = (val >> 8) & 255; dst[2] = (val >> 16) & 255; dst[3] = (val >> 24) & 255; dst[4] = (val >> 32) & 255; dst[5] = (val >> 40) & 255; dst[6] = (val >> 48) & 255; dst[7] = (val >> 56) & 255; } static inline void pack_i8(unsigned char *dst, int8_t val) { pack_u8((uint8_t)val); } static inline void pack_i16(unsigned char *dst, int16_t val) { pack_u16((uint16_t)val); } static inline void pack_i24(unsigned char *dst, int32_t val) { pack_u24((uint32_t)val); } static inline void pack_i32(unsigned char *dst, int32_t val) { pack_u32((uint32_t)val); } static inline void pack_i40(unsigned char *dst, int64_t val) { pack_u40((uint64_t)val); } static inline void pack_i48(unsigned char *dst, int64_t val) { pack_u48((uint64_t)val); } static inline void pack_i56(unsigned char *dst, int64_t val) { pack_u56((uint64_t)val); } static inline void pack_i64(unsigned char *dst, int64_t val) { pack_u64((uint64_t)val); } static inline uint8_t unpack_u8(const unsigned char *src) { return (uint_fast8_t)(src[0] & 255); } static inline uint16_t unpack_u16(const unsigned char *src) { return (uint_fast16_t)(src[0] & 255) | ((uint_fast16_t)(src[1] & 255) << 8); } static inline uint32_t unpack_u24(const unsigned char *src) { return (uint_fast32_t)(src[0] & 255) | ((uint_fast32_t)(src[1] & 255) << 8) | ((uint_fast32_t)(src[2] & 255) << 16); } static inline uint32_t unpack_u32(const unsigned char *src) { return (uint_fast32_t)(src[0] & 255) | ((uint_fast32_t)(src[1] & 255) << 8) | ((uint_fast32_t)(src[2] & 255) << 16) | ((uint_fast32_t)(src[3] & 255) << 24); } static inline uint64_t unpack_u40(const unsigned char *src) { return (uint_fast64_t)(src[0] & 255) | ((uint_fast64_t)(src[1] & 255) << 8) | ((uint_fast64_t)(src[2] & 255) << 16) | ((uint_fast64_t)(src[3] & 255) << 24) | ((uint_fast64_t)(src[4] & 255) << 32); } static inline uint64_t unpack_u48(const unsigned char *src) { return (uint_fast64_t)(src[0] & 255) | ((uint_fast64_t)(src[1] & 255) << 8) | ((uint_fast64_t)(src[2] & 255) << 16) | ((uint_fast64_t)(src[3] & 255) << 24) | ((uint_fast64_t)(src[4] & 255) << 32) | ((uint_fast64_t)(src[5] & 255) << 40); } static inline uint64_t unpack_u56(const unsigned char *src) { return (uint_fast64_t)(src[0] & 255) | ((uint_fast64_t)(src[1] & 255) << 8) | ((uint_fast64_t)(src[2] & 255) << 16) | ((uint_fast64_t)(src[3] & 255) << 24) | ((uint_fast64_t)(src[4] & 255) << 32) | ((uint_fast64_t)(src[5] & 255) << 40) | ((uint_fast64_t)(src[6] & 255) << 48); } static inline uint64_t unpack_u64(const unsigned char *src) { return (uint_fast64_t)(src[0] & 255) | ((uint_fast64_t)(src[1] & 255) << 8) | ((uint_fast64_t)(src[2] & 255) << 16) | ((uint_fast64_t)(src[3] & 255) << 24) | ((uint_fast64_t)(src[4] & 255) << 32) | ((uint_fast64_t)(src[5] & 255) << 40) | ((uint_fast64_t)(src[6] & 255) << 48) | ((uint_fast64_t)(src[7] & 255) << 56); } static inline int8_t unpack_i8(const unsigned char *src) { return (int8_t)(src[0] & 255); } static inline int16_t unpack_i16(const unsigned char *src) { return (int16_t)unpack_u16(src); } static inline int32_t unpack_i24(const unsigned char *src) { uint_fast32_t u = unpack_u24(src); /* Sign extend to 32 bits */ if (u & 0x800000) u |= 0xFF000000; return (int32_t)u; } static inline int32_t unpack_i32(const unsigned char *src) { return (int32_t)unpack_u32(src); } static inline int64_t unpack_i40(const unsigned char *src) { uint_fast64_t u = unpack_u40(src); /* Sign extend to 64 bits */ if (u & UINT64_C(0x0000008000000000)) u |= UINT64_C(0xFFFFFF0000000000); return (int64_t)u; } static inline int64_t unpack_i48(const unsigned char *src) { uint_fast64_t u = unpack_i48(src); /* Sign extend to 64 bits */ if (u & UINT64_C(0x0000800000000000)) u |= UINT64_C(0xFFFF000000000000); return (int64_t)u; } static inline int64_t unpack_i56(const unsigned char *src) { uint_fast64_t u = unpack_u56(src); /* Sign extend to 64 bits */ if (u & UINT64_C(0x0080000000000000)) u |= UINT64_C(0xFF00000000000000); return (int64_t)u; } static inline int64_t unpack_i64(const unsigned char *src) { return (int64_t)unpack_u64(src); } #endif /* PACKING_H */ When packed, these values are in two's complement little-endian byte order. pack_uN() and unpack_uN() work with unsigned integers from 0 to 2N-1, inclusive. pack_iN() and unpack_iN() work with signed integers from -2N-1 to 2N-1-1, inclusive. Let's consider a simple binary protocol, where each message starts with two bytes: first one the total length of this message, and the second one identifying the type of the message. This has the nice feature that if something odd happens, it is always possible to resynchronize by sending at least 256 zeroes. Each zero is an invalid length for the message, so they should just be skipped by the receiver. You probably won't need this, but it may come in handy someday. To receive a message of this form, we can use the following function: /* Receive a single message. 'fd' is the socket descriptor, and 'msg' is a buffer of at least 255 chars. Returns -1 with errno set if an error occurs, or the message type (0 to 255, inclusive) if success. */ int recv_message(const int fd, unsigned char *msg) { ssize_t n; msg[0] = 0; msg[1] = 0; /* Loop to skip zero bytes. */ do { do { n = read(fd, msg, 1); } while (n == -1 && errno == EINTR); if (n == -1) { /* Error; errno already set. */ return -1; } else if (n == 0) { /* Other end closed the socket. */ errno = EPIPE; return -1; } else if (n != 1) { errno = EIO; return -1; } } while (msg[0] == 0); /* Read the rest of the message. */ { unsigned char *const end = msg + msg[0]; unsigned char *ptr = msg + 1; while (ptr < end) { n = read(fd, ptr, (size_t)(end - ptr)); if (n > 0) { ptr += n; } else if (n == 0) { /* Other end closed socket */ errno = EPIPE; return -1; } else if (n != -1) { errno = EIO; return -1; } else if (errno != EINTR) { /* Error; errno already set */ return -1; } } } /* Success, return message type. */ return msg[1]; } In your own code, you can use the above like this: unsigned char buffer[256]; switch(receive_message(fd, buffer)) { case -1: if (errno == EPIPE) { /* The other end closed the connection */ } else { /* Other error; see strerror(errno). */ } break or return or abort; case 0: /* Exit/cancel game */ break or return or abort; case 4: /* Coordinate message */ int x = unpack_i16(buffer + 2); int y = unpack_i16(buffer + 4); /* x,y is the coordinate pair; do something */ break; default: /* Ignore all other message types */ } where I randomly chose 0 as the abort-game message type, and 4 as the coordinate message type. Instead of scattering such statements here and there in your client, put it in a function. You could also consider using a finite-state machine to represent the game state. To send messages, you can use a helper function like /* Send one or more messages; does not verify contents. Returns 0 if success, -1 with errno set if an error occurs. */ int send_message(const int fd, const void *msg, const size_t len) { const unsigned char *const end = (const unsigned char *)msg + len; const unsigned char *ptr = (const unsigned char *)msg; ssize_t n; while (ptr < end) { n = write(fd, ptr, (size_t)(end - ptr)); if (n > 0) { ptr += n; } else if (n != -1) { /* C library bug, should not occur */ errno = EIO; return -1; } else if (errno != EINTR) { /* Other error */ return -1; } } return 0; } so that sending an abort game (type 0) message would be int send_abort_message(const int fd) { unsigned char buffer[2] = { 1, 0 }; return send_message(fd, buffer, 2); } and sending a coordinate (type 4) message would be e.g. int send_coordinates(const int fd, const int x, const int y) { unsigned char buffer[2 + 2 + 2]; buffer[0] = 6; /* Length in bytes/chars */ buffer[1] = 4; /* Type */ pack_i16(buffer + 2, x); pack_i16(buffer + 4, y); return send_message(fd, buffer, 6); } If the game is not turn-based, you won't want to block in the sends or receives, like the above functions do. Nonblocking I/O is the way to go. Essentially, you'll have something like static int server_fd = -1; static size_t send_size = 0; static unsigned char *send_data = NULL; static size_t send_next = 0; /* First unsent byte */ static size_t send_ends = 0; /* End of buffered data */ static size_t recv_size = 0; static unsigned char *recv_data = NULL; static size_t recv_next = 0; /* Start of next message */ static size_t recv_ends = 0; /* End of buffered data */ and you set the server_fd nonblocking using e.g. fcntl(server_fd, F_SETFL, O_NONBLOCK);. A communicator function will try to send and receive as much data as possible. It will return 1 if it sent anything, 2 if it received anything, 3 if both, 0 if neither, and -1 if an error occurred: int communicate(void) { int retval = 0; ssize_t n; while (send_next < send_ends) { n = write(server_fd, send_data + send_next, send_ends - send_next); if (n > 0) { send_next += n; retval |= 1; } else if (n != -1) { /* errno already set */ return -1; } else if (errno == EAGAIN || errno == EWOULDBLOCK) { /* Cannot send more without blocking */ break; } else if (errno != EINTR) { /* Error, errno set */ return -1; } } /* If send buffer became empty, reset it. */ if (send_next >= send_ends) { send_next = 0; send_ends = 0; } /* If receive buffer is empty, reset it. */ if (recv_next >= recv_ends) { recv_next = 0; recv_ends = 0; } /* Receive loop. */ while (1) { /* Receive buffer full? */ if (recv_ends + 256 > recv_ends) { /* First try to repack. */ if (recv_next > 0) { memmove(recv_data, recv_data + recv_next, recv_ends - recv_next); recv_ends -= recv_next; recv_next = 0; } if (recv_ends + 256 > recv_ends) { /* Allocate 16k more (256 messages!) */ size_t new_size = recv_size + 16384; unsigned char *new_data; new_data = realloc(recv_data, new_size); if (!new_data) { errno = ENOMEM; return -1; } recv_data = new_data; recv_size = new_size; } } /* Try to receive incoming data. */ n = read(server_fd, recv_data + recv_ends, recv_size - recv_ends); if (n > 0) { recv_ends += n; retval |= 2; } else if (n == 0) { /* Other end closed the connection. */ errno = EPIPE; return -1; } else if (n != -1) { errno = EIO; return -1; } else if (errno == EAGAIN || errno == EWOULDBLOCK) { break; } else if (errno != EINTR) { return -1; } } return retval; } When there is nothing to do, and you want to wait for a short while (some milliseconds), but interrupt the wait whenever more I/O can be done, use /* Wait for max 'ms' milliseconds for communication to occur. Returns 1 if data received, 2 if sent, 3 if both, 0 if neither (having waited for 'ms' milliseconds), or -1 if an error occurs. */ int communicate_wait(int ms) { struct pollfd fds[1]; int retval; /* Zero timeout is "forever", and we don't want that. */ if (ms < 1) ms = 1; /* We try communicating right now. */ retval = communicate(); if (retval) return retval; /* Poll until I/O possible. */ fds[0].fd = server_fd; if (send_ends > send_next) fds[0].events = POLLIN | POLLOUT; else fds[0].events = POLLIN; fds[0].revents = 0; poll(fds, 1, ms); /* We retry I/O now. */ return communicate(); } To process messages received thus far, you use a loop: while (recv_next < recv_ends && recv_next + recv_data[recv_next] <= recv_ends) { if (recv_data[recv_next] == 0) { recv_next++; continue; } /* recv_data[recv_next+0] is the length of the message, recv_data[recv_next+1] is the type of the message. */ switch (recv_data[recv_next + 1]) { case 4: /* Coordinate message */ if (recv_data[recv_next] >= 6) { int x = unpack_i16(recv_data + recv_next + 2); int y = unpack_i16(recv_data + recv_next + 4); /* Do something with x and y ... */ } break; /* Handle other message types ... */ } recv_next += recv_data[recv_next]; } Then you recalculate game state, update the display, communicate some more, and repeat.
Debug Error - abort() has been called (gcz2tga)
Complete C++/C newbie here. I recompiled gcz2tga (I did not make this code, just found it on the webs) into a debug .exe file using Visual Studio 2019. Everything works well until it gets to "split_images" and then the program spits out this error: Debug Error! Program: C:\Users\Harrison\source\repos\gcz2tga\Debug\gcz2tga.exe abort() has been called (Press Retry to debug the application) When I hit Retry, the program closes. The code is set up like this: /* gcz2tga.c: Slice up a directory full of GCZ (texture) files into TGA files. * * Credit goes to afwefwe for reverse-engineering the texture format * and LZSS compression */ #include <stdlib.h> #include <string.h> #include <stdio.h> /* Assuming x86, usual endian crap is not accounted for */ typedef unsigned char u8_t; typedef unsigned short u16_t; typedef unsigned int u32_t; struct image { unsigned int width, height; u8_t *planes; }; struct clip { short x, y, w, h; }; static u16_t swab16(u16_t in) { /* GC headers are big-endian */ return ((in & 0xFF) << 8) | (in >> 8); } static void unpack_gc(struct image *out, u8_t *in, size_t out_sz) { unsigned int npixels; unsigned int i; unsigned int j; u16_t *pixels; u16_t *pheight; u16_t *pwidth; u16_t *pmagic; u16_t pixel; pmagic = (u16_t *) in; pheight = (u16_t *) (in + 14); pwidth = (u16_t *) (in + 12); if (*pmagic != 0x4347) { fprintf(stderr, "(FAIL: Invalid header)\n"); exit(EXIT_FAILURE); } /* Set up output image struct */ in += 24; out->width = swab16(*pwidth); out->height = swab16(*pheight); out->planes = malloc(out->width * out->height * 4); /* Clamp W/H (don't know why this is necessary but it is) */ out->width = out->width > 1024 ? 1024 : out->width; out->height = out->height > 1024 ? 1024 : out->height; fprintf(stderr, "(%dx%d)", out->width, out->height); /* Unpack pixels */ pixels = (u16_t *) in; npixels = out->width * out->height; if (out_sz > npixels * 4) { /* Deep image (i.e. 32-bit) */ memcpy(out->planes, pixels, npixels * 4); } else { /* Shallow image (i.e. 16-bit) */ for (i = 0, j = 0 ; i < npixels ; i++) { pixel = pixels[i]; out->planes[j++] = ((pixel ) & 0x1F) << 3; /* B */ out->planes[j++] = ((pixel >> 5) & 0x1F) << 3; /* G */ out->planes[j++] = ((pixel >> 10) ) << 3; /* R */ out->planes[j++] = pixel & 0x8000 ? 0xFF : 0x00; /* A */ } } } static u8_t *expand_lzss(u8_t *lzss, size_t *pout_sz) { static u8_t ring[0x1000]; unsigned int ring_pos = 0x0FEE; unsigned int chunk_offset; unsigned int chunk_length; u32_t control_word = 1; size_t length; u8_t cmd1; u8_t cmd2; u8_t *out; u8_t *pos; u8_t *in; /* Header = 32 bit unpacked file length */ length = *((u32_t *) lzss); *pout_sz = length; if (length > 8000000) { fprintf(stderr, "(FAIL: Unreasonably large expanded size %d)\n", length); exit(EXIT_FAILURE); } out = malloc(length * 2); /* Seems to overrun */ pos = out; in = lzss + 4; while (length > 0) { if (control_word == 1) { /* Read a control byte */ control_word = 0x100 | *in++; } /* Decode a byte according to the current control byte bit */ if (control_word & 1) { /* Straight copy */ *pos++ = *in; ring[ring_pos] = *in++; ring_pos = (ring_pos + 1) % 0x1000; length--; } else { /* Reference to data in ring buffer */ cmd1 = *in++; cmd2 = *in++; chunk_length = (cmd2 & 0x0F) + 3; chunk_offset = ((cmd2 & 0xF0) << 4) | cmd1; for ( ; chunk_length > 0 ; chunk_length--) { /* Copy historical data to output AND current ring pos */ *pos++ = ring[chunk_offset]; ring[ring_pos] = ring[chunk_offset]; /* Update counters */ chunk_offset = (chunk_offset + 1) % 0x1000; ring_pos = (ring_pos + 1) % 0x1000; length--; } } /* Get next control bit */ control_word >>= 1; } return out; } static void readfile(const char *filename, u8_t **data, long *nbytes) { FILE *f; f = fopen(filename, "rb"); if (f == NULL) abort(); fseek(f, 0, SEEK_END); *nbytes = ftell(f); fseek(f, 0, SEEK_SET); *data = malloc(*nbytes); fread(*data, *nbytes, 1, f); fclose(f); } void put8(FILE *f, unsigned char val) { fwrite(&val, 1, 1, f); } void put16(FILE *f, unsigned short val) { fwrite(&val, 2, 1, f); } void split_images(const char *in_dir, const char *out_dir, struct image *images, int nimages) { struct clip *clips; char filename[512]; long nbytes; u8_t *data; char *name; FILE *f; int i; int j; int k; /* Read file and get TOC */ sprintf(filename, "%s/system.idx", in_dir); readfile(filename, &data, &nbytes); clips = (struct clip *) (data + 0x01BC); name = (char *) (data + 8 + *((long *) data)); /* Guess how many clips there are with a heuristic */ for (i = 0 ; clips[i].w != 0 && clips[i].h != 0 ; i++) { sprintf(filename, "%s/%s.tga", out_dir, name); name += strlen(name) + 3; f = fopen(filename, "wb"); if (f == NULL) abort(); /* Locate the correct source image */ j = 0; while (clips[i].y > images[j].height) { clips[i].y -= images[j].height; j++; } /* Write header */ put8(f, 0); put8(f, 0); put8(f, 2); put16(f, 0); put16(f, 0); put8(f, 0); put16(f, 0); put16(f, 0); put16(f, clips[i].w); put16(f, clips[i].h); put8(f, 32); put8(f, 32); /* Write scanlines */ for (k = 0 ; k < clips[i].h ; k++) { if (clips[i].y == images[j].height) { clips[i].y = 0; j++; } fwrite(images[j].planes + ((images[j].width * clips[i].y) + clips[i].x) * 4, clips[i].w, 4, f); clips[i].y++; } /* Close output file */ fclose(f); } /* Cleanup */ free(data); } int main(int argc, char **argv) { char *in_dir; char *out_dir; struct image images[32]; char filename[256]; unsigned int i; long filesize; u8_t *lzss, *gc; size_t out_sz; FILE *f; /* Usage */ if (argc != 3) { fprintf(stderr, "Usage: %s [indir] [outdir]\n", argv[0]); return EXIT_FAILURE; } /* Setup */ memset(images, 0, sizeof(images)); in_dir = argv[1]; out_dir = argv[2]; for (i = 0 ; i < 32 ; i++) { /* Open 0.gcz, 1.gcz etc ... */ sprintf(filename, "%s/%d.gcz", in_dir, i); f = fopen(filename, "rb"); if (f == NULL) break; /* Read entire file */ fseek(f, 0, SEEK_END); filesize = ftell(f); fseek(f, 0, SEEK_SET); fprintf(stderr, "%s: fread", filename); lzss = malloc(filesize); fread(lzss, filesize, 1, f); fclose(f); /* Decompress */ fprintf(stderr, "(OK) expand_lzss"); gc = expand_lzss(lzss, &out_sz); free(lzss); /* Unpack GC to 32-bit RGBA */ fprintf(stderr, "(OK) unpack_gc"); unpack_gc(&images[i], gc, out_sz); free(gc); fprintf(stderr, "(OK)\n"); } /* Sanity check */ if (i == 0) { fprintf(stderr, "No GCZ files found\n"); exit(EXIT_FAILURE); } /* Emit pile of TGAs */ fprintf(stderr, "split_images"); split_images(in_dir, out_dir, images, i); fprintf(stderr, "(OK)\n\n"); return 0; } What is wrong with the code that could be causing this? The code is unaltered save for #define _CRT_SECURE_NO_WARNINGS being added to the code before the #include headers and having the program compiled as C.
RSA public key encryption openssl
a question: Vendor says that for some encryption purpose uses PKCS#1 V2.1 OAEP with SHA-256... Is that even possible? I have checked and re-checked openssl and all they have is RSA public key encrypt with OAEP padding which is supposed to be PKCS#1 V2.1 with SHA1 So what can I do? How can I use SHA256 in RSA PUBLIC KEY encryption? IS it even possible? Best regards, EDITED: ANSWER HOW TO USE RSA ENCRYPTION USING OPENSSL OAEP PADDING AND SHA256 DIGEST #include "openssl/rsa.h" #include <openssl/err.h> #define RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1 154 int RSA_padding_add_PKCS1_OAEP_mgf1_SHA256(unsigned char *to, int tlen, const unsigned char *from, int flen, const unsigned char *param, int plen, const EVP_MD *md, const EVP_MD *mgf1md) { int i, emlen = tlen - 1; unsigned char *db, *seed; unsigned char *dbmask, seedmask[EVP_MAX_MD_SIZE]; int mdlen; if (md == NULL) md = EVP_sha256(); //HERE IS THE ACTUAL USE OF SHAR256 digest! if (mgf1md == NULL) mgf1md = md; mdlen = EVP_MD_size(md); if (flen > emlen - 2 * mdlen - 1) { RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE); return 0; } if (emlen < 2 * mdlen + 1) { RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1, RSA_R_KEY_SIZE_TOO_SMALL); return 0; } to[0] = 0; seed = to + 1; db = to + mdlen + 1; if (!EVP_Digest((void *)param, plen, db, NULL, md, NULL)) return 0; memset(db + mdlen, 0, emlen - flen - 2 * mdlen - 1); db[emlen - flen - mdlen - 1] = 0x01; memcpy(db + emlen - flen - mdlen, from, (unsigned int)flen); if (RAND_bytes(seed, mdlen) <= 0) return 0; #ifdef PKCS_TESTVECT memcpy(seed, "\xaa\xfd\x12\xf6\x59\xca\xe6\x34\x89\xb4\x79\xe5\x07\x6d\xde\xc2\xf0\x6c\xb5\x8f", 20); #endif dbmask = (unsigned char*)OPENSSL_malloc(emlen - mdlen); if (dbmask == NULL) { RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP_MGF1, ERR_R_MALLOC_FAILURE); return 0; } if (PKCS1_MGF1(dbmask, emlen - mdlen, seed, mdlen, mgf1md) < 0) return 0; for (i = 0; i < emlen - mdlen; i++) db[i] ^= dbmask[i]; if (PKCS1_MGF1(seedmask, mdlen, db, emlen - mdlen, mgf1md) < 0) return 0; for (i = 0; i < mdlen; i++) seed[i] ^= seedmask[i]; OPENSSL_free(dbmask); return 1; } int RSA_padding_add_PKCS1_OAEP_SHA256(unsigned char *to, int tlen, const unsigned char *from, int flen, const unsigned char *param, int plen) { return RSA_padding_add_PKCS1_OAEP_mgf1_SHA256(to, tlen, from, flen, param, plen, NULL, NULL); } static int RSA_eay_public_encrypt_SHA256(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { BIGNUM *f, *ret; int i, j, k, num = 0, r = -1; unsigned char *buf = NULL; BN_CTX *ctx = NULL; #ifdef OPENSSL_FIPS if (FIPS_selftest_failed()) { FIPSerr(FIPS_F_RSA_EAY_PUBLIC_ENCRYPT, FIPS_R_FIPS_SELFTEST_FAILED); goto err; } if (FIPS_module_mode() && !(rsa->flags & RSA_FLAG_NON_FIPS_ALLOW) && (BN_num_bits(rsa->n) < OPENSSL_RSA_FIPS_MIN_MODULUS_BITS)) { RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_KEY_SIZE_TOO_SMALL); return -1; } #endif if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS) { RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_MODULUS_TOO_LARGE); return -1; } if (BN_ucmp(rsa->n, rsa->e) <= 0) { RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE); return -1; } /* for large moduli, enforce exponent limit */ if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS) { if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS) { RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE); return -1; } } if ((ctx = BN_CTX_new()) == NULL) goto err; BN_CTX_start(ctx); f = BN_CTX_get(ctx); ret = BN_CTX_get(ctx); num = BN_num_bytes(rsa->n); buf = (unsigned char*)OPENSSL_malloc(num); if (!f || !ret || !buf) { RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, ERR_R_MALLOC_FAILURE); goto err; } switch (padding) { case RSA_PKCS1_PADDING: i = RSA_padding_add_PKCS1_type_2(buf, num, from, flen); break; #ifndef OPENSSL_NO_SHA case RSA_PKCS1_OAEP_PADDING: i = RSA_padding_add_PKCS1_OAEP_SHA256(buf, num, from, flen, NULL, 0); break; #endif case RSA_SSLV23_PADDING: i = RSA_padding_add_SSLv23(buf, num, from, flen); break; case RSA_NO_PADDING: i = RSA_padding_add_none(buf, num, from, flen); break; default: RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_UNKNOWN_PADDING_TYPE); goto err; } if (i <= 0) goto err; if (BN_bin2bn(buf, num, f) == NULL) goto err; if (BN_ucmp(f, rsa->n) >= 0) { /* usually the padding functions would catch this */ RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_DATA_TOO_LARGE_FOR_MODULUS); goto err; } if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, CRYPTO_LOCK_RSA, rsa->n, ctx)) goto err; if (!rsa->meth->bn_mod_exp(ret, f, rsa->e, rsa->n, ctx, rsa->_method_mod_n)) goto err; /* put in leading 0 bytes if the number is less than the * length of the modulus */ j = BN_num_bytes(ret); i = BN_bn2bin(ret, &(to[num - j])); for (k = 0; k<(num - i); k++) to[k] = 0; r = num; err: if (ctx != NULL) { BN_CTX_end(ctx); BN_CTX_free(ctx); } if (buf != NULL) { OPENSSL_cleanse(buf, num); OPENSSL_free(buf); } return(r); } int RSA_public_encrypt_sha256(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { return(RSA_eay_public_encrypt_SHA256(flen, from, to, rsa, padding)); } Just add these few functions and call RSA_public_encrypt_sha256 instead of RSA_public_encrypt and voila you have RSA_OAEP_SHA256 Well i know this is abusing the openssl code, but this is a solution if you cannot compile openssl lib yourself, like i cannot because i received this as a part of an ARM platform All thanks go to JARIQ in the answer below! Thank you!
I am not sure about the OpenSSL API but in PKCS#11 API when you are using RSA encryption with OAEP padding you can specify message digest algorithm and also a mask generation function as you can see in my code sample (take a look at _03_EncryptAndDecryptSinglePartOaepTest() method) . It is written in C# but I believe it should be easily understandable. However I have never tried anything else than SHA1. More information can be found in RFC 3447 and PKCS#11 specification (chapter 12.1.7 and chapter 12.1.8). EDIT for OpenSSL: In OpenSSL RSA encryption with public key and OAEP padding is performed in this order: you need to pass RSA_PKCS1_OAEP_PADDING flag to function RSA_public_encrypt() implemented in rsa_crpt.c RSA_public_encrypt() then calls function RSA_eay_public_encrypt() implemented in rsa_eay.c (unless you are using some cryptographic hardware device via ENGINE) RSA_eay_public_encrypt() then calls function RSA_padding_add_PKCS1_OAEP() implemented in rsa_oaep.c This uses SHA1 which seems to be currently the only option implemented in OpenSSL but I believe it should be possible to slightly modify code in rsa_oaep.c file to achieve what you need.
C++ AES OpenSSL
I'm trying to read a string from a file encrypt it with AES and then save it to other file. Later I need to read the new file, decrypt and save the output to a new file again. The problem is some strange character are appearing. int Crypt::__aesEncrypt(const unsigned char *msg, size_t msgLen, unsigned char **encMsg) { EVP_CIPHER_CTX *aesEncryptCtx = (EVP_CIPHER_CTX*)malloc(sizeof(EVP_CIPHER_CTX)); EVP_CIPHER_CTX_init(aesEncryptCtx); unsigned char *aesKey = (unsigned char*)malloc(AES_KEYLEN/8); unsigned char *aesIV = (unsigned char*)malloc(AES_KEYLEN/8); unsigned char *aesPass = (unsigned char*)malloc(AES_KEYLEN/8); unsigned char *aesSalt = (unsigned char*)malloc(8); if(RAND_bytes(aesPass, AES_KEYLEN/8) == 0) { return FAILURE; } if(RAND_bytes(aesSalt, 8) == 0) { return FAILURE; } if(EVP_BytesToKey(EVP_aes_256_cbc(), EVP_sha1(), aesSalt, aesPass, AES_KEYLEN/8, AES_ROUNDS, aesKey, aesIV) == 0) { return FAILURE; } strncpy((char*)aesKey, (const char*)"B374A26A714904AAB374A26A714904AA", AES_KEYLEN/8); strncpy((char*)aesIV, (const char*)"B374A26A714904AA", AES_KEYLEN/16); size_t blockLen = 0; size_t encMsgLen = 0; *encMsg = (unsigned char*)malloc(msgLen + AES_BLOCK_SIZE); if(encMsg == NULL) return FAILURE; (*encMsg)[0] = '\0'; if(!EVP_EncryptInit_ex(aesEncryptCtx, EVP_aes_256_cbc(), NULL, aesKey, aesIV)) { return FAILURE; } if(!EVP_EncryptUpdate(aesEncryptCtx, *encMsg, (int*)&blockLen, (unsigned char*)msg, msgLen)) { return FAILURE; } encMsgLen += blockLen; if(!EVP_EncryptFinal_ex(aesEncryptCtx, *encMsg + encMsgLen, (int*)&blockLen)) { return FAILURE; } EVP_CIPHER_CTX_cleanup(aesEncryptCtx); free(aesEncryptCtx); free(aesKey); free(aesIV); return encMsgLen + blockLen; } int Crypt::__aesDecrypt(unsigned char *encMsg, size_t encMsgLen, char **decMsg) { EVP_CIPHER_CTX *aesDecryptCtx = (EVP_CIPHER_CTX*)malloc(sizeof(EVP_CIPHER_CTX)); EVP_CIPHER_CTX_init(aesDecryptCtx); unsigned char *aesKey; unsigned char *aesIV; aesKey = (unsigned char*)malloc(AES_KEYLEN/8); aesIV = (unsigned char*)malloc(AES_KEYLEN/8); unsigned char *aesPass = (unsigned char*)malloc(AES_KEYLEN/8); unsigned char *aesSalt = (unsigned char*)malloc(8); if(RAND_bytes(aesPass, AES_KEYLEN/8) == 0) { return FAILURE; } if(RAND_bytes(aesSalt, 8) == 0) { return FAILURE; } if(EVP_BytesToKey(EVP_aes_256_cbc(), EVP_sha1(), aesSalt, aesPass, AES_KEYLEN/8, AES_ROUNDS, aesKey, aesIV) == 0) { return FAILURE; } strncpy((char*)aesKey, (const char*)"B374A26A714904AAB374A26A714904AA", AES_KEYLEN/8); strncpy((char*)aesIV, (const char*)"B374A26A714904AA", AES_KEYLEN/16); size_t decLen = 0; size_t blockLen = 0; *decMsg = (char*)malloc(encMsgLen); if(*decMsg == NULL) return FAILURE; if(!EVP_DecryptInit_ex(aesDecryptCtx, EVP_aes_256_cbc(), NULL, aesKey, aesIV)) { return FAILURE; } if(!EVP_DecryptUpdate(aesDecryptCtx, (unsigned char*)*decMsg, (int*)&blockLen, encMsg, (int)encMsgLen)) { return FAILURE; } decLen += blockLen; if(!EVP_DecryptFinal_ex(aesDecryptCtx, (unsigned char*)*decMsg + decLen, (int*)&blockLen)) { return FAILURE; } decLen += blockLen; (*decMsg)[decLen] = '\0'; EVP_CIPHER_CTX_cleanup(aesDecryptCtx); return encMsgLen; } Encrypting: unsigned char *encMsg = NULL; int size = __aesEncrypt((const unsigned char*)decrypted_string.c_str(), decrypted_string.size(), &encMsg); return std::string(reinterpret_cast<const char*>(encMsg), size); Decrypting: char *decMsg = NULL; int size = __aesDecrypt((unsigned char*)encrypted_string.c_str(), encrypted_string.size(), &decMsg); return std::string(reinterpret_cast<const char*>(decMsg), size); I can successfully crypt and decrypt, but some strange characters are appearing at the end of the encrypted file, they are like empty spaces:
AES is a block cypher. It takes blocks of 16 bytes, and encrypt them into a block of 16 byets. If you try to use it with data whose length is not a multiple of 16, padding (usually random data) is added to round it up to a multiple of 16 bytes. You need to manage the length of the data yourself. Example: int encryptHelper(const string& msg, ...) { uint32_t msgSize = msg.length(); newMsg.push_back((msgSize >> 0) & 0xFF); newMsg.push_back((msgSize >> 8) & 0xFF); newMsg.push_back((msgSize >> 16) & 0xFF); newMsg.push_back((msgSize >> 24) & 0xFF); string newMsg(reinterpret_cast<const char*>(&msgSize), sizeof(msgSize)); newMsg += msg; return __aesEncrypt(newMsg.c_str(), newMsg.length(), ...); } int decryptHelper(const string& encrypted, ...) { string msg = ... whatever you are doing to decrypt uint32_t actualSize = 0; // remove signal first, then widen actualSize |= static_cast<uint32_t>(static_cast<unsigned char>(msg[0])) << 0; actualSize |= static_cast<uint32_t>(static_cast<unsigned char>(msg[1])) << 8; actualSize |= static_cast<uint32_t>(static_cast<unsigned char>(msg[2])) << 16; actualSize |= static_cast<uint32_t>(static_cast<unsigned char>(msg[3])) << 24; string actualMsg = msg.substr(4, actualSize); ... } I didn't bother to write the exact code to call your functions because all that casting and memory leaking gave me nausea. Fill in the blanks.