How to compose a PKCS#7 signature file correctly? - c++

Question

I'm trying to use the Botan library to generate a detached signature file. The resulting signature file is not validated by OpenSSL (no other checks). Prompt in what there can be an error of formation of the signature file.
A couple of keys for signing and the certificate is stored in the HSM, it was not difficult to get them. For tests I use RSA keys and SoftHSM, later another key format and physical HSM will be used. PKCS#11 is used to communicate with HSM.
For create PKCS#7:
static const Botan::BigInt CMSVersion(1ull);
std::vector<uint8_t> createAttributes(std::vector<uint8_t> &digestData)
{
std::chrono::time_point<std::chrono::system_clock> time = std::chrono::system_clock::now();
Botan::OID dataOID("1.2.840.113549.1.7.1");
Botan::Attribute contentType(Botan::OIDS::str2oid("PKCS9.ContentType"),
dataOID.BER_encode());
Botan::X509_Time timeASN1(time);
std::vector<uint8_t> attributesData;
Botan::DER_Encoder attrib(attributesData);
attrib.start_cons(Botan::ASN1_Tag(0),
Botan::ASN1_Tag(Botan::ASN1_Tag::CONTEXT_SPECIFIC));
attrib.encode(contentType)
.start_cons(Botan::ASN1_Tag::SEQUENCE)
.encode(Botan::OID("1.2.840.113549.1.9.5"))
.start_cons(Botan::ASN1_Tag::SET).encode(timeASN1).end_cons()
.end_cons()
.start_cons(Botan::ASN1_Tag::SEQUENCE)
.encode(Botan::OIDS::str2oid("PKCS9.MessageDigest"))
.start_cons(Botan::ASN1_Tag::SET)
.encode(digestData, Botan::ASN1_Tag::OCTET_STRING,
Botan::ASN1_Tag::OCTET_STRING, Botan::ASN1_Tag::UNIVERSAL)
.end_cons()
.end_cons();
attrib.end_cons();
return attributesData;
}
std::vector<uint8_t> createCMS(const Botan::AlgorithmIdentifier &digestAlg,
Botan::X509_Certificate &cert,
const Botan::AlgorithmIdentifier &keyAlg,
std::vector<uint8_t> &sigData,
std::vector<uint8_t> &signedAttributes)
{
Botan::secure_vector<uint8_t> msgData;
Botan::DER_Encoder encoder(msgData);
encoder.start_cons(Botan::ASN1_Tag::SEQUENCE).encode(CMSVersion)
.start_cons(Botan::ASN1_Tag::SET).start_cons(Botan::ASN1_Tag::SEQUENCE)
.encode(digestAlg.get_oid()).end_cons().end_cons();
Botan::OID dataOID("1.2.840.113549.1.7.1");
encoder.start_cons(Botan::ASN1_Tag::SEQUENCE).encode(dataOID).end_cons();
encoder.start_cons(Botan::ASN1_Tag::UNIVERSAL, Botan::ASN1_Tag::PRIVATE)
.encode(cert).end_cons();
encoder.start_cons(Botan::ASN1_Tag::SET);
Botan::secure_vector<uint8_t> signerInfoData;
Botan::DER_Encoder signerInfo(signerInfoData);
signerInfo.start_cons(Botan::ASN1_Tag::SEQUENCE);
signerInfo.encode(CMSVersion);
signerInfo.start_cons(Botan::ASN1_Tag::SEQUENCE)
.encode(cert.issuer_dn())
.encode(Botan::BigInt(cert.serial_number())).end_cons();
signerInfo.start_cons(Botan::ASN1_Tag::SEQUENCE).encode(digestAlg.get_oid())
.end_cons();
signerInfo.raw_bytes(signedAttributes);
signerInfo.encode(keyAlg)
.encode(sigData, Botan::ASN1_Tag::OCTET_STRING,
Botan::ASN1_Tag::OCTET_STRING, Botan::ASN1_Tag::UNIVERSAL);
signerInfo.end_cons();
encoder.raw_bytes(signerInfoData).end_cons().end_cons();
std::vector<uint8_t> resulData;
Botan::DER_Encoder result(resulData);
result.start_cons(Botan::ASN1_Tag::SEQUENCE)
.encode(Botan::OID("1.2.840.113549.1.7.2"))
.start_cons(Botan::ASN1_Tag::UNIVERSAL, Botan::ASN1_Tag::PRIVATE)
.raw_bytes(msgData).end_cons().end_cons();
return resulData;
}
To calculate the hash and signature using PKCS#11, as follows:
QFile input(m_content->text()), output(m_sigFile->text());
if(!input.open(QFile::ReadOnly))
{
QMessageBox::critical(this, tr("Error"),
tr("Content file '%1' not open.\n"
"Error message: %2").arg(m_content->text())
.arg(input.errorString()));
return;
}
Botan::PKCS11::PKCS11_X509_Certificate *cert = nullptr;
Botan::Private_Key *key = nullptr;
// извлечение ключа и сертификата из токена
while(!input.atEnd())
{
static const qint64 maxLen = 1024;
QByteArray data = input.read(maxLen);
(*module)->C_DigestUpdate(session->handle(),
reinterpret_cast<uchar*>(data.data()),
data.size(), &rv);
if(rv != Botan::PKCS11::ReturnValue::OK)
{
QMessageBox::critical(this, tr("Error"),
tr("Digest not run.\nError code: 0x%3")
.arg(static_cast<int>(rv), 0, 16));
delete key;
delete cert;
delete session;
delete slot;
delete module;
return;
}
}
digest.resize(102400);
ulong digestLen;
(*module)->C_DigestFinal(session->handle(), digest.data(), &digestLen, &rv);
if(rv != Botan::PKCS11::ReturnValue::OK)
{
QMessageBox::critical(this, tr("Error"),
tr("Digest not start.\nError code: 0x%3")
.arg(static_cast<int>(rv), 0, 16));
delete key;
delete cert;
delete session;
delete slot;
delete module;
return;
}
digest.resize(digestLen);
{
Botan::PKCS11::PKCS11_RNG rng(*session);
std::unique_ptr<Botan::PK_Ops::Signature> signer =
key->create_signature_op(rng,
"EMSA3(SHA-256)",
"");
signer->update(digest.data(), digest.size());
std::vector<uint8_t> attr = createAttributes(digest);
auto signData = signer->sign(rng);
for(uint8_t i : signData)
signature.push_back(i);
Botan::AlgorithmIdentifier digAlg("SHA-256", {});
auto fileData = createCMS(digAlg, *cert, key->algorithm_identifier(),
signature, attr);
output.write(reinterpret_cast<const char*>(fileData.data()),
fileData.size());
output.close();
}
When checking the received signature file, OpenSSL says
Verification failure
140365848428992:error:04091068:rsa routines:int_rsa_verify:bad signature:../crypto/rsa/rsa_sign.c:220:
140365848428992:error:2E09A09E:CMS routines:CMS_SignerInfo_verify_content:verification failure:../crypto/cms/cms_sd.c:842:
140365848428992:error:2E09D06D:CMS routines:CMS_verify:content verify error:../crypto/cms/cms_smime.c:393: