Self sign with SAN with OpenSSL in Chrome and Firefox - c++

Question

I created a minimal code that serves as an HTTPS server with CA created as follows:
$ openssl req -config openssl.conf -x509 -sha256 -nodes -extensions v3_ca -days 3650 -subj '/CN=OpenSSL CA/O=Example Company/C=SE' -newkey rsa:4096 -keyout ca.key -out ca.pem
I added the CA to the Trusted Certificate in Chrome and Firefox, and I added a new line to hosts file to resolve test.com as 127.0.0.1.
In Firefox it works as expected, but Chrome returns a 'NET::ERR_CERT_COMMON_NAME_INVALID
' error. It makes sense that Chrome starting version 58 no longer supports a common name and instead needs to use subjectAltName.
So I added a snippet of code (currently comment-out) to support this, then Firefox returns the error SEC_ERROR_BAD_SIGNATURE, and Chrome returns the NET::ERR_CERT_AUTHORITY_INVALID.
#include <openssl/err.h>
#include <openssl/conf.h>
#include <openssl/pem.h>
#include <openssl/rand.h>
#include <openssl/ssl.h>
#include <netinet/in.h>
#include <sys/socket.h>
#include <openssl/x509v3.h>
#define RSA_KEY_BITS (4096)
#define REQ_DN_C "SE"
#define REQ_DN_ST ""
#define REQ_DN_L ""
#define REQ_DN_O "Example Company"
#define REQ_DN_OU ""
#define REQ_DN_CN "test.com"
static void crt_to_pem(X509 *crt, uint8_t **crt_bytes, size_t *crt_size);
static int generate_key_csr(EVP_PKEY **key, X509_REQ **req);
static int generate_set_random_serial(X509 *crt);
static int generate_signed_key_pair(EVP_PKEY *ca_key, X509 *ca_crt, EVP_PKEY **key, X509 **crt);
static void key_to_pem(EVP_PKEY *key, uint8_t **key_bytes, size_t *key_size);
static int load_ca(const char *ca_key_path, EVP_PKEY **ca_key, const char *ca_crt_path, X509 **ca_crt);
static void print_bytes(uint8_t *data, size_t size);
int create_socket() {
int sockfd, portno;
char buffer[256];
struct sockaddr_in serv_addr{}, cli_addr{};
int n;
/* First call to socket() function */
sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (sockfd < 0) {
perror("ERROR opening socket");
exit(1);
}
portno = 8000;
serv_addr.sin_family = AF_INET;
serv_addr.sin_addr.s_addr = INADDR_ANY;
serv_addr.sin_port = htons(portno);
/* Now bind the host address using bind() call.*/
if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) < 0) {
perror("ERROR on binding");
exit(1);
}
listen(sockfd,5);
return sockfd;
}
SSL_CTX *create_context()
{
const SSL_METHOD *method;
SSL_CTX *ctx;
method = TLS_server_method();
ctx = SSL_CTX_new(method);
if (!ctx) {
exit(EXIT_FAILURE);
}
return ctx;
}
int main(int argc, char **argv)
{
char *ca_key_path = "ca.key";
char *ca_crt_path = "ca.pem";
/* Load CA key and cert. */
EVP_PKEY *ca_key = NULL;
X509 *ca_crt = NULL;
if (!load_ca(ca_key_path, &ca_key, ca_crt_path, &ca_crt)) {
fprintf(stderr, "Failed to load CA certificate and/or key!\n");
return 1;
}
/* Generate keypair and then print it byte-by-byte for demo purposes. */
EVP_PKEY *key = NULL;
X509 *crt = NULL;
int ret = generate_signed_key_pair(ca_key, ca_crt, &key, &crt);
if (!ret) {
fprintf(stderr, "Failed to generate key pair!\n");
return 1;
}
/*
X509_EXTENSION *cert_ex = X509V3_EXT_conf_nid(nullptr, nullptr, NID_subject_alt_name, "DNS:test.com");
X509_add_ext(crt, cert_ex, -1);
*/
/* Convert key and certificate to PEM format. */
uint8_t *key_bytes = NULL;
uint8_t *crt_bytes = NULL;
size_t key_size = 0;
size_t crt_size = 0;
key_to_pem(key, &key_bytes, &key_size);
crt_to_pem(crt, &crt_bytes, &crt_size);
/* Print key and certificate. */
print_bytes(key_bytes, key_size);
print_bytes(crt_bytes, crt_size);
auto ctx = create_context();
auto lfd = create_socket();
while (true) {
int clilen = 0;
struct sockaddr_in serv_addr{}, cli_addr{};
auto s = accept(lfd, (struct sockaddr *)&cli_addr, reinterpret_cast<socklen_t *>(&clilen));
auto ssl = SSL_new(ctx);
SSL_set_fd(ssl, s);
SSL_use_certificate(ssl, crt);
SSL_use_PrivateKey(ssl, key);
SSL_accept(ssl);
char msg_200ok[] = "HTTP/1.1 404 \n"
"Content-Length: 0\n\n";
SSL_write(ssl, msg_200ok, sizeof msg_200ok - 1);
}
/* Free stuff. */
EVP_PKEY_free(ca_key);
EVP_PKEY_free(key);
X509_free(ca_crt);
X509_free(crt);
free(key_bytes);
free(crt_bytes);
return 0;
}
void crt_to_pem(X509 *crt, uint8_t **crt_bytes, size_t *crt_size)
{
/* Convert signed certificate to PEM format. */
BIO *bio = BIO_new(BIO_s_mem());
PEM_write_bio_X509(bio, crt);
*crt_size = BIO_pending(bio);
*crt_bytes = (uint8_t *)malloc(*crt_size + 1);
BIO_read(bio, *crt_bytes, *crt_size);
BIO_free_all(bio);
}
int generate_signed_key_pair(EVP_PKEY *ca_key, X509 *ca_crt, EVP_PKEY **key, X509 **crt)
{
/* Generate the private key and corresponding CSR. */
X509_REQ *req = NULL;
if (!generate_key_csr(key, &req)) {
fprintf(stderr, "Failed to generate key and/or CSR!\n");
return 0;
}
/* Sign with the CA. */
*crt = X509_new();
X509_set_version(*crt, 2); /* Set version to X509v3 */
/* Generate random 20 byte serial. */
generate_set_random_serial(*crt);
/* Set issuer to CA's subject. */
X509_set_issuer_name(*crt, X509_get_subject_name(ca_crt));
/* Set validity of certificate to 2 years. */
X509_gmtime_adj(X509_get_notBefore(*crt), 0);
X509_gmtime_adj(X509_get_notAfter(*crt), (long)2*365*24*3600);
/* Get the request's subject and just use it (we don't bother checking it since we generated
* it ourself). Also take the request's public key. */
X509_set_subject_name(*crt, X509_REQ_get_subject_name(req));
EVP_PKEY *req_pubkey = X509_REQ_get_pubkey(req);
X509_set_pubkey(*crt, req_pubkey);
EVP_PKEY_free(req_pubkey);
/* Now perform the actual signing with the CA. */
if (X509_sign(*crt, ca_key, EVP_sha256()) == 0) goto err;
X509_REQ_free(req);
return 1;
err:
EVP_PKEY_free(*key);
X509_REQ_free(req);
X509_free(*crt);
return 0;
}
int generate_key_csr(EVP_PKEY **key, X509_REQ **req)
{
*key = NULL;
*req = NULL;
RSA *rsa = NULL;
BIGNUM *e = NULL;
*key = EVP_PKEY_new();
*req = X509_REQ_new();
rsa = RSA_new();
e = BN_new();
BN_set_word(e, 65537);
RSA_generate_key_ex(rsa, RSA_KEY_BITS, e, NULL);
EVP_PKEY_assign_RSA(*key, rsa);
X509_REQ_set_pubkey(*req, *key);
/* Set the DN of the request. */
X509_NAME *name = X509_REQ_get_subject_name(*req);
X509_NAME_add_entry_by_txt(name, "C", MBSTRING_ASC, (const unsigned char*)REQ_DN_C, -1, -1, 0);
X509_NAME_add_entry_by_txt(name, "ST", MBSTRING_ASC, (const unsigned char*)REQ_DN_ST, -1, -1, 0);
X509_NAME_add_entry_by_txt(name, "L", MBSTRING_ASC, (const unsigned char*)REQ_DN_L, -1, -1, 0);
X509_NAME_add_entry_by_txt(name, "O", MBSTRING_ASC, (const unsigned char*)REQ_DN_O, -1, -1, 0);
X509_NAME_add_entry_by_txt(name, "OU", MBSTRING_ASC, (const unsigned char*)REQ_DN_OU, -1, -1, 0);
X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC, (const unsigned char*)REQ_DN_CN, -1, -1, 0);
/* Self-sign the request to prove that we posses the key. */
if (!X509_REQ_sign(*req, *key, EVP_sha256())) goto err;
BN_free(e);
return 1;
err:
EVP_PKEY_free(*key);
X509_REQ_free(*req);
RSA_free(rsa);
BN_free(e);
return 0;
}
int generate_set_random_serial(X509 *crt)
{
/* Generates a 20 byte random serial number and sets in certificate. */
unsigned char serial_bytes[20];
if (RAND_bytes(serial_bytes, sizeof(serial_bytes)) != 1) return 0;
serial_bytes[0] &= 0x7f; /* Ensure positive serial! */
BIGNUM *bn = BN_new();
BN_bin2bn(serial_bytes, sizeof(serial_bytes), bn);
ASN1_INTEGER *serial = ASN1_INTEGER_new();
BN_to_ASN1_INTEGER(bn, serial);
X509_set_serialNumber(crt, serial); // Set serial.
ASN1_INTEGER_free(serial);
BN_free(bn);
return 1;
}
void key_to_pem(EVP_PKEY *key, uint8_t **key_bytes, size_t *key_size)
{
/* Convert private key to PEM format. */
BIO *bio = BIO_new(BIO_s_mem());
PEM_write_bio_PrivateKey(bio, key, NULL, NULL, 0, NULL, NULL);
*key_size = BIO_pending(bio);
*key_bytes = (uint8_t *)malloc(*key_size + 1);
BIO_read(bio, *key_bytes, *key_size);
BIO_free_all(bio);
}
int load_ca(const char *ca_key_path, EVP_PKEY **ca_key, const char *ca_crt_path, X509 **ca_crt)
{
BIO *bio = NULL;
*ca_crt = NULL;
*ca_key = NULL;
/* Load CA public key. */
bio = BIO_new(BIO_s_file());
if (!BIO_read_filename(bio, ca_crt_path)) goto err;
*ca_crt = PEM_read_bio_X509(bio, NULL, NULL, NULL);
if (!*ca_crt) goto err;
BIO_free_all(bio);
/* Load CA private key. */
bio = BIO_new(BIO_s_file());
if (!BIO_read_filename(bio, ca_key_path)) goto err;
*ca_key = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL);
if (!ca_key) goto err;
BIO_free_all(bio);
return 1;
err:
BIO_free_all(bio);
X509_free(*ca_crt);
EVP_PKEY_free(*ca_key);
return 0;
}
void print_bytes(uint8_t *data, size_t size)
{
for (size_t i = 0; i < size; i++) {
printf("%c", data[i]);
}
}
What is the problem? How can this be debugged?
Thanks

Answer 1

You are trying to add the extension to the certificate after it was signed. This makes the CA's signature invalid. You have to do that before signing, e.g.:
X509_EXTENSION *cert_ex = X509V3_EXT_conf_nid(nullptr, nullptr, NID_subject_alt_name, "DNS:test.com");
X509_add_ext(*crt, cert_ex, -1);
/* Now perform the actual signing with the CA. */
if (X509_sign(*crt, ca_key, EVP_sha256()) == 0) goto err;
With that it works for me both on Firefox and Chrome.