epoll-based server fails (mutual authentication) - c++

Question

Based on the examples at http://simplestcodings.blogspot.com/2010/08/secure-server-client-using-openssl-in-c.html I have written the following epoll server code to replace the regular tcp sockets with SSL. My code is slightly modified to do mutual authentication and verify the certs since I add
SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL);
SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);.
My OS is Ubuntu 12.04.
I have also generated server cert, client cert, server key, client key, and of course cacert.pem files from the IBM tutorial http://pic.dhe.ibm.com/infocenter/lnxinfo/v3r0m0/index.jsp?topic=%2Fliaat%2Fliaatseccreatecskeycert.htm
I ran the single server - single client code and it runs perfectly fine. Inspired by this I wanted to see how the epoll server code would behave but then I ran into issues. Here is the code for the server
Code that does not work
//(c) 2014 enthusiasticgeek for stackoverflow - epollserver.cc
//============================================================================
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/epoll.h>
#include <errno.h>
#include <iostream>
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <resolv.h>
#include "openssl/ssl.h"
#include "openssl/err.h"
#define FAIL -1
#define MAXEVENTS 64
SSL_CTX* InitServerCTX(void)
{ const SSL_METHOD *method;
SSL_CTX *ctx;
OpenSSL_add_all_algorithms(); /* load & register all cryptos, etc. */
SSL_load_error_strings(); /* load all error messages */
method = SSLv23_server_method(); /* create new server-method instance */
ctx = SSL_CTX_new(method); /* create new context from method */
if ( ctx == NULL )
{
ERR_print_errors_fp(stderr);
abort();
}
return ctx;
}
void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{
SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL);
SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);
/* set the local certificate from CertFile */
if ( SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* set the private key from KeyFile (may be the same as CertFile) */
if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* verify private key */
if ( !SSL_CTX_check_private_key(ctx) )
{
fprintf(stderr, "Private key does not match the public certificate\n");
abort();
}
}
void ShowCerts(SSL* ssl)
{ X509 *cert;
char *line;
cert = SSL_get_peer_certificate(ssl); /* Get certificates (if available) */
if ( cert != NULL )
{
printf("Server certificates:\n");
line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
printf("Subject: %s\n", line);
free(line);
line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
printf("Issuer: %s\n", line);
free(line);
X509_free(cert);
}
else
printf("No certificates.\n");
}
static int
AibSocketNonBlocking (int sfd)
{
int flags, s;
flags = fcntl (sfd, F_GETFL, 0);
if (flags == -1)
{
perror ("fcntl");
return -1;
}
flags |= O_NONBLOCK;
s = fcntl (sfd, F_SETFL, flags);
if (s == -1)
{
perror ("fcntl");
return -1;
}
return 0;
}
static int
AibCreateAndBind (char *port)
{
struct addrinfo hints;
struct addrinfo *result, *rp;
int s, sfd;
memset (&hints, 0, sizeof (struct addrinfo));
hints.ai_family = AF_UNSPEC; /* Return IPv4 and IPv6 choices */
hints.ai_socktype = SOCK_STREAM; /* We want a TCP socket */
hints.ai_flags = AI_PASSIVE; /* All interfaces */
s = getaddrinfo (NULL, port, &hints, &result);
if (s != 0)
{
fprintf (stderr, "getaddrinfo: %s\n", gai_strerror (s));
return -1;
}
for (rp = result; rp != NULL; rp = rp->ai_next)
{
sfd = socket (rp->ai_family, rp->ai_socktype, rp->ai_protocol);
if (sfd == -1)
continue;
s = bind (sfd, rp->ai_addr, rp->ai_addrlen);
if (s == 0)
{
/* We managed to bind successfully! */
break;
}
close (sfd);
}
if (rp == NULL)
{
fprintf (stderr, "Could not bind\n");
return -1;
}
freeaddrinfo (result);
return sfd;
}
int
main (int argc, char *argv[])
{
SSL_CTX *ctx;
SSL_library_init();
ctx = InitServerCTX(); /* initialize SSL */
LoadCertificates(ctx, "/home/test/ssl/cert_files/servercert.pem", "/home/test/ssl/cert_files/serverkey.pem"); /* load certs */
int sfd, s;
int efd;
struct epoll_event aibevent;
struct epoll_event *aibevents;
if (argc != 2) {
fprintf (stderr, "Usage: %s [port]\n", argv[0]);
exit (EXIT_FAILURE);
}
char portt[sizeof (unsigned int)];
snprintf(portt,sizeof portt + 1,"%u",atoi(argv[1]));
printf("sizeof %s argv[1] = %d\n",argv[1], sizeof(argv[1]));
printf("sizeof %s portt = %d\n",portt, sizeof(portt));
sfd = AibCreateAndBind (portt);//argv[1]);
if (sfd == -1) {
abort ();
}
s = AibSocketNonBlocking (sfd);
if (s == -1) {
abort ();
}
s = listen (sfd, SOMAXCONN);
if (s == -1) {
perror ("listen");
abort ();
}
efd = epoll_create1 (0);
if (efd == -1) {
perror ("epoll_create");
abort ();
}
aibevent.data.fd = sfd;
aibevent.events = EPOLLIN | EPOLLET;
s = epoll_ctl (efd, EPOLL_CTL_ADD, sfd, &aibevent);
if (s == -1) {
perror ("epoll_ctl");
abort ();
}
// Buffer where events are returned
//events = static_cast<epoll_event*>(calloc (MAXEVENTS, sizeof event));
//aibevents = static_cast<epoll_event*>(malloc (MAXEVENTS * sizeof aibevent));
aibevents = new epoll_event[MAXEVENTS * sizeof aibevent];
// The event loop
while (true)
{
int n, i;
n = epoll_wait (efd, aibevents, MAXEVENTS, -1);
for (i = 0; i < n; i++)
{
if ((aibevents[i].events & EPOLLERR) ||
(aibevents[i].events & EPOLLHUP) ||
(!(aibevents[i].events & EPOLLIN)))
{
// An error has occured on this fd, or the socket is not
// ready for reading (why were we notified then?)
fprintf (stderr, "epoll error\n");
close (aibevents[i].data.fd);
continue;
} else if (sfd == aibevents[i].data.fd) {
// We have a notification on the listening socket, which
// means one or more incoming connections.
while (1)
{
struct sockaddr in_addr;
socklen_t in_len;
int infd;
char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
in_len = sizeof in_addr;
infd = accept (sfd, &in_addr, &in_len);
if (infd == -1)
{
if ((errno == EAGAIN) ||(errno == EWOULDBLOCK)) {
// We have processed all incoming
// connections.
break;
} else {
perror ("accept");
break;
}
}
s = getnameinfo (&in_addr, in_len,
hbuf, sizeof hbuf,
sbuf, sizeof sbuf,
NI_NUMERICHOST | NI_NUMERICSERV);
if (s == 0) {
printf("Accepted connection on descriptor %d "
"(host=%s, port=%s)\n", infd, hbuf, sbuf);
}
// Make the incoming socket non-blocking and add it to the
// list of fds to monitor.
s = AibSocketNonBlocking (infd);
if (s == -1) {
abort ();
}
aibevent.data.fd = infd;
aibevent.events = EPOLLIN | EPOLLET;
s = epoll_ctl (efd, EPOLL_CTL_ADD, infd, &aibevent);
if (s == -1) {
perror ("epoll_ctl");
abort ();
}
}
continue;
} else {
// We have data on the fd waiting to be read. Read and
// display it. We must read whatever data is available
// completely, as we are running in edge-triggered mode
// and won't get a notification again for the same
// data.
int done = 0;
int sd;
SSL *ssl;
while (1)
{
ssize_t count;
char buf[1024];
char reply[1024];
printf("Performing exchange.\n");
ssl = SSL_new(ctx); /* get new SSL state with context */
SSL_set_fd(ssl, aibevents[i].data.fd); /* set connection socket to SSL state */
printf("Performing exchange 1.\n");
const char* HTMLecho="<html><body><pre>%s</pre></body></html>\n\n";
if ( SSL_accept(ssl) == FAIL ) { /* do SSL-protocol accept */
ERR_print_errors_fp(stderr);
printf("Performing exchange Error 1.\n");
done = 1;
break;
} else {
ShowCerts(ssl); /* get any certificates */
count = SSL_read(ssl, buf, sizeof(buf)); /* get request */
if ( count > 0 )
{
buf[count] = 0;
printf("Client msg: \"%s\"\n", buf);
sprintf(reply, HTMLecho, buf); /* construct reply */
SSL_write(ssl, reply, strlen(reply)); /* send reply */
} else {
ERR_print_errors_fp(stderr);
printf("Performing exchange Error 2.\n");
done = 1;
break;
}
}
sd = SSL_get_fd(ssl); /* get socket connection */
/*
count = read (aibevents[i].data.fd, buf, sizeof buf);
if (count == -1)
{
// If errno == EAGAIN, that means we have read all
// data. So go back to the main loop.
if (errno != EAGAIN)
{
perror ("read");
done = 1;
}
break;
}
else if (count == 0)
{
// End of file. The remote has closed the
// connection.
done = 1;
break;
}
// Write the buffer to standard output
s = write (1, buf, count);
if (s == -1)
{
perror ("write");
abort ();
}
printf(" read correctly (n > 0) n==%d\n",s);
printf("msg: %s\n", buf);
write(aibevents[i].data.fd, buf,s);
memset(buf,'\0', s);
*/
}
if (done)
{
printf("Freeing data.\n");
SSL_free(ssl); /* release SSL state */
close(sd); /* close connection */
//printf ("Closed connection on descriptor %d\n",
// aibevents[i].data.fd);
// Closing the descriptor will make epoll remove it
// from the set of descriptors which are monitored.
//close (aibevents[i].data.fd);
}
}
}
}
//free (aibevents);
delete[] aibevents;
close (sfd);
SSL_CTX_free(ctx); /* release context */
return EXIT_SUCCESS;
}
The single server- single client code are listed below for reference. Note that I am using the same client code to interact with the epoll server.
Single server code. Works
//SSL-Single Server code that works ! used as a reference for epoll server
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <resolv.h>
#include "openssl/ssl.h"
#include "openssl/err.h"
#define FAIL -1
int OpenListener(int port)
{ int sd;
struct sockaddr_in addr;
sd = socket(PF_INET, SOCK_STREAM, 0);
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
addr.sin_addr.s_addr = INADDR_ANY;
if ( bind(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0 )
{
perror("can't bind port");
abort();
}
if ( listen(sd, 10) != 0 )
{
perror("Can't configure listening port");
abort();
}
return sd;
}
SSL_CTX* InitServerCTX(void)
{ const SSL_METHOD *method;
SSL_CTX *ctx;
OpenSSL_add_all_algorithms(); /* load & register all cryptos, etc. */
SSL_load_error_strings(); /* load all error messages */
method = SSLv23_server_method(); /* create new server-method instance */
ctx = SSL_CTX_new(method); /* create new context from method */
if ( ctx == NULL )
{
ERR_print_errors_fp(stderr);
abort();
}
return ctx;
}
void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{
SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL);
SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);
/* set the local certificate from CertFile */
if ( SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* set the private key from KeyFile (may be the same as CertFile) */
if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* verify private key */
if ( !SSL_CTX_check_private_key(ctx) )
{
fprintf(stderr, "Private key does not match the public certificate\n");
abort();
}
}
void ShowCerts(SSL* ssl)
{ X509 *cert;
char *line;
cert = SSL_get_peer_certificate(ssl); /* Get certificates (if available) */
if ( cert != NULL )
{
printf("Server certificates:\n");
line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
printf("Subject: %s\n", line);
free(line);
line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
printf("Issuer: %s\n", line);
free(line);
X509_free(cert);
}
else
printf("No certificates.\n");
}
void Servlet(SSL* ssl) /* Serve the connection -- threadable */
{ char buf[1024];
char reply[1024];
int sd, bytes;
const char* HTMLecho="<html><body><pre>%s</pre></body></html>\n\n";
if ( SSL_accept(ssl) == FAIL ) /* do SSL-protocol accept */
ERR_print_errors_fp(stderr);
else
{
ShowCerts(ssl); /* get any certificates */
bytes = SSL_read(ssl, buf, sizeof(buf)); /* get request */
if ( bytes > 0 )
{
buf[bytes] = 0;
printf("Client msg: \"%s\"\n", buf);
sprintf(reply, HTMLecho, buf); /* construct reply */
SSL_write(ssl, reply, strlen(reply)); /* send reply */
}
else
ERR_print_errors_fp(stderr);
}
sd = SSL_get_fd(ssl); /* get socket connection */
SSL_free(ssl); /* release SSL state */
close(sd); /* close connection */
}
int main(int count, char *strings[])
{ SSL_CTX *ctx;
int server;
char *portnum;
if ( count != 2 )
{
printf("Usage: %s <portnum>\n", strings[0]);
exit(0);
}
SSL_library_init();
portnum = strings[1];
ctx = InitServerCTX(); /* initialize SSL */
LoadCertificates(ctx, "/home/test/ssl/cert_files/servercert.pem", "/home/test/ssl/cert_files/serverkey.pem"); /* load certs */
server = OpenListener(atoi(portnum)); /* create server socket */
while (1)
{ struct sockaddr_in addr;
socklen_t len = sizeof(addr);
SSL *ssl;
int client = accept(server, (struct sockaddr*)&addr, &len); /* accept connection as usual */
printf("Connection: %s:%d\n",inet_ntoa(addr.sin_addr), ntohs(addr.sin_port));
ssl = SSL_new(ctx); /* get new SSL state with context */
SSL_set_fd(ssl, client); /* set connection socket to SSL state */
Servlet(ssl); /* service connection */
}
close(server); /* close server socket */
SSL_CTX_free(ctx); /* release context */
}
Single client code Works
// Single Client Code that works!
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <sys/socket.h>
#include <resolv.h>
#include <netdb.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#define FAIL -1
//Added the LoadCertificates how in the server-side makes.
void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{
SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL);
SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);
/* set the local certificate from CertFile */
if ( SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* set the private key from KeyFile (may be the same as CertFile) */
if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* verify private key */
if ( !SSL_CTX_check_private_key(ctx) )
{
fprintf(stderr, "Private key does not match the public certificate\n");
abort();
}
}
int OpenConnection(const char *hostname, int port)
{ int sd;
struct hostent *host;
struct sockaddr_in addr;
if ( (host = gethostbyname(hostname)) == NULL )
{
perror(hostname);
abort();
}
sd = socket(PF_INET, SOCK_STREAM, 0);
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
addr.sin_addr.s_addr = *(long*)(host->h_addr);
if ( connect(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0 )
{
close(sd);
perror(hostname);
abort();
}
return sd;
}
SSL_CTX* InitCTX(void)
{ const SSL_METHOD *method;
SSL_CTX *ctx;
OpenSSL_add_all_algorithms(); /* Load cryptos, et.al. */
SSL_load_error_strings(); /* Bring in and register error messages */
method = SSLv23_client_method(); /* Create new client-method instance */
ctx = SSL_CTX_new(method); /* Create new context */
if ( ctx == NULL )
{
ERR_print_errors_fp(stderr);
abort();
}
return ctx;
}
void ShowCerts(SSL* ssl)
{ X509 *cert;
char *line;
cert = SSL_get_peer_certificate(ssl); /* get the server's certificate */
if ( cert != NULL )
{
printf("Server certificates:\n");
line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
printf("Subject: %s\n", line);
free(line); /* free the malloc'ed string */
line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
printf("Issuer: %s\n", line);
free(line); /* free the malloc'ed string */
X509_free(cert); /* free the malloc'ed certificate copy */
}
else
printf("No certificates.\n");
}
int main(int count, char *strings[])
{
SSL_CTX *ctx;
int server;
SSL *ssl;
char buf[1024];
int bytes;
if ( count != 3 )
{
printf("Usage: %s <host/ip> <portnum>\n", strings[0]);
exit(0);
}
char* hostname=strings[1];//"127.0.0.1";
char* portnum=strings[2];//"3334";
char CertFile[] = "/home/test/ssl/cert_files/clientcert.pem";
char KeyFile[] = "/home/test/ssl/cert_files/clientkey.pem";
SSL_library_init();
ctx = InitCTX();
LoadCertificates(ctx, CertFile, KeyFile);
server = OpenConnection(hostname, atoi(portnum));
ssl = SSL_new(ctx); /* create new SSL connection state */
SSL_set_fd(ssl, server); /* attach the socket descriptor */
if ( SSL_connect(ssl) == FAIL ) /* perform the connection */
ERR_print_errors_fp(stderr);
else
{ char *msg = "Hello???";
printf("Connected with %s encryption\n", SSL_get_cipher(ssl));
ShowCerts(ssl); /* get any certs */
SSL_write(ssl, msg, strlen(msg)); /* encrypt & send message */
bytes = SSL_read(ssl, buf, sizeof(buf)); /* get reply & decrypt */
buf[bytes] = 0;
printf("Received: \"%s\"\n", buf);
SSL_free(ssl); /* release connection state */
}
close(server); /* close socket */
SSL_CTX_free(ctx); /* release context */
return 0;
}
I did compilation using
g++ -g <src>.cc -o <src> -lssl -lcrypto for each of the three files (I had a Makefile)
The error that see is
./tcpserver_epoll 3334
sizeof 3334 argv[1] = 4
sizeof 3334 portt = 4
Accepted connection on descriptor 5 (host=127.0.0.1, port=44716)
Performing exchange.
Performing exchange 1.
Performing exchange Error 1.
Freeing data.
Performing exchange.
Performing exchange 1.
3072792824:error:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol:s23_srvr.c:628:
Performing exchange Error 1.
Freeing data.
I am wondering if I have to create multiple instances of SSL_CTX *ctx;. I am a bit lost. Any help is appreciated. Thanks.
Update: Thanks to everyone for guidance. I found an example here that closely matches what I want
http://bendecplusplus.googlecode.com/svn/trunk/ssl_mycode/epoll_ssl/server.c
http://bendecplusplus.googlecode.com/svn/trunk/ssl_mycode/epoll_ssl/client.c

Answer 1

Your code is very hard to read (how about proper indentation?) but I think the main problem is, that you consider a return of -1 from SSL_accept to be a fatal error.
According to the man page (which is true in this case) -1 can happen on fatal error or "It can also occur of action is need to continue the operation for non-blocking BIOs. Call SSL_get_error() with the return value ret to find out the reason.". So you have to check the error and if its SSL_ERROR_WANT_READ you have to wait the socket is readable and on SSL_ERROR_WANT_WRITE until its writable.

Answer 2

I am wondering if I have to create multiple instances of SSL_CTX *ctx;
No, you can use a single SSL_CTX*.
The SSL_CTX* is reference counted. It will be destroyed after the last SSL_free.
Since you are using a common SSL_CTX*, you should tune each SSL session with the non-context API calls. For example, SSL_set_cipher_list rather than SSL_CTX_set_cipher_list or SSL_use_certificate rather than SSL_CTX_use_certificate (unless, of course, its common).