How to set a timeout for BIO_do_connect? - c++

I found a SSL/TLS client example here, it works well.
#include <stdio.h>
#include <errno.h>
#include <malloc.h>
#include <string.h>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <openssl/err.h>
#include <openssl/pkcs12.h>
#include <openssl/ssl.h>
#include <openssl/conf.h>
void connect(const char* host, int port) {
BIO* sbio, * out;
int len;
char tmpbuf[1024];
SSL_CTX* ctx;
SSL* ssl;
char server[200];
snprintf(server, sizeof(server), "%s:%d", host, port);
/* XXX Seed the PRNG if needed. */
ctx = SSL_CTX_new(TLS_client_method());
/* XXX Set verify paths and mode here. */
sbio = BIO_new_ssl_connect(ctx);
BIO_get_ssl(sbio, &ssl);
if (ssl == NULL) {
fprintf(stderr, "Can't locate SSL pointer\n");
ERR_print_errors_fp(stderr);
exit(1);
}
/* Don't want any retries */
SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
/* XXX We might want to do other things with ssl here */
/* An empty host part means the loopback address */
BIO_set_conn_hostname(sbio, server);
out = BIO_new_fp(stdout, BIO_NOCLOSE);
if (BIO_do_connect(sbio) <= 0) {
fprintf(stderr, "Error connecting to server\n");
ERR_print_errors_fp(stderr);
exit(1);
}
int ret = 0;
if ((ret = BIO_do_handshake(sbio)) <= 0) {
fprintf(stderr, "Error establishing SSL connection\n");
ERR_print_errors_fp(stderr);
exit(1);
}
/* XXX Could examine ssl here to get connection info */
BIO_puts(sbio, "Hi, this message is from client c++");
for (;;) {
len = BIO_read(sbio, tmpbuf, 1024);
if (len <= 0) {
break;
}
BIO_write(out, tmpbuf, len);
}
BIO_free_all(sbio);
BIO_free(out);
}
int main() {
connect("127.0.0.1", 5555);
}
but i need to set a timeout for this connection. then i found How to set connection timeout and operation timeout in OpenSSL.
so i change the codes
if (BIO_do_connect(sbio) <= 0) {
fprintf(stderr, "Error connecting to server\n");
ERR_print_errors_fp(stderr);
exit(1);
}
to
{
BIO_set_nbio(sbio, 1);
if (1 > BIO_do_connect(sbio)) {
if (!BIO_should_retry(sbio)) {
fprintf(stderr, "Error: should not retry\n");
ERR_print_errors_fp(stderr);
exit(1);
}
int fdSocket = 0;
if (BIO_get_fd(sbio, &fdSocket) < 0) {
fprintf(stderr, "Error: can not get socket\n");
ERR_print_errors_fp(stderr);
exit(1);
}
struct timeval timeout;
fd_set connectionfds;
FD_ZERO(&connectionfds);
FD_SET(fdSocket, &connectionfds);
timeout.tv_usec = 0;
timeout.tv_sec = 4;
if (0 == select(fdSocket + 1, NULL, &connectionfds, NULL, &timeout)) {
fprintf(stderr, "Error: timeout\n");
ERR_print_errors_fp(stderr);
exit(1);
}
}
}
now BIO_do_handshake returns -1 and the program exits.
How can i set a timeout correctly for my ssl connection?
Please give me some advice! help me!

I think you should set a timeout for handshake, not connection. in your code the connection has no problem because "select" returned non-zero value. in fact BIO_do_connect does handshake after connection is available. BIO_do_connect and BIO_do_handshake are the same in header file.
# define BIO_do_connect(b) BIO_do_handshake(b)
So i think this problem is handshake. eg. you connect to a server which uses a normal tcp socket without ssl. the server will not send "server_hallo" and certificate. then the client will wait for these "server_hallo" and certificate. BIO_do_handshake returns -1 if the handshake progress is still not finished.
maybe you can use BIO_set_ssl_renegotiate_timeout to set a timeout.

I would go about this in two steps:
I would deal with connection setup on my own. That way you can use non-blocking socket, connect(2) and select(2) and have complete control over timing of this part.
I would also implement by own BIO. You can use an existing BIO and only implement read, write and puts methods. This will allow you to control socket accesses.
With this in place you can have total control over how much time you spend. You can implement different timeouts for session setup, renegotiation, normal operation...

The problem with BIO_set_nbio is that you set I/O to non blocking mode. So you have to process further steps in non blocking mode.
I made an example how to process the request with sleep and non blocking mode. Maybe it is a bit ugly. But it worked for me.
#include <openssl/err.h>
#include <openssl/ssl.h>
#include <unistd.h>
#include <stdio.h>
void connect(const char* host, int port) {
const long timeout_nsec = 4 * (long)1000000000, dt_nsec = 100000;
char tmpbuf[1024];
char server[200];
snprintf(server, sizeof(server), "%s:%d", host, port);
struct timespec dt;
dt.tv_sec = 0;
dt.tv_nsec = dt_nsec;
/* XXX Seed the PRNG if needed. */
SSL_CTX *ctx = SSL_CTX_new(TLS_client_method());
/* XXX Set verify paths and mode here. */
BIO *sbio = BIO_new_ssl_connect(ctx);
SSL* ssl = nullptr;
BIO_get_ssl(sbio, &ssl);
if (ssl == NULL) {
fprintf(stderr, "Can't locate SSL pointer\n");
ERR_print_errors_fp(stderr);
exit(1);
}
/* Don't want any retries */
SSL_set_mode(ssl, SSL_MODE_AUTO_RETRY);
/* XXX We might want to do other things with ssl here */
/* An empty host part means the loopback address */
BIO_set_conn_hostname(sbio, server);
BIO *out = BIO_new_fp(stdout, BIO_NOCLOSE);
BIO_set_nbio(sbio, 1);
{
long time_remained = timeout_nsec;
while(1) {
int res = BIO_do_connect(sbio);
if (res <= 0 && BIO_should_retry(sbio)) {
clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &dt, NULL);
time_remained -= dt_nsec;
if (time_remained <= 0) {
fprintf(stderr, "Timeout\n");
exit(1);
}
continue;
}
if (res <= 0) {
fprintf(stderr, "BIO_do_connect error\n");
ERR_print_errors_fp(stderr);
exit(1);
}
break;
}
}
{
long time_remained = timeout_nsec;
while(1) {
int res = BIO_do_handshake(sbio);
if (res <= 0 && BIO_should_retry(sbio)) {
clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &dt, NULL);
time_remained -= dt_nsec;
if (time_remained <= 0) {
fprintf(stderr, "Timeout\n");
exit(1);
}
continue;
}
if (res <= 0) {
fprintf(stderr, "BIO_do_handshake error\n");
ERR_print_errors_fp(stderr);
exit(1);
}
break;
}
}
/* XXX Could examine ssl here to get connection info */
int a = BIO_puts(sbio, "Hi, this message is from client c++");
for (;;) {
int len = -1;
{
long time_remained = timeout_nsec;
while(1) {
len = BIO_read(sbio, tmpbuf, 1024);
if (len < 0 && BIO_should_retry(sbio)) {
clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, &dt, NULL);
time_remained -= dt_nsec;
if (time_remained <= 0) {
fprintf(stderr, "Timeout\n");
exit(1);
}
continue;
}
if (len < 0) {
fprintf(stderr, "BIO_read error\n");
ERR_print_errors_fp(stderr);
exit(1);
}
break;
}
}
if (len == 0) {
break;
}
BIO_write(out, tmpbuf, len);
}
BIO_free_all(sbio);
BIO_free(out);
}
int main() {
connect("127.0.0.1", 5555);
}

Related

How to do Unit Testing for the C++ service which performs IPC communication using Unix domain socket in linux?

I am developing an C++ service to achieve IPC using Unix domain socket in Linux.
How can I perform Unit Testing to test CreateSocket and RunServer class method all use case?
Can someone suggest a way to perform Unit Testing for such kind of IPC communication service ? I am new to Unit Testing.
I am not sure how to design class such a way that I can perform Unit Testing.
Need to Test following Functionality for Unit Testing:
CreateSocket - success
CreateSocket - Failed at unlink system call
CreateSocket - Failed at socket system call
CreateSocket - Failed at bind system call
CreateSocket - Failed at listen system call
RunServer - Success
RunServer - Failed at accept system call
RunServer - Failed at read system call
RunServer - Failed at write system call
RunServer - Failed at default switch case error case
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <errno.h>
#include <unistd.h>
#include <sys/select.h>
#include <sys/stat.h>
#define SERVER_SOCKET "/dev/ipcserver"
#define TOTAL_QUEUE_CONNECTION 10
enum message_type
{
REQUEST,
RESPONSE,
DICONNECT
};
struct message
{
message_type type;
char data [50];
};
class Server
{
private:
int server_socket_fd;
fd_set fds, readfds;
int fdmax;
struct message message;
public:
int CreateSocket(void);
int RunServer();
Server()
{
FD_ZERO (&fds);
FD_ZERO (&readfds);
}
~Server()
{
}
};
int Server::CreateSocket()
{
// create a unix domain socket,
struct stat statbuf;
if (stat (SERVER_SOCKET, &statbuf) == 0)
{
if (unlink (SERVER_SOCKET) == -1)
{
printf ("unlink failed \n");
return -1;
}
}
if ((server_socket_fd = socket (AF_UNIX, SOCK_SEQPACKET, 0)) == -1)
{
printf ("socket failed \n ");
return -1;
}
struct sockaddr_un socket_address;
memset (&socket_address, 0, sizeof (struct sockaddr_un));
socket_address.sun_family = AF_UNIX;
strncpy (socket_address.sun_path, SERVER_SOCKET, sizeof(socket_address.sun_path) - 1);
if (bind (server_socket_fd, (const struct sockaddr *) &socket_address, sizeof (struct sockaddr_un)) == -1)
{
printf ("bind failed \n");
return -1;
}
if (listen (server_socket_fd, TOTAL_QUEUE_CONNECTION) == -1)
{
printf ("listen failed \n");
return -1;
}
FD_SET (server_socket_fd, &fds);
fdmax = server_socket_fd;
return 0;
}
int Server::RunServer()
{
while (1)
{
readfds = fds;
// monitor readfds for readiness for reading
if (select (fdmax + 1, &readfds, NULL, NULL, NULL) == -1)
{
printf ("select failed \n");
return -1;
}
// Some sockets are ready. Examine readfds
for (int fd = 0; fd < (fdmax + 1); fd++)
{
if (FD_ISSET (fd, &readfds))
{ // fd is ready for reading
if (fd == server_socket_fd)
{ // request for new connection
int fd_new;
if ((fd_new = accept (server_socket_fd, NULL, NULL)) == -1)
{
printf ("accept failed \n");
}
FD_SET (fd_new, &fds);
if (fd_new > fdmax)
{
fdmax = fd_new;
}
}
else // data from an existing connection
{
memset (&message, '\0', sizeof (struct message));
ssize_t numbytes = read (fd, &message, sizeof (struct message));
if (numbytes == -1)
{
printf ("read failed \n");
}
else
{
printf ("Data received from client : \n");
switch (message.type)
{
case REQUEST:
{
memset (&message, '\0', sizeof (struct message));
char data[] = {0x01,0x02,0x03,0x04,0x05,0x06};
memcpy(message.data,data,sizeof(data));
message.type = RESPONSE;
if (write (fd, &message, sizeof (struct message)) == -1)
{
printf ("write error \n");
}
break;
}
case DICONNECT: // clear client connection from monitored fd set
{
FD_CLR(fd,&fds);
}
default:
{
printf ("Unexpected message from client\n");
break;
}
}
}
}
} // if
} // for
} // while (1)
return 0;
}
int main()
{
Server IPCServer;
int status = IPCServer.CreateSocket();
if(status == 0)
{
IPCServer.RunServer();
}
else
{
printf("IPCServer create fail\n");
return -1;
}
return 0;
}

Two pairs of TCP server/client sharing same code, but only one client receives messages

I have two pairs of simple TCP server/client, i.e., one client per server, running on Windows:
Servers run in a process (app).
Clients run in the other process.
Servers keep sending heartbeats (a string) to their paired client.
The first pair of server/client run their mainloops in their own threads.
Once the first server/client have shaken hands with the first heartbeat, the second pair of server/client start their mainloops in their own threads.
For this test, they run on the same machine with different ports: 2345 and 2346.
Now my problem
The first client receives its server's heartbeat.
The second client does NOT, although the second server sent out heartbeats without errors.
Here is the server code:
// hello_dualchannel_server.cpp : This file contains the 'main' function.
#include "pch.h"
#include <iostream>
#include <thread>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <spdlog/spdlog.h>
#define SPDLOG_WCHAR_TO_UTF8_SUPPORT
#ifdef _DEBUG
#if !defined(SPDLOG_ACTIVE_LEVEL)
#define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_TRACE
#endif // #if !defined(SPDLOG_ACTIVE_LEVEL)
#define SPDLOG_DEBUG_ON
#define SPDLOG_TRACE_ON
#define _trace SPDLOG_TRACE
#endif // #ifdef _DEBUG
using namespace spdlog;
SOCKET g_sockFirst = 0;
SOCKET g_sockClientFirst = 0;
std::thread g_threadFirst;
uint32_t g_timeLatestHeartBeatFirst = 0;
SOCKET g_sockSecond = 0;
SOCKET g_sockClientSecond = 0;
std::thread g_threadSecond;
uint32_t g_timeLatestHeartBeatSecond = 0;
void SetupLogger() {
#ifdef _DEBUG
spdlog::set_level(spdlog::level::trace);
spdlog::set_pattern("[%H:%M:%S%z][%^%L%$][%t:%s:%#] %v");
#else
spdlog::set_level(spdlog::level::info);
spdlog::set_pattern("[%H:%M:%S][%^%L%$][%t] %v");
#endif // #ifdef _DEBUG
}
int InitWinSock() {
WORD wVersionRequested;
WSADATA wsaData;
/* Use the MAKEWORD(lowbyte, highbyte) macro declared in Windef.h */
wVersionRequested = MAKEWORD(2, 2);
int err = WSAStartup(wVersionRequested, &wsaData);
if (err != 0) {
/* Tell the user that we could not find a usable */
/* Winsock DLL. */
printf("WSAStartup failed with error: %d\n", err);
return 1;
}
/* Confirm that the WinSock DLL supports 2.2.*/
/* Note that if the DLL supports versions greater */
/* than 2.2 in addition to 2.2, it will still return */
/* 2.2 in wVersion since that is the version we */
/* requested. */
if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2) {
/* Tell the user that we could not find a usable */
/* WinSock DLL. */
printf("Could not find a usable version of Winsock.dll\n");
WSACleanup();
return 1;
}
else
printf("The Winsock 2.2 dll was found okay\n");
return 0;
}
bool Init(int host_port, SOCKET* p_sockServer, SOCKET*p_sockClient) {
int err = 0;
int* p_int = 0;
std::string host_name("127.0.0.1");
struct sockaddr_in my_addr;
int addr_size = 0;
sockaddr_in sadr_client;
if (!*p_sockServer) {
*p_sockServer = socket(AF_INET, SOCK_STREAM, 0);
if (*p_sockServer == -1) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Server Error initializing socket: {}", log);
goto FINISH;
}
p_int = (int*)malloc(sizeof(int));
*p_int = 1;
if ((setsockopt(*p_sockServer, SOL_SOCKET, SO_REUSEADDR, (char*)p_int, sizeof(int)) == -1)
|| (setsockopt(*p_sockServer, SOL_SOCKET, SO_KEEPALIVE, (char*)p_int, sizeof(int)) == -1)) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Server Error setting options: {}", log);
free(p_int);
goto FINISH;
}
free(p_int);
info("Server socket is set up.");
my_addr.sin_family = AF_INET;
my_addr.sin_port = htons(host_port);
memset(&(my_addr.sin_zero), 0, 8);
my_addr.sin_addr.s_addr = INADDR_ANY;
if (bind(*p_sockServer, (sockaddr*)&my_addr, sizeof(my_addr)) == -1) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Server Error binding to socket, make sure nothing else is listening on this port: {}", log);
goto FINISH;
}
if (listen(*p_sockServer, 10) == -1) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Server Error listening: {}", log);
goto FINISH;
}
info("SUCCESS: Server socket listening ...");
}
info("Server accepting connection ...");
addr_size = sizeof(sockaddr_in);
char sAddress[MAX_PATH];
*p_sockClient = accept(*p_sockServer, (sockaddr*)&sadr_client, &addr_size);
if (*p_sockClient == INVALID_SOCKET) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Server error accepting client connection: {}", log);
// DO NOT close sockets here.
return false;
}
inet_ntop(sadr_client.sin_family, &sadr_client.sin_addr, sAddress, MAX_PATH);
g_timeLatestHeartBeatFirst = GetCurrentTime();
info("SUCCESS: Server accepted client connection.");
return true;
FINISH:
closesocket(*p_sockServer);
return false;
}
bool IsConnected(uint32_t timeLatestHeartBeat) {
// CAUTION: denser than client for sure catch
const unsigned long ConnTimeoutMs = 300;
auto cur = GetCurrentTime();
auto latest = timeLatestHeartBeat;
return cur - latest < ConnTimeoutMs;
}
bool StayInTouch(const char* name, SOCKET* pSockClient, uint32_t* pTimeLatestHeartBeat) {
if (IsConnected(*pTimeLatestHeartBeat))
return true;
char heartBeat[] = "biku";
int nBytesSent = 0;
int flags = 0;
int res = send(*pSockClient, heartBeat, sizeof(heartBeat), flags);
if (res == SOCKET_ERROR) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("{}: Server failed to send heartbeat: {}, Windows error: {}", name, log, GetLastError());
return false;
}
else if (res == 0) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("{}: Server sent zerobyte heartbeat: {}", name, log);
return false;
}
debug("{}: Heartbeat sent: {}", name, heartBeat);
*pTimeLatestHeartBeat = GetCurrentTime();
return true;
}
void Close(SOCKET* pSock) {
closesocket(*pSock);
*pSock = 0;
}
bool Connect() {
if (g_threadFirst.joinable()) {
warn("FirstTunnel already running. Skipped.");
return true;
}
g_threadFirst = std::thread([&]() {
bool isConnected = false;
while (true) {
while (!isConnected) {
isConnected = Init(2345, &g_sockFirst, &g_sockClientFirst);
}
isConnected = StayInTouch("FirstTunnel", &g_sockClientFirst, &g_timeLatestHeartBeatFirst);
if (!isConnected) {
// We don't close as client.
// We keep connecting
error("About to reconnect ...");
Sleep(1000);
continue;
}
if (!g_threadSecond.joinable()) {
g_threadSecond = std::thread([&]() {
while (true) {
while (!isConnected) {
isConnected = Init(2346, &g_sockSecond, &g_sockClientSecond);
}
isConnected = StayInTouch("SecondTunnel", &g_sockClientSecond, &g_timeLatestHeartBeatSecond);
if (!isConnected) {
// We don't close as client.
// We keep connecting
error("About to reconnect ...");
Sleep(1000);
continue;
}
}
info("SecondTunnel quitting...");
Close(&g_sockSecond);
});
}
}
info("FirstTunnel quitting...");
Close(&g_sockFirst);
});
while (true) {
//info("main thread ...");
Sleep(3000);
}
return g_threadFirst.joinable() ? true : false;
}
int main() {
SetupLogger();
info("Hello World!\n");
if (InitWinSock()) {
critical("Failed to initialize Window socket. Aborted.");
}
Connect();
if (g_threadSecond.joinable()) {
g_threadSecond.join();
}
if (g_threadFirst.joinable()) {
g_threadFirst.join();
}
WSACleanup();
info("Bye!");
}
Here is the client code
// hello_dualchannel_client.cpp : This file contains the 'main' function.
//
#include "pch.h"
#include <iostream>
#include <thread>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <spdlog/spdlog.h>
#define SPDLOG_WCHAR_TO_UTF8_SUPPORT
#ifdef _DEBUG
#if !defined(SPDLOG_ACTIVE_LEVEL)
#define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_TRACE
#endif // #if !defined(SPDLOG_ACTIVE_LEVEL)
#define SPDLOG_DEBUG_ON
#define SPDLOG_TRACE_ON
#define _trace SPDLOG_TRACE
#endif // #ifdef _DEBUG
using namespace spdlog;
SOCKET g_sockFirst = 0;
std::thread g_threadFirst;
uint32_t g_timeLatestHeartBeatFirst;
SOCKET g_sockSecond = 0;
std::thread g_threadSecond;
uint32_t g_timeLatestHeartBeatSecond;
void SetupLogger() {
#ifdef _DEBUG
spdlog::set_level(spdlog::level::trace);
spdlog::set_pattern("[%H:%M:%S%z][%^%L%$][%t:%s:%#] %v");
#else
spdlog::set_level(spdlog::level::info);
spdlog::set_pattern("[%H:%M:%S][%^%L%$][%t] %v");
#endif // #ifdef _DEBUG
}
int InitWinSock() {
WORD wVersionRequested;
WSADATA wsaData;
/* Use the MAKEWORD(lowbyte, highbyte) macro declared in Windef.h */
wVersionRequested = MAKEWORD(2, 2);
int err = WSAStartup(wVersionRequested, &wsaData);
if (err != 0) {
/* Tell the user that we could not find a usable */
/* Winsock DLL. */
printf("WSAStartup failed with error: %d\n", err);
return 1;
}
/* Confirm that the WinSock DLL supports 2.2.*/
/* Note that if the DLL supports versions greater */
/* than 2.2 in addition to 2.2, it will still return */
/* 2.2 in wVersion since that is the version we */
/* requested. */
if (LOBYTE(wsaData.wVersion) != 2 || HIBYTE(wsaData.wVersion) != 2) {
/* Tell the user that we could not find a usable */
/* WinSock DLL. */
printf("Could not find a usable version of Winsock.dll\n");
WSACleanup();
return 1;
}
else
printf("The Winsock 2.2 dll was found okay\n");
return 0;
}
bool Init(int host_port, SOCKET* p_sock) {
int err = 0;
int* p_int = 0;
std::string host_name("127.0.0.1");
struct sockaddr_in my_addr;
char handshake[] = "hello";
//int nBytesSent;
if (!*p_sock) {
*p_sock = socket(AF_INET, SOCK_STREAM, 0);
if (*p_sock == -1) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Client Error initializing socket {}", log);
goto FINISH;
}
p_int = (int*)malloc(sizeof(int));
*p_int = 1;
if ((setsockopt(*p_sock, SOL_SOCKET, SO_REUSEADDR, (char*)p_int, sizeof(int)) == -1)
|| (setsockopt(*p_sock, SOL_SOCKET, SO_KEEPALIVE, (char*)p_int, sizeof(int)) == -1)) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Client Error setting options {}", log);
free(p_int);
goto FINISH;
}
free(p_int);
info("SUCCESS: Client socket is set up.");
}
my_addr.sin_family = AF_INET;
my_addr.sin_port = htons(host_port);
memset(&(my_addr.sin_zero), 0, 8);
inet_pton(my_addr.sin_family, host_name.c_str(), &my_addr.sin_addr);
if (connect(*p_sock, (struct sockaddr*)&my_addr, sizeof(my_addr)) == SOCKET_ERROR) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Client Error connecting socket {}", log);
Sleep(1000);
goto FINISH;
}
/*nBytesSent = send(g_sockFirst, handshake, sizeof(handshake), 0);
if (nBytesSent <= 0) {
char log[MAX_PATH];
strerror_s(log, MAX_PATH, errno);
error("Client error sending handshake: {}", log);
goto FINISH;
}*/
g_timeLatestHeartBeatFirst = GetCurrentTime();
info("SUCCESS: Client connected to server.");
return true;
FINISH:
closesocket(*p_sock);
*p_sock = 0;
return false;
}
bool IsConnected(uint32_t timeLatestHeartBeat) {
const unsigned long ConnTimeoutMs = 3000;
auto cur = GetCurrentTime();
auto latest = timeLatestHeartBeat;
//if (cur - latest > ConnTimeoutMs)
//{
// debug("cur: {}, late: {}", cur, latest);
//}
return cur - latest < ConnTimeoutMs;
}
bool StayInTouch(const char* name, SOCKET* pSock, uint32_t* pTimeLatestHeartBeat) {
// Client checks inbox right away and measure timeout later.
char heartBeat[MAX_PATH] = { 0 };
// CAUTION: min 100ms required for receiving heartbeat.
//const uint32_t TimeoutMS = 100;
int flags = 0;
int nBytesRecved = recv(*pSock, heartBeat, sizeof(heartBeat), flags);
bool gotHeartbeat = nBytesRecved > 0;
if (gotHeartbeat) {
debug("{}: Heartbeat received: {}", name, heartBeat);
*pTimeLatestHeartBeat = GetCurrentTime();
}
return IsConnected(*pTimeLatestHeartBeat);
}
void Close(SOCKET* pSock) {
closesocket(*pSock);
*pSock = 0;
}
bool Connect() {
if (g_threadFirst.joinable()) {
warn("FirstTunnel already running. Skipped.");
return true;
}
g_threadFirst = std::thread([&]() {
bool isConnected = false;
while (true) {
while (!isConnected) {
isConnected = Init(2345, &g_sockFirst);
}
isConnected = StayInTouch("FirstTunnel", &g_sockFirst, &g_timeLatestHeartBeatFirst);
if (!isConnected) {
// We don't close as client.
// We keep connecting
Close(&g_sockFirst);
error("About to reconnect ...");
continue;
}
if (!g_threadSecond.joinable()) {
g_threadSecond = std::thread([&]() {
while (true) {
while (!isConnected) {
isConnected = Init(2346, &g_sockSecond);
}
isConnected = StayInTouch("SecondTunnel", &g_sockSecond, &g_timeLatestHeartBeatSecond);
if (!isConnected) {
// We don't close as client.
// We keep connecting
error("About to reconnect ...");
Sleep(1000);
continue;
}
}
info("SecondTunnel quitting...");
Close(&g_sockSecond);
});
}
}
info("FirstTunnel quitting.");
Close(&g_sockFirst);
});
while (true) {
//info("main thread ...");
Sleep(3000);
}
return g_threadFirst.joinable() ? true : false;
}
int main() {
SetupLogger();
info("Hello World!\n");
if (InitWinSock()) {
critical("Failed to initialize Window socket. Aborted.");
}
Connect();
if (g_threadSecond.joinable()) {
g_threadSecond.join();
}
if (g_threadFirst.joinable()) {
g_threadFirst.join();
}
WSACleanup();
info("Bye!");
}
The main connection logic is in the function Connect().
I'd appreciate tips on where I was wrong.
To run the code as is, you need one dependency spdlog
vcpkg install spdlog:x64-Windows
You can also replace all the spdlog-based logging code with your own.
UPDATE
Observation #1
I stepped into the code and confirmed
All the loops are running. So all the threads are spawned.
No redundant threads are spawned due to the joinable() guard.
The only failure point is the recv call of the second client.
So conclusion
No firewalls
No redundant threads
Both threads run on the client and server sides by design.
Observation #2
While running the server and client programs and having let the problem happen, I tried
Keep both programs open.
Run netcat (NMap's netcat port) like this C:\Apps\Nmap\ncat.exe 127.0.0.1 2346
This actually help fix things. It tells me that there is a connection problem, while I could definitely step into the client code and see that the connection is still there.
Observation #3
After I leave breakpoints in only the second connection code of the client program, and run the program, I could not step out into the first connection code. Note that they are in separate threads.
Leaving breakpoints in both connections, and quickly step over between them, I could keep the stepping going without getting trapped in only one thread. This is when I notice that the client starts receiving the messages it should.
So I suspect that there is a threading issue there. If that's the problem, How to fix this then?
I found the problem myself. It was a stupid bug on my part:
The two threads share a state by mistake: isConnected. So the original programs keep kicking each other after one of them gains a new connection or heartbeat. They should have used different states for that.
To give more detail: isConnected, which is initialized by g_threadFirst, falls through the closure of g_threadSecond's anonymous function.

epoll-based server fails (mutual authentication)

Based on the examples at http://simplestcodings.blogspot.com/2010/08/secure-server-client-using-openssl-in-c.html I have written the following epoll server code to replace the regular tcp sockets with SSL. My code is slightly modified to do mutual authentication and verify the certs since I add
SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL);
SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);.
My OS is Ubuntu 12.04.
I have also generated server cert, client cert, server key, client key, and of course cacert.pem files from the IBM tutorial http://pic.dhe.ibm.com/infocenter/lnxinfo/v3r0m0/index.jsp?topic=%2Fliaat%2Fliaatseccreatecskeycert.htm
I ran the single server - single client code and it runs perfectly fine. Inspired by this I wanted to see how the epoll server code would behave but then I ran into issues. Here is the code for the server
Code that does not work
//(c) 2014 enthusiasticgeek for stackoverflow - epollserver.cc
//============================================================================
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netdb.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/epoll.h>
#include <errno.h>
#include <iostream>
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <resolv.h>
#include "openssl/ssl.h"
#include "openssl/err.h"
#define FAIL -1
#define MAXEVENTS 64
SSL_CTX* InitServerCTX(void)
{ const SSL_METHOD *method;
SSL_CTX *ctx;
OpenSSL_add_all_algorithms(); /* load & register all cryptos, etc. */
SSL_load_error_strings(); /* load all error messages */
method = SSLv23_server_method(); /* create new server-method instance */
ctx = SSL_CTX_new(method); /* create new context from method */
if ( ctx == NULL )
{
ERR_print_errors_fp(stderr);
abort();
}
return ctx;
}
void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{
SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL);
SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);
/* set the local certificate from CertFile */
if ( SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* set the private key from KeyFile (may be the same as CertFile) */
if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* verify private key */
if ( !SSL_CTX_check_private_key(ctx) )
{
fprintf(stderr, "Private key does not match the public certificate\n");
abort();
}
}
void ShowCerts(SSL* ssl)
{ X509 *cert;
char *line;
cert = SSL_get_peer_certificate(ssl); /* Get certificates (if available) */
if ( cert != NULL )
{
printf("Server certificates:\n");
line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
printf("Subject: %s\n", line);
free(line);
line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
printf("Issuer: %s\n", line);
free(line);
X509_free(cert);
}
else
printf("No certificates.\n");
}
static int
AibSocketNonBlocking (int sfd)
{
int flags, s;
flags = fcntl (sfd, F_GETFL, 0);
if (flags == -1)
{
perror ("fcntl");
return -1;
}
flags |= O_NONBLOCK;
s = fcntl (sfd, F_SETFL, flags);
if (s == -1)
{
perror ("fcntl");
return -1;
}
return 0;
}
static int
AibCreateAndBind (char *port)
{
struct addrinfo hints;
struct addrinfo *result, *rp;
int s, sfd;
memset (&hints, 0, sizeof (struct addrinfo));
hints.ai_family = AF_UNSPEC; /* Return IPv4 and IPv6 choices */
hints.ai_socktype = SOCK_STREAM; /* We want a TCP socket */
hints.ai_flags = AI_PASSIVE; /* All interfaces */
s = getaddrinfo (NULL, port, &hints, &result);
if (s != 0)
{
fprintf (stderr, "getaddrinfo: %s\n", gai_strerror (s));
return -1;
}
for (rp = result; rp != NULL; rp = rp->ai_next)
{
sfd = socket (rp->ai_family, rp->ai_socktype, rp->ai_protocol);
if (sfd == -1)
continue;
s = bind (sfd, rp->ai_addr, rp->ai_addrlen);
if (s == 0)
{
/* We managed to bind successfully! */
break;
}
close (sfd);
}
if (rp == NULL)
{
fprintf (stderr, "Could not bind\n");
return -1;
}
freeaddrinfo (result);
return sfd;
}
int
main (int argc, char *argv[])
{
SSL_CTX *ctx;
SSL_library_init();
ctx = InitServerCTX(); /* initialize SSL */
LoadCertificates(ctx, "/home/test/ssl/cert_files/servercert.pem", "/home/test/ssl/cert_files/serverkey.pem"); /* load certs */
int sfd, s;
int efd;
struct epoll_event aibevent;
struct epoll_event *aibevents;
if (argc != 2) {
fprintf (stderr, "Usage: %s [port]\n", argv[0]);
exit (EXIT_FAILURE);
}
char portt[sizeof (unsigned int)];
snprintf(portt,sizeof portt + 1,"%u",atoi(argv[1]));
printf("sizeof %s argv[1] = %d\n",argv[1], sizeof(argv[1]));
printf("sizeof %s portt = %d\n",portt, sizeof(portt));
sfd = AibCreateAndBind (portt);//argv[1]);
if (sfd == -1) {
abort ();
}
s = AibSocketNonBlocking (sfd);
if (s == -1) {
abort ();
}
s = listen (sfd, SOMAXCONN);
if (s == -1) {
perror ("listen");
abort ();
}
efd = epoll_create1 (0);
if (efd == -1) {
perror ("epoll_create");
abort ();
}
aibevent.data.fd = sfd;
aibevent.events = EPOLLIN | EPOLLET;
s = epoll_ctl (efd, EPOLL_CTL_ADD, sfd, &aibevent);
if (s == -1) {
perror ("epoll_ctl");
abort ();
}
// Buffer where events are returned
//events = static_cast<epoll_event*>(calloc (MAXEVENTS, sizeof event));
//aibevents = static_cast<epoll_event*>(malloc (MAXEVENTS * sizeof aibevent));
aibevents = new epoll_event[MAXEVENTS * sizeof aibevent];
// The event loop
while (true)
{
int n, i;
n = epoll_wait (efd, aibevents, MAXEVENTS, -1);
for (i = 0; i < n; i++)
{
if ((aibevents[i].events & EPOLLERR) ||
(aibevents[i].events & EPOLLHUP) ||
(!(aibevents[i].events & EPOLLIN)))
{
// An error has occured on this fd, or the socket is not
// ready for reading (why were we notified then?)
fprintf (stderr, "epoll error\n");
close (aibevents[i].data.fd);
continue;
} else if (sfd == aibevents[i].data.fd) {
// We have a notification on the listening socket, which
// means one or more incoming connections.
while (1)
{
struct sockaddr in_addr;
socklen_t in_len;
int infd;
char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
in_len = sizeof in_addr;
infd = accept (sfd, &in_addr, &in_len);
if (infd == -1)
{
if ((errno == EAGAIN) ||(errno == EWOULDBLOCK)) {
// We have processed all incoming
// connections.
break;
} else {
perror ("accept");
break;
}
}
s = getnameinfo (&in_addr, in_len,
hbuf, sizeof hbuf,
sbuf, sizeof sbuf,
NI_NUMERICHOST | NI_NUMERICSERV);
if (s == 0) {
printf("Accepted connection on descriptor %d "
"(host=%s, port=%s)\n", infd, hbuf, sbuf);
}
// Make the incoming socket non-blocking and add it to the
// list of fds to monitor.
s = AibSocketNonBlocking (infd);
if (s == -1) {
abort ();
}
aibevent.data.fd = infd;
aibevent.events = EPOLLIN | EPOLLET;
s = epoll_ctl (efd, EPOLL_CTL_ADD, infd, &aibevent);
if (s == -1) {
perror ("epoll_ctl");
abort ();
}
}
continue;
} else {
// We have data on the fd waiting to be read. Read and
// display it. We must read whatever data is available
// completely, as we are running in edge-triggered mode
// and won't get a notification again for the same
// data.
int done = 0;
int sd;
SSL *ssl;
while (1)
{
ssize_t count;
char buf[1024];
char reply[1024];
printf("Performing exchange.\n");
ssl = SSL_new(ctx); /* get new SSL state with context */
SSL_set_fd(ssl, aibevents[i].data.fd); /* set connection socket to SSL state */
printf("Performing exchange 1.\n");
const char* HTMLecho="<html><body><pre>%s</pre></body></html>\n\n";
if ( SSL_accept(ssl) == FAIL ) { /* do SSL-protocol accept */
ERR_print_errors_fp(stderr);
printf("Performing exchange Error 1.\n");
done = 1;
break;
} else {
ShowCerts(ssl); /* get any certificates */
count = SSL_read(ssl, buf, sizeof(buf)); /* get request */
if ( count > 0 )
{
buf[count] = 0;
printf("Client msg: \"%s\"\n", buf);
sprintf(reply, HTMLecho, buf); /* construct reply */
SSL_write(ssl, reply, strlen(reply)); /* send reply */
} else {
ERR_print_errors_fp(stderr);
printf("Performing exchange Error 2.\n");
done = 1;
break;
}
}
sd = SSL_get_fd(ssl); /* get socket connection */
/*
count = read (aibevents[i].data.fd, buf, sizeof buf);
if (count == -1)
{
// If errno == EAGAIN, that means we have read all
// data. So go back to the main loop.
if (errno != EAGAIN)
{
perror ("read");
done = 1;
}
break;
}
else if (count == 0)
{
// End of file. The remote has closed the
// connection.
done = 1;
break;
}
// Write the buffer to standard output
s = write (1, buf, count);
if (s == -1)
{
perror ("write");
abort ();
}
printf(" read correctly (n > 0) n==%d\n",s);
printf("msg: %s\n", buf);
write(aibevents[i].data.fd, buf,s);
memset(buf,'\0', s);
*/
}
if (done)
{
printf("Freeing data.\n");
SSL_free(ssl); /* release SSL state */
close(sd); /* close connection */
//printf ("Closed connection on descriptor %d\n",
// aibevents[i].data.fd);
// Closing the descriptor will make epoll remove it
// from the set of descriptors which are monitored.
//close (aibevents[i].data.fd);
}
}
}
}
//free (aibevents);
delete[] aibevents;
close (sfd);
SSL_CTX_free(ctx); /* release context */
return EXIT_SUCCESS;
}
The single server- single client code are listed below for reference. Note that I am using the same client code to interact with the epoll server.
Single server code. Works
//SSL-Single Server code that works ! used as a reference for epoll server
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <resolv.h>
#include "openssl/ssl.h"
#include "openssl/err.h"
#define FAIL -1
int OpenListener(int port)
{ int sd;
struct sockaddr_in addr;
sd = socket(PF_INET, SOCK_STREAM, 0);
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
addr.sin_addr.s_addr = INADDR_ANY;
if ( bind(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0 )
{
perror("can't bind port");
abort();
}
if ( listen(sd, 10) != 0 )
{
perror("Can't configure listening port");
abort();
}
return sd;
}
SSL_CTX* InitServerCTX(void)
{ const SSL_METHOD *method;
SSL_CTX *ctx;
OpenSSL_add_all_algorithms(); /* load & register all cryptos, etc. */
SSL_load_error_strings(); /* load all error messages */
method = SSLv23_server_method(); /* create new server-method instance */
ctx = SSL_CTX_new(method); /* create new context from method */
if ( ctx == NULL )
{
ERR_print_errors_fp(stderr);
abort();
}
return ctx;
}
void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{
SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL);
SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);
/* set the local certificate from CertFile */
if ( SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* set the private key from KeyFile (may be the same as CertFile) */
if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* verify private key */
if ( !SSL_CTX_check_private_key(ctx) )
{
fprintf(stderr, "Private key does not match the public certificate\n");
abort();
}
}
void ShowCerts(SSL* ssl)
{ X509 *cert;
char *line;
cert = SSL_get_peer_certificate(ssl); /* Get certificates (if available) */
if ( cert != NULL )
{
printf("Server certificates:\n");
line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
printf("Subject: %s\n", line);
free(line);
line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
printf("Issuer: %s\n", line);
free(line);
X509_free(cert);
}
else
printf("No certificates.\n");
}
void Servlet(SSL* ssl) /* Serve the connection -- threadable */
{ char buf[1024];
char reply[1024];
int sd, bytes;
const char* HTMLecho="<html><body><pre>%s</pre></body></html>\n\n";
if ( SSL_accept(ssl) == FAIL ) /* do SSL-protocol accept */
ERR_print_errors_fp(stderr);
else
{
ShowCerts(ssl); /* get any certificates */
bytes = SSL_read(ssl, buf, sizeof(buf)); /* get request */
if ( bytes > 0 )
{
buf[bytes] = 0;
printf("Client msg: \"%s\"\n", buf);
sprintf(reply, HTMLecho, buf); /* construct reply */
SSL_write(ssl, reply, strlen(reply)); /* send reply */
}
else
ERR_print_errors_fp(stderr);
}
sd = SSL_get_fd(ssl); /* get socket connection */
SSL_free(ssl); /* release SSL state */
close(sd); /* close connection */
}
int main(int count, char *strings[])
{ SSL_CTX *ctx;
int server;
char *portnum;
if ( count != 2 )
{
printf("Usage: %s <portnum>\n", strings[0]);
exit(0);
}
SSL_library_init();
portnum = strings[1];
ctx = InitServerCTX(); /* initialize SSL */
LoadCertificates(ctx, "/home/test/ssl/cert_files/servercert.pem", "/home/test/ssl/cert_files/serverkey.pem"); /* load certs */
server = OpenListener(atoi(portnum)); /* create server socket */
while (1)
{ struct sockaddr_in addr;
socklen_t len = sizeof(addr);
SSL *ssl;
int client = accept(server, (struct sockaddr*)&addr, &len); /* accept connection as usual */
printf("Connection: %s:%d\n",inet_ntoa(addr.sin_addr), ntohs(addr.sin_port));
ssl = SSL_new(ctx); /* get new SSL state with context */
SSL_set_fd(ssl, client); /* set connection socket to SSL state */
Servlet(ssl); /* service connection */
}
close(server); /* close server socket */
SSL_CTX_free(ctx); /* release context */
}
Single client code Works
// Single Client Code that works!
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <sys/socket.h>
#include <resolv.h>
#include <netdb.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#define FAIL -1
//Added the LoadCertificates how in the server-side makes.
void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{
SSL_CTX_set_verify(ctx,SSL_VERIFY_PEER,NULL);
SSL_CTX_load_verify_locations(ctx,"/home/test/ssl/cert_files/cacert.pem",NULL);
/* set the local certificate from CertFile */
if ( SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* set the private key from KeyFile (may be the same as CertFile) */
if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* verify private key */
if ( !SSL_CTX_check_private_key(ctx) )
{
fprintf(stderr, "Private key does not match the public certificate\n");
abort();
}
}
int OpenConnection(const char *hostname, int port)
{ int sd;
struct hostent *host;
struct sockaddr_in addr;
if ( (host = gethostbyname(hostname)) == NULL )
{
perror(hostname);
abort();
}
sd = socket(PF_INET, SOCK_STREAM, 0);
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
addr.sin_addr.s_addr = *(long*)(host->h_addr);
if ( connect(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0 )
{
close(sd);
perror(hostname);
abort();
}
return sd;
}
SSL_CTX* InitCTX(void)
{ const SSL_METHOD *method;
SSL_CTX *ctx;
OpenSSL_add_all_algorithms(); /* Load cryptos, et.al. */
SSL_load_error_strings(); /* Bring in and register error messages */
method = SSLv23_client_method(); /* Create new client-method instance */
ctx = SSL_CTX_new(method); /* Create new context */
if ( ctx == NULL )
{
ERR_print_errors_fp(stderr);
abort();
}
return ctx;
}
void ShowCerts(SSL* ssl)
{ X509 *cert;
char *line;
cert = SSL_get_peer_certificate(ssl); /* get the server's certificate */
if ( cert != NULL )
{
printf("Server certificates:\n");
line = X509_NAME_oneline(X509_get_subject_name(cert), 0, 0);
printf("Subject: %s\n", line);
free(line); /* free the malloc'ed string */
line = X509_NAME_oneline(X509_get_issuer_name(cert), 0, 0);
printf("Issuer: %s\n", line);
free(line); /* free the malloc'ed string */
X509_free(cert); /* free the malloc'ed certificate copy */
}
else
printf("No certificates.\n");
}
int main(int count, char *strings[])
{
SSL_CTX *ctx;
int server;
SSL *ssl;
char buf[1024];
int bytes;
if ( count != 3 )
{
printf("Usage: %s <host/ip> <portnum>\n", strings[0]);
exit(0);
}
char* hostname=strings[1];//"127.0.0.1";
char* portnum=strings[2];//"3334";
char CertFile[] = "/home/test/ssl/cert_files/clientcert.pem";
char KeyFile[] = "/home/test/ssl/cert_files/clientkey.pem";
SSL_library_init();
ctx = InitCTX();
LoadCertificates(ctx, CertFile, KeyFile);
server = OpenConnection(hostname, atoi(portnum));
ssl = SSL_new(ctx); /* create new SSL connection state */
SSL_set_fd(ssl, server); /* attach the socket descriptor */
if ( SSL_connect(ssl) == FAIL ) /* perform the connection */
ERR_print_errors_fp(stderr);
else
{ char *msg = "Hello???";
printf("Connected with %s encryption\n", SSL_get_cipher(ssl));
ShowCerts(ssl); /* get any certs */
SSL_write(ssl, msg, strlen(msg)); /* encrypt & send message */
bytes = SSL_read(ssl, buf, sizeof(buf)); /* get reply & decrypt */
buf[bytes] = 0;
printf("Received: \"%s\"\n", buf);
SSL_free(ssl); /* release connection state */
}
close(server); /* close socket */
SSL_CTX_free(ctx); /* release context */
return 0;
}
I did compilation using
g++ -g <src>.cc -o <src> -lssl -lcrypto for each of the three files (I had a Makefile)
The error that see is
./tcpserver_epoll 3334
sizeof 3334 argv[1] = 4
sizeof 3334 portt = 4
Accepted connection on descriptor 5 (host=127.0.0.1, port=44716)
Performing exchange.
Performing exchange 1.
Performing exchange Error 1.
Freeing data.
Performing exchange.
Performing exchange 1.
3072792824:error:140760FC:SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol:s23_srvr.c:628:
Performing exchange Error 1.
Freeing data.
I am wondering if I have to create multiple instances of SSL_CTX *ctx;. I am a bit lost. Any help is appreciated. Thanks.
Update: Thanks to everyone for guidance. I found an example here that closely matches what I want
http://bendecplusplus.googlecode.com/svn/trunk/ssl_mycode/epoll_ssl/server.c
http://bendecplusplus.googlecode.com/svn/trunk/ssl_mycode/epoll_ssl/client.c
Your code is very hard to read (how about proper indentation?) but I think the main problem is, that you consider a return of -1 from SSL_accept to be a fatal error.
According to the man page (which is true in this case) -1 can happen on fatal error or "It can also occur of action is need to continue the operation for non-blocking BIOs. Call SSL_get_error() with the return value ret to find out the reason.". So you have to check the error and if its SSL_ERROR_WANT_READ you have to wait the socket is readable and on SSL_ERROR_WANT_WRITE until its writable.
I am wondering if I have to create multiple instances of SSL_CTX *ctx;
No, you can use a single SSL_CTX*.
The SSL_CTX* is reference counted. It will be destroyed after the last SSL_free.
Since you are using a common SSL_CTX*, you should tune each SSL session with the non-context API calls. For example, SSL_set_cipher_list rather than SSL_CTX_set_cipher_list or SSL_use_certificate rather than SSL_CTX_use_certificate (unless, of course, its common).

Server and client in Python and C

I've wrote a simple client code in python, and I'm trying to connect to a simple echo server written in C.
I know it shouldn't matter, but for some reason I did manage to connect to a server written in python, but I cannot connect to the C server.
Here's the code of the client:
import socket
import sys
import time
HOST = 'localhost'
PORT = 11000
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
msg = raw_input()
s.send(msg)
data = s.recv(len(msg))
s.close()
print 'Received: ', data
And here's the C code of the echo server:
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <netinet/in.h>
#include <string.h>
#include <netdb.h>
#ifndef AF_INET
#define AF_INET 2
#endif
#ifndef SOCK_DGRAM
#define SOCK_DGRAM 2
#endif
#ifndef INADDR_ANY
#define INADDR_ANY 0
#endif
#ifndef IP_DONTFRAG
#define IP_DONTFRAG 67
#endif
#define BUFFER_SIZE 1024
#define ECHO_PORT_UDP 10000
#define ECHO_PORT_TCP 11000
int main(int argc, char *argv[]) {
int echo_socket = 0;
int echo_socket_child = 0; // for TCP
struct sockaddr_in server;
struct sockaddr_in client;
struct hostent *hostp; // client host info
struct sockaddr_in clientaddr; // client addr
char *hostaddrp; // dotted decimal host addr string
char buffer[BUFFER_SIZE];
unsigned int clientlen = 0;
unsigned int serverlen = 0;
int received = 0;
int port = 0;
char *endptr;
int optval = 1;
int msg_byte_size = 0;
// Parameters check
if (argc == 2) {
port = strtol(argv[1], &endptr, 0);
if ((*endptr) || ((port != ECHO_PORT_UDP) && (port != ECHO_PORT_TCP))) {
printf("EchoServer: Invalid port number.\n Use port %d for UDP, port %d for TCP.\n", ECHO_PORT_UDP, ECHO_PORT_TCP);
return -1;
}
else {
if (port == ECHO_PORT_UDP) {
printf("EchoServer: Running UDP on port %d.\n", port);
}
if (port == ECHO_PORT_TCP) {
printf("EchoServer: Running TCP on port %d.\n", port);
}
}
}
else {
printf("EchoServer: Invalid arguments.\n");
return -1;
}
// Opening UDP socket
if (port == ECHO_PORT_UDP) {
if ((echo_socket = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
printf("EchoServer: Failed opening socket");
return -1;
}
}
if (port == ECHO_PORT_TCP) {
if ((echo_socket = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
printf("EchoServer: Failed opening socket");
return -1;
}
// setsockopt: Handy debugging trick that lets us rerun the server immediately after we kill it; otherwise we have to wait about 20 secs.
// Eliminates "ERROR on binding: Address already in use" error.
setsockopt(echo_socket, SOL_SOCKET, SO_REUSEADDR,(const void *)&optval , sizeof(int));
}
// Construct the server sockaddr_in structure
memset(&server, 0, sizeof(server)); /* Clear struct */
server.sin_family = AF_INET; /* Internet/IP */
server.sin_addr.s_addr = htonl(INADDR_ANY); /* Any IP address */
server.sin_port = htons(atol(argv[1])); /* server port */
// Bind the socket
serverlen = sizeof(server);
if (bind(echo_socket, (struct sockaddr *) &server, serverlen) < 0) {
printf("EchoServer: Failed binding socket");
return -1;
}
// Wait for a datagram until cancelled
if (port == ECHO_PORT_UDP) {
while (1) {
/* Receive a message from the client */
clientlen = sizeof(client);
if ((received = recvfrom(echo_socket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&client, &clientlen)) < 0) {
printf("EchoServer: Failed receiving datagram");
return -1;
}
printf("Client datagram received from: %s\n", inet_ntoa(client.sin_addr));
/* Send the message back to client */
if (sendto(echo_socket, buffer, received, 0, (struct sockaddr *) &client, sizeof(client)) != received) {
printf("Mismatch in number of echoed bytes");
return -1;
}
}
}
// Wait for a connection until cancelled
if (port == ECHO_PORT_TCP) {
while (1) {
echo_socket_child = accept(echo_socket, (struct sockaddr *) &client, &clientlen);
if (echo_socket_child < 0) {
printf("ERROR on accept");
break;
}
// gethostbyaddr: determine who sent the message
hostp = gethostbyaddr((const char *)&clientaddr.sin_addr.s_addr, sizeof(clientaddr.sin_addr.s_addr), AF_INET);
if (hostp == NULL) {
printf("ERROR on gethostbyaddr");
break;
}
hostaddrp = inet_ntoa(clientaddr.sin_addr);
if (hostaddrp == NULL) {
printf("ERROR on inet_ntoa\n");
break;
}
printf("server established connection with %s \n", hostaddrp);
// read: read input string from the client
bzero(buffer, BUFFER_SIZE);
msg_byte_size = read(echo_socket_child, buffer, BUFFER_SIZE);
if (msg_byte_size < 0) {
printf("ERROR reading from socket");
break;
}
printf("server received %d bytes: %s", msg_byte_size, buffer);
// write: echo the input string back to the client
msg_byte_size = write(echo_socket_child, buffer, strlen(buffer));
if (msg_byte_size < 0) {
printf("ERROR writing to socket");
break;
}
} // endof while(1)
close(echo_socket_child);
return -1;
}
return 0;
}
Any ideas why I fail to connect to the server?
edit:
this is the error I receive:
Traceback (most recent call last):
File "s.py", line 8, in <module>
s.connect((HOST, PORT))
File "C:\Python27\lib\socket.py", line 224, in meth
return getattr(self._sock,name)(*args)
socket.error: [Errno 10061]
(1) Add a listen call to the TCP section of the code.
(2) You have to tell accept what the length of the sockaddr you are passing it is and it will in return tell you the length of the address of the client it returned. You were passing it as 0 length so naturally it didn't pass back a client address which subsequently makes your gethostbyaddr fail with unknown address.
(3) If you don't close the client socket within the loop it just remains open (and leaks a file descriptor) for the duration of the server's life. Eventually you will run out of FDs. It doesn't effect your client which just closes after the receipt of one msg but any client who writes more than one message will never have it received by the server and will never receive an eof from the server.
if (port == ECHO_PORT_TCP)
{
if (listen(echo_socket, ECHO_PORT_TCP) == -1)
{
perror("listen");
exit(1);
}
while (1)
{
clientlen = sizeof(client);
echo_socket_child = accept(echo_socket, (struct sockaddr *) &client, &clientlen);
if (echo_socket_child < 0)
{
perror("accept");
break;
}
// gethostbyaddr: determine who sent the message
hostp = gethostbyaddr((const char *) &client.sin_addr.s_addr, sizeof(client.sin_addr.s_addr), AF_INET);
if (hostp == NULL)
{ herror("byaddr");
break;
}
hostaddrp = inet_ntoa(client.sin_addr);
if (hostaddrp == NULL)
{
printf("ERROR on inet_ntoa\n");
break;
}
printf("server established connection with %s (%s)\n", hostp->h_name, hostaddrp);
bzero(buffer, BUFFER_SIZE);
msg_byte_size = read(echo_socket_child, buffer, BUFFER_SIZE);
if (msg_byte_size < 0)
{
printf("ERROR reading from socket");
break;
}
printf("server received %d bytes: %s", msg_byte_size, buffer);
msg_byte_size = write(echo_socket_child, buffer, strlen(buffer));
if (msg_byte_size < 0)
{
printf("ERROR writing to socket");
break;
}
close(echo_socket_child);
} // endof while(1)
return -1;
}

OpenSSL, RSA and Winsock and C++

I found this source floating about the net and I was hoping someone could solve why this program simply shuts down instead of listening for a connection. This source was meant to open a server socket but, doesn't.
#include <openssl/bio.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <iostream>
#include <stdio.h>
#include <winsock2.h>
#define PASSWORD "passme"
int main(int argc, char** argv)
{
CRYPTO_malloc_init(); // Initialize malloc, free, etc for OpenSSL's use
SSL_library_init(); // Initialize OpenSSL's SSL libraries
SSL_load_error_strings(); // Load SSL error strings
ERR_load_BIO_strings(); // Load BIO error strings
OpenSSL_add_all_algorithms(); // Load all available encryption algorithms
return 0;
}
void serverThread()
{
// First, we need to initialize Winsock.
WSADATA wsadata;
int ret = WSAStartup(0x101, &wsadata);
if (ret != 0) {
printf("WSAStartup() failed with: %d!\n", GetLastError());
return;
}
// Next we need to create a server socket.
SOCKET server = socket(AF_INET, SOCK_STREAM, 0);
sockaddr_in sockaddrin;
// Internet socket
sockaddrin.sin_family = AF_INET;
// Accept any IP
sockaddrin.sin_addr.s_addr = INADDR_ANY;
// Use port 6789
sockaddrin.sin_port = htons(6789);
// Valid socket?
if (server == INVALID_SOCKET) {
printf("Error creating server socket!");
return;
}
// Now bind to the port
ret = bind(server, (sockaddr*) &(sockaddrin), sizeof(sockaddrin));
if (ret != 0) {
printf("Error binding to port!\n");
return;
}
// Start listening for connections
// Second param is max number of connections
ret = listen(server, 50);
if (ret != 0) {
printf("Error listening for connections!\n");
return;
}
// Set up to accept connections
SOCKET client;
sockaddr_in clientsockaddrin;
int len = sizeof(clientsockaddrin);
printf("Server ready to accept connections!\n");
while (1) {
// Block until a connection is ready
client = accept(server, (sockaddr*) &clientsockaddrin, &len);
printf("Connection recieved from %s!\n", inet_ntoa(clientsockaddrin.sin_addr));
// Notice that we use server_method instead of client_method
SSL_CTX* ctx = SSL_CTX_new(SSLv23_server_method());
BIO* bio = BIO_new_file("dh1024.pem", "r");
// Did we get a handle to the file?
if (bio == NULL) {
printf("Couldn't open DH param file!\n");
break;
}
// Read in the DH params.
DH* ret = PEM_read_bio_DHparams(bio, NULL, NULL, NULL);
// Free up the BIO object.
BIO_free(bio);
// Set up our SSL_CTX to use the DH parameters.
if (SSL_CTX_set_tmp_dh(ctx, ret) < 0) {
printf("Couldn't set DH parameters!\n");
break;
}
// Now we need to generate a RSA key for use.
// 1024-bit key. If you want to use something stronger, go ahead but it must be a power of 2. Upper limit should be 4096.
RSA* rsa = RSA_generate_key(1024, RSA_F4, NULL, NULL);
// Set up our SSL_CTX to use the generated RSA key.
if (!SSL_CTX_set_tmp_rsa(ctx, rsa)) {
printf("Couldn't set RSA key!\n");
// We don't break out here because it's not a requirement for the RSA key to be set. It does help to have it.
}
// Free up the RSA structure.
RSA_free(rsa);
SSL_CTX_set_cipher_list(ctx, "ALL");
// Set up our SSL object as before
SSL* ssl = SSL_new(ctx);
// Set up our BIO object to use the client socket
BIO* sslclient = BIO_new_socket(client, BIO_NOCLOSE);
// Set up our SSL object to use the BIO.
SSL_set_bio(ssl, sslclient, sslclient);
// Do SSL handshaking.
int r = SSL_accept(ssl);
// Something failed. Print out all the error information, since all of it may be relevant to the problem.
if (r != 1) {
printf("SSL_accept() returned %d\n", r);
printf("Error in SSL_accept(): %d\n", SSL_get_error(ssl, r));
char error[65535];
ERR_error_string_n(ERR_get_error(), error, 65535);
printf("Error: %s\n\n", error);
ERR_print_errors(sslclient);
int err = WSAGetLastError();
printf("WSA: %d\n", err);
break;
}
}
}
int password_callback(char* buffer, int num, int rwflag, void* userdata)
{
if (num < (strlen(PASSWORD) + 1)) {
return(0);
}
strcpy(buffer, PASSWORD);
return strlen(PASSWORD);
}
int verify_callback(int ok, X509_STORE_CTX* store)
{
char data[255];
if (!ok) {
X509* cert = X509_STORE_CTX_get_current_cert(store);
int depth = X509_STORE_CTX_get_error_depth(store);
int err = X509_STORE_CTX_get_error(store);
printf("Error with certificate at depth: %d!\n", depth);
X509_NAME_oneline(X509_get_issuer_name(cert), data, 255);
printf("\tIssuer: %s\n", data);
X509_NAME_oneline(X509_get_subject_name(cert), data, 255);
printf("\tSubject: %s\n", data);
printf("\tError %d: %s\n", err, X509_verify_cert_error_string(err));
}
return ok;
}
You never start the ServerThread in main