Connection between DTLS server and client not working - c++

Question

I am developing a DTLS server and client. But there is no handshake between them.
PS I made a cut from the project in order to reduce the amount of code. If you forgot something - do not kick ...
a common part
static int dtls_verify_callback(int ok, X509_STORE_CTX* ctx) {
return ok;
}
static int generate_cookie(SSL* ssl, unsigned char* cookie, unsigned int* cookie_len) {
unsigned int i;
for (i = 0; i < COOKIE_LEN; i++, cookie++)
*cookie = i;
*cookie_len = COOKIE_LEN;
return 1;
}
static int verify_cookie(SSL* ssl,
#if OPENSSL_VERSION_NUMBER >= 0x10100000L
const
#endif
unsigned char* cookie,
unsigned int cookie_len) {
unsigned int i;
if (cookie_len != COOKIE_LEN)
return 0;
for (i = 0; i < COOKIE_LEN; i++, cookie++) {
if (*cookie != i)
return 0;
}
return 1;
}
Server side
int main() {
int iTimeout = 1000;
ADDRINFOW* pAddressInfo;
ADDRINFOW AddressHints;
WSADATA WinsockData;
LPCWSTR lpwszAddress = L"";
SOCKET socket;
union {
struct sockaddr_storage ss;
struct sockaddr_in s4;
struct sockaddr_in6 s6;
} server_addr, client_addr;
const TIMEVAL Timeout = { iTimeout / 1000, (iTimeout % 1000) * 1000 };
if (1 != OPENSSL_init_ssl(0
| OPENSSL_INIT_NO_LOAD_CONFIG
| OPENSSL_INIT_ASYNC
| OPENSSL_INIT_ENGINE_DYNAMIC
| OPENSSL_INIT_ENGINE_ALL_BUILTIN
| OPENSSL_INIT_NO_ATEXIT
| OPENSSL_INIT_LOAD_SSL_STRINGS
| OPENSSL_INIT_LOAD_CRYPTO_STRINGS
, nullptr))
{
return 1;
}
WSAStartup(MAKEWORD(2, 2), &WinsockData);
auto ctx = unique_ptr_c<SSL_CTX>(SSL_CTX_new(DTLS_server_method()), SSL_CTX_free);
auto cert_store = unique_ptr_c<X509_STORE>(X509_STORE_new(), X509_STORE_free);
X509_STORE_load_locations(cert_store.get(), "certs/ca-cert.pem", nullptr);
SSL_CTX_set1_verify_cert_store(ctx.get(), cert_store.get());
if (!SSL_CTX_use_certificate_chain_file(ctx.get(), "certs/server-cert.pem"))
{
ERR_print_errors_fp(stderr);
}
if (!SSL_CTX_use_PrivateKey_file(ctx.get(), "certs/server-key.pem", SSL_FILETYPE_PEM))
{
ERR_print_errors_fp(stderr);
}
if (!SSL_CTX_check_private_key(ctx.get()))
{
ERR_print_errors_fp(stderr);
}
SSL_CTX_set_verify(ctx.get(), SSL_VERIFY_PEER, dtls_verify_callback);
SSL_CTX_set_read_ahead(ctx.get(), 1);
SSL_CTX_set_cookie_generate_cb(ctx.get(), generate_cookie);
SSL_CTX_set_cookie_verify_cb(ctx.get(), verify_cookie);
memset(&AddressHints, 0, sizeof(AddressHints));
AddressHints.ai_flags = AI_PASSIVE | AI_NUMERICHOST;
AddressHints.ai_family = AF_UNSPEC;
AddressHints.ai_socktype = SOCK_DGRAM;
AddressHints.ai_protocol = 0;
iResult = GetAddrInfoW(lpwszAddress, 12345, &AddressHints, &pAddressInfo);
if (0 != iResult)
{
}
else
{
for (const ADDRINFOW* pCurrentAddressInfo = pAddressInfo; nullptr != pCurrentAddressInfo; pCurrentAddressInfo = pCurrentAddressInfo->ai_next)
{
socket = socket(pCurrentAddressInfo->ai_family, pCurrentAddressInfo->ai_socktype, pCurrentAddressInfo->ai_protocol);
if (INVALID_SOCKET == CurrentSocket) { continue; }
if (SOCKET_ERROR == bind(CurrentSocket, pCurrentAddressInfo->ai_addr, static_cast<int>(pCurrentAddressInfo->ai_addrlen)))
{
(void)closesocket(CurrentSocket);
continue;
}
if (SOCKET_ERROR == setsockopt(CurrentSocket, SOL_SOCKET, SO_SNDTIMEO, reinterpret_cast<const char*>(&iTimeout), sizeof(iTimeout))) { }
break;
}
}
for (;;)
{
if (SOCKET_ERROR == select(0, &socket, nullptr, nullptr, &Timeout)) {}
memset(&client_addr, 0, sizeof(struct sockaddr_storage));
auto bio = BIO_new_dgram(socket, BIO_NOCLOSE);
auto ssl = unique_ptr_c<SSL>(SSL_new(ctx.get()), SSL_close_and_free);
SSL_set_bio(ssl.get(), bio, bio);
SSL_set_options(ssl.get(), SSL_OP_COOKIE_EXCHANGE);
while (iResult <= 0) {
iResult = DTLSv1_listen(ssl.get(), (BIO_ADDR*)&client_addr);
if (iResult <= 0) { }
}
BIO_set_fd(SSL_get_rbio(ssl.get()), socket, BIO_NOCLOSE);
BIO_ctrl(SSL_get_rbio(ssl.get()), BIO_CTRL_DGRAM_SET_CONNECTED, 0, &client_addr);
iResult = SSL_accept(ssl.get());
if (iResult != 1) {
printf("%s\n", ERR_error_string(ERR_peek_last_error(), NULL));
printf("%s\n", ERR_error_string(ERR_get_error(), buf));
}
else
{
// I didn't cut the code further, since it never goes here
}
}
}
Client side
int main() {
ADDRINFOW AddressHints;
SOCKET ClientSocket = INVALID_SOCKET;
memset(&AddressHints, 0, sizeof(AddressHints));
AddressHints.ai_family = AF_UNSPEC;
AddressHints.ai_socktype = SOCK_DGRAM;
AddressHints.ai_protocol = 0;
int iResult = GetAddrInfoW(lpwszHost, wszPort, &AddressHints, &pAddressInfo);
if (0 != iResult)
{
}
else
{
unique_ptr_c<SSL_CTX> ctx;
unique_ptr_c<SSL> ssl;
BIO* bio;
if (1 != OPENSSL_init_ssl(0
| OPENSSL_INIT_NO_LOAD_CONFIG
| OPENSSL_INIT_ASYNC
| OPENSSL_INIT_ENGINE_DYNAMIC
| OPENSSL_INIT_ENGINE_ALL_BUILTIN
| OPENSSL_INIT_NO_ATEXIT
| OPENSSL_INIT_LOAD_SSL_STRINGS
| OPENSSL_INIT_LOAD_CRYPTO_STRINGS
, nullptr))
{
return 1;
}
ctx = unique_ptr_c<SSL_CTX>(SSL_CTX_new(DTLS_client_method()), SSL_CTX_free);
auto cert_store = unique_ptr_c<X509_STORE>(X509_STORE_new(), X509_STORE_free);
X509_STORE_load_locations(cert_store.get(), "certs/ca-cert.crt", nullptr);
SSL_CTX_set1_verify_cert_store(ctx.get(), cert_store.get());
iResult = SSL_CTX_use_certificate_chain_file(ctx.get(), "certs/client-cert.pem");
if (!iResult)
{
ERR_print_errors_fp(stderr);
}
iResult = SSL_CTX_use_PrivateKey_file(ctx.get(), "certs/client-key.pem", SSL_FILETYPE_PEM);
if (!iResult)
{
ERR_print_errors_fp(stderr);
}
if (!SSL_CTX_check_private_key(ctx.get()))
{
ERR_print_errors_fp(stderr);
}
SSL_CTX_set_verify(ctx.get(), SSL_VERIFY_PEER, dtls_verify_callback);
SSL_CTX_set_read_ahead(ctx.get(), 1);
SSL_CTX_set_cookie_generate_cb(ctx.get(), generate_cookie);
SSL_CTX_set_cookie_verify_cb(ctx.get(), verify_cookie);
ssl = unique_ptr_c<SSL>(SSL_new(ctx.get()), SSL_close_and_free);
for (pCurrentAddressInfo = pAddressInfo; nullptr != pCurrentAddressInfo; pCurrentAddressInfo = pCurrentAddressInfo->ai_next)
{
ClientSocket = socket(pCurrentAddressInfo->ai_family, pCurrentAddressInfo->ai_socktype, pCurrentAddressInfo->ai_protocol);
if (INVALID_SOCKET == ClientSocket)
{
continue;
}
iResult = connect(ClientSocket, pCurrentAddressInfo->ai_addr, static_cast<int>(pCurrentAddressInfo->ai_addrlen));
if (SOCKET_ERROR == iResult)
{
}
else
{
break;
}
if (SOCKET_ERROR == closesocket(ClientSocket))
{
}
}
if (nullptr != pCurrentAddressInfo)
{
if (SOCKET_ERROR == setsockopt(ClientSocket, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast<const char*>(&iTimeout), sizeof(iTimeout)))
{
iResult = WSAGetLastError();
}
else
{
bio = BIO_new_dgram(ClientSocket, BIO_NOCLOSE);
BIO_ctrl(bio, BIO_CTRL_DGRAM_SET_CONNECTED, 0, remote_address.get());
SSL_set_bio(ssl.get(), bio, bio);
iResult = SSL_connect(ssl.get()); // Trying 5 handshakes fails with an SSL_ERROR_SYSCALL error
if (iResult <= 0) {
char buf[BUFFER_SIZE];
//print_errors_stderr();
ERR_print_errors_fp(stderr);
printf("%s\n", ERR_error_string(ERR_peek_last_error(), NULL));
printf("%s\n", ERR_error_string(ERR_get_error(), buf));
switch (SSL_get_error(ssl.get(), iResult)) {
...
case SSL_ERROR_SYSCALL:
ERR_print_errors_fp(stderr);
DBG_MSG((DBG_ERR, L"DTLSClient::Send: SSL_connect failed with SSL_ERROR_SYSCALL"));
break;
...
}
fprintf(stderr, "%s\n", ERR_error_string(ERR_get_error(), NULL));
ERR_print_errors_fp(stderr);
}
else
{
// I didn't cut the code further, since it never goes here
}
}
// Close the socket
if (SOCKET_ERROR == closesocket(ClientSocket))
{
}
}
}
}
The client tries 5 times to make a handshake with the server but it doesn't work.
An error occurs from the client side on the line iResult = SSL_connect(ssl.get()) - an error occurs:
error:00000000:lib(0)::reason(0)
ERR: DTLSClient::Send: SSL_connect failed with SSL_ERROR_SYSCALL
An error occurs on the server at the line iResult = SSL_accept(ssl.get()) - with the error:
SSL_accept: No error
error:00000000:lib(0)::reason(0)
Based on traffic
enter image description here
Client sends - Client Hello
The server replies to this - Hello verify Request
And then the client tries to answer something back. Now I don't know what to do next. Are there any ideas?
I tried to do it as in the examples from OpenLL and other examples on github.