ConnectEx fails with WSAEISCONN after calling DisconnectEx with TF_REUSE_SOCKET - c++

Question

I have an IOCP based client for which I wanted to implement HTTP redirects in a following way:
1) When encountering a redirect to a different host call DisconnectEx with TF_REUSE_SOCKET
2) Await the overlapped completion and then call ConnectEx
However my code gets a WSAEISCONN return code upon calling ConnectEx even though if I check GetLastError() when the overlapped result of DisconnectEx is returned it gives 0.
An MVCE would be quite big in this case, but if there are no experience based suggestions I'll post one.
Update
I tried to make a MVCE but encountered different symptoms:
#define VC_EXTRALEAN
#define WIN32_LEAN_AND_MEAN
#include <WinSock2.h>
#include <MSWSock.h>
#include <Windows.h>
#include <Ws2tcpip.h>
#pragma comment(lib, "ws2_32.lib")
#include <stdexcept>
#include <iostream>
#include <sstream>
#include <map>
static inline std::string ErrorMessage(const char* pErrorMessage, ...)
{
std::string sFormattedMessage;
va_list VariableArgumentList;
va_start(VariableArgumentList, pErrorMessage);
sFormattedMessage.resize(_vscprintf(pErrorMessage, VariableArgumentList) + 1);
vsnprintf_s(const_cast<char*>(sFormattedMessage.c_str()), sFormattedMessage.size(), sFormattedMessage.size(), pErrorMessage, VariableArgumentList);
va_end(VariableArgumentList);
return sFormattedMessage;
}
#define CHECK(x, format, ...) { if ((x) == false) throw std::runtime_error(ErrorMessage("%s(%d): "format, __FILE__, __LINE__, __VA_ARGS__)); }
template<typename T>
bool LoadWinsockExtensionFunction(SOCKET Socket, GUID Guid, T* pFunction)
{
DWORD nBytesReturned = 0;
return WSAIoctl(Socket, SIO_GET_EXTENSION_FUNCTION_POINTER, &Guid, sizeof(Guid), pFunction, sizeof(T), &nBytesReturned, NULL, NULL) == 0 && nBytesReturned == sizeof(T);
}
int main(int argc, char** argv)
{
try
{
WORD nRequestedWinsockVersion = MAKEWORD(2, 2);
WSADATA WsaData;
CHECK(WSAStartup(nRequestedWinsockVersion, &WsaData) == 0, "WSAStartup failed");
auto hCompletionPort = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, NULL, 0);
CHECK(hCompletionPort != NULL, "CreateIoCompletionPort failed(%d)", GetLastError());
auto Socket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
CHECK(Socket != INVALID_SOCKET, "WSASocket failed(%d)", WSAGetLastError());
CHECK(CreateIoCompletionPort(reinterpret_cast<HANDLE>(Socket), hCompletionPort, NULL, 0), "CreateIoCompletionPort failed(%d)", GetLastError());
sockaddr_in LocalAddress;
ZeroMemory(&LocalAddress, sizeof(LocalAddress));
LocalAddress.sin_family = AF_INET;
LocalAddress.sin_addr.s_addr = INADDR_ANY;
LocalAddress.sin_port = 0;
CHECK(bind(Socket, reinterpret_cast<SOCKADDR*>(&LocalAddress), sizeof(LocalAddress)) == 0, "bind failed(%d)", WSAGetLastError());
LPFN_CONNECTEX pConnectEx = nullptr;
CHECK(LoadWinsockExtensionFunction(Socket, WSAID_CONNECTEX, &pConnectEx), "WSAIoctl failed to load ConnectEx(%d)", WSAGetLastError());
LPFN_DISCONNECTEX pDisconnectEx = nullptr;
CHECK(LoadWinsockExtensionFunction(Socket, WSAID_DISCONNECTEX, &pDisconnectEx), "WSAIoctl failed to load DisconnectEx(%d)", WSAGetLastError());
addrinfo Hint;
ZeroMemory(&Hint, sizeof(Hint));
Hint.ai_family = AF_INET;
Hint.ai_protocol = IPPROTO_TCP;
Hint.ai_socktype = SOCK_STREAM;
std::map<std::string, PADDRINFOA> Hosts;
// Scenarios:
// one host - failure to connect on the second try to the first host with 52
// two distinct hosts - failure to connect on the second try to the first host with 52
Hosts.emplace("www.google.com", nullptr);
Hosts.emplace("www.facebook.com", nullptr);
for (auto& Host : Hosts)
{
auto nGetAddressInfoResult = getaddrinfo(Host.first.c_str(), "http", &Hint, &Host.second);
CHECK(nGetAddressInfoResult == 0 && &Host.second, "getaddrinfo failed(%d)", nGetAddressInfoResult);
}
auto Host = Hosts.begin();
WSAOVERLAPPED Overlapped;
ZeroMemory(&Overlapped, sizeof(Overlapped));
while (true)
{
if ((*pConnectEx)(Socket, Host->second->ai_addr, Host->second->ai_addrlen, nullptr, 0, nullptr, &Overlapped) == FALSE)
{
auto nWSAError = WSAGetLastError();
CHECK(nWSAError == ERROR_IO_PENDING, "ConnectEx failed(%d)", nWSAError);
DWORD nBytesTransferred = 0;
ULONG_PTR pCompletionKey = 0;
LPOVERLAPPED pOverlapped = nullptr;
CHECK(GetQueuedCompletionStatus(hCompletionPort, &nBytesTransferred, &pCompletionKey, &pOverlapped, INFINITE), "overlapped operation failed(%d)", GetLastError());
}
CHECK(setsockopt(Socket, SOL_SOCKET, SO_UPDATE_CONNECT_CONTEXT, NULL, 0) == 0, "setsockopt failed(%d)", WSAGetLastError());
CHECK(shutdown(Socket, SD_BOTH) == 0, "shutdown failed(%d)", WSAGetLastError());
if ((*pDisconnectEx)(Socket, &Overlapped, TF_REUSE_SOCKET, 0) == FALSE)
{
auto nWSAError = WSAGetLastError();
CHECK(nWSAError == ERROR_IO_PENDING, "ConnectEx failed(%d)", nWSAError);
DWORD nBytesTransferred = 0;
ULONG_PTR pCompletionKey = 0;
LPOVERLAPPED pOverlapped = nullptr;
CHECK(GetQueuedCompletionStatus(hCompletionPort, &nBytesTransferred, &pCompletionKey, &pOverlapped, INFINITE), "overlapped operation failed(%d)", GetLastError());
}
if (++Host == Hosts.end())
{
Host = Hosts.begin();
}
}
closesocket(Socket);
CloseHandle(hCompletionPort);
for (auto& Host : Hosts)
{
freeaddrinfo(Host.second);
}
WSACleanup();
}
catch (std::exception& Exception)
{
OutputDebugStringA(Exception.what());
std::cout << Exception.what();
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
With this snippet what happens is that I get ERROR_DUP_NAME on the second attempt to connect to the same host (TIME_WAIT possibly?). I assume that if I could rebind the socket (but I can't since a second bind call fails with WSAEINVAL which is correct since the socket is already bound) this could even work fine.
What I have in my original code is a redirect from localhost to the actual address of the interface - maybe there simply is a path which in that case gives out WSAEISCONN instead of ERROR_DUP_NAME ? I can't post an MVCE for the original code since then another piece of code that would accept the connections is needed (maybe I'll make one).
I actually found that if DisconnectEx is called by the client then yes, ERROR_DUP_NAME happens because of TIME_WAIT (see this for a detailed analysis). So the bottom line is I should simply not try to reuse sockets in this scenario.