C++ low throughput winsock TCP test application - c++

Question

I'm trying to build a fast server & client which work in localhost. The idea is to send data blobs from another program, and be quick about it. only one client connects to the server at a time.
I first tried implementing this using boost::asio library. everything worked fine, except that the throughput was abysmally slow, 415megabytes/s.
Then I proceeded to create a testcase with winsock, it too had very similar throughput, 434megabytes/s. abysmally slow.
I was expecting more of at range of 40Gigabytes or at least several gigabytes per second.
Id appreaciate any suggestions, as I am way out of my element, with networks programming.
My current client function:
bool sendDataWin(size_t size, const size_t blocksize, size_t port)
{
int result;
struct addrinfo *addressinfo = nullptr, hints;
auto sport = std::to_string(port);
ZeroMemory(&hints, sizeof(hints));
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
// Resolve the local address and port to be used by the server
result = getaddrinfo("localhost", sport.c_str(), &hints, &addressinfo);
if (result != 0) {
printf("Error at client socket(): %ld\n", WSAGetLastError());
return false;
}
SOCKET connection = socket(
addressinfo->ai_family,
addressinfo->ai_socktype,
addressinfo->ai_protocol);
if (connection == INVALID_SOCKET) {
printf("Error at client socket(): %ld\n", WSAGetLastError());
freeaddrinfo(addressinfo);
return false;
}
// Try to put loopback fast path on.
bool value;
DWORD dwBytesRet;
int status =
WSAIoctl(
connection,
SIO_LOOPBACK_FAST_PATH,
&value,
sizeof(bool),
NULL,
0,
&dwBytesRet,
0,
0);
if (status == SOCKET_ERROR) {
DWORD LastError = ::GetLastError();
if (LastError == WSAEOPNOTSUPP) {
printf("client call is not supported :: SIO_LOOPBACK_FAST_PATH\n");
}
}
// Connect to server.
result = connect(connection, addressinfo->ai_addr, (int)addressinfo->ai_addrlen);
if (result == SOCKET_ERROR) {
printf("Error at client socket(): %ld\n", WSAGetLastError());
closesocket(connection);
return false;
}
freeaddrinfo(addressinfo);
std::vector<uint8_t> buffer;
buffer.resize(blocksize);
size_t total = 0;
do {
size_t sendSize = blocksize;
size_t next = total + sendSize;
if (next > size)
{
sendSize -= next - size;
}
// Echo the buffer back to the sender
result = send(connection, (const char*)buffer.data(), (int)sendSize, 0);
if (result == SOCKET_ERROR) {
printf("client send failed: %d\n", WSAGetLastError());
closesocket(connection);
return false;
}
total += sendSize;
} while (total < size);
result = shutdown(connection, SD_SEND);
if (result == SOCKET_ERROR) {
printf("client shutdown failed: %d\n", WSAGetLastError());
closesocket(connection);
return false;
}
// cleanup
closesocket(connection);
return true;
}
server function:
bool serverReceiveDataWin(size_t size, const size_t blocksize, size_t port)
{
int result;
struct addrinfo *addressinfo = nullptr, *ptr = nullptr, hints;
auto sport = std::to_string(port);
ZeroMemory(&hints, sizeof(hints));
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
hints.ai_flags = AI_PASSIVE;
// Resolve the local address and port to be used by the server
result = getaddrinfo(nullptr, sport.c_str(), &hints, &addressinfo);
if (result != 0) {
printf("Error at server socket(): %ld\n", WSAGetLastError());
return false;
}
SOCKET ListenSocket = socket(addressinfo->ai_family, addressinfo->ai_socktype, addressinfo->ai_protocol);
if (ListenSocket == INVALID_SOCKET) {
printf("Error at server socket(): %ld\n", WSAGetLastError());
freeaddrinfo(addressinfo);
return false;
}
// Try to put loopback fast path on.
bool value;
DWORD dwBytesRet;
int status =
WSAIoctl(
ListenSocket,
SIO_LOOPBACK_FAST_PATH,
&value,
sizeof(bool),
NULL,
0,
&dwBytesRet,
0,
0);
if (status == SOCKET_ERROR) {
DWORD LastError = ::GetLastError();
if (LastError == WSAEOPNOTSUPP) {
printf("server call is not supported :: SIO_LOOPBACK_FAST_PATH\n");
}
}
// Setup the TCP listening socket
result = bind(ListenSocket, addressinfo->ai_addr, (int)addressinfo->ai_addrlen);
if (result == SOCKET_ERROR) {
printf("server bind failed with error: %d\n", WSAGetLastError());
freeaddrinfo(addressinfo);
closesocket(ListenSocket);
return false;
}
if (listen(ListenSocket, SOMAXCONN) == SOCKET_ERROR) {
printf("Listen failed with error: %ld\n", WSAGetLastError());
closesocket(ListenSocket);
return false;
}
// Accept a client socket
SOCKET ClientSocket = accept(ListenSocket, nullptr, nullptr);
if (ClientSocket == INVALID_SOCKET) {
printf("server accept failed: %d\n", WSAGetLastError());
closesocket(ListenSocket);
return false;
}
std::vector<uint8_t> buffer;
buffer.resize(blocksize);
size_t total = 0;
// Receive until the peer shuts down the connection
do {
size_t currentSize = blocksize;
size_t next = total + currentSize;
if (next > size)
{
currentSize -= next - size;
}
result = recv(ClientSocket, (char*)buffer.data(), (int)currentSize, 0);
if (result > 0)
{
total += result;
}
else if (result == 0)
{
printf("server Connection closing...\n");
return false;
}
else {
printf("server recv failed: %d\n", WSAGetLastError());
closesocket(ClientSocket);
return false;
}
} while (total < size);
result = shutdown(ClientSocket, SD_SEND);
if (result == SOCKET_ERROR) {
printf("server shutdown failed: %d\n", WSAGetLastError());
closesocket(ClientSocket);
return false;
}
// cleanup
closesocket(ClientSocket);
return true;
}
and the test program itself is:
int main()
{
int width = 1920;
int height = 1080;
const size_t totalBpp = 3;
const size_t totalSize = width * height * totalBpp;
size_t port = 27140;
size_t times = 1000;
size_t expectedData = totalSize * times;
// Initialize Winsock
int iResult;
WSADATA wsaData;
iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
if (iResult != 0) {
std::cout << "WSAStartup failed: " << iResult << std::endl;
return EXIT_FAILURE;
}
std::atomic_bool serverOk{ false };
std::atomic_bool clientOk{ false };
auto start = std::chrono::high_resolution_clock::now();
std::thread server = std::thread([&] {
serverOk = serverReceiveDataWin(expectedData, totalSize, port);
});
std::thread client = std::thread([&] {
clientOk = sendDataWin(expectedData, totalSize, port);
});
client.join();
server.join();
auto end = std::chrono::high_resolution_clock::now();
WSACleanup();
if (!(clientOk && serverOk))
{
if (!serverOk) std::cout << "Server was not OK." << std::endl;
if (!clientOk) std::cout << "Client was not OK." << std::endl;
return EXIT_FAILURE;
}
std::chrono::duration<double> diff = end - start;
double frameTime = diff.count() / times;
double fps = 1.0 / frameTime;
std::cout << "Sent: " << width << "x" << height << "_" << totalBpp << "(" << totalSize << "). times: " << times << std::endl;
std::cout << "frameTime: " << frameTime << "s." << std::endl;
std::cout << "fps: " << fps << "." << std::endl;
std::cout << "transfer rate : " << ((expectedData / diff.count()) / 1048576) << " mebibytes/s." << std::endl;
std::cout << "transfer rate : " << ((expectedData / diff.count()) / 1000000) << " megabytes/s." << std::endl;
std::this_thread::sleep_for(std::chrono::seconds{ 60 });
return EXIT_SUCCESS;
}
on my machine I get these results:
Sent: 1920x1080_3(6220800).times : 1000
frameTime : 0.0138217s.
fps : 72.35.
transfer rate : 429.225 mebibytes / s.
transfer rate : 450.075 megabytes / s.
It seems that the issue was somewhat my firewall/anti-virus, I first unloaded F-secure, the speeds increased to 500megabytes/s.. after uninstall & reboot, the speeds increased to 4000megabytes/s.

Answer 1

A couple points.
the IOCTL is not called correctly.
bool value;
DWORD dwBytesRet;
int status =
WSAIoctl(
ListenSocket,
SIO_LOOPBACK_FAST_PATH,
&value,
sizeof(bool),
NULL,
0,
&dwBytesRet,
0,
0);
This should pass in a DWORD (32 bit integer) set to 1. The above code passes a C++ bool (probably only 1 byte) that was never initialized to anything.
Since this is only a single socket, this is going to be CPU limited, since it's effectively going to just be going from user/kernel and memcopying on a single thread. It's not likely that this will hit 40 Gbps over a single connection. Particularly since this is only sending 6GB worth of data.

Answer 2

You should turn off Nagle's Algorithm when sending large amounts of data like this. Set TCP_NODELAY on the sending socket.