Tcp packets sent by socket via send() slow - c++

Question

I am trying to write an C/C++ application comunicating with a device using tcp socket. My app acts as a server socket.
When received data from device socket, server responses to device using send() function:
ssize_t send(int sockfd, const void *buf, size_t len, int flags);
The buffer length (~4kB) is greater than MTU (1500 byte). TCP divide it to smaller packages. I used wireshark to capture packets. I saw that the time delay between tcp packets is 0.2-0.3s. If I send large data (~100Mb), it takes too long time (several hours).
I have tried to setsockopt to enable TCP_NODELAY, TCP_QUICKACK and TCP_CORK but still cannot reduce the time delay.
Could you guys help me this issue?
-------------------
EDIT:
Here is my code:
#include <iostream>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <string.h>
void createServerSocket();
void acceptConnect(int serverSock);
void receiveDataFromSocket(int socket);
void sendDataOverSocket(int clientSock);
void createServerSocket() {
int serverSockFd = socket(AF_INET6, SOCK_STREAM, IPPROTO_TCP);
std::string address = "fd53:7cb8:383:5::68";
int port = 42654;
if (serverSockFd < 0)
{
printf("Create Server socket fail");
return;
}
struct sockaddr_in6 serverAddress;
(void)memset(&serverAddress, 0, sizeof(sockaddr_in6));
serverAddress.sin6_family = AF_INET6;
serverAddress.sin6_port = htons(port);
int result = inet_pton(AF_INET6, address.c_str(), &serverAddress.sin6_addr);
if (result <= 0)
{
printf("inet_pton() failed portnumber: %d, address: %s \n", port, address.c_str());
return;
}
// setting socket options
int flag = 1;
if(setsockopt(serverSockFd,IPPROTO_TCP,TCP_QUICKACK ,(char *)&flag,sizeof(flag)) == -1)
{
printf("setsockopt TCP_QUICKACK failed for server socket on address %s \n", address.c_str());
}
if(setsockopt(serverSockFd,IPPROTO_TCP,TCP_CORK,(char *)&flag,sizeof(flag)) == -1)
{
printf("setsockopt TCP_CORK failed for server socket on address %s \n", address.c_str());
}
if(setsockopt(serverSockFd,IPPROTO_TCP,TCP_NODELAY,(char *)&flag,sizeof(flag)) == -1)
{
printf("setsockopt TCP_NODELAY failed for server socket on address %s \n", address.c_str());
}
result = bind(serverSockFd, (struct sockaddr*)&serverAddress, sizeof(sockaddr_in6));
if (result != 0)
{
printf("bind() failed portnumber: %d, address: %s \n", port, address.c_str());
return ;
}
result = listen(serverSockFd, 10);
if (result != 0) {
printf("listen() failed portnumber: %d, address: %s \n", port, address.c_str());
return ;
}
acceptConnect(serverSockFd);
}
void acceptConnect(int serverSock)
{
struct sockaddr_in6 clientAddress;
socklen_t len = sizeof(sockaddr_in6);
memset(&clientAddress, 0, sizeof(sockaddr_in6));
const int clientSocket = accept(serverSock, (struct sockaddr*)&clientAddress, &len);
if(clientSocket >= 0) {
char str_addr[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, &(clientAddress.sin6_addr),
str_addr, sizeof(str_addr));
printf("New connection from: %s:%d ...\n", str_addr, ntohs(clientAddress.sin6_port));
receiveDataFromSocket(clientSocket);
}
}
void receiveDataFromSocket(int socket)
{
int SOCKET_BUFFER_MAX_SIZE = 8*1024;
char buffer[SOCKET_BUFFER_MAX_SIZE];
memset(buffer, '\0', SOCKET_BUFFER_MAX_SIZE);
//Receive data from sock
while (true) {
int dataLen = recv(socket, buffer, SOCKET_BUFFER_MAX_SIZE, 0);
printf("Receive data from socket: %d, msgLength = %d\n", socket, dataLen);
sendDataOverSocket(socket);
}
}
void sendDataOverSocket(int clientSock)
{
int dataLen = 4*1024 + 7;
char *buf = new char[dataLen];
memset(buf, 'a', 4*1024 + 7);
int ret;
ret = send(clientSock, buf, dataLen, 0);
if (ret <= 0) {
printf("ERROR Send message over socket");
return;
}
int error_code;
socklen_t error_code_size = sizeof(sockaddr_in6);
getsockopt(clientSock, SOL_SOCKET, SO_ERROR, &error_code, &error_code_size);
printf("Error code size: %d, error code: %d\n", error_code_size, error_code);
}

Answer 1

I think you are exceeding the TCP sending buffers, you are sequentially receiving and sending, if the other side is not reading data fast enough from the socket you will wait in every send() operation until you have space in your sending buffer.
Check if you are sending more information than receiving, I suspect recv() is reading small blocks (because the MTU) but you are sending about 4KB in every loop, so your recv/send calls are not properly balanced.