epoll multi process - c++

Question

I continue learn network programming using c/c++, and after that I have created multi process tcp server, I want to create simple http server, which return static resources, I use epoll so let me show my code
first of all I use fd passing for handle request in workers
so, my main function and head process
struct Descriptors{
int sv[2];
};
class Parent{
public:
static Parent& getInstance(){
static Parent instance;
return instance;
}
Parent(Parent const&) = delete;
void operator=(Parent const&) = delete;
void addFd(int fd){
m_fd.push_back(fd);
};
void run() {
startServer();
size_t index = 0;
while(true){
struct epoll_event Events[MAX_EVENTS];
int N = epoll_wait(m_epoll, Events, MAX_EVENTS, -1);
for (size_t i =0; i < N; ++i){
if (Events[i].events & EPOLLHUP){
epoll_ctl(m_epoll, EPOLL_CTL_DEL, Events[i].data.fd, &(Events[i]));
shutdown(Events[i].data.fd,SHUT_RDWR);
close(Events[i].data.fd);
continue;
}else {
if (Events[i].data.fd == m_masterSocket) {
handleConnection();
}else {
char * arg = "1";
ssize_t size = sock_fd_write(m_fd[index], arg, 1,Events[i].data.fd);
index = (1+index) % m_fd.size();
}
}
}
}
}
private:
Parent(){
m_numCpu = sysconf(_SC_NPROCESSORS_ONLN);
}
void startServer(){
m_masterSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
struct sockaddr_in SockAddr;
SockAddr.sin_family = AF_INET;
SockAddr.sin_port = htons(11141);
SockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
bind(m_masterSocket, (struct sockaddr *)(&SockAddr), sizeof(SockAddr));
set_nonblock(m_masterSocket);
listen(m_masterSocket, SOMAXCONN);
m_epoll = epoll_create1(0);
struct epoll_event Event;
Event.data.fd = m_masterSocket;
Event.events = EPOLLIN | EPOLLRDHUP;
epoll_ctl(m_epoll, EPOLL_CTL_ADD, m_masterSocket, &Event);
}
void handleConnection(){
int SlaveSocket = accept(m_masterSocket, 0, 0);
set_nonblock(SlaveSocket);
struct epoll_event Event;
Event.data.fd = SlaveSocket;
Event.events = EPOLLIN | EPOLLRDHUP;
epoll_ctl(m_epoll, EPOLL_CTL_ADD, SlaveSocket, &Event);
}
int m_epoll;
int m_masterSocket;
int m_numCpu;
std::vector<int> m_fd;
};
void parent(int sock){
Parent::getInstance().addFd(sock);
}
int main(int argc, char **argv){
int numCpu = sysconf(_SC_NPROCESSORS_ONLN);
std::vector<Descriptors> desc;
desc.resize(numCpu);
bool isParent = true;
for (int i = 0; i < numCpu && isParent; ++i){
std::cout << "pid my is = " << getpid() <<std::endl;
int sv[2];
if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) < 0) {
perror("socketpair");
exit(1);
}
pid_t forkId = fork();
switch (forkId){
case 0:{
isParent = false;
close(sv[0]);
child(sv[1]);
break;
}
case -1:
perror("fork");
exit(1);
default:
close(sv[1]);
parent(sv[0]);
break;
}
}
if (isParent){
Parent::getInstance().run();
int status;
waitpid(-1, &status, 0);
}
}
And my worker process is
void respond(int fd)
{
char mesg[99999], *reqline[3], data_to_send[BYTES], path[99999];
int rcvd, fileDesc, bytes_read;
memset( (void*)mesg, (int)'\0', 99999 );
const char *ROOT = "/home/web_server/";
int RecvResult = recv(fd,mesg, 99999, MSG_NOSIGNAL);
if (RecvResult == 0 && errno != EAGAIN){
shutdown(fd,SHUT_RDWR);
close(fd);
}else if (RecvResult >0){
printf("%s", mesg);
reqline[0] = strtok (mesg, " \t\n"); // split on lexemes
if ( strncmp(reqline[0], "GET\0", 4)==0 ) // if first 4 character equal
{
reqline[1] = strtok (NULL, " \t");
reqline[2] = strtok (NULL, " \t\n");
std::cout << "reqline 1 " << reqline[1] << std::endl;
std::cout << "reqline 2 " << reqline[2] << std::endl;
if ( strncmp( reqline[2], "HTTP/1.0", 8)!=0
&& strncmp(reqline[2], "HTTP/1.1", 8 ) !=0 )
{
write(fd, "HTTP/1.0 400 Bad Request\n", 25);
}
else
{
if ( strncmp(reqline[1], "/\0", 2)==0 )
reqline[1] = "/index.html";
strcpy(path, ROOT);
strcpy(&path[strlen(ROOT)], reqline[1]);
printf("file: %s\n", path);
if ( (fileDesc=open(path, O_RDONLY))!=-1 )
{
send(fd, "HTTP/1.0 200 OK\n\n", 17, 0);
while ( (bytes_read=read(fileDesc, data_to_send, BYTES))>0 )
write (fd, data_to_send, bytes_read);
}
else write(fd, "HTTP/1.0 404 Not Found\n", 23);
}
}
}
shutdown(fd,SHUT_RDWR);
close(fd);
}
void child(int sock)
{
int fd;
char buf[16];
ssize_t size;
sleep(1);
for (;;) {
size = sock_fd_read(sock, buf, sizeof(buf), &fd);
if (size <= 0)
break;
if (fd != -1) {
respond(fd);
}
}
printf("child processes is end\n");
}
And when I go in browser http://127.0.0.1:11141/ it is ok, and I get index.html, but when I run in apache benchmark, as
ab -n 10 -c 10 http://127.0.0.1:11141/
I get answer as
This is ApacheBench, Version 2.3
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/
Benchmarking 127.0.0.1 (be patient)...apr_socket_recv: Connection reset by peer (104)
Total of 2 requests completed
I don't understand where is my error, because I I think that my server in theory(because using epoll ) have to resolved C10K problem. but on the practice, my server can not resolved 10 connection. Could you help me please?
Thank you for useful links and any advices!
UPDATE
When I run as
strace -f ./server 2> error.txt
in end of error.txt
[pid 6552] write(6, 0x7ffdbff00390, 757) = -1 EPIPE (Broken pipe)
[pid 6552] --- SIGPIPE {si_signo=SIGPIPE, si_code=SI_USER, si_pid=6552, si_uid=1000} ---
[pid 6552] +++ killed by SIGPIPE +++
write(1, 0x7fc5ffbe3000, 83) = 83
write(1, 0x7fc5ffbe3000, 12) = 12
write(1, 0x7fc5ffbe3000, 20) = 20
write(1, 0x7fc5ffbe3000, 41) = 41
open(0x7ffdbff18e30, O_RDONLY) = 11
sendto(10, 0x403df9, 17, 0, NULL, 0) = 17
read(11, 0x7ffdbff00390, 1024) = 757
write(10, 0x7ffdbff00390, 757) = -1 EPIPE (Broken pipe)
--- SIGPIPE {si_signo=SIGPIPE, si_code=SI_USER, si_pid=6554, si_uid=1000} ---
+++ killed by SIGPIPE +++
So I think that problem in EPipe error, But I don't understand why...
Update
So I think that problem in close descriptor, but I don't understand how to fix it. Thank you for useful advices.
UPDATE
I Get error on function send in worker process

Answer 1

Seems I found my error, right version function void :
void respond(int fd)
{
char mesg[99999], *reqline[3], data_to_send[BYTES], path[99999];
int rcvd, fileDesc, bytes_read;
memset( (void*)mesg, (int)'\0', 99999 );
const char *ROOT = "/home/web_server/";
int RecvResult = recv(fd,mesg, 99999, MSG_NOSIGNAL);
//EAGAIN - "there is no data available right now, try again later
if (RecvResult == 0 && errno != EAGAIN){
shutdown(fd,SHUT_RDWR);
close(fd);
std::cout << "error recv" << std::endl;
return;
}else if (RecvResult >0){
printf("%s", mesg);
reqline[0] = strtok (mesg, " \t\n"); // split on lexemes
if ( strncmp(reqline[0], "GET\0", 4)==0 ) // if first 4 character equal
{
reqline[1] = strtok (NULL, " \t");
reqline[2] = strtok (NULL, " \t\n");
std::cout << "reqline 1 " << reqline[1] << std::endl;
std::cout << "reqline 2 " << reqline[2] << std::endl;
if ( strncmp( reqline[2], "HTTP/1.0", 8)!=0
&& strncmp(reqline[2], "HTTP/1.1", 8 ) !=0 )
{
send(fd, "HTTP/1.0 400 Bad Request\n", 25 , MSG_NOSIGNAL);
}
else
{
if ( strncmp(reqline[1], "/\0", 2)==0 )
reqline[1] = "/index.html";
strcpy(path, ROOT);
strcpy(&path[strlen(ROOT)], reqline[1]);
printf("file: %s\n", path);
if ( (fileDesc=open(path, O_RDONLY))!=-1 )
{
send(fd, "HTTP/1.0 200 OK\n\n", 17, MSG_NOSIGNAL);
while ( (bytes_read=read(fileDesc, data_to_send, BYTES))>0 )
{
if (bytes_read != -1)
send (fd, data_to_send, bytes_read, MSG_NOSIGNAL);
}
}
else send(fd, "HTTP/1.0 404 Not Found\n", 23, MSG_NOSIGNAL);
}
shutdown(fd,SHUT_RDWR);
close(fd);
}
}else {
std::cout << "Client disconnected unexpect" << std::endl;
}
}
Problem was that , that I close socket , and after that I try to read from this socket.