I want to broadcast OpenCV images (coming from a camera) to a distant computer in real time, it has to be done via Ethernet. The images are continuously received in standard OpenCV Mat objects. The final code has to be integrated into a C++ (Qt) application.
I have found this Python script that does the job very well.
Now I'm trying to get a C++ equivalent of that code, I managed to create an HTTP server using Boost Asio and the Simple-Web-Server project. I am able to display a static blue image/webcam camera image (not refreshed).
I have written a code but it's not working. My guess is that the data are sent only at the return of the function (which never returns).
How can I force sending the data after each iteration of the while loop?
#include "server_http.hpp"
#include <thread>
#include <boost/chrono.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/date_time/posix_time/posix_time_io.hpp>
#include <opencv2/opencv.hpp>
//#include <opencv/cv.h>
using namespace boost::posix_time;
typedef SimpleWeb::Server<SimpleWeb::HTTP> HttpServer;
cv::Mat image;
cv::VideoCapture cap;
int main()
{
cap.open(0);
if (!cap.isOpened ())
{
std::cerr << "Could not initialize capturing" << std::endl;
return (-1);
}
cap >> image;
HttpServer server(8080, 2);
// Image resource is requested
server.resource["^/cam.mjpg"]["GET"] =
[=](HttpServer::Response& response, std::shared_ptr<HttpServer::Request> request)
{
time_facet *facet = new time_facet("%d-%b-%Y %H:%M:%S");
std::cout.imbue(std::locale(std::cout.getloc(), facet));
std::cout << second_clock::local_time() << " | " << "Camera image requested!" << std::endl;
response <<
"HTTP/1.1 200 OK\r\n"
"Content-type: multipart/x-mixed-replace; boundary=--jpgboundary";
//TODO: Send header
while (1) // TODO: Allow exiting this
{
std::cout << "Send image" << std::endl;
cap >> image;
// Encode mat to jpg and copy it to content
std::vector<uchar> buf;
cv::imencode(".jpg", image, buf, std::vector<int>());
std::string img_content(buf.begin(), buf.end());
response << "--jpgboundary\r\n" << // Signal we start a new image
"Content-type: image/jpeg" <<
"Content-Length: " << img_content.length() << "\r\n" <<
"\r\n" << img_content << "\r\n";
std::this_thread::sleep_for(std::chrono::milliseconds(400));
}
};
// Anything else is requested
server.default_resource["GET"] = [](HttpServer::Response& response, std::shared_ptr<HttpServer::Request> request)
{
time_facet *facet = new time_facet("%d-%b-%Y %H:%M:%S");
std::cout.imbue(std::locale(std::cout.getloc(), facet));
std::cout << second_clock::local_time() << " | " << request->path << std::endl;
std::string content =
"<html><head></head><body>"
"<img src=\"cam.mjpg\"/>"
"</body></html>";
response <<
"HTTP/1.1 200 OK\r\n"
"Content-Length: " << content.length() << "\r\n"
"\r\n" << content;
};
std::thread server_thread([&server]()
{
server.start();
});
std::this_thread::sleep_for(std::chrono::seconds(1));
server_thread.join();
return 0;
}
EDIT 1
Based on Technik Empire comment I went back to boost examples;
In the HTTP server example the response in sent when the callback returns so I modified the callback to allow do_write() operations on the socket.
The HTML page is correctly displayed but the image is not displayed (the broken image icon is shown instead), I tried to see what happens with Wireshark but I don't know what is wrong.
Here is my handle_request function: (request_handler.cpp):
void request_handler::handle_request(const request& req, reply& rep, connection &con)
{
// Decode url to path.
std::string request_path;
if (!url_decode(req.uri, request_path))
{
rep = reply::stock_reply(reply::bad_request);
return;
}
// Request path must be absolute and not contain "..".
if (request_path.empty() || request_path[0] != '/'
|| request_path.find("..") != std::string::npos)
{
rep = reply::stock_reply(reply::bad_request);
return;
}
// Determine the file extension.
std::size_t last_slash_pos = request_path.find_last_of("/");
std::string filename;
if (last_slash_pos != std::string::npos)
filename = request_path.substr(last_slash_pos + 1);
if (filename == "cam.mjpg") // Image is requested
{
rep.status = reply::ok;
rep.headers.resize(1);
rep.headers[0].name = "Content-Type";
rep.headers[0].value = "multipart/x-mixed-replace; boundary=--jpgboundary";
rep.content.empty();
con.do_write();
rep.status = reply::none;
while (true) // FIXME: How do I handle disconnection from the client?
{
cv::Mat image(200, 300, CV_8UC3);
int random = rand() % 255 + 1;
image = cv::Scalar(random, 0, 0); // Fill image with blue
std::vector<uchar> buf;
cv::imencode(".jpg", image, buf, std::vector<int>());
std::string img_content(buf.begin(), buf.end());
rep.headers.clear();
rep.content.clear();
rep.content.append("--jpgboundary\r\n");
con.do_write();
rep.content.clear();
rep.headers.resize(2);
rep.headers[0].name = "Content-Type";
rep.headers[0].value = mime_types::extension_to_type("jpg");
rep.headers[1].name = "Content-length";
rep.headers[1].value = img_content.size();
rep.content.append(img_content);
rep.content.append("\r\n");
con.do_write();
boost::this_thread::sleep(boost::posix_time::milliseconds(500));
}
}
else // Anything but the image is requested
{
std::string content =
"<html><head></head><body>"
"Hello :)<br>"
"<img src=\"cam.mjpg\"/>"
"</body></html>";
rep.status = reply::ok;
rep.headers.resize(2);
rep.headers[0].name = "Content-Length";
rep.headers[0].value = content.length();
rep.headers[1].name = "Content-Type";
rep.headers[1].value = mime_types::extension_to_type("html");
rep.content.append(content);
con.do_write();
return;
}
}
I got it working by analyzing the packets thanks to the Firefox network analyzer, I replicated the Python headers/content answer and it works fine:
I added a reply::none reply type and made sure that if this reply was provided, no HTTP status is being sent. So in the reply::to_buffers() function I added this:
if (status != none) // Don't add status to buffer if status is "none"
buffers.push_back(status_strings::to_buffer(status));
The request_handler.cpp code looks like this:
if (filename == "cam.mjpg") // Image is requested
{
rep.status = reply::ok;
rep.headers.resize(1);
rep.headers[0].name = "Content-Type";
rep.headers[0].value = "multipart/x-mixed-replace; boundary=--jpgboundary\r\n";
con.do_write();
while (true) // FIXME: How do I handle disconnection from the client?
{
cv::Mat image(200, 300, CV_8UC3);
int random = rand() % 255 + 1;
image = cv::Scalar(random, 0, 0); // Fill image with blue
std::vector<uchar> buf;
cv::imencode(".jpg", image, buf, std::vector<int>());
std::string img_content(buf.begin(), buf.end());
rep.status = reply::none;
rep.headers.resize(0);
rep.content.clear();
rep.content.append("--jpgboundary");
rep.content.append("\r\n");
rep.content.append("Content-Type: image/jpeg");
rep.content.append("\r\n");
rep.content.append("Content-length: "+boost::lexical_cast<std::string>(img_content.length()));
rep.content.append("\r\n");
rep.content.append("\r\n");
rep.content.append(img_content);
rep.content.append("\r\n");
con.do_write();
boost::this_thread::sleep(boost::posix_time::milliseconds(100));
}
}
else // Anything but the image is requested
{
std::string content =
"<html><head></head><body>"
"<img src=\"cam.mjpg\"/>"
"</body></html>";
rep.status = reply::ok;
rep.headers.resize(2);
rep.headers[0].name = "Content-Length";
rep.headers[0].value = content.length();
rep.headers[1].name = "Content-Type";
rep.headers[1].value = mime_types::extension_to_type("html");
rep.content.append(content);
con.do_write();
return;
}
Improvements suggestions are welcome in the comments. I don't know how to handle a disconnection from the client (exiting the while loop) so at the moment the server will only work with 1 request; after that it's stucked in the while loop.
Http_server.hpp
#ifndef HTTPSERVER_HPP_INCLUDED
#define HTTPSERVER_HPP_INCLUDED
#include <boost/bind.hpp>
#include <boost/asio.hpp>
#include <opencv2/core/core.hpp>
#include <boost/thread.hpp>
using boost::asio::ip::tcp;
typedef boost::shared_ptr<tcp::socket> socket_ptr;
class HttpServer
{
public:
std::map<std::string, std::string> winnames;
std::map<std::string,int> requestcounts;
std::map<std::string,std::vector<unsigned char> > jpegbuffers;
short port;
HttpServer();
void run(int portno);
boost::shared_mutex mut;
boost::condition_variable_any cond;
int httpdelay;
void IMSHOW(std::string win, cv::Mat mat);
int compression;
bool is_debug;
private:
int it;
void server(int port);
void session(socket_ptr sock);
void handleinfo(socket_ptr sock);
void handlewindows(socket_ptr sock);
void handlemjpeg(socket_ptr sock,std::string winname);
void handlejpg(socket_ptr sock,std::string winname);
void handle404(socket_ptr sock);
};
#endif
Http_server.cpp
#include "http_server.hpp"
#include <fstream>
#include <boost/filesystem.hpp>
#include <boost/format.hpp>
#include <opencv2/opencv.hpp>
#include <boost/lexical_cast.hpp>
namespace bfs= boost::filesystem;
using namespace std;
using boost::lexical_cast;
// Helper functions
#if defined(unix) || defined(__unix) || defined(__unix__) \
|| defined(linux) || defined(__linux) || defined(__linux__) \
|| defined(sun) || defined(__sun) \
|| defined(BSD) || defined(__OpenBSD__) || defined(__NetBSD__) \
|| defined(__FreeBSD__) || defined __DragonFly__ \
|| defined(sgi) || defined(__sgi) \
|| defined(__MACOSX__) || defined(__APPLE__) \
|| defined(__CYGWIN__)
#define is_nix
#endif
#if defined(_MSC_VER) || defined(WIN32) || defined(_WIN32) || defined(__WIN32__) \
|| defined(WIN64) || defined(_WIN64) || defined(__WIN64__)
#define is_win
#endif
#ifdef is_win
#include <windows.h>
#define SLEEP(ms) Sleep(ms)
#endif
#ifdef is_nix
#define SLEEP(ms) usleep(ms*1000)
#endif
std::vector<std::string> &dssplit(const std::string &s, char delim, std::vector<std::string> &elems) {
std::stringstream ss(s);
std::string item;
while (getline(ss, item, delim)) {
elems.push_back(item);
}
return elems;
}
std::vector<std::string> dssplit(const std::string &s, char delim) {
std::vector<std::string> elems;
return dssplit(s, delim, elems);
}
void removeEmptyStrings(std::vector<std::string>& strings)
{
std::vector<std::string>::iterator it = remove_if(strings.begin(), strings.end(), mem_fun_ref(&std::string::empty));
strings.erase(it, strings.end());
}
bool hasEnding(std::string const &fullString, std::string const &ending)
{
if (fullString.length() >= ending.length()) {
return (0 == fullString.compare(fullString.length() - ending.length(), ending.length(), ending));
}
else {
return false;
}
}
bool startswith(std::string const &src, std::string const &start)
{
if (src.compare(0, start.length(), start) == 0)
{
return true;
}
return false;
}
std::string urldecode(std::string &src) {
std::string ret;
char ch;
int ii;
for (size_t i = 0; i<src.length(); i++) {
if (int(src[i]) == 37) {
sscanf(src.substr(i + 1, 2).c_str(), "%x", &ii);
ch = static_cast<char>(ii);
ret += ch;
i = i + 2;
}
else {
ret += src[i];
}
}
return (ret);
}
// Server implementation
HttpServer::HttpServer() :compression(70), is_debug(true), httpdelay(100)
{
}
void HttpServer::IMSHOW(std::string win, cv::Mat mat)
{
winnames[win] = lexical_cast<string>(mat.cols) + "," + lexical_cast<string>(mat.rows);
if (is_debug)
{
cv::imshow(win, mat);
}
else
{
//cvDestroyWindow(win.c_str());
}
if (requestcounts[win] > 0)
{
cv::Mat towrite;
if (mat.type() == CV_8UC1)
{
cvtColor(mat, towrite, CV_GRAY2BGR);
}
else if (mat.type() == CV_32FC3)
{
double minVal, maxVal;
minMaxLoc(mat, &minVal, &maxVal);
mat.convertTo(towrite, CV_8U, 255.0 / (maxVal - minVal), -minVal * 255.0 / (maxVal - minVal));
}
else{
towrite = mat;
}
std::vector<uchar> buffer;
std::vector<int> param(2);
param[0] = CV_IMWRITE_JPEG_QUALITY;
param[1] = compression;
imencode(".jpg", towrite, buffer, param);
jpegbuffers[win].swap(buffer);
}
}
void HttpServer::run(int portno)
{
port=portno;
boost::thread t(boost::bind(&HttpServer::server,this,port));
}
void HttpServer::server(int port)
{
try
{
boost::asio::io_service io_service;
io_service.run();
tcp::acceptor a(io_service, tcp::endpoint(tcp::v4(), port));
for (;;)
{
socket_ptr sock(new tcp::socket(io_service));
a.accept(*sock);
boost::thread t(boost::bind(&HttpServer::session, this, sock));
}
}
catch (boost::exception & e)
{
std::cout << "OMG!" << boost::diagnostic_information(e)<<endl;
}
}
void HttpServer::session(socket_ptr sock)
{
try
{
boost::system::error_code ec;
boost::asio::streambuf sbuffer;
boost::asio::read_until(* sock, sbuffer, "\0", ec );
const char* header=boost::asio::buffer_cast<const char*>(sbuffer.data());
std::string reqStr(header,header+sbuffer.size());
sbuffer.consume(sbuffer.size());
std::vector<std::string> strs;
strs = dssplit(reqStr,' ');
if(strs.size()>1)
{
std::string requesturl = urldecode(strs[1]);
std::vector<std::string> splited=dssplit(requesturl,'/');
removeEmptyStrings(splited);
if(splited.size()==1)
{
if(startswith(splited[0],"windows"))
{
handlewindows(sock);
}else if(startswith(splited[0],"info"))
{
handleinfo(sock);
}else if(hasEnding(splited[0],".mjpg"))
{
handlemjpeg(sock,splited[0].substr(0,splited[0].size()-5));
}else if(hasEnding(splited[0],".jpg") || splited[0].find(".jpg?")!=string::npos)
{
handlejpg(sock,splited[0]);
}else
{
handle404(sock);
}
}else
{
handle404(sock);
}
sock->shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec);
}
}catch(const std::exception& ex)
{
boost::system::error_code ec;
boost::asio::ip::tcp::endpoint endpoint = sock->remote_endpoint(ec);
if(!ec)
{
sock->shutdown(boost::asio::ip::tcp::socket::shutdown_both);
}
//DPRINTERR(ex.what());
}catch(const std::string& ex)
{
boost::system::error_code ec;
boost::asio::ip::tcp::endpoint endpoint = sock->remote_endpoint(ec);
if(!ec)
{
sock->shutdown(boost::asio::ip::tcp::socket::shutdown_both);
}
}
}
void HttpServer::handleinfo(socket_ptr sock)
{
boost::system::error_code error;
boost::asio::streambuf sbuffer;
std::ostream response_stream(&sbuffer);
string retstr;
for (std::map<std::string,std::string>::iterator it=winnames.begin(); it!=winnames.end(); ++it)
{
string wname =it->first;
int rcnt = 0;
if(requestcounts.find(wname)!=requestcounts.end())
{
rcnt=requestcounts[wname];
}
retstr+=boost::str(boost::format("{"
"\"name\":\"%s\","
"\"reqCnt\":%d,"
"\"size\":\"%s\""
"},"
)
%wname
%rcnt
%it->second
);
}
if(retstr.size()>0) retstr.resize(retstr.size()-1);
retstr=boost::str(boost::format("{"
"\"windows\":[%s],"
"\"version\":\"%s\","
"\"fps\":%s"
"}"
)
%retstr
%"0.0"
% to_string(0.)
);
response_stream << "HTTP/1.1 200 OK\r\n";
response_stream << "Access-Control-Allow-Origin: *\r\n";
response_stream << "Content-Type: text/plain\r\n\r\n";
response_stream << retstr << "\r\n\r\n";
boost::asio::write(*sock, sbuffer);
}
void HttpServer::handlewindows(socket_ptr sock)
{
boost::system::error_code error;
boost::asio::streambuf sbuffer;
std::ostream response_stream(&sbuffer);
string retstr;
for (std::map<std::string,std::string>::iterator it=winnames.begin(); it!=winnames.end(); ++it)
{
string wname =it->first;
int rcnt = 0;
if(requestcounts.find(wname)!=requestcounts.end())
{
rcnt=requestcounts[wname];
}
retstr+=boost::str(boost::format("{"
"\"name\":\"%s\","
"\"reqCnt\":%d,"
"\"size\":\"%s\""
"},"
)
%wname
%rcnt
%it->second
);
}
if(retstr.size()>0) retstr.resize(retstr.size()-1);
retstr="{\"windows\":["+retstr+"]}";
response_stream<<"HTTP/1.1 200 OK\r\n"
"Content-Type: text/plain\r\n\r\n"<<
retstr<<"\r\n\r\n";
boost::asio::write(*sock, sbuffer);
}
void HttpServer::handlemjpeg(socket_ptr sock,std::string winname)
{
if(requestcounts.find(winname)==requestcounts.end())
{
handle404(sock);
return;
}
std::string frame=winname;
//boost::shared_lock<boost::shared_mutex> lock(mut);
//lock.lock();
requestcounts[frame]++;
//lock.unlock();
boost::system::error_code error;
boost::asio::streambuf sbuffer;
std::ostream response_stream(&sbuffer);
response_stream<<"HTTP/1.1 200 OK\r\n";
response_stream<<"Content-Type: multipart/mixed;boundary=b\r\n";
response_stream<<"Cache-Control: no-store\r\n";
response_stream<<"Pragma: no-cache\r\n";
response_stream<<"Audio Mode : None\r\n";
response_stream<<"Connection: close\r\n";
response_stream<<"\r\n";
boost::asio::write(*sock, sbuffer);
for(;;)
{
try
{
if( (jpegbuffers.count(frame)<0 ||
jpegbuffers[frame].size()<4) ||
(jpegbuffers[frame][0]!=0xff && jpegbuffers[frame][1]!=0xd8 &&
jpegbuffers[frame][jpegbuffers[frame].size()-2]!=0xff && jpegbuffers[frame][jpegbuffers[frame]. size()-1]!=0xd9))
{
SLEEP(10);
continue;
}
//boost::shared_lock<boost::shared_mutex> lock(mut);
response_stream<<"--b\r\n";
response_stream<<"Content-Type: image/jpeg\r\n";
response_stream<<"Content-length: "<<jpegbuffers[frame].size()<<"\r\n";
response_stream<<"\r\n";
boost::asio::write(*sock, sbuffer);
boost::asio::write(*sock,boost::asio::buffer(jpegbuffers[frame], jpegbuffers[frame].size()));
//lock.unlock();
SLEEP(httpdelay);
}
catch (std::exception& e)
{
SLEEP(50);
//lock.lock();
requestcounts[frame]--;
//lock.unlock();
return;
}
}
//lock.lock();
requestcounts[frame]--;
//lock.unlock();
}
void HttpServer::handlejpg(socket_ptr sock,std::string winname)
{
if(winname.find("?")!=string::npos)
{
winname = winname.substr(0,winname.find("?"));
}
winname =winname.substr(0,winname.size()-4);
std::string frame=winname;
requestcounts[frame]++;
boost::system::error_code error;
boost::asio::streambuf sbuffer;
std::ostream response_stream(&sbuffer);
jpegbuffers[frame].clear();
for(;;)
{
try
{
if( (jpegbuffers.count(frame)<0 ||
jpegbuffers[frame].size()<4) ||
(jpegbuffers[frame][0]!=0xff && jpegbuffers[frame][1]!=0xd8 &&
jpegbuffers[frame][jpegbuffers[frame].size()-2]!=0xff && jpegbuffers[frame][jpegbuffers[frame]. size()-1]!=0xd9))
{
SLEEP(10);
continue;
}
response_stream<<"HTTP/1.1 200 OK\r\n";
response_stream<<"Content-Type: image/jpeg\r\n";
response_stream<<"Cache-Control: no-store\r\n";
response_stream<<"Access-Control-Allow-Origin: *\r\n";
response_stream<<"Pragma: no-cache\r\n";
response_stream<<"Content-length: "<<jpegbuffers[frame].size()<<"\r\n";
response_stream<<"Connection: close\r\n";
response_stream<<"\r\n";
boost::asio::write(*sock, sbuffer);
boost::asio::write(*sock,boost::asio::buffer(jpegbuffers[frame], jpegbuffers[frame].size()));
break;
}
catch (std::exception& e)
{
//DPRINTERR( "net exceptoin:"+std::string(e.what()));
SLEEP(50);
requestcounts[frame]--;
return;
}
}
requestcounts[frame]--;
}
void HttpServer::handle404(socket_ptr sock)
{
boost::system::error_code error;
boost::asio::streambuf sbuffer;
std::ostream response_stream(&sbuffer);
response_stream<<"HTTP/1.1 404 Not Found\r\n"
"Content-Type: text/html\r\n"
"Connection: close\r\n"
"Content-Length: 132\r\n\r\n"
"<html>\r\n"
"<head><title>404 Not Found</title></head>\r\n"
"<body bgcolor=\"white\">\r\n"
"<center><h1>404 Not Found</h1></center>\r\n"
"</body>\r\n"
"</html>\r\n";
boost::asio::write(*sock, sbuffer);
}
Main.cpp
#include <opencv2/opencv.hpp>
#include "http_server.hpp"
#include <iostream>
#include <fstream>
using namespace cv;
#define MJPGFILE_BUFFER_SIZE 10240
class MjpgFileCapture{
public:
static double lastframeseen;
MjpgFileCapture() {};
MjpgFileCapture(std::string filepath)
{
filepath_ = filepath;
is_inited_ = false;
skip_ = true;
imgready_ = false;
ff_ = false;
readbytes_ = -2;
i_ = 0;
};
void init();
MjpgFileCapture& operator >>(cv::Mat& out);
private:
std::string filepath_;
bool is_inited_;
std::ifstream ifstream_;
std::vector<char> data_;
bool skip_;
bool imgready_;
bool ff_;//have we seen ff byte?
long long readbytes_;
char ca_[MJPGFILE_BUFFER_SIZE];
int i_;//loop index
};
void MjpgFileCapture::init()
{
is_inited_ = true;
ifstream_ = std::ifstream(filepath_.c_str(), std::ios::binary);
}
MjpgFileCapture& MjpgFileCapture::operator >> (cv::Mat& out)
{
out = Mat();
if (!is_inited_)
{
init();
}
while (1)
{
uchar c;
if (readbytes_ != 0 && readbytes_ != -1)
{
if (i_ >= readbytes_)
{
ifstream_.read(ca_, MJPGFILE_BUFFER_SIZE);
readbytes_ = ifstream_.gcount();
i_ = 0;
}
for (; i_ < readbytes_; i_++)
{
c = ca_[i_];
if (ff_ && c == 0xd8)
{
skip_ = false;
data_.push_back((uchar)0xff);
}
if (ff_ && c == 0xd9)
{
imgready_ = true;
data_.push_back((uchar)0xd9);
skip_ = true;
}
ff_ = c == 0xff;
if (!skip_)
{
data_.push_back(c);
}
if (imgready_)
{
if (data_.size() != 0)
{
cv::Mat data_mat(data_);
cv::Mat frame(imdecode(data_mat, 1));
out = frame;
}
else
{
printf("warning:image is ready and data is empty. Likely bug.");
}
imgready_ = false;
skip_ = true;
data_.clear();
return *this;
}
}
}
else
{
//own exception class
throw std::string("zero byte read:probably end of file.");
}
}
return *this;
}
HttpServer* server = 0;
void file_loop()
{
MjpgFileCapture cap("C:/v/frame.mjpg");
while (true)
{
Mat im;
cap >> im;
server->IMSHOW("im", im);
imshow("im", im);
if (waitKey(1) == 27)
exit(0);
}
}
int main(int argc, char** argv)
{
server = new HttpServer;
//server->port = 8080;
server->run(8080);
while (true)
{
try{
file_loop();
}
catch (...)
{
}
}
return 0;
}
Usage
server->IMSHOW("im",mat);//register mat as window name im in server, accessible from browser as http://localhost:8080/im.jpg or http://localhost:8080/im.mjpg
http://localhost:8080/windows (show all registered windows)
Related
I write a C++ dome of tcp server with the libuv. When I check the cpu performance, I found the dome is a single thread running, how can I implement it with multi-thread?
Currently, the dome can hanlde 100,000+ tcp request per second, it can only eat 1 CPU.
Code:
#include <iostream>
#include <atomic>
#include "uv.h"
#include <thread>
#include <mutex>
#include <map>
using namespace std;
auto loop = uv_default_loop();
struct sockaddr_in addr;
typedef struct {
uv_write_t req;
uv_buf_t buf;
} write_req_t;
typedef struct {
uv_stream_t* client;
uv_alloc_cb alloc_cb;
uv_read_cb read_cb;
} begin_read_req;
void alloc_buffer(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) {
buf->base = (char*)malloc(suggested_size);
buf->len = suggested_size;
}
void free_write_req(uv_write_t *req) {
write_req_t *wr = (write_req_t*)req;
free(wr->buf.base);
free(wr);
}
void echo_write(uv_write_t *req, int status) {
if (status) {
fprintf(stderr, "Write error %s\n", uv_strerror(status));
}
free_write_req(req);
}
void echo_read(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
if (nread > 0) {
auto req = (write_req_t*)malloc(sizeof(write_req_t));
auto *aaa = (char*)malloc(5);
aaa[0] = '+';
aaa[1] = 'O';
aaa[2] = 'K';
aaa[3] = '\r';
aaa[4] = '\n';
req->buf = uv_buf_init(aaa, 5);
uv_write((uv_write_t*)req, client, &req->buf, 1, echo_write);
}
if (nread < 0) {
if (nread != UV_EOF)
fprintf(stderr, "Read error %s\n", uv_err_name(static_cast<unsigned int>(nread)));
uv_close((uv_handle_t*)client, nullptr);
}
free(buf->base);
}
void acceptClientRead(uv_work_t *req) {
begin_read_req *data = (begin_read_req *)req->data;
uv_read_start(data->client, data->alloc_cb, data->read_cb);
}
void on_new_connection(uv_stream_t *server, int status) {
if (status < 0) {
cout << "New connection error:" << uv_strerror(status);
return;
}
uv_tcp_t *client = (uv_tcp_t *)malloc(sizeof(uv_tcp_t));
uv_tcp_init(loop, client);
uv_work_t *req = (uv_work_t *)malloc(sizeof(uv_work_t));
begin_read_req *read_req = (begin_read_req *)malloc(sizeof(begin_read_req));
read_req->client = (uv_stream_t *)client;
read_req->read_cb = echo_read;
read_req->alloc_cb = alloc_buffer;
req->data = read_req;
if (uv_accept(server, (uv_stream_t *)client) == 0) {
uv_read_start((uv_stream_t *)client, alloc_buffer, echo_read);
// uv_queue_work(workloop[0], req, acceptClientRead, nullptr);
}
else {
uv_close((uv_handle_t *)client, nullptr);
}
}
void timer_callback(uv_timer_t* handle) {
cout << std::this_thread::get_id() << "---------" << "hello" << endl;
}
int main() {
uv_tcp_t server{};
uv_tcp_init(loop, &server);
uv_ip4_addr("0.0.0.0", 8790, &addr);
uv_tcp_bind(&server, (const struct sockaddr *) &addr, 0);
uv_listen((uv_stream_t *)&server, 511, on_new_connection);
uv_run(loop, UV_RUN_DEFAULT);
return 0;
}
Of course, I can make the write step asynchronous in the method "echo_read", but I didn't do anything before the write, can I make the demo multi-thread in another way to improve the throughput?
I'm writing tests for an inotify library and I'm currently working on capturing the IN_CREATE event. I'm trying to use boost::filesystem::ofstream, which is essentially std::basic_ofstream, but I'm not getting the IN_CREATE event.
There are two threads in execution, the main thread, and one I create that processes events like so:
std::thread thread([&]()
{
boost::asio::io_service::work(g_io_service);
g_io_service.run();
}
Whenever an event is received, the library calls this function in the context of the above thread:
[&](event_type event, std::string const & path)
{
std::lock_guard<std::mutex> lock(ready_mtx);
events.push_back(event);
condition.notify_all();
ready = true;
}
What the test looks like (main thread):
{
boost::filesystem::ofstream file(path);
file << "test";
}
{
std::lock_guard<std::mutex> lock(ready_mtx);
while (!ready)
condition.wait(lock);
REQUIRE(events[0] == event_type::FILE_CREATE);
}
I'm expecting an event for the creation and modification of the file, and the file is in fact being created when I look for it in the terminal, but I do not receive any event.
However, when I manually create the file with echo "test" > test.file I do receive the create and modify events.
Is there a reason for this behavior? Am I approaching this the wrong way?
I created the following without my library and it works just fine. I'm obviously doing something wrong in my own code. Thanks.
#include <atomic>
#include <condition_variable>
#include <fstream>
#include <functional>
#include <iostream>
#include <iterator>
#include <string>
#include <thread>
#include <linux/limits.h>
#include <sys/inotify.h>
#include <unistd.h>
std::mutex cout_mtx;
std::string path("/tmp/test-inotify");
std::string filename("test.file");
void print(std::string const & msg)
{
std::lock_guard<std::mutex> lock(cout_mtx);
std::cout << msg << std::endl;
}
void thread_fn(int fd, std::function<void(int, std::string const & name)> callback)
{
std::string buffer;
buffer.resize(64 * (sizeof(inotify_event) + NAME_MAX + 1));
while (true)
{
int bytes_read = ::read(fd, &buffer[0], buffer.size());
if (bytes_read == -1)
{
if (errno != EAGAIN)
{
print("Fatal error to call read");
break;
}
}
int offset = 0;
inotify_event const * event = nullptr;
for (; offset < bytes_read; offset += sizeof(inotify_event) + event->len)
{
event = reinterpret_cast<inotify_event const*>(&buffer[offset]);
if (event->mask & IN_IGNORED)
{
// rewatch
int wd = ::inotify_add_watch(fd, path.c_str(), IN_CREATE);
if (wd == -1)
{
print("Unable to rewatch directory");
break;
}
}
std::string name;
std::copy(&event->name[0], &event->name[event->len], std::back_inserter(name));
int event_value = event->mask & 0xffff;
callback(event_value, name);
}
}
}
int main()
{
int fd = ::inotify_init1(IN_NONBLOCK);
if (fd == -1)
{
print("inotifiy_init1 failed");
return errno;
}
int wd = ::inotify_add_watch(fd, path.c_str(), IN_CREATE);
if (wd == -1)
{
print("inotify_add_watch failed");
return errno;
}
std::atomic<bool> ready;
std::mutex ready_mtx;
std::condition_variable condition;
int first_event = -1;
std::thread thread([&]()
{
thread_fn(fd,
[&](int event, std::string const & name)
{
std::unique_lock<std::mutex> lock(ready_mtx);
print(std::to_string(event));
if (event == IN_CREATE)
{
first_event = event;
print(name + " was created");
}
condition.notify_all();
ready = true;
});
});
{
std::ofstream file(path + "/" + filename);
}
{
std::unique_lock<std::mutex> lock(ready_mtx);
while (!ready)
condition.wait(lock);
if (first_event == IN_CREATE)
print("success");
else
print("failure");
}
thread.join();
return 0;
}
I am reading data off a socket using boost asio async_read(). I have the following members of the reader class which persist through the lifetime of the program:
boost::asio::streambuf recv_data;
std::istream stream_is(&recv_data);
The async_read call looks like this:
boost::asio::async_read(ep_ptr->get_sock(), recv_data, boost::asio::transfer_exactly(n),
boost::bind(&IProtocol::handle_data_recv, &protocol,
boost::asio::placeholders::error));
My question is, what would happen if I am reading 'n' bytes off the socket and the size of the streambuf is less than 'n' so it resizes itself. Do I need to re-create the std::istream since the internal streambuf buffer that the std::istream is holding might now be freed/deallocated?
No, thankfully, the binding doesn't care that recv_data's internals may get reallocated and it instead binds to the recv_data object itself. Here is a working example of a downloader I wrote which you can see I do not re-allocate the buffer in between reads.
In the same way you can safely share a reference to a vector and not care if the internals of the vector are re-allocated (unless you start pointing at memory addresses of the vector's elements directly, or using iterators after they become invalidated. The handle to the vector remains valid, and in this same way, the handle to the streambuf remains valid to the istream and it works just fine).
download.h
#ifndef _MV_DOWNLOAD_H_
#define _MV_DOWNLOAD_H_
#include <string>
#include <iostream>
#include <istream>
#include <ostream>
#include <fstream>
#include <algorithm>
#include "Network/url.h"
#include "Utility/generalUtility.h"
#include <boost/asio.hpp>
#include <boost/bind.hpp>
namespace MV {
struct HttpHeader {
std::string version;
int status = 0;
std::string message;
std::map<std::string, std::string> values;
std::vector<std::string> bounces;
bool success = false;
std::string errorMessage;
size_t contentLength;
HttpHeader() {
}
HttpHeader(std::istream& response_stream) {
read(response_stream);
}
void read(std::istream& response_stream);
};
inline std::ostream& operator<<(std::ostream& os, const HttpHeader& obj) {
os << "\\/______HTTP_HEADER______\\/\nVersion [" << obj.version << "] Status [" << obj.status << "] Message [" << obj.message << "]\n";
os << "||-----------------------||\n";
for (auto&& kvp : obj.values) {
os << "[" << kvp.first << "]: " << kvp.second << "\n";
}
os << "\n||--------Bounces--------||\n";
for (size_t i = 0; i < obj.bounces.size(); ++i) {
os << i << ": " << obj.bounces[i] << "\n";
}
os << "/\\_______________________/\\" << std::endl;
return os;
}
inline std::istream& operator>>(std::istream& a_is, HttpHeader& a_obj) {
a_obj.read(a_is);
return a_is;
}
class DownloadRequest : public std::enable_shared_from_this<DownloadRequest> {
public:
static std::shared_ptr<DownloadRequest> make(const MV::Url& a_url, const std::shared_ptr<std::ostream> &a_streamOutput) {
auto result = std::shared_ptr<DownloadRequest>(new DownloadRequest(a_streamOutput));
result->perform(a_url);
return result;
}
//onComplete is called on success or error at the end of the download.
static std::shared_ptr<DownloadRequest> make(const std::shared_ptr<boost::asio::io_service> &a_ioService, const MV::Url& a_url, const std::shared_ptr<std::ostream> &a_streamOutput, std::function<void (std::shared_ptr<DownloadRequest>)> a_onComplete) {
auto result = std::shared_ptr<DownloadRequest>(new DownloadRequest(a_streamOutput));
result->onComplete = a_onComplete;
result->ioService = a_ioService;
result->perform(a_url);
return result;
}
HttpHeader& header() {
return headerData;
}
MV::Url finalUrl() {
return currentUrl;
}
MV::Url inputUrl() {
return originalUrl;
}
private:
DownloadRequest(const std::shared_ptr<std::ostream> &a_streamOutput) :
streamOutput(a_streamOutput) {
}
void perform(const MV::Url& a_url);
bool initializeSocket();
void initiateRequest(const MV::Url& a_url);
void handleResolve(const boost::system::error_code& err, boost::asio::ip::tcp::resolver::iterator endpoint_iterator);
void handleConnect(const boost::system::error_code& err, boost::asio::ip::tcp::resolver::iterator endpoint_iterator);
void handleWriteRequest(const boost::system::error_code& err);
void handleReadHeaders(const boost::system::error_code& err);
void handleReadContent(const boost::system::error_code& err);
void readResponseToStream() {
(*streamOutput) << &(*response);
}
std::shared_ptr<boost::asio::io_service> ioService;
std::unique_ptr<boost::asio::ip::tcp::resolver> resolver;
std::unique_ptr<boost::asio::ip::tcp::socket> socket;
std::unique_ptr<std::istream> responseStream;
std::unique_ptr<boost::asio::streambuf> request;
std::unique_ptr<boost::asio::streambuf> response;
std::shared_ptr<std::ostream> streamOutput;
HttpHeader headerData;
MV::Url currentUrl;
MV::Url originalUrl;
std::function<void(std::shared_ptr<DownloadRequest>)> onComplete;
};
std::string DownloadString(const MV::Url& a_url);
HttpHeader DownloadFile(const MV::Url& a_url, const std::string &a_path);
void DownloadFile(const std::shared_ptr<boost::asio::io_service> &a_ioService, const MV::Url& a_url, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete = std::function<void(std::shared_ptr<DownloadRequest>)>());
void DownloadFiles(const std::vector<MV::Url>& a_url, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete = std::function<void(std::shared_ptr<DownloadRequest>)>());
void DownloadFiles(const std::shared_ptr<boost::asio::io_service> &a_ioService, const std::vector<MV::Url>& a_url, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete = std::function<void(std::shared_ptr<DownloadRequest>)>(), std::function<void()> a_onAllComplete = std::function<void()>());
}
#endif
download.cpp
#include "download.h"
#include <boost/filesystem.hpp>
#include <atomic>
namespace MV{
void HttpHeader::read(std::istream& response_stream) {
values.clear();
response_stream >> version;
std::string status_code;
response_stream >> status_code;
try {
status = std::stoi(status_code);
} catch (...) {
status = 0;
}
getline_platform_agnostic(response_stream, message);
if (!message.empty() && message[0] == ' ') { message = message.substr(1); }
std::string header;
while (getline_platform_agnostic(response_stream, header) && !header.empty()) {
auto index = header.find_first_of(':');
if (index != std::string::npos && index > 0) {
auto key = header.substr(0, index);
auto value = (index + 2 >= header.size()) ? "" : header.substr(index + 2);
std::transform(key.begin(), key.end(), key.begin(), [](char c) {return std::tolower(c); });
values[key] = value;
if (toLower(key) == "content-length") {
try {
contentLength = static_cast<size_t>(stol(value));
} catch (std::exception &e) {
std::cerr << e.what() << std::endl;
contentLength = 0;
}
}
}
}
}
std::string DownloadString(const Url& a_url) {
auto result = std::make_shared<std::stringstream>();
if (DownloadRequest::make(a_url, result)->header().success) {
return result->str();
} else {
return "";
}
}
MV::HttpHeader DownloadFile(const Url& a_url, const std::string &a_path) {
HttpHeader header;
{
boost::filesystem::create_directories(boost::filesystem::path(a_path).parent_path());
auto outFile = std::make_shared<std::ofstream>(a_path, std::ofstream::out | std::ofstream::binary);
auto request = DownloadRequest::make(a_url, outFile);
header = request->header();
}
if (!header.success) {
std::remove(a_path.c_str());
}
return header;
}
void DownloadFile(const std::shared_ptr<boost::asio::io_service> &a_ioService, const MV::Url& a_url, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete) {
boost::filesystem::create_directories(boost::filesystem::path(a_path).parent_path());
auto outFile = std::make_shared<std::ofstream>(a_path, std::ofstream::out | std::ofstream::binary);
auto request = DownloadRequest::make(a_ioService, a_url, outFile, [a_path, a_onComplete](std::shared_ptr<DownloadRequest> a_result) {
if (!a_result->header().success) {
std::remove(a_path.c_str());
}
if (a_onComplete) { a_onComplete(a_result); }
});
}
void DownloadFiles(const std::vector<MV::Url>& a_urls, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete) {
auto service = std::make_shared<boost::asio::io_service>();
for (auto&& url : a_urls) {
DownloadFile(service, url, a_path + boost::filesystem::path(url.path()).filename().string(), a_onComplete);
}
service->run();
}
void DownloadFiles(const std::shared_ptr<boost::asio::io_service> &a_ioService, const std::vector<MV::Url>& a_urls, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete, std::function<void()> a_onAllComplete) {
size_t totalFiles = a_urls.size();
for (auto&& url : a_urls) {
auto counter = std::make_shared<std::atomic<size_t>>(0);
DownloadFile(a_ioService, url, a_path + boost::filesystem::path(url.path()).filename().string(), [=](std::shared_ptr<DownloadRequest> a_request) {
a_onComplete(a_request);
if (++(*counter) == totalFiles) {
a_onAllComplete();
}
});
}
}
void DownloadRequest::handleReadContent(const boost::system::error_code& err) {
if (!err) {
readResponseToStream();
if (onComplete) { onComplete(shared_from_this()); }
} else if (err != boost::asio::error::eof) {
headerData.success = false;
headerData.errorMessage = "Download Read Content Failure: " + err.message();
std::cerr << headerData.errorMessage << std::endl;
if (onComplete) { onComplete(shared_from_this()); }
}
}
void DownloadRequest::handleReadHeaders(const boost::system::error_code& err) {
if (!err) {
responseStream = std::make_unique<std::istream>(&(*response));
headerData.read(*responseStream);
headerData.success = true;
headerData.errorMessage = "";
if (headerData.status >= 300 && headerData.status < 400 && headerData.bounces.size() < 32 && headerData.values.find("location") != headerData.values.end()) {
headerData.bounces.push_back(currentUrl.toString());
initiateRequest(headerData.values["location"]);
} else {
auto amountLeftToRead = headerData.contentLength - response->size();
if (response->size() > 0) {
readResponseToStream();
}
if (amountLeftToRead > 0) {
boost::asio::async_read(*socket, *response, boost::asio::transfer_at_least(amountLeftToRead), boost::bind(&DownloadRequest::handleReadContent, shared_from_this(), boost::asio::placeholders::error));
} else {
if (onComplete) { onComplete(shared_from_this()); }
}
}
} else {
headerData.success = false;
headerData.errorMessage = "Download Read Header Failure: " + err.message();
std::cerr << headerData.errorMessage << std::endl;
if (onComplete) { onComplete(shared_from_this()); }
}
}
void DownloadRequest::handleWriteRequest(const boost::system::error_code& err) {
if (!err) {
boost::asio::async_read_until(*socket, *response, "\r\n\r\n", boost::bind(&DownloadRequest::handleReadHeaders, shared_from_this(), boost::asio::placeholders::error));
} else {
headerData.success = false;
headerData.errorMessage = "Download Write Failure: " + err.message();
std::cerr << headerData.errorMessage << std::endl;
if (onComplete) { onComplete(shared_from_this()); }
}
}
void DownloadRequest::handleConnect(const boost::system::error_code& err, boost::asio::ip::tcp::resolver::iterator endpoint_iterator) {
if (!err) {
// The connection was successful. Send the request.
boost::asio::async_write(*socket, *request, boost::bind(&DownloadRequest::handleWriteRequest, shared_from_this(), boost::asio::placeholders::error));
} else if (endpoint_iterator != boost::asio::ip::tcp::resolver::iterator()) {
// The connection failed. Try the next endpoint in the list.
socket->close();
boost::asio::ip::tcp::endpoint endpoint = *endpoint_iterator;
socket->async_connect(endpoint, boost::bind(&DownloadRequest::handleConnect, shared_from_this(), boost::asio::placeholders::error, ++endpoint_iterator));
} else {
headerData.success = false;
headerData.errorMessage = "Download Connection Failure: " + err.message();
std::cerr << headerData.errorMessage << std::endl;
if (onComplete) { onComplete(shared_from_this()); }
}
}
void DownloadRequest::handleResolve(const boost::system::error_code& err, boost::asio::ip::tcp::resolver::iterator endpoint_iterator) {
if (!err) {
// Attempt a connection to the first endpoint in the list. Each endpoint
// will be tried until we successfully establish a connection.
boost::asio::ip::tcp::endpoint endpoint = *endpoint_iterator;
socket->async_connect(endpoint, boost::bind(&DownloadRequest::handleConnect, shared_from_this(), boost::asio::placeholders::error, ++endpoint_iterator));
} else {
headerData.success = false;
headerData.errorMessage = "Download Resolve Failure: " + err.message();
std::cerr << headerData.errorMessage << std::endl;
if (onComplete) { onComplete(shared_from_this()); }
}
}
void DownloadRequest::initiateRequest(const MV::Url& a_url) {
socket->close();
currentUrl = a_url;
request = std::make_unique<boost::asio::streambuf>();
response = std::make_unique<boost::asio::streambuf>();
using boost::asio::ip::tcp;
std::ostream requestStream(&(*request));
requestStream << "GET " << a_url.pathAndQuery() << " HTTP/1.1\r\n";
requestStream << "Host: " << a_url.host() << "\r\n";
requestStream << "Accept: */*\r\n";
requestStream << "Connection: close\r\n\r\n";
tcp::resolver::query query(a_url.host(), "http");
resolver->async_resolve(query, boost::bind(&DownloadRequest::handleResolve, shared_from_this(), boost::asio::placeholders::error, boost::asio::placeholders::iterator));
}
bool DownloadRequest::initializeSocket() {
bool created = false;
if (!ioService) {
ioService = std::make_shared<boost::asio::io_service>();
created = true;
}
resolver = std::make_unique<boost::asio::ip::tcp::resolver>(*ioService);
socket = std::make_unique<boost::asio::ip::tcp::socket>(*ioService);
return created;
}
void DownloadRequest::perform(const MV::Url& a_url) {
originalUrl = a_url;
try {
bool needToCallRun = initializeSocket();
initiateRequest(a_url);
if (needToCallRun) {
ioService->run();
}
} catch (...) {
headerData.success = false;
headerData.errorMessage = "Exception thrown to top level.";
std::cerr << headerData.errorMessage << std::endl;
onComplete(shared_from_this());
}
}
}
generalUtility.h (part of it anyway, just for this reference)
inline std::istream& getline_platform_agnostic(std::istream& is, std::string& t) {
t.clear();
// The characters in the stream are read one-by-one using a std::streambuf.
// That is faster than reading them one-by-one using the std::istream.
// Code that uses streambuf this way must be guarded by a sentry object.
// The sentry object performs various tasks,
// such as thread synchronization and updating the stream state.
std::istream::sentry se(is, true);
std::streambuf* sb = is.rdbuf();
for (;;) {
int c = sb->sbumpc();
switch (c) {
case '\n':
return is;
case '\r':
if (sb->sgetc() == '\n')
sb->sbumpc();
return is;
case EOF:
// Also handle the case when the last line has no line ending
if (t.empty())
is.setstate(std::ios::eofbit);
return is;
default:
t += (char)c;
}
}
}
inline std::string toLower(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](char c) { return std::tolower(c); });
return s;
}
url.h
Modified (slightly, just changed some naming scheme stuff)
https://github.com/keyz182/Poco-1.4.3/blob/master/Foundation/include/Poco/URI.h
https://github.com/keyz182/Poco-1.4.3/blob/master/Foundation/src/URI.cpp
I am reading data off a socket using boost asio async_read(). I have the following members of the reader class which persist through the lifetime of the program:
boost::asio::streambuf recv_data;
std::istream stream_is(&recv_data);
The async_read call looks like this:
boost::asio::async_read(ep_ptr->get_sock(), recv_data, boost::asio::transfer_exactly(n),
boost::bind(&IProtocol::handle_data_recv, &protocol,
boost::asio::placeholders::error));
My question is, what would happen if I am reading 'n' bytes off the socket and the size of the streambuf is less than 'n' so it resizes itself. Do I need to re-create the std::istream since the internal streambuf buffer that the std::istream is holding might now be freed/deallocated?
No, thankfully, the binding doesn't care that recv_data's internals may get reallocated and it instead binds to the recv_data object itself. Here is a working example of a downloader I wrote which you can see I do not re-allocate the buffer in between reads.
In the same way you can safely share a reference to a vector and not care if the internals of the vector are re-allocated (unless you start pointing at memory addresses of the vector's elements directly, or using iterators after they become invalidated. The handle to the vector remains valid, and in this same way, the handle to the streambuf remains valid to the istream and it works just fine).
download.h
#ifndef _MV_DOWNLOAD_H_
#define _MV_DOWNLOAD_H_
#include <string>
#include <iostream>
#include <istream>
#include <ostream>
#include <fstream>
#include <algorithm>
#include "Network/url.h"
#include "Utility/generalUtility.h"
#include <boost/asio.hpp>
#include <boost/bind.hpp>
namespace MV {
struct HttpHeader {
std::string version;
int status = 0;
std::string message;
std::map<std::string, std::string> values;
std::vector<std::string> bounces;
bool success = false;
std::string errorMessage;
size_t contentLength;
HttpHeader() {
}
HttpHeader(std::istream& response_stream) {
read(response_stream);
}
void read(std::istream& response_stream);
};
inline std::ostream& operator<<(std::ostream& os, const HttpHeader& obj) {
os << "\\/______HTTP_HEADER______\\/\nVersion [" << obj.version << "] Status [" << obj.status << "] Message [" << obj.message << "]\n";
os << "||-----------------------||\n";
for (auto&& kvp : obj.values) {
os << "[" << kvp.first << "]: " << kvp.second << "\n";
}
os << "\n||--------Bounces--------||\n";
for (size_t i = 0; i < obj.bounces.size(); ++i) {
os << i << ": " << obj.bounces[i] << "\n";
}
os << "/\\_______________________/\\" << std::endl;
return os;
}
inline std::istream& operator>>(std::istream& a_is, HttpHeader& a_obj) {
a_obj.read(a_is);
return a_is;
}
class DownloadRequest : public std::enable_shared_from_this<DownloadRequest> {
public:
static std::shared_ptr<DownloadRequest> make(const MV::Url& a_url, const std::shared_ptr<std::ostream> &a_streamOutput) {
auto result = std::shared_ptr<DownloadRequest>(new DownloadRequest(a_streamOutput));
result->perform(a_url);
return result;
}
//onComplete is called on success or error at the end of the download.
static std::shared_ptr<DownloadRequest> make(const std::shared_ptr<boost::asio::io_service> &a_ioService, const MV::Url& a_url, const std::shared_ptr<std::ostream> &a_streamOutput, std::function<void (std::shared_ptr<DownloadRequest>)> a_onComplete) {
auto result = std::shared_ptr<DownloadRequest>(new DownloadRequest(a_streamOutput));
result->onComplete = a_onComplete;
result->ioService = a_ioService;
result->perform(a_url);
return result;
}
HttpHeader& header() {
return headerData;
}
MV::Url finalUrl() {
return currentUrl;
}
MV::Url inputUrl() {
return originalUrl;
}
private:
DownloadRequest(const std::shared_ptr<std::ostream> &a_streamOutput) :
streamOutput(a_streamOutput) {
}
void perform(const MV::Url& a_url);
bool initializeSocket();
void initiateRequest(const MV::Url& a_url);
void handleResolve(const boost::system::error_code& err, boost::asio::ip::tcp::resolver::iterator endpoint_iterator);
void handleConnect(const boost::system::error_code& err, boost::asio::ip::tcp::resolver::iterator endpoint_iterator);
void handleWriteRequest(const boost::system::error_code& err);
void handleReadHeaders(const boost::system::error_code& err);
void handleReadContent(const boost::system::error_code& err);
void readResponseToStream() {
(*streamOutput) << &(*response);
}
std::shared_ptr<boost::asio::io_service> ioService;
std::unique_ptr<boost::asio::ip::tcp::resolver> resolver;
std::unique_ptr<boost::asio::ip::tcp::socket> socket;
std::unique_ptr<std::istream> responseStream;
std::unique_ptr<boost::asio::streambuf> request;
std::unique_ptr<boost::asio::streambuf> response;
std::shared_ptr<std::ostream> streamOutput;
HttpHeader headerData;
MV::Url currentUrl;
MV::Url originalUrl;
std::function<void(std::shared_ptr<DownloadRequest>)> onComplete;
};
std::string DownloadString(const MV::Url& a_url);
HttpHeader DownloadFile(const MV::Url& a_url, const std::string &a_path);
void DownloadFile(const std::shared_ptr<boost::asio::io_service> &a_ioService, const MV::Url& a_url, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete = std::function<void(std::shared_ptr<DownloadRequest>)>());
void DownloadFiles(const std::vector<MV::Url>& a_url, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete = std::function<void(std::shared_ptr<DownloadRequest>)>());
void DownloadFiles(const std::shared_ptr<boost::asio::io_service> &a_ioService, const std::vector<MV::Url>& a_url, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete = std::function<void(std::shared_ptr<DownloadRequest>)>(), std::function<void()> a_onAllComplete = std::function<void()>());
}
#endif
download.cpp
#include "download.h"
#include <boost/filesystem.hpp>
#include <atomic>
namespace MV{
void HttpHeader::read(std::istream& response_stream) {
values.clear();
response_stream >> version;
std::string status_code;
response_stream >> status_code;
try {
status = std::stoi(status_code);
} catch (...) {
status = 0;
}
getline_platform_agnostic(response_stream, message);
if (!message.empty() && message[0] == ' ') { message = message.substr(1); }
std::string header;
while (getline_platform_agnostic(response_stream, header) && !header.empty()) {
auto index = header.find_first_of(':');
if (index != std::string::npos && index > 0) {
auto key = header.substr(0, index);
auto value = (index + 2 >= header.size()) ? "" : header.substr(index + 2);
std::transform(key.begin(), key.end(), key.begin(), [](char c) {return std::tolower(c); });
values[key] = value;
if (toLower(key) == "content-length") {
try {
contentLength = static_cast<size_t>(stol(value));
} catch (std::exception &e) {
std::cerr << e.what() << std::endl;
contentLength = 0;
}
}
}
}
}
std::string DownloadString(const Url& a_url) {
auto result = std::make_shared<std::stringstream>();
if (DownloadRequest::make(a_url, result)->header().success) {
return result->str();
} else {
return "";
}
}
MV::HttpHeader DownloadFile(const Url& a_url, const std::string &a_path) {
HttpHeader header;
{
boost::filesystem::create_directories(boost::filesystem::path(a_path).parent_path());
auto outFile = std::make_shared<std::ofstream>(a_path, std::ofstream::out | std::ofstream::binary);
auto request = DownloadRequest::make(a_url, outFile);
header = request->header();
}
if (!header.success) {
std::remove(a_path.c_str());
}
return header;
}
void DownloadFile(const std::shared_ptr<boost::asio::io_service> &a_ioService, const MV::Url& a_url, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete) {
boost::filesystem::create_directories(boost::filesystem::path(a_path).parent_path());
auto outFile = std::make_shared<std::ofstream>(a_path, std::ofstream::out | std::ofstream::binary);
auto request = DownloadRequest::make(a_ioService, a_url, outFile, [a_path, a_onComplete](std::shared_ptr<DownloadRequest> a_result) {
if (!a_result->header().success) {
std::remove(a_path.c_str());
}
if (a_onComplete) { a_onComplete(a_result); }
});
}
void DownloadFiles(const std::vector<MV::Url>& a_urls, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete) {
auto service = std::make_shared<boost::asio::io_service>();
for (auto&& url : a_urls) {
DownloadFile(service, url, a_path + boost::filesystem::path(url.path()).filename().string(), a_onComplete);
}
service->run();
}
void DownloadFiles(const std::shared_ptr<boost::asio::io_service> &a_ioService, const std::vector<MV::Url>& a_urls, const std::string &a_path, std::function<void(std::shared_ptr<DownloadRequest>)> a_onComplete, std::function<void()> a_onAllComplete) {
size_t totalFiles = a_urls.size();
for (auto&& url : a_urls) {
auto counter = std::make_shared<std::atomic<size_t>>(0);
DownloadFile(a_ioService, url, a_path + boost::filesystem::path(url.path()).filename().string(), [=](std::shared_ptr<DownloadRequest> a_request) {
a_onComplete(a_request);
if (++(*counter) == totalFiles) {
a_onAllComplete();
}
});
}
}
void DownloadRequest::handleReadContent(const boost::system::error_code& err) {
if (!err) {
readResponseToStream();
if (onComplete) { onComplete(shared_from_this()); }
} else if (err != boost::asio::error::eof) {
headerData.success = false;
headerData.errorMessage = "Download Read Content Failure: " + err.message();
std::cerr << headerData.errorMessage << std::endl;
if (onComplete) { onComplete(shared_from_this()); }
}
}
void DownloadRequest::handleReadHeaders(const boost::system::error_code& err) {
if (!err) {
responseStream = std::make_unique<std::istream>(&(*response));
headerData.read(*responseStream);
headerData.success = true;
headerData.errorMessage = "";
if (headerData.status >= 300 && headerData.status < 400 && headerData.bounces.size() < 32 && headerData.values.find("location") != headerData.values.end()) {
headerData.bounces.push_back(currentUrl.toString());
initiateRequest(headerData.values["location"]);
} else {
auto amountLeftToRead = headerData.contentLength - response->size();
if (response->size() > 0) {
readResponseToStream();
}
if (amountLeftToRead > 0) {
boost::asio::async_read(*socket, *response, boost::asio::transfer_at_least(amountLeftToRead), boost::bind(&DownloadRequest::handleReadContent, shared_from_this(), boost::asio::placeholders::error));
} else {
if (onComplete) { onComplete(shared_from_this()); }
}
}
} else {
headerData.success = false;
headerData.errorMessage = "Download Read Header Failure: " + err.message();
std::cerr << headerData.errorMessage << std::endl;
if (onComplete) { onComplete(shared_from_this()); }
}
}
void DownloadRequest::handleWriteRequest(const boost::system::error_code& err) {
if (!err) {
boost::asio::async_read_until(*socket, *response, "\r\n\r\n", boost::bind(&DownloadRequest::handleReadHeaders, shared_from_this(), boost::asio::placeholders::error));
} else {
headerData.success = false;
headerData.errorMessage = "Download Write Failure: " + err.message();
std::cerr << headerData.errorMessage << std::endl;
if (onComplete) { onComplete(shared_from_this()); }
}
}
void DownloadRequest::handleConnect(const boost::system::error_code& err, boost::asio::ip::tcp::resolver::iterator endpoint_iterator) {
if (!err) {
// The connection was successful. Send the request.
boost::asio::async_write(*socket, *request, boost::bind(&DownloadRequest::handleWriteRequest, shared_from_this(), boost::asio::placeholders::error));
} else if (endpoint_iterator != boost::asio::ip::tcp::resolver::iterator()) {
// The connection failed. Try the next endpoint in the list.
socket->close();
boost::asio::ip::tcp::endpoint endpoint = *endpoint_iterator;
socket->async_connect(endpoint, boost::bind(&DownloadRequest::handleConnect, shared_from_this(), boost::asio::placeholders::error, ++endpoint_iterator));
} else {
headerData.success = false;
headerData.errorMessage = "Download Connection Failure: " + err.message();
std::cerr << headerData.errorMessage << std::endl;
if (onComplete) { onComplete(shared_from_this()); }
}
}
void DownloadRequest::handleResolve(const boost::system::error_code& err, boost::asio::ip::tcp::resolver::iterator endpoint_iterator) {
if (!err) {
// Attempt a connection to the first endpoint in the list. Each endpoint
// will be tried until we successfully establish a connection.
boost::asio::ip::tcp::endpoint endpoint = *endpoint_iterator;
socket->async_connect(endpoint, boost::bind(&DownloadRequest::handleConnect, shared_from_this(), boost::asio::placeholders::error, ++endpoint_iterator));
} else {
headerData.success = false;
headerData.errorMessage = "Download Resolve Failure: " + err.message();
std::cerr << headerData.errorMessage << std::endl;
if (onComplete) { onComplete(shared_from_this()); }
}
}
void DownloadRequest::initiateRequest(const MV::Url& a_url) {
socket->close();
currentUrl = a_url;
request = std::make_unique<boost::asio::streambuf>();
response = std::make_unique<boost::asio::streambuf>();
using boost::asio::ip::tcp;
std::ostream requestStream(&(*request));
requestStream << "GET " << a_url.pathAndQuery() << " HTTP/1.1\r\n";
requestStream << "Host: " << a_url.host() << "\r\n";
requestStream << "Accept: */*\r\n";
requestStream << "Connection: close\r\n\r\n";
tcp::resolver::query query(a_url.host(), "http");
resolver->async_resolve(query, boost::bind(&DownloadRequest::handleResolve, shared_from_this(), boost::asio::placeholders::error, boost::asio::placeholders::iterator));
}
bool DownloadRequest::initializeSocket() {
bool created = false;
if (!ioService) {
ioService = std::make_shared<boost::asio::io_service>();
created = true;
}
resolver = std::make_unique<boost::asio::ip::tcp::resolver>(*ioService);
socket = std::make_unique<boost::asio::ip::tcp::socket>(*ioService);
return created;
}
void DownloadRequest::perform(const MV::Url& a_url) {
originalUrl = a_url;
try {
bool needToCallRun = initializeSocket();
initiateRequest(a_url);
if (needToCallRun) {
ioService->run();
}
} catch (...) {
headerData.success = false;
headerData.errorMessage = "Exception thrown to top level.";
std::cerr << headerData.errorMessage << std::endl;
onComplete(shared_from_this());
}
}
}
generalUtility.h (part of it anyway, just for this reference)
inline std::istream& getline_platform_agnostic(std::istream& is, std::string& t) {
t.clear();
// The characters in the stream are read one-by-one using a std::streambuf.
// That is faster than reading them one-by-one using the std::istream.
// Code that uses streambuf this way must be guarded by a sentry object.
// The sentry object performs various tasks,
// such as thread synchronization and updating the stream state.
std::istream::sentry se(is, true);
std::streambuf* sb = is.rdbuf();
for (;;) {
int c = sb->sbumpc();
switch (c) {
case '\n':
return is;
case '\r':
if (sb->sgetc() == '\n')
sb->sbumpc();
return is;
case EOF:
// Also handle the case when the last line has no line ending
if (t.empty())
is.setstate(std::ios::eofbit);
return is;
default:
t += (char)c;
}
}
}
inline std::string toLower(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](char c) { return std::tolower(c); });
return s;
}
url.h
Modified (slightly, just changed some naming scheme stuff)
https://github.com/keyz182/Poco-1.4.3/blob/master/Foundation/include/Poco/URI.h
https://github.com/keyz182/Poco-1.4.3/blob/master/Foundation/src/URI.cpp
i solved couple of my redefinition problems but still have one:
Error 2 error LNK2005: "class ConsoleCommandHandler commandHandler" (?commandHandler##3VConsoleCommandHandler##A) already defined in IRC.obj C:\Users\Łukasz\Desktop\IRCClient-master\Magic.obj
Here are the .h files
magic.h
#ifndef Magic_h
#define Magic_h
#include <iostream>
#include <signal.h>
#include <cstdlib>
#include <map>
#include <algorithm>
#include "src\Thread.h"
#include "src\IRCClient.h"
void signalHandler(int signal);
class ConsoleCommandHandler
{
public:
bool AddCommand(std::string name, int argCount, void(*handler)(std::string /*params*/, IRCClient* /*client*/));
void ParseCommand(std::string command, IRCClient* client);
private:
struct CommandEntry
{
int argCount;
void(*handler)(std::string /*arguments*/, IRCClient* /*client*/);
};
std::map<std::string, CommandEntry> _commands;
};
ConsoleCommandHandler commandHandler;
void msgCommand(std::string arguments, IRCClient* client);
void joinCommand(std::string channel, IRCClient* client);
void partCommand(std::string channel, IRCClient* client);
void ctcpCommand(std::string arguments, IRCClient* client);
ThreadReturn inputThread(void* client);
#endif
magic.cpp
#include "Magic.h"
void signalHandler(int signal)
{
volatile bool running;
running = false;
};
bool ConsoleCommandHandler::AddCommand(std::string name, int argCount, void(*handler)(std::string /*params*/, IRCClient* /*client*/))
{
CommandEntry entry;
entry.argCount = argCount;
entry.handler = handler;
std::transform(name.begin(), name.end(), name.begin(), towlower);
_commands.insert(std::pair<std::string, CommandEntry>(name, entry));
return true;
}
void ConsoleCommandHandler::ParseCommand(std::string command, IRCClient* client)
{
if (_commands.empty())
{
std::cout << "No commands available." << std::endl;
return;
}
if (command[0] == '/')
command = command.substr(1); // Remove the slash
std::string name = command.substr(0, command.find(" "));
std::string args = command.substr(command.find(" ") + 1);
int argCount = std::count(args.begin(), args.end(), ' ');
std::transform(name.begin(), name.end(), name.begin(), towlower);
std::map<std::string, CommandEntry>::const_iterator itr = _commands.find(name);
if (itr == _commands.end())
{
std::cout << "Command not found." << std::endl;
return;
}
if (++argCount < itr->second.argCount)
{
std::cout << "Insuficient arguments." << std::endl;
return;
}
(*(itr->second.handler))(args, client);
}
struct CommandEntry
{
int argCount;
void(*handler)(std::string /*arguments*/, IRCClient* /*client*/);
};
std::map<std::string, CommandEntry> _commands;
void msgCommand(std::string arguments, IRCClient* client)
{
std::string to = arguments.substr(0, arguments.find(" "));
std::string text = arguments.substr(arguments.find(" ") + 1);
std::cout << "To " + to + ": " + text << std::endl;
client->SendIRC("PRIVMSG " + to + " :" + text);
};
void joinCommand(std::string channel, IRCClient* client)
{
if (channel[0] != '#')
channel = "#" + channel;
client->SendIRC("JOIN " + channel);
}
void partCommand(std::string channel, IRCClient* client)
{
if (channel[0] != '#')
channel = "#" + channel;
client->SendIRC("PART " + channel);
}
void ctcpCommand(std::string arguments, IRCClient* client)
{
std::string to = arguments.substr(0, arguments.find(" "));
std::string text = arguments.substr(arguments.find(" ") + 1);
std::transform(text.begin(), text.end(), text.begin(), towupper);
client->SendIRC("PRIVMSG " + to + " :\001" + text + "\001");
}
ThreadReturn inputThread(void* client)
{
std::string command;
commandHandler.AddCommand("msg", 2, &msgCommand);
commandHandler.AddCommand("join", 1, &joinCommand);
commandHandler.AddCommand("part", 1, &partCommand);
commandHandler.AddCommand("ctcp", 2, &ctcpCommand);
while (true)
{
getline(std::cin, command);
if (command == "")
continue;
if (command[0] == '/')
commandHandler.ParseCommand(command, (IRCClient*)client);
else
((IRCClient*)client)->SendIRC(command);
if (command == "quit")
break;
}
#ifdef _WIN32
_endthread();
#else
pthread_exit(NULL);
#endif
}
irc.h
#pragma once
#include <iostream>
#include <signal.h>
#include <cstdlib>
#include <map>
#include <algorithm>
#include "Magic.h"
#include <msclr/marshal.h>
#include <msclr/marshal_cppstd.h>
#using <mscorlib.dll>
namespace IRCclient {
using namespace System;
using namespace System::ComponentModel;
using namespace System::Collections;
using namespace System::Windows::Forms;
using namespace System::Data;
using namespace System::Drawing;
using namespace System::IO;
using namespace System::Runtime::InteropServices;
using namespace msclr::interop;
/// <summary>
/// Summary for MyForm
/// </summary>
private: System::Void connect_button_Click(System::Object^ sender, System::EventArgs^ e)
{
if ((server_box->Text == "") || (port_box->Text == "") || (username_box->Text == "") || (channel_box->Text == ""))
{
MessageBox::Show("Wypełnij wszystkie pola", "Puste pola", MessageBoxButtons::OK, MessageBoxIcon::Warning);
server_box->Focus();
}
else
{
String^ host_string = server_box->Text;
char* host = (char*)(void*)Marshal::StringToHGlobalAnsi(host_string);
String^ port_string = port_box->Text;
int port;
//String^ port_string = port.ToString();
String^ nick_string = username_box->Text;
std::string nick(marshal_as<std::string>(nick_string));
std::string user = "test";
IRCClient client;
volatile bool running;
Thread thread;
thread.Start(&inputThread, &client);
if (client.InitSocket())
{
content_box->Text = "Socket initialized. Connecting..." + "\r\n";
if (client.Connect(host, port))
{
content_box->Text = "Connected. Loggin in..." + "\r\n";
if (client.Login(nick, user))
{
content_box->Text = "Logged." + "\r\n";
running = true;
signal(SIGINT, signalHandler);
while (client.Connected() && running)
client.ReceiveData();
}
if (client.Connected())
client.Disconnect();
content_box->Text = "Disconnected." + "\r\n";
}
}
}
};
};
};
In the Magic.h file make this change:
extern ConsoleCommandHandler commandHandler;
Then in the Magic.cpp file add this code:
ConsoleCommandHandler commandHandler;
Otherwise both Magic.obj and IRC.obj will end up with ConsoleCommandHandler commandHandler because the header will reserve it twice, once in each obj file.