Vlan id is set to 0 when TPACKET_V2 is used - c++

I have a problem about the usage of this TPACKET_V2 .
My problem is that after setting of this type of packet on socket, when I try to receive some packets I can't read the vlan id from the packet (of course from the header of the packet) the vlan_tci is ever 0.
Now I'm using open suse sp1 and when I run my program on sless sp2 I 'm able to get the vlan id with the same program that doesn't work on sless sp1 but the weird thing is that tcpdump is able to get the vlan id (on this sless) and tcpdump set the TPACKET_V2 (so this means that TPACKET_2 is supported)
My simple project is based on these functions , all called by the function createSocket , then there is a simple method that is reading packets on the socket and there I try to get informations on vlan id (there there is also the relative part used before with the TPACKET_V1)
#include <linux/if_packet.h>
#include <linux/if_ether.h>
#include <netinet/in.h>
#include <netinet/if_ether.h>
#include <netinet/ether.h>
#include <linux/filter.h>
#include <net/if.h>
#include <arpa/inet.h>
enum INTERFACE_T
{
RX_INTERFACE,
TX_INTERFACE
};
static const char* PKT_TYPE[];
// BLOCK_NUM*BLOCK_SIZE = FRAME_NUM*FRAME_SIZE
enum { RX_BLOCK_SIZE = 8192,
RX_BLOCK_NUM = 256,
RX_FRAME_SIZE = 2048,
RX_FRAME_NUM = 1024
};
enum { TX_BLOCK_SIZE = 8192,
TX_BLOCK_NUM = 256,
TX_FRAME_SIZE = 2048,
TX_FRAME_NUM = 1024
};
struct RxFrame {
struct tpacket2_hdr tp_h; // Packet header
uint8_t tp_pad[TPACKET_ALIGN(sizeof(tpacket2_hdr))-sizeof(tpacket2_hdr)];
struct sockaddr_ll sa_ll; // Link level address information
uint8_t sa_ll_pad[14]; //Alignment padding
struct ethhdr eth_h;
} __attribute__((packed));
struct TxFrame
{
struct tpacket_hdr tp_h; // Packet header
uint8_t tp_pad[TPACKET_ALIGN(sizeof(tpacket_hdr))-sizeof(tpacket_hdr)];
// struct vlan_ethhdr vlan_eth_h;
// struct arp arp;
} __attribute__((packed));
struct ring_buff {
struct tpacket_req req;
size_t size; // mmap size
size_t cur_frame;
struct iovec *ring_buffer_;
void *buffer; // mmap
};
int setIfFlags(short int flags)
{
struct ifreq ifr;
memset(&ifr, 0, sizeof(ifr));
strncpy(ifr.ifr_name, if_name_.c_str(), sizeof(ifr.ifr_name));
ifr.ifr_hwaddr.sa_family=getIfArptype();
ifr.ifr_flags |= flags;
if ( ioctl(socket_, SIOCSIFFLAGS, &ifr) == -1)
{
std::cout << "Error: ioctl(SIOSIFFLAGS) failed!" << std::endl;
return 1;
}
return 0;
}
int bindSocket()
{
struct sockaddr_ll sll;
memset(&sll, 0, sizeof(sll));
sll.sll_family = AF_PACKET;
sll.sll_protocol = htons(ETH_P_ALL);
sll.sll_ifindex = ifIndex_;
sll.sll_hatype = 0;
sll.sll_pkttype = 0;
sll.sll_halen = 0;
if (bind(socket_, (struct sockaddr *)&sll, sizeof(sll)) < 0) {
std::cout << "Error: bind() failed!" << std::endl;
return 1;
}
return 0;
}
int packetMmap(ring_buff * rb)
{
assert(rb);
rb->buffer = mmap(0, rb->size, PROT_READ | PROT_WRITE, MAP_SHARED, socket_, 0);
if (rb->buffer == MAP_FAILED) {
std::cout << "Error: mmap() failed!" << std::endl;
return 1;
}
return 0;
}
void packetMunmap(ring_buff * rb)
{
assert(rb);
if (rb->buffer)
{
munmap(rb->buffer, rb->size);
rb->buffer = NULL;
rb->size = 0;
}
}
int frameBufferCreate(ring_buff * rb)
{
assert(rb);
rb->ring_buffer_ = (struct iovec*) malloc(rb->req.tp_frame_nr * sizeof(*rb->ring_buffer_));
if (!rb->ring_buffer_) {
std::cout << "No memory available !!!" << std::endl;
return 1;
}
memset(rb->ring_buffer_, 0, rb->req.tp_frame_nr * sizeof(*rb->ring_buffer_));
for (unsigned int i = 0; i < rb->req.tp_frame_nr; i++) {
rb->ring_buffer_[i].iov_base = static_cast<void *>(static_cast<char *>(rb->buffer)+(i*rb->req.tp_frame_size));
rb->ring_buffer_[i].iov_len = rb->req.tp_frame_size;
}
return 0;
}
void setRingBuffer(struct ring_buff *ringbuf) { rb_ = ringbuf; }
int setVlanTaggingStripping()
{
socklen_t len;
int val;
unsigned int sk_type, tp_reserve, maclen, tp_hdrlen, netoff, macoff;
unsigned int tp_hdr_len;
unsigned int frame_size = RX_FRAME_SIZE;
val = TPACKET_V2;
len = sizeof(val);
if (getsockopt(socket_, SOL_PACKET, PACKET_HDRLEN, &val, &len) < 0) {
std::cout << "Error: getsockopt(SOL_PACKET, PACKET_HDRLEN) failed (can't get TPACKET_V2 header len on packet)" << std::endl;
return 1;
}
tp_hdr_len = val;
std::cout << "TPACKET_V2 header is supported (hdr len is " << val << ")"<< std::endl;
std::cout << "tpacket2_hdrs header is supported (hdr len is " << sizeof(tpacket2_hdr) << ")"<< std::endl;
val = TPACKET_V2;
if (setsockopt(socket_, SOL_PACKET, PACKET_VERSION, &val, sizeof(val)) < 0) {
std::cout << "Error: setsockopt(SOL_PACKET, PACKET_VERSION) failed (can't activate TPACKET_V2 on packet)" << std::endl;
return 1;
}
std::cout << "TPACKET_V2 version is configured !!! " << std::endl;
/* Reserve space for VLAN tag reconstruction */
val = VLAN_TAG_LEN;
if (setsockopt(socket_, SOL_PACKET, PACKET_RESERVE, &val, sizeof(val)) < 0) {
std::cout << "Error: setsockopt(SOL_PACKET, PACKET_RESERVE) failed (can't set up reserve on packet)" << std::endl;
return 1;
}
std::cout<< "Reserve space for VLAN tag reconstruction is configured !!! " << std::endl;
return 0;
}
int setSoBufforce(int optname, int buffSize)
{
if (setsockopt(socket_, SOL_SOCKET, SO_SNDBUFFORCE, &buffSize, sizeof(buffSize)) == -1)
{
std::cout << "Error: setsocketopt("<< (optname == SO_SNDBUFFORCE ? "SO_SNDBUFFORCE" : "SO_RCVBUFFORCE") << ") failed!" << std::endl;
return 1;
}
return 0;
}
createSocket(std::string ifName, INTERFACE_T ifType)
{
if (ifName.empty())
{
std::cout << "Error: interface is empty!" << std::endl;;
return NULL;
}
//Create the socket
if ( (socket_ = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL))) == -1 )
{
std::cout << "Error: calling socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ALL)) failed!" << std::endl;
}
std::cout << "Creating Socket on interface= " << ifName << " to listen to ETH_P_ALL"<<std::endl;;
s->setIfFlags(IFF_PROMISC|IFF_BROADCAST);
//allocate space for ring buffer
ring_buff *rb = (ring_buff *) malloc(sizeof(ring_buff));
// use the same size for RX/TX ring
//set the version , here I insert the use of TPACKET_V2!
setVlanTaggingStripping();
rb->req.tp_block_size = RX_BLOCK_SIZE;
rb->req.tp_block_nr = RX_BLOCK_NUM;
rb->req.tp_frame_size = RX_FRAME_SIZE;
rb->req.tp_frame_nr = RX_FRAME_NUM;
setPacketRing(PACKET_RX_RING,&rb->req);
rb->size = (rb->req.tp_block_size)*(rb->req.tp_block_nr);
rb->cur_frame = 0;
// Tweak send/rcv buffer size
int sndBufSz = 4194304; // Send buffer in bytes
int rcvBufSz = 4194304; // Receive buffer in bytes
if (setSoBufforce(SO_SNDBUFFORCE, sndBufSz))
{
//close socket
}
if (setSoBufforce(SO_RCVBUFFORCE, rcvBufSz))
{
//close socket
}
// Add ARP filter so we will only receive ARP packet on this socket
struct sock_filter BPF_code[6];
struct sock_fprog filter;
bindSocket();
if (packetMmap(rb))
{
std::cout << "Error: mmap() failed!" << std::endl;
//close socket
}
frameBufferCreate(rb);
setRingBuffer(rb);
}
and in my function for receive packets and I try to read informations and in particular h_vlan_TCI from but I receive ever 0x00 !!! Any suggestions?
struct vlan_ethhdr* vlan_eth_h = (struct vlan_ethhdr*)&frame->eth_h
void readRawSocket(socket_)
{
while (*(unsigned long*)rb->ring_buffer_[rb->cur_frame].iov_base)
{
RxFrame* frame = (RxFrame *)rb->ring_buffer_[rb->cur_frame].iov_base;
#if 0
tpacket_hdr* h = &frame->tp_h;
char buffer[256];
sprintf (buffer, " -tpacket(v1): status=%ld,len=%d,snaplen=%d,mac=%d,net=%d,sec=%d,usec=%d",
h->tp_status, h->tp_len, h->tp_snaplen, h->tp_mac,h->tp_net, h->tp_sec, h->tp_usec);
std::cout << std::string(buffer) << std::endl;
#else
tpacket2_hdr* h = &frame->tp_h;
char buffer[512];
sprintf (buffer, " -tpacket(v2): status=%d,len=%d,snaplen=%d,mac=%d,net=%d,sec=%d,nsec=%d,vlan_tci=%d (vlan_tci=0x%04x)",
h->tp_status, h->tp_len, h->tp_snaplen, h->tp_mac, h->tp_net, h->tp_sec, h->tp_nsec, h->tp_vlan_tci, ntohs(h->tp_vlan_tci));
std::cout << std::string(buffer) << std::endl;
#endif
if ( ETH_P_8021Q == ntohs(frame->eth_h.h_proto) )
{
struct vlan_ethhdr* vlan_eth_h = (struct vlan_ethhdr*)&frame->eth_h;
int vlan_tag = VLAN_TAG(ntohs(vlan_eth_h->h_vlan_TCI));
std::cout << " -Vlan " << vlan_tag << " packet to this host received";
}
rb->cur_frame = ( rb->cur_frame+1) % rx_socket_->getFrameNum();
} // while()
}

When the kernel removes the vlan it also changes eth_h.h_proto to the protocol after de vlan tag so ETH_P_8021Q == ntohs(frame->eth_h.h_proto) will most probably be false.
Also, if you are listening in the tagged interface (ie. eth0.100) instead of the physical interface (eth0) you will not see the tags.

Related

Enumerate removable drives on Linux and macOS

On Windows I am able to enumerate removable drives using FindFirstVolume, FindNextVolume, GetVolumePathNamesForVolumeName and the GetDriveType functions. How can I achieve this on Linux and macOS?
On macOS you need to work with IOKit: https://developer.apple.com/documentation/iokit
It can looks like the following code:
#include <cstdlib>
#include <iostream>
#include <mach/mach_port.h>
#include <IOKit/IOKitLib.h>
#include <IOKit/IOTypes.h>
#include <IOKit/IOKitKeys.h>
#include <IOKit/IOCFPlugIn.h>
#include <IOKit/usb/IOUSBLib.h>
std::string get_descriptor_idx(IOUSBDeviceInterface **dev, UInt8 idx);
int main(int argc, char *argv[]) {
/*
* Get the IO registry
*/
auto entry = IORegistryGetRootEntry(kIOMasterPortDefault);
if (entry == 0)
return EXIT_FAILURE;
io_iterator_t iter{};
auto kret = IORegistryEntryCreateIterator(entry, kIOUSBPlane, kIORegistryIterateRecursively, &iter);
if (kret != KERN_SUCCESS || iter == 0)
return EXIT_FAILURE;
io_service_t service {};
std::cout << std::endl;
while ((service = IOIteratorNext(iter))) {
IOCFPlugInInterface **plug = nullptr;
IOUSBDeviceInterface **dev = nullptr;
io_string_t path;
SInt32 score = 0;
IOReturn ioret;
kret = IOCreatePlugInInterfaceForService(service, kIOUSBDeviceUserClientTypeID, kIOCFPlugInInterfaceID, &plug, &score);
IOObjectRelease(service);
if (kret != KERN_SUCCESS || plug == nullptr) {
continue;
}
/*
* USB
*/
ioret = (*plug)->QueryInterface(plug, CFUUIDGetUUIDBytes(kIOUSBDeviceInterfaceID),
static_cast<LPVOID *>((void *) &dev));
(*plug)->Release(plug);
if (ioret != kIOReturnSuccess || dev == nullptr) {
continue;
}
if (IORegistryEntryGetPath(service, kIOServicePlane, path) != KERN_SUCCESS) {
(*dev)->Release(dev);
continue;
}
std::cout << "USB Path: " << path << std::endl;
UInt8 si;
UInt16 u16v;
if ((*dev)->GetDeviceVendor(dev, &u16v) == kIOReturnSuccess)
std::cout << "VID: "<< u16v << std::endl;
if ((*dev)->GetDeviceProduct(dev, &u16v) == kIOReturnSuccess)
std::cout << "PID: "<< u16v << std::endl;
if ((*dev)->USBGetManufacturerStringIndex(dev, &si) == kIOReturnSuccess) {
std::cout << "Manufacturer: " << get_descriptor_idx(dev, si) << std::endl;
}
if ((*dev)->USBGetProductStringIndex(dev, &si) == kIOReturnSuccess) {
std::cout << "Product: " << get_descriptor_idx(dev, si) << std::endl;
}
(*dev)->Release(dev);
std::cout << std::endl;
}
IOObjectRelease(iter);
return 0;
}
/* a quick version */
std::string get_descriptor_idx(IOUSBDeviceInterface **dev, UInt8 idx)
{
IOUSBDevRequest request;
IOReturn ioret;
char buffer[4086] = { 0 };
CFStringRef cfstr;
CFIndex len;
request.bmRequestType = USBmakebmRequestType(kUSBIn, kUSBStandard, kUSBDevice);
request.bRequest = kUSBRqGetDescriptor;
request.wValue = (kUSBStringDesc << 8) | idx;
request.wIndex = 0x409;
request.wLength = sizeof(buffer);
request.pData = buffer;
ioret = (*dev)->DeviceRequest(dev, &request);
if (ioret != kIOReturnSuccess)
return "n/a";
if (request.wLenDone <= 2)
return "n/a";
cfstr = CFStringCreateWithBytes(nullptr, (const UInt8 *)buffer+2, request.wLenDone-2, kCFStringEncodingUTF16LE, 0);
len = CFStringGetMaximumSizeForEncoding(CFStringGetLength(cfstr), kCFStringEncodingUTF8) + 1;
if (len < 0) {
CFRelease(cfstr);
return "n/a";
}
std::string str; str.resize(len);
CFStringGetCString(cfstr, str.data(), len, kCFStringEncodingUTF8);
CFRelease(cfstr);
return str;
}
In my case the result ıs:
USB Path: IOService:/AppleARMPE/arm-io/xxxx/USB2.1 Hub#01100000
VID: 1507
PID: 1552
Manufacturer: GenesysLogic
Product: USB2.1 USB

Boost Asio, console pinger throws exception on socket.send (...)

I'm trying to figure out how ICMP and Boost Asio work. There was a problem sending the packet to the endpoint. The entered url is translated into ip and a socket connection is made. The problem is that when a packet is sent via socket.send (...), an exception is thrown.
Exception: send: Bad address
#include <algorithm>
#include <chrono>
#include <functional>
#include <iostream>
#include <memory>
#include <tuple>
//BOOST
#include <boost/asio.hpp>
#include <boost/program_options.hpp>
#include <boost/log/trivial.hpp>
//CONSTANTS
#define BUFFER_SIZE_64KB 65536
#define TTL_DEFAULT 64
#define ICMP_HDR_SIZE 8
#define LINUX_PAYLOAD_SIZE 56
#define TIME_BYTE_SIZE 4
#define FILL_BYTE 0X8
template <typename T, typename flag_type = int>
using flagged = std::tuple<flag_type, T>;
using namespace boost::asio;
typedef boost::system::error_code error_code;
typedef unsigned char byte;
enum ICMP : uint8_t {
ECHO_REPLY = 0,
UNREACH = 3,
TIME_EXCEEDED = 11,
ECHO_REQUEST = 8
};
enum class IPtype {IPV4, IPV6, BOTH};
struct icmp_header_t {
uint8_t type;
uint8_t code;
uint16_t checksum;
uint16_t id;
uint16_t seq_num;
};
struct ip_header_t {
uint8_t ver_ihl;
uint8_t tos;
uint16_t total_length;
uint16_t id;
uint16_t flags_fo;
uint8_t ttl;
uint8_t protocol;
uint16_t checksum;
uint32_t src_addr;
uint32_t dst_addr;
};
ip_header_t ip_load(std::istream& stream, bool ntoh ) {
ip_header_t header;
stream.read((char*)&header.ver_ihl, sizeof(header.ver_ihl));
stream.read((char*)&header.tos, sizeof(header.tos));
stream.read((char*)&header.total_length, sizeof(header.total_length));
stream.read((char*)&header.id, sizeof(header.id));
stream.read((char*)&header.flags_fo, sizeof(header.flags_fo));
stream.read((char*)&header.ttl, sizeof(header.ttl));
stream.read((char*)&header.protocol, sizeof(header.protocol));
stream.read((char*)&header.checksum, sizeof(header.checksum));
stream.read((char*)&header.src_addr, sizeof(header.src_addr));
stream.read((char*)&header.dst_addr, sizeof(header.dst_addr));
if (ntoh) {
header.total_length = ntohs(header.total_length);
header.id = ntohs(header.id);
header.flags_fo = ntohs(header.flags_fo);
header.checksum = ntohs(header.checksum);
header.src_addr = ntohl(header.src_addr);
header.dst_addr = ntohl(header.dst_addr);
}
return header;
}
icmp_header_t icmp_load(std::istream& stream) {
icmp_header_t header;
stream.read((char*)&header.type, sizeof(header.type));
stream.read((char*)&header.code, sizeof(header.code));
stream.read((char*)&header.checksum, sizeof(header.checksum));
stream.read((char*)&header.id, sizeof(header.id));
stream.read((char*)&header.seq_num, sizeof(header.seq_num));
return header;
}
flagged<ip::icmp::endpoint> sync_icmp_solver(io_service& ios, std::string host,
IPtype type = IPtype::BOTH) noexcept {
ip::icmp::resolver::query query(host, "");
ip::icmp::resolver resl(ios);
ip::icmp::endpoint ep;
error_code ec;
auto it = resl.resolve(query, ec);
if (ec != boost::system::errc::errc_t::success) {
std::cerr << "Error message = " << ec.message() << std::endl;
return std::make_tuple(ec.value(), ep);
}
ip::icmp::resolver::iterator it_end;
//Finds first available ip.
while (it != it_end) {
ip::icmp::endpoint ep = (it++)->endpoint();
auto addr = ep.address();
switch(type) {
case IPtype::IPV4:
if (addr.is_v4()) return std::make_tuple(0, ep);
break;
case IPtype::IPV6:
if(addr.is_v6()) return std::make_tuple(0, ep);
break;
case IPtype::BOTH:
return std::make_tuple(0, ep);
break;
}
}
return std::make_tuple(-1, ep);
}
unsigned short checksum(void *b, int len) {
unsigned short* buf = reinterpret_cast<unsigned short*>(b);
unsigned int sum = 0;
unsigned short result;
for (sum = 0; len > 1; len -= 2 ) {
sum += *buf++;
}
if (len == 1) sum += *(byte*) buf;
sum = (sum >> 16) + (sum & 0xFFFF);
sum += (sum >> 16);
result = ~sum;
return result;
}
unsigned short get_identifier() {
#if defined(BOOST_WINDOWS)
return static_cast<unsigned short>(::GetCurrentProcessId());
#else
return static_cast<unsigned short>(::getpid());
#endif
}
struct PingInfo {
unsigned short seq_num = 0;
size_t time_out;
size_t reply_time = 1;
size_t payload_size = LINUX_PAYLOAD_SIZE;
size_t packets_rec = 0;
size_t packets_trs = 0;
size_t reps = 0;
};
class PingConnection {
private:
ip::icmp::socket sock;
io_service* ios_ptr;
PingInfo* pi_ptr;
ip::icmp::endpoint dst;
boost::posix_time::ptime timestamp;
streambuf input_buf;
deadline_timer deadtime;
//TODO: Check for memleaks.
void write_icmp_req(std::ostream& os) {
byte* pckt = new byte[ICMP_HDR_SIZE + pi_ptr->payload_size];
unsigned short pid = get_identifier();
pckt[0] = 0x8;
pckt[1] = 0x0;
pckt[2] = 0x0;
pckt[3] = 0x0;
pckt[4] = (byte)((pid & 0xF0) >> 4);
pckt[5] = (byte)(pid & 0x0F);
for (size_t i = ICMP_HDR_SIZE; i < ICMP_HDR_SIZE + pi_ptr->payload_size; i++) {
pckt[i] = FILL_BYTE;
}
pckt[6] = (byte)((pi_ptr->seq_num & 0xF0) >> 4);
pckt[7] = (byte)((pi_ptr->seq_num)++ & 0x0F);
unsigned short cs = checksum(pckt, ICMP_HDR_SIZE);
pckt[2] = (byte)((cs & 0xF0) >> 4);
pckt[3] = (byte)(cs & 0x0F);
os << pckt;
delete [] pckt;
}
void pckt_send() {
streambuf buf;
std::ostream os(&buf);
write_icmp_req(os);
timestamp = boost::posix_time::microsec_clock::universal_time();
std::cout << "begin" << std::endl;
sock.send(buf.data());
std::cout << "sock.send(buf.data())" << std::endl;
deadtime.expires_at(timestamp + boost::posix_time::seconds(pi_ptr->time_out));
deadtime.async_wait(std::bind(&PingConnection::req_timeout_callback, this));
}
void req_timeout_callback() {
if (pi_ptr->reps == 0) {
std::cout << "Time Out:echo req" << std::endl;
}
deadtime.expires_at(timestamp + boost::posix_time::seconds(pi_ptr->reply_time));
deadtime.async_wait(std::bind(&PingConnection::pckt_send, this));
}
void pckt_recv() {
std::cout << "pckt_recv" << std::endl;
input_buf.consume(input_buf.size());
sock.async_receive(input_buf.prepare(BUFFER_SIZE_64KB),
std::bind(&PingConnection::recv_timeout_callback, this, std::placeholders::_2));
}
void recv_timeout_callback(size_t sz) {
std::cout << "recv_timeout_callback" << std::endl;
input_buf.commit(sz);
std::istream is(&input_buf);
ip_header_t iph = ip_load(is, false);
icmp_header_t icmph = icmp_load(is);
if (is &&
icmph.type == ECHO_REQUEST &&
icmph.id == get_identifier() &&
icmph.seq_num == pi_ptr->seq_num) {
// If this is the first reply, interrupt the five second timeout.
if (pi_ptr->reps++ == 0) deadtime.cancel();
boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time();
std::cout << sz - iph.total_length
<< " bytes from " << iph.src_addr
<< ": icmp_seq=" << icmph.seq_num
<< ", ttl=" << iph.ttl
<< ", time=" << (now - timestamp).total_milliseconds() << " ms"
<< std::endl;
}
pckt_recv();
}
public:
PingConnection(io_service& ios, PingInfo& pi_add) : deadtime(ios), sock(ios) {
pi_ptr = &pi_add;
ios_ptr = &ios;
}
void ping(std::string host) {
int err_flag;
error_code error;
std::tie(err_flag, dst) = sync_icmp_solver(*ios_ptr, host);
if (err_flag) return;
std::cout << dst << std::endl;
sock.connect(dst, error);
if(error) {
return;
}
std::cout << "sock.connect(dst)" << error.message() <<std::endl;
pckt_send();
pckt_recv();
}
};
int main(int argc, char** argv) {
try
{
if (argc < 2) {
std::cerr << "Usage: ping [args]* destination\n";
return -1;
}
io_service ios;
PingInfo pi;
pi.time_out = 56;
PingConnection ping(ios, pi);
ping.ping(argv[1]);
ios.run();
} catch(std::exception& e) {
std::cerr << "Exception: " << e.what() << std::endl;
}
}
socket.send() is called in pckt_send()
For development I use WSL2 and Ubuntu image.

Random lack of connection and message delay in DTLS with OpenSSL

Trying to write a server for DTLS that will currently just output the text that it receives. The working client is taken from https://github.com/stepheny/openssl-dtls-custom-bio and it sends and receives to its own server just fine.
However, when it sends to this server something strange is happening. Firstly the connection happens only sometimes, there seems to be no way to determine if the connection will start or not. Secondly, and that is even stranger the data is "delayed". One needs to send 6 messages for 1 message to arrive.
So this is the situation:
Start the server.
Start the client.
Hope for connection.
If connected type 5 messages in client to send to server, they are sent, but the server keeps having an error decoding them.
Once you send the 6th message you can note that the 1st message arrives on server.
Once you send the 7th, you will get the 2nd. Etc.
It should be noted that we are not talking about a time delay, there is no way to simply read 5 empty messages at the start of the server, the queue is empty. Only once the 6th message is sent is the queue populated with the 1st real message.
Code:
//server.cpp
#include "DTLSConnection.hpp"
#include <iostream>
#include <chrono>
#include <thread>
int main(int argc, char *argv[])
{
try
{
DTLSConnection con("192.168.31.177:1235");
std::cout << "Connection created" << std::endl;
ssize_t ret;
for(;;)
{
ret = con.recv([](Client* c) {
try{
std::cout << c->SSL_read_alt() << std::endl;
std::cout << "I am in onmessage" << std::endl;
}
catch(std::string &e)
{
std::cerr << "EXCEPTION: " << e << std::endl;
}
});
std::cout << "Returned value is " << ret << std::endl;
std::this_thread::sleep_for(std::chrono::seconds(1));
}
}
catch(std::string &e)
{
std::cerr << "EXCEPTION: " << e << std::endl;
}
return 0;
}
// CustomBIO.hpp
#include <memory>
#include <deque>
#include <vector>
#include <unordered_map>
#include <cstdio> // temporary
#include <cstring>
#include <cassert>
#include <openssl/ssl.h>
#include <signal.h>
const char *sdump_addr(const struct sockaddr *sa)
{
static char buf[1024];
switch (sa->sa_family)
{
case AF_INET:
memmove(buf, "INET: ", 6);
inet_ntop(AF_INET, &((struct sockaddr_in *)sa)->sin_addr, buf+6, sizeof(buf)-6);
sprintf(buf+strlen(buf), ":%d", ntohs(((struct sockaddr_in *)sa)->sin_port));
break;
case AF_INET6:
memmove(buf, "INET6: [", 8);
inet_ntop(AF_INET6, &((struct sockaddr_in6 *)sa)->sin6_addr, buf+8, sizeof(buf)-8);
sprintf(buf+strlen(buf), "]:%d", ntohs(((struct sockaddr_in6 *)sa)->sin6_port));
break;
default:
memmove(buf, "unknown", 8);
break;
}
return buf;
}
struct Packet
{
size_t capacity = 0;
size_t len = 0;
char * data = nullptr;
Packet() = default;
Packet(size_t cap)
{
init(cap);
}
Packet(char *b, size_t cap)
{
capacity=cap;
len=capacity;
data = new char[capacity];
memcpy(data, b, capacity);
}
Packet(char* b, char* e)
{
capacity=e-b;
len=capacity;
data = new char[capacity];
memcpy(data, b, capacity);
}
char* begin()
{
return data;
}
char* end()
{
return data ? data+len : nullptr;
}
void swap(Packet& that)
{
std::swap(this->capacity, that.capacity);
std::swap(this->len, that.len);
std::swap(this->data, that.data);
}
void init(size_t cap)
{
data = new char[cap];
len = 0;
capacity = cap;
}
void free()
{
if(!data) return;
delete data;
len = 0;
capacity = 0;
data = nullptr;
}
};
// used for both reading and writing
struct CustomBIO
{
using dataBuffer = Packet;
int sockfd;
sockaddr_storage thisAddr{};
socklen_t thisAddr_len{sizeof(sockaddr_storage)};
sockaddr_storage thatAddr{};
socklen_t thatAddr_len{sizeof(sockaddr_storage)};
template<typename T>
T* getThat()
{
return reinterpret_cast<T*>(&thatAddr);
}
std::deque<dataBuffer> receivingQueue{};
bool peekmode{false};
};
inline CustomBIO* BIO_get_CBIO(BIO* b)
{
return reinterpret_cast<CustomBIO *>(BIO_get_data(b));
}
extern "C"
{
int BIO_s_custom_write_ex(BIO *b, const char *data, size_t dlen, size_t *written);
int BIO_s_custom_write(BIO *b, const char *data, int dlen);
int BIO_s_custom_read_ex(BIO *b, char *data, size_t dlen, size_t *readbytes);
int BIO_s_custom_read(BIO *b, char *data, int dlen);
int BIO_s_custom_gets(BIO *b, char *data, int size);
int BIO_s_custom_puts(BIO *b, const char *data);
long BIO_s_custom_ctrl(BIO *b, int cmd, long larg, void *pargs);
int BIO_s_custom_create(BIO *b);
int BIO_s_custom_destroy(BIO *b);
// long BIO_s_custom_callback_ctrl(BIO *, int, BIO_info_cb *);
BIO_METHOD *BIO_s_custom();
void BIO_s_custom_meth_free();
int BIO_s_custom_write_ex(BIO *b, const char *data, size_t dlen, size_t *written)
{
fprintf(stderr, "BIO_s_custom_write_ex(BIO[0x%016lX], data[0x%016lX], dlen[%ld], *written[%ld])\n", (long unsigned int)b, (long unsigned int)data, dlen, *written);
fflush(stderr);
return -1;
}
int BIO_s_custom_write(BIO *b, const char *data, int dlen)
{
int ret;
CustomBIO *cbio;
ret = -1;
fprintf(stderr, "BIO_s_custom_write(BIO[0x%016lX], data[0x%016lX], dlen[%ld])\n", (unsigned long)b, (unsigned long)data, (long)dlen);
fflush(stderr);
cbio = BIO_get_CBIO(b);
// dump_addr((struct sockaddr *)&cbio->txaddr, ">> ");
// dump_hex((unsigned const char *)data, dlen, " ");
ret = sendto(cbio->sockfd, data, dlen, 0, cbio->getThat<const sockaddr>(), cbio->thatAddr_len);
if (ret >= 0)
{
fprintf(stderr, " %d bytes sent\n", ret);
}
else
{
fprintf(stderr, " ret: %d errno: [%d] %s\n", ret, errno, strerror(errno));
fprintf(stderr, " socket: %d\n", cbio->sockfd);
fprintf(stderr, " thatAddrLen: %d\n", cbio->thatAddr_len);
fprintf(stderr, " thatAddr: %s\n", sdump_addr(cbio->getThat<sockaddr>()));
}
return ret;
}
int BIO_s_custom_read_ex(BIO *b, char *data, size_t dlen, size_t *readbytes)
{
fprintf(stderr, "BIO_s_custom_read_ex(BIO[0x%016lX], data[0x%016lX], dlen[%ld], *readbytes[%ld])\n", (long unsigned int)b, (long unsigned int)data, (long int)dlen, *readbytes);
fflush(stderr);
return -1;
}
int BIO_s_custom_read(BIO *b, char *data, int dlen)
{
int ret;
CustomBIO *cbio;
ret = -1;
fprintf(stderr, "BIO_s_custom_read(BIO[0x%016lX], data[0x%016lX], dlen[%ld])\n", (long unsigned int)b, (long unsigned int)data, (long int)dlen);
fprintf(stderr, " probe peekmode %d\n", ((CustomBIO *)BIO_get_data(b))->peekmode);
fflush(stderr);
cbio = BIO_get_CBIO(b);
if(!cbio->receivingQueue.empty())
{
if(cbio->receivingQueue.front().len > (size_t)dlen)
{
fprintf(stderr, "if(cbio->receivingQueue.front().len > (size_t)dlen)");
memmove(data, cbio->receivingQueue.front().data, dlen);
ret = dlen;
if(!cbio->peekmode)
{
CustomBIO::dataBuffer rest{cbio->receivingQueue.front().begin()+ret, cbio->receivingQueue.front().end()};
cbio->receivingQueue.front().swap(rest);
}
}
else
{
Packet &pac = cbio->receivingQueue.front();
ret = pac.len;
memmove(data, pac.data, ret);
if(!cbio->peekmode)
{
pac.free();
cbio->receivingQueue.pop_front();
}
}
fprintf(stderr, " %d bytes read from queue\n", ret);
fflush(stderr);
}
else
{
fprintf(stderr, " The queue is empty\n");
/*ret = recvfrom(cbio->sockfd, data, dlen, 0, cbio->getThat<sockaddr>(), &cbio->thatAddr_len); // not right
if(ret>0 && cbio->peekmode)
{
// todo
}*/
}
return ret;
}
int BIO_s_custom_gets(BIO *b, char *data, int size)
{
fprintf(stderr, "BIO_s_custom_gets(BIO[0x%016lX], data[0x%016lX], size[%d]\n", (long unsigned int)b, (long unsigned int)data, size);
if(size <= 1)
{
return 0;
}
else
{
size = BIO_s_custom_read(b, data, size-1);
data[size] = '\0';
return size;
}
}
int BIO_s_custom_puts(BIO *b, const char *buf)
{
fprintf(stderr, "BIO_s_custom_puts(BIO[0x%016lX], buf[0x%016lX]\n", (long unsigned int)b, (long unsigned int)buf);
size_t size = std::strlen(buf);
return size > 0 ? BIO_s_custom_write(b, buf, size) : 0;
}
long BIO_s_custom_ctrl(BIO *b, int cmd, long larg, void *pargs)
{
long ret = 0;
fprintf(stderr, "BIO_s_custom_ctrl(BIO[0x%016lX], cmd[%d], larg[%ld], pargs[0x%016lX])\n", (long unsigned int)b, cmd, larg, (long unsigned int)pargs);
if(pargs)
{
for(int i=0; ; ++i)
{
fprintf(stderr, "[%d]=%X ", i, (int)((unsigned char*)pargs)[i]);
if(((unsigned char*)pargs)[i] == 0) break;
}
}
fprintf(stderr, "\n");
fflush(stderr);
switch(cmd)
{
case BIO_CTRL_FLUSH: // 11
case BIO_CTRL_DGRAM_SET_CONNECTED: // 32
case BIO_CTRL_DGRAM_SET_PEER: // 44
case BIO_CTRL_DGRAM_GET_PEER: // 46
ret = 1;
break;
case BIO_CTRL_WPENDING: // 13
ret = 0;
break;
case BIO_CTRL_DGRAM_QUERY_MTU: // 40
case BIO_CTRL_DGRAM_GET_FALLBACK_MTU: // 47
ret = 1500;
// ret = 9000; // jumbo?
break;
case BIO_CTRL_DGRAM_GET_MTU_OVERHEAD: // 49
ret = 96; // random guess
break;
case BIO_CTRL_DGRAM_SET_PEEK_MODE: // 71
BIO_get_CBIO(b)->peekmode = (larg != 0);
ret = 1;
break;
case BIO_CTRL_PUSH: // 6
case BIO_CTRL_POP: // 7
case BIO_CTRL_DGRAM_SET_NEXT_TIMEOUT: // 45
ret = 0;
break;
default:
fprintf(stderr, "BIO_s_custom_ctrl(BIO[0x%016lX], cmd[%d], larg[%ld], pargs[0x%016lX])\n", (long unsigned int)b, cmd, larg, (long unsigned int)pargs);
fprintf(stderr, " unknown cmd: %d\n", cmd);
fflush(stderr);
ret = 0;
break;
}
return ret;
}
int BIO_s_custom_create(BIO *b)
{
fprintf(stderr, "BIO_s_custom_create(BIO[0x%016lX])\n", (long unsigned int)b);
fflush(stderr);
return 1;
}
int BIO_s_custom_destroy(BIO *b)
{
fprintf(stderr, "BIO_s_custom_destroy(BIO[0x%016lX])\n", (long unsigned int)b);
fflush(stderr);
return 1;
}
BIO_METHOD *_BIO_s_custom = nullptr;
BIO_METHOD *BIO_s_custom()
{
if (!_BIO_s_custom)
{
_BIO_s_custom = BIO_meth_new(BIO_get_new_index()|BIO_TYPE_SOURCE_SINK, "BIO_s_custom");
//BIO_meth_set_callback_ctrl(_BIO_s_custom, BIO_s_custom_callback_ctrl);
BIO_meth_set_create(_BIO_s_custom, BIO_s_custom_create);
BIO_meth_set_ctrl(_BIO_s_custom, BIO_s_custom_ctrl);
BIO_meth_set_destroy(_BIO_s_custom, BIO_s_custom_destroy);
BIO_meth_set_gets(_BIO_s_custom, BIO_s_custom_gets);
BIO_meth_set_puts(_BIO_s_custom, BIO_s_custom_puts);
BIO_meth_set_read_ex(_BIO_s_custom, BIO_s_custom_read_ex);
BIO_meth_set_read(_BIO_s_custom, BIO_s_custom_read);
BIO_meth_set_write_ex(_BIO_s_custom, BIO_s_custom_write_ex);
BIO_meth_set_write(_BIO_s_custom, BIO_s_custom_write);
}
return _BIO_s_custom;
}
void BIO_s_custom_meth_free()
{
if (_BIO_s_custom)
BIO_meth_free(_BIO_s_custom);
_BIO_s_custom = NULL;
}
}
// DTLSConnection.hpp
#include <string>
#include <list>
#include <functional>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <sys/epoll.h>
#include <cerrno>
#include <iostream> // temp
#include "CustomBIO.hpp"
constexpr int TIME_OUT = 10000; // ms
char cookie_str[] = "BISCUIT!"; // how to change this
//нужен способ, чтобы клиент был извествен (knownclients) но еще не подключен (не прошел ssl_accept())
// (void) BIO_dgram_get_peer(SSL_get_rbio(ssl), &peer);
// see https://github.com/nplab/DTLS-Examples/blob/master/src/dtls_udp_echo.c
int generate_cookie([[maybe_unused]] SSL *ssl, unsigned char *cookie, unsigned int *cookie_len)
{
memmove(cookie, cookie_str, sizeof(cookie_str)-1);
*cookie_len = sizeof(cookie_str)-1;
return 1;
}
int verify_cookie([[maybe_unused]] SSL *ssl, const unsigned char *cookie, unsigned int cookie_len)
{
return sizeof(cookie_str)-1==cookie_len && memcmp(cookie, cookie_str, sizeof(cookie_str)-1)==0;
}
void throw_SSL_error_if_error(SSL* ssl, int ret, std::string str)
{
if(ret>0) return; // SSL_ERROR_NONE
str += " ret="+std::to_string(ret)+' ';
auto sslError = SSL_get_error(ssl, ret);
if(sslError == SSL_ERROR_SYSCALL){
throw std::string{str+"SSL_ERROR_SYSCALL + error "}+std::to_string(errno);
}
}
namespace std
{
template<> struct hash<sockaddr>
{
size_t operator()(sockaddr const& val) const noexcept
{
size_t res = 0;
for(unsigned long h : val.sa_data)
{
res = (res << 1) ^ h;
}
return res;
}
};
}
bool operator==(const sockaddr& l, const sockaddr& r)
{
if(l.sa_family != r.sa_family) return false;
for(int i=0; i<14; ++i)
{
if(l.sa_data[i] != r.sa_data[i]) return false;
}
return true;
}
class DTLSConnection;
class SSLSetterUpper
{
friend DTLSConnection;
SSL_CTX *ctx;
SSLSetterUpper()
{
int ret; // because it is C;
SSL_load_error_strings();
SSL_library_init();
const SSL_METHOD *mtd = DTLS_server_method();
ctx = SSL_CTX_new(mtd);
SSL_CTX_set_min_proto_version(ctx, DTLS1_2_VERSION);
SSL_CTX_use_certificate_chain_file(ctx, "server-cert.pem");
SSL_CTX_use_PrivateKey_file(ctx, "server-key.pem", SSL_FILETYPE_PEM);
ret = SSL_CTX_load_verify_locations(ctx, "root-ca.pem", nullptr);
if(ret != 1)
{
throw std::string{"SSL_CTX_load_verify_locations failed"};
}
ret = SSL_CTX_set_default_verify_file(ctx);
if(ret != 1)
{
throw std::string{"SSL_CTX_set_default_verify_file failed"};
}
SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);
SSL_CTX_set_cookie_generate_cb(ctx, generate_cookie);
SSL_CTX_set_cookie_verify_cb(ctx, verify_cookie);
}
SSL* generateSSL() const
{
return SSL_new(ctx);
}
};
struct Client
{
CustomBIO cbio;
SSL *ssl;
explicit Client(SSL *ssl)
:cbio{}, ssl{ssl}
{
cbio.peekmode = false;
BIO *bio = BIO_new(BIO_s_custom());
BIO_set_data(bio, (void *)&cbio);
BIO_set_init(bio, 1);
SSL_set_bio(ssl, bio, bio);
}
bool on_init() const
{
int ret = DTLSv1_listen(this->ssl, nullptr);
//throw_SSL_error_if_error(ssl, ret, "DTLSv1_listen failed");
std::cout << "DTLSv1_listen " << ret << std::endl;
return (ret==1);
}
bool on_connect() const
{
int ret = SSL_accept(ssl);
if(ret!=1) return false;
std::cout << "ssl = " << SSL_state_string_long(ssl) << std::endl;
std::cout << "SSL_accept successful!" << std::endl;
return true;
}
std::string SSL_read_alt() const
{
Packet p(2000);
std::cout << "sizeA = " << cbio.receivingQueue.size() << std::endl;
int ret = ::SSL_read(ssl,p.data,p.capacity);
std::cout << "ret = " << ret << std::endl;
std::cout << "sizeB = " << cbio.receivingQueue.size() << std::endl;
std::cout << "pdata0 " << p.data[0] << std::endl;
//std::cout << SSL_get_error(ssl, ret) << std::endl;
throw_SSL_error_if_error(ssl, ret, "sslread");
p.len = ret;
std::cerr << "plen" << std::endl;
std::cerr << p.len << std::endl;
std::string result(p.data,p.len);
if(result[0] == '\0') std::cout << "res = " << result << std::endl;
p.free();
return result;
}
};
class DTLSConnection
{
static const SSLSetterUpper sslSetup;
int epoll_fd;
std::unordered_map<sockaddr, std::shared_ptr<Client>> knownClients;
std::unordered_map<sockaddr, std::shared_ptr<Client>> connectedClients;
std::shared_ptr<Client> incomingClient;
public:
// 127.0.0.1:1234 or [::1]:1234
explicit DTLSConnection(std::string thisAddress)
{
std::list<sockaddr_storage> addresses;
if (thisAddress[0]=='[')
{
auto pos = thisAddress.find(']', 1);
if (pos == std::string::npos)
{
throw std::string{"invalid target"};
}
int port = std::stoi(thisAddress.substr(pos+2));
if (port<1||port>65535)
{
throw std::string{"invalid port"};
}
addresses.emplace_back();
auto* thisAddr = (sockaddr_in6 *)&(addresses.back());
thisAddr->sin6_family = AF_INET6;
if ( ! inet_pton(AF_INET6, thisAddress.substr(1, pos).c_str(), &thisAddr->sin6_addr) )
{
throw std::string{"invalid ipv6 address"};
}
thisAddr->sin6_port = htons(port);
}
else
{
auto pos = thisAddress.find(':');
if (pos == std::string::npos)
{
throw std::string{"invalid target"};
}
int port = std::stoi(thisAddress.substr(pos+1));
if (port<1||port>65535)
{
throw std::string{"invalid port"};
}
addresses.emplace_back();
auto * thisAddr = (sockaddr_in *)&(addresses.back());
thisAddr->sin_family = AF_INET;
if ( ! inet_pton(AF_INET, thisAddress.substr(0, pos).c_str(), &thisAddr->sin_addr) )
{
throw std::string{"invalid ipv4 address"};
}
thisAddr->sin_port = htons(port);
}
epoll_fd = epoll_create1(EPOLL_CLOEXEC);
for(auto &address : addresses)
{
epoll_event epe {};
epe.data.fd = socket(address.ss_family, SOCK_DGRAM/*|SOCK_NONBLOCK*/|SOCK_CLOEXEC, 0);
if(bind(epe.data.fd, (const sockaddr*)&address, sizeof(address)) != 0)
{
throw std::string{"failed to bind"};
}
epe.events = EPOLLIN|EPOLLET;
epoll_ctl(epoll_fd, EPOLL_CTL_ADD, epe.data.fd, &epe);
}
//signal(SIGINT, signal_handler); // do i need this?
incomingClient = std::make_shared<Client>(sslSetup.generateSSL());
}
ssize_t recv(const std::function<void(Client *)>& onmessage)
{
std::cout << "recv(onmessage)" << std::endl;
epoll_event epe{};
int ret;
ret = epoll_wait(epoll_fd, &epe, 1, TIME_OUT);
if (ret==-1)
{
throw std::string{"epoll_wait failed"};
}
if(ret==0)
{
return ret; // wait longer
}
Packet packet{2000};
ret = recvfrom(epe.data.fd, packet.data, packet.capacity, 0, incomingClient->cbio.getThat<sockaddr>(), &incomingClient->cbio.thatAddr_len);
packet.len = ret;
if(ret==0) return ret;
if(ret<0)
{
switch(errno)
{
case EAGAIN:
//case EWOULDBLOCK:
return ret;
case EBADF:
throw std::string{"EBADF"};
case ECONNREFUSED:
throw std::string{"ECONNREFUSED"};
case EFAULT:
throw std::string{"EFAULT"};
case EINTR:
throw std::string{"EINTR"};
case EINVAL:
throw std::string{"EINVAL"};
case ENOMEM:
throw std::string{"ENOMEM"};
case ENOTCONN:
throw std::string{"ENOTCONN"};
case ENOTSOCK:
throw std::string{"ENOTSOCK"};
default:
throw std::string{"Unknwon errno with negative return from recvfrom: "}+std::to_string(errno);
}
}
auto known = knownClients.find(*incomingClient->cbio.getThat<sockaddr>());
auto connected = connectedClients.find(*incomingClient->cbio.getThat<sockaddr>());
std::cout << "START" << std::endl;
for(auto &pair : knownClients)
{
std::cout << sdump_addr(&pair.first) << std::endl;
}
std::cout << "END" << std::endl;
if(known == knownClients.end())
{
std::cout << "inetaddr = " << sdump_addr(incomingClient->cbio.getThat<sockaddr>()) << std::endl;
ret = 0;
incomingClient->cbio.receivingQueue.push_back(std::move(packet));
incomingClient->cbio.sockfd = epe.data.fd;
if( incomingClient->on_init() )
{
std::cout << "inc = " << incomingClient->on_connect() << std::endl; //скорее всего, всегда будет ложь
std::cout << "on_init if" << std::endl;
knownClients[*incomingClient->cbio.getThat<sockaddr>()] = incomingClient;
incomingClient = std::make_shared<Client>(sslSetup.generateSSL());
}
}
else if(connected == connectedClients.end())
{
std::cout << "elseif" << std::endl;
ret = 0;
auto cli = known->second;
cli->cbio.receivingQueue.push_back(std::move(packet));
if( cli->on_connect() )
{
std::cout << "cli->cbio.receivingQueue.size()" << cli->cbio.receivingQueue.size() << std::endl;
connectedClients[*cli->cbio.getThat<sockaddr>()] = cli;
SSL_write(cli->ssl, "hello", 6);
}
}
else
{
std::cout << "else" << std::endl;
std::cout << sdump_addr(incomingClient->cbio.getThat<sockaddr>()) << " has been found as connected" << std::endl;
connected->second->cbio.receivingQueue.push_back(std::move(packet));
onmessage(connected->second.get());
}
return ret;
}
};
const SSLSetterUpper DTLSConnection::sslSetup{};
The output from running the server (CustomBIO output truncated) is:
Connection created
recv(onmessage)
START
END
inetaddr = INET: 192.168.31.177:58897
probe peekmode 0
211 bytes read from queue
36 bytes sent
probe peekmode 0
The queue is empty
DTLSv1_listen -1
Returned value is 0
recv(onmessage)
START
END
inetaddr = INET: 192.168.31.177:58897
probe peekmode 0
219 bytes read from queue
DTLSv1_listen 1
1180 bytes sent
probe peekmode 0
The queue is empty
inc = 0
on_init if
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
elseif
86 bytes sent
823 bytes sent
167 bytes sent
79 bytes sent
25 bytes sent
probe peekmode 0
219 bytes read from queue
probe peekmode 0
The queue is empty
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
elseif
probe peekmode 0
1088 bytes read from queue
618 bytes sent
ssl = SSL negotiation finished successfully
SSL_accept successful!
cli->cbio.receivingQueue.size()0
43 bytes sent
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
824 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0 �
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 824
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
58 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0 `
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 58
recv(onmessage)
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
131 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 131
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
14 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0 �
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 14
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
The queue is empty
ret = -1
sizeB = 0
pdata0
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 61
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
55 bytes read from queue
ret = 18
sizeB = 0
pdata0 d
plen
18
dfksaiopfjiaosjfio
I am in onmessage
Returned value is 55
recv(onmessage)
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
46 bytes read from queue
ret = 9
sizeB = 0
pdata0 s
plen
9
sdasdasda
I am in onmessage
Returned value is 46
recv(onmessage)
Returned value is 0
recv(onmessage)
Where you see EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0 that means that SSL_read has returned -1. Sorry for somewhat dirty code.
In case somebody else will have a similar issue. The problem was that the wait between calling server's recv function was 1 second. In that time client thought that server has not responded and began doing weird things. Lowering the delay solved the problem.

gRPC UDP with Deadline

I have created a client-server program based on one of the tests in the gRPC repo.
The UDP code in gRPC is not built on top of its RPC layer, and so there is no notion of stubs, etc.
My code works, though I've noticed that under just a mild stress, a huge fraction of messages get dropped, and I'm not sure if it's entirely due to the lossy nature of UDP or it's something about my code.
I have two questions:
Main question: Is there a gRPC-way to set deadlines for UDP messages? I am familiar with ClientContext and its deadline feature, but I don't know how to use it in a non-TCP RPC-less code. If not, what is the best way to achieve this?
Is a drop rate of %50 for a UDP localhost communication sensible?
My code (It's quite long, so just attaching it for reference. My main question doesn't require reading the code):
#include <netdb.h>
#include <string>
#include <thread>
#include <vector>
// grpc headers
#include <grpcpp/grpcpp.h>
#include "src/core/lib/iomgr/udp_server.h"
#include "src/core/lib/iomgr/socket_utils_posix.h"
#include "src/core/lib/iomgr/unix_sockets_posix.h"
#include "src/core/lib/iomgr/sockaddr_utils.h"
using namespace std;
int client_port = 6666;
int server_port = 5555;
int num_of_msgs = 1000;
int listening_port;
int remote_port;
int fd;
int received_msgs_cnt = 0;
vector<bool> is_received(num_of_msgs, false);
enum Role {
CLIENT,
SERVER
};
struct Request {
int id;
};
struct Response {
int id;
};
Role role;
bool udpServerFinished = false;
void sendUdp(const char *hostname, int port, const char* payload, size_t size) {
auto transferred = write(fd, (void*)payload, size);
assert(size == transferred);
}
/***************************************
* UDP Handler class
* (will be generated by factory class)
* upon receiving a new message, the Read()
* function is invoked
***************************************/
class UdpHandler : public GrpcUdpHandler {
public:
UdpHandler(grpc_fd *emfd, void *user_data):
GrpcUdpHandler(emfd, user_data), emfd_(emfd) {
}
virtual ~UdpHandler() {}
static void startLoop(volatile bool &udpServerFinished) {
grpc_core::ExecCtx exec_ctx;
grpc_millis deadline;
gpr_mu_lock(g_mu);
while (!udpServerFinished) {
deadline = grpc_timespec_to_millis_round_up(gpr_time_add(
gpr_now(GPR_CLOCK_MONOTONIC),
gpr_time_from_millis(10000, GPR_TIMESPAN)));
grpc_pollset_worker *worker = nullptr;
GPR_ASSERT(GRPC_LOG_IF_ERROR(
"pollset_work", grpc_pollset_work(UdpHandler::g_pollset, &worker, deadline)));
gpr_mu_unlock(UdpHandler::g_mu);
grpc_core::ExecCtx::Get()->Flush();
gpr_mu_lock(UdpHandler::g_mu);
}
gpr_mu_unlock(g_mu);
}
static grpc_pollset *g_pollset;
static gpr_mu *g_mu;
public:
static int g_num_listeners;
protected:
bool Read() override {
char read_buffer[512];
ssize_t byte_count;
gpr_mu_lock(UdpHandler::g_mu);
byte_count = recv(grpc_fd_wrapped_fd(emfd()), read_buffer, sizeof(read_buffer), 0);
processIncomingMsg((void*)read_buffer, byte_count);
GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick",
grpc_pollset_kick(UdpHandler::g_pollset, nullptr)));
gpr_mu_unlock(UdpHandler::g_mu);
return false;
}
void processIncomingMsg(void* msg, ssize_t size) {
received_msgs_cnt++;
(void)size;
int id;
if (role == Role::CLIENT) {
Response res;
assert(size == sizeof(Response));
memcpy((void*)&res, (void*)msg, size);
id = res.id;
cout << "Msg: response for request " << res.id << endl;
}
else {
Request req;
assert(size == sizeof(Request));
memcpy((void*)&req, (void*)msg, size);
id = req.id;
cout << "Msg: request " << req.id << endl;
// send response
Response res;
res.id = req.id;
sendUdp("127.0.0.1", remote_port, (const char*)&res, sizeof(Response));
}
// check for termination condition (both for client and server)
if (received_msgs_cnt == num_of_msgs) {
cout << "This is the last msg" << endl;
udpServerFinished = true;
}
// mark the id of the current message
is_received[id] = true;
// if this was the last message, print the missing msg ids
if (id == num_of_msgs - 1) {
cout << "missing ids: ";
for (int i = 0; i < num_of_msgs; i++) {
if (is_received[i] == false)
cout << i << ", ";
}
cout << endl;
cout << "% of missing messages: "
<< 1.0 - ((double)received_msgs_cnt / num_of_msgs) << endl;
}
}
void OnCanWrite(void* /*user_data*/, grpc_closure* /*notify_on_write_closure*/) override {
gpr_mu_lock(g_mu);
GPR_ASSERT(GRPC_LOG_IF_ERROR("pollset_kick",
grpc_pollset_kick(UdpHandler::g_pollset, nullptr)));
gpr_mu_unlock(g_mu);
}
void OnFdAboutToOrphan(grpc_closure *orphan_fd_closure, void* /*user_data*/) override {
grpc_core::ExecCtx::Run(DEBUG_LOCATION, orphan_fd_closure, GRPC_ERROR_NONE);
}
grpc_fd *emfd() { return emfd_; }
private:
grpc_fd *emfd_;
};
int UdpHandler::g_num_listeners = 1;
grpc_pollset *UdpHandler::g_pollset;
gpr_mu *UdpHandler::g_mu;
/****************************************
* Factory class (generated UDP handler)
****************************************/
class UdpHandlerFactory : public GrpcUdpHandlerFactory {
public:
GrpcUdpHandler *CreateUdpHandler(grpc_fd *emfd, void *user_data) override {
UdpHandler *handler = new UdpHandler(emfd, user_data);
return handler;
}
void DestroyUdpHandler(GrpcUdpHandler *handler) override {
delete reinterpret_cast<UdpHandler *>(handler);
}
};
/****************************************
* Main function
****************************************/
int main(int argc, char *argv[]) {
if (argc != 2) {
cerr << "Usage: './run client' or './run server' " << endl;
return 1;
}
string r(argv[1]);
if (r == "client") {
cout << "Client is initializing to send requests!" << endl;
role = Role::CLIENT;
listening_port = client_port;
remote_port = server_port;
}
else if (r == "server") {
cout << "Server is initializing to accept requests!" << endl;
role = Role::SERVER;
listening_port = server_port;
remote_port = client_port;
}
else {
cerr << "Usage: './run client' or './run server' " << endl;
return 1;
}
/********************************************************
* Initialize UDP Listener
********************************************************/
/* Initialize the grpc library. After it's called,
* a matching invocation to grpc_shutdown() is expected. */
grpc_init();
grpc_core::ExecCtx exec_ctx;
UdpHandler::g_pollset = static_cast<grpc_pollset *>(
gpr_zalloc(grpc_pollset_size()));
grpc_pollset_init(UdpHandler::g_pollset, &UdpHandler::g_mu);
grpc_resolved_address resolved_addr;
struct sockaddr_storage *addr =
reinterpret_cast<struct sockaddr_storage *>(resolved_addr.addr);
int svrfd;
grpc_udp_server *s = grpc_udp_server_create(nullptr);
grpc_pollset *pollsets[1];
memset(&resolved_addr, 0, sizeof(resolved_addr));
resolved_addr.len = static_cast<socklen_t>(sizeof(struct sockaddr_storage));
addr->ss_family = AF_INET;
grpc_sockaddr_set_port(&resolved_addr, listening_port);
/* setup UDP server */
UdpHandlerFactory handlerFactory;
int rcv_buf_size = 1024;
int snd_buf_size = 1024;
GPR_ASSERT(grpc_udp_server_add_port(s, &resolved_addr, rcv_buf_size,
snd_buf_size, &handlerFactory,
UdpHandler::g_num_listeners) > 0);
svrfd = grpc_udp_server_get_fd(s, 0);
GPR_ASSERT(svrfd >= 0);
GPR_ASSERT(getsockname(svrfd, (struct sockaddr *) addr,
(socklen_t *) &resolved_addr.len) == 0);
GPR_ASSERT(resolved_addr.len <= sizeof(struct sockaddr_storage));
pollsets[0] = UdpHandler::g_pollset;
grpc_udp_server_start(s, pollsets, 1, nullptr);
string addr_str = grpc_sockaddr_to_string(&resolved_addr, 1);
cout << "UDP Server listening on: " << addr_str << endl;
thread udpPollerThread(
UdpHandler::startLoop, ref(udpServerFinished));
/********************************************************
* Establish connection to the other side
********************************************************/
struct sockaddr_in serv_addr;
struct hostent *server = gethostbyname("127.0.0.1");
bzero((char *) &serv_addr, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
bcopy((char *) server->h_addr,
(char *) &serv_addr.sin_addr.s_addr,
server->h_length);
serv_addr.sin_port = htons(remote_port);
fd = socket(serv_addr.sin_family, SOCK_DGRAM, 0);
GPR_ASSERT(fd >= 0);
GPR_ASSERT(connect(fd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)) == 0);
/********************************************************
* Send requests
********************************************************/
if (role == Role::CLIENT) {
static int counter = 0;
for (int i = 0; i < num_of_msgs; i++) {
Request req;
req.id = counter++;
cout << "Sending request " << req.id << endl;
sendUdp("127.0.0.1", remote_port, (char*)&req, sizeof(Request));
}
}
/********************************************************
* wait for client to finish
********************************************************/
udpPollerThread.join();
/********************************************************
* cleanup
********************************************************/
close(fd);
gpr_free(UdpHandler::g_pollset);
grpc_shutdown();
cout << "finished successfully!" << endl;
return 0;
}
Compiled with:
-std=c++17 -I$(GRPC_DIR) -I$(GRPC_DIR)/third_party/abseil-cpp.
Linked with:
pkg-config --libs grpc++

Get device path based on USB VID:PID in Linux

If I plug in a device, say /dev/ttyUSB0 and I want to get the number 0 based on its VID:PID (found with lsusb), how could I do that in C++ Linux? I have this code to find one printer device, if it's helpful at all:
int printer_open (void)
{
char printer_location[] = "/dev/usb/lpX";
struct stat buf;
// continuously try all numbers until stat returns true for the connected printer
for (int i = 0; i < 10; i++)
{
printer_location[11] = '0' + i;
if (!stat (printer_location, &buf))
break;
}
return 0;
}
You could use libusb
apt-get install build-essential libudev-dev
Here is a good example: http://www.dreamincode.net/forums/topic/148707-introduction-to-using-libusb-10/
and here is the lib description: http://libusb.sourceforge.net/api-1.0/
int main() {
libusb_context *context = NULL;
libusb_device **list = NULL;
int rc = 0;
ssize_t count = 0;
rc = libusb_init(&context);
assert(rc == 0);
count = libusb_get_device_list(context, &list);
assert(count > 0);
for (size_t idx = 0; idx < count; ++idx) {
libusb_device *device = list[idx];
libusb_device_descriptor desc = {0};
rc = libusb_get_device_descriptor(device, &desc);
assert(rc == 0);
printf("Vendor:Device = %04x:%04x\n", desc.idVendor, desc.idProduct);
}
}
And if you compile your code don't forget to add the lib reference -I/usr/include/libusb-1.0/ and - lusb-1.0
libusb can't get it actually. So look at this file instead: /proc/bus/input/devices
Example line from the file:
I: Bus=0003 Vendor=1a2c Product=0c23 Version=0110
N: Name="USB USB Keyboard"
P: Phys=usb-0000:00:14.0-3/input0
S: Sysfs=/devices/pci0000:00/0000:00:14.0/usb1/1-3/1-3:1.0/0003:1A2C:0C23.0015/input/input30
U: Uniq=
H: Handlers=sysrq kbd event10 leds
B: PROP=0
B: EV=120013
B: KEY=1000000000007 ff800000000007ff febeffdff3cfffff fffffffffffffffe
B: MSC=10
B: LED=7
This function gets the event number from the device with the matching VID:PID:
#include <string>
#include <iostream>
#include <fstream>
void open_device (std::string device_vid, std::string device_pid)
{
try
{
std::ifstream file_input;
std::size_t pos;
std::string device_path, current_line, search_str, event_str;
std::string device_list_file = "/proc/bus/input/devices";
bool vid_pid_found = false;
int fd = 0;
bool debug = true;
// 1. open device list file
file_input.open(device_list_file.c_str());
if (!file_input.is_open())
{
std::cerr << "file_input.open >> " << std::strerror(errno) << std::endl;
throw -2;
}
// 2. search for first VID:PID and get event number
search_str = "Vendor=" + device_vid + " Product=" + device_pid;
while (getline(file_input, current_line))
{
if (!vid_pid_found)
{
pos = current_line.find(search_str, 0);
if (pos != std::string::npos)
{
vid_pid_found = true;
search_str = "event";
}
}
else
{
pos = current_line.find(search_str, 0);
if (pos != std::string::npos)
{
event_str = current_line.substr(pos);
// find space and substring event##
pos = event_str.find(' ', 0);
event_str = event_str.substr(0, pos);
break;
}
}
}
// 3. build device path
device_path = "/dev/input/" + event_str;
if (debug) std::cout << "device_path = " << device_path << std::endl;
// 4. connect to device
fd = open (device_path.c_str(), O_RDONLY);
if (fd < 0)
{
std::cerr << "open >> errno = " << std::strerror(errno) << std::endl;
throw -3;
}
}
catch (const std::exception &e)
{
std::cerr << "e.what() = " << e.what() << std::endl;
throw -1;
}
return;
}