I was trying to make up a code that I saw for some distributed application to see its working, but I am getting an error. I am trying to see the working of messages
The code is
class Address {
public:
char addr[6];
Address() {}
Address(string address) {
size_t pos = address.find(":");
int id = stoi(address.substr(0, pos));
short port = (short)stoi(address.substr(pos + 1, address.size()-pos-1));
memcpy(&addr[0], &id, sizeof(int));
memcpy(&addr[4], &port, sizeof(short));
}
};
enum MsgTypes{
JOINREQ,
JOINREPLY,
DUMMYLASTMSGTYPE,
HEARTBEAT
};
/**
* STRUCT NAME: MessageHdr
*
* DESCRIPTION: Header and content of a message
*/
typedef struct MessageHdr {
enum MsgTypes msgType;
}MessageHdr;
typedef struct en_msg {
// Number of bytes after the class
int size;
// Source node
Address from;
// Destination node
Address to;
}en_msg;
void send(Address *myaddr, Address *toaddr, char *data, int size);
int main()
{
MessageHdr *msg;
size_t msgsize = sizeof(MessageHdr) + sizeof(Address) + sizeof(long) + 1;
int id=233;
short port =22;
long heartbeat=1;
string s=to_string(id)+to_string(port);
string s1=to_string(id+1)+to_string(port+1);
Address *addr= new Address(s);
Address *toaddr= new Address(s);
msg->msgType = JOINREQ;
memcpy((char *)(msg+1), addr, sizeof(addr));
memcpy((char *)(msg+1) + 1 + sizeof(addr), (char *)heartbeat, sizeof(long));
send(addr, toaddr, (char *)msg, msgsize);
}
void send(Address *myaddr, Address *toaddr, char *data, int size) {
en_msg *em;
static char temp[2048];
em = (en_msg *)malloc(sizeof(en_msg) + size);
em->size = size;
memcpy(&(em->from), &(myaddr), sizeof(em->from));
memcpy(&(em->to), &(toaddr), sizeof(em->from));
memcpy(em + 1, data, size);
cout<<em;
}
The error is just this line :
Segmentation fault (core dumped)
1) as Retired Ninja said in comments, first line of main must be like
MessageHdr *msg = new MessageHdr();
because msg->msgType = JOINREQ; will cause an error with unassigned msg.
2) the first fix will not help because of expressions like
(char *)(msg+1)
here msg of type MessageHdr * used as char * in address arithmetic. I mean that it is dangerous to calculate addresses by expression of style
(char *)(msg+1) + 1
while msg is of MessageHdr* type, (msg+1) - means a shift to next structure MessageHdr and the additional +1 after casting to char* means shift to one byte. Personally I cannot understand the logic, whereas MessageHdr structure has only one enum field and with so strange address manipulation you are trying to fit instance of Address class (or long int value) to that structure with memcpy.
Conclusion:
Very substantial program redesign is needed with writing comments and using clear logic in operations.
Related
My goal is to create a unique ID for all IP address - port pair. The UID must be same across systems (no conflict for different endian systems). Size of IPV4 UID is 6 bytes and for ipv6 is 18 bytes.
uint8_t sourcePair[18]; /*ipv4=(4+2) bytes or ipv6=(16+2) bytes*/
I have two functions that will take the remote endpoint of a socket and get the desired UID. The design is as follows.
void CmdInterpreter::makeSourcePairV4(asio::ip::tcp::endpoint& remoteEp, unsigned short portNum, unsigned char(&binSourcePair)[18])
{
auto addressClass = remoteEp.address().to_v4();
auto ipBin = addressClass.to_uint();
memcpy(&binSourcePair[0], &ipBin, 4);
memcpy(&binSourcePair[4], &portNum, 2);
}
void CmdInterpreter::makeSourcePairV6(asio::ip::tcp::endpoint& remoteEp, unsigned short portNum, unsigned char(&binSourcePair)[18])
{
auto addressClass = remoteEp.address().to_v6();
auto ipBin = addressClass.to_bytes();
memcpy(&binSourcePair[0], &ipBin[0], 16);
memcpy(&binSourcePair[16], &portNum, 2);
}
This is how these functions are called
remoteEp = socketPtr->remote_endpoint();
if (remoteEp.address().is_v4())
CmdInterpreter::makeSourcePairV4(remoteEp, remoteEp.port(), sourcePair);
else
CmdInterpreter::makeSourcePairV6(remoteEp, remoteEp.port(), sourcePair);
Here the problem is the only way to access the IPv6 underlying data is using to_byte() which will give the data in network byte order. Also, I am doing a memcopy in unsigned short which is multibyte in length. Does this work? Is it a safe way? Is their any workarounds?
void CmdInterpreter::makeSourcePairV4(asio::ip::tcp::endpoint& remoteEp, unsigned short portNum, uint8_t(&sourcePair)[18])
{
auto addressClass = remoteEp.address().to_v4();
auto ipBin = addressClass.to_bytes();
memcpy(&sourcePair[0], &ipBin[0], 4);
#ifdef BOOST_ENDIAN_LITTLE_BYTE
byteSwap(portNum);
#endif
memcpy(&sourcePair[4], &portNum, 2);
}
void CmdInterpreter::makeSourcePairV6(asio::ip::tcp::endpoint& remoteEp, unsigned short portNum, uint8_t(&sourcePair)[18])
{
auto addressClass = remoteEp.address().to_v6();
auto ipBin = addressClass.to_bytes();
memcpy(&sourcePair[0], &ipBin[0], 16);
#ifdef BOOST_ENDIAN_LITTLE_BYTE
byteSwap(portNum);
#endif
memcpy(&sourcePair[16], &portNum, 2);
}
For both IPv4 and IPv6 address, use to_byte() function to get the remote endpoint address in big-endian format. For little-endian host, the port number will make endianness problem which can be fixed by swapping the bytes. To encode it to base 64 I used cppcodec library.
UID = cppcodec::base64_rfc4648::encode(sourcePair, 6);
UID = cppcodec::base64_rfc4648::encode(sourcePair, 18);
The template function used to swap the port number is:
template <typename T>
void byteSwap(T& portNumber)
{
char* startIndex = static_cast<char*>((void*)&portNumber);
char* endIndex = startIndex + sizeof(T);
std::reverse(startIndex, endIndex);
}
I use fuse to build my own file system in MIT 6.824 lab, and the read operation is implemented in this function.
void
fuseserver_read(fuse_req_t req, fuse_ino_t ino, size_t size,
off_t off, struct fuse_file_info *fi)
{
std::string buf;
int r;
if ((r = yfs->read(ino, size, off, buf)) == yfs_client::OK) {
char* retbuf = (char *)malloc(buf.size());
memcpy(retbuf,buf.data(),buf.size());
//Print the information of the result.
printf("debug read in fuse: the content of %lu is %s, size %lu\n",ino,retbuf, buf.size());
fuse_reply_buf(req,retbuf,buf.size());
} else {
fuse_reply_err(req, ENOENT);
}
//global definition
//struct fuse_lowlevel_ops fuseserver_oper;
//In main()
// fuseserver_oper.read = fuseserver_read;
I print the information of the buf before it return.
The write operation is also implemented, of course.
Then I run a simple test to read out some words.
//test.c
int main(){
//./yfs1 is the mount point of my filesystem
int fd = open("./yfs1/test-file",O_RDWR | O_CREAT,0777);
char* buf = "123";
char* readout;
readout = (char *)malloc(3);
int writesize = write(fd,buf,3);
int readsize = read(fd,readout,3);
printf("%s,%d\n",buf,writesize);
printf("%s,%d\n",readout,readsize);
close(fd);
}
I can get nothing by read(fd,readout,3), but the information printed by the fuseserver_read shows that the buffer is read out successfully before fuse_reply_buf
$ ./test
123,3
,0
debug read in fuse: the content of 2 is 123, size 3
So why the read() in test.c can not read anything from my file system??
Firstly, I've made a mistake to write my test file. The file pointer will point to the end of the file after "write" and of course can read nothing later. So simply reopen the file can make the test work.
Secondly, before read() operation of FUSE, the FUSE will getattr() first and truncate the result of the read() operation with the "size" attribute of the file. So it must be very careful to manipulate the attribute of a file.
There is also a need to notify that you have finished reading by sending an empty buffer, as an "EOF". You can do that by using reply_buf_limited.
Take a look at hello_ll example in the fuse source tree:
static void tfs_read(fuse_req_t req, fuse_ino_t ino, size_t size,
off_t off, struct fuse_file_info *fi) {
(void) fi;
assert(ino == FILE_INO);
reply_buf_limited(req, file_contents, file_size, off, size);
}
static int reply_buf_limited(fuse_req_t req, const char *buf, size_t bufsize,
off_t off, size_t maxsize)
{
if (off < bufsize)
return fuse_reply_buf(req, buf + off,
min(bufsize - off, maxsize));
else
return fuse_reply_buf(req, NULL, 0);
}
i was trying to right some socket programming stuff and i faced the following error while trying to make the cpp files , i'm on OS X 10.11 :
user-router/sm.cpp:84:3: error: unknown type name 'iphdr'
here is the code for that specific section :
void SimulatedMachine::run () {
// TODO: write your business logic here...
struct ethernet_header {
byte dst[6];
byte src[6];
uint16 type;
} __attribute__ ((packed));
const int frameLength = sizeof (ethernet_header) + 100;
byte *data = new byte[frameLength];
ethernet_header *eth = (ethernet_header *) data;
memset (eth->dst, 255, 6); // broadcast address
memcpy (eth->src, iface[0].mac, 6);
eth->type = htons (0x0800);
iphdr *packet = (iphdr *) (data + sizeof (ethernet_header));
packet->version = 4;
packet->ihl = 5;
packet->tot_len = htons (100);
Frame frame (frameLength, data);
sendFrame (frame, 0); // sends frame on interface 0
delete[] data;
cerr << "now ./free.sh and check the pcap log file to see the sent packet" << endl;
}
as iphdr is supposed to exist in i have that included as well.
any help is appreciated .
Need to write an application in C/C++ on Linux that receives a stream of bytes from a socket and process them. The total bytes could be close to 1TB. If I have unlimited amount memory, I will just put it all in the memory, so my application can easily process data. It's much easy to do many things on flat memory space, such as memmem(), memcmp() ... On a circular buffer, the application has to be extra smart to be aware of the circular buffer.
I have about 8G of memory, but luckily due to locality, my application never needs to go back by more than 1GB from the latest data it received. Is there a way to have a 1TB buffer, with only the latest 1GB data mapped to physical memory? If so, how to do it?
Any ideas? Thanks.
Here's an example. It sets up a full terabyte mapping, but initially inaccessible (PROT_NONE). You, the programmer, maintain a window that can only extend and move upwards in memory. The example program uses a one and a half gigabyte window, advancing it in steps of 1,023,739,137 bytes (the mapping_use() makes sure the available pages cover at least the desired region), and does actually modify every page in every window, just to be sure.
#define _GNU_SOURCE
#define _POSIX_C_SOURCE 200809L
#include <stdlib.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <stdio.h>
typedef struct mapping mapping;
struct mapping {
unsigned char *head; /* Start of currently accessible region */
unsigned char *tail; /* End of currently accessible region */
unsigned char *ends; /* End of region */
size_t page; /* Page size of this mapping */
};
/* Discard mapping.
*/
void mapping_free(mapping *const m)
{
if (m && m->ends > m->head) {
munmap(m->head, (size_t)(m->ends - m->head));
m->head = NULL;
m->tail = NULL;
m->ends = NULL;
m->page = 0;
}
}
/* Move the accessible part up in memory, to [from..to).
*/
int mapping_use(mapping *const m, void *const from, void *const to)
{
if (m && m->ends > m->head) {
unsigned char *const head = ((unsigned char *)from <= m->head) ? m->head :
((unsigned char *)from >= m->ends) ? m->ends :
m->head + m->page * (size_t)(((size_t)((unsigned char *)from - m->head)) / m->page);
unsigned char *const tail = ((unsigned char *)to <= head) ? head :
((unsigned char *)to >= m->ends) ? m->ends :
m->head + m->page * (size_t)(((size_t)((unsigned char *)to - m->head) + m->page - 1) / m->page);
if (head > m->head) {
munmap(m->head, (size_t)(head - m->head));
m->head = head;
}
if (tail > m->tail) {
#ifdef USE_MPROTECT
mprotect(m->tail, (size_t)(tail - m->tail), PROT_READ | PROT_WRITE);
#else
void *result;
do {
result = mmap(m->tail, (size_t)(tail - m->tail), PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_FIXED | MAP_PRIVATE | MAP_NORESERVE, -1, (off_t)0);
} while (result == MAP_FAILED && errno == EINTR);
if (result == MAP_FAILED)
return errno = ENOMEM;
#endif
m->tail = tail;
}
return 0;
}
return errno = EINVAL;
}
/* Initialize a mapping.
*/
int mapping_create(mapping *const m, const size_t size)
{
void *base;
size_t page, truesize;
if (!m || size < (size_t)1)
return errno = EINVAL;
m->head = NULL;
m->tail = NULL;
m->ends = NULL;
m->page = 0;
/* Obtain default page size. */
{
long value = sysconf(_SC_PAGESIZE);
page = (size_t)value;
if (value < 1L || (long)page != value)
return errno = ENOTSUP;
}
/* Round size up to next multiple of page. */
if (size % page)
truesize = size + page - (size % page);
else
truesize = size;
/* Create mapping. */
do {
errno = ENOTSUP;
base = mmap(NULL, truesize, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, (off_t)0);
} while (base == MAP_FAILED && errno == EINTR);
if (base == MAP_FAILED)
return errno;
/* Success. */
m->head = base;
m->tail = base;
m->ends = (unsigned char *)base + truesize;
m->page = page;
errno = 0;
return 0;
}
static void memtouch(void *const ptr, const size_t size)
{
if (ptr && size > 0) {
unsigned char *mem = (unsigned char *)ptr;
const size_t step = 2048;
size_t n = size / (size_t)step - 1;
mem[0]++;
mem[size-1]++;
while (n-->0) {
mem += step;
mem[0]++;
}
}
}
int main(void)
{
const size_t size = (size_t)1024 * (size_t)1024 * (size_t)1024 * (size_t)1024;
const size_t need = (size_t)1500000000UL;
const size_t step = (size_t)1023739137UL;
unsigned char *base;
mapping map;
size_t i;
if (mapping_create(&map, size)) {
fprintf(stderr, "Cannot create a %zu-byte mapping: %m.\n", size);
return EXIT_FAILURE;
}
printf("Have a %zu-byte mapping at %p to %p.\n", size, (void *)map.head, (void *)map.ends);
fflush(stdout);
base = map.head;
for (i = 0; i <= size - need; i += step) {
printf("Requesting %p to %p .. ", (void *)(base + i), (void *)(base + i + need));
fflush(stdout);
if (mapping_use(&map, base + i, base + i + need)) {
printf("Failed (%m).\n");
fflush(stdout);
return EXIT_FAILURE;
}
printf("received %p to %p.\n", (void *)map.head, (void *)map.tail);
fflush(stdout);
memtouch(base + i, need);
}
mapping_free(&map);
return EXIT_SUCCESS;
}
The approach is twofold. First, an inaccessible (PROT_NONE) mapping is created to reserve the necessary virtual contiguous address space. If we omit this step, it would make it possible for a malloc() call or similar to acquire pages within this range, which would defeat the entire purpose; a single terabyte-long mapping.
Second, when the accessible window extends into the region, either mprotect() (if USE_MPROTECT is defined), or mmap() is used to make the required pages accessible. Pages no longer needed are completely unmapped.
Compile and run using
gcc -Wall -Wextra -std=c99 example.c -o example
time ./example
or, to use mmap() only once and mprotect() to move the window,
gcc -DUSE_MPROTECT=1 -Wall -Wextra -std=c99 example.c -o example
time ./example
Note that you probably don't want to run the test if you don't have at least 4GB of physical RAM.
On this particular machine (i5-4200U laptop with 4GB of RAM, 3.13.0-62-generic kernel on Ubuntu x86_64), quick testing didn't show any kind of performance difference between mprotect() and mmap(), in execution speed or resident set size.
If anyone bothers to compile and run the above, and finds that one of them has a repeatable benefit/drawback (resident set size or time used), I'd very much like to know about it. Please also define your kernel and CPU used.
I'm not sure which details I should expand on, since this is pretty straightforward, really, and the Linux man pages project man 2 mmap and man 2 mprotect pages are quite descriptive. If you have any questions on this approach or program, I'd be happy to try and elaborate.
I'm attempting to use the SoundTouch C++ library for audio speed and pitch changes in an Android app. I have successfully pushed a Java byte[] array (from a .wav) through JNI, returned it, and played it back with an AudioTrack.
The next step is attempting to push a sample byte[] through the SoundTouch pipeline. I have dissected the source of the SoundStretch console program included with the library and have attempted to adapt it. I am using a stereo, 16-bit source for testing purposes.
With my current temporary setup I am ignoring the RIFF header and converting it along with the .wav data because the Java AudioTrack object does not need to read the header, it just plays raw PCM. Playing the raw byte[] without sending through SoundTouch just results in a small click where the header is.
After sending through the SoundTouch pipeline, I am playing back white noise where the beginning of the audio is supposed to be. I assume I am having a problem at the end of my write() function, where I am casting short's to signed chars. Here, the console app is writing to a file, instead of pushing to a vector:
int res = (int)fwrite(temp, 1, numBytes, fptr);
I have read the documentation for fwrite but I don't know enough about bit twiddling or audio processing to know what to do here to correctly get this information in a char[] instead of writing to a file. I know I am loosing information with the cast, but I am unsure of how to correct it.
In case anyone is extra motivated, the SoundStretch source can be found here: http://www.surina.net/soundtouch/sourcecode.html
extern "C" DLL_PUBLIC jbyteArray
Java_net_surina_soundtouch_SoundTouch_getMutatedBytes
(JNIEnv *env, jobject thiz, jbyteArray input, jint length)
{
const int BUFF_SIZE = 2048000;
SoundTouch soundTouch;
jboolean isCopy;
jbyte* ar = env->GetByteArrayElements(input, &isCopy);
signed char* cBufferIn = (signed char*)ar;
SAMPLETYPE* fBufferIn = new SAMPLETYPE[length];
vector<signed char> fBufferOut;
//converts the chars to floats per the SoundTouch console app.
convertInput16(cBufferIn, fBufferIn, length);
//channels, sampling rate, speed, pitch change
setup(&soundTouch, 2, 44100, 1.0, 0);
//transform floats from fBufferIn to fBufferout
process(&soundTouch, fBufferIn, fBufferOut, BUFF_SIZE);
signed char* res = &fBufferOut[0];
jbyteArray result = env->NewByteArray(length);
env->SetByteArrayRegion(result, 0, fBufferOut.size(), res);
LOGV("fBufferOut Size: %d", fBufferOut.size());
delete[] fBufferIn;
return result;
}
process():
static void process(SoundTouch* soundTouch, SAMPLETYPE* fBufferIn, vector<signed char>& fBufferOut, int BUFF_SIZE)
{
int nSamples = BUFF_SIZE / 2; //2 bytes per sample, using 16 bit sample for testing
int buffSizeSamples = BUFF_SIZE / 2; //2 channel stereo
soundTouch->putSamples(fBufferIn, nSamples);
do
{
nSamples = soundTouch->receiveSamples(fBufferIn, buffSizeSamples);
write(fBufferIn, fBufferOut, nSamples / 2); //2 channels
} while (nSamples != 0);
soundTouch->flush();
do
{
nSamples = soundTouch->receiveSamples(fBufferIn, buffSizeSamples);
write(fBufferIn, fBufferOut, nSamples / 2);
LOGV("NUMBER OF SAMPLES: %d", nSamples);
} while (nSamples != 0);
}
write():
static void write(const float *bufferIn, vector<signed char>& bufferOut, int numElems)
{
int numBytes;
int bytesPerSample;
if (numElems == 0) return;
bytesPerSample = 16 / 8; //16 bit test sample / bits in a byte
numBytes = numElems * bytesPerSample;
short *temp = (short*)getConvBuffer(numBytes);
switch (bytesPerSample)
{
case 2: //16 bit encoding per the SoundStretch console app
{
short *temp2 = (short *)temp;
for (int i = 0; i < numElems; i++)
{
short value = (short)saturate(bufferIn[i] * 32768.0f, -32768.0f, 32767.0f); //magic to me
temp2[i] = value; //works for little endian only.
}
break;
}
default:
assert(false);
}
for (int i = 0; i < numElems; ++i)
{
bufferOut.push_back((signed char)temp[i]); //I think my problem is here.
}
delete[] temp;
//bytesWritten += numBytes;
}
I just needed to get all the bits in char[]:
for (int i = 0; i < numElems; ++i)
{
bufferOut.push_back(temp[i] & 0xff);
bufferOut.push_back((temp[i] >> 8) & 0xff);
}