zlib different decompresssion size - c++

I am trying to use zlib for decompression. Im look at a tutorial at zlib site and inflate codes produce different size output.
int CZLib::Inflate() {
int ret;
unsigned int have;
z_stream zstream;
unsigned char in[CHUNK];
unsigned char out[CHUNK];
zstream.zalloc = Z_NULL;
zstream.zfree = Z_NULL;
zstream.opaque = Z_NULL;
zstream.avail_in = 0;
zstream.next_in = Z_NULL;
ret = inflateInit(&zstream);
if (ret != Z_OK)
return ret;
do {
zstream.avail_in = fread(in, 1, CHUNK, fin);
if (ferror(fin)) {
(void)inflateEnd(&zstream);
return Z_ERRNO;
}
if (zstream.avail_in == 0) break;
zstream.next_in = in;
do {
zstream.avail_out = CHUNK;
zstream.next_out = out;
ret = inflate(&zstream, Z_NO_FLUSH);
assert(ret != Z_STREAM_ERROR);
switch (ret) {
case Z_NEED_DICT:
ret = Z_DATA_ERROR;
case Z_DATA_ERROR:
case Z_MEM_ERROR:
(void)inflateEnd(&zstream);
return ret;
}
have = CHUNK - zstream.avail_out;
if (fwrite(out, 1, have, fout) != have || ferror(fout)) {
(void)inflateEnd(&zstream);
return Z_ERRNO;
}
} while (zstream.avail_out == 0);
} while (ret != Z_STREAM_END);
(void)inflateEnd(&zstream);
return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
}
and other
int CZLib::Inflate(const std::string& src) {
std::vector<char> output;
z_stream zstream;
zstream.zalloc = Z_NULL;
zstream.zfree = Z_NULL;
zstream.opaque = Z_NULL;
zstream.avail_in = 0;
zstream.next_in = Z_NULL;
int ret = inflateInit(&zstream);
if (ret != Z_OK)
return ret;
unsigned char in[CHUNK];
unsigned char out[CHUNK];
int have = 0, nByte = CHUNK, off = 0, remaining = src.size();
if (src.size() < CHUNK) nByte = src.size();
do {
memcpy(in, &src[off], nByte);
off += nByte;
remaining -= nByte;
if (nByte > 0) zstream.avail_in = nByte;
if (remaining > CHUNK) { nByte = CHUNK; }
else { nByte = remaining; }
if (zstream.avail_in == 0) break;
zstream.next_in = in;
do {
zstream.avail_out = CHUNK;
zstream.next_out = out;
ret = inflate(&zstream, Z_NO_FLUSH);
have = CHUNK - zstream.avail_out;
output.insert(output.end(), out, out + have);
} while (zstream.avail_out == 0);
} while (ret != Z_STREAM_END);
CFile* file = new CFile("in.out", "wb");
file->Write<char>(&output[0], output.size());
delete file;
return ret;
}
Its uses same data. One of them reads file on disk and other uses memory (buffer method). CHUNK size 16384. First code produce 524288(0x80000) and other 524800 (0x80200) byte. The difference are 512 bytes. Why is it happening ?

In the first code example you have this line
zstream.avail_in = fread(in, 1, CHUNK, fin);
and then you have
if (zstream.avail_in == 0) break;
to stop the loop.
In the second code example, you have the same line to stop the loop but you also have this line:
if (nByte > 0) zstream.avail_in = nByte;
^^^^^^^^^
So you only assign to zstream.avail_in when nByte > 0
....
....
if (zstream.avail_in == 0) break;
^^^^^^^^^^^^^^^^
Consequently this will not be true when nByte is zero and the
code will not exit
Try this instead:
zstream.avail_in = nByte; // Unconditional assignment
....
if (zstream.avail_in <= 0) break; // Less or equal to zero

Related

Add actual timestamp to mp4 using ffmpeg

I'm using ffmpeg to write an h264 stream to a mp4 file.
Everything is working, but now I need to embed to this file the actual timestamp in milliseconds of each frame.
Is it possible?
This is my code:
void mp4_file_create(mp4_par * par, t_image * img_in)
{
AVCodec * codec = NULL;
AVCodecContext * cc_in;
AVFormatContext * av_fmt_ctx_out;
AVStream * av_stream;
AVPacket av_pkt;
AVFormatContext * ifmt_ctx;
unsigned long long last_frame_ts_utc;
unsigned long long last_frame_ts_absolute;
unsigned long long last_pts;
t_mp4_dict_metadata metadata;
char file_name[1024];
char TSstrdate[128];
av_register_all();
cc_in = NULL;
av_stream = NULL;
if (avformat_alloc_output_context2(&mp4h->av_fmt_ctx_out, NULL, NULL, file_name) < 0) {
trace_error("avformat_alloc_output_context2 failed");
goto FnExit;
}
/* find the H264 RAW encoder */
codec = avcodec_find_encoder(AV_CODEC_ID_H264);
if (!codec) {
int ret;
AVStream *in_stream = NULL;
if (av_fmt_ctx_in == NULL)
{
trace_error("av_fmt_ctx_in is NULL");
goto FnExit;
}
in_stream = av_fmt_ctx_in->streams[0];
in_stream->codec->width = par.width;
in_stream->codec->height = par.height;
in_stream->codec->coded_width = par.width;
in_stream->codec->coded_height = par.height;
in_stream->codec->bit_rate = 1024;
in_stream->codec->flags = CODEC_FLAG_GLOBAL_HEADER;
in_stream->codec->time_base.num = 1;
in_stream->codec->time_base.den = par.frame_rate;
in_stream->codec->gop_size = par.gop;
in_stream->codec->pix_fmt = AV_PIX_FMT_YUV420P;
av_stream = avformat_new_stream(mp4h->av_fmt_ctx_out, in_stream->codec->codec);
if (!av_stream) {
trace_error("Failed allocating output stream");
goto FnExit;
}
ret = avcodec_copy_context(av_stream->codec, in_stream->codec);
if (ret != 0) {
goto FnExit;
}
}
else {
int ret;
av_stream = avformat_new_stream(mp4h->av_fmt_ctx_out, NULL);
if (!av_stream) {
goto FnExit;
}
cc_in = avcodec_alloc_context3(codec);
if (cc_in == NULL) {
goto FnExit;
}
cc_in->width = par.width;
cc_in->height = par.height;
cc_in->bit_rate = 1024;
cc_in->flags = CODEC_FLAG_GLOBAL_HEADER;
cc_in->time_base.num = 1;
cc_in->time_base.den = par.frame_rate;
cc_in->gop_size = par.gop;
cc_in->pix_fmt = AV_PIX_FMT_YUVJ420P;
cc_in->extradata = (unsigned char*)av_mallocz(sizeof(sample_spspps));
cc_in->extradata_size = sizeof(sample_spspps);
memcpy(cc_in->extradata, sample_spspps, cc_in->extradata_size);
ret = avcodec_copy_context(av_stream->codec, cc_in);
if (ret != 0) {
goto FnExit;
}
}
av_stream->codec->codec_tag = 0;
if (av_fmt_ctx_out->oformat->flags & AVFMT_GLOBALHEADER)
av_stream->codec->flags |= CODEC_FLAG_GLOBAL_HEADER;
if (!(av_fmt_ctx_out->flags & AVFMT_NOFILE)) {
int ret = avio_open(&av_fmt_ctx_out->pb, file_name, AVIO_FLAG_READ_WRITE);
if (ret < 0) {
trace_error("Could not open output file '%s'", file_name);
goto FnExit;
}
}
av_fmt_ctx_out->streams[0]->time_base.num = 1;
av_fmt_ctx_out->streams[0]->time_base.den = par.frame_rate;
av_fmt_ctx_out->streams[0]->codec->time_base.num = 1;
av_fmt_ctx_out->streams[0]->codec->time_base.den = par.frame_rate;
AVRational fps;
fps.num = 1;
fps.den = par.frame_rate;
av_stream_set_r_frame_rate(av_fmt_ctx_out->streams[0], fps);
mp4h->av_fmt_ctx_out->streams[0]->first_dts = AV_TIME_BASE;
av_dict_set(&pMetaData, "title", par.guid_video_function, 0);
av_dict_set(&pMetaData, "artist", "Test Artist", 0);
av_dict_set(&pMetaData, "date", TSstrdate, 0);
av_fmt_ctx_out->metadata = pMetaData;
if (avformat_write_header(av_fmt_ctx_out, NULL) < 0) {
goto FnExit;
}
//............. Now for each frame_rate........
av_init_packet(&av_pkt);
if (first_frame)
{
av_pkt.pts = 0;
av_pkt.dts = 0;
}
else
{
av_pkt.pts = last_pts + (long long int)((img->timestamp_absolute - last_frame_ts_absolute) * (unsigned long long)av_stream->time_base.den / 1000000ULL);
av_pkt.dts = last_pts + (long long int)((img->timestamp_absolute - last_frame_ts_absolute) * (unsigned long long)av_stream->time_base.den / 1000000ULL);
}
mp4h->av_pkt.duration = 0;
mp4h->av_pkt.pos = -1;
last_frame_ts_utc = img->timestamp_utc.t;
last_frame_ts_absolute = img->timestamp_absolute.t;
last_pts = av_pkt.pts;
if (img->type == keyframe)
{
av_pkt.flags |= AV_PKT_FLAG_KEY;
}
av_pkt.data = img->ptr;
av_pkt.size = img->size;
av_pkt.stream_index = av_stream->index;
ret = av_interleaved_write_frame(av_fmt_ctx_out, &av_pkt);
if (ret != 0) {
char strE[256];
av_strerror(ret, strE, sizeof(strE));
trace_error("av_write_frame returns %d - %s", ret, strE);
return;
}
//........then I will close the file
FnExit:
if (av_fmt_ctx_out && av_fmt_ctx_out->pb != NULL) {
if (av_write_trailer(mp4h->av_fmt_ctx_out) != 0) {
trace_error("av_write_trailer Error!");
}
}
if (ifmt_ctx)
avformat_close_input(&ifmt_ctx);
avio_closep(&av_fmt_ctx_out->pb);
avcodec_close(av_stream->codec);
avformat_free_context(av_fmt_ctx_out);
}
How can I modify it in order to embed the actual timestamp of each frame?
I tried to add the actual timestamp to the first frame pts instead of setting it to zero, but it didn't work.

ZLib won't compress beyond ~600kb output in D

I've tried both std.zlib and using zlib directly, but it doesn't seem to compress anything beyond the 600kb mark. I have mostly followed all online guides except for the input buffer since I have to compress a preesixting memory space rather than a file on the disk.
Forgot the code, this one doesn't go further beyond ~600kb:
int ret, flush;
zlib.z_stream strm;
strm.zalloc = null;
strm.zfree = null;
strm.opaque = null;
ret = zlib.deflateInit(&strm, compLevel);
if (ret != zlib.Z_OK)
throw new Exception("Compressor initialization error");
ubyte[] output;
output.length = cast(uint)imageData.length;
strm.next_in = imageData.ptr;
strm.avail_in = cast(uint)imageData.length;
strm.next_out = output.ptr;
strm.avail_out = cast(uint)output.length;
do{
ret = zlib.deflate(&strm, zlib.Z_FINISH);
if(!(ret == zlib.Z_OK || ret == zlib.Z_STREAM_END)){
//version(unittest) std.stdio.writeln(strm.total_out);
zlib.deflateEnd(&strm);
throw new Exception("Compressor output error: " ~ cast(string)std.string.fromStringz(strm.msg));
}
} while (ret != zlib.Z_STREAM_END);
writeBuffer = cast(void[])[Chunk(cast(uint)strm.total_out, DATA_INIT).nativeToBigEndian] ~ output[0..cast(size_t)strm.total_out];
file.rawWrite(writeBuffer);
crc = crc32Of(writeBuffer[4..$]).dup.reverse;
file.rawWrite(crc);
Previously I tried this, but this immediately has a stream error with no output:
int ret, flush;
//uint have;
zlib.z_stream strm;
strm.zalloc = null;
strm.zfree = null;
strm.opaque = null;
ret = zlib.deflateInit(&strm, compLevel);
if (ret != zlib.Z_OK)
throw new Exception("Compressor initialization error");
ubyte[] output;
static if(writeblocksize < 2048)
output.length = 2048;
strm.next_in = imageData.ptr;
strm.avail_in = cast(uint)imageData.length;
do {
flush = strm.avail_in ? zlib.Z_NO_FLUSH : zlib.Z_FINISH;
strm.next_out = output.ptr;
strm.avail_out = cast(uint)output.length;
ret = zlib.deflate(&strm, flush);
if(ret == zlib.Z_STREAM_ERROR){
version(unittest) std.stdio.writeln(ret);
zlib.deflateEnd(&strm);
throw new Exception("Compressor output error: " ~ cast(string)std.string.fromStringz(strm.msg));
}
//version(unittest) std.stdio.writeln(strm.total_out);
//writeBuffer = output[0..$-strm.avail_out];
writeBuffer = cast(void[])[Chunk(cast(uint)writeBuffer.length, DATA_INIT).nativeToBigEndian] ~ output;
file.rawWrite(writeBuffer);
crc = crc32Of(writeBuffer[4..$]).dup.reverse;
file.rawWrite(crc);
//writeBuffer.length = 0;
} while (flush != zlib.Z_FINISH);
zlib.deflateEnd(&strm);

My Implementation of xmodem receiving function is not working

I would like to ask for an advice about my code and show me implementing model of yours.
I am implementing program in the code below, and make this code the basis.
Now, I implemented receiver function for xmodem, but it's not working.
"xmodem_receive" called in main function just once printed "Sending C ping ...." as printf and , finished.
Sending xmodem is working as being shown below.
Receving Xmodem is not working.It finished before receiving data.
So I would like to ask
1)Why my xmodem receive function is not working?
2)Why STX is present as "K" ,and that could go through IF statement
if(recSTX == X_STX){
printf("STX is %c\n", &recSTX);
}else{
printf("Garabage payload %c\n", &recSTX);
}
This is my implementation function
static int xmodem_receive(int serial_fd, char* filename, int _crc){
int skip = 0;
uint8_t sdCRC = 'C';
uint8_t sdACK = X_ACK;
uint8_t eof = X_EOF;
uint8_t sdNAK = X_NAK;
uint8_t recSTX; // receive SOH from chunk
uint8_t recBlk; // receive blk from chunk
uint8_t recNegBlk; // receive blk negative from chunk
uint8_t recData[1024]; // receive data from chunk
uint16_t recChksum;
FILE *fp;
int ret, fd;
struct stat stat;
// If we want to receive, We have to send NAK to Sendor.
/* Writing as binary */
fp = fopen(filename, "wb");
//Send NAK still read SOH that header of one data chunks
while(1){
char status;
printf("Waiting for sender ping ...");
fflush(stdout);
//
do {
if(_crc){
printf("Send C ping....\n");
ret = write(serial_fd, &sdCRC, sizeof(sdCRC));
}else{
// send NAK before read SOH
printf("Send NAK ping....\n");
ret = write(serial_fd, &sdNAK, sizeof(sdNAK));
} // after sending NAK,receiving SOH from chunk
fflush(stdout);
ret = read(serial_fd, &recSTX, sizeof(recSTX));
if(recSTX == X_STX){
printf("STX is %c\n", &recSTX);
}else{
printf("Garabage payload %c\n", &recSTX);
}
fflush(stdout);
if (ret != sizeof(recSTX)) {
printf("Not working");
fflush(stdout);
perror("read");
return -errno;
}
} while (recSTX != X_STX);
//So next, receiving blk
ret = read(serial_fd, &recBlk, sizeof(recBlk));
printf("Block Num is %d\n", recBlk);
fflush(stdout);
if (ret != sizeof(recBlk)) {
perror("read");
return -errno;
}
ret = read(serial_fd, &recNegBlk, sizeof(recNegBlk));
printf("Negative Block Num is %d\n", recNegBlk);
fflush(stdout);
if (ret != sizeof(recNegBlk)) {
perror("read");
return -errno;
}
ret = read(serial_fd, (void *)&recData, sizeof(recData));
printf("Data buffer is %c\n", &recData);
fflush(stdout);
if (ret != sizeof(recData)) {
perror("read");
return -errno;
}
ret = read(serial_fd, &recChksum, sizeof(recChksum));
printf("Check sum is %c", &recChksum);
fflush(stdout);
if (ret != sizeof(recChksum)) {
perror("read");
return -errno;
}
// data block number check
if(recBlk == 0xff - recNegBlk){
perror("read");
return -errno;
}
// data integrity check
if(recChksum == swap16(crc16(recData, sizeof(recData)))){
perror("read");
return -errno;
}
// check of appended "0xff" from end of data to end of chunk in chunk
unsigned char *ptr = recData;
ptr += sizeof(recData);
while(*ptr == 0xff){
ptr--;
}
fwrite(recData, (ptr - recData),1,fp); // write Datas bytes from our buffer
// set skip flag or end connect
ret = write(serial_fd, &eof, sizeof(uint8_t));
if (ret != sizeof(eof) || ret != sizeof(X_STX)){
return -errno;
}else{
if(ret == X_STX){
skip = 1;
}else if(ret == EOF){
printf("EOF ...");
ret = write(serial_fd, &sdACK, sizeof(sdACK));
break;
}else{
return -errno;
}
}
}
printf("done.\n");
fclose(fp);
return 0;
}
And This is full code.
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <fcntl.h>
#include <termios.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <string>
#define X_STX 0x02
#define X_ACK 0x06
#define X_NAK 0x15
#define X_EOF 0x04
#define min(a, b) ((a) < (b) ? (a) : (b))
struct xmodem_chunk {
uint8_t start;
uint8_t block;
uint8_t block_neg;
uint8_t payload[1024];
uint16_t crc;
} __attribute__((packed));
#define CRC_POLY 0x1021
static uint16_t crc_update(uint16_t crc_in, int incr)
{
uint16_t xor1 = crc_in >> 15;
uint16_t out1 = crc_in << 1;
if(incr)
out1++;
if(xor1)
out1 ^= CRC_POLY; // xor 0b1000000100001
return out1;
}
static uint16_t crc16(const uint8_t *data, uint16_t size)
{
uint16_t crc, i;
for (crc = 0; size > 0; size--, data++)
for (i = 0x80; i; i >>= 1)
crc = crc_update(crc, *data & i);
for (i = 0; i < 16; i++)
crc = crc_update(crc, 0);
return crc;
}
static uint16_t swap16(uint16_t in)
{
return (in >> 8) | ((in & 0xff) << 8);
}
enum {
PROTOCOL_XMODEM,
PROTOCOL_YMODEM,
};
static int xmodem_receive(int serial_fd, char* filename, int _crc){
int skip = 0;
uint8_t sdCRC = 'C';
uint8_t sdACK = X_ACK;
uint8_t eof = X_EOF;
uint8_t sdNAK = X_NAK;
uint8_t recSTX; // receive SOH from chunk
uint8_t recBlk; // receive blk from chunk
uint8_t recNegBlk; // receive blk negative from chunk
uint8_t recData[1024]; // receive data from chunk
uint16_t recChksum;
FILE *fp;
int ret, fd;
struct stat stat;
// If we want to receive, We have to send NAK to Sendor.
/* Writing as binary */
fp = fopen(filename, "wb");
//Send NAK still read SOH that header of one data chunks
while(1){
char status;
printf("Waiting for sender ping ...");
fflush(stdout);
//
do {
if(_crc){
printf("Send C ping....\n");
ret = write(serial_fd, &sdCRC, sizeof(sdCRC));
}else{
// send NAK before read SOH
printf("Send NAK ping....\n");
ret = write(serial_fd, &sdNAK, sizeof(sdNAK));
} // after sending NAK,receiving SOH from chunk
fflush(stdout);
ret = read(serial_fd, &recSTX, sizeof(recSTX));
if(recSTX == X_STX){
printf("STX is %c\n", &recSTX);
}else{
printf("Garabage payload %c\n", &recSTX);
}
fflush(stdout);
if (ret != sizeof(recSTX)) {
printf("Not working");
fflush(stdout);
perror("read");
return -errno;
}
} while (recSTX != X_STX);
//So next, receiving blk
ret = read(serial_fd, &recBlk, sizeof(recBlk));
printf("Block Num is %d\n", recBlk);
fflush(stdout);
if (ret != sizeof(recBlk)) {
perror("read");
return -errno;
}
ret = read(serial_fd, &recNegBlk, sizeof(recNegBlk));
printf("Negative Block Num is %d\n", recNegBlk);
fflush(stdout);
if (ret != sizeof(recNegBlk)) {
perror("read");
return -errno;
}
ret = read(serial_fd, (void *)&recData, sizeof(recData));
printf("Data buffer is %c\n", &recData);
fflush(stdout);
if (ret != sizeof(recData)) {
perror("read");
return -errno;
}
ret = read(serial_fd, &recChksum, sizeof(recChksum));
printf("Check sum is %c", &recChksum);
fflush(stdout);
if (ret != sizeof(recChksum)) {
perror("read");
return -errno;
}
// data block number check
if(recBlk == 0xff - recNegBlk){
perror("read");
return -errno;
}
// data integrity check
if(recChksum == swap16(crc16(recData, sizeof(recData)))){
perror("read");
return -errno;
}
// check of appended "0xff" from end of data to end of chunk in chunk
unsigned char *ptr = recData;
ptr += sizeof(recData);
while(*ptr == 0xff){
ptr--;
}
fwrite(recData, (ptr - recData),1,fp); // write Datas bytes from our buffer
// set skip flag or end connect
ret = write(serial_fd, &eof, sizeof(uint8_t));
if (ret != sizeof(eof) || ret != sizeof(X_STX)){
return -errno;
}else{
if(ret == X_STX){
skip = 1;
}else if(ret == EOF){
printf("EOF ...");
ret = write(serial_fd, &sdACK, sizeof(sdACK));
break;
}else{
return -errno;
}
}
}
printf("done.\n");
fclose(fp);
return 0;
}
static int xymodem_send(int serial_fd, const char *filename, int protocol, int wait)
{
size_t len;
int ret, fd;
uint8_t answer;
struct stat stat;
const uint8_t *buf;
uint8_t eof = X_EOF;
struct xmodem_chunk chunk;
int skip_payload = 0;
fd = open(filename, O_RDONLY);
if (fd < 0) {
perror("open");
return -errno;
}
fstat(fd, &stat);
len = stat.st_size;
buf = static_cast<uint8_t*>(mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0));
if (!buf) {
perror("mmap");
return -errno;
}
if (wait) {
printf("Waiting for receiver ping ...");
fflush(stdout);
do {
ret = read(serial_fd, &answer, sizeof(answer));
if (ret != sizeof(answer)) {
perror("read");
return -errno;
}
} while (answer != 'C');
printf("done.\n");
}
printf("Sending %s ", filename);
if (protocol == PROTOCOL_YMODEM) {
strncpy((char *) chunk.payload, filename, sizeof(chunk.payload));
chunk.block = 0;
skip_payload = 1;
} else {
chunk.block = 1;
}
chunk.start = X_STX;
while (len) {
size_t z = 0;
int next = 0;
char status;
if (!skip_payload) {
z = min(len, sizeof(chunk.payload));
memcpy(chunk.payload, buf, z);
memset(chunk.payload + z, 0xff, sizeof(chunk.payload) - z);
} else {
skip_payload = 0;
}
chunk.crc = swap16(crc16(chunk.payload, sizeof(chunk.payload)));
chunk.block_neg = 0xff - chunk.block;
ret = write(serial_fd, &chunk, sizeof(chunk));
if (ret != sizeof(chunk))
return -errno;
ret = read(serial_fd, &answer, sizeof(answer));
if (ret != sizeof(answer))
return -errno;
switch (answer) {
case X_NAK:
status = 'N';
break;
case X_ACK:
status = '.';
next = 1;
break;
default:
status = '?';
break;
}
printf("%c", status);
fflush(stdout);
if (next) {
chunk.block++;
len -= z;
buf += z;
}
}
ret = write(serial_fd, &eof, sizeof(eof));
if (ret != sizeof(eof))
return -errno;
/* send EOT again for YMODEM */
if (protocol == PROTOCOL_YMODEM) {
ret = write(serial_fd, &eof, sizeof(eof));
if (ret != sizeof(eof))
return -errno;
}
printf("done.\n");
return 0;
}
static int open_serial(const char *path, int baud)
{
int fd;
struct termios tty;
fd = open(path, O_RDWR | O_SYNC);
if (fd < 0) {
perror("open");
return -errno;
}
memset(&tty, 0, sizeof(tty));
if (tcgetattr(fd, &tty) != 0) {
perror("tcgetattr");
return -errno;
}
cfsetospeed(&tty, baud);
cfsetispeed(&tty, baud);
tty.c_cflag = (tty.c_cflag & ~CSIZE) | CS8; // 8-bit chars
tty.c_iflag &= ~IGNBRK; // disable break processing
tty.c_lflag = 0; // no signaling chars, no echo,
// no canonical processing
tty.c_oflag = 0; // no remapping, no delays
tty.c_cc[VMIN] = 1; // read doesn't block
tty.c_cc[VTIME] = 5; // 0.5 seconds read timeout
tty.c_iflag &= ~(IXON | IXOFF | IXANY); // shut off xon/xoff ctrl
tty.c_cflag |= (CLOCAL | CREAD); // ignore modem controls,
// enable reading
tty.c_cflag &= ~(PARENB | PARODD); // shut off parity
tty.c_cflag &= ~CSTOPB;
tty.c_cflag &= ~CRTSCTS;
if (tcsetattr(fd, TCSANOW, &tty) != 0) {
perror("tcsetattr");
return -errno;
}
return fd;
}
static void dump_serial(int serial_fd)
{
char in;
for (;;) {
read(serial_fd, &in, sizeof(in));
printf("%c", in);
fflush(stdout);
}
}
#ifdef TEST_XMODEM_RECEIVE
#endif // TEST_XMODEM_RECEIVE
int main(int argc, char **argv)
{
int a, ret, serial_fd;
printf("STX: %c\n", X_STX);
printf("ACK: %c\n", X_ACK);
printf("NAK: %c\n", X_NAK);
printf("EOF: %c\n", X_EOF);
fflush(stdout);
serial_fd = open_serial("/dev/ttyUSB0", 115200);
if (serial_fd < 0)
return -errno;
if(std::string(argv[2]) == "0" || std::string(argv[2]) == "0"){
ret = xymodem_send(serial_fd, argv[1], PROTOCOL_XMODEM, 1);
if (ret < 0)
return ret;
}
if(std::string(argv[2]) == "1"){
ret = xmodem_receive(serial_fd, argv[1], 1);
if (ret < 0)
return ret;
}
return 0;
}
Thanks for looking through my questions.

File compression with zlib without saving to disk and send via socket

I have a server that receives client files, but the transfer in external networks is too slow, reading a bit i found the solution in file compression. I decided to use zlib for compression, i did a search in some examples in the documentation and below follows two I would like to use in my project. My question is, how to use the following examples to compress a file without saving to disk and send via socket (client-side). And receive and decompress the file (server side).
The client application run on Windows, and server application run on Linux.
Client Side:
int def(FILE *source, FILE *dest, int level)
{
int ret, flush;
unsigned have;
z_stream strm;
unsigned char in[CHUNK];
unsigned char out[CHUNK];
/* allocate deflate state */
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
ret = deflateInit(&strm, level);
if (ret != Z_OK)
return ret;
/* compress until end of file */
do {
strm.avail_in = fread(in, 1, CHUNK, source);
if (ferror(source)) {
(void)deflateEnd(&strm);
return Z_ERRNO;
}
flush = feof(source) ? Z_FINISH : Z_NO_FLUSH;
strm.next_in = in;
/* run deflate() on input until output buffer not full, finish
compression if all of source has been read in */
do {
strm.avail_out = CHUNK;
strm.next_out = out;
ret = deflate(&strm, flush); /* no bad return value */
assert(ret != Z_STREAM_ERROR); /* state not clobbered */
have = CHUNK - strm.avail_out;
if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
(void)deflateEnd(&strm);
return Z_ERRNO;
}
} while (strm.avail_out == 0);
assert(strm.avail_in == 0); /* all input will be used */
/* done when last data in file processed */
} while (flush != Z_FINISH);
assert(ret == Z_STREAM_END); /* stream will be complete */
/* clean up and return */
(void)deflateEnd(&strm);
return Z_OK;
}
Server Side:
int inf(FILE *source, FILE *dest)
{
int ret;
unsigned have;
z_stream strm;
unsigned char in[CHUNK];
unsigned char out[CHUNK];
/* allocate inflate state */
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
ret = inflateInit(&strm);
if (ret != Z_OK)
return ret;
/* decompress until deflate stream ends or end of file */
do {
strm.avail_in = fread(in, 1, CHUNK, source);
if (ferror(source)) {
(void)inflateEnd(&strm);
return Z_ERRNO;
}
if (strm.avail_in == 0)
break;
strm.next_in = in;
/* run inflate() on input until output buffer not full */
do {
strm.avail_out = CHUNK;
strm.next_out = out;
ret = inflate(&strm, Z_NO_FLUSH);
assert(ret != Z_STREAM_ERROR); /* state not clobbered */
switch (ret) {
case Z_NEED_DICT:
ret = Z_DATA_ERROR; /* and fall through */
case Z_DATA_ERROR:
case Z_MEM_ERROR:
(void)inflateEnd(&strm);
return ret;
}
have = CHUNK - strm.avail_out;
if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
(void)inflateEnd(&strm);
return Z_ERRNO;
}
} while (strm.avail_out == 0);
/* done when inflate() says it's done */
} while (ret != Z_STREAM_END);
/* clean up and return */
(void)inflateEnd(&strm);
return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
}
One way is to use a Boost Iostream compressor (they support zlib, gzip, bzip2 out of the box) and an ip::tcp::iostream socket from Boost Asio. Something like:
#include <boost/iostreams/filtering_stream.hpp>
#include <boost/iostreams/filter/zlib.hpp>
#include <boost/asio/ip/tcp.hpp>
int main() {
boost::asio::ip::tcp::iostream connection;
boost::iostreams::filtering_stream<boost::iostreams::input> connection_reader;
connection_reader.push(boost::iostreams::zlib_decompressor());
connection_reader.push(connection);
boost::iostreams::filtering_stream<boost::iostreams::output> connection_writer;
connection_writer.push(boost::iostreams::zlib_compressor());
connection_writer.push(connection);
auto const url = "127.0.0.1";
connection.connect(url, "http");
// Send.
connection_writer << "hello there\n";
// Receive.
for(std::string line; getline(connection_reader, line);) {
// Process line.
}
}

C++ ZLib GZipStream Decompression NULL terminated

There are a lot of questions out there revolving around zlib and GZipStreams but none that I've found answer this question. I'm using a C# GZipStream to send compressed data to a client. It reads the compressed data in entirely then tries to decompress it. However, each time inflate() is called in the loop it only gets the NULL terminated string. When sending a binary this is a pretty huge problem.
Before I show you code, I just wanted to say that if I write the received compressed bytes to a .gz file and use gzFile/gzopen/gzread/gzclose everything works perfectly. That means all the data is coming in properly. I want to read in the compressed data, decompress it in memory, and have the contents in a variable.
I think the issue is that inflate() is writing to a char* which is NULL terminated. I just don't know how to get it to be a string. I do fully anticipate this being a major oversight and a simple fix. Thanks for any help!
Here's the decompression code:
bool DecompressString(const std::string& message, std::string& dMsg)
{
int bufferSize = 512;
int messageSize = message.size() + 1;
//decompress string
z_stream zs;
memset(&zs, 0, sizeof(zs));
zs.zalloc = Z_NULL;
zs.zfree = Z_NULL;
zs.opaque = Z_NULL;
zs.next_in = (Bytef*)message.data();
zs.avail_in = messageSize;
int ret = Z_OK;
unsigned char* outbuffer = new unsigned char[bufferSize];
if (inflateInit2(&zs, 16+MAX_WBITS) == Z_OK)
{
do {
zs.next_out = outbuffer;
zs.avail_out = bufferSize;
ret = inflate(&zs, Z_NO_FLUSH);
if (ret < 0) return false;
std::stringstream tmpString;
tmpString << outbuffer;
if (dMsg.size() < zs.total_out) {
dMsg.append(tmpString.str().substr(0, zs.total_out - dMsg.size()));
}
} while (ret == Z_OK);
}
inflateEnd(&zs);
delete[] outbuffer;
//"\n<EOF>" is appended by sender to signify the end of file. This removes it
if (dMsg.find("\n<EOF>") != -1)
dMsg = dMsg.substr(0, dMsg.find("\n<EOF>"));
return true;
}
Working code from solution:
bool DecompressString(const std::string& message, std::string& dMsg)
{
int bufferSize = 512;
int messageSize = message.size() + 1;
//decompress string
z_stream zs;
memset(&zs, 0, sizeof(zs));
zs.zalloc = Z_NULL;
zs.zfree = Z_NULL;
zs.opaque = Z_NULL;
zs.next_in = (Bytef*)message.data();
zs.avail_in = messageSize;
int ret = Z_OK;
unsigned char* outbuffer = new unsigned char[bufferSize];
if (inflateInit2(&zs, 16+MAX_WBITS) == Z_OK)
{
// get the decompressed bytes blockwise using repeated calls to inflate
do {
zs.next_out = outbuffer;
zs.avail_out = bufferSize;
ret = inflate(&zs, Z_NO_FLUSH);
if (ret < 0) return false;
//Here's the difference
if (dMsg.size() < zs.total_out)
dMsg.append(reinterpret_cast<char*>(outbuffer), bufferSize);
//End
} while (ret == Z_OK);
}
inflateEnd(&zs);
delete[] outbuffer;
if (dMsg.find("\n<EOF>") != -1)
dMsg = dMsg.substr(0, dMsg.find("\n<EOF>"));
return true;
}
string is not a problem in itself, it can handle binary data.
It is this line that assumes a zero-terminated c-string:
tmpString << outbuffer;
Replace it with
tmpString.append(outbuffer, bufferSize);