I'm writing Qt-based client application. It connects to remote server using QTcpSocket. Before sending any actual data it needs to send login info, which is zlib-compressed json.
As far as I know from server sources, to make everything work I need to send X bytes of compressed data following 4 bytes with length of uncompressed data.
Uncompressing on server-side looks like this:
/* look at first 32 bits of buffer, which contains uncompressed len */
unc_len = le32toh(*((uint32_t *)buf));
if (unc_len > CLI_MAX_MSG)
return NULL;
/* alloc buffer for uncompressed data */
obj_unc = malloc(unc_len + 1);
if (!obj_unc)
return NULL;
/* decompress buffer (excluding first 32 bits) */
comp_p = buf + 4;
if (uncompress(obj_unc, &dest_len, comp_p, buflen - 4) != Z_OK)
goto out;
if (dest_len != unc_len)
goto out;
memcpy(obj_unc + unc_len, &zero, 1); /* null terminate */
I'm compressing json using Qt built-in zlib (I've just downloaded headers and placed it in mingw's include folder):
char json[] = "{\"version\":1,\"user\":\"test\"}";
char pass[] = "test";
std::auto_ptr<Bytef> message(new Bytef[ // allocate memory for:
sizeof(ubbp_header) // + msg header
+ sizeof(uLongf) // + uncompressed data size
+ strlen(json) // + compressed data itself
+ 64 // + reserve (if compressed size > uncompressed size)
+ SHA256_DIGEST_LENGTH]);//+ SHA256 digest
uLongf unc_len = strlen(json);
uLongf enc_len = strlen(json) + 64;
// header goes first, so server will determine that we want to login
Bytef* pHdr = message.get();
// after that: uncompressed data length and data itself
Bytef* pLen = pHdr + sizeof(ubbp_header);
Bytef* pDat = pLen + sizeof(uLongf);
// hash of compressed message updated with user pass
Bytef* pSha;
if (Z_OK != compress(pLen, &enc_len, (Bytef*)json, unc_len))
{
qDebug("Compression failed.");
return false;
}
Complete function code here: http://pastebin.com/hMY2C4n5
Even though server correctly recieves uncompressed length, uncompress() returning Z_BUF_ERROR.
P.S.: I'm actually writing pushpool's client to figure out how it's binary protocol works. I've asked this question on official bitcoin forum, but no luck there. http://forum.bitcoin.org/index.php?topic=24257.0
Turns out it was server-side bug. More details in bitcoin forum thread.
Related
I am trying to record audio from an INMP441 which is connected to a ESP32 but returning the buffer containing the bytes the microphone read always leads to something which is NULL.
The code for setting up i2s and the microphone is this:
// i2s config
const i2s_config_t i2s_config = {
.mode = i2s_mode_t(I2S_MODE_MASTER | I2S_MODE_RX), // receive
.sample_rate = SAMPLE_RATE, // 44100 (44,1KHz)
.bits_per_sample = I2S_BITS_PER_SAMPLE_32BIT, // 32 bits per sample
.channel_format = I2S_CHANNEL_FMT_ONLY_LEFT, // use right channel
.communication_format = i2s_comm_format_t(I2S_COMM_FORMAT_I2S | I2S_COMM_FORMAT_I2S_MSB),
.intr_alloc_flags = ESP_INTR_FLAG_LEVEL1, // interrupt level 1
.dma_buf_count = 64, // number of buffers
.dma_buf_len = SAMPLES_PER_BUFFER}; // 512
// pin config
const i2s_pin_config_t pin_config = {
.bck_io_num = gpio_sck, // serial clock, sck (gpio 33)
.ws_io_num = gpio_ws, // word select, ws (gpio 32)
.data_out_num = I2S_PIN_NO_CHANGE, // only used for speakers
.data_in_num = gpio_sd // serial data, sd (gpio 34)
};
// config i2s driver and pins
// fct must be called before any read/write
esp_err_t err = i2s_driver_install(I2S_PORT, &i2s_config, 0, NULL);
if (err != ESP_OK)
{
Serial.printf("Failed installing the driver: %d\n", err);
}
err = i2s_set_pin(I2S_PORT, &pin_config);
if (err != ESP_OK)
{
Serial.printf("Failed setting pin: %d\n", err);
}
Serial.println("I2S driver installed! :-)");
Setting up the i2s stuff is no problem at all. The tricky part for me is reading from the i2s:
// 44KHz * Byte per sample * time in seconds = total size in bytes
const size_t recordSize = (SAMPLE_RATE * I2S_BITS_PER_SAMPLE_32BIT / 8) * recordTime; //recordTime = 5s
// size in bytes
size_t totalReadSize = 0;
// 32 bits per sample set in config * 1024 samples per buffers = total bits per buffer
char *samples = (char *)calloc(totalBitsPerBuffer, sizeof(char));
// number of bytes read
size_t bytesRead;
Serial.println("Start recording...");
// read until wanted size is reached
while (totalReadSize < recordSize)
{
// read to buffer
esp_err_t err = i2s_read(I2S_PORT, (void *)samples, totalBitsPerBuffer, &bytesRead, portMAX_DELAY);
// check if error occurd, if so stop recording
if (err != ESP_OK)
{
Serial.println("Error while recording!");
break;
}
// check if bytes read works → yes
/*
for (int i = 0; i < bytesRead; i++)
{
uint8_t sample = (uint8_t) samples[i];
Serial.print(sample);
} */
// add read size to total read size
totalReadSize += bytesRead;
// Serial.printf("Currently recorded %d%% \n", totalReadSize * 100 / recordSize);
}
// convert bytes to mb
double_t totalReadSizeMB = (double_t)totalReadSize / 1e+6;
Serial.printf("Total read size: %fMb\n", totalReadSizeMB);
Serial.println("Samples deref");
Serial.println(*samples);
Serial.println("Samples");
Serial.println(samples);
return samples;
Using this code leads to the following output:
I2S driver installed! :-)
Start recording...
Total read size: 0.884736Mb
Samples deref
␀
Samples
When I uncomment the part where I iterate over the bytes read part I get something like this:
200224231255255224210022418725525522493000902552550238002241392542552241520020425225508050021624525501286700194120022461104022421711102242271030018010402242510000188970224141930022291022410185022487830021679001127500967200666902241776600246610224895902244757022418353002224802242274302249741022419339009435001223102242432602243322022412120001241402245911022418580084402248325525522461252255044249255224312452552242212372552241272352550342302552241212262552242112212550252216255014621325501682092550112205255224161202255224237198255224235194255224231922552248518725501141832550421812552241951762550144172255018168255034164255224173157255018215525522455152255028148255021014425505214025522487137255014613225522412112825502361252550180120255018011725522451172550252113255224133111255061082550248105255224891042552249910125522439972550138942552242279225503287255224101832552242478125522410178255224231732552244970255224336525501766225501426125502325625522424553255224109492550186[...]
This shows that the microphone is able to record, but I cant return the actual value of the buffer.
While programming this code I looked up at the official doku and some code which seems to work elsewhere.
I am also new to C++ and am not used to work with pointers.
Does anyone know what the problem could be?
I'm working to send frame of 8 bytes to Micro-controller Xmega128a1 (via RS232) the frame looks like this
{header1,header2,CMD,D1,D2,D3,D4,CRC},
for example
{0x55,0xaa,0xFF,0x59,0xfd,0x64,0x68,0x32},
Micro-controller has to resend the frame back to PC, if it's 'correct'.
I built GUI in QT Creator I defined the Headers (header0=0x55, header1=0xaa) and CMD=01 also calculated the CRC,
the user has to enter the data field in the Line_Edit which is value in RPM(Real value) The Micro-controller Receive the frame byte byte and resend the full frame, so I have to send the frame in the form of bytes, when I send the frame I receive the headers, command and CRC correctly, but data field Received not in proper way such in the picture below, my problem is with converting the input value in the Line_Edit to bytes to be send inside the frame, when I tried to send the value 1265 RPM I received the frame {55aa0100209e44fb} but I want to receive the frame look like this {55aa014F109e44fb}, where: (1265)DC=(4F1)HEX, I couldn't figure what's the problem with my code:
the way I read data from serial port:
void MainWindow::read()
{
uint64_t size = serial->bytesAvailable();
if (size > 0)
{
QByteArray data;
data.append(serial->readAll());
ui->termial_textEdit->append(data.toHex());
}
}
the send value in RPM code:
#define CMD_SPEED_REF2 0x01
void MainWindow::on_speed_ref2_lineEdit_returnPressed()
{
uint8_t frame2[8];
frame2[0] = 0x55;
frame2[1] = 0xAA;
frame2[2] = CMD_SPEED_REF2;
float fdata2 = 0.0f;
fdata2 = ui->speed_ref2_lineEdit->text().toFloat();
uint8_t *data2 = new uint8_t();
data2 = (uint8_t*)&fdata2;
frame2[3] = data2[0];
frame2[4] = data2[1];
frame2[5] = data2[2];
frame2[6] = data2[3];
frame2[7] = frame2[2] ^ frame2[3] ^ frame2[4] ^ frame2[5] ^ frame2[6];
serial->write((char*)frame2, 8);
}
this Image Illustrate what happens:recived frame
I think your code mostly looks ok. The one area that looks very suspect is your conversion of the text/string back into binary.
Since you convert your binary into a string with:
ui->termial_textEdit->append(data.toHex());
You should in theory be able to use the following to convert it back:
// Convert back...
QByteArray binaryData = QByteArray::fromHex(ui->speed_ref2_lineEdit->text().toLatin1());
// Print to debug to check it...
qDebug("d1: %02x, d2: %02x...etc...\n", binaryData[0], binaryData[1]);
// or just
qDebug() << "data:" << binaryData.toHex() << endl;
Not on my qt PC until Monday so I can't verify this code, so there may be a bug in there somewhere... I'll check it on Monday!
For serial comms I always use QByteArray's instead of char/uint8_t arrays (when using Qt) because they are so easy to use. You can re-build your array like this:
QByteArray frame2;
frame2.append((char) 0x55); // not sure you need to cast it here
frame2.append((char) 0xAA);
frame2.append((char) CMD_SPEED_REF2);
:
etc
:
If you MUST send as a char * then just do:
serial->write(frame2.data(), 8);
//or
serial->write(frame2.data(), frame2.size()); // if you want to send the whole thing
I am trying to implement the "zlib.h" deflate and inflate functions to compress and decompress streams in PDF-file.
Input: compressed stream from PDF-file. I implemented inflate function -- it's all right, I have uncopressed stream, after that I try to compress this stream again with deflate function, as output I have compressed stream, but it is not equal to input compressed stream and they are not equal to the length. What I'm doing wrong? This is a part of my code:
size_t outsize = (streamend - streamstart) * 10;
char* output = new char[outsize]; ZeroMemory(output, outsize);
z_stream zstrm; ZeroMemory(&zstrm, sizeof(zstrm));
zstrm.avail_in = streamend - streamstart + 1;
zstrm.avail_out = outsize;
zstrm.next_in = (Bytef*)(buffer + streamstart);//block of date to infalte
zstrm.next_out = (Bytef*)output;
int rsti = inflateInit(&zstrm);
if (rsti == Z_OK)
{
int rst2 = inflate(&zstrm, Z_FINISH);
if (rst2 >= 0)
{
cout << output << endl;//inflated data
}
}
char* deflate_output = new char[streamend - streamstart];
ZeroMemory(deflate_output, streamend - streamstart);
z_stream d_zstrm; ZeroMemory(&d_zstrm, sizeof(d_zstrm));
d_zstrm.avail_in = (uInt) (strlen(output)+1);
d_zstrm.avail_out = (uInt) (streamend - streamstart);
d_zstrm.next_in = (Bytef*)(output);
d_zstrm.next_out = (Bytef*)(deflate_output);
int rsti1 = deflateInit(&d_zstrm, Z_DEFAULT_COMPRESSION);
if (rsti1 == Z_OK)
{
int rst22 = deflate(&d_zstrm, Z_FINISH);
out << deflate_output << endl << "**********************" << endl;
//I try to write deflated stream to file
printf("New size of stream: %lu\n", (char*)d_zstrm.next_out - deflate_output);
}
There is nothing wrong. There is not a unique compressed stream for a given uncompressed stream. All that is required is that the decompression give you back exactly what was compressed (hence "lossless").
It may simply be caused by different compression parameters, different compression code, or even a different version of the same compression code.
If you can't reproduce the original compressed data, so what? All that matters is that you can make a valid PDF file that can be decompressed and has the content that you want.
I have to transfer some huge files (2GB-ish) to a web service:
public bool UploadContent(System.Web.HttpContext context)
{
var file = context.Request.Files[0];
var fileName = file.FileName;
byte[] fileBytes = new Byte[file.ContentLength];
file.InputStream.Read(fileBytes, 0, fileBytes.Length);
client.createResource(fileBytes);
}
The HttpContext already has the contents of the file in File[0], but I can't see a way to pass those bytes to the createResource(byte[] contents) method of the web service without making a copy as a byte array... so I am eating memory like candy.
Is there a more efficient way to do this?
EDIT client.createResource() is part of a COTS product and modification is outside our control.
Rather than sending the whole bytes you can send the chunks of the files. Seek the file for step by step upload and merge the next chunk to already save bytes on server.
You need to update your client.CreateResource method only if you're allowed to modify that method :)
Add following parameters:
string fileName // To locate the file name when you start sending the chunks
byte[] buffer // chunk that would be sent to server via webservice
long offset // Information that will tell you how much data is already uploaded, so that you can seek the file and merge the buffer.
Now your method will look like:
public bool CreateResource(string FileName, byte[] buffer, long Offset)
{
bool retVal = false;
try
{
string FilePath = "d:\\temp\\uploadTest.extension";
if (Offset == 0)
File.Create(FilePath).Close();
// open a file stream and write the buffer.
// Don't open with FileMode.Append because the transfer may wish to
// start a different point
using (FileStream fs = new FileStream(FilePath, FileMode.Open,
FileAccess.ReadWrite, FileShare.Read))
{
fs.Seek(Offset, SeekOrigin.Begin);
fs.Write(buffer, 0, buffer.Length);
}
retVal = true;
}
catch (Exception ex)
{
// Log exception or send error message to someone who cares
}
return retVal;
}
Now to read the file in chunks from the InputStream of HttpPostedFile try below code:
public bool UploadContent(System.Web.HttpContext context)
{
//the file that we want to upload
var file = context.Request.Files[0];
var fs = file.InputStream;
int Offset = 0; // starting offset.
//define the chunk size
int ChunkSize = 65536; // 64 * 1024 kb
//define the buffer array according to the chunksize.
byte[] Buffer = new byte[ChunkSize];
//opening the file for read.
try
{
long FileSize = file.ContentLength; // File size of file being uploaded.
// reading the file.
fs.Position = Offset;
int BytesRead = 0;
while (Offset != FileSize) // continue uploading the file chunks until offset = file size.
{
BytesRead = fs.Read(Buffer, 0, ChunkSize); // read the next chunk
if (BytesRead != Buffer.Length)
{
ChunkSize = BytesRead;
byte[] TrimmedBuffer = new byte[BytesRead];
Array.Copy(Buffer, TrimmedBuffer, BytesRead);
Buffer = TrimmedBuffer; // the trimmed buffer should become the new 'buffer'
}
// send this chunk to the server. it is sent as a byte[] parameter,
// but the client and server have been configured to encode byte[] using MTOM.
bool ChunkAppened = client.createResource(file.FileName, Buffer, Offset);
if (!ChunkAppened)
{
break;
}
// Offset is only updated AFTER a successful send of the bytes.
Offset += BytesRead; // save the offset position for resume
}
}
catch (Exception ex)
{
}
finally
{
fs.Close();
}
}
Disclaimer: I haven't tested this code. This is a sample code to show how large file upload can be achieved without hampering the memory.
Ref: Source article.
let's say I need to send, for instance, five images from a client to a server over a socket and that I want to do it at once (not sending one and waiting for an ACK).
Questions:
I'd like to know if there are some best practices or guidelines for delimiting the end of each one.
What would be the safest approach for detecting the delimiters and processing each image once in the server? (In C/C++ if possible)
Thanks in advance!
Since images are binary data, it would be difficult to come up with delimiter that cannot be contained in the image. (And ultimately confusing the receiving side)
I would advice you to create a header that would be placed at the beginning of the transmission, or at the beginning of each image.
An example:
struct Header
{
uint32_t ImageLength;
// char ImageName[128];
} __attribute__(packed);
The sender should prepend this before each image and fill in the length correctly. The receiver would then know when the image ends and would expect another Header structure at that position.
The attribute(packed) is a safety, that makes sure the header will have the same alignment even if you compile server and client with different GCC versions. It's recomended in cases where structures are interpreted by different processes.
Data Stream:
Header
Image Data
Header
Image Data
Header
Image Data
...
You can use these function to send files (from client in java) to a server (in C). The idea is to send 4 bytes which indicates the file's size followed by the file content, when all files have been sent, send 4 bytes (all set to 0 zero) to indicate the end of the transfer.
// Compile with Microsoft Visual Studio 2008
// path, if not empty, must be ended with a path separator '/'
// for example: "C:/MyImages/"
int receiveFiles(SOCKET sck, const char *pathDir)
{
int fd;
long fSize=0;
char buffer[8 * 1024];
char filename[MAX_PATH];
int count=0;
// keep on receiving until we get the appropiate signal
// or the socket has an error
while (true)
{
if (recv(sck, buffer, 4, 0) != 4)
{
// socket is closed or has an error
// return what we've received so far
return count;
}
fSize = (int) ((buffer[0] & 0xff) << 24) |
(int) ((buffer[1] & 0xff) << 16) |
(int) ((buffer[2] & 0xff) << 8) |
(int) (buffer[3] & 0xff);
if (fSize == 0)
{
// received final signal
return count;
}
sprintf(filename, "%sIMAGE_%d.img", pathDir, count+1);
fd = _creat(filename, _S_IREAD | _S_IWRITE);
int iReads;
int iRet;
int iLeft=fSize;
while (iLeft > 0)
{
if (iLeft > sizeof(buffer)) iReads = sizeof(buffer);
else iReads=iLeft;
if ((iRet=recv(sck, buffer, iReads, 0)) <= 0)
{
_close(fd);
// you may delete the file or leave it to inspect
// _unlink(filename);
return count; // socket is closed or has an error
}
iLeft-=iRet;
_write(fd, buffer, iRet);
}
count++;
_close(fd);
}
}
The client part
/**
* Send a file to a connected socket.
* <p>
* First it send the file size if 4 bytes then the file's content.
* </p>
* <p>
* Note: File size is limited to a 32bit signed integer, 2GB
* </p>
*
* #param os
* OutputStream of the connected socket
* #param fileName
* The complete file's path of the image to send
* #throws Exception
* #see {#link receiveFile} for an example on how to receive the file from the other side.
*
*/
public void sendFile(OutputStream os, String fileName) throws Exception
{
// File to send
File myFile = new File(fileName);
int fSize = (int) myFile.length();
if (fSize == 0) return; // No empty files
if (fSize < myFile.length())
{
System.out.println("File is too big'");
throw new IOException("File is too big.");
}
// Send the file's size
byte[] bSize = new byte[4];
bSize[0] = (byte) ((fSize & 0xff000000) >> 24);
bSize[1] = (byte) ((fSize & 0x00ff0000) >> 16);
bSize[2] = (byte) ((fSize & 0x0000ff00) >> 8);
bSize[3] = (byte) (fSize & 0x000000ff);
// 4 bytes containing the file size
os.write(bSize, 0, 4);
// In case of memory limitations set this to false
boolean noMemoryLimitation = true;
FileInputStream fis = new FileInputStream(myFile);
BufferedInputStream bis = new BufferedInputStream(fis);
try
{
if (noMemoryLimitation)
{
// Use to send the whole file in one chunk
byte[] outBuffer = new byte[fSize];
int bRead = bis.read(outBuffer, 0, outBuffer.length);
os.write(outBuffer, 0, bRead);
}
else
{
// Use to send in a small buffer, several chunks
int bRead = 0;
byte[] outBuffer = new byte[8 * 1024];
while ((bRead = bis.read(outBuffer, 0, outBuffer.length)) > 0)
{
os.write(outBuffer, 0, bRead);
}
}
os.flush();
}
finally
{
bis.close();
}
}
To send the files from the client:
try
{
// The file name must be a fully qualified path
sendFile(mySocket.getOutputStream(), "C:/MyImages/orange.png");
sendFile(mySocket.getOutputStream(), "C:/MyImages/lemmon.png");
sendFile(mySocket.getOutputStream(), "C:/MyImages/apple.png");
sendFile(mySocket.getOutputStream(), "C:/MyImages/papaya.png");
// send the end of the transmition
byte[] buff = new byte[4];
buff[0]=0x00;
buff[1]=0x00;
buff[2]=0x00;
buff[3]=0x00;
mySocket.getOutputStream().write(buff, 0, 4);
}
catch (Exception e)
{
e.printStackTrace();
}
If you cannot easily send a header containing the length, use some likely delimiter. If the images are not compressed and consist of bitmap-stype data, maybe 0xFF/0XFFFF/0xFFFFFFF as fully-saturated luminance values are usually rare?
Use an escape-sequence to eliminate any instances of the delimiter that turn up inside your data.
This does mean iterating all the data at both ends, but depending on your data flows, and what is being done anyway, it may be a useful solution :(