Related
I'm making a small client/server based game, on linux in c/c++ and I need to send the player turn to the server.
Here is my problem.
I want to send two integers to the server and sometimes it works perfectly, but sometimes the server receives both integer in the first recv() and its stuck.
I know that the best way is to package the messages.
The problem is I don't know how the syntax should look like.
In theory--> the player input would be like an int column = 4 and a second int row = 1 and I package the message as 4|1 or something like this. Then I send from client to server and encode it on the server.
An example would be great or maybe some advice how stuff like this is handled probably.
I'm still very new to socket programming.
Here is how my function looks like:
Client:
#define BUFFER 512
void send_turn_to_server(int sock, int row, int column)
{
// sends row to server from player turn
char char_row[BUFFER];
sprintf(char_row, "%d", row);
char *message_from_client = char_row;
int len, bytes_sent_row;
len = strlen(message_from_client);
if (sendall(sock, message_from_client, &len) == -1)
{
perror("sendall");
printf("We only sent %d bytes because of the error!\n", len);
}
char char_column[BUFFER];
int bytes_sent_column;
//sends column from player turn
//sprintf converts the int to char
sprintf(char_column, "%d", column);
char *column_from_client = char_column;
len = strlen(column_from_client);
if (sendall(sock, column_from_client, &len) == -1)
{
perror("sendall");
printf("We only sent %d bytes because of the error!\n", len);
}
cout << "send_turn_to_server_complete" << endl;
}
Here I use a function from Beej's Guide to Network Programming, so I can be sure the whole buffer is sent.
Client:
int sendall(int s, char *buf, int *len)
{
int total = 0; // how many bytes we've sent
int bytesleft = *len; // how many we have left to send
int n;
while (total < *len)
{
n = send(s, buf + total, bytesleft, 0);
if (n == -1)
{
break;
}
total += n;
bytesleft -= n;
}
*len = total; // return number actually sent here
return n == -1 ? -1 : 0; // return -1 on failure, 0 on success
}
Server:
int receive_player_turn(int sock, int &int_row, int &int_column)
{
int byte_count;
char buf[BUFFER];
byte_count = recv(sock, buf, sizeof buf, 0);
cout << "The row from player: " << buf << endl;
//The C library function int atoi(const char *str) converts the string argument str to an integer (type int).
int_row = atoi(buf);
//cleans the buffer
bzero(buf, sizeof(buf));
byte_count = recv(sock, buf, sizeof buf, 0);
cout << buf << endl;
cout << "The column from player: " << buf << endl;
//converts the char string to an int
int_column = atoi(buf);
cout << endl
<< "receive player turn worked" << endl
<< "players turn was in the row " << int_row << " and in the column " << int_column + 1 << endl;
return int_row, int_column;
}
output correct from server:
Player connected: SchleichsSalaticus
The row from player: 7
4
The column from player: 4
receive player turn worked
players turn was in the row 7 and in the column 5
7 4
output wrong from server:
Player connected: SchleichsSalaticus
The row from player: 74
The issue is that TCP is a continuous stream, with no concept of the start or end of a ”message” because it is not message-based.
Most times, people use a very simple ”framing protocol” whereby you always send a 4-byte header on every transfer which tells the recipient how many bytes to read, then you send that many bytes as your message.
Use htonl() to send the 4-byte header in network byte order then you will be interoperable. There is a very similar example here.
One possible solution could be defining a format for the message that client send to server. For example you could define a protocol as follow:
[4 bytes length of your message][2 bytes for first player][2 bytes for second one] and in server side you should at first in rcv function get 4 bytes and extract the length of the arrived message and based on the receiving length(L) call again the rcv function with size L after that you should parse received messaged and extract the turn of each players.
If all your messages are expected to be of same length, then you do not need a message header. Something like that given below should work fine. In general you should be prepared to receive less or more than your expected message, as well as for one message to be split across many receives.
Also, I would recommend one function that receives bytes making no assumption about what they mean, and another that interprets them. Then the first one can be applied more broadly.
Treat the following only as pseudo code. not tested.
// use a buffer length double of MESSAGE_LENGTH.
static int offset = 0; // not thread safe.
// loop to receive a message.
while(offset < MESSAGE_LENGTH) {
byte_count = recv(sock, &buf[offset], (sizeof(buf)-offset), 0);
if(byte_count > 0) {
offset += byte_count;
}
else {
// add error handling here. close socket.
break out of loop
}
}
// process buf here, but do not clear it.
// received message always starts at buf[0].
if(no receive error above) {
process_received_message(buf); //
}
// move part of next message (if any) to start of buffer.
if(offset > MESSAGE_LENGTH) {
// copy the start of next message to start of buffer.
// and remember the new offset to avoid overwriting them.
char* pSrc = &buf[MESSAGE_LENGTH];
char* pSrcEnd = &buf[offset];
char* pDest = buf;
while(pSrc < pSrcEnd){
*pDest++ = *pSrc++;
} //or memcpy.
offset -= MESSAGE_LENGTH;
}
else {
offset = 0;
}
On many hardware architectures, integers and other types have alignment requirements. The compiler normally takes care of this, but when in a buffer, unaligned accesses can be an issue. Furthermore, the server and the client might not use the same byte order.
Here is a set of inline helper functions you can use to pack and unpack integer types to/from a buffer:
/* SPDX-License-Identifier: CC0-1.0 */
#ifndef PACKING_H
#define PACKING_H
#include <stdint.h>
/* Packing and unpacking unsigned and signed integers in
little-endian byte order.
Works on all architectures and OSes when compiled
using a standards-conforming C implementation, C99 or later.
*/
static inline void pack_u8(unsigned char *dst, uint8_t val)
{
dst[0] = val & 255;
}
static inline void pack_u16(unsigned char *dst, uint16_t val)
{
dst[0] = val & 255;
dst[1] = (val >> 8) & 255;
}
static inline void pack_u24(unsigned char *dst, uint32_t val)
{
dst[0] = val & 255;
dst[1] = (val >> 8) & 255;
dst[2] = (val >> 16) & 255;
}
static inline void pack_u32(unsigned char *dst, uint32_t val)
{
dst[0] = val & 255;
dst[1] = (val >> 8) & 255;
dst[2] = (val >> 16) & 255;
dst[3] = (val >> 24) & 255;
}
static inline void pack_u40(unsigned char *dst, uint64_t val)
{
dst[0] = val & 255;
dst[1] = (val >> 8) & 255;
dst[2] = (val >> 16) & 255;
dst[3] = (val >> 24) & 255;
dst[4] = (val >> 32) & 255;
}
static inline void pack_u48(unsigned char *dst, uint64_t val)
{
dst[0] = val & 255;
dst[1] = (val >> 8) & 255;
dst[2] = (val >> 16) & 255;
dst[3] = (val >> 24) & 255;
dst[4] = (val >> 32) & 255;
dst[5] = (val >> 40) & 255;
}
static inline void pack_u56(unsigned char *dst, uint64_t val)
{
dst[0] = val & 255;
dst[1] = (val >> 8) & 255;
dst[2] = (val >> 16) & 255;
dst[3] = (val >> 24) & 255;
dst[4] = (val >> 32) & 255;
dst[5] = (val >> 40) & 255;
dst[6] = (val >> 48) & 255;
}
static inline void pack_u64(unsigned char *dst, uint64_t val)
{
dst[0] = val & 255;
dst[1] = (val >> 8) & 255;
dst[2] = (val >> 16) & 255;
dst[3] = (val >> 24) & 255;
dst[4] = (val >> 32) & 255;
dst[5] = (val >> 40) & 255;
dst[6] = (val >> 48) & 255;
dst[7] = (val >> 56) & 255;
}
static inline void pack_i8(unsigned char *dst, int8_t val)
{
pack_u8((uint8_t)val);
}
static inline void pack_i16(unsigned char *dst, int16_t val)
{
pack_u16((uint16_t)val);
}
static inline void pack_i24(unsigned char *dst, int32_t val)
{
pack_u24((uint32_t)val);
}
static inline void pack_i32(unsigned char *dst, int32_t val)
{
pack_u32((uint32_t)val);
}
static inline void pack_i40(unsigned char *dst, int64_t val)
{
pack_u40((uint64_t)val);
}
static inline void pack_i48(unsigned char *dst, int64_t val)
{
pack_u48((uint64_t)val);
}
static inline void pack_i56(unsigned char *dst, int64_t val)
{
pack_u56((uint64_t)val);
}
static inline void pack_i64(unsigned char *dst, int64_t val)
{
pack_u64((uint64_t)val);
}
static inline uint8_t unpack_u8(const unsigned char *src)
{
return (uint_fast8_t)(src[0] & 255);
}
static inline uint16_t unpack_u16(const unsigned char *src)
{
return (uint_fast16_t)(src[0] & 255)
| ((uint_fast16_t)(src[1] & 255) << 8);
}
static inline uint32_t unpack_u24(const unsigned char *src)
{
return (uint_fast32_t)(src[0] & 255)
| ((uint_fast32_t)(src[1] & 255) << 8)
| ((uint_fast32_t)(src[2] & 255) << 16);
}
static inline uint32_t unpack_u32(const unsigned char *src)
{
return (uint_fast32_t)(src[0] & 255)
| ((uint_fast32_t)(src[1] & 255) << 8)
| ((uint_fast32_t)(src[2] & 255) << 16)
| ((uint_fast32_t)(src[3] & 255) << 24);
}
static inline uint64_t unpack_u40(const unsigned char *src)
{
return (uint_fast64_t)(src[0] & 255)
| ((uint_fast64_t)(src[1] & 255) << 8)
| ((uint_fast64_t)(src[2] & 255) << 16)
| ((uint_fast64_t)(src[3] & 255) << 24)
| ((uint_fast64_t)(src[4] & 255) << 32);
}
static inline uint64_t unpack_u48(const unsigned char *src)
{
return (uint_fast64_t)(src[0] & 255)
| ((uint_fast64_t)(src[1] & 255) << 8)
| ((uint_fast64_t)(src[2] & 255) << 16)
| ((uint_fast64_t)(src[3] & 255) << 24)
| ((uint_fast64_t)(src[4] & 255) << 32)
| ((uint_fast64_t)(src[5] & 255) << 40);
}
static inline uint64_t unpack_u56(const unsigned char *src)
{
return (uint_fast64_t)(src[0] & 255)
| ((uint_fast64_t)(src[1] & 255) << 8)
| ((uint_fast64_t)(src[2] & 255) << 16)
| ((uint_fast64_t)(src[3] & 255) << 24)
| ((uint_fast64_t)(src[4] & 255) << 32)
| ((uint_fast64_t)(src[5] & 255) << 40)
| ((uint_fast64_t)(src[6] & 255) << 48);
}
static inline uint64_t unpack_u64(const unsigned char *src)
{
return (uint_fast64_t)(src[0] & 255)
| ((uint_fast64_t)(src[1] & 255) << 8)
| ((uint_fast64_t)(src[2] & 255) << 16)
| ((uint_fast64_t)(src[3] & 255) << 24)
| ((uint_fast64_t)(src[4] & 255) << 32)
| ((uint_fast64_t)(src[5] & 255) << 40)
| ((uint_fast64_t)(src[6] & 255) << 48)
| ((uint_fast64_t)(src[7] & 255) << 56);
}
static inline int8_t unpack_i8(const unsigned char *src)
{
return (int8_t)(src[0] & 255);
}
static inline int16_t unpack_i16(const unsigned char *src)
{
return (int16_t)unpack_u16(src);
}
static inline int32_t unpack_i24(const unsigned char *src)
{
uint_fast32_t u = unpack_u24(src);
/* Sign extend to 32 bits */
if (u & 0x800000)
u |= 0xFF000000;
return (int32_t)u;
}
static inline int32_t unpack_i32(const unsigned char *src)
{
return (int32_t)unpack_u32(src);
}
static inline int64_t unpack_i40(const unsigned char *src)
{
uint_fast64_t u = unpack_u40(src);
/* Sign extend to 64 bits */
if (u & UINT64_C(0x0000008000000000))
u |= UINT64_C(0xFFFFFF0000000000);
return (int64_t)u;
}
static inline int64_t unpack_i48(const unsigned char *src)
{
uint_fast64_t u = unpack_i48(src);
/* Sign extend to 64 bits */
if (u & UINT64_C(0x0000800000000000))
u |= UINT64_C(0xFFFF000000000000);
return (int64_t)u;
}
static inline int64_t unpack_i56(const unsigned char *src)
{
uint_fast64_t u = unpack_u56(src);
/* Sign extend to 64 bits */
if (u & UINT64_C(0x0080000000000000))
u |= UINT64_C(0xFF00000000000000);
return (int64_t)u;
}
static inline int64_t unpack_i64(const unsigned char *src)
{
return (int64_t)unpack_u64(src);
}
#endif /* PACKING_H */
When packed, these values are in two's complement little-endian byte order.
pack_uN() and unpack_uN() work with unsigned integers from 0 to 2N-1, inclusive.
pack_iN() and unpack_iN() work with signed integers from -2N-1 to 2N-1-1, inclusive.
Let's consider a simple binary protocol, where each message starts with two bytes: first one the total length of this message, and the second one identifying the type of the message.
This has the nice feature that if something odd happens, it is always possible to resynchronize by sending at least 256 zeroes. Each zero is an invalid length for the message, so they should just be skipped by the receiver. You probably won't need this, but it may come in handy someday.
To receive a message of this form, we can use the following function:
/* Receive a single message.
'fd' is the socket descriptor, and
'msg' is a buffer of at least 255 chars.
Returns -1 with errno set if an error occurs,
or the message type (0 to 255, inclusive) if success.
*/
int recv_message(const int fd, unsigned char *msg)
{
ssize_t n;
msg[0] = 0;
msg[1] = 0;
/* Loop to skip zero bytes. */
do {
do {
n = read(fd, msg, 1);
} while (n == -1 && errno == EINTR);
if (n == -1) {
/* Error; errno already set. */
return -1;
} else
if (n == 0) {
/* Other end closed the socket. */
errno = EPIPE;
return -1;
} else
if (n != 1) {
errno = EIO;
return -1;
}
} while (msg[0] == 0);
/* Read the rest of the message. */
{
unsigned char *const end = msg + msg[0];
unsigned char *ptr = msg + 1;
while (ptr < end) {
n = read(fd, ptr, (size_t)(end - ptr));
if (n > 0) {
ptr += n;
} else
if (n == 0) {
/* Other end closed socket */
errno = EPIPE;
return -1;
} else
if (n != -1) {
errno = EIO;
return -1;
} else
if (errno != EINTR) {
/* Error; errno already set */
return -1;
}
}
}
/* Success, return message type. */
return msg[1];
}
In your own code, you can use the above like this:
unsigned char buffer[256];
switch(receive_message(fd, buffer)) {
case -1:
if (errno == EPIPE) {
/* The other end closed the connection */
} else {
/* Other error; see strerror(errno). */
}
break or return or abort;
case 0: /* Exit/cancel game */
break or return or abort;
case 4: /* Coordinate message */
int x = unpack_i16(buffer + 2);
int y = unpack_i16(buffer + 4);
/* x,y is the coordinate pair; do something */
break;
default:
/* Ignore all other message types */
}
where I randomly chose 0 as the abort-game message type, and 4 as the coordinate message type.
Instead of scattering such statements here and there in your client, put it in a function. You could also consider using a finite-state machine to represent the game state.
To send messages, you can use a helper function like
/* Send one or more messages; does not verify contents.
Returns 0 if success, -1 with errno set if an error occurs.
*/
int send_message(const int fd, const void *msg, const size_t len)
{
const unsigned char *const end = (const unsigned char *)msg + len;
const unsigned char *ptr = (const unsigned char *)msg;
ssize_t n;
while (ptr < end) {
n = write(fd, ptr, (size_t)(end - ptr));
if (n > 0) {
ptr += n;
} else
if (n != -1) {
/* C library bug, should not occur */
errno = EIO;
return -1;
} else
if (errno != EINTR) {
/* Other error */
return -1;
}
}
return 0;
}
so that sending an abort game (type 0) message would be
int send_abort_message(const int fd)
{
unsigned char buffer[2] = { 1, 0 };
return send_message(fd, buffer, 2);
}
and sending a coordinate (type 4) message would be e.g.
int send_coordinates(const int fd, const int x, const int y)
{
unsigned char buffer[2 + 2 + 2];
buffer[0] = 6; /* Length in bytes/chars */
buffer[1] = 4; /* Type */
pack_i16(buffer + 2, x);
pack_i16(buffer + 4, y);
return send_message(fd, buffer, 6);
}
If the game is not turn-based, you won't want to block in the sends or receives, like the above functions do.
Nonblocking I/O is the way to go. Essentially, you'll have something like
static int server_fd = -1;
static size_t send_size = 0;
static unsigned char *send_data = NULL;
static size_t send_next = 0; /* First unsent byte */
static size_t send_ends = 0; /* End of buffered data */
static size_t recv_size = 0;
static unsigned char *recv_data = NULL;
static size_t recv_next = 0; /* Start of next message */
static size_t recv_ends = 0; /* End of buffered data */
and you set the server_fd nonblocking using e.g. fcntl(server_fd, F_SETFL, O_NONBLOCK);.
A communicator function will try to send and receive as much data as possible. It will return 1 if it sent anything, 2 if it received anything, 3 if both, 0 if neither, and -1 if an error occurred:
int communicate(void) {
int retval = 0;
ssize_t n;
while (send_next < send_ends) {
n = write(server_fd, send_data + send_next, send_ends - send_next);
if (n > 0) {
send_next += n;
retval |= 1;
} else
if (n != -1) {
/* errno already set */
return -1;
} else
if (errno == EAGAIN || errno == EWOULDBLOCK) {
/* Cannot send more without blocking */
break;
} else
if (errno != EINTR) {
/* Error, errno set */
return -1;
}
}
/* If send buffer became empty, reset it. */
if (send_next >= send_ends) {
send_next = 0;
send_ends = 0;
}
/* If receive buffer is empty, reset it. */
if (recv_next >= recv_ends) {
recv_next = 0;
recv_ends = 0;
}
/* Receive loop. */
while (1) {
/* Receive buffer full? */
if (recv_ends + 256 > recv_ends) {
/* First try to repack. */
if (recv_next > 0) {
memmove(recv_data, recv_data + recv_next, recv_ends - recv_next);
recv_ends -= recv_next;
recv_next = 0;
}
if (recv_ends + 256 > recv_ends) {
/* Allocate 16k more (256 messages!) */
size_t new_size = recv_size + 16384;
unsigned char *new_data;
new_data = realloc(recv_data, new_size);
if (!new_data) {
errno = ENOMEM;
return -1;
}
recv_data = new_data;
recv_size = new_size;
}
}
/* Try to receive incoming data. */
n = read(server_fd, recv_data + recv_ends, recv_size - recv_ends);
if (n > 0) {
recv_ends += n;
retval |= 2;
} else
if (n == 0) {
/* Other end closed the connection. */
errno = EPIPE;
return -1;
} else
if (n != -1) {
errno = EIO;
return -1;
} else
if (errno == EAGAIN || errno == EWOULDBLOCK) {
break;
} else
if (errno != EINTR) {
return -1;
}
}
return retval;
}
When there is nothing to do, and you want to wait for a short while (some milliseconds), but interrupt the wait whenever more I/O can be done, use
/* Wait for max 'ms' milliseconds for communication to occur.
Returns 1 if data received, 2 if sent, 3 if both, 0 if neither
(having waited for 'ms' milliseconds), or -1 if an error occurs.
*/
int communicate_wait(int ms)
{
struct pollfd fds[1];
int retval;
/* Zero timeout is "forever", and we don't want that. */
if (ms < 1)
ms = 1;
/* We try communicating right now. */
retval = communicate();
if (retval)
return retval;
/* Poll until I/O possible. */
fds[0].fd = server_fd;
if (send_ends > send_next)
fds[0].events = POLLIN | POLLOUT;
else
fds[0].events = POLLIN;
fds[0].revents = 0;
poll(fds, 1, ms);
/* We retry I/O now. */
return communicate();
}
To process messages received thus far, you use a loop:
while (recv_next < recv_ends && recv_next + recv_data[recv_next] <= recv_ends) {
if (recv_data[recv_next] == 0) {
recv_next++;
continue;
}
/* recv_data[recv_next+0] is the length of the message,
recv_data[recv_next+1] is the type of the message. */
switch (recv_data[recv_next + 1]) {
case 4: /* Coordinate message */
if (recv_data[recv_next] >= 6) {
int x = unpack_i16(recv_data + recv_next + 2);
int y = unpack_i16(recv_data + recv_next + 4);
/* Do something with x and y ... */
}
break;
/* Handle other message types ... */
}
recv_next += recv_data[recv_next];
}
Then you recalculate game state, update the display, communicate some more, and repeat.
Hi and hope everyone is doing good!
I started working on a chrome extension which talks to a cpp exe. At the moment I am just trying to read some data from the extension which works as expected using the following code
unsigned int length = 0;
{
unsigned int read_char = getchar();
length = length | (read_char << index*8);
}
std::string message = "";
for (int index = 0; index < length; index++)
{
message += getchar();
}
When trying to send the same information from the cpp -> extension using the following code
unsigned int len = message.length();
printf("%c%c%c%c", (char) (len & 0xff),
(char) (len << 8 & 0xff),
(char) (len << 16 & 0xff),
(char) (len << 24 & 0xff));
printf("%s", message.c_str());
it works as expected
Screenshot of the working scenario
However, when i try to send back another string rather than the same string that we send it won't work.
Example:
unsigned int length = 0;
for (int index = 0; index < 4; index++)
{
unsigned int read_char = getchar();
cerr << read_char << endl;
length = length | (read_char << index*8);
}
std::string message = "";
for (int index = 0; index < length; index++)
{
message += getchar();
}
std::string outputMessage = "output";
unsigned int len = outputMessage.length();
printf("%c%c%c%c", (char) (len & 0xff),
(char) (len << 8 & 0xff),
(char) (len << 16 & 0xff),
(char) (len << 24 & 0xff));
printf("%s", outputMessage.c_str());
return 0;
Here is also the screenshot of the scenario that is failing. Failed scenario
I am not sure what I missed here, consider that I am just creating a new string and also using it's length for information that the extension needs.
Anyway, it would be great if anyone could look at this one. Thanks in advance :)
UPDATE:
After trying different ways of returning messages I found this on the web and it works as expected
message_t encode_message(json content) {
string encoded_content = content.dump();
message_t m;
m.content = encoded_content;
m.length = (uint32_t)encoded_content.length();
return m;
}
// Send an encoded message to stdout.
void send_message(message_t encoded_message) {
char* raw_length = reinterpret_cast<char*>(&encoded_message.length);
fwrite(raw_length, 4, sizeof(char), stdout);
fwrite(encoded_message.content.c_str(), encoded_message.length, sizeof(char), stdout);
fflush(stdout);
}
You have to explicitly flush the buffer, if those data are to be sent to the native messaging host ASAP as shown below:
std::string outputMessage = "output";
unsigned int len = outputMessage.length();
printf("%c%c%c%c", (char) (len & 0xff),
(char) ((len >> 8) & 0xff),
(char) ((len >> 16) & 0xff),
(char) ((len >> 24) & 0xff));
printf("%s", outputMessage.c_str());
fflush(stdout); // This will flush the buffer
I'm trying to make a FLV muxer. And I'm started to test on a h264 stream I capture from my camera (c920). After encoding the .flv file, it doesn't play correctly.
First I tried to find NALs in h264 searching for the pattern 0x00 0x00 0x00 0x01... but I found in the internet that there are two patterns to find NALs... So I implemented the searching for 0x00 0x00 0x01 and the other...
In firsts tests I found that are few NALs starting with four bytes, but after changing the code to search the 3 bytes pattern I found a lot of NALs...
The references I found in the internet shows code using few NAL types to encapsulate the FLV file, but after the change to detect 3 bytes, the program found a lot of NALs and I don't know how to stream them...
Some code :-)
I'm implementing in C++, so I have the classes.
The FLVWritter
class FLVWritter{
public:
std::vector<uint8_t> buffer;
void flushToFD(int fd) {
if (buffer.size() <= 0)
return;
::write(fd,&buffer[0],buffer.size());
buffer.clear();
}
void reset(){
buffer.clear();
}
void writeFLVFileHeader(bool haveAudio, bool haveVideo){
uint8_t flv_header[13] = {
0x46, 0x4c, 0x56, 0x01, 0x05,
0x00, 0x00, 0x00, 0x09, 0x00,
0x00, 0x00, 0x00
};
if (haveAudio && haveVideo) {
flv_header[4] = 0x05;
} else if (haveAudio && !haveVideo) {
flv_header[4] = 0x04;
} else if (!haveAudio && haveVideo) {
flv_header[4] = 0x01;
} else {
flv_header[4] = 0x00;
}
for(int i=0;i<13;i++)
buffer.push_back(flv_header[i]);
}
void writeUInt8(uint8_t v) {
buffer.push_back(v);
}
void writeUInt16(uint32_t v){
buffer.push_back((uint8_t)(v >> 8));
buffer.push_back((uint8_t)(v));
}
void writeUInt24(uint32_t v){
buffer.push_back((uint8_t)(v >> 16));
buffer.push_back((uint8_t)(v >> 8));
buffer.push_back((uint8_t)(v));
}
void writeUInt32(uint32_t v){
buffer.push_back((uint8_t)(v >> 24));
buffer.push_back((uint8_t)(v >> 16));
buffer.push_back((uint8_t)(v >> 8));
buffer.push_back((uint8_t)(v));
}
void writeUInt32Timestamp(uint32_t v){
buffer.push_back((uint8_t)(v >> 16));
buffer.push_back((uint8_t)(v >> 8));
buffer.push_back((uint8_t)(v));
buffer.push_back((uint8_t)(v >> 24));
}
};
The h264 parser:
class ParserH264 {
std::vector<uint8_t> buffer;
enum State{
None,
Data,
NAL0,
NAL1
};
State nalState; // nal = network abstraction layer
State writingState;
void putByte(uint8_t byte) {
//
// Detect start code 00 00 01 and 00 00 00 01
//
// It returns the buffer right after the start code
//
if (byte == 0x00 && nalState == None)
nalState = NAL0;
else if (byte == 0x00 && (nalState == NAL0 || nalState == NAL1) )
nalState = NAL1;
else if (byte == 0x01 && nalState == NAL1){
nalState = None;
if (writingState == None){
writingState = Data;
return;
} else if (writingState == Data){
buffer.pop_back();// 0x00
buffer.pop_back();// 0x00
//in the case using the format 00 00 00 01, remove the last element detected
if (buffer[buffer.size()-1] == 0x00 &&
buffer[buffer.size()-2] != 0x03 )//keep value, if emulation prevention is present
buffer.pop_back();
chunkDetectedH264(&buffer[0],buffer.size());
buffer.clear();
return;
}
} else
nalState = None;
if (writingState == Data){
//
// increase raw buffer size
//
buffer.push_back(byte);
}
}
public:
ParserH264() {
nalState = None;
writingState = None;
}
virtual ~ParserH264(){
}
virtual void chunkDetectedH264(const uint8_t* ibuffer, int size){
}
void endOfStreamH264() {
if (buffer.size() <= 0)
return;
chunkDetectedH264(&buffer[0],buffer.size());
buffer.clear();
writingState = None;
nalState = None;
}
void parseH264(const uint8_t* ibuffer, int size) {
for(int i=0;i<size;i++){
putByte(ibuffer[i]);
}
}
};
And finally the main class:
class FLVFileWriter: public ParserH264 {
std::vector<uint8_t> lastSPS;
public:
FLVWritter mFLVWritter;
int fd;
bool firstAudioWrite;
uint32_t audioTimestamp_ms;
uint32_t videoTimestamp_ms;
FLVFileWriter ( const char* file ) {
fd = open(file, O_WRONLY | O_CREAT | O_TRUNC , 0644 );
if (fd < 0){
fprintf(stderr,"error to create flv file\n");
exit(-1);
}
//have audio, have video
mFLVWritter.writeFLVFileHeader(false, true);
mFLVWritter.flushToFD(fd);
firstAudioWrite = true;
audioTimestamp_ms = 0;
videoTimestamp_ms = 0;
}
virtual ~FLVFileWriter(){
if (fd >= 0){
mFLVWritter.flushToFD(fd);
close(fd);
}
}
void chunkDetectedH264(const uint8_t* ibuffer, int size) {
printf("[debug] Detected NAL chunk size: %i\n",size);
if (size <= 0){
fprintf(stdout, " error On h264 chunk detection\n");
return;
}
uint8_t nal_bit = ibuffer[0];
uint8_t nal_type = (nal_bit & 0x1f);
//0x67
//if (nal_bit == (NAL_IDC_PICTURE | NAL_TYPE_SPS) ) {
if ( nal_type == (NAL_TYPE_SPS) ) {
fprintf(stdout, " processing: 0x%x (SPS)\n",nal_bit);
//store information to use when arrise PPS nal_bit, probably the next NAL detection
lastSPS.clear();
for(int i=0;i<size;i++)
lastSPS.push_back(ibuffer[i]);
}
//else if (nal_bit == (NAL_IDC_PICTURE | NAL_TYPE_PPS) ) {
else if ( nal_type == (NAL_TYPE_PPS) ) {
fprintf(stdout, " processing: 0x%x (PPS)\n",nal_bit);
//must be called just after the SPS detection
int32_t bodyLength = lastSPS.size() + size + 16;
//
// flv tag header = 11 bytes
//
mFLVWritter.writeUInt8(0x09);//tagtype video
mFLVWritter.writeUInt24( bodyLength );//data len
mFLVWritter.writeUInt32Timestamp( videoTimestamp_ms );//timestamp
mFLVWritter.writeUInt24( 0 );//stream id 0
//
// Message Body = 16 bytes + SPS bytes + PPS bytes
//
//flv VideoTagHeader
mFLVWritter.writeUInt8(0x17);//key frame, AVC 1:keyframe 7:h264
mFLVWritter.writeUInt8(0x00);//avc sequence header
mFLVWritter.writeUInt24( 0x00 );//composit time ??????????
//flv VideoTagBody --AVCDecoderCOnfigurationRecord
mFLVWritter.writeUInt8(0x01);//configurationversion
mFLVWritter.writeUInt8(lastSPS[1]);//avcprofileindication
mFLVWritter.writeUInt8(lastSPS[2]);//profilecompatibilty
mFLVWritter.writeUInt8(lastSPS[3]);//avclevelindication
mFLVWritter.writeUInt8(0xFC | 0x03); //reserved + lengthsizeminusone
mFLVWritter.writeUInt8(0xe0 | 0x01); // first reserved, second number of SPS
mFLVWritter.writeUInt16( lastSPS.size() ); //sequence parameter set length
//H264 sequence parameter set raw data
for(int i=0;i<lastSPS.size();i++)
mFLVWritter.writeUInt8(lastSPS[i]);
mFLVWritter.writeUInt8(0x01); // number of PPS
//sanity check with the packet size...
if ( size-4 > 0xffff ){
fprintf(stderr, "PPS Greater than 64k. This muxer does not support it.\n");
exit(-1);
}
mFLVWritter.writeUInt16(size); //picture parameter set length
//H264 picture parameter set raw data
for(int i=0;i<size;i++)
mFLVWritter.writeUInt8(ibuffer[i]);
//
// previous tag size
//
uint32_t currentSize = mFLVWritter.buffer.size();
if (currentSize != bodyLength + 11 ){
fprintf(stderr, "error to write flv video tag NAL_TYPE_PPS\n");
exit(-1);
}
mFLVWritter.writeUInt32(currentSize);//data len
mFLVWritter.flushToFD(fd);
videoTimestamp_ms += 1000/30;
}
//0x65
//else if (nal_bit == (NAL_IDC_PICTURE | NAL_TYPE_CSIDRP) ) {
else if ( nal_type == (NAL_TYPE_CSIDRP) ) {
fprintf(stdout, " processing: 0x%x (0x65)\n",nal_bit);
uint32_t bodyLength = size + 5 + 4;//flv VideoTagHeader + NALU length
//
// flv tag header = 11 bytes
//
mFLVWritter.writeUInt8(0x09);//tagtype video
mFLVWritter.writeUInt24( bodyLength );//data len
mFLVWritter.writeUInt32Timestamp( videoTimestamp_ms );//timestamp
mFLVWritter.writeUInt24( 0 );//stream id 0
//
// Message Body = VideoTagHeader(5) + NALLength(4) + NAL raw data
//
//flv VideoTagHeader
mFLVWritter.writeUInt8(0x17);//key frame, AVC 1:keyframe 2:inner frame 7:H264
mFLVWritter.writeUInt8(0x01);//avc NALU unit
mFLVWritter.writeUInt24(0x00);//composit time ??????????
mFLVWritter.writeUInt32(size);//nal length
//nal raw data
for(int i=0;i<size;i++)
mFLVWritter.writeUInt8(ibuffer[i]);
//
// previous tag size
//
uint32_t currentSize = mFLVWritter.buffer.size();
if (currentSize != bodyLength + 11 ){
fprintf(stderr, "error to write flv video tag NAL_TYPE_CSIDRP\n");
exit(-1);
}
mFLVWritter.writeUInt32(currentSize);//data len
mFLVWritter.flushToFD(fd);
videoTimestamp_ms += 1000/30;
}
//0x61
//else if (nal_bit == (NAL_IDC_FRAME | NAL_TYPE_CSNIDRP) ) {
else if ( nal_type == (NAL_TYPE_CSNIDRP) ) {
fprintf(stdout, " processing: 0x%x (0x61)\n",nal_bit);
uint32_t bodyLength = size + 5 + 4;//flv VideoTagHeader + NALU length
//
// flv tag header = 11 bytes
//
mFLVWritter.writeUInt8(0x09);//tagtype video
mFLVWritter.writeUInt24( bodyLength );//data len
mFLVWritter.writeUInt32Timestamp( videoTimestamp_ms );//timestamp
mFLVWritter.writeUInt24( 0 );//stream id 0
//
// Message Body = VideoTagHeader(5) + NALLength(4) + NAL raw data
//
//flv VideoTagHeader
mFLVWritter.writeUInt8(0x27);//key frame, AVC 1:keyframe 2:inner frame 7:H264
mFLVWritter.writeUInt8(0x01);//avc NALU unit
mFLVWritter.writeUInt24(0x00);//composit time ??????????
mFLVWritter.writeUInt32(size);//nal length
// raw nal data
for(int i=0;i<size;i++)
mFLVWritter.writeUInt8(ibuffer[i]);
//
// previous tag size
//
uint32_t currentSize = mFLVWritter.buffer.size();
if (currentSize != bodyLength + 11 ){
fprintf(stderr, "error to write flv video tag NAL_TYPE_CSNIDRP\n");
exit(-1);
}
mFLVWritter.writeUInt32(currentSize);//data len
mFLVWritter.flushToFD(fd);
videoTimestamp_ms += 1000/30;
}
else if (nal_type == (NAL_TYPE_SEI)) {
fprintf(stdout, " ignoring SEI bit: 0x%x type: 0x%x\n",nal_bit, nal_type);
} else {
// nal_bit type not implemented...
fprintf(stdout, "Error: unknown NAL bit: 0x%x type: 0x%x\n",nal_bit, nal_type);
exit(-1);
}
}
};
To use these classes in main we can write this:
volatile bool exit_requested = false;
void signal_handler(int signal) {
exit_requested = true;
}
int main(int argc, char* argv[]) {
int fd_stdin = fileno(stdin);
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
signal(SIGQUIT, signal_handler);
unsigned char buffer[65536];
FLVFileWriter flv("out.flv");
while (!exit_requested) {
int readedSize = read(fd_stdin,buffer,65536);
if (readedSize==0)
break;
flv.parseH264(buffer,readedSize);
}
flv.endOfStreamH264();
signal(SIGINT, SIG_DFL);
signal(SIGTERM, SIG_DFL);
signal(SIGQUIT, SIG_DFL);
return 0;
}
Compiling the program above we can execute as follows:
cat test.h264 | ./flv
Then I get this:
I used the ffmpeg from command line to check if my source h264 stream file is corrupted. So I ran the following:
cat test.h264 | ffmpeg -i - -c:v copy out2.flv
With ffmpeg the result is OK:
I put the video I tested with the full source code here.
The h264 file I used for the test.
WIP (work in progress)
But I found a lot of new information :-).
First: The FLV container does not work packing one by one NAL buffers from h264.
Second: The timestamp needs to be increased just when a new frame is available.
The problem with the green screen I got is related to the fragmentation on NAL packages with the timestamp issue.
The difficult part is to detect when the stream is getting a new frame. There are several ways as mentioned here.
But the h264 stream have data members with different bit sizes in its structure.
Reading Bits
ue(v) data type. To read this structure you need to count the leading 0 bits until reach bit 1 and read the same amount of 0 bits after the 1 bit we found.
The content of this field inside the headers of h264 can vary... one common value is called golomb, and it is calculated with this formula: (1 << leadingZeros) - 1 + readbits(leadingZeros).
RBSP - Raw Byte Sequence Payload
We cannot read the stream values directly, because it may contains the emulation prevention bytes. We need to remove the emulation prevention bytes before read the headers.
The Code
So to start mitigate the problem first I rewrite the H264 parser as follow:
struct sps_info{
uint8_t profile_idc;
uint8_t constraints;
uint8_t level_idc;
uint8_t log2_max_frame_num;
bool set;
};
class ParserH264 {
std::vector<uint8_t> buffer;
enum State{
None,
Data,
NAL0,
NAL1,
EMULATION0,
EMULATION1,
EMULATION_FORCE_SKIP
};
State nalState; // nal = network abstraction layer
State writingState;
void putByte(uint8_t byte) {
//
// Detect start code 00 00 01 and 00 00 00 01
//
// It returns the buffer right after the start code
//
if (byte == 0x00 && nalState == None)
nalState = NAL0;
else if (byte == 0x00 && (nalState == NAL0 || nalState == NAL1) )
nalState = NAL1;
else if (byte == 0x01 && nalState == NAL1){
nalState = None;
if (writingState == None){
writingState = Data;
return;
} else if (writingState == Data){
buffer.pop_back();// 0x00
buffer.pop_back();// 0x00
//in the case using the format 00 00 00 01, remove the last element detected
if (buffer.size()-2 >=0 &&
buffer[buffer.size()-1] == 0x00 &&
buffer[buffer.size()-2] != 0x03 )//keep value, if emulation prevention is present
buffer.pop_back();
chunkDetectedH264(buffer);
buffer.clear();
return;
}
} else
nalState = None;
if (writingState == Data){
//
// increase raw buffer size
//
buffer.push_back(byte);
}
}
public:
ParserH264() {
nalState = None;
writingState = None;
}
virtual ~ParserH264(){
}
// rawBuffer might has emulation bytes
// Raw Byte Sequence Payload (RBSP)
virtual void chunkDetectedH264(const std::vector<uint8_t> &nal){
}
void endOfStreamH264() {
if (buffer.size() <= 0)
return;
chunkDetectedH264(buffer);
buffer.clear();
writingState = None;
nalState = None;
}
void parseH264(const uint8_t* ibuffer, int size) {
for(int i=0;i<size;i++){
putByte(ibuffer[i]);
}
}
static inline uint32_t readbit(int bitPos, const uint8_t* data, int size){
int dataPosition = bitPos / 8;
int bitPosition = 7 - bitPos % 8;
if (dataPosition >= size){
fprintf(stderr,"error to access bit...\n");
exit(-1);
}
return (data[dataPosition] >> bitPosition) & 0x01;
}
// leading 0`s count equals the number of next bits after bit 1
//
// Example: 01x 001xx 0001xxx 00001xxxx
//
// The max number of bits is equal 32 in this sintax
//
static inline int bitsToSkip_ue( int start, const uint8_t* data, int size){
int bitPos = start;
int dataPosition = start / 8;
int bitPosition;
while (dataPosition < size){
dataPosition = bitPos / 8;
bitPosition = 7 - bitPos % 8;
int bit = (data[dataPosition] >> bitPosition) & 0x01;
if (bit == 1)
break;
bitPos++;
}
int leadingZeros = bitPos - start;
int totalBits = leadingZeros + 1 + leadingZeros;
if (totalBits > 32){
fprintf(stderr,"bitsToSkip_ue length greated than 32\n");
exit(-1);
}
return totalBits;
}
static inline uint32_t read_golomb_ue(int start, const uint8_t* data, int size){
int bitPos = start;
int dataPosition = start / 8;
int bitPosition;
while (dataPosition < size){
dataPosition = bitPos / 8;
bitPosition = 7 - bitPos % 8;
int bit = (data[dataPosition] >> bitPosition) & 0x01;
if (bit == 1)
break;
bitPos++;
}
uint32_t leadingZeros = (uint32_t)(bitPos - start);
uint32_t num = readbits(bitPos+1, leadingZeros, data, size);
num += (1 << leadingZeros) - 1;
return num;
}
static inline uint32_t readbits(int bitPos, int length, const uint8_t* data, int size){
if (length > 32){
fprintf(stderr,"readbits length greated than 32\n");
exit(-1);
}
uint32_t result = 0;
for(int i=0;i<length;i++){
result <<= 1;
result = result | readbit(bitPos+i, data, size);
}
return result;
}
static inline sps_info parseSPS(const std::vector<uint8_t> sps_rbsp) {
const uint8_t *data = &sps_rbsp[0];
uint32_t size = sps_rbsp.size();
sps_info result;
result.profile_idc = sps_rbsp[1];
result.constraints = sps_rbsp[2];
result.level_idc = sps_rbsp[3];
uint32_t startIndex = 8+24;//NAL bit + profile_idc (8bits) + constraints (8bits) + level_idc (8bits)
startIndex += bitsToSkip_ue(startIndex, data, size);//seq_parameter_set_id (ue)
uint32_t log2_max_frame_num_minus4 = read_golomb_ue(startIndex, data, size);
if (log2_max_frame_num_minus4 < 0 ||
log2_max_frame_num_minus4 > 12){
fprintf(stderr,"parseSPS_log2_max_frame_num_minus4 value not in range [0-12] \n");
exit(-1);
}
result.log2_max_frame_num = log2_max_frame_num_minus4 + 4;
result.set = true;
return result;
}
// Raw Byte Sequence Payload (RBSP) -- without the emulation prevention bytes
// maxSize is used to parse just the beggining of the nal structure...
// avoid process all buffer size on NAL new frame check
static inline void nal2RBSP(const std::vector<uint8_t> &buffer,
std::vector<uint8_t> *rbsp,
int maxSize = -1){
if (maxSize <= 0)
maxSize = buffer.size();
rbsp->resize(maxSize);
State emulationState = None;
int count = 0;
for(int i=0; i < maxSize ;i++){
uint8_t byte = buffer[i];
if (byte == 0x00 && emulationState == None)
emulationState = EMULATION0;
else if (byte == 0x00 && (emulationState == EMULATION0 || emulationState == EMULATION1) )
emulationState = EMULATION1;
else if (byte == 0x03 && emulationState == EMULATION1)
{
emulationState = EMULATION_FORCE_SKIP;
continue;
}
else if (emulationState == EMULATION_FORCE_SKIP) { //skip 00 01 02 or 03
if ( byte != 0x00 && byte != 0x01 && byte != 0x02 && byte != 0x03 ){
fprintf(stdout, "H264 NAL emulation prevention pattern detected error (%u)\n", byte);
exit(-1);
}
emulationState = None;
} else
emulationState = None;
(*rbsp)[count] = byte;
count++;
}
if (count != rbsp->size())
rbsp->resize(count);
}
};
I updated the FLVWriter to write the whole frame (with 1 or more NALs) and to write the video sequence header.
The code:
class FLVWritter{
public:
std::vector<uint8_t> buffer;
void flushToFD(int fd) {
if (buffer.size() <= 0)
return;
::write(fd,&buffer[0],buffer.size());
buffer.clear();
}
void reset(){
buffer.clear();
}
void writeFLVFileHeader(bool haveAudio, bool haveVideo){
uint8_t flv_header[13] = {
0x46, 0x4c, 0x56, 0x01, 0x05,
0x00, 0x00, 0x00, 0x09, 0x00,
0x00, 0x00, 0x00
};
if (haveAudio && haveVideo) {
flv_header[4] = 0x05;
} else if (haveAudio && !haveVideo) {
flv_header[4] = 0x04;
} else if (!haveAudio && haveVideo) {
flv_header[4] = 0x01;
} else {
flv_header[4] = 0x00;
}
for(int i=0;i<13;i++)
buffer.push_back(flv_header[i]);
}
void writeUInt8(uint8_t v) {
buffer.push_back(v);
}
void writeUInt16(uint32_t v){
buffer.push_back((v >> 8) & 0xff);
buffer.push_back((v) & 0xff);
}
void writeUInt24(uint32_t v){
buffer.push_back((v >> 16) & 0xff);
buffer.push_back((v >> 8) & 0xff);
buffer.push_back((v) & 0xff);
}
void writeUInt32(uint32_t v){
buffer.push_back((v >> 24) & 0xff);
buffer.push_back((v >> 16) & 0xff);
buffer.push_back((v >> 8) & 0xff);
buffer.push_back((v) & 0xff);
}
void writeUInt32(uint32_t v, std::vector<uint8_t> *data){
data->push_back((v >> 24) & 0xff);
data->push_back((v >> 16) & 0xff);
data->push_back((v >> 8) & 0xff);
data->push_back((v) & 0xff);
}
void writeUInt32Timestamp(uint32_t v){
buffer.push_back((v >> 16) & 0xff);
buffer.push_back((v >> 8) & 0xff);
buffer.push_back((v) & 0xff);
buffer.push_back((v >> 24) & 0xff);
}
void writeVideoSequenceHeader(const std::vector<uint8_t> &sps, const std::vector<uint8_t> &pps, const sps_info &spsinfo) {
//
// flv tag header = 11 bytes
//
writeUInt8(0x09);//tagtype video
writeUInt24( sps.size() + pps.size() + 16 );//data len
writeUInt32Timestamp( 0 );//timestamp
writeUInt24( 0 );//stream id 0
//
// Message Body = 16 bytes + SPS bytes + PPS bytes
//
//flv VideoTagHeader
writeUInt8(0x17);//key frame, AVC 1:keyframe 7:h264
writeUInt8(0x00);//avc sequence header
writeUInt24( 0x00 );//composition time
//flv VideoTagBody --AVCDecoderCOnfigurationRecord
writeUInt8(0x01);//configurationversion
writeUInt8(spsinfo.profile_idc);//avcprofileindication
writeUInt8(spsinfo.constraints);//profilecompatibilty
writeUInt8(spsinfo.level_idc);//avclevelindication
writeUInt8(0xFC | 0x03); //reserved (6 bits), NULA length size - 1 (2 bits)
writeUInt8(0xe0 | 0x01); // first reserved, second number of SPS
writeUInt16( sps.size() ); //sequence parameter set length
//H264 sequence parameter set raw data
for(int i=0;i<sps.size();i++)
writeUInt8(sps[i]);
writeUInt8(0x01); // number of PPS
writeUInt16(pps.size()); //picture parameter set length
//H264 picture parameter set raw data
for(int i=0;i<pps.size();i++)
writeUInt8(pps[i]);
if (buffer.size() != sps.size() + pps.size() + 16 + 11 ){
fprintf(stderr, "error writeVideoSequenceHeader\n");
exit(-1);
}
// previous tag size
writeUInt32(buffer.size());
}
void writeVideoFrame(const std::vector<uint8_t> &data, bool keyframe, uint32_t timestamp_ms, int streamID){
writeUInt8(0x09);//tagtype video
writeUInt24( data.size() + 5 );//data len
writeUInt32Timestamp( timestamp_ms );//timestamp
writeUInt24( streamID );//stream id 0)
if (keyframe)
writeUInt8(0x17);//key frame, AVC 1:keyframe 2:inner frame 7:H264
else
writeUInt8(0x27);//key frame, AVC 1:keyframe 2:inner frame 7:H264
writeUInt8(0x01);//avc NALU unit
writeUInt24(0x00);//composit time ??????????
for(int i=0;i<data.size();i++)
writeUInt8(data[i]);
if (buffer.size() != data.size() + 5 + 11 ){
fprintf(stderr, "error writeVideoFrame\n");
exit(-1);
}
// previous size
writeUInt32(buffer.size());
}
void writeVideoEndOfStream(uint32_t timestamp_ms, int streamID){
writeUInt8(0x09);//tagtype video
writeUInt24( 5 );//data len
writeUInt32Timestamp( timestamp_ms );//timestamp
writeUInt24( streamID );//stream id 0)
writeUInt8(0x17);//key frame, AVC 1:keyframe 2:inner frame 7:H264
writeUInt8(0x02);//avc EOS
writeUInt24(0x00);//composit time ??????????
if (buffer.size() != 5 + 11 ){
fprintf(stderr, "error writeVideoEOS\n");
exit(-1);
}
// previous size
writeUInt32(buffer.size());
}
};
Next we need to detect new frame from the stream.
I've implemented several methods and the frame_num method also.
To read the frame_num, we need to skip 3 ue(v) structures before read the frame_num bits.
Now the h264 new frame detector class:
class H264NewFrameDetection {
std::vector<uint8_t> aux;
bool newFrameOnNextIDR;
uint32_t old_frame_num;
bool spsinfo_set;
bool firstFrame;
public:
uint32_t currentFrame;
bool newFrameFound;
H264NewFrameDetection() {
currentFrame = 0;
newFrameOnNextIDR = false;
old_frame_num = 0;
spsinfo_set = false;
firstFrame = true;
}
void analyseBufferForNewFrame(const std::vector<uint8_t> &nal, const sps_info &spsinfo){
uint8_t nal_bit = nal[0];
uint8_t nal_type = (nal_bit & 0x1f);
if ( nal_type == (NAL_TYPE_SPS) ||
nal_type == (NAL_TYPE_PPS) ||
nal_type == (NAL_TYPE_AUD) ||
nal_type == (NAL_TYPE_SEI) ||
(nal_type >= 14 && nal_type <= 18)
) {
newFrameOnNextIDR = true;
}
if ((nal_type == NAL_TYPE_CSIDRP ||
nal_type == NAL_TYPE_CSNIDRP)
&& spsinfo.set ){
aux.clear();
//(8 + 3*(32*2+1) + 16) = max header per NALU slice bits = 27.375 bytes
// 32 bytes + 8 (Possibility of Emulation in 32 bytes)
int RBSPMaxBytes = 32 + 8;
if (nal.size() < (32 + 8))
RBSPMaxBytes = nal.size();
ParserH264::nal2RBSP(nal, &aux, RBSPMaxBytes );
uint8_t * rbspBuffer = &aux[0];
uint32_t rbspBufferSize = aux.size();
uint32_t frame_num_index = 8;//start counting after the nal_bit
//first_mb_in_slice (ue)
frame_num_index += ParserH264::bitsToSkip_ue(frame_num_index, rbspBuffer, rbspBufferSize);
//slice_type (ue)
frame_num_index += ParserH264::bitsToSkip_ue(frame_num_index, rbspBuffer, rbspBufferSize);
//pic_parameter_set_id (ue)
frame_num_index += ParserH264::bitsToSkip_ue(frame_num_index, rbspBuffer, rbspBufferSize);
//now can read frame_num
uint32_t frame_num = ParserH264::readbits(frame_num_index,
spsinfo.log2_max_frame_num,
rbspBuffer, rbspBufferSize);
if (!spsinfo_set){
old_frame_num = frame_num;
spsinfo_set = true;//spsinfo.set
}
if (old_frame_num != frame_num){
newFrameOnNextIDR = true;
old_frame_num = frame_num;
}
}
if (newFrameOnNextIDR &&(nal_type == NAL_TYPE_CSIDRP ||
nal_type == NAL_TYPE_CSNIDRP)) {
newFrameOnNextIDR = false;
if (firstFrame){//skip the first frame
firstFrame = false;
} else {
newFrameFound = true;
currentFrame++;
}
}
}
void reset(){
newFrameFound = false;
}
};
Finally the writer class.
In this implementation I store the NALs until a new frame is detected. When it founds a new frame I write the FLV Tag with the NAL set (all NALs stored in the list).
The code:
class FLVFileWriter: public ParserH264 {
std::vector<uint8_t> sps;
std::vector<uint8_t> pps;
sps_info spsInfo;
H264NewFrameDetection mH264NewFrameDetection;
public:
FLVWritter mFLVWritter;
int fd;
uint32_t videoTimestamp_ms;
//contains the several NALs until complete a frame... after that can write to FLV
std::vector<uint8_t> nalBuffer;
FLVFileWriter ( const char* file ) {
fd = open(file, O_WRONLY | O_CREAT | O_TRUNC , 0644 );
if (fd < 0){
fprintf(stderr,"error to create flv file\n");
exit(-1);
}
//have audio, have video
mFLVWritter.writeFLVFileHeader(false, true);
mFLVWritter.flushToFD(fd);
videoTimestamp_ms = 0;
spsInfo.set = false;
}
virtual ~FLVFileWriter(){
if (fd >= 0){
//force write the last frame
if (nalBuffer.size() > 0){
uint8_t firstNALtype = nalBuffer[4] & 0x1f;
bool iskeyframe = (firstNALtype == NAL_TYPE_CSIDRP);
mFLVWritter.writeVideoFrame(nalBuffer, iskeyframe, videoTimestamp_ms, 0);
mFLVWritter.flushToFD(fd);
nalBuffer.clear();
mFLVWritter.writeVideoEndOfStream(videoTimestamp_ms,0);
mFLVWritter.flushToFD(fd);
} else {
if (mH264NewFrameDetection.currentFrame > 0)
videoTimestamp_ms = ((mH264NewFrameDetection.currentFrame-1) * 1000)/30;
mFLVWritter.writeVideoEndOfStream(videoTimestamp_ms,0);
mFLVWritter.flushToFD(fd);
}
close(fd);
}
}
//decoding time stamp (DTS) and presentation time stamp (PTS)
void chunkDetectedH264(const std::vector<uint8_t> &data) {
if (data.size() <= 0){
fprintf(stdout, " error On h264 chunk detection\n");
return;
}
uint8_t nal_bit = data[0];
uint8_t nal_type = (nal_bit & 0x1f);
mH264NewFrameDetection.analyseBufferForNewFrame(data, spsInfo);
if (mH264NewFrameDetection.newFrameFound){
mH264NewFrameDetection.reset();
uint8_t firstNALtype = nalBuffer[4] & 0x1f;
bool iskeyframe = (firstNALtype == NAL_TYPE_CSIDRP);
mFLVWritter.writeVideoFrame(nalBuffer, iskeyframe, videoTimestamp_ms, 0);
mFLVWritter.flushToFD(fd);
nalBuffer.clear();
videoTimestamp_ms = (mH264NewFrameDetection.currentFrame * 1000)/30;
}
//0x67
if ( nal_type == (NAL_TYPE_SPS) ) {
fprintf(stdout, " processing: 0x%x (SPS)\n",nal_bit);
sps.clear();
//max 26 bytes on sps header (read until log2_max_frame_num_minus4)
nal2RBSP(data, &sps, 26 );
spsInfo = parseSPS(sps);
sps.clear();
for(int i=0;i<data.size();i++)
sps.push_back(data[i]);
}
//0x68
else if ( nal_type == (NAL_TYPE_PPS) ) {
fprintf(stdout, " processing: 0x%x (PPS)\n",nal_bit);
pps.clear();
for(int i=0;i<data.size();i++)
pps.push_back(data[i]);
mFLVWritter.writeVideoSequenceHeader(sps, pps, spsInfo);
mFLVWritter.flushToFD(fd);
}
//0x65, 0x61, 0x41
else if ( nal_type == NAL_TYPE_CSIDRP ||
nal_type == NAL_TYPE_CSNIDRP ) {
//convert annexb to AVCC (length before the NAL structure)
mFLVWritter.writeUInt32( data.size(), &nalBuffer );
for(int i=0;i<data.size();i++)
nalBuffer.push_back(data[i]);
} else if (nal_type == (NAL_TYPE_SEI)) {
fprintf(stdout, " ignoring SEI bit: 0x%x type: 0x%x\n",nal_bit, nal_type);
} else {
// nal_bit type not implemented...
fprintf(stdout, "Error: unknown NAL bit: 0x%x type: 0x%x\n",nal_bit, nal_type);
exit(-1);
}
}
};
Conclusions
After testing, the video works correctly with this new implementation.
The only issue I have now is related to PTS (presentation timestamp) and DTS (decoder timestamp). I don't know how to extract them from the stream and how to use them to compute the frame timestamp.
This current implementation uses the hardcoded ratio 1000/30 (e.g., 30fps) as basis to increment the timestamp.
The timestamp is in milliseconds.
I'm trying to communicate with a server program I installed. The server sends and receives all data in the form of constructed packets that follow the setup of:
int int int string nullbyte
like this:
little endian signed int -> The size of the ID (4 bytes) + size of Type (4 bytes) + size of Body (minimum 1 for null terminator) + null byte for a minimum of 10 as the value;
little endian signed int -> id
little endian signed int -> packet type
Null terminiated ascii string -> body
null byte
I've managed to read the packets just fine but when I try to send the packet with the password, the server completely ignores it which means the packet is wrong some how. I construct the packet like this:
void Packet::build(){
/*
* Create unsigned char vector to store
* the data while we build the byte array
* and create a pointer so the byte array can
* be modified by the required functions.
*/
std::vector<unsigned char> packet(m_size);
unsigned char *ptr = packet.data();
/*
* Convert each of the three integers as well
* as the string into bytes which will be stored
* back into the memory that ptr points to.
*
* Packet data follows format:
* int32 -> Size of ID + Server Data + body (minimum of 10).
* int32 -> ID of request. Used to match to response.
* int32 -> Server Data type. Identifies type of request.
* String -> Minimum of 1 byte for null terminator.
* String -> Null terminator.
*/
storeInt32Le(ptr, m_sizeInPacket);
storeInt32Le(ptr, m_requestID);
storeInt32Le(ptr, m_serverData);
storeStringNt(ptr, m_body);
/*
* Store the vector in member variable m_cmdBytes;
*
*/
m_cmdBytes = packet;
}
storeInt32Le:
void Packet::storeInt32Le(unsigned char* &buffer, int32_t value) {
/*
* Copy the integer to a byte array using
* bitwise AND with mask to ensure the right
* bits are copied to each segment then
* increment the pointer by 4 for the next
* iteration.
*/
buffer[0] = value & 0xFF;
buffer[1] = (value >> 8) & 0xFF;
buffer[2] = (value >> 16) & 0xFF;
buffer[3] = (value >> 24) & 0xFF;
buffer += 4;
}
storeStringNt:
void Packet::storeStringNt(unsigned char* &buffer, const string &s) {
/*
* Get the size of the string to be copied
* then do a memcpy of string char array to
* the buffer.
*/
size_t size = s.size() + 1;
memcpy(buffer, s.c_str(), size);
buffer += size;
}
And finally, I send it with:
bool Connection::sendCmd(Packet packet) {
unsigned char *pBytes = packet.bytes().data();
size_t size = packet.size();
while (size > 0){
int val = send(m_socket, pBytes, size, 0);
if (val <= 0) {
return false;
}
pBytes += val;
size -= val;
}
return true;
}
Packet::bytes() just returns m_cmdBytes
As stated in comments, you are:
making assumptions about the byte size and endian of your compiler's int data type. Since your protocol requires a very specific byte size and endian for its integers, you need to force your code to follow those requirements when preparing the packet.
using memcpy() incorrectly. You have the source and destination buffers reversed.
are not ensuring that send() is actually sending the full packet correctly.
Try something more like this:
void store_uint32_le(unsigned char* &buffer, uint32_t value)
{
buffer[0] = value & 0xFF;
buffer[1] = (value >> 8) & 0xFF;
buffer[2] = (value >> 16) & 0xFF;
buffer[3] = (value >> 24) & 0xFF;
buffer += 4;
}
void store_string_nt(unsigned char* &buffer, const std::string &s)
{
size_t size = s.size() + 1;
memcpy(buffer, s.c_str(), size);
buffer += size;
}
...
std::vector<unsigned char> packet(13 + m_body.size());
unsigned char *ptr = packet.data(); // or = &packet[0] prior to C++11
store_uint32_le(ptr, packet.size() - 4);
store_uint32_le(ptr, m_requestID);
store_uint32_le(ptr, m_serverData);
store_string_nt(ptr, m_body);
...
unsigned char *pBytes = packet.data(); // or = &packet[0] prior to C++11
size_t size = packet.size();
while (size > 0)
{
int val = send(m_socket, pBytes, size, 0);
if (val < 0)
{
// if your socket is non-blocking, enable this...
/*
#ifdef _WINDOWS // check your compiler for specific defines...
if (WSAGetLastError() == WSAEWOULDBLOCK)
#else
if ((errno == EAGAIN) || (errno == EWOULDBLOCK) || (errno == EINTR))
#endif
continue;
*/
return false;
}
if (val == 0)
return false;
pBytes += val;
size -= val;
}
return true;
hopefully one of you out there can help me!
I am trying to use the Adafruit SHT31-D (an i2c device) board with my Pi2. I am going off of this datasheet to guide my coding efforts. I am using Wiring Pi (wiringpi.com) to facilitate things.
I am able to successfully open a connection to the device, and sending commands seems to work fine, but I am unable to read data back! Here is the little mini library I have put together. I am hoping that one of you might have some experience with this sort of thing and be able to help me see where I've gone wrong.
To rule out any possible issues with the sensor hardware, I have tested it with my Arduino UNO and it works without issues.
Here is my C++ code:
SHT3x.h
#pragma once
/* Sensor Commands */
#define DEFAULT_SHT_ADDR 0x44
#define MEAS_HREP_STRETCH 0x2C06
#define MEAS_MREP_STRETCH 0x2C0D
#define MEAS_LREP_STRETCH 0x2C10
#define MEAS_HREP 0x2400
#define MEAS_MREP 0x240B
#define MEAS_LREP 0x2416
#include <cstdint>
class SHT3x {
public:
SHT3x(const uint8_t& i2cAddr);
float readHumidity(const uint16_t& command) const;
float readTempC(const uint16_t& command) const;
float readTempF(const uint16_t& command) const;
private:
int8_t _fd;
uint8_t _header;
uint32_t getMeasurement(const uint16_t& command) const;
void sendCommand(const uint16_t& command) const;
uint32_t receiveData(void) const;
};
SHT3x.cpp
#include <stdexcept>
#include <wiringPi.h>
#include <wiringPiI2C.h>
#include "SHT3x.h"
SHT3x::SHT3x(const uint8_t& i2cAddr) {
_fd = wiringPiI2CSetup(i2cAddr);
_header = i2cAddr << 1;
if (_fd < 0) {
throw std::runtime_error("Unable to connect");
}
}
float SHT3x::readHumidity(const uint16_t& command) const {
uint32_t raw_data = getMeasurement(command);
if (!raw_data) {
throw std::runtime_error("Bad Reading.");
}
uint16_t raw_humidity = raw_data & 0xFFFF;
float humidity = 100.0 * ((float) raw_humidity / (float) 0xFFFF);
return humidity;
}
float SHT3x::readTempC(const uint16_t& command) const {
uint32_t raw_data = getMeasurement(command);
if (!raw_data) {
throw std::runtime_error("Bad Reading.");
}
uint16_t raw_temp = raw_data >> 16;
float tempC = -45.0 + (175.0 * ((float) raw_temp / (float) 0xFFFF));
return tempC;
}
float SHT3x::readTempF(const uint16_t& command) const {
uint32_t raw_data = getMeasurement(command);
if (!raw_data) {
throw std::runtime_error("Bad Reading.");
}
uint16_t raw_temp = raw_data >> 16;
float tempF = -49.0 + (315.0 * ((float) raw_temp / (float) 0xFFFF));
return tempF;
}
uint32_t SHT3x::getMeasurement(const uint16_t& command) const {
try {
sendCommand(command);
} catch (std::runtime_error& e) {
throw;
}
return receiveData();
}
void SHT3x::sendCommand(const uint16_t& command) const {
// break command into bytes
uint8_t MSB = command >> 8;
uint8_t LSB = command & 0xFF;
// send header
int8_t ack = wiringPiI2CWrite(_fd, _header);
// send command
ack &= wiringPiI2CWrite(_fd, MSB);
ack &= wiringPiI2CWrite(_fd, LSB);
// handle errors
if (ack) {
throw std::runtime_error("Sending command failed.");
}
}
uint32_t SHT3x::receiveData(void) const {
uint32_t data;
// send header
uint8_t read_header = _header | 0x01;
int8_t ack = wiringPiI2CWrite(_fd, read_header);
// handle errors
if (ack) throw std::runtime_error("Unable to read data.");
// read data
data = wiringPiI2CRead(_fd);
for (size_t i = 0; i < 4; i++) {
printf("Data: %d\n", data);
data <<= 8;
if (i != 1) {
data |= wiringPiI2CRead(_fd);
} else {
wiringPiI2CRead(_fd); // skip checksum
}
}
wiringPiI2CRead(_fd); // second checksum
return data;
}
The SHT31 uses 16bit read and write, rather than using 2 8bit writes you might be better off using wiringpi's 16bit write. wiringPiI2CWriteReg16(). Same thing applies to the read.
Below is a very early copy of what I've done to read the sht31-d on a PI. It has no dependencies except i2c-dev. Heater enable/disable is not working, but softreset, clearstatus, getserial & get temp/humid are all fine.
/*
* Referances
* https://www.kernel.org/doc/Documentation/i2c/dev-interface
* https://github.com/adafruit/Adafruit_SHT31
* https://www.sensirion.com/fileadmin/user_upload/customers/sensirion/Dokumente/Humidity_and_Temperature_Sensors/Sensirion_Humidity_and_Temperature_Sensors_SHT3x_Datasheet_digital.pdf
*
* This depends on i2c dev lib
* sudo apt-get install libi2c-dev
*
* Below is also a good one to have, but be careful i2cdump from the below cause the sht31 interface to become unstable for me
* and requires a hard-reset to recover correctly.
* sudo apt-get install i2c-tools
*
* on PI make sure below 2 commands are in /boot/config.txt
* dtparam=i2c_arm=on
* dtparam=i2c1_baudrate=10000
* I know we are slowing down the baurate from optimal, but it seems to be the most stable setting in my testing.
* add another 0 to the above baudrate for max setting, ie dtparam=i2c1_baudrate=100000
*/
#include <linux/i2c-dev.h>
#include <time.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <fcntl.h>
#include <elf.h>
#include <unistd.h>
#define SHT31_INTERFACE_ADDR 1
#define SHT31_DEFAULT_ADDR 0x44
#define SHT31_READ_SERIALNO 0x3780
#define SHT31_MEAS_HIGHREP_STRETCH 0x2C06 // Doesn't work on PI
#define SHT31_MEAS_MEDREP_STRETCH 0x2C0D // Seems to work on PI but shouldn't
#define SHT31_MEAS_LOWREP_STRETCH 0x2C10 // Seems to work on PI but shouldn't
#define SHT31_MEAS_HIGHREP 0x2400 // Doesn't work on PI
#define SHT31_MEAS_MEDREP 0x240B
#define SHT31_MEAS_LOWREP 0x2416
#define SHT31_READSTATUS 0xF32D
#define SHT31_CLEARSTATUS 0x3041
#define SHT31_SOFTRESET 0x30A2
#define SHT31_HEATER_ENABLE 0x306D
#define SHT31_HEATER_DISABLE 0x3066
#define CHECK_BIT(var,pos) (((var)>>(pos)) & 1)
/*
* delay:
* Wait for some number of milliseconds
*********************************************************************************
*/
void delay (unsigned int howLong)
{
struct timespec sleeper, dummy ;
sleeper.tv_sec = (time_t)(howLong / 1000) ;
sleeper.tv_nsec = (long)(howLong % 1000) * 1000000 ;
nanosleep (&sleeper, &dummy) ;
}
/*
*
* CRC-8 formula from page 14 of SHT spec pdf
*
* Test data 0xBE, 0xEF should yield 0x92
*
* Initialization data 0xFF
* Polynomial 0x31 (x8 + x5 +x4 +1)
* Final XOR 0x00
*/
uint8_t crc8(const uint8_t *data, int len)
{
const uint8_t POLYNOMIAL = 0x31;
uint8_t crc = 0xFF;
int j;
int i;
for (j = len; j; --j ) {
crc ^= *data++;
for ( i = 8; i; --i ) {
crc = ( crc & 0x80 )
? (crc << 1) ^ POLYNOMIAL
: (crc << 1);
}
}
return crc;
}
/*
*
* buffer should return with data read, size defined by readsize
*********************************************************************************
*/
int writeandread(int fd, uint16_t sndword, uint8_t *buffer, int readsize)
{
int rtn;
uint8_t snd[3];
// Split the 16bit word into two 8 bits that are flipped.
snd[0]=(sndword >> 8) & 0xff;
snd[1]=sndword & 0xff;
rtn = write(fd, snd, 2);
if ( rtn != 2 ) {
return 1;
}
if (readsize > 0) {
delay(10);
rtn = read(fd, buffer, readsize);
if ( rtn < readsize) {
return 2;
}
}
return 0;
}
void printserialnum(int file)
{
uint8_t buf[10];
int rtn;
rtn = writeandread(file, SHT31_READ_SERIALNO, buf, 6);
if (rtn != 0)
printf("ERROR:- Get serial i2c %s failed\n",(rtn==1?"write":"read"));
else {
if (buf[2] != crc8(buf, 2) || buf[5] != crc8(buf+3, 2))
printf("WARNING:- Get serial CRC check failed, don't trust result\n");
uint32_t serialNo = ((uint32_t)buf[0] << 24)
| ((uint32_t)buf[1] << 16)
| ((uint32_t)buf[3] << 8)
| (uint32_t)buf[4];
printf("Serial# = %d\n",serialNo);
}
}
void printtempandhumidity(int file)
{
uint8_t buf[10];
int rtn;
rtn = writeandread(file, SHT31_MEAS_MEDREP_STRETCH, buf, 6);
if (rtn != 0)
printf("ERROR:- Get temp/humidity i2c %s failed\n",(rtn==1?"write":"read"));
else {
if ( buf[2] != crc8(buf, 2) || buf[5] != crc8(buf+3, 2))
printf("WARNING:- Get temp/humidity CRC check failed, don't trust results\n");
uint16_t ST, SRH;
ST = buf[0];
ST <<= 8;
ST |= buf[1];
SRH = buf[3];
SRH <<= 8;
SRH |= buf[4];
double stemp = ST;
stemp *= 175;
stemp /= 0xffff;
stemp = -45 + stemp;
double stempf = ST;
stempf *= 315;
stempf /= 0xffff;
stempf = -49 + stempf;
printf("Temperature %.2fc - %.2ff\n",stemp,stempf);
double shum = SRH;
shum *= 100;
shum /= 0xFFFF;
printf("Humidity %.2f%%\n",shum);
}
}
void printBitStatus(uint16_t stat)
{
printf("Status\n");
printf(" Checksum status %d\n", CHECK_BIT(stat,0));
printf(" Last command status %d\n", CHECK_BIT(stat,1));
printf(" Reset detected status %d\n", CHECK_BIT(stat,4));
printf(" 'T' tracking alert %d\n", CHECK_BIT(stat,10));
printf(" 'RH' tracking alert %d\n", CHECK_BIT(stat,11));
printf(" Heater status %d\n", CHECK_BIT(stat,13));
printf(" Alert pending status %d\n", CHECK_BIT(stat,15));
}
void printstatus(int file)
{
uint8_t buf[10];
int rtn;
rtn = writeandread(file, SHT31_READSTATUS, buf, 3);
if (rtn != 0)
printf("ERROR:- readstatus %s failed\n",(rtn==1?"write":"read"));
else {
if ( buf[2] != crc8(buf, 2))
printf("WARNING:- Get status CRC check failed, don't trust results\n");
uint16_t stat = buf[0];
stat <<= 8;
stat |= buf[1];
printBitStatus(stat);
}
}
void clearstatus(int file)
{
if( writeandread(file, SHT31_CLEARSTATUS, NULL, 0) != 0)
printf("ERROR:- sht31 clear status failed\n");
else
printf("Clearing status - ok\n");
}
void softreset(int file)
{
if( writeandread(file, SHT31_SOFTRESET, NULL, 0) != 0)
printf("ERROR:- sht31 soft reset failed\n");
else
printf("Soft reset - ok\n");
}
void enableheater(int file)
{
if( writeandread(file, SHT31_HEATER_ENABLE, NULL, 0) != 0)
printf("ERROR:- sht31 heater enable failed\n");
else
printf("Enabiling heater - ok\n");
}
void disableheater(int file)
{
if( writeandread(file, SHT31_HEATER_DISABLE, NULL, 0) != 0)
printf("ERROR:- sht31 heater enable failed\n");
else
printf("Disableing heater - ok\n");
}
int main()
{
int file;
char filename[20];
snprintf(filename, 19, "/dev/i2c-%d", SHT31_INTERFACE_ADDR);
file = open(filename, O_RDWR);
if (file < 0) {
printf("ERROR:- Can't open %s\n",filename);
exit(1);
}
if (ioctl(file, I2C_SLAVE, SHT31_DEFAULT_ADDR) < 0) {
printf("ERROR:- Connecting to sht31 I2C address 0x%02hhx\n", SHT31_DEFAULT_ADDR);
exit(1);
}
softreset(file);
printtempandhumidity(file);
printstatus(file);
close(file);
return 0;
}