I would like some help as I seem to be unable to calculate the CRC for a single byte, let alone an array of bytes.
Below is the test code that I have been trying to get work. Please note that I used the code from this post, and the example with the array of chars works, but I need to be able to adapt this to work for an array of uint8_t.
When going through the fundamentals of CRC, this seems like it should work but I may be mistaken.
I have been trying to validate the function by checking the outputted remainder (crc) using this site.
If someone could kindly identify where the issue is coming round which is leading to the wrong CRC value being calculated, and educate me as to why this is happening, that would be highly appreciated!
#include <cstdint>
#include <iostream>
// Below 2 functions take from
// https://stackoverflow.com/questions/59486262/issues-calculating-crc16-mcrf4xx-for-more-than-1-byte
int16_t Utils_CRC16_MCRF4XX( uint16_t Crc, uint8_t Byte )
{
Crc ^= Byte ;
for( uint8_t i = 0; i < 8; i++ )
{
Crc = (Crc & 0x0001) != 0 ? (Crc >> 1) ^ 0x8408 :
Crc >> 1 ;
}
return Crc ;
}
uint16_t crc( uint8_t* pData, uint32_t Len )
{
uint16_t Crc = 0xffff;
for( uint32_t i = 0; i < Len; i++ )
{
Crc = Utils_CRC16_MCRF4XX( Crc, pData[i] );
}
return (Crc);
}
int main()
{
int8_t val1 = 30;
int8_t val2 = 30;
// Arbitrary Test Message
uint8_t message[] =
{
(uint8_t)54
};
/*
// Sample Test Message (Actually what I intend to transmit)
uint8_t message[] =
{
(uint8_t)250,
(uint8_t)7,
(uint8_t)21,
(uint8_t)2,
(uint8_t)2,
(uint8_t)val1,
(uint8_t)val2
};
*/
//uint8_t test[] = "123456789";
uint16_t remainder = crc(message, sizeof(message));
// Expected DEC: 28561
// Actual DEC: 23346
std::cout << "CRC: " << remainder << std::endl;
return 0;
}
If you do this:
uint8_t test[] = "123456789";
uint16_t remainder = crc(test, sizeof(test)-1);
then you get the expected result, 28561.
You need to subtract 1 from sizeof(test) to exclude the terminating 0.
Related
I am trying to implement CRC32 calculation of file by splitting it in parts. I used algorithms and ideas from CRC Calculation Of A Mostly Static Data Stream.
Unfortunately, my program gives incorrect answer, although it returns the same value of CRC regardless of number of parts.
Please, tell me where the mistake is and what I do wrong.
Here is the code of program:
#include <iostream>
#include <fstream>
#include <stdint.h>
#include <string>
#include <sstream>
#include <stdlib.h>
#include <stdio.h>
#include <pthread.h>
using namespace std;
struct data {
pthread_t id;
uint8_t *buf;
long int start, end;
long int num_zeros;
uint32_t crc;
};
//Straight function
uint32_t crc_32(ifstream& input) {
input.seekg(0, input.end);
size_t size = input.tellg();
input.seekg(0, input.beg);
uint32_t polynomial = 0xEDB88320;
uint32_t table[256];
for(uint32_t i=0; i<=0xff; i++) {
uint32_t c = i;
for (size_t j = 0; j < 8; j++)
{
if (c & 1) {
c = polynomial ^ (c >> 1);
}
else {
c >>= 1;
}
}
table[i] = c;
}
uint32_t CRC = 0xffffffff;
uint8_t buf;
for(size_t i=0; i<size; i++) {
input.read( (char *) &buf, sizeof(buf));
CRC = (CRC>>8) ^ table[(CRC ^ buf) & 0xff ];
}
CRC ^= 0xffffffff;
return CRC;
}
// generate crc
uint32_t GenCrc(data *work, long int beg, long int end, uint32_t *crctbl) {
uint32_t init = 0x00000000;
for(long int i = beg; i<end; i++) {
init = (init<<8)^crctbl[ (init>>24) ^ work->buf[i] ];
}
return init;
}
// (a*b)%crc
uint32_t MpyModCrc(uint32_t a, uint32_t b) {
uint32_t pd = 0;
uint32_t i;
for(i = 0; i < 32; i++){
pd = (pd<<1)^((0-(pd>>31))&0x04c11db7);
pd ^= (0-(b>>31))&a;
b <<= 1;
}
return pd;
}
// pow(2,p)%crc
uint32_t PowModCrc(uint32_t p) {
uint32_t prd = 0x1u;
uint32_t sqr = 0x2u;
while(p) {
if(p&1)
prd = MpyModCrc(prd, sqr);
sqr = MpyModCrc(sqr, sqr);
p >>= 1;
}
return prd;
}
void do_work(data *work) {
//Generate lookup table:
uint32_t polynomial = 0x04c11db7;
uint32_t crctbl[256];
uint32_t crc;
uint32_t c;
uint32_t i;
for(c=0; c <256; c++) {
crc = c<<24;
/*
for(i=0; i <8; i++) {
if( (crc & 0x80000000) !=0) {
crc <<= 1;
crc ^= polynomial;
}
else {
crc <<=1;
}
}
*/
for(i=0; i<8; i++) {
crc = (crc<<1)^(0-(crc>>31))&polynomial;
}
crctbl[c] = crc;
}
uint32_t pmc;
uint32_t crf;
crf = GenCrc(work, work->start, work->end, crctbl);
if(work->num_zeros > 0) {
pmc = PowModCrc((work->num_zeros)*8);
crf = MpyModCrc(crf, pmc);
}
work->crc = crf;
}
void *do_stuff(void *d) {
data *mydata = (data*)d;
do_work(mydata);
return 0;
}
int main(int argc, char** argv) {
ifstream input("8733718.zip", ios::binary);
if(!input) {
cerr << "Can't open file." <<endl;
}
input.seekg(0, input.end);
long int len = input.tellg();
input.seekg(0, input.beg);
uint8_t *buf = new uint8_t[len];
input.read( (char *) buf, len);
int k;
cout << "Enter number of parts: ";
if(!(cin >> k) || k<1) {
cout << "Error. We need at least one part!" <<endl;
return -1;
}
data *work = new data[k+1];
for(int i=0; i < k; i++) {
work[i].start = len*i;
work[i].start /=k;
work[i].end = len * (i+1);
work[i].end /= k;
work[i].num_zeros = len - work[i].end;
work[i].buf = buf;
}
for(int i=0; i < k; i++) {
void *tmp = (void*)&work[i];
pthread_create(&work[i].id, 0, do_stuff, tmp);
}
for(int i=0; i<k; i++) {
pthread_join(work[i].id, 0);
}
uint32_t crc = work[0].crc;
for(int i=1; i<k; i++) {
crc ^= work[i].crc;
}
delete [] buf;
delete [] work;
cout << "Straigth CRC_32 = ";
uint32_t result;
result = crc_32(input);
cout << hex << result;
cout <<endl <<endl;
cout << "Parallel CRC_32 = ";
uint32_t result2;
result2 = crc;
cout << hex << crc <<endl <<endl;
cout <<endl <<endl;
cout <<"=========================="<<endl<<endl;
input.close();
return 0;
}
"Straight" function gives the answer which coincides with the answer of, for example, website https://emn178.github.io/online-tools/crc32_checksum.html.
But "parallel" procedure gives another answer.
As rcgldr noted, you are mixing up reflected and non-reflected CRC calculations. The original CRC is reflected so you need to stick with that. You need to always be shifting right. You always need to use the reflected polynomial, as in the original, 0xedb88320.
So, GenCRC() needs to shift right, and use the low eight bits of the CRC (which you're calling init) instead of the high eight bits to get the index.
MpyModCrc() needs to shift right, use the low bit instead of the high bit for the decisions, and use the correct polynomial.
PowModCrc() is starting off with the wrong initial polynomials. Since the polynomials are reflected, the initial values need to be as well. So 1 (x0) is 0x80000000, and x (x1) is 0x40000000.
In do_work(), you need to generate the table exactly as you did crc_32(). Of course, why you're generating the exact same table in every thread, I have no idea. Do that once.
Lastly, having done all that, your threads will be computing the CRC with a zero initial value and zero final exclusive or. That's ok, so long as you then exclusive-or with the CRC of len zero bytes with an initial value of 0xffffffff, and then exclusive-or the final result with 0xffffffff to get the same effect. That is:
crc ^= MpyModCrc(0xffffffff, PowModCrc(len * 8)) ^ 0xffffffff;
Alternatively, you could start the first work unit, and only that one, with an initial CRC of 0xffffffff. And then exclusive-or the final result with 0xffffffff.
Another improvement is to not calculate the CRC of so many zeros (even though it is an O(log n) calculation), and to only calculate the power function once. You can combine the CRCs as you join each thread, only needing PowModCrc() of your chunk size, calculating that just once, and applying it each time. And once more for the final chunk which may be smaller.
You don't need to read in the entire file and then do the CRC calculation. You should be calculating in parallel with reading. Instead of deciding how many pieces, decide on a fixed chunk size. Then read chunks and fire off threads for CRC calculations as you read them. Combine the CRCs as the threads complete, joining them in order. Limit the number of threads to something like twice the number of cores. You want to keep the cores busy, but you don't want the overhead of too many threads and too much memory in use.
The final alternative would be to not do any of this, and simply use zlib which provides the CRC combination functions for you.
crc_32() is a right shifting CRC that initial CRC = 0xffffffff and final CRC ^= 0xffffffff, while do_work() gencrc(), ... are using a left shifting CRC.
Change do_work() and gencrc() to be a modified version of crc_32() with initial CRC = 0, and final CRC not changed.
Changes for CRC32 (which is a reflected CRC):
#define CRCPOLY 0xEDB88320u
#define INITXOR 0xFFFFFFFFu
#define FINALXOR 0xFFFFFFFFu
static uint32_t crc_table[256];
void GenCrcTable()
{
uint32_t crc;
for (uint32_t byte = 0; byte <= 0xFFu; byte++ )
{
crc = byte;
for (uint8_t bit = 0; bit < 8; bit++ )
crc = (crc&1) ? (crc>>1)^CRCPOLY : (crc>>1);
crc_table[byte] = crc;
}
}
int Crc32(uint8_t *buffer, size_t length)
{
uint32_t crc = INITXOR;
for (size_t i = 0; i < length; ++i)
crc = crc_table[(crc&0xFFu)^*buffer++]^(crc>>8);
crc ^= FINALXOR;
return crc;
}
// use this one for the multi-thread code
int Crc320(uint8_t *buffer, size_t length)
{
uint32_t crc = 0;
for (size_t i = 0; i < length; ++i)
crc = crc_table[(crc&0xFFu)^*buffer++]^(crc>>8);
return crc;
}
// carryless multiply modulo crc
uint32_t MpyModCrc(uint32_t a, uint32_t b) // (a*b)%crc
{
uint32_t prd = 0;
uint32_t i;
for(i = 0; i < 32; i++){
prd = (prd&1u) ? (prd>>1)^CRCPOLY : (prd>>1);
prd ^= (b&1u) ? a : 0;
b >>= 1;
}
return prd;
}
// exponentiate by repeated squaring modulo crc
uint32_t PowModCrc(uint32_t p) // pow(2,p)%crc
{
uint32_t prd = 0x80000000u; // current product
uint32_t sqr = 0x40000000u; // current square
while(p){
if(p&1)
prd = MpyModCrc(prd, sqr);
sqr = MpyModCrc(sqr, sqr);
p >>= 1;
}
return prd;
}
After these changes try this code at the end (I haven't tested this yet):
uint32_t crc = work[0].crc;
for(int i=1; i<k; i++) {
crc ^= work[i].crc;
}
uint32_t pmc = PowModCrc(len*8); // add initial CRC of 0xffffffff
crc ^= MpyModCrc(0xffffffff, pmc);
crc ^= 0xffffffff; // final_xor = 0xffffffff
If running on a PC in 64 bit mode, you could use a PCLMULQDQ based CRC to speed things up (probably to the point that multi-threading won't help much). You can search github for examples of this. The assembly files are a bit over 500 lines. I have 6 sets of these for Visual Studio | MASM (ML64.EXE). Link to to code for 32 bit CRC reflected. If using my code, you need to change the if defines from 0 to 1 to use CRC32 instead of CRC32C.
https://github.com/jeffareid/crc/tree/master/crc32r
I am trying to implement CRC-7/MMC checksum with pregenerated lookup table. This is the code so far:
#include <iostream>
#include <string>
#include <cstdint>
using namespace std;
/* CRC-7/MMC
Poly: 0x09
Xorout: NO
Init: 0x00
Check value: 0x75 for "123456789"
*/
uint16_t CRC7_table[256];
void generate_crc7_table() {
uint16_t byte;
for (uint16_t i = 0; i < 256; i++) {
byte = i;
for (uint16_t bit = 0; bit < 8; bit++) {
if (byte & 1) { //if lsb is 1
byte >>= 1;
byte ^= 0x09;
}
else
byte >>= 1;
}
CRC7_table[i] = byte >> 1; //to drop off MSB
}
}
uint16_t crc7(string input) {
uint16_t reg = 0;
uint16_t b;
for (uint16_t i = 0; i < input.length(); i++) {
b = (input[i] ^ reg) & 0xFF;
reg = (reg >> 1) ^ CRC7_table[b];
}
return reg;
}
int main()
{
generate_crc7_table();
cout << hex << crc7("123456789") << endl;
return 0;
}
But it gives wrong output. I should get 0x75 but I get 0x07. I used this website to checkout the outputs.
Any suggestion or idea is highly appreciated. Thanks.
Note that the CRC definition you pointed to includes refin=false refout=false. That CRC is not reflected, so it is computed with left shifts, not right shifts.
Given that, and the fact that the CRC is less than eight bits in length, you will also want to keep the seven bits at the top of the byte being used for calculation, as opposed to the bottom. I.e. bits 1 to 7, as opposed to bits 0 to 6. The polynomial is then also shifted up by one for the table calculation.
This allows the table-driven, byte-wise calculation to simply exclusive-or each message byte into the byte being used for the calculations. If you want to return the CRC in the low seven bits, you can shift it down one at the end.
Example (0x12 is 0x09 shifted up one):
#include <iostream>
#include <string>
uint8_t table[256];
void make_table() {
uint8_t octet = 0;
do {
uint8_t crc = octet;
for (int k = 0; k < 8; k++)
crc = crc & 0x80 ? (crc << 1) ^ 0x12 : crc << 1;
table[octet++] = crc;
} while (octet);
}
uint8_t crc7(std::string const& input) {
uint8_t crc = 0;
for (auto octet : input)
crc = table[crc ^ octet];
return crc >> 1;
}
int main() {
make_table();
std::cout << std::hex << (unsigned)crc7("123456789") << '\n';
}
crcany will generate CRC code for you, given the definition. Since your CRC-7/MMC is in Greg's catalogue, crcany will generate that code out of the box, along with the other 100+ CRCs defined there. The generated code includes bit-wise, byte-wise, and word-wise calculations.
I am trying to code a function to match CRC 32 output from a device to the actual CRC-32 sum that I calculate. Following is my code:
#include <iostream>
#include <string.h>
#define CRC32_POLYNOMIAL 0xEDB88320
using namespace std;
unsigned int crc32b(unsigned char *message,size_t l)
{
int i, j;
unsigned int byte, crc, mask;
i = 0;
crc = 0xFFFFFFFF;
while (i<l) {
byte = message[i]; // Get next byte.
crc = crc ^ byte;
for (j = 7; j >= 0; j--) { // Do eight times.
mask = -(crc & 1);
crc = (crc >> 1) ^ (0xEDB88320 & mask);
}
i = i + 1;
}
return ~crc;
}
int main()
{
unsigned char Buff[] = {0x91,0xFF,0xFC,0xEA,0xFF,0xFF,0x70,0xFF,0xFD,0x87,0x00,0xFF,0xF9,0x1B,0xFF,0xF3,0x4E,0x00,0xFB,0x00,0x00,0x02,0x01,0xFB};
unsigned long CRC = crc32b((unsigned char *)Buff,24);
cout << hex << CRC <<endl;
getchar();
return 0;
}
This gives me the 32 bit CRC output of following payload:
91FFFCEAFFFF70FFFD8700FFF91BFFF34E00FB00000201FB
as 1980AC80. However the device is giving the checksum as 8059143D.
Upon further inspection using online CRC calculators I found that the device is sending out CRC-32/MPEG-2 checksum value. (Can be verified here). I have browsed multiple sites but did not find any straight forward implementation of CRC32/MPEG2 which I can integrate in my code. Can anyone help?
As noted in the crcalc web page, crc32/mpeg2 uses a left shifting (not reflected) CRC along with the CRC polynomial 0x104C11DB7 and initial CRC value of 0xFFFFFFFF, and not post complemented:
unsigned int crc32b(unsigned char *message, size_t l)
{
size_t i, j;
unsigned int crc, msb;
crc = 0xFFFFFFFF;
for(i = 0; i < l; i++) {
// xor next byte to upper bits of crc
crc ^= (((unsigned int)message[i])<<24);
for (j = 0; j < 8; j++) { // Do eight times.
msb = crc>>31;
crc <<= 1;
crc ^= (0 - msb) & 0x04C11DB7;
}
}
return crc; // don't complement crc on output
}
Normally one would add a CRC to the end of the data stream. The CRC check would include the CRC itself and return 0 if the CRC is correct.
I need to add a CRC to verify my embedded code. It needs to be checked in place, but the top word in memory space is for an interrupt vector. Is it possible to place a key value midstream such that the CRC check returns 0 for the whole code? (or is this unsolvable?)
It's definitely possible. You can run a CRC backwards, which would be fast and easy. Below is example code.
In fact, you can give me the locations of bits scattered wherever in the stream, and if you give me enough of them I can tell you what to set them to to get a zero CRC at the end, or any other CRC value for that matter. My spoof code solves the linear equations to come up with that answer.
However I would wonder why you'd want to do any of that. Why not just know where the CRC is stored and compute the CRC for everything but that, and then check the result against the stored CRC?
// Example of the generation of a "middle" CRC, which is inserted somewhere in
// the middle of a sequence, where the CRC is generated such that the CRC of
// the complete sequence will be zero. This particular CRC has no pre or post
// processing.
//
// Placed into the public domain by Mark Adler, 11 May 2016.
#include <stddef.h> // for size_t
#include <stdint.h> // for uint32_t and uint64_t
#define POLY 0xedb88320 // CRC polynomial
// Byte-wise CRC tables for forward and reverse calculations.
uint32_t crc_forward_table[256];
uint32_t crc_reverse_table[256];
// Fill in CRC tables using bit-wise calculations.
void crc32_make_tables(void) {
for (uint32_t n = 0; n < 256; n++) {
uint32_t crc = n;
for (int k = 0; k < 8; k++)
crc = crc & 1 ? (crc >> 1) ^ POLY : crc >> 1;
crc_forward_table[n] = crc;
crc_reverse_table[crc >> 24] = (crc << 8) ^ n;
}
}
// Return the forward CRC of buf[0..len-1], starting with crc at the front.
uint32_t crc32(uint32_t crc, unsigned char *buf, size_t len) {
for (size_t n = 0; n < len; n++)
crc = (crc >> 8) ^ crc_forward_table[(crc ^ buf[n]) & 0xff];
return crc;
}
// Return the reverse CRC of buf[0..len-1], starting with crc at the end.
uint32_t crc32_reverse(uint32_t crc, unsigned char *buf, size_t len) {
while (len)
crc = (crc << 8) ^ crc_reverse_table[crc >> 24] ^ buf[--len];
return crc;
}
// Put a 32-bit value into a byte buffer in little-endian order.
void put4(uint32_t word, unsigned char *pos) {
pos[0] = word;
pos[1] = word >> 8;
pos[2] = word >> 16;
pos[3] = word >> 24;
}
#include <stdlib.h> // for random() and srandomdev()
// Fill dat[0..len-1] with uniformly random byte values. All of the bits from
// each random() call are used, except for possibly a few leftover at the end.
void ranfill(unsigned char *dat, size_t len) {
uint64_t ran = 1;
while (len) {
if (ran < 0x100)
ran = (ran << 31) + random();
*dat++ = ran;
ran >>= 8;
len--;
}
}
#include <stdio.h> // for printf()
#define LEN 1024 // length of the message without the CRC
// Demonstrate the generation of a middle-CRC, using the forward and reverse
// CRC computations. Verify that the CRC of the resulting sequence is zero.
int main(void) {
crc32_make_tables();
srandomdev();
unsigned char dat[LEN+4];
ranfill(dat, LEN/2);
put4(0, dat + LEN/2); // put zeros where the CRC will go
ranfill(dat + LEN/2 + 4, (LEN+1)/2);
put4(crc32(0, dat, LEN/2) ^ crc32_reverse(0, dat + LEN/2, (LEN+1)/2 + 4),
dat + LEN/2); // replace the zeros with the CRC
printf("%08x\n", crc32(0, dat, LEN+4));
return 0;
}
I am trying to implement the algorithm of a CRC check, which basically created a value, based on an input message.
So, consider I have a hex message 3F214365876616AB15387D5D59, and I want to obtain the CRC24Q value of the message.
The algorithm that I found to do this is the following:
typedef unsigned long crc24;
crc24 crc_check(unsigned char *input) {
unsigned char *octets;
crc24 crc = 0xb704ce; // CRC24_INIT;
int i;
int len = strlen(input);
octets = input;
while (len--) {
crc ^= ((*octets++) << 16);
for (i = 0; i < 8; i++) {
crc <<= 1;
if (crc & 0x1000000)
crc ^= CRC24_POLY;
}
}
return crc & 0xFFFFFF;
}
where *input=3F214365876616AB15387D5D59.
The problem is that ((*octets++) << 16) will shift by 16 bits the ascii value of the hex character and not the character itself.
So, I made a function to convert the hex numbers to characters.
I know the implementation looks weird, and I wouldn't be surprised if it were wrong.
This is the convert function:
char* convert(unsigned char* message) {
unsigned char* input;
input = message;
int p;
char *xxxx[20];
xxxx[0]="";
for (p = 0; p < length(message) - 1; p = p + 2) {
char* pp[20];
pp[0] = input[0];
char *c[20];
*input++;
c[0]= input[0];
*input++;
strcat(pp,c);
char cc;
char tt[2];
cc = (char ) strtol(pp, &pp, 16);
tt[0]=cc;
strcat(xxxx,tt);
}
return xxxx;
}
SO:
unsigned char *msg_hex="3F214365876616AB15387D5D59";
crc_sum = crc_check(convert((msg_hex)));
printf("CRC-sum: %x\n", crc_sum);
Thank you very much for any suggestions.
Shouldn't the if (crc & 0x8000000) be if (crc & 0x1000000) otherwise you're testing the 28th bit not the 25th for 24-bit overflow