What is wrong in my RC6 implmentation? - c++

Question

can anyone see where i made a mistake here? I know that the algorithm will properly decrypt the encrypted data. however, most of the encrypted data is not the correct output, according to the RC6 paper.
// hexlify(string) turns a string into its hex representation: hexlify("AB") -> "4142"
// unhexlify(string) turns a string into its ASCII representation: unhexlify("4142") -> "AB"
// uint128_t is my own version of uint128, and Im pretty sure that the math is correct
// little_end(string, base) flips a string by bytes to get the little endian version of the string
// ROL/ROR(int, rotate x bits, bitsize of input int) does bitwise rotation
class RC6{
private:
unsigned int w, r, b, lgw;
std::vector <uint32_t> S;
uint128_t mod;
std::string mode;
void keygen(std::string KEY){
uint64_t p, q;
rc_pq(w, p, q);
KEY = hexlify(KEY);
unsigned int u = (unsigned int) ceil(w / 8.);
unsigned int c = (unsigned int) ceil(float(b) / u);
while ((KEY.size() >> 1) % u != 0)
KEY += zero;
std::vector <uint32_t> L;
for(unsigned int x = 0; x < c; x++)
L.push_back(toint(little_end(KEY.substr(2 * u * x, 2 * u), 16), 16));
S.push_back(p);
for(unsigned int i = 0; i < 2 * r + 3; i++)
S.push_back((S[i] + q) % mod);
uint32_t A = 0, B = 0, i = 0, j = 0;
uint32_t v = 3 * std::max(c, 2 * r + 4);
for(unsigned int s = 1; s < v + 1; s++){
A = S[i] = ROL((S[i] + A + B) % mod, 3, w);
B = L[j] = ROL((L[j] + A + B) % mod, (A + B) % w, w);
i = (i + 1) % (2 * r + 4);
j = (j + 1) % c;
}
}
public:
RC6(std::string KEY, std::string MODE, unsigned int W = 32, unsigned int R = 20, unsigned int B = 16){
w = W;
r = R;
b = B;
mod = uint128_t(1) << w;
lgw = (unsigned int) log2(w);
mode = MODE;
keygen(KEY);
}
std::string run(std::string DATA){
DATA = hexlify(DATA);
uint32_t A = toint(little_end(DATA.substr(0, 8), 16), 16), B = toint(little_end(DATA.substr(8, 8), 16), 16), C = toint(little_end(DATA.substr(16, 8), 16), 16), D = toint(little_end(DATA.substr(24, 8), 16), 16);
if (mode == "e"){
B += S[0];
D += S[1];
for(unsigned int i = 1; i < r + 1; i++){
uint64_t t = ROL((uint64_t) ((B * (2 * B + 1)) % mod), lgw, w);
uint64_t u = ROL((uint64_t) ((D * (2 * D + 1)) % mod), lgw, w);
A = ROL(A ^ t, u % w, w) + S[2 * i];
C = ROL(C ^ u, t % w, w) + S[2 * i + 1];
uint64_t temp = A; A = B % mod; B = C % mod; C = D % mod; D = temp % mod;
}
A += S[2 * r + 2];
C += S[2 * r + 3];
}
else{
C -= S[2 * r + 3];
A -= S[2 * r + 2];
for(int i = r; i > 0; i--){
uint64_t temp = D; D = C % mod; C = B % mod; B = A % mod; A = temp % mod;
uint64_t u = ROL((uint64_t) ((D * (2 * D + 1)) % mod), lgw, w);
uint64_t t = ROL((uint64_t) ((B * (2 * B + 1)) % mod), lgw, w);
C = ROR((C - S[2 * i + 1]) % mod, t % w, w) ^ u;
A = ROR((A - S[2 * i]) % mod, u % w, w) ^ t;
}
D -= S[1];
B -= S[0];
}
w >>= 2;
return unhexlify(little_end(makehex(A % mod, w)) + little_end(makehex(B % mod, w)) + little_end(makehex(C % mod, w)) + little_end(makehex(D % mod, w)));
}
};
of these tests vectors, only the first two are correct. the rest are not
data = "00000000000000000000000000000000";
key = "00000000000000000000000000000000";
ciphertext = "8fc3a53656b1f778c129df4e9848a41e";
data = "02132435465768798a9bacbdcedfe0f1";
key = "0123456789abcdef0112233445566778";
ciphertext = "524e192f4715c6231f51f6367ea43f18";
data = "00000000000000000000000000000000";
key = "000000000000000000000000000000000000000000000000";
ciphertext = "6cd61bcb190b30384e8a3f168690ae82";
data = "02132435465768798a9bacbdcedfe0f1";
key = "0123456789abcdef0112233445566778899aabbccddeeff0";
ciphertext = "688329d019e505041e52e92af95291d4";
data = "00000000000000000000000000000000";
key = "0000000000000000000000000000000000000000000000000000000000000000";
ciphertext = "8f5fbd0510d15fa893fa3fda6e857ec2";
data = "02132435465768798a9bacbdcedfe0f1";
key = "0123456789abcdef0112233445566778899aabbccddeeff01032547698badcfe";
ciphertext = "c8241816f0d7e48920ad16a1674e5d48";
did i mess up a uint somewhere? wrong little endian change?

Answer 1

I think I figured it out. Can anyone corroborate? I think that because I set b = 16 by default, I'm causing the errors. My harddrive is dead or I would have tested this already