How to build Perfect Hash in C++? - c++

Question

I'd like to know how to build a Perfect Hash in C++.
Perfect Hash is such a hash that 1) Has no collisions at all, 2) Is built only for fixed set of values, 3) Maps set of N values to a range of numbers of 0 .. N * 1.23 - 1, i.e. it maps not to numbers till N, but till some bigger multiple of N, like N * 1.23.
I've read this Wiki article about Perfect Hash.
And decided to post this short question only to send my own Answer.
So I don't provide any Minimal Reproducible Example, only because answer is fully contained.

Answer 1

Suppose we have set S of N integer elements. We want to perfect-hash this set.
There are different ways of building perfect hash. But one way, according to Wiki, is following way:
First we choose some function g(x) = k * x mod p mod n, where P is some quite large prime. And K is some random constant. N is number of elements in a set.
Then we map through g(x) all elements of a set, these elements map to some integers in a range 0..N-1 which may collide. Collided integers form separate buckets.
We create infinite amount of Hash functions. For example in my below C++ code I use same as g(x) functions equal to Hash[i](x) = RandomConstant[i] * x mod Prime[i] mod M, where M = N * 1.23, here 1.23 is some small constant, it can be something like 1.2-1.5.
Each bucket B_i is hashed separately in such a way that it forms set K_i = Hash[l](x) for x in B_i, so that l is minimal and |K_i| = |B_i|, and K_i doesn't intersect with any previous K_i. Each minimal found l is stored as sigma(i) = l.
sigma(i) is compressed into bit vector in such a way that we can get value l = sigma(i) in O(1) time. Using any compact bit packing technique like Gamma Coding or Unary Coding.
Finally, to get perfect hash of value x we do PerfectHash(x) = Hash[sigma(g(x))](x).
Full code below. It generates N random numbers, then perfect hashes them and finally outputs amount of bits per number.
Try it online!
#include <cstdint>
#include <bit>
#include <vector>
#include <random>
#include <cstring>
#include <stdexcept>
#include <string>
#include <iostream>
#include <iomanip>
#include <algorithm>
#include <unordered_map>
#include <array>
#define ASSERT_MSG(cond, msg) { if (!(cond)) throw std::runtime_error("Assertion (" #cond ") failed at line " + std::to_string(__LINE__) + "! Msg: '" + std::string(msg) + "'."); }
#define ASSERT(cond) ASSERT_MSG(cond, "")
#define DASSERT_MSG(cond, msg) ASSERT_MSG(cond, msg)
#define DASSERT(cond) DASSERT_MSG(cond, "")
using u8 = uint8_t;
using u32 = uint32_t;
using u64 = uint64_t;
template <typename T>
bool IsPrime(T const & n) {
if (n % 2 == 0)
return false;
for (size_t d = 3; d * d <= n; d += 2)
if (n % d == 0)
return false;
return true;
}
template <typename T>
T NextPrime(T const & i) {
if (i <= 2) return 2;
for (T j = i | 1;; j += 2)
if (IsPrime(j))
return j;
}
class BitVector {
static size_t constexpr index_block = 1 << 9;
public:
BitVector() {}
BitVector(size_t size) : size_(size), bits_((size_ + 7) / 8) {}
void Clear() {
size_ = 0;
bits_.clear();
index_.clear();
}
size_t Size() const { return size_; }
bool Get(size_t i) const {
return (bits_[i / 8] >> (i % 8)) & u8(1);
}
void Set(size_t i, bool val = true) {
if (val)
bits_[i / 8] |= u8(1) << (i % 8);
else
bits_[i / 8] &= ~(u8(1) << (i % 8));
}
void Push(bool val) {
++size_;
if (size_ - 1 >= bits_.size() * 8)
bits_.resize((size_ + 7) / 8);
Set(size_ - 1, val);
}
void Index() {
index_.clear();
for (size_t i = 0; i < size_; i += index_block) {
size_t sum = 0;
size_t const portion = std::min(index_block, size_ - i);
for (size_t k = i; k < i + portion; k += 64)
if (i + portion - k >= 64)
sum += std::popcount(*(u64*)&bits_[k / 8]);
else {
u64 x = 0;
std::memcpy(&x, &bits_[k / 8], bits_.size() - k / 8);
sum += std::popcount(x);
}
index_.push_back(index_.empty() ? sum : (index_.back() + sum));
}
}
size_t Select1(size_t idx) const {
size_t const d = std::distance(index_.data(), std::upper_bound(index_.data(), index_.data() + index_.size(), idx));
ASSERT_MSG(d < index_.size(), "idx " + std::to_string(idx));
size_t const prev_sum = d == 0 ? 0 : index_[d - 1], hi = std::min<size_t>(size_, index_block * (d + 1));
size_t csum = 0, i = 0;
u64 word = 0;
for (i = index_block * d; i < hi; i += 64) {
size_t const portion = std::min<size_t>(hi - i, 64);
size_t word_sum = 0;
if (portion == 64)
word = *(u64*)&bits_[i / 8];
else {
word = 0;
std::memcpy(&word, &bits_[i / 8], bits_.size() - i / 8);
}
word_sum = std::popcount(word);
if (prev_sum + csum + word_sum > idx)
break;
csum += word_sum;
}
size_t sum0 = 0;
while (true) {
size_t const i1 = std::countr_zero(word);
ASSERT(i1 < 64);
if (prev_sum + csum + sum0 >= idx) {
ASSERT(prev_sum + csum + sum0 == idx);
ASSERT(Get(i + i1));
return i + i1;
}
word &= word - 1;
++sum0;
}
}
std::string Dump() const {
std::string r;
for (size_t i = 0; i < size_; ++i)
r.append(1, Get(i) ? '1' : '0');
return r;
}
u64 Word(size_t i) const {
return (*(u64*)&bits_[i / 8]) >> (i % 8);
}
private:
size_t size_ = 0;
std::vector<u8> bits_;
std::vector<size_t> index_;
};
class GammaBitVector {
static size_t constexpr index_block = 1 << 7;
public:
template <typename T>
void GammaEncodeVec(std::vector<T> const & nums) {
for (auto n: nums) {
auto [x, b] = GammaEncode(std::max<size_t>(n, 1) - 1);
//std::cout << n << ": " << b << " " << x << std::endl;
for (size_t i = 0; i < b; ++i) {
bv_.Push(bool(x & 1));
x >>= 1;
}
}
//std::cout << "GammaEncodedVec " << bv_.Size() << std::endl;
//std::cout << bv_.Dump() << std::endl << std::endl;
}
void Index() {
size_t i = 0, cnt = 0;
while (i < bv_.Size()) {
auto const [n, ebits, dbits] = GammaDecode(bv_.Word(i));
++cnt;
i += ebits;
if (cnt < index_block && i < bv_.Size())
continue;
index_.push_back(i);
cnt = 0;
}
}
size_t Get(size_t i) const {
size_t j = i / index_block * index_block, sum = i / index_block > 0 ? index_.at(i / index_block - 1) : 0;
while (sum < bv_.Size()) {
auto const [n, ebits, dbits] = GammaDecode(bv_.Word(sum));
if (j >= i)
return n + 1;
++j;
sum += ebits;
}
ASSERT(false);
}
size_t Size() const { return bv_.Size(); }
size_t GetBitOffset(size_t i) const {
size_t j = i / index_block * index_block, sum = i / index_block > 0 ? index_.at(i / index_block - 1) : 0;
while (sum < bv_.Size()) {
auto const [n, ebits, dbits] = GammaDecode(bv_.Word(sum));
if (j >= i)
return sum;
++j;
sum += ebits;
}
ASSERT(false);
}
private:
static u64 Shl(u64 w, size_t cnt) {
return cnt >= 64 ? u64(0) : (w << cnt);
}
static u64 Shr(u64 w, size_t cnt) {
return cnt >= 64 ? u64(0) : (w >> cnt);
}
static u64 Mask(size_t n) {
return n >= 64 ? ~u64(0) : (u64(1) << n) - 1;
}
static size_t NumBits(u64 n) {
return 64 - std::countl_zero(n);
}
static std::tuple<u64, size_t> GammaEncode(u64 n) {
++n;
DASSERT(n != 0);
size_t const nbits = NumBits(n);
static auto lo = []{
std::array<u32, 32> r{};
for (size_t i = 0; i < r.size(); ++i)
r[i] = u32(1) << i;
return r;
}();
size_t const rnbits = nbits - 1;
DASSERT(rnbits < lo.size());
return std::make_tuple((Shl(n & Mask(rnbits), nbits) | u64(lo[rnbits])), rnbits + nbits);
}
static std::tuple<u64, size_t, size_t> GammaDecode(u64 n) {
static size_t constexpr c_tab_bits = 8;
static auto tab = []{
std::array<u8, (1 << c_tab_bits)> r{};
for (size_t i = 0; i < r.size(); ++i) {
size_t j = i, sr = 0;
if (i == 0)
sr = 0xFF;
else
while (!bool(j & 1)) {
++sr;
j >>= 1;
}
r[i] = u8(sr);
}
return r;
}();
size_t cnt = tab[n & Mask(c_tab_bits)];
if (cnt == 0xFF) {
ASSERT(n != 0);
cnt = 0;
u64 m = n;
while (!bool(m & 1)) {
++cnt;
m >>= 1;
}
ASSERT(cnt <= 31);
}
return std::make_tuple(u64((((n >> (cnt + 1)) & Mask(cnt)) | (u64(1) << cnt)) - 1), size_t(2 * cnt + 1), size_t(cnt + 1));
}
BitVector bv_;
std::vector<size_t> index_;
};
class PerfectHash {
public:
// https://en.wikipedia.org/wiki/Perfect_hash_function
void Build(std::vector<u64> const & nums) {
size_t const n = nums.size();
m_ = 1.5 * n;
n_ = n;
primes_.clear();
primes_.push_back({rng_(), NextPrime(n_)});
primes_.push_back({rng_(), NextPrime(m_)});
std::vector<std::vector<size_t>> Bs(n);
for (size_t i = 0; i < n; ++i) {
Bs[g(nums[i])].push_back(nums[i]);
//std::cout << "i " << i << ": " << nums[i] << ": " << g(nums[i]) << ", ";
}
//std::cout << std::endl;
std::vector<u64> K;
BitVector Tb(m_);
std::vector<u32> sigma_l(n);
size_t max_bucket_size = 0;
for (size_t i = 0; i < n; ++i) {
auto const & B = Bs.at(i);
max_bucket_size = std::max<size_t>(max_bucket_size, B.size());
if (B.empty())
continue;
size_t l = 0;
for (l = 1; l < 10'000; ++l) {
bool exists = false;
K.clear();
for (size_t iB = 0; iB < B.size(); ++iB) {
auto const j = B[iB];
auto const h = HashFunc(l, j);
if (Tb.Get(h)) {
exists = true;
break;
}
for (auto k: K)
if (k == h) {
exists = true;
break;
}
if (exists)
break;
K.push_back(h);
}
if (!exists)
break;
}
ASSERT(l < 10'000);
sigma_l[i] = l;
for (auto j: K)
Tb.Set(j);
}
sigma_bv_.Clear();
//std::cout << "MaxBucket " << max_bucket_size << std::endl;
//std::cout << "Sigma: ";
for (size_t i = 0; i < sigma_l.size(); ++i) {
auto const l = sigma_l[i];
//std::cout << l << ", ";
sigma_bv_.Push(1);
for (size_t i = 0; i + 1 < l; ++i)
sigma_bv_.Push(0);
}
//std::cout << std::endl;
sigma_gbv_.GammaEncodeVec(sigma_l);
sigma_gbv_.Index();
//std::cout << "Sigma from GBV: ";
for (size_t i = 0; i < sigma_l.size(); ++i) {
//std::cout << sigma_gbv_.Get(i) << ", ";
ASSERT_MSG(std::max<size_t>(1, sigma_l[i]) == sigma_gbv_.Get(i), "i " + std::to_string(i) + " sigma_l " + std::to_string(std::max<size_t>(1, sigma_l[i])) + " sigma_Get(i) " + std::to_string(sigma_gbv_.Get(i)) + " sigma_GetOff(i) " + std::to_string(sigma_gbv_.GetBitOffset(i)) + " sigma_GetOff(i - 1) " + std::to_string(sigma_gbv_.GetBitOffset(i - 1)));
}
//std::cout << std::endl;
sigma_bv_.Index();
for (size_t i = 0; i < sigma_bv_.Size(); ++i) {
//std::cout << (sigma_bv_.Get(i) ? "1" : "0");
}
//std::cout << std::endl;
}
size_t Hash(u64 const & x) {
return HashFunc(Sigma(g(x)), x);
}
size_t NumBits() const {
return sigma_gbv_.Size();
}
size_t HashFunc(size_t i, u64 const & x) {
while (i >= primes_.size())
primes_.push_back({rng_(), NextPrime(primes_.back().second + 1)});
auto const [k, p] = primes_[i];
auto v = (k * x) % p;
size_t const mod = i == 0 ? n_ : m_;
while (v >= mod)
v -= mod;
return v;
}
size_t g(u64 const & x) {
return HashFunc(0, x);
}
size_t Sigma(size_t i) {
size_t const i1 = sigma_gbv_.Get(i);
//std::cout << "Sigma: " << i << ": " << i1 << std::endl;
return i1;
/*
size_t cnt = 0;
for (size_t i = i1 + 1, size = sigma_bv_.Size(); i < size; ++i, ++cnt) {
std::cout << i << " (" << std::boolalpha << sigma_bv_.Get(i) << "), ";
if (sigma_bv_.Get(i))
break;
}
std::cout << std::endl << "Val: " << (cnt + 1) << std::endl;
return cnt + 1;
*/
}
size_t N() const { return n_; }
size_t M() const { return m_; }
private:
std::mt19937_64 rng_{123};
size_t n_ = 0, m_ = 0;
BitVector sigma_bv_;
GammaBitVector sigma_gbv_;
std::vector<std::pair<u64, u64>> primes_;
};
int main() {
try {
std::mt19937_64 rng{123};
std::vector<u64> nums(1 << 17);
for (size_t i = 0; i < nums.size(); ++i)
nums[i] = rng();
PerfectHash ph;
ph.Build(nums);
std::cout << "Nums " << nums.size() << std::endl;
std::cout << "PerfectHash Bits " << ph.NumBits() << ", " << std::setprecision(3)
<< (double(ph.NumBits()) / nums.size()) << " bits/num" << std::endl;
std::unordered_map<u64, u64> hashes;
for (size_t i = 0; i < nums.size(); ++i) {
//std::cout << "i " << i << std::endl;
auto const hash = ph.Hash(nums[i]);
if (i < 16) {
//std::cout << nums[i] << ": " << hash << std::endl;
}
ASSERT(hash < ph.M());
ASSERT_MSG(!hashes.count(hash),
"i " + std::to_string(i) + " nums[i] " + std::to_string(nums[i]) +
" hash " + std::to_string(hash) + " g(x) " + std::to_string(ph.g(nums[i])) +
" sigma " + std::to_string(ph.Sigma(ph.g(nums[i]))) +
" hash_func " + std::to_string(ph.HashFunc(ph.Sigma(ph.g(nums[i])), nums[i])) +
" prev_i " + std::to_string(hashes.at(hash)) + " nums[hashes.at(hash)] " +
std::to_string(nums[hashes.at(hash)]) + " prev_g(x) " +
std::to_string(ph.g(nums[hashes.at(hash)])) + " prev_sigma " +
std::to_string(ph.Sigma(ph.g(nums[hashes.at(hash)]))) + " prev_hash_func " +
std::to_string(ph.HashFunc(ph.Sigma(ph.g(nums[hashes.at(hash)])), nums[hashes.at(hash)]))
);
hashes[hash] = i;
}
ASSERT(hashes.size() == nums.size());
return 0;
} catch (std::exception const & ex) {
std::cout << "Exception: " << ex.what() << std::endl;
return -1;
}
}
Console Output:
Nums 131072
PerfectHash Bits 244430, 1.86 bits/num