Searching words in a very big file - c++

Question

Given a large file, we need to store the words so that searching of a word can be done in constant time. Also how will we find the 10% most frequently occurring words in the file?
What I have achieved so far is searching the word through trie implementation.
Please suggest some way to find the 10% most frequent words.
#include<iostream>
#include<cstdio>
using namespace std;
class Node
{
public:
char value;
Node* right;
Node* down;
Node()
{
right=down=NULL;
}
};
class Trie
{
public:
Node* head;
Trie()
{
head=NULL;
}
void insert(string s);
void search(string s);
};
void Trie::insert(string s)
{
if(head==NULL)
{
Node* f=new Node();
head=f;
Node* temp=f;
f->value=s[0];
for(int i=1;i<s.length();i++)
{
Node* n=new Node();
n->value=s[i];
temp->down=n;
temp=n;
if(i==s.length()-1)
n->down=NULL;
}
}
else
{
Node* ptr=head;
int i=0;
while(1)
{
if(i==s.length())break;
if(ptr->value==s[i])
{
i++;
if(ptr->down)
ptr=ptr->down;
else
{
Node* temp=new Node();
ptr->down=temp;
temp->value=s[i];
ptr=temp;
}
}
else if(ptr->value!=s[i])
{
if(ptr->right)
ptr=ptr->right;
else
{
Node*temp=new Node();
ptr->right=temp;
temp->value=s[i];
ptr=temp;
}
}
}
}
}
void Trie::search(string s)
{
Node* ptr=head;
int i=0;
while(1)
{
if(ptr->value==s[i])
{
//cout<<ptr->value<<endl;
ptr=ptr->down;
i++;
}
else if(ptr->value!=s[i])
{
ptr=ptr->right;
}
if(ptr==NULL)break;
}
if(i==s.length()+1)cout<<"String found\n";
else cout<<"String not found\n";
}
int main()
{
Trie t;
FILE* input;
char s[100];
input=fopen("big.txt","r");
int i=0;
while( (fgets(s,sizeof(s),input) ) !=NULL)
{
int i=0; int j=0;
char str[47];
while(s[i]!='\0')
{
if(s[i]==' ' || s[i+1]=='\0')
{
str[j]='\0';
j=0;
t.insert(str);
i++;
continue;
}
str[j]=s[i];
j++;
i++;
}
}
t.search("Dates");
//t.search("multinational");
fclose(input);
}

Answer 1

A hash will let you look up the words in constant time.
You could probably use some kind of partitioning like that used in quicksort to find a word that occurs at least 10% out of the file.

Answer 2

The obvious solution is to store the contents of the file in some appropriate STL container such as std::set and then run find() on that container.
If you insist on doing this manually, a binary tree will grow increasingly slower the more data you put inside it. Plus you must keep balancing it. A hash table with chaining would be a more efficient ADT for huge amounts of data.

Answer 3

If you use a tree you cannot get constant time. The binary tree you are building has a logarithmic time complexity.
If it is possible to build an index consider an inverted index. This would still not help you with the constant time (I don't see how you can achieve that anyway), but can help you figure what are the words used the most, since for every word it stores the positions in the file where the word is found. You can actually combine that into your tree.

Answer 4

Here is the similar c++ code using priority queue, map and trie.
for simplicity its reading from vector strings but can be easily modified to read words from file.
//Find the top K frequent words in a file or stream, C++
//This is a working solution for priority_queue for your reference.
#include <iostream>
#include <vector>
#include <queue>
#include <unordered_map>
using namespace std;
#define K_TH 3
class TrieNode;
typedef struct HeapNode
{
string word;
int frequency;
HeapNode(): frequency(0), word(""){} ;
TrieNode *trieNode;
}HeapNode;
class TrieNode
{
private:
int frequency = 0;
bool m_isLeaf = false;
string word = "";
unordered_map<char, TrieNode*> children;
HeapNode *heapNode = NULL;
public:
TrieNode() {}
TrieNode(char c)
{
children[c] = new TrieNode();
this->m_isLeaf = false;
}
void setWord(string word)
{
this->word = word;
}
string getWord()
{
return this->word;
}
bool isLeaf(void)
{
return this->m_isLeaf;
}
void setLeaf(bool leaf)
{
this->m_isLeaf = leaf;
}
TrieNode* getChild(char c)
{
if (children[c] != NULL)
return children[c];
return NULL;
}
void insert(char c)
{
children[c] = new TrieNode();
}
int getFrequency()
{
return this->frequency;
}
void setFrequency(int frequency)
{
this->frequency = frequency;
}
void setHeapNode(HeapNode *heapNode)
{
this->heapNode = heapNode;
}
HeapNode* getHeapNode()
{
return heapNode;
}
bool operator()(HeapNode* &a, HeapNode* &b)
{
return (a->frequency > b->frequency);
}
};
class Trie
{
private:
TrieNode *root = NULL;
public:
Trie()
{
if (!root)
{
this->root = new TrieNode();
}
}
TrieNode* insert(string word)
{
if (!root)
root = new TrieNode();
TrieNode* current = root;
int length = word.length();
//insert "abc"
for(int i = 0; i < length; ++i)
{
if (current->getChild(word.at(i)) == NULL)
{
current->insert(word.at(i));
}
current = current->getChild(word.at(i));
}
current->setLeaf(true);
current->setWord(word);
current->setFrequency(current->getFrequency() + 1);
return current;
}
};
struct cmp
{
bool operator()(HeapNode* &a, HeapNode* &b)
{
return (a->frequency > b->frequency);
}
};
typedef priority_queue<HeapNode*, vector<HeapNode*>, cmp > MinHeap;
void insertUtils(Trie *root, MinHeap &pq, string word )
{
if (!root)
return;
TrieNode* current = root->insert(word);
HeapNode *heapNode = current->getHeapNode();
if(heapNode)// if word already present in heap
{
heapNode->frequency += 1;
}else if (pq.empty() || pq.size() < K_TH)
{// if word not present in heap and heap is not full;
heapNode = new HeapNode();
heapNode->word = word;
heapNode->frequency = 1;
heapNode->trieNode = current;
current->setHeapNode(heapNode);
pq.push(heapNode);
}else if (pq.top()->frequency < current->getFrequency())
{ // if word is not present and heap is full;
HeapNode *temp = pq.top();
//remove first element and add current word
pq.pop();
delete temp;
heapNode = new HeapNode();
current->setHeapNode(heapNode);
pq.push(heapNode);
}
}
void printKMostFrequentWords(vector<std::string> input)
{
Trie *root = new Trie();
MinHeap minHeap;
for (vector<string>::iterator it = input.begin(); it != input.end(); ++it)
{
insertUtils(root, minHeap, *it);
}
while(!minHeap.empty())
{
HeapNode *heapNode = minHeap.top();
cout << heapNode->word << ":" << heapNode->frequency << endl;
minHeap.pop();
}
}
int main() {
vector<std::string>input( {
"abc", "def", "ghi",
"jkl", "abc", "def",
"mno", "xyz", "abc"
} ) ;
printKMostFrequentWords(input);
}