how do i read a csvfile faster in c++ - c++

Question

This is my current code:
std::vector <OrderBookEntry> entries;
std::ifstream csvFile{ csvFilename };
std::string line;
if (csvFile.is_open()) {
while (std::getline(csvFile, line)) {
try {
std::stringstream ss(line);
OrderBookEntry obe = stringsToOBE(tokenise(line, ','));
entries.push_back(obe);
}
catch (const std::exception& e) {
std::cout << "CSVReader::readCSV bad data" << std::endl;
}
}//end of while loop
}
std::cout << "CSVReader::readCSV read " << entries.size() << " entries" << std::endl;
return entries;
}
OrderBookEntry::OrderBookEntry(double _price,
double _amount,
std::string _timestamp,
std::string _product,
OrderBookType _orderType,
std::string _username)
: price(_price),
amount(_amount),
timestamp(_timestamp),
product(_product),
orderType(_orderType),
username(_username)
{
}
example of csv file:
Hi all, I have a csvfile (1 mill+) rows and i am trying to read and store the entries faster. Currently it takes a bit of time before my program is able to read the data and run.
How do i read the csvFile quicker? Thanks!

Answer 1

Difficult to answer.
Please see below a solution that can create a test file with 1 million records.
I then make only the input buffer bigger, without any further optimization.
Then, all lines of the test file are read and parsed in 2.6seconds.
The question is now: Is this fast or is this slow? Please comment.
#include <iostream>
#include <fstream>
#include <string>
#include <unordered_map>
#include <iomanip>
#include <array>
#include <random>
#include <vector>
#include <chrono>
#include <algorithm>
#include <iterator>
const std::array<std::string, 3> OTString{ "bid","offer","trade" };
enum class OrderType : int { bid, offer, trade, invalid };
const std::unordered_map<std::string, OrderType> OrderTypeMap{ {OTString[0],OrderType::bid}, {OTString[1],OrderType::offer}, {OTString[2],OrderType::trade} };
OrderType getOrderType(std::string& ot) {
if (const auto it = OrderTypeMap.find(ot); it != OrderTypeMap.cend())
return it->second;
else
return OrderType::invalid;
}
const std::string testFileName{ "test.txt" };
void createTestFile() {
unsigned ms{};
unsigned second{};
unsigned minute{};
std::random_device rd; // Will be used to obtain a seed for the random number engine
std::mt19937 gen(rd()); // Standard mersenne_twister_engine seeded with rd()
std::uniform_real_distribution<> dis1(0.01, 1);
std::uniform_real_distribution<> dis2(1.0, 1000.0);
std::string product{"AAAAAAAAAAAA"};
if (std::ofstream ofs{ testFileName }; ofs) {
ofs << std::setfill('0') << std::right;
for (std::size_t row{}; row < 1'000'000; ++row) {
ofs << std::setw(2) << minute << ':' << std::setw(2) << second << ':' << ms << ','
<< product << ',' << OTString[row%3] << ',' << dis1(gen) << ',' << dis2(gen) << '\n';
++product[product.length() - 1];
for (std::size_t i{ product.length() - 1 }; i > 0; --i)
if (product[i] > 'Z') {
product[i] = 'A';
++product[i - 1];
}
else
break;
if (row % 100 == 0) {
++ms;
if (ms > 9) {
ms = 0;
++second;
if (second > 59) {
second = 0;
++minute;
}
}
}
}
}
else
std::cerr << "\n\n*** Error: Could not open file '" << testFileName << "' for writing.\n\n";
}
// For measuring durations
struct Timer {
std::chrono::time_point<std::chrono::high_resolution_clock> startTime{};
long long elapsedTime{};
void start() { startTime = std::chrono::high_resolution_clock::now(); }
void stop() { elapsedTime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - startTime).count(); }
friend std::ostream& operator << (std::ostream& os, const Timer& t) { return os << t.elapsedTime << " ms "; }
};
constexpr std::size_t IOBufSize = 5'000'000;
char ioBuf[IOBufSize];
struct OrderBookEntry {
std::string timestamp{};
std::string product{};
OrderType orderType{};
double amount{};
double price{};
friend std::istream& operator >> (std::istream& is, OrderBookEntry& obe) {
std::string ot{}; char c;
std::getline(is >> std::ws, obe.timestamp, ',');
std::getline(is >> std::ws, obe.product, ',');
std::getline(is >> std::ws, ot, ',');
is >> obe.amount >> c >> obe.price;
obe.orderType = getOrderType(ot);
return is;
}
};
int main() {
//createTestFile();
std::vector<OrderBookEntry> obe{};
obe.reserve(1'500'000);
if (std::ifstream ifs{ testFileName }; ifs) {
// To speed up reading of the file, we will set a bigger input buffer
ifs.rdbuf()->pubsetbuf(ioBuf, IOBufSize);
Timer t; t.start();
std::copy(std::istream_iterator<OrderBookEntry>(ifs), {}, std::back_inserter(obe));
std::cout << "\nRecords read: "<< obe.size() << '\n';
t.stop();
std::cout << t << '\n';
}
else
std::cerr << "\n\n*** Error: Could not open file '" << testFileName << "' for reading.\n\n";
}