Convert byte array to time_t - c++

I have a linux time clock, that sending me the clock with bytes, I have no idea how to get the clock out of it, I do know that it's a type of "time_t" (which is probably the total seconds since 1/1/1970), See below a few examples what I'm receiving from the clock
(I'm not sure if it's sending local or GMT time)
This bytes represents 11/09/2017 16:52(Local time) or 11/09/2017 21:52(GMT)
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,129,66,120,53,34,0,0,0
This bytes represents 11/10/2017 10:27(Local time) or 11/10/2017 15:27(GMT)
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,129,162,111,54,34,0,0,0
This bytes represents 11/13/2017 14:20(Local time) or 11/13/2017 19:20(GMT)
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,129,160,154,58,34,0,0,0

With a little reverse engineering, it's possible to see what the data from your Linux clock is doing.
If you read your lines of data, you can see that the final 7 bytes are an incrementing counter, with the least significant byte in the first position. If you do the maths, the 3 counters give the following totals:
573929538
573992866
574266016
Looking at the differences from the first number to the second and third, you get the following:
63328
336478
... which are the elapsed times between your samples, in seconds.
For some reason, the raw values aren't the time in seconds since the Unix Epoch, but the time in seconds since a day early in September 1999.
Anyway, the following source code shows the basics of how you can convert your data into the local time.
#include <time.h>
int main(int argc, char* argv[])
{
const unsigned long localEpoch = 936316800; // local epoch Sept 1999
std::vector<std::vector<int>> timeData =
{
{ 66, 120, 53, 34, 0, 0, 0 },
{ 162, 111, 54, 34, 0, 0, 0 },
{ 160, 154, 58, 34, 0, 0, 0 }
};
for (auto& data : timeData)
{
// convert to total seconds
unsigned long total = 0;
for (std::vector<int>::iterator it = data.begin(); it != data.end(); ++it)
{
total += (*it) << (std::distance(data.begin(), it) * 8);
}
std::cout << "Seconds: " << total << std::endl;
// add to local epoch and display
time_t raw = total + localEpoch;
struct tm timeinfo;
localtime_s(&timeinfo, &raw);
char fmt[100] = { 0 };
asctime_s(fmt, 100, &timeinfo);
std::cout << fmt << std::endl;
}
return 0;
}
And it gives the following outputs:
Seconds: 573929538
Thu Nov 9 16:52:18 2017
Seconds: 573992866
Fri Nov 10 10:27:46 2017
Seconds: 574266016
Mon Nov 13 14:20:16 2017
Hope this is of some help. (NB I did this in VS, so the time formatting functions are of the Windows variety, but you get the idea.)
(Edit): OK as requested, here's some equivalent C# code:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace ConsoleApp2
{
class Program
{
public static void Main(string[] args)
{
int[,] timeData = new int[,]
{
{ 66, 120, 53, 34, 0, 0, 0 },
{ 162, 111, 54, 34, 0, 0, 0 },
{ 160, 154, 58, 34, 0, 0, 0 }
};
for (int line = 0; line < timeData.GetLength(0); ++line)
{
ulong total = 0;
for (int i = 0; i < timeData.GetLength(1); ++i)
{
total += (ulong)timeData[line,i] << (8 * i);
}
System.Console.WriteLine("Seconds: " + total.ToString());
DateTime result = FromUnixTime(total + localEpoch);
System.Console.WriteLine(result.ToString());
}
System.Console.ReadKey();
}
public static DateTime FromUnixTime(ulong unixTime)
{
return epoch.AddSeconds(unixTime);
}
private static readonly DateTime epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
private static readonly ulong localEpoch = 936316800; // local epoch Sept 1999
}
}
(Thanks to this question for the Epoch conversion.)

WOW!!!, that helped me a lot, Here is the code what I used, Thanks #ASMackay
private DateTime convertTime(byte[] bytes)
{
ulong localEpoch = 936316800; // local epoch Sept 1999
DateTime epoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
ulong total = 0;
for (int i = 0; i < bytes.Count(); ++i)
{
total += (ulong)bytes[i] << (8 * i);
}
return epoch.AddSeconds(total + localEpoch);
}

Related

Why is the xTimerPeriodInTicks affecting the code's runtime?

So for a report I'm trying to time the number of cycles it takes to execute a couple seperate functions in a program running on an ESP32, using freeRTOS. The code looks something like this with init_input_generator() being the first function called, here i'm timing onesecwindow.fifo_iir_filter_datapoint():
float geninputarray[SAMPLING_RATE]; //stores the values of an artificial signal we can use to test the algorithm
fifo_buffer onesecwindow = fifo_buffer(SAMPLING_RATE);
esp_err_t init_input_generator() {
dsps_tone_gen_f32(geninputarray, SAMPLING_RATE, 4, 0.04, 0); //10Hz
TimerHandle_t input_signal_timer =
xTimerCreate("input_timer", pdMS_TO_TICKS(10), pdTRUE, (void *)0, input_generator_callback);
xTimerStart(input_signal_timer, 0);
return ESP_OK;
}
void input_generator_callback(TimerHandle_t xTimer)
{
xTaskCreatePinnedToCore(
input_counter,
"input_generator",
4096,
NULL,
tskIDLE_PRIORITY + 11,
NULL,
PRO_CPU_NUM);
}
void input_counter(TimerHandle_t xTimer)
{
stepcounter = (stepcounter + 1) % SAMPLING_RATE;
insert_new_data(geninputarray[stepcounter]);
vTaskDelete(nullptr); // to gracefully end the task as returning is not allowed
}
esp_err_t insert_new_data(float datapoint)
{
onesecwindow.fifo_write(datapoint); // writes the new datapoint into the onesec window
unsigned int start_time = dsp_get_cpu_cycle_count();
onesecwindow.fifo_iir_filter_datapoint();
unsigned int end_time = dsp_get_cpu_cycle_count();
printf("%i\n", (end_time-start_time));
}
The thing that i'm noticing is that whenever I change the xTimerPeriodInTicks to a larger number, I get significantly longer time results, which just really confuses me and get's in the way of proper timing. Although the function doesn't scale with the SAMPLING_RATE and thus should give quite consistent results for each loop, output in cycles with a 10ms timer period looks something like this (every 5 or 6 "loops" for some reason it's longer):
1567, 630, 607, 624, 591, 619, 649, 1606, 607
with a 40ms timer period I get this as a typical output:
1904, 600, 1894, 616, 1928, 1928, 607, 1897, 628
As such I'm confused by the output, as they use the same freeRTOS taskcreate to run with the same priority on the same core, I don't see why there would be any difference. Perhaps I'm misunderstanding something basic/fundamental here so any help would be greatly appreciated.
Update
So based on the comments by #Tarmo i have restructured to approach to having a recurring task, unfortunately the output still seems to suffer from the same problem. The code now looks like this:
#include <esp_dsp.h> //Official ESP-DSP library
float geninputarray[SAMPLING_RATE]; //stores the values of an artificial signal we can use to test the algorithm
fifo_buffer onesecwindow = fifo_buffer(SAMPLING_RATE);
esp_err_t init_input_generator() {
dsps_tone_gen_f32(geninputarray, SAMPLING_RATE, 4, 0.04, 0); //10Hz
xTaskCreatePinnedToCore(
input_counter, // the actual function to be called
"input_generator",
4096,
NULL,
tskIDLE_PRIORITY + 5,
NULL,
APP_CPU_NUM);
return ESP_OK;
}
void input_counter(TimerHandle_t xTimer)
{
while (true)
{
stepcounter = (stepcounter + 1) % SAMPLING_RATE;
insert_new_data(geninputarray[stepcounter]);
vTaskDelay(4);
}
}
esp_err_t insert_new_data(float datapoint)
{
onesecwindow.fifo_write(datapoint); // writes the new datapoint into the onesec window
unsigned int start_time = dsp_get_cpu_cycle_count();
onesecwindow.fifo_iir_filter_datapoint();
unsigned int end_time = dsp_get_cpu_cycle_count();
printf("%i\n", (end_time-start_time));
}

About cryptocurrency on memory pool, Web wallet and Confirmations (LItecoin)

Note : If you don't known the bitcoin/Litecoin Source program, Then This Question is not for you. Please try not to Downgrade
I have created a alternative coin by using the fork of litecoin, and succeeded, But stil there exist some doubts about it. I googled for solution but can't able to find. If someone know the Answers or link please suggest
1. How can i reduce the Default Confirmation '101' for the coins to mature ? Instead that i need only 50 Confirmation.
2. Does the memory pool only works for the raw transaction between to address ? why not for mining transactions and whats its use ?
3. Is the Web wallets like LiteVault, Blockchain.info are working with the property of wallet accounts ? instead of creating each Wallet.dat file ?
Am working on the Litecoin v0.15
and the main class file in chainparam.cpp
class CMainParams : public CChainParams {
public:
CMainParams() {
strNetworkID = "main";
consensus.nSubsidyHalvingInterval = 840000;
consensus.BIP34Height = 710000;
consensus.BIP34Hash = uint256S("fa09d204a83a768ed5a7c8d441fa62f2043abf420cff1226c7b4329aeb9d51cf");
consensus.BIP65Height = 918684; // bab3041e8977e0dc3eeff63fe707b92bde1dd449d8efafb248c27c8264cc311a
consensus.BIP66Height = 811879; // 7aceee012833fa8952f8835d8b1b3ae233cd6ab08fdb27a771d2bd7bdc491894
consensus.powLimit = uint256S("00000fffffffffffffffffffffffffffffffffffffffffffffffffffffffffff");
consensus.nPowTargetTimespan = 3.5 * 24 * 60 * 60; // 3.5 days
consensus.nPowTargetSpacing = 2.5 * 60;
consensus.fPowAllowMinDifficultyBlocks = false;
consensus.fPowNoRetargeting = false;
consensus.nRuleChangeActivationThreshold = 6048; // 75% of 8064
consensus.nMinerConfirmationWindow = 8064; // nPowTargetTimespan / nPowTargetSpacing * 4
consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].bit = 28;
consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nStartTime = 1199145601; // January 1, 2008
consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout = 1230767999; // December 31, 2008
// Deployment of BIP68, BIP112, and BIP113.
consensus.vDeployments[Consensus::DEPLOYMENT_CSV].bit = 0;
consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nStartTime = 1485561600; // January 28, 2017
consensus.vDeployments[Consensus::DEPLOYMENT_CSV].nTimeout = 1517356801; // January 31st, 2018
// Deployment of SegWit (BIP141, BIP143, and BIP147)
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].bit = 1;
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nStartTime = 1485561600; // January 28, 2017
consensus.vDeployments[Consensus::DEPLOYMENT_SEGWIT].nTimeout = 1517356801; // January 31st, 2018
// The best chain should have at least this much work.
consensus.nMinimumChainWork = uint256S("0x00000000000000000000000000000000000000000000002ebcfe2dd9eff82666");
// By default assume that the signatures in ancestors of this block are valid.
consensus.defaultAssumeValid = uint256S("0x59c9b9d3fec105bdc716d84caa7579503d5b05b73618d0bf2d5fa639f780a011"); //1353397
pchMessageStart[0] = 0xfb;
pchMessageStart[1] = 0xc0;
pchMessageStart[2] = 0xb6;
pchMessageStart[3] = 0xdb;
nDefaultPort = 9333;
nPruneAfterHeight = 100000;
genesis = CreateGenesisBlock(1317972665, 2084524493, 0x1e0ffff0, 1, 50 * COIN);
consensus.hashGenesisBlock = genesis.GetHash();
assert(consensus.hashGenesisBlock == uint256S("0x12a765e31ffd4059bada1e25190f6e98c99d9714d334efa41a195a7e7e04bfe2"));
assert(genesis.hashMerkleRoot == uint256S("0x97ddfbbae6be97fd6cdf3e7ca13232a3afff2353e29badfab7f73011edd4ced9"));
// Note that of those with the service bits flag, most only support a subset of possible options
vSeeds.emplace_back("seed-a.altcoin.loshan.co.uk", true);
base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,48);
base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,5);
base58Prefixes[SCRIPT_ADDRESS2] = std::vector<unsigned char>(1,50);
base58Prefixes[SECRET_KEY] = std::vector<unsigned char>(1,176);
base58Prefixes[EXT_PUBLIC_KEY] = {0x04, 0x88, 0xB2, 0x1E};
base58Prefixes[EXT_SECRET_KEY] = {0x04, 0x88, 0xAD, 0xE4};
vFixedSeeds = std::vector<SeedSpec6>(pnSeed6_main, pnSeed6_main + ARRAYLEN(pnSeed6_main));
fDefaultConsistencyChecks = false;
fRequireStandard = true;
fMineBlocksOnDemand = false;
checkpointData = (CCheckpointData) {
{
{ 1500, uint256S("0x841a2965955dd288cfa707a755d05a54e45f8bd476835ec9af4402a2b59a2967")}
}
};
chainTxData = ChainTxData{
// Data as of block 59c9b9d3fec105bdc716d84caa7579503d5b05b73618d0bf2d5fa639f780a011 (height 1353397).
1516406833, // * UNIX timestamp of last known number of transactions
19831879, // * total number of transactions between genesis and that timestamp
// (the tx=... number in the SetBestChain debug.log lines)
0.06 // * estimated number of transactions per second after that timestamp
};
}
};
Thank you in advance

Converting time string into epoch time on windows

I have an interface where I'm getting a file and the file name is the time stamp when the internal data was valid. I'm processing the files in order and providing them to another API which takes the time as milliseconds since Jan 1st, 1970.
So, I get the file name in as a string in the format of YYYYMMDD_HHmmSS.sss and have to turn this into time. I assume the processing and collection occurred in the same timezone and such and simply have to convert the characters to digits.
uint64_t foo::convertFileNameToTimestamp(const std::string& filename) const
{
const uint64_t yyyy = atoi(filename.substr(0,4).c_str());
const uint64_t MM = atoi(filename.substr(4,2).c_str());
const uint64_t DD = atoi(filename.substr(6,2).c_str());
const uint64_t HH = atoi(filename.substr(9,2).c_str());
const uint64_t MI = atoi(filename.substr(11,2).c_str());
const uint64_t SS = atoi(filename.substr(13,2).c_str());
const uint64_t sss = atoi(filename.substr(16,3).c_str());
// number of milliseconds in a day
const uint64_t DAY = 24 * 60 * 60 * 1000;
int MD = 0;
if (MM == 2) MD = 31; // currently feb, so add all of january's days
else if (MM == 3) MD = 31+28; // ...
else if (MM == 4) MD = 31+28+31;
else if (MM == 5) MD = 31+28+31+30;
else if (MM == 6) MD = 31+28+31+30+31;
else if (MM == 7) MD = 31+28+31+30+31+30;
else if (MM == 8) MD = 31+28+31+30+31+30+31;
else if (MM == 9) MD = 31+28+31+30+31+30+31+31;
else if (MM == 10) MD = 31+28+31+30+31+30+31+31+30;
else if (MM == 11) MD = 31+28+31+30+31+30+31+31+30+31;
else if (MM == 12) MD = 31+28+31+30+31+30+31+31+30+31+30;
// year 2000 wasn't a leap year
uint64_t YLD = ((yyyy-1970) / 4);
if (yyyy > 2000) --YLD;
uint64_t temp = sss;
temp += SS * 1000;
temp += MI * 60 * 1000;
temp += HH * 60 * 60 * 1000;
temp += (DD-1) * DAY;
temp += (MD) * DAY;
temp += (yyyy-1970) * 365 * DAY + YLD*DAY;
return temp;
}
Obviously, reinventing the wheel here. Seems like there should be some sort of function for this. Also.. how do I account for leap seconds? It was annoying enough to deal with leap days. The time stamps are all from 2015 and beyond and always will be, but I don't think I can just blindly add 26 seconds. Eventually we will have 27 or back up to 25. In previous functions I have validated the string to be the right format and the file to exist and all that jazz. I'm running on windows 8.1 compilation for 64 bit using VS 2010.
I've looked at Convert Epoch Time string to Time which suggested ctime(), but it doesn't seem to deal with milliseconds in the constructor or even any of the get methods and it doesn't accept generically formatted string inputs. I'm assuming I've got to call some time classes CTOR which will accept the file name string and then call some accessor on it to get the millisecond time since 1970 including leap seconds and such.
I am not using boost and don't have access/permission to use it.
Here is an answer that will work on any platform that supports C++11 or C++14. It builds off of the std::chrono library that was introduced in C++11. It also uses a free, open source, cross platform library to simplify the arithmetic (MIT license which is usually considered lawyer-friendly).
If you don't need to take leap seconds into account, you can use this date library, and it will look like this:
#include <string>
#include "date.h"
using time_stamp = std::chrono::time_point<std::chrono::system_clock,
std::chrono::milliseconds>;
time_stamp
convertFileNameToTimestamp(const std::string& filename)
{
using namespace std::chrono;
using namespace date;
const uint64_t yyyy = atoi(filename.substr(0,4).c_str());
const uint64_t MM = atoi(filename.substr(4,2).c_str());
const uint64_t DD = atoi(filename.substr(6,2).c_str());
const uint64_t HH = atoi(filename.substr(9,2).c_str());
const uint64_t MI = atoi(filename.substr(11,2).c_str());
const uint64_t SS = atoi(filename.substr(13,2).c_str());
const uint64_t sss = atoi(filename.substr(16,3).c_str());
return sys_days{year(yyyy)/MM/DD}
+ hours{HH} + minutes{MI} + seconds{SS} + milliseconds{sss};
}
After parsing the numbers out of the filename, it is very simple to create a type-safe std::chrono::time_point which simply holds an integral number of milliseconds since 1970-01-01 (as an int64_t).
If you want to take leap seconds into account, you need this higher-level library which is a complete parser of the IANA timezone database. You will also need to keep an updated copy of the IANA timezone database downloaded for my timezone/leap-second library to parse. But once set up, the source code for your converter is very similar to that above, and nearly as simple:
#include <string>
#include "tz.h"
using time_stamp_ls = std::chrono::time_point<date::utc_clock,
std::chrono::milliseconds>;
time_stamp_ls
convertFileNameToTimestamp_ls(const std::string& filename)
{
using namespace std::chrono;
using namespace date;
const uint64_t yyyy = atoi(filename.substr(0,4).c_str());
const uint64_t MM = atoi(filename.substr(4,2).c_str());
const uint64_t DD = atoi(filename.substr(6,2).c_str());
const uint64_t HH = atoi(filename.substr(9,2).c_str());
const uint64_t MI = atoi(filename.substr(11,2).c_str());
const uint64_t SS = atoi(filename.substr(13,2).c_str());
const uint64_t sss = atoi(filename.substr(16,3).c_str());
return utc_clock::sys_to_utc(sys_days{year(yyyy)/MM/DD}
+ hours{HH} + minutes{MI} + seconds{SS} + milliseconds{sss});
}
Both of these functions can be exercised with the following HelloWorld:
#include <iostream>
int
main()
{
std::string filename = "20150830_002120.123";
std::cout << convertFileNameToTimestamp (filename).time_since_epoch().count() << '\n';
std::cout << convertFileNameToTimestamp_ls(filename).time_since_epoch().count() << '\n';
}
which outputs:
1440894080123
1440894106123
Note that these timestamps are exactly 26,000ms apart.
Update
The "tz.h" header now includes a parse function which makes writing these functions much easier:
date::sys_time<std::chrono::milliseconds>
convertFileNameToTimestamp(const std::string& filename)
{
using namespace std::chrono;
using namespace date;
std::istringstream in{filename};
sys_time<milliseconds> tp;
parse(in, "%Y%m%d_%H%M%S", tp);
return tp;
}
date::utc_time<std::chrono::milliseconds>
convertFileNameToTimestamp_ls(const std::string& filename)
{
return date::to_utc_time(convertFileNameToTimestamp(filename));
}
You can use this piece of code (you don't have to worry about leap years and all the related stuff).
#Edit1: Modified the code to take leap seconds into account; also restructured it into a class:
Foo.h:
#ifndef __FOO__H__
#define __FOO__H__
#include <string>
#include <Windows.h>
#include <stdint.h>
class CFoo {
private:
const static int kLeapSecsDim = 26;
static uint64_t msecsBetweenEpochs;
static SYSTEMTIME leapSecs[kLeapSecsDim];
ULARGE_INTEGER leapSecsUi[kLeapSecsDim];
int CFoo::getLeapSeconds(ULARGE_INTEGER ui) const;
public:
CFoo();
~CFoo() {};
uint64_t toEpoch(const std::string& filename) const;
};
#endif //__FOO__H__
Foo.cpp:
#include "Foo.h"
uint64_t CFoo::msecsBetweenEpochs = 11644473600000; /* Milliseconds between 1.1.1601 and 1.1.1970 */
SYSTEMTIME CFoo::leapSecs[CFoo::kLeapSecsDim] =
{{1972, 06, 0, 30, 23, 59, 59, 999},
{1972, 12, 0, 31, 23, 59, 59, 999},
{1973, 12, 0, 31, 23, 59, 59, 999},
{1974, 12, 0, 31, 23, 59, 59, 999},
{1975, 12, 0, 31, 23, 59, 59, 999},
{1976, 12, 0, 31, 23, 59, 59, 999},
{1977, 12, 0, 31, 23, 59, 59, 999},
{1978, 12, 0, 31, 23, 59, 59, 999},
{1979, 12, 0, 31, 23, 59, 59, 999},
{1981, 06, 0, 30, 23, 59, 59, 999},
{1982, 06, 0, 30, 23, 59, 59, 999},
{1983, 06, 0, 30, 23, 59, 59, 999},
{1985, 06, 0, 30, 23, 59, 59, 999},
{1987, 12, 0, 31, 23, 59, 59, 999},
{1989, 12, 0, 31, 23, 59, 59, 999},
{1990, 12, 0, 31, 23, 59, 59, 999},
{1992, 06, 0, 30, 23, 59, 59, 999},
{1993, 06, 0, 30, 23, 59, 59, 999},
{1994, 06, 0, 30, 23, 59, 59, 999},
{1995, 12, 0, 31, 23, 59, 59, 999},
{1997, 06, 0, 30, 23, 59, 59, 999},
{1998, 12, 0, 31, 23, 59, 59, 999},
{2005, 12, 0, 31, 23, 59, 59, 999},
{2008, 12, 0, 31, 23, 59, 59, 999},
{2012, 06, 0, 30, 23, 59, 59, 999},
{2015, 06, 0, 30, 23, 59, 59, 999},
};
int CFoo::getLeapSeconds(ULARGE_INTEGER ui) const {
int ret = 0;
for (int i = 0; i < kLeapSecsDim; i++) {
if (ui.QuadPart <= this->leapSecsUi[i].QuadPart)
break;
ret++;
}
return ret;
}
CFoo::CFoo() {
FILETIME ft;
BOOL res;
for (int i = 0; i < this->kLeapSecsDim; i++) {
res = SystemTimeToFileTime(&(this->leapSecs[i]), &ft);
if (res == FALSE)
throw std::exception("SystemTimeToFileTime error", GetLastError());
this->leapSecsUi[i].LowPart = ft.dwLowDateTime;
this->leapSecsUi[i].HighPart = ft.dwHighDateTime;
}
}
uint64_t CFoo::toEpoch(const std::string& filename) const {
SYSTEMTIME st;
FILETIME ft;
ULARGE_INTEGER ui;
st.wYear = atoi(filename.substr(0, 4).c_str());
st.wMonth = atoi(filename.substr(4, 2).c_str());
st.wDay = atoi(filename.substr(6, 2).c_str());
st.wHour = atoi(filename.substr(9, 2).c_str());
st.wMinute = atoi(filename.substr(11, 2).c_str());
st.wSecond = atoi(filename.substr(13, 2).c_str());
st.wMilliseconds = atoi(filename.substr(16, 3).c_str());
BOOL result = SystemTimeToFileTime(&st, &ft);
if (result == FALSE)
throw std::exception("SystemTimeToFileTime error", GetLastError());
ui.HighPart = ft.dwHighDateTime;
ui.LowPart = ft.dwLowDateTime;
//printf("%016I64X - %I64u\n", ui.QuadPart, ui.QuadPart);
//printf("%016I64X - %I64u\n", ui.QuadPart/10000, ui.QuadPart/10000);
return (ui.QuadPart / 10000) - this->msecsBetweenEpochs + this->getLeapSeconds(ui) * 1000;
}
Notes:
For invalid dates/times SystemTimeToFileTime will fail
The constant CFoo::msecsBetweenEpochs I think it can be found on Google; I took it from Python(2.7.10)'s posixmodule.c (actually there's the seconds number; I only had to multiply it by 1000)
Your implementation yields results that are not very accurate (I used http://www.epochconverter.com for reference).
According to SystemTimeToFileTime, the timestamp is in UTC.

Lookup on nested boost multi_index_container

Consider following code
struct VersionData
{
VersionData();
VersionData(VersionData&& rhs);
int m_versionId;
int m_weight;
int m_pId;
bool m_hdi;
};
struct VersionId{};
typedef boost::multi_index_container<
VersionData,
bmi::indexed_by<
bmi::ordered_non_unique<
bmi::tag<VersionId>,
bmi::member<VersionData, int, &VersionData::m_versionId>
>
>
> VersionDataContainer;
struct VersionsData
{
VersionsData();
VersionsData(VersionsData&& rhs);
int m_sdGroupId;
int m_retId;
VersionDataContainer m_versionData;
};
struct mvKey{};
typedef boost::multi_index_container<
VersionsData,
bmi::indexed_by<
bmi::ordered_unique<
bmi::tag<mvKey>,
bmi::composite_key<
VersionsData,
bmi::member<VersionsData,int, &VersionsData::m_subdeliveryGroupId>,
bmi::member<VersionsData,int, &VersionsData::m_retargetingId>
>
>
>
> mvDataContainer;
mvDataContainer m_data;
The intention is to lookup using the composite key in mvDataContainer but in some cases I need to lookup in VersionData across all VersionsData. Something like m_data.get<mvKey>.equal_range(make_tuple(ignore, ignore, ignore)).get<VersionId>.equal_range(123456);
First question, is it achievable?
Second, is this the right approach to use nested multi_index_containers? any performance impacts/gains?
In addition to the other answer suggesting a single container for the whole table, here's the idea based on Boost Intrusive multiset
See it Live On Coliru
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/composite_key.hpp>
// for intrusive multiset
#include <boost/intrusive/set.hpp>
#include <boost/range/iterator_range.hpp>
#include <iostream>
namespace bmi = boost::multi_index;
namespace bi = boost::intrusive;
struct VersionData : bi::set_base_hook<bi::link_mode<bi::auto_unlink> > {
VersionData(int versionId, int weight=0, int pId=0, bool hdi=false) :
m_versionId(versionId), m_weight(weight), m_pId(pId), m_hdi(hdi) { }
int m_versionId;
int m_weight;
int m_pId;
bool m_hdi;
friend std::ostream& operator<<(std::ostream& os, VersionData const& vd) {
return os << "{ " << vd.m_versionId << " " << vd.m_weight << " " << vd.m_pId << " " << vd.m_hdi << " }";
}
struct ById {
bool operator()(VersionData const& a, VersionData const& b) const { return a.m_versionId < b.m_versionId; }
};
};
typedef bi::multiset<VersionData, bi::constant_time_size<false>, bi::compare<VersionData::ById> > VersionIndex;
typedef boost::multi_index_container<
VersionData,
bmi::indexed_by<
bmi::ordered_non_unique<
bmi::tag<struct VersionId>,
bmi::member<VersionData, int, &VersionData::m_versionId>
>
>
> VersionDataContainer;
struct VersionsData
{
int m_subdeliveryGroupId;
int m_retargetingId;
VersionDataContainer m_versionData;
};
typedef boost::multi_index_container<
VersionsData,
bmi::indexed_by<
bmi::ordered_unique<
bmi::tag<struct mvKey>,
bmi::composite_key<
VersionsData,
bmi::member<VersionsData,int, &VersionsData::m_subdeliveryGroupId>,
bmi::member<VersionsData,int, &VersionsData::m_retargetingId>
>
>
>
> mvDataContainer;
void insert(
mvDataContainer& into, VersionIndex& global_version_index,
int subdeliveryGroupId, int retargetingId, int
versionId, int weight, int pId, bool hdi)
{
auto& mainIdx = into.get<mvKey>();
auto insertion = mainIdx.insert(VersionsData { subdeliveryGroupId, retargetingId, VersionDataContainer {} });
mainIdx.modify(insertion.first, [&](VersionsData& record) {
auto insertion = record.m_versionData.insert(VersionData { versionId, weight, pId, hdi });
global_version_index.insert(const_cast<VersionData&>(*insertion.first));
});
}
int main() {
VersionIndex global_version_index;
mvDataContainer table;
insert(table, global_version_index, 21, 10, 1, 100, 123, false);
insert(table, global_version_index, 9, 27, 2, 90, 123, false);
insert(table, global_version_index, 12, 25, 3, 110, 123, true);
insert(table, global_version_index, 10, 33, /*version 8:*/ 8, 80, 123, false);
insert(table, global_version_index, 4, 38, 5, 101, 124, false);
insert(table, global_version_index, 33, 7, 6, 91, 124, false);
insert(table, global_version_index, 34, 27, 7, 111, 124, true);
insert(table, global_version_index, 9, 11, /*version 8:*/ 8, 81, 124, false);
insert(table, global_version_index, 0, 12, 9, 99, 125, false);
insert(table, global_version_index, 35, 39, /*version 8:*/ 8, 89, 125, false);
insert(table, global_version_index, 15, 15, 11, 109, 125, true);
insert(table, global_version_index, 25, 32, /*version 8:*/ 8, 79, 125, false);
// debug table output
assert(table.size()==12);
// so now you can do:
std::cout << "---\nQuerying for version id 8:\n";
for (auto& record : boost::make_iterator_range(global_version_index.equal_range(8)))
std::cout << record << "\n";
table.erase(table.find(boost::make_tuple(10,33))); // auto unlinks from global_version_index
std::cout << "---\nQuerying for version id 8:\n";
for (auto& record : boost::make_iterator_range(global_version_index.equal_range(8)))
std::cout << record << "\n";
}
Prints:
---
Querying for version id 8:
{ 8 80 123 0 }
{ 8 81 124 0 }
{ 8 89 125 0 }
{ 8 79 125 0 }
---
Querying for version id 8:
{ 8 81 124 0 }
{ 8 89 125 0 }
{ 8 79 125 0 }
So indeed, instead of using nested containers, like that (live on Coliru)
You could/should implement it as a single /table/ (after all, this is a table with several indices):
Live On Coliru
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/composite_key.hpp>
namespace bmi = boost::multi_index;
struct VersionRecord {
int m_subdeliveryGroupId;
int m_retargetingId;
int m_versionId;
int m_weight;
int m_pId;
bool m_hdi;
friend std::ostream& operator<<(std::ostream& os, VersionRecord const& record) {
return os << "{ " << record.m_subdeliveryGroupId << " " << record.m_retargetingId << " "
<< record.m_versionId << " " << record.m_weight << " " << record.m_pId << " " << record.m_hdi << " }";
}
};
typedef boost::multi_index_container<
VersionRecord,
bmi::indexed_by<
bmi::ordered_unique<
bmi::tag<struct mvKey>,
bmi::composite_key<
VersionRecord,
bmi::member<VersionRecord,int, &VersionRecord::m_subdeliveryGroupId>,
bmi::member<VersionRecord,int, &VersionRecord::m_retargetingId>
>
>,
bmi::ordered_non_unique<
bmi::tag<struct VersionId>,
bmi::member<VersionRecord, int, &VersionRecord::m_versionId>
>
>
> VersionTable;
#include <iostream>
#include <boost/range/iterator_range.hpp>
int main() {
auto table = VersionTable {
{ 21, 10, 1, 100, 123, false },
{ 9, 27, 2, 90, 123, false },
{ 12, 25, 3, 110, 123, true },
{ 10, 33, /*version 8:*/ 8, 80, 123, false },
{ 4, 38, 5, 101, 124, false },
{ 33, 7, 6, 91, 124, false },
{ 34, 27, 7, 111, 124, true },
{ 9, 11, /*version 8:*/ 8, 81, 124, false },
{ 0, 12, 9, 99, 125, false },
{ 35, 39, /*version 8:*/ 8, 89, 125, false },
{ 15, 15, 11, 109, 125, true },
{ 25, 32, /*version 8:*/ 8, 79, 125, false },
};
// debug table output
assert(table.size()==12);
for (auto& record : table) std::cout << record << "\n";
// so now you can do:
auto& idx = table.get<VersionId>();
std::cout << "---\nQuerying for version id 8:\n";
for (auto& record : boost::make_iterator_range(idx.equal_range(8)))
std::cout << record << "\n";
}
Which prints:
{ 0 12 9 99 125 0 }
{ 4 38 5 101 124 0 }
{ 9 11 8 81 124 0 }
{ 9 27 2 90 123 0 }
{ 10 33 8 80 123 0 }
{ 12 25 3 110 123 1 }
{ 15 15 11 109 125 1 }
{ 21 10 1 100 123 0 }
{ 25 32 8 79 125 0 }
{ 33 7 6 91 124 0 }
{ 34 27 7 111 124 1 }
{ 35 39 8 89 125 0 }
---
Querying for version id 8:
{ 25 32 8 79 125 0 }
{ 35 39 8 89 125 0 }
{ 10 33 8 80 123 0 }
{ 9 11 8 81 124 0 }
Alternatively, you can bolt an intrusive container on top of the VersionsData records. This however, complicates the design (you either have to use auto_unlink node hooks (sacrificing thread safety control) or you have to make sure the containers are in synch all the time.
It is not the exact answer which I originally asked but since the performance issue was mentioned and in light of discussion with #sehe this is what I found.
1) use flat structure, you can save wasting memory on the same keys using boost::flyweight
2) use MIC instead of tailored containers, MIC might be slightly slower (depends on test scenario) when searching on simple indexes, but once you use composite keys (and implement similar behavior for your tailored datastructure) it is from slightly to significantly faster than tailored DS
My previous statement that tailored one is faster is wrong, since I was using MIC from boost 1.52 and looks like there was a bug when using composite keys with strings (5 orders of magnitude slower than composite without string). When switched to 1.57 everything started to work as expected.
Tests on Coliru
Have a nice indexing, guys! :)

Daylight Saving Time and UTC-to-local time conversions with WinAPIs

I'm trying to see if WindAPIs that convert from local to UTC time and vice versa are Daylight Saving Time accurate. For instance, let's take LocalFileTimeToFileTime API. Its description states:
LocalFileTimeToFileTime uses the current settings for the time zone
and daylight saving time. Therefore, if it is daylight saving time,
this function will take daylight saving time into account, even if the
time you are converting is in standard time.
So I'm testing it with this code:
//Say, if DST change takes place on Mar-8-2015 at 2:00:00 AM
//when the clock is set 1 hr forward
//Let's check the difference between two times:
SYSTEMTIME st1_local = {2015, 3, 0, 8, 1, 30, 0, 0}; //Mar-8-2015 1:30:00 AM
SYSTEMTIME st2_local = {2015, 3, 0, 8, 3, 30, 0, 0}; //Mar-8-2015 3:30:00 AM
//Convert to file-time format
FILETIME ft1_local, ft2_local;
VERIFY(::SystemTimeToFileTime(&st1_local, &ft1_local));
VERIFY(::SystemTimeToFileTime(&st2_local, &ft2_local));
//Then convert from local to UTC time
FILETIME ft1_utc, ft2_utc;
VERIFY(::LocalFileTimeToFileTime(&ft1_local, &ft1_utc));
VERIFY(::LocalFileTimeToFileTime(&ft2_local, &ft2_utc));
//Get the difference
LONGLONG iiDiff100ns = (((LONGLONG)ft2_utc.dwHighDateTime << 32) | ft2_utc.dwLowDateTime) -
(((LONGLONG)ft1_utc.dwHighDateTime << 32) | ft1_utc.dwLowDateTime);
//Convert from 100ns to seconds
LONGLONG iiDiffSecs = iiDiff100ns / 10000000LL;
//I would expect 1 hr
ASSERT(iiDiffSecs == 3600); //But I get 7200, which is 2 hrs!
So what am I missing here?
SystemTimeToFileTime() interprets its first argument as a UTC time (which has no concept of DST), so your ft1_local and ft2_local objects will always be two hours apart since you're changing the data format, but not the actual point in time. LocalFileTimeToFileTime() will then apply the same offset to whatever you pass to it, so ft1_utc and ft2_utc will always end up two hours apart, also.
As the documentation says, "LocalFileTimeToFileTime uses the current settings for the time zone and daylight saving time" (emphasis mine), so if at the current time you're four hours behind UTC, for instance, it'll just deduct four hours from whatever time you pass to it, regardless of whether that time originally represented some time at the other side of DST.
EDIT: Per the comments, here's how you'd get the seconds difference between two local times in standard C:
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
int main(void) {
struct tm start_time;
start_time.tm_year = 115;
start_time.tm_mon = 2;
start_time.tm_mday = 8;
start_time.tm_hour = 1;
start_time.tm_min = 30;
start_time.tm_sec = 0;
start_time.tm_isdst = -1;
struct tm end_time;
end_time.tm_year = 115;
end_time.tm_mon = 2;
end_time.tm_mday = 8;
end_time.tm_hour = 3;
end_time.tm_min = 30;
end_time.tm_sec = 0;
end_time.tm_isdst = -1;
time_t start_tm = mktime(&start_time);
time_t end_tm = mktime(&end_time);
if ( start_tm == -1 || end_tm == -1 ) {
fputs("Couldn't get local time.", stderr);
exit(EXIT_FAILURE);
}
double seconds_diff = difftime(end_tm, start_tm);
printf("There are %.1f seconds difference.\n", seconds_diff);
return EXIT_SUCCESS;
}
which outputs:
paul#thoth:~/src$ ./difftime
There are 3600.0 seconds difference.
paul#thoth:~/src$
as you're expecting.
Note that, with struct tm:
tm_year is expressed in years since 1900, so to get 2015 we write 115
tm_mon is in the range 0 though 11, so March is 2, not 3.
The other time members are as you'd expect
When tm_isdst is set to -1, mktime() will attempt to find out for itself whether DST was in effect at the local time we supplied, which is what we want it to do, here.
In despite of all the beauty of Paul Griffiths' solution, I can't use it due to an apparent locale limitation. (C is obviously showing its age.) So I had to go with a pure WinAPI approach. Next is what I came up with. Correct me if I'm wrong (especially people with access to time zones other than the US one that Microsoft's mktime seems to be favoring):
SYSTEMTIME st1 = {2015, 3, 0, 8, 1, 30, 0, 0}; //Mar-8-2015 1:30:00 AM
SYSTEMTIME st2 = {2015, 3, 0, 8, 3, 30, 0, 0}; //Mar-8-2015 3:30:00 AM
LONGLONG iiDiffNs;
if(GetLocalDateTimeDifference(&st1, &st2, &iiDiffNs))
{
_tprintf(L"Difference is %.02f sec\n", (double)iiDiffNs / 1000.0);
}
else
{
_tprintf(L"ERROR (%d) calculating the difference.\n", ::GetLastError());
}
Then this is the actual implementation. One important aspect to note here is that the method below may not work reliably on Windows XP due to the lack of APIs to retrieve time zone info for a specific year.
Some declarations first:
enum DST_STATUS{
DST_ERROR = TIME_ZONE_ID_INVALID, //Error
DST_NONE = TIME_ZONE_ID_UNKNOWN, //Daylight Saving Time is NOT observed
DST_OFF = TIME_ZONE_ID_STANDARD, //Daylight Saving Time is observed, but the system is currently not on it
DST_ON = TIME_ZONE_ID_DAYLIGHT, //Daylight Saving Time is observed, and the system is currently on it
};
#define FILETIME_TO_100NS(f) (((LONGLONG)f.dwHighDateTime << 32) | f.dwLowDateTime)
BOOL GetLocalDateTimeDifference(SYSTEMTIME* pStBegin_Local, SYSTEMTIME* pStEnd_Local, LONGLONG* pOutDiffMs = NULL);
BOOL ConvertLocalTimeToUTCTime(SYSTEMTIME* pSt_Local, SYSTEMTIME* pOutSt_UTC = NULL);
DST_STATUS GetDSTInfoForYear(USHORT uYear, TIME_ZONE_INFORMATION* pTZI = NULL);
And the implementation:
BOOL GetLocalDateTimeDifference(SYSTEMTIME* pStBegin_Local, SYSTEMTIME* pStEnd_Local, LONGLONG* pOutDiffMs)
{
//Calculate difference between two local dates considering DST adjustments between them
//INFO: May not work correctly on Windows XP for a year other than the current year!
//'pStBegin_Local' = local date/time to start from
//'pStEnd_Local' = local date/time to end with
//'pOutDiffMs' = if not NULL, receives the difference in milliseconds (if success)
//RETURN:
// = TRUE if success
// = FALSE if error (check GetLastError() for info)
BOOL bRes = FALSE;
LONGLONG iiDiffMs = 0;
int nOSError = NO_ERROR;
if(pStBegin_Local &&
pStEnd_Local)
{
//Convert both dates to UTC
SYSTEMTIME stBeginUTC;
if(ConvertLocalTimeToUTCTime(pStBegin_Local, &stBeginUTC))
{
SYSTEMTIME stEndUTC;
if(ConvertLocalTimeToUTCTime(pStEnd_Local, &stEndUTC))
{
//Then convert into a more manageable format: FILETIME
//It will represent number of 100-nanosecond intervals since January 1, 1601 for each date
FILETIME ftBeginUTC;
if(::SystemTimeToFileTime(&stBeginUTC, &ftBeginUTC))
{
FILETIME ftEndUTC;
if(::SystemTimeToFileTime(&stEndUTC, &ftEndUTC))
{
//Now get the difference in ms
//Convert from 100-ns intervals = 10^7, where ms = 10^3
iiDiffMs = (FILETIME_TO_100NS(ftEndUTC) - FILETIME_TO_100NS(ftBeginUTC)) / 10000LL;
//Done
bRes = TRUE;
}
else
nOSError = ::GetLastError();
}
else
nOSError = ::GetLastError();
}
else
nOSError = ::GetLastError();
}
else
nOSError = ::GetLastError();
}
else
nOSError = ERROR_INVALID_PARAMETER;
if(pOutDiffMs)
*pOutDiffMs = iiDiffMs;
::SetLastError(nOSError);
return bRes;
}
BOOL ConvertLocalTimeToUTCTime(SYSTEMTIME* pSt_Local, SYSTEMTIME* pOutSt_UTC)
{
//Convert local date/time from 'pSt_Local'
//'pOutSt_UTC' = if not NULL, receives converted UTC time
//RETURN:
// = TRUE if success
// = FALSE if error (check GetLastError() for info)
BOOL bRes = FALSE;
SYSTEMTIME stUTC = {0};
int nOSError = NO_ERROR;
if(pSt_Local)
{
//First get time zone info
TIME_ZONE_INFORMATION tzi;
if(GetDSTInfoForYear(pSt_Local->wYear, &tzi) != DST_ERROR)
{
if(::TzSpecificLocalTimeToSystemTime(&tzi, pSt_Local, &stUTC))
{
//Done
bRes = TRUE;
}
else
nOSError = ::GetLastError();
}
else
nOSError = ::GetLastError();
}
else
nOSError = ERROR_INVALID_PARAMETER;
if(pOutSt_UTC)
*pOutSt_UTC = stUTC;
::SetLastError(nOSError);
return bRes;
}
DST_STATUS GetDSTInfoForYear(USHORT uYear, TIME_ZONE_INFORMATION* pTZI)
{
//Get DST info for specific 'uYear'
//INFO: Year is not used on the OS prior to Vista SP1
//'pTZI' = if not NULL, will receive the DST data currently set for the time zone for the year
//RETURN:
// = Current DST status, or an error
// If error (check GetLastError() for info)
DST_STATUS tzStat = DST_ERROR;
int nOSError = NO_ERROR;
//Define newer APIs
DWORD (WINAPI *pfnGetDynamicTimeZoneInformation)(PDYNAMIC_TIME_ZONE_INFORMATION);
BOOL (WINAPI *pfnGetTimeZoneInformationForYear)(USHORT, PDYNAMIC_TIME_ZONE_INFORMATION, LPTIME_ZONE_INFORMATION);
//Load APIs dynamically (in case of Windows XP)
HMODULE hKernel32 = ::GetModuleHandle(L"Kernel32.dll");
ASSERT(hKernel32);
(FARPROC&)pfnGetDynamicTimeZoneInformation = ::GetProcAddress(hKernel32, "GetDynamicTimeZoneInformation");
(FARPROC&)pfnGetTimeZoneInformationForYear = ::GetProcAddress(hKernel32, "GetTimeZoneInformationForYear");
TIME_ZONE_INFORMATION tzi = {0};
//Use newer API if possible
if(pfnGetDynamicTimeZoneInformation &&
pfnGetTimeZoneInformationForYear)
{
//Use new API for dynamic time zone
DYNAMIC_TIME_ZONE_INFORMATION dtzi = {0};
tzStat = (DST_STATUS)pfnGetDynamicTimeZoneInformation(&dtzi);
if(tzStat == DST_ERROR)
{
//Failed -- try old method
goto lbl_fallback_method;
}
//Get TZ info for a year
if(!pfnGetTimeZoneInformationForYear(uYear, &dtzi, &tzi))
{
//Failed -- try old method
goto lbl_fallback_method;
}
}
else
{
lbl_fallback_method:
//Older API (also used as a fall-back method)
tzStat = (DST_STATUS)GetTimeZoneInformation(&tzi);
if(tzStat == DST_ERROR)
nOSError = ::GetLastError();
else
nOSError = ERROR_NOT_SUPPORTED;
}
if(pTZI)
{
*pTZI = tzi;
}
::SetLastError(nOSError);
return tzStat;
}