Input/Output Error in FUSE when mknod callback invoked - c++

I am developing sample fuse application.
In my application, Input/Output error has raise when create new file with touch command.
ll_mknod function has invoked when create file with touch command.
How I can fix it?
#define FUSE_USE_VERSION 310
#include <fuse3/fuse_lowlevel.h>
#include <cerrno>
#include <cstdio>
#include <cstring>
#include <mutex>
#include <regex>
#include <unordered_map>
struct mutex_map {
int counter = 2;
std::mutex _mtx;
std::unordered_map<int, const char*> _data;
std::unordered_map<const char*, int> _rev_data;
public:
int set_value(const char* value) {
std::lock_guard<std::mutex> lock(_mtx);
counter++;
_data[counter] = value;
return counter;
}
const char* get_value(int key) { return _data[key]; }
int get_ino(const char* name) { return _rev_data[name]; }
};
static mutex_map mm;
static int sendmailfs_stat(fuse_ino_t ino, struct stat* stbuf,
size_t name_length) {
stbuf->st_ino = ino;
if (ino == 1) {
stbuf->st_mode = S_IFDIR | 0755;
stbuf->st_nlink = 2;
} else {
stbuf->st_mode = S_IFCHR | 0666;
stbuf->st_nlink = 1;
stbuf->st_size = name_length;
}
return 0;
}
static void ll_lookup(fuse_req_t req, fuse_ino_t parent, const char* name) {
struct fuse_entry_param e;
if (parent != 1) {
fuse_reply_err(req, ENOENT);
return;
}
if (mm._rev_data.find(name) == mm._rev_data.end()) {
fuse_reply_err(req, ENOENT);
return;
}
memset(&e, 0, sizeof(e));
int new_ino = mm.set_value(name);
e.ino = (uint64_t)new_ino;
e.attr_timeout = 1.0;
e.entry_timeout = 1.0;
sendmailfs_stat(e.ino, &e.attr, strlen(name));
puts("Helloworld2");
printf("strlen:%zu: %s\n", strlen(name), name);
fuse_reply_entry(req, &e);
}
static void ll_mknod(fuse_req_t req, fuse_ino_t parent, const char* name,
mode_t mode, dev_t rdev) {
if (parent != 1) {
fuse_reply_err(req, ENOENT);
return;
}
std::regex re("(\\w+)(\\.|_)?(\\w*)#(\\w+)(\\.(\\w+))+");
if (std::regex_match(name, re)) {
puts("Matched");
int inode_num = mm.set_value(name);
struct fuse_entry_param e;
e.ino = inode_num;
e.attr_timeout = 1.0;
e.entry_timeout = 1.0;
sendmailfs_stat(e.ino, &e.attr, strlen(name));
fuse_reply_entry(req, &e);
} else {
puts("Unmatched");
fuse_reply_err(req, EINVAL);
}
}
static void ll_write(fuse_req_t req, fuse_ino_t ino, const char* buf,
size_t size, off_t off, struct fuse_file_info* fi) {
if (mm.get_value(ino) == NULL) {
fuse_reply_err(req, EINVAL);
return;
}
fuse_reply_write(req, size);
}
static void ll_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info* fi) {
if (ino == 1 || !mm.get_value(ino)) {
puts("invalid access due to hello");
fuse_reply_err(req, ENOENT);
} else if ((fi->flags && O_WRONLY)) {
fuse_reply_err(req, EINVAL);
}
fuse_reply_open(req, fi);
}
static void ll_setattr(fuse_req_t req, fuse_ino_t ino, struct stat* attr,
int to_set, struct fuse_file_info* fi) {
puts("setattr");
fuse_reply_err(req, EINVAL);
}
static void ll_setxattr(fuse_req_t req, fuse_ino_t ino, const char* name,
const char* value, size_t size, int flags) {
puts("setxattr");
fuse_reply_err(req, EINVAL);
}
static void ll_getattr(fuse_req_t req, fuse_ino_t ino,
struct fuse_file_info* fi) {
puts("getattr");
fuse_reply_err(req, EINVAL);
}
static void ll_getxattr(fuse_req_t req, fuse_ino_t ino, const char* name,
size_t size) {
puts("getxattr");
fuse_reply_err(req, EINVAL);
}
static const struct fuse_lowlevel_ops opener = {
.lookup = ll_lookup,
.mknod = ll_mknod,
.open = ll_open,
.write = ll_write,
};
int main(int argc, char* argv[]) {
struct fuse_args args = FUSE_ARGS_INIT(argc, argv);
struct fuse_session* se;
struct fuse_cmdline_opts opts;
struct fuse_loop_config config;
int ret = -1;
if (fuse_parse_cmdline(&args, &opts) != 0) {
return 1;
}
if (opts.show_help) {
printf("usage: %s [options] <mountpoint>\n\n", argv[0]);
fuse_cmdline_help();
fuse_lowlevel_help();
ret = 0;
goto err_out1;
} else if (opts.show_version) {
printf("FUSE library version %d\n", fuse_version());
fuse_lowlevel_version();
ret = 0;
goto err_out1;
}
if (opts.mountpoint == NULL) {
printf("usage: %s [options] <mountpoint>\n\n", argv[0]);
printf(" %s --help\n", argv[0]);
ret = 1;
goto err_out1;
}
se = fuse_session_new(&args, &opener, sizeof(opener), NULL);
if (fuse_set_signal_handlers(se) != 0) {
goto err_out2;
}
if (fuse_session_mount(se, opts.mountpoint) != 0) {
goto err_out3;
}
fuse_daemonize(opts.foreground);
if (opts.singlethread) {
ret = fuse_session_loop(se);
} else {
config.clone_fd = opts.clone_fd;
config.max_idle_threads = opts.max_idle_threads;
ret = fuse_session_loop_mt(se, &config);
}
fuse_session_unmount(se);
err_out3:
fuse_remove_signal_handlers(se);
err_out2:
fuse_session_destroy(se);
err_out1:
free(opts.mountpoint);
fuse_opt_free_args(&args);
return ret ? 1 : 0;
}
I using:
Linux 5.12.15
libfuse 3.10.4

Related

error: invalid operands of types 'bool' and 'int' to binary 'operator<=> - (GCC 10 - std=gnu++20) [closed]

Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 1 year ago.
Improve this question
i have some c++ project after a release of support c++20, i want to upgrade my makefile std support 17 to 20 after that point my compiler (gcc10.2) give me a error like this ;
Error
In file included from /usr/local/lib/gcc10/include/c++/bits/node_handle.h:39,
from /usr/local/lib/gcc10/include/c++/bits/stl_tree.h:72,
from /usr/local/lib/gcc10/include/c++/map:60,
from AsyncSQL.h:10,
from AsyncSQL.cpp:4:
/usr/local/lib/gcc10/include/c++/optional: In function 'constexpr std::strong_ordering std::operator<=>(const std::optional<_Tp>&, std::nullopt_t)':
/usr/local/lib/gcc10/include/c++/optional:1052:24: error: invalid operands of types 'bool' and 'int' to binary 'operator<=>'
1052 | { return bool(__x) <=> false; }
| ~~~~~~~~~ ^~~
| |
| bool
gmake[2]: *** [Makefile:23: AsyncSQL.o] Error 1
This is my AsyncSQL.cpp ;
#include <sys/time.h>
#include <cstdlib>
#include <cstring>
#include "AsyncSQL.h"
#define MUTEX_LOCK(mtx) pthread_mutex_lock(mtx)
#define MUTEX_UNLOCK(mtx) pthread_mutex_unlock(mtx)
CAsyncSQL::CAsyncSQL(): m_stHost (""), m_stUser (""), m_stPassword (""), m_stDB (""), m_stLocale (""), m_iMsgCount (0), m_iPort (0), m_bEnd (false), m_hThread (0), m_mtxQuery (NULL), m_mtxResult (NULL), m_iQueryFinished (0), m_ulThreadID (0), m_bConnected (false), m_iCopiedQuery (0)
{
memset (&m_hDB, 0, sizeof (m_hDB));
m_aiPipe[0] = 0;
m_aiPipe[1] = 0;
}
CAsyncSQL::~CAsyncSQL()
{
Quit();
Destroy();
}
void CAsyncSQL::Destroy()
{
if (m_hDB.host)
{
sys_log (0, "AsyncSQL: closing mysql connection.");
mysql_close (&m_hDB);
m_hDB.host = NULL;
}
if (m_mtxQuery)
{
pthread_mutex_destroy (m_mtxQuery);
delete m_mtxQuery;
m_mtxQuery = NULL;
}
if (m_mtxResult)
{
pthread_mutex_destroy (m_mtxResult);
delete m_mtxResult;
m_mtxQuery = NULL;
}
}
void* AsyncSQLThread (void* arg)
{
CAsyncSQL* pSQL = ((CAsyncSQL*) arg);
if (!pSQL->Connect())
{
return NULL;
}
pSQL->ChildLoop();
return NULL;
}
bool CAsyncSQL::QueryLocaleSet()
{
if (0 == m_stLocale.length())
{
sys_err ("m_stLocale == 0");
return true;
}
if (mysql_set_character_set (&m_hDB, m_stLocale.c_str()))
{
sys_err ("cannot set locale %s by 'mysql_set_character_set', errno %u %s", m_stLocale.c_str(), mysql_errno (&m_hDB) , mysql_error (&m_hDB));
return false;
}
sys_log (0, "\t--mysql_set_character_set(%s)", m_stLocale.c_str());
return true;
}
bool CAsyncSQL::Connect()
{
if (0 == mysql_init (&m_hDB))
{
fprintf (stderr, "mysql_init failed\n");
return false;
}
if (!m_stLocale.empty())
{
if (mysql_options (&m_hDB, MYSQL_SET_CHARSET_NAME, m_stLocale.c_str()) != 0)
{
fprintf (stderr, "mysql_option failed : MYSQL_SET_CHARSET_NAME %s ", mysql_error(&m_hDB));
}
}
if (!mysql_real_connect (&m_hDB, m_stHost.c_str(), m_stUser.c_str(), m_stPassword.c_str(), m_stDB.c_str(), m_iPort, NULL, CLIENT_MULTI_STATEMENTS))
{
fprintf (stderr, "mysql_real_connect: %s\n", mysql_error(&m_hDB));
return false;
}
my_bool reconnect = true;
if (0 != mysql_options (&m_hDB, MYSQL_OPT_RECONNECT, &reconnect))
{
fprintf (stderr, "mysql_option: %s\n", mysql_error(&m_hDB));
}
m_ulThreadID = mysql_thread_id (&m_hDB);
m_bConnected = true;
return true;
}
bool CAsyncSQL::Setup (CAsyncSQL* sql, bool bNoThread)
{
return Setup (sql->m_stHost.c_str(), sql->m_stUser.c_str(), sql->m_stPassword.c_str(), sql->m_stDB.c_str(), sql->m_stLocale.c_str(), bNoThread, sql->m_iPort);
}
bool CAsyncSQL::Setup (const char* c_pszHost, const char* c_pszUser, const char* c_pszPassword, const char* c_pszDB, const char* c_pszLocale, bool bNoThread, int iPort)
{
m_stHost = c_pszHost;
m_stUser = c_pszUser;
m_stPassword = c_pszPassword;
m_stDB = c_pszDB;
m_iPort = iPort;
if (c_pszLocale)
{
m_stLocale = c_pszLocale;
sys_log (0, "AsyncSQL: locale %s", m_stLocale.c_str());
}
if (!bNoThread)
{
m_mtxQuery = new pthread_mutex_t;
m_mtxResult = new pthread_mutex_t;
if (0 != pthread_mutex_init (m_mtxQuery, NULL))
{
perror ("pthread_mutex_init");
exit (0);
}
if (0 != pthread_mutex_init (m_mtxResult, NULL))
{
perror ("pthread_mutex_init");
exit (0);
}
pthread_create (&m_hThread, NULL, AsyncSQLThread, this);
return true;
}
else
{
return Connect();
}
}
void CAsyncSQL::Quit()
{
m_bEnd = true;
m_sem.Release();
if (m_hThread)
{
pthread_join (m_hThread, NULL);
m_hThread = NULL;
}
}
SQLMsg* CAsyncSQL::DirectQuery (const char* c_pszQuery)
{
if (m_ulThreadID != mysql_thread_id (&m_hDB))
{
sys_log (0, "MySQL connection was reconnected. querying locale set");
while (!QueryLocaleSet());
m_ulThreadID = mysql_thread_id (&m_hDB);
}
SQLMsg* p = new SQLMsg;
p->m_pkSQL = &m_hDB;
p->iID = ++m_iMsgCount;
p->stQuery = c_pszQuery;
if (mysql_real_query (&m_hDB, p->stQuery.c_str(), p->stQuery.length()))
{
char buf[1024];
snprintf (buf, sizeof(buf), "AsyncSQL::DirectQuery : mysql_query error: %s\nquery: %s", mysql_error (&m_hDB), p->stQuery.c_str());
sys_err (buf);
p->uiSQLErrno = mysql_errno (&m_hDB);
}
p->Store();
return p;
}
void CAsyncSQL::AsyncQuery (const char* c_pszQuery)
{
auto p = new SQLMsg;
p->m_pkSQL = &m_hDB;
p->iID = ++m_iMsgCount;
p->stQuery = c_pszQuery;
PushQuery (p);
}
void CAsyncSQL::ReturnQuery (const char* c_pszQuery, void* pvUserData)
{
auto p = new SQLMsg;
p->m_pkSQL = &m_hDB;
p->iID = ++m_iMsgCount;
p->stQuery = c_pszQuery;
p->bReturn = true;
p->pvUserData = pvUserData;
PushQuery (p);
}
void CAsyncSQL::PushResult (SQLMsg* p)
{
MUTEX_LOCK (m_mtxResult);
m_queue_result.push (p);
MUTEX_UNLOCK (m_mtxResult);
}
bool CAsyncSQL::PopResult(SQLMsg** pp)
{
MUTEX_LOCK (m_mtxResult);
if (m_queue_result.empty())
{
MUTEX_UNLOCK (m_mtxResult);
return false;
}
*pp = m_queue_result.front();
m_queue_result.pop();
MUTEX_UNLOCK (m_mtxResult);
return true;
}
void CAsyncSQL::PushQuery (SQLMsg* p)
{
MUTEX_LOCK (m_mtxQuery);
m_queue_query.push (p);
m_sem.Release();
MUTEX_UNLOCK (m_mtxQuery);
}
bool CAsyncSQL::PeekQuery (SQLMsg** pp)
{
MUTEX_LOCK (m_mtxQuery);
if (m_queue_query.empty())
{
MUTEX_UNLOCK (m_mtxQuery);
return false;
}
*pp = m_queue_query.front();
MUTEX_UNLOCK (m_mtxQuery);
return true;
}
bool CAsyncSQL::PopQuery (int iID)
{
MUTEX_LOCK (m_mtxQuery);
if (m_queue_query.empty())
{
MUTEX_UNLOCK (m_mtxQuery);
return false;
}
m_queue_query.pop();
MUTEX_UNLOCK (m_mtxQuery);
return true;
}
bool CAsyncSQL::PeekQueryFromCopyQueue (SQLMsg** pp)
{
if (m_queue_query_copy.empty())
{
return false;
}
*pp = m_queue_query_copy.front();
return true;
}
int CAsyncSQL::CopyQuery()
{
MUTEX_LOCK (m_mtxQuery);
if (m_queue_query.empty())
{
MUTEX_UNLOCK (m_mtxQuery);
return -1;
}
while (!m_queue_query.empty())
{
SQLMsg* p = m_queue_query.front();
m_queue_query_copy.push (p);
m_queue_query.pop();
}
int count = m_queue_query_copy.size();
MUTEX_UNLOCK (m_mtxQuery);
return count;
}
bool CAsyncSQL::PopQueryFromCopyQueue()
{
if (m_queue_query_copy.empty())
{
return false;
}
m_queue_query_copy.pop();
return true;
}
int CAsyncSQL::GetCopiedQueryCount()
{
return m_iCopiedQuery;
}
void CAsyncSQL::ResetCopiedQueryCount()
{
m_iCopiedQuery = 0;
}
void CAsyncSQL::AddCopiedQueryCount (int iCopiedQuery)
{
m_iCopiedQuery += iCopiedQuery;
}
DWORD CAsyncSQL::CountQuery()
{
return m_queue_query.size();
}
DWORD CAsyncSQL::CountResult()
{
return m_queue_result.size();
}
void __timediff (struct timeval* a, struct timeval* b, struct timeval* rslt)
{
if (a->tv_sec < b->tv_sec)
{
rslt->tv_sec = rslt->tv_usec = 0;
}
else if (a->tv_sec == b->tv_sec)
{
if (a->tv_usec < b->tv_usec)
{
rslt->tv_sec = rslt->tv_usec = 0;
}
else
{
rslt->tv_sec = 0;
rslt->tv_usec = a->tv_usec - b->tv_usec;
}
}
else
{
rslt->tv_sec = a->tv_sec - b->tv_sec;
if (a->tv_usec < b->tv_usec)
{
rslt->tv_usec = a->tv_usec + 1000000 - b->tv_usec;
rslt->tv_sec--;
}
else
{
rslt->tv_usec = a->tv_usec - b->tv_usec;
}
}
}
class cProfiler
{
public:
cProfiler()
{
m_nInterval = 0 ;
memset (&prev, 0, sizeof (prev));
memset (&now, 0, sizeof (now));
memset (&interval, 0, sizeof (interval));
Start();
}
cProfiler (int nInterval = 100000)
{
m_nInterval = nInterval;
memset (&prev, 0, sizeof (prev));
memset (&now, 0, sizeof (now));
memset (&interval, 0, sizeof (interval));
Start();
}
void Start()
{
gettimeofday (&prev , (struct timezone*) 0);
}
void Stop()
{
gettimeofday (&now, (struct timezone*) 0);
__timediff (&now, &prev, &interval);
}
bool IsOk()
{
if (interval.tv_sec > (m_nInterval / 1000000))
{
return false;
}
if (interval.tv_usec > m_nInterval)
{
return false;
}
return true;
}
struct timeval* GetResult()
{
return &interval;
}
long GetResultSec()
{
return interval.tv_sec;
}
long GetResultUSec()
{
return interval.tv_usec;
}
private:
int m_nInterval;
struct timeval prev;
struct timeval now;
struct timeval interval;
};
void CAsyncSQL::ChildLoop()
{
cProfiler profiler(500000);
while (!m_bEnd)
{
m_sem.Wait();
int count = CopyQuery();
if (count <= 0)
{
continue;
}
AddCopiedQueryCount (count);
SQLMsg* p;
while (count--)
{
profiler.Start();
if (!PeekQueryFromCopyQueue (&p))
{
continue;
}
if (m_ulThreadID != mysql_thread_id (&m_hDB))
{
sys_log (0, "MySQL connection was reconnected. querying locale set");
while (!QueryLocaleSet());
m_ulThreadID = mysql_thread_id (&m_hDB);
}
if (mysql_real_query (&m_hDB, p->stQuery.c_str(), p->stQuery.length()))
{
p->uiSQLErrno = mysql_errno (&m_hDB);
sys_err ("AsyncSQL: query failed: %s (query: %s errno: %d)", mysql_error (&m_hDB), p->stQuery.c_str(), p->uiSQLErrno);
switch (p->uiSQLErrno)
{
case CR_SOCKET_CREATE_ERROR:
case CR_CONNECTION_ERROR:
case CR_IPSOCK_ERROR:
case CR_UNKNOWN_HOST:
case CR_SERVER_GONE_ERROR:
case CR_CONN_HOST_ERROR:
case ER_NOT_KEYFILE:
case ER_CRASHED_ON_USAGE:
case ER_CANT_OPEN_FILE:
case ER_HOST_NOT_PRIVILEGED:
case ER_HOST_IS_BLOCKED:
case ER_PASSWORD_NOT_ALLOWED:
case ER_PASSWORD_NO_MATCH:
case ER_CANT_CREATE_THREAD:
case ER_INVALID_USE_OF_NULL:
m_sem.Release();
sys_err ("AsyncSQL: retrying");
continue;
}
}
profiler.Stop();
if (!profiler.IsOk())
{
sys_log (0, "[QUERY : LONG INTERVAL(OverSec %ld.%ld)] : %s", profiler.GetResultSec(), profiler.GetResultUSec(), p->stQuery.c_str());
}
PopQueryFromCopyQueue();
if (p->bReturn)
{
p->Store();
PushResult (p);
}
else
{
delete p;
}
++m_iQueryFinished;
}
}
SQLMsg* p;
while (PeekQuery (&p))
{
if (m_ulThreadID != mysql_thread_id (&m_hDB))
{
sys_log (0, "MySQL connection was reconnected. querying locale set");
while (!QueryLocaleSet());
m_ulThreadID = mysql_thread_id (&m_hDB);
}
if (mysql_real_query (&m_hDB, p->stQuery.c_str(), p->stQuery.length()))
{
p->uiSQLErrno = mysql_errno (&m_hDB);
sys_err ("AsyncSQL::ChildLoop : mysql_query error: %s:\nquery: %s", mysql_error (&m_hDB), p->stQuery.c_str());
switch (p->uiSQLErrno)
{
case CR_SOCKET_CREATE_ERROR:
case CR_CONNECTION_ERROR:
case CR_IPSOCK_ERROR:
case CR_UNKNOWN_HOST:
case CR_SERVER_GONE_ERROR:
case CR_CONN_HOST_ERROR:
case ER_NOT_KEYFILE:
case ER_CRASHED_ON_USAGE:
case ER_CANT_OPEN_FILE:
case ER_HOST_NOT_PRIVILEGED:
case ER_HOST_IS_BLOCKED:
case ER_PASSWORD_NOT_ALLOWED:
case ER_PASSWORD_NO_MATCH:
case ER_CANT_CREATE_THREAD:
case ER_INVALID_USE_OF_NULL:
continue;
}
}
sys_log (0, "QUERY_FLUSH: %s", p->stQuery.c_str());
PopQuery (p->iID);
if (p->bReturn)
{
p->Store();
PushResult (p);
}
else
{
delete p;
}
++m_iQueryFinished;
}
}
int CAsyncSQL::CountQueryFinished()
{
return m_iQueryFinished;
}
void CAsyncSQL::ResetQueryFinished()
{
m_iQueryFinished = 0;
}
MYSQL* CAsyncSQL::GetSQLHandle()
{
return &m_hDB;
}
size_t CAsyncSQL::EscapeString (char* dst, size_t dstSize, const char* src, size_t srcSize)
{
if (0 == srcSize)
{
memset (dst, 0, dstSize);
return 0;
}
if (0 == dstSize)
{
return 0;
}
if (dstSize < srcSize * 2 + 1)
{
char tmp[256];
size_t tmpLen = sizeof (tmp) > srcSize ? srcSize : sizeof (tmp);
strlcpy (tmp, src, tmpLen);
sys_err ("FATAL ERROR!! not enough buffer size (dstSize %u srcSize %u src%s: %s)", dstSize, srcSize, tmpLen != srcSize ? "(trimmed to 255 characters)" : "", tmp);
dst[0] = '\0';
return 0;
}
return mysql_real_escape_string (GetSQLHandle(), dst, src, srcSize);
}
void CAsyncSQL2::SetLocale (const std::string & stLocale)
{
m_stLocale = stLocale;
QueryLocaleSet();
}
This is my AsyncSQL.h
#ifndef __INC_METIN_II_ASYNCSQL_H__
#define __INC_METIN_II_ASYNCSQL_H__
#include "../../libthecore/src/stdafx.h"
#include "../../libthecore/src/log.h"
#include "../../Ayarlar.h"
#include <string>
#include <queue>
#include <vector>
#include <map>
#include <mysql/server/mysql.h>
#include <mysql/server/errmsg.h>
#include <mysql/server/mysqld_error.h>
#include "Semaphore.h"
typedef struct _SQLResult
{
_SQLResult(): pSQLResult (NULL), uiNumRows (0), uiAffectedRows (0), uiInsertID (0) {}
~_SQLResult()
{
if (pSQLResult)
{
mysql_free_result (pSQLResult);
pSQLResult = NULL;
}
}
MYSQL_RES* pSQLResult;
uint32_t uiNumRows;
uint32_t uiAffectedRows;
uint32_t uiInsertID;
} SQLResult;
typedef struct _SQLMsg
{
_SQLMsg() : m_pkSQL (NULL), iID (0), uiResultPos (0), pvUserData (NULL), bReturn (false), uiSQLErrno (0) {}
~_SQLMsg()
{
auto first = vec_pkResult.begin();
auto past = vec_pkResult.end();
while (first != past)
{
delete * (first++);
}
vec_pkResult.clear();
}
void Store()
{
do
{
SQLResult* pRes = new SQLResult;
pRes->pSQLResult = mysql_store_result (m_pkSQL);
pRes->uiInsertID = mysql_insert_id (m_pkSQL);
pRes->uiAffectedRows = mysql_affected_rows (m_pkSQL);
if (pRes->pSQLResult)
{
pRes->uiNumRows = mysql_num_rows (pRes->pSQLResult);
}
else
{
pRes->uiNumRows = 0;
}
vec_pkResult.push_back (pRes);
}
while (!mysql_next_result (m_pkSQL));
}
SQLResult* Get()
{
if (uiResultPos >= vec_pkResult.size())
{
return NULL;
}
return vec_pkResult[uiResultPos];
}
bool Next()
{
if (uiResultPos + 1 >= vec_pkResult.size())
{
return false;
}
++uiResultPos;
return true;
}
MYSQL* m_pkSQL;
int iID;
std::string stQuery;
std::vector<SQLResult *> vec_pkResult;
unsigned int uiResultPos;
void* pvUserData;
bool bReturn;
unsigned int uiSQLErrno;
} SQLMsg;
class CAsyncSQL
{
public:
CAsyncSQL();
virtual ~CAsyncSQL();
void Quit();
bool Setup (const char* c_pszHost, const char* c_pszUser, const char* c_pszPassword, const char* c_pszDB, const char* c_pszLocale, bool bNoThread = false, int iPort = 0);
bool Setup (CAsyncSQL* sql, bool bNoThread = false);
bool Connect();
bool IsConnected()
{
return m_bConnected;
}
bool QueryLocaleSet();
void AsyncQuery (const char* c_pszQuery);
void ReturnQuery (const char* c_pszQuery, void* pvUserData);
SQLMsg* DirectQuery (const char* c_pszQuery);
DWORD CountQuery();
DWORD CountResult();
void PushResult (SQLMsg* p);
bool PopResult (SQLMsg** pp);
void ChildLoop();
MYSQL* GetSQLHandle();
int CountQueryFinished();
void ResetQueryFinished();
size_t EscapeString (char* dst, size_t dstSize, const char* src, size_t srcSize);
protected:
void Destroy();
void PushQuery (SQLMsg* p);
bool PeekQuery (SQLMsg** pp);
bool PopQuery (int iID);
bool PeekQueryFromCopyQueue (SQLMsg** pp );
INT CopyQuery();
bool PopQueryFromCopyQueue();
public:
int GetCopiedQueryCount();
void ResetCopiedQueryCount();
void AddCopiedQueryCount (int iCopiedQuery);
protected:
MYSQL m_hDB;
std::string m_stHost;
std::string m_stUser;
std::string m_stPassword;
std::string m_stDB;
std::string m_stLocale;
int m_iMsgCount;
int m_aiPipe[2];
int m_iPort;
std::queue<SQLMsg*> m_queue_query;
std::queue<SQLMsg*> m_queue_query_copy;
std::queue<SQLMsg*> m_queue_result;
volatile bool m_bEnd;
pthread_t m_hThread;
pthread_mutex_t* m_mtxQuery;
pthread_mutex_t* m_mtxResult;
CSemaphore m_sem;
int m_iQueryFinished;
unsigned long m_ulThreadID;
bool m_bConnected;
int m_iCopiedQuery;
};
class CAsyncSQL2 : public CAsyncSQL
{
public:
void SetLocale (const std::string & stLocale);
};
#endif
And this is the function the reason of the error ;
optional:1052 ;
#ifdef __cpp_lib_three_way_comparison
template<typename _Tp>
constexpr strong_ordering
operator<=>(const optional<_Tp>& __x, nullopt_t) noexcept
{ return bool(__x) <=> false; }
#else
After a see a document the microsoft release i'm gonna try <= > false; like this and take a error again..
Best Regards.
I ve no idea why it looks is getting bool(__x) <=> false as an bool and int comparison.
I would think you got some strange macro in your files included before to include the header that is going to break the standard code.
I would suggest you try to move above the standard headers and below them your 'user defined' headers.
#include <string>
#include <queue>
#include <vector>
#include <map>
#include <mysql/server/mysql.h>
#include <mysql/server/errmsg.h>
#include <mysql/server/mysqld_error.h>
#include "../../libthecore/src/stdafx.h"
#include "../../libthecore/src/log.h"
#include "../../Ayarlar.h"
#include "Semaphore.h"
EDIT:
i ve found the cause of the problem.
a macro defined in "libthrecore/stdafx.h" (i own the files that is using the author, they are public).
#ifndef false
#define false 0
#define true (!false)
#endif
it is causing false to be read as a int and is causing the spaceship operator to fails with the error shown by the author. Move up the standard headers or remove the macro to solve the error.

Firebird database UDF for encrypt/decrypt not freeing memory

I'm testing a UDF external function in Firebird 3 database, I made a C++ DLL which performs a simple XOR to a given string using a given key.
This is the code:
#include <windows.h>
#include <iostream>
#include <string>
#include <stdio.h>
#include <vector>
#include <math.h>
#include "../FirebirdLib/src/include/ibase.h"
#include "ib_util.h"
using namespace std;
//------------------------------------------------------------------------------------
typedef void (__stdcall * FCallback)(const char * message);
FCallback g_messageCallback = 0;
FCallback g_errorCallback = 0;
//------------------------------------------------------------------------------------
#define ON_MESSAGE(mess) { if(g_messageCallback) g_messageCallback(mess); }
#define ON_ERROR(mess) { if(g_errorCallback) g_errorCallback(mess); }
//------------------------------------------------------------------------------------
extern "C" __declspec(dllexport) void RegisterCallbacks(FCallback messageCallback, FCallback errorCallback)
{
g_messageCallback = messageCallback;
g_errorCallback = errorCallback;
}
//------------------------------------------------------------------------------------
class EncryptionUDF
{
public:
EncryptionUDF()
{
//ON_MESSAGE("--EncryptionUDF created--")
}
~EncryptionUDF()
{
//ON_MESSAGE("--EncryptionUDF destroyed--")
}
char* XORCipher(const char* data, const char* key, int dataLen, int keyLen) {
char* output = (char*)ib_util_malloc(2000 + 1L);
output[dataLen] = '\0';
for (int i = 0; i < dataLen; ++i) {
if (data[i] != key[i % keyLen])
output[i] = data[i] ^ key[i % keyLen];
else
output[i] = data[i];
}
return output;
}
char * Encrypt(const char * str, const char * key) {
int dataLen = strlen(str);
int keyLen = strlen(key);
char* output = (char*)ib_util_malloc(2000 + 1L);
output[dataLen] = '\0';
try {
if ((str == NULL) || (str[0] == '\0')) {
return NULL;
}
else {
try {
if ((key != NULL) && (key[0] == '\0')) {
strncpy(output, str, dataLen);
}
else if (key != NULL) {
output = XORCipher(str, key, dataLen, keyLen);
}
else strncpy(output, str, dataLen);
}
catch (...) { strncpy(output, str, dataLen); }
return output;
}
}
catch (...) { strncpy(output, str, dataLen); }
return output;
}
char * Decrypt(const char * str, const char * key) {
int dataLen = strlen(str);
int keyLen = strlen(key);
char* output = (char*)ib_util_malloc(2000 + 1L);
output[dataLen] = '\0';
try {
if ((str == NULL) || (str[0] == '\0')) {
return NULL;
}
else {
try {
if ((key != NULL) && (key[0] == '\0')) {
strncpy(output, str, dataLen);
}
else if (key != NULL) {
output = XORCipher(str, key, dataLen, keyLen);
}
else strncpy(output, str, dataLen);
}
catch (...) { strncpy(output, str, dataLen); }
return output;
}
}
catch (...) { strncpy(output, str, dataLen); }
return output;
}
};
//------------------------------------------------------------------------------------
extern "C" __declspec(dllexport) char * EncryptUDF_DesEncrypt(const char *str, const char *key)
{
try
{
EncryptionUDF self = EncryptionUDF();
return self.Encrypt(str, key);
}
catch (std::exception & ex)
{
ON_ERROR(ex.what());
}
catch (...)
{
ON_ERROR("Unknown error");
}
return 0;
}
//------------------------------------------------------------------------------------
extern "C" __declspec(dllexport) char * EncryptUDF_DesDecrypt(const char *str, const char *key)
{
try
{
EncryptionUDF self = EncryptionUDF();
return self.Decrypt(str, key);
}
catch (std::exception & ex)
{
ON_ERROR(ex.what());
}
catch (...)
{
ON_ERROR("Unknown error");
}
return 0;
}
//------------------------------------------------------------------------------------
BOOL APIENTRY DllMain( HMODULE hModule,
DWORD ul_reason_for_call,
LPVOID lpReserved
)
{
return TRUE;
}
//------------------------------------------------------------------------------------
The UDF is defined in database as:
DECLARE EXTERNAL FUNCTION X_DECRYPT
CSTRING(2000),
CSTRING(64)
RETURNS CSTRING(2000) FREE_IT
ENTRY_POINT 'EncryptUDF_DesDecrypt' MODULE_NAME 'EncryptUDF';
DECLARE EXTERNAL FUNCTION X_ENCRYPT
CSTRING(2000),
CSTRING(64)
RETURNS CSTRING(2000) FREE_IT
ENTRY_POINT 'EncryptUDF_DesEncrypt' MODULE_NAME 'EncryptUDF';
When using this UDF in SQL select commands, the ram used by firebird server tends to increase continously. When using embedded the RAM goes up quickly, when in server mode, the RAM is increasing but slowly and somehow more controlled.
Please help to understand where the error is.
After some investigation, I decided to change the parts on code where the string is copied using:
strncpy(output, str, dataLen);
with:
strncpy_s(output, dataLen, str, dataLen);
and after this change, the memory has been in normal levels, either in embedded firebird or in server mode.
It seems that was a memory leak in releasing or managing those string copies.

Memory usage with IOCP [closed]

Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 5 years ago.
Improve this question
I am converting our code to use IOCP and I got the communication relatively stable, but the memory usage of the application is increasing. Looks like I am getting back (on completion function calls) much fewer objects of OverlappedEx than I create. My code is below. What am I doing wrong?
#ifndef NETWORK_DATA
#define NETWORK_DATA
#include <afxwin.h>
#include <vector>
#include <string>
#include "CriticalSectionLocker.h"
using namespace std;
DWORD NetworkManager::NetworkThread(void* param)
{
bool bRun = true;
while (bRun)
{
DWORD wait = ::WaitForSingleObject(CCommunicationManager::s_hShutdownEvent, 0);
if (WAIT_OBJECT_0 == wait)
{
bRun = false;
DEBUG_LOG0("Shutdown event was signalled thread");
}
else
{
DWORD dwBytesTransfered = 0;
void* lpContext = nullptr;
OVERLAPPED* pOverlapped = nullptr;
BOOL bReturn = GetQueuedCompletionStatus(s_IOCompletionPort,
&dwBytesTransfered,
(LPDWORD)&lpContext,
&pOverlapped,
INFINITE);
if (nullptr == lpContext)
{
DEBUG_LOG0("invalid context");
/*continue;*/
}
else
{
if (bReturn && dwBytesTransfered > 0)
{
OverlappedEx* data = reinterpret_cast<OverlappedEx*>(pOverlapped);
ServerData* networkData = reinterpret_cast<ServerData*>(lpContext);
if (networkData && data)
{
switch(data->m_opType)
{
case OverlappedEx::OP_READ:
/*DEBUG_LOG4("device name: %s bytes received: %d socket: %d handle: %d",
networkData->Name().c_str(), dwBytesTransfered, networkData->Socket(), networkData->Handle());*/
networkData->CompleteReceive(dwBytesTransfered, data);
break;
case OverlappedEx::OP_WRITE:
/*DEBUG_LOG4("device name: %s bytes sent: %d socket: %d handle: %d",
networkData->Name().c_str(), dwBytesTransfered, networkData->Socket(), networkData->Handle());*/
networkData->CompleteSend(dwBytesTransfered, data);
break;
}
}
}
else
{
/*DEBUG_LOG2("GetQueuedCompletionStatus failed: bReturn: %d dwBytesTransferred: %u", bReturn, dwBytesTransfered);*/
}
}
}
}
return 0;
}
enum NetworkType
{
UDP,
TCP
};
struct OverlappedEx : public OVERLAPPED
{
enum OperationType
{
OP_READ,
OP_WRITE
};
const static int MAX_PACKET_SIZE = 2048;
WSABUF m_wBuf;
char m_buffer[MAX_PACKET_SIZE];
OperationType m_opType;
OverlappedEx()
{
Clear();
m_refCount = 1;
}
void AddRef()
{
::InterlockedIncrement(&m_refCount);
}
void Release()
{
::InterlockedDecrement(&m_refCount);
}
int Refcount() const
{
return InterlockedExchangeAdd((unsigned long*)&m_refCount, 0UL);
}
~OverlappedEx()
{
Clear();
}
void Clear()
{
memset(m_buffer, 0, MAX_PACKET_SIZE);
m_wBuf.buf = m_buffer;
m_wBuf.len = MAX_PACKET_SIZE;
Internal = 0;
InternalHigh = 0;
Offset = 0;
OffsetHigh = 0;
hEvent = nullptr;
m_opType = OP_READ;
}
private:
volatile LONG m_refCount;
};
class ServerData
{
public:
const static int MAX_REVEIVE_QUEUE_SIZE = 100;
const static int MAX_PACKET_SIZE = 2048;
const static int MAX_SEND_QUEUE_SIZE = 10;
const static int MAX_RECEIVE_QUEUE_SIZE = 100;
const static int MAX_OVERLAPPED_STRUCTS = 20;
ServerData(NetworkType netType, const string& sName, CCommunicationManager::CommHandle handle,
SOCKET sock, HANDLE IOPort) :
m_sName(sName)
{
InitializeCriticalSection(&m_receiveQueLock);
InitializeCriticalSection(&m_objectLock);
m_Handle = handle;
m_Socket = sock;
m_nIPAddress = 0;
m_netType = netType;
m_bEnabled = true;
m_ovlpIndex = 0;
for (int i = 0; i < MAX_OVERLAPPED_STRUCTS; ++i)
{
m_olps.push_back(new OverlappedEx);
}
/* Associate socket with completion handle */
if (m_Socket != 0)
{
CreateIoCompletionPort( reinterpret_cast<HANDLE>(m_Socket), IOPort, reinterpret_cast<ULONG_PTR>(this), 0 );
}
}
~ServerData()
{
CriticalSectionLocker lock(&m_receiveQueLock);
DeleteCriticalSection(&m_receiveQueLock);
DeleteCriticalSection(&m_objectLock);
closesocket(m_Socket);
}
const string& Name() const { return m_sName; }
bool Enabled() const { return m_bEnabled; }
void SetEnabled(bool bEnabled)
{
m_bEnabled = bEnabled;
}
int Handle() const { return m_Handle; }
void SetHandle(int handle)
{
m_Handle = handle;
}
unsigned long IPAddress() const { return m_nIPAddress; }
SOCKET Socket() const
{
return m_Socket;
}
void SetSocket(SOCKET sock)
{
m_Socket = sock;
}
void SetIPAddress(unsigned long nIP)
{
m_nIPAddress = nIP;
}
bool ValidTelegram(const vector<char>& telegram) const
{
return false;
}
OverlappedEx* GetBuffer()
{
OverlappedEx* ret = nullptr;
if (!m_olps.empty())
{
ret = m_olps.front();
m_olps.pop_front();
}
return ret;
}
void CompleteReceive(size_t numBytes, OverlappedEx* data)
{
//DEBUG_LOG1("%d buffers are available", AvailableBufferCount());
if (numBytes > 0)
{
vector<char> v(data->m_buffer, data->m_buffer + numBytes);
ReceivedData rd;
rd.SetData(v);
EnqueReceiveMessage(rd);
}
data->Release();
{
CriticalSectionLocker lock(&m_objectLock);
m_olps.push_back(data);
// DEBUG_LOG1("Queue size: %d", m_olps.size());
}
StartReceiving();
}
void CompleteSend(size_t numBytes, OverlappedEx* data)
{
data->Release();
{
CriticalSectionLocker lock(&m_objectLock);
m_olps.push_back(data);
//DEBUG_LOG1("Queue size: %d", m_olps.size());
}
//DEBUG_LOG2("Object: %s num sent: %d", Name().c_str(), numBytes);
}
void StartReceiving()
{
DWORD bytesRecv = 0;
sockaddr_in senderAddr;
DWORD flags = 0;
int senderAddrSize = sizeof(senderAddr);
int rc = 0;
CriticalSectionLocker lock(&m_objectLock);
auto olp = GetBuffer();
if (!olp)
{
if (...)
{
m_olps.push_back(new OverlappedEx);
olp = GetBuffer();
}
else
{
if (...)
{
DEBUG_LOG1("Name: %s ************* NO AVAILABLE BUFFERS - bailing ***************", Name().c_str());
}
return;
}
}
olp->Clear();
olp->m_opType = OverlappedEx::OP_READ;
olp->AddRef();
switch(GetNetworkType())
{
case UDP:
{
rc = WSARecvFrom(Socket(),
&olp->m_wBuf,
1,
&bytesRecv,
&flags,
(SOCKADDR *)&senderAddr,
&senderAddrSize, (OVERLAPPED*)olp, NULL);
}
break;
case TCP:
{
rc = WSARecv(Socket(),
&olp->m_wBuf,
1,
&bytesRecv,
&flags,
(OVERLAPPED*)olp, NULL);
}
break;
}
if (SOCKET_ERROR == rc)
{
DWORD err = WSAGetLastError();
if (err != WSA_IO_PENDING)
{
olp->Release();
m_olps.push_back(olp);
}
}
}
void SetWriteBuf(const SendData& msg, OverlappedEx* data)
{
int len = min(msg.Data().size(), MAX_PACKET_SIZE);
memcpy(data->m_buffer, &msg.Data()[0], len);
data->m_wBuf.buf = data->m_buffer;
data->m_wBuf.len = len;
}
void StartSending(const SendData& msg)
{
DEBUG_LOG1("device name: %s", Name().c_str());
int rc = 0;
DWORD bytesSent = 0;
DWORD flags = 0;
SOCKET sock = Socket();
int addrSize = sizeof(sockaddr_in);
CriticalSectionLocker lock(&m_objectLock);
//UpdateOverlapped(OverlappedEx::OP_WRITE);
auto olp = GetBuffer();
if (!olp)
{
if (...)
{
m_olps.push_back(new OverlappedEx);
olp = GetBuffer();
DEBUG_LOG2("name: %s ************* NO AVAILABLE BUFFERS new size: %d ***************", Name().c_str(), m_olps.size());
}
else
{
if (...)
{
DEBUG_LOG1("Name: %s ************* NO AVAILABLE BUFFERS - bailing ***************", Name().c_str());
}
return;
}
}
olp->Clear();
olp->m_opType = OverlappedEx::OP_WRITE;
olp->AddRef();
SetWriteBuf(msg, olp);
switch(GetNetworkType())
{
case UDP:
rc = WSASendTo(Socket(), &olp->m_wBuf, 1,
&bytesSent, flags, (sockaddr*)&msg.SendAddress(),
addrSize, (OVERLAPPED*)olp, NULL);
break;
case TCP:
rc = WSASend(Socket(), &olp->m_wBuf, 1,
&bytesSent, flags, (OVERLAPPED*)olp, NULL);
break;
}
if (SOCKET_ERROR == rc)
{
DWORD err = WSAGetLastError();
if (err != WSA_IO_PENDING)
{
olp->Release();
m_olps.push_back(olp);
}
}
}
size_t ReceiveQueueSize()
{
CriticalSectionLocker lock(&m_receiveQueLock);
return m_receiveDataQueue.size();
}
void GetAllData(vector <ReceivedData> & data)
{
CriticalSectionLocker lock(&m_receiveQueLock);
while (m_receiveDataQueue.size() > 0)
{
data.push_back(m_receiveDataQueue.front());
m_receiveDataQueue.pop_front();
}
}
void DequeReceiveMessage(ReceivedData& msg)
{
CriticalSectionLocker lock(&m_receiveQueLock);
if (m_receiveDataQueue.size() > 0)
{
msg = m_receiveDataQueue.front();
m_receiveDataQueue.pop_front();
}
}
template <class T>
void EnqueReceiveMessage(T&& data)
{
CriticalSectionLocker lock(&m_receiveQueLock);
if (m_receiveDataQueue.size() <= MAX_RECEIVE_QUEUE_SIZE)
{
m_receiveDataQueue.push_back(data);
}
else
{
static int s_nLogCount = 0;
if (s_nLogCount % 100 == 0)
{
DEBUG_LOG2("Max queue size was reached handle id: %d in %s", Handle(), Name().c_str());
}
s_nLogCount++;
}
}
NetworkType GetNetworkType() const
{
return m_netType;
}
private:
ServerData(const ServerData&);
ServerData& operator=(const ServerData&);
private:
bool m_bEnabled; //!< This member flags if this reciever is enabled for receiving incoming connections.
int m_Handle; //!< This member holds the handle for this receiver.
SOCKET m_Socket; //!< This member holds the socket information for this receiver.
unsigned long m_nIPAddress; //!< This member holds an IP address the socket is bound to.
deque < ReceivedData > m_receiveDataQueue;
CRITICAL_SECTION m_receiveQueLock;
CRITICAL_SECTION m_objectLock;
string m_sName;
NetworkType m_netType;
deque<OverlappedEx*> m_olps;
size_t m_ovlpIndex;
};
#endif
your implementation of void Release() have no sense - you decrement m_refCount and so what ? must be
void Release()
{
if (!InterlockedDecrement(&m_refCount)) delete this;
}
as result you never free OverlappedEx* data - this what i just view and this give memory leak.
also can advice - use WaitForSingleObject(CCommunicationManager::s_hShutdownEvent, 0); this is bad idea for detect shutdown. call only GetQueuedCompletionStatus and for shutdown call PostQueuedCompletionStatus(s_IOCompletionPort, 0, 0, 0) several times(number or threads listen on s_IOCompletionPort) and if thread view pOverlapped==0 - just exit.
use
OverlappedEx* data = static_cast<OverlappedEx*>(pOverlapped);
instead of reinterpret_cast
make ~OverlappedEx() private - it must not be direct called, only via Release
olp->Release();
m_olps.push_back(olp);
after you call Release() on object you must not it more access here, so or olp->Release() or m_olps.push_back(olp); but not both. this kill all logic of Release may be you need overwrite operator delete of OverlappedEx and inside it call m_olps.push_back(olp); and of course overwrite operator new too
again (OVERLAPPED*)olp - for what reinterpret_cast here ? because you inherit own struct from OVERLAPPED compiler auto do type cast here

How to asynchronously read/write in C++?

How do you copy one stream to another using dedicated read/write threads in C++?
Let's say I have these methods (not real, but to illustrate the point) to read/write data from. These read/write functions could represent anything (network/file/USB/serial/etc).
// returns the number of bytes read
void read(char* buffer, int bufferSize, int* bytesRead);
// returns the number of bytes written
void write(char* buffer, int bufferSize, int* bytesWritten);
The solution should also be portable.
NOTE: I am aware that Windows has a FILE_FLAG_OVERLAPPED feature, but this assumes that the read/write is file IO. Remember, these read/write methods could represent anything.
Here is the solution I came up with.
Header
#pragma once
#include <stdlib.h>
#include <queue>
#include <mutex>
#include <thread>
#include <chrono>
#include <list>
#include <thread>
#define ASYNC_COPY_READ_WRITE_SUCCESS 0
struct BufferBlock;
struct ReadStream
{
// read a stream to a buffer.
// return non-zero if error occured
virtual int read(char* buffer, int bufferSize, int* bytesRead) = 0;
};
struct WriteStream
{
// write a buffer to a stream.
// return non-zero if error occured
virtual int write(char* buffer, int bufferSize, int* bytesWritten) = 0;
};
class BufferBlockManager
{
public:
BufferBlockManager(int numberOfBlocks, int bufferSize);
~BufferBlockManager();
void enqueueBlockForRead(BufferBlock* block);
void dequeueBlockForRead(BufferBlock** block);
void enqueueBlockForWrite(BufferBlock* block);
void dequeueBlockForWrite(BufferBlock** block);
void resetState();
private:
std::list<BufferBlock*> blocks;
std::queue<BufferBlock*> blocksPendingRead;
std::queue<BufferBlock*> blocksPendingWrite;
std::mutex queueLock;
std::chrono::milliseconds dequeueSleepTime;
};
void AsyncCopyStream(BufferBlockManager* bufferBlockManager, ReadStream* readStream, WriteStream* writeStream, int* readResult, int* writeResult);
CPP
#include "AsyncReadWrite.h"
struct BufferBlock
{
BufferBlock(int bufferSize) : buffer(NULL)
{
this->bufferSize = bufferSize;
this->buffer = new char[bufferSize];
this->actualSize = 0;
this->isLastBlock = false;
}
~BufferBlock()
{
this->bufferSize = 0;
free(this->buffer);
this->buffer = NULL;
this->actualSize = 0;
}
char* buffer;
int bufferSize;
int actualSize;
bool isLastBlock;
};
BufferBlockManager::BufferBlockManager(int numberOfBlocks, int bufferSize)
{
dequeueSleepTime = std::chrono::milliseconds(100);
for (int x = 0; x < numberOfBlocks; x++)
{
BufferBlock* block = new BufferBlock(bufferSize);
blocks.push_front(block);
blocksPendingRead.push(block);
}
}
BufferBlockManager::~BufferBlockManager()
{
for (std::list<BufferBlock*>::const_iterator iterator = blocks.begin(), end = blocks.end(); iterator != end; ++iterator) {
delete (*iterator);
}
}
void BufferBlockManager::enqueueBlockForRead(BufferBlock* block)
{
queueLock.lock();
block->actualSize = 0;
block->isLastBlock = false;
blocksPendingRead.push(block);
queueLock.unlock();
}
void BufferBlockManager::dequeueBlockForRead(BufferBlock** block)
{
WAITFOR:
while (blocksPendingRead.size() == 0)
std::this_thread::sleep_for(dequeueSleepTime);
queueLock.lock();
if (blocksPendingRead.size() == 0)
{
queueLock.unlock();
goto WAITFOR;
}
*block = blocksPendingRead.front();
blocksPendingRead.pop();
queueLock.unlock();
}
void BufferBlockManager::enqueueBlockForWrite(BufferBlock* block)
{
queueLock.lock();
blocksPendingWrite.push(block);
queueLock.unlock();
}
void BufferBlockManager::dequeueBlockForWrite(BufferBlock** block)
{
WAITFOR:
while (blocksPendingWrite.size() == 0)
std::this_thread::sleep_for(dequeueSleepTime);
queueLock.lock();
if (blocksPendingWrite.size() == 0)
{
queueLock.unlock();
goto WAITFOR;
}
*block = blocksPendingWrite.front();
blocksPendingWrite.pop();
queueLock.unlock();
}
void BufferBlockManager::resetState()
{
queueLock.lock();
blocksPendingRead = std::queue<BufferBlock*>();
blocksPendingWrite = std::queue<BufferBlock*>();
for (std::list<BufferBlock*>::const_iterator iterator = blocks.begin(), end = blocks.end(); iterator != end; ++iterator) {
(*iterator)->actualSize = 0;
}
queueLock.unlock();
}
struct AsyncCopyContext
{
AsyncCopyContext(BufferBlockManager* bufferBlockManager, ReadStream* readStream, WriteStream* writeStream)
{
this->bufferBlockManager = bufferBlockManager;
this->readStream = readStream;
this->writeStream = writeStream;
this->readResult = ASYNC_COPY_READ_WRITE_SUCCESS;
this->writeResult = ASYNC_COPY_READ_WRITE_SUCCESS;
}
BufferBlockManager* bufferBlockManager;
ReadStream* readStream;
WriteStream* writeStream;
int readResult;
int writeResult;
};
void ReadStreamThread(AsyncCopyContext* asyncContext)
{
int bytesRead = 0;
BufferBlock* readBuffer = NULL;
int readResult = ASYNC_COPY_READ_WRITE_SUCCESS;
while (
// as long there hasn't been any write errors
asyncContext->writeResult == ASYNC_COPY_READ_WRITE_SUCCESS
// and we haven't had an error reading yet
&& readResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
// let's deque a block to read to!
asyncContext->bufferBlockManager->dequeueBlockForRead(&readBuffer);
readResult = asyncContext->readStream->read(readBuffer->buffer, readBuffer->bufferSize, &bytesRead);
readBuffer->actualSize = bytesRead;
readBuffer->isLastBlock = bytesRead == 0;
if (readResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
// this was a valid read, go ahead and queue it for writing
asyncContext->bufferBlockManager->enqueueBlockForWrite(readBuffer);
}
else
{
// an error occured reading
asyncContext->readResult = readResult;
// since an error occured, lets queue an block to write indicatiting we are done and there are no more bytes to read
readBuffer->isLastBlock = true;
readBuffer->actualSize = 0;
asyncContext->bufferBlockManager->enqueueBlockForWrite(readBuffer);
}
if (readBuffer->isLastBlock) return;
}
}
void WriteStreamThread(AsyncCopyContext* asyncContext)
{
int bytesWritten = 0;
BufferBlock* writeBuffer = NULL;
int writeResult = ASYNC_COPY_READ_WRITE_SUCCESS;
bool isLastWriteBlock = false;
while (
// as long as there are no errors during reading
asyncContext->readResult == ASYNC_COPY_READ_WRITE_SUCCESS
// and we haven't had an error writing yet
&& writeResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
// lets dequeue a block for writing!
asyncContext->bufferBlockManager->dequeueBlockForWrite(&writeBuffer);
isLastWriteBlock = writeBuffer->isLastBlock;
if (writeBuffer->actualSize > 0)
writeResult = asyncContext->writeStream->write(writeBuffer->buffer, writeBuffer->actualSize, &bytesWritten);
if (writeResult == ASYNC_COPY_READ_WRITE_SUCCESS)
{
asyncContext->bufferBlockManager->enqueueBlockForRead(writeBuffer);
if (isLastWriteBlock) return;
}
else
{
asyncContext->writeResult = writeResult;
asyncContext->bufferBlockManager->enqueueBlockForRead(writeBuffer);
return;
}
}
}
void AsyncCopyStream(BufferBlockManager* bufferBlockManager, ReadStream* readStream, WriteStream* writeStream, int* readResult, int* writeResult)
{
AsyncCopyContext asyncContext(bufferBlockManager, readStream, writeStream);
std::thread readThread(ReadStreamThread, &asyncContext);
std::thread writeThread(WriteStreamThread, &asyncContext);
readThread.join();
writeThread.join();
*readResult = asyncContext.readResult;
*writeResult = asyncContext.writeResult;
}
Usage
#include <stdio.h>
#include <tchar.h>
#include "AsyncReadWrite.h"
struct ReadTestStream : ReadStream
{
int readCount = 0;
int read(char* buffer, int bufferSize, int* bytesRead)
{
printf("Starting read...\n");
memset(buffer, bufferSize, 0);
if (readCount == 10)
{
*bytesRead = 0;
return 0;
}
// pretend this function takes a while!
std::this_thread::sleep_for(std::chrono::milliseconds(100));
char buff[100];
sprintf_s(buff, "This is read number %d\n", readCount);
strcpy_s(buffer, sizeof(buff), buff);
*bytesRead = strlen(buffer);
readCount++;
printf("Finished read...\n");
return 0;
}
};
struct WriteTestStream : WriteStream
{
int write(char* buffer, int bufferSize, int* bytesWritten)
{
printf("Starting write...\n");
// pretend this function takes a while!
std::this_thread::sleep_for(std::chrono::milliseconds(500));
printf(buffer);
printf("Finished write...\n");
return 0;
}
};
int _tmain(int argc, _TCHAR* argv[])
{
BufferBlockManager bufferBlockManager(5, 4096);
ReadTestStream readStream;
WriteTestStream writeStream;
int readResult = 0;
int writeResult = 0;
printf("Starting copy...\n");
AsyncCopyStream(&bufferBlockManager, &readStream, &writeStream, &readResult, &writeResult);
printf("Finished copy... readResult=%d writeResult=%d \n", readResult, writeResult);
getchar();
return 0;
}
EDIT: I put my solution into a GitHub repository here. If you wish to use this code, refer to the repository since it may be more updated than this answer.
Typically, you would just have one thread for each direction that alternates between reads and writes.

How to use LZMA SDK in C++?

i have difficulties in using LZMA SDK in my application.
I would like to create a kind of single file compression tool. I dont need any directory support, just need only the LZMA2 stream. But i have no idea on how LZMA SDK is to be used for this.
Please can anyone give me a little example on how the LZMA SDK can be used under C++?
I think that it's a properly little example to use LZMA SDK.
/* LzmaUtil.c -- Test application for LZMA compression
2008-08-05
Igor Pavlov
public domain */
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../LzmaDec.h"
#include "../LzmaEnc.h"
#include "../Alloc.h"
const char *kCantReadMessage = "Can not read input file";
const char *kCantWriteMessage = "Can not write output file";
const char *kCantAllocateMessage = "Can not allocate memory";
const char *kDataErrorMessage = "Data error";
static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); }
static void SzFree(void *p, void *address) { p = p; MyFree(address); }
static ISzAlloc g_Alloc = { SzAlloc, SzFree };
#define kInBufferSize (1 << 15)
#define kOutBufferSize (1 << 15)
unsigned char g_InBuffer[kInBufferSize];
unsigned char g_OutBuffer[kOutBufferSize];
size_t MyReadFile(FILE *file, void *data, size_t size)
{ return fread(data, 1, size, file); }
int MyReadFileAndCheck(FILE *file, void *data, size_t size)
{ return (MyReadFile(file, data, size) == size); }
size_t MyWriteFile(FILE *file, const void *data, size_t size)
{
if (size == 0)
return 0;
return fwrite(data, 1, size, file);
}
int MyWriteFileAndCheck(FILE *file, const void *data, size_t size)
{ return (MyWriteFile(file, data, size) == size); }
long MyGetFileLength(FILE *file)
{
long length;
fseek(file, 0, SEEK_END);
length = ftell(file);
fseek(file, 0, SEEK_SET);
return length;
}
void PrintHelp(char *buffer)
{
strcat(buffer, "\nLZMA Utility 4.58 Copyright (c) 1999-2008 Igor Pavlov 2008-04-11\n"
"\nUsage: lzma <e|d> inputFile outputFile\n"
" e: encode file\n"
" d: decode file\n");
}
int PrintError(char *buffer, const char *message)
{
strcat(buffer, "\nError: ");
strcat(buffer, message);
strcat(buffer, "\n");
return 1;
}
int PrintErrorNumber(char *buffer, SRes val)
{
sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val);
return 1;
}
int PrintUserError(char *buffer)
{
return PrintError(buffer, "Incorrect command");
}
#define IN_BUF_SIZE (1 << 16)
#define OUT_BUF_SIZE (1 << 16)
static int Decode(FILE *inFile, FILE *outFile, char *rs)
{
UInt64 unpackSize;
int thereIsSize; /* = 1, if there is uncompressed size in headers */
int i;
int res = 0;
CLzmaDec state;
/* header: 5 bytes of LZMA properties and 8 bytes of uncompressed size */
unsigned char header[LZMA_PROPS_SIZE + 8];
/* Read and parse header */
if (!MyReadFileAndCheck(inFile, header, sizeof(header)))
return PrintError(rs, kCantReadMessage);
unpackSize = 0;
thereIsSize = 0;
for (i = 0; i < 8; i++)
{
unsigned char b = header[LZMA_PROPS_SIZE + i];
if (b != 0xFF)
thereIsSize = 1;
unpackSize += (UInt64)b << (i * 8);
}
LzmaDec_Construct(&state);
res = LzmaDec_Allocate(&state, header, LZMA_PROPS_SIZE, &g_Alloc);
if (res != SZ_OK)
return res;
{
Byte inBuf[IN_BUF_SIZE];
Byte outBuf[OUT_BUF_SIZE];
size_t inPos = 0, inSize = 0, outPos = 0;
LzmaDec_Init(&state);
for (;;)
{
if (inPos == inSize)
{
inSize = MyReadFile(inFile, inBuf, IN_BUF_SIZE);
inPos = 0;
}
{
SizeT inProcessed = inSize - inPos;
SizeT outProcessed = OUT_BUF_SIZE - outPos;
ELzmaFinishMode finishMode = LZMA_FINISH_ANY;
ELzmaStatus status;
if (thereIsSize && outProcessed > unpackSize)
{
outProcessed = (SizeT)unpackSize;
finishMode = LZMA_FINISH_END;
}
res = LzmaDec_DecodeToBuf(&state, outBuf + outPos, &outProcessed,
inBuf + inPos, &inProcessed, finishMode, &status);
inPos += (UInt32)inProcessed;
outPos += outProcessed;
unpackSize -= outProcessed;
if (outFile != 0)
MyWriteFile(outFile, outBuf, outPos);
outPos = 0;
if (res != SZ_OK || thereIsSize && unpackSize == 0)
break;
if (inProcessed == 0 && outProcessed == 0)
{
if (thereIsSize || status != LZMA_STATUS_FINISHED_WITH_MARK)
res = SZ_ERROR_DATA;
break;
}
}
}
}
LzmaDec_Free(&state, &g_Alloc);
return res;
}
typedef struct _CFileSeqInStream
{
ISeqInStream funcTable;
FILE *file;
} CFileSeqInStream;
static SRes MyRead(void *p, void *buf, size_t *size)
{
if (*size == 0)
return SZ_OK;
*size = MyReadFile(((CFileSeqInStream*)p)->file, buf, *size);
/*
if (*size == 0)
return SZE_FAIL;
*/
return SZ_OK;
}
typedef struct _CFileSeqOutStream
{
ISeqOutStream funcTable;
FILE *file;
} CFileSeqOutStream;
static size_t MyWrite(void *pp, const void *buf, size_t size)
{
return MyWriteFile(((CFileSeqOutStream *)pp)->file, buf, size);
}
static SRes Encode(FILE *inFile, FILE *outFile, char *rs)
{
CLzmaEncHandle enc;
SRes res;
CFileSeqInStream inStream;
CFileSeqOutStream outStream;
CLzmaEncProps props;
enc = LzmaEnc_Create(&g_Alloc);
if (enc == 0)
return SZ_ERROR_MEM;
inStream.funcTable.Read = MyRead;
inStream.file = inFile;
outStream.funcTable.Write = MyWrite;
outStream.file = outFile;
LzmaEncProps_Init(&props);
res = LzmaEnc_SetProps(enc, &props);
if (res == SZ_OK)
{
Byte header[LZMA_PROPS_SIZE + 8];
size_t headerSize = LZMA_PROPS_SIZE;
UInt64 fileSize;
int i;
res = LzmaEnc_WriteProperties(enc, header, &headerSize);
fileSize = MyGetFileLength(inFile);
for (i = 0; i < 8; i++)
header[headerSize++] = (Byte)(fileSize >> (8 * i));
if (!MyWriteFileAndCheck(outFile, header, headerSize))
return PrintError(rs, "writing error");
if (res == SZ_OK)
res = LzmaEnc_Encode(enc, &outStream.funcTable, &inStream.funcTable,
NULL, &g_Alloc, &g_Alloc);
}
LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc);
return res;
}
int main2(int numArgs, const char *args[], char *rs)
{
FILE *inFile = 0;
FILE *outFile = 0;
char c;
int res;
int encodeMode;
if (numArgs == 1)
{
PrintHelp(rs);
return 0;
}
if (numArgs < 3 || numArgs > 4 || strlen(args[1]) != 1)
return PrintUserError(rs);
c = args[1][0];
encodeMode = (c == 'e' || c == 'E');
if (!encodeMode && c != 'd' && c != 'D')
return PrintUserError(rs);
{
size_t t4 = sizeof(UInt32);
size_t t8 = sizeof(UInt64);
if (t4 != 4 || t8 != 8)
return PrintError(rs, "LZMA UTil needs correct UInt32 and UInt64");
}
inFile = fopen(args[2], "rb");
if (inFile == 0)
return PrintError(rs, "Can not open input file");
if (numArgs > 3)
{
outFile = fopen(args[3], "wb+");
if (outFile == 0)
return PrintError(rs, "Can not open output file");
}
else if (encodeMode)
PrintUserError(rs);
if (encodeMode)
{
res = Encode(inFile, outFile, rs);
}
else
{
res = Decode(inFile, outFile, rs);
}
if (outFile != 0)
fclose(outFile);
fclose(inFile);
if (res != SZ_OK)
{
if (res == SZ_ERROR_MEM)
return PrintError(rs, kCantAllocateMessage);
else if (res == SZ_ERROR_DATA)
return PrintError(rs, kDataErrorMessage);
else
return PrintErrorNumber(rs, res);
}
return 0;
}
int MY_CDECL main(int numArgs, const char *args[])
{
char rs[800] = { 0 };
int res = main2(numArgs, args, rs);
printf(rs);
return res;
}
Also you can see it at:
http://read.pudn.com/downloads151/sourcecode/zip/656407/7z460/C/LzmaUtil/LzmaUtil.c__.htm
http://read.pudn.com/downloads157/sourcecode/zip/698262/LZMA/LzmaUtil.c__.htm
I recently found a nice example, written in C++. Credit goes to GH user Treeki who published the original gist:
// note: -D_7ZIP_ST is required when compiling on non-Windows platforms
// g++ -o lzma_sample -std=c++14 -D_7ZIP_ST lzma_sample.cpp LzmaDec.c LzmaEnc.c LzFind.c
#include <stdio.h>
#include <stdint.h>
#include <string.h>
#include <memory>
#include "LzmaEnc.h"
#include "LzmaDec.h"
static void *_lzmaAlloc(ISzAllocPtr, size_t size) {
return new uint8_t[size];
}
static void _lzmaFree(ISzAllocPtr, void *addr) {
if (!addr)
return;
delete[] reinterpret_cast<uint8_t *>(addr);
}
static ISzAlloc _allocFuncs = {
_lzmaAlloc, _lzmaFree
};
std::unique_ptr<uint8_t[]> lzmaCompress(const uint8_t *input, uint32_t inputSize, uint32_t *outputSize) {
std::unique_ptr<uint8_t[]> result;
// set up properties
CLzmaEncProps props;
LzmaEncProps_Init(&props);
if (inputSize >= (1 << 20))
props.dictSize = 1 << 20; // 1mb dictionary
else
props.dictSize = inputSize; // smaller dictionary = faster!
props.fb = 40;
// prepare space for the encoded properties
SizeT propsSize = 5;
uint8_t propsEncoded[5];
// allocate some space for the compression output
// this is way more than necessary in most cases...
// but better safe than sorry
// (a smarter implementation would use a growing buffer,
// but this requires a bunch of fuckery that is out of
/// scope for this simple example)
SizeT outputSize64 = inputSize * 1.5;
if (outputSize64 < 1024)
outputSize64 = 1024;
auto output = std::make_unique<uint8_t[]>(outputSize64);
int lzmaStatus = LzmaEncode(
output.get(), &outputSize64, input, inputSize,
&props, propsEncoded, &propsSize, 0,
NULL,
&_allocFuncs, &_allocFuncs);
*outputSize = outputSize64 + 13;
if (lzmaStatus == SZ_OK) {
// tricky: we have to generate the LZMA header
// 5 bytes properties + 8 byte uncompressed size
result = std::make_unique<uint8_t[]>(outputSize64 + 13);
uint8_t *resultData = result.get();
memcpy(resultData, propsEncoded, 5);
for (int i = 0; i < 8; i++)
resultData[5 + i] = (inputSize >> (i * 8)) & 0xFF;
memcpy(resultData + 13, output.get(), outputSize64);
}
return result;
}
std::unique_ptr<uint8_t[]> lzmaDecompress(const uint8_t *input, uint32_t inputSize, uint32_t *outputSize) {
if (inputSize < 13)
return NULL; // invalid header!
// extract the size from the header
UInt64 size = 0;
for (int i = 0; i < 8; i++)
size |= (input[5 + i] << (i * 8));
if (size <= (256 * 1024 * 1024)) {
auto blob = std::make_unique<uint8_t[]>(size);
ELzmaStatus lzmaStatus;
SizeT procOutSize = size, procInSize = inputSize - 13;
int status = LzmaDecode(blob.get(), &procOutSize, &input[13], &procInSize, input, 5, LZMA_FINISH_END, &lzmaStatus, &_allocFuncs);
if (status == SZ_OK && procOutSize == size) {
*outputSize = size;
return blob;
}
}
return NULL;
}
void hexdump(const uint8_t *buf, int size) {
int lines = (size + 15) / 16;
for (int i = 0; i < lines; i++) {
printf("%08x | ", i * 16);
int lineMin = i * 16;
int lineMax = lineMin + 16;
int lineCappedMax = (lineMax > size) ? size : lineMax;
for (int j = lineMin; j < lineCappedMax; j++)
printf("%02x ", buf[j]);
for (int j = lineCappedMax; j < lineMax; j++)
printf(" ");
printf("| ");
for (int j = lineMin; j < lineCappedMax; j++) {
if (buf[j] >= 32 && buf[j] <= 127)
printf("%c", buf[j]);
else
printf(".");
}
printf("\n");
}
}
void testIt(const uint8_t *input, int size) {
printf("Test Input:\n");
hexdump(input, size);
uint32_t compressedSize;
auto compressedBlob = lzmaCompress(input, size, &compressedSize);
if (compressedBlob) {
printf("Compressed:\n");
hexdump(compressedBlob.get(), compressedSize);
} else {
printf("Nope, we screwed it\n");
return;
}
// let's try decompressing it now
uint32_t decompressedSize;
auto decompressedBlob = lzmaDecompress(compressedBlob.get(), compressedSize, &decompressedSize);
if (decompressedBlob) {
printf("Decompressed:\n");
hexdump(decompressedBlob.get(), decompressedSize);
} else {
printf("Nope, we screwed it (part 2)\n");
return;
}
printf("----------\n");
}
void testIt(const char *string) {
testIt((const uint8_t *)string, strlen(string));
}
int main(int argc, char **argv) {
testIt("a");
testIt("here is a cool string");
testIt("here's something that should compress pretty well: abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdef");
return 0;
}
You can refer to this file on how to use lzma2。
https://github.com/Tencent/libpag/blob/aab6391e455193c8ec5b8e2031b495b3fe77b034/test/framework/utils/LzmaUtil.cpp
/////////////////////////////////////////////////////////////////////////////////////////////////
//
// Tencent is pleased to support the open source community by making libpag available.
//
// Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// unless required by applicable law or agreed to in writing, software distributed under the
// license is distributed on an "as is" basis, without warranties or conditions of any kind,
// either express or implied. see the license for the specific language governing permissions
// and limitations under the license.
//
/////////////////////////////////////////////////////////////////////////////////////////////////
#include "LzmaUtil.h"
#include "test/framework/lzma/Lzma2DecMt.h"
#include "test/framework/lzma/Lzma2Enc.h"
namespace pag {
static void* LzmaAlloc(ISzAllocPtr, size_t size) {
return new uint8_t[size];
}
static void LzmaFree(ISzAllocPtr, void* address) {
if (!address) {
return;
}
delete[] reinterpret_cast<uint8_t*>(address);
}
static ISzAlloc gAllocFuncs = {LzmaAlloc, LzmaFree};
class SequentialOutStream {
public:
virtual ~SequentialOutStream() = default;
virtual bool write(const void* data, size_t size) = 0;
};
class SequentialInStream {
public:
virtual ~SequentialInStream() = default;
virtual bool read(void* data, size_t size, size_t* processedSize) = 0;
};
struct CSeqInStreamWrap {
ISeqInStream vt;
std::unique_ptr<SequentialInStream> inStream;
};
struct CSeqOutStreamWrap {
ISeqOutStream vt;
std::unique_ptr<SequentialOutStream> outStream;
};
class BuffPtrInStream : public SequentialInStream {
public:
explicit BuffPtrInStream(const uint8_t* buffer, size_t bufferSize)
: buffer(buffer), bufferSize(bufferSize) {
}
bool read(void* data, size_t size, size_t* processedSize) override {
if (processedSize) {
*processedSize = 0;
}
if (size == 0 || position >= bufferSize) {
return true;
}
auto remain = bufferSize - position;
if (remain > size) {
remain = size;
}
memcpy(data, static_cast<const uint8_t*>(buffer) + position, remain);
position += remain;
if (processedSize) {
*processedSize = remain;
}
return true;
}
private:
const uint8_t* buffer = nullptr;
size_t bufferSize = 0;
size_t position = 0;
};
class VectorOutStream : public SequentialOutStream {
public:
explicit VectorOutStream(std::vector<uint8_t>* buffer) : buffer(buffer) {
}
bool write(const void* data, size_t size) override {
auto oldSize = buffer->size();
buffer->resize(oldSize + size);
memcpy(&(*buffer)[oldSize], data, size);
return true;
}
private:
std::vector<uint8_t>* buffer;
};
class BuffPtrSeqOutStream : public SequentialOutStream {
public:
BuffPtrSeqOutStream(uint8_t* buffer, size_t size) : buffer(buffer), bufferSize(size) {
}
bool write(const void* data, size_t size) override {
auto remain = bufferSize - position;
if (remain > size) {
remain = size;
}
if (remain != 0) {
memcpy(buffer + position, data, remain);
position += remain;
}
return remain != 0 || size == 0;
}
private:
uint8_t* buffer = nullptr;
size_t bufferSize = 0;
size_t position = 0;
};
static const size_t kStreamStepSize = 1 << 31;
static SRes MyRead(const ISeqInStream* p, void* data, size_t* size) {
CSeqInStreamWrap* wrap = CONTAINER_FROM_VTBL(p, CSeqInStreamWrap, vt);
auto curSize = (*size < kStreamStepSize) ? *size : kStreamStepSize;
if (!wrap->inStream->read(data, curSize, &curSize)) {
return SZ_ERROR_READ;
}
*size = curSize;
return SZ_OK;
}
static size_t MyWrite(const ISeqOutStream* p, const void* buf, size_t size) {
auto* wrap = CONTAINER_FROM_VTBL(p, CSeqOutStreamWrap, vt);
if (wrap->outStream->write(buf, size)) {
return size;
}
return 0;
}
class Lzma2Encoder {
public:
Lzma2Encoder() {
encoder = Lzma2Enc_Create(&gAllocFuncs, &gAllocFuncs);
}
~Lzma2Encoder() {
Lzma2Enc_Destroy(encoder);
}
std::shared_ptr<Data> code(const std::shared_ptr<Data>& inputData) {
if (encoder == nullptr || inputData == nullptr || inputData->size() == 0) {
return nullptr;
}
auto inputSize = inputData->size();
CLzma2EncProps lzma2Props;
Lzma2EncProps_Init(&lzma2Props);
lzma2Props.lzmaProps.dictSize = inputSize;
lzma2Props.lzmaProps.level = 9;
lzma2Props.numTotalThreads = 4;
Lzma2Enc_SetProps(encoder, &lzma2Props);
std::vector<uint8_t> outBuf;
outBuf.resize(1 + 8);
outBuf[0] = Lzma2Enc_WriteProperties(encoder);
for (int i = 0; i < 8; i++) {
outBuf[1 + i] = static_cast<uint8_t>(inputSize >> (8 * i));
}
CSeqInStreamWrap inWrap = {};
inWrap.vt.Read = MyRead;
inWrap.inStream = std::make_unique<BuffPtrInStream>(
static_cast<const uint8_t*>(inputData->data()), inputSize);
CSeqOutStreamWrap outStream = {};
outStream.vt.Write = MyWrite;
outStream.outStream = std::make_unique<VectorOutStream>(&outBuf);
auto status =
Lzma2Enc_Encode2(encoder, &outStream.vt, nullptr, nullptr, &inWrap.vt, nullptr, 0, nullptr);
if (status != SZ_OK) {
return nullptr;
}
return Data::MakeWithCopy(&outBuf[0], outBuf.size());
}
private:
CLzma2EncHandle encoder = nullptr;
};
std::shared_ptr<Data> LzmaUtil::Compress(const std::shared_ptr<Data>& pixelData) {
Lzma2Encoder encoder;
return encoder.code(pixelData);
}
class Lzma2Decoder {
public:
Lzma2Decoder() {
decoder = Lzma2DecMt_Create(&gAllocFuncs, &gAllocFuncs);
}
~Lzma2Decoder() {
if (decoder) {
Lzma2DecMt_Destroy(decoder);
}
}
std::shared_ptr<Data> code(const std::shared_ptr<Data>& inputData) {
if (decoder == nullptr || inputData == nullptr || inputData->size() == 0) {
return nullptr;
}
auto input = static_cast<const uint8_t*>(inputData->data());
auto inputSize = inputData->size() - 9;
Byte prop = static_cast<const Byte*>(input)[0];
CLzma2DecMtProps props;
Lzma2DecMtProps_Init(&props);
props.inBufSize_ST = inputSize;
props.numThreads = 1;
UInt64 outBufferSize = 0;
for (int i = 0; i < 8; i++) {
outBufferSize |= (input[1 + i] << (i * 8));
}
auto outBuffer = new uint8_t[outBufferSize];
CSeqInStreamWrap inWrap = {};
inWrap.vt.Read = MyRead;
inWrap.inStream = std::make_unique<BuffPtrInStream>(input + 9, inputSize);
CSeqOutStreamWrap outWrap = {};
outWrap.vt.Write = MyWrite;
outWrap.outStream = std::make_unique<BuffPtrSeqOutStream>(outBuffer, outBufferSize);
UInt64 inProcessed = 0;
int isMT = false;
auto res = Lzma2DecMt_Decode(decoder, prop, &props, &outWrap.vt, &outBufferSize, 1, &inWrap.vt,
&inProcessed, &isMT, nullptr);
if (res == SZ_OK && inputSize == inProcessed) {
return Data::MakeAdopted(outBuffer, outBufferSize, Data::DeleteProc);
}
delete[] outBuffer;
return nullptr;
}
private:
CLzma2DecMtHandle decoder = nullptr;
};
std::shared_ptr<Data> LzmaUtil::Decompress(const std::shared_ptr<Data>& data) {
Lzma2Decoder decoder;
return decoder.code(data);
}
} // namespace pag