i need to make a program that will handle binary data, lots of it.
in short, the C++ program will load a binary file (some of them exceed 20mb) into a buffer.
then it will run a Lua script and pass all this loaded data to the script, which will do some manipulation and return the result to the C++ program.
I need to do this as quickly as possible, perform at the best and get the job done faster.
A while ago I already made this program using the conventional methods of Lua, but it was extremely slow.
So I lost the files, and now I want to redo it in a better, faster way that doesn't compromise performance.
Searching a bit, I found this.
I had to make some small changes to adapt to the new version of Lua, but I can't get it to work.
Can you help me with this?
And if there's a better way to do the job I said, what would it be?
#include "stdafx.h"
// metatable method for handling "array[index]"
static int array_index(lua_State* L) {
int** parray = (int**)luaL_checkudata(L, 1, "array");
int index = luaL_checkinteger(L, 2);
lua_pushnumber(L, (*parray)[index - 1]);
return 1;
}
// metatable method for handle "array[index] = value"
static int array_newindex(lua_State* L) {
int** parray = (int**)luaL_checkudata(L, 1, "array");
int index = luaL_checkinteger(L, 2);
int value = luaL_checkinteger(L, 3);
(*parray)[index - 1] = value;
return 0;
}
// create a metatable for our array type
static void create_array_type(lua_State* L) {
static const struct luaL_Reg array[] = {
{ "__index", array_index },
{ "__newindex", array_newindex },
NULL, NULL
};
luaL_newmetatable(L, "array");
luaL_setfuncs(L, array, 0);
}
// expose an array to lua, by storing it in a userdata with the array metatable
static int expose_array(lua_State* L, int array[]) {
int** parray = (int**)lua_newuserdata(L, sizeof(int**));
*parray = array;
luaL_getmetatable(L, "array");
lua_setmetatable(L, -2);
return 1;
}
// test data
int mydata[] = { 1, 2, 3, 4 };
// test routine which exposes our test array to Lua
static int getarray(lua_State* L) {
return expose_array(L, mydata);
}
int __declspec(dllexport) __cdecl luaopen_array(lua_State* L) {
create_array_type(L);
// make our test routine available to Lua
lua_register(L, "array", getarray);
return 0;
}
int main()
{
lua_State* L = luaL_newstate();
luaL_dostring(L, "require 'array'");
luaL_dostring(L, "foo = array()");
luaL_dostring(L, "x = foo[1]");
lua_getglobal(L, "x");
lua_Number x = lua_tonumber(L, 1);
printf("foo[1] = %d\n", (int)x);
}
Consider using lightuserdata to avoid copying file contents excessively.
lightuserdata is just a pointer, so you need to define some methods to work with it as well.
The idea looks like this:
#include <lauxlib.h>
#include <lualib.h>
#define BIN_DATA_MT_ID "bin data"
int get_byte(lua_State *L) {
const char *file_contents = luaL_checkudata(L, 1, BIN_DATA_MT_ID);
size_t byte_index = luaL_checknumber(L, 2);
// checking OOB is your custody here, omitted for simplicity
lua_pushlstring(L, file_contents + byte_index, 1);
return 1;
}
static const luaL_Reg __index[] = {
{"get_byte", get_byte},
{NULL, NULL}
};
int main() {
const char file_contents[4] = { 0x25, 0xAA, 0xBB, 0xCC };
lua_State *L = luaL_newstate();
luaopen_base(L); // adds "print" function
lua_pushlightuserdata(L, (void *)file_contents);
luaL_newmetatable(L, BIN_DATA_MT_ID);
luaL_newlib(L, __index);
lua_setfield(L, -2, "__index");
lua_setmetatable(L, -2);
lua_setglobal(L, "mybindata");
luaL_dostring(L, "print(mybindata:get_byte(0))");
lua_close(L);
return 0;
}
The fastest way for Lua to access the bytes of a large byte array is to expose that array directly as a string within Lua. Now because Lua does reference counting for strings, this means that Lua will always allocate its own storage for the string. So to efficiently load the string into Lua (ie: avoiding a 20+MiB copy), you need to use the lua_Buffer-based API to load it directly into Lua's storage.
But outside of that quirk, it will certainly be faster inside Lua to use array accesses of a string to access bytes compared to doing a function call for each byte accessed from the buffer.
I need an idea, how I can store lua closures to invoke them asynchronously later.
my first idea was lua_tocfunction but a closure is not a cfunction and cannot be invoked from C directly
second idea was to save the closure in the metatable, that I can push it and call it later, but it seems, that I cannot copy a closure. (Error: attempt to index a function value).
So I need your help please. How can I store a closure?
I admit, that I did not completely understand why there is an __index field in my lua ctor as I've copied that part from somewhere.
By the way: the program without onrender worked as expected. I'm using qt gui and the lua-states are closed, after qt's main loop, thus the created window is not going to be delete by __gc after the script.
bootstrap.lua
local w = w_render() -- create window object
w:show()
w:onrender(function()
print('render')
end)
w_lua.cpp
// chlua_* are helper macros/templates/methods
// 1: self
// 2: render closure
int w_render_onrender(lua_State *L) {
auto *self = chlua_this<GLWindow *>(L, 1, w_render_table);
lua_pushvalue(L, 2); // copy closure to top
lua_setfield(L, 2, "onrender_cb"); // save closure in metatable
// !!! ERROR: attempt to index a function value
self->onrender([L](){
lua_getfield(L, 2, "onrender_cb");
qDebug() << "onrender";
lua_call(L, 0, 0);
});
return 0;
}
// Creates the object
int w_render(lua_State *L) {
auto *&self = chlua_newuserdata<GLWindow *>(L);
self = new GLWindow;
if (luaL_newmetatable(L, w_render_table)) {
luaL_setfuncs(L, w_render_methods, 0);
lua_pushvalue(L, -1);
lua_setfield(L, -2, "__index");
}
lua_setmetatable(L, -2);
return 1;
}
It looks like your problem is stemming from using the wrong indices and attempting to set/get fields on the wrong lua object on the stack. Assuming the udata representing your GLWindow * is first followed by the lua closure second, try changing the code like this:
int w_render_onrender(lua_State *L)
{
luaL_checkudata(L, 1, w_render_table);
luaL_checktype(L, 2, LUA_TFUNCTION);
auto *self = chlua_this<GLWindow *>(L, 1, w_render_table);
lua_getmetatable(L, 1);
lua_insert(L, -2); // GLWindow GLWindow_mt lua_closure
lua_setfield(L, -2, "onrender_cb"); // save closure in metatable
self->onrender([L]()
{
luaL_checkudata(L, 1, w_render_table);
// assuming GLWindow udata is self and onrender_cb is your lua closure above
// access GLWindow.onrender_cb through GLWindows's metatable
lua_getfield(L, 1, "onrender_cb");
qDebug() << "onrender";
luaL_checktype(L, -1, LUA_TFUNCTION); // Just to be sure
lua_call(L, 0, 0);
});
return 0;
}
Edit: After thinking about this some more, it probably makes more sense to create a lua reference using luaL_ref. This way you don't have to care what happens to be on the stack when self->onrender actually runs, which I'm assuming is async:
int w_render_onrender(lua_State *L)
{
luaL_checkudata(L, 1, w_render_table);
luaL_checktype(L, 2, LUA_TFUNCTION);
auto *self = chlua_this<GLWindow *>(L, 1, w_render_table);
auto lua_cb = luaL_ref(L, LUA_REGISTRYINDEX);
// just to check that what's on the stack shouldn't matter
lua_settop(L, 0);
self->onrender([L, lua_cb]()
{
lua_rawgeti(L, LUA_REGISTRYINDEX, lua_cb);
luaL_checktype(L, -1, LUA_TFUNCTION); // Just to be sure
qDebug() << "onrender";
lua_call(L, 0, 0);
luaL_unref(L, LUA_REGISTRYINDEX, lua_cb); // assuming you're done with it
});
return 0;
}
Closed. This question does not meet Stack Overflow guidelines. It is not currently accepting answers.
We don’t allow questions seeking recommendations for books, tools, software libraries, and more. You can edit the question so it can be answered with facts and citations.
Closed 6 years ago.
Improve this question
I've been working on this problem off and on for a few months, and now wanted to really come up with a proper solution that will handle the case of creating new user-defined classes (and instances of those classes) with member functions/properties at run-time in a C++11 project.
So far, I've been using SWIG (formerly with Python, now with Lua, exploring Squirrel). Like all the C++ binding/embedding libraries I've encountered so far (Luna*, luabinder, luabind, OOLua, Sqrat/Sqext, Squall), all expect your classes to be predefined in C++ prior to code execution because they either rely on preprocessor directives or templates.
So my question is, are there any libraries out there that use a more procedural approach to wrapping a language, or are there any good tutorials/examples for something like Lua or Squirrel, that one would recommend for handling the creation of custom-named classes with custom members and functions? Some direction would be greatly appreciated.
Even simply a good example showing how to create a custom class with a function and a property, in either Lua, Squirrel, via their respective C++ APIs without the use of macros/templates/dynamically-generated code, would be hugely helpful.
EDIT: I have gone as far as creating an Instance class that contains a std::vector of members key/value pairs, and a member identifying the type so functions can be looked up. However, there is very little documentation out there on creating simple classes in Lua/Squirrel without the use of static code.
EDIT 2: I would like a solution that works on any platform and without having to dynamically link.
Creating a class derived from some existing C++ class is the only way (known to me) to bring a new class into a running C++ program. Short of dynamically compiling actual C++ source and loading the resulting library, there is no way to physically add a new class. The next best thing is to create a proxy object in C++ that wraps a Python (Lua etc) object, and make that Python (Lua) object an instance of a class that extends an existing C++ class mirrored to the Python (Lua) side.
C++
+---------+ mirrors +--------------+
| class X | ...............................> | class X |
+---------+ | mirrored to |
| | Python |
| inherits +--------------+
v inherits |
+-----------------+ v
| class X_Wrapper | references +--------------+
| | python obj -------------------------> | class CX(X): |
+-----------------+ | def met() |
+--------------+
Here's an example of extending a C++ class with Python, using boost::python as a bridge.
C++ side:
#include <boost/python.hpp>
#include <iostream>
using namespace boost::python;
// this is the interface we will implement in Python
struct World
{
virtual std::string greet() = 0;
virtual ~World() {}
};
// this is a helper class needed to access Python-overrided methods
struct WorldWrap : World, wrapper<World>
{
std::string greet()
{
return this->get_override("greet")();
}
};
// This function tests our class implemented in Python
std::string test(World* w)
{
std::cout << "Calling w->greet() on some World-derived object\n";
return w->greet();
}
// This is what the Python side will see
BOOST_PYTHON_MODULE(hello)
{
class_<WorldWrap, boost::noncopyable>("World")
.def("greet", pure_virtual(&World::greet));
def("test", test);
}
Python side:
import hello
class HomeWorld(hello.World):
""" Implements a function defined in C++ as pure virtual """
def greet(self):
return "howdy"
home = HomeWorld()
print (hello.test(home))
Consider following Lua multimap example.
Multimap = {};
function Multimap:__index(key)
if (key == 'keys') then
local ret = {}
for k,_ in pairs(self) do
ret[#ret+1] = k;
end
return ret;
else
return rawget(getmetatable(self), key)
end
end
function Multimap.Create()
local self = {};
setmetatable(self, Multimap);
return self;
end
function Multimap:Insert(key, value)
local list = self[key];
if (list == nil) then
list = {};
self[key] = list;
end
table.insert(list, value);
end
function Multimap:Remove(key, value)
local list = self[key];
assert(list ~= nil, "key not found");
for i = 1,#list do
if (list[i] == value) then
table.remove(list, i);
if (#list == 0) then
self[key] = nil;
end
return;
end
end
error("value not found");
end
-- testing
m = Multimap.Create()
m:Insert(1,5)
m:Insert(2,6)
m:Insert(3,7)
m:Insert(1,8)
m:Remove(2,6)
print(pcall(function()
m:Remove(2,6) -- will produce assert exception
end))
print("keys left: ", table.concat(m.keys, ','))
You can implement this in C++ in several ways.
Use heavy Lua API. The code below is almost exact to Lua.
#include <Lua/src/lua.hpp>
int Multimap_Index(lua_State* L) {
lua_settop(L, 2); // force 2 arguments
const char *key_value = "key";
size_t key_len;
const char *key = lua_tolstring(L, 2, &key_len);
if (!strncmp(key, key_value, strlen(key_value))) {
int i = 0;
lua_newtable(L); // stack : self, key, ret = {}
int ret = lua_gettop(L);
lua_pushnil(L); // stack : self, key, ret, nil
while (lua_next(L, 1) != 0) { // stack : self, key, ret, k, v
lua_pop(L, 1); // stack : self, key, ret, k
lua_len(L, ret); // stack : self, key, ret, k, #ret
lua_pushvalue(L, -2); // stack : self, key, ret, k, #ret, k
lua_rawseti(L, ret, lua_tointeger(L, -2)+1); // ret[#ret+1] = k ; || stack : self, key, ret, k, #ret
lua_pop(L, 1); // stack : self, key, ret, k
}
// stack : self, key, ret
return 1;
}
else {
lua_getmetatable(L, 1); // stack : self, key, metatable(self)
lua_pushvalue(L, 2); // stack : self, key, metatable(self), key
lua_rawget(L, -2); // stack : self, key, metatable(self), rawget(metatable(self), key)
return 1;
}
}
int Multimap_Remove(lua_State* L) {
lua_settop(L, 3); // force 3 arguments: self, key, value
lua_checkstack(L, 12); // reserve 12 arguments on stack (just in case)
lua_pushvalue(L, 2); // stack: self, key, value, key
lua_gettable(L, 1); // stack: self, key, value, list = self[key]
if (lua_isnil(L, -1))
luaL_error(L, "key not found");
lua_len(L, -1); // stack: self, key, value, list, #list
int count = lua_tointeger(L, -1);
lua_pop(L, 1); // stack: self, key, value, list
for (int i = 1; i <= count; ++i) {
lua_rawgeti(L, -1, i); // stack: self, key, value, list, v = list[i]
if (lua_compare(L, 3, 5, LUA_OPEQ)) { // if (list[i] == value)
lua_getglobal(L, "table"); // stack : self, key, value, list, v, table
lua_getfield(L, -1, "remove"); // stack : self, key, value, list, v, table, table.remove
lua_pushvalue(L, 4);
lua_pushinteger(L, i); // stack : self, key, value, list, v, table, table.remove, list, i
lua_call(L, 2, 0); // table.remove(list, i); || stack : self, key, value, list, v, table
lua_pushnil(L);
if (lua_next(L, 4) == 0) { // if list is empty table
lua_pushvalue(L, 2);
lua_pushnil(L);
lua_settable(L, 1); // self[key] = nil
}
return 0;
}
}
luaL_error(L, "value not found");
}
int main() {
auto L = luaL_newstate();
luaL_openlibs(L);
lua_newtable(L);
int Multimap = lua_gettop(L); // Multimap = {}
lua_pushvalue(L, Multimap);
lua_setglobal(L, "Multimap"); // _G.Multimap = Multimap;
// option 1: create a C function for operation
// Multimap.__index = &Multimap_Index
lua_pushcfunction(L, Multimap_Index);
lua_setfield(L, Multimap, "__index");
// option 2: compile Lua code and use it
luaL_loadstring(L,
"local self = {};\n"
"setmetatable(self, Multimap);\n"
"return self;\n"
);
lua_setfield(L, Multimap, "Create"); // Multimap.Create = &Multimap_Create
luaL_loadstring(L,
"local self, key, value = ...;\n" // initialize local variables from parameters here
"local list = self[key];\n"
"if (list == nil) then\n"
" list = {};\n"
" self[key] = list;\n"
"end\n"
"table.insert(list, value);\n"
);
lua_setfield(L, Multimap, "Insert"); // Multimap.Create = &Multimap_Insert
lua_pushcfunction(L, Multimap_Remove);
lua_setfield(L, Multimap, "Remove"); // Multimap.Create = &Multimap_Remove
lua_getfield(L, Multimap, "Create");
lua_call(L, 0, 1);
int m = lua_gettop(L);
lua_getfield(L, m, "Insert"); // stack : m, m.insert
int Insert = lua_gettop(L);
// m.Insert(m, 1, 5)
lua_pushvalue(L, Insert);
lua_pushvalue(L, m);
lua_pushinteger(L, 1);
lua_pushinteger(L, 5);
lua_call(L, 3, 0);
// m.Insert(m, 2, 6)
lua_pushvalue(L, Insert);
lua_pushvalue(L, m);
lua_pushinteger(L, 2);
lua_pushinteger(L, 6);
lua_call(L, 3, 0);
// m.Insert(m, 3, 7)
lua_pushvalue(L, Insert);
lua_pushvalue(L, m);
lua_pushinteger(L, 3);
lua_pushinteger(L, 7);
lua_call(L, 3, 0);
// m.Insert(m, 1, 8)
lua_pushvalue(L, Insert);
lua_pushvalue(L, m);
lua_pushinteger(L, 1);
lua_pushinteger(L, 8);
lua_call(L, 3, 0);
// m.Remove(m, 2, 6)
lua_getfield(L, m, "Remove");
lua_pushvalue(L, m);
lua_pushinteger(L, 2);
lua_pushinteger(L, 6);
lua_call(L, 3, 0);
// m.Remove(m, 2, 6)
lua_getfield(L, m, "Remove");
lua_pushvalue(L, m);
lua_pushinteger(L, 2);
lua_pushinteger(L, 6);
lua_pcall(L, 3, 0, 0);
printf("%s\n", lua_tostring(L, -1));
lua_getglobal(L, "table");
lua_getfield(L, -1, "concat");
lua_getfield(L, m, "keys");
lua_pushstring(L, ",");
lua_call(L, 2, 1);
printf("keys left: %s\n", lua_tostring(L, -1));
lua_close(L);
return 0;
}
OR you can use Lua userdata that uses std::multimap (I would need another hour to implement this, so ask if you really need that -- that doesn't follow from your question)
Disclaimer: I'm posting this contribution as an answer because I don't have enough reputation points to add a comment.
Comment: Setting aside the problematic of binding with a specific scripting language, it seems that you are facing a fundamental limitation of the C++ language: it is not "dynamic" (as pointed out by other comments). That is, the language does not provide any functionality to extend or modify a compiled program.
Maybe all hope is not lost, though. Searching the web for "c++ dynamic loading" reveals that some systems (such and Linux and Windows) do seem to implement a dynamic loading mechanism.
Here are the links to two (old) articles that talk about the subject.
Dynamic Class Loading for C++ on Linux in the Linux Journal.
Dynamically Loaded C++ Objects in Dr.Dobb's.
They seem interesting at first glance. I'm not sure they are still relevant, though.
This is but a shot in the dark.
How to return C++ object to lua?
My C++ code is following:
class MyClass
{
public:
void say()
{
print("Hello\r\n");
}
};
int test(lua_State* l)
{
MyClass* obj = new MyClass();
lua_pushlightuserdata(l, obj);
return 1;
}
Lua Test is following:
local a = MyClass:new()
a:say() <--- OK, beacause I set metatable!!
local b = test()
b:say() <--- ERROR: attempt to index local 'b' (a userdata value)
How to modify test() function to work fine?
obj will auto destory by lua ?
PS: I has set MyClass metatable is following
void l_registerClass()
{
lua_newtable(l);
int methods = lua_gettop(l);
luaL_newmetatable(l, "MyClass");
int metatable = lua_gettop(l);
lua_pushvalue(l, methods);
lua_setglobal(l, "MyClass");
lua_pushvalue(l, methods);
l_set(l, metatable, "__metatable");
//set metatable __index
lua_pushvalue(l, methods);
l_set(l, metatable, "__index");
//set metatable __gc
lua_pushcfunction(l, l_destructor);
l_set(l, metatable, "__gc");
//set method table
lua_newtable(l); // mt for method table
lua_pushcfunction(l, l_constructor);
lua_pushvalue(l, -1); // dup new_T function
l_set(l, methods, "new"); // add new_T to method table
l_set(l, -3, "__call"); // mt.__call = new_T
lua_setmetatable(l, methods);
// set methods metatable
lua_pushstring(l, "say");
lua_pushcclosure(l, l_proxy, 1);
lua_settable(l, methods);
lua_pop(l, 2);
}
int l_proxy(lua_State* l)
{
int i = (int)lua_tonumber(l, lua_upvalueindex(1));
lua_remove(l, 1); // remove self so member function args start at index 1
//call real function
MyClass* obj = getInstance();
obj->say();
return 1;
}
I should don't lost step ?
I don't use any Lua Binding Framework, I am using pure Lua Library.
==== update 1 ====
Thanks for user1520427's answer, but....
int test(lua_State* l)
{
MyClass** c = (MyClass**)lua_newuserdata(l, sizeof(MyClass*));
*c = new MyClass(); // we manage this
lua_getglobal(l, "MyClass");
lua_setmetatable(l, -2);
return 1;
}
and I test it in Lua
local b = test()
print( type(b) )
local meta = getmetatable(b)
for k,v in pairs(meta) do
print(" ", k, v)
end
Lua show metatable is correct.
userdata
say function: 00602860
new function: 00493665
But lua still shows the same error in
b:say() <-- attempt to index local 'b' (a userdata value)
=== update 2 ===
int test(lua_State* l)
{
MyClass** c = (MyClass**)lua_newuserdata(l, sizeof(MyClass*));
*c = new MyClass(); // we manage this
luaL_getmetatable(l, "MyClass"); //
lua_getglobal(l, "MyClass");
lua_setmetatable(l, -2);
return 1;
}
the lua test result:
b:say() <-- attempt to call method 'say' (a nil value)
=== update 3 ===
int test(lua_State* l)
{
MyClass** c = (MyClass**)lua_newuserdata(l, sizeof(MyClass*));
*c = new MyClass(); // we manage this
luaL_getmetatable(l, "MyClass");
luaL_setmetatable(l, "MyClass"); //modify
return 1;
}
Lua test result:
b:say() <-- calling 'say' on bad self
You're not associating what you return from test with the class you registered. Try something like:
int test(lua_state* l) {
MyClass** c = lua_newuserdata(l, sizeof(MyClass*)); // lua will manage the MyClass** ptr
*c = new MyClass(); // we manage this
luaL_getmetatable(l, "MyClass");
lua_setmetatable(l, -2);
return 1;
}
That's off the top of my head but you get the idea. You've already set the destructor, so when Lua garbage collects the userdata, it will call your __gc func which should then cast, dereference and delete the data.
thanks for user1520427 and lhf
correct code sould is following:
int test(lua_State* l)
{
MyClass** c = (MyClass**)lua_newuserdata(l, sizeof(MyClass*));
*c = new MyClass(); // we manage this
luaL_setmetatable(l, "MyClass"); //assign MyClass metatable
return 1;
}
Lua test code is work fine.