Handling binary data between C++ and Lua - c++

Question

i need to make a program that will handle binary data, lots of it.
in short, the C++ program will load a binary file (some of them exceed 20mb) into a buffer.
then it will run a Lua script and pass all this loaded data to the script, which will do some manipulation and return the result to the C++ program.
I need to do this as quickly as possible, perform at the best and get the job done faster.
A while ago I already made this program using the conventional methods of Lua, but it was extremely slow.
So I lost the files, and now I want to redo it in a better, faster way that doesn't compromise performance.
Searching a bit, I found this.
I had to make some small changes to adapt to the new version of Lua, but I can't get it to work.
Can you help me with this?
And if there's a better way to do the job I said, what would it be?
#include "stdafx.h"
// metatable method for handling "array[index]"
static int array_index(lua_State* L) {
int** parray = (int**)luaL_checkudata(L, 1, "array");
int index = luaL_checkinteger(L, 2);
lua_pushnumber(L, (*parray)[index - 1]);
return 1;
}
// metatable method for handle "array[index] = value"
static int array_newindex(lua_State* L) {
int** parray = (int**)luaL_checkudata(L, 1, "array");
int index = luaL_checkinteger(L, 2);
int value = luaL_checkinteger(L, 3);
(*parray)[index - 1] = value;
return 0;
}
// create a metatable for our array type
static void create_array_type(lua_State* L) {
static const struct luaL_Reg array[] = {
{ "__index", array_index },
{ "__newindex", array_newindex },
NULL, NULL
};
luaL_newmetatable(L, "array");
luaL_setfuncs(L, array, 0);
}
// expose an array to lua, by storing it in a userdata with the array metatable
static int expose_array(lua_State* L, int array[]) {
int** parray = (int**)lua_newuserdata(L, sizeof(int**));
*parray = array;
luaL_getmetatable(L, "array");
lua_setmetatable(L, -2);
return 1;
}
// test data
int mydata[] = { 1, 2, 3, 4 };
// test routine which exposes our test array to Lua
static int getarray(lua_State* L) {
return expose_array(L, mydata);
}
int __declspec(dllexport) __cdecl luaopen_array(lua_State* L) {
create_array_type(L);
// make our test routine available to Lua
lua_register(L, "array", getarray);
return 0;
}
int main()
{
lua_State* L = luaL_newstate();
luaL_dostring(L, "require 'array'");
luaL_dostring(L, "foo = array()");
luaL_dostring(L, "x = foo[1]");
lua_getglobal(L, "x");
lua_Number x = lua_tonumber(L, 1);
printf("foo[1] = %d\n", (int)x);
}

Answer 1

Consider using lightuserdata to avoid copying file contents excessively.
lightuserdata is just a pointer, so you need to define some methods to work with it as well.
The idea looks like this:
#include <lauxlib.h>
#include <lualib.h>
#define BIN_DATA_MT_ID "bin data"
int get_byte(lua_State *L) {
const char *file_contents = luaL_checkudata(L, 1, BIN_DATA_MT_ID);
size_t byte_index = luaL_checknumber(L, 2);
// checking OOB is your custody here, omitted for simplicity
lua_pushlstring(L, file_contents + byte_index, 1);
return 1;
}
static const luaL_Reg __index[] = {
{"get_byte", get_byte},
{NULL, NULL}
};
int main() {
const char file_contents[4] = { 0x25, 0xAA, 0xBB, 0xCC };
lua_State *L = luaL_newstate();
luaopen_base(L); // adds "print" function
lua_pushlightuserdata(L, (void *)file_contents);
luaL_newmetatable(L, BIN_DATA_MT_ID);
luaL_newlib(L, __index);
lua_setfield(L, -2, "__index");
lua_setmetatable(L, -2);
lua_setglobal(L, "mybindata");
luaL_dostring(L, "print(mybindata:get_byte(0))");
lua_close(L);
return 0;
}

Answer 2

The fastest way for Lua to access the bytes of a large byte array is to expose that array directly as a string within Lua. Now because Lua does reference counting for strings, this means that Lua will always allocate its own storage for the string. So to efficiently load the string into Lua (ie: avoiding a 20+MiB copy), you need to use the lua_Buffer-based API to load it directly into Lua's storage.
But outside of that quirk, it will certainly be faster inside Lua to use array accesses of a string to access bytes compared to doing a function call for each byte accessed from the buffer.