Fails to load Python module with Python 3 - c++

Question

#include <Python.h>
#include <fstream>
#include <iostream>
#include <string>
#include <filesystem>
#include <sys/types.h>
#include <dirent.h>
static const char * sPythonCode =
"class Test :\n"
" def __init__(self) : \n"
" self.Disc_ = 0. \n"
" def getset(self) : \n"
" self.Disc_ = 7. \n"
" return self.Disc_ \n";
std::string writeFile()
{
static int iFile = 0;
std::string sFileName(std::string("test") + std::to_string(iFile));
std::ofstream out("py/" + sFileName + ".py");
out << sPythonCode;
out.flush();
out.close();
iFile++;
return sFileName;
}
static bool bPythonOpen = false;
#define PYTHONPATHLEN 501
static void _PyInit()
{
if (!Py_IsInitialized())
{
Py_InitializeEx(0);
}
}
void openPython(void)
{
if (!bPythonOpen)
{
const size_t szBufferN = 1000;
char acLoadPath[szBufferN];
const char *pypath = "./py";
_PyInit();
PyRun_SimpleString("import sys");
PyRun_SimpleString("print('python (%d.%d.%d) initialized' % (sys.version_info.major, sys.version_info.minor, sys.version_info.micro))");
PyRun_SimpleString("print('--------------------------')");
snprintf(acLoadPath, szBufferN, "sys.path.append('%s')", pypath);
PyRun_SimpleString(acLoadPath);
bPythonOpen = true;
}
}
PyObject *loadPythonModule(const char *acModule)
{
PyObject *pyModule = NULL;
if (bPythonOpen && acModule && strcmp(acModule, ""))
{
printf("%s\n", acModule);
pyModule = PyImport_ImportModule(acModule);
if (!pyModule)
{
PyErr_Print();
}
}
return pyModule;
}
void loadPython()
{
std::string sFileName = writeFile();
openPython();
//sleep(1);
PyObject *pPythonModule = loadPythonModule(sFileName.c_str());
if (pPythonModule)
PyDict_DelItemString(PyImport_GetModuleDict(), PyModule_GetName((PyObject *)pPythonModule));
}
int main(int argc, char **argv)
{
for (int i = 0; i < 10; i++)
{
loadPython();
}
}
My working env:
gcc version 8.3.1 20190311 (Red Hat 8.3.1-3) (GCC)
Red Hat Enterprise Linux Server release 7.6 (Maipo)
problem with python 3.6.10 / 3.8.3
Command to compile:
g++ pythontest.cpp -I/opt/python/python3.6.10/include/python3.6m -L/opt/python/python3.6.10/lib -lpython3.6m
create py directory:
mkdir py
When I run this code I have random error on different test file that I load.
Example of output:
python (3.6.10) initialized
--------------------------
test0
test1
test2
test3
ModuleNotFoundError: No module named 'test3'
test4
test5
ModuleNotFoundError: No module named 'test5'
test6
test7
ModuleNotFoundError: No module named 'test7'
test8
test9
ModuleNotFoundError: No module named 'test9'
Good to know:
If I uncomment the line with the sleep it works well
If I remove the iFile++, it works also as it used after an already created file
If I relaunch a second without rm -rf py directory it works also
If I erase file after each run in the loadPython function and remove iFile++ it works also
If I use strace to launch the executable I don't see the problem
For the moment it seems that the Python loader does not see the file on disk, however in case of failure if I print what I have in the directory thanks to dirent I see the testx.py
Please note that we reproduce the error on different Linux servers (not a hardware problem and even on Windows), with Python 2.7.x it works perfectly well.

Answer 1

You should call __import__('importlib').invalidate_caches() each time you modify modules folders to let C Python knows it must read directories again.