C++ function not exported by .so seemingly because of pybind11 parameters - c++

I am currently trying to define a shared library that I aim to use from a Python C++ extension as well as from vanilla C++ applications.
I managed to build the shared library, and tried to link a simple C++ application against it to test its functionalities, but one of the functions of the shared library is treated as an undefined reference by the linker. After checking with nm --demangle --dynamic --defined-only --extern-only libmylib.so, I realized the function is not being exported by the shared library, but I have no idea why.
The function's signature is as follows:
void bootstrap_mylib(std::vector<std::string> python_path,
std::vector<std::string> python_scripts,
std::string interface_module,
std::function<void (pybind11::module_, pybind11::object)> interface_module_initializer);
Everything goes well if I comment out the last parameter, so the problem seems to be coming from the way I declare the dependencies with pybind11 somehow.
Here are the relevant parts of my CMakeLists.txt:
set(CMAKE_CXX_COMPILER /usr/bin/g++)
project(monilog LANGUAGES CXX VERSION 0.0.1)
set(PYBIND11_PYTHON_VERSION 3.8)
find_package(pybind11 REQUIRED)
include_directories(${PYTHON_INCLUDE_DIRS})
add_library(mylib SHARED MyLib.cc MyLib.h)
set_property(TARGET mylib PROPERTY CXX_STANDARD 17)
target_link_libraries(mylib ${PYTHON_LIBRARIES})
set_target_properties(mylib PROPERTIES VERSION ${PROJECT_VERSION})
set_target_properties(mylib PROPERTIES SOVERSION 1)
set_target_properties(mylib PROPERTIES PUBLIC_HEADER MyLib.h)
Any idea of what I might be doing wrong?
Edit: minimal working example
Here is a minimal example of my problem, consisting of the following files:
example.h
#include <pybind11/stl.h>
namespace Example
{
void simple_func(std::string some_string);
void pybind11_func(pybind11::function some_func);
}
example.cc
#include "example.h"
namespace Example
{
void simple_func(std::string some_string)
{
std::cout << "Simple function" << '\n';
}
void pybind11_func(pybind11::function some_func)
{
std::cout << "Pybind11 function" << '\n';
}
}
CMakeLists.txt
cmake_minimum_required(VERSION 3.10)
cmake_policy(SET CMP0074 NEW)
# SET VARIABLES
set(CMAKE_CXX_COMPILER /usr/bin/g++)
set(CMAKE_CXX_STANDARD 17)
project(example CXX)
set(PYBIND11_PYTHON_VERSION 3.8)
find_package(pybind11 REQUIRED)
# include_directories(${PYTHON_INCLUDE_DIRS})
include_directories(${pybind11_INCLUDE_DIRS})
add_library(example SHARED example.cc example.h)
target_link_libraries(example ${PYTHON_LIBRARIES})
When I build the project, if I then search for func in the exposed symbols, I get the following result:
> nm -D libexample.so | grep "func"
00000000000039d9 T _ZN7Example11simple_funcENSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE
pybind11_func is thus not exported, while simple_func is correctly exported.

Namespace pybind11 has hidden visibility.
/usr/include/pybind11/detail/common.h:#
define PYBIND11_NAMESPACE pybind11 __attribute__((visibility("hidden")))
so all functions that have anything to do with that namespace (like having an argument type from it) are also hidden.
You can override this by explicitly setting visibility to default:
__attribute__((visibility("default")))
void bootstrap_mylib( ... )

Related

pybind11 and another libs in one project [duplicate]

I want to be able to call my C++ code as a python package. To do this I am using pybind11 with CMakelists (following this example https://github.com/pybind/cmake_example). My problem is that I have to include GSL libraries in the compilation of the code, and these need an explicit linker -lgsl .
If I were just to compile and run the C++ without wrapping it with python, the following Cmakelists.txt file does the job
cmake_minimum_required(VERSION 3.0)
set(CMAKE_BUILD_TYPE Debug)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
project(myProject)
add_executable(
myexecutable
main.cpp
function1.cpp
)
find_package(GSL REQUIRED)
target_link_libraries(myexecutable GSL::gsl GSL::gslcblas)
but when using pybind11 the template I found doesn't allow the add_executable therefore target_link_libraries doesn't work.
I have trie this
project(myProject)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED YES) # See below (1)
# Set source directory
set(SOURCE_DIR "project")
# Tell CMake that headers are also in SOURCE_DIR
include_directories(${SOURCE_DIR})
set(SOURCES "${SOURCE_DIR}/functions.cpp")
# Generate Python module
add_subdirectory(lib/pybind11)
pybind11_add_module(namr ${SOURCES} "${SOURCE_DIR}/bindings.cpp")
FIND_PACKAGE(GSL REQUIRED)
target_link_libraries(GSL::gsl GSL::gslcblas)
but this produces errors in the building.
Any idea ?
Function pybind11_add_module creates a library target, which can be used for link added module with other libraries:
pybind11_add_module(namr ${SOURCES} "${SOURCE_DIR}/bindings.cpp")
target_link_libraries(namr PUBLIC GSL::gsl GSL::gslcblas)
This is explicitely stated in documentation:
This function behaves very much like CMake’s builtin add_library (in fact, it’s a wrapper function around that command). It will add a library target called <name> to be built from the listed source files. In addition, it will take care of all the Python-specific compiler and linker flags as well as the OS- and Python-version-specific file extension. The produced target <name> can be further manipulated with regular CMake commands.

pybind11 type as function parameters in library yielding compile error in application linking against that library

I'm not sure where I'm going wrong, but in any case, here's what I'm seeing:
Lets say I have a library that's defined something like this:
# minimal-lib/CMakeLists.txt
cmake_minimum_required(VERSION 3.14)
project(minimallib
LANGUAGES CXX
VERSION 1.0.0
DESCRIPTION "minimal library")
include(GNUInstallDirs)
set(CMAKE_INCLUDE_CURRENT_DIR ON)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CPP_COMPILER clang++-13)
# get rid of the pybind11 warning spam
include(CheckCXXCompilerFlag)
check_cxx_compiler_flag(-Wno-attributes HAVE_ATTRIBUTES)
if (HAVE_ATTRIBUTES)
add_compile_options(-Wno-attributes)
endif()
IF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
SET(CMAKE_INSTALL_PREFIX /opt/experiment CACHE PATH "comment" FORCE)
ENDIF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
set(
PSOURCES
src/fun.cpp
)
set(
PHEADERS
include/fun.hpp
)
find_package(pybind11 REQUIRED)
add_library(${PROJECT_NAME} SHARED
${PSOURCES}
${PHEADERS}
)
set_target_properties(${PROJECT_NAME}
PROPERTIES
VERSION 1.0.0
SOVERSION 1.0.0
LINKER_LANGUAGE CXX
MAP_IMPORTED_CONFIG_COVERAGE "DEBUG"
)
target_include_directories(${PROJECT_NAME}
PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<INSTALL_INTERFACE:include>
PRIVATE src)
target_link_libraries(${PROJECT_NAME} PUBLIC pybind11::embed)
install(
TARGETS ${PROJECT_NAME}
EXPORT ${PROJECT_NAME}Config
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_BINDIR}
)
install(
DIRECTORY include/
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME}
)
export(
TARGETS ${PROJECT_NAME}
FILE ${PROJECT_NAME}Config.cmake
NAMESPACE ${PROJECT_NAME}::
FILE "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake"
)
install(
EXPORT ${PROJECT_NAME}Config
DESTINATION "${CMAKE_INSTALL_DATADIR}/${PROJECT_NAME}/cmake"
)
//minimal-lib/include/fun.hpp
#ifndef FUN_HPP
#define FUN_HPP
#include <pybind11/pybind11.h>
#include <optional>
#include <string>
namespace minimallib
{
void write(
const std::optional<std::string> &file,
const pybind11::args &args = pybind11::none(),
const pybind11::kwargs &kwargs = pybind11::none());
}//namespace minimallib
#endif //FUN_HPP
//minimal-lib/src/fun.cpp
#include <iostream>
#include "fun.hpp"
void minimallib::write(
const std::optional<std::string> &file,
const pybind11::args &args,
const pybind11::kwargs &kwargs)
{
std::cout << "I should do something more useful." << std::endl;
}
and an application project that uses that library that looks something like this:
# rms/CMakeLists.txt
cmake_minimum_required(VERSION 3.16.0)
project(minimalapp
LANGUAGES CXX
VERSION 1.0.0)
include(GNUInstallDirs)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_INCLUDE_CURRENT_DIR ON)
IF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
SET(CMAKE_INSTALL_PREFIX /opt/experiment CACHE PATH "comment" FORCE)
ENDIF(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
list(APPEND CMAKE_PREFIX_PATH "/opt/experiment/")
find_package(pybind11 REQUIRED)
find_package(minimallib REQUIRED)
set(
PSOURCES
src/main.cpp
)
set(
MAIN_HEADERS
)
add_executable(${PROJECT_NAME}
${PHEADERS}
${PSOURCES}
)
set_target_properties(${PROJECT_NAME}
PROPERTIES
VERSION 1.0.0
MAP_IMPORTED_CONFIG_COVERAGE "DEBUG")
set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX)
target_include_directories(${PROJECT_NAME}
PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<INSTALL_INTERFACE:include>
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/src)
target_link_libraries(${PROJECT_NAME} PRIVATE pybind11::embed)
target_link_libraries(${PROJECT_NAME} PRIVATE minimallib)
install(TARGETS ${PROJECT_NAME}
EXPORT ${PROJECT_NAME}Targets
RUNTIME DESTINATION bin)
#include <minimallib/fun.hpp>
int main()
{
minimallib::write("foobar.txt");
return 0;
}
The library compiles and links without complaints.
In any case, linking the application using that library fails with an error like this:
FAILED: minimalapp-1.0.0
: && /usr/bin/c++ CMakeFiles/minimalapp.dir/src/main.cpp.o -o minimalapp-1.0.0 -Wl,-rpath,/opt/experiment/lib: /opt/experiment/lib/libminimallib.so.1.0.0 /usr/lib/x86_64-linux-gnu/libpython3.9.so && :
/usr/bin/ld: CMakeFiles/minimalapp.dir/src/main.cpp.o: in function `main':
main.cpp:(.text+0x1bc): undefined reference to `minimallib::write(std::optional<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > const&, pybind11::args const&, pybind11::kwargs const&)'
collect2: error: ld returned 1 exit status
ninja: build stopped: subcommand failed.
If I remove the pybind11::kwargs and pybind11::args from the signature in the function in the library, the application that uses it compiles without issue.
I'm somewhat ignorant to looking at the symbols included in the *.so file, but I took a look anyway and here's what I observed.
Case 1) pybind11::args and pybind11::kwargs included in signature: can see the relevant symbol with objdump -t but not with objdump -T.
0000000000006259 l F .text 0000000000000046 _ZN10minimallib5writeERKSt8optionalINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEERKN8pybind114argsERKNSA_6kwargsE
Case 2) not including those: can see the relevant symbol with both objdump -t and objdump -T. I note that it's also marked (g)lobal instead of (l)ocal, and marked (D)ynamic instead of ( )normal
0000000000006259 g DF .text 000000000000003e Base _ZN10minimallib5writeERKSt8optionalINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEE
Does this mean that the relevant symbol is invisible to the application trying to link to the library, or am I chasing my tail?
Does anyone have any insight or ideas that might help me fix this?
I've found a solution, though I still don't know why the behavior changed the way it did with the addition of a pair of parameters to a function ... I can only imagine that the library defining the types sets the visibility to hidden or something...
For posterity, for a library that doesn't need to exist outside GCC/Clang, can append this after the namespace name
__attribute__ ((visibility("default")))
or
can prepend #pragma GCC visibility push(default) and postpend #pragma GCC visibility pop this for larger blocks where you need everything visible. (I effectively boxed in all of my include files with this ... probably crude, but was effective)

How to use dynamic link library with CMake?

I have simple program as follow:
CMakeLists.txt:
cmake_minimum_required(VERSION 3.5)
project(test LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
LINK_DIRECTORIES(${PROJECT_SOURCE_DIR})
add_executable(test main.cpp)
target_include_directories(test PRIVATE ${PROJECT_SOURCE_DIR})
target_link_libraries(test PRIVATE power.dll)
main.cpp:
#include <iostream>
#include "power.h"
using namespace std;
int main()
{
cout << "Hello World!" << endl;
power(4.);
return 0;
}
power.h:
#ifndef POWER_H
#define POWER_H
double power(double number) noexcept;
#endif // POWER_H
Implementation of power.h is in a .dll named power.dll.
If I compile this project with MinGW 7.3.0 X64 says:
error: undefined reference to `power(double)'
If I compile it with MSVC 2017 X64 says:
error: LNK1104: cannot open file 'power.lib'
both errors show that power.dll can't detect by the linker.
I did many searches but none of solutions worked for me!
Can anyone help about this?
Thanks in advance!
Your modelling of the dynamic library is incorrect, both on CMake and on the source level.
As a starting point, try building the dll as part of the same CMake project as the consuming executable:
cmake_minimum_required(VERSION 3.5)
project(test LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
include(GenerateExportHeader)
add_library(power SHARED power_sources.cpp power.h)
generate_export_header(power)
target_include_directories(power PUBLIC ${PROJECT_BINARY_DIR} ${PROJECT_SOURCE_DIR})
add_executable(test main.cpp)
target_link_libraries(test PRIVATE power)
Note the use of the generate_export_header function, which instructs CMake to generate macros for exporting functions on shared library interfaces in a portable way. Since generated files go to the binary directory tree, we have to adjust the include directories for the library accordingly.
To make sure the function gets properly exported, change your header as follows:
#ifndef POWER_H
#define POWER_H
#include <power_export.h>
POWER_EXPORT double power(double number) noexcept;
#endif // POWER_H
Note that generare_export_header allows you to customize the generated export header extensively.
Be sure you get the project to build and run from this baseline.
If you want to build the dll externally (which is not strictly necessary, but since that's what your question is about...), we have to modify the CMake file to something like:
cmake_minimum_required(VERSION 3.5)
project(test LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
find_package(power)
add_executable(test main.cpp)
target_link_libraries(test PRIVATE power)
With all the magic here happening in the find_package call. That call is now responsible for providing all the information that was previously handled by the lines for building the library:
Providing of an imported target power for consumption by the target_link_libraries call
Association of the library name of the import library (the power.lib file) via that imported target
Exposure of the public include directories for both power.h and power_export.h via that imported target
You can either construct such an imported target manually in the find script, or have CMake do it for you. In the first case, create a FindPower.cmake script file, make sure it's location is part of the CMAKE_MODULE_PATH and write the code for finding the library and header files and constructing the imported target in there. Note that getting this right in a portable way can be very tricky and goes far beyond the scope of a StackOverflow question. In the second case, have the CMake script that builds the power library perform an install step during which a config file package will get generated, which can then be consumed by your test project. Note that this approach is not viable if the power library is not itself being built with CMake, so in that case you will have to stick with the first option.
Dynamic linking in Windows requires that externally visible symbols are declared with the keyword __declspec. Your header "power.h" should be modified:
#ifndef POWER_H
#define POWER_H
#if defined(__WIN32__) && !defined(__CYGWIN__)
# if (defined(_MSC_VER) || defined(__MINGW32__)) && defined(BUILD_DLL)
# define POWERAPI __declspec(dllexport)
# elif (defined(_MSC_VER) || defined(__MINGW32__))
# define POWERAPI __declspec(dllimport)
# endif
#endif
POWERAPI double power(double number) noexcept;
#endif // POWER_H
In the project building the DLL power.dll with CMake, you should define the symbol BUILD_DLL:
add_definitions(-DBUILD_DLL)
then it should generate a power.lib file when the MSVC compiler and a power.a when using MINGW. Don't define BUILD_DLL in the project using the DLL, and it should work.

How to generate a DLL linked to a static library with CMake

The main objective with this question is to write an CMakeLists.txt to generate a dynamic library, "containing/linked" a static library.
Let me create the scenario:
My C++ code is written in mycode.cpp
In mycode.cpp, I call some functions from libthirdparty.a (static library)
I want to generate libmylib.so (shared library) to be dynamically linked by others
libmylib.so must to "contain" libthirdparty.a
My attempt to write this script is at the lines bellow:
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Werror -m64 -fPIC ")
add_executable(myapp mycode.cpp)
target_link_libraries(myapp thirdparty)
add_library(mylib SHARED myapp)
But of course this is not working and I would like some help to write it correctly.
For now, let's remove the myapp and focus only on the library you are trying to create.
That said, here is what you could do
cmake_minimum_required(VERSION 3.12)
project(AwesomeLib)
include(GenerateExportHeader)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_STANDARD 11)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
find_package(ThirdParty REQUIRED)
add_library(mylib SHARED mylib.cpp)
target_link_libraries(mylib PUBLIC ThirdParty)
# Note: If you library will be cross-platform, flag should be conditionally specified. Have a look at generator-expression
target_compile_options(mylib PRIVATE -Wall -Werror)
generate_export_header(mylib)
# TODO:
# * add install rules
# * generate config-file package
# * add tests
Notes:
generate_export_header will generate "mylib_export.h" header with the MYLIB_EXPORT macro for exporting symbols. See https://cmake.org/cmake/help/latest/module/GenerateExportHeader.html
to understand the idea behind find_package(ThirdParty REQUIRED), I recommend you read config-file package. See Correct way to use third-party libraries in cmake project
to learn more about generator expression. See https://cmake.org/cmake/help/latest/manual/cmake-generator-expressions.7.html

Linking freetype with cmake

I'm having troubles with linking freetype 2 under linux using cmake when building a C++11 project with an extern C library.
With cmake and freetype 2 I basically have 2 options :
use the utility freetype-config like freetype-config --libs
use the FindFreetype cmake module
Now I'm trying to implement the second option and I'm not very skilled with cmake nor I understand the logic of it.
My problem is the linking phase, I have no idea how to do that properly plus this module is not as complete as the result of freetype-config --libs which really includes all the libraries and flags that I need, and not just the path of a file; so I'm assuming that I have to do the same for zlib and libpng.
CMakeLists.txt
cmake_minimum_required (VERSION 2.6)
project (FreetypeTutorials1)
include(FindFreetype)
include_directories(${FREETYPE_INCLUDE_DIRS})
SET(CMAKE_CXX_FLAGS "-O2 -std=c++11")
SET(CMAKE_EXE_LINKER_FLAGS "-v -lfreetype")
add_executable( demo "${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp")
./src/main.cpp ( just some random code so I have something to feed to the compiler )
extern "C" {
#include <ft2build.h>
#include FT_FREETYPE_H
}
#include <iostream>
int main()
{
FT_Library library;
auto error = FT_Init_FreeType(&library);
if (error)
{
std::cout << "An error !\n";
}
}
To load a module like FindFreetype.cmake you need to use it in cmake with the find_package-command. The first argument is the package name. The "Find" of its corresponding filename is added automatically by cmake.
While include might work with find_package you can add some flags. For example, as shown below, REQUIRED, to make cmake fail when freetype wasn't found.
Additionally linking with cmake should be done with the command target_link_libraries.
This is how I would write you CMakeLists.txt:
cmake_minimum_required (VERSION 2.6)
project (FreetypeTutorials1)
find_package(Freetype REQUIRED)
SET(CMAKE_CXX_FLAGS "-O2 -std=c++11")
SET(CMAKE_EXE_LINKER_FLAGS "-v")
add_executable( demo src/main.cpp) # CMAKE_CURRENT_SOURCE_DIR is implicit here
target_link_libraries(demo ${FREETYPE_LIBRARIES})
target_include_directories(demo PRIVATE ${FREETYPE_INCLUDE_DIRS})
target_link_libraries is platform independent whereas '-lfreetype in CMAKE_EXE_LINKER_FLAGS is not.
The CMakeLists.txt will work on other platforms where freetype is available.
(Edit 2019: use target_include_directories() instead of include_directories()