How to build and link external CMake dependencies with CMake? - c++

I'd like to use the following third party library:
https://github.com/Corvusoft/restbed
I'd like to create a folder structure similar to this:
- root
- CMakeLists.txt
- src
- third_party
- restbed
- src
- CMakeLists.txt
- CMakeLists.txt
I've found out that I should use the ExternalProject CMake module to achieve this but it does not seem that I can do it correctly. For some reason the external lib's (restbed's) source does not appear in my XCode project when I generate it out with the use of the root CMakeLists.txt. This root CMakeLists.txt includes in the third_party folder's CMakeLists.txt. The third_party folder's CMakeLists.txt looks like this:
include(ExternalProject)
# CMAKE_CURRENT_LIST_DIR -> third_party folder
set(RESTBED_DIR ${CMAKE_CURRENT_LIST_DIR}/restbed)
ExternalProject_Add(restbed
PREFIX ${RESTBED_DIR}
SOURCE_DIR ${RESTBED_DIR}
CMAKE_ARGS -DBUILD_SSL=NO
DOWNLOAD_COMMAND ""
UPDATE_COMMAND ""
BUILD_COMMAND cmake ..
BINARY_DIR ${RESTBED_DIR}/build
INSTALL_DIR ${RESTBED_DIR}/distribution
INSTALL_COMMAND make install
)
set(RESTBED_INCLUDE_DIR ${RESTBED_DIR}/distribution/include)
set(RESTBED_LIB_DIR ${RESTBED_DIR}/distribution/lib)
add_library(librestbed IMPORTED STATIC)
add_dependencies(librestbed restbed)
After this I'd like to use target_include_dir and target_link_libraries to add librestbed as a dependency to my root project. After this when I build my XCode project nothing happens, the external lib won't get built. What am I doing wrong? I would not like to use the built-in CMake git functionality because I'd like to store my external dependencies as git submodules in the third_party lib. I'd rather not let CMake figure out if it needs to re-download a given dependency. Thanks in advance!

I think you mixed up the BUILD_COMMAND option with the CONFIGURE_COMMAND option. The cmake .. should be placed after the option CONFIGURE_COMMAND.
ExternalProject_Add(restbed
PREFIX ${RESTBED_DIR}
SOURCE_DIR ${RESTBED_DIR}
CMAKE_ARGS -DBUILD_SSL=NO
DOWNLOAD_COMMAND ""
UPDATE_COMMAND ""
CONFIGURE_COMMAND cmake ..
BINARY_DIR ${RESTBED_DIR}/build
INSTALL_DIR ${RESTBED_DIR}/distribution
INSTALL_COMMAND make install
)
Per the documentation, there are defaults to each of these commands, so you likely don't have to specify the CONFIGURE_COMMAND at all. For example, for BUILD_COMMAND:
If this option is not given, the default build command will be chosen to integrate with the main build in the most appropriate way (e.g. using recursive make for Makefile generators or cmake --build if the project uses a CMake build).
Also, if you do specify the CONFIGURE_COMMAND, you likely expand the arguments you pass it to something like this:
...
CONFIGURE_COMMAND cd restbed && mkdir build && cd build && cmake ..
...

Related

Integrate pre-compiled libraries into C++ codebase with CMake ExternalProject

I want to integrate CasADi into a CMake-based C++ codebase as an ExternalProject. For this purpose, I would like to use pre-compiled libraries because building from source is not recommended. So far, I have only managed to write the following:
ExternalProject_Add(
casadi-3.5.5
URL https://github.com/casadi/casadi/releases/download/3.5.5/casadi-linux-py39-v3.5.5-64bit.tar.gz
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
PREFIX ${CMAKE_BINARY_DIR}/external/casadi)
and I noticed that all the binaries are correctly downloaded in the specified folder. However, I do not know how to link my targets to CasADi, nor how to find the package.
There is a natural problem with ExternalProject_Add:
ExternalProject_Add executes commands only on build.
Hence, download will not happen at the configure stage of your project which makes it difficult to use find_package, because the files cannot be found during your first configure run.
Take this CMakeLists.txt:
cmake_minimum_required(VERSION 3.21)
project(untitled)
set(CMAKE_CXX_STANDARD 17)
add_executable(untitled main.cpp)
include(ExternalProject)
ExternalProject_Add(
casadi-3.5.5
URL https://github.com/casadi/casadi/releases/download/3.5.5/casadi-linux-py39-v3.5.5-64bit.tar.gz
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
PREFIX ${CMAKE_BINARY_DIR}/external/casadi)
find_package(casadi HINTS ${CMAKE_BINARY_DIR}/external/casadi/src/casadi-3.5.5/casadi)
target_link_libraries(untitled casadi)
In order to use it you have to do the following:
Configure your project
mkdir build
cd build
cmake ..
Build (download) casadi-3.5.5
cmake --build . --target casadi-3.5.5
Reconfigure your project, because now find_package will find the needed files
cmake ..
Build your targets
cmake --build .
If you want a one step build, there are ways to get around this problem
Use FetchContent
Create a sub-cmake-project in a subfolder with all the ExternalProject_Add commands and execute the approriate build (download) steps manually in your own CMakeLists.txt via execute_process calls: https://stackoverflow.com/a/37554269/8088550
Here is an example for the second option, which might be better since FetchContent doesn't have the full functionality of ExternalProject.
main.cpp
#include <casadi/casadi.hpp>
int main()
{
casadi_printf("This works!");
return 0;
}
CMakeLists.txt
cmake_minimum_required(VERSION 3.20)
project(untitled)
set(CMAKE_CXX_STANDARD 17)
# some default target
add_executable(untitled main.cpp)
# Configure and build external project
file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/external)
execute_process(
COMMAND ${CMAKE_COMMAND} ${CMAKE_SOURCE_DIR}/external
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/external
)
execute_process(
COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR}/external
)
# find and link externals
find_package(casadi REQUIRED HINTS ${CMAKE_BINARY_DIR}/external/external/casadi/src/casadi-3.5.5/casadi)
target_link_libraries(untitled casadi)
external/CMakeLists.txt
cmake_minimum_required(VERSION 3.20)
project(external)
include(ExternalProject)
ExternalProject_Add(
casadi-3.5.5
URL https://github.com/casadi/casadi/releases/download/3.5.5/casadi-linux-py39-v3.5.5-64bit.tar.gz
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
PREFIX ${CMAKE_BINARY_DIR}/external/casadi)
The point is to have another cmake project under external/CMakeLists.txt, which gets configured and build via execute_process calls from the main cmake project. Do note, that you can now have find_package(casadi REQUIRED ...) at configure stage, because the download will happen just before.

Linking against an ExternalProject_add dependency in CMAKE

I'm getting this ninja build error below while running Ninja. My CMAKE build command iscmake -G"Ninja" -DCMAKE_BUILD_TYPE=Release.
ninja: error: 'ext_deps/api/src/ext_api/build/src/lib/libapi.a', needed by 'Project', missing and no known rule to make it
Let's say my project consists of an API (downloaded via CMAKE from GitHub) and the implementation (the Project).
The layout would look like:
Project/
-- build/
-- cmake/modules
----- ExternalDep.cmake
----- FindAPI.cmake
-- CMakeLists.txt
-- src/
---- CMakeLists.txt
-- include/
Let's say that in the top-level CMakeLists.txt I do the usual business of setting build settings, CXX flags, et cetera, and then I call include(ExternalDep), which checks if the "API" library is in the user's system (if not it is downloaded via CMAKE).
In src/CMakeLists.txt I try to link against the API library using a
target_link_libraries(${PROJECT_NAME} PRIVATE ${API_LIBRARY})
The first issue I'm having is that before the "API" library can even be downloaded and built, I get the ninja build error I posted above. I'm positive the ExternalDep.cmake is included before I try to add the Project executable and link against the "API" library.
Here's a simplified version of ExternalDep.cmake:
set(EXT_DEPS_PREFIX "ext_deps")
ExternalProject_Add(ext_lib
GIT_REPOSITORY "https://github.com/fake/api.git"
GIT_TAG "master"
PREFIX "${CMAKE_BINARY_DIR}/${EXT_DEPS_PREFIX}/api"
TMP_DIR "${CMAKE_BINARY_DIR}/${EXT_DEPS_PREFIX}/api-tmp"
STAMP_DIR "${CMAKE_BINARY_DIR}/${EXT_DEPS_PREFIX}/api-stamp"
CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release
SOURCE_DIR "${CMAKE_BINARY_DIR}/${EXT_DEPS_PREFIX}/api/ext_api"
BINARY_DIR "${CMAKE_BINARY_DIR}/${EXT_DEPS_PREFIX}/api/ext_api-build"
BUILD_ALWAYS true
TEST_COMMAND "")
add_dependencies(ext_projects ext_api)
set(API_LIBRARY "${CMAKE_BINARY_DIR}/${EXT_DEPS_PREFIX}/api/ext_api-build/src/lib/libapi.a")
I ran into the same issue with Ninja while it worked fine with Unix Makefiles, and I managed to get it to work with Ninja by adding a BUILD_BYPRODUCTS line to my ExternalProject_Add block. Example:
ExternalProject_Add(SDL2_PROJECT
PREFIX 3rdparty
URL https://www.libsdl.org/release/SDL2-2.0.5.tar.gz
URL_MD5 d4055424d556b4a908aa76fad63abd3c
CONFIGURE_COMMAND <SOURCE_DIR>/configure ${SDL2_configure_args} --prefix=<INSTALL_DIR> --disable-shared
INSTALL_COMMAND make install -j9
BUILD_BYPRODUCTS <INSTALL_DIR>/lib/libSDL2.a
)
ExternalProject_Get_Property(SDL2_PROJECT INSTALL_DIR)
set(SDL2_INSTALL_DIR ${INSTALL_DIR})
add_library(SDL2_LIBRARY STATIC IMPORTED GLOBAL)
set_property(TARGET SDL2_LIBRARY PROPERTY IMPORTED_LOCATION ${SDL2_INSTALL_DIR}/lib/libSDL2.a)
add_dependencies(SDL2_LIBRARY SDL2_PROJECT)
I was able to solve this by generating Unix Makefiles instead of Ninja. I'm still not exactly sure if this was the singular issue but it was definitely one of the issues.

Installing an ExternalProject with CMake

I have the following code in one of my CMakeLists.txt:
FIND_PACKAGE(sphinxbase)
if (${SPHINXBASE_FOUND})
INCLUDE_DIRECTORIES(${SPHINXBASE_INCLUDE_DIR}/sphinxbase/)
else ()
ExternalProject_Add(
sphinxbase
GIT_REPOSITORY "https://github.com/cmusphinx/sphinxbase.git"
GIT_TAG "e34b1c632392276101ed16e8a05862e43f038a7c"
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/sphinxbase
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lib/sphinxbase/autogen.sh
BUILD_COMMAND ${MAKE}
UPDATE_COMMAND ""
INSTALL_COMMAND ""
BUILD_IN_SOURCE ON
LOG_DOWNLOAD ON
LOG_UPDATE ON
LOG_CONFIGURE ON
LOG_BUILD ON
LOG_TEST ON
LOG_INSTALL ON
)
ExternalProject_Get_Property(sphinxbase SOURCE_DIR)
ExternalProject_Get_Property(sphinxbase BINARY_DIR)
SET(SPHINXBASE_SOURCE_DIR ${SOURCE_DIR})
SET(SPHINXBASE_BINARY_DIR ${BINARY_DIR})
SET(SPHINXBASE_LIBRARIES ${SPHINXBASE_BINARY_DIR}/src/libsphinxbase/.libs/libsphinxbase.a)
INCLUDE_DIRECTORIES(${SPHINXBASE_SOURCE_DIR}/include)
endif ()
SET(DEPENDENCIES ${DEPENDENCIES} sphinxbase)
SET(LIBS ${LIBS} ${SPHINXBASE_LIBRARIES})
In addition to the TARGET that I'm trying to install, how would I go about installing this ExternalProject? Right now I'm trying to do it like this:
install(TARGETS ${LIBS}
LIBRARY DESTINATION lib
ARCHIVE DESTINATION lib
OPTIONAL
)
install(TARGETS ${PROJECT_NAME}
RUNTIME DESTINATION bin
)
But I'm getting the following error thrown at me:
CMake Error at CMakeLists.txt:197 (install):
install TARGETS given target
"/Users/syb0rg/Dropbox/Development/Khronos/Khronos/lib/sphinxbase/src/libsphinxbase/.libs/libsphinxbase.a"
which does not exist in this directory.
Any suggestions?
Command flow install(TARGETS) installs targets, which are created with add_executable or add_library commands. For install concrete files, you need to use command flow install(FILES).
Instead of installing selected files from subproject's build directory, you may install subproject as is. For that you can use
make DESTDIR=<...> install
command as INSTALL option of ExeternalProject_Add. This command will install whole subproject into directory given as DESTDIR parameter for make (more information about this parameter can be found here).
Then you may use install(DIRECTORY) command for install all subproject's files at once:
# It is better to use binary directory for download or build 3d-party project
set(sphinxbase_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/lib/sphinxbase)
# This will be used as DESTDIR on subproject's `make install`.
set(sphinxbase_DESTDIR ${CMAKE_CURRENT_BINARY_DIR}/lib/sphinxbase_install)
ExternalProject_Add(
sphinxbase
GIT_REPOSITORY "https://github.com/cmusphinx/sphinxbase.git"
GIT_TAG "e34b1c632392276101ed16e8a05862e43f038a7c"
SOURCE_DIR ${sphinxbase_SOURCE_DIR}
# Specify installation prefix for configure.sh (autogen.sh).
CONFIGURE_COMMAND ./autogen.sh --prefix=${CMAKE_INSTALL_PREFIX}
BUILD_COMMAND ${MAKE}
UPDATE_COMMAND ""
# Fake installation: copy installed files into DESTDIR.
INSTALL_COMMAND make DESTDIR=${sphinxbase_DESTDIR} install
...
)
# Actually install subproject.
install(
DIRECTORY ${sphinxbase_DESTDIR}/
DESTINATION "/"
USE_SOURCE_PERMISSIONS # Remain permissions (rwx) for installed files
)
Note, that root directory is used as DESTINATION. This is because files under sphinxbase_DESTDIR directory already located in accordance to CMAKE_INSTALL_PREFIX, which is passed as --prefix option to configure.sh (autogen.sh in the given case).
If you know, that subprojects installs its files only under installation prefix (given by --prefix), you can make main project installation more packaging-friendly:
install(
DIRECTORY ${sphinxbase_DESTDIR}/${CMAKE_INSTALL_PREFIX}/
DESTINATION "."
USE_SOURCE_PERMISSIONS
)
Because now relative path is used for DESTINATION option, the project will correctly support packaging. For example, make DESTDIR=<...> install will work for main project too.
Note, that even if ExternalProject_Add is used for build CMake subproject, targets created in this subroject are not seen by the main project.

CMakeLists configuration to link two C++ projects

I have the following situation: a Project A depends on a Project B, but both are built at the same time. Project A has to include the includes of project B and it needs also to link its libraries. Up to now I've tried this way:
ADD_SUBDIRECTORY(${CMAKE_SOURCE_DIR}/other_project other_project)
and then:
INCLUDE_DIRECTORIES(includ ${CMAKE_SOURCE_DIR}/other_project/include})
LIST(APPEND LINK_LIBS other_project)
in the CMakeLists.txt of Project A
but it doesn't seem to work, the compiler also gives me error when including the headers of Project B saying that they do not exist.
What is the right way to add dependencies in A? How should the CMakeLists.txt look like?
EDIT:
as suggested in the comments, this question was addressed in this, however I'd like to see an example of how to use it in a CMakeList.txt file.
The following is a simple example which builds zlib and then builds libxml2 which depends on the zlib we build. One thing to note, I quickly pulled this example from stuff I've done before. The libxml2 example uses make, thus will only actually build on a system which has it, e.g. Linux, Mac ...
Here is the proposed directory structure for this example ...
src/
-- CMakeLists.txt
CMake/
---- External_ZLib.cmake
---- External_libxml2.cmake
Dowloads/ ( no need to create this directory, CMake will do it for you )
build/ ( Out of source build to prevent littering source tree )
Files:
CMakeLists.txt
# Any version that supports ExternalProject should do
cmake_minimum_required( VERSION 3.1)
project(test_ext_proj)
set(test_ext_proj_CMAKE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/CMake")
set(CMAKE_MODULE_PATH ${test_ext_proj_CMAKE_DIR} ${CMAKE_MODULE_PATH})
set(test_ext_proj_BUILD_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/install)
set(test_ext_proj_DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/Downloads CACHE PATH "Directory to store downloaded tarballs.")
include(ExternalProject)
include(External_ZLib)
include(External_libxml2)
External_ZLib.cmake:
set(ZLib_version 1.2.8)
set(ZLib_url "http://zlib.net/zlib-${ZLib_version}.tar.gz")
set(ZLib_md5 "44d667c142d7cda120332623eab69f40")
ExternalProject_Add(ZLib
URL ${ZLib_url}
URL_MD5 ${ZLib_md5}
PREFIX ${vision-tpl_BUILD_PREFIX}
DOWNLOAD_DIR ${test_ext_proj_DOWNLOAD_DIR}
INSTALL_DIR ${test_ext_proj_BUILD_INSTALL_PREFIX}
CMAKE_GENERATOR ${gen}
CMAKE_ARGS
-DCMAKE_INSTALL_PREFIX:PATH=${test_ext_proj_BUILD_INSTALL_PREFIX}
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
)
#This variable is required so other packages can find it.
set(ZLIB_ROOT ${test_ext_proj_BUILD_INSTALL_PREFIX} CACHE PATH "" FORCE)
External_libxml2.cmake:
set(libxml2_release "2.9")
set(libxml2_patch_version 0)
set(libxml2_url "ftp://xmlsoft.org/libxml2/libxml2-sources-${libxml2_release}.${libxml2_patch_version}.tar.gz")
set(libxml2_md5 "7da7af8f62e111497d5a2b61d01bd811")
#We need to state that we're dependent on ZLib so build order is correct
set(_XML2_DEPENDS ZLib)
#This build requires make, ensure we have it, or error out.
if(CMAKE_GENERATOR MATCHES ".*Makefiles")
set(MAKE_EXECUTABLE "$(MAKE)")
else()
find_program(MAKE_EXECUTABLE make)
if(NOT MAKE_EXECUTABLE)
message(FATAL_ERROR "Could not find 'make', required to build libxml2.")
endif()
endif()
ExternalProject_Add(libxml2
DEPENDS ${_XML2_DEPENDS}
URL ${libxml2_url}
URL_MD5 ${libxml2_md5}
PREFIX ${test_ext_proj_BUILD_PREFIX}
DOWNLOAD_DIR ${test_ext_proj_DOWNLOAD_DIR}
INSTALL_DIR ${test_ext_proj_BUILD_INSTALL_PREFIX}
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND ./configure
--prefix=${test_ext_proj_BUILD_INSTALL_PREFIX}
--with-zlib=${ZLIB_ROOT}
BUILD_COMMAND ${MAKE_EXECUTABLE}
INSTALL_COMMAND ${MAKE_EXECUTABLE} install
)

CMake ExternalProject_Add() and FindPackage()

Is there are proper way to find a library (via FindPackage()) which was built with ExternalProject_Add()?
The problem is that CMake cannot find the library at CMake-time because the external library gets build at compile time. I know that it is possible to combine these two CMake function when building the library and the project in a superbuild but I want to use it in a normal CMake project.
In fact I would like to build VTK 6 with ExternalProject_Add and find it with FindPackage all inside my CMake project.
there is a way to do this. but it´s kind of hackish.
you basically add a custom target, that reruns cmake during build.
you will have to try this in a small test project, to decide if it works for you
find_package(Beaengine)
############################################
#
# BeaEngine
#
include(ExternalProject)
externalproject_add(BeaEngine
SOURCE_DIR ${PROJECT_SOURCE_DIR}/beaengine
SVN_REPOSITORY http://beaengine.googlecode.com/svn/trunk/
CMAKE_ARGS -DoptHAS_OPTIMIZED=TRUE -DoptHAS_SYMBOLS=FALSE -DoptBUILD_64BIT=FALSE -DoptBUILD_DLL=FALSE -DoptBUILD_LITE=FALSE
INSTALL_COMMAND ""
)
if(NOT ${Beaengine_FOUND})
#rerun cmake in initial build
#will update cmakecache/project files on first build
#so you may have to reload project after first build
add_custom_target(Rescan ${CMAKE_COMMAND} ${CMAKE_SOURCE_DIR} DEPENDS BeaEngine)
else()
#Rescan becomes a dummy target after first build
#this prevents cmake from rebuilding cache/projects on subsequent builds
add_custom_target(Rescan)
endif()
add_executable(testapp testapp.cpp )
add_dependencies(testapp Rescan)
if(${Beaengine_FOUND})
target_link_libraries(testapp ${Beaengine_LIBRARY})
endif()
this seems to work well for mingw makefiles / eclipse makefile projects.
vs will request to reload all projects after first build.
You can force a build using the build_external_project function below.
It works by generating a simple helper project inside the build tree and then calling the cmake configuration and the cmake build on the helper.
Customize at will for the actual ExternalProject_add command.
Note that the trailing arguments are used to pass CMAKE_ARGS. Furthur enhancements are left as an exercise to the reader :-)
# This function is used to force a build on a dependant project at cmake configuration phase.
#
function (build_external_project target prefix url) #FOLLOWING ARGUMENTS are the CMAKE_ARGS of ExternalProject_Add
set(trigger_build_dir ${CMAKE_BINARY_DIR}/force_${target})
#mktemp dir in build tree
file(MAKE_DIRECTORY ${trigger_build_dir} ${trigger_build_dir}/build)
#generate false dependency project
set(CMAKE_LIST_CONTENT "
cmake_minimum_required(VERSION 2.8)
include(ExternalProject)
ExternalProject_add(${target}
PREFIX ${prefix}/${target}
URL ${url}
CMAKE_ARGS ${ARGN}
INSTALL_COMMAND \"\"
)
add_custom_target(trigger_${target})
add_dependencies(trigger_${target} ${target})
")
file(WRITE ${trigger_build_dir}/CMakeLists.txt "${CMAKE_LIST_CONTENT}")
execute_process(COMMAND ${CMAKE_COMMAND} ..
WORKING_DIRECTORY ${trigger_build_dir}/build
)
execute_process(COMMAND ${CMAKE_COMMAND} --build .
WORKING_DIRECTORY ${trigger_build_dir}/build
)
endfunction()
Time has passed and CMake implemented a native version allowing to reference targets from an ExternalProject_Add.
This feature is implemented in the FetchContent module. It allows downloading and immediately consuming targets defined at configure time.
It uses a scratch build dir as hinted by my previous answer, but in a more integrated API.