Build external library only once with CMake - c++

My C++ projects includes the source code of a third-party library (currently as a git submodule).
This library is added to the project by our main CMakelists through the use of add_subdirectory, and then the library is linked with the main target.
Here is a reduced version of my current Cmake file :
add_subdirectory(foo)
set(FOO_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/libfoo/libfoo.so)
add_executable(target main.cpp)
add_dependencies(target foo)
target_link_libraries(target ${FOO_LIBRARY})
This library takes a long time to build and, since I don't change its code I need it built only once (per build configuration). But when I clean and rebuild my code it also cleans the library files and recompile them.
I have tried to set the property CLEAN_NO_CUSTOM in the library's directory, but according to the documentation it only works for custom command targets.
Is there a mechanism in CMake through which it is possible to specify that this library target needs to be generated only once, or alternatively not cleaned by make clean ?

As #Tsyvarev said, in your case ExternalProject_Add is better than add_subdirectory. add_subdirectory is good when you want project to be essential part of your build system because target it creates can be used in in the right-hand-side of the target_link_libraries() command while target created by ExternalProject_Add cannot.
Here is the approach I used in one of my projects. You try to find required library and build it only if it was not found. I use INTERFACE library to turn FOO_EXTERNAL into a target acceptable by target_link_libraries().
add_library(foo INTERFACE)
find_package(foo ${FOO_VER})
if(NOT foo_FOUND)
include(ExternalProject)
include(GNUInstallDirs)
ExternalProject_Add(FOO_EXTERNAL
SOURCE_DIR "${FOO_SOURCE_DIR}"
BINARY_DIR "${FOO_BINARY_DIR}"
INSTALL_DIR "${FOO_INSTALL_DIR}"
CMAKE_ARGS "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}"
"-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}"
"-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
"-DCMAKE_INSTALL_PREFIX=${FOO_INSTALL_DIR}"
)
add_dependencies(foo FOO_EXTERNAL)
set(foo_LIBRARY
"${FOO_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/${CMAKE_STATIC_LIBRARY_PREFIX}foo${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(foo_INCLUDE_DIR "${FOO_INSTALL_DIR}/include")
endif()
target_link_libraries(foo INTERFACE ${foo_LIBRARY})
target_include_directories(foo INTERFACE ${foo_INCLUDE_DIR})

Based on the excellent answer by #Hikke, I wrote two macros to simplify using external projects.
Code
include(ExternalProject)
#
# Add external project.
#
# \param name Name of external project
# \param path Path to source directory
# \param external Name of the external target
#
macro(add_external_project name path)
# Create external project
set(${name}_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${path})
set(${name}_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/${path})
ExternalProject_Add(${name}
SOURCE_DIR "${${name}_SOURCE_DIR}"
BINARY_DIR "${${name}_BINARY_DIR}"
CMAKE_ARGS "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}"
"-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}"
"-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
# These are only useful if you're cross-compiling.
# They, however, will not hurt regardless.
"-DCMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME}"
"-DCMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}"
"-DCMAKE_AR=${CMAKE_AR}"
"-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
"-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
"-DCMAKE_RC_COMPILER=${CMAKE_RC_COMPILER}"
"-DCMAKE_COMPILER_PREFIX=${CMAKE_COMPILER_PREFIX}"
"-DCMAKE_FIND_ROOT_PATH=${CMAKE_FIND_ROOT_PATH}"
INSTALL_COMMAND ""
)
endmacro(add_external_project)
#
# Add external target to external project.
#
# \param name Name of external project
# \param includedir Path to include directory
# \param libdir Path to library directory
# \param build_type Build type {STATIC, SHARED}
# \param external Name of the external target
#
macro(add_external_target name includedir libdir build_type external)
# Configurations
set(${name}_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/${libdir})
# Create external library
add_library(${name} ${build_type} IMPORTED)
set(${name}_LIBRARY "${${name}_BINARY_DIR}/${CMAKE_CFG_INTDIR}/${CMAKE_${build_type}_LIBRARY_PREFIX}${name}${CMAKE_${build_type}_LIBRARY_SUFFIX}")
# Find paths and set dependencies
add_dependencies(${name} ${external})
set(${name}_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/${includedir}")
# Set interface properties
set_target_properties(${name} PROPERTIES IMPORTED_LOCATION ${${name}_LIBRARY})
set_target_properties(${name} PROPERTIES INCLUDE_DIRECTORIES ${${name}_INCLUDE_DIR})
endmacro(add_external_target)
Explanation
The first macro creates the external project, which does the entire external build step, while the second step sets the necessary dependencies and defines the interface. Separating the two is important, because most projects have more than one interface/library.
Example
Say I have GoogleTest as a submodule in my project, located in the googletest subfolder. I can use the following interface to define the gtest and gtest_main macros, very similar to how Googletest itself does it.
add_external_project(googletest_external googletest)
add_external_target(gtest googletest/googletest/include googletest/googlemock/gtest STATIC googletest_external)
add_external_target(gtest_main googletest/googletest/include googletest/googlemock/gtest STATIC googletest_external)
I can then link my target to googletest much like before:
target_link_libraries(target_tests
gtest
gtest_main
# The CMAKE_THREAD_LIBS_INIT can be found from `find_package(Threads)`
# and is required for all but MinGW builds.
${CMAKE_THREAD_LIBS_INIT}
)
This should provide sufficient boilerplate to simplify the actual external build process, even with CMake-driven projects.

Related

How to get the output path of a target defined by ExternalProject?

I'm building Google's FlatBuffers as a dependency for my own project and I need to compile a schema at build-time. I don't want to use BuildFlatBuffers.cmake or FindFlatBuffers.cmake because I'm using a specific version and I can't rely on it being locally installed.
This is a simplified version of my CMakeLists.txt:
ExternalProject_Add (
flatbuf
URL "https://github.com/google/flatbuffers/archive/v1.8.0.tar.gz"
CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
)
add_custom_target (
flatbuf_schema
PREFIX ${FLATBUF_PREFIX}
DEPENDS flatbuf
COMMAND ${FLATBUF_PREFIX}/src/flatbuf-build/flatc --cpp ${FLATBUF_SCHEMA}
)
It works fine for Make and Ninja but fails in Xcode, which builds flatc in the Debug directory.
I thought about these possible solutions:
use add_subdirectory instead of ExternalProject_Add so that I can use ${FLATBUFFERS_FLATC_EXECUTABLE} or $<TARGET_FILE:flatc>;
manually assign a RUNTIME_OUTPUT_DIRECTORY for flatbuf;
search for flatc in multiple paths (not portable; I also don't know how to make it happen at build-time).
I tried (2) and (3) but without success. As for (1), I'm not sure it's a good idea. How can I build schemas in a portable manner?
You can use ExternalProject_Get_Property, something like this...
note: I suppose you don't even need to install flatbuf, just build it and use it.
ExternalProject_Add (
flatbuf_project
URL "https://github.com/google/flatbuffers/archive/v1.8.0.tar.gz"
CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
INSTALL_COMMAND ""
)
ExternalProject_Get_Property(flatbuf_project source_dir)
ExternalProject_Get_Property(flatbuf_project binary_dir)
# Export flatbuf executable to consume schema file during build
add_executable(flatbuf::flatbuf IMPORTED)
set_target_properties(flatbuf::flatbuf PROPERTIES IMPORTED_LOCATION
"${binary_dir}/flatc")
add_dependencies(flatbuf::flatbuf flatbuf_project)
add_custom_target(flatbuf_schema
PREFIX ${FLATBUF_PREFIX}
COMMAND flatbuf::flatbuff --cpp ${FLATBUF_SCHEMA}
)
note2:
If COMMAND specifies an executable target name (created by the add_executable() command) it will automatically be replaced by the location of the executable created at build time. If set, the CROSSCOMPILING_EMULATOR executable target property will also be prepended to the command to allow the executable to run on the host. Additionally a target-level dependency will be added so that the executable target will be built before this custom target.
note3:
target ALIAS are not working on IMPORTED target unfortunately...
Apparently CMake provides a variable to solve this exact problem, i.e. CMAKE_CFG_INTDIR (docs). The path to the flatc executable should then be
ExternalProject_Get_Property(flatbuf BINARY_DIR)
set (FLATC "${BINARY_DIR}/${CMAKE_CFG_INTDIR}/flatc")

Specify RPATH for externally built CMAKE project

How do I specify an RPATH for a project that was built externally in CMake (using 3.0.0) via ExternalProject_Add() macro?
For reference, let's say my ExternalProject is SFML. My external project call looks like:
set(SFML_INSTALL_PATH "${CMAKE_CURRENT_BINARY_DIR}/ext-deps/ext-sfml-build")
set(SFML_RPATH "${SFML_INSTALL_PATH}/src/lib")
set(SFML_CMAKE_ARGS -DCMAKE_BUILD_TYPE:STRING=Release -DUSE_STATIC_LIBS:BOOL=true
-DCMAKE_INSTALL_PREFIX:PATH=${SFML_INSTALL_PATH}
-DCMAKE_INSTALL_RPATH:PATH=${SFML_RPATH})
ExternalProject_Add(ext-sfml
GIT_REPOSITORY "${SFML_REPO}"
GIT_TAG "${SFML_TAG}"
URL SFML_URL
URL_HASH 256=${SFML_SHA256}
CMAKE_ARGS "${SFML_CMAKE_ARGS}"
TMP_DIR "${CMAKE_CURRENT_BINARY_DIR}/${EXT_DEPS_PREFIX}/ext-sfml-tmp"
STAMP_DIR "${CMAKE_CURRENT_BINARY_DIR}/${EXT_DEPS_PREFIX}/ext-sfml-stamp"
INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/${EXT_DEPS_PREFIX}/ext-sfml-build"
SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/${EXT_DEPS_PREFIX}/ext-sfml"
BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/${EXT_DEPS_PREFIX}/ext-sfml-build"
TEST_COMMAND "")
The error command I get when I run the build step (configuration is fine) is
Make Error at src/cmake_install.cmake:45 (file):
file RPATH_CHANGE could not write new RPATH:
/home/hendrix/repo/project/build/ext_deps/ext_sfml-build/lib
to the file:
/home/hendrix/repo/project/build/ext_deps/ext_sfml-build/bin/exe
Finally, this "/home/hendrix/repo/project/build/ext_deps/ext_sfml-build/lib" is the line I'm trying to modify. It appears that it is the value of my ${CMAKE_INSTALL_RPATH} variable but containing the install prefix of my ext-dep project and not the top-level master project.
I figured it out (thanks to Tsyvarev's comments). The externally built project that I was using set it's own RPATH. So the string that I found in the error message was that RPATH, not any RPATH which I set in my "top-level" project.
It appears that setting the RPATH of an external project can not override that external project's internally set RPATH (if it is using CMAKE).

How to get a header only library portably from version control using CMake?

For a typical C++ header only library located on, e.g., github, at: https://github.com/username/library_name,
has a directory structure with an include/library_name folder like:
include/library_name
containing all the library sources. This is typically installed by users to, e.g., under Linux: /usr/local/include/library_name.
I need a cmake script for using the library in external projects portably (across Linux, MacOs, BSD, Windows).
It should:
find if the library is installed, if the version is over a threshold, use the installed library
otherwise, get the library from github, configure it as an external project, and put it in the system include path so that it can be used by the project as if it were installed.
What is the correct way of achieving this with CMake?
I finally got it to work, this FindLibraryName.cmake tries to find the library and if it doesn't find it, gets it from github and sets it up as an external project:
# Find the Library_Name include directory
# The following variables are set if Library_Name is found.
# Library_Name_FOUND - True when the Library_Name include directory is found.
# Library_Name_INCLUDE_DIRS - The path to where the poco include files are.
# If Library_Name is not found, Library_Name_FOUND is set to false.
find_package(PkgConfig)
# Allow the user can specify the include directory manually:
if(NOT EXISTS "${LIBRARY_NAME_INCLUDE_DIR}")
find_path(LIBRARY_NAME_INCLUDE_DIR
NAMES library_name/library_name.hpp
DOC "Library_Name library header files"
)
endif()
if(EXISTS "${LIBRARY_NAME_INCLUDE_DIR}")
include(FindPackageHandleStandardArgs)
mark_as_advanced(LIBRARY_NAME_INCLUDE_DIR)
else()
include(ExternalProject)
ExternalProject_Add(library_name
GIT_REPOSITORY https://github.com/username/library_name.git
TIMEOUT 5
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
PREFIX "${CMAKE_CURRENT_BINARY_DIR}"
INSTALL_COMMAND "" # Disable install step, is a header only lib!
)
# Specify include dir
ExternalProject_Get_Property(library_name source_dir)
set(LIBRARY_NAME_INCLUDE_DIRS ${source_dir}/include)
endif()
if(EXISTS "${LIBRARY_NAME_INCLUDE_DIR}")
set(Library_Name_FOUND 1)
else()
set(Library_Name_FOUND 0)
endif()
Then in the projects CMakeLists.txt, just add:
find_package(Library_Name)
include_directories(${LIBRARY_NAME_INCLUDE_DIRS})
You can replace GIT with SVN above and provide the URL of an svn repo and it will work as well. Other version control systems are also available.

How to work around CMake + XCode 4 paths dependencies?

I have projects structured like so:
Libs/
Apps1/
Apps2/
In each folder is a CMakeLists.txt. I would like to generate a project file for each of the folders, and each AppsN references Libs. My method of doing that is by calling CMake's add_subdirectory(../Libs/Source/LibN) etc.
Now when I do this, CMake says add_subdirectory must specify a unique absolute path for the binary output folder.
See this post:
Xcode dependencies across different build directories?
XCode can not handle dependencies when the build output folder is unique per target. It needs one folder. And CMake does this by default, it just refuses to when the folder is not a subdir.
I tried altering and changing the output path after the target is created. This will build the objects to the output folder, XCode sees them, but all references to this target in the CMake script will use the unique path.
Proposed solutions are:
include project files in App1/Projects/Subdir and duplicate projects in an irrelevant location
reorganize my folders to a shared parent folder to avoid this CMake craziness, which presents some security problems for me (as some dirs are not public)
never refer to the target by its CMake name, instead using the shared path name. Not sure how to do this properly
try and get this patched on the CMake side somehow
switch to premake
Try to add the following to the root CMakeLists.txt:
CMAKE_MINIMUM_REQUIRED(VERSION 2.8.0)
PROJECT (ContainerProject)
SET (LIBRARY_OUTPUT_PATH ${ContainerProject_BINARY_DIR}/bin CACHE PATH
"Single output directory for building all libraries.")
SET (EXECUTABLE_OUTPUT_PATH ${ContainerProject_BINARY_DIR}/bin CACHE PATH
"Single output directory for building all executables.")
MARK_AS_ADVANCED(LIBRARY_OUTPUT_PATH EXECUTABLE_OUTPUT_PATH)
# for common headers (all project could include them, off topic)
INCLUDE_DIRECTORIES(ContainerProject_SOURCE_DIR/include)
# for add_subdirectory:
# 1) do not use relative paths (just as an addition to absolute path),
# 2) include your stuffs in build order, so your path structure should
# depend on build order,
# 3) you could use all variables what are already loaded in previous
# add_subdirectory commands.
#
# - inside here you should make CMakeLists.txt for all libs and for the
# container folders, too.
add_subdirectory(Libs)
# you could use Libs inside Apps, because they have been in this point of
# the script
add_subdirectory(Apps1)
add_subdirectory(Apps2)
In Libs CMakeLists.txt:
add_subdirectory(Source)
In Source CMakeLists.txt:
add_subdirectory(Lib1)
# Lib2 could depend on Lib1
add_subdirectory(Lib2)
In this way all Apps could use all libraries. All binary will be made to your binary ${root}/bin.
An example lib:
PROJECT(ExampleLib)
INCLUDE_DIRECTORIES(
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_SOURCE_DIR}
)
SET(ExampleLibSrcs
...
)
ADD_LIBRARY(ExampleLib SHARED ${ExampleLibSrcs})
An example executable (with dependency):
PROJECT(ExampleBin)
INCLUDE_DIRECTORIES(
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_SOURCE_DIR}
${ExampleLib_SOURCE_DIR}
)
SET(ExampleBinSrcs
...
)
# OSX gui style executable (Finder could use it)
ADD_EXECUTABLE(ExampleBin MACOSX_BUNDLE ${ExampleBinSrcs})
TARGET_LINK_LIBRARIES(ExampleBin
ExampleLib
)
Here is a stupid and working example.

splitting a project into a library and a application

I'm using cmake for my project. No I want to split some parts into a library and use this for 2 different applications.
Now I don't how how to do this subprojects in cmake. My first attempt was to use the add_subdirectory command:
cmake_minimum_required(VERSION 2.8)
add_subdirectory(MSI)
message("Building MsiQtWizard with: ${MSI_INCLUDE_DIR}")
add_subdirectory(MsiQtWizard)
So MSI would be my library. Inside the MSI folder is another cmakelists which is basically a standalone list for building the library. I thought I could make the MsiQtWizard project also a standalone cmakelists, so I could theoretically build MSI and use the library to build MsiQtWizard (and other projects).
The cmakelists in the root directory would just be a helper to build the library and the GUI in one single step.
The problem is, for building MsiQtWizard, I need the include path to msi and the static library binaries. I tried to do something like that at the end of MIS/CMakelists.txt:
### Set variables, other scripts may use ###
SET(MSI_INCLUDE_DIR include)
MESSAGE("Include directory is: ${MSI_INCLUDE_DIR}")
and in the MsiQtWizard/CMakelists:
##### external libraries #####
#MSI
find_path(MSI_INCLUDE_DIR REQUIRED msi/Image.hpp
PATH_SUFFIXES MSI/include include)
My intend is, that MsiQtWizard will search for msi, if the varaible was not previously set (so that you could use this cmakelists as a standalone). When building MSI, I want to save the include path (and later binary locations) and pass it to MsiQtWizard - but the value is gone as soon as I'm back in my root cmakelists.
So that is, what I tried. My Question is now: How would I correctly split my project into a library and a (later multiple) application and can I do it in a way that I can also build them independently?
Or, more specific: How can I pass values from a node CMakelist to the root CMakeList (like I tried with MSI_INCLUDE_DIR)?
If your building a library - its best to completely separate it from the application build. Otherwise you are coupling your library with your application with cmake, which in my view defeats the purpose of building a library.
When building your library you will want something like
project (MSILibrary)
ADD_LIBRARY(MSILibrary src/MSI1.cpp src/MSI2.cpp)
install (TARGETS MSILibrary DESTINATION lib)
where src contains your library code. You can then make then sudo make install your library to your standard library location (e.g. /usr/lib).
You can then use your library in any subsequent project. Put these in a new directory and create a new CMakeLists.txt for them.
You will want something like,
#include find modules
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
project (MSI-project-1)
find_package(MSILibrary REQUIRED)
IF(MSILibrary_FOUND)
include_directories(${MSILibrary_INCLUDE_DIRS}
ENDIF(MSILibrary_FOUND )
target_link_libraries (MSI-project-1 ${MSILibrary_LIBRARIES})
install (TARGETS MSI-project-1 DESTINATION bin)
Now all you need to do is help cmake find you library.
You can include a module for this. In the file ./cmake/Modules/FindMSILibrary.cmake type something like:
# - Try to find MSILibrary library
# Once done, this will define
#
# MSILibrary_FOUND - system has MSILibrary
# MSILibrary_INCLUDE_DIRS - the MSILibrary include directories
# MSILibrary_LIBRARIES - link these to use MSILibrary
## Google this script (I think its fairly standard, but was not bundled with my CMAKE) - it helps find the macros.
include(LibFindMacros)
# Dependencies
libfind_package(MSILibrary)
# Use pkg-config to get hints about paths
libfind_pkg_check_modules(MSILibrary_PKGCONF MSILibrary)
# Include dir
find_path(MSILibrary_INCLUDE_DIR
NAMES MSI.hpp
PATHS ${MSI_Library_PKGCONF_INCLUDE_DIRS}
)
# Finally the library itself
find_library(MSILibrary_LIBRARY
NAMES MSILibrary
PATHS ${MSILibrary_PKGCONF_LIBRARY_DIRS}
)
# Set the include dir variables and the libraries and let libfind_process do the rest.
# NOTE: Singular variables for this library, plural for libraries this this lib depends on.
set(MSILibrary_PROCESS_INCLUDES MSILibrary_INCLUDE_DIR MSILibrary_INCLUDE_DIRS)
set(MSILibrary_PROCESS_LIBS MSILibrary_LIBRARY MSILibrary_LIBRARIES)
libfind_process(MSILibrary)
That should be it.
EDIT:
If you really want to package your applications with your library (perhaps some example applications), then you can do something like so:
In your root CMakeLists.txt
cmake_minimum_required (VERSION 2.6)
project (MSIProject)
# The version number.
set (MSIProject_VERSION_MAJOR 0)
set (MSIProject_VERSION_MINOR 1)
set (MSIProject_PATCH_LEVEL 3 )
# project options
OPTION( BUILD_SHARED_LIBS "Set to OFF to build static libraries" ON )
OPTION( BUILD_EXAMPLES "Set to OFF to skip building the examples" ON )
# Put the libaries and binaries that get built into directories at the
# top of the build tree rather than in hard-to-find leaf
# directories.
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
##########################################################################
# Build the library
##########################################################################
add_subdirectory(MSI-src)
##################
# Build your example Apps if requested
############
IF( BUILD_EXAMPLES )
add_subdirectory(example/MSI-project-1)
add_subdirectory(example/MSI-project-2)
ENDIF( BUILD_EXAMPLES )
Your library MSI-src/CMakeFiles.txt will be as before, and your example/MSI-project-1/CMakeLists.txt will be something like
## Make the InferData example project
project (MSI-project-1)
#include MSI library
include_directories ("${MSILibrary_SOURCE_DIR}/include")
#include the includes of this project
include_directories ("${MSI-project-1_SOURCE_DIR}/../include")
#build
add_executable(MSI-project-1 src/P1.cpp)
target_link_libraries (MSI-project-1 MSILibrary) #link
install (TARGETS MSI-project-1 DESTINATION bin)