cmake: external project update and a work offline - c++

I was using external projects to keep libraries up to date for a long time
something like:
include(AddAsio)
and script AddAsio.cmake like this:
cmake_minimum_required(VERSION 2.8.8)
include(ExternalProject)
find_package(Git REQUIRED)
ExternalProject_Add(
asio
PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/include/ext
GIT_REPOSITORY https://github.com/chriskohlhoff/asio.git
GIT_TAG master
TIMEOUT 10
UPDATE_COMMAND ${GIT_EXECUTABLE} pull
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
LOG_DOWNLOAD ON
)
ExternalProject_Get_Property(asio source_dir)
set(ASIO_INCLUDE_DIR ${source_dir}/asio/include CACHE INTERNAL "Path to include folder for Asio")
include_directories(${INCLUDE_DIRECTORIES} ${ASIO_INCLUDE_DIR})
message("asio source dir" ${ASIO_INCLUDE_DIR})
there was no issue for a long time, however yesterday I've faced with a problem when my internet provider was not working and I tried to work offline.
this script tries to update (git pull, look at external project definition) each time of build, even all required headers do exist there. and breaks build process in case of no connection.
any idea how to keep this mechanism of components update but do not stop a building in case of libraries already exist locally?

AFAIK, there is no easy way to make an external project not to update by examining if certain files exist. However, you could set UPDATE_DISCONNECTED to an external project or set EP_UPDATE_DISCONNECTED property of a directory to disable update of external projects. Along with a build option, the process could be made dynamically. For example, you could build the following project without updating foo and bar by configuring with the command cmake -DBUILD_OFFLINE=ON
cmake_minimum_required (VERSION 2.8.11)
project (myrootproject C)
add_library(hello SHARED hello.c)
option(BUILD_OFFLINE "Build offline" OFF)
if (${BUILD_OFFLINE})
set_property(DIRECTORY ${myrootproject_SOURCE_DIR}
PROPERTY EP_UPDATE_DISCONNECTED 1)
endif()
include(ExternalProject)
ExternalProject_Add(foo
SVN_REPOSITORY svn://foobar.org/
)
ExternalProject_Add(bar
GIT_REPOSITORY https://github.com/blah/blah
)

Related

CMake - Alternate option to FetchContent when the source file already exists locally

I tried building the ORTools github package locally using cmake and it builds without any errors. However the environment which we are planning to ultimately use does not have an outbound network connection. The problem I see is that https://github.com/google/or-tools/blob/v9.4/cmake/dependencies/CMakeLists.txt performs a Git Clone to download the dependencies and add them. Since there is no outbound network access this step fails and I'm unable to build the dependency. To circumvent this we are planning to manually download the dependencies and add them to https://github.com/google/or-tools/blob/v9.4/cmake/dependencies/ folder.
I'm pretty new to CMake and I'm not sure what changes have to be made to accommodate this.
For example, what would the following snippet need to be changed to if I cloned Zlib v1.2.11 repository and added it to https://github.com/google/or-tools/blob/v9.4/cmake/dependencies/?
# ##############################################################################
# ZLIB
# ##############################################################################
if(BUILD_ZLIB)
message(CHECK_START "Fetching ZLIB")
list(APPEND CMAKE_MESSAGE_INDENT " ")
FetchContent_Declare(
zlib
GIT_REPOSITORY "https://github.com/madler/ZLIB.git"
GIT_TAG "v1.2.11"
PATCH_COMMAND git apply --ignore-whitespace "${CMAKE_CURRENT_LIST_DIR}/../../patches/ZLIB.patch")
FetchContent_MakeAvailable(zlib)
list(POP_BACK CMAKE_MESSAGE_INDENT)
message(CHECK_PASS "fetched")
endif()
Can FetchContent_Declare be used to point to a directory that already contains the source files? What's the alternative?
You could specify SOURCE_DIR parameter for FetchContent_Declare and omit download options:
FetchContent_Declare(
zlib
SOURCE_DIR <path/to/existing/directory>
PATCH_COMMAND git apply --ignore-whitespace "${CMAKE_CURRENT_LIST_DIR}/../../patches/ZLIB.patch"
)
This works in the same way as in ExternalProject_Add command, which options are accepted by FetchContent_Declare.

CMake declare dependency of function on ExternalProject_Add

TLDR:
My problem is that CMake starts executing this function before
downloading the repository. I would like to declare a dependency for
that function on ExternalProject_Add so that CMake understands that it
should download, build and then run the function.
Context:
I have a cmake module SomeModule.cmake which is supposed to add flatbuffers as an external project from its repository and build it. The build would produce flatbuffers compiler executable which I intend to use in some/directory/CMakeLists.txt file to generate c++ header files from a flatbuffers schema. So in that same CMake module that I use ExternalProject_Add, I have declared a CMake function that generates header files from a given set of schema files and somewhere in some/directory/CMakeLists.txt I call that function.
My problem is that CMake starts executing this function before downloading the repository. I would like to declare a dependency for that function on ExternalProject_Add so that CMake understands that it should download, build and then run the function.
Enough talk. Here's relevant parts of the code:
SomeModule.cmake:
include(ExternalProject)
set(flatbuffers_CMAKE_ARGS
"-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
"-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
"-DFLATBUFFERS_BUILD_TESTS=OFF"
"-DFLATBUFFERS_BUILD_FLATC=ON"
"-DFLATBUFFERS_BUILD_FLATHASH=OFF"
"-DCMAKE_INSTALL_PREFIX=${OTS_DEPENDENCIES}"
)
ExternalProject_Add(
flatbuffers
GIT_REPOSITORY "https://github.com/google/flatbuffers.git"
GIT_TAG "v1.9.0"
SOURCE_DIR "${OTS_DEPDENDENCIES_DIR}/flatbuffers"
BINARY_DIR "${OTS_DEPDENDENCIES_DIR}/flatbuffers"
CMAKE_ARGS "${flatbuffers_CMAKE_ARGS}"
INSTALL_COMMAND ""
)
ExternalProject_Get_Property(flatbuffers SOURCE_DIR)
ExternalProject_Get_Property(flatbuffers BINARY_DIR)
set(flatbuffers_SOURCE_DIR ${SOURCE_DIR})
set(flatbuffers_BINARY_DIR ${BINARY_DIR})
set(flatbuffers_INCLUDE_DIR ${flatbuffers_SOURCE_DIR}/include)
set(flatbuffers_FLATC_EXECUTABLE ${flatbuffers_BINARY_DIR}/flatc)
# please assume that the variables above are all set to appropriate values
function(FlatbuffersGenerateCpp SCHEMA_FILES GENERATED_DIR GENERATED_CXX)
foreach(SCHEMA_FILE ${SCHEMA_FILES})
get_filename_component(NAME ${SCHEMA_FILE} NAME_WE)
set(GENERATED_HEADER_FILE_PATH ${GENERATED_DIR}/${NAME}_generated.h)
message(STATUS "attempting to generate: ${GENERATED_HEADER_FILE_PATH}")
add_custom_command(
DEPENDS ${flatbuffers_FLATC_EXECUTABLE}
OUTPUT ${GENERATED_HEADER_FILE_PATH}
COMMAND ${flatbuffers_FLATC_EXECUTABLE} -o ${GENERATED_DIR} -c ${SCHEMA_FILE}
COMMENT "generating flatbuffers c++ header file: ${GENERATED_HEADER_FILE_PATH}"
)
list(APPEND GENERATED_FILES ${GENERATED_HEADER_FILE_PATH})
endforeach()
message(STATUS "generated c++ header files: ${GENERATED_FILES}")
set(${GENERATED_CXX} ${GENERATED_FILES} PARENT_SCOPE)
endfunction()
And some/directory/CMakeLists.txt:
# cmake module path is properly set so the following works:
include(SomeModule)
set(flatbuffers_GENERATED_INCLUDES_DIR
${CMAKE_BINARY_DIR}/generated/config/flatbuffers
)
FlatbuffersGenerateCpp(
"${flatbuffers_SCHEMAS}"
"${flatbuffers_GENERATED_INCLUDES_DIR}"
flatbuffers_GENERATED_CXX
)
add_library(
my_framework
SHARED
${THE_PUBLIC_HEADER_FILES}
${THE_IMPL_SOURCE_FILES}
${THE_IMPL_HEADER_FILES}
${flatbuffers_GENERATED_CXX}
)
add_dependencies(my_framework flatbuffers ${flatbuffers_GENERATED_CXX})
target_include_directories(my_framework PRIVATE ${flatbuffers_INCLUDE_DIR})
target_include_directories(my_framework PRIVATE ${CMAKE_SOURCE_DIR})
target_include_directories(my_framework PRIVATE ${CMAKE_BINARY_DIR}/generated)
set_source_files_properties(${flatbuffers_GENERATED_CXX} PROPERTIES GENERATED TRUE)
I did start modifying my code based on comment posted by Tsyvarev:
CMake functions are executed at configure stage, so you need to build the external project at configure stage too.
While I trusted that his proposed solution would work, I was slightly uncomfortable and kept thinking that there has to be a more elegant solution. I consulted with a colleague and came up with a better solution which is as simple as the following diff (which removes ${flatbuffers_GENERATED_CXX}).
- add_dependencies(my_framework flatbuffers ${flatbuffers_GENERATED_CXX})
+ add_dependencies(my_framework flatbuffers)
we reviewed that the problem with the code in question is that as is, CMake reads add_dependencies(my_framework flatbuffers ${flatbuffers_GENERATED_CXX}) and understands that it needs ${flatbuffers_GENERATED_CXX} as target to build my_framework so it proceeds with running the function. But there is no way for it to understand that the function depends on the external project. Now if we remove the explicit dependency declaration of ${flatbuffers_GENERATED_CXX}, CMake defers running the function to after resolving other dependencies (flatbuffers target) which will effectively download and build the external project prior to running the project.

Specify RPATH for externally built CMAKE project

How do I specify an RPATH for a project that was built externally in CMake (using 3.0.0) via ExternalProject_Add() macro?
For reference, let's say my ExternalProject is SFML. My external project call looks like:
set(SFML_INSTALL_PATH "${CMAKE_CURRENT_BINARY_DIR}/ext-deps/ext-sfml-build")
set(SFML_RPATH "${SFML_INSTALL_PATH}/src/lib")
set(SFML_CMAKE_ARGS -DCMAKE_BUILD_TYPE:STRING=Release -DUSE_STATIC_LIBS:BOOL=true
-DCMAKE_INSTALL_PREFIX:PATH=${SFML_INSTALL_PATH}
-DCMAKE_INSTALL_RPATH:PATH=${SFML_RPATH})
ExternalProject_Add(ext-sfml
GIT_REPOSITORY "${SFML_REPO}"
GIT_TAG "${SFML_TAG}"
URL SFML_URL
URL_HASH 256=${SFML_SHA256}
CMAKE_ARGS "${SFML_CMAKE_ARGS}"
TMP_DIR "${CMAKE_CURRENT_BINARY_DIR}/${EXT_DEPS_PREFIX}/ext-sfml-tmp"
STAMP_DIR "${CMAKE_CURRENT_BINARY_DIR}/${EXT_DEPS_PREFIX}/ext-sfml-stamp"
INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/${EXT_DEPS_PREFIX}/ext-sfml-build"
SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/${EXT_DEPS_PREFIX}/ext-sfml"
BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/${EXT_DEPS_PREFIX}/ext-sfml-build"
TEST_COMMAND "")
The error command I get when I run the build step (configuration is fine) is
Make Error at src/cmake_install.cmake:45 (file):
file RPATH_CHANGE could not write new RPATH:
/home/hendrix/repo/project/build/ext_deps/ext_sfml-build/lib
to the file:
/home/hendrix/repo/project/build/ext_deps/ext_sfml-build/bin/exe
Finally, this "/home/hendrix/repo/project/build/ext_deps/ext_sfml-build/lib" is the line I'm trying to modify. It appears that it is the value of my ${CMAKE_INSTALL_RPATH} variable but containing the install prefix of my ext-dep project and not the top-level master project.
I figured it out (thanks to Tsyvarev's comments). The externally built project that I was using set it's own RPATH. So the string that I found in the error message was that RPATH, not any RPATH which I set in my "top-level" project.
It appears that setting the RPATH of an external project can not override that external project's internally set RPATH (if it is using CMAKE).

cmake: retrieve library from git before using in a main cmakelists

I'm getting Catch library from github via additional cmakelists.txt which is included into main one:
cmake_minimum_required(VERSION 3.2)
project(MyProject)
add_subdirectory(ext/catch)
include_directories(${CATCH_INCLUDE_DIR} ${CXXOPTS_INCLUDE_DIR} src)
...
Where CmakeLists.txt for Catch is:
cmake_minimum_required(VERSION 2.8.8)
project(catch_builder CXX)
include(ExternalProject)
find_package(Git REQUIRED)
ExternalProject_Add(
catch
PREFIX ${CMAKE_CURRENT_SOURCE_DIR}
GIT_REPOSITORY https://github.com/philsquared/Catch.git
TIMEOUT 10
UPDATE_COMMAND ${GIT_EXECUTABLE} pull
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
LOG_DOWNLOAD ON
)
# Expose required variable (CATCH_INCLUDE_DIR) to parent scope
ExternalProject_Get_Property(catch source_dir)
set(CATCH_INCLUDE_DIR ${source_dir}/include CACHE INTERNAL "Path to include folder for Catch")
message(${CATCH_INCLUDE_DIR})
However, the building of main project is started earlier than Catch will be retrieved from Git.
How to fix this? What is wrong in my cmake script?
I think I had the same problem, what I did is:
Put AddCatch.cmake (The cmakelist of catch) in the CMAKE_MODULE_PATH
e.g.: set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
Remove the line: project(catch_builder CXX)
Replace add_subdirectory(ext/catch) by include(AddCatch)
And it should work just fine ;)
In your case, the download performed by ExternalProject_Add happens at _build_time, not when CMake is run, so the download won't have happened yet when add_subdirectory is called. With just a little bit of work, you can use CMake's ExternalProject_Add command to perform the download of Catch from github at CMake/configure time instead of at build time. This then means it will be present when you call add_subdirectory. There's an article here showing how to do this (it uses GoogleTest as the example and it links through to a fully generalised implementation you should be able to use directly for your situation). The general approach is that it creates a small script which is invoked via CMake's script mode to force the ExternalProject_Add call to happen immediately.

CMAKE with ROS to create external projects locally?

I am working with ROS and wanted to modify the ardrone driver of autonomy lab. Unfortunately I have problems to build external libraries with CMake.
The problem is following:
The compiling original CMakeLists includes an external project:
include(ExternalProject)
ExternalProject_Add(ardronelib
GIT_REPOSITORY git://github.com/AutonomyLab/ardronelib.git
GIT_TAG 2f987029c55531e4c0119c3600f9c57f935851ed
PREFIX ${CATKIN_DEVEL_PREFIX}
CONFIGURE_COMMAND echo "No configure"
BUILD_COMMAND make
INSTALL_COMMAND make install INSTALL_PREFIX=${CATKIN_DEVEL_PREFIX}/lib/
BUILD_IN_SOURCE 1
)
To modify this package, I downloaded it into my sourcefolder:
catkin_ws/src/my_project/
ardronelib/
src/ ...
launch/ ...
...
the ardronelib folder contains the makefile of the downloaded git project.
ardronelib/
ARDroneLib
.git
.travis.yml
LICENSE
Makefile
README.md
Basically I want to have the same libraries installed and linked as in the original version, but from this local folder.
My approach with:
include(ExternalProject)
ExternalProject_Add(ardronelib
SOURCE_DIR=${PROJECT_SOURCE_DIR}/ardronelib
PREFIX ${CATKIN_DEVEL_PREFIX}
CONFIGURE_COMMAND echo "No configure"
BUILD_COMMAND make
INSTALL_COMMAND make install INSTALL_PREFIX=${CATKIN_DEVEL_PREFIX}/lib/
BUILD_IN_SOURCE 1
)
and several other attempts didn't work. Any ideas how to build the project so that later I can use the locally build "ardronelib" within my project?
As the library is quite complex, the direct linking of the source files
would be to costly. Any ideas how I can use the same build structure, but instead of
GIT_REPOSITORY git://github.com/AutonomyLab/ardronelib.git
GIT_TAG 2f987029c55531e4c0119c3600f9c57f935851ed
us the local version:
${PROJECT_SOURCE_DIR}/ardronelib
I would be reaaally happy for any clues.
Thanks a lot in advance!