CMake ExternalProject does not unpack dependencies - c++

Question

I am building an application which uses tinyxml2 and few other dependencies (namely Irrlicht and IrrKlang) that I provide as .zip files in the Dependency subdirectory of my project:
.
├── CMakeLists.txt
├── Dependencies
│   ├── irrKlang-1.5.0.zip
│   ├── irrKlang-32bit-1.5.0.zip
│   ├── irrKlang-64bit-1.5.0.zip
│   ├── irrlicht-1.8.4.zip
│   └── tinyxml2-master.zip
├── Editor
│   ├── CMakeLists.txt
│   └── Sources
│   └── main.cpp
└── Game
   ├── CMakeLists.txt
   └── Sources
   └── main.cpp
NOTE: for a reference, full sources are available on GitHub, here I cut some corners to make the question shorter.
The top-level CMakeFiles.txt is set up is:
cmake_minimum_required(VERSION 3.16 FATAL_ERROR)
project(shoot-them VERSION 1.0.1 LANGUAGES CXX)
include(ExternalProject)
# few platform-specific variables like MAKE_COMMAND, PLATFORM_ARCH, etc.
# libraries (see below)
add_subdirectory(Game)
add_subdirectory(Editor
Both Irrlicht and IrrKlang come with pre-built libraries for Windows for x86, but not for Windows x64 and not for OSX. Hence I add it as a dependency like this (using the if(NOT IRRLICHT_LIBRARY_PATH) just to separate the code into a block:
if(NOT IRRLICHT_LIBRARY_PATH)
ExternalProject_Add(irrlicht-dep
URL ${CMAKE_CURRENT_LIST_DIR}/Dependencies/irrlicht-1.8.4.zip
PREFIX Dependencies/irrlicht
SOURCE_SUBDIR source/Irrlicht
CONFIGURE_COMMAND ""
BUILD_COMMAND "${MAKE_COMMAND}"
INSTALL_COMMAND ""
)
ExternalProject_Get_Property(irrlicht-dep SOURCE_DIR)
set(IRRLICHT_PATH ${SOURCE_DIR})
add_library(irrlicht SHARED IMPORTED GLOBAL)
set_target_properties(
irrlicht PROPERTIES
IMPORTED_LOCATION ${IRRLICHT_PATH}/lib/${IRRLICHT_PATH_SUFFIX}/Irrlicht${CMAKE_LINK_LIBRARY_SUFFIX}
IMPORTED_IMPLIB ${IRRLICHT_PATH}/lib/${IRRLICHT_PATH_SUFFIX}/Irrlicht${CMAKE_IMPORT_LIBRARY_SUFFIX}
INTERFACE_INCLUDE_DIRECTORIES ${IRRLICHT_PATH}/include
)
endif()
I follow the same principles for IrrKlang. But since tinyxml2 comes as a header and a source file and it comes packed with CMakeLists.txt, I just include it like this:
ExternalProject_Add(tinyxml2-dep
URL ${CMAKE_CURRENT_LIST_DIR}/Dependencies/tinyxml2-master.zip
PREFIX Dependencies/tinyxml2
INSTALL_COMMAND ""
)
ExternalProject_Get_Property(tinyxml2-dep SOURCE_DIR)
set(TINYXML2_PATH ${SOURCE_DIR})
add_subdirectory(${TINYXML2_PATH})
I define both Game and Editor sub-projects as follows:
cmake_minimum_required(VERSION 3.16 FATAL_ERROR)
project(Editor VERSION 1.0.1 LANGUAGES CXX)
set(EXECUTABLE_NAME Editor)
set(SOURCES Sources/main.cpp)
# platform-specific variables
set_target_properties(${EXECUTABLE_NAME} PROPERTIES
CXX_STANDARD 17
CXX_STANDARD_REQUIRED ON
CXX_EXTENSIONS ON
)
add_dependencies(${EXECUTABLE_NAME} tinyxml2 irrlicht)
target_link_libraries(${EXECUTABLE_NAME} PUBLIC ${LIBRARIES} tinyxml2 irrlicht)
if(NOT APPLE)
# Copy libraries' DLLs
add_custom_command(
TARGET ${EXECUTABLE_NAME} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:tinyxml2> $<TARGET_FILE_DIR:${EXECUTABLE_NAME}>
)
endif()
# TODO: copy DLLs, not LIBs
add_custom_command(
TARGET ${EXECUTABLE_NAME} POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:irrlicht> $<TARGET_FILE_DIR:${EXECUTABLE_NAME}>
)
The way I use ExternalProject specifically is because
I wanted to fix the versions I build my project with
I did not want to pollute my system with installing the libraries (hence no INSTALL steps, although I may completely misunderstand that concept)
I do not depend on 3rd party repositories being dead (hence shipping the ZIP files with the sources)
I do not rely on unbelievably outdated (and thus potentially non-working) find modules for CMake
To be fair: I am not aware of any best practices of building projects with CMake, so I might very well be completely wrong about all of the above, so please do correct me.
When I build this project in Visual Studio 2019 on Windows, it works like a charm. But whenever I try building the thing on OSX, I get failures:
none of the dependencies gets even unpacked
(because of p.1) the ${TINYXML2_DIR} is never set
(because of p.2) the tinyxml2 directory could not be found and thus added via add_subdirectory()
(because of p.3) the $<TARGET_FILE:tinyxml2> expression does not evaluate
(as a global consequence) the project does not build
The way I build project is rather simple:
cmake -Bbuild -H. && cmake --build build
What am I doing wrong?
Also, what is the right way to handle 3rd party dependencies with CMake?
I am very well aware that CMake is technically just a makefile (roughly speaking, since it is different for every build toolchain) generator, so my question is more about how do I tell CMake to generate the correct build files for each type of dependency that should be built with my project (pre-built, build from sources with CMake, build from sources with a custom command). I thought ExternalProject is supposed to handle just that, but apparently something went horribly wrong along the way.

Answer 1

I have played with both a solution suggested by #Mizux in their comment and had some success with two different approaches.
1. vcpkg
This is arguably the easier of the two. It requires vcpkg installed.
See this commit for example.
Create a manifest file, vcpkg.json in the project root directory, listing all the dependencies used by the project:
{
"name": "PROJECT_NAME",
"version-string": "0.1.0",
"dependencies": [
"irrlicht",
"tinyxml2"
]
}
You can also use CLI to generate the manifest by using vcpkg install command.
Use the find_package from CMake to link libraries to each target - in the child CMakeLists.txt:
find_package(irrlicht CONFIG REQUIRED)
find_package(tinyxml2 CONFIG REQUIRED)
target_link_libraries(${EXECUTABLE_NAME} PUBLIC ${LIBRARIES} tinyxml2::tinyxml2 Irrlicht)
Important: when configuring the project, pass the path to the vcpkg CMake module:
cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=[vcpkg root]/scripts/buildsystems/vcpkg.cmake
2. FetchContent + ExternalProject
Since my project depends on both a library that ships with CMakeLists.txt and a library that does not, this is a nice example.
See this commit for example.
First, as #Mizux mentioned, FetchContent works at configure time - it downloads and unpacks the dependency when you configure the project (call cmake -S . -B build). Then, since irrlicht does not ship with CMakeLists.txt, you either use ExternalProject_Add to build it with custom command (make in my case) or add it as a sub-directory to the project.
FetchContent part:
include(FetchContent)
FetchContent_Declare(irrlicht
URL ${CMAKE_CURRENT_LIST_DIR}/Dependencies/irrlicht-1.8.4.zip
)
FetchContent_GetProperties(irrlicht)
if(NOT irrlicht_POPULATED)
FetchContent_Populate(irrlicht)
endif()
set(IRRLICHT_PATH ${irrlicht_SOURCE_DIR})
ExternalProject part:
include(ExternalProject)
if(NOT WIN32)
ExternalProject_Add(irrlicht-dep
SOURCE_DIR "${IRRLICHT_PATH}/source/Irrlicht"
BUILD_IN_SOURCE true
CONFIGURE_COMMAND ""
BUILD_COMMAND "${MAKE_COMMAND}"
)
endif()
add_library(irrlicht SHARED IMPORTED GLOBAL)
set_target_properties(
irrlicht PROPERTIES
IMPORTED_LOCATION ${IRRLICHT_PATH}/lib/${IRRLICHT_PATH_SUFFIX}/Irrlicht${CMAKE_LINK_LIBRARY_SUFFIX}
IMPORTED_IMPLIB ${IRRLICHT_PATH}/lib/${IRRLICHT_PATH_SUFFIX}/Irrlicht${CMAKE_IMPORT_LIBRARY_SUFFIX}
INTERFACE_INCLUDE_DIRECTORIES ${IRRLICHT_PATH}/include
)
Note the important bits in the ExternalProject configuration:
irrlicht_SOURCE_DIR variable is populated by FetchContent
SOURCE_DIR "${irrlicht_SOURCE_DIR}/source/Irrlicht" - tells ExternalProject where the unpacked sources are; if this directory is not empty (which it should be not, since FetchContent should have populated it at project configuration time), ExternalProject will skip the download phase
BUILD_IN_SOURCE true - builds from within the source directory
CONFIGURE_COMMAND "" - skips the dependency configuration phase, so not cmake will be executed for the dependency
BUILD_COMMAND "make" - uses a specific command to build the dependency from sources
if(NOT WIN32) - only uses ExternalProject to build the dependency; since Irrlicht comes with pre-built libraries for Win32, this should build the thing for the other platforms (including Win64)