Linking waLBerla to my own code using Cmake - c++

Question

I want to play around with waLBerla (website and gitlab), which is a C++ library that can be used to simulate certain physical behavior, like fluid dynamics using the Lattice Boltzmann method, but having major difficulties on how to properly set up my CMakeLists.txt, as I don't have much experience with that.
I followed the instructions to install the library, using some of the dependencies like OpenMP, CUDA and OpenMESH, which went fine.
But when trying to compile for example one of the tutorials, I just can't get it to work. So far I have the following Cmake file, where I manually list all the static libraries that have been build when compiling waLBerla:
CMAKE_MINIMUM_REQUIRED (VERSION 3.20)
set (CMAKE_CXX_STANDARD 20)
PROJECT (sugar_dissolving)
add_executable(${PROJECT_NAME} main.cpp)
find_package( waLBerla REQUIRED)
find_package(MPI REQUIRED)
find_package(OpenMP REQUIRED)
target_link_libraries(sugar_dissolving PRIVATE OpenMP::OpenMP_CXX)
target_link_libraries(sugar_dissolving PRIVATE
${MPI_LIBRARIES}
/usr/local/lib/libcore.a
/usr/local/lib/libblockforest.a
/usr/local/lib/liblbm.a
/usr/local/lib/libboundary.a
/usr/local/lib/libdomain_decomposition.a
/usr/local/lib/libblas.a
/usr/local/lib/libfield.a
/usr/local/lib/libexecutiontree.a
/usr/local/lib/liblapack.a
/usr/local/lib/libgeometry.a
/usr/local/lib/libmesh.a
/usr/local/lib/libgather.a
/usr/local/lib/libtimeloop.a
/usr/local/lib/libvtk.a
/usr/local/lib/libcuda.a
/usr/local/lib/libgui.a
/usr/local/lib/liblbm_mesapd_coupling.a
/usr/local/lib/libmesa_pd.a
/usr/local/lib/libpde.a
/usr/local/lib/libpe.a
/usr/local/lib/libpe_coupling.a
/usr/local/lib/libpostprocessing.a
/usr/local/lib/libpython_coupling.a
/usr/local/lib/libsqlite.a
<private>/walberla/build/extern/lodepng/liblodepng.a
)
waLBerla exports a package when building it, so I am using find_package(). At first it couldn't find the waLBerla-config.cmake file, so I manually copied it into the package registery, but I think this is not the way to do it? In general, it feels very weird to have to manually list all the the libraries. In this case I don't know if I can know in advance which libraries I need.
It does manage to compile the tutorial, i.e. the header files are found correctly, but then it fails at the linking stage. It knows where to find most functions, but for some reason some of the math functions, which are in libcore.a, don't get found properly. Similarly for some MPI functions, defined by waLBerla:
/usr/bin/ld: /usr/local/lib/libblockforest.a(Initialization.cpp.o): in function `walberla::blockforest::calculateCellDistribution(walberla::math::Vector3<unsigned long> const&, unsigned long, walberla::math::Vector3<unsigned long>&, walberla::math::Vector3<unsigned long>&)':
Initialization.cpp:(.text+0xbfc): undefined reference to `walberla::math::getFactors(unsigned long, unsigned long, std::vector<double, std::allocator<double> > const&)'
/usr/bin/ld: /usr/local/lib/libblockforest.a(SetupBlockForest.cpp.o): in function `walberla::blockforest::SetupBlockForest::getNextBlock(walberla::blockforest::SetupBlock const*) const':
SetupBlockForest.cpp:(.text+0x117f): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: /usr/local/lib/libblockforest.a(SetupBlockForest.cpp.o): in function `walberla::blockforest::SetupBlockForest::getNextBlock(walberla::blockforest::SetupBlock const*)':
SetupBlockForest.cpp:(.text+0x1287): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: /usr/local/lib/libblockforest.a(SetupBlockForest.cpp.o): in function `walberla::blockforest::SetupBlockForest::getBlock(walberla::blockforest::BlockID const&) const':
SetupBlockForest.cpp:(.text+0x132d): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0x13b8): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0x142c): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: /usr/local/lib/libblockforest.a(SetupBlockForest.cpp.o):SetupBlockForest.cpp:(.text+0x3e06): more undefined references to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)' follow
/usr/bin/ld: /usr/local/lib/libblockforest.a(SetupBlockForest.cpp.o): in function `walberla::blockforest::SetupBlockForest::toStream(std::ostream&) const':
SetupBlockForest.cpp:(.text+0x9379): undefined reference to `walberla::math::Sample::variance(double) const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0x93cd): undefined reference to `walberla::math::Sample::relativeStdDeviation() const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0x95d9): undefined reference to `walberla::math::Sample::variance(double) const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0x962d): undefined reference to `walberla::math::Sample::relativeStdDeviation() const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0x9851): undefined reference to `walberla::math::Sample::variance(double) const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0x98a8): undefined reference to `walberla::math::Sample::relativeStdDeviation() const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0x9f00): undefined reference to `walberla::math::Sample::variance(double) const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0x9f53): undefined reference to `walberla::math::Sample::relativeStdDeviation() const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0xa160): undefined reference to `walberla::math::Sample::variance(double) const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0xa1b3): undefined reference to `walberla::math::Sample::relativeStdDeviation() const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0xa3d0): undefined reference to `walberla::math::Sample::variance(double) const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0xa426): undefined reference to `walberla::math::Sample::relativeStdDeviation() const'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0xa576): undefined reference to `walberla::real_comparison::Epsilon<double>::value'
/usr/bin/ld: /usr/local/lib/libblockforest.a(SetupBlockForest.cpp.o): in function `walberla::blockforest::SetupBlockForest::init(walberla::math::GenericAABB<double> const&, unsigned long, unsigned long, unsigned long, bool, bool, bool, walberla::Set<walberla::uid::UID<walberla::uid::suidgenerator::S> > const&)':
SetupBlockForest.cpp:(.text+0x14267): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0x1454a): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: SetupBlockForest.cpp:(.text+0x159c1): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: /usr/local/lib/libblockforest.a(StructuredBlockForest.cpp.o): in function `walberla::blockforest::StructuredBlockForest::blockExists(walberla::cell::Cell const&, unsigned long) const':
StructuredBlockForest.cpp:(.text+0x24d): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: /usr/local/lib/libblockforest.a(StructuredBlockForest.cpp.o): in function `walberla::blockforest::StructuredBlockForest::getBlockID(walberla::domain_decomposition::IBlockID&, walberla::cell::Cell const&, unsigned long) const':
StructuredBlockForest.cpp:(.text+0x410): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: /usr/local/lib/libblockforest.a(BlockForest.cpp.o):BlockForest.cpp:(.text+0x72a0): more undefined references to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)' follow
/usr/bin/ld: /usr/local/lib/libblockforest.a(BlockForest.cpp.o): in function `walberla::blockforest::BlockForest::constructBlockInformation()':
BlockForest.cpp:(.text+0x1c62a): undefined reference to `walberla::mpi::allGathervBuffer(walberla::mpi::GenericSendBuffer<unsigned char, walberla::mpi::OptimalGrowth> const&, walberla::mpi::GenericRecvBuffer<unsigned char>&, ompi_communicator_t*)'
/usr/bin/ld: /usr/local/lib/libblockforest.a(BlockForest.cpp.o): in function `void walberla::blockforest::BlockID::toBuffer<walberla::mpi::GenericSendBuffer<unsigned char, walberla::mpi::OptimalGrowth> >(walberla::mpi::GenericSendBuffer<unsigned char, walberla::mpi::OptimalGrowth>&) const':
BlockForest.cpp:(.text._ZNK8walberla11blockforest7BlockID8toBufferINS_3mpi17GenericSendBufferIhNS3_13OptimalGrowthEEEEEvRT_[_ZNK8walberla11blockforest7BlockID8toBufferINS_3mpi17GenericSendBufferIhNS3_13OptimalGrowthEEEEEvRT_]+0x1a): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: /usr/local/lib/libblockforest.a(BlockForest.cpp.o): in function `std::vector<walberla::uid::UID<walberla::uid::suidgenerator::S>, std::allocator<walberla::uid::UID<walberla::uid::suidgenerator::S> > > walberla::mpi::allReduceSet<walberla::uid::UID<walberla::uid::suidgenerator::S> >(std::vector<walberla::uid::UID<walberla::uid::suidgenerator::S>, std::allocator<walberla::uid::UID<walberla::uid::suidgenerator::S> > >, walberla::mpi::SetOperation, ompi_communicator_t*, int)':
BlockForest.cpp:(.text._ZN8walberla3mpi12allReduceSetINS_3uid3UIDINS2_13suidgenerator1SEEEEESt6vectorIT_SaIS8_EESA_NS0_12SetOperationEP19ompi_communicator_ti[_ZN8walberla3mpi12allReduceSetINS_3uid3UIDINS2_13suidgenerator1SEEEEESt6vectorIT_SaIS8_EESA_NS0_12SetOperationEP19ompi_communicator_ti]+0x3b2): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: /usr/local/lib/libblockforest.a(BlockReconstruction.cpp.o): in function `walberla::blockforest::BlockReconstruction::reconstructAABB(walberla::math::GenericAABB<double>&, walberla::blockforest::BlockID const&, walberla::math::GenericAABB<double> const&, unsigned long, unsigned long, unsigned long, unsigned long)':
BlockReconstruction.cpp:(.text+0x8d): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: BlockReconstruction.cpp:(.text+0x129): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: BlockReconstruction.cpp:(.text+0x4eb): undefined reference to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)'
/usr/bin/ld: /usr/local/lib/libblockforest.a(PhantomBlockForest.cpp.o):PhantomBlockForest.cpp:(.text+0x6586): more undefined references to `unsigned long walberla::math::uintMSBPosition<unsigned long>(unsigned long)' follow
/usr/bin/ld: /usr/local/lib/libdomain_decomposition.a(BlockStorage.cpp.o): in function `walberla::domain_decomposition::BlockStorage::loadBlockData(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, walberla::selectable::SetSelectableObject<std::shared_ptr<walberla::domain_decomposition::internal::BlockDataHandlingWrapper>, walberla::uid::UID<walberla::uid::suidgenerator::S> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)':
BlockStorage.cpp:(.text+0x31cf): undefined reference to `walberla::mpi::readMPIIO(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, walberla::mpi::GenericRecvBuffer<unsigned char>&)'
/usr/bin/ld: /usr/local/lib/libdomain_decomposition.a(BlockStorage.cpp.o): in function `walberla::domain_decomposition::BlockStorage::saveBlockData(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, walberla::domain_decomposition::BlockDataID const&)':
BlockStorage.cpp:(.text+0x5f8d): undefined reference to `walberla::mpi::writeMPIIO(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, walberla::mpi::GenericSendBuffer<unsigned char, walberla::mpi::OptimalGrowth>&)'
/usr/bin/ld: /usr/local/lib/libdomain_decomposition.a(IBlock.cpp.o): in function `walberla::domain_decomposition::IBlock::operator==(walberla::domain_decomposition::IBlock const&) const':
IBlock.cpp:(.text+0x1d7): undefined reference to `walberla::real_comparison::Epsilon<double>::value'
/usr/bin/ld: /usr/local/lib/libdomain_decomposition.a(StructuredBlockStorage.cpp.o): in function `walberla::domain_decomposition::StructuredBlockStorage::isCellAlignedAABB(walberla::math::GenericAABB<double> const&, unsigned long) const':
StructuredBlockStorage.cpp:(.text+0x721): undefined reference to `walberla::real_comparison::Epsilon<double>::value'
collect2: error: ld returned 1 exit status
make[2]: *** [CMakeFiles/sugar_dissolving.dir/build.make:126: sugar_dissolving] Error 1
make[1]: *** [CMakeFiles/Makefile2:2335: CMakeFiles/sugar_dissolving.dir/all] Error 2
make: *** [Makefile:136: all] Error 2
When building waLBerla, it also compiled these tutorials, so I know I have all the necessary packages and libraries, but how do I get it to compile, preferably in a modern Cmake way, in a different directory.
For completeness, here is the output of tree:
.
├── build
│   ├── CMakeCache.txt
│   ├── CMakeFiles
│   │   ├── 3.20.0
│   │   │   ├── CMakeCCompiler.cmake
│   │   │   ├── CMakeCXXCompiler.cmake
│   │   │   ├── CMakeDetermineCompilerABI_C.bin
│   │   │   ├── CMakeDetermineCompilerABI_CXX.bin
│   │   │   ├── CMakeSystem.cmake
│   │   │   ├── CompilerIdC
│   │   │   │   ├── a.out
│   │   │   │   ├── CMakeCCompilerId.c
│   │   │   │   └── tmp
│   │   │   └── CompilerIdCXX
│   │   │   ├── a.out
│   │   │   ├── CMakeCXXCompilerId.cpp
│   │   │   └── tmp
│   │   ├── cmake.check_cache
│   │   ├── CMakeDirectoryInformation.cmake
│   │   ├── CMakeError.log
│   │   ├── CMakeOutput.log
│   │   ├── CMakeRuleHashes.txt
│   │   ├── CMakeTmp
│   │   ├── FindMPI
│   │   │   ├── test_mpi_C.bin
│   │   │   ├── test_mpi.cpp
│   │   │   └── test_mpi_CXX.bin
│   │   ├── FindOpenMP
│   │   │   ├── ompver_C.bin
│   │   │   ├── ompver_CXX.bin
│   │   │   ├── OpenMPCheckVersion.c
│   │   │   ├── OpenMPCheckVersion.cpp
│   │   │   ├── OpenMPTryFlag.c
│   │   │   └── OpenMPTryFlag.cpp
│   │   ├── Makefile2
│   │   ├── Makefile.cmake
│   │   ├── Progress
│   │   │   └── count.txt
│   │   ├── progress.marks
│   │   ├── sugar_dissolving.dir
│   │   │   ├── build.make
│   │   │   ├── cmake_clean.cmake
│   │   │   ├── compiler_depend.internal
│   │   │   ├── compiler_depend.make
│   │   │   ├── compiler_depend.ts
│   │   │   ├── DependInfo.cmake
│   │   │   ├── depend.make
│   │   │   ├── flags.make
│   │   │   ├── link.txt
│   │   │   ├── main.cpp.o
│   │   │   ├── main.cpp.o.d
│   │   │   └── progress.make
│   │   └── TargetDirectories.txt
│   ├── cmake_install.cmake
│   ├── compile_commands.json
│   └── Makefile
├── CMakeLists.txt
└── main.cpp
Many thanks!

Answer 1

En example project structure can be found here.
For completeness, the file structure looks like:
example_app/
├── apps
│   ├── CMakeLists.txt (1)
│   └── example_app
│   ├── CMakeLists.txt (2)
│   └── ExampleApp.cpp
├── CMakeLists.txt (3)
├── FindwaLBerla.cmake (4)
├── README
├── src
│   └── CMakeLists.txt (5)
└── tests
└── CMakeLists.txt (6)
Going from top to bottom, the content of the CMakeLists.txt is
(1):
add_subdirectory( example_app )
(2):
waLBerla_link_files_to_builddir( *.prm *.py)
waLBerla_add_executable ( NAME ExampleApp
FILES ExampleApp.cpp
DEPENDS blockforest core field lbm geometry timeloop gui mesh)
(3):
CMAKE_MINIMUM_REQUIRED (VERSION 3.0)
PROJECT ( your_project_name )
enable_testing()
include_directories( src )
include_directories ( ${your_project_name_BINARY_DIR}/src )
# Extends cmake module path - so that FindwaLBerla.cmake in the current directory is found
set ( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${your_project_name_SOURCE_DIR} )
find_package( waLBerla )
add_subdirectory( src )
add_subdirectory( apps )
add_subdirectory( tests )
(4):
set( WALBERLA_DIR WALBERLA_DIR-NOTFOUND CACHE PATH "waLBerla path" )
if ( WALBERLA_DIR )
# WALBERLA_DIR has to point to the waLBerla source directory
# this command builds waLBerla (again) in the current build directory in the subfolder "walberla" (second argument)
add_subdirectory( ${WALBERLA_DIR} walberla )
waLBerla_import()
# Adds the 'src' and 'tests' directory of current app
list( APPEND WALBERLA_MODULE_DIRS "${CMAKE_SOURCE_DIR}/src" "${CMAKE_SOURCE_DIR}/tests" )
list( REMOVE_DUPLICATES WALBERLA_MODULE_DIRS )
set ( WALBERLA_MODULE_DIRS ${WALBERLA_MODULE_DIRS} CACHE INTERNAL "All folders that contain modules or tests" )
else()
message( FATAL_ERROR "waLBerla not found - Use 'cmake -DWALBERLA_DIR=path_to_waLBerla_sources pathToApplicationSources' " )
endif()
(5) + (6):
#add_subdirectory( name_of_your_module_subfolder )