Qt6+Cmake bad linking MacOS - c++

Question

I am trying to do a C++ project using CMake, Conan and Qt6.
At first, I was able to add Qt Core in the project (hello world at the time) and call qDebug() instead of std::cout but then I added a lib with Conan (libgphoto2) and now I can't use Qt in the project.
The build does not fail, but running the app gives this error :
dyld[68458]: Library not loaded: '#rpath/QtCore.framework/Versions/A/QtCore'
Referenced from: '/Users/chell/code/skytracker/cmake-build-debug/bin/skytracker'
Reason: tried: '/Library/Frameworks/QtCore.framework/Versions/A/QtCore' (no such file), '/System/Library/Frameworks/QtCore.framework/Versions/A/QtCore' (no such file)
If I understand correctly what I read on internet when trying to find a solution, is that at runtime, the linker will replace #rpath with a list of path to look for the lib. Here, it fails because it doesn't have ~/Qt/6.4.2/macos/lib where the QtCore.framework folder is ?
I'm not using Qt Creator at all, I'm using CLion, I'll just need some Qt libs.
Here's the CMakeLists.txt :
cmake_minimum_required(VERSION 3.22)
project(skytracker)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
#Conan integration
if(NOT EXISTS "${CMAKE_BINARY_DIR}/conan.cmake")
message(STATUS "Downloading conan.cmake from https://github.com/conan-io/cmake-conan")
file(DOWNLOAD "https://raw.githubusercontent.com/conan-io/cmake-conan/0.18.1/conan.cmake"
"${CMAKE_BINARY_DIR}/conan.cmake"
TLS_VERIFY ON)
endif()
include(${CMAKE_BINARY_DIR}/conan.cmake)
set(CI false CACHE BOOL "Set true if CI build")
if(${CI})
message("Using CI profile")
set(CMAKE_C_COMPILER /usr/bin/gcc)
set(CMAKE_CXX_COMPILER /usr/bin/g++)
set(CONANPROFILE ../CI.conanprofile)
endif()
conan_cmake_install(PATH_OR_REFERENCE ..
BUILD missing
REMOTE conancenter
PROFILE ${CONANPROFILE})
include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
conan_basic_setup(TARGETS)
set(sources src/cpp/main.cpp
src/cpp/services/PhotoService.cpp src/cpp/services/PhotoService.h
# src/cpp/entities/Camera.h src/cpp/entities/Camera.cpp
# src/cpp/entities/NikonCamera.cpp src/cpp/entities/NikonCamera.h
)
set(CMAKE_AUTOMOC ON) #Qt C++ extensions
set(CMAKE_AUTOUIC OFF) #Enable GUI compile
set(CMAKE_AUTORCC OFF) #Enable QRC files
if(APPLE)
set(Qt6_DIR ~/Qt/6.4.2/macos/lib/cmake/Qt6/)
set(CONANPROFILE macos.conanprofile)
set(CMAKE_MACOSX_RPATH 1)
elseif(UNIX)
set(Qt6_DIR ~/Qt/6.4.2/gcc_64/lib/cmake/Qt6/)
set(CONANPROFILE unix.conanprofile)
elseif(WIN32)
set(Qt6_DIR C:/Qt/6.4.2/mingw_64/lib/cmake/Qt6/)
set(CONANPROFILE windows.conanprofile)
endif()
#find the Qt libs used by the project
find_package(Qt6 6.4.2 REQUIRED COMPONENTS Core)
include(${CMAKE_BINARY_DIR}/conan_paths.cmake)
# give the compiler the path to the libs
link_directories("${CONAN_LIB_DIRS}")
# and to the headers
include_directories("${CONAN_INCLUDE_DIRS}")
add_executable(skytracker ${sources})
#link all Qt libs to the executable
target_link_libraries(skytracker PRIVATE Qt6::Core)
target_link_libraries(skytracker PRIVATE ${CONAN_LIBS})
Running otool -L skytracker gives me this :
skytracker:
#rpath/QtCore.framework/Versions/A/QtCore (compatibility version 6.0.0, current version 6.4.2)
/Users/chell/.conan/data/libgphoto2/2.5.27/_/_/package/8880a3412c5909fb46130c128d3ff83f4602ce9e/lib/libgphoto2.6.dylib (compatibility version 8.0.0, current version 8.0.0)
/Users/chell/.conan/data/libgphoto2/2.5.27/_/_/package/8880a3412c5909fb46130c128d3ff83f4602ce9e/lib/libgphoto2_port.12.dylib (compatibility version 13.0.0, current version 13.0.0)
/usr/lib/libobjc.A.dylib (compatibility version 1.0.0, current version 228.0.0)
/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit (compatibility version 1.0.0, current version 275.0.0)
/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation (compatibility version 150.0.0, current version 1953.255.0)
/System/Library/Frameworks/Security.framework/Versions/A/Security (compatibility version 1.0.0, current version 60420.60.24)
/System/Library/Frameworks/DiskArbitration.framework/Versions/A/DiskArbitration (compatibility version 1.0.0, current version 1.0.0)
/usr/lib/libc++.1.dylib (compatibility version 1.0.0, current version 1300.36.0)
/usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1319.0.0)
The logs of CMake loading the project contains this -- Conan: Adjusting default RPATHs Conan policies which makes me believe maybe Conan erases the default value that worked before ?
Here are some more details of my system :
MacOS 12.6.3 (but I'll be crosscompiling later to raspberry and I created a github action CI)
Qt6 v6.4.2 installed in home folder
Conan 1.58.0
Cmake 3.25.1
Thank you very much for your help.
Benjamin