CMake - linking to library downloaded from ExternalProject_add()

Question

I am trying to use ExternalProject_add() to download/install dependencies. It installs fine, but I can\'t figure out how to actually link the libraries after they are downlo

Answer 1

When you're using ExternalProject_Add, you can't use find_package, since there's nothing to find when CMake runs to configure the outer project.

So, if library locations vary by platform you will need conditional logic based on your platform. (I don't know protobuf's libraries or structure here, so this is just an example, but it should get you headed in the right direction...) Something like this:

if(WIN32)
  set(PROTOBUF_LIB_DIR "${MYPROJ_SOURCE_DIR}/dependencies/win"
  set(prefix "")
  set(suffix ".lib")
elseif(APPLE)
  set(PROTOBUF_LIB_DIR "${MYPROJ_SOURCE_DIR}/dependencies/mac"
  set(prefix "lib")
  set(suffix ".a")
else()
  set(PROTOBUF_LIB_DIR "${MYPROJ_SOURCE_DIR}/dependencies/linux"
  set(prefix "lib")
  set(suffix ".a")
endif()

set(PROTOBUF_LIBRARIES
  "${PROTOBUF_LIB_DIR}/${prefix}protobufLib1${suffix}"
  "${PROTOBUF_LIB_DIR}/${prefix}protobufLib2${suffix}")

Granted, this is less convenient than using find_package. If you can use a pre-built/pre-installed package, you should, so that you can use find_package. If you must build the other package from source code as part of your project, though, ExternalProject_Add is useful, even though it is unable to abstract away all the details for you.

Answer 2

You can use find_package with ExternalProject_add like the following code snippet:

# --------------------------------------------------

function (build_external_project target file_name)

    set(CMAKELIST_CONTENT "
        cmake_minimum_required(VERSION ${CMAKE_MINIMUM_REQUIRED_VERSION})

        project(build_external_project)

        file(MD5 \"${file_name}\" FILE_HASH)

        include(ExternalProject)
        ExternalProject_add(${target}
            URL \"${file_name}\"
            URL_MD5 \${FILE_HASH}
            CMAKE_GENERATOR \"${CMAKE_GENERATOR}\"
            CMAKE_GENERATOR_PLATFORM \"${CMAKE_GENERATOR_PLATFORM}\"
            CMAKE_GENERATOR_TOOLSET \"${CMAKE_GENERATOR_TOOLSET}\"
            CMAKE_GENERATOR_INSTANCE \"${CMAKE_GENERATOR_INSTANCE}\"
            CMAKE_ARGS ${ARGN})

        add_custom_target(build_external_project)
        add_dependencies(build_external_project ${target})
    ")

    set(TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/ExternalProjects/${target}")

    file(WRITE "${TARGET_DIR}/CMakeLists.txt" "${CMAKELIST_CONTENT}")

    file(MAKE_DIRECTORY "${TARGET_DIR}" "${TARGET_DIR}/build")

    execute_process(COMMAND ${CMAKE_COMMAND}
        -G "${CMAKE_GENERATOR}"
        -A "${CMAKE_GENERATOR_PLATFORM}"
        -T "${CMAKE_GENERATOR_TOOLSET}"
        ..
        WORKING_DIRECTORY "${TARGET_DIR}/build")

    execute_process(COMMAND ${CMAKE_COMMAND}
        --build .
        --config ${CMAKE_BUILD_TYPE}
        WORKING_DIRECTORY "${TARGET_DIR}/build")

endfunction()

#----------------------------------------------------------------------------------------------------

set(THIDR_PARTY_DIR "${CMAKE_CURRENT_LIST_DIR}")
set(THIDR_PARTY_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/ThirdParty")

#----------------------------------------------------------------------------------------------------

# OpenCV.

set(OPENCV_OPTIONS "")

list(APPEND OPENCV_OPTIONS -D WITH_1394=ON)
list(APPEND OPENCV_OPTIONS -D WITH_ADE=ON)
list(APPEND OPENCV_OPTIONS -D WITH_CUDA=OFF)
list(APPEND OPENCV_OPTIONS -D WITH_EIGEN=ON)
list(APPEND OPENCV_OPTIONS -D WITH_FFMPEG=ON)
list(APPEND OPENCV_OPTIONS -D WITH_IPP=ON)
list(APPEND OPENCV_OPTIONS -D WITH_ITT=ON)
list(APPEND OPENCV_OPTIONS -D WITH_JASPER=ON)
list(APPEND OPENCV_OPTIONS -D WITH_JPEG=ON)
list(APPEND OPENCV_OPTIONS -D WITH_OPENCL=ON)
list(APPEND OPENCV_OPTIONS -D WITH_OPENEXR=ON)
list(APPEND OPENCV_OPTIONS -D WITH_OPENJPEG=ON)
list(APPEND OPENCV_OPTIONS -D WITH_PNG=ON)
list(APPEND OPENCV_OPTIONS -D WITH_PROTOBUF=ON)
list(APPEND OPENCV_OPTIONS -D WITH_QT=OFF)
list(APPEND OPENCV_OPTIONS -D WITH_TBB=ON)
list(APPEND OPENCV_OPTIONS -D WITH_TIFF=ON)
list(APPEND OPENCV_OPTIONS -D WITH_WEBP=ON)

list(APPEND OPENCV_OPTIONS -D BUILD_CUDA_STUBS=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_DOCS=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_EXAMPLES=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_IPP_IW=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_ITT=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_JASPER=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_JAVA=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_JPEG=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_OPENEXR=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_PACKAGE=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_PERF_TESTS=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_PNG=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_PROTOBUF=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_TBB=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_TESTS=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_TIFF=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_WEBP=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_ZLIB=ON)

list(APPEND OPENCV_OPTIONS -D BUILD_opencv_apps=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_opencv_java_bindings_generator=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_opencv_js=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_opencv_python_bindings_generator=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_opencv_python_tests=OFF)
list(APPEND OPENCV_OPTIONS -D BUILD_opencv_world=OFF)

list(APPEND OPENCV_OPTIONS -D OPENCV_ENABLE_NONFREE=OFF)
list(APPEND OPENCV_OPTIONS -D OPENCV_FORCE_3RDPARTY_BUILD=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_WITH_STATIC_CRT=OFF)

list(APPEND OPENCV_OPTIONS -D CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
list(APPEND OPENCV_OPTIONS -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
list(APPEND OPENCV_OPTIONS -D CMAKE_POSITION_INDEPENDENT_CODE=ON)
list(APPEND OPENCV_OPTIONS -D BUILD_SHARED_LIBS=OFF)
list(APPEND OPENCV_OPTIONS -D CMAKE_INSTALL_PREFIX=${THIDR_PARTY_INSTALL_DIR}/OpenCV)

build_external_project(OpenCV "${THIDR_PARTY_DIR}/OpenCV/opencv-4.3.0+cache.zip" ${OPENCV_OPTIONS})

set(OpenCV_STATIC ON)

#----------------------------------------------------------------------------------------------------

# Dlib.

set(DLIB_OPTIONS "")

list(APPEND DLIB_OPTIONS -D USE_SSE2_INSTRUCTIONS=ON)
list(APPEND DLIB_OPTIONS -D USE_SSE4_INSTRUCTIONS=ON)
list(APPEND DLIB_OPTIONS -D USE_AVX_INSTRUCTIONS=ON)

list(APPEND DLIB_OPTIONS -D DLIB_GIF_SUPPORT=ON)
list(APPEND DLIB_OPTIONS -D DLIB_JPEG_SUPPORT=ON)
list(APPEND DLIB_OPTIONS -D DLIB_PNG_SUPPORT=ON)
list(APPEND DLIB_OPTIONS -D DLIB_USE_BLAS=ON)
list(APPEND DLIB_OPTIONS -D DLIB_USE_CUDA=OFF)
list(APPEND DLIB_OPTIONS -D DLIB_USE_LAPACK=ON)
list(APPEND DLIB_OPTIONS -D DLIB_USE_MKL_FFT=OFF)
list(APPEND DLIB_OPTIONS -D DLIB_LINK_WITH_SQLITE3=OFF)
list(APPEND DLIB_OPTIONS -D DLIB_NO_GUI_SUPPORT=OFF)
list(APPEND DLIB_OPTIONS -D DLIB_ENABLE_ASSERTS=OFF)
list(APPEND DLIB_OPTIONS -D DLIB_ENABLE_STACK_TRACE=OFF)

list(APPEND DLIB_OPTIONS -D CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
list(APPEND DLIB_OPTIONS -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
list(APPEND DLIB_OPTIONS -D CMAKE_POSITION_INDEPENDENT_CODE=ON)
list(APPEND DLIB_OPTIONS -D BUILD_SHARED_LIBS=OFF)
list(APPEND DLIB_OPTIONS -D CMAKE_INSTALL_PREFIX=${THIDR_PARTY_INSTALL_DIR}/dlib)

build_external_project(dlib ${THIDR_PARTY_DIR}/Dlib/dlib-19.20.tar.bz2 ${DLIB_OPTIONS})

#----------------------------------------------------------------------------------------------------

####################################################################################################
# Now, you can use find_package:
find_package(OpenCV REQUIRED PATHS "${THIDR_PARTY_INSTALL_DIR}/OpenCV" NO_DEFAULT_PATH)
find_package(dlib REQUIRED PATHS "${THIDR_PARTY_INSTALL_DIR}/dlib" NO_DEFAULT_PATH)
####################################################################################################

https://gist.github.com/amir-saniyan/4339e6f3ef109c75eda8018f7d5192a7

Answer 3

To expand on DLRdave answer above you don't need set manually prefixes and suffixes for static libraries because CMAKE provides variables with the right ones for each platform.

See CMake Useful Variables for more information.

For example:

• CMAKE_SHARED_LIBRARY_PREFIX
• CMAKE_SHARED_LIBRARY_SUFFIX
• CMAKE_STATIC_LIBRARY_PREFIX
• CMAKE_STATIC_LIBRARY_SUFFIX

Answer 4

Another idiom you can use to solve this problem:

1. Make your find_package commands optional (remove 'REQUIRED')
2. Add some condition code to only build your targets if find_package succeeds.
3. find_package will fail and your targets won't be built the first time around, but the External Projects will be built.
4. Run cmake/make again, this time find_package will succeed and your targets will be built.

You can see this idiom in action in https://github.com/biometrics/likely.

Answer 5

Because you're downloading the external project, you already know where everything is because you just downloaded it, so it doesn't need 'finding'.

I got it working with add_library. This is my actual code that works:

ExternalProject_Add(ForexConnectDownload
    PREFIX 3rd_party
    #--Download step--------------
    URL http://fxcodebase.com/bin/forexconnect/1.3.1/ForexConnectAPI-1.3.1-Linux-x86_64.tar.gz
    URL_HASH SHA1=7fdb90a2d45085feb8b76167cae419ad4c211d6b
    #--Configure step-------------
    CONFIGURE_COMMAND ""
    #--Build step-----------------
    BUILD_COMMAND ""
    #--Install step---------------
    UPDATE_COMMAND "" # Skip annoying updates for every build
    INSTALL_COMMAND ""
)

SET(FXCM_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/3rd_party/src/ForexConnectDownload/include)
SET(FXCM_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR}/3rd_party/src/ForexConnectDownload/lib)

add_library(ForexConnect SHARED IMPORTED)
set_target_properties(ForexConnect PROPERTIES IMPORTED_LOCATION ${FXCM_LIB_DIR}/libForexConnect.so)

From there, each program that depends on it needs a add_dependencies and of course target_link_libraries. Eg:

include_directories(${FXCM_INCLUDE_DIR})
add_executable(syncDatabase syncDatabase.cpp trader/database.cpp trader/fxcm.cpp)
target_link_libraries(syncDatabase ForexConnect)
add_dependencies(syncDatabase ForexConnectDownload)

• include_directories - tells it to search for directories there
• target_link_libraries - just add your library, as you named it (not a variable)

The add_dependencies makes it wait before trying to include the dirs needed.

That does the trick for me. Works with make -j4. Get's all the dependencies right.

Answer 6

You can use the link_directories command to link libraries within a specific directory. In your case the directory were your external project is build.

ExternalProject_Add(MyExternalLibrary ...)

Add the output directory to the search path:

link_directories(${CMAKE_BINARY_DIR}/lib/MyExternalLibrary-prefix/lib)

Make sure to add the executable after specifying the link directory:

add_executable(MyProgram main.c)

Specify the libraries your project should be linked to:

target_link_libraries(MyProgram ExternalLibraryName)

Don't forget to depend on the external project:

add_dependencies(MyProgram MyExternalLibrary)