How can I statically link Arrow when building parquet-cpp? - c++

Question

From the parquet-cpp home page:
By default, Parquet links to Arrow's shared libraries. If you wish to statically-link the Arrow symbols instead, pass -DPARQUET_ARROW_LINKAGE=static.
I do want to statically link Arrow, because I want to use my program on other servers that won't have Arrow installed. I tried -DPARQUET_ARROW_LINKAGE=static, but I get an error about "missing transitive dependencies":
# cmake -DPARQUET_BUILD_TESTS=Off -DCMAKE_BUILD_TYPE=Release -DPARQUET_MINIMAL_DEPENDENCY=ON -DPARQUET_ARROW_LINKAGE=static .
-- The C compiler identification is GNU 4.8.5
...
-- [ /usr/local/share/cmake-3.9/Modules/FindBoost.cmake:1717 ] Boost_FOUND = 1
-- Boost version: 1.55.0
...
-- THRIFT_HOME:
-- Thrift compiler/libraries NOT found: (THRIFT_INCLUDE_DIR-NOTFOUND, THRIFT_STATIC_LIB-NOTFOUND). Looked in system search paths.
-- Thrift include dir: /root/tmp/parquet-cpp-master/thrift_ep/src/thrift_ep-install/include
-- Thrift static library: /root/tmp/parquet-cpp-master/thrift_ep/src/thrift_ep-install/lib/libthrift.a
-- Thrift compiler: /root/tmp/parquet-cpp-master/thrift_ep/src/thrift_ep-install/bin/thrift
-- Checking for module 'arrow'
-- No package 'arrow' found
-- Could not find the Arrow library. Looked for headers in , and for libs in
-- Building Apache Arrow from commit: 501d60e918bd4d10c429ab34e0b8e8a87dffb732
-- CMAKE_CXX_FLAGS: -O3 -DNDEBUG -Wall -std=c++11
-- Found cpplint executable at /root/tmp/parquet-cpp-master/build-support/cpplint.py
CMake Error at CMakeLists.txt:515 (message):
Missing transitive dependencies for Arrow static linking
So I found the code that generates the error:
if (NOT DEFINED ENV{BROTLI_STATIC_LIB_ENC} OR
NOT DEFINED ENV{BROTLI_STATIC_LIB_DEC} OR
NOT DEFINED ENV{BROTLI_STATIC_LIB_COMMON} OR
NOT DEFINED ENV{SNAPPY_STATIC_LIB} OR
NOT DEFINED ENV{ZLIB_STATIC_LIB} OR
NOT DEFINED ENV{LZ4_STATIC_LIB} OR
NOT DEFINED ENV{ZSTD_STATIC_LIB})
message(FATAL_ERROR "Missing transitive dependencies for Arrow static linking")
But that doesn't really help me, since I don't know what to do to get those environment variable defined.
Do I need to compile Arrow and install myself first? (I would rather have parquet-cpp do it for me.)

Answer 1

I arranged a script to download dependencies sources, set the environment variables and run your cmake line at the end. Just change the DEPDIR variable value, setting it to a directory of choice.
#!/bin/bash
CMKDIR=$PWD
DEPDIR=/tmp
cd $DEPDIR
#snappy
git clone https://github.com/google/snappy.git
cd snappy
mkdir build
cd build
cmake ..
make
export SNAPPY_STATIC_LIB=$DEPDIR/snappy/build/libsnappy.a
cd $DEPDIR
#brotli
git clone https://github.com/google/brotli.git
cd brotli
mkdir out
cd out
../configure-cmake
make
export BROTLI_STATIC_LIB_ENC=$DEPDIR/brotli/out/libbrotlienc-static.a
export BROTLI_STATIC_LIB_DEC=$DEPDIR/brotli/out/libbrotlidec-static.a
export BROTLI_STATIC_LIB_COMMON=$DEPDIR/brotli/out/libbrotlicommon-static.a
cd $DEPDIR
#zlib
git clone https://github.com/madler/zlib.git
cd zlib
./configure
make
export ZLIB_STATIC_LIB=$DEPDIR/zlib/libz.a
cd $DEPDIR
#lz4
git clone https://github.com/lz4/lz4.git
cd lz4
make
export LZ4_STATIC_LIB=$DEPDIR/lz4/lib/liblz4.a
cd $DEPDIR
#zstd
git clone https://github.com/facebook/zstd.git
cd zstd
make
export ZSTD_STATIC_LIB=$DEPDIR/zstd/lib/libzstd.a
cd $CMKDIR
cmake -DPARQUET_BUILD_TESTS=Off -DCMAKE_BUILD_TYPE=Release -DPARQUET_MINIMAL_DEPENDENCY=ON -DPARQUET_ARROW_LINKAGE=static
This script is very simple but should be effective. Just copy it in a new file (in the same CMakeLists.txt directory), give the file the execute permissions (i.e. sudo chmod +x filename) and execute it like this:
./filename.sh
About the fPIC option issue, you have to edit some files:
snappy: add this line in CMakeLists.txt, at the beginning, after the first two lines:
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
lz4 and zstd: edit the Makefile in lib sub-directory, after this line
CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS)
add this line:
CFLAGS += -fPIC
zlib: edit the Makefile, after this line
CFLAGS=-O3 -D_LARGEFILE64_SOURCE=1 -DHAVE_HIDDEN
add this line:
CFLAGS += -fPIC
brotli: as far as I can see from make output, the option is already set.
Before running make again, execute this script:
#!/bin/bash
DEPDIR=/tmp
cd $DEPDIR/snappy/build
cmake ..
make clean
make
cd $DEPDIR/lz4
make clean
make
cd $DEPDIR/zstd
make clean
make