No symbols in google breakpad stack trace when applying compiler optimizations - c++

Question

The main point of a crash reporter tool like Google breakpad is to generate core dump or minidump files from stripped binaries to process later with debugging symbols. normally these binaries are release builds with compiler optimizations applied and also stripped.
To reproduce the problem on Linux:
1.Build + install google breakpad:
git clone https://chromium.googlesource.com/breakpad/breakpad && cd breakpad
git clone https://chromium.googlesource.com/linux-syscall-support src/third_party/lss
./configure --prefix=/usr/local
make -j$(nproc) && sudo make install
2.The Code:
CMakeLists.txt:
cmake_minimum_required(VERSION 3.10)
project(BreakPadTest)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_VERBOSE_MAKEFILE TRUE)
set(BREAKPAD_DIR "/usr/local/include/breakpad")
option(OPTION_WITH_O1 "With -O1" OFF)
if(OPTION_WITH_O1)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O1 -fno-omit-frame-pointer")
endif()
include_directories(
${BREAKPAD_DIR}
)
add_executable(${PROJECT_NAME} "main.cc")
target_link_libraries(${PROJECT_NAME}
-lstdc++fs
-pthread
libbreakpad_client.a
)
main.cc:
#include <thread>
#include <experimental/filesystem>
#include <client/linux/handler/exception_handler.h>
namespace breakpad = google_breakpad;
namespace filesystem = std::experimental::filesystem;
static bool DumpCallBack(const breakpad::MinidumpDescriptor& md,
void* context,
bool success) {
(void)md;
(void)context;
return success;
}
static void fault(unsigned after) {
std::this_thread::sleep_for(std::chrono::seconds{after});
delete reinterpret_cast<std::string*>(0xFEE1DEAD);
}
int32_t main(int argc, char** argv) {
(void)argc;
(void)argv;
auto pwd = filesystem::current_path();
const auto dumpDir = pwd.string() + "/dumps";
filesystem::create_directory(dumpDir);
breakpad::MinidumpDescriptor md(dumpDir);
new google_breakpad::ExceptionHandler(
md,
/* FilterCallback */ nullptr,
DumpCallBack,
/* callback_context */ nullptr,
true,
-1
);
fault(1U);
return EXIT_SUCCESS;
}
dump.sh:
#!/bin/bash
#
# e.g ./dump.sh ./exec $PWD/dumps
#
set -e
set -u
DBG_INFO=$(realpath ${1})
DUMPS_DIR=$(realpath ${2:-$PWD/dumps})
DUMP_SYMS=${3:-~/WorkSpace/libraries/breakpad/src/tools/linux/dump_syms/dump_syms}
STAK_WALK=${4:-~/WorkSpace/libraries/breakpad/src/processor/minidump_stackwalk}
#
# Generate debug symbols
#
base=$(basename $DBG_INFO)
$DUMP_SYMS $DBG_INFO > $DUMPS_DIR/$base.sym
#
# Create dump dir structure
#
list=($(head -n1 $DUMPS_DIR/$base.sym))
hash=${list[3]}
mkdir -p $DUMPS_DIR/symbols/$base/$hash
mv $DUMPS_DIR/$base.sym $DUMPS_DIR/symbols/$base/$hash
#
# Produce stack trace
#
RED='\033[0;36m'
NC='\033[0m' # No Color
tree $DUMPS_DIR
for dmp in $DUMPS_DIR/*.dmp ; do
filename=$(basename -- "${dmp}")
filename="${filename%.*}"
echo -e "generating stack trace for -> ${RED}${dmp}${NC}"
$STAK_WALK ${dmp} $DUMPS_DIR/symbols > $DUMPS_DIR/${filename}.txt 2>/dev/null
done
3.Run normal Debug version:
cmake -DCMAKE_BUILD_TYPE=Debug . && make
./BreakPadTest
4.Process minidump generated from stage 3:
./dump.sh ./BreakPadTest ./dumps
stackwalk:
Operating system: Linux
0.0.0 Linux 4.19.0-16-amd64 #1 SMP Debian 4.19.181-1 (2021-03-19) x86_64
CPU: amd64
family 6 model 58 stepping 9
1 CPU
GPU: UNKNOWN
Crash reason: SIGSEGV /SEGV_MAPERR
Crash address: 0xfee1dead
Process uptime: not available
Thread 0 (crashed)
0 BreakPadTest!std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_data() const [basic_string.h : 176 + 0x4]
rax = 0x00000000fee1dead rdx = 0x00007ffcfccfcb90
rcx = 0x00007f5b0cbb4bc1 rbx = 0x0000557d577eb8e0
rsi = 0x00007ffcfccfcb90 rdi = 0x00000000fee1dead
rbp = 0x00007ffcfccfcb50 rsp = 0x00007ffcfccfcb50
r8 = 0x0000000000000000 r9 = 0x0000557d577efaf8
r10 = 0xfffffffffffff60b r11 = 0x0000000000000246
r12 = 0x0000557d56d3d2c0 r13 = 0x00007ffcfccfce50
r14 = 0x0000000000000000 r15 = 0x0000000000000000
rip = 0x0000557d56d3dfda
Found by: given as instruction pointer in context
1 BreakPadTest!std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_is_local() const [basic_string.h : 211 + 0xc]
rbx = 0x0000557d577eb8e0 rbp = 0x00007ffcfccfcb80
rsp = 0x00007ffcfccfcb60 r12 = 0x0000557d56d3d2c0
r13 = 0x00007ffcfccfce50 r14 = 0x0000000000000000
r15 = 0x0000000000000000 rip = 0x0000557d56d3e2c5
Found by: call frame info
2 BreakPadTest!std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_dispose() [basic_string.h : 220 + 0xc]
rbx = 0x0000557d577eb8e0 rbp = 0x00007ffcfccfcba0
rsp = 0x00007ffcfccfcb90 r12 = 0x0000557d56d3d2c0
r13 = 0x00007ffcfccfce50 r14 = 0x0000000000000000
r15 = 0x0000000000000000 rip = 0x0000557d56d3dff8
Found by: call frame info
3 BreakPadTest!std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string() [basic_string.h : 657 + 0xc]
rbx = 0x0000557d577eb8e0 rbp = 0x00007ffcfccfcbc0
rsp = 0x00007ffcfccfcbb0 r12 = 0x0000557d56d3d2c0
r13 = 0x00007ffcfccfce50 r14 = 0x0000000000000000
r15 = 0x0000000000000000 rip = 0x0000557d56d3d930
Found by: call frame info
4 BreakPadTest!fault [main.cc : 19 + 0xa]
rbx = 0x0000557d577eb8e0 rbp = 0x00007ffcfccfcbf0
rsp = 0x00007ffcfccfcbd0 r12 = 0x0000557d56d3d2c0
r13 = 0x00007ffcfccfce50 r14 = 0x0000000000000000
r15 = 0x0000000000000000 rip = 0x0000557d56d3d3f0
Found by: call frame info
5 BreakPadTest!main [main.cc : 39 + 0xa]
rbx = 0x0000557d577eb8e0 rbp = 0x00007ffcfccfcd70
rsp = 0x00007ffcfccfcc00 r12 = 0x0000557d56d3d2c0
r13 = 0x00007ffcfccfce50 r14 = 0x0000000000000000
r15 = 0x0000000000000000 rip = 0x0000557d56d3d4fd
Found by: call frame info
6 libc.so.6 + 0x2409b
rbx = 0x0000000000000000 rbp = 0x0000557d56d78b80
rsp = 0x00007ffcfccfcd80 r12 = 0x0000557d56d3d2c0
r13 = 0x00007ffcfccfce50 r14 = 0x0000000000000000
r15 = 0x0000000000000000 rip = 0x00007f5b0ca0609b
Found by: call frame info
7 BreakPadTest!fault [main.cc : 20 + 0x3]
rsp = 0x00007ffcfccfcda0 rip = 0x0000557d56d3d402
Found by: stack scanning
8 ld-linux-x86-64.so.2 + 0xf476
rsp = 0x00007ffcfccfce10 rip = 0x00007f5b0cf1c476
Found by: stack scanning
9 BreakPadTest!_start + 0x2a
rsp = 0x00007ffcfccfce40 rip = 0x0000557d56d3d2ea
Found by: stack scanning
10 0x7ffcfccfce48
rsp = 0x00007ffcfccfce48 rip = 0x00007ffcfccfce48
Found by: call frame info
Loaded modules:
0x557d56d34000 - 0x557d56d78fff BreakPadTest ??? (main)
0x7f5b0c9e2000 - 0x7f5b0cb4bfff libc.so.6 ??? (WARNING: No symbols, libc.so.6, A8A9B91823C5CFE5E5B5D946D605D0920)
0x7f5b0cba3000 - 0x7f5b0cbb7fff libpthread.so.0 ???
0x7f5b0cbc4000 - 0x7f5b0cbd7fff libgcc_s.so.1 ???
0x7f5b0cbde000 - 0x7f5b0cc89fff libm.so.6 ???
0x7f5b0cd61000 - 0x7f5b0ce95fff libstdc++.so.6 ???
0x7f5b0cf0d000 - 0x7f5b0cf2bfff ld-linux-x86-64.so.2 ??? (WARNING: No symbols, ld-linux-x86-64.so.2, 7BFD5DF2BE95A34B86FD71080ACCAE8C0)
0x7ffcfcdc5000 - 0x7ffcfcdc6fff linux-gate.so ???
5.Run stage 3 with -O1:
cmake -DCMAKE_BUILD_TYPE=Debug -DOPTION_WITH_O1=ON . && make
./BreakPadTest
6.Process minidump like stage 4:
stackwalk:
Operating system: Linux
0.0.0 Linux 4.19.0-16-amd64 #1 SMP Debian 4.19.181-1 (2021-03-19) x86_64
CPU: amd64
family 6 model 58 stepping 9
1 CPU
GPU: UNKNOWN
Crash reason: SIGSEGV /SEGV_MAPERR
Crash address: 0xfee1dead
Process uptime: not available
Thread 0 (crashed)
0 BreakPadTest!main [basic_string.h : 176 + 0x0]
rax = 0x0000000000000000 rdx = 0x000055bd46f66a40
rcx = 0x00007f7633ea8bc1 rbx = 0x00007ffde4cc7c40
rsi = 0x00007ffde4cc7c40 rdi = 0x00007ffde4cc7c40
rbp = 0x00007ffde4cc7d90 rsp = 0x00007ffde4cc7c20
r8 = 0x0000000000000000 r9 = 0x000055bd474caaf8
r10 = 0x0000000000000000 r11 = 0x0000000000000246
r12 = 0x000055bd474c64f0 r13 = 0x000055bd474c64f0
r14 = 0x0000000000000000 r15 = 0x0000000000000000
rip = 0x000055bd46f1b8dd
Found by: given as instruction pointer in context
1 libc.so.6 + 0x2409b
rbx = 0x0000000000000000 rbp = 0x000055bd46f555e0
rsp = 0x00007ffde4cc7da0 r12 = 0x000055bd46f1b270
r13 = 0x00007ffde4cc7e70 r14 = 0x0000000000000000
r15 = 0x0000000000000000 rip = 0x00007f7633cfa09b
Found by: call frame info
2 BreakPadTest!DumpCallBack [main.cc : 15 + 0x3]
rsp = 0x00007ffde4cc7dc0 rip = 0x000055bd46f1b358
Found by: stack scanning
3 ld-linux-x86-64.so.2 + 0xf476
rsp = 0x00007ffde4cc7e30 rip = 0x00007f7634210476
Found by: stack scanning
4 BreakPadTest!_start + 0x2a
rsp = 0x00007ffde4cc7e60 rip = 0x000055bd46f1b29a
Found by: stack scanning
5 0x7ffde4cc7e68
rsp = 0x00007ffde4cc7e68 rip = 0x00007ffde4cc7e68
Found by: call frame info
Loaded modules:
0x55bd46f14000 - 0x55bd46f55fff BreakPadTest ??? (main)
0x7f7633cd6000 - 0x7f7633e3ffff libc.so.6 ??? (WARNING: No symbols, libc.so.6, A8A9B91823C5CFE5E5B5D946D605D0920)
0x7f7633e97000 - 0x7f7633eabfff libpthread.so.0 ???
0x7f7633eb8000 - 0x7f7633ecbfff libgcc_s.so.1 ???
0x7f7633ed2000 - 0x7f7633f7dfff libm.so.6 ???
0x7f7634055000 - 0x7f7634189fff libstdc++.so.6 ???
0x7f7634201000 - 0x7f763421ffff ld-linux-x86-64.so.2 ??? (WARNING: No symbols, ld-linux-x86-64.so.2, 7BFD5DF2BE95A34B86FD71080ACCAE8C0)
0x7ffde4d9a000 - 0x7ffde4d9bfff linux-gate.so ???
As we can see correct symbols got disappeared from the stack walk of stage 6.
While in other tools like GDB we have correct trace pointing to right location even with -O1 as in stage 5:
Program received signal SIGSEGV, Segmentation fault.
fault (after=1) at /home/iman/WorkSpace/projects/BreakPadTest/src/main.cc:26
26 delete reinterpret_cast<std::string*>(0xFEE1DEAD);
Or in other tools like backward-cpp :
Stack trace (most recent call last):
#3 Object "", at 0xffffffffffffffff, in
#2 Object "/home/iman/WorkSpace/projects/build-CrashReporter-Desktop_Qt_5_11_3_GCC-Debug/CrashReporter", at 0x55f32a66b579, in _start
#1 Source "/build/glibc-vjB4T1/glibc-2.28/csu/../csu/libc-start.c", line 308, in __libc_start_main [0x7f56288be09a]
#0 | Source "/home/iman/WorkSpace/projects/BreakPadTest/src/main.cc", line 48, in main
| 46: #endif // WITH_BREAKPAD
| 47:
| > 48: fault(1U);
| 49:
| 50: return EXIT_SUCCESS;
| Source "/home/iman/WorkSpace/projects/BreakPadTest/src/main.cc", line 26, in fault
| 24: static void fault(unsigned after) {
| 25: std::this_thread::sleep_for(std::chrono::seconds{after});
| > 26: delete reinterpret_cast<std::string*>(0xFEE1DEAD);
| 27: }
| Source "/usr/include/c++/8/bits/basic_string.h", line 657, in
| 655: */
| 656: ~basic_string()
| > 657: { _M_dispose(); }
| 658:
| 659: /**
| Source "/usr/include/c++/8/bits/basic_string.h", line 220, in
| 218: _M_dispose()
| 219: {
| > 220: if (!_M_is_local())
| 221: _M_destroy(_M_allocated_capacity);
| 222: }
| Source "/usr/include/c++/8/bits/basic_string.h", line 211, in
| 209: bool
| 210: _M_is_local() const
| > 211: { return _M_data() == _M_local_data(); }
| 212:
| 213: // Create & Destroy
Source "/usr/include/c++/8/bits/basic_string.h", line 176, in main [0x55f32a66b66c]
174: pointer
175: _M_data() const
> 176: { return _M_dataplus._M_p; }
177:
178: pointer
179: _M_local_data()
Segmentation fault (Address not mapped to object [0xfee1dead])
Segmentation fault
Any idea or hint?

Answer 1

With the optimisation -O1 almost all std::basic_string functions are inline. These functions are inline because std::basic_string is a template and defined in heder files.

Answer 2

There's an old and active issue with google breakpad processor sub-system to handle inline functions or extract inline functions metadata from DWARF debugging information on linux as discussed in these topics:
https://bugzilla.mozilla.org/show_bug.cgi?id=524410
https://bugzilla.mozilla.org/show_bug.cgi?id=563776
https://bugzilla.mozilla.org/show_bug.cgi?id=1665367
https://bugzilla.mozilla.org/show_bug.cgi?id=1398533
https://bugzilla.mozilla.org/show_bug.cgi?id=1636194
https://groups.google.com/g/google-breakpad-discuss/c/ZQOSZBbdF7U/m/58tpSnM1EBsJ
Which requires modification on breakpad internal symbols representation and stack walker but as a workaround you could generate core dump for GDB and get a backtrace with it, for that to happen there's utility inside breakpad code base(src/tools/linux/md2core) called minidump-2-core, so after building your release binary with debugging information:
$ objcopy --only-keep-debug BreakPadTest BreakPadTest.debug
$ strip BreakPadTest
$ ./BreakPadTest
Segmentation fault
$ minidump-2-core -o dumps/45a855b5-8931-4e7e-5f508496-3fe1cacc.core dumps/45a855b5-8931-4e7e-5f508496-3fe1cacc.dmp
$ gdb -c dumps/45a855b5-8931-4e7e-5f508496-3fe1cacc.core BreakPadTest.debug
Program terminated with signal SIGSEGV, Segmentation fault.
#0 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_is_local (this=0xfee1dead) at /usr/include/c++/8/bits/basic_string.h:656
656 ~basic_string()
(gdb) backtrace
#0 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_is_local (this=0xfee1dead) at /usr/include/c++/8/bits/basic_string.h:656
#1 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_dispose (this=0xfee1dead) at /usr/include/c++/8/bits/basic_string.h:220
#2 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string (this=0xfee1dead, __in_chrg=<optimized out>)
at /usr/include/c++/8/bits/basic_string.h:657
#3 fault (after=1) at /home/iman/WorkSpace/projects/BreakPadTest/src/main.cc:19
#4 main (argc=<optimized out>, argv=<optimized out>) at /home/iman/WorkSpace/projects/BreakPadTest/src/main.cc:39
(gdb)