Segmentation fault on libc

[Mohammad Ewais]

Hello,

I had a complex client running with DR 8.0.0-1 for a while now with no issues at all. Recently though, I have had to move my work to another server, running the following:

- Ubuntu 22.04
- kernel 5.15.0-79-generic

- gcc/g++ 11.4.0

- libc 2.35

- DR 8.0.0-1 (though similar issues arise on DR 10.0.0)

My older system was:
- Ubuntu 21.04

- kernel 5.11.0-49-generic

- gcc/g++ 10.3.0

- libc 2.33

- DR 8.0.0-1

For some reason, the client which was working fine on an Ubuntu 21.04 system (Unfortunately, I no longer have access to the other system info) now suffers from segmentation faults in libc calls!

I was able to generate a minimal example, which works fine on the old system, but fails in the new. This failure only happens when I run DR with the -debug flag. 

```cpp
#include <iostream>
#include <dr_api.h>

DR_EXPORT void dr_client_main(client_id_t id, int argc, const char** argv)
{
    std::cout << "Test1\n";
    std::cerr << "Test1\n";
    std::cout << "Test2\n";
}

```

```cmake

cmake_minimum_required(VERSION 3.22)
project(Test)
set(CMAKE_CXX_STANDARD 20)

if (NOT DEFINED ENV{DYNAMORIO_PATH})
    message( SEND_ERROR "DYNAMORIO_PATH must be defined and point to DynamoRIO" )
endif()
find_package(DynamoRIO REQUIRED PATHS $ENV{DYNAMORIO_PATH}/)

add_library(Test SHARED Test.cpp)
configure_DynamoRIO_client(Test)

```

```running

/path/to/DR8.0.0-1/bin64/drrun -debug -c libTest.so -- /path/to/apsp 1 256 16

```

```output

<log dir=/tools/DynamoRIO/DynamoRIO-Linux-8.0.0-1/bin64/../logs/apsp.149430.00000000>
<Starting application /work/mewais/DCArch/DCSim/Bench/CRONO/apps/apsp/apsp (149430)>
<Initial options = -no_dynamic_options -loglevel 4 -client_lib '/home/mewais/Test/Build/libTest.so;0;' -code_api -stack_size 56K -signal_stack_size 32K -max_elide_jmp 0 -max_elide_call 0 -early_inject -emulate_brk -no_inline_ignored_syscalls -native_exec_default_list '' -no_native_exec_managed_code -no_indcall2direct >
<Paste into GDB to debug DynamoRIO clients:
set confirm off
add-symbol-file '/home/mewais/Test/Build/libTest.so' 0x00007f107b97c0a0
add-symbol-file '/tools/DynamoRIO/DynamoRIO-Linux-8.0.0-1/lib64/debug/libdynamorio.so' 0x0000000071040fe0
add-symbol-file '/lib/x86_64-linux-gnu/libstdc++.so.6' 0x00007f10bf5aa420
add-symbol-file '/lib/x86_64-linux-gnu/libm.so.6' 0x00007f10bf8833a0
add-symbol-file '/lib/x86_64-linux-gnu/libc.so.6' 0x00007f10bf0e9700
add-symbol-file '/usr/lib64/ld-linux-x86-64.so.2' 0x00007f10bf83a090
add-symbol-file '/lib/x86_64-linux-gnu/libgcc_s.so.1' 0x00007f10bf81a660
>
<(1+x) Handling our fault in a TRY at 0x000000007129f6b2>
<Application /work/mewais/DCArch/DCSim/Bench/CRONO/apps/apsp/apsp (149430).  Tool internal crash at PC 0x00007f10bf151a7f.  Please report this at your tool's issue tracker.  Program aborted.
Received SIGSEGV at pc 0x00007f10bf151a7f in thread 149430
Base: 0x0000000071000000
Registers:eax=0x0000000000000000 ebx=0x0000000000000006 ecx=0x00007f0ffb9f7700 edx=0x0000000000000006
esi=0x00007f0ffba3bc80 edi=0x0000000000000001 esp=0x00007ffc83b13078 ebp=0x00007f0ffba3bc80
r8 =0x0000000000000000 r9 =0x00007f0ffba3c000 r10=0x00007ffc83b12240 r11=0x0000000000000246
r12=0x0000000000000006 r13=0x00007f10bf2db780 r14=0x00007f10bf2d7600 r15=0x00007f10bf2d6a00
eflags=0x0000000000010246
version 8.0.0, build 1
-no_dynamic_options -loglevel 4 -client_lib '/home/mewais/Test/Build/libTest.so;0;' -code_api -stack_size 56K -signal_stack_size 32K -max_elide_jmp 0 -max_elide_call 0 -early_inject -emulate_brk -no_inline_ignored_syscalls -native_exec_default_list '' -no_native_exec_managed_code -no_indcall2direct
0x00007f0ffba3bc80 0xcdcdcdcdcdcdcdcd>
```

```gdb

(gdb) c
Continuing.
<(1+x) Handling our fault in a TRY at 0x000000007129f6b2>

Program received signal SIGSEGV, Segmentation fault.
0x00007ffff7732a7f in __GI___pthread_enable_asynccancel () at ./nptl/cancellation.c:34
34 ./nptl/cancellation.c: No such file or directory.
(gdb) where
#0  0x00007ffff7732a7f in __GI___pthread_enable_asynccancel () at ./nptl/cancellation.c:34
#1  0x00007ffff77b6a57 in __GI___libc_write (nbytes=6, buf=0x7fff33fdfc80, fd=1) at ../sysdeps/unix/sysv/linux/write.c:26
#2  __GI___libc_write (fd=1, buf=0x7fff33fdfc80, nbytes=6) at ../sysdeps/unix/sysv/linux/write.c:24
#3  0x00007ffff772cf6d in _IO_new_file_write (f=0x7ffff78bc780, data=0x7fff33fdfc80, n=6) at ./libio/fileops.c:1180
#4  0x00007ffff772ea61 in new_do_write (to_do=6, data=0x7fff33fdfc80 "Test1\n", '\315' <repeats 193 times>, <incomplete sequence \315>..., fp=0x7ffff78bc780) at ./libio/libioP.h:947
#5  _IO_new_do_write (to_do=6, data=0x7fff33fdfc80 "Test1\n", '\315' <repeats 193 times>, <incomplete sequence \315>..., fp=0x7ffff78bc780) at ./libio/fileops.c:425
#6  _IO_new_do_write (fp=fp@entry=0x7ffff78bc780, data=0x7fff33fdfc80 "Test1\n", '\315' <repeats 193 times>, <incomplete sequence \315>..., to_do=6) at ./libio/fileops.c:422
#7  0x00007ffff772ef43 in _IO_new_file_overflow (f=0x7ffff78bc780, ch=10) at ./libio/fileops.c:783
#8  0x00007ffff772feb4 in __GI__IO_default_xsputn (n=<optimized out>, data=<optimized out>, f=<optimized out>) at ./libio/genops.c:399
#9  __GI__IO_default_xsputn (f=f@entry=0x7ffff78bc780, data=<optimized out>, n=n@entry=6) at ./libio/genops.c:370
#10 0x00007ffff772d79b in _IO_new_file_xsputn (n=6, data=<optimized out>, f=<optimized out>) at ./libio/fileops.c:1264
#11 _IO_new_file_xsputn (f=0x7ffff78bc780, data=<optimized out>, n=6) at ./libio/fileops.c:1196
#12 0x00007ffff7722057 in __GI__IO_fwrite (buf=0x7fffb3f21000, size=1, count=6, fp=0x7ffff78bc780) at ./libio/libioP.h:947
#13 0x00007ffff7c25b65 in do_forward_check_eflags (inst=0x7ffff7d13da0, eflags=32767, eflags_valid=4157678696, eflags_invalid=0, state=0x0) at /home/travis/build/DynamoRIO/dynamorio/core/arch/x86/optimize.c:1862
Backtrace stopped: previous frame inner to this frame (corrupt stack?)
(gdb) c
Continuing.
<Application /work/mewais/DCArch/DCSim/Bench/CRONO/apps/apsp/apsp (149399).  Tool internal crash at PC 0x00007ffff7732a7f.  Please report this at your tool's issue tracker.  Program aborted.
Received SIGSEGV at pc 0x00007ffff7732a7f in thread 149399
Base: 0x0000000071000000
Registers:eax=0x0000000000000000 ebx=0x0000000000000006 ecx=0x00007fff33f9b700 edx=0x0000000000000006
esi=0x00007fff33fdfc80 edi=0x0000000000000001 esp=0x00007fffffffcd08 ebp=0x00007fff33fdfc80
r8 =0x0000000000000000 r9 =0x00007fff33fe0000 r10=0x00007fffffffbed0 r11=0x0000000000000246
r12=0x0000000000000006 r13=0x00007ffff78bc780 r14=0x00007ffff78b8600 r15=0x00007ffff78b7a00
eflags=0x0000000000010246
version 8.0.0, build 1
-no_dynamic_options -loglevel 4 -client_lib '/home/mewais/Test/Build/libTest.so;0;' -code_api -stack_size 56K -signal_stack_size 32K -max_elide_jmp 0 -max_elide_call 0 -early_inject -emulate_brk -no_inline_ignored_syscalls -native_exec_default_list '' -no_native_exec_managed_code -no_indcall2direct
0x00007fff33fdfc80 0xcdcdcdcdcdcdcdcd>
[Inferior 1 (process 149399) exited with code 0377]

```

In my real client, the issues arise even without use of the -debug flag. I assume this is because my client is much more complex and more heavily dependent on the standard C++ library either directly or through third party libs.

[Derek Bruening]

I assume this is due to glibc 2.34+ changes where the glibc devs made a significant change to add inter-library dependencies between ld.so, libc. and libpthread with no public interface (hidden dependencies) which made it very difficult to use private libraries as DR does to isolate tool and application resources.  Please see https://github.com/DynamoRIO/dynamorio/issues/5437#issuecomment-1287425471 and the workaround in that issue; further workarounds may be needed if there are other dependencies being hit but given that long-term these hidden dependencies are going to cause a maintenance nightmare it is not clear what the future solution should be, as discussed in that issue.

--
You received this message because you are subscribed to the Google Groups "DynamoRIO Users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to dynamorio-use...@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/dynamorio-users/9b1b73f7-5d28-4aa2-8fbd-3a2391df6796n%40googlegroups.com.

[Mohammad Ewais]

I am yet to try the hacky workaround of the link you posted.

However, I have made some attempts to link against fPIC versions of standard libraries, here's what I learned:
- libgcc and libstd-c++ already exist as static libraries (at least on my ubuntu system) that are already built with -fPIC. You can directly use these by simply adding the compiler flags: `-static-libgcc -static-libstdc++`. So, probably no issue here, the real problem is libc.
- I tried compiling libc from scratch with additional -fPIC flag. Once it is actually used, it complains with ` can not be used when making a shared object; recompile with -fPIC` errors, obviously indicating that my solution was a failure.

- I noticed there is a libc_pic.a within the libc build folder, all attempts to use it were a failure as well. Apparently, this is an intermediate output used to build libc.so and is not supposed to be used as a direct static library.

- If the internet is to be believed, every answer I have found on stackoverflow and various forums/mailing lists as to building libc with fPIC says it is impossible, and even if it worked, it would potentially break at run time.

- I thought I could give a try to MUSL libc instead of glibc. MUSL apparently is friendly to static linking and can be easily built with fPIC. That said, when I tried it with the simple client above, it failed with the following warnings:
```
<WARNING! symbol lookup error: libTest.so undefined symbol __libc_single_threaded>
<WARNING! symbol lookup error: libTest.so undefined symbol fopen64>
<WARNING! symbol lookup error: libTest.so undefined symbol lseek64>
<WARNING! symbol lookup error: libTest.so undefined symbol ftello64>
<WARNING! symbol lookup error: libTest.so undefined symbol __mbsrtowcs_chk>
<WARNING! symbol lookup error: libTest.so undefined symbol __wmemmove_chk>
<WARNING! symbol lookup error: libTest.so undefined symbol __sprintf_chk>
<WARNING! symbol lookup error: libTest.so undefined symbol __mbsnrtowcs_chk>
<WARNING! symbol lookup error: libTest.so undefined symbol fstat64>
<WARNING! symbol lookup error: libTest.so undefined symbol __wmemset_chk>
<WARNING! symbol lookup error: libTest.so undefined symbol fseeko64>
<WARNING! symbol lookup error: libTest.so undefined symbol __wmemcpy_chk>
```
and errors:
```
<Application /work/mewais/DCArch/DCSim/Bench/CRONO/apps/apsp/apsp (5159).  Client internal crash at PC 0x00007f44aa5d2150.  Please report this at http://dynamorio.org/issues/.  Program aborted.
Received SIGSEGV at client library pc 0x00007f44aa5d2150 in thread 5159
Base: 0x0000000071000000
Registers:eax=0x0000000000000000 ebx=0x0000000071668890 ecx=0x00007ffd55f06ba0 edx=0x00007ffd55f087b0
esi=0x00007ffd55f06ba0 edi=0x00007f44aa6e8fe2 esp=0x00007ffd55f06b30 ebp=0x00007f44aa6ebb14
r8 =0x000000000000004b r9 =0x00007f44aa5b38ff r10=0x00007ffd55f06240 r11=0x0000000000000246
r12=0x0000000000000000 r13=0x0000000000000000 r14=0x0000000000000000 r15=0x0000000000000000
eflags=0x0000000000010206
version 8.0.0, build 1
-no_dynamic_options -loglevel 4 -client_lib '/home/mewais/Test/Build/libTest.so;0;' -code_api -stack_size 56K -signal_stack_size 32K -max_elide_jmp 0 -max_elide_call 0 -early_inject -emulate_brk -no_inline_ignored_syscalls -disable_rseq -native_exec_default_list '' -no_native_exec_managed_code -no_indcall2direct
0x00007f44aa6ebb14 0x0000000000000000
/home/mewais/Test/Build/libTest.so=0x00007f44aa551000>
<rank order violation shared_cache_lock(mutex)@/home/travis/build/DynamoRIO/dynamorio/core/fcache.c:1590 acquired after privload_lock(recursive)@/home/travis/build/DynamoRIO/dynamorio/core/loader_shared.c:63 in tid:1427>
<Application /work/mewais/DCArch/DCSim/Bench/CRONO/apps/apsp/apsp (5159).  Internal Error: DynamoRIO debug check failure: /home/travis/build/DynamoRIO/dynamorio/core/utils.c:624 (dcontext->thread_owned_locks->last_lock->rank < lock->rank IF_CLIENT_INTERFACE( || first_client || both_client)) && "rank order violation"
(Error occurred @0 frags)
version 8.0.0, build 1
-no_dynamic_options -loglevel 4 -client_lib '/home/mewais/Test/Build/libTest.so;0;' -code_api -stack_size 56K -signal_stack_size 32K -max_elide_jmp 0 -max_elide_call 0 -early_inject -emulate_brk -no_inline_ignored_syscalls -disable_rseq -native_exec_default_list '' -no_native_exec_managed_code -no_indcall2direct
0x00007f442a5f9240 0x00000000710dc095
0x00007f442a5f9490 0x00000000710de27e
0x00007f442a5f94d0 0x00000000710deb75
0x00007f442a5f9500 0x00000000710dea54
0x00007f442a5f9530 0x00000000710deb28
0x00007f442a5f9550 0x00000000710892b6
0x00007f442a5f9580 0x000000007115fc64
0x00007f442a5f95a0 0x00000000710e1deb
0x00007f442a5f9710 0x00000000712d9ef0
0x00007f442a5f98c0 0x00000000712df3a5
0x00007f442a5f9ab0 0x000000007129f1d3
0x00007f44aa6ebb14 0x0000000000000000
/home/mewais/Test/Build/libTest.so=0x00007f44aa551000>
```

In this case, the libc was MUSL, while libstd-c++ and libgcc were originally coded against glibc. According to MUSL guides the two libraries will function fine in this scenario for "most" cases, I don't think they are the cause of these issues above. I am not sure, but I think MUSL is probably the best long-term solution candidate?

[Mohammad Ewais]

Oh, and here are the full link options I used to build against my custom libcs, in case you find them useful:
```

cmake_minimum_required(VERSION 3.22)
project(Test)
set(CMAKE_CXX_STANDARD 20)

set(CMAKE_CXX_FLAGS "-Wall -Werror -O3")

if (NOT DEFINED ENV{DYNAMORIO_PATH})
    message( SEND_ERROR "DYNAMORIO_PATH must be defined and point to DynamoRIO" )
endif()
find_package(DynamoRIO REQUIRED PATHS $ENV{DYNAMORIO_PATH}/)

set(LIBC "/tools/MUSL/install")

add_library(Test SHARED Test.cpp)
configure_DynamoRIO_client(Test)
target_link_options(Test PUBLIC -L${LIBC}/lib -I${LIBC}/include -static-libgcc -static-libstdc++ -Wl,--rpath=${LIBC}/lib -Wl,--dynamic-linker=${LIBC}/lib/ld-linux-x86-64.so.2 -Wl,-t)

```
where ${LIBC} either points to MUSL or glibc