From d8c67b61dc6c81f8ac41c19d3cd7f3b8d208a282 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Wed, 25 Mar 2026 09:39:12 +0100 Subject: [PATCH 1/2] fix: promote loader symbols to global scope before loading embedded .so (glibc) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When libdd_profiling.so is dlopen'd with RTLD_GLOBAL, glibc defers symbol promotion until after dlopen returns. The inner dlopen of the embedded .so therefore cannot resolve ddprof_lib_state from the loader, causing it to fail silently. Fix: in the loader constructor, use RTLD_NOLOAD | RTLD_GLOBAL to re-open the loader itself and upgrade its binding to global before loading the embedded library. The soname is passed via DDPROF_LOADER_SONAME from CMake (defined when USE_LOADER is enabled). Guarded by #ifdef so the call is omitted in builds where USE_LOADER is not enabled. Note: dlopen with RTLD_GLOBAL is not supported on musl — musl rejects initial-exec TLS cross-library relocations for dlopen'd libraries. The test is skipped on musl and LD_PRELOAD is documented as the alternative. Add loader_rtld_global_test to verify the fix on glibc. Document the dlopen usage and musl limitation in Troubleshooting.md. Co-Authored-By: Claude Sonnet 4.6 --- CMakeLists.txt | 5 ++- docs/Troubleshooting.md | 19 +++++++++ src/lib/loader.c | 30 ++++++++++++++ test/CMakeLists.txt | 16 ++++++++ test/loader_rtld_global_test.c | 72 ++++++++++++++++++++++++++++++++++ 5 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 test/loader_rtld_global_test.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 71bd6d739..af60b7326 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -389,8 +389,9 @@ if(USE_LOADER) target_link_libraries(dd_profiling-shared PRIVATE libddprofiling_embedded_object ddprof_exe_object) target_compile_definitions( - dd_profiling-shared PRIVATE DDPROF_EMBEDDED_LIB_DATA DDPROF_EMBEDDED_EXE_DATA - DDPROF_PROFILING_LIBRARY) + dd_profiling-shared + PRIVATE DDPROF_EMBEDDED_LIB_DATA DDPROF_EMBEDDED_EXE_DATA DDPROF_PROFILING_LIBRARY + DDPROF_LOADER_SONAME="lib$.so") else() # Without loader, libdd_profiling.so is basically the same as libdd_profiling-embedded.so plus an # embedded ddprof executable. diff --git a/docs/Troubleshooting.md b/docs/Troubleshooting.md index 15eeac7da..696c4fc23 100644 --- a/docs/Troubleshooting.md +++ b/docs/Troubleshooting.md @@ -83,6 +83,25 @@ example: MALLOC_CONF=stats_print:true ./ddprof -S test-service service_cmd ``` +### Loading libdd_profiling.so via dlopen + +The recommended way to use `libdd_profiling.so` is to link against it directly +or use `LD_PRELOAD`. If `dlopen` is required (e.g. conditional profiling based +on a config flag), use `RTLD_GLOBAL`: + +```c +dlopen("libdd_profiling.so", RTLD_LAZY | RTLD_GLOBAL); +``` + +**`RTLD_GLOBAL` is required** so that the loader's symbols are visible to the +embedded library it loads internally. `dlopen` with `RTLD_GLOBAL` is supported +on glibc systems. + +**musl limitation:** loading `libdd_profiling.so` with `dlopen` is not supported +on musl-based systems (e.g. Alpine Linux). musl rejects initial-exec TLS +cross-library relocations for `dlopen`'d libraries, which causes the embedded +library to fail to load. Use `LD_PRELOAD` instead on musl systems. + ### Library issues (LD_PRELOAD or allocation profiling) You will want to instrument the loader function (loader.c) to figure out what is going on. diff --git a/src/lib/loader.c b/src/lib/loader.c index ee47b1972..feecbfd49 100644 --- a/src/lib/loader.c +++ b/src/lib/loader.c @@ -145,6 +145,35 @@ static void ensure_librt_is_loaded() { } } +// When the loader is dlopen'd with RTLD_GLOBAL, glibc does not promote its +// symbols to global scope until dlopen returns. The embedded .so references +// ddprof_lib_state (defined here in the loader) as an undefined symbol, so +// loading it with RTLD_NOW during our constructor fails with +// "undefined symbol: ddprof_lib_state". +// +// Fix: re-open ourselves with RTLD_NOLOAD | RTLD_GLOBAL to promote our +// symbols before loading the embedded .so. RTLD_NOLOAD is a no-op when the +// loader was opened with RTLD_LOCAL (the common LD_PRELOAD case). +// +// Note: on musl, dlopen with RTLD_GLOBAL is not supported for this library +// because musl rejects initial-exec TLS cross-library relocations for +// dlopen'd libraries entirely. +static void ensure_loader_symbols_promoted() { +#ifdef DDPROF_LOADER_SONAME + void *self = + my_dlopen(DDPROF_LOADER_SONAME, RTLD_GLOBAL | RTLD_NOLOAD | RTLD_NOW); + if (!self) { + // RTLD_NOLOAD should always find the loader (we are running inside it). + // NULL means the soname has changed or something is seriously wrong; + // the embedded .so will likely fail to load next. + fprintf(stderr, + "ddprof loader: failed to promote symbols to global scope " + "(RTLD_NOLOAD on " DDPROF_LOADER_SONAME + " returned NULL) -- embedded library may fail to load\n"); + } +#endif +} + static const char *temp_directory_path() { const char *tmpdir = NULL; const char *env[] = {"TMPDIR", "TMP", "TEMP", "TEMPDIR", NULL}; @@ -279,6 +308,7 @@ static void __attribute__((constructor)) loader() { ensure_libm_is_loaded(); ensure_libpthread_is_loaded(); ensure_librt_is_loaded(); + ensure_loader_symbols_promoted(); s_profiling_lib_handle = my_dlopen(lib_profiling_path, RTLD_LOCAL | RTLD_NOW); free(lib_profiling_path); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b5c02deec..3bdbebb77 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -534,6 +534,22 @@ if(NOT CMAKE_BUILD_TYPE STREQUAL "SanitizedDebug") COMMAND ${CMAKE_SOURCE_DIR}/tools/check_for_unsafe_libc_functions.py $) endif() + + # Test that the loader works when dlopen'd with RTLD_GLOBAL (the pattern used by applications that + # load libdd_profiling.so at runtime). Skipped on musl: RTLD_GLOBAL dlopen is unsupported on musl + # because musl rejects initial-exec TLS cross-library relocations for dlopen'd libraries. + if(NOT LIBC_TYPE STREQUAL "musl" AND USE_LOADER) + add_executable(loader_rtld_global_test loader_rtld_global_test.c) + target_link_libraries(loader_rtld_global_test PRIVATE dl) + add_dependencies(loader_rtld_global_test dd_profiling-shared) + add_test( + NAME loader_rtld_global + COMMAND loader_rtld_global_test + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + set_tests_properties( + loader_rtld_global PROPERTIES ENVIRONMENT + "TEST_DD_PROFILING_LIB=$") + endif() endif() if(NOT CMAKE_BUILD_TYPE STREQUAL "SanitizedDebug") diff --git a/test/loader_rtld_global_test.c b/test/loader_rtld_global_test.c new file mode 100644 index 000000000..535ba4167 --- /dev/null +++ b/test/loader_rtld_global_test.c @@ -0,0 +1,72 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present +// Datadog, Inc. + +// Test that libdd_profiling.so (the loader) works when loaded at runtime +// with RTLD_GLOBAL. +// +// This reproduces the pattern used by applications that dlopen the profiling +// library at startup (e.g., after reading a config flag). +// The loader's constructor extracts and dlopen's the embedded .so, which +// references ddprof_lib_state as an extern symbol defined in the loader. +// On glibc, RTLD_GLOBAL only takes effect after dlopen returns, so without +// the self-promotion fix the embedded library fails with: +// "undefined symbol: ddprof_lib_state" +// +// Note: this test is skipped on musl. RTLD_GLOBAL dlopen of the loader is +// unsupported on musl because musl rejects initial-exec TLS cross-library +// relocations for dlopen'd libraries entirely. + +#include +#include +#include + +typedef int (*start_fn_t)(void); +typedef void (*stop_fn_t)(int); + +int main(void) { + const char *lib_path = getenv("TEST_DD_PROFILING_LIB"); + if (!lib_path) { + lib_path = "./libdd_profiling.so"; + } + + fprintf(stderr, "loading %s with RTLD_LAZY | RTLD_GLOBAL...\n", lib_path); + + void *handle = dlopen(lib_path, RTLD_LAZY | RTLD_GLOBAL); + if (!handle) { + fprintf(stderr, "FAIL: dlopen: %s\n", dlerror()); + return 1; + } + + // Call ddprof_start_profiling. If the embedded library failed to load + // (the bug we're testing for), this returns -1. + start_fn_t start = (start_fn_t)dlsym(handle, "ddprof_start_profiling"); + if (!start) { + fprintf(stderr, "FAIL: ddprof_start_profiling not found\n"); + dlclose(handle); + return 1; + } + + int rc = start(); + // The profiler may fail to start for environment reasons (no perf events, + // etc.), but a return of -1 specifically means the embedded library was + // never loaded (the loader's start function returns -1 when its function + // pointer is NULL). + if (rc == -1) { + fprintf(stderr, + "FAIL: ddprof_start_profiling returned -1 " + "(embedded library not loaded)\n"); + dlclose(handle); + return 1; + } + + stop_fn_t stop = (stop_fn_t)dlsym(handle, "ddprof_stop_profiling"); + if (stop) { + stop(1000); + } + + fprintf(stderr, "PASS: loader constructor succeeded with RTLD_GLOBAL\n"); + dlclose(handle); + return 0; +} From 9a0ff499d1f0818725716de63206137fec3a1205 Mon Sep 17 00:00:00 2001 From: r1viollet Date: Wed, 25 Mar 2026 12:05:25 +0100 Subject: [PATCH 2/2] dlopen support - Minor fixes - Clean up documentation - Remove the test which was moved to ddprof-build --- docs/Troubleshooting.md | 9 ++--- src/lib/loader.c | 4 +- test/CMakeLists.txt | 15 ------- test/loader_rtld_global_test.c | 72 ---------------------------------- 4 files changed, 6 insertions(+), 94 deletions(-) delete mode 100644 test/loader_rtld_global_test.c diff --git a/docs/Troubleshooting.md b/docs/Troubleshooting.md index 696c4fc23..d3e56accc 100644 --- a/docs/Troubleshooting.md +++ b/docs/Troubleshooting.md @@ -87,15 +87,14 @@ MALLOC_CONF=stats_print:true ./ddprof -S test-service service_cmd The recommended way to use `libdd_profiling.so` is to link against it directly or use `LD_PRELOAD`. If `dlopen` is required (e.g. conditional profiling based -on a config flag), use `RTLD_GLOBAL`: +on a config flag): ```c -dlopen("libdd_profiling.so", RTLD_LAZY | RTLD_GLOBAL); +dlopen("libdd_profiling.so", RTLD_NOW); ``` -**`RTLD_GLOBAL` is required** so that the loader's symbols are visible to the -embedded library it loads internally. `dlopen` with `RTLD_GLOBAL` is supported -on glibc systems. +The loader internally promotes its own symbols to global scope so that the +embedded library it loads can resolve them. **musl limitation:** loading `libdd_profiling.so` with `dlopen` is not supported on musl-based systems (e.g. Alpine Linux). musl rejects initial-exec TLS diff --git a/src/lib/loader.c b/src/lib/loader.c index feecbfd49..3368b0aaf 100644 --- a/src/lib/loader.c +++ b/src/lib/loader.c @@ -152,8 +152,8 @@ static void ensure_librt_is_loaded() { // "undefined symbol: ddprof_lib_state". // // Fix: re-open ourselves with RTLD_NOLOAD | RTLD_GLOBAL to promote our -// symbols before loading the embedded .so. RTLD_NOLOAD is a no-op when the -// loader was opened with RTLD_LOCAL (the common LD_PRELOAD case). +// symbols before loading the embedded .so. When loaded via LD_PRELOAD, +// symbols are already in global scope so this is a harmless no-op. // // Note: on musl, dlopen with RTLD_GLOBAL is not supported for this library // because musl rejects initial-exec TLS cross-library relocations for diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3bdbebb77..eca30a430 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -535,21 +535,6 @@ if(NOT CMAKE_BUILD_TYPE STREQUAL "SanitizedDebug") $) endif() - # Test that the loader works when dlopen'd with RTLD_GLOBAL (the pattern used by applications that - # load libdd_profiling.so at runtime). Skipped on musl: RTLD_GLOBAL dlopen is unsupported on musl - # because musl rejects initial-exec TLS cross-library relocations for dlopen'd libraries. - if(NOT LIBC_TYPE STREQUAL "musl" AND USE_LOADER) - add_executable(loader_rtld_global_test loader_rtld_global_test.c) - target_link_libraries(loader_rtld_global_test PRIVATE dl) - add_dependencies(loader_rtld_global_test dd_profiling-shared) - add_test( - NAME loader_rtld_global - COMMAND loader_rtld_global_test - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) - set_tests_properties( - loader_rtld_global PROPERTIES ENVIRONMENT - "TEST_DD_PROFILING_LIB=$") - endif() endif() if(NOT CMAKE_BUILD_TYPE STREQUAL "SanitizedDebug") diff --git a/test/loader_rtld_global_test.c b/test/loader_rtld_global_test.c deleted file mode 100644 index 535ba4167..000000000 --- a/test/loader_rtld_global_test.c +++ /dev/null @@ -1,72 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. This product includes software -// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present -// Datadog, Inc. - -// Test that libdd_profiling.so (the loader) works when loaded at runtime -// with RTLD_GLOBAL. -// -// This reproduces the pattern used by applications that dlopen the profiling -// library at startup (e.g., after reading a config flag). -// The loader's constructor extracts and dlopen's the embedded .so, which -// references ddprof_lib_state as an extern symbol defined in the loader. -// On glibc, RTLD_GLOBAL only takes effect after dlopen returns, so without -// the self-promotion fix the embedded library fails with: -// "undefined symbol: ddprof_lib_state" -// -// Note: this test is skipped on musl. RTLD_GLOBAL dlopen of the loader is -// unsupported on musl because musl rejects initial-exec TLS cross-library -// relocations for dlopen'd libraries entirely. - -#include -#include -#include - -typedef int (*start_fn_t)(void); -typedef void (*stop_fn_t)(int); - -int main(void) { - const char *lib_path = getenv("TEST_DD_PROFILING_LIB"); - if (!lib_path) { - lib_path = "./libdd_profiling.so"; - } - - fprintf(stderr, "loading %s with RTLD_LAZY | RTLD_GLOBAL...\n", lib_path); - - void *handle = dlopen(lib_path, RTLD_LAZY | RTLD_GLOBAL); - if (!handle) { - fprintf(stderr, "FAIL: dlopen: %s\n", dlerror()); - return 1; - } - - // Call ddprof_start_profiling. If the embedded library failed to load - // (the bug we're testing for), this returns -1. - start_fn_t start = (start_fn_t)dlsym(handle, "ddprof_start_profiling"); - if (!start) { - fprintf(stderr, "FAIL: ddprof_start_profiling not found\n"); - dlclose(handle); - return 1; - } - - int rc = start(); - // The profiler may fail to start for environment reasons (no perf events, - // etc.), but a return of -1 specifically means the embedded library was - // never loaded (the loader's start function returns -1 when its function - // pointer is NULL). - if (rc == -1) { - fprintf(stderr, - "FAIL: ddprof_start_profiling returned -1 " - "(embedded library not loaded)\n"); - dlclose(handle); - return 1; - } - - stop_fn_t stop = (stop_fn_t)dlsym(handle, "ddprof_stop_profiling"); - if (stop) { - stop(1000); - } - - fprintf(stderr, "PASS: loader constructor succeeded with RTLD_GLOBAL\n"); - dlclose(handle); - return 0; -}