Add image cache to avoid calling dladdr() when doing offline symbol resolution. This cache can also be used in the runtime symbol resolution case to detect we should recreate "backtrace_state" when new images have been loaded

CMakeLists.txt

@@ -86,6 +86,7 @@ set_option(TRACY_NO_CRASH_HANDLER "Disable crash handling" OFF)
 set_option(TRACY_TIMER_FALLBACK "Use lower resolution timers" OFF)
 set_option(TRACE_CLIENT_LIBUNWIND_BACKTRACE "Use libunwind backtracing where supported" OFF)
 set_option(TRACY_SYMBOL_OFFLINE_RESOLVE "Instead of full runtime symbol resolution, only resolve the image path and offset to enable offline symbol resolution" OFF)
+set_option(TRACY_ENABLE_IMAGE_CACHE "On glibc platforms, when doing offline symbol resolution, use an cache to determine the image path and offset instead of dladdr()" OFF)
 
 if(NOT TRACY_STATIC)
     target_compile_definitions(TracyClient PRIVATE TRACY_EXPORTS)

public/client/TracyCallstack.cpp

@@ -90,6 +90,124 @@ extern "C" const char* ___tracy_demangle( const char* mangled )
 #endif
 #endif
 
+#if defined( TRACY_ENABLE_IMAGE_CACHE) && (TRACY_HAS_CALLSTACK == 3)
+#   define TRACY_USE_IMAGE_CACHE
+#   include <link.h>
+#endif
+
+#ifdef TRACY_USE_IMAGE_CACHE
+// when we have access to dl_iterate_phdr(), we can build a cache of address ranges to image paths
+// so we can quickly determine which image an address falls into.
+// We refresh this cache only when we hit an address that doesn't fall into any known range.
+class ImageCache
+{
+public:
+    struct ImageEntry
+    {
+        ImageEntry( void* startAddress, void* endAddress, const char* name )
+        : m_startAddress( startAddress ), m_endAddress( endAddress ), m_name( name ) {}
+
+        void* m_startAddress;
+        void* m_endAddress;
+        const char* m_name;
+    };
+
+    ImageCache()
+    {
+        m_images = (tracy::FastVector<ImageEntry>*)tracy::tracy_malloc( sizeof( tracy::FastVector<ImageEntry> ) );
+        new(m_images) tracy::FastVector<ImageEntry>( 512 );
+
+        Refresh();
+    }
+    const ImageEntry* GetImageForAddress( void* address ) 
+    {
+        const ImageEntry* entry = GetImageEntryForAddress( address );
+        if( !entry )
+        {
+            //printf("* addr not found: %p (%d entries) refreshing (m_numberOfRefreshes: %d)\n",
+            //       address, m_images->size(), m_numberOfRefreshes);
+            Refresh();
+            return GetImageEntryForAddress( address );
+        }
+        return entry;
+    }
+
+    typedef void (*RefreshCallback)();
+    void SetOnRefreshCallback( RefreshCallback callback )
+    {
+        m_onRefreshCallback = callback;
+    }
+
+private:
+    tracy::FastVector<ImageEntry>* m_images;
+    const char* m_imageName = nullptr;
+    RefreshCallback m_onRefreshCallback;
+    uint32_t m_numberOfRefreshes = 0;
+
+    static int Callback( struct dl_phdr_info* info, size_t size, void* data ) 
+    {
+        ImageCache* cache = reinterpret_cast<ImageCache*>( data );
+
+        ImageEntry* image = cache->m_images->push_next();
+        image->m_startAddress = reinterpret_cast<void*>( info->dlpi_addr );
+        const uint32_t headerCount = info->dlpi_phnum;
+        assert( headerCount > 0);
+        image->m_endAddress = reinterpret_cast<void*>( info->dlpi_addr + 
+            info->dlpi_phdr[info->dlpi_phnum - 1].p_vaddr + info->dlpi_phdr[info->dlpi_phnum - 1].p_memsz);
+
+        // the base executable name isn't provided when iterating with dl_iterate_phdr,
+        // so we must get it in an alternative way and cache it
+        if( info->dlpi_name && info->dlpi_name[0] != '\0' )
+        {
+            image->m_name = info->dlpi_name;
+        }
+        else
+        {
+            if( !cache->m_imageName )
+            {
+                Dl_info dlInfo;
+                if( dladdr( (void *)info->dlpi_addr, &dlInfo ) )
+                {
+                    cache->m_imageName = dlInfo.dli_fname;
+                }
+            }
+            image->m_name = cache->m_imageName;
+        }
+
+        //printf("\tdl_iterate_phdr:'%s', start:%p, end:%p (headerCount: %d)'\n",
+        //       image->m_name, image->m_startAddress, image->m_endAddress, headerCount);
+
+        return 0;
+    }
+
+    void Refresh()
+    {
+        m_images->clear();
+
+        dl_iterate_phdr( Callback, this );
+        
+        std::sort( m_images->begin(), m_images->end(), 
+            []( const ImageEntry& lhs, const ImageEntry& rhs ) { return lhs.m_startAddress > rhs.m_startAddress; } );
+
+        if( m_onRefreshCallback ) m_onRefreshCallback();
+
+        m_numberOfRefreshes++;
+    }
+
+    const ImageEntry* GetImageEntryForAddress( void* address ) const 
+    {
+        auto it = std::lower_bound( m_images->begin(), m_images->end(), address, 
+            []( const ImageEntry& lhs, const void* rhs ) { return lhs.m_startAddress > rhs; } );
+
+        if( it != m_images->end() && address < it->m_endAddress )
+        {
+            return &(*it);
+        }
+        return nullptr;
+    }
+};
+#endif //#ifdef TRACY_USE_IMAGE_CACHE
+
 namespace tracy
 {
 
@@ -607,6 +725,9 @@ struct backtrace_state* cb_bts = nullptr;
 int cb_num;
 CallstackEntry cb_data[MaxCbTrace];
 int cb_fixup;
+#ifdef TRACY_USE_IMAGE_CACHE
+static ImageCache* s_imageCache = nullptr;
+#endif //#ifdef TRACY_USE_IMAGE_CACHE
 
 #ifdef TRACY_DEBUGINFOD
 debuginfod_client* s_debuginfod;
@@ -779,6 +900,13 @@ void InitCallstackCritical()
 
 void InitCallstack()
 {
+    InitRpmalloc();
+
+#ifdef TRACY_USE_IMAGE_CACHE
+    s_imageCache = (ImageCache*)tracy::tracy_malloc( sizeof( ImageCache ) );
+    new(s_imageCache) ImageCache();
+#endif //#ifdef TRACY_USE_IMAGE_CACHE
+    
 #ifndef TRACY_SYMBOL_OFFLINE_RESOLVE
     s_shouldResolveSymbolsOffline = ShouldResolveSymbolsOffline();
 #endif //#ifndef TRACY_SYMBOL_OFFLINE_RESOLVE
@@ -789,7 +917,15 @@ void InitCallstack()
     }
     else
     {
+#ifdef TRACY_USE_IMAGE_CACHE
+        s_imageCache->SetOnRefreshCallback([]()
+        {
+            // FIXME: there is no backtrace_destroy_state(), so we are forced to leak...
+            cb_bts = backtrace_create_state( nullptr, 0, nullptr, nullptr );
+        });
+#else
         cb_bts = backtrace_create_state( nullptr, 0, nullptr, nullptr );
+#endif // #ifdef TRACY_USE_IMAGE_CACHE
     }
 
 #ifndef TRACY_DEMANGLE
@@ -869,6 +1005,9 @@ debuginfod_client* GetDebuginfodClient()
 
 void EndCallstack()
 {
+#ifdef TRACY_USE_IMAGE_CACHE
+    tracy::tracy_free(s_imageCache);
+#endif //#ifdef TRACY_USE_IMAGE_CACHE
 #ifndef TRACY_DEMANGLE
     ___tracy_free_demangle_buffer();
 #endif
@@ -1041,12 +1180,12 @@ void SymInfoError( void* /*data*/, const char* /*msg*/, int /*errnum*/ )
     cb_data[cb_num-1].symAddr = 0;
 }
 
-void GetSymbolForOfflineResolve(void* address, Dl_info& dlinfo, CallstackEntry& cbEntry)
+void GetSymbolForOfflineResolve(void* address, uint64_t imageBaseAddress, CallstackEntry& cbEntry)
 {
     // tagged with a string that we can identify as an unresolved symbol
     cbEntry.name = CopyStringFast( "[unresolved]" );
     // set .so relative offset so it can be resolved offline
-    cbEntry.symAddr = (uint64_t)address - (uint64_t)(dlinfo.dli_fbase);
+    cbEntry.symAddr = (uint64_t)address - imageBaseAddress;
     cbEntry.symLen = 0x0;
     cbEntry.file = CopyStringFast( "[unknown]" );
     cbEntry.line = 0;
@@ -1057,17 +1196,29 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr )
     InitRpmalloc();
     if( ptr >> 63 == 0 )
     {
-        const char* symloc = nullptr;
+        const char* imageName = nullptr;
+        uint64_t imageBaseAddress = 0x0;
+
+#ifdef TRACY_USE_IMAGE_CACHE
+        const auto* image = s_imageCache->GetImageForAddress((void*)ptr);
+        if( image )
+        {
+            imageName = image->m_name;
+            imageBaseAddress = uint64_t(image->m_startAddress);
+        }
+#else
         Dl_info dlinfo;
         if( dladdr( (void*)ptr, &dlinfo ) )
         {
-            symloc = dlinfo.dli_fname;
+            imageName = dlinfo.dli_fname;
+            imageBaseAddress = uint64_t( dlinfo.dli_fbase );
         }
+#endif
 
         if( s_shouldResolveSymbolsOffline )
         {
             cb_num = 1;
-            GetSymbolForOfflineResolve( (void*)ptr, dlinfo, cb_data[0] );
+            GetSymbolForOfflineResolve( (void*)ptr, imageBaseAddress, cb_data[0] );
         }
         else
         {
@@ -1078,7 +1229,7 @@ CallstackEntryData DecodeCallstackPtr( uint64_t ptr )
             backtrace_syminfo( cb_bts, ptr, SymInfoCallback, SymInfoError, nullptr );
         }
 
-        return { cb_data, uint8_t( cb_num ), symloc ? symloc : "[unknown]" };
+        return { cb_data, uint8_t( cb_num ), imageName ? imageName : "[unknown]" };
     }
 #ifdef __linux
     else if( s_kernelSym )