Fix rpmalloc init for TRACY_MANUAL_LIFETIME path.

The C++11 spec states in [basic.stc.thread] thread storage duration:

2. A variable with thread storage duration shall be initialized before its
   first odr-use (3.2) and, if constructed, shall be destroyed on thread exit.

Previously Tracy relied on the TLS data being initialized:
- During thread creation (MSVC).
- Or during first use in a thread, but the initialization was performed for
  the whole TLS block.

It seems that new compilers are more granular with how they perform the
initialization, hence rpmalloc init has to be checked before each allocation,
as it cannot be "folded" into, for example, initialization of the profiler
itself.

NEWS

@@ -9,6 +9,9 @@ v0.x.x (xxxx-xx-xx)
 - Added TRACY_NO_CALLSTACK_INLINES macro to disable inline functions
   resolution in call stacks on Windows.
 - Limited client query response rate.
+- Improved function matching algorithm in compare traces view.
+- Added minimal CMake integration layer.
+- Reworked rpmalloc initialization.
 
 
 v0.7.8 (2021-05-19)

TracyC.h

@@ -101,7 +101,6 @@ struct ___tracy_c_zone_context
 // This struct, as visible to user, is immutable, so treat it as if const was declared here.
 typedef /*const*/ struct ___tracy_c_zone_context TracyCZoneCtx;
 
-TRACY_API void ___tracy_init_thread(void);
 TRACY_API uint64_t ___tracy_alloc_srcloc( uint32_t line, const char* source, size_t sourceSz, const char* function, size_t functionSz );
 TRACY_API uint64_t ___tracy_alloc_srcloc_name( uint32_t line, const char* source, size_t sourceSz, const char* function, size_t functionSz, const char* name, size_t nameSz );
 

TracyClient.cpp

@@ -27,6 +27,7 @@
 #include "common/TracySocket.cpp"
 #include "client/tracy_rpmalloc.cpp"
 #include "client/TracyDxt1.cpp"
+#include "client/TracyAlloc.cpp"
 
 #if TRACY_HAS_CALLSTACK == 2 || TRACY_HAS_CALLSTACK == 3 || TRACY_HAS_CALLSTACK == 4 || TRACY_HAS_CALLSTACK == 6
 #  include "libbacktrace/alloc.cpp"

TracyD3D11.hpp

@@ -271,7 +271,7 @@ public:
         MemWrite( &item->gpuZoneBegin.thread, GetThreadHandle() );
         MemWrite( &item->gpuZoneBegin.queryId, uint16_t( queryId ) );
         MemWrite( &item->gpuZoneBegin.context, ctx->GetId() );
-        
+
         Profiler::QueueSerialFinish();
     }
 
@@ -376,7 +376,7 @@ public:
         MemWrite( &item->gpuZoneEnd.thread, GetThreadHandle() );
         MemWrite( &item->gpuZoneEnd.queryId, uint16_t( queryId ) );
         MemWrite( &item->gpuZoneEnd.context, m_ctx->GetId() );
-        
+
         Profiler::QueueSerialFinish();
     }
 
@@ -389,7 +389,6 @@ private:
 
 static inline D3D11Ctx* CreateD3D11Context( ID3D11Device* device, ID3D11DeviceContext* devicectx )
 {
-    InitRPMallocThread();
     auto ctx = (D3D11Ctx*)tracy_malloc( sizeof( D3D11Ctx ) );
     new(ctx) D3D11Ctx( device, devicectx );
     return ctx;

TracyD3D12.hpp

@@ -451,8 +451,6 @@ namespace tracy
 
 	static inline D3D12QueueCtx* CreateD3D12Context(ID3D12Device* device, ID3D12CommandQueue* queue)
 	{
-		InitRPMallocThread();
-
 		auto* ctx = static_cast<D3D12QueueCtx*>(tracy_malloc(sizeof(D3D12QueueCtx)));
 		new (ctx) D3D12QueueCtx{ device, queue };
 

TracyOpenCL.hpp

@@ -286,7 +286,6 @@ namespace tracy {
 
     static inline OpenCLCtx* CreateCLContext(cl_context context, cl_device_id device)
     {
-        InitRPMallocThread();
         auto ctx = (OpenCLCtx*)tracy_malloc(sizeof(OpenCLCtx));
         new (ctx) OpenCLCtx(context, device);
         return ctx;

TracyOpenGL.hpp

@@ -53,7 +53,7 @@ public:
 #  define glQueryCounter glQueryCounterEXT
 #endif
 
-#define TracyGpuContext tracy::InitRPMallocThread(); tracy::GetGpuCtx().ptr = (tracy::GpuCtx*)tracy::tracy_malloc( sizeof( tracy::GpuCtx ) ); new(tracy::GetGpuCtx().ptr) tracy::GpuCtx;
+#define TracyGpuContext tracy::GetGpuCtx().ptr = (tracy::GpuCtx*)tracy::tracy_malloc( sizeof( tracy::GpuCtx ) ); new(tracy::GetGpuCtx().ptr) tracy::GpuCtx;
 #define TracyGpuContextName( name, size ) tracy::GetGpuCtx().ptr->Name( name, size );
 #if defined TRACY_HAS_CALLSTACK && defined TRACY_CALLSTACK
 #  define TracyGpuNamedZone( varname, name, active ) static constexpr tracy::SourceLocationData TracyConcat(__tracy_gpu_source_location,__LINE__) { name, __FUNCTION__,  __FILE__, (uint32_t)__LINE__, 0 }; tracy::GpuCtxScope varname( &TracyConcat(__tracy_gpu_source_location,__LINE__), TRACY_CALLSTACK, active );

TracyVulkan.hpp

@@ -456,7 +456,6 @@ private:
 
 static inline VkCtx* CreateVkContext( VkPhysicalDevice physdev, VkDevice device, VkQueue queue, VkCommandBuffer cmdbuf, PFN_vkGetPhysicalDeviceCalibrateableTimeDomainsEXT gpdctd, PFN_vkGetCalibratedTimestampsEXT gct )
 {
-    InitRPMallocThread();
     auto ctx = (VkCtx*)tracy_malloc( sizeof( VkCtx ) );
     new(ctx) VkCtx( physdev, device, queue, cmdbuf, gpdctd, gct );
     return ctx;

client/TracyAlloc.cpp

@@ -0,0 +1,35 @@
+#ifdef TRACY_ENABLE
+
+#include <atomic>
+
+#include "../common/TracyAlloc.hpp"
+#include "../common/TracyYield.hpp"
+
+namespace tracy
+{
+
+extern std::atomic<int> RpInitDone;
+extern std::atomic<int> RpInitLock;
+
+TRACY_API void InitRpmallocPlumbing()
+{
+    const auto done = RpInitDone.load( std::memory_order_acquire );
+    if( !done )
+    {
+        int expected = 0;
+        while( !RpInitLock.compare_exchange_weak( expected, 1, std::memory_order_release, std::memory_order_relaxed ) ) { expected = 0; YieldThread(); }
+        const auto done = RpInitDone.load( std::memory_order_acquire );
+        if( !done )
+        {
+            rpmalloc_initialize();
+            RpInitDone.store( 1, std::memory_order_release );
+        }
+        RpInitLock.store( 0, std::memory_order_release );
+    }
+    rpmalloc_thread_initialize();
+    RpThreadInitDone = true;
+}
+
+}
+
+#endif

client/TracyProfiler.cpp

@@ -64,6 +64,7 @@
 #include "../common/TracyAlign.hpp"
 #include "../common/TracySocket.hpp"
 #include "../common/TracySystem.hpp"
+#include "../common/TracyYield.hpp"
 #include "../common/tracy_lz4.hpp"
 #include "tracy_rpmalloc.hpp"
 #include "TracyCallstack.hpp"
@@ -118,45 +119,6 @@ extern "C" typedef BOOL (WINAPI *t_GetLogicalProcessorInformationEx)( LOGICAL_PR
 namespace tracy
 {
 
-namespace
-{
-#  if ( defined _WIN32 || defined __CYGWIN__ ) && _WIN32_WINNT >= _WIN32_WINNT_VISTA
-    BOOL CALLBACK InitOnceCallback( PINIT_ONCE /*initOnce*/, PVOID /*Parameter*/, PVOID* /*Context*/)
-    {
-        rpmalloc_initialize();
-        return TRUE;
-    }
-    INIT_ONCE InitOnce = INIT_ONCE_STATIC_INIT;
-#  elif defined __linux__
-    void InitOnceCallback()
-    {
-        rpmalloc_initialize();
-    }
-    pthread_once_t once_control = PTHREAD_ONCE_INIT;
-#  else
-    void InitOnceCallback()
-    {
-        rpmalloc_initialize();
-    }
-    std::once_flag once_flag;
-#  endif
-}
-
-struct RPMallocInit
-{
-    RPMallocInit()
-    {
-#  if ( defined _WIN32 || defined __CYGWIN__ ) && _WIN32_WINNT >= _WIN32_WINNT_VISTA
-        InitOnceExecuteOnce( &InitOnce, InitOnceCallback, nullptr, nullptr );
-#  elif defined __linux__
-        pthread_once( &once_control, InitOnceCallback );
-#  else
-        std::call_once( once_flag, InitOnceCallback );
-#  endif
-        rpmalloc_thread_initialize();
-    }
-};
-
 #ifndef TRACY_DELAYED_INIT
 
 struct InitTimeWrapper
@@ -964,12 +926,6 @@ TRACY_API int64_t GetFrequencyQpc()
 #ifdef TRACY_DELAYED_INIT
 struct ThreadNameData;
 TRACY_API moodycamel::ConcurrentQueue<QueueItem>& GetQueue();
-TRACY_API void InitRPMallocThread();
-
-void InitRPMallocThread()
-{
-    RPMallocInit rpinit;
-}
 
 struct ProfilerData
 {
@@ -991,7 +947,6 @@ struct ProducerWrapper
 struct ProfilerThreadData
 {
     ProfilerThreadData( ProfilerData& data ) : token( data ), gpuCtx( { nullptr } ) {}
-    RPMallocInit rpmalloc_init;
     ProducerWrapper token;
     GpuCtxWrapper gpuCtx;
 #  ifdef TRACY_ON_DEMAND
@@ -999,11 +954,14 @@ struct ProfilerThreadData
 #  endif
 };
 
+std::atomic<int> RpInitDone { 0 };
+std::atomic<int> RpInitLock { 0 };
+thread_local bool RpThreadInitDone = false;
+
 #  ifdef TRACY_MANUAL_LIFETIME
 ProfilerData* s_profilerData = nullptr;
 TRACY_API void StartupProfiler()
 {
-    RPMallocInit init;
     s_profilerData = (ProfilerData*)tracy_malloc( sizeof( ProfilerData ) );
     new (s_profilerData) ProfilerData();
     s_profilerData->profiler.SpawnWorkerThreads();
@@ -1030,11 +988,10 @@ static ProfilerData& GetProfilerData()
     if( !ptr )
     {
         int expected = 0;
-        while( !profilerDataLock.compare_exchange_strong( expected, 1, std::memory_order_release, std::memory_order_relaxed ) ) { expected = 0; }
+        while( !profilerDataLock.compare_exchange_weak( expected, 1, std::memory_order_release, std::memory_order_relaxed ) ) { expected = 0; YieldThread(); }
         ptr = profilerData.load( std::memory_order_acquire );
         if( !ptr )
         {
-            RPMallocInit init;
             ptr = (ProfilerData*)tracy_malloc( sizeof( ProfilerData ) );
             new (ptr) ProfilerData();
             profilerData.store( ptr, std::memory_order_release );
@@ -1071,7 +1028,6 @@ public:
         void* p = pthread_getspecific(m_key);
         if (!p)
         {
-            RPMallocInit init;
             p = (ProfilerThreadData*)tracy_malloc( sizeof( ProfilerThreadData ) );
             new (p) ProfilerThreadData(GetProfilerData());
             pthread_setspecific(m_key, p);
@@ -1123,18 +1079,12 @@ namespace
 #  endif
 
 #else
-TRACY_API void InitRPMallocThread()
-{
-    rpmalloc_thread_initialize();
-}
 
 // MSVC static initialization order solution. gcc/clang uses init_order() to avoid all this.
 
 // 1a. But s_queue is needed for initialization of variables in point 2.
 extern moodycamel::ConcurrentQueue<QueueItem> s_queue;
 
-thread_local RPMallocInit init_order(106) s_rpmalloc_thread_init;
-
 // 2. If these variables would be in the .CRT$XCB section, they would be initialized only in main thread.
 thread_local moodycamel::ProducerToken init_order(107) s_token_detail( s_queue );
 thread_local ProducerWrapper init_order(108) s_token { s_queue.get_explicit_producer( s_token_detail ) };
@@ -1147,7 +1097,9 @@ thread_local ThreadHandleWrapper init_order(104) s_threadHandle { detail::GetThr
 #  endif
 
 static InitTimeWrapper init_order(101) s_initTime { SetupHwTimer() };
-static RPMallocInit init_order(102) s_rpmalloc_init;
+std::atomic<int> init_order(102) RpInitDone( 0 );
+std::atomic<int> init_order(102) RpInitLock( 0 );
+thread_local bool RpThreadInitDone = false;
 moodycamel::ConcurrentQueue<QueueItem> init_order(103) s_queue( QueuePrealloc );
 std::atomic<uint32_t> init_order(104) s_lockCounter( 0 );
 std::atomic<uint8_t> init_order(104) s_gpuCtxCounter( 0 );
@@ -3613,19 +3565,6 @@ TRACY_API uint64_t ___tracy_alloc_srcloc_name( uint32_t line, const char* source
     return tracy::Profiler::AllocSourceLocation( line, source, sourceSz, function, functionSz, name, nameSz );
 }
 
-// thread_locals are not initialized on thread creation. At least on GNU/Linux. Instead they are
-// initialized on their first ODR-use. This means that the allocator is not automagically
-// initialized every time a thread is created. As thus, expose to the C API users a simple API to
-// call every time they create a thread. Here we can then put all sorts of per-thread
-// initialization.
-TRACY_API void ___tracy_init_thread(void) {
-#ifdef TRACY_DELAYED_INIT
-    (void)tracy::GetProfilerThreadData();
-#else
-    (void)tracy::s_rpmalloc_thread_init;
-#endif
-}
-
 #ifdef __cplusplus
 }
 #endif

client/TracyProfiler.hpp

@@ -63,7 +63,6 @@ TRACY_API std::atomic<uint32_t>& GetLockCounter();
 TRACY_API std::atomic<uint8_t>& GetGpuCtxCounter();
 TRACY_API GpuCtxWrapper& GetGpuCtx();
 TRACY_API uint64_t GetThreadHandle();
-TRACY_API void InitRPMallocThread();
 TRACY_API bool ProfilerAvailable();
 TRACY_API int64_t GetFrequencyQpc();
 
@@ -295,7 +294,6 @@ public:
 #endif
         if( callstack != 0 )
         {
-            InitRPMallocThread();
             tracy::GetProfiler().SendCallstack( callstack );
         }
 
@@ -315,7 +313,6 @@ public:
 #endif
         if( callstack != 0 )
         {
-            InitRPMallocThread();
             tracy::GetProfiler().SendCallstack( callstack );
         }
 
@@ -333,7 +330,6 @@ public:
 #endif
         if( callstack != 0 )
         {
-            InitRPMallocThread();
             tracy::GetProfiler().SendCallstack( callstack );
         }
 
@@ -356,7 +352,6 @@ public:
 #endif
         if( callstack != 0 )
         {
-            InitRPMallocThread();
             tracy::GetProfiler().SendCallstack( callstack );
         }
 
@@ -372,7 +367,6 @@ public:
     static tracy_force_inline void MessageAppInfo( const char* txt, size_t size )
     {
         assert( size < std::numeric_limits<uint16_t>::max() );
-        InitRPMallocThread();
         auto ptr = (char*)tracy_malloc( size );
         memcpy( ptr, txt, size );
         TracyLfqPrepare( QueueType::MessageAppInfo );
@@ -423,7 +417,6 @@ public:
 #  endif
         const auto thread = GetThreadHandle();
 
-        InitRPMallocThread();
         auto callstack = Callstack( depth );
 
         profiler.m_serialLock.lock();
@@ -445,7 +438,6 @@ public:
 #  endif
         const auto thread = GetThreadHandle();
 
-        InitRPMallocThread();
         auto callstack = Callstack( depth );
 
         profiler.m_serialLock.lock();
@@ -495,7 +487,6 @@ public:
 #  endif
         const auto thread = GetThreadHandle();
 
-        InitRPMallocThread();
         auto callstack = Callstack( depth );
 
         profiler.m_serialLock.lock();
@@ -518,7 +509,6 @@ public:
 #  endif
         const auto thread = GetThreadHandle();
 
-        InitRPMallocThread();
         auto callstack = Callstack( depth );
 
         profiler.m_serialLock.lock();

client/tracy_rpmalloc.cpp

@@ -130,7 +130,9 @@
 #  include <stdio.h>
 #  include <stdlib.h>
 #  if defined(__APPLE__)
-#    include <mach/mach_vm.h>
+#    if !TARGET_OS_IPHONE && !TARGET_OS_SIMULATOR
+#      include <mach/mach_vm.h>
+#    endif
 #    include <mach/vm_statistics.h>
 #    include <pthread.h>
 #  endif

common/TracyAlloc.hpp

@@ -4,15 +4,28 @@
 #include <stdlib.h>
 
 #ifdef TRACY_ENABLE
+#  include "TracyApi.h"
+#  include "TracyForceInline.hpp"
 #  include "../client/tracy_rpmalloc.hpp"
 #endif
 
 namespace tracy
 {
 
+#ifdef TRACY_ENABLE
+extern thread_local bool RpThreadInitDone;
+TRACY_API void InitRpmallocPlumbing();
+
+static tracy_force_inline void InitRpmalloc()
+{
+    if( !RpThreadInitDone ) InitRpmallocPlumbing();
+}
+#endif
+
 static inline void* tracy_malloc( size_t size )
 {
 #ifdef TRACY_ENABLE
+    InitRpmalloc();
     return rpmalloc( size );
 #else
     return malloc( size );
@@ -22,6 +35,7 @@ static inline void* tracy_malloc( size_t size )
 static inline void tracy_free( void* ptr )
 {
 #ifdef TRACY_ENABLE
+    InitRpmalloc();
     rpfree( ptr );
 #else
     free( ptr );
@@ -31,6 +45,7 @@ static inline void tracy_free( void* ptr )
 static inline void* tracy_realloc( void* ptr, size_t size )
 {
 #ifdef TRACY_ENABLE
+    InitRpmalloc();
     return rprealloc( ptr, size );
 #else
     return realloc( ptr, size );

common/TracySystem.cpp

@@ -96,7 +96,6 @@ struct ThreadNameData
     ThreadNameData* next;
 };
 std::atomic<ThreadNameData*>& GetThreadNameData();
-TRACY_API void InitRPMallocThread();
 #endif
 
 #ifdef _MSC_VER
@@ -161,7 +160,6 @@ TRACY_API void SetThreadName( const char* name )
 #endif
 #ifdef TRACY_ENABLE
     {
-        InitRPMallocThread();
         const auto sz = strlen( name );
         char* buf = (char*)tracy_malloc( sz+1 );
         memcpy( buf, name, sz );

common/TracyYield.hpp

@@ -7,7 +7,7 @@
 #  include <thread>
 #endif
 
-#include "../common/TracyForceInline.hpp"
+#include "TracyForceInline.hpp"
 
 namespace tracy
 {

profiler/build/win32/Tracy.vcxproj

@@ -185,6 +185,7 @@
     <ClInclude Include="..\..\..\common\TracyQueue.hpp" />
     <ClInclude Include="..\..\..\common\TracySocket.hpp" />
     <ClInclude Include="..\..\..\common\TracySystem.hpp" />
+    <ClInclude Include="..\..\..\common\TracyYield.hpp" />
     <ClInclude Include="..\..\..\common\tracy_lz4.hpp" />
     <ClInclude Include="..\..\..\common\tracy_lz4hc.hpp" />
     <ClInclude Include="..\..\..\imgui\imconfig.h" />
@@ -236,7 +237,6 @@
     <ClInclude Include="..\..\..\server\TracyView.hpp" />
     <ClInclude Include="..\..\..\server\TracyViewData.hpp" />
     <ClInclude Include="..\..\..\server\TracyWorker.hpp" />
-    <ClInclude Include="..\..\..\server\TracyYield.hpp" />
     <ClInclude Include="..\..\..\server\tracy_pdqsort.h" />
     <ClInclude Include="..\..\..\server\tracy_robin_hood.h" />
     <ClInclude Include="..\..\..\server\tracy_xxh3.h" />

profiler/build/win32/Tracy.vcxproj.filters

@@ -425,9 +425,6 @@
     <ClInclude Include="..\..\..\server\tracy_xxh3.h">
       <Filter>server</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\..\server\TracyYield.hpp">
-      <Filter>server</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\..\server\TracySort.hpp">
       <Filter>server</Filter>
     </ClInclude>
@@ -578,6 +575,9 @@
     <ClInclude Include="..\..\..\zstd\dictBuilder\divsufsort.h">
       <Filter>zstd\dictBuilder</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\common\TracyYield.hpp">
+      <Filter>common</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Natvis Include="DebugVis.natvis" />

server/TracyFileRead.hpp

@@ -22,7 +22,7 @@
 
 #include "TracyFileHeader.hpp"
 #include "TracyMmap.hpp"
-#include "TracyYield.hpp"
+#include "../common/TracyYield.hpp"
 #include "../common/tracy_lz4.hpp"
 #include "../common/TracyForceInline.hpp"
 #include "../zstd/zstd.h"

server/TracyWorker.cpp

@@ -26,13 +26,13 @@
 
 #include "../common/TracyProtocol.hpp"
 #include "../common/TracySystem.hpp"
+#include "../common/TracyYield.hpp"
 #include "TracyFileRead.hpp"
 #include "TracyFileWrite.hpp"
 #include "TracySort.hpp"
 #include "TracyTaskDispatch.hpp"
 #include "TracyVersion.hpp"
 #include "TracyWorker.hpp"
-#include "TracyYield.hpp"
 
 namespace tracy
 {