Merge pull request #1097 from erieaton-amd/rocprofv3-2

Collect dispatches and counter values with Rocprofv3

Author: wolfpld

CMakeLists.txt

@@ -33,6 +33,7 @@ else()
 endif()
 
 find_package(Threads REQUIRED)
+find_package(rocprofiler-sdk PATHS "/opt/rocm/lib/cmake")
 
 set(TRACY_PUBLIC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/public)
 
@@ -56,6 +57,10 @@ target_link_libraries(
         Threads::Threads
         ${CMAKE_DL_LIBS}
 )
+if(rocprofiler-sdk_FOUND)
+    target_compile_definitions(TracyClient PUBLIC TRACY_ROCPROF)
+    target_link_libraries(TracyClient PUBLIC rocprofiler-sdk::rocprofiler-sdk)
+endif()
 
 if(TRACY_Fortran)
     add_library(TracyClientF90 ${TRACY_VISIBILITY} "${TRACY_PUBLIC_DIR}/TracyClient.F90")
@@ -142,6 +147,10 @@ set_option(TRACY_VERBOSE "[advanced] Verbose output from the profiler" OFF)
 mark_as_advanced(TRACY_VERBOSE)
 set_option(TRACY_DEMANGLE "[advanced] Don't use default demangling function - You'll need to provide your own" OFF)
 mark_as_advanced(TRACY_DEMANGLE)
+if(rocprofiler-sdk_FOUND)
+    set_option(TRACY_ROCPROF_CALIBRATION "[advanced] Use continuous calibration of the Rocprof GPU time." OFF)
+    mark_as_advanced(TRACY_ROCPROF_CALIBRATION)
+endif()
 
 # handle incompatible combinations
 if(TRACY_MANUAL_LIFETIME AND NOT TRACY_DELAYED_INIT)

manual/tracy.tex

@@ -1707,6 +1707,35 @@ \subsubsection{CUDA}
 
 To stop profiling, call the \texttt{TracyCUDAStopProfiling(ctx)} macro.
 
+\subsubsection{ROCm}
+
+On Linux, if rocprofiler-sdk is installed, tracy can automatically trace GPU dispatches and collect
+performance counter values. If CMake can't find rocprofiler-sdk, you can set the CMake variable
+\texttt{rocprofiler-sdk\_DIR} to point it at the correct module directory. Use the
+\texttt{TRACY\_ROCPROF\_COUNTERS} environment variable with the desired counters separated by commas
+to control what values are collected. The results will appear for each dispatch in the tool tip and
+zone detail window. Results are summed across dimensions. You can get a list of the counters
+available for your hardware with this command:
+\begin{lstlisting}[language=sh]
+rocprofv3 -L
+\end{lstlisting}
+
+\subparagraph{Troubleshooting}
+\begin{itemize}
+\item If you are taking very long captures, you may see drift between the GPU and
+  CPU timelines. This may be mitigated by setting the CMake variable
+  \texttt{TRACY\_ROCPROF\_CALIBRATION}, which will refresh the time synchronization about every
+  second.
+\item The timeline drift may also be affected by network time synchronization, in which case the
+  drift will be reduced by disabling that, with the advantage that there is no application performance
+  cost.
+\item On some GPUs, you will need to change the the performance level to see non-zero results from
+  some counters. Use this command:
+\begin{lstlisting}[language=sh]
+sudo amd-smi set -g 0 -l stable_std
+\end{lstlisting}
+\end{itemize}
+
 \subsubsection{Multiple zones in one scope}
 
 Putting more than one GPU zone macro in a single scope features the same issue as with the \texttt{ZoneScoped} macros, described in section~\ref{multizone} (but this time the variable name is \texttt{\_\_\_tracy\_gpu\_zone}).

profiler/src/profiler/TracyView.hpp

@@ -46,7 +46,8 @@ constexpr const char* GpuContextNames[] = {
     "Direct3D 11",
     "Metal",
     "Custom",
-    "CUDA"
+    "CUDA",
+    "Rocprof"
 };
 
 struct MemoryPage;

profiler/src/profiler/TracyView_ZoneInfo.cpp

@@ -1580,6 +1580,21 @@ void View::DrawGpuInfoWindow()
             TextFocused( "Delay to execution:", TimeToString( AdjustGpuTime( ev.GpuStart(), begin, drift ) - ev.CpuStart() ) );
         }
 
+        if( ctx->notes.contains( ev.query_id ) )
+        {
+            for( auto& p : ctx->notes.at( ev.query_id ) )
+            {
+                if( ctx->noteNames.count( p.first ) )
+                {
+                    TextFocused( m_worker.GetString( ctx->noteNames.at( p.first ) ), RealToString( p.second ) );
+                }
+                else
+                {
+                    TextFocused( RealToString( p.first ), RealToString( p.second ) );
+                }
+            }
+        }
+
         ImGui::Separator();
 
         std::vector<const GpuEvent*> zoneTrace;
@@ -2047,6 +2062,21 @@ void View::ZoneTooltip( const GpuEvent& ev )
         TextFocused( "Delay to execution:", TimeToString( AdjustGpuTime( ev.GpuStart(), begin, drift ) - ev.CpuStart() ) );
     }
 
+    if( ctx->notes.contains( ev.query_id ) )
+    {
+        for( auto& p : ctx->notes.at( ev.query_id ) )
+        {
+            if( ctx->noteNames.count( p.first ) )
+            {
+                TextFocused( m_worker.GetString( ctx->noteNames.at( p.first ) ), RealToString( p.second ) );
+            }
+            else
+            {
+                TextFocused( RealToString( p.first ), RealToString( p.second ) );
+            }
+        }
+    }
+
     ImGui::EndTooltip();
 }
 

public/TracyClient.cpp

@@ -32,6 +32,10 @@
 #include "client/TracyOverride.cpp"
 #include "client/TracyKCore.cpp"
 
+#ifdef TRACY_ROCPROF
+#  include "client/TracyRocprof.cpp"
+#endif
+
 #if defined(TRACY_HAS_CALLSTACK)
 #  if TRACY_HAS_CALLSTACK == 2 || TRACY_HAS_CALLSTACK == 3 || TRACY_HAS_CALLSTACK == 4 || TRACY_HAS_CALLSTACK == 6
 #    include "libbacktrace/alloc.cpp"

public/client/TracyProfiler.cpp

@@ -2383,6 +2383,10 @@ static void FreeAssociatedMemory( const QueueItem& item )
         tracy_free( (void*)ptr );
         break;
 #endif
+    case QueueType::GpuAnnotationName:
+        ptr = MemRead<uint64_t>( &item.gpuAnnotationNameFat.ptr );
+        tracy_free( (void*)ptr );
+        break;
 #ifdef TRACY_ON_DEMAND
     case QueueType::MessageAppInfo:
     case QueueType::GpuContextName:
@@ -2598,6 +2602,12 @@ Profiler::DequeueStatus Profiler::Dequeue( moodycamel::ConsumerToken& token )
                         tracy_free_fast( (void*)ptr );
 #endif
                         break;
+                    case QueueType::GpuAnnotationName:
+                        ptr = MemRead<uint64_t>( &item->gpuAnnotationNameFat.ptr );
+                        size = MemRead<uint16_t>( &item->gpuAnnotationNameFat.size );
+                        SendSingleString( (const char*)ptr, size );
+                        tracy_free_fast( (void*)ptr );
+                        break;
                     case QueueType::PlotDataInt:
                     case QueueType::PlotDataFloat:
                     case QueueType::PlotDataDouble:
@@ -2956,6 +2966,14 @@ Profiler::DequeueStatus Profiler::DequeueSerial()
 #endif
                     break;
                 }
+                case QueueType::GpuAnnotationName:
+                {
+                    ptr = MemRead<uint64_t>( &item->gpuAnnotationNameFat.ptr );
+                    uint16_t size = MemRead<uint16_t>( &item->gpuAnnotationNameFat.size );
+                    SendSingleString( (const char*)ptr, size );
+                    tracy_free_fast( (void*)ptr );
+                    break;
+                }
 #ifdef TRACY_FIBERS
                 case QueueType::ZoneBegin:
                 case QueueType::ZoneBeginCallstack: