initial support for dumping a command buffer to a dot file

Author: bashbaug

intercept/src/cmdbuftracer.h

@@ -0,0 +1,168 @@
+/*
+// Copyright (c) 2025 Intel Corporation
+//
+// SPDX-License-Identifier: MIT
+*/
+#pragma once
+
+#include <atomic>
+#include <cinttypes>
+#include <map>
+#include <vector>
+#include <sstream>
+
+#include <stdint.h>
+
+#include "common.h"
+
+struct SCommandBufferTraceInfo
+{
+    std::ostringstream trace;
+
+    void    create(
+                cl_command_buffer_khr cmdbuf,
+                bool isInOrder)
+    {
+        queueIsInOrder = isInOrder;
+        trace << "digraph {\n";
+        trace << "  // " << (queueIsInOrder ? "in-order" : "out-of-order") << " command-buffer\n";
+    }
+
+    void    traceCommand(
+                cl_command_queue queue,
+                const char* cmd,
+                const std::string& tag,
+                cl_uint num_sync_points_in_wait_list,
+                const cl_sync_point_khr* sync_point_wait_list,
+                cl_sync_point_khr* sync_point)
+    {
+        SCommandBufferTraceId id =
+            sync_point == nullptr ?
+            makeInternalId() :
+            makeSyncPointId(*sync_point);
+
+        trace << "  " << (id.isInternal ? "internal" : "syncpoint") << id.id
+              << " [shape=oval, label=\"" << cmd;
+        if( !tag.empty() )
+        {
+            trace << "( " << tag << " )";
+        }
+        trace << "\"]\n";
+
+        for( cl_uint s = 0; s < num_sync_points_in_wait_list; s++ )
+        {
+            trace << "  syncpoint" << sync_point_wait_list[s]
+                  << " -> "
+                  << (id.isInternal ? "internal" : "syncpoint") << id.id
+                  << " // explicit dependency\n";
+        }
+
+        for( const auto& dep : implicitDeps )
+        {
+            trace << "  " << (dep.isInternal ? "internal" : "syncpoint") << dep.id
+                  << " -> "
+                  << (id.isInternal ? "internal" : "syncpoint") << id.id
+                  << " [style=dashed] // implicit dependency\n";
+        }
+
+        if( queueIsInOrder )
+        {
+            implicitDeps.clear();
+            implicitDeps.push_back(id);
+        }
+        else
+        {
+            outstandingIds.push_back(id);
+        }
+    }
+
+    void    traceBarrier(
+                cl_command_queue queue,
+                const char* cmd,
+                cl_uint num_sync_points_in_wait_list,
+                const cl_sync_point_khr* sync_point_wait_list,
+                cl_sync_point_khr* sync_point)
+    {
+        SCommandBufferTraceId id =
+            sync_point == nullptr ?
+            makeInternalId() :
+            makeSyncPointId(*sync_point);
+
+        trace << "  " << (id.isInternal ? "internal" : "syncpoint") << id.id
+              << " [shape=octagon, label=\"" << cmd << "\"]\n";
+
+        // If there is a sync point wait list, then the barrier depends on all
+        // of the commands in the sync point wait list. Otherwise, the barrier
+        // depends on all of the outstanding ids.
+        if( num_sync_points_in_wait_list > 0 )
+        {
+            for( cl_uint s = 0; s < num_sync_points_in_wait_list; s++ )
+            {
+                trace << "  syncpoint" << sync_point_wait_list[s]
+                      << " -> "
+                      << (id.isInternal ? "internal" : "syncpoint") << id.id
+                      << " // explicit dependency\n";
+            }
+        }
+        else
+        {
+            for( const auto& dep : outstandingIds )
+            {
+                trace << "  " << (dep.isInternal ? "internal" : "syncpoint") << dep.id
+                      << " -> "
+                      << (id.isInternal ? "internal" : "syncpoint") << id.id
+                      << " [style=dotted] // barrier dependency\n";
+            }
+            outstandingIds.clear();
+        }
+
+        // Add the implicit dependencies.
+        for( const auto& dep : implicitDeps )
+        {
+            trace << "  " << (dep.isInternal ? "internal" : "syncpoint") << dep.id
+                  << " -> "
+                  << (id.isInternal ? "internal" : "syncpoint") << id.id
+                  << " [style=dashed] // implicit dependency\n";
+        }
+
+        // Now, the only implicit dependency that remains is this barrier.
+        implicitDeps.clear();
+        implicitDeps.push_back(id);
+    }
+
+    void finalize()
+    {
+        trace << "}\n";
+    }
+
+private:
+    struct SCommandBufferTraceId
+    {
+        bool        isInternal = false;
+        uint32_t    id = 0;
+    };
+
+    std::atomic<uint32_t> nextInternalId;
+
+    bool    queueIsInOrder;
+
+    std::vector<SCommandBufferTraceId>  implicitDeps;
+    std::vector<SCommandBufferTraceId>  outstandingIds;
+
+    SCommandBufferTraceId makeInternalId()
+    {
+        SCommandBufferTraceId id;
+        id.isInternal = true;
+        id.id = nextInternalId.fetch_add(1, std::memory_order_relaxed);
+        return id;
+    }
+
+    SCommandBufferTraceId makeSyncPointId(
+        cl_sync_point_khr sync_point)
+    {
+        SCommandBufferTraceId id;
+        id.isInternal = false;
+        id.id = sync_point;
+        return id;
+    }
+};

intercept/src/controls.h

@@ -120,6 +120,9 @@ CLI_CONTROL( std::string,   SPIRVDis,                               "spirv-dis",
 CLI_CONTROL( std::string,   DefaultOptions,                         "-cc1 -x cl -cl-std=CL1.2 -D__OPENCL_C_VERSION__=120 -D__OPENCL_VERSION__=120 -emit-spirv -triple=spir", "This is the list of options that is implicitly passed to CLANG to build a non-OpenCL 2.0 SPIR-V module.  Any application-provided build options will be appended to these build options." )
 CLI_CONTROL( std::string,   OpenCL2Options,                         "-cc1 -x cl -cl-std=CL2.0 -D__OPENCL_C_VERSION__=200 -D__OPENCL_VERSION__=200 -emit-spirv -triple=spir", "This is the list of options that is implicitly passed to CLANG to build an OpenCL 2.0 SPIR-V module.  Any application-provided build options will be appended to these build options." )
 
+CLI_CONTROL_SEPARATOR( Controls for Dumping Command Buffers: )
+CLI_CONTROL( bool,          DumpCommandBuffers,                     false, "TODO" )
+
 CLI_CONTROL_SEPARATOR( Controls for Dumping and Injecting Buffers and Images: )
 CLI_CONTROL( bool,          DumpBufferHashes,                       false, "If set to a nonzero value, the Intercept Layer for OpenCL Applications will dump hashes of a buffer, SVM, or USM allocation rather than the full contents of the buffer.  This can be useful to identify which kernel enqueues generate different results without requiring a large amount of disk space." )
 CLI_CONTROL( bool,          DumpImageHashes,                        false, "If set to a nonzero value, the Intercept Layer for OpenCL Applications will dump hashes of an image rather than the full contents of the image.  This can be useful to identify which kernel enqueues generate different results without requiring a large amount of disk space." )

intercept/src/dispatch.cpp

@@ -10469,6 +10469,7 @@ CL_API_ENTRY cl_command_buffer_khr CL_API_CALL clCreateCommandBufferKHR(
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( errcode_ret[0] );
             ADD_OBJECT_ALLOCATION( retVal );
+            TRACE_COMMAND_BUFFER_CREATE( retVal, num_queues, queues );
             CALL_LOGGING_EXIT( errcode_ret[0], "returned %p", retVal );
 
             if( retVal != NULL )
@@ -10509,6 +10510,7 @@ CL_API_ENTRY cl_int CL_API_CALL clFinalizeCommandBufferKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_FINALIZE( retVal, command_buffer );
             CALL_LOGGING_EXIT( retVal );
 
             return retVal;
@@ -10715,6 +10717,12 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandBarrierWithWaitListKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_BARRIER(
+                retVal,
+                command_buffer,
+                num_sync_points_in_wait_list,
+                sync_point_wait_list,
+                sync_point );
             CALL_LOGGING_EXIT_SYNC_POINT( retVal, sync_point );
             ADD_MUTABLE_COMMAND( mutable_handle, command_buffer );
 
@@ -10779,6 +10787,12 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandCopyBufferKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_COMMAND(
+                retVal,
+                command_buffer,
+                num_sync_points_in_wait_list,
+                sync_point_wait_list,
+                sync_point );
             CALL_LOGGING_EXIT_SYNC_POINT( retVal, sync_point );
             ADD_MUTABLE_COMMAND( mutable_handle, command_buffer );
 
@@ -10851,6 +10865,12 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandCopyBufferRectKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_COMMAND(
+                retVal,
+                command_buffer,
+                num_sync_points_in_wait_list,
+                sync_point_wait_list,
+                sync_point );
             CALL_LOGGING_EXIT_SYNC_POINT( retVal, sync_point );
             ADD_MUTABLE_COMMAND( mutable_handle, command_buffer );
 
@@ -10915,6 +10935,12 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandCopyBufferToImageKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_COMMAND(
+                retVal,
+                command_buffer,
+                num_sync_points_in_wait_list,
+                sync_point_wait_list,
+                sync_point );
             CALL_LOGGING_EXIT_SYNC_POINT( retVal, sync_point );
             ADD_MUTABLE_COMMAND( mutable_handle, command_buffer );
 
@@ -10979,6 +11005,12 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandCopyImageKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_COMMAND(
+                retVal,
+                command_buffer,
+                num_sync_points_in_wait_list,
+                sync_point_wait_list,
+                sync_point );
             CALL_LOGGING_EXIT_SYNC_POINT( retVal, sync_point );
             ADD_MUTABLE_COMMAND( mutable_handle, command_buffer );
 
@@ -11045,6 +11077,12 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandCopyImageToBufferKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_COMMAND(
+                retVal,
+                command_buffer,
+                num_sync_points_in_wait_list,
+                sync_point_wait_list,
+                sync_point );
             CALL_LOGGING_EXIT_SYNC_POINT( retVal, sync_point );
             ADD_MUTABLE_COMMAND( mutable_handle, command_buffer );
 
@@ -11110,6 +11148,12 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandFillBufferKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_COMMAND(
+                retVal,
+                command_buffer,
+                num_sync_points_in_wait_list,
+                sync_point_wait_list,
+                sync_point );
             CALL_LOGGING_EXIT_SYNC_POINT( retVal, sync_point );
             ADD_MUTABLE_COMMAND( mutable_handle, command_buffer );
 
@@ -11173,6 +11217,12 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandFillImageKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_COMMAND(
+                retVal,
+                command_buffer,
+                num_sync_points_in_wait_list,
+                sync_point_wait_list,
+                sync_point );
             CALL_LOGGING_EXIT_SYNC_POINT( retVal, sync_point );
             ADD_MUTABLE_COMMAND( mutable_handle, command_buffer );
 
@@ -11236,6 +11286,12 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandSVMMemcpyKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_COMMAND(
+                retVal,
+                command_buffer,
+                num_sync_points_in_wait_list,
+                sync_point_wait_list,
+                sync_point );
             CALL_LOGGING_EXIT_SYNC_POINT( retVal, sync_point );
             ADD_MUTABLE_COMMAND( mutable_handle, command_buffer );
 
@@ -11301,6 +11357,12 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandSVMMemFillKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_COMMAND(
+                retVal,
+                command_buffer,
+                num_sync_points_in_wait_list,
+                sync_point_wait_list,
+                sync_point );
             CALL_LOGGING_EXIT_SYNC_POINT( retVal, sync_point );
             ADD_MUTABLE_COMMAND( mutable_handle, command_buffer );
 
@@ -11362,6 +11424,13 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandNDRangeKernelKHR(
                 command_queue,
                 kernel,
                 argsString.c_str() );
+            GET_TIMING_TAGS_COMMAND_BUFFER_KERNEL(
+                command_buffer,
+                kernel,
+                work_dim,
+                global_work_offset,
+                global_work_size,
+                local_work_size );
             HOST_PERFORMANCE_TIMING_START();
 
             cl_int  retVal = dispatchX.clCommandNDRangeKernelKHR(
@@ -11380,6 +11449,12 @@ CL_API_ENTRY cl_int CL_API_CALL clCommandNDRangeKernelKHR(
 
             HOST_PERFORMANCE_TIMING_END();
             CHECK_ERROR( retVal );
+            TRACE_COMMAND_BUFFER_COMMAND_WITH_TAG(
+                retVal,
+                command_buffer,
+                num_sync_points_in_wait_list,
+                sync_point_wait_list,
+                sync_point );
             CALL_LOGGING_EXIT_SYNC_POINT( retVal, sync_point );
             ADD_MUTABLE_COMMAND_NDRANGE( mutable_handle, command_buffer, work_dim );