Better DRED output for DX12 device-removal crashes

trinity/TriDevice12.cpp

@@ -212,13 +212,23 @@ void TriDevice::HandleRenderTick( Be::Time realTime, Be::Time simTime )
 
 				CCP_LOGERR( "[DRED] Last tracked GPU operations:" );
 				std::map<UINT, std::wstring> contextStrings;
-				D3D12_AUTO_BREADCRUMB_NODE1 const* pNode = dredAutoBreadcrumbsOutput.pHeadAutoBreadcrumbNode;
-				while( pNode && pNode->pLastBreadcrumbValue )
+				for( D3D12_AUTO_BREADCRUMB_NODE1 const* pNode = dredAutoBreadcrumbsOutput.pHeadAutoBreadcrumbNode; pNode; pNode = pNode->pNext )
 				{
+					if( !pNode->pLastBreadcrumbValue )
+					{
+						continue;
+					}
 					UINT lastCompletedOp = *pNode->pLastBreadcrumbValue;
-					if( lastCompletedOp != (int)pNode->BreadcrumbCount && lastCompletedOp != 0 )
+					// Only lists in flight at removal time; 0 = never started, BreadcrumbCount = fully retired
+					if( lastCompletedOp != pNode->BreadcrumbCount && lastCompletedOp != 0 )
 					{
-						CCP_LOGERR( "[DRED] Commandlist completed %d of %d commands", lastCompletedOp, pNode->BreadcrumbCount );
+						CCP_LOGERR( "[DRED] Commandlist '%s' (%p) on queue '%s' completed %d of %d commands (%d contexts)",
+									pNode->pCommandListDebugNameA ? pNode->pCommandListDebugNameA : "<unnamed>",
+									pNode->pCommandList,
+									pNode->pCommandQueueDebugNameA ? pNode->pCommandQueueDebugNameA : "<unnamed>",
+									lastCompletedOp,
+									pNode->BreadcrumbCount,
+									pNode->BreadcrumbContextsCount );
 
 						UINT firstOp = lastCompletedOp > 100 ? lastCompletedOp - 100 : 0;
 						UINT lastOp = std::min<UINT>( lastCompletedOp + 20, UINT( pNode->BreadcrumbCount ) - 1 );
@@ -227,7 +237,10 @@ void TriDevice::HandleRenderTick( Be::Time realTime, Be::Time simTime )
 						for( UINT breadcrumbContext = 0; breadcrumbContext < pNode->BreadcrumbContextsCount; ++breadcrumbContext )
 						{
 							const D3D12_DRED_BREADCRUMB_CONTEXT& context = pNode->pBreadcrumbContexts[breadcrumbContext];
-							contextStrings[context.BreadcrumbIndex] = context.pContextString;
+							if( context.BreadcrumbIndex >= firstOp && context.BreadcrumbIndex <= lastOp )
+							{
+								contextStrings[context.BreadcrumbIndex] = context.pContextString;
+							}
 						}
 
 						for( UINT op = firstOp; op <= lastOp; ++op )
@@ -241,28 +254,24 @@ void TriDevice::HandleRenderTick( Be::Time realTime, Be::Time simTime )
 								contextString = it->second;
 							}
 
-							char const* opName = DredBreadcrumbOpName( breadcrumbOp );
-							CCP_LOGERR( "\tOp: %d, %s%ls%s", op, opName, contextString.c_str(), ( op + 1 == lastCompletedOp ) ? " - Last completed" : "" );
+							// Markers with a context string are our own annotations, not GPU work
+							char const* opName = breadcrumbOp == D3D12_AUTO_BREADCRUMB_OP_SETMARKER && !contextString.empty() ? "[Trinity]" : DredBreadcrumbOpName( breadcrumbOp );
+							char const* status = op == lastCompletedOp ? " - IN FLIGHT" : ( op + 1 == lastCompletedOp ) ? " - Last completed" :
+																														  "";
+							CCP_LOGERR( "\tOp: %d, %s %ls%s", op, opName, contextString.c_str(), status );
 						}
 					}
-					pNode = pNode->pNext;
 				}
 			}
 			if( SUCCEEDED( pDred->GetPageFaultAllocationOutput1( &dredPageFaultOutput ) ) )
 			{
 				for( auto node = dredPageFaultOutput.pHeadExistingAllocationNode; node != nullptr; node = node->pNext )
 				{
-					if( node->ObjectNameW )
-					{
-						CCP_LOGERR( "Page Fault Allocation on: %ls", node->ObjectNameW );
-					}
+					CCP_LOGERR( "Page Fault Allocation on: %ls (type %d)", node->ObjectNameW ? node->ObjectNameW : L"<unnamed>", node->AllocationType );
 				}
 				for( auto node = dredPageFaultOutput.pHeadRecentFreedAllocationNode; node != nullptr; node = node->pNext )
 				{
-					if( node->ObjectNameW )
-					{
-						CCP_LOGERR( "Page Fault Free on: %ls", node->ObjectNameW );
-					}
+					CCP_LOGERR( "Page Fault Free on: %ls (type %d)", node->ObjectNameW ? node->ObjectNameW : L"<unnamed>", node->AllocationType );
 				}
 			}
 		}

trinityal/Tr2RenderContextEnum.cpp

@@ -6,6 +6,7 @@
 #include "ALLog.h"
 
 bool g_requestDebugMarkers = false;
+bool g_dredBreadcrumbsEnabled = false;
 bool g_skipNvidiaStreamline = false;
 bool g_brokenMacOSNvidiaDrivers = false;
 

trinityal/dx12/Tr2PrimaryRenderContextDx12.cpp

@@ -16,6 +16,7 @@
 extern bool g_requestDeviceDebugLayer;
 extern bool g_requestDebugMarkers;
 extern bool g_requestDred;
+extern bool g_dredBreadcrumbsEnabled;
 bool g_gatherPipelineStatistics = false;
 extern ICrashReporter* TrinityALCrashes;
 
@@ -45,6 +46,9 @@ bool EnableDred()
 		// Turn on auto-breadcrumbs and page fault reporting.
 		pDredSettings->SetAutoBreadcrumbsEnablement( D3D12_DRED_ENABLEMENT_FORCED_ON );
 		pDredSettings->SetPageFaultEnablement( D3D12_DRED_ENABLEMENT_FORCED_ON );
+		// Capture SetMarker/BeginEvent strings alongside the breadcrumb ops
+		pDredSettings->SetBreadcrumbContextEnablement( D3D12_DRED_ENABLEMENT_FORCED_ON );
+		g_dredBreadcrumbsEnabled = true;
 		return true;
 	}
 	return false;
@@ -410,6 +414,7 @@ ALResult Tr2PrimaryRenderContextAL::CreateDevice(
 	desc.NodeMask = 0;
 
 	CR_RETURN_HR( CreateCommandQueue( device, &desc, commandQueue ) );
+	TrinityALImpl::SetDebugName( commandQueue, "PrimaryDirectQueue" );
 
 	const bool isWindowless = ( focusWindow == 0 ) && presentationParameters.software;
 

trinityal/dx12/Tr2RenderContextDx12.cpp

@@ -17,6 +17,7 @@
 #include "util/AmdExtDevice.h"
 
 extern bool g_requestDebugMarkers;
+extern bool g_dredBreadcrumbsEnabled;
 
 CCP_STATS_DECLARE( primitiveCount, "Trinity/AL/primitiveCount", true, CST_COUNTER_HIGH, "Primitive count in DrawPrimitive calls." );
 CCP_STATS_DECLARE( vertexCount, "Trinity/AL/vertexCount", true, CST_COUNTER_HIGH, "Vertex count in DrawPrimitive calls." );
@@ -139,6 +140,7 @@ ALResult Tr2RenderContextAL::CreateDx12( ID3D12CommandAllocator* commandAllocato
 		commandAllocator,
 		nullptr,
 		IID_PPV_ARGS( &m_commandList ) ) );
+	TrinityALImpl::SetDebugName( m_commandList, "Tr2RenderContext CommandList" );
 	CR_RETURN_HR( m_commandList->Close() );
 	m_commandList.QueryInterface( &m_commandList2 );
 
@@ -1568,6 +1570,25 @@ void Tr2RenderContextAL::ResetDx12()
 	m_srgbWriteEnable = false;
 }
 
+namespace
+{
+// DRED breadcrumb contexts are only captured from PIX3-blob markers (Metadata=2,
+// WinPixEventRuntime encoding); legacy ANSI/unicode markers are ignored
+void SetDredMarker( ID3D12GraphicsCommandList* commandList, const char* text )
+{
+	constexpr UINT64 PIXEvent_SetMarker_NoArgs = 0x008;
+	UINT64 blob[64];
+	blob[0] = PIXEvent_SetMarker_NoArgs << 10; // timestamp 0, event type
+	blob[1] = 0xFF000000; // ARGB color
+	blob[2] = ( UINT64( 8 ) << 55 ) | ( UINT64( 1 ) << 54 ); // string info: copy chunk 8, isANSI
+	size_t lenBytes = strlen( text ) + 1;
+	size_t qwords = std::min( ( lenBytes + 7 ) / 8, size_t( 60 ) );
+	memset( &blob[3], 0, qwords * 8 );
+	memcpy( &blob[3], text, std::min( lenBytes, qwords * 8 - 1 ) );
+	commandList->SetMarker( 2, blob, UINT( ( 3 + qwords ) * 8 ) );
+}
+}
+
 void Tr2RenderContextAL::AddGpuMarker( const char* marker )
 {
 	m_ownerDevice->GetMarkerBuffer().PutMarker( m_commandList2, marker );
@@ -1576,6 +1597,10 @@ void Tr2RenderContextAL::AddGpuMarker( const char* marker )
 	{
 		crashTracker->PutMarker( m_commandList2, marker );
 	}
+	if( g_dredBreadcrumbsEnabled )
+	{
+		SetDredMarker( m_commandList, marker );
+	}
 }
 
 void Tr2RenderContextAL::PushGpuMarker( const char* marker )
@@ -1720,10 +1745,109 @@ void Tr2RenderContextAL::ResourceBarrierDx12( const D3D12_RESOURCE_BARRIER& barr
 	ResourceBarrierDx12( 1, &barrier );
 }
 
+namespace
+{
+// snprintf returns the untruncated length; clamp so pos never passes the terminator
+size_t AdvanceFormatPos( size_t pos, int written, size_t size )
+{
+	return written < 0 || pos + size_t( written ) >= size ? size - 1 : pos + size_t( written );
+}
+
+const char* FormatResourceStates( char* buf, size_t size, D3D12_RESOURCE_STATES states )
+{
+	if( states == D3D12_RESOURCE_STATE_COMMON )
+	{
+		return "COMMON";
+	}
+	if( states == D3D12_RESOURCE_STATE_GENERIC_READ )
+	{
+		return "GENERIC_READ";
+	}
+	static const struct
+	{
+		D3D12_RESOURCE_STATES bit;
+		const char* name;
+	} s_stateNames[] = {
+		{ D3D12_RESOURCE_STATE_VERTEX_AND_CONSTANT_BUFFER, "VB_CB" },
+		{ D3D12_RESOURCE_STATE_INDEX_BUFFER, "IB" },
+		{ D3D12_RESOURCE_STATE_RENDER_TARGET, "RT" },
+		{ D3D12_RESOURCE_STATE_UNORDERED_ACCESS, "UAV" },
+		{ D3D12_RESOURCE_STATE_DEPTH_WRITE, "DEPTH_W" },
+		{ D3D12_RESOURCE_STATE_DEPTH_READ, "DEPTH_R" },
+		{ D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE, "SRV_NONPX" },
+		{ D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE, "SRV_PX" },
+		{ D3D12_RESOURCE_STATE_STREAM_OUT, "STREAM_OUT" },
+		{ D3D12_RESOURCE_STATE_INDIRECT_ARGUMENT, "INDIRECT" },
+		{ D3D12_RESOURCE_STATE_COPY_DEST, "COPY_DST" },
+		{ D3D12_RESOURCE_STATE_COPY_SOURCE, "COPY_SRC" },
+		{ D3D12_RESOURCE_STATE_RESOLVE_DEST, "RESOLVE_DST" },
+		{ D3D12_RESOURCE_STATE_RESOLVE_SOURCE, "RESOLVE_SRC" },
+		{ D3D12_RESOURCE_STATE_RAYTRACING_ACCELERATION_STRUCTURE, "RTAS" },
+	};
+	size_t pos = 0;
+	UINT remaining = UINT( states );
+	for( const auto& state : s_stateNames )
+	{
+		if( ( remaining & UINT( state.bit ) ) == UINT( state.bit ) )
+		{
+			pos = AdvanceFormatPos( pos, snprintf( buf + pos, size - pos, "%s%s", pos ? "|" : "", state.name ), size );
+			remaining &= ~UINT( state.bit );
+		}
+	}
+	if( remaining )
+	{
+		snprintf( buf + pos, size - pos, "%s0x%x", pos ? "|" : "", remaining );
+	}
+	return buf;
+}
+
+// Recorded immediately before each ResourceBarrier so the DRED breadcrumb context
+// identifies which resources/states the otherwise anonymous RESOURCEBARRIER op contains
+void EmitBarrierBreadcrumb( ID3D12GraphicsCommandList* commandList, const D3D12_RESOURCE_BARRIER* barriers, size_t count )
+{
+	if( !g_dredBreadcrumbsEnabled )
+	{
+		return;
+	}
+	char buf[512];
+	size_t pos = AdvanceFormatPos( 0, snprintf( buf, sizeof( buf ), "[Barrier]" ), sizeof( buf ) );
+	for( size_t i = 0; i < count && pos < sizeof( buf ) - 1; ++i )
+	{
+		const auto& barrier = barriers[i];
+		ID3D12Resource* resource = barrier.Type == D3D12_RESOURCE_BARRIER_TYPE_UAV ? barrier.UAV.pResource : barrier.Type == D3D12_RESOURCE_BARRIER_TYPE_ALIASING ? barrier.Aliasing.pResourceAfter :
+																																									barrier.Transition.pResource;
+		char name[128];
+		UINT nameSize = sizeof( name ) - 1;
+		if( !resource || FAILED( resource->GetPrivateData( WKPDID_D3DDebugObjectName, &nameSize, name ) ) || nameSize >= sizeof( name ) )
+		{
+			nameSize = UINT( snprintf( name, sizeof( name ), "%p", resource ) );
+		}
+		name[nameSize] = 0;
+		if( barrier.Type == D3D12_RESOURCE_BARRIER_TYPE_TRANSITION )
+		{
+			char before[96], after[96];
+			const char* beforeStates = FormatResourceStates( before, sizeof( before ), barrier.Transition.StateBefore );
+			const char* afterStates = FormatResourceStates( after, sizeof( after ), barrier.Transition.StateAfter );
+			pos = AdvanceFormatPos( pos, snprintf( buf + pos, sizeof( buf ) - pos, " %s(%s->%s)", name, beforeStates, afterStates ), sizeof( buf ) );
+		}
+		else if( barrier.Type == D3D12_RESOURCE_BARRIER_TYPE_UAV )
+		{
+			pos = AdvanceFormatPos( pos, snprintf( buf + pos, sizeof( buf ) - pos, " UAV(%s)", name ), sizeof( buf ) );
+		}
+		else
+		{
+			pos = AdvanceFormatPos( pos, snprintf( buf + pos, sizeof( buf ) - pos, " Alias(%s)", name ), sizeof( buf ) );
+		}
+	}
+	SetDredMarker( commandList, buf );
+}
+}
+
 void Tr2RenderContextAL::FlushBarriersDx12()
 {
 	if( !m_barriers.empty() )
 	{
+		EmitBarrierBreadcrumb( m_commandList, m_barriers.data(), m_barriers.size() );
 		m_commandList->ResourceBarrier( UINT( m_barriers.size() ), m_barriers.data() );
 		m_barriers.clear();
 	}
@@ -1771,6 +1895,7 @@ void Tr2RenderContextAL::FlushBarriersDx12( size_t count, ID3D12Resource** resou
 		}
 		if( barrierCount )
 		{
+			EmitBarrierBreadcrumb( m_commandList, barriers, barrierCount );
 			m_commandList->ResourceBarrier( UINT( barrierCount ), barriers );
 		}
 	}
@@ -1797,6 +1922,7 @@ void Tr2RenderContextAL::FlushBarriersDx12( size_t count, ID3D12Resource** resou
 		}
 		if( !barriers.empty() )
 		{
+			EmitBarrierBreadcrumb( m_commandList, barriers.data(), barriers.size() );
 			m_commandList->ResourceBarrier( UINT( barriers.size() ), barriers.data() );
 		}
 	}