From bc1c4ee02709d1b0c877946058a98a1062e8f681 Mon Sep 17 00:00:00 2001
From: Sam Edwards <CFSworks@gmail.com>
Date: Wed, 1 Mar 2023 19:22:24 -0700
Subject: [PATCH] [dxvk] Periodically flush handles for long-running queries

Some apps (e.g. Halo: MCC) misuse D3D11 queries: they'll Begin(...) a
query, and then only afterward decide that they don't actually want
the results, returning the query object back to the engine's internal
query pool without first calling End(...). The query thus continues
to remain active until the engine decides to allocate the query for
another purpose... if it ever does.

While this is misuse of the D3D API and is definitely a bug in the
app, DXVK ought to tolerate this situation at least as well as D3D
itself apparently does. This change will try to free up Vulkan query
handles if they're in the available state, tracking their totals on
the DxvkGpuQuery object itself instead.

As a bonus, this change should also reduce the CPU time consumed by
repeated GetData(...) calls, since it avoids redundantly summing
query handle results.
---
 src/dxvk/dxvk_gpu_query.cpp | 52 +++++++++++++++++++++++++++----------
 src/dxvk/dxvk_gpu_query.h   | 10 ++++---
 2 files changed, 45 insertions(+), 17 deletions(-)

diff --git a/src/dxvk/dxvk_gpu_query.cpp b/src/dxvk/dxvk_gpu_query.cpp
index 426b58f6e70..df89b1a2c90 100644
--- a/src/dxvk/dxvk_gpu_query.cpp
+++ b/src/dxvk/dxvk_gpu_query.cpp
@@ -32,22 +32,12 @@ namespace dxvk {
 
 
   DxvkGpuQueryStatus DxvkGpuQuery::getData(DxvkQueryData& queryData) const {
-    queryData = DxvkQueryData();
-
     if (!m_ended)
       return DxvkGpuQueryStatus::Invalid;
-    
-    // Empty begin/end pair
-    if (!m_handle.queryPool)
-      return DxvkGpuQueryStatus::Available;
-    
+
     // Get query data from all associated handles
-    DxvkGpuQueryStatus status = getDataForHandle(queryData, m_handle);
+    DxvkGpuQueryStatus status = flushHandles();
 
-    for (size_t i = 0; i < m_handles.size()
-        && status == DxvkGpuQueryStatus::Available; i++)
-      status = getDataForHandle(queryData, m_handles[i]);
-    
     // Treat non-precise occlusion queries as available
     // if we already know the result will be non-zero
     if ((status == DxvkGpuQueryStatus::Pending)
@@ -55,7 +45,10 @@ namespace dxvk {
      && !(m_flags & VK_QUERY_CONTROL_PRECISE_BIT)
      && (queryData.occlusion.samplesPassed))
       status = DxvkGpuQueryStatus::Available;
-    
+
+    if (status == DxvkGpuQueryStatus::Available)
+      queryData = m_queryData;
+
     return status;
   }
 
@@ -69,6 +62,8 @@ namespace dxvk {
     for (const auto& handle : m_handles)
       cmd->trackGpuQuery(handle);
     m_handles.clear();
+
+    m_queryData = DxvkQueryData();
   }
 
   
@@ -78,6 +73,11 @@ namespace dxvk {
 
 
   void DxvkGpuQuery::addQueryHandle(const DxvkGpuQueryHandle& handle) {
+    // Some apps forget to End() their queries; instead of accumulating query
+    // handles indefinitely, try to clean them up periodically.
+    if (m_handles.size() >= 256)
+      flushHandles();
+
     if (m_handle.queryPool)
       m_handles.push_back(m_handle);
     
@@ -85,6 +85,32 @@ namespace dxvk {
   }
 
 
+  DxvkGpuQueryStatus DxvkGpuQuery::flushHandles() const {
+    // Consume handles in FIFO order, aggregating them into m_queryData
+    while (!m_handles.empty()) {
+      DxvkGpuQueryHandle handle = m_handles.front();
+
+      DxvkGpuQueryStatus status = getDataForHandle(m_queryData, handle);
+      if (status != DxvkGpuQueryStatus::Available)
+        return status;
+
+      handle.allocator->freeQuery(handle);
+      m_handles.pop_front();
+    }
+
+    if (m_handle.queryPool) {
+      DxvkGpuQueryStatus status = getDataForHandle(m_queryData, m_handle);
+      if (status != DxvkGpuQueryStatus::Available)
+        return status;
+
+      m_handle.allocator->freeQuery(m_handle);
+      m_handle = DxvkGpuQueryHandle();
+    }
+
+    return DxvkGpuQueryStatus::Available;
+  }
+
+
   DxvkGpuQueryStatus DxvkGpuQuery::getDataForHandle(
           DxvkQueryData&      queryData,
     const DxvkGpuQueryHandle& handle) const {
diff --git a/src/dxvk/dxvk_gpu_query.h b/src/dxvk/dxvk_gpu_query.h
index 0c45d120006..482a7cf904a 100644
--- a/src/dxvk/dxvk_gpu_query.h
+++ b/src/dxvk/dxvk_gpu_query.h
@@ -224,10 +224,12 @@ namespace dxvk {
     uint32_t            m_index;
     bool                m_ended;
 
-    DxvkGpuQueryHandle  m_handle;
-    
-    std::vector<DxvkGpuQueryHandle> m_handles;
-    
+    mutable DxvkQueryData                  m_queryData;
+    mutable DxvkGpuQueryHandle             m_handle;
+    mutable std::deque<DxvkGpuQueryHandle> m_handles;
+
+    DxvkGpuQueryStatus flushHandles() const;
+
     DxvkGpuQueryStatus getDataForHandle(
             DxvkQueryData&      queryData,
       const DxvkGpuQueryHandle& handle) const;