Add a 'garbage collection' for cached state objects

* In extreme cases a program can create lots and lots of unique short-
  lived state objects, we cache them all and run out of the 4096 state
  objects that can be around, then further creates fail out.
* To handle this, if we have 4000 total state objects, we release any
  that are purely being held by us (not the program). This isn't an
  ideal solution but it's quite simple and non-invasive, and only has
  a slight impact on rare Create calls.
* Most programs will either create a few up-front (as intended), or at
  worst create a few dozen or even few hundred then re-use them as they
  are cached.

renderdoc/driver/d3d11/d3d11_device.cpp

@@ -3156,6 +3156,39 @@ HRESULT WrappedID3D11Device::Present(IDXGISwapChain *swap, UINT SyncInterval, UI
 	return S_OK;
 }
 
+void WrappedID3D11Device::CachedObjectsGarbageCollect()
+{
+	// 4000 is a fairly arbitrary number, chosen to make sure this garbage
+	// collection kicks in as rarely as possible (4000 is a *lot* of unique
+	// state objects to have), while still meaning that we'll never
+	// accidentally cause a state object to fail to create because the app
+	// expects only N to be alive but we're caching M more causing M+N>4096
+	if(m_CachedStateObjects.size() < 4000) return;
+
+	// Now release all purely cached objects that have no external refcounts.
+	// This will thrash if we have e.g. 2000 rasterizer state objects, all
+	// referenced, and 2000 sampler state objects, all referenced.
+
+	for(auto it=m_CachedStateObjects.begin(); it != m_CachedStateObjects.end();)
+	{
+		ID3D11DeviceChild *o = *it;
+
+		o->AddRef();
+		if(o->Release() == 1)
+		{
+			auto eraseit = it;
+			++it;
+			o->Release();
+			InternalRelease();
+			m_CachedStateObjects.erase(eraseit);
+		}
+		else
+		{
+			++it;
+		}
+	}
+}
+
 void WrappedID3D11Device::AddDeferredContext(WrappedID3D11DeviceContext *defctx)
 {
 	RDCASSERT(m_DeferredContexts.find(defctx) == m_DeferredContexts.end());

renderdoc/driver/d3d11/d3d11_device.h

@@ -228,6 +228,22 @@ class WrappedID3D11Device : public IFrameCapturer, public D3DDEVICEPARENT
 	bool m_AppControlledCapture;
 
 	set<ID3D11DeviceChild *> m_CachedStateObjects;
+
+	// This function will check if m_CachedStateObjects is growing too large, and if so
+	// go through m_CachedStateObjects and release any state objects that are purely
+	// cached (refcount == 1). This prevents us from aggressively caching and running
+	// out of state objects (D3D11 has a max of 4096).
+	//
+	// This isn't the ideal solution as it means some Create calls will be slightly
+	// more expensive while they run this garbage collect, but it is the simplest.
+	//
+	// For cases where cached objects are repeatedly created and released this will
+	// rarely kick in - only in the case where a lot of unique state objects are
+	// created then released and never re-used.
+	//
+	// Must be called while m_D3DLock is held.
+	void CachedObjectsGarbageCollect();
+
 	set<WrappedID3D11DeviceContext *> m_DeferredContexts;
 	map<ID3D11InputLayout *, vector<D3D11_INPUT_ELEMENT_DESC> > m_LayoutDescs;
 	map<ID3D11InputLayout *, ShaderReflection *> m_LayoutDXBC;

renderdoc/driver/d3d11/d3d11_device1_wrap.cpp

@@ -116,6 +116,8 @@ HRESULT WrappedID3D11Device::CreateBlendState1(const D3D11_BLEND_DESC1 *pBlendSt
 
 		ID3D11BlendState1 *wrapped = new WrappedID3D11BlendState1(real, this);
 
+		CachedObjectsGarbageCollect();
+
 		{
 			RDCASSERT(m_CachedStateObjects.find(wrapped) == m_CachedStateObjects.end());
 			wrapped->AddRef();
@@ -190,6 +192,8 @@ HRESULT WrappedID3D11Device::CreateRasterizerState1(const D3D11_RASTERIZER_DESC1
 
 		ID3D11RasterizerState1 *wrapped = new WrappedID3D11RasterizerState1(real, this);
 
+		CachedObjectsGarbageCollect();
+
 		{
 			RDCASSERT(m_CachedStateObjects.find(wrapped) == m_CachedStateObjects.end());
 			wrapped->AddRef();

renderdoc/driver/d3d11/d3d11_device_wrap.cpp

@@ -2036,6 +2036,8 @@ HRESULT WrappedID3D11Device::CreateBlendState(
 
 		ID3D11BlendState *wrapped = new WrappedID3D11BlendState(real, this);
 
+		CachedObjectsGarbageCollect();
+
 		{
 			RDCASSERT(m_CachedStateObjects.find(wrapped) == m_CachedStateObjects.end());
 			wrapped->AddRef();
@@ -2109,6 +2111,8 @@ HRESULT WrappedID3D11Device::CreateDepthStencilState(
 
 		ID3D11DepthStencilState *wrapped = new WrappedID3D11DepthStencilState(real, this);
 
+		CachedObjectsGarbageCollect();
+
 		{
 			RDCASSERT(m_CachedStateObjects.find(wrapped) == m_CachedStateObjects.end());
 			wrapped->AddRef();
@@ -2182,6 +2186,8 @@ HRESULT WrappedID3D11Device::CreateRasterizerState(
 
 		ID3D11RasterizerState *wrapped = new WrappedID3D11RasterizerState(real, this);
 
+		CachedObjectsGarbageCollect();
+
 		{
 			RDCASSERT(m_CachedStateObjects.find(wrapped) == m_CachedStateObjects.end());
 			wrapped->AddRef();
@@ -2255,6 +2261,8 @@ HRESULT WrappedID3D11Device::CreateSamplerState(
 
 		ID3D11SamplerState *wrapped = new WrappedID3D11SamplerState(real, this);
 
+		CachedObjectsGarbageCollect();
+
 		{
 			RDCASSERT(m_CachedStateObjects.find(wrapped) == m_CachedStateObjects.end());
 			wrapped->AddRef();