fix: periodically clean up recent_dropped_traces cache (#1312)

## Which problem is this PR solving?

The recent_dropped_traces cache wasn't getting cleaned up

## Short description of the changes

- update documentation on the `SetWithTTL`
- make sure the cache is cleaned up periodically

Co-authored-by: Kent Quirk <kentquirk@honeycomb.io>

collect/cache/cuckooSentCache.go

@@ -140,6 +140,7 @@ func (t *cuckooDroppedRecord) Reason() uint {
 var _ TraceSentRecord = (*cuckooDroppedRecord)(nil)
 
 type cuckooSentCache struct {
+	met              metrics.Metrics
 	kept             *lru.Cache[string, *keptTraceCacheEntry]
 	dropped          *CuckooTraceChecker
 	recentDroppedIDs *generics.SetWithTTL[string]
@@ -179,7 +180,10 @@ func NewCuckooSentCache(cfg config.SampleCacheConfig, met metrics.Metrics) (Trac
 	// request.
 	recentDroppedIDs := generics.NewSetWithTTL[string](3 * time.Second)
 
+	met.Register("cache_recent_dropped_traces", "gauge")
+
 	cache := &cuckooSentCache{
+		met:              met,
 		kept:             stc,
 		dropped:          dropped,
 		recentDroppedIDs: recentDroppedIDs,
@@ -198,6 +202,10 @@ func (c *cuckooSentCache) monitor() {
 		select {
 		case <-ticker.C:
 			c.dropped.Maintain()
+			// Length() returns the number of items in the cache and it will
+			// clean up any expired items.
+			numOfDroppedIDs := c.recentDroppedIDs.Length()
+			c.met.Gauge("cache_recent_dropped_traces", numOfDroppedIDs)
 		case <-c.done:
 			return
 		}

generics/setttl.go

@@ -11,7 +11,7 @@ import (
 )
 
 // SetWithTTL is a unique set of items with a TTL (time to live) for each item.
-// After the TTL expires, the item is automatically removed from the set.
+// After the TTL expires, the item is automatically removed from the set when either Members or Length is called.
 // It is safe for concurrent use.
 type SetWithTTL[T cmp.Ordered] struct {
 	Items map[T]time.Time