Merge pull request #201 from ipfs/feat/log-memory-blockstore-consumption

Log unverified blockstore memory consumption

requestmanager/asyncloader/unverifiedblockstore/unverifiedblockstore.go

@@ -2,10 +2,12 @@ package unverifiedblockstore
 
 import (
 	"fmt"
-
+	logging "github.com/ipfs/go-log/v2"
 	ipld "github.com/ipld/go-ipld-prime"
 )
 
+var log = logging.Logger("gs-unverifiedbs")
+
 type settableWriter interface {
 	SetBytes([]byte) error
 }
@@ -15,6 +17,7 @@ type settableWriter interface {
 type UnverifiedBlockStore struct {
 	inMemoryBlocks map[ipld.Link][]byte
 	storer         ipld.Storer
+	dataSize uint64
 }
 
 // New initializes a new unverified store with the given storer function for writing
@@ -30,21 +33,26 @@ func New(storer ipld.Storer) *UnverifiedBlockStore {
 // comes in as part of a traversal.
 func (ubs *UnverifiedBlockStore) AddUnverifiedBlock(lnk ipld.Link, data []byte) {
 	ubs.inMemoryBlocks[lnk] = data
+	ubs.dataSize = ubs.dataSize + uint64(len(data))
+	log.Debugw("added in-memory block", "total_queued_bytes", ubs.dataSize)
 }
 
 // PruneBlocks removes blocks from the unverified store without committing them,
 // if the passed in function returns true for the given link
 func (ubs *UnverifiedBlockStore) PruneBlocks(shouldPrune func(ipld.Link) bool) {
-	for link := range ubs.inMemoryBlocks {
+	for link,data := range ubs.inMemoryBlocks {
 		if shouldPrune(link) {
 			delete(ubs.inMemoryBlocks, link)
+			ubs.dataSize = ubs.dataSize - uint64(len(data))
 		}
 	}
+	log.Debugw("finished pruning in-memory blocks", "total_queued_bytes", ubs.dataSize)
 }
 
 // PruneBlock deletes an individual block from the store
 func (ubs *UnverifiedBlockStore) PruneBlock(link ipld.Link) {
 	delete(ubs.inMemoryBlocks, link)
+	ubs.dataSize = ubs.dataSize - uint64(len(ubs.inMemoryBlocks[link]))
 }
 
 // VerifyBlock verifies the data for the given link as being part of a traversal,
@@ -55,6 +63,8 @@ func (ubs *UnverifiedBlockStore) VerifyBlock(lnk ipld.Link) ([]byte, error) {
 		return nil, fmt.Errorf("Block not found")
 	}
 	delete(ubs.inMemoryBlocks, lnk)
+	ubs.dataSize = ubs.dataSize - uint64(len(data))
+
 	buffer, committer, err := ubs.storer(ipld.LinkContext{})
 	if err != nil {
 		return nil, err