From 9146c8b08427864ccb4fb6657065d500c985e140 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Wed, 18 Nov 2020 05:03:02 +0100
Subject: [PATCH 01/51] update quic

don't wait for a blob to be written to disk before sending it downstream
don't wait for the disk store to be walked before starting everything up
---
 go.mod                       |  6 +++---
 go.sum                       | 15 +++++++--------
 store/caching.go             | 13 ++++++++++---
 store/lru.go                 | 13 +++++++------
 store/speedwalk/speedwalk.go |  3 +--
 5 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/go.mod b/go.mod
index 9df120d..b98b2ad 100644
--- a/go.mod
+++ b/go.mod
@@ -12,6 +12,7 @@ require (
 	github.com/golang/protobuf v1.4.2
 	github.com/google/btree v1.0.0 // indirect
 	github.com/google/gops v0.3.7
+	github.com/google/martian v2.1.0+incompatible
 	github.com/gorilla/mux v1.7.4
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/hashicorp/golang-lru v0.5.4
@@ -24,11 +25,11 @@ require (
 	github.com/lbryio/lbry.go v1.1.2 // indirect
 	github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128
 	github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec
-	github.com/lucas-clemente/quic-go v0.18.1
+	github.com/lucas-clemente/quic-go v0.19.1
 	github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
 	github.com/prometheus/client_golang v0.9.2
 	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/afero v1.4.1
+	github.com/spf13/afero v1.4.1 // indirect
 	github.com/spf13/cast v1.3.0
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3 // indirect
@@ -40,7 +41,6 @@ require (
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect
 	google.golang.org/appengine v1.6.2 // indirect
-	gotest.tools v2.2.0+incompatible
 )
 
 go 1.15
diff --git a/go.sum b/go.sum
index 84b207a..fceb2df 100644
--- a/go.sum
+++ b/go.sum
@@ -109,6 +109,7 @@ github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+u
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gops v0.3.7 h1:KtVAagOM0FIq+02DiQrKBTnLhYpWBMowaufcj+W1Exw=
 github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
@@ -150,8 +151,6 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
-github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce h1:xdsDDbiBDQTKASoGEZ+pEmF1OnWuu8AQ9I8iNbHNeno=
@@ -222,8 +221,8 @@ github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiV
 github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c=
 github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lucas-clemente/quic-go v0.18.1 h1:DMR7guC0NtVS8zNZR3IO7NARZvZygkSC56GGtC6cyys=
-github.com/lucas-clemente/quic-go v0.18.1/go.mod h1:yXttHsSNxQi8AWijC/vLP+OJczXqzHSOcJrM5ITUlCg=
+github.com/lucas-clemente/quic-go v0.19.1 h1:J9TkQJGJVOR3UmGhd4zdVYwKSA0EoXbLRf15uQJ6gT4=
+github.com/lucas-clemente/quic-go v0.19.1/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
 github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
@@ -232,12 +231,12 @@ github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCW
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/marten-seemann/qpack v0.2.0 h1:/r1rhZoOmgxVKBqPNnYilZBDEyw+6OUHCbBzA5jc2y0=
-github.com/marten-seemann/qpack v0.2.0/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
+github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs=
+github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
 github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc=
 github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs=
-github.com/marten-seemann/qtls-go1-15 v0.1.0 h1:i/YPXVxz8q9umso/5y474CNcHmTpA+5DH+mFPjx6PZg=
-github.com/marten-seemann/qtls-go1-15 v0.1.0/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
+github.com/marten-seemann/qtls-go1-15 v0.1.1 h1:LIH6K34bPVttyXnUWixk0bzH6/N07VxbSabxn5A5gZQ=
+github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
diff --git a/store/caching.go b/store/caching.go
index 0a06395..c8e7d45 100644
--- a/store/caching.go
+++ b/store/caching.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/lbryio/reflector.go/internal/metrics"
 )
@@ -62,9 +63,15 @@ func (c *CachingStore) Get(hash string) (stream.Blob, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	err = c.cache.Put(hash, blob)
-	return blob, err
+	// there is no need to wait for the blob to be stored before we return it
+	// TODO: however this should be refactored to limit the amount of routines that the process can spawn to avoid a possible DoS
+	go func() {
+		err = c.cache.Put(hash, blob)
+		if err != nil {
+			log.Errorf("error saving blob to underlying cache: %s", err.Error())
+		}
+	}()
+	return blob, nil
 }
 
 // Put stores the blob in the origin and the cache
diff --git a/store/lru.go b/store/lru.go
index 4ef4908..6d06aa6 100644
--- a/store/lru.go
+++ b/store/lru.go
@@ -30,13 +30,14 @@ func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore {
 		store: store,
 		lru:   lru,
 	}
-
-	if lstr, ok := store.(lister); ok {
-		err = l.loadExisting(lstr, maxItems)
-		if err != nil {
-			panic(err) // TODO: what should happen here? panic? return nil? just keep going?
+	go func() {
+		if lstr, ok := store.(lister); ok {
+			err = l.loadExisting(lstr, maxItems)
+			if err != nil {
+				panic(err) // TODO: what should happen here? panic? return nil? just keep going?
+			}
 		}
-	}
+	}()
 
 	return l
 }
diff --git a/store/speedwalk/speedwalk.go b/store/speedwalk/speedwalk.go
index e2563ba..23230df 100644
--- a/store/speedwalk/speedwalk.go
+++ b/store/speedwalk/speedwalk.go
@@ -60,7 +60,6 @@ func AllFiles(startDir string, basename bool) ([]string, error) {
 				walkerWG.Done()
 				goroutineLimiter <- struct{}{}
 			}()
-
 			err = godirwalk.Walk(filepath.Join(startDir, dir), &godirwalk.Options{
 				Unsorted: true, // faster this way
 				Callback: func(osPathname string, de *godirwalk.Dirent) error {
@@ -84,6 +83,6 @@ func AllFiles(startDir string, basename bool) ([]string, error) {
 
 	close(pathChan)
 	pathWG.Wait()
-
+	logrus.Infoln("loaded LRU")
 	return paths, nil
 }

From a574fecf4ee3595325b2f84d711b9532c527838f Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Fri, 20 Nov 2020 19:39:08 +0100
Subject: [PATCH 02/51] add buffer cache for nvme drive

---
 cmd/reflector.go | 50 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 18 deletions(-)

diff --git a/cmd/reflector.go b/cmd/reflector.go
index 2f687d0..cff66f5 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -22,19 +22,20 @@ import (
 )
 
 var (
-	tcpPeerPort           int
-	http3PeerPort         int
-	receiverPort          int
-	metricsPort           int
-	disableUploads        bool
-	disableBlocklist      bool
-	proxyAddress          string
-	proxyPort             string
-	proxyProtocol         string
-	useDB                 bool
-	cloudFrontEndpoint    string
-	reflectorCmdDiskCache string
-	reflectorCmdMemCache  int
+	tcpPeerPort                 int
+	http3PeerPort               int
+	receiverPort                int
+	metricsPort                 int
+	disableUploads              bool
+	disableBlocklist            bool
+	proxyAddress                string
+	proxyPort                   string
+	proxyProtocol               string
+	useDB                       bool
+	cloudFrontEndpoint          string
+	reflectorCmdDiskCache       string
+	bufferReflectorCmdDiskCache string
+	reflectorCmdMemCache        int
 )
 
 func init() {
@@ -56,6 +57,8 @@ func init() {
 	cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not")
 	cmd.Flags().StringVar(&reflectorCmdDiskCache, "disk-cache", "",
 		"enable disk cache, setting max size and path where to store blobs. format is 'MAX_BLOBS:CACHE_PATH'")
+	cmd.Flags().StringVar(&bufferReflectorCmdDiskCache, "buffer-disk-cache", "",
+		"enable buffer disk cache, setting max size and path where to store blobs. format is 'MAX_BLOBS:CACHE_PATH'")
 	cmd.Flags().IntVar(&reflectorCmdMemCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
 	rootCmd.AddCommand(cmd)
 }
@@ -147,7 +150,19 @@ func setupStore() store.BlobStore {
 func wrapWithCache(s store.BlobStore) store.BlobStore {
 	wrapped := s
 
-	diskCacheMaxSize, diskCachePath := diskCacheParams()
+	diskCacheMaxSize, diskCachePath := diskCacheParams(reflectorCmdDiskCache)
+	if diskCacheMaxSize > 0 {
+		err := os.MkdirAll(diskCachePath, os.ModePerm)
+		if err != nil {
+			log.Fatal(err)
+		}
+		wrapped = store.NewCachingStore(
+			"reflector",
+			wrapped,
+			store.NewLRUStore("peer_server", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize),
+		)
+	}
+	diskCacheMaxSize, diskCachePath = diskCacheParams(bufferReflectorCmdDiskCache)
 	if diskCacheMaxSize > 0 {
 		err := os.MkdirAll(diskCachePath, os.ModePerm)
 		if err != nil {
@@ -159,7 +174,6 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 			store.NewLRUStore("peer_server", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize),
 		)
 	}
-
 	if reflectorCmdMemCache > 0 {
 		wrapped = store.NewCachingStore(
 			"reflector",
@@ -171,12 +185,12 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 	return wrapped
 }
 
-func diskCacheParams() (int, string) {
-	if reflectorCmdDiskCache == "" {
+func diskCacheParams(diskParams string) (int, string) {
+	if diskParams == "" {
 		return 0, ""
 	}
 
-	parts := strings.Split(reflectorCmdDiskCache, ":")
+	parts := strings.Split(diskParams, ":")
 	if len(parts) != 2 {
 		log.Fatalf("--disk-cache must be a number, followed by ':', followed by a string")
 	}

From bb41a84bb7ac4335497cae43fa3f00535edfe02b Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Fri, 20 Nov 2020 15:01:33 -0500
Subject: [PATCH 03/51] rename cahces

---
 cmd/reflector.go      |  8 +++++---
 store/lru.go          | 18 ++++++++++--------
 store/singleflight.go | 10 +++++-----
 3 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/cmd/reflector.go b/cmd/reflector.go
index cff66f5..a047b85 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -159,9 +159,10 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 		wrapped = store.NewCachingStore(
 			"reflector",
 			wrapped,
-			store.NewLRUStore("peer_server", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize),
+			store.NewLRUStore("hdd", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize),
 		)
 	}
+
 	diskCacheMaxSize, diskCachePath = diskCacheParams(bufferReflectorCmdDiskCache)
 	if diskCacheMaxSize > 0 {
 		err := os.MkdirAll(diskCachePath, os.ModePerm)
@@ -171,14 +172,15 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 		wrapped = store.NewCachingStore(
 			"reflector",
 			wrapped,
-			store.NewLRUStore("peer_server", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize),
+			store.NewLRUStore("nvme", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize),
 		)
 	}
+
 	if reflectorCmdMemCache > 0 {
 		wrapped = store.NewCachingStore(
 			"reflector",
 			wrapped,
-			store.NewLRUStore("peer_server", store.NewMemStore(), reflectorCmdMemCache),
+			store.NewLRUStore("mem", store.NewMemStore(), reflectorCmdMemCache),
 		)
 	}
 
diff --git a/store/lru.go b/store/lru.go
index 6d06aa6..edbe3a5 100644
--- a/store/lru.go
+++ b/store/lru.go
@@ -18,18 +18,20 @@ type LRUStore struct {
 
 // NewLRUStore initialize a new LRUStore
 func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore {
+	l := &LRUStore{
+		store: store,
+	}
+
 	lru, err := golru.NewWithEvict(maxItems, func(key interface{}, value interface{}) {
-		metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
+		metrics.CacheLRUEvictCount.With(metrics.CacheLabels(l.Name(), component)).Inc()
 		_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
 	})
 	if err != nil {
 		panic(err)
 	}
 
-	l := &LRUStore{
-		store: store,
-		lru:   lru,
-	}
+	l.lru = lru
+
 	go func() {
 		if lstr, ok := store.(lister); ok {
 			err = l.loadExisting(lstr, maxItems)
@@ -42,10 +44,10 @@ func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore {
 	return l
 }
 
-const nameLRU = "lru"
-
 // Name is the cache type name
-func (l *LRUStore) Name() string { return nameLRU }
+func (l *LRUStore) Name() string {
+	return "lru_" + l.store.Name()
+}
 
 // Has returns whether the blob is in the store, without updating the recent-ness.
 func (l *LRUStore) Has(hash string) (bool, error) {
diff --git a/store/singleflight.go b/store/singleflight.go
index fbe314f..cca1e2f 100644
--- a/store/singleflight.go
+++ b/store/singleflight.go
@@ -32,8 +32,8 @@ func (s *singleflightStore) Name() string {
 // Get ensures that only one request per hash is sent to the origin at a time,
 // thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem
 func (s *singleflightStore) Get(hash string) (stream.Blob, error) {
-	metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Inc()
-	defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Dec()
+	metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
+	defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
 
 	blob, err, _ := s.sf.Do(hash, s.getter(hash))
 	if err != nil {
@@ -46,8 +46,8 @@ func (s *singleflightStore) Get(hash string) (stream.Blob, error) {
 // only one getter per hash will be executing at a time
 func (s *singleflightStore) getter(hash string) func() (interface{}, error) {
 	return func() (interface{}, error) {
-		metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Inc()
-		defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.BlobStore.Name(), s.component)).Dec()
+		metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
+		defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
 
 		start := time.Now()
 		blob, err := s.BlobStore.Get(hash)
@@ -57,7 +57,7 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) {
 
 		rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
 		metrics.CacheRetrievalSpeed.With(map[string]string{
-			metrics.LabelCacheType: s.BlobStore.Name(),
+			metrics.LabelCacheType: s.Name(),
 			metrics.LabelComponent: s.component,
 			metrics.LabelSource:    "origin",
 		}).Set(rate)

From bc54601dde0b295cef801c62263b6321fe15bfc0 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Sat, 21 Nov 2020 01:26:32 +0100
Subject: [PATCH 04/51] add LFUDA store

update quic
fix tests
---
 go.mod                |   3 +-
 go.sum                |   4 ++
 store/caching_test.go |   1 +
 store/lfuda.go        | 118 +++++++++++++++++++++++++++++++++++
 store/lfuda_test.go   | 139 ++++++++++++++++++++++++++++++++++++++++++
 store/lru_test.go     |   4 +-
 6 files changed, 267 insertions(+), 2 deletions(-)
 create mode 100644 store/lfuda.go
 create mode 100644 store/lfuda_test.go

diff --git a/go.mod b/go.mod
index b98b2ad..5ab0eb5 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203
 require (
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
 	github.com/aws/aws-sdk-go v1.16.11
+	github.com/bparli/lfuda-go v0.3.0
 	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
 	github.com/davecgh/go-spew v1.1.1
@@ -25,7 +26,7 @@ require (
 	github.com/lbryio/lbry.go v1.1.2 // indirect
 	github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128
 	github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec
-	github.com/lucas-clemente/quic-go v0.19.1
+	github.com/lucas-clemente/quic-go v0.19.2
 	github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
 	github.com/prometheus/client_golang v0.9.2
 	github.com/sirupsen/logrus v1.4.2
diff --git a/go.sum b/go.sum
index fceb2df..25e908f 100644
--- a/go.sum
+++ b/go.sum
@@ -25,6 +25,8 @@ github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bparli/lfuda-go v0.3.0 h1:b6qPjEb0BN006oQnj2nuGfz94yY3iYo0bmuFM079tQg=
+github.com/bparli/lfuda-go v0.3.0/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
@@ -223,6 +225,8 @@ github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiV
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lucas-clemente/quic-go v0.19.1 h1:J9TkQJGJVOR3UmGhd4zdVYwKSA0EoXbLRf15uQJ6gT4=
 github.com/lucas-clemente/quic-go v0.19.1/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0=
+github.com/lucas-clemente/quic-go v0.19.2 h1:w8BBYUx5Z+kNpeaOeQW/KzcNsKWhh4O6PeQhb0nURPg=
+github.com/lucas-clemente/quic-go v0.19.2/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
 github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
diff --git a/store/caching_test.go b/store/caching_test.go
index 34f928c..1168a92 100644
--- a/store/caching_test.go
+++ b/store/caching_test.go
@@ -58,6 +58,7 @@ func TestCachingStore_CacheMiss(t *testing.T) {
 	if !bytes.Equal(b, res) {
 		t.Errorf("expected Get() to return %s, got %s", string(b), string(res))
 	}
+	time.Sleep(10 * time.Millisecond) //storing to cache is done async so let's give it some time
 
 	has, err := cache.Has(hash)
 	if err != nil {
diff --git a/store/lfuda.go b/store/lfuda.go
new file mode 100644
index 0000000..9715954
--- /dev/null
+++ b/store/lfuda.go
@@ -0,0 +1,118 @@
+package store
+
+import (
+	"github.com/bparli/lfuda-go"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/internal/metrics"
+)
+
+// LRUStore adds a max cache size and LRU eviction to a BlobStore
+type LFUDAStore struct {
+	// underlying store
+	store BlobStore
+	// lfuda implementation
+	lfuda *lfuda.Cache
+}
+
+// NewLRUStore initialize a new LRUStore
+func NewLFUDAStore(component string, store BlobStore, maxSize float64) *LFUDAStore {
+	lfuda := lfuda.NewGDSFWithEvict(maxSize, func(key interface{}, value interface{}) {
+		metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
+		_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
+	})
+	lfuda.Age()
+	l := &LFUDAStore{
+		store: store,
+		lfuda: lfuda,
+	}
+	go func() {
+		if lstr, ok := store.(lister); ok {
+			err := l.loadExisting(lstr, int(maxSize/2000000.0))
+			if err != nil {
+				panic(err) // TODO: what should happen here? panic? return nil? just keep going?
+			}
+		}
+	}()
+
+	return l
+}
+
+const nameLFUDA = "lfuda"
+
+var fakeTrue = []byte{'t'}
+
+// Name is the cache type name
+func (l *LFUDAStore) Name() string { return nameLFUDA }
+
+// Has returns whether the blob is in the store, without updating the recent-ness.
+func (l *LFUDAStore) Has(hash string) (bool, error) {
+	return l.lfuda.Contains(hash), nil
+}
+
+// Get returns the blob or an error if the blob doesn't exist.
+func (l *LFUDAStore) Get(hash string) (stream.Blob, error) {
+	_, has := l.lfuda.Get(hash)
+	if !has {
+		return nil, errors.Err(ErrBlobNotFound)
+	}
+	blob, err := l.store.Get(hash)
+	if errors.Is(err, ErrBlobNotFound) {
+		// Blob disappeared from underlying store
+		l.lfuda.Remove(hash)
+	}
+	return blob, err
+}
+
+// Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
+func (l *LFUDAStore) Put(hash string, blob stream.Blob) error {
+	err := l.store.Put(hash, blob)
+	if err != nil {
+		return err
+	}
+	l.lfuda.Set(hash, fakeTrue)
+	return nil
+}
+
+// PutSD stores the sd blob
+func (l *LFUDAStore) PutSD(hash string, blob stream.Blob) error {
+	err := l.store.PutSD(hash, blob)
+	if err != nil {
+		return err
+	}
+
+	l.lfuda.Set(hash, fakeTrue)
+	return nil
+}
+
+// Delete deletes the blob from the store
+func (l *LFUDAStore) Delete(hash string) error {
+	err := l.store.Delete(hash)
+	if err != nil {
+		return err
+	}
+
+	// This must come after store.Delete()
+	// Remove triggers onEvict function, which also tries to delete blob from store
+	// We need to delete it manually first so any errors can be propagated up
+	l.lfuda.Remove(hash)
+	return nil
+}
+
+// loadExisting imports existing blobs from the underlying store into the LRU cache
+func (l *LFUDAStore) loadExisting(store lister, maxItems int) error {
+	existing, err := store.list()
+	if err != nil {
+		return err
+	}
+
+	added := 0
+	for _, h := range existing {
+		l.lfuda.Set(h, fakeTrue)
+		added++
+		if maxItems > 0 && added >= maxItems { // underlying cache is bigger than LRU cache
+			break
+		}
+	}
+	return nil
+}
diff --git a/store/lfuda_test.go b/store/lfuda_test.go
new file mode 100644
index 0000000..9b1eb58
--- /dev/null
+++ b/store/lfuda_test.go
@@ -0,0 +1,139 @@
+package store
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const cacheMaxSize = 3
+
+func getTestLFUDAStore() (*LFUDAStore, *MemStore) {
+	m := NewMemStore()
+	return NewLFUDAStore("test", m, cacheMaxSize), m
+}
+
+func TestFUDAStore_Eviction(t *testing.T) {
+	lfuda, mem := getTestLFUDAStore()
+	b := []byte("x")
+	err := lfuda.Put("one", b)
+	require.NoError(t, err)
+	err = lfuda.Put("two", b)
+	require.NoError(t, err)
+	err = lfuda.Put("three", b)
+	require.NoError(t, err)
+	err = lfuda.Put("four", b)
+	require.NoError(t, err)
+	err = lfuda.Put("five", b)
+	require.NoError(t, err)
+	err = lfuda.Put("five", b)
+	require.NoError(t, err)
+	err = lfuda.Put("four", b)
+	require.NoError(t, err)
+	err = lfuda.Put("two", b)
+	require.NoError(t, err)
+
+	_, err = lfuda.Get("five")
+	require.NoError(t, err)
+	_, err = lfuda.Get("four")
+	require.NoError(t, err)
+	_, err = lfuda.Get("two")
+	require.NoError(t, err)
+	assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
+
+	for k, v := range map[string]bool{
+		"one":   false,
+		"two":   true,
+		"three": false,
+		"four":  true,
+		"five":  true,
+		"six":   false,
+	} {
+		has, err := lfuda.Has(k)
+		assert.NoError(t, err)
+		assert.Equal(t, v, has)
+	}
+
+	lfuda.Get("two")  // touch so it stays in cache
+	lfuda.Get("five") // touch so it stays in cache
+	lfuda.Put("six", b)
+
+	assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
+
+	keys := lfuda.lfuda.Keys()
+	log.Infof("%+v", keys)
+	for k, v := range map[string]bool{
+		"one":   false,
+		"two":   true,
+		"three": false,
+		"four":  false,
+		"five":  true,
+		"six":   true,
+	} {
+		has, err := lfuda.Has(k)
+		assert.NoError(t, err)
+		assert.Equal(t, v, has)
+	}
+
+	err = lfuda.Delete("six")
+	assert.NoError(t, err)
+	err = lfuda.Delete("five")
+	assert.NoError(t, err)
+	err = lfuda.Delete("two")
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(mem.Debug()))
+}
+
+func TestFUDAStore_UnderlyingBlobMissing(t *testing.T) {
+	lfuda, mem := getTestLFUDAStore()
+	hash := "hash"
+	b := []byte("this is a blob of stuff")
+	err := lfuda.Put(hash, b)
+	require.NoError(t, err)
+
+	err = mem.Delete(hash)
+	require.NoError(t, err)
+
+	// hash still exists in lru
+	assert.True(t, lfuda.lfuda.Contains(hash))
+
+	blob, err := lfuda.Get(hash)
+	assert.Nil(t, blob)
+	assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s",
+		reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(),
+		reflect.TypeOf(err).String(), err.Error())
+
+	// lru.Get() removes hash if underlying store doesn't have it
+	assert.False(t, lfuda.lfuda.Contains(hash))
+}
+
+func TestFUDAStore_loadExisting(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "reflector_test_*")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+	d := NewDiskStore(tmpDir, 2)
+
+	hash := "hash"
+	b := []byte("this is a blob of stuff")
+	err = d.Put(hash, b)
+	require.NoError(t, err)
+
+	existing, err := d.list()
+	require.NoError(t, err)
+	require.Equal(t, 1, len(existing), "blob should exist in cache")
+	assert.Equal(t, hash, existing[0])
+
+	lfuda := NewLFUDAStore("test", d, 3) // lru should load existing blobs when it's created
+	time.Sleep(100 * time.Millisecond)   // async load so let's wait...
+	has, err := lfuda.Has(hash)
+	require.NoError(t, err)
+	assert.True(t, has, "hash should be loaded from disk store but it's not")
+}
diff --git a/store/lru_test.go b/store/lru_test.go
index 968956c..2bd934c 100644
--- a/store/lru_test.go
+++ b/store/lru_test.go
@@ -5,6 +5,7 @@ import (
 	"os"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 
@@ -114,7 +115,8 @@ func TestLRUStore_loadExisting(t *testing.T) {
 	require.Equal(t, 1, len(existing), "blob should exist in cache")
 	assert.Equal(t, hash, existing[0])
 
-	lru := NewLRUStore("test", d, 3) // lru should load existing blobs when it's created
+	lru := NewLRUStore("test", d, 3)   // lru should load existing blobs when it's created
+	time.Sleep(100 * time.Millisecond) // async load so let's wait...
 	has, err := lru.Has(hash)
 	require.NoError(t, err)
 	assert.True(t, has, "hash should be loaded from disk store but it's not")

From d45abdbdb0e0351e212940f56021437ae2e3836a Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Sat, 21 Nov 2020 01:39:50 +0100
Subject: [PATCH 05/51] use LFUDA store

swap size to bytes
---
 cmd/reflector.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cmd/reflector.go b/cmd/reflector.go
index a047b85..4d47970 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -151,6 +151,7 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 	wrapped := s
 
 	diskCacheMaxSize, diskCachePath := diskCacheParams(reflectorCmdDiskCache)
+	cacheMaxSizeInBytes := float64(diskCacheMaxSize * 2 * 1000 * 1000)
 	if diskCacheMaxSize > 0 {
 		err := os.MkdirAll(diskCachePath, os.ModePerm)
 		if err != nil {
@@ -159,7 +160,7 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 		wrapped = store.NewCachingStore(
 			"reflector",
 			wrapped,
-			store.NewLRUStore("hdd", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize),
+			store.NewLFUDAStore("hdd", store.NewDiskStore(diskCachePath, 2), cacheMaxSizeInBytes),
 		)
 	}
 
@@ -172,7 +173,7 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 		wrapped = store.NewCachingStore(
 			"reflector",
 			wrapped,
-			store.NewLRUStore("nvme", store.NewDiskStore(diskCachePath, 2), diskCacheMaxSize),
+			store.NewLFUDAStore("nvme", store.NewDiskStore(diskCachePath, 2), cacheMaxSizeInBytes),
 		)
 	}
 

From 7b80b2d4d24f82f115e26bdc0d2e8ae6edd2a49f Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Sun, 22 Nov 2020 04:04:44 +0100
Subject: [PATCH 06/51] fix buffer cache running out of space

---
 cmd/reflector.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmd/reflector.go b/cmd/reflector.go
index 4d47970..55d77b3 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -165,6 +165,7 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 	}
 
 	diskCacheMaxSize, diskCachePath = diskCacheParams(bufferReflectorCmdDiskCache)
+	cacheMaxSizeInBytes = float64(diskCacheMaxSize * 2 * 1000 * 1000)
 	if diskCacheMaxSize > 0 {
 		err := os.MkdirAll(diskCachePath, os.ModePerm)
 		if err != nil {

From ff9b61b034bd75ed23b2c2826b5b5d4690e67cae Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Sun, 22 Nov 2020 05:06:36 +0100
Subject: [PATCH 07/51] fix cache size mess

---
 cmd/reflector.go | 34 +++++++++++++++++++++-------------
 go.mod           |  2 +-
 go.sum           |  4 ++--
 store/lfuda.go   |  5 +++--
 4 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/cmd/reflector.go b/cmd/reflector.go
index 55d77b3..7551f77 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -16,8 +16,10 @@ import (
 	"github.com/lbryio/reflector.go/reflector"
 	"github.com/lbryio/reflector.go/store"
 
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	"github.com/c2h5oh/datasize"
 	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cast"
 	"github.com/spf13/cobra"
 )
 
@@ -56,9 +58,9 @@ func init() {
 	cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
 	cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not")
 	cmd.Flags().StringVar(&reflectorCmdDiskCache, "disk-cache", "",
-		"enable disk cache, setting max size and path where to store blobs. format is 'MAX_BLOBS:CACHE_PATH'")
+		"enable disk cache, setting max size and path where to store blobs. format is 'sizeGB:CACHE_PATH'")
 	cmd.Flags().StringVar(&bufferReflectorCmdDiskCache, "buffer-disk-cache", "",
-		"enable buffer disk cache, setting max size and path where to store blobs. format is 'MAX_BLOBS:CACHE_PATH'")
+		"enable buffer disk cache, setting max size and path where to store blobs. format is 'sizeGB:CACHE_PATH'")
 	cmd.Flags().IntVar(&reflectorCmdMemCache, "mem-cache", 0, "enable in-memory cache with a max size of this many blobs")
 	rootCmd.AddCommand(cmd)
 }
@@ -151,7 +153,9 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 	wrapped := s
 
 	diskCacheMaxSize, diskCachePath := diskCacheParams(reflectorCmdDiskCache)
-	cacheMaxSizeInBytes := float64(diskCacheMaxSize * 2 * 1000 * 1000)
+	//we are tracking blobs in memory with a 1 byte long boolean, which means that for each 2MB (a blob) we need 1Byte
+	// so if the underlying cache holds 10MB, 10MB/2MB=5Bytes which is also the exact count of objects to restore on startup
+	realCacheSize := float64(diskCacheMaxSize) / float64(stream.MaxBlobSize)
 	if diskCacheMaxSize > 0 {
 		err := os.MkdirAll(diskCachePath, os.ModePerm)
 		if err != nil {
@@ -160,12 +164,12 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 		wrapped = store.NewCachingStore(
 			"reflector",
 			wrapped,
-			store.NewLFUDAStore("hdd", store.NewDiskStore(diskCachePath, 2), cacheMaxSizeInBytes),
+			store.NewLFUDAStore("hdd", store.NewDiskStore(diskCachePath, 2), realCacheSize),
 		)
 	}
 
 	diskCacheMaxSize, diskCachePath = diskCacheParams(bufferReflectorCmdDiskCache)
-	cacheMaxSizeInBytes = float64(diskCacheMaxSize * 2 * 1000 * 1000)
+	realCacheSize = float64(diskCacheMaxSize) / float64(stream.MaxBlobSize)
 	if diskCacheMaxSize > 0 {
 		err := os.MkdirAll(diskCachePath, os.ModePerm)
 		if err != nil {
@@ -174,7 +178,7 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 		wrapped = store.NewCachingStore(
 			"reflector",
 			wrapped,
-			store.NewLFUDAStore("nvme", store.NewDiskStore(diskCachePath, 2), cacheMaxSizeInBytes),
+			store.NewLFUDAStore("nvme", store.NewDiskStore(diskCachePath, 2), realCacheSize),
 		)
 	}
 
@@ -199,15 +203,19 @@ func diskCacheParams(diskParams string) (int, string) {
 		log.Fatalf("--disk-cache must be a number, followed by ':', followed by a string")
 	}
 
-	maxSize := cast.ToInt(parts[0])
-	if maxSize <= 0 {
-		log.Fatalf("--disk-cache max size must be more than 0")
-	}
-
+	diskCacheSize := parts[0]
 	path := parts[1]
 	if len(path) == 0 || path[0] != '/' {
 		log.Fatalf("--disk-cache path must start with '/'")
 	}
 
-	return maxSize, path
+	var maxSize datasize.ByteSize
+	err := maxSize.UnmarshalText([]byte(diskCacheSize))
+	if err != nil {
+		log.Fatal(err)
+	}
+	if maxSize <= 0 {
+		log.Fatal("--disk-cache size must be more than 0")
+	}
+	return int(maxSize), path
 }
diff --git a/go.mod b/go.mod
index 5ab0eb5..df0c53d 100644
--- a/go.mod
+++ b/go.mod
@@ -8,12 +8,12 @@ require (
 	github.com/bparli/lfuda-go v0.3.0
 	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
+	github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/golang/protobuf v1.4.2
 	github.com/google/btree v1.0.0 // indirect
 	github.com/google/gops v0.3.7
-	github.com/google/martian v2.1.0+incompatible
 	github.com/gorilla/mux v1.7.4
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/hashicorp/golang-lru v0.5.4
diff --git a/go.sum b/go.sum
index 25e908f..5965f78 100644
--- a/go.sum
+++ b/go.sum
@@ -41,6 +41,8 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3
 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak=
+github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
@@ -223,8 +225,6 @@ github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiV
 github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c=
 github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lucas-clemente/quic-go v0.19.1 h1:J9TkQJGJVOR3UmGhd4zdVYwKSA0EoXbLRf15uQJ6gT4=
-github.com/lucas-clemente/quic-go v0.19.1/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0=
 github.com/lucas-clemente/quic-go v0.19.2 h1:w8BBYUx5Z+kNpeaOeQW/KzcNsKWhh4O6PeQhb0nURPg=
 github.com/lucas-clemente/quic-go v0.19.2/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
diff --git a/store/lfuda.go b/store/lfuda.go
index 9715954..2a12e99 100644
--- a/store/lfuda.go
+++ b/store/lfuda.go
@@ -5,6 +5,7 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/sirupsen/logrus"
 )
 
 // LRUStore adds a max cache size and LRU eviction to a BlobStore
@@ -21,14 +22,13 @@ func NewLFUDAStore(component string, store BlobStore, maxSize float64) *LFUDASto
 		metrics.CacheLRUEvictCount.With(metrics.CacheLabels(store.Name(), component)).Inc()
 		_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
 	})
-	lfuda.Age()
 	l := &LFUDAStore{
 		store: store,
 		lfuda: lfuda,
 	}
 	go func() {
 		if lstr, ok := store.(lister); ok {
-			err := l.loadExisting(lstr, int(maxSize/2000000.0))
+			err := l.loadExisting(lstr, int(maxSize))
 			if err != nil {
 				panic(err) // TODO: what should happen here? panic? return nil? just keep going?
 			}
@@ -101,6 +101,7 @@ func (l *LFUDAStore) Delete(hash string) error {
 
 // loadExisting imports existing blobs from the underlying store into the LRU cache
 func (l *LFUDAStore) loadExisting(store lister, maxItems int) error {
+	logrus.Infof("loading at most %d items", maxItems)
 	existing, err := store.list()
 	if err != nil {
 		return err

From 9fc96ac01b61607bde39a66f407e54283eb8d351 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Wed, 25 Nov 2020 19:24:32 +0100
Subject: [PATCH 08/51] only store the blobs in the underlying storage if LFUDA
 accepted them

---
 store/lfuda.go | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/store/lfuda.go b/store/lfuda.go
index 2a12e99..e33afa0 100644
--- a/store/lfuda.go
+++ b/store/lfuda.go
@@ -66,11 +66,14 @@ func (l *LFUDAStore) Get(hash string) (stream.Blob, error) {
 
 // Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
 func (l *LFUDAStore) Put(hash string, blob stream.Blob) error {
-	err := l.store.Put(hash, blob)
-	if err != nil {
-		return err
-	}
 	l.lfuda.Set(hash, fakeTrue)
+	has, _ := l.Has(hash)
+	if has {
+		err := l.store.Put(hash, blob)
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }
 

From 2c0df2ca8a497b13829e13790fe9b1b9106464a2 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Fri, 27 Nov 2020 16:19:17 +0100
Subject: [PATCH 09/51] update lfuda library

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index df0c53d..95a4a5a 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203
 require (
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
 	github.com/aws/aws-sdk-go v1.16.11
-	github.com/bparli/lfuda-go v0.3.0
+	github.com/bparli/lfuda-go v0.3.1
 	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
 	github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
diff --git a/go.sum b/go.sum
index 5965f78..4879450 100644
--- a/go.sum
+++ b/go.sum
@@ -25,8 +25,8 @@ github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bparli/lfuda-go v0.3.0 h1:b6qPjEb0BN006oQnj2nuGfz94yY3iYo0bmuFM079tQg=
-github.com/bparli/lfuda-go v0.3.0/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo=
+github.com/bparli/lfuda-go v0.3.1 h1:nO9Szo627RC8/z+R+MMPBItNwHCOonchmpjQuQi8jVY=
+github.com/bparli/lfuda-go v0.3.1/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=

From 74b76a11e47360023059883fc9635d6a7ffb800f Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Thu, 17 Dec 2020 23:49:37 +0100
Subject: [PATCH 10/51] upgrade quic

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 95a4a5a..04d07c2 100644
--- a/go.mod
+++ b/go.mod
@@ -26,7 +26,7 @@ require (
 	github.com/lbryio/lbry.go v1.1.2 // indirect
 	github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128
 	github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec
-	github.com/lucas-clemente/quic-go v0.19.2
+	github.com/lucas-clemente/quic-go v0.19.3
 	github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
 	github.com/prometheus/client_golang v0.9.2
 	github.com/sirupsen/logrus v1.4.2
diff --git a/go.sum b/go.sum
index 4879450..18420b2 100644
--- a/go.sum
+++ b/go.sum
@@ -225,8 +225,8 @@ github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiV
 github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c=
 github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lucas-clemente/quic-go v0.19.2 h1:w8BBYUx5Z+kNpeaOeQW/KzcNsKWhh4O6PeQhb0nURPg=
-github.com/lucas-clemente/quic-go v0.19.2/go.mod h1:ZUygOqIoai0ASXXLJ92LTnKdbqh9MHCLTX6Nr1jUrK0=
+github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4=
+github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
 github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=

From def551cc895b078aa01714daeb49f8e943559a5a Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 22 Dec 2020 20:53:48 +0100
Subject: [PATCH 11/51] add option to run with RO-CF only as upstream

increase idle timeout to avoid errors downstream
add option to delete blobs from DB if storage doesn't have it (for future local tracking)
---
 cmd/peer.go          |   2 +-
 cmd/reflector.go     |  18 ++++--
 cmd/start.go         |   2 +-
 cmd/upload.go        |   2 +-
 go.mod               |   5 +-
 go.sum               | 149 +++++++++++++++++++++++++++++++++++++++++++
 peer/http3/server.go |   2 +-
 peer/http3/store.go  |   2 +-
 store/dbbacked.go    |  23 +++++--
 9 files changed, 188 insertions(+), 17 deletions(-)

diff --git a/cmd/peer.go b/cmd/peer.go
index 3cb2a19..86af279 100644
--- a/cmd/peer.go
+++ b/cmd/peer.go
@@ -37,7 +37,7 @@ func peerCmd(cmd *cobra.Command, args []string) {
 		err = db.Connect(globalConfig.DBConn)
 		checkErr(err)
 
-		combo := store.NewDBBackedStore(s3, db)
+		combo := store.NewDBBackedStore(s3, db, false)
 		peerServer = peer.NewServer(combo)
 	}
 
diff --git a/cmd/reflector.go b/cmd/reflector.go
index 7551f77..58c3f6f 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -127,11 +127,21 @@ func setupStore() store.BlobStore {
 			log.Fatalf("protocol is not recognized: %s", proxyProtocol)
 		}
 	} else {
-		s3Store := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
+		var s3Store *store.S3Store
+		if conf != "none" {
+			s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
+		}
 		if cloudFrontEndpoint != "" {
-			s = store.NewCloudFrontRWStore(store.NewCloudFrontROStore(cloudFrontEndpoint), s3Store)
-		} else {
+			cfs := store.NewCloudFrontROStore(cloudFrontEndpoint)
+			if s3Store != nil {
+				s = store.NewCloudFrontRWStore(cfs, s3Store)
+			} else {
+				s = cfs
+			}
+		} else if s3Store != nil {
 			s = s3Store
+		} else {
+			log.Fatalf("this configuration does not include a valid upstream source")
 		}
 	}
 
@@ -143,7 +153,7 @@ func setupStore() store.BlobStore {
 			log.Fatal(err)
 		}
 
-		s = store.NewDBBackedStore(s, db)
+		s = store.NewDBBackedStore(s, db, false)
 	}
 
 	return s
diff --git a/cmd/start.go b/cmd/start.go
index 4d4a56b..476e9fd 100644
--- a/cmd/start.go
+++ b/cmd/start.go
@@ -56,7 +56,7 @@ func startCmd(cmd *cobra.Command, args []string) {
 	err := db.Connect(globalConfig.DBConn)
 	checkErr(err)
 	s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
-	comboStore := store.NewDBBackedStore(s3, db)
+	comboStore := store.NewDBBackedStore(s3, db, false)
 
 	conf := prism.DefaultConf()
 
diff --git a/cmd/upload.go b/cmd/upload.go
index 636e824..e5e2cec 100644
--- a/cmd/upload.go
+++ b/cmd/upload.go
@@ -36,7 +36,7 @@ func uploadCmd(cmd *cobra.Command, args []string) {
 
 	st := store.NewDBBackedStore(
 		store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName),
-		db)
+		db, false)
 
 	uploader := reflector.NewUploader(db, st, uploadWorkers, uploadSkipExistsCheck, uploadDeleteBlobsAfterUpload)
 
diff --git a/go.mod b/go.mod
index 04d07c2..d3384a1 100644
--- a/go.mod
+++ b/go.mod
@@ -13,7 +13,7 @@ require (
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/golang/protobuf v1.4.2
 	github.com/google/btree v1.0.0 // indirect
-	github.com/google/gops v0.3.7
+	github.com/google/gops v0.3.7 // indirect
 	github.com/gorilla/mux v1.7.4
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/hashicorp/golang-lru v0.5.4
@@ -28,12 +28,13 @@ require (
 	github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec
 	github.com/lucas-clemente/quic-go v0.19.3
 	github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
-	github.com/prometheus/client_golang v0.9.2
+	github.com/prometheus/client_golang v0.9.3
 	github.com/sirupsen/logrus v1.4.2
 	github.com/spf13/afero v1.4.1 // indirect
 	github.com/spf13/cast v1.3.0
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3 // indirect
+	github.com/spf13/viper v1.7.1 // indirect
 	github.com/stretchr/testify v1.4.0
 	github.com/volatiletech/null v8.0.0+incompatible
 	go.uber.org/atomic v1.5.1
diff --git a/go.sum b/go.sum
index 18420b2..b7dbdb2 100644
--- a/go.sum
+++ b/go.sum
@@ -2,16 +2,31 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
 dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
 dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -24,7 +39,10 @@ github.com/aws/aws-sdk-go v1.16.11 h1:g/c7gJeVyHoXCxM2fddS85bPGVkBF8s2q8t3fyEleg
 github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bparli/lfuda-go v0.3.1 h1:nO9Szo627RC8/z+R+MMPBItNwHCOonchmpjQuQi8jVY=
 github.com/bparli/lfuda-go v0.3.1/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
@@ -43,17 +61,24 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 h1:t8KYCwSKsOEZBFELI4Pn/phbp38iJ1RRAkDFNin1aak=
 github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
@@ -68,23 +93,31 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
 github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
@@ -116,8 +149,12 @@ github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQl
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
 github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20190915194858-d3ddacdb130f h1:TyqzGm2z1h3AGhjOoRYyeLcW4WlW81MDQkWa+rx/000=
@@ -134,11 +171,19 @@ github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlI
 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
@@ -147,6 +192,7 @@ github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
@@ -155,10 +201,13 @@ github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce h1:xdsDDbiBDQTKASoGEZ+pEmF1OnWuu8AQ9I8iNbHNeno=
 github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
@@ -182,6 +231,7 @@ github.com/johntdyer/slack-go v0.0.0-20180213144715-95fac1160b22/go.mod h1:u0Jo4
 github.com/johntdyer/slackrus v0.0.0-20170926115001-3992f319fd0a/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07 h1:+kBG/8rjCa6vxJZbUjAiE4MQmBEBYc8nLEb51frnvBY=
 github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/8f3jowErEq4XyeypkCdvg5EeHkf0YCKCcq5Ybo=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -189,17 +239,20 @@ github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro=
 github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -234,6 +287,8 @@ github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcX
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs=
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
@@ -250,6 +305,8 @@ github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00v
 github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -258,6 +315,7 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
 github.com/nlopes/slack v0.5.0/go.mod h1:jVI4BBK3lSktibKahxBF74txcK2vyvkza1z/+rRnVAM=
@@ -265,6 +323,7 @@ github.com/nlopes/slack v0.6.0 h1:jt0jxVQGhssx1Ib7naAOZEZcGdtIhTzkP0nopK0AsRA=
 github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -293,16 +352,30 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
+github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rubenv/sql-migrate v0.0.0-20170330050058-38004e7a77f2/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -351,10 +424,15 @@ github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUr
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.1 h1:Lt3ihYMlE+lreX1GS4Qw4ZsNpYQLxIXKBTEOXm3nt6I=
 github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ=
 github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
@@ -364,18 +442,25 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig=
 github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/viper v1.0.2 h1:Ncr3ZIuJn322w2k1qmzXDnkLAdQMlJqBa9kfAH+irso=
 github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
+github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
@@ -386,15 +471,21 @@ github.com/volatiletech/null v8.0.0+incompatible h1:7wP8m5d/gZ6kW/9GnrLtMCRre2dl
 github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05VDbXbafIb0KU0MdVhCzmOQ=
 github.com/volatiletech/sqlboiler v3.4.0+incompatible h1:saQ6WxZ9wEJp33q3w/DHs7an7SYi1H7Yzf4/moxCbJU=
 github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
 github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
 go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -403,6 +494,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -412,13 +504,26 @@ golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -428,11 +533,14 @@ golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -445,6 +553,7 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -461,13 +570,18 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190520201301-c432e742b0af/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -487,19 +601,30 @@ golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -510,10 +635,17 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.2 h1:j8RI1yW0SkI+paT6uGwMlrMI/6zwYA6/CFil8rxOzGI=
 google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -522,13 +654,21 @@ google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoA
 google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
 google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
 google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -537,10 +677,12 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw=
@@ -548,10 +690,14 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8=
 gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79 h1:FpCr9V8wuOei4BAen+93HtVJ+XSi+KPbaPKm0Vj5R64=
 gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79/go.mod h1:gWkaRU7CoXpezCBWfWjm3999QqS+1pYPXGbqQCTMzo8=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
@@ -564,7 +710,10 @@ grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJd
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/peer/http3/server.go b/peer/http3/server.go
index 55ab5fc..aaadac0 100644
--- a/peer/http3/server.go
+++ b/peer/http3/server.go
@@ -65,7 +65,7 @@ func (s *Server) Start(address string) error {
 	log.Println("HTTP3 peer listening on " + address)
 	quicConf := &quic.Config{
 		HandshakeTimeout: 4 * time.Second,
-		MaxIdleTimeout:   10 * time.Second,
+		MaxIdleTimeout:   20 * time.Second,
 	}
 	r := mux.NewRouter()
 	r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
diff --git a/peer/http3/store.go b/peer/http3/store.go
index 4749ab0..76a58c1 100644
--- a/peer/http3/store.go
+++ b/peer/http3/store.go
@@ -32,7 +32,7 @@ func NewStore(opts StoreOpts) *Store {
 func (p *Store) getClient() (*Client, error) {
 	var qconf quic.Config
 	qconf.HandshakeTimeout = 4 * time.Second
-	qconf.MaxIdleTimeout = 10 * time.Second
+	qconf.MaxIdleTimeout = 20 * time.Second
 	pool, err := x509.SystemCertPool()
 	if err != nil {
 		return nil, err
diff --git a/store/dbbacked.go b/store/dbbacked.go
index 59c47b5..25cc446 100644
--- a/store/dbbacked.go
+++ b/store/dbbacked.go
@@ -14,15 +14,16 @@ import (
 
 // DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
 type DBBackedStore struct {
-	blobs     BlobStore
-	db        *db.SQL
-	blockedMu sync.RWMutex
-	blocked   map[string]bool
+	blobs        BlobStore
+	db           *db.SQL
+	blockedMu    sync.RWMutex
+	blocked      map[string]bool
+	deleteOnMiss bool
 }
 
 // NewDBBackedStore returns an initialized store pointer.
-func NewDBBackedStore(blobs BlobStore, db *db.SQL) *DBBackedStore {
-	return &DBBackedStore{blobs: blobs, db: db}
+func NewDBBackedStore(blobs BlobStore, db *db.SQL, deleteOnMiss bool) *DBBackedStore {
+	return &DBBackedStore{blobs: blobs, db: db, deleteOnMiss: deleteOnMiss}
 }
 
 const nameDBBacked = "db-backed"
@@ -44,6 +45,16 @@ func (d *DBBackedStore) Get(hash string) (stream.Blob, error) {
 	if !has {
 		return nil, ErrBlobNotFound
 	}
+	if d.deleteOnMiss {
+		b, err := d.blobs.Get(hash)
+		if err != nil && errors.Is(err, ErrBlobNotFound) {
+			e2 := d.Delete(hash)
+			if e2 != nil {
+				log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
+			}
+			return b, err
+		}
+	}
 
 	return d.blobs.Get(hash)
 }

From 869030fc584ce8db93f2817e56f604eefa52c391 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 22 Dec 2020 21:19:48 +0100
Subject: [PATCH 12/51] address some review comments

---
 store/lfuda.go               | 23 ++++++++++++-----------
 store/lfuda_test.go          |  3 ---
 store/lru.go                 |  4 +++-
 store/speedwalk/speedwalk.go |  1 -
 4 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/store/lfuda.go b/store/lfuda.go
index e33afa0..cfdb155 100644
--- a/store/lfuda.go
+++ b/store/lfuda.go
@@ -40,8 +40,6 @@ func NewLFUDAStore(component string, store BlobStore, maxSize float64) *LFUDASto
 
 const nameLFUDA = "lfuda"
 
-var fakeTrue = []byte{'t'}
-
 // Name is the cache type name
 func (l *LFUDAStore) Name() string { return nameLFUDA }
 
@@ -66,7 +64,7 @@ func (l *LFUDAStore) Get(hash string) (stream.Blob, error) {
 
 // Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
 func (l *LFUDAStore) Put(hash string, blob stream.Blob) error {
-	l.lfuda.Set(hash, fakeTrue)
+	l.lfuda.Set(hash, true)
 	has, _ := l.Has(hash)
 	if has {
 		err := l.store.Put(hash, blob)
@@ -77,14 +75,16 @@ func (l *LFUDAStore) Put(hash string, blob stream.Blob) error {
 	return nil
 }
 
-// PutSD stores the sd blob
+// PutSD stores the sd blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
 func (l *LFUDAStore) PutSD(hash string, blob stream.Blob) error {
-	err := l.store.PutSD(hash, blob)
-	if err != nil {
-		return err
+	l.lfuda.Set(hash, true)
+	has, _ := l.Has(hash)
+	if has {
+		err := l.store.PutSD(hash, blob)
+		if err != nil {
+			return err
+		}
 	}
-
-	l.lfuda.Set(hash, fakeTrue)
 	return nil
 }
 
@@ -109,12 +109,13 @@ func (l *LFUDAStore) loadExisting(store lister, maxItems int) error {
 	if err != nil {
 		return err
 	}
+	logrus.Infof("read %d files from disk", len(existing))
 
 	added := 0
 	for _, h := range existing {
-		l.lfuda.Set(h, fakeTrue)
+		l.lfuda.Set(h, true)
 		added++
-		if maxItems > 0 && added >= maxItems { // underlying cache is bigger than LRU cache
+		if maxItems > 0 && added >= maxItems { // underlying cache is bigger than the cache
 			break
 		}
 	}
diff --git a/store/lfuda_test.go b/store/lfuda_test.go
index 9b1eb58..d5eae47 100644
--- a/store/lfuda_test.go
+++ b/store/lfuda_test.go
@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
-	log "github.com/sirupsen/logrus"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -68,8 +67,6 @@ func TestFUDAStore_Eviction(t *testing.T) {
 
 	assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
 
-	keys := lfuda.lfuda.Keys()
-	log.Infof("%+v", keys)
 	for k, v := range map[string]bool{
 		"one":   false,
 		"two":   true,
diff --git a/store/lru.go b/store/lru.go
index edbe3a5..071e2b9 100644
--- a/store/lru.go
+++ b/store/lru.go
@@ -6,6 +6,7 @@ import (
 	"github.com/lbryio/reflector.go/internal/metrics"
 
 	golru "github.com/hashicorp/golang-lru"
+	"github.com/sirupsen/logrus"
 )
 
 // LRUStore adds a max cache size and LRU eviction to a BlobStore
@@ -106,11 +107,12 @@ func (l *LRUStore) Delete(hash string) error {
 
 // loadExisting imports existing blobs from the underlying store into the LRU cache
 func (l *LRUStore) loadExisting(store lister, maxItems int) error {
+	logrus.Infof("loading at most %d items", maxItems)
 	existing, err := store.list()
 	if err != nil {
 		return err
 	}
-
+	logrus.Infof("read %d files from disk", len(existing))
 	added := 0
 	for _, h := range existing {
 		l.lru.Add(h, true)
diff --git a/store/speedwalk/speedwalk.go b/store/speedwalk/speedwalk.go
index 23230df..9b899e4 100644
--- a/store/speedwalk/speedwalk.go
+++ b/store/speedwalk/speedwalk.go
@@ -83,6 +83,5 @@ func AllFiles(startDir string, basename bool) ([]string, error) {
 
 	close(pathChan)
 	pathWG.Wait()
-	logrus.Infoln("loaded LRU")
 	return paths, nil
 }

From 03304312e8e4f032bf135696e2113c1f3d5f5ff6 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Wed, 23 Dec 2020 06:04:42 +0100
Subject: [PATCH 13/51] add PoC for litedb to avoid all the overhead

---
 cmd/reflector.go       |  11 +-
 lite_db/db.go          | 348 +++++++++++++++++++++++++++++++++++++++++
 peer/http3/store.go    |   5 +
 peer/store.go          |   5 +
 store/caching.go       |   9 +-
 store/cloudfront_ro.go |   5 +
 store/cloudfront_rw.go |   7 +
 store/dbbacked.go      |   6 +
 store/disk.go          |   5 +
 store/lfuda.go         |   7 +-
 store/litedbbacked.go  | 160 +++++++++++++++++++
 store/lru.go           |   5 +
 store/memory.go        |   5 +
 store/noop.go          |   1 +
 store/s3.go            |   5 +
 store/singleflight.go  |   6 +
 store/store.go         |   2 +
 17 files changed, 589 insertions(+), 3 deletions(-)
 create mode 100644 lite_db/db.go
 create mode 100644 store/litedbbacked.go

diff --git a/cmd/reflector.go b/cmd/reflector.go
index 58c3f6f..3bb6b4e 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -10,6 +10,7 @@ import (
 
 	"github.com/lbryio/reflector.go/db"
 	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/lite_db"
 	"github.com/lbryio/reflector.go/meta"
 	"github.com/lbryio/reflector.go/peer"
 	"github.com/lbryio/reflector.go/peer/http3"
@@ -101,6 +102,8 @@ func reflectorCmd(cmd *cobra.Command, args []string) {
 	metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
 	metricsServer.Start()
 	defer metricsServer.Shutdown()
+	defer underlyingStore.Shutdown()
+	defer outerStore.Shutdown()
 
 	interruptChan := make(chan os.Signal, 1)
 	signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
@@ -171,10 +174,16 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 		if err != nil {
 			log.Fatal(err)
 		}
+		localDb := new(lite_db.SQL)
+		localDb.TrackAccessTime = true
+		err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
+		if err != nil {
+			log.Fatal(err)
+		}
 		wrapped = store.NewCachingStore(
 			"reflector",
 			wrapped,
-			store.NewLFUDAStore("hdd", store.NewDiskStore(diskCachePath, 2), realCacheSize),
+			store.NewLiteDBBackedStore("hdd", store.NewDiskStore(diskCachePath, 2), localDb, int(realCacheSize)),
 		)
 	}
 
diff --git a/lite_db/db.go b/lite_db/db.go
new file mode 100644
index 0000000..6be9837
--- /dev/null
+++ b/lite_db/db.go
@@ -0,0 +1,348 @@
+package lite_db
+
+import (
+	"database/sql"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	qt "github.com/lbryio/lbry.go/v2/extras/query"
+
+	"github.com/go-sql-driver/mysql"
+	_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
+	log "github.com/sirupsen/logrus"
+	"github.com/volatiletech/null"
+)
+
+// SdBlob is a special blob that contains information on the rest of the blobs in the stream
+type SdBlob struct {
+	StreamName string `json:"stream_name"`
+	Blobs      []struct {
+		Length   int    `json:"length"`
+		BlobNum  int    `json:"blob_num"`
+		BlobHash string `json:"blob_hash,omitempty"`
+		IV       string `json:"iv"`
+	} `json:"blobs"`
+	StreamType        string `json:"stream_type"`
+	Key               string `json:"key"`
+	SuggestedFileName string `json:"suggested_file_name"`
+	StreamHash        string `json:"stream_hash"`
+}
+
+// SQL implements the DB interface
+type SQL struct {
+	conn *sql.DB
+
+	TrackAccessTime bool
+}
+
+func logQuery(query string, args ...interface{}) {
+	s, err := qt.InterpolateParams(query, args...)
+	if err != nil {
+		log.Errorln(err)
+	} else {
+		log.Debugln(s)
+	}
+}
+
+// Connect will create a connection to the database
+func (s *SQL) Connect(dsn string) error {
+	var err error
+	// interpolateParams is necessary. otherwise uploading a stream with thousands of blobs
+	// will hit MySQL's max_prepared_stmt_count limit because the prepared statements are all
+	// opened inside a transaction. closing them manually doesn't seem to help
+	dsn += "?parseTime=1&collation=utf8mb4_unicode_ci&interpolateParams=1"
+	s.conn, err = sql.Open("mysql", dsn)
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	s.conn.SetMaxIdleConns(12)
+
+	return errors.Err(s.conn.Ping())
+}
+
+// AddBlob adds a blob to the database.
+func (s *SQL) AddBlob(hash string, length int) error {
+	if s.conn == nil {
+		return errors.Err("not connected")
+	}
+
+	_, err := s.insertBlob(hash, length)
+	return err
+}
+
+func (s *SQL) insertBlob(hash string, length int) (int64, error) {
+	if length <= 0 {
+		return 0, errors.Err("length must be positive")
+	}
+	const isStored = true
+	now := time.Now()
+	args := []interface{}{hash, isStored, length, now}
+	blobID, err := s.exec(
+		"INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at=VALUES(last_accessed_at)",
+		args...,
+	)
+	if err != nil {
+		return 0, err
+	}
+
+	if blobID == 0 {
+		err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
+		if err != nil {
+			return 0, errors.Err(err)
+		}
+		if blobID == 0 {
+			return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
+		}
+	}
+
+	return blobID, nil
+}
+
+// HasBlob checks if the database contains the blob information.
+func (s *SQL) HasBlob(hash string) (bool, error) {
+	exists, err := s.HasBlobs([]string{hash})
+	if err != nil {
+		return false, err
+	}
+	return exists[hash], nil
+}
+
+// HasBlobs checks if the database contains the set of blobs and returns a bool map.
+func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
+	exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
+	s.touch(streamsNeedingTouch)
+	return exists, err
+}
+
+func (s *SQL) touch(blobIDs []uint64) error {
+	if len(blobIDs) == 0 {
+		return nil
+	}
+
+	query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
+	args := make([]interface{}, len(blobIDs)+1)
+	args[0] = time.Now()
+	for i := range blobIDs {
+		args[i+1] = blobIDs[i]
+	}
+
+	startTime := time.Now()
+	_, err := s.exec(query, args...)
+	log.Debugf("blobs access query touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
+	return errors.Err(err)
+}
+
+func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
+	if s.conn == nil {
+		return nil, nil, errors.Err("not connected")
+	}
+
+	var (
+		hash           string
+		blobID         uint64
+		lastAccessedAt null.Time
+	)
+
+	var needsTouch []uint64
+	exists := make(map[string]bool)
+
+	touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
+	maxBatchSize := 10000
+	doneIndex := 0
+
+	for len(hashes) > doneIndex {
+		sliceEnd := doneIndex + maxBatchSize
+		if sliceEnd > len(hashes) {
+			sliceEnd = len(hashes)
+		}
+		log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
+		batch := hashes[doneIndex:sliceEnd]
+
+		// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
+
+		query := `SELECT hash, id, last_accessed_at
+FROM blob_ 
+WHERE is_stored = ? and hash IN (` + qt.Qs(len(batch)) + `)`
+		args := make([]interface{}, len(batch)+1)
+		args[0] = true
+		for i := range batch {
+			args[i+1] = batch[i]
+		}
+
+		logQuery(query, args...)
+
+		err := func() error {
+			startTime := time.Now()
+			rows, err := s.conn.Query(query, args...)
+			log.Debugf("hashes query took %s", time.Since(startTime))
+			if err != nil {
+				return errors.Err(err)
+			}
+			defer closeRows(rows)
+
+			for rows.Next() {
+				err := rows.Scan(&hash, &blobID, &lastAccessedAt)
+				if err != nil {
+					return errors.Err(err)
+				}
+				exists[hash] = true
+				if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) {
+					needsTouch = append(needsTouch, blobID)
+				}
+			}
+
+			err = rows.Err()
+			if err != nil {
+				return errors.Err(err)
+			}
+
+			doneIndex += len(batch)
+			return nil
+		}()
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	return exists, needsTouch, nil
+}
+
+// Delete will remove the blob from the db
+func (s *SQL) Delete(hash string) error {
+	_, err := s.exec("UPDATE blob_ set is_stored = ? WHERE hash = ?", 0, hash)
+	return errors.Err(err)
+}
+
+// AddSDBlob insert the SD blob and all the content blobs. The content blobs are marked as "not stored",
+// but they are tracked so reflector knows what it is missing.
+func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int) error {
+	if s.conn == nil {
+		return errors.Err("not connected")
+	}
+
+	_, err := s.insertBlob(sdHash, sdBlobLength)
+	return err
+}
+
+// GetHashRange gets the smallest and biggest hashes in the db
+func (s *SQL) GetLRUBlobs(maxBlobs int) ([]string, error) {
+	if s.conn == nil {
+		return nil, errors.Err("not connected")
+	}
+
+	query := "SELECT hash from blob_ where is_stored = ? order by last_accessed_at limit ?"
+	const isStored = true
+	logQuery(query, isStored, maxBlobs)
+	rows, err := s.conn.Query(query, isStored, maxBlobs)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer closeRows(rows)
+	blobs := make([]string, 0, maxBlobs)
+	for rows.Next() {
+		var hash string
+		err := rows.Scan(&hash)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		blobs = append(blobs, hash)
+	}
+	return blobs, nil
+}
+
+func (s *SQL) AllBlobs() ([]string, error) {
+	if s.conn == nil {
+		return nil, errors.Err("not connected")
+	}
+
+	query := "SELECT hash from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
+	const isStored = true
+	logQuery(query, isStored)
+	rows, err := s.conn.Query(query, isStored)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer closeRows(rows)
+	totalBlobs, err := s.BlobsCount()
+	if err != nil {
+		return nil, err
+	}
+	blobs := make([]string, 0, totalBlobs)
+	for rows.Next() {
+		var hash string
+		err := rows.Scan(&hash)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		blobs = append(blobs, hash)
+	}
+	return blobs, nil
+}
+
+func (s *SQL) BlobsCount() (int, error) {
+	if s.conn == nil {
+		return 0, errors.Err("not connected")
+	}
+
+	query := "SELECT count(id) from blob_ where is_stored = ?" //TODO: maybe sorting them makes more sense?
+	const isStored = true
+	logQuery(query, isStored)
+	var count int
+	err := s.conn.QueryRow(query, isStored).Scan(&count)
+	return count, errors.Err(err)
+}
+
+func closeRows(rows *sql.Rows) {
+	if rows != nil {
+		err := rows.Close()
+		if err != nil {
+			log.Error("error closing rows: ", err)
+		}
+	}
+}
+
+func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
+	logQuery(query, args...)
+	attempt, maxAttempts := 0, 3
+Retry:
+	attempt++
+	result, err := s.conn.Exec(query, args...)
+	if isLockTimeoutError(err) {
+		if attempt <= maxAttempts {
+			//Error 1205: Lock wait timeout exceeded; try restarting transaction
+			goto Retry
+		}
+		err = errors.Prefix("Lock timeout for query "+query, err)
+	}
+
+	if err != nil {
+		return 0, errors.Err(err)
+	}
+
+	lastID, err := result.LastInsertId()
+	return lastID, errors.Err(err)
+}
+
+func isLockTimeoutError(err error) bool {
+	e, ok := err.(*mysql.MySQLError)
+	return ok && e != nil && e.Number == 1205
+}
+
+/*  SQL schema
+
+in prod make sure you use latin1 or utf8 charset, NOT utf8mb4. that's a waste of space.
+
+CREATE TABLE `blob_` (
+  `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+  `hash` char(96) NOT NULL,
+  `is_stored` tinyint(1) NOT NULL DEFAULT '0',
+  `length` bigint unsigned DEFAULT NULL,
+  `last_accessed_at` datetime DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `id` (`id`),
+  UNIQUE KEY `blob_hash_idx` (`hash`),
+  KEY `blob_last_accessed_idx` (`last_accessed_at`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1
+
+*/
diff --git a/peer/http3/store.go b/peer/http3/store.go
index 76a58c1..b681443 100644
--- a/peer/http3/store.go
+++ b/peer/http3/store.go
@@ -91,3 +91,8 @@ func (p *Store) PutSD(hash string, blob stream.Blob) error {
 func (p *Store) Delete(hash string) error {
 	panic("http3Store cannot put or delete blobs")
 }
+
+// Delete is not supported
+func (p *Store) Shutdown() {
+	return
+}
diff --git a/peer/store.go b/peer/store.go
index b8abedd..3ef9622 100644
--- a/peer/store.go
+++ b/peer/store.go
@@ -66,3 +66,8 @@ func (p *Store) PutSD(hash string, blob stream.Blob) error {
 func (p *Store) Delete(hash string) error {
 	panic("PeerStore cannot put or delete blobs")
 }
+
+// Delete is not supported
+func (p *Store) Shutdown() {
+	return
+}
diff --git a/store/caching.go b/store/caching.go
index c8e7d45..53b692c 100644
--- a/store/caching.go
+++ b/store/caching.go
@@ -68,7 +68,7 @@ func (c *CachingStore) Get(hash string) (stream.Blob, error) {
 	go func() {
 		err = c.cache.Put(hash, blob)
 		if err != nil {
-			log.Errorf("error saving blob to underlying cache: %s", err.Error())
+			log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
 		}
 	}()
 	return blob, nil
@@ -100,3 +100,10 @@ func (c *CachingStore) Delete(hash string) error {
 	}
 	return c.cache.Delete(hash)
 }
+
+// Shutdown shuts down the store gracefully
+func (c *CachingStore) Shutdown() {
+	c.origin.Shutdown()
+	c.cache.Shutdown()
+	return
+}
diff --git a/store/cloudfront_ro.go b/store/cloudfront_ro.go
index a914285..3dcffc0 100644
--- a/store/cloudfront_ro.go
+++ b/store/cloudfront_ro.go
@@ -103,3 +103,8 @@ func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
 func (c *CloudFrontROStore) Delete(_ string) error {
 	panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore")
 }
+
+// Shutdown shuts down the store gracefully
+func (c *CloudFrontROStore) Shutdown() {
+	return
+}
diff --git a/store/cloudfront_rw.go b/store/cloudfront_rw.go
index ee771da..6de64af 100644
--- a/store/cloudfront_rw.go
+++ b/store/cloudfront_rw.go
@@ -48,3 +48,10 @@ func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error {
 func (c *CloudFrontRWStore) Delete(hash string) error {
 	return c.s3.Delete(hash)
 }
+
+// Shutdown shuts down the store gracefully
+func (c *CloudFrontRWStore) Shutdown() {
+	c.s3.Shutdown()
+	c.cf.Shutdown()
+	return
+}
diff --git a/store/dbbacked.go b/store/dbbacked.go
index 25cc446..0ebaaed 100644
--- a/store/dbbacked.go
+++ b/store/dbbacked.go
@@ -193,3 +193,9 @@ func (d *DBBackedStore) initBlocked() error {
 
 	return err
 }
+
+// Shutdown shuts down the store gracefully
+func (d *DBBackedStore) Shutdown() {
+	d.blobs.Shutdown()
+	return
+}
diff --git a/store/disk.go b/store/disk.go
index 2234971..03cc393 100644
--- a/store/disk.go
+++ b/store/disk.go
@@ -147,3 +147,8 @@ func (d *DiskStore) initOnce() error {
 	d.initialized = true
 	return nil
 }
+
+// Shutdown shuts down the store gracefully
+func (d *DiskStore) Shutdown() {
+	return
+}
diff --git a/store/lfuda.go b/store/lfuda.go
index cfdb155..0eb7296 100644
--- a/store/lfuda.go
+++ b/store/lfuda.go
@@ -109,7 +109,7 @@ func (l *LFUDAStore) loadExisting(store lister, maxItems int) error {
 	if err != nil {
 		return err
 	}
-	logrus.Infof("read %d files from disk", len(existing))
+	logrus.Infof("read %d files from underlying store", len(existing))
 
 	added := 0
 	for _, h := range existing {
@@ -121,3 +121,8 @@ func (l *LFUDAStore) loadExisting(store lister, maxItems int) error {
 	}
 	return nil
 }
+
+// Shutdown shuts down the store gracefully
+func (l *LFUDAStore) Shutdown() {
+	return
+}
diff --git a/store/litedbbacked.go b/store/litedbbacked.go
new file mode 100644
index 0000000..93790d6
--- /dev/null
+++ b/store/litedbbacked.go
@@ -0,0 +1,160 @@
+package store
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/reflector.go/db"
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/lite_db"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
+type LiteDBBackedStore struct {
+	blobs     BlobStore
+	db        *lite_db.SQL
+	maxItems  int
+	stopper   *stop.Stopper
+	component string
+}
+
+// NewDBBackedStore returns an initialized store pointer.
+func NewLiteDBBackedStore(component string, blobs BlobStore, db *lite_db.SQL, maxItems int) *LiteDBBackedStore {
+	instance := &LiteDBBackedStore{blobs: blobs, db: db, maxItems: maxItems, stopper: stop.New(), component: component}
+	go func() {
+		instance.selfClean()
+	}()
+	return instance
+}
+
+const nameLiteDBBacked = "lite-db-backed"
+
+// Name is the cache type name
+func (d *LiteDBBackedStore) Name() string { return nameDBBacked }
+
+// Has returns true if the blob is in the store
+func (d *LiteDBBackedStore) Has(hash string) (bool, error) {
+	return d.db.HasBlob(hash)
+}
+
+// Get gets the blob
+func (d *LiteDBBackedStore) Get(hash string) (stream.Blob, error) {
+	has, err := d.db.HasBlob(hash)
+	if err != nil {
+		return nil, err
+	}
+	if !has {
+		return nil, ErrBlobNotFound
+	}
+	b, err := d.blobs.Get(hash)
+	if err != nil && errors.Is(err, ErrBlobNotFound) {
+		e2 := d.db.Delete(hash)
+		if e2 != nil {
+			log.Errorf("error while deleting blob from db: %s", errors.FullTrace(e2))
+		}
+		return b, err
+	}
+
+	return d.blobs.Get(hash)
+}
+
+// Put stores the blob in the S3 store and stores the blob information in the DB.
+func (d *LiteDBBackedStore) Put(hash string, blob stream.Blob) error {
+	err := d.blobs.Put(hash, blob)
+	if err != nil {
+		return err
+	}
+
+	return d.db.AddBlob(hash, len(blob))
+}
+
+// PutSD stores the SDBlob in the S3 store. It will return an error if the sd blob is missing the stream hash or if
+// there is an error storing the blob information in the DB.
+func (d *LiteDBBackedStore) PutSD(hash string, blob stream.Blob) error {
+	var blobContents db.SdBlob
+	err := json.Unmarshal(blob, &blobContents)
+	if err != nil {
+		return errors.Err(err)
+	}
+	if blobContents.StreamHash == "" {
+		return errors.Err("sd blob is missing stream hash")
+	}
+
+	err = d.blobs.PutSD(hash, blob)
+	if err != nil {
+		return err
+	}
+
+	return d.db.AddSDBlob(hash, len(blob))
+}
+
+func (d *LiteDBBackedStore) Delete(hash string) error {
+	err := d.blobs.Delete(hash)
+	if err != nil {
+		return err
+	}
+
+	return d.db.Delete(hash)
+}
+
+// list returns the hashes of blobs that already exist in the database
+func (d *LiteDBBackedStore) list() ([]string, error) {
+	blobs, err := d.db.AllBlobs()
+	return blobs, err
+}
+
+func (d *LiteDBBackedStore) selfClean() {
+	d.stopper.Add(1)
+	defer d.stopper.Done()
+	lastCleanup := time.Now()
+	const cleanupInterval = 10 * time.Second
+	for {
+		select {
+		case <-d.stopper.Ch():
+			log.Infoln("stopping self cleanup")
+			return
+		default:
+			time.Sleep(1 * time.Second)
+		}
+		if time.Since(lastCleanup) < cleanupInterval {
+			continue
+		}
+		blobsCount, err := d.db.BlobsCount()
+		if err != nil {
+			log.Errorf(errors.FullTrace(err))
+		}
+		if blobsCount >= d.maxItems {
+			itemsToDelete := blobsCount / 100 * 10
+			blobs, err := d.db.GetLRUBlobs(itemsToDelete)
+			if err != nil {
+				log.Errorf(errors.FullTrace(err))
+			}
+			for _, hash := range blobs {
+				select {
+				case <-d.stopper.Ch():
+					return
+				default:
+
+				}
+				err = d.Delete(hash)
+				if err != nil {
+					log.Errorf(errors.FullTrace(err))
+				}
+				metrics.CacheLRUEvictCount.With(metrics.CacheLabels(d.Name(), d.component)).Inc()
+			}
+		}
+		lastCleanup = time.Now()
+	}
+}
+
+// Shutdown shuts down the store gracefully
+func (d *LiteDBBackedStore) Shutdown() {
+	d.stopper.StopAndWait()
+	return
+}
diff --git a/store/lru.go b/store/lru.go
index 071e2b9..ce0d83d 100644
--- a/store/lru.go
+++ b/store/lru.go
@@ -123,3 +123,8 @@ func (l *LRUStore) loadExisting(store lister, maxItems int) error {
 	}
 	return nil
 }
+
+// Shutdown shuts down the store gracefully
+func (l *LRUStore) Shutdown() {
+	return
+}
diff --git a/store/memory.go b/store/memory.go
index f462a8d..62b93b6 100644
--- a/store/memory.go
+++ b/store/memory.go
@@ -71,3 +71,8 @@ func (m *MemStore) Debug() map[string]stream.Blob {
 	defer m.mu.RUnlock()
 	return m.blobs
 }
+
+// Shutdown shuts down the store gracefully
+func (m *MemStore) Shutdown() {
+	return
+}
diff --git a/store/noop.go b/store/noop.go
index 9e5d815..2792a0d 100644
--- a/store/noop.go
+++ b/store/noop.go
@@ -13,3 +13,4 @@ func (n *NoopStore) Get(_ string) (stream.Blob, error)   { return nil, nil }
 func (n *NoopStore) Put(_ string, _ stream.Blob) error   { return nil }
 func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil }
 func (n *NoopStore) Delete(_ string) error               { return nil }
+func (n *NoopStore) Shutdown()                           { return }
diff --git a/store/s3.go b/store/s3.go
index 53451be..02a23c5 100644
--- a/store/s3.go
+++ b/store/s3.go
@@ -158,3 +158,8 @@ func (s *S3Store) initOnce() error {
 	s.session = sess
 	return nil
 }
+
+// Shutdown shuts down the store gracefully
+func (s *S3Store) Shutdown() {
+	return
+}
diff --git a/store/singleflight.go b/store/singleflight.go
index cca1e2f..7d0139d 100644
--- a/store/singleflight.go
+++ b/store/singleflight.go
@@ -65,3 +65,9 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) {
 		return blob, nil
 	}
 }
+
+// Shutdown shuts down the store gracefully
+func (s *singleflightStore) Shutdown() {
+	s.BlobStore.Shutdown()
+	return
+}
diff --git a/store/store.go b/store/store.go
index 200ff61..36d388d 100644
--- a/store/store.go
+++ b/store/store.go
@@ -19,6 +19,8 @@ type BlobStore interface {
 	PutSD(hash string, blob stream.Blob) error
 	// Delete the blob from the store.
 	Delete(hash string) error
+	// Shutdown the store gracefully
+	Shutdown()
 }
 
 // Blocklister is a store that supports blocking blobs to prevent their inclusion in the store.

From 3a1d9d33041b9d43adf2a08cb83e17014ce8a86e Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Wed, 23 Dec 2020 17:08:13 -0500
Subject: [PATCH 14/51] something like this

---
 cmd/reflector.go       |  74 ++++++++++--
 db/db.go               | 265 ++++++++++++++++++++++++++++++++++++-----
 go.mod                 |   4 +-
 peer/http3/store.go    |   5 +
 peer/store.go          |   5 +
 store/caching.go       |   9 +-
 store/caching_test.go  |   4 +
 store/cloudfront_ro.go |   5 +
 store/cloudfront_rw.go |   7 ++
 store/dbbacked.go      |   9 +-
 store/disk.go          |   5 +
 store/lfuda.go         |   7 +-
 store/lru.go           |   5 +
 store/memory.go        |   5 +
 store/noop.go          |   1 +
 store/s3.go            |   5 +
 store/singleflight.go  |   6 +
 store/store.go         |   2 +
 18 files changed, 380 insertions(+), 43 deletions(-)

diff --git a/cmd/reflector.go b/cmd/reflector.go
index 58c3f6f..47a27ff 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -8,6 +8,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/reflector.go/db"
 	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/meta"
@@ -67,10 +69,11 @@ func init() {
 
 func reflectorCmd(cmd *cobra.Command, args []string) {
 	log.Printf("reflector %s", meta.VersionString())
+	cleanerStopper := stop.New()
 
 	// the blocklist logic requires the db backed store to be the outer-most store
 	underlyingStore := setupStore()
-	outerStore := wrapWithCache(underlyingStore)
+	outerStore := wrapWithCache(underlyingStore, cleanerStopper)
 
 	if !disableUploads {
 		reflectorServer := reflector.NewServer(underlyingStore)
@@ -101,11 +104,14 @@ func reflectorCmd(cmd *cobra.Command, args []string) {
 	metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
 	metricsServer.Start()
 	defer metricsServer.Shutdown()
+	defer outerStore.Shutdown()
+	defer underlyingStore.Shutdown()
 
 	interruptChan := make(chan os.Signal, 1)
 	signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
 	<-interruptChan
 	// deferred shutdowns happen now
+	cleanerStopper.StopAndWait()
 }
 
 func setupStore() store.BlobStore {
@@ -146,20 +152,20 @@ func setupStore() store.BlobStore {
 	}
 
 	if useDB {
-		db := new(db.SQL)
-		db.TrackAccessTime = true
-		err := db.Connect(globalConfig.DBConn)
+		dbInst := new(db.SQL)
+		dbInst.TrackAccess = db.TrackAccessStreams
+		err := dbInst.Connect(globalConfig.DBConn)
 		if err != nil {
 			log.Fatal(err)
 		}
 
-		s = store.NewDBBackedStore(s, db, false)
+		s = store.NewDBBackedStore(s, dbInst, false)
 	}
 
 	return s
 }
 
-func wrapWithCache(s store.BlobStore) store.BlobStore {
+func wrapWithCache(s store.BlobStore, cleanerStopper *stop.Group) store.BlobStore {
 	wrapped := s
 
 	diskCacheMaxSize, diskCachePath := diskCacheParams(reflectorCmdDiskCache)
@@ -171,11 +177,20 @@ func wrapWithCache(s store.BlobStore) store.BlobStore {
 		if err != nil {
 			log.Fatal(err)
 		}
+
+		localDb := new(db.SQL)
+		localDb.TrackAccess = db.TrackAccessBlobs
+		err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
+		if err != nil {
+			log.Fatal(err)
+		}
 		wrapped = store.NewCachingStore(
 			"reflector",
 			wrapped,
-			store.NewLFUDAStore("hdd", store.NewDiskStore(diskCachePath, 2), realCacheSize),
+			store.NewDBBackedStore(store.NewDiskStore(diskCachePath, 2), localDb, false),
 		)
+
+		go cleanOldestBlobs(int(realCacheSize), localDb, wrapped, cleanerStopper)
 	}
 
 	diskCacheMaxSize, diskCachePath = diskCacheParams(bufferReflectorCmdDiskCache)
@@ -229,3 +244,48 @@ func diskCacheParams(diskParams string) (int, string) {
 	}
 	return int(maxSize), path
 }
+
+func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) {
+	const cleanupInterval = 10 * time.Second
+
+	for {
+		select {
+		case <-stopper.Ch():
+			log.Infoln("stopping self cleanup")
+			return
+		case <-time.After(cleanupInterval):
+			err := doClean(maxItems, db, store, stopper)
+			if err != nil {
+				log.Error(errors.FullTrace(err))
+			}
+		}
+	}
+}
+
+func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) error {
+	blobsCount, err := db.Count()
+	if err != nil {
+		return err
+	}
+
+	if blobsCount >= maxItems {
+		itemsToDelete := blobsCount / 10
+		blobs, err := db.LeastRecentlyAccessedHashes(itemsToDelete)
+		if err != nil {
+			return err
+		}
+
+		for _, hash := range blobs {
+			select {
+			case <-stopper.Ch():
+				return nil
+			default:
+			}
+
+			err = store.Delete(hash)
+			if err != nil {
+				return err
+			}
+		}
+	}
+}
diff --git a/db/db.go b/db/db.go
index c3bea24..3ac2dab 100644
--- a/db/db.go
+++ b/db/db.go
@@ -30,11 +30,26 @@ type SdBlob struct {
 	StreamHash        string `json:"stream_hash"`
 }
 
+type trackAccess int
+
+const (
+	// Don't track accesses
+	TrackAccessNone trackAccess = iota
+	// Track accesses at the stream level
+	TrackAccessStreams
+	// Track accesses at the blob level
+	TrackAccessBlobs
+)
+
 // SQL implements the DB interface
 type SQL struct {
 	conn *sql.DB
 
-	TrackAccessTime bool
+	// Track the approx last time a blob or stream was accessed
+	TrackAccess trackAccess
+
+	// Instead of deleting a blob, marked it as not stored in the db
+	SoftDelete bool
 }
 
 func logQuery(query string, args ...interface{}) {
@@ -78,11 +93,19 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
 		return 0, errors.Err("length must be positive")
 	}
 
-	args := []interface{}{hash, isStored, length}
-	blobID, err := s.exec(
-		"INSERT INTO blob_ (hash, is_stored, length) VALUES ("+qt.Qs(len(args))+") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))",
-		args...,
+	var (
+		q    string
+		args []interface{}
 	)
+	if s.TrackAccess == TrackAccessBlobs {
+		args = []interface{}{hash, isStored, length, time.Now()}
+		q = "INSERT INTO blob_ (hash, is_stored, length, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored)), last_accessed_at = VALUES(last_accessed_at)"
+	} else {
+		args = []interface{}{hash, isStored, length}
+		q = "INSERT INTO blob_ (hash, is_stored, length) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))"
+	}
+
+	blobID, err := s.exec(q, args...)
 	if err != nil {
 		return 0, err
 	}
@@ -95,17 +118,33 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
 		if blobID == 0 {
 			return 0, errors.Err("blob ID is 0 even after INSERTing and SELECTing")
 		}
+
+		if s.TrackAccess == TrackAccessBlobs {
+			err := s.touchBlobs([]uint64{uint64(blobID)})
+			if err != nil {
+				return 0, errors.Err(err)
+			}
+		}
 	}
 
 	return blobID, nil
 }
 
 func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
-	args := []interface{}{hash, sdBlobID, time.Now()}
-	streamID, err := s.exec(
-		"INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES ("+qt.Qs(len(args))+")",
-		args...,
+	var (
+		q    string
+		args []interface{}
 	)
+
+	if s.TrackAccess == TrackAccessStreams {
+		args = []interface{}{hash, sdBlobID, time.Now()}
+		q = "INSERT IGNORE INTO stream (hash, sd_blob_id, last_accessed_at) VALUES (" + qt.Qs(len(args)) + ")"
+	} else {
+		args = []interface{}{hash, sdBlobID}
+		q = "INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (" + qt.Qs(len(args)) + ")"
+	}
+
+	streamID, err := s.exec(q, args...)
 	if err != nil {
 		return 0, errors.Err(err)
 	}
@@ -119,8 +158,8 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
 			return 0, errors.Err("stream ID is 0 even after INSERTing and SELECTing")
 		}
 
-		if s.TrackAccessTime {
-			err := s.touch([]uint64{uint64(streamID)})
+		if s.TrackAccess == TrackAccessStreams {
+			err := s.touchStreams([]uint64{uint64(streamID)})
 			if err != nil {
 				return 0, errors.Err(err)
 			}
@@ -140,12 +179,36 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
 
 // HasBlobs checks if the database contains the set of blobs and returns a bool map.
 func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
-	exists, streamsNeedingTouch, err := s.hasBlobs(hashes)
-	s.touch(streamsNeedingTouch)
+	exists, idsNeedingTouch, err := s.hasBlobs(hashes)
+
+	if s.TrackAccess == TrackAccessBlobs {
+		s.touchBlobs(idsNeedingTouch)
+	} else if s.TrackAccess == TrackAccessStreams {
+		s.touchStreams(idsNeedingTouch)
+	}
+
 	return exists, err
 }
 
-func (s *SQL) touch(streamIDs []uint64) error {
+func (s *SQL) touchBlobs(blobIDs []uint64) error {
+	if len(blobIDs) == 0 {
+		return nil
+	}
+
+	query := "UPDATE blob_ SET last_accessed_at = ? WHERE id IN (" + qt.Qs(len(blobIDs)) + ")"
+	args := make([]interface{}, len(blobIDs)+1)
+	args[0] = time.Now()
+	for i := range blobIDs {
+		args[i+1] = blobIDs[i]
+	}
+
+	startTime := time.Now()
+	_, err := s.exec(query, args...)
+	log.Debugf("touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
+	return errors.Err(err)
+}
+
+func (s *SQL) touchStreams(streamIDs []uint64) error {
 	if len(streamIDs) == 0 {
 		return nil
 	}
@@ -159,7 +222,7 @@ func (s *SQL) touch(streamIDs []uint64) error {
 
 	startTime := time.Now()
 	_, err := s.exec(query, args...)
-	log.Debugf("stream access query touched %d streams and took %s", len(streamIDs), time.Since(startTime))
+	log.Debugf("touched %d streams and took %s", len(streamIDs), time.Since(startTime))
 	return errors.Err(err)
 }
 
@@ -169,9 +232,9 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
 	}
 
 	var (
-		hash           string
-		streamID       uint64
-		lastAccessedAt null.Time
+		hash             string
+		blobID, streamID uint64
+		lastAccessedAt   null.Time
 	)
 
 	var needsTouch []uint64
@@ -189,17 +252,23 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
 		log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
 		batch := hashes[doneIndex:sliceEnd]
 
-		// TODO: this query doesn't work for SD blobs, which are not in the stream_blob table
+		var lastAccessedAtSelect string
+		if s.TrackAccess == TrackAccessBlobs {
+			lastAccessedAtSelect = "b.last_accessed_at"
+		} else if s.TrackAccess == TrackAccessStreams {
+			lastAccessedAtSelect = "s.last_accessed_at"
+		} else {
+			lastAccessedAtSelect = "NULL"
+		}
 
-		query := `SELECT b.hash, s.id, s.last_accessed_at
+		query := `SELECT b.hash, b.id, s.id, ` + lastAccessedAtSelect + `
 FROM blob_ b
 LEFT JOIN stream_blob sb ON b.id = sb.blob_id
-INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
-WHERE b.is_stored = ? and b.hash IN (` + qt.Qs(len(batch)) + `)`
-		args := make([]interface{}, len(batch)+1)
-		args[0] = true
+LEFT JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
+WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
+		args := make([]interface{}, len(batch))
 		for i := range batch {
-			args[i+1] = batch[i]
+			args[i] = batch[i]
 		}
 
 		logQuery(query, args...)
@@ -214,13 +283,17 @@ WHERE b.is_stored = ? and b.hash IN (` + qt.Qs(len(batch)) + `)`
 			defer closeRows(rows)
 
 			for rows.Next() {
-				err := rows.Scan(&hash, &streamID, &lastAccessedAt)
+				err := rows.Scan(&hash, &blobID, &streamID, &lastAccessedAt)
 				if err != nil {
 					return errors.Err(err)
 				}
 				exists[hash] = true
-				if s.TrackAccessTime && (!lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline)) {
-					needsTouch = append(needsTouch, streamID)
+				if !lastAccessedAt.Valid || lastAccessedAt.Time.Before(touchDeadline) {
+					if s.TrackAccess == TrackAccessBlobs {
+						needsTouch = append(needsTouch, blobID)
+					} else if s.TrackAccess == TrackAccessStreams {
+						needsTouch = append(needsTouch, streamID)
+					}
 				}
 			}
 
@@ -240,8 +313,14 @@ WHERE b.is_stored = ? and b.hash IN (` + qt.Qs(len(batch)) + `)`
 	return exists, needsTouch, nil
 }
 
-// Delete will remove the blob from the db
+// Delete will remove (or soft-delete) the blob from the db
+// NOTE: If SoftDelete is enabled, streams will never be deleted
 func (s *SQL) Delete(hash string) error {
+	if s.SoftDelete {
+		_, err := s.exec("UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash)
+		return errors.Err(err)
+	}
+
 	_, err := s.exec("DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash)
 	if err != nil {
 		return errors.Err(err)
@@ -251,6 +330,88 @@ func (s *SQL) Delete(hash string) error {
 	return errors.Err(err)
 }
 
+// GetHashRange gets the smallest and biggest hashes in the db
+func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
+	if s.conn == nil {
+		return nil, errors.Err("not connected")
+	}
+
+	if s.TrackAccess != TrackAccessBlobs {
+		return nil, errors.Err("blob access tracking is disabled")
+	}
+
+	query := "SELECT hash from blob_ where is_stored = 1 order by last_accessed_at limit ?"
+	logQuery(query, maxBlobs)
+
+	rows, err := s.conn.Query(query, maxBlobs)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	defer closeRows(rows)
+
+	blobs := make([]string, 0, maxBlobs)
+	for rows.Next() {
+		var hash string
+		err := rows.Scan(&hash)
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		blobs = append(blobs, hash)
+	}
+
+	return blobs, nil
+}
+
+// AllHashes writes all hashes from the db into the channel.
+// It does not close the channel when it finishes.
+//func (s *SQL) AllHashes(ch chan<- string) error {
+//	if s.conn == nil {
+//		return errors.Err("not connected")
+//	}
+//
+//	query := "SELECT hash from blob_"
+//	if s.SoftDelete {
+//		query += " where is_stored = 1"
+//	}
+//	logQuery(query)
+//
+//	rows, err := s.conn.Query(query)
+//	if err != nil {
+//		return errors.Err(err)
+//	}
+//	defer closeRows(rows)
+//
+//	for rows.Next() {
+//		var hash string
+//		err := rows.Scan(&hash)
+//		if err != nil {
+//			return errors.Err(err)
+//		}
+//		ch <- hash
+//      // TODO: this needs testing
+//		// TODO: need a way to cancel this early (e.g. in case of shutdown)
+//	}
+//
+//	close(ch)
+//	return nil
+//}
+
+func (s *SQL) Count() (int, error) {
+	if s.conn == nil {
+		return 0, errors.Err("not connected")
+	}
+
+	query := "SELECT count(id) from blob_"
+	if s.SoftDelete {
+		query += " where is_stored = 1"
+	}
+	logQuery(query)
+
+	var count int
+	err := s.conn.QueryRow(query).Scan(&count)
+	return count, errors.Err(err)
+}
+
 // Block will mark a blob as blocked
 func (s *SQL) Block(hash string) error {
 	query := "INSERT IGNORE INTO blocked SET hash = ?"
@@ -528,8 +689,10 @@ CREATE TABLE blob_ (
   hash char(96) NOT NULL,
   is_stored TINYINT(1) NOT NULL DEFAULT 0,
   length bigint(20) unsigned DEFAULT NULL,
+  last_accessed_at TIMESTAMP NULL DEFAULT NULL,
   PRIMARY KEY (id),
   UNIQUE KEY blob_hash_idx (hash)
+  KEY `blob_last_accessed_idx` (`last_accessed_at`)
 );
 
 CREATE TABLE stream (
@@ -560,3 +723,47 @@ CREATE TABLE blocked (
 );
 
 */
+
+//func (d *LiteDBBackedStore) selfClean() {
+//	d.stopper.Add(1)
+//	defer d.stopper.Done()
+//	lastCleanup := time.Now()
+//	const cleanupInterval = 10 * time.Second
+//	for {
+//		select {
+//		case <-d.stopper.Ch():
+//			log.Infoln("stopping self cleanup")
+//			return
+//		default:
+//			time.Sleep(1 * time.Second)
+//		}
+//		if time.Since(lastCleanup) < cleanupInterval {
+//			continue
+//
+//		blobsCount, err := d.db.BlobsCount()
+//		if err != nil {
+//			log.Errorf(errors.FullTrace(err))
+//		}
+//		if blobsCount >= d.maxItems {
+//			itemsToDelete := blobsCount / 100 * 10
+//			blobs, err := d.db.GetLRUBlobs(itemsToDelete)
+//			if err != nil {
+//				log.Errorf(errors.FullTrace(err))
+//			}
+//			for _, hash := range blobs {
+//				select {
+//				case <-d.stopper.Ch():
+//					return
+//				default:
+//
+//				}
+//				err = d.Delete(hash)
+//				if err != nil {
+//					log.Errorf(errors.FullTrace(err))
+//				}
+//				metrics.CacheLRUEvictCount.With(metrics.CacheLabels(d.Name(), d.component)).Inc()
+//			}
+//		}
+//		lastCleanup = time.Now()
+//	}
+//}
diff --git a/go.mod b/go.mod
index d3384a1..4570f1d 100644
--- a/go.mod
+++ b/go.mod
@@ -12,8 +12,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/golang/protobuf v1.4.2
-	github.com/google/btree v1.0.0 // indirect
-	github.com/google/gops v0.3.7 // indirect
+	github.com/google/gops v0.3.7
 	github.com/gorilla/mux v1.7.4
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/hashicorp/golang-lru v0.5.4
@@ -33,7 +32,6 @@ require (
 	github.com/spf13/afero v1.4.1 // indirect
 	github.com/spf13/cast v1.3.0
 	github.com/spf13/cobra v0.0.3
-	github.com/spf13/pflag v1.0.3 // indirect
 	github.com/spf13/viper v1.7.1 // indirect
 	github.com/stretchr/testify v1.4.0
 	github.com/volatiletech/null v8.0.0+incompatible
diff --git a/peer/http3/store.go b/peer/http3/store.go
index 76a58c1..b681443 100644
--- a/peer/http3/store.go
+++ b/peer/http3/store.go
@@ -91,3 +91,8 @@ func (p *Store) PutSD(hash string, blob stream.Blob) error {
 func (p *Store) Delete(hash string) error {
 	panic("http3Store cannot put or delete blobs")
 }
+
+// Delete is not supported
+func (p *Store) Shutdown() {
+	return
+}
diff --git a/peer/store.go b/peer/store.go
index b8abedd..3ef9622 100644
--- a/peer/store.go
+++ b/peer/store.go
@@ -66,3 +66,8 @@ func (p *Store) PutSD(hash string, blob stream.Blob) error {
 func (p *Store) Delete(hash string) error {
 	panic("PeerStore cannot put or delete blobs")
 }
+
+// Delete is not supported
+func (p *Store) Shutdown() {
+	return
+}
diff --git a/store/caching.go b/store/caching.go
index c8e7d45..53b692c 100644
--- a/store/caching.go
+++ b/store/caching.go
@@ -68,7 +68,7 @@ func (c *CachingStore) Get(hash string) (stream.Blob, error) {
 	go func() {
 		err = c.cache.Put(hash, blob)
 		if err != nil {
-			log.Errorf("error saving blob to underlying cache: %s", err.Error())
+			log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
 		}
 	}()
 	return blob, nil
@@ -100,3 +100,10 @@ func (c *CachingStore) Delete(hash string) error {
 	}
 	return c.cache.Delete(hash)
 }
+
+// Shutdown shuts down the store gracefully
+func (c *CachingStore) Shutdown() {
+	c.origin.Shutdown()
+	c.cache.Shutdown()
+	return
+}
diff --git a/store/caching_test.go b/store/caching_test.go
index 1168a92..0636583 100644
--- a/store/caching_test.go
+++ b/store/caching_test.go
@@ -168,3 +168,7 @@ func (s *SlowBlobStore) Delete(hash string) error {
 	time.Sleep(s.delay)
 	return s.mem.Delete(hash)
 }
+
+func (s *SlowBlobStore) Shutdown() {
+	return
+}
diff --git a/store/cloudfront_ro.go b/store/cloudfront_ro.go
index a914285..3dcffc0 100644
--- a/store/cloudfront_ro.go
+++ b/store/cloudfront_ro.go
@@ -103,3 +103,8 @@ func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
 func (c *CloudFrontROStore) Delete(_ string) error {
 	panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore")
 }
+
+// Shutdown shuts down the store gracefully
+func (c *CloudFrontROStore) Shutdown() {
+	return
+}
diff --git a/store/cloudfront_rw.go b/store/cloudfront_rw.go
index ee771da..6de64af 100644
--- a/store/cloudfront_rw.go
+++ b/store/cloudfront_rw.go
@@ -48,3 +48,10 @@ func (c *CloudFrontRWStore) PutSD(hash string, blob stream.Blob) error {
 func (c *CloudFrontRWStore) Delete(hash string) error {
 	return c.s3.Delete(hash)
 }
+
+// Shutdown shuts down the store gracefully
+func (c *CloudFrontRWStore) Shutdown() {
+	c.s3.Shutdown()
+	c.cf.Shutdown()
+	return
+}
diff --git a/store/dbbacked.go b/store/dbbacked.go
index 25cc446..1c5088a 100644
--- a/store/dbbacked.go
+++ b/store/dbbacked.go
@@ -4,10 +4,9 @@ import (
 	"encoding/json"
 	"sync"
 
-	"github.com/lbryio/reflector.go/db"
-
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/db"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -193,3 +192,9 @@ func (d *DBBackedStore) initBlocked() error {
 
 	return err
 }
+
+// Shutdown shuts down the store gracefully
+func (d *DBBackedStore) Shutdown() {
+	d.blobs.Shutdown()
+	return
+}
diff --git a/store/disk.go b/store/disk.go
index 2234971..03cc393 100644
--- a/store/disk.go
+++ b/store/disk.go
@@ -147,3 +147,8 @@ func (d *DiskStore) initOnce() error {
 	d.initialized = true
 	return nil
 }
+
+// Shutdown shuts down the store gracefully
+func (d *DiskStore) Shutdown() {
+	return
+}
diff --git a/store/lfuda.go b/store/lfuda.go
index cfdb155..0eb7296 100644
--- a/store/lfuda.go
+++ b/store/lfuda.go
@@ -109,7 +109,7 @@ func (l *LFUDAStore) loadExisting(store lister, maxItems int) error {
 	if err != nil {
 		return err
 	}
-	logrus.Infof("read %d files from disk", len(existing))
+	logrus.Infof("read %d files from underlying store", len(existing))
 
 	added := 0
 	for _, h := range existing {
@@ -121,3 +121,8 @@ func (l *LFUDAStore) loadExisting(store lister, maxItems int) error {
 	}
 	return nil
 }
+
+// Shutdown shuts down the store gracefully
+func (l *LFUDAStore) Shutdown() {
+	return
+}
diff --git a/store/lru.go b/store/lru.go
index 071e2b9..ce0d83d 100644
--- a/store/lru.go
+++ b/store/lru.go
@@ -123,3 +123,8 @@ func (l *LRUStore) loadExisting(store lister, maxItems int) error {
 	}
 	return nil
 }
+
+// Shutdown shuts down the store gracefully
+func (l *LRUStore) Shutdown() {
+	return
+}
diff --git a/store/memory.go b/store/memory.go
index f462a8d..62b93b6 100644
--- a/store/memory.go
+++ b/store/memory.go
@@ -71,3 +71,8 @@ func (m *MemStore) Debug() map[string]stream.Blob {
 	defer m.mu.RUnlock()
 	return m.blobs
 }
+
+// Shutdown shuts down the store gracefully
+func (m *MemStore) Shutdown() {
+	return
+}
diff --git a/store/noop.go b/store/noop.go
index 9e5d815..2792a0d 100644
--- a/store/noop.go
+++ b/store/noop.go
@@ -13,3 +13,4 @@ func (n *NoopStore) Get(_ string) (stream.Blob, error)   { return nil, nil }
 func (n *NoopStore) Put(_ string, _ stream.Blob) error   { return nil }
 func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil }
 func (n *NoopStore) Delete(_ string) error               { return nil }
+func (n *NoopStore) Shutdown()                           { return }
diff --git a/store/s3.go b/store/s3.go
index 53451be..02a23c5 100644
--- a/store/s3.go
+++ b/store/s3.go
@@ -158,3 +158,8 @@ func (s *S3Store) initOnce() error {
 	s.session = sess
 	return nil
 }
+
+// Shutdown shuts down the store gracefully
+func (s *S3Store) Shutdown() {
+	return
+}
diff --git a/store/singleflight.go b/store/singleflight.go
index cca1e2f..7d0139d 100644
--- a/store/singleflight.go
+++ b/store/singleflight.go
@@ -65,3 +65,9 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) {
 		return blob, nil
 	}
 }
+
+// Shutdown shuts down the store gracefully
+func (s *singleflightStore) Shutdown() {
+	s.BlobStore.Shutdown()
+	return
+}
diff --git a/store/store.go b/store/store.go
index 200ff61..36d388d 100644
--- a/store/store.go
+++ b/store/store.go
@@ -19,6 +19,8 @@ type BlobStore interface {
 	PutSD(hash string, blob stream.Blob) error
 	// Delete the blob from the store.
 	Delete(hash string) error
+	// Shutdown the store gracefully
+	Shutdown()
 }
 
 // Blocklister is a store that supports blocking blobs to prevent their inclusion in the store.

From 0d5004a83bc5cd6eaca1d05976403f74ae1dadfb Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Wed, 30 Dec 2020 04:24:11 +0100
Subject: [PATCH 15/51] add cmd to populate db

fix store init
try fixing unreasonable db bottleneck
---
 cmd/populatedb.go     |  45 ++++++++++++
 cmd/reflector.go      |   7 +-
 db/db.go              |  56 +++++++++++++--
 store/dbbacked.go     |   6 +-
 store/litedbbacked.go | 155 ------------------------------------------
 5 files changed, 101 insertions(+), 168 deletions(-)
 create mode 100644 cmd/populatedb.go
 delete mode 100644 store/litedbbacked.go

diff --git a/cmd/populatedb.go b/cmd/populatedb.go
new file mode 100644
index 0000000..d30bafe
--- /dev/null
+++ b/cmd/populatedb.go
@@ -0,0 +1,45 @@
+package cmd
+
+import (
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/reflector.go/db"
+	"github.com/lbryio/reflector.go/meta"
+	"github.com/lbryio/reflector.go/store/speedwalk"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var (
+	diskStorePath string
+)
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:   "populate-db",
+		Short: "populate local database with blobs from a disk storage",
+		Run:   populateDbCmd,
+	}
+	cmd.Flags().StringVar(&diskStorePath, "store-path", "",
+		"path of the store where all blobs are cached")
+	rootCmd.AddCommand(cmd)
+}
+
+func populateDbCmd(cmd *cobra.Command, args []string) {
+	log.Printf("reflector %s", meta.VersionString())
+	if diskStorePath == "" {
+		log.Fatal("store-path must be defined")
+	}
+	localDb := new(db.SQL)
+	localDb.SoftDelete = true
+	localDb.TrackAccess = db.TrackAccessBlobs
+	err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
+	if err != nil {
+		log.Fatal(err)
+	}
+	blobs, err := speedwalk.AllFiles(diskStorePath, true)
+	err = localDb.AddBlobs(blobs)
+	if err != nil {
+		log.Errorf("error while storing to db: %s", errors.FullTrace(err))
+	}
+}
diff --git a/cmd/reflector.go b/cmd/reflector.go
index ab9f53e..aacef08 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -185,11 +185,11 @@ func wrapWithCache(s store.BlobStore, cleanerStopper *stop.Group) store.BlobStor
 		if err != nil {
 			log.Fatal(err)
 		}
-		dbBackedDiskStore := store.NewDBBackedStore(store.NewDiskStore(diskCachePath, 2), localDb, false)
+		dbBackedDiskStore := store.NewDBBackedStore(store.NewDiskStore(diskCachePath, 2), localDb, true)
 		wrapped = store.NewCachingStore(
 			"reflector",
 			wrapped,
-			store.NewDBBackedStore(store.NewDiskStore(diskCachePath, 2), localDb, false),
+			dbBackedDiskStore,
 		)
 
 		go cleanOldestBlobs(int(realCacheSize), localDb, dbBackedDiskStore, cleanerStopper)
@@ -248,8 +248,7 @@ func diskCacheParams(diskParams string) (int, string) {
 }
 
 func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) {
-	const cleanupInterval = 10 * time.Second
-
+	const cleanupInterval = 10 * time.Minute
 	for {
 		select {
 		case <-stopper.Ch():
diff --git a/db/db.go b/db/db.go
index 40b5c7d..9e19eba 100644
--- a/db/db.go
+++ b/db/db.go
@@ -3,11 +3,13 @@ package db
 import (
 	"context"
 	"database/sql"
+	"strings"
 	"time"
 
 	"github.com/lbryio/lbry.go/v2/dht/bits"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	qt "github.com/lbryio/lbry.go/v2/extras/query"
+	"github.com/lbryio/lbry.go/v2/stream"
 
 	"github.com/go-sql-driver/mysql"
 	_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
@@ -88,6 +90,27 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
 	return err
 }
 
+// AddBlob adds a blob to the database.
+func (s *SQL) AddBlobs(hash []string) error {
+	if s.conn == nil {
+		return errors.Err("not connected")
+	}
+	// Split the slice into batches of 20 items.
+	batch := 10000
+
+	for i := 0; i < len(hash); i += batch {
+		j := i + batch
+		if j > len(hash) {
+			j = len(hash)
+		}
+		err := s.insertBlobs(hash[i:j]) // Process the batch.
+		if err != nil {
+			log.Errorf("error while inserting batch: %s", errors.FullTrace(err))
+		}
+	}
+	return nil
+}
+
 func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
 	if length <= 0 {
 		return 0, errors.Err("length must be positive")
@@ -130,6 +153,26 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
 	return blobID, nil
 }
 
+func (s *SQL) insertBlobs(hashes []string) error {
+	var (
+		q    string
+		args []interface{}
+	)
+	dayAgo := time.Now().AddDate(0, 0, -1)
+	q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
+	for _, hash := range hashes {
+		q += "(?,?,?,?),"
+		args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
+	}
+	q = strings.TrimSuffix(q, ",")
+	_, err := s.exec(q, args...)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
 	var (
 		q    string
@@ -180,12 +223,13 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
 // HasBlobs checks if the database contains the set of blobs and returns a bool map.
 func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
 	exists, idsNeedingTouch, err := s.hasBlobs(hashes)
-
-	if s.TrackAccess == TrackAccessBlobs {
-		s.touchBlobs(idsNeedingTouch)
-	} else if s.TrackAccess == TrackAccessStreams {
-		s.touchStreams(idsNeedingTouch)
-	}
+	go func() {
+		if s.TrackAccess == TrackAccessBlobs {
+			s.touchBlobs(idsNeedingTouch)
+		} else if s.TrackAccess == TrackAccessStreams {
+			s.touchStreams(idsNeedingTouch)
+		}
+	}()
 
 	return exists, err
 }
diff --git a/store/dbbacked.go b/store/dbbacked.go
index 1c5088a..dc88ad2 100644
--- a/store/dbbacked.go
+++ b/store/dbbacked.go
@@ -44,18 +44,18 @@ func (d *DBBackedStore) Get(hash string) (stream.Blob, error) {
 	if !has {
 		return nil, ErrBlobNotFound
 	}
+
+	b, err := d.blobs.Get(hash)
 	if d.deleteOnMiss {
-		b, err := d.blobs.Get(hash)
 		if err != nil && errors.Is(err, ErrBlobNotFound) {
 			e2 := d.Delete(hash)
 			if e2 != nil {
 				log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
 			}
-			return b, err
 		}
 	}
 
-	return d.blobs.Get(hash)
+	return b, err
 }
 
 // Put stores the blob in the S3 store and stores the blob information in the DB.
diff --git a/store/litedbbacked.go b/store/litedbbacked.go
deleted file mode 100644
index 2e2d726..0000000
--- a/store/litedbbacked.go
+++ /dev/null
@@ -1,155 +0,0 @@
-package store
-
-import (
-	"encoding/json"
-
-	"github.com/lbryio/lbry.go/v2/extras/stop"
-	"github.com/lbryio/reflector.go/db"
-
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/stream"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// DBBackedStore is a store that's backed by a DB. The DB contains data about what's in the store.
-type LiteDBBackedStore struct {
-	blobs     BlobStore
-	db        *db.SQL
-	maxItems  int
-	stopper   *stop.Stopper
-	component string
-}
-
-// NewDBBackedStore returns an initialized store pointer.
-func NewLiteDBBackedStore(component string, blobs BlobStore, db *db.SQL, maxItems int) *LiteDBBackedStore {
-	instance := &LiteDBBackedStore{blobs: blobs, db: db, maxItems: maxItems, stopper: stop.New(), component: component}
-	return instance
-}
-
-const nameLiteDBBacked = "lite-db-backed"
-
-// Name is the cache type name
-func (d *LiteDBBackedStore) Name() string { return nameDBBacked }
-
-// Has returns true if the blob is in the store
-func (d *LiteDBBackedStore) Has(hash string) (bool, error) {
-	return d.db.HasBlob(hash)
-}
-
-// Get gets the blob
-func (d *LiteDBBackedStore) Get(hash string) (stream.Blob, error) {
-	has, err := d.db.HasBlob(hash)
-	if err != nil {
-		return nil, err
-	}
-	if !has {
-		return nil, ErrBlobNotFound
-	}
-	b, err := d.blobs.Get(hash)
-	if err != nil && errors.Is(err, ErrBlobNotFound) {
-		e2 := d.db.Delete(hash)
-		if e2 != nil {
-			log.Errorf("error while deleting blob from db: %s", errors.FullTrace(e2))
-		}
-		return b, err
-	}
-
-	return d.blobs.Get(hash)
-}
-
-// Put stores the blob in the S3 store and stores the blob information in the DB.
-func (d *LiteDBBackedStore) Put(hash string, blob stream.Blob) error {
-	err := d.blobs.Put(hash, blob)
-	if err != nil {
-		return err
-	}
-
-	return d.db.AddBlob(hash, len(blob), true)
-}
-
-// PutSD stores the SDBlob in the S3 store. It will return an error if the sd blob is missing the stream hash or if
-// there is an error storing the blob information in the DB.
-func (d *LiteDBBackedStore) PutSD(hash string, blob stream.Blob) error {
-	var blobContents db.SdBlob
-	err := json.Unmarshal(blob, &blobContents)
-	if err != nil {
-		return errors.Err(err)
-	}
-	if blobContents.StreamHash == "" {
-		return errors.Err("sd blob is missing stream hash")
-	}
-
-	err = d.blobs.PutSD(hash, blob)
-	if err != nil {
-		return err
-	}
-
-	return d.db.AddSDBlob(hash, len(blob), blobContents)
-}
-
-func (d *LiteDBBackedStore) Delete(hash string) error {
-	err := d.blobs.Delete(hash)
-	if err != nil {
-		return err
-	}
-
-	return d.db.Delete(hash)
-}
-
-// list returns the hashes of blobs that already exist in the database
-func (d *LiteDBBackedStore) list() ([]string, error) {
-	//blobs, err := d.db.AllBlobs()
-	//return blobs, err
-	return nil, nil
-}
-
-//func (d *LiteDBBackedStore) selfClean() {
-//	d.stopper.Add(1)
-//	defer d.stopper.Done()
-//	lastCleanup := time.Now()
-//	const cleanupInterval = 10 * time.Second
-//	for {
-//		select {
-//		case <-d.stopper.Ch():
-//			log.Infoln("stopping self cleanup")
-//			return
-//		default:
-//			time.Sleep(1 * time.Second)
-//		}
-//		if time.Since(lastCleanup) < cleanupInterval {
-//			continue
-//		}
-//		blobsCount, err := d.db.BlobsCount()
-//		if err != nil {
-//			log.Errorf(errors.FullTrace(err))
-//		}
-//		if blobsCount >= d.maxItems {
-//			itemsToDelete := blobsCount / 100 * 10
-//			blobs, err := d.db.GetLRUBlobs(itemsToDelete)
-//			if err != nil {
-//				log.Errorf(errors.FullTrace(err))
-//			}
-//			for _, hash := range blobs {
-//				select {
-//				case <-d.stopper.Ch():
-//					return
-//				default:
-//
-//				}
-//				err = d.Delete(hash)
-//				if err != nil {
-//					log.Errorf(errors.FullTrace(err))
-//				}
-//				metrics.CacheLRUEvictCount.With(metrics.CacheLabels(d.Name(), d.component)).Inc()
-//			}
-//		}
-//		lastCleanup = time.Now()
-//	}
-//}
-
-// Shutdown shuts down the store gracefully
-func (d *LiteDBBackedStore) Shutdown() {
-	d.stopper.StopAndWait()
-	return
-}

From b33651ae2640012da93f8d29ea986fa0c3913d0e Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 5 Jan 2021 05:09:55 +0100
Subject: [PATCH 16/51] save uploaded blobs and work around the blocklist issue

---
 cmd/reflector.go    |  2 +-
 cmd/test.go         |  2 +-
 prism/prism.go      |  2 +-
 reflector/server.go | 26 ++++++++++++++------------
 4 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/cmd/reflector.go b/cmd/reflector.go
index aacef08..49bef3e 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -76,7 +76,7 @@ func reflectorCmd(cmd *cobra.Command, args []string) {
 	outerStore := wrapWithCache(underlyingStore, cleanerStopper)
 
 	if !disableUploads {
-		reflectorServer := reflector.NewServer(underlyingStore)
+		reflectorServer := reflector.NewServer(underlyingStore, outerStore)
 		reflectorServer.Timeout = 3 * time.Minute
 		reflectorServer.EnableBlocklist = !disableBlocklist
 
diff --git a/cmd/test.go b/cmd/test.go
index a330856..afa2ff2 100644
--- a/cmd/test.go
+++ b/cmd/test.go
@@ -31,7 +31,7 @@ func testCmd(cmd *cobra.Command, args []string) {
 
 	memStore := store.NewMemStore()
 
-	reflectorServer := reflector.NewServer(memStore)
+	reflectorServer := reflector.NewServer(memStore, memStore)
 	reflectorServer.Timeout = 3 * time.Minute
 
 	err := reflectorServer.Start(":" + strconv.Itoa(reflector.DefaultPort))
diff --git a/prism/prism.go b/prism/prism.go
index 12d0329..ee18ba1 100644
--- a/prism/prism.go
+++ b/prism/prism.go
@@ -79,7 +79,7 @@ func New(conf *Config) *Prism {
 		dht:       d,
 		cluster:   c,
 		peer:      peer.NewServer(conf.Blobs),
-		reflector: reflector.NewServer(conf.Blobs),
+		reflector: reflector.NewServer(conf.Blobs, conf.Blobs),
 
 		grp: stop.New(),
 	}
diff --git a/reflector/server.go b/reflector/server.go
index bbe3d33..592ba3d 100644
--- a/reflector/server.go
+++ b/reflector/server.go
@@ -40,16 +40,18 @@ type Server struct {
 
 	EnableBlocklist bool // if true, blocklist checking and blob deletion will be enabled
 
-	store store.BlobStore
-	grp   *stop.Group
+	underlyingStore store.BlobStore
+	outerStore      store.BlobStore
+	grp             *stop.Group
 }
 
 // NewServer returns an initialized reflector server pointer.
-func NewServer(store store.BlobStore) *Server {
+func NewServer(underlying store.BlobStore, outer store.BlobStore) *Server {
 	return &Server{
-		Timeout: DefaultTimeout,
-		store:   store,
-		grp:     stop.New(),
+		Timeout:         DefaultTimeout,
+		underlyingStore: underlying,
+		outerStore:      outer,
+		grp:             stop.New(),
 	}
 }
 
@@ -85,7 +87,7 @@ func (s *Server) Start(address string) error {
 	}()
 
 	if s.EnableBlocklist {
-		if b, ok := s.store.(store.Blocklister); ok {
+		if b, ok := s.underlyingStore.(store.Blocklister); ok {
 			s.grp.Add(1)
 			go func() {
 				s.enableBlocklist(b)
@@ -190,13 +192,13 @@ func (s *Server) receiveBlob(conn net.Conn) error {
 	}
 
 	var wantsBlob bool
-	if bl, ok := s.store.(store.Blocklister); ok {
+	if bl, ok := s.underlyingStore.(store.Blocklister); ok {
 		wantsBlob, err = bl.Wants(blobHash)
 		if err != nil {
 			return err
 		}
 	} else {
-		blobExists, err := s.store.Has(blobHash)
+		blobExists, err := s.underlyingStore.Has(blobHash)
 		if err != nil {
 			return err
 		}
@@ -206,7 +208,7 @@ func (s *Server) receiveBlob(conn net.Conn) error {
 	var neededBlobs []string
 
 	if isSdBlob && !wantsBlob {
-		if nbc, ok := s.store.(neededBlobChecker); ok {
+		if nbc, ok := s.underlyingStore.(neededBlobChecker); ok {
 			neededBlobs, err = nbc.MissingBlobsForKnownStream(blobHash)
 			if err != nil {
 				return err
@@ -249,9 +251,9 @@ func (s *Server) receiveBlob(conn net.Conn) error {
 	log.Debugln("Got blob " + blobHash[:8])
 
 	if isSdBlob {
-		err = s.store.PutSD(blobHash, blob)
+		err = s.outerStore.PutSD(blobHash, blob)
 	} else {
-		err = s.store.Put(blobHash, blob)
+		err = s.outerStore.Put(blobHash, blob)
 	}
 	if err != nil {
 		return err

From 49714c02a61256d0af8303f3ed749cf56d5b7977 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Tue, 5 Jan 2021 11:36:33 -0500
Subject: [PATCH 17/51] only touch blobs when you get them

---
 db/db.go              | 51 ++++++++++++++++++++++---------------------
 reflector/uploader.go |  2 +-
 store/dbbacked.go     | 16 ++++++--------
 3 files changed, 34 insertions(+), 35 deletions(-)

diff --git a/db/db.go b/db/db.go
index 9e19eba..e48bcaf 100644
--- a/db/db.go
+++ b/db/db.go
@@ -111,6 +111,26 @@ func (s *SQL) AddBlobs(hash []string) error {
 	return nil
 }
 
+func (s *SQL) insertBlobs(hashes []string) error {
+	var (
+		q    string
+		args []interface{}
+	)
+	dayAgo := time.Now().AddDate(0, 0, -1)
+	q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
+	for _, hash := range hashes {
+		q += "(?,?,?,?),"
+		args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
+	}
+	q = strings.TrimSuffix(q, ",")
+	_, err := s.exec(q, args...)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
 	if length <= 0 {
 		return 0, errors.Err("length must be positive")
@@ -153,26 +173,6 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
 	return blobID, nil
 }
 
-func (s *SQL) insertBlobs(hashes []string) error {
-	var (
-		q    string
-		args []interface{}
-	)
-	dayAgo := time.Now().AddDate(0, 0, -1)
-	q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
-	for _, hash := range hashes {
-		q += "(?,?,?,?),"
-		args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
-	}
-	q = strings.TrimSuffix(q, ",")
-	_, err := s.exec(q, args...)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
 	var (
 		q    string
@@ -212,8 +212,8 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
 }
 
 // HasBlob checks if the database contains the blob information.
-func (s *SQL) HasBlob(hash string) (bool, error) {
-	exists, err := s.HasBlobs([]string{hash})
+func (s *SQL) HasBlob(hash string, touch bool) (bool, error) {
+	exists, err := s.HasBlobs([]string{hash}, touch)
 	if err != nil {
 		return false, err
 	}
@@ -221,15 +221,16 @@ func (s *SQL) HasBlob(hash string) (bool, error) {
 }
 
 // HasBlobs checks if the database contains the set of blobs and returns a bool map.
-func (s *SQL) HasBlobs(hashes []string) (map[string]bool, error) {
+func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
 	exists, idsNeedingTouch, err := s.hasBlobs(hashes)
-	go func() {
+
+	if touch {
 		if s.TrackAccess == TrackAccessBlobs {
 			s.touchBlobs(idsNeedingTouch)
 		} else if s.TrackAccess == TrackAccessStreams {
 			s.touchStreams(idsNeedingTouch)
 		}
-	}()
+	}
 
 	return exists, err
 }
diff --git a/reflector/uploader.go b/reflector/uploader.go
index ae762f4..e5b2688 100644
--- a/reflector/uploader.go
+++ b/reflector/uploader.go
@@ -74,7 +74,7 @@ func (u *Uploader) Upload(dirOrFilePath string) error {
 
 	var exists map[string]bool
 	if !u.skipExistsCheck {
-		exists, err = u.db.HasBlobs(hashes)
+		exists, err = u.db.HasBlobs(hashes, false)
 		if err != nil {
 			return err
 		}
diff --git a/store/dbbacked.go b/store/dbbacked.go
index dc88ad2..1484c49 100644
--- a/store/dbbacked.go
+++ b/store/dbbacked.go
@@ -32,12 +32,12 @@ func (d *DBBackedStore) Name() string { return nameDBBacked }
 
 // Has returns true if the blob is in the store
 func (d *DBBackedStore) Has(hash string) (bool, error) {
-	return d.db.HasBlob(hash)
+	return d.db.HasBlob(hash, false)
 }
 
 // Get gets the blob
 func (d *DBBackedStore) Get(hash string) (stream.Blob, error) {
-	has, err := d.db.HasBlob(hash)
+	has, err := d.db.HasBlob(hash, true)
 	if err != nil {
 		return nil, err
 	}
@@ -46,12 +46,10 @@ func (d *DBBackedStore) Get(hash string) (stream.Blob, error) {
 	}
 
 	b, err := d.blobs.Get(hash)
-	if d.deleteOnMiss {
-		if err != nil && errors.Is(err, ErrBlobNotFound) {
-			e2 := d.Delete(hash)
-			if e2 != nil {
-				log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
-			}
+	if d.deleteOnMiss && errors.Is(err, ErrBlobNotFound) {
+		e2 := d.Delete(hash)
+		if e2 != nil {
+			log.Errorf("error while deleting blob from db: %s", errors.FullTrace(err))
 		}
 	}
 
@@ -110,7 +108,7 @@ func (d *DBBackedStore) Block(hash string) error {
 		return err
 	}
 
-	has, err := d.db.HasBlob(hash)
+	has, err := d.db.HasBlob(hash, false)
 	if err != nil {
 		return err
 	}

From cc504e6c4493c74be4fc2f1be4a86823ccf37627 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Tue, 5 Jan 2021 12:16:44 -0500
Subject: [PATCH 18/51] fix long query

---
 db/db.go | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/db/db.go b/db/db.go
index e48bcaf..d1c8b7b 100644
--- a/db/db.go
+++ b/db/db.go
@@ -298,20 +298,23 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
 		log.Debugf("getting hashes[%d:%d] of %d", doneIndex, sliceEnd, len(hashes))
 		batch := hashes[doneIndex:sliceEnd]
 
-		var lastAccessedAtSelect string
+		var query string
 		if s.TrackAccess == TrackAccessBlobs {
-			lastAccessedAtSelect = "b.last_accessed_at"
+			query = `SELECT b.hash, b.id, NULL, b.last_accessed_at
+FROM blob_ b
+WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
 		} else if s.TrackAccess == TrackAccessStreams {
-			lastAccessedAtSelect = "s.last_accessed_at"
-		} else {
-			lastAccessedAtSelect = "NULL"
-		}
-
-		query := `SELECT b.hash, b.id, s.id, ` + lastAccessedAtSelect + `
+			query = `SELECT b.hash, b.id, s.id, s.last_accessed_at
 FROM blob_ b
 LEFT JOIN stream_blob sb ON b.id = sb.blob_id
-LEFT JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
+INNER JOIN stream s on (sb.stream_id = s.id or s.sd_blob_id = b.id)
 WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
+		} else {
+			query = `SELECT b.hash, b.id, NULL, NULL
+FROM blob_ b
+WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
+		}
+
 		args := make([]interface{}, len(batch))
 		for i := range batch {
 			args[i] = batch[i]

From c4504631bc896b67611fa4a9ff02ddfedefbcc39 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Wed, 6 Jan 2021 10:43:35 -0500
Subject: [PATCH 19/51] avoid heavy interpolateparams call

---
 cmd/peer.go              |  4 +++-
 cmd/populatedb.go        |  8 +++++---
 cmd/reflector.go         | 14 +++++++++-----
 cmd/start.go             |  4 +++-
 cmd/upload.go            |  5 ++++-
 db/db.go                 | 33 ++++++++++++++++++++-------------
 reflector/server_test.go |  6 +++---
 7 files changed, 47 insertions(+), 27 deletions(-)

diff --git a/cmd/peer.go b/cmd/peer.go
index 86af279..c631bfe 100644
--- a/cmd/peer.go
+++ b/cmd/peer.go
@@ -33,7 +33,9 @@ func peerCmd(cmd *cobra.Command, args []string) {
 	peerServer := peer.NewServer(s3)
 
 	if !peerNoDB {
-		db := new(db.SQL)
+		db := &db.SQL{
+			LogQueries: log.GetLevel() == log.DebugLevel,
+		}
 		err = db.Connect(globalConfig.DBConn)
 		checkErr(err)
 
diff --git a/cmd/populatedb.go b/cmd/populatedb.go
index d30bafe..7199e42 100644
--- a/cmd/populatedb.go
+++ b/cmd/populatedb.go
@@ -30,9 +30,11 @@ func populateDbCmd(cmd *cobra.Command, args []string) {
 	if diskStorePath == "" {
 		log.Fatal("store-path must be defined")
 	}
-	localDb := new(db.SQL)
-	localDb.SoftDelete = true
-	localDb.TrackAccess = db.TrackAccessBlobs
+	localDb := &db.SQL{
+		SoftDelete:  true,
+		TrackAccess: db.TrackAccessBlobs,
+		LogQueries:  log.GetLevel() == log.DebugLevel,
+	}
 	err := localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
 	if err != nil {
 		log.Fatal(err)
diff --git a/cmd/reflector.go b/cmd/reflector.go
index 49bef3e..738a066 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -152,8 +152,10 @@ func setupStore() store.BlobStore {
 	}
 
 	if useDB {
-		dbInst := new(db.SQL)
-		dbInst.TrackAccess = db.TrackAccessStreams
+		dbInst := &db.SQL{
+			TrackAccess: db.TrackAccessStreams,
+			LogQueries:  log.GetLevel() == log.DebugLevel,
+		}
 		err := dbInst.Connect(globalConfig.DBConn)
 		if err != nil {
 			log.Fatal(err)
@@ -178,9 +180,11 @@ func wrapWithCache(s store.BlobStore, cleanerStopper *stop.Group) store.BlobStor
 			log.Fatal(err)
 		}
 
-		localDb := new(db.SQL)
-		localDb.SoftDelete = true
-		localDb.TrackAccess = db.TrackAccessBlobs
+		localDb := &db.SQL{
+			SoftDelete:  true,
+			TrackAccess: db.TrackAccessBlobs,
+			LogQueries:  log.GetLevel() == log.DebugLevel,
+		}
 		err = localDb.Connect("reflector:reflector@tcp(localhost:3306)/reflector")
 		if err != nil {
 			log.Fatal(err)
diff --git a/cmd/start.go b/cmd/start.go
index 476e9fd..7b5facd 100644
--- a/cmd/start.go
+++ b/cmd/start.go
@@ -52,7 +52,9 @@ func init() {
 }
 
 func startCmd(cmd *cobra.Command, args []string) {
-	db := new(db.SQL)
+	db := &db.SQL{
+		LogQueries: log.GetLevel() == log.DebugLevel,
+	}
 	err := db.Connect(globalConfig.DBConn)
 	checkErr(err)
 	s3 := store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
diff --git a/cmd/upload.go b/cmd/upload.go
index e5e2cec..e2a49cd 100644
--- a/cmd/upload.go
+++ b/cmd/upload.go
@@ -8,6 +8,7 @@ import (
 	"github.com/lbryio/reflector.go/db"
 	"github.com/lbryio/reflector.go/reflector"
 	"github.com/lbryio/reflector.go/store"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
 )
@@ -30,7 +31,9 @@ func init() {
 }
 
 func uploadCmd(cmd *cobra.Command, args []string) {
-	db := new(db.SQL)
+	db := &db.SQL{
+		LogQueries: log.GetLevel() == log.DebugLevel,
+	}
 	err := db.Connect(globalConfig.DBConn)
 	checkErr(err)
 
diff --git a/db/db.go b/db/db.go
index d1c8b7b..afa708b 100644
--- a/db/db.go
+++ b/db/db.go
@@ -52,14 +52,21 @@ type SQL struct {
 
 	// Instead of deleting a blob, marked it as not stored in the db
 	SoftDelete bool
+
+	// Log executed queries. qt.InterpolateParams is cpu-heavy. This avoids that call if not needed.
+	LogQueries bool
 }
 
-func logQuery(query string, args ...interface{}) {
-	s, err := qt.InterpolateParams(query, args...)
+func (s SQL) logQuery(query string, args ...interface{}) {
+	if !s.LogQueries {
+		return
+	}
+
+	qStr, err := qt.InterpolateParams(query, args...)
 	if err != nil {
 		log.Errorln(err)
 	} else {
-		log.Debugln(s)
+		log.Debugln(qStr)
 	}
 }
 
@@ -320,7 +327,7 @@ WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
 			args[i] = batch[i]
 		}
 
-		logQuery(query, args...)
+		s.logQuery(query, args...)
 
 		err := func() error {
 			startTime := time.Now()
@@ -390,7 +397,7 @@ func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
 	}
 
 	query := "SELECT hash from blob_ where is_stored = 1 order by last_accessed_at limit ?"
-	logQuery(query, maxBlobs)
+	s.logQuery(query, maxBlobs)
 
 	rows, err := s.conn.Query(query, maxBlobs)
 	if err != nil {
@@ -422,7 +429,7 @@ func (s *SQL) LeastRecentlyAccessedHashes(maxBlobs int) ([]string, error) {
 //	if s.SoftDelete {
 //		query += " where is_stored = 1"
 //	}
-//	logQuery(query)
+//	s.logQuery(query)
 //
 //	rows, err := s.conn.Query(query)
 //	if err != nil {
@@ -454,7 +461,7 @@ func (s *SQL) Count() (int, error) {
 	if s.SoftDelete {
 		query += " where is_stored = 1"
 	}
-	logQuery(query)
+	s.logQuery(query)
 
 	var count int
 	err := s.conn.QueryRow(query).Scan(&count)
@@ -465,7 +472,7 @@ func (s *SQL) Count() (int, error) {
 func (s *SQL) Block(hash string) error {
 	query := "INSERT IGNORE INTO blocked SET hash = ?"
 	args := []interface{}{hash}
-	logQuery(query, args...)
+	s.logQuery(query, args...)
 	_, err := s.conn.Exec(query, args...)
 	return errors.Err(err)
 }
@@ -473,7 +480,7 @@ func (s *SQL) Block(hash string) error {
 // GetBlocked will return a list of blocked hashes
 func (s *SQL) GetBlocked() (map[string]bool, error) {
 	query := "SELECT hash FROM blocked"
-	logQuery(query)
+	s.logQuery(query)
 	rows, err := s.conn.Query(query)
 	if err != nil {
 		return nil, errors.Err(err)
@@ -516,7 +523,7 @@ func (s *SQL) MissingBlobsForKnownStream(sdHash string) ([]string, error) {
 	`
 	args := []interface{}{sdHash}
 
-	logQuery(query, args...)
+	s.logQuery(query, args...)
 
 	rows, err := s.conn.Query(query, args...)
 	if err != nil {
@@ -595,7 +602,7 @@ func (s *SQL) GetHashRange() (string, string, error) {
 
 	query := "SELECT MIN(hash), MAX(hash) from blob_"
 
-	logQuery(query)
+	s.logQuery(query)
 
 	err := s.conn.QueryRow(query).Scan(&min, &max)
 	return min, max, err
@@ -619,7 +626,7 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
 		query := "SELECT hash FROM blob_ WHERE hash >= ? AND hash <= ? AND is_stored = 1"
 		args := []interface{}{start.Hex(), end.Hex()}
 
-		logQuery(query, args...)
+		s.logQuery(query, args...)
 
 		rows, err := s.conn.Query(query, args...)
 		defer closeRows(rows)
@@ -700,7 +707,7 @@ func closeRows(rows *sql.Rows) {
 }
 
 func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
-	logQuery(query, args...)
+	s.logQuery(query, args...)
 	attempt, maxAttempts := 0, 3
 Retry:
 	attempt++
diff --git a/reflector/server_test.go b/reflector/server_test.go
index 0de200e..9004d84 100644
--- a/reflector/server_test.go
+++ b/reflector/server_test.go
@@ -22,7 +22,7 @@ func startServerOnRandomPort(t *testing.T) (*Server, int) {
 		t.Fatal(err)
 	}
 
-	srv := NewServer(store.NewMemStore())
+	srv := NewServer(store.NewMemStore(), store.NewMemStore())
 	err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
 	if err != nil {
 		t.Fatal(err)
@@ -119,7 +119,7 @@ func TestServer_Timeout(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	srv := NewServer(store.NewMemStore())
+	srv := NewServer(store.NewMemStore(), store.NewMemStore())
 	srv.Timeout = testTimeout
 	err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
 	if err != nil {
@@ -190,7 +190,7 @@ func TestServer_PartialUpload(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	srv := NewServer(st)
+	srv := NewServer(st, st)
 	err = srv.Start("127.0.0.1:" + strconv.Itoa(port))
 	if err != nil {
 		t.Fatal(err)

From 3e475e537b37566ac3c35ab1649f6cfd694516ec Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Thu, 7 Jan 2021 01:26:30 +0100
Subject: [PATCH 20/51] optimize batch insertions

reduce touch time to every 6 hours
---
 db/db.go | 65 ++++++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 49 insertions(+), 16 deletions(-)

diff --git a/db/db.go b/db/db.go
index afa708b..3e7d4ca 100644
--- a/db/db.go
+++ b/db/db.go
@@ -3,18 +3,22 @@ package db
 import (
 	"context"
 	"database/sql"
+	"fmt"
+	"runtime"
 	"strings"
 	"time"
 
 	"github.com/lbryio/lbry.go/v2/dht/bits"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	qt "github.com/lbryio/lbry.go/v2/extras/query"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
 	"github.com/lbryio/lbry.go/v2/stream"
 
 	"github.com/go-sql-driver/mysql"
 	_ "github.com/go-sql-driver/mysql" // blank import for db driver ensures its imported even if its not used
 	log "github.com/sirupsen/logrus"
 	"github.com/volatiletech/null"
+	"go.uber.org/atomic"
 )
 
 // SdBlob is a special blob that contains information on the rest of the blobs in the stream
@@ -104,33 +108,62 @@ func (s *SQL) AddBlobs(hash []string) error {
 	}
 	// Split the slice into batches of 20 items.
 	batch := 10000
-
-	for i := 0; i < len(hash); i += batch {
-		j := i + batch
-		if j > len(hash) {
-			j = len(hash)
-		}
-		err := s.insertBlobs(hash[i:j]) // Process the batch.
-		if err != nil {
-			log.Errorf("error while inserting batch: %s", errors.FullTrace(err))
+	totalBlobs := int64(len(hash))
+	work := make(chan []string, 1000)
+	stopper := stop.New()
+	var totalInserted atomic.Int64
+	start := time.Now()
+	go func() {
+		for i := 0; i < len(hash); i += batch {
+			j := i + batch
+			if j > len(hash) {
+				j = len(hash)
+			}
+			work <- hash[i:j]
 		}
+		log.Infof("done loading %d hashes in the work queue", len(hash))
+		close(work)
+	}()
+	for i := 0; i < runtime.NumCPU(); i++ {
+		stopper.Add(1)
+		go func(worker int) {
+			log.Infof("starting worker %d", worker)
+			defer stopper.Done()
+			for hashes := range work {
+				inserted := totalInserted.Load()
+				remaining := totalBlobs - inserted
+				if inserted > 0 {
+					timePerBlob := time.Since(start).Microseconds() / inserted
+					remainingTime := time.Duration(remaining*timePerBlob) * time.Microsecond
+					log.Infof("[T%d] processing batch of %d items. ETA: %s", worker, len(hashes), remainingTime.String())
+				}
+				err := s.insertBlobs(hashes) // Process the batch.
+				if err != nil {
+					log.Errorf("error while inserting batch: %s", errors.FullTrace(err))
+				}
+				totalInserted.Add(int64(len(hashes)))
+			}
+		}(i)
 	}
+	stopper.Wait()
 	return nil
 }
 
 func (s *SQL) insertBlobs(hashes []string) error {
 	var (
-		q    string
-		args []interface{}
+		q string
+		//args []interface{}
 	)
-	dayAgo := time.Now().AddDate(0, 0, -1)
+	dayAgo := time.Now().AddDate(0, 0, -1).Format("2006-01-02 15:04:05")
 	q = "insert into blob_ (hash, is_stored, length, last_accessed_at) values "
 	for _, hash := range hashes {
-		q += "(?,?,?,?),"
-		args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
+		// prepared statements slow everything down by a lot due to reflection
+		// for this specific instance we'll go ahead and hardcode the query to make it go faster
+		q += fmt.Sprintf("('%s',1,%d,'%s'),", hash, stream.MaxBlobSize, dayAgo)
+		//args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
 	}
 	q = strings.TrimSuffix(q, ",")
-	_, err := s.exec(q, args...)
+	_, err := s.exec(q)
 	if err != nil {
 		return err
 	}
@@ -293,7 +326,7 @@ func (s *SQL) hasBlobs(hashes []string) (map[string]bool, []uint64, error) {
 	var needsTouch []uint64
 	exists := make(map[string]bool)
 
-	touchDeadline := time.Now().AddDate(0, 0, -1) // touch blob if last accessed before this time
+	touchDeadline := time.Now().Add(-6 * time.Hour) // touch blob if last accessed before this time
 	maxBatchSize := 10000
 	doneIndex := 0
 

From 6291e33ee184ad334f993ea062384787db966643 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Sat, 9 Jan 2021 05:08:20 +0100
Subject: [PATCH 21/51] add tracing to blobs

---
 cmd/getstream.go       |  4 +--
 peer/client.go         | 32 ++++++++++-------
 peer/http3/client.go   | 37 ++++++++++++-------
 peer/http3/server.go   | 22 +++++++++++-
 peer/http3/store.go    |  6 ++--
 peer/server.go         |  6 +++-
 peer/store.go          |  6 ++--
 shared/shared.go       | 82 ++++++++++++++++++++++++++++++++++++++++++
 shared/shared_test.go  | 32 +++++++++++++++++
 store/caching.go       | 13 +++----
 store/caching_test.go  | 11 +++---
 store/cloudfront_ro.go | 22 ++++++------
 store/cloudfront_rw.go |  9 +++--
 store/dbbacked.go      | 13 ++++---
 store/disk.go          | 13 ++++---
 store/disk_test.go     |  4 +--
 store/lfuda.go         | 12 ++++---
 store/lfuda_test.go    |  8 ++---
 store/lru.go           | 12 ++++---
 store/lru_test.go      |  2 +-
 store/memory.go        |  9 +++--
 store/memory_test.go   |  4 +--
 store/noop.go          | 15 +++++---
 store/s3.go            | 16 +++++----
 store/singleflight.go  | 28 +++++++++++----
 store/store.go         |  3 +-
 26 files changed, 317 insertions(+), 104 deletions(-)
 create mode 100644 shared/shared.go
 create mode 100644 shared/shared_test.go

diff --git a/cmd/getstream.go b/cmd/getstream.go
index c02ad0b..1c25df5 100644
--- a/cmd/getstream.go
+++ b/cmd/getstream.go
@@ -41,7 +41,7 @@ func getStreamCmd(cmd *cobra.Command, args []string) {
 
 	var sd stream.SDBlob
 
-	sdb, err := s.Get(sdHash)
+	sdb, _, err := s.Get(sdHash)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -62,7 +62,7 @@ func getStreamCmd(cmd *cobra.Command, args []string) {
 	}
 
 	for i := 0; i < len(sd.BlobInfos)-1; i++ {
-		b, err := s.Get(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
+		b, _, err := s.Get(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
 		if err != nil {
 			log.Fatal(err)
 		}
diff --git a/peer/client.go b/peer/client.go
index cdafd1c..9eae088 100644
--- a/peer/client.go
+++ b/peer/client.go
@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/shared"
 	"github.com/lbryio/reflector.go/store"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -57,10 +58,11 @@ func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Str
 
 	var sd stream.SDBlob
 
-	b, err := c.GetBlob(sdHash)
+	b, trace, err := c.GetBlob(sdHash)
 	if err != nil {
 		return nil, err
 	}
+	log.Debug(trace.String())
 
 	err = sd.FromBlob(b)
 	if err != nil {
@@ -71,10 +73,11 @@ func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Str
 	s[0] = b
 
 	for i := 0; i < len(sd.BlobInfos)-1; i++ {
-		s[i+1], err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
+		s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
 		if err != nil {
 			return nil, err
 		}
+		log.Debug(trace.String())
 	}
 
 	return s, nil
@@ -114,47 +117,52 @@ func (c *Client) HasBlob(hash string) (bool, error) {
 }
 
 // GetBlob gets a blob
-func (c *Client) GetBlob(hash string) (stream.Blob, error) {
+func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
 	if !c.connected {
-		return nil, errors.Err("not connected")
+		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), errors.Err("not connected")
 	}
 
 	sendRequest, err := json.Marshal(blobRequest{
 		RequestedBlob: hash,
 	})
 	if err != nil {
-		return nil, err
+		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
 	}
 
 	err = c.write(sendRequest)
 	if err != nil {
-		return nil, err
+		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
 	}
 
 	var resp blobResponse
 	err = c.read(&resp)
 	if err != nil {
-		return nil, err
+		return nil, shared.NewBlobTrace(time.Since(start), "tcp"), err
 	}
 
+	trace := shared.NewBlobTrace(time.Since(start), "tcp")
+	if resp.RequestTrace != nil {
+		trace = *resp.RequestTrace
+	}
 	if resp.IncomingBlob.Error != "" {
-		return nil, errors.Prefix(hash[:8], resp.IncomingBlob.Error)
+		return nil, trace, errors.Prefix(hash[:8], resp.IncomingBlob.Error)
 	}
 	if resp.IncomingBlob.BlobHash != hash {
-		return nil, errors.Prefix(hash[:8], "blob hash in response does not match requested hash")
+		return nil, trace.Stack(time.Since(start), "tcp"), errors.Prefix(hash[:8], "blob hash in response does not match requested hash")
 	}
 	if resp.IncomingBlob.Length <= 0 {
-		return nil, errors.Prefix(hash[:8], "length reported as <= 0")
+		return nil, trace, errors.Prefix(hash[:8], "length reported as <= 0")
 	}
 
 	log.Debugf("receiving blob %s from %s", hash[:8], c.conn.RemoteAddr())
 
 	blob, err := c.readRawBlob(resp.IncomingBlob.Length)
 	if err != nil {
-		return nil, err
+		return nil, (*resp.RequestTrace).Stack(time.Since(start), "tcp"), err
 	}
 	metrics.MtrInBytesTcp.Add(float64(len(blob)))
-	return blob, nil
+	return blob, trace.Stack(time.Since(start), "tcp"), nil
 }
 
 func (c *Client) read(v interface{}) error {
diff --git a/peer/http3/client.go b/peer/http3/client.go
index ad5c741..d9b1495 100644
--- a/peer/http3/client.go
+++ b/peer/http3/client.go
@@ -9,12 +9,14 @@ import (
 	"sync"
 	"time"
 
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/shared"
 	"github.com/lbryio/reflector.go/store"
 
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lucas-clemente/quic-go/http3"
+	log "github.com/sirupsen/logrus"
 )
 
 // Client is an instance of a client connected to a server.
@@ -35,7 +37,7 @@ func (c *Client) Close() error {
 func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Stream, error) {
 	var sd stream.SDBlob
 
-	b, err := c.GetBlob(sdHash)
+	b, _, err := c.GetBlob(sdHash)
 	if err != nil {
 		return nil, err
 	}
@@ -49,10 +51,12 @@ func (c *Client) GetStream(sdHash string, blobCache store.BlobStore) (stream.Str
 	s[0] = b
 
 	for i := 0; i < len(sd.BlobInfos)-1; i++ {
-		s[i+1], err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
+		var trace shared.BlobTrace
+		s[i+1], trace, err = c.GetBlob(hex.EncodeToString(sd.BlobInfos[i].BlobHash))
 		if err != nil {
 			return nil, err
 		}
+		log.Debug(trace.String())
 	}
 
 	return s, nil
@@ -75,26 +79,35 @@ func (c *Client) HasBlob(hash string) (bool, error) {
 }
 
 // GetBlob gets a blob
-func (c *Client) GetBlob(hash string) (stream.Blob, error) {
-	resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s", c.ServerAddr, hash))
+func (c *Client) GetBlob(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	resp, err := c.conn.Get(fmt.Sprintf("https://%s/get/%s?trace=true", c.ServerAddr, hash))
 	if err != nil {
-		return nil, errors.Err(err)
+		return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(err)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode == http.StatusNotFound {
 		fmt.Printf("%s blob not found %d\n", hash, resp.StatusCode)
-		return nil, errors.Err(store.ErrBlobNotFound)
+		return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err(store.ErrBlobNotFound)
 	} else if resp.StatusCode != http.StatusOK {
-		return nil, errors.Err("non 200 status code returned: %d", resp.StatusCode)
+		return nil, shared.NewBlobTrace(time.Since(start), "http3"), errors.Err("non 200 status code returned: %d", resp.StatusCode)
 	}
 
 	tmp := getBuffer()
 	defer putBuffer(tmp)
-
+	serialized := resp.Header.Get("Via")
+	trace := shared.NewBlobTrace(time.Since(start), "http3")
+	if serialized != "" {
+		parsedTrace, err := shared.Deserialize(serialized)
+		if err != nil {
+			return nil, shared.NewBlobTrace(time.Since(start), "http3"), err
+		}
+		trace = *parsedTrace
+	}
 	written, err := io.Copy(tmp, resp.Body)
 	if err != nil {
-		return nil, errors.Err(err)
+		return nil, trace.Stack(time.Since(start), "http3"), errors.Err(err)
 	}
 
 	blob := make([]byte, written)
@@ -102,7 +115,7 @@ func (c *Client) GetBlob(hash string) (stream.Blob, error) {
 
 	metrics.MtrInBytesUdp.Add(float64(len(blob)))
 
-	return blob, nil
+	return blob, trace.Stack(time.Since(start), "http3"), nil
 }
 
 // buffer pool to reduce GC
diff --git a/peer/http3/server.go b/peer/http3/server.go
index aaadac0..c33a676 100644
--- a/peer/http3/server.go
+++ b/peer/http3/server.go
@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"math/big"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/lbryio/reflector.go/internal/metrics"
@@ -71,7 +72,26 @@ func (s *Server) Start(address string) error {
 	r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
 		vars := mux.Vars(r)
 		requestedBlob := vars["hash"]
-		blob, err := s.store.Get(requestedBlob)
+		traceParam := r.URL.Query().Get("trace")
+		var err error
+		wantsTrace := false
+		if traceParam != "" {
+			wantsTrace, err = strconv.ParseBool(traceParam)
+			if err != nil {
+				wantsTrace = false
+			}
+		}
+		blob, trace, err := s.store.Get(requestedBlob)
+
+		if wantsTrace {
+			serialized, err := trace.Serialize()
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusNotFound)
+				return
+			}
+			w.Header().Add("Via", serialized)
+			log.Debug(trace.String())
+		}
 		if err != nil {
 			if errors.Is(err, store.ErrBlobNotFound) {
 				http.Error(w, err.Error(), http.StatusNotFound)
diff --git a/peer/http3/store.go b/peer/http3/store.go
index b681443..2d9a23c 100644
--- a/peer/http3/store.go
+++ b/peer/http3/store.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/shared"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/lucas-clemente/quic-go/http3"
 )
@@ -68,10 +69,11 @@ func (p *Store) Has(hash string) (bool, error) {
 }
 
 // Get downloads the blob from the peer
-func (p *Store) Get(hash string) (stream.Blob, error) {
+func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
 	c, err := p.getClient()
 	if err != nil {
-		return nil, err
+		return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
 	}
 	defer c.Close()
 	return c.GetBlob(hash)
diff --git a/peer/server.go b/peer/server.go
index 44916cb..708fe53 100644
--- a/peer/server.go
+++ b/peer/server.go
@@ -12,6 +12,7 @@ import (
 
 	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/reflector"
+	"github.com/lbryio/reflector.go/shared"
 	"github.com/lbryio/reflector.go/store"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -253,6 +254,7 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
 	}
 
 	var blob []byte
+	var trace shared.BlobTrace
 	if request.RequestedBlob != "" {
 		if len(request.RequestedBlob) != stream.BlobHashHexLength {
 			return nil, errors.Err("Invalid blob hash length")
@@ -260,7 +262,8 @@ func (s *Server) handleCompositeRequest(data []byte) ([]byte, error) {
 
 		log.Debugln("Sending blob " + request.RequestedBlob[:8])
 
-		blob, err = s.store.Get(request.RequestedBlob)
+		blob, trace, err = s.store.Get(request.RequestedBlob)
+		log.Debug(trace.String())
 		if errors.Is(err, store.ErrBlobNotFound) {
 			response.IncomingBlob = incomingBlob{
 				Error: err.Error(),
@@ -382,6 +385,7 @@ type incomingBlob struct {
 }
 type blobResponse struct {
 	IncomingBlob incomingBlob `json:"incoming_blob"`
+	RequestTrace *shared.BlobTrace
 }
 
 type compositeRequest struct {
diff --git a/peer/store.go b/peer/store.go
index 3ef9622..c227884 100644
--- a/peer/store.go
+++ b/peer/store.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/shared"
 )
 
 // Store is a blob store that gets blobs from a peer.
@@ -43,10 +44,11 @@ func (p *Store) Has(hash string) (bool, error) {
 }
 
 // Get downloads the blob from the peer
-func (p *Store) Get(hash string) (stream.Blob, error) {
+func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
 	c, err := p.getClient()
 	if err != nil {
-		return nil, err
+		return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
 	}
 	defer c.Close()
 	return c.GetBlob(hash)
diff --git a/shared/shared.go b/shared/shared.go
new file mode 100644
index 0000000..c232cf6
--- /dev/null
+++ b/shared/shared.go
@@ -0,0 +1,82 @@
+package shared
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+)
+
+type BlobStack struct {
+	Timing     time.Duration `json:"timing"`
+	OriginName string        `json:"origin_name"`
+	HostName   string        `json:"host_name"`
+}
+type BlobTrace struct {
+	Stacks []BlobStack `json:"stacks"`
+}
+
+var hostName *string
+
+func getHostName() string {
+	if hostName == nil {
+		hn, err := os.Hostname()
+		if err != nil {
+			hn = "unknown"
+		}
+		hostName = &hn
+	}
+	return *hostName
+}
+func (b *BlobTrace) Stack(timing time.Duration, originName string) BlobTrace {
+	b.Stacks = append(b.Stacks, BlobStack{
+		Timing:     timing,
+		OriginName: originName,
+		HostName:   getHostName(),
+	})
+	return *b
+}
+func (b *BlobTrace) Merge(otherTrance BlobTrace) BlobTrace {
+	b.Stacks = append(b.Stacks, otherTrance.Stacks...)
+	return *b
+}
+func NewBlobTrace(timing time.Duration, originName string) BlobTrace {
+	b := BlobTrace{}
+	b.Stacks = append(b.Stacks, BlobStack{
+		Timing:     timing,
+		OriginName: originName,
+		HostName:   getHostName(),
+	})
+	return b
+}
+
+func (b BlobTrace) String() string {
+	var fullTrace string
+	for i, stack := range b.Stacks {
+		delta := time.Duration(0)
+		if i > 0 {
+			delta = stack.Timing - b.Stacks[i-1].Timing
+		}
+		fullTrace += fmt.Sprintf("[%d](%s) origin: %s - timing: %s - delta: %s\n", i, stack.HostName, stack.OriginName, stack.Timing.String(), delta.String())
+	}
+	return fullTrace
+}
+
+func (b BlobTrace) Serialize() (string, error) {
+	t, err := json.Marshal(b)
+	if err != nil {
+		return "", errors.Err(err)
+	}
+	return string(t), nil
+}
+
+func Deserialize(serializedData string) (*BlobTrace, error) {
+	var trace BlobTrace
+	err := json.Unmarshal([]byte(serializedData), &trace)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	return &trace, nil
+}
diff --git a/shared/shared_test.go b/shared/shared_test.go
new file mode 100644
index 0000000..47c8eef
--- /dev/null
+++ b/shared/shared_test.go
@@ -0,0 +1,32 @@
+package shared
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBlobTrace_Serialize(t *testing.T) {
+	stack := NewBlobTrace(10*time.Second, "test")
+	stack.Stack(20*time.Second, "test2")
+	stack.Stack(30*time.Second, "test3")
+	serialized, err := stack.Serialize()
+	assert.NoError(t, err)
+	t.Log(serialized)
+	expected := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}"
+	assert.Equal(t, expected, serialized)
+}
+
+func TestBlobTrace_Deserialize(t *testing.T) {
+	serialized := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}"
+	stack, err := Deserialize(serialized)
+	assert.NoError(t, err)
+	assert.Len(t, stack.Stacks, 3)
+	assert.Equal(t, stack.Stacks[0].Timing, 10*time.Second)
+	assert.Equal(t, stack.Stacks[1].Timing, 20*time.Second)
+	assert.Equal(t, stack.Stacks[2].Timing, 30*time.Second)
+	assert.Equal(t, stack.Stacks[0].OriginName, "test")
+	assert.Equal(t, stack.Stacks[1].OriginName, "test2")
+	assert.Equal(t, stack.Stacks[2].OriginName, "test3")
+}
diff --git a/store/caching.go b/store/caching.go
index 53b692c..6f1d4d4 100644
--- a/store/caching.go
+++ b/store/caching.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/shared"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/lbryio/reflector.go/internal/metrics"
@@ -43,9 +44,9 @@ func (c *CachingStore) Has(hash string) (bool, error) {
 
 // Get tries to get the blob from the cache first, falling back to the origin. If the blob comes
 // from the origin, it is also stored in the cache.
-func (c *CachingStore) Get(hash string) (stream.Blob, error) {
+func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 	start := time.Now()
-	blob, err := c.cache.Get(hash)
+	blob, trace, err := c.cache.Get(hash)
 	if err == nil || !errors.Is(err, ErrBlobNotFound) {
 		metrics.CacheHitCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
 		rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
@@ -54,14 +55,14 @@ func (c *CachingStore) Get(hash string) (stream.Blob, error) {
 			metrics.LabelComponent: c.component,
 			metrics.LabelSource:    "cache",
 		}).Set(rate)
-		return blob, err
+		return blob, trace.Stack(time.Since(start), c.Name()), err
 	}
 
 	metrics.CacheMissCount.With(metrics.CacheLabels(c.cache.Name(), c.component)).Inc()
 
-	blob, err = c.origin.Get(hash)
+	blob, trace, err = c.origin.Get(hash)
 	if err != nil {
-		return nil, err
+		return nil, trace.Stack(time.Since(start), c.Name()), err
 	}
 	// there is no need to wait for the blob to be stored before we return it
 	// TODO: however this should be refactored to limit the amount of routines that the process can spawn to avoid a possible DoS
@@ -71,7 +72,7 @@ func (c *CachingStore) Get(hash string) (stream.Blob, error) {
 			log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
 		}
 	}()
-	return blob, nil
+	return blob, trace.Stack(time.Since(start), c.Name()), nil
 }
 
 // Put stores the blob in the origin and the cache
diff --git a/store/caching_test.go b/store/caching_test.go
index 0636583..66a42fd 100644
--- a/store/caching_test.go
+++ b/store/caching_test.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/shared"
 )
 
 func TestCachingStore_Put(t *testing.T) {
@@ -51,7 +52,7 @@ func TestCachingStore_CacheMiss(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	res, err := s.Get(hash)
+	res, stack, err := s.Get(hash)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -67,14 +68,16 @@ func TestCachingStore_CacheMiss(t *testing.T) {
 	if !has {
 		t.Errorf("Get() did not copy blob to cache")
 	}
+	t.Logf("stack: %s", stack.String())
 
-	res, err = cache.Get(hash)
+	res, stack, err = cache.Get(hash)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if !bytes.Equal(b, res) {
 		t.Errorf("expected cached Get() to return %s, got %s", string(b), string(res))
 	}
+	t.Logf("stack: %s", stack.String())
 }
 
 func TestCachingStore_ThunderingHerd(t *testing.T) {
@@ -93,7 +96,7 @@ func TestCachingStore_ThunderingHerd(t *testing.T) {
 	wg := &sync.WaitGroup{}
 
 	getNoErr := func() {
-		res, err := s.Get(hash)
+		res, _, err := s.Get(hash)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -149,7 +152,7 @@ func (s *SlowBlobStore) Has(hash string) (bool, error) {
 	return s.mem.Has(hash)
 }
 
-func (s *SlowBlobStore) Get(hash string) (stream.Blob, error) {
+func (s *SlowBlobStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 	time.Sleep(s.delay)
 	return s.mem.Get(hash)
 }
diff --git a/store/cloudfront_ro.go b/store/cloudfront_ro.go
index 3dcffc0..51fe4b8 100644
--- a/store/cloudfront_ro.go
+++ b/store/cloudfront_ro.go
@@ -6,11 +6,11 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/lbryio/reflector.go/internal/metrics"
-	"github.com/lbryio/reflector.go/meta"
-
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/meta"
+	"github.com/lbryio/reflector.go/shared"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -49,30 +49,30 @@ func (c *CloudFrontROStore) Has(hash string) (bool, error) {
 }
 
 // Get gets the blob from Cloudfront.
-func (c *CloudFrontROStore) Get(hash string) (stream.Blob, error) {
+func (c *CloudFrontROStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 	log.Debugf("Getting %s from S3", hash[:8])
+	start := time.Now()
 	defer func(t time.Time) {
 		log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
-	}(time.Now())
+	}(start)
 
 	status, body, err := c.cfRequest(http.MethodGet, hash)
 	if err != nil {
-		return nil, err
+		return nil, shared.NewBlobTrace(time.Since(start), c.Name()), err
 	}
 	defer body.Close()
-
 	switch status {
 	case http.StatusNotFound, http.StatusForbidden:
-		return nil, errors.Err(ErrBlobNotFound)
+		return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(ErrBlobNotFound)
 	case http.StatusOK:
 		b, err := ioutil.ReadAll(body)
 		if err != nil {
-			return nil, errors.Err(err)
+			return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err(err)
 		}
 		metrics.MtrInBytesS3.Add(float64(len(b)))
-		return b, nil
+		return b, shared.NewBlobTrace(time.Since(start), c.Name()), nil
 	default:
-		return nil, errors.Err("unexpected status %d", status)
+		return nil, shared.NewBlobTrace(time.Since(start), c.Name()), errors.Err("unexpected status %d", status)
 	}
 }
 
diff --git a/store/cloudfront_rw.go b/store/cloudfront_rw.go
index 6de64af..32ee418 100644
--- a/store/cloudfront_rw.go
+++ b/store/cloudfront_rw.go
@@ -1,7 +1,10 @@
 package store
 
 import (
+	"time"
+
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/shared"
 )
 
 // CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront, writes go to S3.
@@ -30,8 +33,10 @@ func (c *CloudFrontRWStore) Has(hash string) (bool, error) {
 }
 
 // Get gets the blob from Cloudfront.
-func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, error) {
-	return c.cf.Get(hash)
+func (c *CloudFrontRWStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	blob, trace, err := c.cf.Get(hash)
+	return blob, trace.Stack(time.Since(start), c.Name()), err
 }
 
 // Put stores the blob on S3
diff --git a/store/dbbacked.go b/store/dbbacked.go
index 1484c49..90c9fed 100644
--- a/store/dbbacked.go
+++ b/store/dbbacked.go
@@ -3,10 +3,12 @@ package store
 import (
 	"encoding/json"
 	"sync"
+	"time"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lbryio/reflector.go/db"
+	"github.com/lbryio/reflector.go/shared"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -36,16 +38,17 @@ func (d *DBBackedStore) Has(hash string) (bool, error) {
 }
 
 // Get gets the blob
-func (d *DBBackedStore) Get(hash string) (stream.Blob, error) {
+func (d *DBBackedStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
 	has, err := d.db.HasBlob(hash, true)
 	if err != nil {
-		return nil, err
+		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
 	}
 	if !has {
-		return nil, ErrBlobNotFound
+		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), ErrBlobNotFound
 	}
 
-	b, err := d.blobs.Get(hash)
+	b, stack, err := d.blobs.Get(hash)
 	if d.deleteOnMiss && errors.Is(err, ErrBlobNotFound) {
 		e2 := d.Delete(hash)
 		if e2 != nil {
@@ -53,7 +56,7 @@ func (d *DBBackedStore) Get(hash string) (stream.Blob, error) {
 		}
 	}
 
-	return b, err
+	return b, stack.Stack(time.Since(start), d.Name()), err
 }
 
 // Put stores the blob in the S3 store and stores the blob information in the DB.
diff --git a/store/disk.go b/store/disk.go
index 03cc393..412d326 100644
--- a/store/disk.go
+++ b/store/disk.go
@@ -4,9 +4,11 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"time"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/shared"
 	"github.com/lbryio/reflector.go/store/speedwalk"
 )
 
@@ -52,21 +54,22 @@ func (d *DiskStore) Has(hash string) (bool, error) {
 }
 
 // Get returns the blob or an error if the blob doesn't exist.
-func (d *DiskStore) Get(hash string) (stream.Blob, error) {
+func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
 	err := d.initOnce()
 	if err != nil {
-		return nil, err
+		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
 	}
 
 	blob, err := ioutil.ReadFile(d.path(hash))
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, errors.Err(ErrBlobNotFound)
+			return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(ErrBlobNotFound)
 		}
-		return nil, errors.Err(err)
+		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
 	}
 
-	return blob, nil
+	return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
 }
 
 // Put stores the blob on disk
diff --git a/store/disk_test.go b/store/disk_test.go
index 3bc088a..72c7755 100644
--- a/store/disk_test.go
+++ b/store/disk_test.go
@@ -28,7 +28,7 @@ func TestDiskStore_Get(t *testing.T) {
 	err = ioutil.WriteFile(expectedPath, data, os.ModePerm)
 	require.NoError(t, err)
 
-	blob, err := d.Get(hash)
+	blob, _, err := d.Get(hash)
 	assert.NoError(t, err)
 	assert.EqualValues(t, data, blob)
 }
@@ -39,7 +39,7 @@ func TestDiskStore_GetNonexistentBlob(t *testing.T) {
 	defer os.RemoveAll(tmpDir)
 	d := NewDiskStore(tmpDir, 2)
 
-	blob, err := d.Get("nonexistent")
+	blob, _, err := d.Get("nonexistent")
 	assert.Nil(t, blob)
 	assert.True(t, errors.Is(err, ErrBlobNotFound))
 }
diff --git a/store/lfuda.go b/store/lfuda.go
index 0eb7296..b0437b0 100644
--- a/store/lfuda.go
+++ b/store/lfuda.go
@@ -1,10 +1,13 @@
 package store
 
 import (
+	"time"
+
 	"github.com/bparli/lfuda-go"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/shared"
 	"github.com/sirupsen/logrus"
 )
 
@@ -49,17 +52,18 @@ func (l *LFUDAStore) Has(hash string) (bool, error) {
 }
 
 // Get returns the blob or an error if the blob doesn't exist.
-func (l *LFUDAStore) Get(hash string) (stream.Blob, error) {
+func (l *LFUDAStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
 	_, has := l.lfuda.Get(hash)
 	if !has {
-		return nil, errors.Err(ErrBlobNotFound)
+		return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
 	}
-	blob, err := l.store.Get(hash)
+	blob, stack, err := l.store.Get(hash)
 	if errors.Is(err, ErrBlobNotFound) {
 		// Blob disappeared from underlying store
 		l.lfuda.Remove(hash)
 	}
-	return blob, err
+	return blob, stack.Stack(time.Since(start), l.Name()), err
 }
 
 // Put stores the blob. Following LFUDA rules it's not guaranteed that a SET will store the value!!!
diff --git a/store/lfuda_test.go b/store/lfuda_test.go
index d5eae47..d8657ac 100644
--- a/store/lfuda_test.go
+++ b/store/lfuda_test.go
@@ -40,11 +40,11 @@ func TestFUDAStore_Eviction(t *testing.T) {
 	err = lfuda.Put("two", b)
 	require.NoError(t, err)
 
-	_, err = lfuda.Get("five")
+	_, _, err = lfuda.Get("five")
 	require.NoError(t, err)
-	_, err = lfuda.Get("four")
+	_, _, err = lfuda.Get("four")
 	require.NoError(t, err)
-	_, err = lfuda.Get("two")
+	_, _, err = lfuda.Get("two")
 	require.NoError(t, err)
 	assert.Equal(t, cacheMaxBlobs, len(mem.Debug()))
 
@@ -102,7 +102,7 @@ func TestFUDAStore_UnderlyingBlobMissing(t *testing.T) {
 	// hash still exists in lru
 	assert.True(t, lfuda.lfuda.Contains(hash))
 
-	blob, err := lfuda.Get(hash)
+	blob, _, err := lfuda.Get(hash)
 	assert.Nil(t, blob)
 	assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s",
 		reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(),
diff --git a/store/lru.go b/store/lru.go
index ce0d83d..e96931b 100644
--- a/store/lru.go
+++ b/store/lru.go
@@ -1,9 +1,12 @@
 package store
 
 import (
+	"time"
+
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/shared"
 
 	golru "github.com/hashicorp/golang-lru"
 	"github.com/sirupsen/logrus"
@@ -56,17 +59,18 @@ func (l *LRUStore) Has(hash string) (bool, error) {
 }
 
 // Get returns the blob or an error if the blob doesn't exist.
-func (l *LRUStore) Get(hash string) (stream.Blob, error) {
+func (l *LRUStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
 	_, has := l.lru.Get(hash)
 	if !has {
-		return nil, errors.Err(ErrBlobNotFound)
+		return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
 	}
-	blob, err := l.store.Get(hash)
+	blob, stack, err := l.store.Get(hash)
 	if errors.Is(err, ErrBlobNotFound) {
 		// Blob disappeared from underlying store
 		l.lru.Remove(hash)
 	}
-	return blob, err
+	return blob, stack.Stack(time.Since(start), l.Name()), err
 }
 
 // Put stores the blob
diff --git a/store/lru_test.go b/store/lru_test.go
index 2bd934c..6aec768 100644
--- a/store/lru_test.go
+++ b/store/lru_test.go
@@ -89,7 +89,7 @@ func TestLRUStore_UnderlyingBlobMissing(t *testing.T) {
 	// hash still exists in lru
 	assert.True(t, lru.lru.Contains(hash))
 
-	blob, err := lru.Get(hash)
+	blob, _, err := lru.Get(hash)
 	assert.Nil(t, blob)
 	assert.True(t, errors.Is(err, ErrBlobNotFound), "expected (%s) %s, got (%s) %s",
 		reflect.TypeOf(ErrBlobNotFound).String(), ErrBlobNotFound.Error(),
diff --git a/store/memory.go b/store/memory.go
index 62b93b6..30d8dea 100644
--- a/store/memory.go
+++ b/store/memory.go
@@ -2,9 +2,11 @@ package store
 
 import (
 	"sync"
+	"time"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/shared"
 )
 
 // MemStore is an in memory only blob store with no persistence.
@@ -34,14 +36,15 @@ func (m *MemStore) Has(hash string) (bool, error) {
 }
 
 // Get returns the blob byte slice if present and errors if the blob is not found.
-func (m *MemStore) Get(hash string) (stream.Blob, error) {
+func (m *MemStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
 	m.mu.RLock()
 	defer m.mu.RUnlock()
 	blob, ok := m.blobs[hash]
 	if !ok {
-		return nil, errors.Err(ErrBlobNotFound)
+		return nil, shared.NewBlobTrace(time.Since(start), m.Name()), errors.Err(ErrBlobNotFound)
 	}
-	return blob, nil
+	return blob, shared.NewBlobTrace(time.Since(start), m.Name()), nil
 }
 
 // Put stores the blob in memory
diff --git a/store/memory_test.go b/store/memory_test.go
index 8d85114..775850b 100644
--- a/store/memory_test.go
+++ b/store/memory_test.go
@@ -25,7 +25,7 @@ func TestMemStore_Get(t *testing.T) {
 		t.Error("error getting memory blob - ", err)
 	}
 
-	gotBlob, err := s.Get(hash)
+	gotBlob, _, err := s.Get(hash)
 	if err != nil {
 		t.Errorf("Expected no error, got %v", err)
 	}
@@ -33,7 +33,7 @@ func TestMemStore_Get(t *testing.T) {
 		t.Error("Got blob that is different from expected blob")
 	}
 
-	missingBlob, err := s.Get("nonexistent hash")
+	missingBlob, _, err := s.Get("nonexistent hash")
 	if err == nil {
 		t.Errorf("Expected ErrBlobNotFound, got nil")
 	}
diff --git a/store/noop.go b/store/noop.go
index 2792a0d..9cadf5c 100644
--- a/store/noop.go
+++ b/store/noop.go
@@ -1,15 +1,22 @@
 package store
 
-import "github.com/lbryio/lbry.go/v2/stream"
+import (
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/shared"
+)
 
 // NoopStore is a store that does nothing
 type NoopStore struct{}
 
 const nameNoop = "noop"
 
-func (n *NoopStore) Name() string                        { return nameNoop }
-func (n *NoopStore) Has(_ string) (bool, error)          { return false, nil }
-func (n *NoopStore) Get(_ string) (stream.Blob, error)   { return nil, nil }
+func (n *NoopStore) Name() string               { return nameNoop }
+func (n *NoopStore) Has(_ string) (bool, error) { return false, nil }
+func (n *NoopStore) Get(_ string) (stream.Blob, shared.BlobTrace, error) {
+	return nil, shared.NewBlobTrace(time.Since(time.Now()), n.Name()), nil
+}
 func (n *NoopStore) Put(_ string, _ stream.Blob) error   { return nil }
 func (n *NoopStore) PutSD(_ string, _ stream.Blob) error { return nil }
 func (n *NoopStore) Delete(_ string) error               { return nil }
diff --git a/store/s3.go b/store/s3.go
index 02a23c5..75d66cc 100644
--- a/store/s3.go
+++ b/store/s3.go
@@ -8,6 +8,7 @@ import (
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/shared"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -65,17 +66,18 @@ func (s *S3Store) Has(hash string) (bool, error) {
 }
 
 // Get returns the blob slice if present or errors on S3.
-func (s *S3Store) Get(hash string) (stream.Blob, error) {
+func (s *S3Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
 	//Todo-Need to handle error for blob doesn't exist for consistency.
 	err := s.initOnce()
 	if err != nil {
-		return nil, err
+		return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
 	}
 
 	log.Debugf("Getting %s from S3", hash[:8])
 	defer func(t time.Time) {
 		log.Debugf("Getting %s from S3 took %s", hash[:8], time.Since(t).String())
-	}(time.Now())
+	}(start)
 
 	buf := &aws.WriteAtBuffer{}
 	_, err = s3manager.NewDownloader(s.session).Download(buf, &s3.GetObjectInput{
@@ -86,15 +88,15 @@ func (s *S3Store) Get(hash string) (stream.Blob, error) {
 		if aerr, ok := err.(awserr.Error); ok {
 			switch aerr.Code() {
 			case s3.ErrCodeNoSuchBucket:
-				return nil, errors.Err("bucket %s does not exist", s.bucket)
+				return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("bucket %s does not exist", s.bucket)
 			case s3.ErrCodeNoSuchKey:
-				return nil, errors.Err(ErrBlobNotFound)
+				return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err(ErrBlobNotFound)
 			}
 		}
-		return buf.Bytes(), err
+		return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), err
 	}
 
-	return buf.Bytes(), nil
+	return buf.Bytes(), shared.NewBlobTrace(time.Since(start), s.Name()), nil
 }
 
 // Put stores the blob on S3 or errors if S3 connection errors.
diff --git a/store/singleflight.go b/store/singleflight.go
index 7d0139d..362182d 100644
--- a/store/singleflight.go
+++ b/store/singleflight.go
@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/shared"
 
 	"github.com/lbryio/lbry.go/v2/stream"
 
@@ -29,17 +30,24 @@ func (s *singleflightStore) Name() string {
 	return "sf_" + s.BlobStore.Name()
 }
 
+type getterResponse struct {
+	blob  stream.Blob
+	stack shared.BlobTrace
+}
+
 // Get ensures that only one request per hash is sent to the origin at a time,
 // thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem
-func (s *singleflightStore) Get(hash string) (stream.Blob, error) {
+func (s *singleflightStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
 	metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
 	defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
 
-	blob, err, _ := s.sf.Do(hash, s.getter(hash))
+	gr, err, _ := s.sf.Do(hash, s.getter(hash))
 	if err != nil {
-		return nil, err
+		return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
 	}
-	return blob.(stream.Blob), nil
+	rsp := gr.(getterResponse)
+	return rsp.blob, rsp.stack, nil
 }
 
 // getter returns a function that gets a blob from the origin
@@ -50,9 +58,12 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) {
 		defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
 
 		start := time.Now()
-		blob, err := s.BlobStore.Get(hash)
+		blob, stack, err := s.BlobStore.Get(hash)
 		if err != nil {
-			return nil, err
+			return getterResponse{
+				blob:  nil,
+				stack: stack.Stack(time.Since(start), s.Name()),
+			}, err
 		}
 
 		rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
@@ -62,7 +73,10 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) {
 			metrics.LabelSource:    "origin",
 		}).Set(rate)
 
-		return blob, nil
+		return getterResponse{
+			blob:  blob,
+			stack: stack.Stack(time.Since(start), s.Name()),
+		}, nil
 	}
 }
 
diff --git a/store/store.go b/store/store.go
index 36d388d..bd9223b 100644
--- a/store/store.go
+++ b/store/store.go
@@ -3,6 +3,7 @@ package store
 import (
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/shared"
 )
 
 // BlobStore is an interface for handling blob storage.
@@ -12,7 +13,7 @@ type BlobStore interface {
 	// Does blob exist in the store.
 	Has(hash string) (bool, error)
 	// Get the blob from the store. Must return ErrBlobNotFound if blob is not in store.
-	Get(hash string) (stream.Blob, error)
+	Get(hash string) (stream.Blob, shared.BlobTrace, error)
 	// Put the blob into the store.
 	Put(hash string, blob stream.Blob) error
 	// Put an SD blob into the store.

From 7b49dd115b75b1fd16362248dcee1d7e631926eb Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 23 Feb 2021 15:08:32 +0100
Subject: [PATCH 22/51] remove panics

---
 peer/http3/store.go    | 9 ++++++---
 peer/store.go          | 9 ++++++---
 store/cloudfront_ro.go | 6 +++---
 store/store.go         | 3 +++
 4 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/peer/http3/store.go b/peer/http3/store.go
index 2d9a23c..f842437 100644
--- a/peer/http3/store.go
+++ b/peer/http3/store.go
@@ -13,6 +13,9 @@ import (
 	"github.com/lucas-clemente/quic-go/http3"
 )
 
+//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
+var ErrNotImplemented = errors.Base("this store does not implement this method")
+
 // Store is a blob store that gets blobs from a peer.
 // It satisfies the store.BlobStore interface but cannot put or delete blobs.
 type Store struct {
@@ -81,17 +84,17 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 
 // Put is not supported
 func (p *Store) Put(hash string, blob stream.Blob) error {
-	panic("http3Store cannot put or delete blobs")
+	return errors.Err(ErrNotImplemented)
 }
 
 // PutSD is not supported
 func (p *Store) PutSD(hash string, blob stream.Blob) error {
-	panic("http3Store cannot put or delete blobs")
+	return errors.Err(ErrNotImplemented)
 }
 
 // Delete is not supported
 func (p *Store) Delete(hash string) error {
-	panic("http3Store cannot put or delete blobs")
+	return errors.Err(ErrNotImplemented)
 }
 
 // Delete is not supported
diff --git a/peer/store.go b/peer/store.go
index c227884..6bf3b38 100644
--- a/peer/store.go
+++ b/peer/store.go
@@ -8,6 +8,9 @@ import (
 	"github.com/lbryio/reflector.go/shared"
 )
 
+//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
+var ErrNotImplemented = errors.Base("this store does not implement this method")
+
 // Store is a blob store that gets blobs from a peer.
 // It satisfies the store.BlobStore interface but cannot put or delete blobs.
 type Store struct {
@@ -56,17 +59,17 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 
 // Put is not supported
 func (p *Store) Put(hash string, blob stream.Blob) error {
-	panic("PeerStore cannot put or delete blobs")
+	return errors.Err(ErrNotImplemented)
 }
 
 // PutSD is not supported
 func (p *Store) PutSD(hash string, blob stream.Blob) error {
-	panic("PeerStore cannot put or delete blobs")
+	return errors.Err(ErrNotImplemented)
 }
 
 // Delete is not supported
 func (p *Store) Delete(hash string) error {
-	panic("PeerStore cannot put or delete blobs")
+	return errors.Err(ErrNotImplemented)
 }
 
 // Delete is not supported
diff --git a/store/cloudfront_ro.go b/store/cloudfront_ro.go
index 51fe4b8..fcc8ed0 100644
--- a/store/cloudfront_ro.go
+++ b/store/cloudfront_ro.go
@@ -93,15 +93,15 @@ func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser,
 }
 
 func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error {
-	panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore")
+	return errors.Err(ErrNotImplemented)
 }
 
 func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
-	panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore")
+	return errors.Err(ErrNotImplemented)
 }
 
 func (c *CloudFrontROStore) Delete(_ string) error {
-	panic("CloudFrontROStore cannot do writes. Use CloudFrontRWStore")
+	return errors.Err(ErrNotImplemented)
 }
 
 // Shutdown shuts down the store gracefully
diff --git a/store/store.go b/store/store.go
index bd9223b..3a2fc6f 100644
--- a/store/store.go
+++ b/store/store.go
@@ -40,3 +40,6 @@ type lister interface {
 
 //ErrBlobNotFound is a standard error when a blob is not found in the store.
 var ErrBlobNotFound = errors.Base("blob not found")
+
+//ErrNotImplemented is a standard error when a store that implements this interface does not implement a method
+var ErrNotImplemented = errors.Base("this store does not implement this method")

From 8cb73896199b9ec808505e6b42435d40f75654b3 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 23 Feb 2021 15:23:46 +0100
Subject: [PATCH 23/51] make it simpler

---
 peer/http3/store.go    | 9 +++------
 peer/store.go          | 9 +++------
 shared/errors.go       | 6 ++++++
 store/cloudfront_ro.go | 6 +++---
 store/store.go         | 3 ---
 5 files changed, 15 insertions(+), 18 deletions(-)
 create mode 100644 shared/errors.go

diff --git a/peer/http3/store.go b/peer/http3/store.go
index f842437..8a5c812 100644
--- a/peer/http3/store.go
+++ b/peer/http3/store.go
@@ -13,9 +13,6 @@ import (
 	"github.com/lucas-clemente/quic-go/http3"
 )
 
-//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
-var ErrNotImplemented = errors.Base("this store does not implement this method")
-
 // Store is a blob store that gets blobs from a peer.
 // It satisfies the store.BlobStore interface but cannot put or delete blobs.
 type Store struct {
@@ -84,17 +81,17 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 
 // Put is not supported
 func (p *Store) Put(hash string, blob stream.Blob) error {
-	return errors.Err(ErrNotImplemented)
+	return errors.Err(shared.ErrNotImplemented)
 }
 
 // PutSD is not supported
 func (p *Store) PutSD(hash string, blob stream.Blob) error {
-	return errors.Err(ErrNotImplemented)
+	return errors.Err(shared.ErrNotImplemented)
 }
 
 // Delete is not supported
 func (p *Store) Delete(hash string) error {
-	return errors.Err(ErrNotImplemented)
+	return errors.Err(shared.ErrNotImplemented)
 }
 
 // Delete is not supported
diff --git a/peer/store.go b/peer/store.go
index 6bf3b38..3068794 100644
--- a/peer/store.go
+++ b/peer/store.go
@@ -8,9 +8,6 @@ import (
 	"github.com/lbryio/reflector.go/shared"
 )
 
-//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
-var ErrNotImplemented = errors.Base("this store does not implement this method")
-
 // Store is a blob store that gets blobs from a peer.
 // It satisfies the store.BlobStore interface but cannot put or delete blobs.
 type Store struct {
@@ -59,17 +56,17 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 
 // Put is not supported
 func (p *Store) Put(hash string, blob stream.Blob) error {
-	return errors.Err(ErrNotImplemented)
+	return errors.Err(shared.ErrNotImplemented)
 }
 
 // PutSD is not supported
 func (p *Store) PutSD(hash string, blob stream.Blob) error {
-	return errors.Err(ErrNotImplemented)
+	return errors.Err(shared.ErrNotImplemented)
 }
 
 // Delete is not supported
 func (p *Store) Delete(hash string) error {
-	return errors.Err(ErrNotImplemented)
+	return errors.Err(shared.ErrNotImplemented)
 }
 
 // Delete is not supported
diff --git a/shared/errors.go b/shared/errors.go
new file mode 100644
index 0000000..f103394
--- /dev/null
+++ b/shared/errors.go
@@ -0,0 +1,6 @@
+package shared
+
+import "github.com/lbryio/lbry.go/v2/extras/errors"
+
+//ErrNotImplemented is a standard error when a store that implements the store interface does not implement a method
+var ErrNotImplemented = errors.Base("this store does not implement this method")
diff --git a/store/cloudfront_ro.go b/store/cloudfront_ro.go
index fcc8ed0..757174c 100644
--- a/store/cloudfront_ro.go
+++ b/store/cloudfront_ro.go
@@ -93,15 +93,15 @@ func (c *CloudFrontROStore) cfRequest(method, hash string) (int, io.ReadCloser,
 }
 
 func (c *CloudFrontROStore) Put(_ string, _ stream.Blob) error {
-	return errors.Err(ErrNotImplemented)
+	return errors.Err(shared.ErrNotImplemented)
 }
 
 func (c *CloudFrontROStore) PutSD(_ string, _ stream.Blob) error {
-	return errors.Err(ErrNotImplemented)
+	return errors.Err(shared.ErrNotImplemented)
 }
 
 func (c *CloudFrontROStore) Delete(_ string) error {
-	return errors.Err(ErrNotImplemented)
+	return errors.Err(shared.ErrNotImplemented)
 }
 
 // Shutdown shuts down the store gracefully
diff --git a/store/store.go b/store/store.go
index 3a2fc6f..bd9223b 100644
--- a/store/store.go
+++ b/store/store.go
@@ -40,6 +40,3 @@ type lister interface {
 
 //ErrBlobNotFound is a standard error when a blob is not found in the store.
 var ErrBlobNotFound = errors.Base("blob not found")
-
-//ErrNotImplemented is a standard error when a store that implements this interface does not implement a method
-var ErrNotImplemented = errors.Base("this store does not implement this method")

From ebb62d0a245e256fdeb7c9ae4affcf92a4eac5d5 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Mon, 29 Mar 2021 19:44:27 +0200
Subject: [PATCH 24/51] run go mod tidy

---
 go.sum | 26 --------------------------
 1 file changed, 26 deletions(-)

diff --git a/go.sum b/go.sum
index b7dbdb2..2b6ef87 100644
--- a/go.sum
+++ b/go.sum
@@ -37,7 +37,6 @@ github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:l
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.16.11 h1:g/c7gJeVyHoXCxM2fddS85bPGVkBF8s2q8t3fyElegc=
 github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -83,9 +82,7 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -118,7 +115,6 @@ github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200j
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0 h1:Rd1kQnQu0Hq3qvJppYSG0HtP+f5LPPUiDswTLiEegLg=
 github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
@@ -130,7 +126,6 @@ github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
@@ -146,7 +141,6 @@ github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+u
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gops v0.3.7 h1:KtVAagOM0FIq+02DiQrKBTnLhYpWBMowaufcj+W1Exw=
 github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -169,7 +163,6 @@ github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
 github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
 github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -204,7 +197,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce h1:xdsDDbiBDQTKASoGEZ+pEmF1OnWuu8AQ9I8iNbHNeno=
 github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
@@ -215,7 +207,6 @@ github.com/hashicorp/memberlist v0.1.4 h1:gkyML/r71w3FL8gUi74Vk76avkj/9lYAY9lvg0
 github.com/hashicorp/memberlist v0.1.4/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8=
 github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
@@ -262,7 +253,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lbryio/chainquery v1.9.0 h1:NfBZ3eKYwD3PqXU/vt+2tF3ox3WUWoW4J5YdEQ0rxw0=
 github.com/lbryio/chainquery v1.9.0/go.mod h1:7G8l7jNtANS1I7fQOvtzbiHsv6qKVmN4codXHc3C4kk=
 github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpUqE8hRA3OrYYosw9+Sl681BF9cwcjzE+OCNK8=
-github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f h1:ovd2wPXzkT80vdP/FX5xcQeXu0i9RAo80SQ6qIsrAjM=
 github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
 github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
 github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
@@ -285,7 +275,6 @@ github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKR
 github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5 h1:mG83tLXWSRdcXMWfkoumVwhcCbf3jHF9QKv/m37BkM0=
 github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5/go.mod h1:H0aPCWffGOaDcjkw1iB7W9DVLp6GXmfcJY/7YZCWPA4=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -353,23 +342,19 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
 github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -422,7 +407,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
 github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
-github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -430,7 +414,6 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.1 h1:Lt3ihYMlE+lreX1GS4Qw4ZsNpYQLxIXKBTEOXm3nt6I=
 github.com/spf13/afero v1.1.1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ=
@@ -440,14 +423,12 @@ github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec h1:2ZXvIUGghLpdTVHR1UfvfrzoVlZaE/yOWC5LueIHZig=
 github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.0.2 h1:Ncr3ZIuJn322w2k1qmzXDnkLAdQMlJqBa9kfAH+irso=
 github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
 github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
 github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
@@ -544,7 +525,6 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191009170851-d66e71096ffb h1:TR699M2v0qoKTOHxeLgp6zPqaQNs74f01a/ob9W0qko=
 golang.org/x/net v0.0.0-20191009170851-d66e71096ffb/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
@@ -559,7 +539,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -587,7 +566,6 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
@@ -595,7 +573,6 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -683,12 +660,10 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/gorp.v1 v1.7.1/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.41.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.48.0 h1:URjZc+8ugRY5mL5uUeQH/a63JcHwdX9xZaWvmNWD7z8=
 gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
 gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
@@ -700,7 +675,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

From 3a441aed3abbb13a685e72acb32957353c0a5432 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Mon, 29 Mar 2021 19:56:18 +0200
Subject: [PATCH 25/51] fix issues caused by beamer's renaming

---
 cmd/decode.go    |  4 ++--
 go.mod           |  6 +++---
 go.sum           | 14 ++++++++------
 wallet/client.go |  4 ++--
 4 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/cmd/decode.go b/cmd/decode.go
index 4a54668..7cb677f 100644
--- a/cmd/decode.go
+++ b/cmd/decode.go
@@ -4,7 +4,7 @@ import (
 	"encoding/hex"
 	"fmt"
 
-	"github.com/lbryio/lbry.go/v2/schema/claim"
+	"github.com/lbryio/lbry.go/v2/schema/stake"
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/golang/protobuf/jsonpb"
@@ -23,7 +23,7 @@ func init() {
 }
 
 func decodeCmd(cmd *cobra.Command, args []string) {
-	c, err := claim.DecodeClaimHex(args[0], "")
+	c, err := stake.DecodeClaimHex(args[0], "")
 	if err != nil {
 		log.Fatal(err)
 	}
diff --git a/go.mod b/go.mod
index 4570f1d..ba4f821 100644
--- a/go.mod
+++ b/go.mod
@@ -23,8 +23,8 @@ require (
 	github.com/karrick/godirwalk v1.16.1
 	github.com/lbryio/chainquery v1.9.0
 	github.com/lbryio/lbry.go v1.1.2 // indirect
-	github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128
-	github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec
+	github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011
+	github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
 	github.com/lucas-clemente/quic-go v0.19.3
 	github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
 	github.com/prometheus/client_golang v0.9.3
@@ -33,7 +33,7 @@ require (
 	github.com/spf13/cast v1.3.0
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/viper v1.7.1 // indirect
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.7.0
 	github.com/volatiletech/null v8.0.0+incompatible
 	go.uber.org/atomic v1.5.1
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
diff --git a/go.sum b/go.sum
index 2b6ef87..5895055 100644
--- a/go.sum
+++ b/go.sum
@@ -256,17 +256,16 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
 github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
 github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
 github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
-github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128 h1:VL209c+AGKixMFpxT+TsOAPzBPuoUzyjXf47iNe7OzY=
-github.com/lbryio/lbry.go/v2 v2.6.1-0.20200901175808-73382bb02128/go.mod h1:RqOv4V5eWY/JGmduCPcQVdN19SEYnNY3SuF+arTKIU4=
+github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011 h1:r1NoX3NQu/Me+/qw4OzJGw8bhOUnTZHUneCbIV6SC+Y=
+github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011/go.mod h1:sUhhSKqPNkiwgBqvBzJIqfLLzGH8hkDGrrO/HcaXzFc=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
 github.com/lbryio/lbryschema.go v0.0.0-20190602173230-6d2f69a36f46/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
 github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:fbG/dzobG8r95KzMwckXiLMHfFjZaBRQqC9hPs2XAQ4=
 github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
-github.com/lbryio/types v0.0.0-20191009145016-1bb8107e04f8/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
-github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec h1:2xk/qg4VTOCJ8RzV/ED5AKqDcJ00zVb08ltf9V+sr3c=
-github.com/lbryio/types v0.0.0-20191228214437-05a22073b4ec/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M=
+github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4=
 github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8=
@@ -436,8 +435,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
@@ -678,6 +678,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
diff --git a/wallet/client.go b/wallet/client.go
index a4cb165..3dc0e8c 100644
--- a/wallet/client.go
+++ b/wallet/client.go
@@ -6,7 +6,7 @@ import (
 
 	"github.com/lbryio/chainquery/lbrycrd"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/schema/claim"
+	"github.com/lbryio/lbry.go/v2/schema/stake"
 	types "github.com/lbryio/types/v2/go"
 
 	"github.com/btcsuite/btcutil"
@@ -140,7 +140,7 @@ func (n *Node) GetClaimInTx(txid string, nout int) (*types.Claim, error) {
 		return nil, errors.Err(err)
 	}
 
-	ch, err := claim.DecodeClaimBytes(value, "")
+	ch, err := stake.DecodeClaimBytes(value, "")
 	if err != nil {
 		return nil, errors.Err(err)
 	}

From 90c36fbe243f0f2fc0560206685792682bc68ae9 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Wed, 31 Mar 2021 04:53:27 +0200
Subject: [PATCH 26/51] upgrade quic-go

add cache for blobs not found
---
 go.mod               |  2 +-
 go.sum               | 27 ++++++++++++---------------
 peer/http3/server.go |  4 ++--
 peer/http3/store.go  | 18 +++++++++++++++---
 4 files changed, 30 insertions(+), 21 deletions(-)

diff --git a/go.mod b/go.mod
index ba4f821..ed3ebcb 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@ require (
 	github.com/lbryio/lbry.go v1.1.2 // indirect
 	github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011
 	github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
-	github.com/lucas-clemente/quic-go v0.19.3
+	github.com/lucas-clemente/quic-go v0.20.0
 	github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
 	github.com/prometheus/client_golang v0.9.3
 	github.com/sirupsen/logrus v1.4.2
diff --git a/go.sum b/go.sum
index 5895055..f61f06b 100644
--- a/go.sum
+++ b/go.sum
@@ -109,15 +109,13 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -267,8 +265,8 @@ github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiV
 github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M=
 github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lucas-clemente/quic-go v0.19.3 h1:eCDQqvGBB+kCTkA0XrAFtNe81FMa0/fn4QSoeAbmiF4=
-github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8=
+github.com/lucas-clemente/quic-go v0.20.0 h1:FSU3YN5VnLafHR27Ejs1r1CYMS7XMyIVDzRewkDLNBw=
+github.com/lucas-clemente/quic-go v0.20.0/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
 github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
@@ -280,10 +278,10 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/qpack v0.2.1 h1:jvTsT/HpCn2UZJdP+UUB53FfUUgeOyG5K1ns0OJOGVs=
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
-github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl55b2pc=
-github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs=
-github.com/marten-seemann/qtls-go1-15 v0.1.1 h1:LIH6K34bPVttyXnUWixk0bzH6/N07VxbSabxn5A5gZQ=
-github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
+github.com/marten-seemann/qtls-go1-15 v0.1.4 h1:RehYMOyRW8hPVEja1KBVsFVNSm35Jj9Mvs5yNoZZ28A=
+github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
+github.com/marten-seemann/qtls-go1-16 v0.1.3 h1:XEZ1xGorVy9u+lJq+WXNE+hiqRYLNvJGYmwfwKQN2gU=
+github.com/marten-seemann/qtls-go1-16 v0.1.3/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
@@ -460,7 +458,6 @@ go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
@@ -506,6 +503,7 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -568,9 +566,9 @@ golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/sys v0.0.0-20201231184435-2d18734c6014 h1:joucsQqXmyBVxViHCPFjG3hx8JzIFSaym3l3MM/Jsdg=
+golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -602,6 +600,7 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 h1:Toz2IK7k8rbltAXwNAxKcn9OzqyNfMUhUNjz3sL0NMk=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -691,7 +690,5 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
diff --git a/peer/http3/server.go b/peer/http3/server.go
index c33a676..16fa156 100644
--- a/peer/http3/server.go
+++ b/peer/http3/server.go
@@ -65,8 +65,8 @@ type availabilityResponse struct {
 func (s *Server) Start(address string) error {
 	log.Println("HTTP3 peer listening on " + address)
 	quicConf := &quic.Config{
-		HandshakeTimeout: 4 * time.Second,
-		MaxIdleTimeout:   20 * time.Second,
+		HandshakeIdleTimeout: 4 * time.Second,
+		MaxIdleTimeout:       20 * time.Second,
 	}
 	r := mux.NewRouter()
 	r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
diff --git a/peer/http3/store.go b/peer/http3/store.go
index 8a5c812..bcc7cae 100644
--- a/peer/http3/store.go
+++ b/peer/http3/store.go
@@ -4,11 +4,14 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"net/http"
+	"strings"
+	"sync"
 	"time"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lbryio/reflector.go/shared"
+	"github.com/lbryio/reflector.go/store"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/lucas-clemente/quic-go/http3"
 )
@@ -16,7 +19,8 @@ import (
 // Store is a blob store that gets blobs from a peer.
 // It satisfies the store.BlobStore interface but cannot put or delete blobs.
 type Store struct {
-	opts StoreOpts
+	opts          StoreOpts
+	NotFoundCache *sync.Map
 }
 
 // StoreOpts allows to set options for a new Store.
@@ -27,12 +31,12 @@ type StoreOpts struct {
 
 // NewStore makes a new peer store.
 func NewStore(opts StoreOpts) *Store {
-	return &Store{opts: opts}
+	return &Store{opts: opts, NotFoundCache: &sync.Map{}}
 }
 
 func (p *Store) getClient() (*Client, error) {
 	var qconf quic.Config
-	qconf.HandshakeTimeout = 4 * time.Second
+	qconf.HandshakeIdleTimeout = 4 * time.Second
 	qconf.MaxIdleTimeout = 20 * time.Second
 	pool, err := x509.SystemCertPool()
 	if err != nil {
@@ -71,7 +75,15 @@ func (p *Store) Has(hash string) (bool, error) {
 // Get downloads the blob from the peer
 func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 	start := time.Now()
+	if lastChecked, ok := p.NotFoundCache.Load(hash); ok {
+		if lastChecked.(time.Time).After(time.Now().Add(-5 * time.Minute)) {
+			return nil, shared.NewBlobTrace(time.Since(start), p.Name()+"-notfoundcache"), store.ErrBlobNotFound
+		}
+	}
 	c, err := p.getClient()
+	if err != nil && strings.Contains(err.Error(), "blob not found") {
+		p.NotFoundCache.Store(hash, time.Now())
+	}
 	if err != nil {
 		return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
 	}

From 38b44218f2a6fd843faa643165f9df431fd78672 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Mon, 5 Apr 2021 23:34:45 +0200
Subject: [PATCH 27/51] check blobs when reading them

---
 store/disk.go | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/store/disk.go b/store/disk.go
index 412d326..08cdbd1 100644
--- a/store/disk.go
+++ b/store/disk.go
@@ -1,6 +1,9 @@
 package store
 
 import (
+	"crypto/sha512"
+	"encoding/hex"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
@@ -10,6 +13,7 @@ import (
 	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lbryio/reflector.go/shared"
 	"github.com/lbryio/reflector.go/store/speedwalk"
+	log "github.com/sirupsen/logrus"
 )
 
 // DiskStore stores blobs on a local disk
@@ -68,6 +72,17 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 		}
 		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
 	}
+	hashBytes := sha512.Sum384(blob)
+	readHash := hex.EncodeToString(hashBytes[:])
+	if hash != readHash {
+		message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash)
+		log.Errorf("%s", message)
+		err := d.Delete(hash)
+		if err != nil {
+			return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
+		}
+		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message)
+	}
 
 	return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil
 }

From bd13836897b0b69e2216517415e665b6f15687e5 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Tue, 6 Apr 2021 14:00:36 -0400
Subject: [PATCH 28/51] Add request queue for blob cache

---
 internal/metrics/metrics.go |  5 +++
 peer/http3/server.go        | 87 ++++++++++++++++++++-----------------
 peer/http3/worker.go        | 40 +++++++++++++++++
 3 files changed, 91 insertions(+), 41 deletions(-)
 create mode 100644 peer/http3/worker.go

diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
index 8d46266..2793fc0 100644
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@@ -201,6 +201,11 @@ var (
 		Name:      "s3_in_bytes",
 		Help:      "Total number of incoming bytes (from S3-CF)",
 	})
+	Http3BlobReqQueue = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: ns,
+		Name:      "http3_blob_request_queue_size",
+		Help:      "Blob requests of https queue size",
+	})
 )
 
 func CacheLabels(name, component string) prometheus.Labels {
diff --git a/peer/http3/server.go b/peer/http3/server.go
index 16fa156..da8f485 100644
--- a/peer/http3/server.go
+++ b/peer/http3/server.go
@@ -70,46 +70,7 @@ func (s *Server) Start(address string) error {
 	}
 	r := mux.NewRouter()
 	r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
-		vars := mux.Vars(r)
-		requestedBlob := vars["hash"]
-		traceParam := r.URL.Query().Get("trace")
-		var err error
-		wantsTrace := false
-		if traceParam != "" {
-			wantsTrace, err = strconv.ParseBool(traceParam)
-			if err != nil {
-				wantsTrace = false
-			}
-		}
-		blob, trace, err := s.store.Get(requestedBlob)
-
-		if wantsTrace {
-			serialized, err := trace.Serialize()
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusNotFound)
-				return
-			}
-			w.Header().Add("Via", serialized)
-			log.Debug(trace.String())
-		}
-		if err != nil {
-			if errors.Is(err, store.ErrBlobNotFound) {
-				http.Error(w, err.Error(), http.StatusNotFound)
-				return
-			}
-			fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err))
-			s.logError(err)
-			http.Error(w, err.Error(), http.StatusBadRequest)
-			return
-		}
-
-		_, err = w.Write(blob)
-		if err != nil {
-			s.logError(err)
-		}
-		metrics.MtrOutBytesUdp.Add(float64(len(blob)))
-		metrics.BlobDownloadCount.Inc()
-		metrics.Http3DownloadCount.Inc()
+		enqueue(&blobRequest{request: r, reply: w})
 	})
 	r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) {
 		vars := mux.Vars(r)
@@ -147,7 +108,7 @@ func (s *Server) Start(address string) error {
 		},
 		QuicConfig: quicConf,
 	}
-
+	go InitWorkers(s, 100)
 	go s.listenForShutdown(&server)
 	s.grp.Add(1)
 	go func() {
@@ -196,3 +157,47 @@ func (s *Server) listenForShutdown(listener *http3.Server) {
 		log.Error("error closing listener for peer server - ", err)
 	}
 }
+
+func (s *Server) HandleGetBlob(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	requestedBlob := vars["hash"]
+	traceParam := r.URL.Query().Get("trace")
+	var err error
+	wantsTrace := false
+	if traceParam != "" {
+		wantsTrace, err = strconv.ParseBool(traceParam)
+		if err != nil {
+			wantsTrace = false
+		}
+	}
+
+	blob, trace, err := s.store.Get(requestedBlob)
+
+	if wantsTrace {
+		serialized, err := trace.Serialize()
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusNotFound)
+			return
+		}
+		w.Header().Add("Via", serialized)
+		log.Debug(trace.String())
+	}
+	if err != nil {
+		if errors.Is(err, store.ErrBlobNotFound) {
+			http.Error(w, err.Error(), http.StatusNotFound)
+			return
+		}
+		fmt.Printf("%s: %s", requestedBlob, errors.FullTrace(err))
+		s.logError(err)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	_, err = w.Write(blob)
+	if err != nil {
+		s.logError(err)
+	}
+	metrics.MtrOutBytesUdp.Add(float64(len(blob)))
+	metrics.BlobDownloadCount.Inc()
+	metrics.Http3DownloadCount.Inc()
+}
diff --git a/peer/http3/worker.go b/peer/http3/worker.go
new file mode 100644
index 0000000..182ca08
--- /dev/null
+++ b/peer/http3/worker.go
@@ -0,0 +1,40 @@
+package http3
+
+import (
+	"net/http"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+)
+
+type blobRequest struct {
+	request *http.Request
+	reply   http.ResponseWriter
+}
+
+var getReqCh = make(chan *blobRequest)
+
+func InitWorkers(server *Server, workers int) error {
+	stopper := stop.New(server.grp)
+	for i := 0; i < workers; i++ {
+		go func(worker int) {
+			select {
+			case <-stopper.Ch():
+			case r := <-getReqCh:
+				metrics.Http3BlobReqQueue.Dec()
+				process(server, r)
+			}
+		}(i)
+	}
+	return nil
+}
+
+func enqueue(b *blobRequest) {
+	metrics.Http3BlobReqQueue.Inc()
+	getReqCh <- b
+}
+
+func process(server *Server, r *blobRequest) {
+	server.HandleGetBlob(r.reply, r.request)
+}

From b97595311f077b6a40b11fdb182d92dcd07a4efa Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Tue, 6 Apr 2021 14:21:05 -0400
Subject: [PATCH 29/51] Wait for request to be handled before returning

---
 peer/http3/server.go | 6 +++++-
 peer/http3/worker.go | 6 ++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/peer/http3/server.go b/peer/http3/server.go
index da8f485..2147421 100644
--- a/peer/http3/server.go
+++ b/peer/http3/server.go
@@ -70,7 +70,11 @@ func (s *Server) Start(address string) error {
 	}
 	r := mux.NewRouter()
 	r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
-		enqueue(&blobRequest{request: r, reply: w})
+		waiter := stop.New()
+		enqueue(&blobRequest{request: r, reply: w, finished: waiter})
+		select {
+		case <-waiter.Ch():
+		}
 	})
 	r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) {
 		vars := mux.Vars(r)
diff --git a/peer/http3/worker.go b/peer/http3/worker.go
index 182ca08..62c4c4d 100644
--- a/peer/http3/worker.go
+++ b/peer/http3/worker.go
@@ -9,8 +9,9 @@ import (
 )
 
 type blobRequest struct {
-	request *http.Request
-	reply   http.ResponseWriter
+	request  *http.Request
+	reply    http.ResponseWriter
+	finished *stop.Group
 }
 
 var getReqCh = make(chan *blobRequest)
@@ -37,4 +38,5 @@ func enqueue(b *blobRequest) {
 
 func process(server *Server, r *blobRequest) {
 	server.HandleGetBlob(r.reply, r.request)
+	r.finished.Done()
 }

From 25a7fac4f0a2cd6287bab7a6da7be4250f81d956 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Tue, 6 Apr 2021 14:28:29 -0400
Subject: [PATCH 30/51] use wait group not stopper

---
 peer/http3/server.go | 8 ++++----
 peer/http3/worker.go | 3 ++-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/peer/http3/server.go b/peer/http3/server.go
index 2147421..89a64db 100644
--- a/peer/http3/server.go
+++ b/peer/http3/server.go
@@ -11,6 +11,7 @@ import (
 	"math/big"
 	"net/http"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/lbryio/reflector.go/internal/metrics"
@@ -70,11 +71,10 @@ func (s *Server) Start(address string) error {
 	}
 	r := mux.NewRouter()
 	r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
-		waiter := stop.New()
+		waiter := &sync.WaitGroup{}
+		waiter.Add(1)
 		enqueue(&blobRequest{request: r, reply: w, finished: waiter})
-		select {
-		case <-waiter.Ch():
-		}
+		waiter.Wait()
 	})
 	r.HandleFunc("/has/{hash}", func(w http.ResponseWriter, r *http.Request) {
 		vars := mux.Vars(r)
diff --git a/peer/http3/worker.go b/peer/http3/worker.go
index 62c4c4d..dac2893 100644
--- a/peer/http3/worker.go
+++ b/peer/http3/worker.go
@@ -2,6 +2,7 @@ package http3
 
 import (
 	"net/http"
+	"sync"
 
 	"github.com/lbryio/reflector.go/internal/metrics"
 
@@ -11,7 +12,7 @@ import (
 type blobRequest struct {
 	request  *http.Request
 	reply    http.ResponseWriter
-	finished *stop.Group
+	finished *sync.WaitGroup
 }
 
 var getReqCh = make(chan *blobRequest)

From dc95351cf30c3332185c929b18fc0eba29f54afa Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Wed, 7 Apr 2021 04:46:18 +0200
Subject: [PATCH 31/51] add integrity check cmd

throttle live integrity checks
bug fixes
---
 cmd/integrity.go     | 93 ++++++++++++++++++++++++++++++++++++++++++++
 peer/http3/worker.go | 18 +++++----
 store/disk.go        | 30 +++++++++-----
 3 files changed, 124 insertions(+), 17 deletions(-)
 create mode 100644 cmd/integrity.go

diff --git a/cmd/integrity.go b/cmd/integrity.go
new file mode 100644
index 0000000..8be9e2b
--- /dev/null
+++ b/cmd/integrity.go
@@ -0,0 +1,93 @@
+package cmd
+
+import (
+	"crypto/sha512"
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"path"
+	"runtime"
+	"sync/atomic"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/reflector.go/meta"
+	"github.com/lbryio/reflector.go/store/speedwalk"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var threads int
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:   "check-integrity",
+		Short: "check blobs integrity for a given path",
+		Run:   integrityCheckCmd,
+	}
+	cmd.Flags().StringVar(&diskStorePath, "store-path", "", "path of the store where all blobs are cached")
+	cmd.Flags().IntVar(&threads, "threads", runtime.NumCPU()-1, "number of concurrent threads to process blobs")
+	rootCmd.AddCommand(cmd)
+}
+
+func integrityCheckCmd(cmd *cobra.Command, args []string) {
+	log.Printf("reflector %s", meta.VersionString())
+	if diskStorePath == "" {
+		log.Fatal("store-path must be defined")
+	}
+
+	blobs, err := speedwalk.AllFiles(diskStorePath, true)
+	if err != nil {
+		log.Errorf("error while reading blobs from disk %s", errors.FullTrace(err))
+	}
+	tasks := make(chan string, len(blobs))
+	done := make(chan bool)
+	processed := new(int32)
+	go produce(tasks, blobs)
+	cpus := runtime.NumCPU()
+	for i := 0; i < cpus-1; i++ {
+		go consume(i, tasks, done, len(blobs), processed)
+	}
+	<-done
+}
+
+func produce(tasks chan<- string, blobs []string) {
+	for _, b := range blobs {
+		tasks <- b
+	}
+	close(tasks)
+}
+
+func consume(worker int, tasks <-chan string, done chan<- bool, totalTasks int, processed *int32) {
+	start := time.Now()
+
+	for b := range tasks {
+		checked := atomic.AddInt32(processed, 1)
+		if worker == 0 {
+			remaining := int32(totalTasks) - checked
+			timePerBlob := time.Since(start).Microseconds() / int64(checked)
+			remainingTime := time.Duration(int64(remaining)*timePerBlob) * time.Microsecond
+			log.Infof("[T%d] %d/%d blobs checked. ETA: %s", worker, checked, totalTasks, remainingTime.String())
+		}
+		blobPath := path.Join(diskStorePath, b[:2], b)
+		blob, err := ioutil.ReadFile(blobPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			log.Errorf("[Worker %d] Error looking up blob %s: %s", worker, b, err.Error())
+			continue
+		}
+		hashBytes := sha512.Sum384(blob)
+		readHash := hex.EncodeToString(hashBytes[:])
+		if readHash != b {
+			log.Infof("[%s] found a broken blob while reading from disk. Actual hash: %s", b, readHash)
+			err := os.Remove(blobPath)
+			if err != nil {
+				log.Errorf("Error while deleting broken blob %s: %s", b, err.Error())
+			}
+		}
+	}
+	done <- true
+}
diff --git a/peer/http3/worker.go b/peer/http3/worker.go
index dac2893..d2b625b 100644
--- a/peer/http3/worker.go
+++ b/peer/http3/worker.go
@@ -15,21 +15,23 @@ type blobRequest struct {
 	finished *sync.WaitGroup
 }
 
-var getReqCh = make(chan *blobRequest)
+var getReqCh = make(chan *blobRequest, 20000)
 
-func InitWorkers(server *Server, workers int) error {
+func InitWorkers(server *Server, workers int) {
 	stopper := stop.New(server.grp)
 	for i := 0; i < workers; i++ {
 		go func(worker int) {
-			select {
-			case <-stopper.Ch():
-			case r := <-getReqCh:
-				metrics.Http3BlobReqQueue.Dec()
-				process(server, r)
+			for {
+				select {
+				case <-stopper.Ch():
+				case r := <-getReqCh:
+					metrics.Http3BlobReqQueue.Dec()
+					process(server, r)
+				}
 			}
 		}(i)
 	}
-	return nil
+	return
 }
 
 func enqueue(b *blobRequest) {
diff --git a/store/disk.go b/store/disk.go
index 08cdbd1..eaecfbb 100644
--- a/store/disk.go
+++ b/store/disk.go
@@ -14,6 +14,7 @@ import (
 	"github.com/lbryio/reflector.go/shared"
 	"github.com/lbryio/reflector.go/store/speedwalk"
 	log "github.com/sirupsen/logrus"
+	"go.uber.org/atomic"
 )
 
 // DiskStore stores blobs on a local disk
@@ -25,8 +26,12 @@ type DiskStore struct {
 
 	// true if initOnce ran, false otherwise
 	initialized bool
+
+	concurrentChecks atomic.Int32
 }
 
+const maxConcurrentChecks = 3
+
 // NewDiskStore returns an initialized file disk store pointer.
 func NewDiskStore(dir string, prefixLength int) *DiskStore {
 	return &DiskStore{
@@ -72,16 +77,23 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 		}
 		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(err)
 	}
-	hashBytes := sha512.Sum384(blob)
-	readHash := hex.EncodeToString(hashBytes[:])
-	if hash != readHash {
-		message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash)
-		log.Errorf("%s", message)
-		err := d.Delete(hash)
-		if err != nil {
-			return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
+
+	// this is a rather poor yet effective way of throttling how many blobs can be checked concurrently
+	// poor because there is a possible race condition between the check and the actual +1
+	if d.concurrentChecks.Load() < maxConcurrentChecks {
+		d.concurrentChecks.Add(1)
+		defer d.concurrentChecks.Sub(1)
+		hashBytes := sha512.Sum384(blob)
+		readHash := hex.EncodeToString(hashBytes[:])
+		if hash != readHash {
+			message := fmt.Sprintf("[%s] found a broken blob while reading from disk. Actual hash: %s", hash, readHash)
+			log.Errorf("%s", message)
+			err := d.Delete(hash)
+			if err != nil {
+				return nil, shared.NewBlobTrace(time.Since(start), d.Name()), err
+			}
+			return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message)
 		}
-		return nil, shared.NewBlobTrace(time.Since(start), d.Name()), errors.Err(message)
 	}
 
 	return blob, shared.NewBlobTrace(time.Since(start), d.Name()), nil

From ec3aae33ba8d091c85c73fb384f9fb43c3686496 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Mon, 12 Apr 2021 23:05:50 +0200
Subject: [PATCH 32/51] add if this than that store

switch to wasabi for uploads
---
 cmd/reflector.go            | 10 +++--
 internal/metrics/metrics.go | 13 +++++++
 peer/http3/server.go        |  2 +-
 store/cloudfront_rw.go      |  6 +--
 store/ittt.go               | 75 +++++++++++++++++++++++++++++++++++++
 store/s3.go                 | 10 +++--
 6 files changed, 104 insertions(+), 12 deletions(-)
 create mode 100644 store/ittt.go

diff --git a/cmd/reflector.go b/cmd/reflector.go
index 738a066..6fe44ad 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -37,6 +37,7 @@ var (
 	proxyProtocol               string
 	useDB                       bool
 	cloudFrontEndpoint          string
+	WasabiEndpoint              string
 	reflectorCmdDiskCache       string
 	bufferReflectorCmdDiskCache string
 	reflectorCmdMemCache        int
@@ -52,6 +53,7 @@ func init() {
 	cmd.Flags().StringVar(&proxyPort, "proxy-port", "5567", "port of another reflector server where blobs are fetched from")
 	cmd.Flags().StringVar(&proxyProtocol, "proxy-protocol", "http3", "protocol used to fetch blobs from another reflector server (tcp/http3)")
 	cmd.Flags().StringVar(&cloudFrontEndpoint, "cloudfront-endpoint", "", "CloudFront edge endpoint for standard HTTP retrieval")
+	cmd.Flags().StringVar(&WasabiEndpoint, "wasabi-endpoint", "", "Wasabi edge endpoint for standard HTTP retrieval")
 	cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from")
 	cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
 	cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
@@ -137,12 +139,12 @@ func setupStore() store.BlobStore {
 		if conf != "none" {
 			s3Store = store.NewS3Store(globalConfig.AwsID, globalConfig.AwsSecret, globalConfig.BucketRegion, globalConfig.BucketName)
 		}
-		if cloudFrontEndpoint != "" {
-			cfs := store.NewCloudFrontROStore(cloudFrontEndpoint)
+		if cloudFrontEndpoint != "" && WasabiEndpoint != "" {
+			ittt := store.NewITTTStore(store.NewCloudFrontROStore(WasabiEndpoint), store.NewCloudFrontROStore(cloudFrontEndpoint))
 			if s3Store != nil {
-				s = store.NewCloudFrontRWStore(cfs, s3Store)
+				s = store.NewCloudFrontRWStore(ittt, s3Store)
 			} else {
-				s = cfs
+				s = ittt
 			}
 		} else if s3Store != nil {
 			s = s3Store
diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
index 2793fc0..2570677 100644
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@@ -60,6 +60,7 @@ func (s *Server) Shutdown() {
 const (
 	ns             = "reflector"
 	subsystemCache = "cache"
+	subsystemITTT  = "ittt"
 
 	labelDirection = "direction"
 	labelErrorType = "error_type"
@@ -124,6 +125,18 @@ var (
 		Name:      "hit_total",
 		Help:      "Total number of blobs retrieved from the cache storage",
 	}, []string{LabelCacheType, LabelComponent})
+	ThisHitCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Subsystem: subsystemITTT,
+		Name:      "this_hit_total",
+		Help:      "Total number of blobs retrieved from the this storage",
+	})
+	ThatHitCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Subsystem: subsystemITTT,
+		Name:      "that_hit_total",
+		Help:      "Total number of blobs retrieved from the that storage",
+	})
 	CacheMissCount = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: ns,
 		Subsystem: subsystemCache,
diff --git a/peer/http3/server.go b/peer/http3/server.go
index 89a64db..30ae3fd 100644
--- a/peer/http3/server.go
+++ b/peer/http3/server.go
@@ -112,7 +112,7 @@ func (s *Server) Start(address string) error {
 		},
 		QuicConfig: quicConf,
 	}
-	go InitWorkers(s, 100)
+	go InitWorkers(s, 200)
 	go s.listenForShutdown(&server)
 	s.grp.Add(1)
 	go func() {
diff --git a/store/cloudfront_rw.go b/store/cloudfront_rw.go
index 32ee418..6b293a8 100644
--- a/store/cloudfront_rw.go
+++ b/store/cloudfront_rw.go
@@ -7,15 +7,15 @@ import (
 	"github.com/lbryio/reflector.go/shared"
 )
 
-// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront, writes go to S3.
+// CloudFrontRWStore combines a Cloudfront and an S3 store. Reads go to Cloudfront/Wasabi, writes go to S3.
 type CloudFrontRWStore struct {
-	cf *CloudFrontROStore
+	cf *ITTTStore
 	s3 *S3Store
 }
 
 // NewCloudFrontRWStore returns an initialized CloudFrontRWStore store pointer.
 // NOTE: It panics if either argument is nil.
-func NewCloudFrontRWStore(cf *CloudFrontROStore, s3 *S3Store) *CloudFrontRWStore {
+func NewCloudFrontRWStore(cf *ITTTStore, s3 *S3Store) *CloudFrontRWStore {
 	if cf == nil || s3 == nil {
 		panic("both stores must be set")
 	}
diff --git a/store/ittt.go b/store/ittt.go
new file mode 100644
index 0000000..5edc67d
--- /dev/null
+++ b/store/ittt.go
@@ -0,0 +1,75 @@
+package store
+
+import (
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/shared"
+)
+
+// ITTT store performs an operation on this storage, if this fails, it attempts to run it on that
+type ITTTStore struct {
+	this, that BlobStore
+}
+
+// NewCachingStore makes a new caching disk store and returns a pointer to it.
+func NewITTTStore(this, that BlobStore) *ITTTStore {
+	return &ITTTStore{
+		this: this,
+		that: that,
+	}
+}
+
+const nameIttt = "ittt"
+
+// Name is the cache type name
+func (c *ITTTStore) Name() string { return nameIttt }
+
+// Has checks the cache and then the origin for a hash. It returns true if either store has it.
+func (c *ITTTStore) Has(hash string) (bool, error) {
+	has, err := c.this.Has(hash)
+	if err != nil || !has {
+		has, err = c.that.Has(hash)
+	}
+	return has, err
+}
+
+// Get tries to get the blob from this first, falling back to that.
+func (c *ITTTStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	blob, trace, err := c.this.Get(hash)
+	if err == nil {
+		metrics.ThisHitCount.Inc()
+		return blob, trace.Stack(time.Since(start), c.Name()), err
+	}
+
+	blob, trace, err = c.that.Get(hash)
+	if err != nil {
+		return nil, trace.Stack(time.Since(start), c.Name()), err
+	}
+	metrics.ThatHitCount.Inc()
+	return blob, trace.Stack(time.Since(start), c.Name()), nil
+}
+
+// Put not implemented
+func (c *ITTTStore) Put(hash string, blob stream.Blob) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+// PutSD not implemented
+func (c *ITTTStore) PutSD(hash string, blob stream.Blob) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+// Delete not implemented
+func (c *ITTTStore) Delete(hash string) error {
+	return errors.Err(shared.ErrNotImplemented)
+}
+
+// Shutdown shuts down the store gracefully
+func (c *ITTTStore) Shutdown() {
+	return
+}
diff --git a/store/s3.go b/store/s3.go
index 75d66cc..75ba459 100644
--- a/store/s3.go
+++ b/store/s3.go
@@ -112,10 +112,11 @@ func (s *S3Store) Put(hash string, blob stream.Blob) error {
 	}(time.Now())
 
 	_, err = s3manager.NewUploader(s.session).Upload(&s3manager.UploadInput{
-		Bucket:       aws.String(s.bucket),
-		Key:          aws.String(hash),
-		Body:         bytes.NewBuffer(blob),
-		StorageClass: aws.String(s3.StorageClassIntelligentTiering),
+		Bucket: aws.String(s.bucket),
+		Key:    aws.String(hash),
+		Body:   bytes.NewBuffer(blob),
+		ACL:    aws.String("public-read"),
+		//StorageClass: aws.String(s3.StorageClassIntelligentTiering),
 	})
 	metrics.MtrOutBytesReflector.Add(float64(blob.Size()))
 
@@ -152,6 +153,7 @@ func (s *S3Store) initOnce() error {
 	sess, err := session.NewSession(&aws.Config{
 		Credentials: credentials.NewStaticCredentials(s.awsID, s.awsSecret, ""),
 		Region:      aws.String(s.region),
+		Endpoint:    aws.String("https://s3.wasabisys.com"),
 	})
 	if err != nil {
 		return err

From 4392c9724262b60ed378cd727c2a2d9b35e4c638 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Tue, 13 Apr 2021 00:52:56 +0200
Subject: [PATCH 33/51] fix mess with lbry.go

---
 cmd/root.go        |   6 ++
 cmd/send.go        | 160 +++++++++++++++++++++++++++++++++++++++++++++
 cmd/sendblob.go    |   6 +-
 go.mod             |   7 +-
 go.sum             |  11 ++--
 publish/publish.go |  74 +++++++++------------
 6 files changed, 210 insertions(+), 54 deletions(-)
 create mode 100644 cmd/send.go

diff --git a/cmd/root.go b/cmd/root.go
index ce28f8f..1e422a7 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -163,3 +163,9 @@ func mustGetFlagInt64(cmd *cobra.Command, name string) int64 {
 	checkErr(err)
 	return v
 }
+
+func mustGetFlagBool(cmd *cobra.Command, name string) bool {
+	v, err := cmd.Flags().GetBool(name)
+	checkErr(err)
+	return v
+}
diff --git a/cmd/send.go b/cmd/send.go
new file mode 100644
index 0000000..7021866
--- /dev/null
+++ b/cmd/send.go
@@ -0,0 +1,160 @@
+package cmd
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/signal"
+	"path"
+	"syscall"
+
+	"github.com/lbryio/reflector.go/reflector"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	var cmd = &cobra.Command{
+		Use:   "send ADDRESS:PORT PATH",
+		Short: "Send a file to a reflector",
+		Args:  cobra.ExactArgs(2),
+		Run:   sendCmd,
+	}
+	cmd.PersistentFlags().String("sd-cache", "", "path to dir where sd blobs will be cached")
+	rootCmd.AddCommand(cmd)
+}
+
+// todo: if retrying a large file is slow, we can add the ability to seek ahead in the file so we're not
+// re-uploading blobs that already exist
+
+var hackyReflector reflector.Client
+
+func sendCmd(cmd *cobra.Command, args []string) {
+	reflectorAddress := args[0]
+	err := hackyReflector.Connect(reflectorAddress)
+	checkErr(err)
+	defer hackyReflector.Close()
+
+	filePath := args[1]
+	file, err := os.Open(filePath)
+	checkErr(err)
+	defer file.Close()
+
+	sdCachePath := ""
+	sdCacheDir := mustGetFlagString(cmd, "sd-cache")
+	if sdCacheDir != "" {
+		if _, err := os.Stat(sdCacheDir); os.IsNotExist(err) {
+			err = os.MkdirAll(sdCacheDir, 0777)
+			checkErr(err)
+		}
+		sdCachePath = path.Join(sdCacheDir, filePath+".sdblob")
+	}
+
+	var enc *stream.Encoder
+
+	if sdCachePath != "" {
+		if _, err := os.Stat(sdCachePath); !os.IsNotExist(err) {
+			sdBlob, err := ioutil.ReadFile(sdCachePath)
+			checkErr(err)
+			cachedSDBlob := &stream.SDBlob{}
+			err = cachedSDBlob.FromBlob(sdBlob)
+			checkErr(err)
+			enc = stream.NewEncoderFromSD(file, cachedSDBlob)
+		}
+	}
+	if enc == nil {
+		enc = stream.NewEncoder(file)
+	}
+
+	exitCode := 0
+
+	var killed bool
+	interruptChan := make(chan os.Signal, 1)
+	signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		sig := <-interruptChan
+		fmt.Printf("caught %s, exiting...\n", sig.String())
+		killed = true
+		exitCode = 1
+	}()
+
+	for {
+		if killed {
+			break
+		}
+
+		b, err := enc.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			fmt.Printf("error reading next blob: %v\n", err)
+			exitCode = 1
+			break
+		}
+
+		err = hackyReflect(b, false)
+		if err != nil {
+			fmt.Printf("error reflecting blob %s: %v\n", b.HashHex()[:8], err)
+			exitCode = 1
+			break
+		}
+	}
+
+	sd := enc.SDBlob()
+	//sd.StreamName = filepath.Base(filePath)
+	//sd.SuggestedFileName = filepath.Base(filePath)
+	err = ioutil.WriteFile(sdCachePath, sd.ToBlob(), 0666)
+	if err != nil {
+		fmt.Printf("error saving sd blob: %v\n", err)
+		fmt.Println(sd.ToJson())
+		exitCode = 1
+	}
+
+	if killed {
+		os.Exit(exitCode)
+	}
+
+	if reflectorAddress != "" {
+		err = hackyReflect(sd.ToBlob(), true)
+		if err != nil {
+			fmt.Printf("error reflecting sd blob %s: %v\n", sd.HashHex()[:8], err)
+			exitCode = 1
+		}
+	}
+
+	ret := struct {
+		SDHash     string `json:"sd_hash"`
+		SourceHash string `json:"source_hash"`
+	}{
+		SDHash:     sd.HashHex(),
+		SourceHash: hex.EncodeToString(enc.SourceHash()),
+	}
+
+	j, err := json.MarshalIndent(ret, "", "  ")
+	checkErr(err)
+	fmt.Println(string(j))
+	os.Exit(exitCode)
+}
+
+func hackyReflect(b stream.Blob, sd bool) error {
+	var err error
+	if sd {
+		err = hackyReflector.SendSDBlob(b)
+	} else {
+		err = hackyReflector.SendBlob(b)
+	}
+
+	if errors.Is(err, reflector.ErrBlobExists) {
+		//fmt.Printf("%s already reflected\n", b.HashHex()[:8])
+		return nil
+	}
+
+	return err
+}
diff --git a/cmd/sendblob.go b/cmd/sendblob.go
index 3cd9f05..f37f390 100644
--- a/cmd/sendblob.go
+++ b/cmd/sendblob.go
@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"crypto/rand"
-	"io/ioutil"
 	"os"
 
 	"github.com/lbryio/reflector.go/reflector"
@@ -52,9 +51,8 @@ func sendBlobCmd(cmd *cobra.Command, args []string) {
 
 	file, err := os.Open(path)
 	checkErr(err)
-	data, err := ioutil.ReadAll(file)
-	checkErr(err)
-	s, err := stream.New(data)
+	defer file.Close()
+	s, err := stream.New(file)
 	checkErr(err)
 
 	sdBlob := &stream.SDBlob{}
diff --git a/go.mod b/go.mod
index ed3ebcb..4ed4f23 100644
--- a/go.mod
+++ b/go.mod
@@ -2,6 +2,8 @@ module github.com/lbryio/reflector.go
 
 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
 
+//replace github.com/lbryio/lbry.go/v2 => ../lbry.go
+
 require (
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
 	github.com/aws/aws-sdk-go v1.16.11
@@ -23,9 +25,9 @@ require (
 	github.com/karrick/godirwalk v1.16.1
 	github.com/lbryio/chainquery v1.9.0
 	github.com/lbryio/lbry.go v1.1.2 // indirect
-	github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011
+	github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d
 	github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
-	github.com/lucas-clemente/quic-go v0.20.0
+	github.com/lucas-clemente/quic-go v0.20.1
 	github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
 	github.com/prometheus/client_golang v0.9.3
 	github.com/sirupsen/logrus v1.4.2
@@ -36,7 +38,6 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/volatiletech/null v8.0.0+incompatible
 	go.uber.org/atomic v1.5.1
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect
diff --git a/go.sum b/go.sum
index f61f06b..c51b403 100644
--- a/go.sum
+++ b/go.sum
@@ -88,8 +88,9 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
-github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg=
+github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-ini/ini v1.48.0 h1:TvO60hO/2xgaaTWp2P0wUe4CFxwdMzfbkv3+343Xzqw=
@@ -254,8 +255,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
 github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
 github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
 github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
-github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011 h1:r1NoX3NQu/Me+/qw4OzJGw8bhOUnTZHUneCbIV6SC+Y=
-github.com/lbryio/lbry.go/v2 v2.7.2-0.20210316000044-988178df5011/go.mod h1:sUhhSKqPNkiwgBqvBzJIqfLLzGH8hkDGrrO/HcaXzFc=
+github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d h1:VUaOZ3cbCe7gfpycN/srCOk6U2bBS9NZHEz9RiRxd4E=
+github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=
@@ -265,8 +266,8 @@ github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiV
 github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M=
 github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lucas-clemente/quic-go v0.20.0 h1:FSU3YN5VnLafHR27Ejs1r1CYMS7XMyIVDzRewkDLNBw=
-github.com/lucas-clemente/quic-go v0.20.0/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk=
+github.com/lucas-clemente/quic-go v0.20.1 h1:hb5m76V8QS/8Nw/suHvXqo3BMHAozvIkcnzpJdpanSk=
+github.com/lucas-clemente/quic-go v0.20.1/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5/go.mod h1:c2mYKRyMb1BPkO5St0c/ps62L4S0W2NAkaTXj9qEI+0=
 github.com/lusis/slack-test v0.0.0-20180109053238-3c758769bfa6/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg=
diff --git a/publish/publish.go b/publish/publish.go
index 417978e..9e585e8 100644
--- a/publish/publish.go
+++ b/publish/publish.go
@@ -3,7 +3,6 @@ package publish
 import (
 	"bytes"
 	"encoding/json"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -21,7 +20,6 @@ import (
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
 	"github.com/golang/protobuf/proto"
-	"golang.org/x/crypto/sha3"
 )
 
 var TODO = `
@@ -43,6 +41,14 @@ var TODO = `
 }
 `
 
+type Details struct {
+	Title       string
+	Description string
+	Author      string
+	Tags        []string
+	ReleaseTime int64
+}
+
 func Publish(client *lbrycrd.Client, path, name, address string, details Details, reflectorAddress string) (*wire.MsgTx, *chainhash.Hash, error) {
 	if name == "" {
 		return nil, nil, errors.Err("name required")
@@ -69,11 +75,20 @@ func Publish(client *lbrycrd.Client, path, name, address string, details Details
 		return nil, nil, err
 	}
 
-	claim, st, err := makeClaimAndStream(path, details)
+	st, stPB, err := makeStream(path)
 	if err != nil {
 		return nil, nil, err
 	}
 
+	stPB.Author = details.Author
+	stPB.ReleaseTime = details.ReleaseTime
+
+	claim := &pb.Claim{
+		Title:       details.Title,
+		Description: details.Description,
+		Type:        &pb.Claim_Stream{Stream: stPB},
+	}
+
 	err = addClaimToTx(tx, claim, name, amount, addr)
 	if err != nil {
 		return nil, nil, err
@@ -203,50 +218,31 @@ func reflect(st stream.Stream, reflectorAddress string) error {
 	return nil
 }
 
-type Details struct {
-	Title       string
-	Description string
-	Author      string
-	Tags        []string
-	ReleaseTime int64
-}
-
-func makeClaimAndStream(path string, details Details) (*pb.Claim, stream.Stream, error) {
+func makeStream(path string) (stream.Stream, *pb.Stream, error) {
 	file, err := os.Open(path)
 	if err != nil {
 		return nil, nil, errors.Err(err)
 	}
-	data, err := ioutil.ReadAll(file)
-	if err != nil {
-		return nil, nil, errors.Err(err)
-	}
-	s, err := stream.New(data)
-	if err != nil {
-		return nil, nil, errors.Err(err)
-	}
+	defer file.Close()
+
+	enc := stream.NewEncoder(file)
 
-	// make the claim
-	sdBlob := &stream.SDBlob{}
-	err = sdBlob.FromBlob(s[0])
+	s, err := enc.Stream()
 	if err != nil {
 		return nil, nil, errors.Err(err)
 	}
 
-	filehash := sha3.Sum384(data)
-
-	streamPB := &pb.Stream{
-		Author:      details.Author,
-		ReleaseTime: details.ReleaseTime,
+	streamProto := &pb.Stream{
 		Source: &pb.Source{
-			SdHash: s[0].Hash(),
+			SdHash: enc.SDBlob().Hash(),
 			Name:   filepath.Base(file.Name()),
-			Size:   uint64(len(data)),
-			Hash:   filehash[:],
+			Size:   uint64(enc.SourceLen()),
+			Hash:   enc.SourceHash(),
 		},
 	}
 
 	mimeType, category := guessMimeType(filepath.Ext(file.Name()))
-	streamPB.Source.MediaType = mimeType
+	streamProto.Source.MediaType = mimeType
 
 	switch category {
 	case "video":
@@ -254,20 +250,14 @@ func makeClaimAndStream(path string, details Details) (*pb.Claim, stream.Stream,
 		//if err != nil {
 		//	return nil, nil, err
 		//}
-		streamPB.Type = &pb.Stream_Video{}
+		streamProto.Type = &pb.Stream_Video{}
 	case "audio":
-		streamPB.Type = &pb.Stream_Audio{}
+		streamProto.Type = &pb.Stream_Audio{}
 	case "image":
-		streamPB.Type = &pb.Stream_Image{}
-	}
-
-	claim := &pb.Claim{
-		Title:       details.Title,
-		Description: details.Description,
-		Type:        &pb.Claim_Stream{Stream: streamPB},
+		streamProto.Type = &pb.Stream_Image{}
 	}
 
-	return claim, s, nil
+	return s, streamProto, nil
 }
 
 func getClaimPayoutScript(name string, value []byte, address btcutil.Address) ([]byte, error) {

From c4084eeb6849c4e6903285b18463e1c1c0432cd7 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Thu, 29 Apr 2021 03:41:18 +0200
Subject: [PATCH 34/51] improve disk cleanup

add index to is_stored
fix test
replace LRU cache
---
 cmd/reflector.go   | 46 +++++++++++++++++++++++++++++++++++-----------
 db/db.go           |  3 ++-
 go.mod             |  3 ++-
 go.sum             |  2 ++
 store/disk_test.go |  2 +-
 store/lru.go       | 35 +++++++++++++++--------------------
 store/lru_test.go  |  4 ++--
 7 files changed, 59 insertions(+), 36 deletions(-)

diff --git a/cmd/reflector.go b/cmd/reflector.go
index 6fe44ad..a642c7a 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -254,6 +254,11 @@ func diskCacheParams(diskParams string) (int, string) {
 }
 
 func cleanOldestBlobs(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Group) {
+	// this is so that it runs on startup without having to wait for 10 minutes
+	err := doClean(maxItems, db, store, stopper)
+	if err != nil {
+		log.Error(errors.FullTrace(err))
+	}
 	const cleanupInterval = 10 * time.Minute
 	for {
 		select {
@@ -281,19 +286,38 @@ func doClean(maxItems int, db *db.SQL, store store.BlobStore, stopper *stop.Grou
 		if err != nil {
 			return err
 		}
-
-		for _, hash := range blobs {
-			select {
-			case <-stopper.Ch():
-				return nil
-			default:
-			}
-
-			err = store.Delete(hash)
-			if err != nil {
-				return err
+		blobsChan := make(chan string, len(blobs))
+		wg := &stop.Group{}
+		go func() {
+			for _, hash := range blobs {
+				select {
+				case <-stopper.Ch():
+					return
+				default:
+				}
+				blobsChan <- hash
 			}
+			close(blobsChan)
+		}()
+		for i := 0; i < 3; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for h := range blobsChan {
+					select {
+					case <-stopper.Ch():
+						return
+					default:
+					}
+					err = store.Delete(h)
+					if err != nil {
+						log.Errorf("error pruning %s: %s", h, errors.FullTrace(err))
+						continue
+					}
+				}
+			}()
 		}
+		wg.Wait()
 	}
 	return nil
 }
diff --git a/db/db.go b/db/db.go
index 3e7d4ca..babe641 100644
--- a/db/db.go
+++ b/db/db.go
@@ -781,7 +781,8 @@ CREATE TABLE blob_ (
   last_accessed_at TIMESTAMP NULL DEFAULT NULL,
   PRIMARY KEY (id),
   UNIQUE KEY blob_hash_idx (hash),
-  KEY `blob_last_accessed_idx` (`last_accessed_at`)
+  KEY `blob_last_accessed_idx` (`last_accessed_at`),
+  KEY `is_stored_idx` (`is_stored`)
 );
 
 CREATE TABLE stream (
diff --git a/go.mod b/go.mod
index 4ed4f23..eceb7ba 100644
--- a/go.mod
+++ b/go.mod
@@ -7,6 +7,7 @@ replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203
 require (
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
 	github.com/aws/aws-sdk-go v1.16.11
+	github.com/bluele/gcache v0.0.2
 	github.com/bparli/lfuda-go v0.3.1
 	github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
@@ -17,7 +18,7 @@ require (
 	github.com/google/gops v0.3.7
 	github.com/gorilla/mux v1.7.4
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
-	github.com/hashicorp/golang-lru v0.5.4
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/memberlist v0.1.4 // indirect
 	github.com/hashicorp/serf v0.8.2
 	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
diff --git a/go.sum b/go.sum
index c51b403..48680c9 100644
--- a/go.sum
+++ b/go.sum
@@ -42,6 +42,8 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
+github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
 github.com/bparli/lfuda-go v0.3.1 h1:nO9Szo627RC8/z+R+MMPBItNwHCOonchmpjQuQi8jVY=
 github.com/bparli/lfuda-go v0.3.1/go.mod h1:BR5a9lwlqRqnPhU3F5ojFK3VhTKg8iFVtJJKgZBQhAo=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
diff --git a/store/disk_test.go b/store/disk_test.go
index 72c7755..1ab3f05 100644
--- a/store/disk_test.go
+++ b/store/disk_test.go
@@ -19,7 +19,7 @@ func TestDiskStore_Get(t *testing.T) {
 	defer os.RemoveAll(tmpDir)
 	d := NewDiskStore(tmpDir, 2)
 
-	hash := "1234567890"
+	hash := "f428b8265d65dad7f8ffa52922bba836404cbd62f3ecfe10adba6b444f8f658938e54f5981ac4de39644d5b93d89a94b"
 	data := []byte("oyuntyausntoyaunpdoyruoyduanrstjwfjyuwf")
 
 	expectedPath := path.Join(tmpDir, hash[:2], hash)
diff --git a/store/lru.go b/store/lru.go
index e96931b..be6cd82 100644
--- a/store/lru.go
+++ b/store/lru.go
@@ -3,12 +3,13 @@ package store
 import (
 	"time"
 
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/shared"
 
-	golru "github.com/hashicorp/golang-lru"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	"github.com/bluele/gcache"
 	"github.com/sirupsen/logrus"
 )
 
@@ -17,7 +18,7 @@ type LRUStore struct {
 	// underlying store
 	store BlobStore
 	// lru implementation
-	lru *golru.Cache
+	lru gcache.Cache
 }
 
 // NewLRUStore initialize a new LRUStore
@@ -25,20 +26,14 @@ func NewLRUStore(component string, store BlobStore, maxItems int) *LRUStore {
 	l := &LRUStore{
 		store: store,
 	}
-
-	lru, err := golru.NewWithEvict(maxItems, func(key interface{}, value interface{}) {
+	l.lru = gcache.New(maxItems).ARC().EvictedFunc(func(key, value interface{}) {
 		metrics.CacheLRUEvictCount.With(metrics.CacheLabels(l.Name(), component)).Inc()
-		_ = store.Delete(key.(string)) // TODO: log this error. may happen if underlying entry is gone but cache entry still there
-	})
-	if err != nil {
-		panic(err)
-	}
-
-	l.lru = lru
+		_ = store.Delete(key.(string))
+	}).Build()
 
 	go func() {
 		if lstr, ok := store.(lister); ok {
-			err = l.loadExisting(lstr, maxItems)
+			err := l.loadExisting(lstr, maxItems)
 			if err != nil {
 				panic(err) // TODO: what should happen here? panic? return nil? just keep going?
 			}
@@ -55,14 +50,14 @@ func (l *LRUStore) Name() string {
 
 // Has returns whether the blob is in the store, without updating the recent-ness.
 func (l *LRUStore) Has(hash string) (bool, error) {
-	return l.lru.Contains(hash), nil
+	return l.lru.Has(hash), nil
 }
 
 // Get returns the blob or an error if the blob doesn't exist.
 func (l *LRUStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 	start := time.Now()
-	_, has := l.lru.Get(hash)
-	if !has {
+	_, err := l.lru.Get(hash)
+	if err != nil {
 		return nil, shared.NewBlobTrace(time.Since(start), l.Name()), errors.Err(ErrBlobNotFound)
 	}
 	blob, stack, err := l.store.Get(hash)
@@ -80,7 +75,7 @@ func (l *LRUStore) Put(hash string, blob stream.Blob) error {
 		return err
 	}
 
-	l.lru.Add(hash, true)
+	l.lru.Set(hash, true)
 	return nil
 }
 
@@ -91,7 +86,7 @@ func (l *LRUStore) PutSD(hash string, blob stream.Blob) error {
 		return err
 	}
 
-	l.lru.Add(hash, true)
+	_ = l.lru.Set(hash, true)
 	return nil
 }
 
@@ -119,7 +114,7 @@ func (l *LRUStore) loadExisting(store lister, maxItems int) error {
 	logrus.Infof("read %d files from disk", len(existing))
 	added := 0
 	for _, h := range existing {
-		l.lru.Add(h, true)
+		l.lru.Set(h, true)
 		added++
 		if maxItems > 0 && added >= maxItems { // underlying cache is bigger than LRU cache
 			break
diff --git a/store/lru_test.go b/store/lru_test.go
index 6aec768..798d5c2 100644
--- a/store/lru_test.go
+++ b/store/lru_test.go
@@ -87,7 +87,7 @@ func TestLRUStore_UnderlyingBlobMissing(t *testing.T) {
 	require.NoError(t, err)
 
 	// hash still exists in lru
-	assert.True(t, lru.lru.Contains(hash))
+	assert.True(t, lru.lru.Has(hash))
 
 	blob, _, err := lru.Get(hash)
 	assert.Nil(t, blob)
@@ -96,7 +96,7 @@ func TestLRUStore_UnderlyingBlobMissing(t *testing.T) {
 		reflect.TypeOf(err).String(), err.Error())
 
 	// lru.Get() removes hash if underlying store doesn't have it
-	assert.False(t, lru.lru.Contains(hash))
+	assert.False(t, lru.lru.Has(hash))
 }
 
 func TestLRUStore_loadExisting(t *testing.T) {

From 070938e12afba8e26881259e04a6654102443413 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Thu, 6 May 2021 22:53:18 +0200
Subject: [PATCH 35/51] increase window size

---
 peer/http3/server.go | 9 +++++++--
 peer/http3/store.go  | 4 ++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/peer/http3/server.go b/peer/http3/server.go
index 30ae3fd..d81f0f8 100644
--- a/peer/http3/server.go
+++ b/peer/http3/server.go
@@ -65,9 +65,14 @@ type availabilityResponse struct {
 // Start starts the server listener to handle connections.
 func (s *Server) Start(address string) error {
 	log.Println("HTTP3 peer listening on " + address)
+	window500M := 500 * 1 << 20
+
 	quicConf := &quic.Config{
-		HandshakeIdleTimeout: 4 * time.Second,
-		MaxIdleTimeout:       20 * time.Second,
+		MaxStreamReceiveWindow:     uint64(window500M),
+		MaxConnectionReceiveWindow: uint64(window500M),
+		EnableDatagrams:            true,
+		HandshakeIdleTimeout:       4 * time.Second,
+		MaxIdleTimeout:             20 * time.Second,
 	}
 	r := mux.NewRouter()
 	r.HandleFunc("/get/{hash}", func(w http.ResponseWriter, r *http.Request) {
diff --git a/peer/http3/store.go b/peer/http3/store.go
index bcc7cae..7d8fc34 100644
--- a/peer/http3/store.go
+++ b/peer/http3/store.go
@@ -36,6 +36,10 @@ func NewStore(opts StoreOpts) *Store {
 
 func (p *Store) getClient() (*Client, error) {
 	var qconf quic.Config
+	window500M := 500 * 1 << 20
+	qconf.MaxStreamReceiveWindow = uint64(window500M)
+	qconf.MaxConnectionReceiveWindow = uint64(window500M)
+	qconf.EnableDatagrams = true
 	qconf.HandshakeIdleTimeout = 4 * time.Second
 	qconf.MaxIdleTimeout = 20 * time.Second
 	pool, err := x509.SystemCertPool()

From 50c077a9cba8d0ac22d2be4328308ed611895ac6 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Thu, 20 May 2021 23:17:18 +0200
Subject: [PATCH 36/51] request queue size param

---
 cmd/reflector.go     | 4 +++-
 peer/http3/server.go | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/cmd/reflector.go b/cmd/reflector.go
index a642c7a..d2acfed 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -41,6 +41,7 @@ var (
 	reflectorCmdDiskCache       string
 	bufferReflectorCmdDiskCache string
 	reflectorCmdMemCache        int
+	requestQueueSize            int
 )
 
 func init() {
@@ -58,6 +59,7 @@ func init() {
 	cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
 	cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
 	cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for metrics")
+	cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests should be submitted to upstream")
 	cmd.Flags().BoolVar(&disableUploads, "disable-uploads", false, "Disable uploads to this reflector server")
 	cmd.Flags().BoolVar(&disableBlocklist, "disable-blocklist", false, "Disable blocklist watching/updating")
 	cmd.Flags().BoolVar(&useDB, "use-db", true, "whether to connect to the reflector db or not")
@@ -96,7 +98,7 @@ func reflectorCmd(cmd *cobra.Command, args []string) {
 	}
 	defer peerServer.Shutdown()
 
-	http3PeerServer := http3.NewServer(outerStore)
+	http3PeerServer := http3.NewServer(outerStore, requestQueueSize)
 	err = http3PeerServer.Start(":" + strconv.Itoa(http3PeerPort))
 	if err != nil {
 		log.Fatal(err)
diff --git a/peer/http3/server.go b/peer/http3/server.go
index d81f0f8..3d49d17 100644
--- a/peer/http3/server.go
+++ b/peer/http3/server.go
@@ -33,7 +33,7 @@ type Server struct {
 }
 
 // NewServer returns an initialized Server pointer.
-func NewServer(store store.BlobStore) *Server {
+func NewServer(store store.BlobStore, requestQueueSize int) *Server {
 	return &Server{
 		store: store,
 		grp:   stop.New(),

From eafc62f2a6103ed1435c906553cf0f5b7e42ebbd Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 20 May 2021 17:21:35 -0400
Subject: [PATCH 37/51] add gops to reflector server

---
 go.mod              |  2 +-
 go.sum              | 25 +++++++++++++------------
 reflector/server.go |  5 ++++-
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/go.mod b/go.mod
index eceb7ba..82eaa1e 100644
--- a/go.mod
+++ b/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/golang/protobuf v1.4.2
-	github.com/google/gops v0.3.7
+	github.com/google/gops v0.3.18
 	github.com/gorilla/mux v1.7.4
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
diff --git a/go.sum b/go.sum
index 48680c9..6af413e 100644
--- a/go.sum
+++ b/go.sum
@@ -23,7 +23,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -100,7 +100,7 @@ github.com/go-ini/ini v1.48.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
@@ -140,8 +140,8 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/google/gops v0.3.7 h1:KtVAagOM0FIq+02DiQrKBTnLhYpWBMowaufcj+W1Exw=
-github.com/google/gops v0.3.7/go.mod h1:bj0cwMmX1X4XIJFTjR99R5sCxNssNJ8HebFNvoQlmgY=
+github.com/google/gops v0.3.18 h1:my259V+172PVFmduS2RAsq4FKH+HjKqdh7pLr17Ot8c=
+github.com/google/gops v0.3.18/go.mod h1:Pfp8hWGIFdV/7rY9/O/U5WgdjYQXf/GiEK4NVuVd2ZE=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -232,11 +232,9 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1 h1:PJPDf8OUfOK1bb/NeTKd4f1QXZItOX389VN3B6qC8ro=
-github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
-github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
+github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
@@ -372,8 +370,7 @@ github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77/go.mod h1:jXP4hmWywN
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sfreiberg/gotwilio v0.0.0-20180612161623-8fb7259ba8bf/go.mod h1:60PiR0SAnAcYSiwrXB6BaxeqHdXMf172toCosHfV+Yk=
-github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
-github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shirou/gopsutil/v3 v3.21.2/go.mod h1:ghfMypLDrFSWN2c9cDYFLHyynQ+QUht0cv/18ZqVczw=
 github.com/shopspring/decimal v0.0.0-20180607144847-19e3cb6c2930/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v0.0.0-20191009025716-f1972eb1d1f5 h1:Gojs/hac/DoYEM7WEICT45+hNWczIeuL5D21e5/HPAw=
@@ -437,11 +434,14 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
+github.com/tklauser/go-sysconf v0.3.4/go.mod h1:Cl2c8ZRWfHD5IrfHo9VN+FX9kCFjIOyVklgXycLB6ek=
+github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1gBkr4QyP8=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
@@ -454,7 +454,7 @@ github.com/volatiletech/null v8.0.0+incompatible/go.mod h1:0wD98JzdqB+rLyZ70fN05
 github.com/volatiletech/sqlboiler v3.4.0+incompatible h1:saQ6WxZ9wEJp33q3w/DHs7an7SYi1H7Yzf4/moxCbJU=
 github.com/volatiletech/sqlboiler v3.4.0+incompatible/go.mod h1:jLfDkkHWPbS2cWRLkyC20vQWaIQsASEY7gM7zSo11Yw=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
+github.com/xlab/treeprint v1.0.0/go.mod h1:IoImgRak9i3zJyuxOKUP1v4UZd1tMoKkq/Cimt1uhCg=
 github.com/ybbus/jsonrpc v0.0.0-20180411222309-2a548b7d822d/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE=
 github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -543,7 +543,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20171017063910-8dbc5d05d6ed/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -572,6 +571,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201231184435-2d18734c6014 h1:joucsQqXmyBVxViHCPFjG3hx8JzIFSaym3l3MM/Jsdg=
 golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210217105451-b926d437f341 h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU=
+golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -692,6 +693,6 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
+rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
 sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
diff --git a/reflector/server.go b/reflector/server.go
index 592ba3d..8e78e22 100644
--- a/reflector/server.go
+++ b/reflector/server.go
@@ -10,6 +10,7 @@ import (
 	"net"
 	"time"
 
+	"github.com/google/gops/agent"
 	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/store"
 
@@ -69,7 +70,9 @@ func (s *Server) Start(address string) error {
 		return errors.Err(err)
 	}
 	log.Println("reflector listening on " + address)
-
+	if err := agent.Listen(agent.Options{}); err != nil {
+		log.Fatal(err)
+	}
 	s.grp.Add(1)
 	go func() {
 		<-s.grp.Ch()

From 0152300d8df2570344b333161be3c915e83cdcf4 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 20 May 2021 18:01:13 -0400
Subject: [PATCH 38/51] add guage metrics for go routines in reflector package

---
 internal/metrics/metrics.go |  5 +++++
 reflector/blocklist.go      |  5 ++++-
 reflector/server.go         | 10 ++++++++++
 reflector/uploader.go       |  6 ++++++
 4 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
index 2570677..4c30606 100644
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@@ -219,6 +219,11 @@ var (
 		Name:      "http3_blob_request_queue_size",
 		Help:      "Blob requests of https queue size",
 	})
+	RoutinesQueue = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: ns,
+		Name:      "routines",
+		Help:      "routines running by type",
+	}, []string{"package", "kind"})
 )
 
 func CacheLabels(name, component string) prometheus.Labels {
diff --git a/reflector/blocklist.go b/reflector/blocklist.go
index 9abfc4a..26edff6 100644
--- a/reflector/blocklist.go
+++ b/reflector/blocklist.go
@@ -8,6 +8,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/lbryio/reflector.go/internal/metrics"
+
 	"github.com/lbryio/reflector.go/store"
 	"github.com/lbryio/reflector.go/wallet"
 
@@ -109,8 +111,9 @@ func sdHashesForOutpoints(walletServers, outpoints []string, stopper stop.Chan)
 	}
 
 	done := make(chan bool)
-
+	metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Inc()
 	go func() {
+		defer metrics.RoutinesQueue.WithLabelValues("reflector", "sdhashesforoutput").Dec()
 		select {
 		case <-done:
 		case <-stopper:
diff --git a/reflector/server.go b/reflector/server.go
index 8e78e22..3171ced 100644
--- a/reflector/server.go
+++ b/reflector/server.go
@@ -74,7 +74,9 @@ func (s *Server) Start(address string) error {
 		log.Fatal(err)
 	}
 	s.grp.Add(1)
+	metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Inc()
 	go func() {
+		defer metrics.RoutinesQueue.WithLabelValues("reflector", "listener").Dec()
 		<-s.grp.Ch()
 		err := l.Close()
 		if err != nil {
@@ -84,7 +86,9 @@ func (s *Server) Start(address string) error {
 	}()
 
 	s.grp.Add(1)
+	metrics.RoutinesQueue.WithLabelValues("reflector", "start").Inc()
 	go func() {
+		defer metrics.RoutinesQueue.WithLabelValues("reflector", "start").Dec()
 		s.listenAndServe(l)
 		s.grp.Done()
 	}()
@@ -92,7 +96,9 @@ func (s *Server) Start(address string) error {
 	if s.EnableBlocklist {
 		if b, ok := s.underlyingStore.(store.Blocklister); ok {
 			s.grp.Add(1)
+			metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Inc()
 			go func() {
+				defer metrics.RoutinesQueue.WithLabelValues("reflector", "enableblocklist").Dec()
 				s.enableBlocklist(b)
 				s.grp.Done()
 			}()
@@ -115,7 +121,9 @@ func (s *Server) listenAndServe(listener net.Listener) {
 			log.Error(err)
 		} else {
 			s.grp.Add(1)
+			metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc()
 			go func() {
+				defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-listenandserve").Inc()
 				s.handleConn(conn)
 				s.grp.Done()
 			}()
@@ -130,7 +138,9 @@ func (s *Server) handleConn(conn net.Conn) {
 		close(connNeedsClosing)
 	}()
 	s.grp.Add(1)
+	metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Inc()
 	go func() {
+		defer metrics.RoutinesQueue.WithLabelValues("reflector", "server-handleconn").Dec()
 		defer s.grp.Done()
 		select {
 		case <-connNeedsClosing:
diff --git a/reflector/uploader.go b/reflector/uploader.go
index e5b2688..b421272 100644
--- a/reflector/uploader.go
+++ b/reflector/uploader.go
@@ -7,6 +7,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lbryio/reflector.go/internal/metrics"
+
 	"github.com/lbryio/reflector.go/db"
 	"github.com/lbryio/reflector.go/store"
 
@@ -88,7 +90,9 @@ func (u *Uploader) Upload(dirOrFilePath string) error {
 
 	for i := 0; i < u.workers; i++ {
 		workerWG.Add(1)
+		metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Inc()
 		go func(i int) {
+			defer metrics.RoutinesQueue.WithLabelValues("reflector", "upload").Dec()
 			defer workerWG.Done()
 			defer func(i int) { log.Debugf("worker %d quitting", i) }(i)
 			u.worker(pathChan)
@@ -97,7 +101,9 @@ func (u *Uploader) Upload(dirOrFilePath string) error {
 
 	countWG := sync.WaitGroup{}
 	countWG.Add(1)
+	metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Inc()
 	go func() {
+		defer metrics.RoutinesQueue.WithLabelValues("reflector", "uploader").Dec()
 		defer countWG.Done()
 		u.counter()
 	}()

From 4ecce75e2372c690fc85ea801dedbaa81e90cd78 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 20 May 2021 18:12:30 -0400
Subject: [PATCH 39/51] add metric calls for other packages

---
 peer/http3/worker.go         | 2 ++
 peer/server.go               | 2 ++
 store/caching.go             | 2 ++
 store/speedwalk/speedwalk.go | 3 +++
 4 files changed, 9 insertions(+)

diff --git a/peer/http3/worker.go b/peer/http3/worker.go
index d2b625b..91dfd02 100644
--- a/peer/http3/worker.go
+++ b/peer/http3/worker.go
@@ -20,7 +20,9 @@ var getReqCh = make(chan *blobRequest, 20000)
 func InitWorkers(server *Server, workers int) {
 	stopper := stop.New(server.grp)
 	for i := 0; i < workers; i++ {
+		metrics.RoutinesQueue.WithLabelValues("http3", "worker").Inc()
 		go func(worker int) {
+			defer metrics.RoutinesQueue.WithLabelValues("http3", "worker").Dec()
 			for {
 				select {
 				case <-stopper.Ch():
diff --git a/peer/server.go b/peer/server.go
index 708fe53..4061ea5 100644
--- a/peer/server.go
+++ b/peer/server.go
@@ -89,7 +89,9 @@ func (s *Server) listenAndServe(listener net.Listener) {
 			log.Error(errors.Prefix("accepting conn", err))
 		} else {
 			s.grp.Add(1)
+			metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Inc()
 			go func() {
+				defer metrics.RoutinesQueue.WithLabelValues("peer", "server-handleconn").Dec()
 				s.handleConnection(conn)
 				s.grp.Done()
 			}()
diff --git a/store/caching.go b/store/caching.go
index 6f1d4d4..cd05b41 100644
--- a/store/caching.go
+++ b/store/caching.go
@@ -66,7 +66,9 @@ func (c *CachingStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 	}
 	// there is no need to wait for the blob to be stored before we return it
 	// TODO: however this should be refactored to limit the amount of routines that the process can spawn to avoid a possible DoS
+	metrics.RoutinesQueue.WithLabelValues("store", "cache-put").Inc()
 	go func() {
+		defer metrics.RoutinesQueue.WithLabelValues("store", "cache-put").Dec()
 		err = c.cache.Put(hash, blob)
 		if err != nil {
 			log.Errorf("error saving blob to underlying cache: %s", errors.FullTrace(err))
diff --git a/store/speedwalk/speedwalk.go b/store/speedwalk/speedwalk.go
index 9b899e4..1d4b981 100644
--- a/store/speedwalk/speedwalk.go
+++ b/store/speedwalk/speedwalk.go
@@ -6,6 +6,8 @@ import (
 	"runtime"
 	"sync"
 
+	"github.com/lbryio/reflector.go/internal/metrics"
+
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 
 	"github.com/karrick/godirwalk"
@@ -24,6 +26,7 @@ func AllFiles(startDir string, basename bool) ([]string, error) {
 	paths := make([]string, 0, 1000)
 	pathWG := &sync.WaitGroup{}
 	pathWG.Add(1)
+	metrics.RoutinesQueue.WithLabelValues("speedwalk", "worker").Inc()
 	go func() {
 		defer pathWG.Done()
 		for {

From 45130499cd2e2fbfd566832c92baf802c5dce8a2 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 20 May 2021 19:05:48 -0400
Subject: [PATCH 40/51] Add single flight for cache not just origin

---
 store/caching.go      |  2 +-
 store/singleflight.go | 37 +++++++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/store/caching.go b/store/caching.go
index cd05b41..65b6397 100644
--- a/store/caching.go
+++ b/store/caching.go
@@ -24,7 +24,7 @@ func NewCachingStore(component string, origin, cache BlobStore) *CachingStore {
 	return &CachingStore{
 		component: component,
 		origin:    WithSingleFlight(component, origin),
-		cache:     cache,
+		cache:     WithSingleFlight(component, cache),
 	}
 }
 
diff --git a/store/singleflight.go b/store/singleflight.go
index 362182d..6de91f4 100644
--- a/store/singleflight.go
+++ b/store/singleflight.go
@@ -80,6 +80,43 @@ func (s *singleflightStore) getter(hash string) func() (interface{}, error) {
 	}
 }
 
+// Put ensures that only one request per hash is sent to the origin at a time,
+// thereby protecting against https://en.wikipedia.org/wiki/Thundering_herd_problem
+func (s *singleflightStore) Put(hash string, blob stream.Blob) error {
+	metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
+	defer metrics.CacheWaitingRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
+
+	_, err, _ := s.sf.Do(hash, s.putter(hash, blob))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// putter returns a function that puts a blob from the origin
+// only one putter per hash will be executing at a time
+func (s *singleflightStore) putter(hash string, blob stream.Blob) func() (interface{}, error) {
+	return func() (interface{}, error) {
+		metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Inc()
+		defer metrics.CacheOriginRequestsCount.With(metrics.CacheLabels(s.Name(), s.component)).Dec()
+
+		start := time.Now()
+		err := s.BlobStore.Put(hash, blob)
+		if err != nil {
+			return nil, err
+		}
+
+		rate := float64(len(blob)) / 1024 / 1024 / time.Since(start).Seconds()
+		metrics.CacheRetrievalSpeed.With(map[string]string{
+			metrics.LabelCacheType: s.Name(),
+			metrics.LabelComponent: s.component,
+			metrics.LabelSource:    "origin",
+		}).Set(rate)
+
+		return nil, nil
+	}
+}
+
 // Shutdown shuts down the store gracefully
 func (s *singleflightStore) Shutdown() {
 	s.BlobStore.Shutdown()

From 006b04f6e9bc140572c4a9b1acdb2a7a1232adb6 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Fri, 21 May 2021 01:48:46 +0200
Subject: [PATCH 41/51] add a lot of extra heavy debugging

---
 store/disk.go | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/store/disk.go b/store/disk.go
index eaecfbb..f108204 100644
--- a/store/disk.go
+++ b/store/disk.go
@@ -101,6 +101,12 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 
 // Put stores the blob on disk
 func (d *DiskStore) Put(hash string, blob stream.Blob) error {
+	start := time.Now()
+	defer func() {
+		if time.Since(start) > 100*time.Millisecond {
+			log.Infof("it took %s to write %s", time.Since(start), hash)
+		}
+	}()
 	err := d.initOnce()
 	if err != nil {
 		return err
@@ -110,8 +116,26 @@ func (d *DiskStore) Put(hash string, blob stream.Blob) error {
 	if err != nil {
 		return err
 	}
-
+	hashBytes := sha512.Sum384(blob)
+	readHash := hex.EncodeToString(hashBytes[:])
+	matchesBeforeWriting := readHash == hash
 	err = ioutil.WriteFile(d.path(hash), blob, 0644)
+	if err != nil {
+		log.Errorf("Error saving to disk: %s", err.Error())
+	}
+	readBlob, err := ioutil.ReadFile(d.path(hash))
+	matchesAfterReading := false
+	if err != nil {
+		log.Errorf("for some fucking reasons I can't read the blob I just wrote %s", err.Error())
+	} else {
+		hashBytes = sha512.Sum384(readBlob)
+		readHash = hex.EncodeToString(hashBytes[:])
+		matchesAfterReading = readHash == hash
+	}
+
+	log.Infof(`writing %s to disk: hash match: %t, error: %t
+reading after writing: hash match: %t`, hash, matchesBeforeWriting, err == nil, matchesAfterReading)
+
 	return errors.Err(err)
 }
 

From 213d21b0214f210c581e744e30afc0e158826c75 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 20 May 2021 19:59:50 -0400
Subject: [PATCH 42/51] Add locks to disk store.

---
 locks/multiplelock.go      | 131 +++++++++++++++++++++++++++++++++++++
 locks/multiplelock_test.go |  56 ++++++++++++++++
 store/disk.go              |  13 ++++
 3 files changed, 200 insertions(+)
 create mode 100644 locks/multiplelock.go
 create mode 100644 locks/multiplelock_test.go

diff --git a/locks/multiplelock.go b/locks/multiplelock.go
new file mode 100644
index 0000000..c87b645
--- /dev/null
+++ b/locks/multiplelock.go
@@ -0,0 +1,131 @@
+package locks
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+// MultipleLock is the main interface for multiLock based on key
+type MultipleLock interface {
+	Lock(key string)
+	RLock(key string)
+	Unlock(key string)
+	RUnlock(key string)
+}
+
+func NewMultipleLock() MultipleLock {
+	return &multiLock{
+		locks: make(map[string]*itemLock),
+		mu:    sync.Mutex{},
+	}
+}
+
+type itemLock struct {
+	lk  *sync.RWMutex
+	cnt int64
+}
+
+// multiLock is an optimized locking system per locking key
+type multiLock struct {
+	locks map[string]*itemLock
+	mu    sync.Mutex // synchronize reads/writes to locks map
+}
+
+func debugPrint(format string, a ...interface{}) {
+	debugEnabled := false
+	if debugEnabled {
+		a = append(a, time.Now().Format("04:05.000000"))
+		fmt.Printf(format+" at %s\n", a...)
+	}
+}
+
+func (ml *multiLock) Lock(key string) {
+	debugPrint("mutex requested %s", key)
+	ml.mu.Lock()
+	debugPrint("mutex acquired %s", key)
+
+	itmLock, exists := ml.locks[key]
+	if !exists {
+		debugPrint("new lock created %s", key)
+		itmLock = &itemLock{&sync.RWMutex{}, 0}
+		ml.locks[key] = itmLock
+	}
+	itmLock.cnt++
+	debugPrint("releasing mutex %s", key)
+	ml.mu.Unlock()
+
+	debugPrint("Lock requested %s", key)
+	itmLock.lk.Lock()
+	debugPrint("Lock acquired %s", key)
+}
+
+func (ml *multiLock) RLock(key string) {
+	debugPrint("mutex requested %s", key)
+	ml.mu.Lock()
+	debugPrint("mutex acquired %s", key)
+
+	itmLock, exists := ml.locks[key]
+	if !exists {
+		debugPrint("new lock created for %s", key)
+		itmLock = &itemLock{&sync.RWMutex{}, 0}
+		ml.locks[key] = itmLock
+	}
+	itmLock.cnt++
+	debugPrint("releasing mutex %s", key)
+	ml.mu.Unlock()
+
+	debugPrint("RLock requested %s", key)
+	itmLock.lk.RLock()
+	debugPrint("RLock acquired %s", key)
+}
+
+func (ml *multiLock) Unlock(key string) {
+	debugPrint("mutex requested %s", key)
+	ml.mu.Lock()
+	debugPrint("mutex acquired %s", key)
+
+	itmLock, exists := ml.locks[key]
+	if !exists {
+		panic("sync Unlock of non existent lock!!")
+	}
+
+	debugPrint("Unlock %s", key)
+	itmLock.lk.Unlock()
+	itmLock.cnt--
+	if itmLock.cnt == 0 {
+		debugPrint("delete lock %s", key)
+		delete(ml.locks, key)
+	}
+	if itmLock.cnt < 0 {
+		panic("sync Unlock of free Lock!!")
+	}
+
+	debugPrint("releasing mutex %s", key)
+	ml.mu.Unlock()
+}
+
+func (ml *multiLock) RUnlock(key string) {
+	debugPrint("mutex requested %s", key)
+	ml.mu.Lock()
+	debugPrint("mutex acquired %s", key)
+
+	itmLock, exists := ml.locks[key]
+	if !exists {
+		panic("sync Unlock of non existent lock!!")
+	}
+
+	debugPrint("RUnlock %s", key)
+	itmLock.lk.RUnlock()
+	itmLock.cnt--
+	if itmLock.cnt == 0 {
+		debugPrint("delete lock %s", key)
+		delete(ml.locks, key)
+	}
+	if itmLock.cnt < 0 {
+		panic("sync Unlock of free Lock!!")
+	}
+
+	debugPrint("releasing mutex %s", key)
+	ml.mu.Unlock()
+}
diff --git a/locks/multiplelock_test.go b/locks/multiplelock_test.go
new file mode 100644
index 0000000..a7a42c7
--- /dev/null
+++ b/locks/multiplelock_test.go
@@ -0,0 +1,56 @@
+package locks
+
+import (
+	"math/rand"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+)
+
+var lock = NewMultipleLock()
+
+func TestNewMultipleLock(t *testing.T) {
+	grp := stop.New()
+	for i := 0; i < 100; i++ {
+		grp.Add(2)
+		go doRWWork(i, i%10, grp)
+		go doRWork(i, i%10, grp)
+	}
+	time.Sleep(5 * time.Second)
+	grp.StopAndWait()
+}
+func doRWWork(worker int, resource int, grp *stop.Group) {
+	for {
+		select {
+		case <-grp.Ch():
+			grp.Done()
+			return
+		default:
+			//log.Printf("RW - worker %d doing work on resource %d\n", worker, resource)
+			lock.Lock(strconv.Itoa(resource))
+			randomTime := time.Duration(rand.Int()%10+1) * time.Microsecond
+			time.Sleep(randomTime)
+			//log.Printf("RW - worker %d releasing %d\n", worker, resource)
+			lock.Unlock(strconv.Itoa(resource))
+		}
+	}
+}
+
+func doRWork(worker int, resource int, grp *stop.Group) {
+	for {
+		select {
+		case <-grp.Ch():
+			grp.Done()
+			return
+		default:
+			//log.Printf("R  - worker %d doing work on resource %d\n", worker, resource)
+			lock.RLock(strconv.Itoa(resource))
+			randomTime := time.Duration(rand.Int()%10+1) * time.Microsecond
+			time.Sleep(randomTime)
+			//log.Printf("R  - worker %d releasing %d\n", worker, resource)
+			lock.RUnlock(strconv.Itoa(resource))
+		}
+	}
+}
diff --git a/store/disk.go b/store/disk.go
index f108204..1515135 100644
--- a/store/disk.go
+++ b/store/disk.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/locks"
 	"github.com/lbryio/reflector.go/shared"
 	"github.com/lbryio/reflector.go/store/speedwalk"
 	log "github.com/sirupsen/logrus"
@@ -28,6 +29,7 @@ type DiskStore struct {
 	initialized bool
 
 	concurrentChecks atomic.Int32
+	lock             locks.MultipleLock
 }
 
 const maxConcurrentChecks = 3
@@ -37,6 +39,7 @@ func NewDiskStore(dir string, prefixLength int) *DiskStore {
 	return &DiskStore{
 		blobDir:      dir,
 		prefixLength: prefixLength,
+		lock:         locks.NewMultipleLock(),
 	}
 }
 
@@ -47,6 +50,8 @@ func (d *DiskStore) Name() string { return nameDisk }
 
 // Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
 func (d *DiskStore) Has(hash string) (bool, error) {
+	d.lock.RLock(hash)
+	defer d.lock.RUnlock(hash)
 	err := d.initOnce()
 	if err != nil {
 		return false, err
@@ -64,6 +69,8 @@ func (d *DiskStore) Has(hash string) (bool, error) {
 
 // Get returns the blob or an error if the blob doesn't exist.
 func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	d.lock.RLock(hash)
+	defer d.lock.RUnlock(hash)
 	start := time.Now()
 	err := d.initOnce()
 	if err != nil {
@@ -101,6 +108,8 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 
 // Put stores the blob on disk
 func (d *DiskStore) Put(hash string, blob stream.Blob) error {
+	d.lock.Lock(hash)
+	defer d.lock.Unlock(hash)
 	start := time.Now()
 	defer func() {
 		if time.Since(start) > 100*time.Millisecond {
@@ -141,11 +150,15 @@ reading after writing: hash match: %t`, hash, matchesBeforeWriting, err == nil,
 
 // PutSD stores the sd blob on the disk
 func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
+	d.lock.Lock(hash)
+	defer d.lock.Unlock(hash)
 	return d.Put(hash, blob)
 }
 
 // Delete deletes the blob from the store
 func (d *DiskStore) Delete(hash string) error {
+	d.lock.Lock(hash)
+	defer d.lock.Unlock(hash)
 	err := d.initOnce()
 	if err != nil {
 		return err

From c1caf1938c35082fe18920f59cbabb276297fba4 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 20 May 2021 20:41:47 -0400
Subject: [PATCH 43/51] Add queue to prevent writing too many files at once.

---
 store/disk.go | 37 +++++++++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/store/disk.go b/store/disk.go
index 1515135..e52731a 100644
--- a/store/disk.go
+++ b/store/disk.go
@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"runtime"
 	"time"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
@@ -18,6 +19,23 @@ import (
 	"go.uber.org/atomic"
 )
 
+func init() {
+	writeCh = make(chan writeRequest)
+	for i := 0; i < runtime.NumCPU(); i++ {
+		go func() {
+			select {
+			case r := <-writeCh:
+				err := ioutil.WriteFile(r.filename, r.data, r.perm)
+				if err != nil {
+					log.Errorf("could not write file %s to disk, failed with error: %s", r.filename, err.Error())
+				}
+			}
+		}()
+	}
+}
+
+var writeCh chan writeRequest
+
 // DiskStore stores blobs on a local disk
 type DiskStore struct {
 	// the location of blobs on disk
@@ -128,10 +146,7 @@ func (d *DiskStore) Put(hash string, blob stream.Blob) error {
 	hashBytes := sha512.Sum384(blob)
 	readHash := hex.EncodeToString(hashBytes[:])
 	matchesBeforeWriting := readHash == hash
-	err = ioutil.WriteFile(d.path(hash), blob, 0644)
-	if err != nil {
-		log.Errorf("Error saving to disk: %s", err.Error())
-	}
+	writeFile(d.path(hash), blob, 0644)
 	readBlob, err := ioutil.ReadFile(d.path(hash))
 	matchesAfterReading := false
 	if err != nil {
@@ -215,7 +230,21 @@ func (d *DiskStore) initOnce() error {
 	return nil
 }
 
+type writeRequest struct {
+	filename string
+	data     []byte
+	perm     os.FileMode
+}
+
 // Shutdown shuts down the store gracefully
 func (d *DiskStore) Shutdown() {
 	return
 }
+
+func writeFile(filename string, data []byte, perm os.FileMode) {
+	writeCh <- writeRequest{
+		filename: filename,
+		data:     data,
+		perm:     perm,
+	}
+}

From 76ece1e1178377373093dba8ae4b8e9ea23cfdc7 Mon Sep 17 00:00:00 2001
From: Mark Beamer Jr <markbeamerjr@gmail.com>
Date: Thu, 20 May 2021 20:43:01 -0400
Subject: [PATCH 44/51] Add queue to prevent writing too many files at once.

---
 store/disk.go | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/store/disk.go b/store/disk.go
index e52731a..7123b65 100644
--- a/store/disk.go
+++ b/store/disk.go
@@ -23,11 +23,13 @@ func init() {
 	writeCh = make(chan writeRequest)
 	for i := 0; i < runtime.NumCPU(); i++ {
 		go func() {
-			select {
-			case r := <-writeCh:
-				err := ioutil.WriteFile(r.filename, r.data, r.perm)
-				if err != nil {
-					log.Errorf("could not write file %s to disk, failed with error: %s", r.filename, err.Error())
+			for {
+				select {
+				case r := <-writeCh:
+					err := ioutil.WriteFile(r.filename, r.data, r.perm)
+					if err != nil {
+						log.Errorf("could not write file %s to disk, failed with error: %s", r.filename, err.Error())
+					}
 				}
 			}
 		}()

From a7086a00f3a5afa9b164db59b62078550a17071f Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Fri, 21 May 2021 05:49:02 +0200
Subject: [PATCH 45/51] add http server/client

---
 cmd/reflector.go            |  12 +++
 go.mod                      |   1 +
 go.sum                      |  29 +++++++
 internal/metrics/metrics.go |   5 ++
 peer/store.go               |   9 ++-
 server/http/routes.go       |  53 +++++++++++++
 server/http/server.go       |  68 ++++++++++++++++
 store/disk.go               |  22 ------
 store/http.go               | 152 ++++++++++++++++++++++++++++++++++++
 store/singleflight.go       |   4 +
 10 files changed, 332 insertions(+), 23 deletions(-)
 create mode 100644 server/http/routes.go
 create mode 100644 server/http/server.go
 create mode 100644 store/http.go

diff --git a/cmd/reflector.go b/cmd/reflector.go
index d2acfed..087c918 100644
--- a/cmd/reflector.go
+++ b/cmd/reflector.go
@@ -16,6 +16,7 @@ import (
 	"github.com/lbryio/reflector.go/peer"
 	"github.com/lbryio/reflector.go/peer/http3"
 	"github.com/lbryio/reflector.go/reflector"
+	"github.com/lbryio/reflector.go/server/http"
 	"github.com/lbryio/reflector.go/store"
 
 	"github.com/lbryio/lbry.go/v2/stream"
@@ -28,6 +29,7 @@ import (
 var (
 	tcpPeerPort                 int
 	http3PeerPort               int
+	httpPort                    int
 	receiverPort                int
 	metricsPort                 int
 	disableUploads              bool
@@ -57,6 +59,7 @@ func init() {
 	cmd.Flags().StringVar(&WasabiEndpoint, "wasabi-endpoint", "", "Wasabi edge endpoint for standard HTTP retrieval")
 	cmd.Flags().IntVar(&tcpPeerPort, "tcp-peer-port", 5567, "The port reflector will distribute content from")
 	cmd.Flags().IntVar(&http3PeerPort, "http3-peer-port", 5568, "The port reflector will distribute content from over HTTP3 protocol")
+	cmd.Flags().IntVar(&httpPort, "http-port", 5569, "The port reflector will distribute content from over HTTP protocol")
 	cmd.Flags().IntVar(&receiverPort, "receiver-port", 5566, "The port reflector will receive content from")
 	cmd.Flags().IntVar(&metricsPort, "metrics-port", 2112, "The port reflector will use for metrics")
 	cmd.Flags().IntVar(&requestQueueSize, "request-queue-size", 200, "How many concurrent requests should be submitted to upstream")
@@ -105,6 +108,13 @@ func reflectorCmd(cmd *cobra.Command, args []string) {
 	}
 	defer http3PeerServer.Shutdown()
 
+	httpServer := http.NewServer(outerStore)
+	err = httpServer.Start(":" + strconv.Itoa(httpPort))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer httpServer.Shutdown()
+
 	metricsServer := metrics.NewServer(":"+strconv.Itoa(metricsPort), "/metrics")
 	metricsServer.Start()
 	defer metricsServer.Shutdown()
@@ -133,6 +143,8 @@ func setupStore() store.BlobStore {
 				Address: proxyAddress + ":" + proxyPort,
 				Timeout: 30 * time.Second,
 			})
+		case "http":
+			s = store.NewHttpStore(proxyAddress + ":" + proxyPort)
 		default:
 			log.Fatalf("protocol is not recognized: %s", proxyProtocol)
 		}
diff --git a/go.mod b/go.mod
index 82eaa1e..08137fa 100644
--- a/go.mod
+++ b/go.mod
@@ -13,6 +13,7 @@ require (
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
 	github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
 	github.com/davecgh/go-spew v1.1.1
+	github.com/gin-gonic/gin v1.7.1 // indirect
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/golang/protobuf v1.4.2
 	github.com/google/gops v0.3.18
diff --git a/go.sum b/go.sum
index 6af413e..810e719 100644
--- a/go.sum
+++ b/go.sum
@@ -89,6 +89,10 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
+github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-gonic/gin v1.7.1 h1:qC89GU3p8TvKWMAVhEpmpB2CIb1hnqt2UdKZaP93mS8=
+github.com/gin-gonic/gin v1.7.1/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg=
@@ -103,6 +107,13 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -123,6 +134,7 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -140,6 +152,7 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gops v0.3.18 h1:my259V+172PVFmduS2RAsq4FKH+HjKqdh7pLr17Ot8c=
 github.com/google/gops v0.3.18/go.mod h1:Pfp8hWGIFdV/7rY9/O/U5WgdjYQXf/GiEK4NVuVd2ZE=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -226,6 +239,8 @@ github.com/johntdyer/slackrus v0.0.0-20180518184837-f7aae3243a07/go.mod h1:j1kV/
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jteeuwen/go-bindata v3.0.7+incompatible/go.mod h1:JVvhzYOiGBnFSYRyV00iY8q7/0PThjIYav1p9h5dmKs=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@@ -265,6 +280,8 @@ github.com/lbryio/ozzo-validation v0.0.0-20170323141101-d1008ad1fd04/go.mod h1:f
 github.com/lbryio/types v0.0.0-20190422033210-321fb2abda9c/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
 github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386 h1:JOQkGpeCM9FWkEHRx+kRPqySPCXElNW1em1++7tVS4M=
 github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386/go.mod h1:CG3wsDv5BiVYQd5i1Jp7wGsaVyjZTJshqXeWMVKsISE=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lucas-clemente/quic-go v0.20.1 h1:hb5m76V8QS/8Nw/suHvXqo3BMHAozvIkcnzpJdpanSk=
 github.com/lucas-clemente/quic-go v0.20.1/go.mod h1:fZq/HUDIM+mW6X6wtzORjC0E/WDBMKe5Hf9bgjISwLk=
@@ -285,6 +302,8 @@ github.com/marten-seemann/qtls-go1-16 v0.1.3 h1:XEZ1xGorVy9u+lJq+WXNE+hiqRYLNvJG
 github.com/marten-seemann/qtls-go1-16 v0.1.3/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -300,7 +319,11 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
 github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
@@ -445,6 +468,10 @@ github.com/tklauser/numcpus v0.2.1/go.mod h1:9aU+wOc6WjUIZEwWMP62PL/41d65P+iks1g
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
+github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
+github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
+github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
+github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
 github.com/volatiletech/inflect v0.0.0-20170731032912-e7201282ae8d h1:gI4/tqP6lCY5k6Sg+4k9qSoBXmPwG+xXgMpK7jivD4M=
@@ -566,6 +593,7 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -679,6 +707,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
index 4c30606..ca65021 100644
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@@ -194,6 +194,11 @@ var (
 		Name:      "udp_in_bytes",
 		Help:      "Total number of bytes downloaded through UDP",
 	})
+	MtrInBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "http_in_bytes",
+		Help:      "Total number of bytes downloaded through HTTP",
+	})
 	MtrOutBytesUdp = promauto.NewCounter(prometheus.CounterOpts{
 		Namespace: ns,
 		Name:      "udp_out_bytes",
diff --git a/peer/store.go b/peer/store.go
index 3068794..689d1c0 100644
--- a/peer/store.go
+++ b/peer/store.go
@@ -1,11 +1,13 @@
 package peer
 
 import (
+	"strings"
 	"time"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lbryio/reflector.go/shared"
+	"github.com/lbryio/reflector.go/store"
 )
 
 // Store is a blob store that gets blobs from a peer.
@@ -51,7 +53,12 @@ func (p *Store) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 		return nil, shared.NewBlobTrace(time.Since(start), p.Name()), err
 	}
 	defer c.Close()
-	return c.GetBlob(hash)
+	blob, trace, err := c.GetBlob(hash)
+	if err != nil && strings.Contains(err.Error(), "blob not found") {
+		return nil, trace, store.ErrBlobNotFound
+	}
+
+	return blob, trace, err
 }
 
 // Put is not supported
diff --git a/server/http/routes.go b/server/http/routes.go
new file mode 100644
index 0000000..eee696f
--- /dev/null
+++ b/server/http/routes.go
@@ -0,0 +1,53 @@
+package http
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/reflector.go/store"
+	log "github.com/sirupsen/logrus"
+)
+
+func (s *Server) getBlob(c *gin.Context) {
+	hash := c.Query("hash")
+	blob, trace, err := s.store.Get(hash)
+	if err != nil {
+		serialized, serializeErr := trace.Serialize()
+		if serializeErr != nil {
+			_ = c.AbortWithError(http.StatusInternalServerError, errors.Prefix(serializeErr.Error(), err))
+			return
+		}
+		c.Header("Via", serialized)
+
+		if errors.Is(err, store.ErrBlobNotFound) {
+			log.Errorf("wtf: %s", err.Error())
+			c.AbortWithStatus(http.StatusNotFound)
+			return
+		}
+		_ = c.AbortWithError(http.StatusInternalServerError, err)
+		return
+	}
+	serialized, err := trace.Serialize()
+	if err != nil {
+		_ = c.AbortWithError(http.StatusInternalServerError, err)
+		return
+	}
+	c.Header("Via", serialized)
+	c.Header("Content-Disposition", "filename="+hash)
+	c.Data(http.StatusOK, "application/octet-stream", blob)
+}
+
+func (s *Server) hasBlob(c *gin.Context) {
+	hash := c.Query("hash")
+	has, err := s.store.Has(hash)
+	if err != nil {
+		_ = c.AbortWithError(http.StatusInternalServerError, err)
+		return
+	}
+	if has {
+		c.Status(http.StatusNoContent)
+		return
+	}
+	c.Status(http.StatusNotFound)
+}
diff --git a/server/http/server.go b/server/http/server.go
new file mode 100644
index 0000000..adc104f
--- /dev/null
+++ b/server/http/server.go
@@ -0,0 +1,68 @@
+package http
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/reflector.go/store"
+	log "github.com/sirupsen/logrus"
+)
+
+// Server is an instance of a peer server that houses the listener and store.
+type Server struct {
+	store store.BlobStore
+	grp   *stop.Group
+}
+
+// NewServer returns an initialized Server pointer.
+func NewServer(store store.BlobStore) *Server {
+	return &Server{
+		store: store,
+		grp:   stop.New(),
+	}
+}
+
+// Shutdown gracefully shuts down the peer server.
+func (s *Server) Shutdown() {
+	log.Debug("shutting down HTTP server")
+	s.grp.StopAndWait()
+	log.Debug("HTTP server stopped")
+}
+
+// Start starts the server listener to handle connections.
+func (s *Server) Start(address string) error {
+	gin.SetMode(gin.ReleaseMode)
+	router := gin.Default()
+	router.GET("/blob", s.getBlob)
+	router.HEAD("/blob", s.hasBlob)
+	srv := &http.Server{
+		Addr:    address,
+		Handler: router,
+	}
+	go s.listenForShutdown(srv)
+	// Initializing the server in a goroutine so that
+	// it won't block the graceful shutdown handling below
+	s.grp.Add(1)
+	go func() {
+		defer s.grp.Done()
+		log.Println("HTTP server listening on " + address)
+		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+			log.Fatalf("listen: %s\n", err)
+		}
+	}()
+	return nil
+}
+
+func (s *Server) listenForShutdown(listener *http.Server) {
+	<-s.grp.Ch()
+	// The context is used to inform the server it has 5 seconds to finish
+	// the request it is currently handling
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if err := listener.Shutdown(ctx); err != nil {
+		log.Fatal("Server forced to shutdown:", err)
+	}
+}
diff --git a/store/disk.go b/store/disk.go
index 7123b65..fb5e326 100644
--- a/store/disk.go
+++ b/store/disk.go
@@ -130,12 +130,6 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 func (d *DiskStore) Put(hash string, blob stream.Blob) error {
 	d.lock.Lock(hash)
 	defer d.lock.Unlock(hash)
-	start := time.Now()
-	defer func() {
-		if time.Since(start) > 100*time.Millisecond {
-			log.Infof("it took %s to write %s", time.Since(start), hash)
-		}
-	}()
 	err := d.initOnce()
 	if err != nil {
 		return err
@@ -145,23 +139,7 @@ func (d *DiskStore) Put(hash string, blob stream.Blob) error {
 	if err != nil {
 		return err
 	}
-	hashBytes := sha512.Sum384(blob)
-	readHash := hex.EncodeToString(hashBytes[:])
-	matchesBeforeWriting := readHash == hash
 	writeFile(d.path(hash), blob, 0644)
-	readBlob, err := ioutil.ReadFile(d.path(hash))
-	matchesAfterReading := false
-	if err != nil {
-		log.Errorf("for some fucking reasons I can't read the blob I just wrote %s", err.Error())
-	} else {
-		hashBytes = sha512.Sum384(readBlob)
-		readHash = hex.EncodeToString(hashBytes[:])
-		matchesAfterReading = readHash == hash
-	}
-
-	log.Infof(`writing %s to disk: hash match: %t, error: %t
-reading after writing: hash match: %t`, hash, matchesBeforeWriting, err == nil, matchesAfterReading)
-
 	return errors.Err(err)
 }
 
diff --git a/store/http.go b/store/http.go
new file mode 100644
index 0000000..8a934d7
--- /dev/null
+++ b/store/http.go
@@ -0,0 +1,152 @@
+package store
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+	"github.com/lbryio/reflector.go/internal/metrics"
+	"github.com/lbryio/reflector.go/shared"
+)
+
+// NoopStore is a store that does nothing
+type HttpStore struct {
+	upstream   string
+	httpClient *http.Client
+}
+
+func NewHttpStore(upstream string) *HttpStore {
+	return &HttpStore{
+		upstream:   upstream,
+		httpClient: getClient(),
+	}
+}
+
+const nameHttp = "http"
+
+func (n *HttpStore) Name() string { return nameNoop }
+func (n *HttpStore) Has(hash string) (bool, error) {
+	url := n.upstream + "/blob?hash=" + hash
+
+	req, err := http.NewRequest("HEAD", url, nil)
+	if err != nil {
+		return false, errors.Err(err)
+	}
+
+	res, err := n.httpClient.Do(req)
+	if err != nil {
+		return false, errors.Err(err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode == http.StatusNotFound {
+		return false, nil
+	}
+	if res.StatusCode == http.StatusNoContent {
+		return true, nil
+	}
+	var body []byte
+	if res.Body != nil {
+		body, _ = ioutil.ReadAll(res.Body)
+	}
+	return false, errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
+}
+
+func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
+	start := time.Now()
+	url := n.upstream + "/blob?hash=" + hash
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
+	}
+
+	res, err := n.httpClient.Do(req)
+	if err != nil {
+		return nil, shared.NewBlobTrace(time.Since(start), n.Name()), errors.Err(err)
+	}
+	defer res.Body.Close()
+	tmp := getBuffer()
+	defer putBuffer(tmp)
+	serialized := res.Header.Get("Via")
+	trace := shared.NewBlobTrace(time.Since(start), n.Name())
+	if serialized != "" {
+		parsedTrace, err := shared.Deserialize(serialized)
+		if err != nil {
+			return nil, shared.NewBlobTrace(time.Since(start), n.Name()), err
+		}
+		trace = *parsedTrace
+	}
+
+	if res.StatusCode == http.StatusNotFound {
+		return nil, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
+	}
+	if res.StatusCode == http.StatusOK {
+		written, err := io.Copy(tmp, res.Body)
+		if err != nil {
+			return nil, trace.Stack(time.Since(start), n.Name()), errors.Err(err)
+		}
+
+		blob := make([]byte, written)
+		copy(blob, tmp.Bytes())
+		metrics.MtrInBytesHttp.Add(float64(len(blob)))
+		return blob, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
+	}
+	var body []byte
+	if res.Body != nil {
+		body, _ = ioutil.ReadAll(res.Body)
+	}
+
+	return nil, trace.Stack(time.Since(start), n.Name()), errors.Err("upstream error. Status code: %d (%s)", res.StatusCode, string(body))
+}
+
+func (n *HttpStore) Put(string, stream.Blob) error {
+	return shared.ErrNotImplemented
+}
+func (n *HttpStore) PutSD(string, stream.Blob) error {
+	return shared.ErrNotImplemented
+}
+func (n *HttpStore) Delete(string) error {
+	return shared.ErrNotImplemented
+}
+func (n *HttpStore) Shutdown() { return }
+
+// buffer pool to reduce GC
+// https://www.captaincodeman.com/2017/06/02/golang-buffer-pool-gotcha
+var buffers = sync.Pool{
+	// New is called when a new instance is needed
+	New: func() interface{} {
+		buf := make([]byte, 0, stream.MaxBlobSize)
+		return bytes.NewBuffer(buf)
+	},
+}
+
+// getBuffer fetches a buffer from the pool
+func getBuffer() *bytes.Buffer {
+	return buffers.Get().(*bytes.Buffer)
+}
+
+// putBuffer returns a buffer to the pool
+func putBuffer(buf *bytes.Buffer) {
+	buf.Reset()
+	buffers.Put(buf)
+}
+
+// getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
+func getClient() *http.Client {
+	// Customize the Transport to have larger connection pool
+	defaultRoundTripper := http.DefaultTransport
+	defaultTransportPointer := defaultRoundTripper.(*http.Transport)
+
+	defaultTransport := *defaultTransportPointer // dereference it to get a copy of the struct that the pointer points to
+	defaultTransport.MaxIdleConns = 100
+	defaultTransport.DisableCompression = true
+	defaultTransport.MaxIdleConnsPerHost = 100
+	defaultTransport.ReadBufferSize = stream.MaxBlobSize + 1024*10 //add an extra few KBs to make sure it fits the extra information
+
+	return &http.Client{Transport: &defaultTransport}
+}
diff --git a/store/singleflight.go b/store/singleflight.go
index 6de91f4..02337a2 100644
--- a/store/singleflight.go
+++ b/store/singleflight.go
@@ -3,6 +3,7 @@ package store
 import (
 	"time"
 
+	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/shared"
 
@@ -46,6 +47,9 @@ func (s *singleflightStore) Get(hash string) (stream.Blob, shared.BlobTrace, err
 	if err != nil {
 		return nil, shared.NewBlobTrace(time.Since(start), s.Name()), err
 	}
+	if gr == nil {
+		return nil, shared.NewBlobTrace(time.Since(start), s.Name()), errors.Err("getter response is nil")
+	}
 	rsp := gr.(getterResponse)
 	return rsp.blob, rsp.stack, nil
 }

From 5cc1e84adb77756bfa83b5e32e44ad7aa38ebf45 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Fri, 21 May 2021 05:53:13 +0200
Subject: [PATCH 46/51] remove locks causing deadlocks

---
 locks/multiplelock.go      | 131 -------------------------------------
 locks/multiplelock_test.go |  56 ----------------
 store/disk.go              |  13 ----
 store/http.go              |   2 +-
 4 files changed, 1 insertion(+), 201 deletions(-)
 delete mode 100644 locks/multiplelock.go
 delete mode 100644 locks/multiplelock_test.go

diff --git a/locks/multiplelock.go b/locks/multiplelock.go
deleted file mode 100644
index c87b645..0000000
--- a/locks/multiplelock.go
+++ /dev/null
@@ -1,131 +0,0 @@
-package locks
-
-import (
-	"fmt"
-	"sync"
-	"time"
-)
-
-// MultipleLock is the main interface for multiLock based on key
-type MultipleLock interface {
-	Lock(key string)
-	RLock(key string)
-	Unlock(key string)
-	RUnlock(key string)
-}
-
-func NewMultipleLock() MultipleLock {
-	return &multiLock{
-		locks: make(map[string]*itemLock),
-		mu:    sync.Mutex{},
-	}
-}
-
-type itemLock struct {
-	lk  *sync.RWMutex
-	cnt int64
-}
-
-// multiLock is an optimized locking system per locking key
-type multiLock struct {
-	locks map[string]*itemLock
-	mu    sync.Mutex // synchronize reads/writes to locks map
-}
-
-func debugPrint(format string, a ...interface{}) {
-	debugEnabled := false
-	if debugEnabled {
-		a = append(a, time.Now().Format("04:05.000000"))
-		fmt.Printf(format+" at %s\n", a...)
-	}
-}
-
-func (ml *multiLock) Lock(key string) {
-	debugPrint("mutex requested %s", key)
-	ml.mu.Lock()
-	debugPrint("mutex acquired %s", key)
-
-	itmLock, exists := ml.locks[key]
-	if !exists {
-		debugPrint("new lock created %s", key)
-		itmLock = &itemLock{&sync.RWMutex{}, 0}
-		ml.locks[key] = itmLock
-	}
-	itmLock.cnt++
-	debugPrint("releasing mutex %s", key)
-	ml.mu.Unlock()
-
-	debugPrint("Lock requested %s", key)
-	itmLock.lk.Lock()
-	debugPrint("Lock acquired %s", key)
-}
-
-func (ml *multiLock) RLock(key string) {
-	debugPrint("mutex requested %s", key)
-	ml.mu.Lock()
-	debugPrint("mutex acquired %s", key)
-
-	itmLock, exists := ml.locks[key]
-	if !exists {
-		debugPrint("new lock created for %s", key)
-		itmLock = &itemLock{&sync.RWMutex{}, 0}
-		ml.locks[key] = itmLock
-	}
-	itmLock.cnt++
-	debugPrint("releasing mutex %s", key)
-	ml.mu.Unlock()
-
-	debugPrint("RLock requested %s", key)
-	itmLock.lk.RLock()
-	debugPrint("RLock acquired %s", key)
-}
-
-func (ml *multiLock) Unlock(key string) {
-	debugPrint("mutex requested %s", key)
-	ml.mu.Lock()
-	debugPrint("mutex acquired %s", key)
-
-	itmLock, exists := ml.locks[key]
-	if !exists {
-		panic("sync Unlock of non existent lock!!")
-	}
-
-	debugPrint("Unlock %s", key)
-	itmLock.lk.Unlock()
-	itmLock.cnt--
-	if itmLock.cnt == 0 {
-		debugPrint("delete lock %s", key)
-		delete(ml.locks, key)
-	}
-	if itmLock.cnt < 0 {
-		panic("sync Unlock of free Lock!!")
-	}
-
-	debugPrint("releasing mutex %s", key)
-	ml.mu.Unlock()
-}
-
-func (ml *multiLock) RUnlock(key string) {
-	debugPrint("mutex requested %s", key)
-	ml.mu.Lock()
-	debugPrint("mutex acquired %s", key)
-
-	itmLock, exists := ml.locks[key]
-	if !exists {
-		panic("sync Unlock of non existent lock!!")
-	}
-
-	debugPrint("RUnlock %s", key)
-	itmLock.lk.RUnlock()
-	itmLock.cnt--
-	if itmLock.cnt == 0 {
-		debugPrint("delete lock %s", key)
-		delete(ml.locks, key)
-	}
-	if itmLock.cnt < 0 {
-		panic("sync Unlock of free Lock!!")
-	}
-
-	debugPrint("releasing mutex %s", key)
-	ml.mu.Unlock()
-}
diff --git a/locks/multiplelock_test.go b/locks/multiplelock_test.go
deleted file mode 100644
index a7a42c7..0000000
--- a/locks/multiplelock_test.go
+++ /dev/null
@@ -1,56 +0,0 @@
-package locks
-
-import (
-	"math/rand"
-	"strconv"
-	"testing"
-	"time"
-
-	"github.com/lbryio/lbry.go/v2/extras/stop"
-)
-
-var lock = NewMultipleLock()
-
-func TestNewMultipleLock(t *testing.T) {
-	grp := stop.New()
-	for i := 0; i < 100; i++ {
-		grp.Add(2)
-		go doRWWork(i, i%10, grp)
-		go doRWork(i, i%10, grp)
-	}
-	time.Sleep(5 * time.Second)
-	grp.StopAndWait()
-}
-func doRWWork(worker int, resource int, grp *stop.Group) {
-	for {
-		select {
-		case <-grp.Ch():
-			grp.Done()
-			return
-		default:
-			//log.Printf("RW - worker %d doing work on resource %d\n", worker, resource)
-			lock.Lock(strconv.Itoa(resource))
-			randomTime := time.Duration(rand.Int()%10+1) * time.Microsecond
-			time.Sleep(randomTime)
-			//log.Printf("RW - worker %d releasing %d\n", worker, resource)
-			lock.Unlock(strconv.Itoa(resource))
-		}
-	}
-}
-
-func doRWork(worker int, resource int, grp *stop.Group) {
-	for {
-		select {
-		case <-grp.Ch():
-			grp.Done()
-			return
-		default:
-			//log.Printf("R  - worker %d doing work on resource %d\n", worker, resource)
-			lock.RLock(strconv.Itoa(resource))
-			randomTime := time.Duration(rand.Int()%10+1) * time.Microsecond
-			time.Sleep(randomTime)
-			//log.Printf("R  - worker %d releasing %d\n", worker, resource)
-			lock.RUnlock(strconv.Itoa(resource))
-		}
-	}
-}
diff --git a/store/disk.go b/store/disk.go
index fb5e326..5ef7006 100644
--- a/store/disk.go
+++ b/store/disk.go
@@ -12,7 +12,6 @@ import (
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/lbry.go/v2/stream"
-	"github.com/lbryio/reflector.go/locks"
 	"github.com/lbryio/reflector.go/shared"
 	"github.com/lbryio/reflector.go/store/speedwalk"
 	log "github.com/sirupsen/logrus"
@@ -49,7 +48,6 @@ type DiskStore struct {
 	initialized bool
 
 	concurrentChecks atomic.Int32
-	lock             locks.MultipleLock
 }
 
 const maxConcurrentChecks = 3
@@ -59,7 +57,6 @@ func NewDiskStore(dir string, prefixLength int) *DiskStore {
 	return &DiskStore{
 		blobDir:      dir,
 		prefixLength: prefixLength,
-		lock:         locks.NewMultipleLock(),
 	}
 }
 
@@ -70,8 +67,6 @@ func (d *DiskStore) Name() string { return nameDisk }
 
 // Has returns T/F or Error if it the blob stored already. It will error with any IO disk error.
 func (d *DiskStore) Has(hash string) (bool, error) {
-	d.lock.RLock(hash)
-	defer d.lock.RUnlock(hash)
 	err := d.initOnce()
 	if err != nil {
 		return false, err
@@ -89,8 +84,6 @@ func (d *DiskStore) Has(hash string) (bool, error) {
 
 // Get returns the blob or an error if the blob doesn't exist.
 func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
-	d.lock.RLock(hash)
-	defer d.lock.RUnlock(hash)
 	start := time.Now()
 	err := d.initOnce()
 	if err != nil {
@@ -128,8 +121,6 @@ func (d *DiskStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 
 // Put stores the blob on disk
 func (d *DiskStore) Put(hash string, blob stream.Blob) error {
-	d.lock.Lock(hash)
-	defer d.lock.Unlock(hash)
 	err := d.initOnce()
 	if err != nil {
 		return err
@@ -145,15 +136,11 @@ func (d *DiskStore) Put(hash string, blob stream.Blob) error {
 
 // PutSD stores the sd blob on the disk
 func (d *DiskStore) PutSD(hash string, blob stream.Blob) error {
-	d.lock.Lock(hash)
-	defer d.lock.Unlock(hash)
 	return d.Put(hash, blob)
 }
 
 // Delete deletes the blob from the store
 func (d *DiskStore) Delete(hash string) error {
-	d.lock.Lock(hash)
-	defer d.lock.Unlock(hash)
 	err := d.initOnce()
 	if err != nil {
 		return err
diff --git a/store/http.go b/store/http.go
index 8a934d7..34461de 100644
--- a/store/http.go
+++ b/store/http.go
@@ -94,7 +94,7 @@ func (n *HttpStore) Get(hash string) (stream.Blob, shared.BlobTrace, error) {
 		blob := make([]byte, written)
 		copy(blob, tmp.Bytes())
 		metrics.MtrInBytesHttp.Add(float64(len(blob)))
-		return blob, trace.Stack(time.Since(start), n.Name()), ErrBlobNotFound
+		return blob, trace.Stack(time.Since(start), n.Name()), nil
 	}
 	var body []byte
 	if res.Body != nil {

From 1ec218483397f6e472030e4ab9b5b7d86a4217eb Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Fri, 21 May 2021 17:58:33 +0200
Subject: [PATCH 47/51] upgrade singleflight

http store fix
---
 go.mod        | 4 ++--
 go.sum        | 5 +++--
 store/http.go | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/go.mod b/go.mod
index 08137fa..0bc0bd2 100644
--- a/go.mod
+++ b/go.mod
@@ -13,7 +13,7 @@ require (
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
 	github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2
 	github.com/davecgh/go-spew v1.1.1
-	github.com/gin-gonic/gin v1.7.1 // indirect
+	github.com/gin-gonic/gin v1.7.1
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/golang/protobuf v1.4.2
 	github.com/google/gops v0.3.18
@@ -41,7 +41,7 @@ require (
 	github.com/volatiletech/null v8.0.0+incompatible
 	go.uber.org/atomic v1.5.1
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
-	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4 // indirect
 	google.golang.org/appengine v1.6.2 // indirect
 )
diff --git a/go.sum b/go.sum
index 810e719..64c9a07 100644
--- a/go.sum
+++ b/go.sum
@@ -107,6 +107,7 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-ozzo/ozzo-validation v3.6.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
@@ -568,8 +569,9 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -597,7 +599,6 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201231184435-2d18734c6014 h1:joucsQqXmyBVxViHCPFjG3hx8JzIFSaym3l3MM/Jsdg=
 golang.org/x/sys v0.0.0-20201231184435-2d18734c6014/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210217105451-b926d437f341 h1:2/QtM1mL37YmcsT8HaDNHDgTqqFVw+zr8UzMiBVLzYU=
 golang.org/x/sys v0.0.0-20210217105451-b926d437f341/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/store/http.go b/store/http.go
index 34461de..11f506d 100644
--- a/store/http.go
+++ b/store/http.go
@@ -22,7 +22,7 @@ type HttpStore struct {
 
 func NewHttpStore(upstream string) *HttpStore {
 	return &HttpStore{
-		upstream:   upstream,
+		upstream:   "http://" + upstream,
 		httpClient: getClient(),
 	}
 }

From bc889001bbe924ff1dad13852b66352b070e9ea6 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Fri, 21 May 2021 18:05:37 +0200
Subject: [PATCH 48/51] update lbry.go dep

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 0bc0bd2..31386d5 100644
--- a/go.mod
+++ b/go.mod
@@ -27,7 +27,7 @@ require (
 	github.com/karrick/godirwalk v1.16.1
 	github.com/lbryio/chainquery v1.9.0
 	github.com/lbryio/lbry.go v1.1.2 // indirect
-	github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d
+	github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3
 	github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
 	github.com/lucas-clemente/quic-go v0.20.1
 	github.com/phayes/freeport v0.0.0-20171002185219-e27662a4a9d6
diff --git a/go.sum b/go.sum
index 64c9a07..351f737 100644
--- a/go.sum
+++ b/go.sum
@@ -271,8 +271,8 @@ github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c/go.mod h1:muH7wpU
 github.com/lbryio/lbry.go v1.1.1-0.20190825202001-8fa28d3d656f/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
 github.com/lbryio/lbry.go v1.1.2 h1:Dyxc+glT/rVWJwHfIf7vjlPYYbjzrQz5ARmJd5Hp69c=
 github.com/lbryio/lbry.go v1.1.2/go.mod h1:JtyI30bU51rm0LZ/po3mQuzf++14OWb6kR/6mMRAmKU=
-github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d h1:VUaOZ3cbCe7gfpycN/srCOk6U2bBS9NZHEz9RiRxd4E=
-github.com/lbryio/lbry.go/v2 v2.7.2-0.20210412222918-ed51ece75c3d/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk=
+github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3 h1:hkVViG8qbOKJMf/M6mN+Hrtns4j55IU2dRavpJZWbxw=
+github.com/lbryio/lbry.go/v2 v2.7.2-0.20210416195322-6516df1418e3/go.mod h1:I1q8W9fwU+t0IWNiprPgE1SorWQwcO6ser0nzP3L5Pk=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19 h1:/zWD8dVIl7bV1TdJWqPqy9tpqixzX2Qxgit48h3hQcY=
 github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ=
 github.com/lbryio/lbryschema.go v0.0.0-20190428231007-c54836bca002/go.mod h1:dAzPCBj3CKKWBGYBZxK6tKBP5SCgY2tqd9SnQd/OyKo=

From df881e16b5b39cb8dee4d29a6517f48e7ccfa6d5 Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Fri, 21 May 2021 19:09:02 +0200
Subject: [PATCH 49/51] add metrics

---
 internal/metrics/metrics.go | 10 ++++++++++
 server/http/routes.go       |  6 ++++++
 store/http.go               |  5 +++--
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
index ca65021..b6d5a57 100644
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@@ -118,6 +118,11 @@ var (
 		Name:      "http3_blob_download_total",
 		Help:      "Total number of blobs downloaded from reflector through QUIC protocol",
 	})
+	HttpDownloadCount = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "http_blob_download_total",
+		Help:      "Total number of blobs downloaded from reflector through HTTP protocol",
+	})
 
 	CacheHitCount = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: ns,
@@ -204,6 +209,11 @@ var (
 		Name:      "udp_out_bytes",
 		Help:      "Total number of bytes streamed out through UDP",
 	})
+	MtrOutBytesHttp = promauto.NewCounter(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "http_out_bytes",
+		Help:      "Total number of bytes streamed out through UDP",
+	})
 	MtrInBytesReflector = promauto.NewCounter(prometheus.CounterOpts{
 		Namespace: ns,
 		Name:      "reflector_in_bytes",
diff --git a/server/http/routes.go b/server/http/routes.go
index eee696f..e9124a9 100644
--- a/server/http/routes.go
+++ b/server/http/routes.go
@@ -5,7 +5,10 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/store"
+
 	log "github.com/sirupsen/logrus"
 )
 
@@ -33,6 +36,9 @@ func (s *Server) getBlob(c *gin.Context) {
 		_ = c.AbortWithError(http.StatusInternalServerError, err)
 		return
 	}
+	metrics.MtrOutBytesHttp.Add(float64(len(blob)))
+	metrics.BlobDownloadCount.Inc()
+	metrics.HttpDownloadCount.Inc()
 	c.Header("Via", serialized)
 	c.Header("Content-Disposition", "filename="+hash)
 	c.Data(http.StatusOK, "application/octet-stream", blob)
diff --git a/store/http.go b/store/http.go
index 11f506d..f79612e 100644
--- a/store/http.go
+++ b/store/http.go
@@ -8,10 +8,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/lbryio/lbry.go/v2/stream"
 	"github.com/lbryio/reflector.go/internal/metrics"
 	"github.com/lbryio/reflector.go/shared"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
 )
 
 // NoopStore is a store that does nothing

From 9670bc14f88d835dca248dc030a7001dc773350b Mon Sep 17 00:00:00 2001
From: Niko Storni <niko@lbry.io>
Date: Fri, 21 May 2021 21:06:59 +0200
Subject: [PATCH 50/51] fix unsafe dereference

---
 store/http.go | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/store/http.go b/store/http.go
index f79612e..c24169a 100644
--- a/store/http.go
+++ b/store/http.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"sync"
 	"time"
@@ -140,14 +141,20 @@ func putBuffer(buf *bytes.Buffer) {
 // getClient gets an http client that's customized to be more performant when dealing with blobs of 2MB in size (most of our blobs)
 func getClient() *http.Client {
 	// Customize the Transport to have larger connection pool
-	defaultRoundTripper := http.DefaultTransport
-	defaultTransportPointer := defaultRoundTripper.(*http.Transport)
-
-	defaultTransport := *defaultTransportPointer // dereference it to get a copy of the struct that the pointer points to
-	defaultTransport.MaxIdleConns = 100
-	defaultTransport.DisableCompression = true
-	defaultTransport.MaxIdleConnsPerHost = 100
-	defaultTransport.ReadBufferSize = stream.MaxBlobSize + 1024*10 //add an extra few KBs to make sure it fits the extra information
+	defaultTransport := &http.Transport{
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		DisableCompression:    true,
+		MaxIdleConnsPerHost:   100,
+		ReadBufferSize:        stream.MaxBlobSize + 1024*10, //add an extra few KBs to make sure it fits the extra information
+	}
 
-	return &http.Client{Transport: &defaultTransport}
+	return &http.Client{Transport: defaultTransport}
 }

From 2a1557845da5e6e4acf06882d4976cd5f0e01092 Mon Sep 17 00:00:00 2001
From: Alex Grintsvayg <grin@lbry.com>
Date: Thu, 20 May 2021 17:15:45 -0400
Subject: [PATCH 51/51] wrap blob insertion in tx. fixes lbryio/lbry-sdk#3296

The problem is that inserting an sd blob with ~5k
blobs takes longer than 30 seconds. So the client
times out and retries the request. At that point,
reflector is not done inserting so it replies with
a smaller number of blobs than it should. The client
uploads that many blobs and marks the stream as
reflected. The remaining blobs never get uploaded.

Doing the insert inside a transaction should be
faster than doing 10k (2 per blob) inserts
independently.
---
 db/db.go              | 125 +++++++++++++++++++++---------------------
 db/interfaces.go      |  45 +++++++++++++++
 shared/shared_test.go |  22 ++++++--
 3 files changed, 125 insertions(+), 67 deletions(-)
 create mode 100644 db/interfaces.go

diff --git a/db/db.go b/db/db.go
index babe641..11f4d53 100644
--- a/db/db.go
+++ b/db/db.go
@@ -97,7 +97,7 @@ func (s *SQL) AddBlob(hash string, length int, isStored bool) error {
 		return errors.Err("not connected")
 	}
 
-	_, err := s.insertBlob(hash, length, isStored)
+	_, err := s.insertBlob(s.conn, hash, length, isStored)
 	return err
 }
 
@@ -163,7 +163,7 @@ func (s *SQL) insertBlobs(hashes []string) error {
 		//args = append(args, hash, true, stream.MaxBlobSize, dayAgo)
 	}
 	q = strings.TrimSuffix(q, ",")
-	_, err := s.exec(q)
+	_, err := s.exec(s.conn, q)
 	if err != nil {
 		return err
 	}
@@ -171,7 +171,7 @@ func (s *SQL) insertBlobs(hashes []string) error {
 	return nil
 }
 
-func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error) {
+func (s *SQL) insertBlob(ex Executor, hash string, length int, isStored bool) (int64, error) {
 	if length <= 0 {
 		return 0, errors.Err("length must be positive")
 	}
@@ -188,13 +188,13 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
 		q = "INSERT INTO blob_ (hash, is_stored, length) VALUES (" + qt.Qs(len(args)) + ") ON DUPLICATE KEY UPDATE is_stored = (is_stored or VALUES(is_stored))"
 	}
 
-	blobID, err := s.exec(q, args...)
+	blobID, err := s.exec(ex, q, args...)
 	if err != nil {
 		return 0, err
 	}
 
 	if blobID == 0 {
-		err = s.conn.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
+		err = ex.QueryRow("SELECT id FROM blob_ WHERE hash = ?", hash).Scan(&blobID)
 		if err != nil {
 			return 0, errors.Err(err)
 		}
@@ -203,7 +203,7 @@ func (s *SQL) insertBlob(hash string, length int, isStored bool) (int64, error)
 		}
 
 		if s.TrackAccess == TrackAccessBlobs {
-			err := s.touchBlobs([]uint64{uint64(blobID)})
+			err := s.touchBlobs(ex, []uint64{uint64(blobID)})
 			if err != nil {
 				return 0, errors.Err(err)
 			}
@@ -227,7 +227,7 @@ func (s *SQL) insertStream(hash string, sdBlobID int64) (int64, error) {
 		q = "INSERT IGNORE INTO stream (hash, sd_blob_id) VALUES (" + qt.Qs(len(args)) + ")"
 	}
 
-	streamID, err := s.exec(q, args...)
+	streamID, err := s.exec(s.conn, q, args...)
 	if err != nil {
 		return 0, errors.Err(err)
 	}
@@ -266,16 +266,16 @@ func (s *SQL) HasBlobs(hashes []string, touch bool) (map[string]bool, error) {
 
 	if touch {
 		if s.TrackAccess == TrackAccessBlobs {
-			s.touchBlobs(idsNeedingTouch)
+			_ = s.touchBlobs(s.conn, idsNeedingTouch)
 		} else if s.TrackAccess == TrackAccessStreams {
-			s.touchStreams(idsNeedingTouch)
+			_ = s.touchStreams(idsNeedingTouch)
 		}
 	}
 
 	return exists, err
 }
 
-func (s *SQL) touchBlobs(blobIDs []uint64) error {
+func (s *SQL) touchBlobs(ex Executor, blobIDs []uint64) error {
 	if len(blobIDs) == 0 {
 		return nil
 	}
@@ -288,7 +288,7 @@ func (s *SQL) touchBlobs(blobIDs []uint64) error {
 	}
 
 	startTime := time.Now()
-	_, err := s.exec(query, args...)
+	_, err := s.exec(ex, query, args...)
 	log.Debugf("touched %d blobs and took %s", len(blobIDs), time.Since(startTime))
 	return errors.Err(err)
 }
@@ -306,7 +306,7 @@ func (s *SQL) touchStreams(streamIDs []uint64) error {
 	}
 
 	startTime := time.Now()
-	_, err := s.exec(query, args...)
+	_, err := s.exec(s.conn, query, args...)
 	log.Debugf("touched %d streams and took %s", len(streamIDs), time.Since(startTime))
 	return errors.Err(err)
 }
@@ -406,16 +406,16 @@ WHERE b.is_stored = 1 and b.hash IN (` + qt.Qs(len(batch)) + `)`
 // NOTE: If SoftDelete is enabled, streams will never be deleted
 func (s *SQL) Delete(hash string) error {
 	if s.SoftDelete {
-		_, err := s.exec("UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash)
+		_, err := s.exec(s.conn, "UPDATE blob_ SET is_stored = 0 WHERE hash = ?", hash)
 		return errors.Err(err)
 	}
 
-	_, err := s.exec("DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash)
+	_, err := s.exec(s.conn, "DELETE FROM stream WHERE sd_blob_id = (SELECT id FROM blob_ WHERE hash = ?)", hash)
 	if err != nil {
 		return errors.Err(err)
 	}
 
-	_, err = s.exec("DELETE FROM blob_ WHERE hash = ?", hash)
+	_, err = s.exec(s.conn, "DELETE FROM blob_ WHERE hash = ?", hash)
 	return errors.Err(err)
 }
 
@@ -590,7 +590,7 @@ func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
 		return errors.Err("not connected")
 	}
 
-	sdBlobID, err := s.insertBlob(sdHash, sdBlobLength, true)
+	sdBlobID, err := s.insertBlob(s.conn, sdHash, sdBlobLength, true)
 	if err != nil {
 		return err
 	}
@@ -600,28 +600,30 @@ func (s *SQL) AddSDBlob(sdHash string, sdBlobLength int, sdBlob SdBlob) error {
 		return err
 	}
 
-	// insert content blobs and connect them to stream
-	for _, contentBlob := range sdBlob.Blobs {
-		if contentBlob.BlobHash == "" {
-			// null terminator blob
-			continue
-		}
+	return withTx(s.conn, func(tx Transactor) error {
+		// insert content blobs and connect them to stream
+		for _, contentBlob := range sdBlob.Blobs {
+			if contentBlob.BlobHash == "" {
+				// null terminator blob
+				continue
+			}
 
-		blobID, err := s.insertBlob(contentBlob.BlobHash, contentBlob.Length, false)
-		if err != nil {
-			return err
-		}
+			blobID, err := s.insertBlob(tx, contentBlob.BlobHash, contentBlob.Length, false)
+			if err != nil {
+				return err
+			}
 
-		args := []interface{}{streamID, blobID, contentBlob.BlobNum}
-		_, err = s.exec(
-			"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")",
-			args...,
-		)
-		if err != nil {
-			return errors.Err(err)
+			args := []interface{}{streamID, blobID, contentBlob.BlobNum}
+			_, err = s.exec(tx,
+				"INSERT IGNORE INTO stream_blob (stream_id, blob_id, num) VALUES ("+qt.Qs(len(args))+")",
+				args...,
+			)
+			if err != nil {
+				return errors.Err(err)
+			}
 		}
-	}
-	return nil
+		return nil
+	})
 }
 
 // GetHashRange gets the smallest and biggest hashes in the db
@@ -694,39 +696,38 @@ func (s *SQL) GetStoredHashesInRange(ctx context.Context, start, end bits.Bitmap
 }
 
 // txFunc is a function that can be wrapped in a transaction
-type txFunc func(tx *sql.Tx) error
+type txFunc func(tx Transactor) error
 
-// withTx wraps a function in an sql transaction. the transaction is committed if there's no error, or rolled back if there is one.
-// if dbOrTx is an sql.DB, a new transaction is started
+// withTx wraps a function in an sql transaction. the transaction is committed if there's
+// no error, or rolled back if there is one. if dbOrTx is not a Transactor (e.g. if it's
+// an *sql.DB), withTx attempts to start a new transaction to use.
 func withTx(dbOrTx interface{}, f txFunc) (err error) {
-	var tx *sql.Tx
+	var tx Transactor
+	var ok bool
 
-	switch t := dbOrTx.(type) {
-	case *sql.Tx:
-		tx = t
-	case *sql.DB:
-		tx, err = t.Begin()
+	tx, ok = dbOrTx.(Transactor)
+	if !ok {
+		tx, err = Begin(dbOrTx)
 		if err != nil {
 			return err
 		}
-		defer func() {
-			if p := recover(); p != nil {
-				if rollBackError := tx.Rollback(); rollBackError != nil {
-					log.Error("failed to rollback tx on panic - ", rollBackError)
-				}
-				panic(p)
-			} else if err != nil {
-				if rollBackError := tx.Rollback(); rollBackError != nil {
-					log.Error("failed to rollback tx on panic - ", rollBackError)
-				}
-			} else {
-				err = errors.Err(tx.Commit())
-			}
-		}()
-	default:
-		return errors.Err("db or tx required")
 	}
 
+	defer func() {
+		if p := recover(); p != nil {
+			if rollBackError := tx.Rollback(); rollBackError != nil {
+				log.Error("failed to rollback tx on panic: ", rollBackError)
+			}
+			err = errors.Prefix("panic", p)
+		} else if err != nil {
+			if rollBackError := tx.Rollback(); rollBackError != nil {
+				log.Error("failed to rollback tx: ", rollBackError)
+			}
+		} else {
+			err = errors.Err(tx.Commit())
+		}
+	}()
+
 	return f(tx)
 }
 
@@ -739,12 +740,12 @@ func closeRows(rows *sql.Rows) {
 	}
 }
 
-func (s *SQL) exec(query string, args ...interface{}) (int64, error) {
+func (s *SQL) exec(ex Executor, query string, args ...interface{}) (int64, error) {
 	s.logQuery(query, args...)
 	attempt, maxAttempts := 0, 3
 Retry:
 	attempt++
-	result, err := s.conn.Exec(query, args...)
+	result, err := ex.Exec(query, args...)
 	if isLockTimeoutError(err) {
 		if attempt <= maxAttempts {
 			//Error 1205: Lock wait timeout exceeded; try restarting transaction
diff --git a/db/interfaces.go b/db/interfaces.go
new file mode 100644
index 0000000..11b2ee6
--- /dev/null
+++ b/db/interfaces.go
@@ -0,0 +1,45 @@
+package db
+
+import (
+	"database/sql"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+)
+
+// Executor can perform SQL queries.
+type Executor interface {
+	Exec(query string, args ...interface{}) (sql.Result, error)
+	Query(query string, args ...interface{}) (*sql.Rows, error)
+	QueryRow(query string, args ...interface{}) *sql.Row
+}
+
+// Transactor can commit and rollback, on top of being able to execute queries.
+type Transactor interface {
+	Commit() error
+	Rollback() error
+
+	Executor
+}
+
+// Begin begins a transaction
+func Begin(db interface{}) (Transactor, error) {
+	type beginner interface {
+		Begin() (Transactor, error)
+	}
+
+	creator, ok := db.(beginner)
+	if ok {
+		return creator.Begin()
+	}
+
+	type sqlBeginner interface {
+		Begin() (*sql.Tx, error)
+	}
+
+	creator2, ok := db.(sqlBeginner)
+	if ok {
+		return creator2.Begin()
+	}
+
+	return nil, errors.Err("database does not support transactions")
+}
diff --git a/shared/shared_test.go b/shared/shared_test.go
index 47c8eef..50a4eb7 100644
--- a/shared/shared_test.go
+++ b/shared/shared_test.go
@@ -1,27 +1,39 @@
 package shared
 
 import (
+	"os"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestBlobTrace_Serialize(t *testing.T) {
+	hostName, err := os.Hostname()
+	require.NoError(t, err)
+
 	stack := NewBlobTrace(10*time.Second, "test")
 	stack.Stack(20*time.Second, "test2")
 	stack.Stack(30*time.Second, "test3")
 	serialized, err := stack.Serialize()
-	assert.NoError(t, err)
-	t.Log(serialized)
-	expected := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}"
+	require.NoError(t, err)
+
+	expected := `{"stacks":[{"timing":10000000000,"origin_name":"test","host_name":"` +
+		hostName +
+		`"},{"timing":20000000000,"origin_name":"test2","host_name":"` +
+		hostName +
+		`"},{"timing":30000000000,"origin_name":"test3","host_name":"` +
+		hostName +
+		`"}]}`
 	assert.Equal(t, expected, serialized)
 }
 
 func TestBlobTrace_Deserialize(t *testing.T) {
-	serialized := "{\"stacks\":[{\"timing\":10000000000,\"origin_name\":\"test\"},{\"timing\":20000000000,\"origin_name\":\"test2\"},{\"timing\":30000000000,\"origin_name\":\"test3\"}]}"
+	serialized := `{"stacks":[{"timing":10000000000,"origin_name":"test"},{"timing":20000000000,"origin_name":"test2"},{"timing":30000000000,"origin_name":"test3"}]}`
 	stack, err := Deserialize(serialized)
-	assert.NoError(t, err)
+	require.NoError(t, err)
+
 	assert.Len(t, stack.Stacks, 3)
 	assert.Equal(t, stack.Stacks[0].Timing, 10*time.Second)
 	assert.Equal(t, stack.Stacks[1].Timing, 20*time.Second)