From 3c1cbaf18e3e67dfa9e6f0c5609a572d5922ead3 Mon Sep 17 00:00:00 2001 From: Artem Date: Sun, 1 May 2022 19:34:27 +0300 Subject: [PATCH] Fix: retry logic. IPFS rate limit --- cmd/metadata/config/config.go | 4 + cmd/metadata/main.go | 4 +- cmd/metadata/models/db.go | 4 +- cmd/metadata/models/elastic.go | 582 --------------------------------- cmd/metadata/models/source.go | 7 +- docker-compose.search.yml | 44 --- go.mod | 5 +- go.sum | 31 +- internal/ipfs/pool.go | 6 + 9 files changed, 21 insertions(+), 666 deletions(-) delete mode 100644 cmd/metadata/models/elastic.go delete mode 100644 docker-compose.search.yml diff --git a/cmd/metadata/config/config.go b/cmd/metadata/config/config.go index 518656b..758b667 100644 --- a/cmd/metadata/config/config.go +++ b/cmd/metadata/config/config.go @@ -15,6 +15,10 @@ type Config struct { // Substitute - func (c *Config) Substitute() error { + if err := c.Config.Substitute(); err != nil { + return err + } + for _, indexer := range c.Metadata.Indexers { if err := substituteContracts(c, &indexer.Filters); err != nil { return err diff --git a/cmd/metadata/main.go b/cmd/metadata/main.go index 4160214..cac0956 100644 --- a/cmd/metadata/main.go +++ b/cmd/metadata/main.go @@ -81,7 +81,7 @@ func main() { return } - if err := hasura.Create(ctx, cfg.Hasura, cfg.Database, views, new(models.TokenMetadata), new(models.ContractMetadata)); err != nil { + if err := hasura.Create(ctx, cfg.Hasura, cfg.Database, views, nil, new(models.TokenMetadata), new(models.ContractMetadata)); err != nil { log.Err(err).Msg("hasura.Create") } }) @@ -109,7 +109,7 @@ func main() { return } - if err := hasura.Create(ctx, cfg.Hasura, cfg.Database, views, new(models.TokenMetadata), new(models.ContractMetadata)); err != nil { + if err := hasura.Create(ctx, cfg.Hasura, cfg.Database, views, nil, new(models.TokenMetadata), new(models.ContractMetadata)); err != nil { log.Err(err).Msg("hasura.Create") } }) diff --git a/cmd/metadata/models/db.go b/cmd/metadata/models/db.go index e76d6a3..f627abd 100644 --- a/cmd/metadata/models/db.go +++ b/cmd/metadata/models/db.go @@ -77,7 +77,7 @@ func (d *dbLogger) AfterQuery(ctx context.Context, event *pg.QueryEvent) error { // GetContractMetadata - func (db *RelativeDatabase) GetContractMetadata(network string, status Status, limit, offset, retryCount int) (all []ContractMetadata, err error) { - query := db.DB().Model(&all).Where("status = ?", status).Where("network = ?", network) + query := db.DB().Model(&all).Where("status = ?", status).Where("network = ?", network).Where("created_at < (extract(epoch from current_timestamp) - 10 * retry_count)") if limit > 0 { query.Limit(limit) } @@ -126,7 +126,7 @@ func (db *RelativeDatabase) CountContractsByStatus(network string, status Status // GetTokenMetadata - func (db *RelativeDatabase) GetTokenMetadata(network string, status Status, limit, offset, retryCount int) (all []TokenMetadata, err error) { - query := db.DB().Model(&all).Where("status = ?", status).Where("network = ?", network) + query := db.DB().Model(&all).Where("status = ?", status).Where("network = ?", network).Where(" created_at < (extract(epoch from current_timestamp) - 10 * retry_count)") if limit > 0 { query.Limit(limit) } diff --git a/cmd/metadata/models/elastic.go b/cmd/metadata/models/elastic.go deleted file mode 100644 index 30f3477..0000000 --- a/cmd/metadata/models/elastic.go +++ /dev/null @@ -1,582 +0,0 @@ -package models - -import ( - "bytes" - "context" - stdJSON "encoding/json" - "fmt" - "io/ioutil" - "os" - "strconv" - "strings" - "time" - - "github.com/cenkalti/backoff" - "github.com/dipdup-net/go-lib/config" - "github.com/dipdup-net/go-lib/database" - "github.com/elastic/go-elasticsearch/v8" - "github.com/elastic/go-elasticsearch/v8/esapi" - "github.com/go-pg/pg/v10" - "github.com/pkg/errors" - - jsoniter "github.com/json-iterator/go" -) - -var json = jsoniter.ConfigCompatibleWithStandardLibrary - -// Elastic - -type Elastic struct { - *elasticsearch.Client -} - -// NewElastic - -func NewElastic(cfg config.Database) (*Elastic, error) { - retryBackoff := backoff.NewExponentialBackOff() - elasticConfig := elasticsearch.Config{ - Addresses: []string{cfg.Path}, - RetryOnStatus: []int{502, 503, 504, 429}, - RetryBackoff: func(i int) time.Duration { - if i == 1 { - retryBackoff.Reset() - } - return retryBackoff.NextBackOff() - }, - MaxRetries: 5, - } - es, err := elasticsearch.NewClient(elasticConfig) - if err != nil { - return nil, err - } - - entity := &Elastic{es} - - response, err := entity.Ping() - if err != nil { - return nil, err - } - defer response.Body.Close() - - if err := entity.createIndices(); err != nil { - return nil, err - } - - return entity, nil -} - -type elasticResponse struct { - Hits struct { - Hits []struct { - ID string `json:"_id"` - Source stdJSON.RawMessage `json:"_source"` - } `json:"hits"` - } `json:"hits"` -} - -type bulkResponse struct { - Took int64 `json:"took"` - Errors bool `json:"errors"` - Items stdJSON.RawMessage `json:"items,omitempty"` -} - -func (e *Elastic) search(query string, opts ...func(*esapi.SearchRequest)) (*elasticResponse, error) { - body := strings.NewReader(query) - opts = append(opts, e.Search.WithBody(body)) - response, err := e.Search(opts...) - if err != nil { - return nil, err - } - defer response.Body.Close() - if response.IsError() { - return nil, errors.New(response.String()) - } - - var hits elasticResponse - err = json.NewDecoder(response.Body).Decode(&hits) - return &hits, err -} - -func (e *Elastic) bulk(buf *bytes.Buffer) error { - req := esapi.BulkRequest{ - Body: bytes.NewReader(buf.Bytes()), - Refresh: "true", - } - - res, err := req.Do(context.Background(), e) - if err != nil { - return err - } - defer res.Body.Close() - - var response bulkResponse - if err := json.NewDecoder(res.Body).Decode(&response); err != nil { - return err - } - if response.Errors { - return errors.Errorf("Bulk error: %s", string(response.Items)) - } - return nil -} - -// GetContractMetadata - -// TODO: filter network -func (e *Elastic) GetContractMetadata(network string, status Status, limit, offset, retryCount int) ([]ContractMetadata, error) { - hits, err := e.search( - fmt.Sprintf(`{"query":{"term":{"status": %d}}}`, status), - e.Search.WithIndex(ContractMetadata{}.TableName()), - e.Search.WithSort("retry_count:asc"), - e.Search.WithSize(limit), - e.Search.WithFrom(offset), - ) - if err != nil { - return nil, err - } - - contracts := make([]ContractMetadata, len(hits.Hits.Hits)) - for i, hit := range hits.Hits.Hits { - if err := json.Unmarshal(hit.Source, &contracts[i]); err != nil { - return nil, err - } - id, err := strconv.ParseUint(hit.ID, 10, 64) - if err != nil { - return nil, err - } - contracts[i].ID = id - } - return contracts, nil -} - -// UpdateContractMetadata - -func (e *Elastic) UpdateContractMetadata(ctx context.Context, metadata []*ContractMetadata) error { - if len(metadata) == 0 { - return nil - } - bulk := bytes.NewBuffer([]byte{}) - for i := range metadata { - metadata[i].UpdatedAt = metadata[i].CreatedAt - meta := fmt.Sprintf(`{"update":{"_id":"%d","_index":"%s"}}`, time.Now().UnixNano(), metadata[i].TableName()) - if _, err := bulk.WriteString(meta); err != nil { - return err - } - - if err := bulk.WriteByte('\n'); err != nil { - return err - } - - data, err := json.Marshal(metadata[i]) - if err != nil { - return err - } - if err := stdJSON.Compact(bulk, data); err != nil { - return err - } - if err := bulk.WriteByte('\n'); err != nil { - return err - } - - if (i%1000 == 0 && i > 0) || i == len(metadata)-1 { - if err := e.bulk(bulk); err != nil { - return err - } - bulk.Reset() - } - } - return nil -} - -// SaveContractMetadata - -func (e *Elastic) SaveContractMetadata(ctx context.Context, metadata []*ContractMetadata) error { - if len(metadata) == 0 { - return nil - } - bulk := bytes.NewBuffer([]byte{}) - for i := range metadata { - metadata[i].CreatedAt = time.Now().Unix() - metadata[i].UpdatedAt = metadata[i].CreatedAt - meta := fmt.Sprintf(`{"index":{"_id":"%d","_index":"%s"}}`, time.Now().UnixNano(), metadata[i].TableName()) - if _, err := bulk.WriteString(meta); err != nil { - return err - } - - if err := bulk.WriteByte('\n'); err != nil { - return err - } - - data, err := json.Marshal(metadata[i]) - if err != nil { - return err - } - if err := stdJSON.Compact(bulk, data); err != nil { - return err - } - if err := bulk.WriteByte('\n'); err != nil { - return err - } - - if (i%1000 == 0 && i > 0) || i == len(metadata)-1 { - if err := e.bulk(bulk); err != nil { - return err - } - bulk.Reset() - } - } - return nil -} - -// LastContractUpdateID - -func (e *Elastic) LastContractUpdateID() (value int64, err error) { - // TODO: realize LastContractUpdateID - return -} - -// CountContractsByStatus - -func (e *Elastic) CountContractsByStatus(network string, status Status) (int, error) { - // TODO: realize CountContractsByStatus - return 0, nil -} - -// GetContractMetadata - -// TODO: filter network -func (e *Elastic) GetTokenMetadata(network string, status Status, limit, offset, retryCount int) ([]TokenMetadata, error) { - hits, err := e.search( - fmt.Sprintf(`{"query":{"term":{"status": %d}}}`, status), - e.Search.WithIndex(TokenMetadata{}.TableName()), - e.Search.WithSort("retry_count:asc"), - e.Search.WithSize(limit), - e.Search.WithFrom(offset), - ) - if err != nil { - return nil, err - } - - tokens := make([]TokenMetadata, len(hits.Hits.Hits)) - for i, hit := range hits.Hits.Hits { - if err := json.Unmarshal(hit.Source, &tokens[i]); err != nil { - return nil, err - } - id, err := strconv.ParseUint(hit.ID, 10, 64) - if err != nil { - return nil, err - } - tokens[i].ID = id - } - return tokens, nil -} - -// UpdateTokenMetadata - -func (e *Elastic) UpdateTokenMetadata(ctx context.Context, metadata []*TokenMetadata) error { - if len(metadata) == 0 { - return nil - } - bulk := bytes.NewBuffer([]byte{}) - for i := range metadata { - metadata[i].UpdatedAt = metadata[i].CreatedAt - meta := fmt.Sprintf(`{"update":{"_id":"%d","_index":"%s"}}`, time.Now().UnixNano(), metadata[i].TableName()) - if _, err := bulk.WriteString(meta); err != nil { - return err - } - - if err := bulk.WriteByte('\n'); err != nil { - return err - } - - data, err := json.Marshal(metadata[i]) - if err != nil { - return err - } - if err := stdJSON.Compact(bulk, data); err != nil { - return err - } - if err := bulk.WriteByte('\n'); err != nil { - return err - } - - if (i%1000 == 0 && i > 0) || i == len(metadata)-1 { - if err := e.bulk(bulk); err != nil { - return err - } - bulk.Reset() - } - } - return nil -} - -// SaveContractMetadata - -func (e *Elastic) SaveTokenMetadata(ctx context.Context, metadata []*TokenMetadata) error { - if len(metadata) == 0 { - return nil - } - bulk := bytes.NewBuffer([]byte{}) - for i := range metadata { - metadata[i].CreatedAt = time.Now().Unix() - metadata[i].UpdatedAt = metadata[i].CreatedAt - meta := fmt.Sprintf(`{"index":{"_id":"%d","_index":"%s"}}`, time.Now().UnixNano(), metadata[i].TableName()) - if _, err := bulk.WriteString(meta); err != nil { - return err - } - - if err := bulk.WriteByte('\n'); err != nil { - return err - } - - data, err := json.Marshal(metadata[i]) - if err != nil { - return err - } - if err := stdJSON.Compact(bulk, data); err != nil { - return err - } - if err := bulk.WriteByte('\n'); err != nil { - return err - } - - if (i%1000 == 0 && i > 0) || i == len(metadata)-1 { - if err := e.bulk(bulk); err != nil { - return err - } - bulk.Reset() - } - } - return nil -} - -// LastTokenUpdateID - -func (e *Elastic) LastTokenUpdateID() (value int64, err error) { - // TODO: realize LastTokenUpdateID - return -} - -// CountTokensByStatus - -func (e *Elastic) CountTokensByStatus(network string, status Status) (int, error) { - // TODO: realize CountTokensByStatus - return 0, nil -} - -// SetImageProcessed - -func (e *Elastic) SetImageProcessed(token TokenMetadata) error { - token.UpdatedAt = time.Now().Unix() - response, err := e.Update( - token.TableName(), - fmt.Sprintf("%d", token.ID), - strings.NewReader(`{"doc":{"image_processed":true}}`), - e.Update.WithRefresh("true"), - ) - if err != nil { - return err - } - defer response.Body.Close() - - if response.IsError() { - return errors.New(response.String()) - } - return nil -} - -// GetUnprocessedImage - -func (e *Elastic) GetUnprocessedImage(from uint64, limit int) ([]TokenMetadata, error) { - var b strings.Builder - b.WriteString(`{"query":{"bool":{"filter":[{"term":{"status": 3}},{"term":{"image_processed":false}}`) - if from > 0 { - b.WriteString(fmt.Sprintf(`,{"range":{"id":{"gte":%d}}}`, from)) - } - b.WriteString(`]}}}`) - hits, err := e.search( - b.String(), - e.Search.WithIndex(TokenMetadata{}.TableName()), - e.Search.WithSort("retry_count:asc"), - e.Search.WithSize(limit), - ) - if err != nil { - return nil, err - } - - tokens := make([]TokenMetadata, len(hits.Hits.Hits)) - for i, hit := range hits.Hits.Hits { - if err := json.Unmarshal(hit.Source, &tokens[i]); err != nil { - return nil, err - } - id, err := strconv.ParseUint(hit.ID, 10, 64) - if err != nil { - return nil, err - } - tokens[i].ID = id - } - return tokens, nil -} - -// State - -func (e *Elastic) State(indexName string) (s *database.State, err error) { - hits, err := e.search( - fmt.Sprintf(`{"query":{"term":{"index_name":"%s"}}}`, indexName), - e.Search.WithIndex(s.TableName()), - e.Search.WithSize(1), - ) - if err != nil { - return - } - - if len(hits.Hits.Hits) != 1 { - return s, errors.Wrapf(pg.ErrNoRows, "%s %s", indexName, s.TableName()) - } - err = json.Unmarshal(hits.Hits.Hits[0].Source, s) - return -} - -// UpdateState - -func (e *Elastic) UpdateState(s *database.State) error { - data, err := json.Marshal(s) - if err != nil { - return err - } - var b strings.Builder - b.WriteString(`{"doc_as_upsert":true,"doc":`) - b.Write(data) - b.WriteString(`}`) - - resp, err := e.Update( - s.TableName(), - s.IndexName, - strings.NewReader(b.String()), - e.Update.WithRefresh("true"), - ) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.IsError() { - return errors.New(resp.String()) - } - return nil -} - -// CreateState - -func (e *Elastic) CreateState(s *database.State) error { - // TODO: implement - return nil -} - -// DeleteState - -func (e *Elastic) DeleteState(s *database.State) error { - // TODO: implement - return nil -} - -// Close - -func (e *Elastic) Close() error { - return nil -} - -func (e *Elastic) createIndices() error { - if err := e.createIndex(database.State{}.TableName()); err != nil { - return err - } - if err := e.createIndex(ContractMetadata{}.TableName()); err != nil { - return err - } - if err := e.createIndex(TokenMetadata{}.TableName()); err != nil { - return err - } - if err := e.createIndex(TezosKey{}.TableName()); err != nil { - return err - } - return nil -} - -func (e *Elastic) createIndex(name string) error { - req := esapi.IndicesExistsRequest{ - Index: []string{name}, - } - resp, err := req.Do(context.Background(), e) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode == 200 { - return nil - } - - opts := []func(*esapi.IndicesCreateRequest){} - mapping, err := os.Open(fmt.Sprintf("./mappings/%s.json", name)) - if err == nil { - data, err := ioutil.ReadAll(mapping) - if err != nil { - return err - } - if err := mapping.Close(); err != nil { - return err - } - opts = append(opts, e.Indices.Create.WithBody(bytes.NewReader(data))) - } - - response, err := e.Indices.Create(name, opts...) - if err != nil { - return err - } - defer response.Body.Close() - - if response.IsError() { - return errors.New(response.String()) - } - return nil -} - -// IPFSLink - -func (e *Elastic) IPFSLink(id int64) (link IPFSLink, err error) { - // TODO: implement - return -} - -// IPFSLinks - -func (e *Elastic) IPFSLinks(limit, offset int) (links []IPFSLink, err error) { - // TODO: implement - return -} - -// SaveIPFSLink - -func (e *Elastic) SaveIPFSLink(link IPFSLink) error { - // TODO: implement - return nil -} - -// UpdateIPFSLink - -func (e *Elastic) UpdateIPFSLink(link IPFSLink) error { - // TODO: implement - return nil -} - -// IPFSLinkByURL - -func (e *Elastic) IPFSLinkByURLs(url ...string) (links []IPFSLink, err error) { - // TODO: implement - return -} - -// CreateIndices - -func (e *Elastic) CreateIndices() error { - // TODO: implement - return nil -} - -// Exec - -func (e *Elastic) Exec(sql string) error { - // TODO: implement - return nil -} - -// GetTezosKey - -func (e *Elastic) GetTezosKey(network, address, key string) (tk TezosKey, err error) { - // TODO: implement - return -} - -// SaveTezosKey - -func (e *Elastic) SaveTezosKey(tk TezosKey) error { - // TODO: implement - return nil -} - -// DeleteTezosKey - -func (e *Elastic) DeleteTezosKey(tk TezosKey) error { - // TODO: implement - return nil -} diff --git a/cmd/metadata/models/source.go b/cmd/metadata/models/source.go index 6695ab5..8a195e8 100644 --- a/cmd/metadata/models/source.go +++ b/cmd/metadata/models/source.go @@ -23,10 +23,5 @@ type Database interface { // NewDatabase - func NewDatabase(ctx context.Context, cfg config.Database) (Database, error) { - switch cfg.Kind { - case "elastic": - return NewElastic(cfg) - default: - return NewRelativeDatabase(ctx, cfg) - } + return NewRelativeDatabase(ctx, cfg) } diff --git a/docker-compose.search.yml b/docker-compose.search.yml deleted file mode 100644 index ad4778e..0000000 --- a/docker-compose.search.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: "3.6" - -services: - indexer: - image: dipdup/metadata:latest - command: -c dipdup.search.yml - build: - dockerfile: build/metadata/Dockerfile - context: . - env_file: - - .env - depends_on: - - elastic - restart: always - - elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1 - restart: always - volumes: - - esdata:/usr/share/elasticsearch/data - environment: - - bootstrap.memory_lock=true - - discovery.type=single-node - env_file: - - .env - ports: - - 127.0.0.1:9200:9200 - - api: - image: dipdup/metadata-api:latest - command: -c dipdup.search.yml - build: - dockerfile: build/api/Dockerfile - context: . - env_file: - - .env - depends_on: - - elastic - restart: always - ports: - - 127.0.0.1:11111:11111 - -volumes: - esdata: diff --git a/go.mod b/go.mod index 1ac2ef7..3cc0160 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( github.com/aws/aws-sdk-go v1.43.31 github.com/cenkalti/backoff v2.2.1+incompatible - github.com/dipdup-net/go-lib v0.2.0 + github.com/dipdup-net/go-lib v0.2.8 github.com/disintegration/imaging v1.6.2 github.com/elastic/go-elasticsearch/v8 v8.1.0 github.com/go-pg/pg/v10 v10.10.6 @@ -17,6 +17,7 @@ require ( github.com/rs/zerolog v1.26.1 github.com/shopspring/decimal v1.3.1 github.com/stretchr/testify v1.7.1 + golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 ) require ( @@ -25,6 +26,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/ebellocchia/go-base58 v0.1.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect github.com/go-pg/zerochecker v0.2.0 // indirect github.com/go-playground/locales v0.14.0 // indirect @@ -92,7 +94,6 @@ require ( golang.org/x/net v0.0.0-20220403103023-749bd193bc2b // indirect golang.org/x/sys v0.0.0-20220403020550-483a9cbc67c0 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect diff --git a/go.sum b/go.sum index 9088120..ec59fc0 100644 --- a/go.sum +++ b/go.sum @@ -83,18 +83,16 @@ github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuv github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/dipdup-net/go-lib v0.1.56 h1:Jr0+lDCaOc9Y9XfKoi1F22peVaB1mZe5XsIJKg5mUHk= -github.com/dipdup-net/go-lib v0.1.56/go.mod h1:ZOAyo2sqNLCCVdvHvPbiWfgImvmrMW7/sdQiSsGrD44= -github.com/dipdup-net/go-lib v0.2.0 h1:RGreU/ydxeaU5Eyad1g1NF44uMpgqVR6kgmY+2Imvg0= -github.com/dipdup-net/go-lib v0.2.0/go.mod h1:ZOAyo2sqNLCCVdvHvPbiWfgImvmrMW7/sdQiSsGrD44= +github.com/dipdup-net/go-lib v0.2.8 h1:P3u5SGfPy983kFOmdCjN1NWzDMLWzVeRO4hooHKv9Xo= +github.com/dipdup-net/go-lib v0.2.8/go.mod h1:RjVhNv63RE3pOAeHLwnspa2UjJi80k/Kelwv7aN1ETE= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= +github.com/ebellocchia/go-base58 v0.1.0 h1:0w/ODEfZnOPW5KW0QY/Xpb1fxba/BxQJMUa5iYzpljk= github.com/ebellocchia/go-base58 v0.1.0/go.mod h1:RHE/6C6Ru6YAH9Tc+A9eHQ6ZKEooLC0jw+YLnpt3CAU= github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE= github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= @@ -128,7 +126,6 @@ github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-playground/validator/v10 v10.9.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX91eVn1JYXMWt7ig= github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= @@ -197,7 +194,6 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -215,7 +211,6 @@ github.com/ipfs/go-ipfs-files v0.0.9/go.mod h1:aFv2uQ/qxWpL/6lidWvnSQmaVqCrf0TBG github.com/ipfs/go-ipfs-files v0.1.1 h1:/MbEowmpLo9PJTEQk16m9rKzUHjeP4KRU9nWJyJO324= github.com/ipfs/go-ipfs-files v0.1.1/go.mod h1:8xkIrMWH+Y5P7HvJ4Yc5XWwIW2e52dyXUiC0tZyjDbM= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -226,7 +221,6 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ= github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= @@ -237,7 +231,6 @@ github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5W github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= @@ -253,20 +246,17 @@ github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01C github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.9.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38= github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.14.0/go.mod h1:jT3ibf/A0ZVCp89rtCIN0zCJxcE74ypROmHEZYsG/j8= github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w= github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= @@ -274,8 +264,6 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.3/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -444,11 +432,9 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo= github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= @@ -472,9 +458,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -493,7 +476,6 @@ github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs= github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b h1:wA3QeTsaAXybLL2kb2cKhCAQTHgYTMwuI8lBlJSv5V8= github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b/go.mod h1:xT1Y5p2JR2PfSZihE0s4mjdJaRGp1waCTf5JzhQLBck= -github.com/yhirose/go-peg v0.0.0-20210804202551-de25d6753cf1/go.mod h1:q2QWLflHsZxT6ixYcXveTYicEvxGh5Uv6CnI7f7BfjQ= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -535,7 +517,6 @@ golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o= @@ -690,7 +671,6 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220403020550-483a9cbc67c0 h1:PgUUmg0gNMIPY2WafhL/oLyQGw+kdTNPlVWOjltpp3w= @@ -868,17 +848,12 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.2.1/go.mod h1:qsiz+XcAyMrS6QY+X3M9R6b/lKM1imKmcuK9kac5LTo= gorm.io/driver/mysql v1.3.3 h1:jXG9ANrwBc4+bMvBcSl8zCfPBaVoPyBEBshA8dA93X8= gorm.io/driver/mysql v1.3.3/go.mod h1:ChK6AHbHgDCFZyJp0F+BmVGb06PSIoh9uVYKAlRbb2U= -gorm.io/driver/postgres v1.2.3/go.mod h1:pJV6RgYQPG47aM1f0QeOzFH9HxQc8JcmAgjRCgS0wjs= gorm.io/driver/postgres v1.3.3 h1:y6DU2kJgDNisxfAlmxRaQZOIy4ytnuYrpzpSFYnSfCY= gorm.io/driver/postgres v1.3.3/go.mod h1:y0vEuInFKJtijuSGu9e5bs5hzzSzPK+LancpKpvbRBw= -gorm.io/driver/sqlite v1.2.6/go.mod h1:gyoX0vHiiwi0g49tv+x2E7l8ksauLK0U/gShcdUsjWY= gorm.io/driver/sqlite v1.3.1 h1:bwfE+zTEWklBYoEodIOIBwuWHpnx52Z9zJFW5F33WLk= gorm.io/driver/sqlite v1.3.1/go.mod h1:wJx0hJspfycZ6myN38x1O/AqLtNS6c5o9TndewFbELg= -gorm.io/gorm v1.22.3/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0= -gorm.io/gorm v1.22.4/go.mod h1:1aeVC+pe9ZmvKZban/gW4QPra7PRoTEssyc922qCAkk= gorm.io/gorm v1.23.1/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.23.4 h1:1BKWM67O6CflSLcwGQR7ccfmC4ebOxQrTfOQGRE9wjg= gorm.io/gorm v1.23.4/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= diff --git a/internal/ipfs/pool.go b/internal/ipfs/pool.go index 9ee1317..b85715d 100644 --- a/internal/ipfs/pool.go +++ b/internal/ipfs/pool.go @@ -10,6 +10,7 @@ import ( "time" "github.com/pkg/errors" + "golang.org/x/time/rate" ) // Pool - @@ -17,6 +18,7 @@ type Pool struct { gateways []string limit int64 client *http.Client + limiter *rate.Limiter } // NewPool - @@ -38,6 +40,7 @@ func NewPool(gateways []string, limit int64) (*Pool, error) { DisableKeepAlives: true, }, }, + limiter: rate.NewLimiter(rate.Limit(10), 1), }, nil } @@ -85,6 +88,9 @@ func (pool *Pool) GetFromNode(ctx context.Context, link, node string) (Data, err } func (pool *Pool) request(ctx context.Context, link, node string) ([]byte, error) { + if err := pool.limiter.Wait(ctx); err != nil { + return nil, err + } path := Path(link) gatewayURL := Link(node, path)