Remove support for data-transfer/graphsync

- Worker numbering
- More debug logging
- Remove graphsync config and comments

.github/workflows/ecr-publisher.yml

@@ -14,6 +14,7 @@ on:
       - 'doc/**'
     branches:
       - main
+      - no-graphsync
 
 jobs:
   publisher:

command/gc/daemon.go

@@ -128,7 +128,6 @@ func daemonAction(cctx *cli.Context) error {
 		reaper.WithDatastoreDir(dsDir),
 		reaper.WithDatastoreTempDir(dsTmpDir),
 		reaper.WithPCache(pc),
-		reaper.WithTopicName(cfg.Ingest.PubSubTopic),
 		reaper.WithHttpTimeout(time.Duration(cfg.Ingest.HttpSyncTimeout)),
 		reaper.WithSyncSegmentSize(cctx.Int("sync-segment-size")),
 	)

command/gc/provider.go

@@ -164,7 +164,6 @@ func providerAction(cctx *cli.Context) error {
 		reaper.WithEntriesFromPublisher(cctx.Bool("ents-from-pub")),
 		reaper.WithPCache(pc),
 		reaper.WithSegmentSize(cctx.Int("segment-size")),
-		reaper.WithTopicName(cfg.Ingest.PubSubTopic),
 		reaper.WithHttpTimeout(time.Duration(cfg.Ingest.HttpSyncTimeout)),
 		reaper.WithSyncSegmentSize(cctx.Int("sync-segment-size")),
 	)

config/ingest.go

@@ -37,12 +37,6 @@ type Ingest struct {
 	// recent advertisement, set this to 1. A value of 0, the default, means
 	// unlimited depth.
 	FirstSyncDepth int
-	// GsMaxInRequests is the maximum number of incoming in-progress graphsync
-	// requests. Default is 1024.
-	GsMaxInRequests uint64
-	// GsMaxOutRequests is the maximum number of outgoing in-progress graphsync
-	// requests. Default is 1024.
-	GsMaxOutRequests uint64
 	// HttpSyncRetryMax sets the maximum number of times HTTP sync requests
 	// should be retried. A value of zero, the default, means no retry.
 	HttpSyncRetryMax int
@@ -97,8 +91,6 @@ func NewIngest() Ingest {
 		AdvertisementDepthLimit: 33554432,
 		AdvertisementMirror:     NewMirror(),
 		EntriesDepthLimit:       65536,
-		GsMaxInRequests:         1024,
-		GsMaxOutRequests:        1024,
 		HttpSyncRetryWaitMax:    Duration(30 * time.Second),
 		HttpSyncRetryWaitMin:    Duration(1 * time.Second),
 		HttpSyncTimeout:         Duration(10 * time.Second),
@@ -122,12 +114,6 @@ func (c *Ingest) populateUnset() {
 	if c.EntriesDepthLimit == 0 {
 		c.EntriesDepthLimit = def.EntriesDepthLimit
 	}
-	if c.GsMaxInRequests == 0 {
-		c.GsMaxInRequests = def.GsMaxInRequests
-	}
-	if c.GsMaxOutRequests == 0 {
-		c.GsMaxOutRequests = def.GsMaxOutRequests
-	}
 	if c.HttpSyncRetryWaitMax == 0 {
 		c.HttpSyncRetryWaitMax = def.HttpSyncRetryWaitMax
 	}

config/logging.go

@@ -16,11 +16,8 @@ func NewLogging() Logging {
 	return Logging{
 		Level: "info",
 		Loggers: map[string]string{
-			"basichost":    "warn",
-			"bootstrap":    "warn",
-			"dt_graphsync": "warn",
-			"dt-impl":      "warn",
-			"graphsync":    "warn",
+			"basichost": "warn",
+			"bootstrap": "warn",
 		},
 	}
 }

deploy/manifests/base/storetheindex-single/config.json

@@ -71,10 +71,7 @@
     "Level": "info",
     "Loggers": {
       "basichost": "warn",
-      "bootstrap": "warn",
-      "dt-impl": "warn",
-      "dt_graphsync": "warn",
-      "graphsync": "warn"
+      "bootstrap": "warn"
     }
   },
   "Peering": {

deploy/manifests/base/storetheindex/indexer-config.yaml

@@ -81,10 +81,7 @@ data:
         "Level": "info",
           "Loggers": {
           "basichost": "warn",
-          "bootstrap": "warn",
-          "dt-impl": "warn",
-          "dt_graphsync": "warn",
-          "graphsync": "warn"
+          "bootstrap": "warn"
         }
       },
       "Peering": {

deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/inga/config.json

@@ -97,10 +97,7 @@
     "Level": "info",
     "Loggers": {
       "basichost": "warn",
-      "bootstrap": "warn",
-      "dt-impl": "warn",
-      "dt_graphsync": "warn",
-      "graphsync": "warn"
+      "bootstrap": "warn"
     }
   },
   "Peering": {

deploy/manifests/prod/us-east-2/tenant/storetheindex/instances/inga/deployment.yaml

@@ -7,9 +7,6 @@ spec:
     spec:
       serviceAccountName: storetheindex
       # Wait at most 3 minutes for storetheindex process to exit.
-      # This value used to be 10 minutes which we hit every time the shutdown took longer than 3 minutes.
-      # The most likely cause for this is lingering GraphSync syncs which do not clear after 10 minutes of wait,
-      # hence no point to wait that long.
       terminationGracePeriodSeconds: 180
       containers:
         - name: indexer

e2e_test.go

@@ -55,11 +55,6 @@ func TestEndToEndWithAllProviderTypes(t *testing.T) {
 	t.Run("Libp2pWithHTTPProvider", func(t *testing.T) {
 		testEndToEndWithReferenceProvider(t, "libp2phttp")
 	})
-
-	// Test with publisher running dtsync over libp2p.
-	t.Run("DTSyncProvider", func(t *testing.T) {
-		testEndToEndWithReferenceProvider(t, "dtsync")
-	})
 }
 
 func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) {
@@ -88,13 +83,10 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) {
 
 	// install index-provider
 	switch publisherProto {
-	case "dtsync":
-		// Install index-provider that supports dtsync.
-		e.Run("go", "install", "github.com/ipni/index-provider/cmd/provider@v0.13.6")
 	case "libp2p", "libp2phttp", "http":
 		e.Run("go", "install", "github.com/ipni/index-provider/cmd/provider@latest")
 	default:
-		panic("providerProto must be one of: libp2phttp, http, dtsync")
+		panic("providerProto must be one of: libp2p, libp2phttp, http")
 	}
 	// install dhstore
 	e.Run("go", "install", "-tags", "nofdb", "github.com/ipni/dhstore/cmd/dhstore@latest")
@@ -107,8 +99,6 @@ func testEndToEndWithReferenceProvider(t *testing.T, publisherProto string) {
 
 	// initialize index-provider
 	switch publisherProto {
-	case "dtsync":
-		e.Run(provider, "init")
 	case "http":
 		e.Run(provider, "init", "--pubkind=http")
 	case "libp2p":

gc/reaper/option.go

@@ -14,7 +14,6 @@ const (
 	defaultHttpTimeout = 10 * time.Second
 	defaultSegmentSize = 16384
 	defaultSyncSegSize = 4096
-	defaultTopic       = "/indexer/ingest/mainnet"
 )
 
 type config struct {
@@ -31,7 +30,6 @@ type config struct {
 	pcache            *pcache.ProviderCache
 	segmentSize       int
 	syncSegSize       int
-	topic             string
 }
 
 // Option is a function that sets a value in a config.
@@ -46,7 +44,6 @@ func getOpts(opts []Option) (config, error) {
 		httpTimeout: defaultHttpTimeout,
 		segmentSize: defaultSegmentSize,
 		syncSegSize: defaultSyncSegSize,
-		topic:       defaultTopic,
 	}
 
 	for i, opt := range opts {
@@ -158,15 +155,6 @@ func WithSyncSegmentSize(size int) Option {
 	}
 }
 
-// WithTopicName sets the topic name on which the provider announces advertised
-// content. Defaults to '/indexer/ingest/mainnet'.
-func WithTopicName(topic string) Option {
-	return func(c *config) error {
-		c.topic = topic
-		return nil
-	}
-}
-
 // WithEntriesDepthLimit sets the depth limit when syncing an
 // advertisement entries chain. Setting to 0 means no limit.
 func WithEntriesDepthLimit(depthLimit int64) Option {

gc/reaper/reaper.go

@@ -98,7 +98,6 @@ type Reaper struct {
 	stats       GCStats
 	statsMutex  sync.Mutex
 	syncSegSize int
-	topic       string
 }
 
 type scythe struct {
@@ -193,7 +192,6 @@ func New(idxr indexer.Interface, fileStore filestore.Interface, options ...Optio
 		pcache:      opts.pcache,
 		segmentSize: opts.segmentSize,
 		syncSegSize: opts.syncSegSize,
-		topic:       opts.topic,
 	}, nil
 }
 
@@ -616,7 +614,7 @@ func (r *Reaper) makeSubscriber(dstoreTmp datastore.Batching) (*dagsync.Subscrib
 		}, nil
 	}
 
-	return dagsync.NewSubscriber(r.host, dstoreTmp, linksys, r.topic,
+	return dagsync.NewSubscriber(r.host, linksys,
 		dagsync.HttpTimeout(r.httpTimeout),
 		dagsync.SegmentDepthLimit(int64(r.syncSegSize)))
 }

gc/reaper/reaper_test.go

@@ -20,8 +20,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const testTopic = "/indexer/ingest/test"
-
 var pid1, pid2, pid3 peer.ID
 var adCid cid.Cid
 
@@ -68,7 +66,6 @@ func TestReaper(t *testing.T) {
 		reaper.WithDatastoreDir(dsDir),
 		reaper.WithDatastoreTempDir(dsTmpDir),
 		reaper.WithPCache(pc),
-		reaper.WithTopicName(testTopic),
 	)
 	require.NoError(t, err)
 	defer gc.Close()
@@ -94,7 +91,6 @@ func TestReaper(t *testing.T) {
 		reaper.WithDatastoreTempDir(dsTmpDir),
 		reaper.WithDeleteNotFound(true),
 		reaper.WithPCache(pc),
-		reaper.WithTopicName(testTopic),
 	)
 	require.NoError(t, err)
 	defer gc2.Close()
@@ -116,7 +112,6 @@ func TestReaper(t *testing.T) {
 		reaper.WithDatastoreTempDir(dsTmpDir),
 		reaper.WithDeleteNotFound(true),
 		reaper.WithPCache(pc),
-		reaper.WithTopicName(testTopic),
 	)
 	require.NoError(t, err)
 

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/ipld/go-ipld-prime v0.21.0
 	github.com/ipld/go-ipld-prime/storage/dsadapter v0.0.0-20230102063945-1a409dc236dd
 	github.com/ipni/go-indexer-core v0.8.10
-	github.com/ipni/go-libipni v0.5.16
+	github.com/ipni/go-libipni v0.6.5
 	github.com/libp2p/go-libp2p v0.33.2
 	github.com/libp2p/go-msgio v0.3.0
 	github.com/mitchellh/go-homedir v1.1.0
@@ -64,7 +64,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bep/debounce v1.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cockroachdb/errors v1.11.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
@@ -83,11 +82,6 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
-	github.com/filecoin-project/go-cbor-util v0.0.1 // indirect
-	github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 // indirect
-	github.com/filecoin-project/go-ds-versioning v0.1.2 // indirect
-	github.com/filecoin-project/go-statemachine v1.0.2 // indirect
-	github.com/filecoin-project/go-statestore v0.2.0 // indirect
 	github.com/flynn/noise v1.1.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -106,8 +100,6 @@ require (
 	github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect
 	github.com/google/uuid v1.5.0 // indirect
 	github.com/gorilla/websocket v1.5.1 // indirect
-	github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect
-	github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -118,14 +110,12 @@ require (
 	github.com/ipfs/bbloom v0.0.4 // indirect
 	github.com/ipfs/go-block-format v0.2.0 // indirect
 	github.com/ipfs/go-blockservice v0.5.1 // indirect
-	github.com/ipfs/go-graphsync v0.16.0 // indirect
 	github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
 	github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
 	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
 	github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
 	github.com/ipfs/go-ipfs-files v0.3.0 // indirect
 	github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
-	github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
 	github.com/ipfs/go-ipfs-util v0.0.3 // indirect
 	github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
 	github.com/ipfs/go-ipld-format v0.6.0 // indirect
@@ -134,15 +124,13 @@ require (
 	github.com/ipfs/go-log v1.0.5 // indirect
 	github.com/ipfs/go-merkledag v0.11.0 // indirect
 	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
-	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
 	github.com/ipfs/go-unixfs v0.4.5 // indirect
 	github.com/ipfs/go-verifcid v0.0.2 // indirect
 	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/klauspost/compress v1.17.6 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect