3510 notify flag (#3744)

* Makes NotifyChunkWritten a method on ScrapeOptions

* Separates out NotifyConfigured from GetNotifyEndpoint

* Moves validation to validate.go where it belongs. Requires --notify flag to enable notifications.

* Surrounds Notify with tests for enablement

* Fixes tests

* Fixes tests

src/apps/chifra/internal/scrape/notify.go

@@ -16,20 +16,24 @@ import (
 
 var ErrConfiguredButNotRunning = fmt.Errorf("listener is configured but not running")
 
-// NotifyConfigured returns true if notification feature is configured
-func NotifyConfigured() (bool, string) {
+// GetNotifyEndpoint returns the notification endpoint
+func GetNotifyEndpoint() string {
 	endpoint := config.GetSettings().Notify.Url
-
 	// If protocol is not specified, use http by default
 	if endpoint != "" && !strings.HasPrefix(endpoint, "http") {
 		endpoint = "http://" + endpoint
 	}
-	return endpoint != "", endpoint
+	return endpoint
+}
+
+// NotifyConfigured returns true if notification feature is configured
+func NotifyConfigured() bool {
+	return GetNotifyEndpoint() != ""
 }
 
 // Notify may be used to tell other processes about progress.
 func Notify[T notify.NotificationPayload](notification notify.Notification[T]) error {
-	_, endpoint := NotifyConfigured()
+	endpoint := GetNotifyEndpoint()
 	if endpoint == "" {
 		return nil
 	}

src/apps/chifra/internal/scrape/scrape_blaze.go

@@ -192,13 +192,15 @@ func (bm *BlazeManager) WriteAppearances(bn base.Blknum, addrMap uniq.AddressBoo
 		}
 	}
 
-	if bn <= bm.ripeBlock {
+	if bm.opts.Notify && bn <= bm.ripeBlock {
 		err = Notify(notify.Notification[[]notify.NotificationPayloadAppearance]{
 			Msg:     notify.MessageAppearance,
 			Meta:    bm.meta,
 			Payload: notificationPayload,
 		})
 		if err != nil {
+			// We need this warning, otherwise errors don't show up for 2,000 blocks
+			logger.Error("error sending notification", err)
 			return err
 		}
 	}

src/apps/chifra/internal/scrape/scrape_chunk_notify.go

@@ -7,13 +7,19 @@ import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/notify"
 )
 
-func NotifyChunkWritten(chunk index.Chunk, chunkPath string) (err error) {
+func (opts *ScrapeOptions) NotifyChunkWritten(chunk index.Chunk, chunkPath string) (err error) {
+	if !opts.Notify {
+		return nil
+	}
+
+	// If --notify is on, it's properly configured and IPFS is running
 	var cidString string
-	if ok, _ := NotifyConfigured(); ok {
+	if config.IpfsRunning() { // probablyh redundant
 		if cidString, err = index.ChunkCid(chunkPath); err != nil {
 			return err
 		}
 	}
+
 	// Generate range from path, as chunks sometimes don't have Range set
 	chunkRange := base.RangeFromFilename(index.ToIndexPath(chunkPath))
 	return Notify(notify.Notification[[]notify.NotificationPayloadChunkWritten]{

src/apps/chifra/internal/scrape/scrape_consolidate.go

@@ -117,7 +117,7 @@ func (bm *BlazeManager) Consolidate(blocks []base.Blknum) (error, bool) {
 				report.FileSize = file.FileSize(chunkPath)
 				report.Report()
 			}
-			if err = NotifyChunkWritten(chunk, chunkPath); err != nil {
+			if err = bm.opts.NotifyChunkWritten(chunk, chunkPath); err != nil {
 				return err, true
 			}
 
@@ -159,12 +159,14 @@ func (bm *BlazeManager) Consolidate(blocks []base.Blknum) (error, bool) {
 	nAppsNow := int(file.FileSize(stageFn) / asciiAppearanceSize)
 	bm.report(len(blocks), int(bm.PerChunk()), nChunks, nAppsNow, nAppsFound, nAddrsFound)
 
-	if err := Notify(notify.Notification[string]{
-		Msg:     notify.MessageStageUpdated,
-		Meta:    bm.meta,
-		Payload: newRange.String(),
-	}); err != nil {
-		return err, true
+	if bm.opts.Notify {
+		if err := Notify(notify.Notification[string]{
+			Msg:     notify.MessageStageUpdated,
+			Meta:    bm.meta,
+			Payload: newRange.String(),
+		}); err != nil {
+			return err, true
+		}
 	}
 
 	// Commit the change by deleting the backup file.

src/apps/chifra/internal/scrape/scrape_prepare.go

@@ -23,13 +23,6 @@ import (
 func (opts *ScrapeOptions) Prepare() (ok bool, err error) {
 	chain := opts.Globals.Chain
 
-	// Notify feature requires IPFS daemon to be running.
-	if ok, _ := NotifyConfigured(); ok {
-		if !config.IpfsRunning() {
-			logger.Fatal("notify requires IPFS daemon")
-		}
-	}
-
 	// We always clean the temporary folders (other than staging) when starting
 	_ = cleanEphemeralIndexFolders(chain)
 
@@ -74,7 +67,7 @@ func (opts *ScrapeOptions) Prepare() (ok bool, err error) {
 		report.FileSize = file.FileSize(indexPath)
 		report.Report()
 	}
-	if err = NotifyChunkWritten(chunk, indexPath); err != nil {
+	if err = opts.NotifyChunkWritten(chunk, indexPath); err != nil {
 		return false, err
 	}
 

src/apps/chifra/internal/scrape/validate.go

@@ -12,6 +12,7 @@ import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/validate"
 )
@@ -21,6 +22,7 @@ import (
 
 func (opts *ScrapeOptions) validateScrape() error {
 	chain := opts.Globals.Chain
+	testMode := opts.Globals.TestMode
 
 	opts.testLog()
 
@@ -32,6 +34,18 @@ func (opts *ScrapeOptions) validateScrape() error {
 		return validate.Usage("chain {0} is not properly configured.", chain)
 	}
 
+	if opts.Notify {
+		if !NotifyConfigured() {
+			return validate.Usage("The {0} feature is {1}.", "--notify", "not properly configured. See the README.md")
+		}
+		if !config.IpfsRunning() {
+			return validate.Usage("The {0} option requires {1}.", "--notify", "a locally running IPFS daemon")
+		}
+	} else if !testMode && NotifyConfigured() {
+		msg := validate.Usage("The notify feature is configured but not running. Enable it with the {0} flag.", "--notify").Error()
+		logger.Warn(msg)
+	}
+
 	err, ok := opts.Conn.IsNodeTracing()
 	if !ok {
 		return validate.Usage("{0} requires {1}, try {2} instead. Error: {3}", "chifra scrape", "tracing", "chifra init", err.Error())