Final 3 (#3731)

* Fixes an issue with back level GoLang version on user's machine * Fixes an issue with chifra chunks --count * Updates tests * Fixes an issue with goMaker and duplicate members in a structure * Fixes issues with testRunner * Fixes issue #3712 * Fixes issue #3712 * Fixes issue #3696 * Fixing tests * Fixing tests * Fixing tests * Fixing tests * Cleaning * Improves goMaker to eliminate duplicate longNames and distinquish enums better. Prepares for auto-gen of deprecatoin. * Fixes tests for chifra daemon * Updates tests
TrueBlocks · May 31, 2024 · 7c76975 · 7c76975
1 parent 459b670
commit 7c76975
Show file tree

Hide file tree

Showing 18 changed files with 162 additions and 73 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -103,6 +103,8 @@ This file details changes made to TrueBlocks over time. See the [migration notes
 - chifra monitors
   - enabled chifra monitors --decache
 - chifra slurp
+  - deprecated `--types` in favor of `--parts` to be consistent with other endpoints
+  - exposes previously hidden `--page`, `--page_id`, and `--per_page`.
   - adds `--source` which allows specification of a provider: etherscan, key, covalent, or alchemy
   - Removes `--raw` option.
   - Adds `--ether` option to all sources shows ether for all commands.

diff --git a/docs/content/chifra/admin.md b/docs/content/chifra/admin.md
@@ -153,6 +153,7 @@ Flags:
 Notes:
   - To start API open terminal window and run chifra daemon.
   - See the API documentation (https://trueblocks.io/api) for more information.
+  - The --port option is deprecated, use --url instead.
 ```
 
 Data models produced by this tool:

diff --git a/sdk/daemon.go b/sdk/daemon.go
@@ -26,7 +26,6 @@ type DaemonOptions struct {
 	Scrape  DaemonScrape `json:"scrape,omitempty"`
 	Monitor bool         `json:"monitor,omitempty"`
 	Grpc    bool         `json:"grpc,omitempty"`
-	Port    string       `json:"port,omitempty"`
 	Silent  bool         `json:"silent,omitempty"`
 	Globals
 }

diff --git a/sdk/daemon_internal.go b/sdk/daemon_internal.go
@@ -25,7 +25,6 @@ type daemonOptionsInternal struct {
 	Scrape  DaemonScrape `json:"scrape,omitempty"`
 	Monitor bool         `json:"monitor,omitempty"`
 	Grpc    bool         `json:"grpc,omitempty"`
-	Port    string       `json:"port,omitempty"`
 	Silent  bool         `json:"silent,omitempty"`
 	Globals
 }
@@ -97,7 +96,6 @@ func (opts *DaemonOptions) toInternal() *daemonOptionsInternal {
 		Scrape:  opts.Scrape,
 		Monitor: opts.Monitor,
 		Grpc:    opts.Grpc,
-		Port:    opts.Port,
 		Silent:  opts.Silent,
 		Globals: opts.Globals,
 	}

diff --git a/sdk/state.go b/sdk/state.go
@@ -52,24 +52,24 @@ func (opts *StateOptions) StateCall(val string) ([]types.Result, *types.MetaData
 type StateParts int
 
 const (
-	NoSP      StateParts = 0
+	NoSTP     StateParts = 0
 	SPBalance            = 1 << iota
 	SPNonce
 	SPCode
 	SPProxy
 	SPDeployed
 	SPAccttype
-	SPSome = SPBalance | SPProxy | SPDeployed | SPAccttype
-	SPAll  = SPBalance | SPNonce | SPCode | SPProxy | SPDeployed | SPAccttype
+	STPSome = SPBalance | SPProxy | SPDeployed | SPAccttype
+	STPAll  = SPBalance | SPNonce | SPCode | SPProxy | SPDeployed | SPAccttype
 )
 
 func (v StateParts) String() string {
 	switch v {
-	case NoSP:
+	case NoSTP:
 		return "none"
-	case SPSome:
+	case STPSome:
 		return "some"
-	case SPAll:
+	case STPAll:
 		return "all"
 	}
 
@@ -94,13 +94,13 @@ func (v StateParts) String() string {
 
 func enumFromStateParts(values []string) (StateParts, error) {
 	if len(values) == 0 {
-		return NoSP, fmt.Errorf("no value provided for parts option")
+		return NoSTP, fmt.Errorf("no value provided for parts option")
 	}
 
 	if len(values) == 1 && values[0] == "all" {
-		return SPAll, nil
+		return STPAll, nil
 	} else if len(values) == 1 && values[0] == "some" {
-		return SPSome, nil
+		return STPSome, nil
 	}
 
 	var result StateParts
@@ -119,7 +119,7 @@ func enumFromStateParts(values []string) (StateParts, error) {
 		case "accttype":
 			result |= SPAccttype
 		default:
-			return NoSP, fmt.Errorf("unknown parts: %s", val)
+			return NoSTP, fmt.Errorf("unknown parts: %s", val)
 		}
 	}
 

diff --git a/src/apps/chifra/cmd/daemon.go b/src/apps/chifra/cmd/daemon.go
@@ -44,7 +44,8 @@ const longDaemon = `Purpose:
 const notesDaemon = `
 Notes:
   - To start API open terminal window and run chifra daemon.
-  - See the API documentation (https://trueblocks.io/api) for more information.`
+  - See the API documentation (https://trueblocks.io/api) for more information.
+  - The --port option is deprecated, use --url instead.`
 
 func init() {
 	var capabilities caps.Capability // capabilities for chifra daemon
@@ -62,20 +63,21 @@ One of [ off | on ]`)
 One of [ off | blooms | index ]`)
 	daemonCmd.Flags().BoolVarP(&daemonPkg.GetOptions().Monitor, "monitor", "m", false, `instruct the node to start the monitors tool (hidden)`)
 	daemonCmd.Flags().BoolVarP(&daemonPkg.GetOptions().Grpc, "grpc", "g", false, `run gRPC server to serve names`)
-	daemonCmd.Flags().StringVarP(&daemonPkg.GetOptions().Port, "port", "p", ":8080", `deprecated please use --url flag instead`)
 	daemonCmd.Flags().BoolVarP(&daemonPkg.GetOptions().Silent, "silent", "", false, `disable logging (for use in SDK for example)`)
+	daemonCmd.Flags().StringVarP(&daemonPkg.GetOptions().Port, "port", "p", ":8080", `deprecated, use --url instead (hidden)`)
 	if os.Getenv("TEST_MODE") != "true" {
 		_ = daemonCmd.Flags().MarkHidden("api")
 		_ = daemonCmd.Flags().MarkHidden("scrape")
 		_ = daemonCmd.Flags().MarkHidden("monitor")
+		_ = daemonCmd.Flags().MarkHidden("port")
 	}
+	_ = daemonCmd.Flags().MarkDeprecated("port", "The --port option has been deprecated.")
 	globals.InitGlobals("daemon", daemonCmd, &daemonPkg.GetOptions().Globals, capabilities)
 
 	daemonCmd.SetUsageTemplate(UsageWithNotes(notesDaemon))
 	daemonCmd.SetOut(os.Stderr)
 
 	// EXISTING_CODE
-	_ = daemonCmd.Flags().MarkDeprecated("port", "The --port option has been deprecated. Please use --url instead.") //cmd is the parent/top level command
 	// EXISTING_CODE
 
 	chifraCmd.AddCommand(daemonCmd)

diff --git a/src/apps/chifra/internal/daemon/README.md b/src/apps/chifra/internal/daemon/README.md
@@ -39,6 +39,7 @@ Flags:
 Notes:
   - To start API open terminal window and run chifra daemon.
   - See the API documentation (https://trueblocks.io/api) for more information.
+  - The --port option is deprecated, use --url instead.
 ```
 
 Data models produced by this tool:

diff --git a/src/apps/chifra/internal/daemon/options.go b/src/apps/chifra/internal/daemon/options.go
@@ -30,8 +30,8 @@ type DaemonOptions struct {
 	Scrape  string                `json:"scrape,omitempty"`  // Start the scraper, initialize it with either just blooms or entire index, generate for new blocks
 	Monitor bool                  `json:"monitor,omitempty"` // Instruct the node to start the monitors tool
 	Grpc    bool                  `json:"grpc,omitempty"`    // Run gRPC server to serve names
-	Port    string                `json:"port,omitempty"`    // Deprecated please use --url flag instead
 	Silent  bool                  `json:"silent,omitempty"`  // Disable logging (for use in SDK for example)
+	Port    string                `json:"port,omitempty"`    // Deprecated, use --url instead
 	Globals globals.GlobalOptions `json:"globals,omitempty"` // The global options
 	Conn    *rpc.Connection       `json:"conn,omitempty"`    // The connection to the RPC server
 	BadFlag error                 `json:"badFlag,omitempty"` // An error flag if needed
@@ -52,8 +52,8 @@ func (opts *DaemonOptions) testLog() {
 	logger.TestLog(len(opts.Scrape) > 0, "Scrape: ", opts.Scrape)
 	logger.TestLog(opts.Monitor, "Monitor: ", opts.Monitor)
 	logger.TestLog(opts.Grpc, "Grpc: ", opts.Grpc)
-	logger.TestLog(len(opts.Port) > 0 && opts.Port != ":8080", "Port: ", opts.Port)
 	logger.TestLog(opts.Silent, "Silent: ", opts.Silent)
+	logger.TestLog(len(opts.Port) > 0 && opts.Port != ":8080", "Port: ", opts.Port)
 	opts.Conn.TestLog(opts.getCaches())
 	opts.Globals.TestLog()
 }
@@ -92,10 +92,10 @@ func DaemonFinishParseInternal(w io.Writer, values url.Values) *DaemonOptions {
 			opts.Monitor = true
 		case "grpc":
 			opts.Grpc = true
-		case "port":
-			opts.Port = value[0]
 		case "silent":
 			opts.Silent = true
+		case "port":
+			opts.Port = value[0]
 		default:
 			if !copy.Globals.Caps.HasKey(key) {
 				err := validate.Usage("Invalid key ({0}) in {1} route.", key, "daemon")
@@ -107,12 +107,13 @@ func DaemonFinishParseInternal(w io.Writer, values url.Values) *DaemonOptions {
 	}
 	opts.Conn = opts.Globals.FinishParseApi(w, values, opts.getCaches())
 
-	// EXISTING_CODE
-	if len(opts.Port) > 0 && opts.Port != ":8080" && opts.Url == "localhost:8080" {
-		// deprecated, but still supported
+	// Deprecated, but still supported
+	if opts.Port != ":8080" && opts.Url == "localhost:8080" {
 		logger.Warn("The --port flag is deprecated. Please use --url instead.")
 		opts.Url = opts.Port
 	}
+
+	// EXISTING_CODE
 	// EXISTING_CODE
 
 	return opts
@@ -137,12 +138,13 @@ func daemonFinishParse(args []string) *DaemonOptions {
 	opts := GetOptions()
 	opts.Conn = opts.Globals.FinishParse(args, opts.getCaches())
 
-	// EXISTING_CODE
-	if len(opts.Port) > 0 && opts.Port != ":8080" && opts.Url == "localhost:8080" {
-		// deprecated, but still supported
+	// Deprecated, but still supported
+	if opts.Port != ":8080" && opts.Url == "localhost:8080" {
 		logger.Warn("The --port flag is deprecated. Please use --url instead.")
 		opts.Url = opts.Port
 	}
+
+	// EXISTING_CODE
 	// EXISTING_CODE
 	if len(opts.Globals.Format) == 0 || opts.Globals.Format == "none" {
 		opts.Globals.Format = defFmt

diff --git a/src/apps/chifra/pkg/usage/usage.go b/src/apps/chifra/pkg/usage/usage.go
@@ -21,12 +21,3 @@ func Replace(msg string, values ...string) string {
 func Usage(msg string, values ...string) error {
 	return errors.New(Replace(msg, values...))
 }
-
-// Deprecated can be used to mark a command option as deprecated.
-func Deprecated(cmd string, rep string) error {
-	msg := "The {0} flag has been deprecated."
-	if len(rep) > 0 {
-		msg += " Use {1} instead."
-	}
-	return Usage(msg, cmd, rep)
-}
diff --git a/src/dev_tools/goMaker/templates/cmd-line-options.csv b/src/dev_tools/goMaker/templates/cmd-line-options.csv
@@ -244,8 +244,8 @@ num,folder,group,route,tool,longName,hotKey,def_val,attributes,handler,option_ty
 44040,apps,Admin,daemon,flame,scrape,s,,,,flag,enum[off|blooms|index]>,,,,,start the scraper&#44; initialize it with either just blooms or entire index&#44; generate for new blocks
 44050,apps,Admin,daemon,flame,monitor,m,,,,switch,<boolean>,,,,,instruct the node to start the monitors tool
 44060,apps,Admin,daemon,flame,grpc,g,,visible|docs,,switch,<boolean>,,,,,run gRPC server to serve names
-44070,apps,Admin,daemon,flame,port,p,:8080,visible|docs,,flag,<string>,,,,,deprecated please use --url flag instead
 44070,apps,Admin,daemon,flame,silent,,,visible|docs,,switch,<boolean>,,,,,disable logging (for use in SDK for example)
+44070,apps,Admin,daemon,flame,port,p,:8080,deprecated=url,,flag,<string>,,,,,deprecated
 44080,apps,Admin,daemon,flame,n1,,,,,note,,,,,,To start API open terminal window and run chifra daemon.
 44090,apps,Admin,daemon,flame,n2,,,,,note,,,,,,See the API documentation (https://trueblocks.io/api) for more information.
 44100,apps,Admin,daemon,flame,a1,,,,,alias,,,,,,serve
@@ -291,11 +291,11 @@ num,folder,group,route,tool,longName,hotKey,def_val,attributes,handler,option_ty
 46240,apps,Admin,chunks,chunkMan,n3,,,,,note,,,,,,If blocks are provided&#44; only chunks intersecting with those blocks are displayed.
 46250,apps,Admin,chunks,chunkMan,n5,,,,,note,,,,,,The --truncate option updates the manifest and removes local data&#44; but does not alter remote pins.
 46260,apps,Admin,chunks,chunkMan,n6,,,,,note,,,,,,The --belongs option is only available in the index mode.
-46270,apps,Admin,chunks,chunkMan,n5,,,,,note,,,,,,The --first_block and --last_block options apply only to addresses&#44; appearances&#44; and index --belongs mode.
-46280,apps,Admin,chunks,chunkMan,n7,,,,,note,,,,,,The --pin option requires a locally running IPFS node or a pinning service API key.
-46290,apps,Admin,chunks,chunkMan,n8,,,,,note,,,,,,The --publish option requires a private key.
-46300,apps,Admin,chunks,chunkMan,n9,,,,,note,,,,,,The --publisher option is ignored with the --publish option since the sender of the transaction is recorded as the publisher.
-46310,apps,Admin,chunks,chunkMan,n10,,,,,note,,,,,,Without --rewrite&#44; the manifest is written to the temporary cache. With it&#44; the manifest is rewritten to the index folder.
+46270,apps,Admin,chunks,chunkMan,n7,,,,,note,,,,,,The --first_block and --last_block options apply only to addresses&#44; appearances&#44; and index --belongs mode.
+46280,apps,Admin,chunks,chunkMan,n8,,,,,note,,,,,,The --pin option requires a locally running IPFS node or a pinning service API key.
+46290,apps,Admin,chunks,chunkMan,n9,,,,,note,,,,,,The --publish option requires a private key.
+46300,apps,Admin,chunks,chunkMan,n10,,,,,note,,,,,,The --publisher option is ignored with the --publish option since the sender of the transaction is recorded as the publisher.
+46310,apps,Admin,chunks,chunkMan,n11,,,,,note,,,,,,Without --rewrite&#44; the manifest is written to the temporary cache. With it&#44; the manifest is rewritten to the index folder.
 #
 47000,apps,Admin,init,init,,,,visible|docs,,command,,,Initialize index,[flags],verbose|version|noop|noColor|chain|,Initialize the TrueBlocks system by downloading the Unchained Index from IPFS.
 47020,apps,Admin,init,init,all,a,,visible|docs,3,switch,<boolean>,message,,,,in addition to Bloom filters&#44; download full index chunks (recommended)
@@ -330,5 +330,5 @@ num,folder,group,route,tool,longName,hotKey,def_val,attributes,handler,option_ty
 53120,tools,Other,slurp,ethslurp,n1,,,,,note,,,,,,An `address` must be either an ENS name or start with '0x' and be forty-two characters long.
 53130,tools,Other,slurp,ethslurp,n2,,,,,note,,,,,,Portions of this software are Powered by Etherscan.io&#44; Covalent&#44; Alchemy&#44; TrueBlocks Key APIs.
 53131,tools,Other,slurp,ethslurp,n3,,,,,note,,,,,,See slurp/README on how to configure keys for API providers
-53140,tools,Other,slurp,ethslurp,n3,,,,,note,,,,,,The withdrawals option is only available on certain chains. It is ignored otherwise.
-53150,tools,Other,slurp,ethslurp,n4,,,,,note,,,,,,If the value of --source is key&#44; --types is ignored.
+53140,tools,Other,slurp,ethslurp,n4,,,,,note,,,,,,The withdrawals option is only available on certain chains. It is ignored otherwise.
+53150,tools,Other,slurp,ethslurp,n5,,,,,note,,,,,,If the value of --source is key&#44; --types is ignored.
diff --git a/src/dev_tools/goMaker/templates/sdk_route+internal.go.tmpl b/src/dev_tools/goMaker/templates/sdk_route+internal.go.tmpl
@@ -14,8 +14,8 @@ import (
 )
 
 type {{toCamel .Route}}OptionsInternal struct {
-{{range .Options}}	{{.GoSdkName}} {{.GoSdkType}} {{.JsonTag}}
-{{end}}	Globals
+{{range .Options}}{{if not .IsDeprecated}}	{{.GoSdkName}} {{.GoSdkType}} {{.JsonTag}}
+{{end}}{{end}}	Globals
 }
 
 // String implements the stringer interface

diff --git a/src/dev_tools/goMaker/templates/src_apps_chifra_cmd_route.go.tmpl b/src/dev_tools/goMaker/templates/src_apps_chifra_cmd_route.go.tmpl
@@ -49,7 +49,7 @@ func init() {
 {{if .HasHidden}}	if os.Getenv("TEST_MODE") != "true" {
 {{range .Options}}{{if .IsHidden}}	_ = {{.Route}}Cmd.Flags().MarkHidden("{{.LongName}}")
 {{end}}{{end}}	}
-{{end}}	globals.InitGlobals("{{.Route}}", {{.Route}}Cmd, &{{.Route}}Pkg.GetOptions().Globals, capabilities)
+{{end}}{{if .HasDeprecated}}{{.Deprecated}}{{end}}	globals.InitGlobals("{{.Route}}", {{.Route}}Cmd, &{{.Route}}Pkg.GetOptions().Globals, capabilities)
 
 	{{.Route}}Cmd.SetUsageTemplate(UsageWithNotes(notes{{toProper .Route}}))
 	{{.Route}}Cmd.SetOut(os.Stderr)

diff --git a/src/dev_tools/goMaker/templates/src_apps_chifra_internal_route_options.go.tmpl b/src/dev_tools/goMaker/templates/src_apps_chifra_internal_route_options.go.tmpl
@@ -60,7 +60,7 @@ func {{toProper .Route}}FinishParseInternal(w io.Writer, values url.Values) *{{t
 		}
 	}
 	opts.Conn = opts.Globals.FinishParseApi(w, values, opts.getCaches())
-{{.EnsConvert1}}
+{{.EnsConvert1}}{{if .HasDeprecated}}{{.DeprecatedTransfer}}{{end}}
 	// EXISTING_CODE
 	// EXISTING_CODE
 {{.EnsConvert2}}
@@ -85,7 +85,7 @@ func {{toLower .Route}}FinishParse(args []string) *{{toProper .Route}}Options {
 	defFmt := "txt"
 	opts := GetOptions()
 	opts.Conn = opts.Globals.FinishParse(args, opts.getCaches())
-{{.EnsConvert1}}
+{{.EnsConvert1}}{{if .HasDeprecated}}{{.DeprecatedTransfer}}{{end}}
 	// EXISTING_CODE
 	// EXISTING_CODE
 {{.EnsConvert2}}	if len(opts.Globals.Format) == 0 || opts.Globals.Format == "none" {

diff --git a/src/dev_tools/goMaker/types/load.go b/src/dev_tools/goMaker/types/load.go
@@ -30,12 +30,20 @@ func LoadCodebase() (CodeBase, error) {
 		return CodeBase{}, err
 	}
 
+	var cb CodeBase
 	options, err := LoadCsv[Option, any](thePath+"cmd-line-options.csv", readCmdOption, nil)
 	if err != nil {
-		return CodeBase{}, err
+		return cb, err
+	}
+	dupMap := make(map[string]bool, len(options))
+	for _, op := range options {
+		key := op.Route + ":" + op.LongName
+		if len(key) > 1 && dupMap[key] {
+			return cb, fmt.Errorf("duplicate option %s", key)
+		}
+		dupMap[key] = true
 	}
 
-	var cb CodeBase
 	structMap := make(map[string]Structure)
 	err = cb.LoadStructures(thePath+"classDefinitions/", readStructure, structMap)
 	if err != nil {