chore: hardcode worker options and remove flags

docs/_data/curio_scan.yaml

@@ -18,18 +18,10 @@ options:
       default_value: 3s
       usage: |
         Set timeout when attempting to resolve detected domains during classification, e.g. --domain-resolution-timeout=3s
-    - name: existing-worker
-      usage: Specify the URL of an existing worker.
     - name: external-rule-dir
       default_value: '[]'
       usage: |
         Specify directories paths that contain .yaml files with external rules configuration
-    - name: file-size-max
-      default_value: "2000000"
-      usage: Ignore files larger than the specified value.
-    - name: files-to-batch
-      default_value: "1"
-      usage: Specify the number of files to batch per worker.
     - name: force
       default_value: "false"
       usage: Disable the cache and runs the detections again
@@ -44,10 +36,6 @@ options:
       default_value: '[]'
       usage: |
         Define regular expressions for better classification of private or unreachable domains e.g. --internal-domains=".*.my-company.com,private.sh"
-    - name: memory-max
-      default_value: "800000000"
-      usage: |
-        If the memory needed to scan a file surpasses the specified limit, skip the file.
     - name: only-rule
       default_value: '[]'
       usage: |
@@ -68,24 +56,6 @@ options:
       default_value: '[]'
       usage: |
         Specify the comma-separated ids of the rules you would like to skip. Runs all other rules.
-    - name: timeout
-      default_value: 10m0s
-      usage: The maximum time alloted to complete the scan.
-    - name: timeout-file-max
-      default_value: 30s
-      usage: |
-        Maximum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes.
-    - name: timeout-file-min
-      default_value: 5s
-      usage: |
-        Minimum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes.
-    - name: timeout-file-second-per-bytes
-      default_value: "10000"
-      usage: |
-        number of file size bytes producing a second of timeout assigned to scanning a file
-    - name: timeout-worker-online
-      default_value: 1m0s
-      usage: Maximum time to wait for a worker process to come online.
 example: |4-
       # Scan a local project, including language-specific files
       $ curio scan /path/to/your_project

integration/flags/.snapshots/TestInitCommand

@@ -15,14 +15,4 @@ scan:
     internal-domains: []
     quiet: false
     skip-path: []
-worker:
-    existing-worker: ""
-    file-size-max: 2000000
-    files-to-batch: 1
-    memory-max: 800000000
-    timeout: 10m0s
-    timeout-file-max: 30s
-    timeout-file-min: 5s
-    timeout-file-second-per-bytes: 10000
-    timeout-worker-online: 1m0s
 

integration/flags/.snapshots/TestMetadataFlags-help-scan

@@ -32,17 +32,6 @@ Scan Flags
 General Flags
       --config-file string   Load configuration from the specified path.
 
-Worker Flags
-      --existing-worker string              Specify the URL of an existing worker.
-      --file-size-max int                   Ignore files larger than the specified value. (default 2000000)
-      --files-to-batch int                  Specify the number of files to batch per worker. (default 1)
-      --memory-max int                      If the memory needed to scan a file surpasses the specified limit, skip the file. (default 800000000)
-      --timeout duration                    The maximum time alloted to complete the scan. (default 10m0s)
-      --timeout-file-max duration           Maximum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes. (default 30s)
-      --timeout-file-min duration           Minimum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes. (default 5s)
-      --timeout-file-second-per-bytes int   number of file size bytes producing a second of timeout assigned to scanning a file (default 10000)
-      --timeout-worker-online duration      Maximum time to wait for a worker process to come online. (default 1m0s)
-
 
 --
 

integration/flags/.snapshots/TestMetadataFlags-scan-help

@@ -32,17 +32,6 @@ Scan Flags
 General Flags
       --config-file string   Load configuration from the specified path.
 
-Worker Flags
-      --existing-worker string              Specify the URL of an existing worker.
-      --file-size-max int                   Ignore files larger than the specified value. (default 2000000)
-      --files-to-batch int                  Specify the number of files to batch per worker. (default 1)
-      --memory-max int                      If the memory needed to scan a file surpasses the specified limit, skip the file. (default 800000000)
-      --timeout duration                    The maximum time alloted to complete the scan. (default 10m0s)
-      --timeout-file-max duration           Maximum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes. (default 30s)
-      --timeout-file-min duration           Minimum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes. (default 5s)
-      --timeout-file-second-per-bytes int   number of file size bytes producing a second of timeout assigned to scanning a file (default 10000)
-      --timeout-worker-online duration      Maximum time to wait for a worker process to come online. (default 1m0s)
-
 
 --
 

integration/flags/.snapshots/TestReportFlagsShouldFail-invalid-context-flag

@@ -33,17 +33,6 @@ Scan Flags
 General Flags
       --config-file string   Load configuration from the specified path.
 
-Worker Flags
-      --existing-worker string              Specify the URL of an existing worker.
-      --file-size-max int                   Ignore files larger than the specified value. (default 2000000)
-      --files-to-batch int                  Specify the number of files to batch per worker. (default 1)
-      --memory-max int                      If the memory needed to scan a file surpasses the specified limit, skip the file. (default 800000000)
-      --timeout duration                    The maximum time alloted to complete the scan. (default 10m0s)
-      --timeout-file-max duration           Maximum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes. (default 30s)
-      --timeout-file-min duration           Minimum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes. (default 5s)
-      --timeout-file-second-per-bytes int   number of file size bytes producing a second of timeout assigned to scanning a file (default 10000)
-      --timeout-worker-online duration      Maximum time to wait for a worker process to come online. (default 1m0s)
-
 
 flag error: scan flag error: invalid context argument; supported values: health
 

integration/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag

@@ -33,17 +33,6 @@ Scan Flags
 General Flags
       --config-file string   Load configuration from the specified path.
 
-Worker Flags
-      --existing-worker string              Specify the URL of an existing worker.
-      --file-size-max int                   Ignore files larger than the specified value. (default 2000000)
-      --files-to-batch int                  Specify the number of files to batch per worker. (default 1)
-      --memory-max int                      If the memory needed to scan a file surpasses the specified limit, skip the file. (default 800000000)
-      --timeout duration                    The maximum time alloted to complete the scan. (default 10m0s)
-      --timeout-file-max duration           Maximum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes. (default 30s)
-      --timeout-file-min duration           Minimum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes. (default 5s)
-      --timeout-file-second-per-bytes int   number of file size bytes producing a second of timeout assigned to scanning a file (default 10000)
-      --timeout-worker-online duration      Maximum time to wait for a worker process to come online. (default 1m0s)
-
 
 flag error: report flags error: invalid format argument; supported values: json, yaml
 

integration/flags/.snapshots/TestReportFlagsShouldFail-invalid-report-flag

@@ -33,17 +33,6 @@ Scan Flags
 General Flags
       --config-file string   Load configuration from the specified path.
 
-Worker Flags
-      --existing-worker string              Specify the URL of an existing worker.
-      --file-size-max int                   Ignore files larger than the specified value. (default 2000000)
-      --files-to-batch int                  Specify the number of files to batch per worker. (default 1)
-      --memory-max int                      If the memory needed to scan a file surpasses the specified limit, skip the file. (default 800000000)
-      --timeout duration                    The maximum time alloted to complete the scan. (default 10m0s)
-      --timeout-file-max duration           Maximum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes. (default 30s)
-      --timeout-file-min duration           Minimum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes. (default 5s)
-      --timeout-file-second-per-bytes int   number of file size bytes producing a second of timeout assigned to scanning a file (default 10000)
-      --timeout-worker-online duration      Maximum time to wait for a worker process to come online. (default 1m0s)
-
 
 flag error: report flags error: invalid report argument; supported values: summary, dataflow, stats
 

pkg/commands/artifact/run.go

@@ -196,7 +196,13 @@ func (r *runner) scanArtifact(ctx context.Context, opts flag.Options) (types.Rep
 
 // Run performs artifact scanning
 func Run(ctx context.Context, opts flag.Options, targetKind TargetKind) (err error) {
-	ctx, cancel := context.WithTimeout(ctx, opts.Timeout)
+	scanSettings, err := settings.FromOptions(opts)
+	scanSettings.Target = opts.Target
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, scanSettings.Worker.Timeout)
 	defer cancel()
 
 	defer func() {
@@ -205,13 +211,6 @@ func Run(ctx context.Context, opts flag.Options, targetKind TargetKind) (err err
 		}
 	}()
 
-	scanSettings, err := settings.FromOptions(opts)
-	scanSettings.Target = opts.Target
-
-	if err != nil {
-		return err
-	}
-
 	r := NewRunner(ctx, scanSettings)
 	defer r.Close(ctx)
 

pkg/commands/process/balancer/filelist/filelist_test.go

@@ -30,7 +30,7 @@ func TestFileList(t *testing.T) {
 			Input: input{
 				projectPath: filepath.Join("testdata", "happy_path", "standard"),
 				config: settings.Config{
-					Worker: flag.WorkerOptions{
+					Worker: settings.WorkerOptions{
 						FileSizeMaximum:           100000,
 						TimeoutFileSecondPerBytes: 1,
 					},
@@ -51,7 +51,7 @@ func TestFileList(t *testing.T) {
 					Scan: flag.ScanOptions{
 						SkipPath: []string{"users/admin.go"},
 					},
-					Worker: flag.WorkerOptions{
+					Worker: settings.WorkerOptions{
 						FileSizeMaximum:           100000,
 						TimeoutFileSecondPerBytes: 1,
 					},
@@ -72,7 +72,7 @@ func TestFileList(t *testing.T) {
 					Scan: flag.ScanOptions{
 						SkipPath: []string{"users"},
 					},
-					Worker: flag.WorkerOptions{
+					Worker: settings.WorkerOptions{
 						FileSizeMaximum:           100000,
 						TimeoutFileSecondPerBytes: 1,
 					},

pkg/commands/process/settings/settings.go

@@ -3,6 +3,7 @@ package settings
 import (
 	"embed"
 	"fmt"
+	"time"
 
 	"github.com/rs/zerolog/log"
 	"gopkg.in/yaml.v2"
@@ -11,8 +12,34 @@ import (
 	"github.com/bearer/curio/pkg/util/rego"
 )
 
+var (
+	Workers                   = 1                 // The number of processing workers to spawn
+	Timeout                   = 10 * time.Minute  // "The maximum time alloted to complete the scan
+	TimeoutFileMinimum        = 5 * time.Second   // Minimum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes
+	TimeoutFileMaximum        = 30 * time.Second  // Maximum timeout assigned for scanning each file. This config superseeds timeout-second-per-bytes
+	TimeoutFileSecondPerBytes = 10 * 1000         // 10kb/s number of file size bytes producing a second of timeout assigned to scanning a file
+	TimeoutWorkerOnline       = 60 * time.Second  // Maximum time to wait for a worker process to come online
+	FileSizeMaximum           = 2 * 1000 * 1000   // 2MB Ignore files larger than the specified value
+	FilesToBatch              = 1                 // Specify the number of files to batch per worker
+	MemoryMaximum             = 800 * 1000 * 1000 // 800 MB If the memory needed to scan a file surpasses the specified limit, skip the file.
+	ExistingWorker            = ""                // Specify the URL of an existing worker
+)
+
+type WorkerOptions struct {
+	Workers                   int           `mapstructure:"workers" json:"workers" yaml:"workers"`
+	Timeout                   time.Duration `mapstructure:"timeout" json:"timeout" yaml:"timeout"`
+	TimeoutFileMinimum        time.Duration `mapstructure:"timeout-file-min" json:"timeout-file-min" yaml:"timeout-file-min"`
+	TimeoutFileMaximum        time.Duration `mapstructure:"timeout-file-max"  json:"timeout-file-max" yaml:"timeout-file-max"`
+	TimeoutFileSecondPerBytes int           `mapstructure:"timeout-file-second-per-bytes" json:"timeout-file-second-per-bytes" yaml:"timeout-file-second-per-bytes"`
+	TimeoutWorkerOnline       time.Duration `mapstructure:"timeout-worker-online" json:"timeout-worker-online" yaml:"timeout-worker-online"`
+	FileSizeMaximum           int           `mapstructure:"file-size-max" json:"file-size-max" yaml:"file-size-max"`
+	FilesToBatch              int           `mapstructure:"files-to-batch" json:"files-to-batch" yaml:"files-to-batch"`
+	MemoryMaximum             int           `mapstructure:"memory-max" json:"memory-max" yaml:"memory-max"`
+	ExistingWorker            string        `mapstructure:"existing-worker" json:"existing-worker" yaml:"existing-worker"`
+}
+
 type Config struct {
-	Worker       flag.WorkerOptions `mapstructure:"worker" json:"worker" yaml:"worker"`
+	Worker       WorkerOptions      `mapstructure:"worker" json:"worker" yaml:"worker"`
 	Scan         flag.ScanOptions   `mapstructure:"scan" json:"scan" yaml:"scan"`
 	Report       flag.ReportOptions `mapstructure:"report" json:"report" yaml:"report"`
 	Policies     map[string]*Policy `mapstructure:"policies" json:"policies" yaml:"policies"`
@@ -168,8 +195,24 @@ func (rule *Rule) PolicyType() bool {
 	return true
 }
 
+func defaultWorkerOptions() WorkerOptions {
+	return WorkerOptions{
+		Workers:                   Workers,
+		Timeout:                   Timeout,
+		TimeoutFileMinimum:        TimeoutFileMinimum,
+		TimeoutFileMaximum:        TimeoutFileMaximum,
+		TimeoutFileSecondPerBytes: TimeoutFileSecondPerBytes,
+		TimeoutWorkerOnline:       TimeoutWorkerOnline,
+		FilesToBatch:              FilesToBatch,
+		FileSizeMaximum:           FileSizeMaximum,
+		MemoryMaximum:             MemoryMaximum,
+		ExistingWorker:            ExistingWorker,
+	}
+}
+
 func FromOptions(opts flag.Options) (Config, error) {
 	policies := DefaultPolicies()
+	workerOptions := defaultWorkerOptions()
 
 	builtInRules, rules, err := loadRules(opts.ExternalRuleDir, opts.RuleOptions)
 	if err != nil {
@@ -191,7 +234,7 @@ func FromOptions(opts flag.Options) (Config, error) {
 	}
 
 	config := Config{
-		Worker:       opts.WorkerOptions,
+		Worker:       workerOptions,
 		Scan:         opts.ScanOptions,
 		Report:       opts.ReportOptions,
 		Policies:     policies,

pkg/commands/scan.go

@@ -31,7 +31,6 @@ Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "he
 var scanFlags = &flag.Flags{
 	ScanFlagGroup:    flag.NewScanFlagGroup(),
 	RuleFlagGroup:    flag.NewRuleFlagGroup(),
-	WorkerFlagGroup:  flag.NewWorkerFlagGroup(),
 	ReportFlagGroup:  flag.NewReportFlagGroup(),
 	GeneralFlagGroup: flag.NewGeneralFlagGroup(),
 }

pkg/flag/options.go

@@ -46,7 +46,6 @@ type Flags struct {
 	RuleFlagGroup    *RuleFlagGroup
 	ProcessFlagGroup *ProcessFlagGroup
 	ScanFlagGroup    *ScanFlagGroup
-	WorkerFlagGroup  *WorkerFlagGroup
 	GeneralFlagGroup *GeneralFlagGroup
 }
 
@@ -55,7 +54,6 @@ type Options struct {
 	RepoOptions
 	ReportOptions
 	RuleOptions
-	WorkerOptions
 	ScanOptions
 	GeneralOptions
 }
@@ -137,13 +135,6 @@ func getStringSlice(flag *Flag) []string {
 	return v
 }
 
-func getInt(flag *Flag) int {
-	if flag == nil {
-		return 0
-	}
-	return viper.GetInt(flag.ConfigName)
-}
-
 func getBool(flag *Flag) bool {
 	if flag == nil {
 		return false
@@ -173,9 +164,6 @@ func (f *Flags) groups() []FlagGroup {
 	if f.GeneralFlagGroup != nil {
 		groups = append(groups, f.GeneralFlagGroup)
 	}
-	if f.WorkerFlagGroup != nil {
-		groups = append(groups, f.WorkerFlagGroup)
-	}
 	if f.ProcessFlagGroup != nil {
 		groups = append(groups, f.ProcessFlagGroup)
 	}
@@ -266,10 +254,6 @@ func (f *Flags) ToOptions(args []string) (Options, error) {
 		opts.RuleOptions = f.RuleFlagGroup.ToOptions(args)
 	}
 
-	if f.WorkerFlagGroup != nil {
-		opts.WorkerOptions = f.WorkerFlagGroup.ToOptions()
-	}
-
 	if f.ScanFlagGroup != nil {
 		opts.ScanOptions, err = f.ScanFlagGroup.ToOptions(args)
 		if err != nil {