Added recursive, all flags. More tuning of process

pkg/passive/sources.go

@@ -34,8 +34,48 @@ import (
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/zoomeye"
 )
 
-// DefaultSources contains the list of sources used by default
+// DefaultSources contains the list of fast sources used by default.
 var DefaultSources = []string{
+	"alienvault",
+	"binaryedge",
+	"bufferover",
+	"certspotter",
+	"certspotterold",
+	"censys",
+	"chaos",
+	"crtsh",
+	"dnsdumpster",
+	"hackertarget",
+	"intelx",
+	"ipv4info",
+	"passivetotal",
+	"securitytrails",
+	"shodan",
+	"spyse",
+	"sublist3r",
+	"threatcrowd",
+	"threatminer",
+	"virustotal",
+}
+
+// DefaultRecursiveSources contains list of default recursive sources
+var DefaultRecursiveSources = []string{
+	"alienvault",
+	"bufferover",
+	"certspotter",
+	"certspotterold",
+	"crtsh",
+	"dnsdumpster",
+	"hackertarget",
+	"ipv4info",
+	"passivetotal",
+	"securitytrails",
+	"sublist3r",
+	"virustotal",
+}
+
+// DefaultAllSources contains list of all sources
+var DefaultAllSources = []string{
 	"alienvault",
 	"archiveis",
 	"binaryedge",

pkg/runner/banners.go

@@ -32,6 +32,20 @@ func (options *Options) normalRunTasks() {
 	if err != nil {
 		gologger.Fatalf("Could not read configuration file %s: %s\n", options.ConfigFile, err)
 	}
+
+	// If we have a different version of subfinder installed
+	// previously, use the new iteration of config file.
+	if configFile.Version != Version {
+		configFile.Sources = passive.DefaultSources
+		configFile.AllSources = passive.DefaultAllSources
+		configFile.Recursive = passive.DefaultRecursiveSources
+		configFile.Version = Version
+
+		err = configFile.MarshalWrite(options.ConfigFile)
+		if err != nil {
+			gologger.Fatalf("Could not update configuration file to %s: %s\n", options.ConfigFile, err)
+		}
+	}
 	options.YAMLConfig = configFile
 }
 
@@ -45,6 +59,10 @@ func (options *Options) firstRunTasks() {
 		Resolvers: resolve.DefaultResolvers,
 		// Use the default list of passive sources
 		Sources: passive.DefaultSources,
+		// Use the default list of all passive sources
+		AllSources: passive.DefaultAllSources,
+		// Use the default list of recursive sources
+		Recursive: passive.DefaultRecursiveSources,
 	}
 
 	err := config.MarshalWrite(options.ConfigFile)

pkg/runner/config.go

@@ -22,6 +22,10 @@ type ConfigFile struct {
 	Resolvers []string `yaml:"resolvers,omitempty"`
 	// Sources contains a list of sources to use for enumeration
 	Sources []string `yaml:"sources,omitempty"`
+	// AllSources contains the list of all sources for enumeration (slow)
+	AllSources []string `yaml:"all-sources,omitempty"`
+	// Recrusive contains the list of recursive subdomain enum sources
+	Recursive []string `yaml:"recursive,omitempty"`
 	// ExcludeSources contains the sources to not include in the enumeration process
 	ExcludeSources []string `yaml:"exclude-sources,omitempty"`
 	// API keys for different sources
@@ -39,6 +43,8 @@ type ConfigFile struct {
 	URLScan        []string `yaml:"urlscan"`
 	Virustotal     []string `yaml:"virustotal"`
 	ZoomEye        []string `yaml:"zoomeye"`
+	// Version indicates the version of subfinder installed.
+	Version string `yaml:"subfinder-version"`
 }
 
 // GetConfigDirectory gets the subfinder config directory for a user

pkg/runner/initialize.go

@@ -11,20 +11,31 @@ import (
 func (r *Runner) initializePassiveEngine() {
 	var sources, exclusions []string
 
-	// If there are any sources from CLI, only use them
-	// Otherwise, use the yaml file sources
-	if r.options.Sources != "" {
-		sources = append(sources, strings.Split(r.options.Sources, ",")...)
-	} else {
-		sources = append(sources, r.options.YAMLConfig.Sources...)
-	}
-
 	if r.options.ExcludeSources != "" {
 		exclusions = append(exclusions, strings.Split(r.options.ExcludeSources, ",")...)
 	} else {
 		exclusions = append(exclusions, r.options.YAMLConfig.ExcludeSources...)
 	}
 
+	// Use all sources if asked by the user
+	if r.options.All {
+		sources = append(sources, r.options.YAMLConfig.AllSources...)
+	}
+
+	// If only recursive sources are wanted, use them only.
+	if r.options.Recursive {
+		sources = append(sources, r.options.YAMLConfig.Recursive...)
+	}
+
+	// If there are any sources from CLI, only use them
+	// Otherwise, use the yaml file sources
+	if !r.options.All && !r.options.Recursive {
+		if r.options.Sources != "" {
+			sources = append(sources, strings.Split(r.options.Sources, ",")...)
+		} else {
+			sources = append(sources, r.options.YAMLConfig.Sources...)
+		}
+	}
 	r.passiveAgent = passive.New(sources, exclusions)
 }
 

pkg/runner/options.go

@@ -23,6 +23,8 @@ type Options struct {
 	RemoveWildcard     bool   // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results.
 	Stdin              bool   // Stdin specifies whether stdin input was given to the process
 	Version            bool   // Version specifies if we should just show version and exit
+	Recursive          bool   // Recursive specifies whether to use only recursive subdomain enumeration sources
+	All                bool   // All specifies whether to use all (slow) sources.
 	Threads            int    // Thread controls the number of threads to use for active enumerations
 	Timeout            int    // Timeout is the seconds to wait for sources to respond
 	MaxEnumerationTime int    // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration
@@ -62,6 +64,8 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.JSON, "oJ", false, "Write output in JSON lines Format")
 	flag.BoolVar(&options.HostIP, "oI", false, "Write output in Host,IP format")
 	flag.BoolVar(&options.Silent, "silent", false, "Show only subdomains in output")
+	flag.BoolVar(&options.Recursive, "recursive", false, "Use only recursive subdomain enumeration sources")
+	flag.BoolVar(&options.All, "all", false, "Use all sources (slow) for enumeration")
 	flag.StringVar(&options.Sources, "sources", "", "Comma separated list of sources to use")
 	flag.BoolVar(&options.ListSources, "ls", false, "List all available sources")
 	flag.StringVar(&options.ExcludeSources, "exclude-sources", "", "List of sources to exclude from enumeration")
@@ -133,7 +137,7 @@ func listSources(options *Options) {
 		needsKey[strings.ToLower(keysElem.Type().Field(i).Name)] = keysElem.Field(i).Interface()
 	}
 
-	for _, source := range options.YAMLConfig.Sources {
+	for _, source := range options.YAMLConfig.AllSources {
 		message := "%s\n"
 		if _, ok := needsKey[source]; ok {
 			message = "%s *\n"