Add Netlas Search Engine (#97)

* Add Netlas search engine

* Fix if engine cond

* update go*

* fixed env typo

* workflow token update

Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>

.github/workflows/provider-integration.yml

@@ -25,6 +25,7 @@ jobs:
           SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}}
           ZOOMEYE_API_KEY: ${{secrets.ZOOMEYE_API_KEY}}
           QUAKE_API_KEY: ${{secrets.QUAKE_TOKEN}}
+          NETLAS_API_KEY: ${{secrets.NETLAS_API_KEY}}
         run: bash run.sh
         working-directory: integration-tests/
 

.gitignore

@@ -1,3 +1,7 @@
+# Uncover
+cmd/uncover/uncover
+integration-tests/integration-test
+
 # Binaries for programs and plugins
 *.exe
 *.exe~

README.md

@@ -42,6 +42,7 @@
   - **[Hunter](https://hunter.qianxin.com)**
   - **[Quake](https://quake.360.net/quake/#/index)**
   - **[Zoomeye](https://www.zoomeye.org)**
+  - **[Netlas](https://netlas.io/)**
 - Multiple API key input support
 - Automatic API key randomization
 - **stdin** / **stdout** support for input
@@ -69,7 +70,7 @@ Usage:
 Flags:
 INPUT:
    -q, -query string[]   search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')
-   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye) (default shodan)
+   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas) (default shodan)
 
 SEARCH-ENGINE:
    -s, -shodan string[]       search query for shodan (example: -shodan 'query.txt')
@@ -79,6 +80,7 @@ SEARCH-ENGINE:
    -qk, -quake string[]       search query for quake (example: -quake 'query.txt')
    -ht, -hunter string[]      search query for hunter (example: -hunter 'query.txt')
    -ze, -zoomeye string[]     search query for zoomeye (example: -zoomeye 'query.txt')
+   -ne, -netlas string[]      search query for netlas (example: -netlas 'query.txt')
 
 CONFIG:
    -pc, -provider string  provider configuration file (default "$HOME/.config/uncover/provider-config.yaml")
@@ -127,6 +129,9 @@ hunter:
 zoomeye:
   - ZOOMEYE_API_KEY_1
   - ZOOMEYE_API_KEY_2
+netlas:
+  - NETLAS_API_KEY_1
+  - NETLAS_API_KEY_2
 ```
 
 When multiple keys/credentials are specified for same provider in the config file, random key will be used for each execution.
@@ -142,9 +147,10 @@ export FOFA_KEY=xxx
 export QUAKE_TOKEN=xxx
 export HUNTER_API_KEY=xxx
 export ZOOMEYE_API_KEY=xxx
+export NETLAS_API_KEY=xxx
 ```
 
-Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1) and [ZoomEye](https://www.zoomeye.org/login) .
+Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index), [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1), [ZoomEye](https://www.zoomeye.org/login) and [Netlas](https://app.netlas.io/registration/).
 
 ## Running Uncover
 
@@ -158,7 +164,7 @@ echo 'ssl:"Uber Technologies, Inc."' | uncover
   __  ______  _________ _   _____  _____
  / / / / __ \/ ___/ __ \ | / / _ \/ ___/
 / /_/ / / / / /__/ /_/ / |/ /  __/ /    
-\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.1    
+\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.9    
                                         
 
 		projectdiscovery.io
@@ -195,7 +201,7 @@ uncover -q dorks.txt
   __  ______  _________ _   _____  _____
  / / / / __ \/ ___/ __ \ | / / _ \/ ___/
 / /_/ / / / / /__/ /_/ / |/ /  __/ /    
-\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.1    
+\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.9    
                                         
 
     projectdiscovery.io
@@ -223,12 +229,12 @@ uncover -q dorks.txt
 **uncover** supports multiple search engine, as default **shodan** is used, `-e` flag can be used to run same query against any or all search engines.
 
 ```console
-echo jira | uncover -e shodan,censys,fofa,quake,hunter,zoomeye
+echo jira | uncover -e shodan,censys,fofa,quake,hunter,zoomeye,netlas
 
   __  ______  _________ _   _____  _____
  / / / / __ \/ ___/ __ \ | / / _ \/ ___/
 / /_/ / / / / /__/ /_/ / |/ /  __/ /    
-\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.3  
+\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.9  
                                         
 
     projectdiscovery.io
@@ -255,12 +261,12 @@ echo jira | uncover -e shodan,censys,fofa,quake,hunter,zoomeye
 
 
 ```console
-uncover -shodan 'http.component:"Atlassian Jira"' -censys 'services.software.product=`Jira`' -fofa 'app="ATLASSIAN-JIRA"' -quake 'Jira' -hunter 'Jira' -zoomeye 'app:"Atlassian JIRA"'
+uncover -shodan 'http.component:"Atlassian Jira"' -censys 'services.software.product=`Jira`' -fofa 'app="ATLASSIAN-JIRA"' -quake 'Jira' -hunter 'Jira' -zoomeye 'app:"Atlassian JIRA"' -netlas 'jira'
 
   __  ______  _________ _   _____  _____
  / / / / __ \/ ___/ __ \ | / / _ \/ ___/
 / /_/ / / / / /__/ /_/ / |/ /  __/ /    
-\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.7
+\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.9
                                         
 
     projectdiscovery.io
@@ -300,7 +306,7 @@ echo 51.83.59.99/24 | uncover
   __  ______  _________ _   _____  _____
  / / / / __ \/ ___/ __ \ | / / _ \/ ___/
 / /_/ / / / / /__/ /_/ / |/ /  __/ /    
-\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.3  
+\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.9  
                                         
 
     projectdiscovery.io

go.mod

@@ -46,6 +46,7 @@ require (
 	github.com/mholt/archiver v3.1.1+incompatible // indirect
 	github.com/nwaples/rardecode v1.1.0 // indirect
 	github.com/pierrec/lz4 v2.6.0+incompatible // indirect
+	github.com/projectdiscovery/iputil v0.0.2
 	github.com/ulikunitz/xz v0.5.7 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect

go.sum

@@ -110,6 +110,8 @@ github.com/projectdiscovery/ipranger v0.0.3/go.mod h1:pb7qOZyXI6n9Z5izLZmkTiunaB
 github.com/projectdiscovery/iputil v0.0.0-20210414194613-4b4d2517acf0/go.mod h1:PQAqn5h5NXsQTF4ZA00ZTYLRzGCjOtcCq8llAqrsd1A=
 github.com/projectdiscovery/iputil v0.0.0-20220625072148-037479960416/go.mod h1:bst7imnXn0ckAw4sToXLHLm62HisFGnS4QlxOkE4mYQ=
 github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8/go.mod h1:vHRC+9exsfSbEngMKDl0xiWqkxlLk3lHQZpbS2yFT8U=
+github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI=
+github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ=
 github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
 github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM=
 github.com/projectdiscovery/mapcidr v1.0.0/go.mod h1:5QkKrV6rNQQurCZI3nNedFsAOYp04mRDkC5yht+znYA=

integration-tests/integration-test.go

@@ -24,7 +24,8 @@ var (
 		"zoomeye": zoomeyeTestcases{},
 		"fofa":    fofaTestcases{},
 		//"hunter":  hunterTestcases{},
-		"quake": quakeTestcases{},
+		"quake":  quakeTestcases{},
+		"netlas": netlasTestcases{},
 	}
 )
 

integration-tests/source-test.go

@@ -116,3 +116,20 @@ func (h quakeTestcases) Execute() error {
 	}
 	return expectResultsGreaterThanCount(results, 0)
 }
+
+type netlasTestcases struct{}
+
+func (h netlasTestcases) Execute() error {
+	token := os.Getenv("NETLAS_API_KEY")
+	if token == "" {
+		return errors.New("missing netlas api key")
+	}
+	netlasToken := fmt.Sprintf(`netlas: [%s]`, token)
+	_ = os.WriteFile(ConfigFile, []byte(netlasToken), 0644)
+	defer os.RemoveAll(ConfigFile)
+	results, err := testutils.RunUncoverAndGetResults(debug, "-netlas", "'Grafana'")
+	if err != nil {
+		return err
+	}
+	return expectResultsGreaterThanCount(results, 0)
+}

runner/options.go

@@ -46,6 +46,7 @@ type Options struct {
 	Fofa         goflags.StringSlice
 	Censys       goflags.StringSlice
 	Quake        goflags.StringSlice
+	Netlas       goflags.StringSlice
 	Hunter       goflags.StringSlice
 	ZoomEye      goflags.StringSlice
 }
@@ -59,7 +60,7 @@ func ParseOptions() *Options {
 
 	flagSet.CreateGroup("input", "Input",
 		flagSet.StringSliceVarP(&options.Query, "query", "q", nil, "search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')", goflags.FileStringSliceOptions),
-		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye) (default shodan)", goflags.FileNormalizedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas) (default shodan)", goflags.FileNormalizedStringSliceOptions),
 	)
 
 	flagSet.CreateGroup("search-engine", "Search-Engine",
@@ -70,6 +71,7 @@ func ParseOptions() *Options {
 		flagSet.StringSliceVarP(&options.Quake, "quake", "qk", nil, "search query for quake (example: -quake 'query.txt')", goflags.FileStringSliceOptions),
 		flagSet.StringSliceVarP(&options.Hunter, "hunter", "ht", nil, "search query for hunter (example: -hunter 'query.txt')", goflags.FileStringSliceOptions),
 		flagSet.StringSliceVarP(&options.ZoomEye, "zoomeye", "ze", nil, "search query for zoomeye (example: -zoomeye 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Netlas, "netlas", "ne", nil, "search query for netlas (example: -netlas 'query.txt')", goflags.FileStringSliceOptions),
 	)
 
 	flagSet.CreateGroup("config", "Config",
@@ -126,7 +128,15 @@ func ParseOptions() *Options {
 		gologger.Warning().Msgf("couldn't parse env vars: %s\n", err)
 	}
 
-	if len(options.Engine) == 0 && len(options.Shodan) == 0 && len(options.Censys) == 0 && len(options.Quake) == 0 && len(options.Fofa) == 0 && len(options.ShodanIdb) == 0 && len(options.Hunter) == 0 && len(options.ZoomEye) == 0 {
+	if len(options.Engine) == 0 &&
+		len(options.Shodan) == 0 &&
+		len(options.Censys) == 0 &&
+		len(options.Quake) == 0 &&
+		len(options.Fofa) == 0 &&
+		len(options.ShodanIdb) == 0 &&
+		len(options.Hunter) == 0 &&
+		len(options.ZoomEye) == 0 &&
+		len(options.Netlas) == 0 {
 		options.Engine = append(options.Engine, "shodan")
 		options.Engine = append(options.Engine, "shodan-idb")
 	}
@@ -201,14 +211,25 @@ func (options *Options) loadProvidersFromEnv() error {
 	if key, exists := os.LookupEnv("ZOOMEYE_API_KEY"); exists {
 		options.Provider.ZoomEye = append(options.Provider.ZoomEye, key)
 	}
+	if key, exists := os.LookupEnv("NETLAS_API_KEY"); exists {
+		options.Provider.Netlas = append(options.Provider.Netlas, key)
+	}
 	return nil
 }
 
 // validateOptions validates the configuration options passed
 func (options *Options) validateOptions() error {
 	// Check if domain, list of domains, or stdin info was provided.
 	// If none was provided, then return.
-	if len(options.Query) == 0 && len(options.Shodan) == 0 && len(options.Censys) == 0 && len(options.Quake) == 0 && len(options.Fofa) == 0 && len(options.ShodanIdb) == 0 && len(options.Hunter) == 0 && len(options.ZoomEye) == 0 {
+	if len(options.Query) == 0 &&
+		len(options.Shodan) == 0 &&
+		len(options.Censys) == 0 &&
+		len(options.Quake) == 0 &&
+		len(options.Fofa) == 0 &&
+		len(options.ShodanIdb) == 0 &&
+		len(options.Hunter) == 0 &&
+		len(options.ZoomEye) == 0 &&
+		len(options.Netlas) == 0 {
 		return errors.New("no query provided")
 	}
 
@@ -218,7 +239,15 @@ func (options *Options) validateOptions() error {
 	}
 
 	// Validate threads and options
-	if len(options.Engine) == 0 && len(options.Shodan) == 0 && len(options.Censys) == 0 && len(options.Quake) == 0 && len(options.Fofa) == 0 && len(options.ShodanIdb) == 0 && len(options.Hunter) == 0 && len(options.ZoomEye) == 0 {
+	if len(options.Engine) == 0 &&
+		len(options.Shodan) == 0 &&
+		len(options.Censys) == 0 &&
+		len(options.Quake) == 0 &&
+		len(options.Fofa) == 0 &&
+		len(options.ShodanIdb) == 0 &&
+		len(options.Hunter) == 0 &&
+		len(options.ZoomEye) == 0 &&
+		len(options.Netlas) == 0 {
 		return errors.New("no engine specified")
 	}
 

runner/provider.go

@@ -14,6 +14,7 @@ type Provider struct {
 	Quake   []string `yaml:"quake"`
 	Hunter  []string `yaml:"hunter"`
 	ZoomEye []string `yaml:"zoomeye"`
+	Netlas  []string `yaml:"netlas"`
 }
 
 func (provider *Provider) GetKeys() uncover.Keys {
@@ -53,9 +54,19 @@ func (provider *Provider) GetKeys() uncover.Keys {
 		keys.ZoomEyeToken = provider.ZoomEye[rand.Intn(len(provider.ZoomEye))]
 	}
 
+	if len(provider.Netlas) > 0 {
+		keys.NetlasToken = provider.Netlas[rand.Intn(len(provider.Netlas))]
+	}
+
 	return keys
 }
 
 func (provider *Provider) HasKeys() bool {
-	return len(provider.Censys) > 0 || len(provider.Shodan) > 0 || len(provider.Fofa) > 0 || len(provider.Quake) > 0 || len(provider.Hunter) > 0 || len(provider.ZoomEye) > 0
+	return len(provider.Censys) > 0 ||
+		len(provider.Shodan) > 0 ||
+		len(provider.Fofa) > 0 ||
+		len(provider.Quake) > 0 ||
+		len(provider.Hunter) > 0 ||
+		len(provider.ZoomEye) > 0 ||
+		len(provider.Netlas) > 0
 }

runner/runner.go

@@ -11,17 +11,18 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/iputil"
 	"github.com/projectdiscovery/ratelimit"
+	"github.com/projectdiscovery/stringsutil"
 	"github.com/projectdiscovery/uncover/uncover"
 	"github.com/projectdiscovery/uncover/uncover/agent/censys"
 	"github.com/projectdiscovery/uncover/uncover/agent/fofa"
 	"github.com/projectdiscovery/uncover/uncover/agent/hunter"
+	"github.com/projectdiscovery/uncover/uncover/agent/netlas"
 	"github.com/projectdiscovery/uncover/uncover/agent/quake"
 	"github.com/projectdiscovery/uncover/uncover/agent/shodan"
 	"github.com/projectdiscovery/uncover/uncover/agent/shodanidb"
 	"github.com/projectdiscovery/uncover/uncover/agent/zoomeye"
-	iputil "github.com/projectdiscovery/utils/ip"
-	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
 func init() {
@@ -48,7 +49,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 		return errors.New("no keys provided")
 	}
 
-	var censysRateLimiter, fofaRateLimiter, shodanRateLimiter, shodanIdbRateLimiter, quakeRatelimiter, hunterRatelimiter, zoomeyeRatelimiter *ratelimit.Limiter
+	var censysRateLimiter, fofaRateLimiter, shodanRateLimiter, shodanIdbRateLimiter, quakeRatelimiter, hunterRatelimiter, zoomeyeRatelimiter, netlasRatelimiter *ratelimit.Limiter
 	if r.options.Delay > 0 {
 		censysRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay))
 		fofaRateLimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay))
@@ -57,6 +58,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 		quakeRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay))
 		hunterRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay))
 		zoomeyeRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay))
+		netlasRatelimiter = ratelimit.New(context.Background(), 1, time.Duration(r.options.Delay))
 	} else {
 		censysRateLimiter = ratelimit.NewUnlimited(context.Background())
 		fofaRateLimiter = ratelimit.NewUnlimited(context.Background())
@@ -65,6 +67,7 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 		quakeRatelimiter = ratelimit.NewUnlimited(context.Background())
 		hunterRatelimiter = ratelimit.NewUnlimited(context.Background())
 		zoomeyeRatelimiter = ratelimit.NewUnlimited(context.Background())
+		netlasRatelimiter = ratelimit.NewUnlimited(context.Background())
 	}
 
 	var agents []uncover.Agent
@@ -96,6 +99,10 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 		r.options.Engine = append(r.options.Engine, "zoomeye")
 		query = append(query, r.options.ZoomEye...)
 	}
+	if len(r.options.Netlas) > 0 {
+		r.options.Engine = append(r.options.Engine, "netlas")
+		query = append(query, r.options.Netlas...)
+	}
 
 	// declare clients
 	for _, engine := range r.options.Engine {
@@ -118,6 +125,8 @@ func (r *Runner) Run(ctx context.Context, query ...string) error {
 			agent, err = hunter.NewWithOptions(&uncover.AgentOptions{RateLimiter: hunterRatelimiter})
 		case "zoomeye":
 			agent, err = zoomeye.NewWithOptions(&uncover.AgentOptions{RateLimiter: zoomeyeRatelimiter})
+		case "netlas":
+			agent, err = netlas.NewWithOptions(&uncover.AgentOptions{RateLimiter: netlasRatelimiter})
 		default:
 			err = errors.New("unknown agent type")
 		}