Merge pull request #69 from projectdiscovery/dev

v0.0.7

Dockerfile

@@ -1,7 +1,7 @@
 FROM golang:1.18.2-alpine3.14 AS build-env
 RUN go install -v github.com/projectdiscovery/uncover/cmd/uncover@latest
 
-FROM alpine:3.16.1
+FROM alpine:3.16.2
 RUN apk add --no-cache bind-tools ca-certificates
 COPY --from=build-env /go/bin/uncover /usr/local/bin/uncover
 ENTRYPOINT ["uncover"]

README.md

@@ -25,7 +25,7 @@
 
 ---
 
-**uncover** is a go wrapper using APIs of well known search engines to quickly discover exposed hosts on the internet. It is built with automation in mind, so you can query it and utilize the results with your current pipeline tools. Currently, it supports **shodan**,**shodan-internetdb**, **censys**, and **fofa** search API.
+**uncover** is a go wrapper using APIs of well known search engines to quickly discover exposed hosts on the internet. It is built with automation in mind, so you can query it and utilize the results with your current pipeline tools.
 
 # Features
 
@@ -34,20 +34,26 @@
   <br>
 </h1>
 
-- Simple and Handy utility to query multiple search engine
-- Multiple Search engine support (**Shodan**, **Censys**, **Fofa**, **Shodan-InternetDB**)
-- Automatic key/credential randomization
-- **stdin** / **stdout** support for input and output
+- Query multiple search engine at once
+- Available Search engine support
+  - **[Shodan](https://www.shodan.io)**
+  - **[Censys](https://search.censys.io)**
+  - **[FOFA](https://fofa.info)**
+  - **[Hunter](https://hunter.qianxin.com)**
+  - **[Quake](https://quake.360.net/quake/#/index)**
+- Multiple API key input support
+- Automatic API key randomization
+- **stdin** / **stdout** support for input
 
-# Installation Instructions
+## Installation Instructions
 
 uncover requires **go1.17** to install successfully. Run the following command to get the repo -
 
 ```sh
 go install -v github.com/projectdiscovery/uncover/cmd/uncover@latest
 ```
 
-# Usage
+## Usage
 
 ```sh
 uncover -h
@@ -61,15 +67,23 @@ Usage:
 
 Flags:
 INPUT:
-   -q, -query string[]   search query or list (file or comma separated or stdin)
-   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys) (default shodan)
+   -q, -query string[]   search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')
+   -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter) (default shodan)
+
+SEARCH-ENGINE:
+   -s, -shodan string[]       search query for shodan (example: -shodan 'query.txt')
+   -sd, -shodan-idb string[]  search query for shodan-idb (example: -shodan-idb 'query.txt')
+   -ff, -fofa string[]        search query for fofa (example: -fofa 'query.txt')
+   -cs, -censys string[]      search query for censys (example: -censys 'query.txt')
+   -qk, -quake string[]       search query for quake (example: -quake 'query.txt')
+   -ht, -hunter string[]      search query for hunter (example: -hunter 'query.txt')
 
 CONFIG:
    -pc, -provider string  provider configuration file (default "$HOME/.config/uncover/provider-config.yaml")
    -config string         flag configuration file (default "$HOME/.config/uncover/config.yaml")
    -timeout int           timeout in seconds (default 30)
    -delay int             delay between requests in seconds (0 to disable) (default 1)
-   -retries int           number of times to retry a failed request
+   -retry int             number of times to retry a failed request (default 2)
 
 OUTPUT:
    -o, -output string  output file to write found results
@@ -85,18 +99,29 @@ DEBUG:
    -v        show verbose output
 ```
 
-# Provider Configuration
+## Provider Configuration
+
+The default provider configuration file should be located at `$HOME/.config/uncover/provider-config.yaml` and has the following contents as an example.
+
 
-The default provider configuration file should be located at `$HOME/.config/uncover/provider-config.yaml` and has the following contents as an example. **In order to run this tool, the API keys / credentials needs to be added in this config file or set as environment variable.**
+> **Note**: API keys are required needs to be configured before running uncover.
 
 ```yaml
 shodan:
-  - SHODAN_API_KEY1
-  - SHODAN_API_KEY2
+  - SHODAN_API_KEY_1
+  - SHODAN_API_KEY_2
 censys:
-  - CENSYS_API_ID:CENSYS_API_SECRET
+  - CENSYS_API_ID_1:CENSYS_API_SECRET_1
+  - CENSYS_API_ID_2:CENSYS_API_SECRET_2
 fofa:
-  - FOFA_EMAIL:FOFA_KEY
+  - FOFA_EMAIL_1:FOFA_KEY_2
+  - FOFA_EMAIL_2:FOFA_KEY_2
+quake:
+  - QUAKE_TOKEN_1
+  - QUAKE_TOKEN_2
+hunter:
+  - HUNTER_API_KEY_1
+  - HUNTER_API_KEY_2
 ```
 
 When multiple keys/credentials are specified for same provider in the config file, random key will be used for each execution.
@@ -109,13 +134,16 @@ export CENSYS_API_ID=xxx
 export CENSYS_API_SECRET=xxx
 export FOFA_EMAIL=xxx
 export FOFA_KEY=xxx
+export HUNTER_API_KEY=xxx
 ```
 
-Required keys can be obtained by signing up on [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin).
+Required API keys can be obtained by signing up on following platform [Shodan](https://account.shodan.io/register), [Censys](https://censys.io/register), [Fofa](https://fofa.info/toLogin), [Quake](https://quake.360.net/quake/#/index) and [Hunter](https://user.skyeye.qianxin.com/user/register?next=https%3A//hunter.qianxin.com/api/uLogin&fromLogin=1) .
 
 ## Running Uncover
 
-**uncover** supports multiple ways to make the query including **stdin** or `q` flag
+### Default run:
+
+**uncover** supports multiple ways to make the query including **stdin** or `q` flag, as default `shodan` engine is used for search if no engine is specified.
 
 ```console
 echo 'ssl:"Uber Technologies, Inc."' | uncover 
@@ -182,13 +210,13 @@ uncover -q dorks.txt
 138.197.147.213:8086
 ```
 
-### Multiple Search Engine API (Shodan,Censys,Fofa)
+### Single query against multiple search engine
 
 
-**uncover** supports multiple search engine, as default **shodan** is used, `engine` flag can be used to specify any available search engines.
+**uncover** supports multiple search engine, as default **shodan** is used, `-e` flag can be used to run same query against any or all search engines.
 
 ```console
-echo jira | uncover -e shodan,censys,fofa
+echo jira | uncover -e shodan,censys,fofa,quake,hunter
 
   __  ______  _________ _   _____  _____
  / / / / __ \/ ___/ __ \ | / / _ \/ ___/
@@ -216,6 +244,43 @@ echo jira | uncover -e shodan,censys,fofa
 42.194.226.30:2626
 ```
 
+### Multiple query against multiple search engine
+
+
+```console
+uncover -shodan 'http.component:"Atlassian Jira"' -censys 'services.software.product=`Jira`' -fofa 'app="ATLASSIAN-JIRA"' -quake 'Jira' -hunter 'Jira'
+
+  __  ______  _________ _   _____  _____
+ / / / / __ \/ ___/ __ \ | / / _ \/ ___/
+/ /_/ / / / / /__/ /_/ / |/ /  __/ /    
+\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.7
+                                        
+
+    projectdiscovery.io
+
+[WRN] Use with caution. You are responsible for your actions
+[WRN] Developers assume no liability and are not responsible for any misuse or damage.
+[WRN] By using uncover, you also agree to the terms of the APIs used.
+
+104.68.37.129:443
+162.222.160.42:443
+34.255.84.133:443
+52.204.121.166:443
+23.198.29.120:443
+136.156.180.95:443
+54.194.233.15:443
+104.117.55.155:443
+149.81.4.6:443
+54.255.218.95:443
+3.223.137.57:443
+83.228.124.171:443
+23.202.195.82:443
+52.16.59.25:443
+18.159.145.227:443
+104.105.53.236:443
+```
+
+
 ### Shodan-InternetDB API
 
 **uncover** supports [shodan-internetdb](https://internetdb.shodan.io) API to pull available ports for given IP/CIDR input.
@@ -249,7 +314,7 @@ echo 51.83.59.99/24 | uncover
 51.83.59.3:993
 ```
 
-### Field Filters
+### Field Format
 
 `-f, -field` flag can be used to indicate which fields to return, currently, `ip`, `port`, and `host` are supported and can be used to return desired fields.
 
@@ -311,8 +376,7 @@ https://129.206.117.248
 ## Notes:
 
 -  **keys/ credentials** are required to configure before running or using this project.
-- `query` flag supports all the filters supported by underlying API in use.
-- `query` flag input needs be compatible with search engine in use.
+- `query` flag supports **all and only filters supported by search engine.**
 - results are limited to `100` as default and can be increased with `limit` flag.
 - `shodan-idb` API doesn't requires an API key and works out of the box.
 - `shodan-idb` API is used as **default** engine when **IP/CIDR** is provided as input.

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/projectdiscovery/fdmax v0.0.3
 	github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c
 	github.com/projectdiscovery/folderutil v0.0.0-20220212074351-38f1c1d2fdd4
-	github.com/projectdiscovery/goflags v0.0.8
+	github.com/projectdiscovery/goflags v0.0.9
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/iputil v0.0.0-20220613112553-9b6873b2c619
 	github.com/projectdiscovery/mapcidr v1.0.1

go.sum

@@ -63,8 +63,9 @@ github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h
 github.com/projectdiscovery/folderutil v0.0.0-20220212074351-38f1c1d2fdd4 h1:rPA+enZtrHCBO0XVGsvoGzBKv9TlqZyqZguh1AOT+gs=
 github.com/projectdiscovery/folderutil v0.0.0-20220212074351-38f1c1d2fdd4/go.mod h1:BMqXH4jNGByVdE2iLtKvc/6XStaiZRuCIaKv1vw9PnI=
 github.com/projectdiscovery/goflags v0.0.8-0.20220426153734-2ffbfbff923c/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA=
-github.com/projectdiscovery/goflags v0.0.8 h1:IhTmnEGSKtBHfD23tSwpnzai5CRYfLKVOWlVq9ngYvY=
 github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80=
+github.com/projectdiscovery/goflags v0.0.9 h1:bPsYIPE1LvdgYaM3XNX0YmS68e6huv22W22rKh5IscI=
+github.com/projectdiscovery/goflags v0.0.9/go.mod h1:t/dEhv2VDOzayugXZCkbkX8n+pPeVmRD+WgQRSgReeI=
 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=

runner/banners.go

@@ -8,11 +8,11 @@ const banner = `
   __  ______  _________ _   _____  _____
  / / / / __ \/ ___/ __ \ | / / _ \/ ___/
 / /_/ / / / / /__/ /_/ / |/ /  __/ /    
-\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.6
+\__,_/_/ /_/\___/\____/|___/\___/_/ v0.0.7
 `
 
 // Version is the current version of uncover
-const Version = `v0.0.6`
+const Version = `v0.0.7`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

runner/options.go

@@ -41,6 +41,12 @@ type Options struct {
 	delay        time.Duration
 	Provider     *Provider
 	Retries      int
+	Shodan       goflags.StringSlice
+	ShodanIdb    goflags.StringSlice
+	Fofa         goflags.StringSlice
+	Censys       goflags.StringSlice
+	Quake        goflags.StringSlice
+	Hunter       goflags.StringSlice
 }
 
 // ParseOptions parses the command line flags provided by a user
@@ -52,15 +58,24 @@ func ParseOptions() *Options {
 
 	flagSet.CreateGroup("input", "Input",
 		flagSet.StringSliceVarP(&options.Query, "query", "q", nil, "search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')", goflags.FileStringSliceOptions),
-		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake) (default shodan)", goflags.FileNormalizedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter) (default shodan)", goflags.FileNormalizedStringSliceOptions),
+	)
+
+	flagSet.CreateGroup("search-engine", "Search-Engine",
+		flagSet.StringSliceVarP(&options.Shodan, "shodan", "s", nil, "search query for shodan (example: -shodan 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.ShodanIdb, "shodan-idb", "sd", nil, "search query for shodan-idb (example: -shodan-idb 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Fofa, "fofa", "ff", nil, "search query for fofa (example: -fofa 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Censys, "censys", "cs", nil, "search query for censys (example: -censys 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Quake, "quake", "qk", nil, "search query for quake (example: -quake 'query.txt')", goflags.FileStringSliceOptions),
+		flagSet.StringSliceVarP(&options.Hunter, "hunter", "ht", nil, "search query for hunter (example: -hunter 'query.txt')", goflags.FileStringSliceOptions),
 	)
 
 	flagSet.CreateGroup("config", "Config",
 		flagSet.StringVarP(&options.ProviderFile, "provider", "pc", defaultProviderConfigLocation, "provider configuration file"),
 		flagSet.StringVar(&options.ConfigFile, "config", defaultConfigLocation, "flag configuration file"),
 		flagSet.IntVar(&options.Timeout, "timeout", 30, "timeout in seconds"),
 		flagSet.IntVar(&options.Delay, "delay", 1, "delay between requests in seconds (0 to disable)"),
-		flagSet.IntVar(&options.Retries, "retries", 1, "number of times to retry a failed request"),
+		flagSet.IntVar(&options.Retries, "retry", 2, "number of times to retry a failed request"),
 	)
 
 	flagSet.CreateGroup("output", "Output",
@@ -175,14 +190,17 @@ func (options *Options) loadProvidersFromEnv() error {
 			return errors.New("missing fofa key")
 		}
 	}
+	if key, exists := os.LookupEnv("HUNTER_API_KEY"); exists {
+		options.Provider.Hunter = append(options.Provider.Hunter, key)
+	}
 	return nil
 }
 
 // validateOptions validates the configuration options passed
 func (options *Options) validateOptions() error {
 	// Check if domain, list of domains, or stdin info was provided.
 	// If none was provided, then return.
-	if len(options.Query) == 0 {
+	if len(options.Query) == 0 && len(options.Shodan) == 0 && len(options.Censys) == 0 && len(options.Quake) == 0 && len(options.Fofa) == 0 && len(options.ShodanIdb) == 0 && len(options.Hunter) == 0 {
 		return errors.New("no query provided")
 	}
 

runner/output_writer.go

@@ -2,10 +2,12 @@ package runner
 
 import (
 	"crypto/sha1"
+	"fmt"
 	"io"
 	"sync"
 
 	lru "github.com/hashicorp/golang-lru"
+	"github.com/projectdiscovery/uncover/uncover"
 )
 
 type OutputWriter struct {
@@ -30,19 +32,30 @@ func (o *OutputWriter) Write(data []byte) {
 	o.Lock()
 	defer o.Unlock()
 
-	// skip duplicates in the last 2048 printed items
-	itemHash := sha1.Sum(data)
-	if o.cache.Contains(itemHash) {
-		return
-	}
-	o.cache.Add(itemHash, struct{}{})
-
 	for _, writer := range o.writers {
 		_, _ = writer.Write(data)
 		_, _ = writer.Write([]byte("\n"))
 	}
 }
+func (o *OutputWriter) findDuplicate(data string) bool {
+	// check if we've already printed this data
+	itemHash := sha1.Sum([]byte(data))
+	if o.cache.Contains(itemHash) {
+		return true
+	}
+	o.cache.Add(itemHash, struct{}{})
+	return false
+}
 
 func (o *OutputWriter) WriteString(data string) {
+	if o.findDuplicate(data) {
+		return
+	}
 	o.Write([]byte(data))
 }
+func (o *OutputWriter) WriteJsonData(data uncover.Result) {
+	if o.findDuplicate(fmt.Sprintf("%s:%d", data.IP, data.Port)) {
+		return
+	}
+	o.Write([]byte(data.JSON()))
+}

runner/provider.go

@@ -12,6 +12,7 @@ type Provider struct {
 	Censys []string `yaml:"censys"`
 	Fofa   []string `yaml:"fofa"`
 	Quake  []string `yaml:"quake"`
+	Hunter []string `yaml:"hunter"`
 }
 
 func (provider *Provider) GetKeys() uncover.Keys {
@@ -43,9 +44,13 @@ func (provider *Provider) GetKeys() uncover.Keys {
 		keys.QuakeToken = provider.Quake[rand.Intn(len(provider.Quake))]
 	}
 
+	if len(provider.Hunter) > 0 {
+		keys.HunterToken = provider.Hunter[rand.Intn(len(provider.Hunter))]
+	}
+
 	return keys
 }
 
 func (provider *Provider) HasKeys() bool {
-	return len(provider.Censys) > 0 || len(provider.Shodan) > 0 || len(provider.Fofa) > 0 || len(provider.Quake) > 0
+	return len(provider.Censys) > 0 || len(provider.Shodan) > 0 || len(provider.Fofa) > 0 || len(provider.Quake) > 0 || len(provider.Hunter) > 0
 }