Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixing provider keys load #201

Merged
merged 5 commits into from
Apr 21, 2023
Merged

Fixing provider keys load #201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
175356e
Fixing provider keys load
Mzack9999 Apr 20, 2023
72d2db4
removing unused fields + eof skip
Mzack9999 Apr 20, 2023
63012c8
early exit + verbose output
Mzack9999 Apr 20, 2023
2078a9d
quake: show resp body in error
tarunKoyalwar Apr 21, 2023
5835ae5
quake: show unescaped error
tarunKoyalwar Apr 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions runner/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/projectdiscovery/gologger"
"github.com/projectdiscovery/gologger/formatter"
"github.com/projectdiscovery/gologger/levels"
"github.com/projectdiscovery/uncover/sources"
fileutil "github.com/projectdiscovery/utils/file"
folderutil "github.com/projectdiscovery/utils/folder"
genericutil "github.com/projectdiscovery/utils/generic"
Expand Down Expand Up @@ -78,7 +79,7 @@ func ParseOptions() *Options {
)

flagSet.CreateGroup("config", "Config",
flagSet.StringVarP(&options.ProviderFile, "provider", "pc", "", "provider configuration file"),
flagSet.StringVarP(&options.ProviderFile, "provider", "pc", sources.DefaultProviderConfigLocation, "provider configuration file"),
flagSet.StringVar(&options.ConfigFile, "config", defaultConfigLocation, "flag configuration file"),
flagSet.IntVar(&options.Timeout, "timeout", 30, "timeout in seconds"),
flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second"),
Expand Down Expand Up @@ -128,6 +129,10 @@ func ParseOptions() *Options {
_ = options.loadConfigFrom(options.ConfigFile)
}

if options.ProviderFile != sources.DefaultProviderConfigLocation {
sources.DefaultProviderConfigLocation = options.ProviderFile
}

if genericutil.EqualsAll(0,
len(options.Engine),
len(options.Shodan),
Expand Down Expand Up @@ -240,7 +245,6 @@ func appendQuery(options *Options, name string, queries ...string) {
}

func appendAllQueries(options *Options) {
var query []string = options.Query
appendQuery(options, "shodan", options.Shodan...)
appendQuery(options, "shodan-idb", options.ShodanIdb...)
appendQuery(options, "fofa", options.Fofa...)
Expand All @@ -252,5 +256,4 @@ func appendAllQueries(options *Options) {
appendQuery(options, "criminalip", options.CriminalIP...)
appendQuery(options, "publicwww", options.Publicwww...)
appendQuery(options, "hunterhow", options.HunterHow...)
options.Query = query
}
25 changes: 23 additions & 2 deletions sources/agent/quake/quake.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,12 @@ import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"

"github.com/projectdiscovery/uncover/sources"
errorutil "github.com/projectdiscovery/utils/errors"
)

const (
Expand Down Expand Up @@ -50,6 +53,11 @@ func (agent *Agent) Query(session *sources.Session, query *sources.Query) (chan
}

numberOfResults += len(quakeResponse.Data)

// early exit without more results
if quakeResponse.Meta.Pagination.Count > 0 && numberOfResults >= quakeResponse.Meta.Pagination.Total {
break
}
}
}()

Expand All @@ -64,8 +72,21 @@ func (agent *Agent) query(URL string, session *sources.Session, quakeRequest *Re
}

quakeResponse := &Response{}
if err := json.NewDecoder(resp.Body).Decode(quakeResponse); err != nil {
results <- sources.Result{Source: agent.Name(), Error: err}
respdata, err := io.ReadAll(resp.Body)
if err != nil {
results <- sources.Result{Source: agent.Name(), Error: fmt.Errorf("%v: %v", err, string(respdata))}
return nil
}
if err := json.NewDecoder(bytes.NewReader(respdata)).Decode(quakeResponse); err != nil {
errx := errorutil.NewWithErr(err)
// quake has different json format for error messages try to unmarshal it in map and print map
var errMap map[string]interface{}
if err := json.NewDecoder(bytes.NewReader(respdata)).Decode(&errMap); err == nil {
errx = errx.Msgf("failed to decode quake response: %v", errMap)
} else {
errx = errx.Msgf("failed to decode quake response: %s", string(respdata))
}
results <- sources.Result{Source: agent.Name(), Error: errx}
return nil
}

Expand Down
9 changes: 4 additions & 5 deletions sources/agent/quake/response.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,17 @@ type responseData struct {
}

type pagination struct {
Count int `json:"count"`
PageIndex int `json:"page_index"`
PageSize int `json:"page_size"`
Total int64 `json:"total"`
Count int `json:"count"`
PageIndex int `json:"page_index"`
PageSize int `json:"page_size"`
Total int `json:"total"`
}

type meta struct {
Pagination pagination `json:"pagination"`
}

type Response struct {
Code int `json:"code"`
Data []responseData `json:"data"`
Message string `json:"message"`
Meta meta `json:"meta"`
Expand Down
16 changes: 8 additions & 8 deletions sources/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,13 +108,13 @@ func (provider *Provider) LoadProviderKeysFromEnv() {
return arr
}
provider.Shodan = appendIfExists(provider.Shodan, "SHODAN_API_KEY")
provider.Hunter = appendIfExists(provider.Shodan, "HUNTER_API_KEY")
provider.Quake = appendIfExists(provider.Shodan, "QUAKE_TOKEN")
provider.ZoomEye = appendIfExists(provider.Shodan, "ZOOMEYE_API_KEY")
provider.Netlas = appendIfExists(provider.Shodan, "NETLAS_API_KEY")
provider.CriminalIP = appendIfExists(provider.Shodan, "CRIMINALIP_API_KEY")
provider.Publicwww = appendIfExists(provider.Shodan, "PUBLICWWW_API_KEY")
provider.HunterHow = appendIfExists(provider.Shodan, "HUNTERHOW_API_KEY")
provider.Hunter = appendIfExists(provider.Hunter, "HUNTER_API_KEY")
provider.Quake = appendIfExists(provider.Quake, "QUAKE_TOKEN")
provider.ZoomEye = appendIfExists(provider.ZoomEye, "ZOOMEYE_API_KEY")
provider.Netlas = appendIfExists(provider.Netlas, "NETLAS_API_KEY")
provider.CriminalIP = appendIfExists(provider.CriminalIP, "CRIMINALIP_API_KEY")
provider.Publicwww = appendIfExists(provider.Publicwww, "PUBLICWWW_API_KEY")
provider.HunterHow = appendIfExists(provider.HunterHow, "HUNTERHOW_API_KEY")

appendIfAllExists := func(arr []string, env1 string, env2 string) []string {
if val1, ok := os.LookupEnv(env1); ok {
Expand All @@ -127,7 +127,7 @@ func (provider *Provider) LoadProviderKeysFromEnv() {
return arr
}
provider.Fofa = appendIfAllExists(provider.Fofa, "FOFA_EMAIL", "FOFA_KEY")
provider.Fofa = appendIfAllExists(provider.Fofa, "CENSYS_API_ID", "CENSYS_API_SECRET")
provider.Censys = appendIfAllExists(provider.Censys, "CENSYS_API_ID", "CENSYS_API_SECRET")
}

// HasKeys returns true if at least one agent/source has keys
Expand Down