From d4f490ddab8c919dd44b20e4fe35db5ed8f27780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 19 Dec 2024 15:58:36 +0300 Subject: [PATCH 1/2] use dnsdumpster api --- v2/pkg/passive/sources_test.go | 3 +- .../sources/dnsdumpster/dnsdumpster.go | 94 ++++++------------- 2 files changed, 30 insertions(+), 67 deletions(-) diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 551a60740..d7076e16b 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -68,7 +68,7 @@ var ( "chinaz", "crtsh", "digitorus", - // "dnsdumpster", //failing with "unexpected status code 403 received" + "dnsdumpster", "dnsrepo", "fofa", "fullhunt", @@ -98,7 +98,6 @@ var ( "bufferover", "certspotter", "crtsh", - "dnsdumpster", "dnsdb", "digitorus", "hackertarget", diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 6298d2c2d..953413ae7 100644 --- a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -3,67 +3,29 @@ package dnsdumpster import ( "context" + "encoding/json" "fmt" - "io" - "net/url" - "regexp" - "strings" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) -// CSRFSubMatchLength CSRF regex submatch length -const CSRFSubMatchLength = 2 - -var re = regexp.MustCompile("") - -// getCSRFToken gets the CSRF Token from the page -func getCSRFToken(page string) string { - if subs := re.FindStringSubmatch(page); len(subs) == CSRFSubMatchLength { - return strings.TrimSpace(subs[1]) - } - return "" -} - -// postForm posts a form for a domain and returns the response -func postForm(ctx context.Context, session *subscraping.Session, token, domain string) (string, error) { - params := url.Values{ - "csrfmiddlewaretoken": {token}, - "targetip": {domain}, - "user": {"free"}, - } - - resp, err := session.HTTPRequest( - ctx, - "POST", - "https://dnsdumpster.com/", - fmt.Sprintf("csrftoken=%s; Domain=dnsdumpster.com", token), - map[string]string{ - "Content-Type": "application/x-www-form-urlencoded", - "Referer": "https://dnsdumpster.com", - "X-CSRF-Token": token, - }, - strings.NewReader(params.Encode()), - subscraping.BasicAuth{}, - ) - - if err != nil { - session.DiscardHTTPResponse(resp) - return "", err - } - - // Now, grab the entire page - in, err := io.ReadAll(resp.Body) - resp.Body.Close() - return string(in), err +type response struct { + A []struct { + Host string `json:"host"` + } `json:"a"` + Ns []struct { + Host string `json:"host"` + } `json:"ns"` } // Source is the passive scraping agent type Source struct { + apiKeys []string timeTaken time.Duration errors int results int + skipped bool } // Run function returns all subdomains found with the service @@ -78,35 +40,36 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) - resp, err := session.SimpleGet(ctx, "https://dnsdumpster.com/") - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true return } - body, err := io.ReadAll(resp.Body) + resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain), "", map[string]string{"X-API-Key": randomApiKey}) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + defer resp.Body.Close() - csrfToken := getCSRFToken(string(body)) - data, err := postForm(ctx, session, csrfToken, domain) + var response response + err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ + resp.Body.Close() return } + fmt.Println(response) - for _, subdomain := range session.Extractor.Extract(data) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + for _, record := range append(response.A, response.Ns...) { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host} s.results++ } + }() return results @@ -118,19 +81,19 @@ func (s *Source) Name() string { } func (s *Source) IsDefault() bool { - return false + return true } func (s *Source) HasRecursiveSupport() bool { - return true + return false } func (s *Source) NeedsKey() bool { - return false + return true } -func (s *Source) AddApiKeys(_ []string) { - // no key needed +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys } func (s *Source) Statistics() subscraping.Statistics { @@ -138,5 +101,6 @@ func (s *Source) Statistics() subscraping.Statistics { Errors: s.errors, Results: s.results, TimeTaken: s.timeTaken, + Skipped: s.skipped, } } From 3e9f308d283a60d9cb827ae6b64bb4e54ec946a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 14 Jan 2025 11:56:15 +0300 Subject: [PATCH 2/2] remove debug code --- v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 953413ae7..2155e31cf 100644 --- a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -63,7 +63,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resp.Body.Close() return } - fmt.Println(response) for _, record := range append(response.A, response.Ns...) { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Host}