Merge pull request #274 from vzamanillo/improved-http-error-handling

Improved HTTP error handling

config.yaml

@@ -19,7 +19,6 @@ sources:
   - certspotterold
   - commoncrawl
   - crtsh
-  - digicert
   - dnsdumpster
   - dnsdb
   - entrust

pkg/passive/sources.go

@@ -11,7 +11,6 @@ import (
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/certspotterold"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/commoncrawl"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/crtsh"
-	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/digicert"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdb"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdumpster"
 	"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/entrust"
@@ -45,10 +44,10 @@ var DefaultSources = []string{
 	"certspotterold",
 	"commoncrawl",
 	"crtsh",
-	"digicert",
 	"dnsdumpster",
 	"dnsdb",
 	"entrust",
+	"github",
 	"hackertarget",
 	"ipv4info",
 	"intelx",
@@ -107,8 +106,6 @@ func (a *Agent) addSources(sources []string) {
 			a.sources[source] = &commoncrawl.Source{}
 		case "crtsh":
 			a.sources[source] = &crtsh.Source{}
-		case "digicert":
-			a.sources[source] = &digicert.Source{}
 		case "dnsdumpster":
 			a.sources[source] = &dnsdumpster.Source{}
 		case "dnsdb":

pkg/subscraping/agent.go

@@ -3,7 +3,11 @@ package subscraping
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
+	"net/url"
 	"time"
 )
 
@@ -44,12 +48,7 @@ func (s *Session) NormalGetWithContext(ctx context.Context, url string) (*http.R
 	req.Header.Set("Accept", "*/*")
 	req.Header.Set("Accept-Language", "en")
 
-	resp, err := s.Client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-
-	return resp, nil
+	return httpRequestWrapper(s.Client, req)
 }
 
 // Get makes a GET request to a URL
@@ -73,10 +72,25 @@ func (s *Session) Get(ctx context.Context, url string, cookies string, headers m
 		}
 	}
 
-	resp, err := s.Client.Do(req)
+	return httpRequestWrapper(s.Client, req)
+}
+
+func (s *Session) DiscardHttpResponse(response *http.Response) {
+	if response != nil {
+		io.Copy(ioutil.Discard, response.Body)
+		response.Body.Close()
+	}
+}
+
+func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) {
+	resp, err := client.Do(request)
 	if err != nil {
 		return nil, err
 	}
 
+	if resp.StatusCode != http.StatusOK {
+		requestUrl, _ := url.QueryUnescape(request.URL.String())
+		return resp, fmt.Errorf("Unexpected status code %d received from %s", resp.StatusCode, requestUrl)
+	}
 	return resp, nil
 }

pkg/subscraping/sources/alienvault/alienvault.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
-	"io/ioutil"
 
 	"github.com/projectdiscovery/subfinder/pkg/subscraping"
 )
@@ -27,16 +25,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain))
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHttpResponse(resp)
 			close(results)
 			return
 		}
-		if resp.StatusCode != 200 {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("invalid status code received: %d", resp.StatusCode)}
-			io.Copy(ioutil.Discard, resp.Body)
-			resp.Body.Close()
-			close(results)
-			return
-		}
+
 		otxResp := &alienvaultResponse{}
 		// Get the response body and decode
 		err = json.NewDecoder(resp.Body).Decode(&otxResp)

pkg/subscraping/sources/archiveis/archiveis.go

@@ -27,6 +27,7 @@ func (a *ArchiveIs) enumerate(ctx context.Context, baseURL string) {
 	resp, err := a.Session.NormalGetWithContext(ctx, baseURL)
 	if err != nil {
 		a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
+		a.Session.DiscardHttpResponse(resp)
 		return
 	}
 

pkg/subscraping/sources/binaryedge/binaryedge.go

@@ -29,6 +29,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		resp, err := session.Get(ctx, fmt.Sprintf("https://api.binaryedge.io/v2/query/domains/subdomain/%s", domain), "", map[string]string{"X-Key": session.Keys.Binaryedge})
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHttpResponse(resp)
 			close(results)
 			return
 		}

pkg/subscraping/sources/bufferover/bufferover.go

@@ -31,6 +31,7 @@ func (s *Source) getData(ctx context.Context, URL string, session *subscraping.S
 	resp, err := session.NormalGetWithContext(ctx, URL)
 	if err != nil {
 		results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+		session.DiscardHttpResponse(resp)
 		return
 	}
 

pkg/subscraping/sources/certspotter/certspotter.go

@@ -29,6 +29,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter})
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHttpResponse(resp)
 			close(results)
 			return
 		}

pkg/subscraping/sources/certspotterold/certspotterold.go

@@ -19,6 +19,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain))
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHttpResponse(resp)
 			close(results)
 			return
 		}

pkg/subscraping/sources/commoncrawl/commoncrawl.go

@@ -2,9 +2,7 @@ package commoncrawl
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net/url"
 	"strings"
@@ -33,14 +31,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		resp, err := session.NormalGetWithContext(ctx, indexURL)
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
-			close(results)
-			return
-		}
-
-		if resp.StatusCode == 500 {
-			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("internal server error")}
-			io.Copy(ioutil.Discard, resp.Body)
-			resp.Body.Close()
+			session.DiscardHttpResponse(resp)
 			close(results)
 			return
 		}

pkg/subscraping/sources/crtsh/crtsh.go

@@ -66,6 +66,7 @@ func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, sessi
 	resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain))
 	if err != nil {
 		results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+		session.DiscardHttpResponse(resp)
 		return false
 	}
 

pkg/subscraping/sources/dnsdb/dnsdb.go

@@ -5,8 +5,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"strings"
 
 	"github.com/projectdiscovery/subfinder/pkg/subscraping"
@@ -36,15 +34,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 			resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdb.info/lookup/rrset/name/*.%s?limit=1000000000000", domain), "", headers)
 			if err != nil {
 				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
-				close(results)
-				return
-			}
-
-			// Check status code
-			if resp.StatusCode != 200 {
-				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("invalid status code received: %d", resp.StatusCode)}
-				io.Copy(ioutil.Discard, resp.Body)
-				resp.Body.Close()
+				session.DiscardHttpResponse(resp)				
 				close(results)
 				return
 			}

pkg/subscraping/sources/dnsdumpster/dnsdumpster.go

@@ -76,6 +76,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		resp, err := session.NormalGetWithContext(ctx, "https://dnsdumpster.com/")
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHttpResponse(resp)
 			close(results)
 			return
 		}

pkg/subscraping/sources/entrust/entrust.go

@@ -20,6 +20,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
 		resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://ctsearch.entrust.com/api/v1/certificates?fields=issuerCN,subjectO,issuerDN,issuerO,subjectDN,signAlg,san,publicKeyType,publicKeySize,validFrom,validTo,sn,ev,logEntries.logName,subjectCNReversed,cert&domain=%s&includeExpired=true&exactMatch=false&limit=5000", domain))
 		if err != nil {
 			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			session.DiscardHttpResponse(resp)
 			close(results)
 			return
 		}