Skip to content

Commit

Permalink
Implemented robtex source
Browse files Browse the repository at this point in the history
  • Loading branch information
vzamanillo committed Sep 13, 2020
1 parent 5eee087 commit 0518d13
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 0 deletions.
5 changes: 5 additions & 0 deletions pkg/passive/sources.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/passivetotal"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/rapiddns"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/recon"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/robtex"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/securitytrails"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/shodan"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sitedossier"
Expand Down Expand Up @@ -49,6 +50,7 @@ var DefaultSources = []string{
"intelx",
"ipv4info",
"passivetotal",
"robtex",
"securitytrails",
"shodan",
"spyse",
Expand Down Expand Up @@ -96,6 +98,7 @@ var DefaultAllSources = []string{
"passivetotal",
"rapiddns",
"recon",
"robtex",
"securitytrails",
"shodan",
"sitedossier",
Expand Down Expand Up @@ -170,6 +173,8 @@ func (a *Agent) addSources(sources []string) {
a.sources[source] = &rapiddns.Source{}
case "recon":
a.sources[source] = &recon.Source{}
case "robtex":
a.sources[source] = &robtex.Source{}
case "securitytrails":
a.sources[source] = &securitytrails.Source{}
case "shodan":
Expand Down
92 changes: 92 additions & 0 deletions pkg/subscraping/sources/robtex/robtext.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package robtex

import (
"bufio"
"bytes"
"context"
"fmt"

jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/pkg/subscraping"
)

const (
addrRecord = "A"
iPv6AddrRecord = "AAAA"
baseURL = "https://freeapi.robtex.com/pdns"
)

// Source is the passive scraping agent
type Source struct{}

type result struct {
Rrname string `json:"rrname"`
Rrdata string `json:"rrdata"`
Rrtype string `json:"rrtype"`
}

// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)

go func() {
defer close(results)

headers := map[string]string{"Content-Type": "application/x-ndjson"}

ips, err := enumerate(ctx, session, fmt.Sprintf("%s/forward/%s", baseURL, domain), headers)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}

for _, result := range ips {
if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord {
domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s", baseURL, result.Rrdata), headers)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}
for _, result := range domains {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata}
}
}
}

}()
return results
}

func enumerate(ctx context.Context, session *subscraping.Session, targetURL string, headers map[string]string) ([]result, error) {
var results []result

resp, err := session.Get(ctx, targetURL, "", headers)
if err != nil {
session.DiscardHTTPResponse(resp)
return results, err
}

scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
var response result
err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response)
if err != nil {
return results, err
}

results = append(results, response)
}

resp.Body.Close()

return results, nil
}

// Name returns the name of the source
func (s *Source) Name() string {
return "robtex"
}

0 comments on commit 0518d13

Please sign in to comment.