Skip to content

Commit

Permalink
Implemented ThreatBook source
Browse files Browse the repository at this point in the history
  • Loading branch information
vzamanillo committed Sep 13, 2020
1 parent 5eee087 commit 8b83c05
Show file tree
Hide file tree
Showing 4 changed files with 88 additions and 0 deletions.
5 changes: 5 additions & 0 deletions pkg/passive/sources.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sitedossier"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/spyse"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sublist3r"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatbook"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatcrowd"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatminer"
"github.com/projectdiscovery/subfinder/pkg/subscraping/sources/virustotal"
Expand All @@ -53,6 +54,7 @@ var DefaultSources = []string{
"shodan",
"spyse",
"sublist3r",
"threatbook",
"threatcrowd",
"threatminer",
"virustotal",
Expand Down Expand Up @@ -101,6 +103,7 @@ var DefaultAllSources = []string{
"sitedossier",
"spyse",
"sublist3r",
"threatbook",
"threatcrowd",
"threatminer",
"virustotal",
Expand Down Expand Up @@ -180,6 +183,8 @@ func (a *Agent) addSources(sources []string) {
a.sources[source] = &spyse.Source{}
case "sublist3r":
a.sources[source] = &sublist3r.Source{}
case "threatbook":
a.sources[source] = &threatbook.Source{}
case "threatcrowd":
a.sources[source] = &threatcrowd.Source{}
case "threatminer":
Expand Down
4 changes: 4 additions & 0 deletions pkg/runner/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type ConfigFile struct {
SecurityTrails []string `yaml:"securitytrails"`
Shodan []string `yaml:"shodan"`
Spyse []string `yaml:"spyse"`
ThreatBook []string `yaml:"threatbook"`
URLScan []string `yaml:"urlscan"`
Virustotal []string `yaml:"virustotal"`
ZoomEye []string `yaml:"zoomeye"`
Expand Down Expand Up @@ -171,6 +172,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys {
if len(c.Spyse) > 0 {
keys.Spyse = c.Spyse[rand.Intn(len(c.Spyse))]
}
if len(c.ThreatBook) > 0 {
keys.ThreatBook = c.ThreatBook[rand.Intn(len(c.ThreatBook))]
}
if len(c.URLScan) > 0 {
keys.URLScan = c.URLScan[rand.Intn(len(c.URLScan))]
}
Expand Down
78 changes: 78 additions & 0 deletions pkg/subscraping/sources/threatbook/threatbook.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package threatbook

import (
"context"
"fmt"
"strconv"

jsoniter "github.com/json-iterator/go"
"github.com/projectdiscovery/subfinder/pkg/subscraping"
)

type threatBookResponse struct {
ResponseCode int64 `json:"response_code"`
VerboseMsg string `json:"verbose_msg"`
Data struct {
Domain string `json:"domain"`
SubDomains struct {
Total string `json:"total"`
Data []string `json:"data"`
} `json:"sub_domains"`
} `json:"data"`
}

// Source is the passive scraping agent
type Source struct{}

// Run function returns all subdomains found with the service
func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
results := make(chan subscraping.Result)

go func() {
defer close(results)

if session.Keys.ThreatBook == "" {
return
}

resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", session.Keys.ThreatBook, domain))
if err != nil && resp == nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
session.DiscardHTTPResponse(resp)
return
}

var response threatBookResponse
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
resp.Body.Close()
return
}
resp.Body.Close()

if response.ResponseCode != 0 {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("code %d, %s", response.ResponseCode, response.VerboseMsg)}
return
}

total, err := strconv.ParseInt(response.Data.SubDomains.Total, 10, 64)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return
}

if total > 0 {
for _, subdomain := range response.Data.SubDomains.Data {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
}
}
}()

return results
}

// Name returns the name of the source
func (s *Source) Name() string {
return "threatbook"
}
1 change: 1 addition & 0 deletions pkg/subscraping/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ type Keys struct {
Securitytrails string `json:"securitytrails"`
Shodan string `json:"shodan"`
Spyse string `json:"spyse"`
ThreatBook string `json:"threatbook"`
URLScan string `json:"urlscan"`
Virustotal string `json:"virustotal"`
ZoomEyeUsername string `json:"zoomeye_username"`
Expand Down

0 comments on commit 8b83c05

Please sign in to comment.