Skip to content

Commit

Permalink
updates related to integration with PostgreSQL
Browse files Browse the repository at this point in the history
  • Loading branch information
@caffix
caffix committed Jul 20, 2023
1 parent 7f0ede2 commit b488859
Show file tree
Hide file tree
Showing 5 changed files with 39 additions and 20 deletions.
2 changes: 1 addition & 1 deletion cmd/amass/enum.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ func printOutput(e *enum.Enumeration, args *enumArgs, output chan string, wg *sy
}

if total == 0 {
r.Println("No names were discovered")
r.Println("No assets were discovered")
}
}

Expand Down
18 changes: 14 additions & 4 deletions cmd/amass/io.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,9 @@ func extractAssetName(a *types.Asset, since time.Time) string {
// ExtractOutput is a convenience method for obtaining new discoveries made by the enumeration process.
func ExtractOutput(ctx context.Context, g *netmap.Graph, e *enum.Enumeration, filter *stringset.Set, asinfo bool) []*requests.Output {
if e.Config.Passive {
return EventNames(ctx, g, e.Config.Domains(), e.Config.CollectionStartTime.UTC(), filter)
return EventNames(ctx, g, e.Config.Domains(), e.Config.CollectionStartTime, filter)
}
return EventOutput(ctx, g, e.Config.Domains(), e.Config.CollectionStartTime.UTC(), filter, asinfo, e.Sys.Cache())
return EventOutput(ctx, g, e.Config.Domains(), e.Config.CollectionStartTime, filter, asinfo, e.Sys.Cache())
}

type outLookup map[string]*requests.Output
Expand All @@ -118,7 +118,12 @@ func EventOutput(ctx context.Context, g *netmap.Graph, domains []string, since t
fqdns = append(fqdns, domain.FQDN{Name: d})
}

assets, err := g.DB.FindByScope(fqdns, since)
qtime := time.Time{}
if !since.IsZero() {
qtime = since.UTC()
}

assets, err := g.DB.FindByScope(fqdns, qtime)
if err != nil {
return res
}
Expand Down Expand Up @@ -226,7 +231,12 @@ func EventNames(ctx context.Context, g *netmap.Graph, domains []string, since ti
fqdns = append(fqdns, domain.FQDN{Name: d})
}

assets, err := g.DB.FindByScope(fqdns, since)
qtime := time.Time{}
if !since.IsZero() {
qtime = since.UTC()
}

assets, err := g.DB.FindByScope(fqdns, qtime)
if err != nil {
return res
}
Expand Down
23 changes: 18 additions & 5 deletions cmd/amass/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ import (

const (
mainUsageMsg = "intel|enum|db [options]"
exampleConfigFileURL = "https://github.com/owasp-amass/amass/blob/master/examples/config.ini"
exampleConfigFileURL = "https://github.com/owasp-amass/amass/blob/master/examples/config.yaml"
userGuideURL = "https://github.com/owasp-amass/amass/blob/master/doc/user_guide.md"
tutorialURL = "https://github.com/owasp-amass/amass/blob/master/doc/tutorial.md"
)
Expand Down Expand Up @@ -190,11 +190,24 @@ func createOutputDirectory(cfg *config.Config) {
}

func openGraphDatabase(dir string, cfg *config.Config) *netmap.Graph {
if db := cfg.LocalDatabaseSettings(cfg.GraphDBs); db != nil {
f := filepath.Join(config.OutputDirectory(dir), "amass.sqlite")
// Add the local database settings to the configuration
cfg.GraphDBs = append(cfg.GraphDBs, cfg.LocalDatabaseSettings(cfg.GraphDBs))

for _, db := range cfg.GraphDBs {
if db.Primary {
var g *netmap.Graph

if db.System == "local" {
g = netmap.NewGraph(db.System, filepath.Join(config.OutputDirectory(cfg.Dir), "amass.sqlite"), db.Options)
} else {
connStr := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s", db.Host, db.Port, db.Username, db.Password, db.DBName)
g = netmap.NewGraph(db.System, connStr, db.Options)
}

if g := netmap.NewGraph("local", f, ""); g != nil {
return g
if g != nil {
return g
}
break
}
}

Expand Down
6 changes: 2 additions & 4 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.19

require (
github.com/PuerkitoBio/goquery v1.8.1
github.com/caffix/netmap v0.2.3
github.com/caffix/netmap v0.2.4
github.com/caffix/pipeline v0.2.2
github.com/caffix/queue v0.1.4
github.com/caffix/service v0.3.0
Expand Down Expand Up @@ -64,7 +64,6 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-sqlite3 v1.14.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
Expand All @@ -73,7 +72,7 @@ require (
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rubenv/sql-migrate v1.5.1 // indirect
github.com/rubenv/sql-migrate v1.5.2 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/temoto/robotstxt v1.1.2 // indirect
go.uber.org/ratelimit v0.3.0 // indirect
Expand All @@ -88,7 +87,6 @@ require (
gorm.io/datatypes v1.2.0 // indirect
gorm.io/driver/mysql v1.5.1 // indirect
gorm.io/driver/postgres v1.5.2 // indirect
gorm.io/driver/sqlite v1.5.2 // indirect
gorm.io/gorm v1.25.2 // indirect
modernc.org/libc v1.24.1 // indirect
modernc.org/mathutil v1.6.0 // indirect
Expand Down
10 changes: 4 additions & 6 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/caffix/netmap v0.2.3 h1:QFPh6yKdAnt0Flq3k1pJm/hNuXoKzTYGrcbX3d/BTlM=
github.com/caffix/netmap v0.2.3/go.mod h1:IApB4oO5vb3Oq1Slv4xlfZLrBBzuRor8xW7hAJ0C/hE=
github.com/caffix/netmap v0.2.4 h1:mzgmk0u+khenSw2qjG8T/KwDmt9Pdb7X4MRPjU99Ghw=
github.com/caffix/netmap v0.2.4/go.mod h1:Qnm5FUdqf7Ey7IiScgdg8FT7m/6y3XhoG8Z+ZJZtYI0=
github.com/caffix/pipeline v0.2.2 h1:d1l7CiBe7jFDc4ksvnNgHVb3XlVlCRFXFo3I6guksuQ=
github.com/caffix/pipeline v0.2.2/go.mod h1:NlUnsifT5h0B9C51DPdp99KjlZjREnE0Iev7R8gEn/s=
github.com/caffix/queue v0.1.4 h1:sQbFzwGaPM1tRnQHWCgHOwj7hLuhDQ3BhY1/1TFbBiE=
Expand Down Expand Up @@ -384,7 +384,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
Expand Down Expand Up @@ -496,8 +495,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rubenv/sql-migrate v1.5.1 h1:WsZo4jPQfjmddDTh/suANP2aKPA7/ekN0LzuuajgQEo=
github.com/rubenv/sql-migrate v1.5.1/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is=
github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0=
github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
Expand Down Expand Up @@ -979,7 +978,6 @@ gorm.io/driver/mysql v1.5.1/go.mod h1:Jo3Xu7mMhCyj8dlrb3WoCaRd1FhsVh+yMXb1jUInf5
gorm.io/driver/postgres v1.5.2 h1:ytTDxxEv+MplXOfFe3Lzm7SjG09fcdb3Z/c056DTBx0=
gorm.io/driver/postgres v1.5.2/go.mod h1:fmpX0m2I1PKuR7mKZiEluwrP3hbs+ps7JIGMUBpCgl8=
gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc=
gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
gorm.io/driver/sqlserver v1.4.1 h1:t4r4r6Jam5E6ejqP7N82qAJIJAht27EGT41HyPfXRw0=
gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho=
Expand Down

0 comments on commit b488859

Please sign in to comment.