diff --git a/go.mod b/go.mod index f86942d..7f98448 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module github.com/LeakIX/l9tcpid go 1.11 require ( - github.com/LeakIX/l9format v1.0.0 - github.com/PuerkitoBio/goquery v1.6.1 + github.com/LeakIX/l9format v1.3.0 + github.com/PuerkitoBio/goquery v1.7.0 github.com/RumbleDiscovery/jarm-go v0.0.6 github.com/alecthomas/kong v0.2.12 github.com/gboddin/goccm v0.0.0-20201205182630-c48e23986e1e gitlab.nobody.run/tbi/bannerid v0.0.0-20200517091717-5cd63f20b794 gitlab.nobody.run/tbi/socksme v0.0.0-20201130024528-3ad02d4108c5 - golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect + golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect ) diff --git a/go.sum b/go.sum index f63af58..58fc223 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,14 @@ github.com/LeakIX/l9format v1.0.0-alpha.1 h1:QCDDH+AJy5V2VwNo/gxDAFx3wLpkDkk6qoL github.com/LeakIX/l9format v1.0.0-alpha.1/go.mod h1:zAMhvA0dNLwo66UmcL6e9vxKo2JxPQEVMDwNcc9Il9w= github.com/LeakIX/l9format v1.0.0 h1:zHqGvnwvr5iuI55nCSQbmVlPdiEfiNrjn890MMlX2rk= github.com/LeakIX/l9format v1.0.0/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k= +github.com/LeakIX/l9format v1.2.0-alpha.1 h1:wluoAX8aV2t+Oc6lx553KKlvDkVDmRB2af4uYutgp50= +github.com/LeakIX/l9format v1.2.0-alpha.1/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k= +github.com/LeakIX/l9format v1.3.0 h1:IrWuvYQ6ayEwf3oTTWG54D3oVfLApQDd0qyKdKK5mw8= +github.com/LeakIX/l9format v1.3.0/go.mod h1:eKQn32c5PgUM7806Un2v6WTSmJcdcixed+cRHsPEp0k= github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk= github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/PuerkitoBio/goquery v1.7.0 h1:O5SP3b9JWqMSVMG69zMfj577zwkSNpxrFf7ybS74eiw= +github.com/PuerkitoBio/goquery v1.7.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/RumbleDiscovery/jarm-go v0.0.6 h1:n3JEmOhPyfhmu1aeDEK/10Y2F+GMUYrtGFZmp4Yj0s4= github.com/RumbleDiscovery/jarm-go v0.0.6/go.mod h1:dXV7z5vBXQI0cNaHXwzGtq2PJ2LgM3XgcFiX32FU3bg= github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= @@ -47,6 +53,8 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG0 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/httpbanner.go b/httpbanner.go index 755fe91..2df9fe0 100644 --- a/httpbanner.go +++ b/httpbanner.go @@ -41,7 +41,7 @@ func GetHttpClient(event *l9format.L9Event) *http.Client { var HttpTestRequest = "GET %s HTTP/1.1\r\n" + "Host: %s\r\n" + - "User-Agent: l9tcpid/v1.0.0\r\n" + + "User-Agent: l9tcpid/v1.1.0\r\n" + "Connection: close\r\n" func SendHttpTestRequest(hostname string, path string, connection net.Conn) (err error) { @@ -150,9 +150,7 @@ func GetHttpBanner(event *l9format.L9Event, path string) (err error) { } for _, matchFunc := range HttpIdentifiers { - if matchFunc(event, body, document) { - break - } + matchFunc(event, body, document) } if len(event.Http.Title) < 1 && len(body) < 16*1024 { event.Summary += body diff --git a/identifiers/http/common_app.go b/identifiers/http/common_app.go new file mode 100644 index 0000000..cb88ed1 --- /dev/null +++ b/identifiers/http/common_app.go @@ -0,0 +1,44 @@ +package http + +import ( + "github.com/LeakIX/l9format" + "github.com/PuerkitoBio/goquery" + "strings" +) + +// This stage is useful if a plugin relies on tags ( eg nuclei plugin ) + +func TagDrupal(event *l9format.L9Event, body string, document *goquery.Document) bool { + if strings.Contains(body, "Drupal.settings") || strings.Contains(body, "content=\"Drupal"){ + event.AddTag("drupal") + event.AddTag("php") + return true + } + return false +} + +func TagWordpress(event *l9format.L9Event, body string, document *goquery.Document) bool { + if strings.Contains(body,"wp-content/") { + event.AddTag("wordpress") + event.AddTag("php") + return true + } + return false +} + +func TagJoomla(event *l9format.L9Event,body string, document *goquery.Document) bool { + if strings.Contains(body,"content=\"Joomla!") { + event.AddTag("joomla") + event.AddTag("php") + return true + } + return false +} + +func TagVMWare(event *l9format.L9Event,body string, document *goquery.Document) bool { + if strings.Contains(body,"vmware.vsphere.client") { + event.AddTag("vmware") + return true + } + return false +} diff --git a/identifiers/http/http_server.go b/identifiers/http/http_server.go new file mode 100644 index 0000000..f7155cf --- /dev/null +++ b/identifiers/http/http_server.go @@ -0,0 +1,46 @@ +package http + +import ( + "github.com/LeakIX/l9format" + "github.com/PuerkitoBio/goquery" + "strings" +) + +func TagNginx(event *l9format.L9Event, body string, document *goquery.Document) bool { + if serverHeader, hasServerHeader := event.Http.Headers["server"]; hasServerHeader { + if strings.Contains(serverHeader, "nginx") { + event.AddTag("nginx") + return true + } + } + return false +} + +func TagApache(event *l9format.L9Event, body string, document *goquery.Document) bool { + if serverHeader, hasServerHeader := event.Http.Headers["server"]; hasServerHeader && len(serverHeader) < 128 { + if strings.Contains(serverHeader, "Apache") { + event.AddTag("apache") + if strings.Contains(serverHeader,"Coyote") { + event.AddTag("tomcat") + } + return true + } + } + return false +} + +func TagPHP(event *l9format.L9Event, body string, document *goquery.Document) bool { + if serverHeader, hasServerHeader := event.Http.Headers["server"]; hasServerHeader && len(serverHeader) < 128 { + if strings.Contains(serverHeader, "PHP/") { + event.AddTag("php") + return true + } + } + if powerHeader, hasPowerHeader := event.Http.Headers["x-powered-by"]; hasPowerHeader && len(powerHeader) < 128 { + if strings.HasPrefix(powerHeader, "PHP") { + event.AddTag("php") + return true + } + } + return false +} \ No newline at end of file diff --git a/identifiers/http/plc.go b/identifiers/http/plc.go index da87339..6384537 100644 --- a/identifiers/http/plc.go +++ b/identifiers/http/plc.go @@ -7,7 +7,7 @@ import ( ) func TagPLC(event *l9format.L9Event, body string, document *goquery.Document) bool { - if serverHeader, hasServerHeader := event.Http.Headers["server"]; hasServerHeader { + if serverHeader, hasServerHeader := event.Http.Headers["server"]; hasServerHeader && len(serverHeader) < 128 { if strings.Contains(serverHeader, "A-B WWW/") { event.AddTag("plc") return true diff --git a/identifiers/http/printer.go b/identifiers/http/printer.go index 6ec0898..796e0e2 100644 --- a/identifiers/http/printer.go +++ b/identifiers/http/printer.go @@ -8,7 +8,7 @@ import ( func TagPrinter(event *l9format.L9Event, body string, document *goquery.Document) bool { - if serverHeader, hasServerHeader := event.Http.Headers["server"]; hasServerHeader { + if serverHeader, hasServerHeader := event.Http.Headers["server"]; hasServerHeader && len(serverHeader) < 128 { if strings.HasPrefix(serverHeader, "HP HTTP Server;") { event.AddTag("printer") return true diff --git a/network.go b/network.go index 24c836f..bdc617d 100644 --- a/network.go +++ b/network.go @@ -1,6 +1,7 @@ package l9tcpid import ( + "bytes" "context" "crypto/ecdsa" "crypto/rsa" @@ -12,14 +13,17 @@ import ( "github.com/LeakIX/l9format" "github.com/RumbleDiscovery/jarm-go" "gitlab.nobody.run/tbi/socksme" + "io" "log" "net" "strconv" "strings" + "sync" "time" "unicode" ) + func GetNetworkConnection(event *l9format.L9Event) (conn net.Conn, err error) { taskContext, _ := context.WithDeadline(context.Background(), time.Now().Add(20*time.Second)) conn, err = net.DialTimeout("tcp", net.JoinHostPort(event.Ip, event.Port), 3*time.Second) @@ -154,51 +158,31 @@ func UpgradeConnection(protocol string, connection net.Conn) (err error) { // takes a connection and populates hostService with findings func FuzzConnection(connection net.Conn, event *l9format.L9Event) (err error) { - err = connection.SetReadDeadline(time.Now().Add(2 * time.Second)) + err = connection.SetReadDeadline(time.Now().Add(5 * time.Second)) if err != nil { return err } defer connection.Close() - var buffer []byte + buffer := &bytes.Buffer{} + wg := &sync.WaitGroup{} + wg.Add(1) // read input until deadline or error - for { - recvBuf := make([]byte, 16) - n, err := connection.Read(recvBuf[:]) - if err != nil { - break - } - if n > 0 && len(buffer) < 512 { - buffer = append(buffer, recvBuf...) - } - } + go func(connection net.Conn, buffer *bytes.Buffer, wg *sync.WaitGroup) { + io.Copy(buffer, io.LimitReader(connection, 1024*512)) + wg.Done() + }(connection, buffer, wg) err = connection.SetWriteDeadline(time.Now().Add(3 * time.Second)) if err != nil { return err } _, err = connection.Write( []byte("GET / HTTP/1.1\r\nHost: " + event.Host + "\r\n\r\nHELP\r\nEHLO leakix.net\r\n?\r\n\r\n")) - if err == nil { - err = connection.SetReadDeadline(time.Now().Add(5 * time.Second)) - if err != nil { - return err - } - for { - recvBuf := make([]byte, 16) - n, err := connection.Read(recvBuf[:]) - if err != nil { - break - } - if n > 0 && len(buffer) < 512 { - buffer = append(buffer, recvBuf...) - } - } - } - - if len(buffer) < 1 { + wg.Wait() + if buffer.Len() < 1 { // So far we have written without issue (aka NO RST came back) return errors.New("empty") } - printables := strings.FieldsFunc(string(buffer), func(r rune) bool { + printables := strings.FieldsFunc(buffer.String(), func(r rune) bool { if r == '\n' || r == '\r' { return true } @@ -209,14 +193,14 @@ func FuzzConnection(connection net.Conn, event *l9format.L9Event) (err error) { event.Summary += strings.TrimSpace(result) + "\n" } for _, matchFunc := range TCPIdentifiers { - if matchFunc(event, buffer, printables) { + if matchFunc(event, buffer.Bytes(), printables) { break } } // We couldn't identify, add connection dump if event.Protocol == "tcp" { event.Summary += "\nRaw connection:\n" - event.Summary += hex.Dump(buffer) + event.Summary += hex.Dump(buffer.Bytes()) } return nil } diff --git a/service_map.go b/service_map.go index 70c8580..b85dc64 100644 --- a/service_map.go +++ b/service_map.go @@ -24,11 +24,18 @@ var TCPIdentifiers = []TcpIdentifier{ tcp.IdentifyCassandra, } var HttpIdentifiers = []HttpIdentifier{ + http.TagDrupal, + http.TagJoomla, + http.TagWordpress, + http.TagVMWare, http.TagPLC, + http.TagPHP, + http.TagNginx, http.TagPrinter, http.IdentifyElasticSearch, http.IdentifyKibana, http.IdentifyCouchDb, + } var serviceMap = map[string]string{ "9092": "kafka",