Added http probe

go.mod

@@ -4,7 +4,9 @@ go 1.11
 
 require (
 	github.com/Machiel/slugify v1.0.1 // indirect
+	github.com/RumbleDiscovery/jarm-go v0.0.5-0.20201125030722-2ca10cdc1b63
 	github.com/zenthangplus/goccm v0.0.0-20200608171100-39e9e08b694a
+	gitlab.nobody.run/tbi/bannerid v0.0.0-20200517091717-5cd63f20b794
 	gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e
 	gitlab.nobody.run/tbi/socksme v0.0.0-20201130024528-3ad02d4108c5
 	golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7 // indirect

go.sum

@@ -1,19 +1,47 @@
 github.com/Machiel/slugify v1.0.1 h1:EfWSlRWstMadsgzmiV7d0yVd2IFlagWH68Q+DcYCm4E=
 github.com/Machiel/slugify v1.0.1/go.mod h1:fTFGn5uWEynW4CUMG7sWkYXOf1UgDxyTM3DbR6Qfg3k=
+github.com/RumbleDiscovery/jarm-go v0.0.5-0.20201125030722-2ca10cdc1b63 h1:FAX/JIXwSwWYH/uqwjLr7Vt8m0aTWP+m3qzuc6d4DGc=
+github.com/RumbleDiscovery/jarm-go v0.0.5-0.20201125030722-2ca10cdc1b63/go.mod h1:J1ZxKveX8XXs7gyF147IWgZ11AJ/a9BvGmVakx/o+To=
+github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/zenthangplus/goccm v0.0.0-20200608171100-39e9e08b694a h1:zxtrEyAn+PXzLBBXUrjamhu1+ZdkOt0nltXvCWh7JvY=
 github.com/zenthangplus/goccm v0.0.0-20200608171100-39e9e08b694a/go.mod h1:PPYr3s9FhH/9fs7kfozlHKs2VXzk4Foyzb3Mke/Bg0U=
+gitlab.nobody.run/tbi/bannerid v0.0.0-20200517091717-5cd63f20b794 h1:bjn2lfo8i1yNnuSpLk5mGlsvWvDawOiuGUC1YvL400Y=
+gitlab.nobody.run/tbi/bannerid v0.0.0-20200517091717-5cd63f20b794/go.mod h1:4jjebVNFNFJPl332Gnrj/jCfQoXB4PWWZURCRJzxjE4=
 gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e h1:0EEfNlVSZtV1q/CqAXpoPcAgyim7s04kAKVmRkhUOfY=
 gitlab.nobody.run/tbi/core v0.0.0-20201124161030-42e21c6deb8e/go.mod h1:obLTFd7diUMk/cXeXJpgYdsj5OKn5c0EARFPatcZQCk=
 gitlab.nobody.run/tbi/socksme v0.0.0-20201130024528-3ad02d4108c5 h1:OUBbWsRDwvD6Nu6rP8PN6xGxIxXpVPBPBE9iuvDQH90=
 gitlab.nobody.run/tbi/socksme v0.0.0-20201130024528-3ad02d4108c5/go.mod h1:2R0aT7bhH9+xiSielZpKS/TL4nTki1jxOCCgXN8BzV8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7 h1:3uJsdck53FDIpWwLeAXlia9p4C8j0BO2xZrqzKpL0D8=
 golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

httpbanner.go

@@ -0,0 +1,187 @@
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/rsa"
+	"crypto/sha256"
+	"crypto/tls"
+	"fmt"
+	"github.com/RumbleDiscovery/jarm-go"
+	"gitlab.nobody.run/tbi/core"
+	"gitlab.nobody.run/tbi/bannerid"
+	"log"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var ValidHeaders = []string{
+	"Server",
+	"Set-Cookie",
+	"X-Powered-By",
+	"X-Mod-Pagespeed",
+	"X-Aspnet-Version",
+	"Www-Authenticate",
+	"Via",
+	"Upgrade",
+	"Status",
+	"Pragma",
+	"Location",
+	"Keep-Alive",
+	"Expires",
+	"Content-Type",
+	"Date",
+	"Content-Length",
+	"Content-Language",
+	"Cache-Control",
+	"Access-Control-Allow-Origin",
+	"Access-Control-Expose-Headers",
+	"Access-Control-Max-Age",
+	"Access-Control-Allow-Credentials",
+	"Accept-Ranges",
+	"Strict-Transport-Security",
+	"X-Debug-Token",
+	"Link",
+	"X-Debug",
+	"Debug",
+	"Kbn-Version",
+	"Kbn-Name",
+	"X-Couch-Request-Id",
+}
+var tlsVersionMap = map[uint16]string{
+	0x0301 : "TLSv1.0",
+	0x0302 : "TLSv1.1",
+	0x0303 : "TLSv1.2",
+	0x0304 : "TLSv1.3",
+	0x0300 : "SSLv3",
+}
+var plugin = core.ProxiedPlugin{}
+
+func GetTLSVersionName(version uint16) string {
+	if name, found := tlsVersionMap[version]; found {
+		return name
+	}
+	return ""
+}
+// Get info
+func GetHttpBanner(service *core.HostService) error {
+	log.Printf("Discovering http://%s:%s ...", service.Ip, service.Port)
+	req, err := http.NewRequest("GET", fmt.Sprintf("%s://%s:%s/", service.Scheme, service.Ip, service.Port), nil)
+	req.Header["User-Agent"] = []string{"l9serviceid/0.1.0 (+https://leakix.net/)"}
+	if service.Hostname != service.Ip && len(service.Hostname) > 1 {
+		req.Host = service.Hostname
+		req.URL.Host = service.Hostname
+	}
+	if err != nil {
+		return err
+	}
+	// use the http client to fetch the page
+	resp, err := plugin.GetHttpClient(service.Ip, service.Port).Do(req)
+	if err != nil {
+		return err
+
+	}
+	defer resp.Body.Close()
+	log.Printf("Found %d headers for http://%s:%s", len(resp.Header), service.Ip, service.Port)
+	service.Headers = make(map[string][]string)
+	// Sanity check to store only sanitized headers
+	if resp.TLS != nil {
+		if len(resp.TLS.PeerCertificates) > 0 {
+			err := resp.TLS.PeerCertificates[0].VerifyHostname(service.Hostname)
+			if err == nil {
+				service.Certificate.Valid = true
+			} else {
+				log.Println(err.Error())
+			}
+			service.Certificate.CommonName = resp.TLS.PeerCertificates[0].Subject.CommonName
+			for _, domain := range resp.TLS.PeerCertificates[0].DNSNames {
+				service.Certificate.Domains = append(service.Certificate.Domains, domain)
+			}
+
+			service.Certificate.Fingerprint = fmt.Sprintf("%02x", sha256.Sum256( resp.TLS.PeerCertificates[0].Raw))
+			service.Certificate.CypherSuite = tls.CipherSuiteName(resp.TLS.CipherSuite)
+			service.Certificate.Version = GetTLSVersionName(resp.TLS.Version)
+			service.Certificate.KeyAlgo = resp.TLS.PeerCertificates[0].PublicKeyAlgorithm.String()
+			if publicKey, isECDSA :=  resp.TLS.PeerCertificates[0].PublicKey.(*ecdsa.PublicKey); isECDSA {
+				service.Certificate.KeySize = publicKey.Params().BitSize
+			} else if publicKey, isRSA :=  resp.TLS.PeerCertificates[0].PublicKey.(*rsa.PublicKey); isRSA {
+				service.Certificate.KeySize = publicKey.Size()*8
+			}
+			service.Certificate.NotBefore = resp.TLS.PeerCertificates[0].NotBefore
+			service.Certificate.NotAfter = resp.TLS.PeerCertificates[0].NotAfter
+			service.Certificate.IssuerName = resp.TLS.PeerCertificates[0].Issuer.CommonName
+			// Only do IPs
+			if service.Ip == service.Hostname {
+				service.Certificate.JARM, _ = GetJARM(service.Ip, service.Hostname, service.Port)
+			}
+		}
+	}
+
+	for header, values := range resp.Header {
+		service.Data += fmt.Sprintf("\r\n%s : %s\n", header, strings.Join(values, ", "))
+		for _, validHeader := range ValidHeaders {
+			if strings.EqualFold(validHeader, header) && len(values) == 1 && len(values[0]) < 512 {
+				service.Headers[validHeader] = values
+				if validHeader == "Server" {
+					software, err := bannerid.ParseWebServerBanner(values[0])
+					if err == nil {
+						service.Software.Name = software.Name
+						service.Software.Version = software.Version
+						service.Software.OperatingSystem = software.OS
+						for _, module := range software.Modules {
+							service.Software.Modules = append(service.Software.Modules, &core.SoftwareModule{
+								Name:    module.Name,
+								Version: module.Version,
+							})
+						}
+					}
+				}
+				if validHeader == "X-Powered-By" {
+					software, err := bannerid.ParseWebServerBanner(values[0])
+					if err == nil {
+						service.Software.Modules = append(service.Software.Modules, &core.SoftwareModule{
+							Name:    software.Name,
+							Version: software.Version,
+						})
+					}
+				}
+			}
+		}
+	}
+	service.DeferSave()
+	return nil
+}
+
+
+func GetJARM(ip, host, servicePort string) (string, error) {
+	results := []string{}
+	port, _ := strconv.Atoi(servicePort)
+	for _, probe := range jarm.GetProbes(host, port) {
+		c, err := plugin.GetProxiedDialer().Dial("tcp", net.JoinHostPort(ip, fmt.Sprintf("%d", port)))
+		if err != nil {
+			return "", err
+		}
+		data := jarm.BuildProbe(probe)
+		c.SetWriteDeadline(time.Now().Add(time.Second * 3))
+		_, err = c.Write(data)
+		if err != nil {
+			results = append(results, "")
+			continue
+		}
+
+		c.SetReadDeadline(time.Now().Add(time.Second * 3))
+		buff := make([]byte, 1484)
+		c.Read(buff)
+		c.Close()
+
+		ans, err := jarm.ParseServerHello(buff, probe)
+		if err != nil {
+			results = append(results, "")
+		}
+
+		results = append(results, ans)
+	}
+	return jarm.RawHashToFuzzyHash(strings.Join(results, ",")), nil
+}

network.go

@@ -83,8 +83,8 @@ func FuzzConnection(connection net.Conn, hostService *core.HostService) (err err
 		}
 	}
 
-	// Try an SSL payload
-	if len(buffer) < 1 || hostService.Scheme != "ssl" {
+	// If we wrote and got no reply , try an "SSL payload"
+	if len(buffer) < 1 && hostService.Scheme != "ssl" {
 		err = connection.SetWriteDeadline(time.Now().Add(3*time.Second))
 		if err != nil {
 			return err
@@ -108,6 +108,7 @@ func FuzzConnection(connection net.Conn, hostService *core.HostService) (err err
 		}
 	}
 	if len(buffer) < 1 {
+		// Couldn't make up anything, return empty
 		return errors.New("empty")
 	}
 	printables := strings.FieldsFunc(string(buffer), func(r rune) bool {
@@ -139,6 +140,7 @@ func GetBanner(hostService *core.HostService) (err error) {
 	//   - upgrade connection
 	//   - Fuzz connection
 	//   - Fill certificate info
+	// - if http, request again, parse headers ( needed for SNI hosts anyway )
 	hostService.Type = "tcp"
 	// Open connection
 	connection, err := GetNetworkConnection(hostService)
@@ -237,7 +239,7 @@ func GetBanner(hostService *core.HostService) (err error) {
 			}
 			hostService.Certificate.Fingerprint = fmt.Sprintf("%02x", sha256.Sum256( tlsConnection.ConnectionState().PeerCertificates[0].Raw))
 			hostService.Certificate.CypherSuite = tls.CipherSuiteName(tlsConnection.ConnectionState().CipherSuite)
-			//hostService.Certificate.Version = plugin.GetTLSVersionName(tlsConnection.ConnectionState().Version)
+			hostService.Certificate.Version = GetTLSVersionName(tlsConnection.ConnectionState().Version)
 			hostService.Certificate.KeyAlgo = tlsConnection.ConnectionState().PeerCertificates[0].PublicKeyAlgorithm.String()
 			if publicKey, isECDSA :=  tlsConnection.ConnectionState().PeerCertificates[0].PublicKey.(*ecdsa.PublicKey); isECDSA {
 				hostService.Certificate.KeySize = publicKey.Params().BitSize
@@ -249,5 +251,15 @@ func GetBanner(hostService *core.HostService) (err error) {
 			hostService.Certificate.IssuerName = tlsConnection.ConnectionState().PeerCertificates[0].Issuer.CommonName
 		}
 	}
+	if hostService.Type == "http" {
+		//Grab HTTP banner
+		err = GetHttpBanner(hostService)
+		for _, matchFunc := range Matches {
+			if matchFunc(hostService, []byte{}, []string{}) {
+				return nil
+			}
+		}
+	}
+	hostService.DeferSave()
 	return nil
 }

service_map.go

@@ -114,4 +114,13 @@ var Matches = []func(*core.HostService, []byte, []string) bool{
 		}
 		return false
 	},
+	//elasticsearch
+	func(hostService *core.HostService, bytes []byte, i []string) bool {
+		if hostService.Type == "http" &&
+			(strings.Contains(hostService.Data,"lucene") && strings.Contains(hostService.Data,"cluster_uuid") ) {
+			hostService.Type = "elasticsearch"
+			return true
+		}
+		return false
+	},
 }