diff --git a/.travis.yml b/.travis.yml index a2f4dcf..89dd88c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,4 +17,5 @@ install: - go get github.com/jessevdk/go-flags - go get gopkg.in/src-d/go-git.v4 - go get github.com/jaytaylor/html2text + - go get gopkg.in/ns3777k/go-shodan.v2/shodan - go build diff --git a/README.md b/README.md index 9b293ae..100a409 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,25 @@ -# gOSINT [![Build Status](https://travis-ci.org/Nhoya/gOSINT.svg?branch=master)](https://travis-ci.org/Nhoya/gOSINT) [![GitHub stars](https://img.shields.io/github/stars/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/stargazers) [![GitHub forks](https://img.shields.io/github/forks/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/network) [![Twitter](https://img.shields.io/twitter/url/https/github.com/Nhoya/gOSINT.svg?style=social&style=plastic)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2FNhoya%2FgOSINT) [![Go Report Card](https://goreportcard.com/badge/github.com/Nhoya/gOSINT)](https://goreportcard.com/report/github.com/Nhoya/gOSINT) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/76673062a30e48bd99d499d32c0c6af0)](https://www.codacy.com/app/Nhoya/gOSINT?utm_source=github.com&utm_medium=referral&utm_content=Nhoya/gOSINT&utm_campaign=Badge_Grade) +# gOSINT [![Build Status](https://travis-ci.org/Nhoya/gOSINT.svg?branch=master)](https://travis-ci.org/Nhoya/gOSINT) [![Build status](https://ci.appveyor.com/api/projects/status/9qn2y2f8t5up8ww2?svg=true)](https://ci.appveyor.com/project/Nhoya/gosint) [![GitHub stars](https://img.shields.io/github/stars/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/stargazers) [![GitHub forks](https://img.shields.io/github/forks/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/network) [![Twitter](https://img.shields.io/twitter/url/https/github.com/Nhoya/gOSINT.svg?style=social&style=plastic)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2FNhoya%2FgOSINT) [![Go Report Card](https://goreportcard.com/badge/github.com/Nhoya/gOSINT)](https://goreportcard.com/report/github.com/Nhoya/gOSINT) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/76673062a30e48bd99d499d32c0c6af0)](https://www.codacy.com/app/Nhoya/gOSINT?utm_source=github.com&utm_medium=referral&utm_content=Nhoya/gOSINT&utm_campaign=Badge_Grade) OSINT framework in Go Take a look at the [develop branch](https://github.com/Nhoya/gOSINT/tree/develop) for more updates. ## Introduction +gOSINT is a multiplatform OSINT Swiss army knife in Golang. If you want, feel free to contribute and/or leave a feedback! -gOSINT is a small OSINT framework in Golang. If you want, feel free to contribute and/or leave a feedback! +## Like my project? Please consider donation :) -## Like my project? Consider donation :) - -[![Paypal Badge](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.me/Nhoya) [![BTC Badge](https://img.shields.io/badge/Donate-BTC-yellow.svg)](https://pastebin.com/raw/nyDDPwaM) [![Monero Badge](https://img.shields.io/badge/Donate-XMR-yellow.svg)](https://pastebin.com/raw/dNUFqwuC) +[![Paypal Badge](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.me/Nhoya) [![BTC Badge](https://img.shields.io/badge/Donate-BTC-yellow.svg)](https://pastebin.com/raw/nyDDPwaM) [![Monero Badge](https://img.shields.io/badge/Donate-XMR-yellow.svg)](https://pastebin.com/raw/dNUFqwuC) [![Ethereum Badge](https://img.shields.io/badge/Donate-Ethereum-yellow.svg)](https://pastebin.com/raw/S6XMmSiv) ## What gOSINT can do - [x] Find mails from git repository - [x] Find Dumps for mail address - [x] Search for mail address linked to domain/mail address in PGP keyring -- [x] Retrieve Info from domain whois (waiting to be implemented) - [x] Search for mail address in source code - [x] Retrieve Telegram Public Groups History +- [x] Retrieve info about hosts via shodan scan -## Building - +## Building on Linux You can use the building script, just clone the directory and execute it ``` @@ -36,7 +34,7 @@ You can then call `gOSINT` from command line `$ gOSINT --help` -## Manual Building +## Manual Building on Linux #### Dependecies Before building `gOSINT` manually you need to solve the dependencies: @@ -47,21 +45,24 @@ go get "github.com/nhoya/goPwned" go get "github.com/jessevdk/go-flags" go get "gopkg.in/src-d/go-git.v4" go get "github.com/jaytaylor/html2text" +go get "gopkg.in/ns3777k/go-shodan.v2/shodan" ``` `git clone https://github.com/Nhoya/gOSINT && cd gOSINT && go build` +## Binaries for Windows +Check the AppVeyor Build page for builds + ## Modules -Currently `gOSINT` is still an early version and few modules are supported +Currently `gOSINT` has different modules: - [x] git support for mail retriving (using github API, bitbucket API or RAW clone and search) - [x] Search for mails in PGP Server - [x] [https://haveibeenpwned.com/](http://haveibeenpwned.com/) search for mail in databreach -- [x] Retrive Telegram Public Group Messages -- [ ] WHOIS support (the module is ready but has to be integrated) +- [x] Retrieve Telegram Public Group Messages - [x] Search for mail address in source -- [ ] [https://shodan.io](https://shodan.io) search +- [x] [https://shodan.io](https://shodan.io) search - [ ] Social Media search - [ ] Search Engine search @@ -72,28 +73,50 @@ Usage: gOSINT [OPTIONS] Application Options: - -m, --module=[pgp|pwnd|git|plainSearch|telegram] Specify module - -v, --version Print version - --url= Specify target URL - --gitAPI=[github|bitbucket] Specify git website API to use (for git module,optional) - -c, --clone Enable clone function for plainSearch module (need to specify repo URL) - --mail= Specify mail target (for pgp and pwnd module) - --grace= Specify telegram messages grace period (default: 15) - -g, --tgroup= Specify Telegram group/channel name - -s, --tgstart= Specify first message to scrape - -e, --tgend= Specify last message to scrape - --dumpfile Create and resume messages from dumpfile - --ask-confirmation Ask confirmation before adding mail to set (for plainSearch module) - -p, --path= Specify target path (for plainSearch module) - -f, --full Make deep search using linked modules + -m, --module=[pgp|pwnd|git|plainSearch|telegram|shodan] Specify module + -v, --version Print version + --url= Specify target URL + --gitAPI=[github|bitbucket] Specify git website API to use (for git module,optional) + -c, --clone Enable clone function for plainSearch module (need to specify repo URL) + --mail= Specify mail target (for pgp and pwnd module) + --grace= Specify telegram messages grace period (default: 15) + -g, --tgroup= Specify Telegram group/channel name + -s, --tgstart= Specify first message to scrape + -e, --tgend= Specify last message to scrape + --dumpfile Create and resume messages from dumpfile + --ask-confirmation Ask confirmation before adding mail to set (for plainSearch module) + -p, --path= Specify target path (for plainSearch module) + -t, --target= Specify shodan target host + --newscan Ask shodan for a new scan (-1 Scan credit) + --honeypot Check Honeypot probability + -f, --full Make deep search using linked modules Help Options: - -h, --help Show this help message + -h, --help Show this help message ``` +## Configuration file + +The configuration file is in `$HOME/.config/gOSINT.conf` + +If some API Keys are missing insert it there + +## PGP module Demo +[![asciicast](https://asciinema.org/a/21PCpbgFqyHiTbPINexHKEywj.png)](https://asciinema.org/a/21PCpbgFqyHiTbPINexHKEywj) + +## Pwnd module Demo +[![asciicast](https://asciinema.org/a/x9Ap0IRcNNcLfriVujkNUhFSF.png)](https://asciinema.org/a/x9Ap0IRcNNcLfriVujkNUhFSF) + +## Telegram Crawler Demo +[![asciicast](https://asciinema.org/a/nbRO9FNpjiYXAKeI87xn29j9z.png)](https://asciinema.org/a/nbRO9FNpjiYXAKeI87xn29j9z) + +## Shodan module Demo +[![asciicast](https://asciinema.org/a/9lfzAZ65n9MJCkrUrxoHZQYwP.png)](https://asciinema.org/a/9lfzAZ65n9MJCkrUrxoHZQYwP) + + ## Examples -Currently `gOSINT` supports the following actions +Currently `gOSINT` supports the following actions: `gOSINT -m git --url=[RepoURL] --gitAPI [github|bitbucket] (optional)` @@ -152,11 +175,14 @@ the output will be stored in a file, if the file is already populated it will re Set start and end messages for scraping -## PGP module Demo -[![asciicast](https://asciinema.org/a/21PCpbgFqyHiTbPINexHKEywj.png)](https://asciinema.org/a/21PCpbgFqyHiTbPINexHKEywj) +`gOSINT -m shodan -t [HOST IP]` -## Pwnd module Demo -[![asciicast](https://asciinema.org/a/x9Ap0IRcNNcLfriVujkNUhFSF.png)](https://asciinema.org/a/x9Ap0IRcNNcLfriVujkNUhFSF) +Get Shodan services report for Host -## Telegram Crawler Demo -[![asciicast](https://asciinema.org/a/nbRO9FNpjiYXAKeI87xn29j9z.png)](https://asciinema.org/a/nbRO9FNpjiYXAKeI87xn29j9z) +`gOSINT -m shodan -t [HOST IP] --honeypot` + +Start Shodan service report for host and honeypot probability + +`gOSINT -m shodan -t [HOST IP] --newscan` + +Send request for new shodan scan (1 scan credit will be removed) diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 0000000..bd9dd35 --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,27 @@ +version: '{branch} - {build}' + +branches: + only: + - master + - develop + + +environment: + GOPATH: c:\gopath + GOVERSION: 1.8 + +install: + - go version + - go env + - go get -v github.com/deckarep/golang-set + - go get -v github.com/nhoya/goPwned + - go get -v github.com/jessevdk/go-flags + - go get -v gopkg.in/src-d/go-git.v4 + - go get -v github.com/jaytaylor/html2text + - go get -v gopkg.in/ns3777k/go-shodan.v2/shodan + +build_script: + - go build + +artifacts: + - path: gosint.exe diff --git a/build.sh b/build.sh index f1e29f7..977756f 100755 --- a/build.sh +++ b/build.sh @@ -22,7 +22,7 @@ if [[ "$version" =~ $go_version_regex ]]; then fi fi -dependencies=( github.com/deckarep/golang-set github.com/nhoya/goPwned github.com/jessevdk/go-flags gopkg.in/src-d/go-git.v4 github.com/jaytaylor/html2text) +dependencies=( github.com/deckarep/golang-set github.com/nhoya/goPwned github.com/jessevdk/go-flags gopkg.in/src-d/go-git.v4 github.com/jaytaylor/html2text gopkg.in/ns3777k/go-shodan.v2/shodan ) echo -e "${GREEN}[+] Installing dependencies${END}" @@ -30,7 +30,7 @@ for i in "${dependencies[@]}" do depname=$(echo $i |awk -F / '{print$3}') echo -e "${BLUE}[+] Installing $depname${END}" - go get -v "$i" + go get -u -v "$i" if [ $? != 0 ]; then echo -e "${RED}[-]$i raised error during installation${END}" exit 2 @@ -41,3 +41,4 @@ echo -e "${GREEN}[+] Building gOSINT${END}" go build echo -e "${GREEN}[+] Installing gOSINT${END}" sudo mv gOSINT /usr/local/bin +mv config/config.json $HOME/.config/gOSINT.conf diff --git a/config/config.json b/config/config.json new file mode 100644 index 0000000..f47c21c --- /dev/null +++ b/config/config.json @@ -0,0 +1,3 @@ +{ + "ShodanAPIKey": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +} diff --git a/gOSINT.go b/gOSINT.go index c6fdc25..3d36b9e 100644 --- a/gOSINT.go +++ b/gOSINT.go @@ -8,10 +8,10 @@ import ( "github.com/jessevdk/go-flags" ) -const ver = "v0.4c" +const ver = "v0.5" var opts struct { - Module string `short:"m" long:"module" description:"Specify module" choice:"pgp" choice:"pwnd" choice:"git" choice:"plainSearch" choice:"telegram"` + Module string `short:"m" long:"module" description:"Specify module" choice:"pgp" choice:"pwnd" choice:"git" choice:"plainSearch" choice:"telegram" choice:"shodan"` Version bool `short:"v" long:"version" description:"Print version"` // git module URL string `long:"url" default:"" description:"Specify target URL"` @@ -28,6 +28,10 @@ var opts struct { // plainSearch module Confirm bool `long:"ask-confirmation" description:"Ask confirmation before adding mail to set (for plainSearch module)"` Path string `short:"p" long:"path" description:"Specify target path (for plainSearch module)"` + // shodan module + ShodanTarget []string `short:"t" long:"target" description:"Specify shodan target host"` + ShodanScan bool `long:"newscan" description:"Ask shodan for a new scan (-1 Scan credit)"` + ShodanHoneyPotFlag bool `long:"honeypot" description:"Check Honeypot probability"` // generic Mode bool `short:"f" long:"full" description:"Make deep search using linked modules"` } @@ -48,6 +52,11 @@ func main() { os.Exit(1) } + if opts.Module == "" { + fmt.Println("You need to specify the module you want to use, -h for more info") + os.Exit(1) + } + if opts.Version { fmt.Println("gOSINT " + ver) os.Exit(0) @@ -67,5 +76,7 @@ func main() { initPlainSearch(mailSet) case "telegram": initTelegram() + case "shodan": + initShodan() } } diff --git a/shodan.go b/shodan.go new file mode 100644 index 0000000..1b02d3a --- /dev/null +++ b/shodan.go @@ -0,0 +1,105 @@ +package main + +import ( + "fmt" + "os" + "strconv" + + "gopkg.in/ns3777k/go-shodan.v2/shodan" +) + +func initShodan() { + if len(opts.ShodanTarget) < 1 { + fmt.Println("[-] You need to specify the target") + os.Exit(1) + } + APIKey := getConfigFile().ShodanAPIKey + if APIKey == "" { + fmt.Println("[-] Unable to retrive Shodan API Key from config file") + os.Exit(1) + } + fmt.Println("[+] APIKey Found") + client := shodan.NewClient(nil, APIKey) + if opts.ShodanScan { + newShodanScan(client, opts.ShodanTarget) + } else { + for _, host := range opts.ShodanTarget { + getShodanHostInfo(host, client) + } + } +} + +func getShodanHostInfo(host string, client *shodan.Client) { + fmt.Println("==== REPORT FOR " + host + " ====") + report, err := client.GetServicesForHost(host, &shodan.HostServicesOptions{false, false}) + if err != nil { + fmt.Println("[-] Unable to get Report") + fmt.Println(err) + os.Exit(1) + } + fmt.Println("ISP: " + report.ISP) + fmt.Println("Organization: " + report.Organization) + if report.OS != "" { + fmt.Println("OS: " + report.OS) + } + fmt.Println("Ports:", report.Ports) + fmt.Println("Hostnames:", report.Hostnames) + if len(report.Vulnerabilities) > 0 { + fmt.Println("Vulnerabilities:", report.Vulnerabilities) + } + fmt.Println("Country:", report.HostLocation.Country) + fmt.Println("City:", report.HostLocation.City) + fmt.Println("Last Update: " + report.LastUpdate) + getShodanServicesData(report.Data) + + if opts.ShodanHoneyPotFlag { + checkHoneyPotProbability(client, host) + } +} + +func getShodanServicesData(services []*shodan.HostData) { + for _, service := range services { + if service.Product == "" { + service.Product = "Unknown" + } + fmt.Println("Service on port " + strconv.Itoa(service.Port) + ": " + service.Product + " " + string(service.Version)) + if service.Title != "" { + fmt.Println("\tTitle: " + service.Title) + } + if service.OS != "" { + fmt.Println("\tOS " + service.OS) + } + } +} + +func newShodanScan(client *shodan.Client, hosts []string) { + info, _ := client.GetAPIInfo() + if info.ScanCredits < len(hosts) { + fmt.Println("[-] Insufficient credits") + os.Exit(1) + } + fmt.Println("[+] Current Scan credits:", info.ScanCredits) + fmt.Println("[+] Requesting new scan") + scan, err := client.Scan(hosts) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Printf("[+] Scan request ID: %s (1 credit will be deducted)\n", scan.ID) + for { + status, _ := client.GetScanStatus(scan.ID) + if status.Status == shodan.ScanStatusDone { + fmt.Println("[+] Scan started, the new result will be available in ~30 minutes") + break + } + } +} + +func checkHoneyPotProbability(client *shodan.Client, host string) { + honeyscore, err := client.CalcHoneyScore(host) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Println("Honeypot Score (0-1):", honeyscore) +} diff --git a/telegram.go b/telegram.go index 5209246..63cd0f0 100644 --- a/telegram.go +++ b/telegram.go @@ -33,8 +33,8 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag fmt.Println("[?] End message set, grace time will be ignored") } - //path for dumps - dumpfile := "tgdumps/" + group + ".dump" + //dump file + dumpfile := os.Getenv("HOME") + "/.local/share/gOSINT/tgdumps/" + group + ".dump" //counter for deleted messages dmCounter := 0 //set messageCounter as startMessage, is -e is not used the default value of startMessage is 0 (Note: the first message on group is id:1) @@ -42,7 +42,7 @@ func getTelegramGroupHistory(group string, grace int, dumpFlag bool, startMessag readFromTelegramDump(&startMessage, dumpfile, dumpFlag, &messageCounter) //add a counter to remember the first message firstMessageCounter := messageCounter - startMessage - //this is needed because if a file is availabe it will start from the next to the last found + //this is needed because if a file is available it will start from the next to the last found messageCounter++ //if -e or - s is set but on the dumpfile the message is already scraped if dumpFlag && ((endMessage != 0 && messageCounter >= endMessage) || (startMessage != 0 && messageCounter >= startMessage)) { @@ -219,7 +219,7 @@ func createMessage(body string, message string) string { func readFromTelegramDump(startMessage *int, dumpfile string, dumpFlag bool, messageCounter *int) { if dumpFlag { - createDirectory("tgdumps") + createDirectory(os.Getenv("HOME") + "/.local/share/gOSINT/tgdumps/") fmt.Println("[=] --dumpfile used, ignoring --startpoint") *startMessage = 0 if fileExists(dumpfile) { diff --git a/utils.go b/utils.go index 0303566..67bea36 100644 --- a/utils.go +++ b/utils.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -11,6 +12,12 @@ import ( "github.com/deckarep/golang-set" ) +//Configuration struct will contain the configuration parameters +//the config file is available in $HOME/.config/gOSINT.conf +type Configuration struct { + ShodanAPIKey string +} + func retrieveRequestBody(domain string) string { resp, err := http.Get(domain) if err != nil { @@ -55,6 +62,7 @@ func writeOnFile(filename string, text string) { f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) if err != nil { fmt.Println("Unabale to open file") + fmt.Println(err) os.Exit(1) } _, err = f.WriteString(text) @@ -70,9 +78,9 @@ func fileExists(file string) bool { return false } func createDirectory(dirname string) { - if !fileExists("tgdumps") { + if !fileExists(dirname) { fmt.Println("[+] Creating directory " + dirname) - os.Mkdir(dirname, os.ModePerm) + os.MkdirAll(dirname, os.ModePerm) } } @@ -89,3 +97,19 @@ func simpleQuestion(question string) bool { } return false } + +func getConfigFile() Configuration { + file, err := os.Open(os.Getenv("HOME") + "/.config/gOSINT.conf") + if err != nil { + fmt.Println("[-] Unable to open config file, be sure it exists") + os.Exit(1) + } + decoder := json.NewDecoder(file) + config := Configuration{} + err = decoder.Decode(&config) + if err != nil { + fmt.Println("[-] Unable to read config file") + os.Exit(1) + } + return config +}