From 702ec9915d4e7747934685d693c880253b3c1099 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Feb 2023 15:39:27 +0000 Subject: [PATCH 01/19] Add JSON lines output capability to cariddi --- cmd/cariddi/main.go | 12 ++-- pkg/crawler/colly.go | 24 ++++++-- pkg/input/flags.go | 3 + pkg/output/beautify.go | 11 ++-- pkg/output/json.go | 122 ++++++++++++++++++++++++++++++++++++++ pkg/scanner/endpoints.go | 4 +- pkg/scanner/error.go | 10 ++-- pkg/scanner/extensions.go | 4 +- pkg/scanner/info.go | 10 ++-- pkg/scanner/secrets.go | 16 ++--- 10 files changed, 179 insertions(+), 37 deletions(-) create mode 100644 pkg/output/json.go diff --git a/cmd/cariddi/main.go b/cmd/cariddi/main.go index ca5cd1a..c1c1896 100644 --- a/cmd/cariddi/main.go +++ b/cmd/cariddi/main.go @@ -121,7 +121,7 @@ func main() { // For each target generate a crawler and collect all the results. for _, inp := range targets { - results, secrets, endpoints, extensions, errors, infos := crawler.New(inp, ResultTxt, ResultHTML, flags.Delay, + results, secrets, endpoints, extensions, errors, infos := crawler.New(inp, flags.JSON, ResultTxt, ResultHTML, flags.Delay, flags.Concurrency, flags.Ignore, flags.IgnoreTXT, flags.Cache, flags.Timeout, flags.Intensive, flags.Rua, flags.Proxy, flags.Insecure, flags.Secrets, secretsFileSlice, flags.Plain, flags.Endpoints, endpointsFileSlice, flags.Extensions, headers, flags.Errors, flags.Info, flags.Debug, flags.UserAgent) @@ -155,14 +155,14 @@ func main() { } // If needed print secrets. - if !flags.Plain && len(finalSecret) != 0 { + if !flags.JSON && !flags.Plain && len(finalSecret) != 0 { for _, elem := range finalSecret { output.EncapsulateCustomGreen(elem.Secret.Name, elem.Match+" in "+elem.URL) } } // If needed print endpoints. - if !flags.Plain && len(finalEndpoints) != 0 { + if !flags.JSON && !flags.Plain && len(finalEndpoints) != 0 { for _, elem := range finalEndpoints { for _, parameter := range elem.Parameters { finalString := "" + parameter.Parameter @@ -179,21 +179,21 @@ func main() { } // If needed print extensions. - if !flags.Plain && len(finalExtensions) != 0 { + if !flags.JSON && !flags.Plain && len(finalExtensions) != 0 { for _, elem := range finalExtensions { output.EncapsulateCustomGreen(elem.Filetype.Extension, elem.URL+" matched!") } } // If needed print errors. - if !flags.Plain && len(finalErrors) != 0 { + if !flags.JSON && !flags.Plain && len(finalErrors) != 0 { for _, elem := range finalErrors { output.EncapsulateCustomGreen(elem.Error.ErrorName, elem.Match+" in "+elem.URL) } } // If needed print infos. - if !flags.Plain && len(finalInfos) != 0 { + if !flags.JSON && !flags.Plain && len(finalInfos) != 0 { for _, elem := range finalInfos { output.EncapsulateCustomGreen(elem.Info.Name, elem.Match+" in "+elem.URL) } diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index 03e46b8..91886dd 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -52,7 +52,7 @@ import ( // New it's the actual crawler engine. // It controls all the behaviours of a scan // (event handlers, secrets, errors, extensions and endpoints scanning). -func New(target string, txt string, html string, delayTime int, concurrency int, +func New(target string, jsonl bool, txt string, html string, delayTime int, concurrency int, ignore string, ignoreTxt string, cache bool, timeout int, intensive bool, rua bool, proxy string, insecure bool, secretsFlag bool, secretsFile []string, plain bool, endpointsFlag bool, endpointsFile []string, fileType int, headers map[string]string, errorsFlag bool, infoFlag bool, @@ -116,7 +116,9 @@ func New(target string, txt string, html string, delayTime int, concurrency int, // On every request that Colly is making, print the URL it's currently visiting c.OnRequest(func(e *colly.Request) { - fmt.Println(e.URL.String()) + if jsonl == false { + fmt.Println(e.URL.String()) + } }) // On every a element which has href attribute call callback @@ -204,8 +206,12 @@ func New(target string, txt string, html string, delayTime int, concurrency int, c.OnResponse(func(r *colly.Response) { minBodyLentgh := 10 - lengthOk := len(string(r.Body)) > minBodyLentgh + secrets := []scanner.SecretMatched{} + parameters := []scanner.Parameter{} + errors := []scanner.ErrorMatched{} + infos := []scanner.InfoMatched{} + filetype := scanner.FileType{} // if endpoints or secrets or filetype: scan if endpointsFlag || secretsFlag || (1 <= fileType && fileType <= 7) || errorsFlag || infoFlag { @@ -214,6 +220,7 @@ func New(target string, txt string, html string, delayTime int, concurrency int, secretsSlice := huntSecrets(secretsFile, r.Request.URL.String(), string(r.Body)) for _, elem := range secretsSlice { FinalSecrets = append(FinalSecrets, elem) + secrets = append(secrets, elem) } } // HERE SCAN FOR ENDPOINTS @@ -222,6 +229,7 @@ func New(target string, txt string, html string, delayTime int, concurrency int, for _, elem := range endpointsSlice { if len(elem.Parameters) != 0 { FinalEndpoints = append(FinalEndpoints, elem) + parameters = append(parameters, elem.Parameters...) } } } @@ -230,6 +238,7 @@ func New(target string, txt string, html string, delayTime int, concurrency int, extension := huntExtensions(r.Request.URL.String(), fileType) if extension.URL != "" { FinalExtensions = append(FinalExtensions, extension) + filetype = extension.Filetype } } // HERE SCAN FOR ERRORS @@ -237,6 +246,7 @@ func New(target string, txt string, html string, delayTime int, concurrency int, errorsSlice := huntErrors(r.Request.URL.String(), string(r.Body)) for _, elem := range errorsSlice { FinalErrors = append(FinalErrors, elem) + errors = append(errors, elem) } } @@ -245,9 +255,13 @@ func New(target string, txt string, html string, delayTime int, concurrency int, infosSlice := huntInfos(r.Request.URL.String(), string(r.Body)) for _, elem := range infosSlice { FinalInfos = append(FinalInfos, elem) + infos = append(infos, elem) } } } + if jsonl == true { + output.GetJsonString(r, secrets, parameters, filetype, errors, infos) + } }) // Start scraping on target @@ -489,18 +503,18 @@ func EndpointsMatch(target string, endpointsFile []string) []scanner.EndpointMat if strings.ToLower(param) == parameter.Parameter { matched = append(matched, parameter) } - endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target}) } } + endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target}) } else { for _, parameter := range endpointsFile { for _, param := range parameters { if param == parameter { matched = append(matched, scanner.Parameter{Parameter: parameter, Attacks: []string{}}) } - endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target}) } } + endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target}) } return endpoints diff --git a/pkg/input/flags.go b/pkg/input/flags.go index 32c5352..8514978 100644 --- a/pkg/input/flags.go +++ b/pkg/input/flags.go @@ -44,6 +44,7 @@ type Input struct { Help bool Examples bool Plain bool + JSON bool HTML string TXT string Ignore string @@ -77,6 +78,7 @@ func ScanFlag() Input { helpPtr := flag.Bool("h", false, "Print the help.") examplesPtr := flag.Bool("examples", false, "Print the examples.") plainPtr := flag.Bool("plain", false, "Print only the results.") + outputJSONPtr := flag.Bool("json", false, "Write the output as JSON lines.") outputHTMLPtr := flag.String("oh", "", "Write the output into an HTML file.") outputTXTPtr := flag.String("ot", "", "Write the output into a TXT file.") ignorePtr := flag.String("i", "", "Ignore the URL containing at least one of the elements of this array.") @@ -120,6 +122,7 @@ func ScanFlag() Input { *helpPtr, *examplesPtr, *plainPtr, + *outputJSONPtr, *outputHTMLPtr, *outputTXTPtr, *ignorePtr, diff --git a/pkg/output/beautify.go b/pkg/output/beautify.go index 9fae40f..f6deecf 100644 --- a/pkg/output/beautify.go +++ b/pkg/output/beautify.go @@ -28,6 +28,7 @@ package output import ( "fmt" + "os" "github.com/fatih/color" ) @@ -39,14 +40,16 @@ func Beautify() { banner3 := " / __/ _` | '__| |/ _` |/ _` | |\n" banner4 := " | (_| (_| | | | | (_| | (_| | |\n" banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.2.1\n" - banner6 := "" + banner6 := "\n" banner7 := " > github.com/edoardottt/cariddi\n" banner8 := " > edoardoottavianelli.it\n" - banner9 := "========================================" + banner9 := "========================================\n" bannerPart1 := banner1 + banner2 + banner3 + banner4 + banner5 bannerPart2 := banner6 + banner7 + banner8 + banner9 - color.Cyan("%s\n", bannerPart1) - fmt.Println(bannerPart2) + color.Set(color.FgCyan) + fmt.Fprintf(os.Stderr, bannerPart1) + color.Unset() + fmt.Fprintf(os.Stderr, bannerPart2) } diff --git a/pkg/output/json.go b/pkg/output/json.go new file mode 100644 index 0000000..6a547dd --- /dev/null +++ b/pkg/output/json.go @@ -0,0 +1,122 @@ +/* +========== +Cariddi +========== + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see http://www.gnu.org/licenses/. + + @Repository: https://github.com/edoardottt/cariddi + + @Author: edoardottt, https://www.edoardoottavianelli.it + + @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE + +*/ + +package output + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/edoardottt/cariddi/pkg/scanner" + "github.com/gocolly/colly" +) + +type jsonData struct { + URL string `json:"url"` + Method string `json:"method"` + StatusCode int `json:"status_code"` + Words int `json:"words"` + Lines int `json:"lines"` + ContentType string `json:"content_type,omit_empty"` + ContentLength int `json:"content_length,omit_empty"` + Matches MatcherResults `json:"matches,omitempty"` + // Host string `json:"host"` # TODO: Add when migrating to Colly 2.x +} + +type MatcherResults struct { + Secrets []scanner.SecretMatched `json:"secrets,omit_empty"` + Parameters []scanner.Parameter `json:"parameters,omit_empty"` + FileType scanner.FileType `json:"filetype,omit_empty"` + Errors []scanner.ErrorMatched `json:"errors,omit_empty"` + Infos []scanner.InfoMatched `json:"infos,omit_empty"` +} + +func GetJsonString( + r *colly.Response, + secrets []scanner.SecretMatched, + parameters []scanner.Parameter, + filetype scanner.FileType, + errors []scanner.ErrorMatched, + infos []scanner.InfoMatched, +) ([]byte, error) { + + // Parse response headers + headers := r.Headers + contentTypes := (*headers)["Content-Type"] + contentType := "" + if len(contentTypes) > 0 { + contentType = contentTypes[0] + } + + contentLength := 0 + contentLengths := (*headers)["Content-Length"] + if len(contentLengths) > 0 { + ret, err := strconv.Atoi(contentLengths[0]) + if err != nil { + return nil, err + } + contentLength = ret + } + + // Parse words from body + words := len(strings.Fields(string(r.Body))) + + // Parse lines from body + lines := len(strings.Split(string(r.Body), "\n")) + + // Construct JSON response + data := MatcherResults{ + Secrets: secrets, + Parameters: parameters, + FileType: filetype, + Errors: errors, + Infos: infos, + } + resp := &jsonData{ + URL: r.Request.URL.String(), + Method: r.Request.Method, + StatusCode: r.StatusCode, + Words: words, + Lines: lines, + ContentType: contentType, + ContentLength: contentLength, + Matches: data, + // Host: "", // TODO: this is available in Colly 2.x but not in 1.2 + } + + // Convert struct to JSON string + jsonOutput, err := json.Marshal(resp) + if err != nil { + return nil, err + } + + // Output JSON string + fmt.Println(string(jsonOutput)) + + return jsonOutput, nil +} diff --git a/pkg/scanner/endpoints.go b/pkg/scanner/endpoints.go index 463c7c6..ba4fad3 100644 --- a/pkg/scanner/endpoints.go +++ b/pkg/scanner/endpoints.go @@ -30,8 +30,8 @@ package scanner // Parameter = the name of the parameter. // Attacks = Possible attacks. type Parameter struct { - Parameter string - Attacks []string + Parameter string `json:"name,omitempty"` + Attacks []string `json:"attacks,omitempty"` } // EndpointMatched struct. diff --git a/pkg/scanner/error.go b/pkg/scanner/error.go index 9d3870e..44c6855 100644 --- a/pkg/scanner/error.go +++ b/pkg/scanner/error.go @@ -27,8 +27,8 @@ package scanner // ErrorName = the name that identifies the error. // Regex = The regular expression to be matched. type Error struct { - ErrorName string - Regex []string + ErrorName string `json:"name,omitempty"` + Regex []string `json:"regex,omitempty"` } // ErrorMatched struct. @@ -36,9 +36,9 @@ type Error struct { // Url = url in which the error is found. // Match = the string matching the regex. type ErrorMatched struct { - Error Error - URL string - Match string + Error Error `json:"details,omitempty"` + URL string `json:"-"` + Match string `json:"match,omitempty"` } // GetErrorRegexes returns all the error structs. diff --git a/pkg/scanner/extensions.go b/pkg/scanner/extensions.go index f84feb2..9aef287 100644 --- a/pkg/scanner/extensions.go +++ b/pkg/scanner/extensions.go @@ -30,8 +30,8 @@ package scanner // Extension = the file extension (doc, txt ..etc..). // Severity = the 'importance' of the file found. Higher is better. type FileType struct { - Extension string - Severity int + Extension string `json:"extension,omitempty"` + Severity int `json:"severity,omitempty"` } // FileTypeMatched struct. diff --git a/pkg/scanner/info.go b/pkg/scanner/info.go index f45e9d5..2754861 100644 --- a/pkg/scanner/info.go +++ b/pkg/scanner/info.go @@ -27,8 +27,8 @@ package scanner // Name = the name that identifies the information. // Regex = The regular expression to be matched. type Info struct { - Name string - Regex []string + Name string `json:"name,omitempty"` + Regex []string `json:"regex,omitempty"` } // InfoMatched struct. @@ -36,9 +36,9 @@ type Info struct { // Url = url in which the information is found. // Match = the string matching the regex. type InfoMatched struct { - Info Info - URL string - Match string + Info Info `json:"details,omitempty"` + URL string `json:"-"` + Match string `json:"match,omitempty"` } // GetInfoRegexes returns all the info structs. diff --git a/pkg/scanner/secrets.go b/pkg/scanner/secrets.go index b005d74..835e149 100644 --- a/pkg/scanner/secrets.go +++ b/pkg/scanner/secrets.go @@ -33,11 +33,11 @@ package scanner // FalsePositives = A list of known false positives. // PoC = cli command to check if the secret is valid or not. type Secret struct { - Name string - Description string - Regex string - FalsePositives []string - Poc string + Name string `json:"name"` + Description string `json:"description"` + Regex string `json:"regex"` + FalsePositives []string `json:"-"` + Poc string `json:"-"` } // SecretMatched struct. @@ -45,9 +45,9 @@ type Secret struct { // Url = url in which is present the secret. // Match = the string matching the regex. type SecretMatched struct { - Secret Secret - URL string - Match string + Secret Secret `json:"details,omitempty"` + URL string `json:"-"` + Match string `json:"match,omitempty"` } // GetSecretRegexes returns a slice of all From b22d12e9fa3ac5ae40f5c162ce64f36ca847b075 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Feb 2023 16:43:05 +0000 Subject: [PATCH 02/19] update --- pkg/output/json.go | 45 +++++++++++++++++++++++++++++---------- pkg/scanner/endpoints.go | 4 ++-- pkg/scanner/error.go | 10 ++++----- pkg/scanner/extensions.go | 4 ++-- pkg/scanner/info.go | 10 ++++----- pkg/scanner/secrets.go | 16 +++++++------- 6 files changed, 56 insertions(+), 33 deletions(-) diff --git a/pkg/output/json.go b/pkg/output/json.go index 6a547dd..47a0421 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -42,18 +42,23 @@ type jsonData struct { StatusCode int `json:"status_code"` Words int `json:"words"` Lines int `json:"lines"` - ContentType string `json:"content_type,omit_empty"` - ContentLength int `json:"content_length,omit_empty"` + ContentType string `json:"content_type,omitempty"` + ContentLength int `json:"content_length,omitempty"` Matches MatcherResults `json:"matches,omitempty"` // Host string `json:"host"` # TODO: Add when migrating to Colly 2.x } type MatcherResults struct { - Secrets []scanner.SecretMatched `json:"secrets,omit_empty"` - Parameters []scanner.Parameter `json:"parameters,omit_empty"` - FileType scanner.FileType `json:"filetype,omit_empty"` - Errors []scanner.ErrorMatched `json:"errors,omit_empty"` - Infos []scanner.InfoMatched `json:"infos,omit_empty"` + FileType scanner.FileType `json:"filetype,omitempty"` + Parameters []scanner.Parameter `json:"parameters,omitempty"` + Errors []MatcherResult `json:"errors,omitempty"` + Infos []MatcherResult `json:"infos,omitempty"` + Secrets []MatcherResult `json:"secrets,omitempty"` +} + +type MatcherResult struct { + Name string `json:"name"` + Match string `json:"match"` } func GetJsonString( @@ -89,13 +94,31 @@ func GetJsonString( // Parse lines from body lines := len(strings.Split(string(r.Body), "\n")) + // Process secrets + secretList := []MatcherResult{} + for _, secret := range secrets { + secretList = append(secretList, MatcherResult{secret.Secret.Name, secret.Match}) + } + + // Process infos + infoList := []MatcherResult{} + for _, info := range infos { + infoList = append(infoList, MatcherResult{info.Info.Name, info.Match}) + } + + // Process + errorList := []MatcherResult{} + for _, error := range errors { + errorList = append(errorList, MatcherResult{error.Error.ErrorName, error.Match}) + } + // Construct JSON response data := MatcherResults{ - Secrets: secrets, - Parameters: parameters, FileType: filetype, - Errors: errors, - Infos: infos, + Parameters: parameters, + Errors: errorList, + Infos: infoList, + Secrets: secretList, } resp := &jsonData{ URL: r.Request.URL.String(), diff --git a/pkg/scanner/endpoints.go b/pkg/scanner/endpoints.go index ba4fad3..463c7c6 100644 --- a/pkg/scanner/endpoints.go +++ b/pkg/scanner/endpoints.go @@ -30,8 +30,8 @@ package scanner // Parameter = the name of the parameter. // Attacks = Possible attacks. type Parameter struct { - Parameter string `json:"name,omitempty"` - Attacks []string `json:"attacks,omitempty"` + Parameter string + Attacks []string } // EndpointMatched struct. diff --git a/pkg/scanner/error.go b/pkg/scanner/error.go index 44c6855..9d3870e 100644 --- a/pkg/scanner/error.go +++ b/pkg/scanner/error.go @@ -27,8 +27,8 @@ package scanner // ErrorName = the name that identifies the error. // Regex = The regular expression to be matched. type Error struct { - ErrorName string `json:"name,omitempty"` - Regex []string `json:"regex,omitempty"` + ErrorName string + Regex []string } // ErrorMatched struct. @@ -36,9 +36,9 @@ type Error struct { // Url = url in which the error is found. // Match = the string matching the regex. type ErrorMatched struct { - Error Error `json:"details,omitempty"` - URL string `json:"-"` - Match string `json:"match,omitempty"` + Error Error + URL string + Match string } // GetErrorRegexes returns all the error structs. diff --git a/pkg/scanner/extensions.go b/pkg/scanner/extensions.go index 9aef287..f84feb2 100644 --- a/pkg/scanner/extensions.go +++ b/pkg/scanner/extensions.go @@ -30,8 +30,8 @@ package scanner // Extension = the file extension (doc, txt ..etc..). // Severity = the 'importance' of the file found. Higher is better. type FileType struct { - Extension string `json:"extension,omitempty"` - Severity int `json:"severity,omitempty"` + Extension string + Severity int } // FileTypeMatched struct. diff --git a/pkg/scanner/info.go b/pkg/scanner/info.go index 2754861..f45e9d5 100644 --- a/pkg/scanner/info.go +++ b/pkg/scanner/info.go @@ -27,8 +27,8 @@ package scanner // Name = the name that identifies the information. // Regex = The regular expression to be matched. type Info struct { - Name string `json:"name,omitempty"` - Regex []string `json:"regex,omitempty"` + Name string + Regex []string } // InfoMatched struct. @@ -36,9 +36,9 @@ type Info struct { // Url = url in which the information is found. // Match = the string matching the regex. type InfoMatched struct { - Info Info `json:"details,omitempty"` - URL string `json:"-"` - Match string `json:"match,omitempty"` + Info Info + URL string + Match string } // GetInfoRegexes returns all the info structs. diff --git a/pkg/scanner/secrets.go b/pkg/scanner/secrets.go index 835e149..b005d74 100644 --- a/pkg/scanner/secrets.go +++ b/pkg/scanner/secrets.go @@ -33,11 +33,11 @@ package scanner // FalsePositives = A list of known false positives. // PoC = cli command to check if the secret is valid or not. type Secret struct { - Name string `json:"name"` - Description string `json:"description"` - Regex string `json:"regex"` - FalsePositives []string `json:"-"` - Poc string `json:"-"` + Name string + Description string + Regex string + FalsePositives []string + Poc string } // SecretMatched struct. @@ -45,9 +45,9 @@ type Secret struct { // Url = url in which is present the secret. // Match = the string matching the regex. type SecretMatched struct { - Secret Secret `json:"details,omitempty"` - URL string `json:"-"` - Match string `json:"match,omitempty"` + Secret Secret + URL string + Match string } // GetSecretRegexes returns a slice of all From bc927a1e2328f000435163b6d59ba1fe969cdf4f Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Feb 2023 16:45:32 +0000 Subject: [PATCH 03/19] update --- pkg/scanner/extensions.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/scanner/extensions.go b/pkg/scanner/extensions.go index f84feb2..9aef287 100644 --- a/pkg/scanner/extensions.go +++ b/pkg/scanner/extensions.go @@ -30,8 +30,8 @@ package scanner // Extension = the file extension (doc, txt ..etc..). // Severity = the 'importance' of the file found. Higher is better. type FileType struct { - Extension string - Severity int + Extension string `json:"extension,omitempty"` + Severity int `json:"severity,omitempty"` } // FileTypeMatched struct. From 7d76368b8213a3582001efaa6f5630c0e2cd5846 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Feb 2023 16:47:01 +0000 Subject: [PATCH 04/19] update --- pkg/output/json.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/output/json.go b/pkg/output/json.go index 47a0421..d34e8dd 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -36,7 +36,7 @@ import ( "github.com/gocolly/colly" ) -type jsonData struct { +type JsonData struct { URL string `json:"url"` Method string `json:"method"` StatusCode int `json:"status_code"` @@ -120,7 +120,7 @@ func GetJsonString( Infos: infoList, Secrets: secretList, } - resp := &jsonData{ + resp := &JsonData{ URL: r.Request.URL.String(), Method: r.Request.Method, StatusCode: r.StatusCode, From 3baf1946bfa5e0634b4b81c35dd650372ffd83c4 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Feb 2023 17:15:22 +0000 Subject: [PATCH 05/19] add json output to Parameter --- pkg/scanner/endpoints.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/scanner/endpoints.go b/pkg/scanner/endpoints.go index 463c7c6..a4a3d61 100644 --- a/pkg/scanner/endpoints.go +++ b/pkg/scanner/endpoints.go @@ -30,8 +30,8 @@ package scanner // Parameter = the name of the parameter. // Attacks = Possible attacks. type Parameter struct { - Parameter string - Attacks []string + Parameter string `json:"name"` + Attacks []string `json:"attacks"` } // EndpointMatched struct. From 7f3077a24e28a14854be627a8d2c6b7d98fc982c Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Feb 2023 17:38:09 +0000 Subject: [PATCH 06/19] update to devel branch --- cmd/cariddi/main.go | 1 + pkg/crawler/colly.go | 16 ++++++++-------- pkg/crawler/utils.go | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cmd/cariddi/main.go b/cmd/cariddi/main.go index da9a3dd..b517fdc 100644 --- a/cmd/cariddi/main.go +++ b/cmd/cariddi/main.go @@ -73,6 +73,7 @@ func main() { Ignore: flags.Ignore, IgnoreTxt: flags.IgnoreTXT, Cache: flags.Cache, + JSON: flags.JSON, Timeout: flags.Timeout, Intensive: flags.Intensive, Rua: flags.Rua, diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index d0ea780..ee3336d 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -69,7 +69,7 @@ type Scan struct { SecretsFlag bool Ignore string IgnoreTxt string - JSON bool + JSON bool HTML string Proxy string Target string @@ -146,7 +146,7 @@ func New(scan *Scan) *Results { // On every request that Colly is making, print the URL it's currently visiting c.OnRequest(func(e *colly.Request) { - if jsonl == false { + if scan.JSON == false { fmt.Println(e.URL.String()) } }) @@ -262,7 +262,7 @@ func New(scan *Scan) *Results { if scan.SecretsFlag && lengthOk { secretsSlice := huntSecrets(scan.SecretsSlice, r.Request.URL.String(), string(r.Body)) results.Secrets = append(results.Secrets, secretsSlice...) - secrets = append(secrets, secretsSlice...) + secrets = append(secrets, secretsSlice...) } // HERE SCAN FOR ENDPOINTS if scan.EndpointsFlag { @@ -270,7 +270,7 @@ func New(scan *Scan) *Results { for _, elem := range endpointsSlice { if len(elem.Parameters) != 0 { results.Endpoints = append(results.Endpoints, elem) - parameters = append(parameters, elem.Parameters...) + parameters = append(parameters, elem.Parameters...) } } } @@ -279,24 +279,24 @@ func New(scan *Scan) *Results { extension := huntExtensions(r.Request.URL.String(), scan.FileType) if extension.URL != "" { results.Extensions = append(results.Extensions, extension) - filetype = extension.Filetype + filetype = extension.Filetype } } // HERE SCAN FOR ERRORS if scan.ErrorsFlag { errorsSlice := huntErrors(r.Request.URL.String(), string(r.Body)) results.Errors = append(results.Errors, errorsSlice...) - errors = append(errors, errorsSlice...) + errors = append(errors, errorsSlice...) } // HERE SCAN FOR INFOS if scan.InfoFlag { infosSlice := huntInfos(r.Request.URL.String(), string(r.Body)) results.Infos = append(results.Infos, infosSlice...) - infos = append(infos, infosSlice) + infos = append(infos, infosSlice...) } } - if jsonl == true { + if scan.JSON == true { output.GetJsonString(r, secrets, parameters, filetype, errors, infos) } }) diff --git a/pkg/crawler/utils.go b/pkg/crawler/utils.go index 988bc6d..7f63e53 100644 --- a/pkg/crawler/utils.go +++ b/pkg/crawler/utils.go @@ -123,18 +123,18 @@ func EndpointsMatch(target string, endpointsFile []string) []scanner.EndpointMat if strings.ToLower(param) == parameter.Parameter { matched = append(matched, parameter) } - endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target}) } } + endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target}) } else { for _, parameter := range endpointsFile { for _, param := range parameters { if param == parameter { matched = append(matched, scanner.Parameter{Parameter: parameter, Attacks: []string{}}) } - endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target}) } } + endpoints = append(endpoints, scanner.EndpointMatched{Parameters: matched, URL: target}) } return endpoints From 81aef1ecd71fc581ad45dbb0459a85c97c3e32ce Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Feb 2023 17:49:28 +0000 Subject: [PATCH 07/19] update --- pkg/crawler/colly.go | 9 ++++++++- pkg/output/json.go | 4 ---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index ee3336d..aba693b 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -297,7 +297,14 @@ func New(scan *Scan) *Results { } } if scan.JSON == true { - output.GetJsonString(r, secrets, parameters, filetype, errors, infos) + jsonOutput, err := output.GetJsonString( + r, secrets, parameters, filetype, errors, infos, + ) + if err == nil { + fmt.Println(string(jsonOutput)) + } else { + log.Println(err) + } } }) diff --git a/pkg/output/json.go b/pkg/output/json.go index d34e8dd..a1317bf 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -28,7 +28,6 @@ package output import ( "encoding/json" - "fmt" "strconv" "strings" @@ -138,8 +137,5 @@ func GetJsonString( return nil, err } - // Output JSON string - fmt.Println(string(jsonOutput)) - return jsonOutput, nil } From 4b0689bbff1a72f442761591ba1bf7ea82b6a8d3 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Feb 2023 17:53:20 +0000 Subject: [PATCH 08/19] update --- pkg/output/json.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/output/json.go b/pkg/output/json.go index a1317bf..673d8ff 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -44,7 +44,7 @@ type JsonData struct { ContentType string `json:"content_type,omitempty"` ContentLength int `json:"content_length,omitempty"` Matches MatcherResults `json:"matches,omitempty"` - // Host string `json:"host"` # TODO: Add when migrating to Colly 2.x + // Host string `json:"host"` # TODO: Available in Colly 2.x } type MatcherResults struct { @@ -96,19 +96,22 @@ func GetJsonString( // Process secrets secretList := []MatcherResult{} for _, secret := range secrets { - secretList = append(secretList, MatcherResult{secret.Secret.Name, secret.Match}) + secretMatch := MatcherResult{secret.Secret.Name, secret.Match} + secretList = append(secretList, secretMatch) } // Process infos infoList := []MatcherResult{} for _, info := range infos { - infoList = append(infoList, MatcherResult{info.Info.Name, info.Match}) + secretMatch := MatcherResult{info.Info.Name, info.Match} + infoList = append(infoList, secretMatch) } // Process errorList := []MatcherResult{} for _, error := range errors { - errorList = append(errorList, MatcherResult{error.Error.ErrorName, error.Match}) + errorMatch := MatcherResult{error.Error.ErrorName, error.Match} + errorList = append(errorList, errorMatch) } // Construct JSON response @@ -128,7 +131,7 @@ func GetJsonString( ContentType: contentType, ContentLength: contentLength, Matches: data, - // Host: "", // TODO: this is available in Colly 2.x but not in 1.2 + // Host: "", // TODO: this is available in Colly 2.x } // Convert struct to JSON string From a6b6288de6a6db742eae4da771e8705dece4f879 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Fri, 3 Feb 2023 17:54:56 +0000 Subject: [PATCH 09/19] fix boolean flag --- pkg/crawler/colly.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index aba693b..32dc98e 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -296,7 +296,7 @@ func New(scan *Scan) *Results { infos = append(infos, infosSlice...) } } - if scan.JSON == true { + if scan.JSON { jsonOutput, err := output.GetJsonString( r, secrets, parameters, filetype, errors, infos, ) From 4f40416f8cca96a8d52a03133df18f3a13f7afc7 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 21 Mar 2023 10:41:41 +0000 Subject: [PATCH 10/19] update event structure to add JSON bool flag --- go.mod | 18 +++++++++--------- go.sum | 39 +++++++++++++++++++++++++++++++++++++++ pkg/crawler/colly.go | 5 ++++- pkg/crawler/utils.go | 1 + 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 1216819..373ee40 100644 --- a/go.mod +++ b/go.mod @@ -8,22 +8,22 @@ require ( ) require ( - github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect - github.com/antchfx/htmlquery v1.2.5 // indirect - github.com/antchfx/xmlquery v1.3.13 // indirect - github.com/antchfx/xpath v1.2.1 // indirect + github.com/antchfx/htmlquery v1.3.0 // indirect + github.com/antchfx/xmlquery v1.3.15 // indirect + github.com/antchfx/xpath v1.2.4 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect - github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/temoto/robotstxt v1.1.2 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/text v0.8.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.30.0 // indirect ) diff --git a/go.sum b/go.sum index 3aeffa3..3a6b9a3 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,22 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/antchfx/htmlquery v1.2.5 h1:1lXnx46/1wtv1E/kzmH8vrfMuUKYgkdDBA9pIdMJnk4= github.com/antchfx/htmlquery v1.2.5/go.mod h1:2MCVBzYVafPBmKbrmwB9F5xdd+IEgRY61ci2oOsOQVw= +github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= +github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= github.com/antchfx/xmlquery v1.3.13 h1:wqhTv2BN5MzYg9rnPVtZb3IWP8kW6WV/ebAY0FCTI7Y= github.com/antchfx/xmlquery v1.3.13/go.mod h1:3w2RvQvTz+DaT5fSgsELkSJcdNgkmg6vuXDEuhdwsPQ= +github.com/antchfx/xmlquery v1.3.15 h1:aJConNMi1sMha5G8YJoAIF5P+H+qG1L73bSItWHo8Tw= +github.com/antchfx/xmlquery v1.3.15/go.mod h1:zMDv5tIGjOxY/JCNNinnle7V/EwthZ5IT8eeCGJKRWA= github.com/antchfx/xpath v1.2.1 h1:qhp4EW6aCOVr5XIkT+l6LJ9ck/JsUH/yyauNgTQkBF8= github.com/antchfx/xpath v1.2.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= +github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= @@ -23,6 +32,8 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= @@ -36,36 +47,62 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= @@ -74,3 +111,5 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index b5356e0..0e9b4ac 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -151,6 +151,7 @@ func New(scan *Scan) *Results { Intensive: scan.Intensive, Ignore: ignoreBool, Debug: scan.Debug, + JSON: scan.JSON, IgnoreSlice: ignoreSlice, URLs: &results.URLs, } @@ -362,7 +363,9 @@ func CreateColly(delayTime int, concurrency int, cache bool, timeout int, func registerHTMLEvents(c *colly.Collector, event *Event) { // On every request that Colly is making, print the URL it's currently visiting c.OnRequest(func(e *colly.Request) { - fmt.Println(e.URL.String()) + if (!event.JSON){ + fmt.Println(e.URL.String()) + } }) // On every a element which has href attribute call callback diff --git a/pkg/crawler/utils.go b/pkg/crawler/utils.go index 484f25b..7854162 100644 --- a/pkg/crawler/utils.go +++ b/pkg/crawler/utils.go @@ -19,6 +19,7 @@ type Event struct { Intensive bool Ignore bool Debug bool + JSON bool IgnoreSlice []string URLs *[]string } From 03d9ce059bfeb1622b5780faaf886340fff75b6e Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 21 Mar 2023 14:10:16 +0000 Subject: [PATCH 11/19] fix linting --- pkg/crawler/colly.go | 2 +- pkg/output/beautify.go | 4 ++-- pkg/output/json.go | 24 ++++++++++++++---------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index 0e9b4ac..986316e 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -219,7 +219,7 @@ func New(scan *Scan) *Results { } } if scan.JSON { - jsonOutput, err := output.GetJsonString( + jsonOutput, err := output.GetJSONString( r, secrets, parameters, filetype, errors, infos, ) if err == nil { diff --git a/pkg/output/beautify.go b/pkg/output/beautify.go index bb59bd3..29ed3f7 100644 --- a/pkg/output/beautify.go +++ b/pkg/output/beautify.go @@ -49,7 +49,7 @@ func Beautify() { bannerPart2 := banner6 + banner7 + banner8 + banner9 color.Set(color.FgCyan) - fmt.Fprintf(os.Stderr, bannerPart1) + fmt.Fprint(os.Stderr, bannerPart1) color.Unset() - fmt.Fprintf(os.Stderr, bannerPart2) + fmt.Fprint(os.Stderr, bannerPart2) } diff --git a/pkg/output/json.go b/pkg/output/json.go index 673d8ff..5b3e269 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -35,7 +35,7 @@ import ( "github.com/gocolly/colly" ) -type JsonData struct { +type JSONData struct { URL string `json:"url"` Method string `json:"method"` StatusCode int `json:"status_code"` @@ -60,7 +60,7 @@ type MatcherResult struct { Match string `json:"match"` } -func GetJsonString( +func GetJSONString( r *colly.Response, secrets []scanner.SecretMatched, parameters []scanner.Parameter, @@ -68,22 +68,28 @@ func GetJsonString( errors []scanner.ErrorMatched, infos []scanner.InfoMatched, ) ([]byte, error) { - // Parse response headers headers := r.Headers contentTypes := (*headers)["Content-Type"] + contentLengths := (*headers)["Content-Length"] contentType := "" + contentLength := 0 + errorList := []MatcherResult{} + infoList := []MatcherResult{} + secretList := []MatcherResult{} + + // Set content type if len(contentTypes) > 0 { contentType = contentTypes[0] } - contentLength := 0 - contentLengths := (*headers)["Content-Length"] + // Set content length if len(contentLengths) > 0 { ret, err := strconv.Atoi(contentLengths[0]) if err != nil { return nil, err } + contentLength = ret } @@ -94,21 +100,19 @@ func GetJsonString( lines := len(strings.Split(string(r.Body), "\n")) // Process secrets - secretList := []MatcherResult{} for _, secret := range secrets { secretMatch := MatcherResult{secret.Secret.Name, secret.Match} secretList = append(secretList, secretMatch) } // Process infos - infoList := []MatcherResult{} + for _, info := range infos { secretMatch := MatcherResult{info.Info.Name, info.Match} infoList = append(infoList, secretMatch) } - // Process - errorList := []MatcherResult{} + // Process errors for _, error := range errors { errorMatch := MatcherResult{error.Error.ErrorName, error.Match} errorList = append(errorList, errorMatch) @@ -122,7 +126,7 @@ func GetJsonString( Infos: infoList, Secrets: secretList, } - resp := &JsonData{ + resp := &JSONData{ URL: r.Request.URL.String(), Method: r.Request.Method, StatusCode: r.StatusCode, From 1d28243dab462f122a6811e34823aba5f1191c70 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Tue, 21 Mar 2023 14:20:44 +0000 Subject: [PATCH 12/19] split content-type to remove charsets --- pkg/output/json.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/output/json.go b/pkg/output/json.go index 5b3e269..7122d07 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -78,9 +78,9 @@ func GetJSONString( infoList := []MatcherResult{} secretList := []MatcherResult{} - // Set content type + // Set content type if len(contentTypes) > 0 { - contentType = contentTypes[0] + contentType = strings.Split(contentTypes[0], "; ")[0] } // Set content length @@ -106,7 +106,6 @@ func GetJSONString( } // Process infos - for _, info := range infos { secretMatch := MatcherResult{info.Info.Name, info.Match} infoList = append(infoList, secretMatch) From 04a4d97896e2568032d8b3d54fef7530a6acb1ee Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 22 Mar 2023 13:35:54 +0000 Subject: [PATCH 13/19] omit values when empty structs --- pkg/crawler/colly.go | 4 ++-- pkg/output/json.go | 26 ++++++++++++++++++++------ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/pkg/crawler/colly.go b/pkg/crawler/colly.go index 32dc98e..118e05e 100644 --- a/pkg/crawler/colly.go +++ b/pkg/crawler/colly.go @@ -253,7 +253,7 @@ func New(scan *Scan) *Results { parameters := []scanner.Parameter{} errors := []scanner.ErrorMatched{} infos := []scanner.InfoMatched{} - filetype := scanner.FileType{} + filetype := &scanner.FileType{} // if endpoints or secrets or filetype: scan if scan.EndpointsFlag || scan.SecretsFlag || @@ -279,7 +279,7 @@ func New(scan *Scan) *Results { extension := huntExtensions(r.Request.URL.String(), scan.FileType) if extension.URL != "" { results.Extensions = append(results.Extensions, extension) - filetype = extension.Filetype + filetype = &extension.Filetype } } // HERE SCAN FOR ERRORS diff --git a/pkg/output/json.go b/pkg/output/json.go index 673d8ff..d72fd2a 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -43,12 +43,12 @@ type JsonData struct { Lines int `json:"lines"` ContentType string `json:"content_type,omitempty"` ContentLength int `json:"content_length,omitempty"` - Matches MatcherResults `json:"matches,omitempty"` + Matches *MatcherResults `json:"matches,omitempty"` // Host string `json:"host"` # TODO: Available in Colly 2.x } type MatcherResults struct { - FileType scanner.FileType `json:"filetype,omitempty"` + FileType *scanner.FileType `json:"filetype,omitempty"` Parameters []scanner.Parameter `json:"parameters,omitempty"` Errors []MatcherResult `json:"errors,omitempty"` Infos []MatcherResult `json:"infos,omitempty"` @@ -64,7 +64,7 @@ func GetJsonString( r *colly.Response, secrets []scanner.SecretMatched, parameters []scanner.Parameter, - filetype scanner.FileType, + filetype *scanner.FileType, errors []scanner.ErrorMatched, infos []scanner.InfoMatched, ) ([]byte, error) { @@ -107,21 +107,22 @@ func GetJsonString( infoList = append(infoList, secretMatch) } - // Process + // Process error list errorList := []MatcherResult{} for _, error := range errors { errorMatch := MatcherResult{error.Error.ErrorName, error.Match} errorList = append(errorList, errorMatch) } - // Construct JSON response - data := MatcherResults{ + data := &MatcherResults{ FileType: filetype, Parameters: parameters, Errors: errorList, Infos: infoList, Secrets: secretList, } + + // Construct JSON response resp := &JsonData{ URL: r.Request.URL.String(), Method: r.Request.Method, @@ -134,6 +135,19 @@ func GetJsonString( // Host: "", // TODO: this is available in Colly 2.x } + // Set empty data if no matches to bridge the omitempty gap for empty structs + var isFileTypeNill bool = false + var isParametersEmpty bool = len(parameters) == 0 + var isErrorsEmpty bool = len(errorList) == 0 + var isInfoEmpty bool = len(infoList) == 0 + if (*filetype == scanner.FileType{}){ + data.FileType = nil + isFileTypeNill = true + } + if (isFileTypeNill && isParametersEmpty && isErrorsEmpty && isInfoEmpty){ + resp.Matches = nil + } + // Convert struct to JSON string jsonOutput, err := json.Marshal(resp) if err != nil { From babee044872240e54b360bc68087042f2d4b31f2 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 22 Mar 2023 13:38:18 +0000 Subject: [PATCH 14/19] fix error --- pkg/output/json.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/output/json.go b/pkg/output/json.go index 0e821ca..786eca4 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -112,7 +112,6 @@ func GetJSONString( } // Process error list - errorList := []MatcherResult{} for _, error := range errors { errorMatch := MatcherResult{error.Error.ErrorName, error.Match} errorList = append(errorList, errorMatch) @@ -127,7 +126,7 @@ func GetJSONString( } // Construct JSON response - resp := &JsonData{ + resp := &JSONData{ URL: r.Request.URL.String(), Method: r.Request.Method, StatusCode: r.StatusCode, From 1ab142d7e695afdca84f93f741ab9be2c73d1b1d Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 22 Mar 2023 13:42:01 +0000 Subject: [PATCH 15/19] fix lint --- pkg/output/json.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/output/json.go b/pkg/output/json.go index 786eca4..01da29f 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -139,14 +139,18 @@ func GetJSONString( } // Set empty data if no matches to bridge the omitempty gap for empty structs - var isFileTypeNill bool = false - var isParametersEmpty bool = len(parameters) == 0 - var isErrorsEmpty bool = len(errorList) == 0 - var isInfoEmpty bool = len(infoList) == 0 + var ( + isFileTypeNill = false + isParametersEmpty = len(parameters) == 0 + isErrorsEmpty = len(errorList) == 0 + isInfoEmpty = len(infoList) == 0 + ) + if (*filetype == scanner.FileType{}){ data.FileType = nil isFileTypeNill = true } + if (isFileTypeNill && isParametersEmpty && isErrorsEmpty && isInfoEmpty){ resp.Matches = nil } From 3fd65225d8f14c6e48b5744b799e61e8a8785053 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 22 Mar 2023 15:02:40 +0000 Subject: [PATCH 16/19] add test case --- internal/json/json_test.go | 120 +++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 internal/json/json_test.go diff --git a/internal/json/json_test.go b/internal/json/json_test.go new file mode 100644 index 0000000..770016b --- /dev/null +++ b/internal/json/json_test.go @@ -0,0 +1,120 @@ +/* +========== +Cariddi +========== + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see http://www.gnu.org/licenses/. + + @Repository: https://github.com/edoardottt/cariddi + + @Author: edoardottt, https://www.edoardoottavianelli.it + + @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE + +*/ + +package utils + +import ( + "net/http" + "reflect" + "testing" + "net/url" + "github.com/gocolly/colly" + "github.com/edoardottt/cariddi/pkg/output" + "github.com/edoardottt/cariddi/pkg/scanner" +) + +func TestHasProtocol(t *testing.T) { + headers := http.Header{} + headers.Set("Content-Type", "application/pdf") + headers.Set("Content-Length", "128") + req := colly.Request{} + u, _ := url.Parse("http://test.com.pdf/id=5") + req.Method = "GET" + req.URL = u + tests := []struct { + name string + r *colly.Response + secrets []scanner.SecretMatched + filetype *scanner.FileType + parameters []scanner.Parameter + errors []scanner.ErrorMatched + infos []scanner.InfoMatched + want string + }{ + { + name: "ok1", + r: &colly.Response{ + StatusCode: 200, + Body: []byte("abcd"), + Ctx: nil, + Request: &req, + Headers: &headers, + }, + secrets: []scanner.SecretMatched{ + scanner.SecretMatched{ + Secret: scanner.Secret{ + Name: "mysecret", + Description: "My Secret", + Regex: "random.*regex", + FalsePositives: []string{}, + Poc: "POC", + }, + URL: "http://test.com/id=5", + Match: "it's a random day for my secret regex to be found", + }, + }, + parameters: []scanner.Parameter{ + scanner.Parameter{ + Parameter: "id", + Attacks: []string{}, + }, + }, + filetype: &scanner.FileType{ + Extension: "pdf", + Severity: 7, + }, + errors: []scanner.ErrorMatched{ + scanner.ErrorMatched{ + Error: scanner.Error{ + ErrorName: "MySQL error", + Regex: []string{"MySQL.*error"}, + }, + URL: "http://test.com/id=5", + Match: "it is a MySQL error happening", + }, + }, + infos: []scanner.InfoMatched{ + scanner.InfoMatched{ + Info: scanner.Info{ + Name: "info1", + Regex: []string{"my.*great"}, + }, + URL: "http://test.com/id=5", + Match: "its my pleasure to inform you on this great day", + }, + }, + want: `{"url":"http://test.com.pdf/id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, _ := output.GetJSONString(tt.r, tt.secrets, tt.parameters, tt.filetype, tt.errors, tt.infos); !reflect.DeepEqual(string(got), tt.want) { + t.Errorf("GetJSONString\n%v", string(got)) + t.Errorf("want\n%v", tt.want) + } + }) + } +} From 893b7a6a9e73ad27ddef38ce2e36fa722504d29f Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 22 Mar 2023 16:51:56 +0000 Subject: [PATCH 17/19] add more test cases --- internal/json/json_test.go | 120 ----------------------- pkg/output/json.go | 14 +-- pkg/output/json_test.go | 193 +++++++++++++++++++++++++++++++++++++ 3 files changed, 201 insertions(+), 126 deletions(-) delete mode 100644 internal/json/json_test.go create mode 100644 pkg/output/json_test.go diff --git a/internal/json/json_test.go b/internal/json/json_test.go deleted file mode 100644 index 770016b..0000000 --- a/internal/json/json_test.go +++ /dev/null @@ -1,120 +0,0 @@ -/* -========== -Cariddi -========== - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see http://www.gnu.org/licenses/. - - @Repository: https://github.com/edoardottt/cariddi - - @Author: edoardottt, https://www.edoardoottavianelli.it - - @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE - -*/ - -package utils - -import ( - "net/http" - "reflect" - "testing" - "net/url" - "github.com/gocolly/colly" - "github.com/edoardottt/cariddi/pkg/output" - "github.com/edoardottt/cariddi/pkg/scanner" -) - -func TestHasProtocol(t *testing.T) { - headers := http.Header{} - headers.Set("Content-Type", "application/pdf") - headers.Set("Content-Length", "128") - req := colly.Request{} - u, _ := url.Parse("http://test.com.pdf/id=5") - req.Method = "GET" - req.URL = u - tests := []struct { - name string - r *colly.Response - secrets []scanner.SecretMatched - filetype *scanner.FileType - parameters []scanner.Parameter - errors []scanner.ErrorMatched - infos []scanner.InfoMatched - want string - }{ - { - name: "ok1", - r: &colly.Response{ - StatusCode: 200, - Body: []byte("abcd"), - Ctx: nil, - Request: &req, - Headers: &headers, - }, - secrets: []scanner.SecretMatched{ - scanner.SecretMatched{ - Secret: scanner.Secret{ - Name: "mysecret", - Description: "My Secret", - Regex: "random.*regex", - FalsePositives: []string{}, - Poc: "POC", - }, - URL: "http://test.com/id=5", - Match: "it's a random day for my secret regex to be found", - }, - }, - parameters: []scanner.Parameter{ - scanner.Parameter{ - Parameter: "id", - Attacks: []string{}, - }, - }, - filetype: &scanner.FileType{ - Extension: "pdf", - Severity: 7, - }, - errors: []scanner.ErrorMatched{ - scanner.ErrorMatched{ - Error: scanner.Error{ - ErrorName: "MySQL error", - Regex: []string{"MySQL.*error"}, - }, - URL: "http://test.com/id=5", - Match: "it is a MySQL error happening", - }, - }, - infos: []scanner.InfoMatched{ - scanner.InfoMatched{ - Info: scanner.Info{ - Name: "info1", - Regex: []string{"my.*great"}, - }, - URL: "http://test.com/id=5", - Match: "its my pleasure to inform you on this great day", - }, - }, - want: `{"url":"http://test.com.pdf/id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got, _ := output.GetJSONString(tt.r, tt.secrets, tt.parameters, tt.filetype, tt.errors, tt.infos); !reflect.DeepEqual(string(got), tt.want) { - t.Errorf("GetJSONString\n%v", string(got)) - t.Errorf("want\n%v", tt.want) - } - }) - } -} diff --git a/pkg/output/json.go b/pkg/output/json.go index 01da29f..7b874b7 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -107,8 +107,8 @@ func GetJSONString( // Process infos for _, info := range infos { - secretMatch := MatcherResult{info.Info.Name, info.Match} - infoList = append(infoList, secretMatch) + infoMatch := MatcherResult{info.Info.Name, info.Match} + infoList = append(infoList, infoMatch) } // Process error list @@ -117,7 +117,8 @@ func GetJSONString( errorList = append(errorList, errorMatch) } - data := &MatcherResults{ + // Construct matcher results + matcherResults := &MatcherResults{ FileType: filetype, Parameters: parameters, Errors: errorList, @@ -134,7 +135,7 @@ func GetJSONString( Lines: lines, ContentType: contentType, ContentLength: contentLength, - Matches: data, + Matches: matcherResults, // Host: "", // TODO: this is available in Colly 2.x } @@ -144,14 +145,15 @@ func GetJSONString( isParametersEmpty = len(parameters) == 0 isErrorsEmpty = len(errorList) == 0 isInfoEmpty = len(infoList) == 0 + isSecretsEmpty = len(secretList) == 0 ) if (*filetype == scanner.FileType{}){ - data.FileType = nil + matcherResults.FileType = nil isFileTypeNill = true } - if (isFileTypeNill && isParametersEmpty && isErrorsEmpty && isInfoEmpty){ + if (isFileTypeNill && isParametersEmpty && isErrorsEmpty && isInfoEmpty && isSecretsEmpty){ resp.Matches = nil } diff --git a/pkg/output/json_test.go b/pkg/output/json_test.go new file mode 100644 index 0000000..b135531 --- /dev/null +++ b/pkg/output/json_test.go @@ -0,0 +1,193 @@ +/* +========== +Cariddi +========== + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see http://www.gnu.org/licenses/. + + @Repository: https://github.com/edoardottt/cariddi + + @Author: edoardottt, https://www.edoardoottavianelli.it + + @License: https://github.com/edoardottt/cariddi/blob/main/LICENSE + +*/ + +package output + +import ( + "net/http" + "reflect" + "testing" + "net/url" + "github.com/gocolly/colly" + "github.com/edoardottt/cariddi/pkg/scanner" +) + +func TestJSONOutput(t *testing.T) { + headers := http.Header{} + headers.Set("Content-Type", "application/pdf") + headers.Set("Content-Length", "128") + secrets := []scanner.SecretMatched{ + scanner.SecretMatched{ + Secret: scanner.Secret{ + Name: "mysecret", + Description: "My Secret", + Regex: "random.*regex", + FalsePositives: []string{}, + Poc: "POC", + }, + URL: "http://test.com?id=5", + Match: "it's a random day for my secret regex to be found", + }, + } + parameters := []scanner.Parameter{ + scanner.Parameter{ + Parameter: "id", + Attacks: []string{}, + }, + } + filetype := &scanner.FileType{ + Extension: "pdf", + Severity: 7, + } + errors := []scanner.ErrorMatched{ + scanner.ErrorMatched{ + Error: scanner.Error{ + ErrorName: "MySQL error", + Regex: []string{"MySQL.*error"}, + }, + URL: "http://test.com?id=5", + Match: "it is a MySQL error happening", + }, + } + infos := []scanner.InfoMatched{ + scanner.InfoMatched{ + Info: scanner.Info{ + Name: "info1", + Regex: []string{"my.*great"}, + }, + URL: "http://test.com?id=5", + Match: "its my pleasure to inform you on this great day", + }, + } + req := colly.Request{} + u, _ := url.Parse("http://test.com.pdf?id=5") + req.Method = "GET" + req.URL = u + resp := &colly.Response{ + StatusCode: 200, + Body: []byte("abcd"), + Ctx: nil, + Request: &req, + Headers: &headers, + } + headers_nocontent := http.Header{} + resp2 := &colly.Response{ + StatusCode: 200, + Body: []byte("abcd"), + Ctx: nil, + Request: &req, + Headers: &headers_nocontent, + } + tests := []struct { + name string + r *colly.Response + secrets []scanner.SecretMatched + filetype *scanner.FileType + parameters []scanner.Parameter + errors []scanner.ErrorMatched + infos []scanner.InfoMatched + want string + }{ + { + name: "test_all_findings", + r: resp, + secrets: secrets, + parameters: parameters, + filetype: filetype, + errors: errors, + infos: infos, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, + }, + { + name: "test_all_findings_nocontent", + r: resp2, + secrets: secrets, + parameters: parameters, + filetype: filetype, + errors: errors, + infos: infos, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, + }, + { + name: "test_no_findings", + r: resp, + secrets: []scanner.SecretMatched{}, + parameters: []scanner.Parameter{}, + filetype: &scanner.FileType{}, + errors: []scanner.ErrorMatched{}, + infos: []scanner.InfoMatched{}, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128}`, + }, + { + name: "test_only_secrets", + r: resp, + secrets: secrets, + parameters: []scanner.Parameter{}, + filetype: &scanner.FileType{}, + errors: []scanner.ErrorMatched{}, + infos: []scanner.InfoMatched{}, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, + }, + { + name: "test_only_params", + r: resp, + secrets: []scanner.SecretMatched{}, + parameters: parameters, + filetype: &scanner.FileType{}, + errors: []scanner.ErrorMatched{}, + infos: []scanner.InfoMatched{}, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"parameters":[{"name":"id","attacks":[]}]}}`, + }, + { + name: "test_only_errors", + r: resp, + secrets: []scanner.SecretMatched{}, + parameters: []scanner.Parameter{}, + filetype: &scanner.FileType{}, + errors: errors, + infos: []scanner.InfoMatched{}, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}]}}`, + }, + { + name: "test_only_infos", + r: resp, + secrets: []scanner.SecretMatched{}, + parameters: []scanner.Parameter{}, + filetype: &scanner.FileType{}, + errors: []scanner.ErrorMatched{}, + infos: infos, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}]}}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, _ := GetJSONString(tt.r, tt.secrets, tt.parameters, tt.filetype, tt.errors, tt.infos); !reflect.DeepEqual(string(got), tt.want) { + t.Errorf("GetJSONString\n%v", string(got)) + t.Errorf("want\n%v", tt.want) + } + }) + } +} From c1f3d67ee193c8211f052c7a4e96bd12f26db52b Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 22 Mar 2023 17:04:33 +0000 Subject: [PATCH 18/19] update tests --- pkg/output/json_test.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/pkg/output/json_test.go b/pkg/output/json_test.go index b135531..bdb4e1c 100644 --- a/pkg/output/json_test.go +++ b/pkg/output/json_test.go @@ -24,7 +24,7 @@ along with this program. If not, see http://www.gnu.org/licenses/. */ -package output +package output_test import ( "net/http" @@ -32,6 +32,7 @@ import ( "testing" "net/url" "github.com/gocolly/colly" + "github.com/edoardottt/cariddi/pkg/output" "github.com/edoardottt/cariddi/pkg/scanner" ) @@ -39,6 +40,7 @@ func TestJSONOutput(t *testing.T) { headers := http.Header{} headers.Set("Content-Type", "application/pdf") headers.Set("Content-Length", "128") + secrets := []scanner.SecretMatched{ scanner.SecretMatched{ Secret: scanner.Secret{ @@ -93,13 +95,13 @@ func TestJSONOutput(t *testing.T) { Request: &req, Headers: &headers, } - headers_nocontent := http.Header{} + headersNoContent := http.Header{} resp2 := &colly.Response{ StatusCode: 200, Body: []byte("abcd"), Ctx: nil, Request: &req, - Headers: &headers_nocontent, + Headers: &headersNoContent, } tests := []struct { name string @@ -119,7 +121,7 @@ func TestJSONOutput(t *testing.T) { filetype: filetype, errors: errors, infos: infos, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll }, { name: "test_all_findings_nocontent", @@ -129,7 +131,7 @@ func TestJSONOutput(t *testing.T) { filetype: filetype, errors: errors, infos: infos, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll }, { name: "test_no_findings", @@ -139,7 +141,7 @@ func TestJSONOutput(t *testing.T) { filetype: &scanner.FileType{}, errors: []scanner.ErrorMatched{}, infos: []scanner.InfoMatched{}, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128}`, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128}`, //nolint: all }, { name: "test_only_secrets", @@ -149,7 +151,7 @@ func TestJSONOutput(t *testing.T) { filetype: &scanner.FileType{}, errors: []scanner.ErrorMatched{}, infos: []scanner.InfoMatched{}, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll }, { name: "test_only_params", @@ -159,7 +161,7 @@ func TestJSONOutput(t *testing.T) { filetype: &scanner.FileType{}, errors: []scanner.ErrorMatched{}, infos: []scanner.InfoMatched{}, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"parameters":[{"name":"id","attacks":[]}]}}`, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"parameters":[{"name":"id","attacks":[]}]}}`, //nolint:lll }, { name: "test_only_errors", @@ -169,7 +171,7 @@ func TestJSONOutput(t *testing.T) { filetype: &scanner.FileType{}, errors: errors, infos: []scanner.InfoMatched{}, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}]}}`, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}]}}`, //nolint:lll }, { name: "test_only_infos", @@ -179,12 +181,13 @@ func TestJSONOutput(t *testing.T) { filetype: &scanner.FileType{}, errors: []scanner.ErrorMatched{}, infos: infos, - want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}]}}`, + want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}]}}`, //nolint:lll }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got, _ := GetJSONString(tt.r, tt.secrets, tt.parameters, tt.filetype, tt.errors, tt.infos); !reflect.DeepEqual(string(got), tt.want) { + if got, _ := output.GetJSONString(tt.r, tt.secrets, tt.parameters, tt.filetype, tt.errors, tt.infos); !reflect.DeepEqual(string(got), tt.want) { //nolint:lll t.Errorf("GetJSONString\n%v", string(got)) t.Errorf("want\n%v", tt.want) } From a78b9b7e12483e56d69054312c3db1d3e8864e33 Mon Sep 17 00:00:00 2001 From: Olivier Cervello Date: Wed, 22 Mar 2023 17:06:20 +0000 Subject: [PATCH 19/19] update banner --- pkg/output/beautify.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/output/beautify.go b/pkg/output/beautify.go index 29ed3f7..03e3653 100644 --- a/pkg/output/beautify.go +++ b/pkg/output/beautify.go @@ -40,7 +40,7 @@ func Beautify() { banner3 := " / __/ _` | '__| |/ _` |/ _` | |\n" banner4 := " | (_| (_| | | | | (_| | (_| | |\n" banner5 := " \\___\\__,_|_| |_|\\__,_|\\__,_|_| v1.3.0\n" - banner6 := "" + banner6 := "\n" banner7 := " > github.com/edoardottt/cariddi\n" banner8 := " > edoardoottavianelli.it\n" banner9 := "========================================\n"