Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/save responses to output directory provided in new -srd flag and also print if in json if -json flag enabled fixes #128, #129, #130 #161

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Feat/save responses to output directory provided in new -srd flag and also print if in json if -json flag enabled fixes #128, #129, #130 #161

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7001edc
feat: responses saving in output folder (for -sr / -srd) and printing…
kartikeysemwal Aug 15, 2024
6a6bfb4
refactoring the code changes
kartikeysemwal Aug 17, 2024
3ad8778
fix: resolve linter and test case errors in workflow
kartikeysemwal Aug 27, 2024
2bd3ae4
code review 1 changes
kartikeysemwal Sep 8, 2024
0bfbd90
Add validation for output folder and enhance unit tests for edge cases
kartikeysemwal Sep 18, 2024
34a15ff
linux OS build failure fixes
kartikeysemwal Sep 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions cmd/cariddi/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,12 @@ func main() {
StoreResp: flags.StoreResp,
}

config.OutputDir = output.CariddiOutputFolder
if flags.StoredRespDir != "" {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should perform some tests. What if config.StoreResp is not specified?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With the two tags -sr and -srd, there are four possible scenarios:

  1. Both -sr and -srd are not set:

    • No HTTP response will be saved.
    • OutputDir will be set to its default value, and any other data that needs to be saved will be stored in this directory.

  2. Only -sr is set:

    • The HTTP response, along with any other data, will be saved in the OutputDir, which will be the default directory.

  3. Only -srd is set:

    • Everything, including the HTTP response, will be saved to the OutputDir provided with the -srd flag.

  4. Both -sr and -srd are set:

    • Everything will be saved to the OutputDir provided with the -srd flag.

In the config struct, I have used the StoreResp boolean as the key indicator of whether to store the response. Therefore, if the user provides only the -srd flag, we will set StoreResp to true.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As far as I can understand point 3 and 4 are the same right?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, just in point 3 we will set the StoreResp value to true

config.OutputDir = flags.StoredRespDir
config.StoreResp = true
}

// Read the targets from standard input.
targets := input.ScanTargets()

Expand Down Expand Up @@ -118,18 +124,18 @@ func main() {
// Create output files if needed (txt / html).
config.Txt = ""
if flags.TXTout != "" {
config.Txt = fileUtils.CreateOutputFile(flags.TXTout, "results", "txt")
config.Txt = fileUtils.CreateOutputFile(flags.TXTout, "results", "txt", config.OutputDir)
}

var ResultHTML = ""
if flags.HTMLout != "" {
ResultHTML = fileUtils.CreateOutputFile(flags.HTMLout, "", "html")
ResultHTML = fileUtils.CreateOutputFile(flags.HTMLout, "", "html", config.OutputDir)
output.BannerHTML(ResultHTML)
output.HeaderHTML("Results", ResultHTML)
}

if config.StoreResp {
fileUtils.CreateIndexOutputFile("index.responses.txt")
fileUtils.CreateIndexOutputFile("index.responses.txt", config.OutputDir)
}

// Read headers if needed
Expand Down Expand Up @@ -167,13 +173,13 @@ func main() {
// IF TXT OUTPUT >
if flags.TXTout != "" {
output.TxtOutput(flags, finalResults, finalSecret, finalEndpoints,
finalExtensions, finalErrors, finalInfos)
finalExtensions, finalErrors, finalInfos, config.OutputDir)
}

// IF HTML OUTPUT >
if flags.HTMLout != "" {
output.HTMLOutput(flags, ResultHTML, finalResults, finalSecret,
finalEndpoints, finalExtensions, finalErrors, finalInfos)
finalEndpoints, finalExtensions, finalErrors, finalInfos, config.OutputDir)
}

// If needed print secrets.
Expand Down
27 changes: 14 additions & 13 deletions internal/file/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,9 @@ const (

// CreateOutputFolder creates the output folder
// If it fails exits with an error message.
func CreateOutputFolder() {
func CreateOutputFolder(outputDir string) {
// Create a folder/directory at a full qualified path
err := os.Mkdir("output-cariddi", Permission0755)
err := os.MkdirAll(outputDir, Permission0755)
if err != nil {
fmt.Println("Can't create output folder.")
os.Exit(1)
Expand All @@ -56,9 +56,9 @@ func CreateOutputFolder() {
// CreateHostOutputFolder creates the host output folder
// for the HTTP responses.
// If it fails exits with an error message.
func CreateHostOutputFolder(host string) {
func CreateHostOutputFolder(host string, outputDir string) {
// Create a folder/directory at a full qualified path
err := os.MkdirAll(filepath.Join("output-cariddi", host), Permission0755)
err := os.MkdirAll(filepath.Join(outputDir, host), Permission0755)
if err != nil {
fmt.Println("Can't create host output folder.")
os.Exit(1)
Expand All @@ -71,21 +71,21 @@ func CreateHostOutputFolder(host string) {
// already exists, if yes asks the user if cariddi has to overwrite it;
// if no cariddi creates it.
// Whenever an instruction fails, it exits with an error message.
func CreateOutputFile(target string, subcommand string, format string) string {
func CreateOutputFile(target string, subcommand string, format string, outputDir string) string {
target = ReplaceBadCharacterOutput(target)

var filename string
if subcommand != "" {
filename = filepath.Join("output-cariddi", target+"."+subcommand+"."+format)
filename = filepath.Join(outputDir, target+"."+subcommand+"."+format)
} else {
filename = filepath.Join("output-cariddi", target+"."+format)
filename = filepath.Join(outputDir, target+"."+format)
}

_, err := os.Stat(filename)

if os.IsNotExist(err) {
if _, err := os.Stat("output-cariddi/"); os.IsNotExist(err) {
CreateOutputFolder()
if _, err := os.Stat(fmt.Sprintf("%s/", outputDir)); os.IsNotExist(err) {
CreateOutputFolder(outputDir)
}
// If the file doesn't exist, create it.
f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, Permission0644)
Expand Down Expand Up @@ -119,15 +119,15 @@ func CreateOutputFile(target string, subcommand string, format string) string {
// It creates the output folder if needed, then checks if the index output file
// already exists, if no cariddi creates it.
// Whenever an instruction fails, it exits with an error message.
func CreateIndexOutputFile(filename string) {
func CreateIndexOutputFile(filename string, outputDir string) {
_, err := os.Stat(filename)

if os.IsNotExist(err) {
if _, err := os.Stat("output-cariddi/"); os.IsNotExist(err) {
CreateOutputFolder()
if _, err := os.Stat(fmt.Sprintf("%s/", outputDir)); os.IsNotExist(err) {
CreateOutputFolder(outputDir)
}
// If the file doesn't exist, create it.
filename = filepath.Join("output-cariddi", filename)
filename = filepath.Join(outputDir, filename)

f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, Permission0644)
if err != nil {
Expand Down Expand Up @@ -161,6 +161,7 @@ func ReadFile(inputFile string) []string {
for scanner.Scan() {
text = append(text, scanner.Text())
}

file.Close()

return text
Expand Down
8 changes: 6 additions & 2 deletions pkg/crawler/colly.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,12 @@ func New(scan *Scan) *Results {
fmt.Println(r.Request.URL)
}

var outputPath string

if scan.StoreResp {
err := output.StoreHTTPResponse(r)
var err error
outputPath, err = output.StoreHTTPResponse(r, scan.OutputDir)

if err != nil {
log.Println(err)
}
Expand Down Expand Up @@ -193,7 +197,7 @@ func New(scan *Scan) *Results {

if scan.JSON {
jsonOutput, err := output.GetJSONString(
r, secrets, parameters, filetype, errors, infos,
r, secrets, parameters, filetype, errors, infos, outputPath,
)

if err == nil {
Expand Down
1 change: 1 addition & 0 deletions pkg/crawler/options.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ type Scan struct {
FileType int
Headers map[string]string
StoreResp bool
OutputDir string

// Settings
Concurrency int
Expand Down
6 changes: 5 additions & 1 deletion pkg/crawler/useragents.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ import (
"time"
)

const (
maxRandomValue = 100
)

// genOsString generates a random OS string for a User Agent.
func genOsString() string {
source := rand.NewSource(time.Now().UnixNano())
Expand Down Expand Up @@ -151,7 +155,7 @@ func GenerateRandomUserAgent() string {
source := rand.NewSource(time.Now().UnixNano())
rng := rand.New(source)

decision := rng.Intn(100)
decision := rng.Intn(maxRandomValue)

var ua string
if decision%2 == 0 {
Expand Down
5 changes: 5 additions & 0 deletions pkg/input/flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,8 @@ type Input struct {
UserAgent string
// StoreResp stores HTTP responses.
StoreResp bool
// StoredRespDir stores HTTP responses to the directory provided.
StoredRespDir string
}

// ScanFlag defines all the options taken
Expand Down Expand Up @@ -142,6 +144,8 @@ func ScanFlag() Input {

storeRespPtr := flag.Bool("sr", false, "Store HTTP responses.")

storedRespDirPtr := flag.String("srd", "", "Stores HTTP responses to the directory provided.")

flag.Parse()

result := Input{
Expand Down Expand Up @@ -173,6 +177,7 @@ func ScanFlag() Input {
*debugPtr,
*userAgentPtr,
*storeRespPtr,
*storedRespDirPtr,
}

return result
Expand Down
3 changes: 3 additions & 0 deletions pkg/output/jsonl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ type JSONData struct {
ContentType string `json:"content_type,omitempty"`
ContentLength int `json:"content_length,omitempty"`
Matches *MatcherResults `json:"matches,omitempty"`
OutputPath string `json:"output_path,omitempty"`
// Host string `json:"host"` # TODO: Available in Colly 2.x
}

Expand All @@ -67,6 +68,7 @@ func GetJSONString(
filetype *scanner.FileType,
errors []scanner.ErrorMatched,
infos []scanner.InfoMatched,
outputPath string,
) ([]byte, error) {
// Parse response headers
headers := r.Headers
Expand Down Expand Up @@ -136,6 +138,7 @@ func GetJSONString(
ContentType: contentType,
ContentLength: contentLength,
Matches: matcherResults,
OutputPath: outputPath,
// Host: "", // TODO
}

Expand Down
33 changes: 26 additions & 7 deletions pkg/output/jsonl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ func TestJSONOutput(t *testing.T) {
errors []scanner.ErrorMatched
infos []scanner.InfoMatched
want string
outputPath string
}{
{
name: "test_all_findings",
Expand All @@ -122,7 +123,8 @@ func TestJSONOutput(t *testing.T) {
filetype: filetype,
errors: errors,
infos: infos,
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]},"output_path":"C:\\testDir1\\testDir2"}`, //nolint:lll
outputPath: "C:\\testDir1\\testDir2",
},
{
name: "test_all_findings_nocontent",
Expand All @@ -132,7 +134,8 @@ func TestJSONOutput(t *testing.T) {
filetype: filetype,
errors: errors,
infos: infos,
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"matches":{"filetype":{"extension":"pdf","severity":7},"parameters":[{"name":"id","attacks":[]}],"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}],"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}],"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]},"output_path":"C:\\testDir1\\testDir2"}`, //nolint:lll
outputPath: "C:\\testDir1\\testDir2",
},
{
name: "test_no_findings",
Expand All @@ -142,7 +145,8 @@ func TestJSONOutput(t *testing.T) {
filetype: &scanner.FileType{},
errors: []scanner.ErrorMatched{},
infos: []scanner.InfoMatched{},
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128}`, //nolint: all
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"output_path":"C:\\testDir1\\testDir2"}`, //nolint: all
outputPath: "C:\\testDir1\\testDir2",
},
{
name: "test_only_secrets",
Expand All @@ -152,7 +156,8 @@ func TestJSONOutput(t *testing.T) {
filetype: &scanner.FileType{},
errors: []scanner.ErrorMatched{},
infos: []scanner.InfoMatched{},
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]}}`, //nolint:lll
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"secrets":[{"name":"mysecret","match":"it's a random day for my secret regex to be found"}]},"output_path":"C:\\testDir1\\testDir2"}`, //nolint:lll
outputPath: "C:\\testDir1\\testDir2",
},
{
name: "test_only_params",
Expand All @@ -162,7 +167,8 @@ func TestJSONOutput(t *testing.T) {
filetype: &scanner.FileType{},
errors: []scanner.ErrorMatched{},
infos: []scanner.InfoMatched{},
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"parameters":[{"name":"id","attacks":[]}]}}`, //nolint:lll
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"parameters":[{"name":"id","attacks":[]}]},"output_path":"C:\\testDir1\\testDir2"}`, //nolint:lll
outputPath: "C:\\testDir1\\testDir2",
},
{
name: "test_only_errors",
Expand All @@ -172,7 +178,8 @@ func TestJSONOutput(t *testing.T) {
filetype: &scanner.FileType{},
errors: errors,
infos: []scanner.InfoMatched{},
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}]}}`, //nolint:lll
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"errors":[{"name":"MySQL error","match":"it is a MySQL error happening"}]},"output_path":"C:\\testDir1\\testDir2"}`, //nolint:lll
outputPath: "C:\\testDir1\\testDir2",
},
{
name: "test_only_infos",
Expand All @@ -182,13 +189,25 @@ func TestJSONOutput(t *testing.T) {
filetype: &scanner.FileType{},
errors: []scanner.ErrorMatched{},
infos: infos,
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}]},"output_path":"C:\\testDir1\\testDir2"}`, //nolint:lll
outputPath: "C:\\testDir1\\testDir2",
},
{
name: "test_no_outputPath",
r: resp,
secrets: []scanner.SecretMatched{},
parameters: []scanner.Parameter{},
filetype: &scanner.FileType{},
errors: []scanner.ErrorMatched{},
infos: infos,
want: `{"url":"http://test.com.pdf?id=5","method":"GET","status_code":200,"words":1,"lines":1,"content_type":"application/pdf","content_length":128,"matches":{"infos":[{"name":"info1","match":"its my pleasure to inform you on this great day"}]}}`, //nolint:lll
outputPath: "",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got, _ := output.GetJSONString(tt.r, tt.secrets, tt.parameters, tt.filetype, tt.errors, tt.infos); !reflect.DeepEqual(string(got), tt.want) { //nolint:lll
if got, _ := output.GetJSONString(tt.r, tt.secrets, tt.parameters, tt.filetype, tt.errors, tt.infos, tt.outputPath); !reflect.DeepEqual(string(got), tt.want) { //nolint:lll
t.Errorf("GetJSONString\n%v", string(got))
t.Errorf("want\n%v", tt.want)
}
Expand Down
Loading
Loading