diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 37fd22bd..9a62837f 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -56,10 +56,37 @@ jobs: - name: run extract-scriptblocks run: cd takajo && ./takajo extract-scriptblocks -t timeline.jsonl + - name: run extract-scriptblocks(-o) + run: cd takajo && ./takajo extract-scriptblocks -t timeline.jsonl -o scriptblocks + + - name: run extract-scriptblocks(-l) + run: cd takajo && ./takajo extract-scriptblocks -t timeline.jsonl -o scriptblocks -l informational + + - name: run list-domain(-o) + run: cd takajo && ./takajo list-domains -t timeline.jsonl -o domains.txt + + - name: run list-domain(-s) + run: cd takajo && ./takajo list-domains -t timeline.jsonl -o domains.txt -s + + - name: run list-hashes(-o) + run: cd takajo && ./takajo list-hashes -t timeline.jsonl -o case-1 + + - name: run list-ip-addresses(-o) + run: cd takajo && ./takajo list-ip-addresses -t timeline.jsonl -o ipAddresses.txt + + - name: run list-ip-addresses(-i) + run: cd takajo && ./takajo list-ip-addresses -t timeline.jsonl -o ipAddresses.txt -i=false + - name: run list-undetected-evtx + run: cd takajo && ./takajo list-undetected-evtx -t timeline.csv -e ../hayabusa-sample-evtx + + - name: run list-undetected-evtx(-o) run: cd takajo && ./takajo list-undetected-evtx -t timeline.csv -e ../hayabusa-sample-evtx -o undetected-evtx.txt - name: run list-unused-rules + run: cd takajo && ./takajo list-unused-rules -t timeline.csv -r ../hayabusa/tmp/rules + + - name: run list-unused-rules(-o) run: cd takajo && ./takajo list-unused-rules -t timeline.csv -r ../hayabusa/tmp/rules -o unused-rules.txt - name: run split-csv-timeline @@ -71,21 +98,48 @@ jobs: - name: run stack-cmdlines run: cd takajo && ./takajo stack-cmdlines -t timeline.jsonl + - name: run stack-cmdlines(-o) + run: cd takajo && ./takajo stack-cmdlines -t timeline.jsonl -o cmdlines.csv + + - name: run stack-cmdlines(-l) + run: cd takajo && ./takajo stack-cmdlines -t timeline.jsonl -o cmdlines.csv -l informational + - name: run stack-dns run: cd takajo && ./takajo stack-dns -t timeline.jsonl + - name: run stack-dns(-o) + run: cd takajo && ./takajo stack-dns -t timeline.jsonl -o dns.csv + - name: run stack-logons run: cd takajo && ./takajo stack-logons -t timeline.jsonl + - name: run stack-logons(-l) + run: cd takajo && ./takajo stack-logons -t timeline.jsonl -l + + - name: run stack-logons(-o) + run: cd takajo && ./takajo stack-logons -t timeline.jsonl -o logon.csv + - name: run stack-processes run: cd takajo && ./takajo stack-processes -t timeline.jsonl + - name: run stack-processes(-o) + run: cd takajo && ./takajo stack-processes -t timeline.jsonl -o processes.csv + + - name: run stack-processes(-l) + run: cd takajo && ./takajo stack-processes -t timeline.jsonl -o processes.csv -l informational + - name: run stack-services run: cd takajo && ./takajo stack-services -t timeline.jsonl + - name: run stack-services(-o) + run: cd takajo && ./takajo stack-services -t timeline.jsonl -o services.csv + - name: run stack-tasks run: cd takajo && ./takajo stack-tasks -t timeline.jsonl + - name: run stack-tasks(-o) + run: cd takajo && ./takajo stack-tasks -t timeline.jsonl -o tasks.csv + - name: run sysmon-process-tree run: cd takajo && ./takajo sysmon-process-tree -t timeline.jsonl -p "365ABB72-3D4A-5CEB-0000-0010FA93FD00" -o process-tree.txt diff --git a/src/takajo.nim b/src/takajo.nim index cef69dfa..43651b3d 100644 --- a/src/takajo.nim +++ b/src/takajo.nim @@ -11,7 +11,6 @@ import strformat import strutils import tables import terminal -import termstyle import times import threadpool import uri @@ -21,6 +20,8 @@ import std/xmlparser import std/xmltree import suru import takajopkg/general +import takajopkg/stackUtil +import takajopkg/takajoTerminal include takajopkg/extractScriptblocks include takajopkg/listDomains include takajopkg/listIpAddresses @@ -56,12 +57,12 @@ when isMainModule: const example_list_unused_rules = " list-unused-rules -t ../hayabusa/timeline.csv -r ../hayabusa/rules\p" const example_split_csv_timeline = " split-csv-timeline -t ../hayabusa/timeline.csv [--makeMultiline] -o case-1-csv\p" const example_split_json_timeline = " split-json-timeline -t ../hayabusa/timeline.jsonl -o case-1-json\p" - const example_stack_cmdlines = " stack-cmdlines -t ../hayabusa/timeline.jsonl -o cmdlines.csv\p" - const example_stack_dns = " stack-dns -t ../hayabusa/timeline.jsonl -o dns.csv\p" + const example_stack_cmdlines = " stack-cmdlines -t ../hayabusa/timeline.jsonl [--level low] -o cmdlines.csv\p" + const example_stack_dns = " stack-dns -t ../hayabusa/timeline.jsonl [--level infomational] -o dns.csv\p" const example_stack_logons = " stack-logons -t ../hayabusa/timeline.jsonl -o logons.csv\p" - const example_stack_services = " stack-services -t ../hayabusa/timeline.jsonl -o services.csv\p" - const example_stack_tasks = " stack-tasks -t ../hayabusa/timeline.jsonl -o tasks.csv\p" - const example_stack_processes = " stack-processes -t ../hayabusa/timeline.jsonl -o processes.csv\p" + const example_stack_services = " stack-services -t ../hayabusa/timeline.jsonl [--level infomational] -o services.csv\p" + const example_stack_tasks = " stack-tasks -t ../hayabusa/timeline.jsonl [--level infomational] -o tasks.csv\p" + const example_stack_processes = " stack-processes -t ../hayabusa/timeline.jsonl [--level low] -o processes.csv\p" const example_list_hashes = " list-hashes -t ../hayabusa/case-1.jsonl -o case-1\p" const example_sysmon_process_tree = " sysmon-process-tree -t ../hayabusa/timeline.jsonl -p [-o process-tree.txt]\p" const example_timeline_logon = " timeline-logon -t ../hayabusa/timeline.jsonl -o logon-timeline.csv\p" @@ -91,7 +92,7 @@ when isMainModule: extractScriptblocks, cmdName = "extract-scriptblocks", doc = "extract and reassemble PowerShell EID 4104 script block logs", help = { - "level": "specify the minimum alert level", + "level": "specify the minimum alert level (default: low)", "output": "output directory (default: scriptblock-logs)", "quiet": "do not display the launch banner", "timeline": "Hayabusa JSONL timeline (profile: any)", @@ -183,6 +184,7 @@ when isMainModule: stackCmdlines, cmdName = "stack-cmdlines", doc = "stack executed command lines", help = { + "level": "specify the minimum alert level (default: low)", "ignoreSysmon": "exclude Sysmon 1 events", "ignoreSecurity": "exclude Security 4688 events", "output": "save results to a CSV file", @@ -198,6 +200,7 @@ when isMainModule: stackDNS, cmdName = "stack-dns", doc = "stack DNS queries and responses", help = { + "level": "specify the minimum alert level (default: informational)", "output": "save results to a CSV file", "quiet": "do not display the launch banner", "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", @@ -217,6 +220,7 @@ when isMainModule: stackProcesses, cmdName = "stack-processes", doc = "stack executed processes", help = { + "level": "specify the minimum alert level (default: low)", "ignoreSysmon": "exclude Sysmon 1 events", "ignoreSecurity": "exclude Security 4688 events", "output": "save results to a CSV file", @@ -232,12 +236,13 @@ when isMainModule: stackServices, cmdName = "stack-services", doc = "stack service names and paths", help = { + "level": "specify the minimum alert level (default: informational)", "output": "save results to a CSV file", "quiet": "do not display the launch banner", "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", }, short = { - "ignoreSysmon": 'y', + "ignoreSystem": 'y', "ignoreSecurity": 'e' } ], @@ -245,6 +250,7 @@ when isMainModule: stackTasks, cmdName = "stack-tasks", doc = "stack new scheduled tasks", help = { + "level": "specify the minimum alert level (default: informational)", "output": "save results to a CSV file", "quiet": "do not display the launch banner", "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index 487e54c0..003b732e 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -164,16 +164,8 @@ proc extractScriptblocks(level: string = "low", output: string = "scriptblock-lo else: outputFile.write(escapeCsvField(val) & "\p") for v in summaryRecords.values: - if v[5] == "crit": - table.add red v[0], red v[1], red v[2], red v[3], red v[4], red v[5], red v[6] - elif v[5] == "high": - table.add yellow v[0], yellow v[1], yellow v[2], yellow v[3], yellow v[4], yellow v[5], yellow v[6] - elif v[5] == "med": - table.add cyan v[0], cyan v[1], cyan v[2], cyan v[3], cyan v[4], cyan v[5], cyan v[6] - elif v[5] == "low": - table.add green v[0], green v[1], green v[2], green v[3], green v[4], green v[5], green v[6] - else: - table.add v + let color = levelColor(v[5]) + table.add color v[0], color v[1], color v[2], color v[3], color v[4], color v[5], color v[6] for i, cell in v: if i < 6: outputFile.write(escapeCsvField(cell) & ",") diff --git a/src/takajopkg/general.nim b/src/takajopkg/general.nim index c56b6d87..ff9e476a 100644 --- a/src/takajopkg/general.nim +++ b/src/takajopkg/general.nim @@ -1,7 +1,4 @@ import json -import nancy -import terminal -import termstyle import re import std/os import std/parsecsv @@ -9,20 +6,11 @@ import std/sequtils import std/strformat import std/strutils import std/tables +import terminal import times +import takajoTerminal from std/streams import newFileStream -proc outputLogo*(): string = - let logo = """ -╔════╦═══╦╗╔═╦═══╗ ╔╦═══╗ -║╔╗╔╗║╔═╗║║║╔╣╔═╗║ ║║╔═╗║ -╚╝║║╚╣║ ║║╚╝╝║║ ║║ ║║║ ║║ - ║║ ║╚═╝║╔╗╖║╚═╝╠╗║║║ ║║ - ╔╝╚╗║╔═╗║║║╚╣╔═╗║╚╝║╚═╝║ - ╚══╝╚╝ ╚╩╝╚═╩╝ ╚╩══╩═══╝ - by Yamato Security -""" - return logo proc getJsonValue*(jsonResponse: JsonNode, keys: seq[string], default: string = "Unknown"): string = var value = jsonResponse @@ -331,7 +319,6 @@ proc extractDomain*(domain: string): string = proc isLocalIP*(ip: string): bool = return ip == "127.0.0.1" or ip == "-" or ip == "::1" - proc isJsonConvertible*(timeline: string) : bool = var file: File @@ -352,42 +339,47 @@ proc isJsonConvertible*(timeline: string) : bool = echo "Failed to open '" & timeline & "'. Please specify a valid file path.\p" return false -proc colorWrite*(color: ForegroundColor, ansiEscape: string, txt: string) = - # Remove ANSI escape sequences and use stdout.styledWrite instead - let replacedTxt = txt.replace(ansiEscape,"").replace(termClear,"") - if "│" in replacedTxt: - stdout.styledWrite(color, replacedTxt.replace("│ ","")) - stdout.write "│ " - else: - stdout.styledWrite(color, replacedTxt) - -proc stdoutStyledWrite*(txt: string) = - if txt.startsWith(termRed): - colorWrite(fgRed, termRed,txt) - elif txt.startsWith(termGreen): - colorWrite(fgGreen, termGreen, txt) - elif txt.startsWith(termYellow): - colorWrite(fgYellow, termYellow, txt) - elif txt.startsWith(termCyan): - colorWrite(fgCyan, termCyan, txt) - else: - stdout.write txt.replace(termClear,"") - -proc echoTableSepsWithStyled*(table: TerminalTable, maxSize = terminalWidth(), seps = defaultSeps) = - # This function equivalent to echoTableSeps without using ANSI escape to avoid the following issue. - # https://github.com/PMunch/nancy/issues/4 - # https://github.com/PMunch/nancy/blob/9918716a563f64d740df6a02db42662781e94fc8/src/nancy.nim#L195C6-L195C19 - let sizes = table.getColumnSizes(maxSize - 4, padding = 3) - printSeparator(top) - for k, entry in table.entries(sizes): - for _, row in entry(): - stdout.write seps.vertical & " " - for i, cell in row(): - stdoutStyledWrite cell & (if i != sizes.high: " " & seps.vertical & " " else: "") - stdout.write " " & seps.vertical & "\n" - if k != table.rows - 1: - printSeparator(center) - printSeparator(bottom) +proc outputElasptedTime*(startTime: float) = + let endTime = epochTime() + let elapsedTime2 = int(endTime - startTime) + let hours = elapsedTime2 div 3600 + let minutes = (elapsedTime2 mod 3600) div 60 + let seconds = elapsedTime2 mod 60 + echo "" + echo "Elapsed time: ", $hours & " hours, " & $minutes & " minutes, " & $seconds & " seconds" + echo "" + +proc checkArgs*(quiet: bool = false, timeline: string, level:string) = + if not quiet: + styledEcho(fgGreen, outputLogo()) + + if not os.fileExists(timeline): + echo "The file '" & timeline & "' does not exist. Please specify a valid file path." + quit(1) + + if not isJsonConvertible(timeline): + quit(1) + + if level != "critical" and level != "high" and level != "medium" and level != "low" and level != "informational": + echo "You must specify a minimum level of critical, high, medium, low or informational. (default: low)" + echo "" + quit(1) + + +proc countJsonlAndStartMsg*(cmdName:string, msg:string, timeline:string):int = + echo "Started the Stack " & cmdName & " command" + echo "" + echo "This command will stack " & msg & "." + echo "" + + echo "Counting total lines. Please wait." + echo "" + let totalLines = countLinesInTimeline(timeline) + echo "Total lines: ", totalLines + echo "" + echo "Scanning the Hayabusa timeline. Please wait." + echo "" + return totalLines type VirusTotalResult* = object resTable*: TableRef[string, string] diff --git a/src/takajopkg/stackCmdlines.nim b/src/takajopkg/stackCmdlines.nim index 7e77479a..e72a6c8e 100644 --- a/src/takajopkg/stackCmdlines.nim +++ b/src/takajopkg/stackCmdlines.nim @@ -1,36 +1,13 @@ -proc stackCmdlines(ignoreSysmon: bool = false, ignoreSecurity: bool = false, output: string = "", quiet: bool = false, timeline: string) = +proc stackCmdlines(level: string = "low", ignoreSysmon: bool = false, ignoreSecurity: bool = false, output: string = "", quiet: bool = false, timeline: string) = let startTime = epochTime() - if not quiet: - styledEcho(fgGreen, outputLogo()) - - if not os.fileExists(timeline): - echo "The file '" & timeline & "' does not exist. Please specify a valid file path." - quit(1) - - if not isJsonConvertible(timeline): - quit(1) - - echo "Started the Stack Cmdlines command" - echo "" - echo "This command will stack executed command lines." - echo "" - - echo "Counting total lines. Please wait." - echo "" - let totalLines = countLinesInTimeline(timeline) - echo "Total lines: ", totalLines - echo "" - echo "Scanning the Hayabusa timeline. Please wait." - echo "" - + checkArgs(quiet, timeline, level) + let totalLines = countJsonlAndStartMsg("Cmdlines", "executed command lines from Sysmon 1 and Security 4688 events", timeline) var bar: SuruBar = initSuruBar() - stackCmdlines = initCountTable[string]() - uniqueCmd = 0 + stack = initTable[string, StackRecord]() bar[0].total = totalLines bar.setup() - # Loop through JSON lines for line in lines(timeline): inc bar @@ -39,43 +16,9 @@ proc stackCmdlines(ignoreSysmon: bool = false, ignoreSecurity: bool = false, out let eventId = jsonLine["EventID"].getInt(0) let channel = jsonLine["Channel"].getStr("N/A") if (eventId == 1 and not ignoreSysmon and channel == "Sysmon") or - (eventId == 4688 and not ignoreSecurity and channel == "Security"): - let command = jsonLine["Details"]["Cmdline"].getStr("N/A") - stackCmdlines.inc(command) - + (eventId == 4688 and not ignoreSecurity and channel == "Sec"): + let stackKey = jsonLine["Details"]["Cmdline"].getStr("N/A") + stackResult(stackKey, stack, level, jsonLine) bar.finish() - echo "" - - stackCmdlines.sort() - - # Print results to screen - var outputFileSize = 0 - if output == "": - for commandline, count in stackCmdlines: - inc uniqueCmd - var commaDelimitedStr = $count & "," & commandline - commaDelimitedStr = replace(commaDelimitedStr, ",", " | ") - echo commaDelimitedStr - # Save to CSV file - else: - let outputFile = open(output, fmWrite) - writeLine(outputFile, "Count,Cmdlines") - - # Write results - for commandline, count in stackCmdlines: - inc uniqueCmd - writeLine(outputFile, $count & "," & commandline) - outputFileSize = getFileSize(outputFile) - close(outputFile) - - echo "" - echo "Saved file: " & output & " (" & formatFileSize(outputFileSize) & ")" - - let endTime = epochTime() - let elapsedTime2 = int(endTime - startTime) - let hours = elapsedTime2 div 3600 - let minutes = (elapsedTime2 mod 3600) div 60 - let seconds = elapsedTime2 mod 60 - echo "" - echo "Elapsed time: ", $hours & " hours, " & $minutes & " minutes, " & $seconds & " seconds" - echo "" \ No newline at end of file + outputResult(output, "Cmdline", stack) + outputElasptedTime(startTime) \ No newline at end of file diff --git a/src/takajopkg/stackDNS.nim b/src/takajopkg/stackDNS.nim index f7e5765f..8c1b0152 100644 --- a/src/takajopkg/stackDNS.nim +++ b/src/takajopkg/stackDNS.nim @@ -1,35 +1,13 @@ -proc stackDNS(output: string = "", quiet: bool = false, timeline: string) = +proc stackDNS(level: string = "informational", output: string = "", quiet: bool = false, timeline: string) = let startTime = epochTime() - if not quiet: - styledEcho(fgGreen, outputLogo()) - - if not os.fileExists(timeline): - echo "The file '" & timeline & "' does not exist. Please specify a valid file path." - quit(1) - - if not isJsonConvertible(timeline): - quit(1) - - echo "Started the Stack DNS command" - echo "" - echo "This command will stack DNS queries and responses." - echo "" - - echo "Counting total lines. Please wait." - echo "" - let totalLines = countLinesInTimeline(timeline) - echo "Total lines: ", totalLines - echo "" - echo "Scanning the Hayabusa timeline. Please wait." - echo "" - + checkArgs(quiet, timeline, level) + let totalLines = countJsonlAndStartMsg("DNS", "DNS queries and responses from Sysmon 22 events", timeline) var bar: SuruBar = initSuruBar() - stackDNS = initCountTable[string]() + stack = initTable[string, StackRecord]() bar[0].total = totalLines bar.setup() - # Loop through JSON lines for line in lines(timeline): inc bar @@ -41,40 +19,8 @@ proc stackDNS(output: string = "", quiet: bool = false, timeline: string) = let prog = jsonLine["Details"]["Proc"].getStr("N/A") let query = jsonLine["Details"]["Query"].getStr("N/A") let res = jsonLine["Details"]["Result"].getStr("N/A") - let result = prog & " -> " & query & " -> " & res - stackDNS.inc(result) - + let stackKey = prog & " -> " & query & " -> " & res + stackResult(stackKey, stack, level, jsonLine, @[prog, query, res]) bar.finish() - echo "" - - stackDNS.sort() - - # Print results to screen - var outputFileSize = 0 - if output == "": - for res, count in stackDNS: - var commaDelimitedStr = $count & "," & res - commaDelimitedStr = replace(commaDelimitedStr, ",", " | ") - echo commaDelimitedStr - # Save to CSV file - else: - let outputFile = open(output, fmWrite) - writeLine(outputFile, "Count,DNS query and response") - - # Write results - for res, count in stackDNS: - writeLine(outputFile, $count & "," & res) - outputFileSize = getFileSize(outputFile) - close(outputFile) - - echo "" - echo "Saved file: " & output & " (" & formatFileSize(outputFileSize) & ")" - - let endTime = epochTime() - let elapsedTime2 = int(endTime - startTime) - let hours = elapsedTime2 div 3600 - let minutes = (elapsedTime2 mod 3600) div 60 - let seconds = elapsedTime2 mod 60 - echo "" - echo "Elapsed time: ", $hours & " hours, " & $minutes & " minutes, " & $seconds & " seconds" - echo "" \ No newline at end of file + outputResult(output, "DNS", stack, @["Image", "Query", "Result"]) + outputElasptedTime(startTime) \ No newline at end of file diff --git a/src/takajopkg/stackLogons.nim b/src/takajopkg/stackLogons.nim index e809540e..82146ae0 100644 --- a/src/takajopkg/stackLogons.nim +++ b/src/takajopkg/stackLogons.nim @@ -4,19 +4,11 @@ proc stackLogons(localSrcIpAddresses = false, output: string = "", quiet: bool = false, timeline: string) = let startTime = epochTime() - if not quiet: - styledEcho(fgGreen, outputLogo()) - - if not os.fileExists(timeline): - echo "The file '" & timeline & "' does not exist. Please specify a valid file path." - quit(1) - - if not isJsonConvertible(timeline): - quit(1) + checkArgs(quiet, timeline, "informational") echo "Started the Stack Logons command" echo "" - echo "This command will stack logons based on target user, target computer, source IP address and source computer." + echo "This command will stack logons based on target user, target computer, source IP address and source computer from Security 4624 events." echo "Local source IP addresses are not included by default but can be enabled with -l, --localSrcIpAddresses." echo "" @@ -105,11 +97,4 @@ proc stackLogons(localSrcIpAddresses = false, output: string = "", quiet: bool = echo "Saved file: " & output & " (" & formatFileSize(outputFileSize) & ")" echo "" - let endTime = epochTime() - let elapsedTime2 = int(endTime - startTime) - let hours = elapsedTime2 div 3600 - let minutes = (elapsedTime2 mod 3600) div 60 - let seconds = elapsedTime2 mod 60 - echo "" - echo "Elapsed time: ", $hours & " hours, " & $minutes & " minutes, " & $seconds & " seconds" - echo "" \ No newline at end of file + outputElasptedTime(startTime) \ No newline at end of file diff --git a/src/takajopkg/stackProcesses.nim b/src/takajopkg/stackProcesses.nim index f73892ed..1fe4e933 100644 --- a/src/takajopkg/stackProcesses.nim +++ b/src/takajopkg/stackProcesses.nim @@ -1,36 +1,12 @@ -proc stackProcesses(ignoreSysmon: bool = false, ignoreSecurity: bool = false, output: string = "", quiet: bool = false, timeline: string) = +proc stackProcesses(level: string = "low", ignoreSysmon: bool = false, ignoreSecurity: bool = false, output: string = "", quiet: bool = false, timeline: string) = let startTime = epochTime() - if not quiet: - styledEcho(fgGreen, outputLogo()) - - if not os.fileExists(timeline): - echo "The file '" & timeline & "' does not exist. Please specify a valid file path." - quit(1) - - if not isJsonConvertible(timeline): - quit(1) - - echo "Started the Stack Processes command" - echo "" - echo "This command will stack executed processes." - echo "" - - echo "Counting total lines. Please wait." - echo "" - let totalLines = countLinesInTimeline(timeline) - echo "Total lines: ", totalLines - echo "" - echo "Scanning the Hayabusa timeline. Please wait." - echo "" - + checkArgs(quiet, timeline, level) + let totalLines = countJsonlAndStartMsg("Processes", "executed processes from Sysmon 1 and Security 4688 events", timeline) var bar: SuruBar = initSuruBar() - stackProcesses = initCountTable[string]() - uniqueProcesses = 0 - + stack = initTable[string, StackRecord]() bar[0].total = totalLines bar.setup() - # Loop through JSON lines for line in lines(timeline): inc bar @@ -40,42 +16,8 @@ proc stackProcesses(ignoreSysmon: bool = false, ignoreSecurity: bool = false, ou let channel = jsonLine["Channel"].getStr("N/A") if (eventId == 1 and not ignoreSysmon and channel == "Sysmon") or (eventId == 4688 and not ignoreSecurity and channel == "Sec"): - let process = jsonLine["Details"]["Proc"].getStr("N/A") - stackProcesses.inc(process) - + let stackKey = jsonLine["Details"]["Proc"].getStr("N/A") + stackResult(stackKey, stack, level, jsonLine) bar.finish() - echo "" - - stackProcesses.sort() - - # Print results to screen - var outputFileSize = 0 - if output == "": - for process, count in stackProcesses: - inc uniqueProcesses - var commaDelimitedStr = $count & "," & process - commaDelimitedStr = replace(commaDelimitedStr, ",", " | ") - echo commaDelimitedStr - # Save to CSV file - else: - let outputFile = open(output, fmWrite) - writeLine(outputFile, "Count,Processes") - - # Write results - for process, count in stackProcesses: - inc uniqueProcesses - writeLine(outputFile, $count & "," & process) - outputFileSize = getFileSize(outputFile) - close(outputFile) - - echo "" - echo "Saved file: " & output & " (" & formatFileSize(outputFileSize) & ")" - - let endTime = epochTime() - let elapsedTime2 = int(endTime - startTime) - let hours = elapsedTime2 div 3600 - let minutes = (elapsedTime2 mod 3600) div 60 - let seconds = elapsedTime2 mod 60 - echo "" - echo "Elapsed time: ", $hours & " hours, " & $minutes & " minutes, " & $seconds & " seconds" - echo "" \ No newline at end of file + outputResult(output, "Process", stack) + outputElasptedTime(startTime) \ No newline at end of file diff --git a/src/takajopkg/stackServices.nim b/src/takajopkg/stackServices.nim index 078af8ac..0c99fb0a 100644 --- a/src/takajopkg/stackServices.nim +++ b/src/takajopkg/stackServices.nim @@ -1,35 +1,14 @@ -proc stackServices(ignoreSysmon: bool = false, ignoreSecurity: bool = false,output: string = "", quiet: bool = false, timeline: string) = +proc stackServices(level: string = "informational", ignoreSystem: bool = false, ignoreSecurity: bool = false,output: string = "", quiet: bool = false, timeline: string) = let startTime = epochTime() - if not quiet: - styledEcho(fgGreen, outputLogo()) - - if not os.fileExists(timeline): - echo "The file '" & timeline & "' does not exist. Please specify a valid file path." - quit(1) - - if not isJsonConvertible(timeline): - quit(1) - - echo "Started the Stack Services command" - echo "" - echo "This command will stack service names and paths." - echo "" - - echo "Counting total lines. Please wait." - echo "" - let totalLines = countLinesInTimeline(timeline) - echo "Total lines: ", totalLines - echo "" - echo "Scanning the Hayabusa timeline. Please wait." - echo "" + checkArgs(quiet, timeline, level) + let totalLines = countJsonlAndStartMsg("Services", "service names and paths from System 7045 and Security 4697 events", timeline) var bar: SuruBar = initSuruBar() - stackServices = initCountTable[string]() + stack = initTable[string, StackRecord]() bar[0].total = totalLines bar.setup() - # Loop through JSON lines for line in lines(timeline): inc bar @@ -37,61 +16,12 @@ proc stackServices(ignoreSysmon: bool = false, ignoreSecurity: bool = false,outp let jsonLine = parseJson(line) let eventId = jsonLine["EventID"].getInt(0) let channel = jsonLine["Channel"].getStr("N/A") - if (eventId == 7040 and not ignoreSysmon and channel == "Sysmon") or + if (eventId == 7045 and not ignoreSystem and channel == "Sys") or (eventId == 4697 and not ignoreSecurity and channel == "Sec"): let svc = jsonLine["Details"]["Svc"].getStr("N/A") let path = jsonLine["Details"]["Path"].getStr("N/A") - let res = svc & " -> " & path - stackServices.inc(res) - + let stackKey = svc & " -> " & path + stackResult(stackKey, stack, level, jsonLine, @[svc, path]) bar.finish() - echo "" - if stackServices.len == 0: - echo "No results where found." - else: - stackServices.sort() - var stackServicesSorted = newOrderedTable[string, int]() - var stack: seq[string] = newSeq[string]() - var prevCount = 0 - for service, count in stackServices: - stack.add(service) - if prevCount == count: - continue - stack.sort() - for s in stack: - stackServicesSorted[s] = count - stack = newSeq[string]() - prevCount = count - stack.sort() - for s in stack: - stackServicesSorted[s] = prevCount - - # Print results to screen - var outputFileSize = 0 - if output == "": - for service, count in stackServicesSorted: - var commaDelimitedStr = $count & "," & service - commaDelimitedStr = replace(commaDelimitedStr, ",", " | ") - echo commaDelimitedStr - # Save to CSV file - else: - let outputFile = open(output, fmWrite) - writeLine(outputFile, "Count,Services") - - # Write results - for service, count in stackServicesSorted: - writeLine(outputFile, $count & "," & service) - outputFileSize = getFileSize(outputFile) - close(outputFile) - - echo "" - echo "Saved file: " & output & " (" & formatFileSize(outputFileSize) & ")" - - let endTime = epochTime() - let elapsedTime2 = int(endTime - startTime) - let hours = elapsedTime2 div 3600 - let minutes = (elapsedTime2 mod 3600) div 60 - let seconds = elapsedTime2 mod 60 - echo "" - echo "Elapsed time: ", $hours & " hours, " & $minutes & " minutes, " & $seconds & " seconds" - echo "" \ No newline at end of file + outputResult(output, "Service", stack, @["ServiceName", "Path"]) + outputElasptedTime(startTime) \ No newline at end of file diff --git a/src/takajopkg/stackTasks.nim b/src/takajopkg/stackTasks.nim index 824d91a9..ed5022f0 100644 --- a/src/takajopkg/stackTasks.nim +++ b/src/takajopkg/stackTasks.nim @@ -1,39 +1,16 @@ proc decodeEntity(txt: string): string = return txt.replace("&","&").replace("<","<").replace(">",">").replace(""","\"").replace("'","'") -proc stackTasks(ignoreSysmon: bool = false, ignoreSecurity: bool = false, output: string = "", quiet: bool = false, timeline: string) = +proc stackTasks(level: string = "informational", ignoreSysmon: bool = false, ignoreSecurity: bool = false, output: string = "", quiet: bool = false, timeline: string) = let startTime = epochTime() - if not quiet: - styledEcho(fgGreen, outputLogo()) - - if not os.fileExists(timeline): - echo "The file '" & timeline & "' does not exist. Please specify a valid file path." - quit(1) - - if not isJsonConvertible(timeline): - quit(1) - - echo "Started the Stack Tasks command" - echo "" - echo "This command will stack new scheduled tasks." - echo "" - - echo "Counting total lines. Please wait." - echo "" - let totalLines = countLinesInTimeline(timeline) - echo "Total lines: ", totalLines - echo "" - echo "Scanning the Hayabusa timeline. Please wait." - echo "" - + checkArgs(quiet, timeline, level) + let totalLines = countJsonlAndStartMsg("Tasks", "new scheduled tasks from Security 4698 events", timeline) var bar: SuruBar = initSuruBar() - stackTasks = initCountTable[string]() + stack = initTable[string, StackRecord]() bar[0].total = totalLines bar.setup() - - # Loop through JSON lines for line in lines(timeline): inc bar @@ -47,48 +24,17 @@ proc stackTasks(ignoreSysmon: bool = false, ignoreSecurity: bool = false, output let content = jsonLine["Details"]["Content"].getStr("N/A").replace("\\r\\n", "") let node = parseXml(content) let commands = node.findAll("Command") - var commandAndArgs = "" + var command = "" + var args = "" if len(commands) > 0: - commandAndArgs = $commands[0] - commandAndArgs = commandAndArgs.replace("", "").replace("","") + command = $commands[0] + command = command.replace("", "").replace("","") let arguments = node.findAll("Arguments") if len(arguments) > 0: - commandAndArgs = commandAndArgs & " " & $arguments[0] - commandAndArgs = commandAndArgs.replace("", "").replace("","") - let result = user & " -> " & name & " -> " & decodeEntity(commandAndArgs) - stackTasks.inc(result) - + args = $arguments[0] + args = args.replace("", "").replace("","") + let stackKey = user & " -> " & name & " -> " & decodeEntity(command & " " & args) + stackResult(stackKey, stack, level, jsonLine, @[name, command, args]) bar.finish() - echo "" - - stackTasks.sort() - if stackTasks.len == 0: - echo "No results where found." - else: - # Print results to screen - var outputFileSize = 0 - if output == "": - for task, count in stackTasks: - var commaDelimitedStr = $count & "," & task - commaDelimitedStr = replace(commaDelimitedStr, ",", " | ") - echo commaDelimitedStr - # Save to CSV file - else: - let outputFile = open(output, fmWrite) - writeLine(outputFile, "Count,Tasks") - - # Write results - for task, count in stackTasks: - writeLine(outputFile, $count & "," & task) - outputFileSize = getFileSize(outputFile) - close(outputFile) - echo "Saved file: " & output & " (" & formatFileSize(outputFileSize) & ")" - - let endTime = epochTime() - let elapsedTime2 = int(endTime - startTime) - let hours = elapsedTime2 div 3600 - let minutes = (elapsedTime2 mod 3600) div 60 - let seconds = elapsedTime2 mod 60 - echo "" - echo "Elapsed time: ", $hours & " hours, " & $minutes & " minutes, " & $seconds & " seconds" - echo "" \ No newline at end of file + outputResult(output, "Task", stack, @["TaskName", "Command", "Arguments"]) + outputElasptedTime(startTime) \ No newline at end of file diff --git a/src/takajopkg/stackUtil.nim b/src/takajopkg/stackUtil.nim new file mode 100644 index 00000000..ce5328b4 --- /dev/null +++ b/src/takajopkg/stackUtil.nim @@ -0,0 +1,103 @@ +import general +import json +import nancy +import takajoTerminal +import std/algorithm +import std/enumerate +import std/sets +import std/sequtils +import std/strutils +import std/tables + +let LEVEL_ORDER = {"crit": 5, "high": 4, "med": 3, "low": 2, "info": 1}.toTable + +type StackRecord* = object + key*: string + count* = 0 + eid* = "" + channel* = "" + levelsOrder* = 0 + levels* = initHashSet[string]() + ruleTitles* = initHashSet[string]() + otherColumn = newSeq[string]() + +proc calcLevelOrder(x: StackRecord): int = + result = 0 + if "crit" in x.levels: + result = LEVEL_ORDER["crit"] + elif "high" in x.levels: + result = LEVEL_ORDER["high"] + elif "med" in x.levels: + result = LEVEL_ORDER["med"] + elif "low" in x.levels: + result = LEVEL_ORDER["low"] + elif "info" in x.levels: + result = LEVEL_ORDER["info"] + +proc levelCmp(x, y: string): int = + cmp(LEVEL_ORDER[x], LEVEL_ORDER[y]) * -1 + +proc recordCmp(x, y: StackRecord): int = + result = cmp(x.levelsOrder, y.levelsOrder) * -1 + if result == 0: + result = cmp(x.count, y.count) * - 1 + if result == 0: + result = cmp(x.channel, y.channel) + if result == 0: + result = cmp(x.eid, y.eid) + if result == 0: + result = cmp(x.key, y.key) + +proc buildCSVRecord(x: StackRecord): seq[string] = + let levelsStr = toSeq(x.levels).sorted(levelCmp).join(" | ") + let ruleTitlesStr = toSeq(x.ruleTitles).sorted.join(" | ") + if x.otherColumn.len == 0: + return @[intToStr(x.count), x.channel, x.eid, x.key, levelsStr, ruleTitlesStr] + return concat(@[intToStr(x.count), x.channel, x.eid], x.otherColumn, @[levelsStr, ruleTitlesStr]) + +proc stackResult*(key:string, stack: var Table[string, StackRecord], minLevel:string, jsonLine:JsonNode, otherColumn:seq[string] = @[]) = + let level = jsonLine["Level"].getStr("N/A") + if not isMinLevel(level, minLevel): + return + var val: StackRecord + if key notin stack: + val = StackRecord(key: key, eid: intToStr(jsonLine["EventID"].getInt(0)), channel: jsonLine["Channel"].getStr("N/A")) + else: + val = stack[key] + val.count += 1 + val.levels.incl(level) + val.levelsOrder = val.calcLevelOrder() + val.ruleTitles.incl(jsonLine["RuleTitle"].getStr("N/A")) + val.otherColumn = otherColumn + stack[key] = val + +proc outputResult*(output:string, culumnName: string, stack: Table[string, StackRecord], otherHeader:seq[string] = newSeq[string]()) = + echo "" + if stack.len == 0: + echo "No results where found." + else: + let stackRecords = toSeq(stack.values).sorted(recordCmp).map(buildCSVRecord) + var header = @["Count", "Channel", "EventID", culumnName, "Levels", "Alerts"] + if otherHeader.len > 0: + header = concat(@["Count", "Channel", "EventID"], otherHeader, @["Levels", "Alerts"]) + var table: TerminalTable + table.add header + for row in stackRecords: + let color = levelColor(row[^2]) + table.add map(row, proc(col: string): string = color col) + table.echoTableSepsWithStyled(seps = boxSeps) + echo "" + if output == "": + return + let outputFile = open(output, fmWrite) + writeLine(outputFile, header.join(",")) + for row in stackRecords: + for i, col in enumerate(row): + if i < row.len - 1: + outputFile.write(escapeCsvField(col) & ",") + else: + outputFile.write(escapeCsvField(col) & "\p") + let outputFileSize = getFileSize(outputFile) + close(outputFile) + echo "" + echo "Saved file: " & output & " (" & formatFileSize(outputFileSize) & ")" \ No newline at end of file diff --git a/src/takajopkg/takajoTerminal.nim b/src/takajopkg/takajoTerminal.nim new file mode 100644 index 00000000..b6e39a6e --- /dev/null +++ b/src/takajopkg/takajoTerminal.nim @@ -0,0 +1,67 @@ +import nancy +import terminal +import termstyle +import std/strutils + +proc outputLogo*(): string = + let logo = """ +╔════╦═══╦╗╔═╦═══╗ ╔╦═══╗ +║╔╗╔╗║╔═╗║║║╔╣╔═╗║ ║║╔═╗║ +╚╝║║╚╣║ ║║╚╝╝║║ ║║ ║║║ ║║ + ║║ ║╚═╝║╔╗╖║╚═╝╠╗║║║ ║║ + ╔╝╚╗║╔═╗║║║╚╣╔═╗║╚╝║╚═╝║ + ╚══╝╚╝ ╚╩╝╚═╩╝ ╚╩══╩═══╝ + by Yamato Security +""" + return logo + + +proc colorWrite*(color: ForegroundColor, ansiEscape: string, txt: string) = + # Remove ANSI escape sequences and use stdout.styledWrite instead + let replacedTxt = txt.replace(ansiEscape,"").replace(termClear,"") + if "│" in replacedTxt: + stdout.styledWrite(color, replacedTxt.replace("│ ","")) + stdout.write "│ " + else: + stdout.styledWrite(color, replacedTxt) + + +proc stdoutStyledWrite*(txt: string) = + if txt.startsWith(termRed): + colorWrite(fgRed, termRed,txt) + elif txt.startsWith(termGreen): + colorWrite(fgGreen, termGreen, txt) + elif txt.startsWith(termYellow): + colorWrite(fgYellow, termYellow, txt) + elif txt.startsWith(termCyan): + colorWrite(fgCyan, termCyan, txt) + else: + stdout.write txt.replace(termClear,"") + + +proc echoTableSepsWithStyled*(table: TerminalTable, maxSize = terminalWidth(), seps = defaultSeps) = + # This function equivalent to echoTableSeps without using ANSI escape to avoid the following issue. + # https://github.com/PMunch/nancy/issues/4 + # https://github.com/PMunch/nancy/blob/9918716a563f64d740df6a02db42662781e94fc8/src/nancy.nim#L195C6-L195C19 + let sizes = table.getColumnSizes(maxSize - 4, padding = 3) + printSeparator(top) + for k, entry in table.entries(sizes): + for _, row in entry(): + stdout.write seps.vertical & " " + for i, cell in row(): + stdoutStyledWrite cell & (if i != sizes.high: " " & seps.vertical & " " else: "") + stdout.write " " & seps.vertical & "\n" + if k != table.rows - 1: + printSeparator(center) + printSeparator(bottom) + +proc levelColor*(level:string): proc(ss: varargs[string, `$`]): string = + if "crit" in level: + return red + elif "high" in level: + return yellow + elif "med" in level: + return cyan + elif "low" in level: + return green + return white \ No newline at end of file