From 2e292b0eaa88eb7d038fafd7d1b084fe55b61f10 Mon Sep 17 00:00:00 2001 From: Jake Wnuk Date: Wed, 31 Jul 2024 18:11:13 -0400 Subject: [PATCH] v0.2.5 (#27) * Add replace mode and bug fixes - Added a new mode that allows for all replacements in a transformation file to be applied at once. This balances the swap mode by providing another option for text swapping. - Added unit tests - Updated documentation * Update USAGE.md * Feature: add more plots - added more plots to the -vvv function by default - added missing debug flag for HexEncodeMap() --- README.md | 4 +- docs/USAGE.md | 88 ++++++++++++++++++++------------- main.go | 3 +- pkg/format/format.go | 83 +++++++++++++++++++++++++++++++ pkg/transform/transform.go | 43 +++++++++++++++- pkg/transform/transform_test.go | 30 ++++++++++- pkg/utils/utils.go | 31 ++++++++++++ pkg/utils/utils_test.go | 39 +++++++++++++++ 8 files changed, 284 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 2b13cdb..76183cd 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ - Supports multiple transformations and operations with a template file. ``` -Usage of Password Transformation Tool (ptt) version (0.2.4): +Usage of Password Transformation Tool (ptt) version (0.2.5): ptt [options] [...] Accepts standard input and/or additonal arguments. @@ -94,6 +94,8 @@ Transformation Modes: Transforms input into prepend-shift rules. -t remove -rm [uldsb] Transforms input by removing characters with provided mask characters. + -t replace -tf [file] + Transforms input by replacing all strings with all matches from a ':' separated file. -t substring -i [index] Transforms input by extracting substrings starting at index and ending at index. -t swap -tf [file] diff --git a/docs/USAGE.md b/docs/USAGE.md index 7f2f261..0d230c8 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -1,5 +1,5 @@ # Password Transformation Tool (PTT) Usage Guide -## Version 0.2.4 +## Version 0.2.5 ### Table of Contents #### Getting Started @@ -28,9 +28,10 @@ ### Wordlist Creation Guide 1. [Wordlist Creation Introduction](#wordlist-creation-introduction) 2. [Direct Swapping](#direct-swapping) -3. [Token Popping](#token-popping) -4. [Token Swapping](#token-swapping) -5. [Passphrases](#passphrases) +3. [Replacing Text and Characters](#replacing-text-and-characters) +4. [Token Popping](#token-popping) +5. [Token Swapping](#token-swapping) +6. [Passphrases](#passphrases) ### Misc Creation Guide 1. [Misc Creation Introduction](#misc-creation-introduction) @@ -174,10 +175,11 @@ keywords above: - `dehex`: `dh`, `unhex` - `mask`: `m`, `partial-mask`, `partial` - `remove`: `rm`, `remove-all`, `delete`, `delete-all` +- `replace`: `rp`, `rep` - `substring`: `sub`, `sb` - `retain`: `r`, `retain-mask`, - `match`: `mt`, `match-mask` -- `swap`: `s`, `replace` +- `swap`: `sw`, `swp` - `pop`: `po`, `split`, `boundary-split`, `boundary-pop`, `pop-split`, `split-pop` - `mask-swap`: `ms`, `shuf`, `shuffle`, `token-swap` - `passphrase`: `pp`, `phrase` @@ -267,45 +269,51 @@ life [14496]======================== ``` - `ptt -f rockyou.txt -t pop -l 4-5 -vvv`: ```shell -$ ptt -f rockyou.txt -t pop -l 4-5 -vvv [*] Starting statistics generation. Please wait... Verbose Statistics: max=25 -------------------------------------------------- General Stats: -Total Items: 4730675 -Total Unique items: 585203 -Total Characters: 2758719 -Total Words: 585199 +Total Items: 4695779 +Total Unique items: 613210 +Total Words: 613206 Largest frequency: 29529 Smallest frequency: 1 +Plots: +Item Length: |[|==========]| +Min: 4, Q1: 4, Q2: 4, Q3: 5, Max: 5 +Item Frequency: |[|]--------------------------------------------------| +Min: 1, Q1: 1, Q2: 1, Q3: 3, Max: 29529 +Item Complexity: |[|]----------------------------------| +Min: 1, Q1: 1, Q2: 1, Q3: 1, Max: 3 + Category Counts: -alphabetical: 496547 -all-lowercase: 524227 -short-non-complex: 585203 -high-numeric-ratio: 86021 -greek-characters: 16 -hex-string: 11208 -non-complex: 585203 -numeric: 86028 -non-ASCII: 566 -cyrillic-characters: 15 +all-uppercase: 58433 +non-ASCII: 547 alphanumeric-with-special: 8 -starts-uppercase: 60976 -all-uppercase: 149650 -arabic-characters: 17 -thai-characters: 14 +alphabetical: 524554 +short-non-complex: 613210 +numeric: 86028 +all-lowercase: 410494 +non-complex: 613210 hebrew-characters: 3 +hex-string: 11395 +thai-characters: 14 +arabic-characters: 17 +cyrillic-characters: 13 +starts-uppercase: 114042 +high-numeric-ratio: 86014 +greek-characters: 16 -------------------------------------------------- 1234 [29529]================================================== 2007 [24459]========================================= 2006 [22002]===================================== -love [21516]==================================== +love [20435]================================== 2008 [20022]================================= -ever [17694]============================= +ever [17605]============================= 1994 [14514]======================== -life [14496]======================== +life [14460]======================== 2005 [14300]======================== 1992 [14159]======================= 1993 [14070]======================= @@ -320,7 +328,7 @@ life [14496]======================== 1988 [9718]================ 2009 [9257]=============== 2004 [9091]=============== -yahoo [8953]=============== +yahoo [8942]=============== 1986 [8860]=============== 1985 [8513]============== ``` @@ -496,13 +504,15 @@ print output for the overwrite transformation starting from index 1 to 5. This document describes the ways to use PTT to create password cracking wordlists. There are several ways to generate wordlists using PTT: -- `direct-swapping`: Swapping characters directly with a `:` separated file. +- `Direct Swapping`: Swapping characters directly with a `:` separated file. This is implemented in the `swap` module. -- `token-popping`: Generates tokens by popping strings at character boundaries. +- `Replacing Text and Characters`: Replacing text and characters in a string. + This is implemented in the `replace` module +- `Token Popping`: Generates tokens by popping strings at character boundaries. This is implemented in the `pop` module. -- `token-swapping`: Generates tokens by swapping characters in a string. This is +- `Token Swapping`: Generates tokens by swapping characters in a string. This is implemented in the `mask-swap` module. -- `passphrases`: Generates passphrases by combining words from a wordlist. This +- `Passphrases`: Generates passphrases by combining words from a wordlist. This is implemented in the `passphrase` module. All modes support multibyte characters and can properly convert them. One @@ -518,7 +528,18 @@ syntax is as follows: ptt -f -t swap -tf ``` The replacement file should contain the strings to be transformed as `PRIOR:POST` -pairs. The replacements will be applied to the first instance in each line. +pairs. The replacements will be applied to the all instance in each line but +only one swap is applied at once. This mode is ideal for subsituting words or characters in a string. + +### Replacing Text and Characters +The `replace` module replaces text and characters in a string. This mode replaces all strings with all matches from a ':' separated file. The syntax is as follows: +``` +ptt -f -t replace -tf +``` +The replacement file should contain the strings to be transformed as +`PRIOR:POST` pairs. The replacements will be applied to all instances in each +line and all replacements are applied to the string. This mode is ideal for replacing all instances of a word or character in +a string. ### Token Popping The `pop` module generates tokens by popping strings at character boundaries. @@ -581,6 +602,7 @@ There are several types that can be created using PTT: - `Encoding and Decoding`: This transforms input to and from URL, HTML, and Unicode escaped strings. - `Hex and Dehex`: This transforms input to and from `$HEX[....]` strings. +- `Substrings`: This extracts substrings from the input based on position. All modes support multibyte characters and can properly convert them. One transformation can be used at a time. diff --git a/main.go b/main.go index e2ddd4d..7d8d1bc 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,7 @@ import ( "github.com/jakewnuk/ptt/pkg/utils" ) -var version = "0.2.4" +var version = "0.2.5" var wg sync.WaitGroup var mutex = &sync.Mutex{} var retain models.FileArgumentFlag @@ -61,6 +61,7 @@ func main() { "mask-swap -tf [file]": "Transforms input by swapping tokens from a partial mask file and a input file.", "passphrase -w [words] -tf [file]": "Transforms input by randomly generating passphrases with a given number of words and separators from a file.", "substring -i [index]": "Transforms input by extracting substrings starting at index and ending at index.", + "replace -tf [file]": "Transforms input by replacing all strings with all matches from a ':' separated file.", } // Sort and print transformation modes diff --git a/pkg/format/format.go b/pkg/format/format.go index bc12cbc..a0a914e 100644 --- a/pkg/format/format.go +++ b/pkg/format/format.go @@ -14,6 +14,7 @@ import ( "strings" "unicode" + "github.com/jakewnuk/ptt/pkg/mask" "github.com/jakewnuk/ptt/pkg/models" ) @@ -144,11 +145,24 @@ func CreateVerboseStats(freq map[string]int) string { // Pull stats totalWords := 0 totalItems := 0 + lengths := make([]int, 0) + frequencies := make([]int, 0) + complexities := make([]int, 0) categoryCounts := make(map[string]int) for k, v := range freq { totalWords += len(strings.Fields(k)) categories := StatClassifyToken(k) + frequencies = append(frequencies, v) totalItems += v + + for i := 0; i < v; i++ { + lengths = append(lengths, len(k)) + } + + m := mask.MakeMaskedString(k, "uldbs") + complexity := mask.TestMaskComplexity(m) + complexities = append(complexities, complexity) + for _, category := range categories { categoryCounts[category]++ } @@ -160,6 +174,17 @@ func CreateVerboseStats(freq map[string]int) string { stats += fmt.Sprintf("Largest frequency: %d\n", p[0].Value) stats += fmt.Sprintf("Smallest frequency: %d\n", p[len(p)-1].Value) + stats += "\nPlots:\n" + plot, min, q1, q2, q3, max := CreateBoxAndWhiskersPlot(lengths) + stats += fmt.Sprintf("Item Length: %s\n", plot) + stats += fmt.Sprintf("Min: %d, Q1: %d, Q2: %d, Q3: %d, Max: %d\n", min, q1, q2, q3, max) + plot, min, q1, q2, q3, max = CreateBoxAndWhiskersPlot(frequencies) + stats += fmt.Sprintf("Item Frequency: %s\n", plot) + stats += fmt.Sprintf("Min: %d, Q1: %d, Q2: %d, Q3: %d, Max: %d\n", min, q1, q2, q3, max) + plot, min, q1, q2, q3, max = CreateBoxAndWhiskersPlot(complexities) + stats += fmt.Sprintf("Item Complexity: %s\n", plot) + stats += fmt.Sprintf("Min: %d, Q1: %d, Q2: %d, Q3: %d, Max: %d\n", min, q1, q2, q3, max) + stats += "\nCategory Counts:\n" for category, count := range categoryCounts { stats += fmt.Sprintf("%s: %d\n", category, count) @@ -318,6 +343,58 @@ func StatClassifyToken(s string) []string { return categories } +// CalculateQuartiles calculates the first, second, and third quartiles of a +// list of integers and returns the values. +// +// Args: +// data ([]int): A list of integers +// +// Returns: +// int: The first quartile value +// int: The second quartile value +// int: The third quartile value +func CalculateQuartiles(data []int) (int, int, int) { + sort.Ints(data) + n := len(data) + + q1 := data[n/4] + q2 := data[n/2] + q3 := data[3*n/4] + + return q1, q2, q3 +} + +// CreateBoxAndWhiskersPlot creates a box and whiskers plot from a list of +// integers. +// +// Args: +// +// data ([]int): A list of integers +// +// Returns: +// string: A string representation of the box and whiskers plot +// int: The minimum value +// int: The first quartile value +// int: The second quartile value +// int: The third quartile value +// int: The maximum value +func CreateBoxAndWhiskersPlot(data []int) (string, int, int, int, int, int) { + q1, q2, q3 := CalculateQuartiles(data) + min := data[0] + max := data[len(data)-1] + + // Normalize the plot + largest := max + normalizedQ1 := q1 * 50 / largest + normalizedQ2 := q2 * 50 / largest + normalizedQ3 := q3 * 50 / largest + normalizedMin := min * 50 / largest + normalizedMax := max * 50 / largest + + plot := fmt.Sprintf("|%s[%s|%s]%s|", strings.Repeat("-", normalizedQ1-normalizedMin), strings.Repeat("=", normalizedQ2-normalizedQ1), strings.Repeat("=", normalizedQ3-normalizedQ2), strings.Repeat("-", normalizedMax-normalizedQ3)) + return plot, min, q1, q2, q3, max +} + // SaveArrayToJSON saves an array of items to a JSON file at the specified path // with a generated filename. The filename is generated with the format "ptt-.json" // where the timestamp is the current time in RFC3339 format. @@ -715,6 +792,12 @@ func HexEncodeMap(input map[string]int, bypass bool, debug bool) map[string]int output := make(map[string]int) for k, v := range input { encoded := hex.EncodeToString([]byte(k)) + if debug { + fmt.Fprintf(os.Stderr, "[?] HexEncodeMap:\n") + fmt.Fprintf(os.Stderr, "Input: %s\n", k) + fmt.Fprintf(os.Stderr, "Encoded: %s\n", encoded) + } + if !bypass { output["$HEX["+encoded+"]"] = v } else { diff --git a/pkg/transform/transform.go b/pkg/transform/transform.go index 16ad585..0a7b9be 100644 --- a/pkg/transform/transform.go +++ b/pkg/transform/transform.go @@ -99,7 +99,7 @@ func TransformationController(input map[string]int, mode string, startingIndex i os.Exit(1) } output = mask.MakeMatchedMaskedMap(input, replacementMask, transformationFilesMap, bypass, functionDebug) - case "swap", "replace", "s": + case "swap", "sw", "swp": if len(transformationFilesMap) == 0 { fmt.Fprintf(os.Stderr, "[!] Swap operations require use of one or more -tf flags to specify one or more files\n") fmt.Fprintf(os.Stderr, "[!] This transformation mode requires a ':' separated list of keys to swap\n") @@ -123,6 +123,12 @@ func TransformationController(input map[string]int, mode string, startingIndex i output = MakePassphraseMap(input, transformationFilesMap, bypass, functionDebug, passphraseWords) case "substring", "sub", "sb": output = utils.SubstringMap(input, startingIndex, endingIndex, bypass, functionDebug) + case "replace", "rp", "rep": + if len(transformationFilesMap) == 0 { + fmt.Fprintf(os.Stderr, "[!] Replace operations require use of one or more -tf flags to specify one or more files\n") + os.Exit(1) + } + output = ReplaceAllKeysInMap(input, transformationFilesMap, bypass, functionDebug) default: output = input } @@ -174,6 +180,41 @@ func ReplaceKeysInMap(originalMap map[string]int, replacements map[string]int, b return newMap } +// ReplaceAllKeysInMap takes a map of keys and values and replaces the keys +// with replacements based on the replacement map. This is useful for +// replacing all instances of a key with a new key. +// +// Args: +// +// originalMap (map[string]int): The original map to replace keys in +// replacements (map[string]int): The map of replacements to use +// bypass (bool): If true, the map is not used for output or filtering +// debug (bool): If true, print additional debug information to stderr +// +// Returns: +// +// (map[string]int): A new map with the keys replaced +func ReplaceAllKeysInMap(originalMap map[string]int, replacements map[string]int, bypass bool, debug bool) map[string]int { + newMap := make(map[string]int) + for key, value := range originalMap { + newKeyArray := utils.ReplaceAllSubstring(key, replacements) + for _, newKey := range newKeyArray { + + if debug { + fmt.Fprintf(os.Stderr, "Key: %s\n", key) + fmt.Fprintf(os.Stderr, "New Key: %s\n", newKey) + } + + if !bypass { + newMap[newKey] = value + } else { + fmt.Println(newKey) + } + } + } + return newMap +} + // MakePassphraseMap takes a map of keys and creates a new map with new // passphrases for each key. The transformation file is used to insert // separators between the words. If the replacement mask is set to blank, then diff --git a/pkg/transform/transform_test.go b/pkg/transform/transform_test.go index be60377..a45cd2a 100644 --- a/pkg/transform/transform_test.go +++ b/pkg/transform/transform_test.go @@ -10,7 +10,8 @@ import ( // Functions with Unit Tests // ---------------------------------------------------------------------------- // ** Generation Functions ** -// - ReplaceKeysInMap +// - ReplaceKeysInMap() +// - ReplaceAllKeysInMap() // // ---------------------------------------------------------------------------- // Functions without Unit Tests @@ -46,3 +47,30 @@ func TestReplaceKeysInMap(t *testing.T) { } } } + +// Unit Test for ReplaceAllKeysInMap +func TestReplaceAllKeysInMap(t *testing.T) { + + // Define a test case struct + type testCase struct { + input map[string]int + replace map[string]int + output map[string]int + } + + type testCases []testCase + + // Define the test cases + tests := testCases{ + {map[string]int{"abc": 1}, map[string]int{"a:1": 1, "b:2": 2, "c:3": 3}, map[string]int{"123": 1}}, + {map[string]int{"123Testing": 1, "456Testing": 2, "789Testing": 3}, map[string]int{"123:foo": 1, "456:bar": 2, "789:love": 3}, map[string]int{"fooTesting": 1, "barTesting": 2, "loveTesting": 3}}, + } + + // Run the test cases + for _, test := range tests { + result := ReplaceAllKeysInMap(test.input, test.replace, false, false) + if utils.CheckAreMapsEqual(result, test.output) == false { + t.Errorf("Test case failed. Expected %v, got %v", test.output, result) + } + } +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 6422b90..55cd330 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -579,6 +579,37 @@ func ReplaceSubstring(original string, replacements map[string]int) []string { return newStrings } +// ReplaceAllSubstring replaces all instances of a substring in a string with +// a new substring if the substring is found in the original string. All of the +// replacements are applied to the original string. The new substring is +// determined by the key in the replacements map separated by a colon +// character. +// +// Args: +// +// original (string): The original string +// replacements (map[string]int): A map of substrings to replace +// +// Returns: +// +// []string: The original string with all instances of the substring replaced +func ReplaceAllSubstring(original string, replacements map[string]int) []string { + newStrings := []string{original} + for newSubstr := range replacements { + // Split the new substring into the old and new strings by the colon character + if !strings.Contains(newSubstr, ":") { + continue + } + oldStr, newStr := strings.Split(newSubstr, ":")[0], strings.Split(newSubstr, ":")[1] + var tempStrings []string + for _, s := range newStrings { + tempStrings = append(tempStrings, strings.Replace(s, oldStr, newStr, -1)) + } + newStrings = tempStrings + } + return newStrings +} + // SubstringMap returns a map of substrings from a map of strings starting at // the start index and ending at the end index. If the bypass flag is set to // true, the function will print to stdout and return an empty map. If the diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index 589e281..6fa751c 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -22,6 +22,7 @@ import ( // - ConvertMultiByteCharToIteratingRule() // - SplitBySeparatorString() // - ReplaceSubstring() +// - ReplaceAllSubstring() // - SubstringMap() // // ** Validation Functions ** @@ -35,6 +36,7 @@ import ( // ---------------------------------------------------------------------------- // - ReadURLsToMap() (Loading and Processing Functions) // - ProcessURL() (Loading and Processing Functions) +// - IsFileSystemDirectory() (Validation Functions) // Unit Test for ReadFilesToMap() func TestReadFilesToMap(t *testing.T) { @@ -401,6 +403,43 @@ func TestReplaceSubstring(t *testing.T) { } } +// Unit Test for ReplaceAllSubstring() +func TestReplaceAllSubstring(t *testing.T) { + + // Define a test case struct + type TestCase struct { + Input string + Replacements map[string]int + Output []string + } + + type TestCases []TestCase + + // Define test cases + testCases := TestCases{ + {"I love you", map[string]int{"love:miss": 1}, []string{"I miss you"}}, + {"I <3 you", map[string]int{"<3:heart": 1}, []string{"I heart you"}}, + {"I 爱 you", map[string]int{"爱:love": 1}, []string{"I love you"}}, + {"I love you", map[string]int{"love:爱": 1}, []string{"I 爱 you"}}, + {"13Teststreet31p", map[string]int{"street:road": 1}, []string{"13Testroad31p"}}, + {"123131asdasd", map[string]int{"131:313": 1}, []string{"123313asdasd"}}, + {"12313zxczxc", map[string]int{"13:31": 1}, []string{"12331zxczxc"}}, + } + + // Run test cases + for _, testCase := range testCases { + input := testCase.Input + replacements := testCase.Replacements + output := testCase.Output + + given := ReplaceAllSubstring(input, replacements) + if given[0] != output[0] { + t.Errorf("ReplaceAllSubstring(%v, %v) = %v; want %v", input, replacements, given, output) + } + } + +} + // Unit Test for SubstringMap() func TestSubstringMap(t *testing.T) {