Update to v0.2.3 (#25)

* Bug fix: Remove move improper byte handling

- remove mode failed to display multibyte characters back to the user due the user supplied mask not reaching the removal function.

* Feature: Add title case to pop

- Added the ability to include "t" into the replacement mask for the pop mode which will pop title case words instead of skipping them
- changed the default replacement mask to include "t"
- fix unit tests

* Add templates directory to docker image

- adding templates to docker image

* add swap templates

- added a swap-encode template
- added a swap lists directory for examples

Author: JakeWnuk

Dockerfile

@@ -5,6 +5,7 @@ ADD ./main.go .
 ADD ./go.mod .
 ADD ./go.sum .
 ADD ./pkg ./pkg
+ADD ./templates/ ./templates
 RUN go build .
 
 FROM alpine

docs/WORDLISTS.md

@@ -48,9 +48,10 @@ Where `<mask_characters>` can be any of the following:
 - `d`: Digits
 - `s`: Special characters
 - `b`: Byte characters
+- `t`: Title case words (requires `u` and `l`)
 - Multiple characters can be combined to create a mask.
 
-The default value is `uldsb` for all characters. This mode will create tokens
+The default value is `uldsbt` for all characters. This mode will create tokens
 by popping characters from the input string then aggregating the results.
 
 ### Token Swapping

main.go

@@ -55,7 +55,7 @@ func main() {
 			"mask -rm [uldsb] -v":                "Transforms input by masking characters with provided mask.",
 			"remove -rm [uldsb]":                 "Transforms input by removing characters with provided mask characters.",
 			"mask-retain -rm [uldsb] -tf [file]": "Transforms input by creating masks that still retain strings from file.",
-			"pop -rm [uldsb]":                    "Transforms input by generating tokens from popping strings at character boundaries.",
+			"pop -rm [uldsbt]":                   "Transforms input by generating tokens from popping strings at character boundaries.",
 			"mask-match -tf [file]":              "Transforms input by keeping only strings with matching masks from a mask file.",
 			"swap -tf [file]":                    "Transforms input by swapping tokens with exact matches from a ':' separated file.",
 			"mask-swap -tf [file]":               "Transforms input by swapping tokens from a partial mask file and a input file.",
@@ -83,7 +83,7 @@ func main() {
 	minimum := flag.Int("m", 0, "Minimum numerical frequency to include in output.")
 	verboseStatsMax := flag.Int("n", 25, "Maximum number of items to display in verbose statistics output.")
 	transformation := flag.String("t", "", "Transformation to apply to input.")
-	replacementMask := flag.String("rm", "uldsb", "Replacement mask for transformations if applicable.")
+	replacementMask := flag.String("rm", "uldsbt", "Replacement mask for transformations if applicable.")
 	jsonOutput := flag.String("o", "", "Output to JSON file in addition to stdout.")
 	bypassMap := flag.Bool("b", false, "Bypass map creation and use stdout as primary output.")
 	debugMode := flag.Int("d", 0, "Enable debug mode with verbosity levels [0-2].")

pkg/mask/mask.go

@@ -267,20 +267,21 @@ func TestMaskComplexity(str string) int {
 // Args:
 //
 //	input (map[string]int): Input map
+//	replacementMask (string): Mask characters to apply
 //	bypass (bool): If true, the map is not used for output or filtering
 //	debug (bool): If true, print additional debug information to stderr
 //
 // Returns:
 //
 //	(map[string]int): Masked map
-func RemoveMaskedCharacters(input map[string]int, bypass bool, debug bool) map[string]int {
+func RemoveMaskedCharacters(input map[string]int, replacementMask string, bypass bool, debug bool) map[string]int {
 	maskedMap := make(map[string]int)
 	replacer := strings.NewReplacer("?u", "", "?l", "", "?d", "", "?b", "", "?s", "")
 
 	for key, value := range input {
 		newKey := replacer.Replace(key)
 
-		if !utils.CheckASCIIString(newKey) {
+		if !utils.CheckASCIIString(newKey) && strings.Contains(replacementMask, "b") {
 			newKey = ConvertMultiByteMask(newKey)
 		}
 
@@ -393,7 +394,9 @@ func BoundarySplitPopMap(input map[string]int, replacementMask string, bypass bo
 			}
 
 			if (lastRuneType != 0 && lastRuneType != runeType) || !strings.ContainsRune(replacementMask, runeType) {
-				if token != "" {
+				if strings.ContainsRune(replacementMask, 't') && lastRuneType == 'u' && runeType == 'l' {
+					// do nothing so the token continues
+				} else if token != "" {
 					result[token]++
 					token = ""
 				}

pkg/mask/mask_test.go

@@ -225,7 +225,7 @@ func TestRemoveMaskedCharacters(t *testing.T) {
 
 	// Run test cases
 	for _, test := range tests {
-		output := RemoveMaskedCharacters(test.input, false, false)
+		output := RemoveMaskedCharacters(test.input, "ulsbd", false, false)
 		if !reflect.DeepEqual(output, test.output) {
 			t.Errorf("Test failed: %v inputted, %v expected, %v returned", test.input, test.output, output)
 		}

pkg/transform/transform.go

@@ -86,7 +86,7 @@ func TransformationController(input map[string]int, mode string, startingIndex i
 		output = format.HexEncodeMap(input, bypass, functionDebug)
 	case "remove", "remove-all", "delete", "delete-all", "rm":
 		input = mask.MakeMaskedMap(input, replacementMask, false, false, false)
-		output = mask.RemoveMaskedCharacters(input, bypass, functionDebug)
+		output = mask.RemoveMaskedCharacters(input, replacementMask, bypass, functionDebug)
 	case "retain-mask", "retain", "r", "mask-retain":
 		if len(transformationFilesMap) == 0 {
 			fmt.Fprintf(os.Stderr, "[!] Retain masks require use of one or more -tf flags to specify one or more files\n")

templates/swap-encode.json

@@ -0,0 +1,20 @@
+[
+    {
+        "StartIndex": 0,
+        "EndIndex": 0,
+        "Verbose": false,
+        "ReplacementMask": "uldsb",
+        "Bypass": false,
+        "TransformationMode": "swap",
+        "PassphraseWords": 0
+    },
+    {
+        "StartIndex": 0,
+        "EndIndex": 0,
+        "Verbose": false,
+        "ReplacementMask": "uldsb",
+        "Bypass": false,
+        "TransformationMode": "encode",
+        "PassphraseWords": 0
+    }
+]

templates/swap-lists/love.txt

@@ -0,0 +1,12 @@
+love:<3
+heart:<3
+love:♥
+heart:♥
+love:❤
+heart:❤
+love:😍
+heart:😍
+love:😘
+heart:😘
+love:🥰
+heart:🥰