Merge pull request #10 from jamesduncombe/features/adds-xxhash64

Adds xxHash64 plus extras

Author: boyter

README.md

@@ -36,7 +36,7 @@ Why use `hashit`?
  - It is very fast
  - You can get multiple hashes "for free" on any CPU with multiple cores
  - Works very well across multiple platforms without slowdown (Windows, Linux, macOS)
- - Supports many hashes `hashit --hashes` MD4, MD5, SHA1, SHA256, SHA512, Blake2b-256, Blake2b-512, Blake3, SHA3-224, SHA3-256, SHA3-384, SHA3-512
+ - Supports many hashes `hashit --hashes` CRC32, xxHash64, MD4, MD5, SHA1, SHA256, SHA512, Blake2b-256, Blake2b-512, Blake3, SHA3-224, SHA3-256, SHA3-384, SHA3-512
  - Output is compatible with `hashdeep`
 
 ### Usage
@@ -81,19 +81,22 @@ README.md (6616 bytes)
      SHA512 e4d108219def5c36089f7cd45d4531548d52a41cc5992004c78e816fbe64d63a21bc7ea1303d7a31bd693bf4f5435c916fbbe4e9d3e1fd0b1982a9734b4ec739
 
 $ hashit --hash all README.md
-README.md (8025 bytes)
-        MD4 1fe6069ad2ad0e14d5760f680563f7ad
-        MD5 76132c9b8bd4abd9968bf77aecdd5212
-       SHA1 b7700fc98999af299270ad5797adff8c765188bb
-     SHA256 a11989771927275e29f66e9f6486126231bf3bc397bdecebb54d06967718a8af
-     SHA512 e7a81bfb5b53c38eda7caab0a158db0067ebfbeabe32db52c97d09b75560199c4ef1ab055f812621f328e0348b861b9bae06dd5d1e6e30c0ebe87cf4145c8eae
-Blake2b-256 04f5f1d2437a2b6627c8ebd365173323bfff698e94882a15d364a88e8e213d40
-Blake2b-512 96fade8d2cb0a894612c8d1f013d22939a7f744efe2d7f71b60d68a5d38697886f53b702f8be159e528c6212cbad4f562c4209236d82f146e77692f2a059e95e
-     Blake3 5e7e9d827d67ffa1813f2611b6ea5db52b8ffc1f21733b7ebff7968b7a7d154b
-   SHA3-224 49c7678edc07785dd86d57eac2a5c55e31b28696b0c65c0a5685427f
-   SHA3-256 b2490b9afc453dac7464e575fd7aac4258e632cbd77c3a371ad72443a21971bd
-   SHA3-384 4091ab0487ca4f6ac2f08e2c4d323c775c441cdf5995e7148e1d3310115e3f99348882e620df4c6959d5074121e50a53
-   SHA3-512 61950eb9fdbaec2add28b46da7ae628c985989453c41f5570335537eff62cb7c135d34baef3a16fdf5d823ec653ce00bc3a70f564b5562f4a7a7fabebdd9c903    
+README.md (8964 bytes)
+      CRC32 a6f59c84
+   xxHash64 d3b87287c02996de
+        MD4 f106887bd63a7b7c5269a039c501b6b6
+        MD5 22ed507c2a5fc7b822c8080340d799ab
+       SHA1 2e74fa58da1f19b66b5826cf73d0c28a6c48aed6
+     SHA256 c84475732426797adfc3b1f14717d1aeeee976ca19ddd596f2ea5e59c932a062
+     SHA512 4f77e1d43cfcdb04c1ad99b5d53087b960ea7301da21f76de42cfefe77eb6f0bb3d2781db6c99f3f99d161e76c68d5a74a990b81e4e908a7104aef7d512efd31
+Blake2b-256 59627f6afdb0bcd783a9fe1f77763ff87c5768135e839fd9f16b440828f2a8e6
+Blake2b-512 9d00cd557c20b15ee85fdcc292d53604f4783d79dde6d3aada67391423633f28a1e5279519260b85171558bafc4b3785adc1854b88b004e402e49962426d86b4
+     Blake3 dcef77d73bc12c8d931b253835eae13854d18b8b16c784c089929815592a4506
+   SHA3-224 3912468ee2c0dfe3a4bf5f2bf4c2c36b684a465a73445d9651cf5e4d
+   SHA3-256 ca7bcbd1ad389727a2502cfd5b785968ebff7453551eb44ffd1e879f47fc00dc
+   SHA3-384 b97839411747e2174ed9d4dbf415e7945160746962f3208a4fd05ded8013c569da66db2fa8e8001c438896e160a153c7
+   SHA3-512 be6ac69cbb38fd2dc2646e7ec3e04878c0bc8824fe196da63ac577fae35a026251ef59618e9584bc2fbb274424a6a5251aea0ab64148469eb1b704a6c836627e
+
 
 $ hashit scripts
 scripts/include.go (1835 bytes)

go.mod

@@ -3,6 +3,7 @@ module github.com/boyter/hashit
 go 1.22
 
 require (
+	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/djherbis/times v1.6.0
 	github.com/gosuri/uiprogress v0.0.1
 	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1

go.sum

@@ -1,3 +1,5 @@
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
 github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=

processor/formatters.go

@@ -72,7 +72,7 @@ func toSum(input chan Result) string {
 	first := true
 
 	for res := range input {
-		if first == false {
+		if !first {
 			str.WriteString("\n")
 		} else {
 			first = false
@@ -81,6 +81,9 @@ func toSum(input chan Result) string {
 		if hasHash(HashNames.CRC32) {
 			str.WriteString(res.CRC32 + "  " + res.File + "\n")
 		}
+		if hasHash(HashNames.XxHash64) {
+			str.WriteString(res.XxHash64 + "  " + res.File + "\n")
+		}
 		if hasHash(HashNames.MD4) {
 			str.WriteString(res.MD4 + "  " + res.File + "\n")
 		}
@@ -118,7 +121,7 @@ func toSum(input chan Result) string {
 			str.WriteString(res.Sha3512 + "  " + res.File + "\n")
 		}
 
-		if NoStream == false && FileOutput == "" {
+		if !NoStream && FileOutput == "" {
 			fmt.Print(str.String())
 			str.Reset()
 		}
@@ -135,6 +138,9 @@ func toHashOnly(input chan Result) (string, bool) {
 		if hasHash(HashNames.CRC32) {
 			str.WriteString(res.CRC32 + "\n")
 		}
+		if hasHash(HashNames.XxHash64) {
+			str.WriteString(res.XxHash64 + "\n")
+		}
 		if hasHash(HashNames.MD4) {
 			str.WriteString(res.MD4 + "\n")
 		}
@@ -172,7 +178,7 @@ func toHashOnly(input chan Result) (string, bool) {
 			str.WriteString(res.Sha3512 + "\n")
 		}
 
-		if NoStream == false && FileOutput == "" {
+		if !NoStream && FileOutput == "" {
 			fmt.Print(str.String())
 			str.Reset()
 		}
@@ -187,7 +193,7 @@ func toText(input chan Result) (string, bool) {
 	first := true
 
 	for res := range input {
-		if first == false {
+		if !first {
 			str.WriteString("\n")
 		} else {
 			first = false
@@ -198,6 +204,9 @@ func toText(input chan Result) (string, bool) {
 		if hasHash(HashNames.CRC32) {
 			str.WriteString("      CRC32 " + res.CRC32 + "\n")
 		}
+		if hasHash(HashNames.XxHash64) {
+			str.WriteString("   xxHash64 " + res.XxHash64 + "\n")
+		}
 		if hasHash(HashNames.MD4) {
 			str.WriteString("        MD4 " + res.MD4 + "\n")
 		}
@@ -235,7 +244,7 @@ func toText(input chan Result) (string, bool) {
 			str.WriteString("   SHA3-512 " + res.Sha3512 + "\n")
 		}
 
-		if NoStream == false && FileOutput == "" {
+		if !NoStream && FileOutput == "" {
 			fmt.Print(str.String())
 			str.Reset()
 		}
@@ -302,6 +311,7 @@ func toHashDeep(input chan Result) string {
 
 func printHashes() {
 	fmt.Println(fmt.Sprintf("      CRC32 (%s)", HashNames.CRC32))
+	fmt.Println(fmt.Sprintf("   xxHash64 (%s)", HashNames.XxHash64))
 	fmt.Println(fmt.Sprintf("        MD4 (%s)", HashNames.MD4))
 	fmt.Println(fmt.Sprintf("        MD5 (%s)", HashNames.MD5))
 	fmt.Println(fmt.Sprintf("       SHA1 (%s)", HashNames.SHA1))

processor/processor.go

@@ -3,13 +3,14 @@ package processor
 import (
 	"bufio"
 	"fmt"
-	"github.com/gosuri/uiprogress"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
+
+	"github.com/gosuri/uiprogress"
 )
 
 // Global Version
@@ -56,7 +57,6 @@ var AuditFile = ""
 
 // DirFilePaths is not set via flags but by arguments following the flags for file or directory to process
 var DirFilePaths = []string{}
-var isDir = false
 
 // FileListQueueSize is the queue of files found and ready to be processed
 var FileListQueueSize = 1000
@@ -75,6 +75,7 @@ var NoThreads = runtime.NumCPU()
 // String mapping for hash names
 var HashNames = Result{
 	CRC32:      "crc32",
+	XxHash64:   "xxhash64",
 	MD4:        "md4",
 	MD5:        "md5",
 	SHA1:       "sha1",
@@ -89,12 +90,6 @@ var HashNames = Result{
 	Sha3512:    "sha3512",
 }
 
-// Raw hashDatabase loaded
-var hashDatabase = map[string]Result{}
-
-// Hash to name lookup
-var hashLookup = map[string]string{}
-
 // Process is the main entry point of the command line it sets everything up and starts running
 func Process() {
 	// Display the supported hashes then bail out
@@ -150,7 +145,6 @@ func Process() {
 					} else {
 						if fi.IsDir() {
 							if Recursive {
-								isDir = true
 								walkDirectory(fp, fileListQueue)
 							}
 						} else {

processor/structs.go

@@ -6,6 +6,7 @@ import "time"
 type Result struct {
 	File       string
 	CRC32      string
+	XxHash64   string
 	MD4        string
 	MD5        string
 	SHA1       string