feat: match prefix or suffix

Author: rgmz

pkg/detectors/detectors.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/rand"
 	"errors"
+	"fmt"
 	"math/big"
 	"net/url"
 	"strings"
@@ -233,6 +234,27 @@ func PrefixRegex(keywords []string) string {
 	return pre + middle + post
 }
 
+func PrefixOrSuffixRegex(keywords []string, pattern string) string {
+	prefixPat := `(?i:` + strings.Join(keywords, "|") + `)(?:.{0,100}?|.*?(?:[\r\n]{1,2}.*?){1,15})` + pattern
+	suffixPat := pattern + `(?:.{0,100}?|.*?(?:[\r\n]{1,2}.*?){0,15})` + `(?i:` + strings.Join(keywords, "|") + `)`
+
+	return fmt.Sprintf(`(?:%s|%s)`, prefixPat, suffixPat)
+}
+
+// FirstNonEmptyMatch returns the index and value of the first non-empty match.
+func FirstNonEmptyMatch(matches []string) string {
+	if len(matches) <= 1 {
+		return ""
+	}
+	// The first index is the entire matched string.
+	for _, val := range matches[1:] {
+		if val != "" {
+			return val
+		}
+	}
+	return ""
+}
+
 // KeyIsRandom is a Low cost check to make sure that 'keys' include a number to reduce FPs.
 // Golang doesn't support regex lookaheads, so must be done in separate calls.
 // TODO improve checks. Shannon entropy did not work well.

pkg/detectors/github/v1/github_old.go

@@ -36,7 +36,7 @@ func (Scanner) Version() int          { return 1 }
 func (Scanner) CloudEndpoint() string { return "https://api.github.com" }
 
 var (
-	keyPat = regexp.MustCompile(`(?:(?i:github|token)|(?-i:GH|gh|HUB|[Hh]ub|PAT|[Pp]at|OCTO|[Oo]cto))[^\.].{0,40}[ =:'"]+([a-f0-9]{40})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixOrSuffixRegex([]string{`(?:(?i:github|token)|(?-i:GH|gh|HUB|[Hh]ub|PAT|[Pp]at|OCTO|[Oo]cto))`}, `\b([a-f0-9]{40})\b`))
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -51,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 	uniqueMatches := make(map[string]struct{})
 	for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
-		m := match[1]
+		m := detectors.FirstNonEmptyMatch(match)
 		// Ignore low-entropy matches.
 		if detectors.StringShannonEntropy(m) < 3 {
 			continue

pkg/detectors/snykkey/snykkey.go

@@ -22,7 +22,7 @@ type Scanner struct {
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
-var keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"snyk"}) + `\b([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`)
+var keyPat = regexp.MustCompile(detectors.PrefixOrSuffixRegex([]string{"snyk"}, `\b([0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})\b`))
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
@@ -36,7 +36,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 	tokens := make(map[string]struct{})
 	for _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {
-		tokens[match[1]] = struct{}{}
+		m := detectors.FirstNonEmptyMatch(match)
+		if detectors.StringShannonEntropy(m) < 3 {
+			continue
+		}
+		tokens[m] = struct{}{}
 	}
 
 	for token := range tokens {

pkg/detectors/snykkey/snykkey_test.go

@@ -25,14 +25,15 @@ set PATH=%PATH%;C:\Program Files\nodejs\;C:\Program Files\Git\cmd`,
 			want: []string{"885953dc-2469-443c-983d-5243d2d54116"},
 		},
 		// https://docs.snyk.io/snyk-api/get-a-projects-sbom-document-endpoint#how-to-generate-the-sbom-for-a-project
-		//	{
-		//		name: "curl example",
-		//		`curl --get \
-		// -H "Authorization: token ccc9ae71-913f-46bd-9d23-03356323400a" \
-		// --data-urlencode "version=2023-03-20" \
-		// --data-urlencode "format=cyclonedx1.4%2Bjson" \
-		// https://api.snyk.io/rest/orgs/1234/projects/1234/sbom`,
-		//	},
+		{
+			name: "suffix example",
+			input: `curl --get \
+		-H "Authorization: token ccc9ae71-913f-46bd-9d23-03356323400a" \
+		--data-urlencode "version=2023-03-20" \
+		--data-urlencode "format=cyclonedx1.4%2Bjson" \
+		https://api.snyk.io/rest/orgs/1234/projects/1234/sbom`,
+			want: []string{"ccc9ae71-913f-46bd-9d23-03356323400a"},
+		},
 	}
 
 	for _, test := range tests {