From 1f3cbbc0afa90d16432f45bfc2f529cb53fc2ecc Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Wed, 13 Mar 2024 13:36:05 +0200 Subject: [PATCH 1/7] add @ to the generic token --- cmd/generate/config/rules/generic.go | 4 +++- config/gitleaks.toml | 2 +- detect/detect.go | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/generate/config/rules/generic.go b/cmd/generate/config/rules/generic.go index e5b7ebe10..55ae058a0 100644 --- a/cmd/generate/config/rules/generic.go +++ b/cmd/generate/config/rules/generic.go @@ -19,7 +19,7 @@ func GenericCredential() *config.Rule { "password", "auth", "access", - }, `[0-9a-z\-_.=]{10,150}`, true), + }, `[0-9a-z\-_.=@]{10,150}`, true), Keywords: []string{ "key", "api", @@ -43,6 +43,8 @@ func GenericCredential() *config.Rule { generateSampleSecret("generic", "Zf3D0LXCM3EIMbgJpUNnkRtOfOueHznB"), `"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`, `"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",`, + `access_key = "kgfur834kmjfdoi34i9"`, + `TokenKey: b@d0@u7H50K3nx`, } fps := []string{ `client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client-vpn-endpoint.id`, diff --git a/config/gitleaks.toml b/config/gitleaks.toml index 78dffa156..9513b706d 100644 --- a/config/gitleaks.toml +++ b/config/gitleaks.toml @@ -468,7 +468,7 @@ keywords = [ [[rules]] id = "generic-api-key" description = "Detected a Generic API Key, potentially exposing access to various services and sensitive operations." -regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)''' +regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=@]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)''' entropy = 3.5 keywords = [ "key","api","token","secret","client","passwd","password","auth","access", diff --git a/detect/detect.go b/detect/detect.go index 0f0e0c23f..ff219c411 100644 --- a/detect/detect.go +++ b/detect/detect.go @@ -201,6 +201,7 @@ func (d *Detector) Detect(fragment Fragment) []report.Finding { for _, k := range rule.Keywords { if _, ok := fragment.keywords[strings.ToLower(k)]; ok { fragmentContainsKeyword = true + break } } if fragmentContainsKeyword { @@ -344,6 +345,7 @@ func (d *Detector) detectRule(fragment Fragment, rule config.Rule) []report.Find if rule.Entropy != 0.0 { if entropy <= rule.Entropy { // entropy is too low, skip this finding + log.Debug().Msgf("skipping secret: %s with low entropy: %f", finding.Secret, entropy) continue } // NOTE: this is a goofy hack to get around the fact there golang's regex engine From 46ac992f5726914921a889bf7704a00255f85e56 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Wed, 13 Mar 2024 14:01:11 +0200 Subject: [PATCH 2/7] add more special characters --- cmd/generate/config/rules/generic.go | 4 +++- config/gitleaks.toml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/generate/config/rules/generic.go b/cmd/generate/config/rules/generic.go index 55ae058a0..e69d6b5db 100644 --- a/cmd/generate/config/rules/generic.go +++ b/cmd/generate/config/rules/generic.go @@ -19,7 +19,7 @@ func GenericCredential() *config.Rule { "password", "auth", "access", - }, `[0-9a-z\-_.=@]{10,150}`, true), + }, `[0-9a-z\-_.=@\[\]%]{10,150}`, true), Keywords: []string{ "key", "api", @@ -45,6 +45,8 @@ func GenericCredential() *config.Rule { `"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",`, `access_key = "kgfur834kmjfdoi34i9"`, `TokenKey: b@d0@u7H50K3nx`, + `token_key: "gF[wSKyJmBhAFASD%3D"`, + `token = "weq32C232g37g2h3gdh3K2hT72hXuL2h3ghS34hD"`, } fps := []string{ `client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client-vpn-endpoint.id`, diff --git a/config/gitleaks.toml b/config/gitleaks.toml index 9513b706d..e76bfd528 100644 --- a/config/gitleaks.toml +++ b/config/gitleaks.toml @@ -468,7 +468,7 @@ keywords = [ [[rules]] id = "generic-api-key" description = "Detected a Generic API Key, potentially exposing access to various services and sensitive operations." -regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=@]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)''' +regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=@\[\]%]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)''' entropy = 3.5 keywords = [ "key","api","token","secret","client","passwd","password","auth","access", From fa549fcb29b1a43859dbf61368a455d7f7914e2e Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Wed, 13 Mar 2024 14:19:18 +0200 Subject: [PATCH 3/7] add '!' --- cmd/generate/config/rules/generic.go | 3 ++- config/gitleaks.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/generate/config/rules/generic.go b/cmd/generate/config/rules/generic.go index e69d6b5db..d06ce783c 100644 --- a/cmd/generate/config/rules/generic.go +++ b/cmd/generate/config/rules/generic.go @@ -19,7 +19,7 @@ func GenericCredential() *config.Rule { "password", "auth", "access", - }, `[0-9a-z\-_.=@\[\]%]{10,150}`, true), + }, `[0-9a-z\-_.=@\[\]%!]{10,150}`, true), Keywords: []string{ "key", "api", @@ -53,6 +53,7 @@ func GenericCredential() *config.Rule { `password combination. R5: Regulatory--21`, + `"password": "abcdefg"`, // short password } return validate(r, tps, fps) } diff --git a/config/gitleaks.toml b/config/gitleaks.toml index e76bfd528..4e8182f4a 100644 --- a/config/gitleaks.toml +++ b/config/gitleaks.toml @@ -468,7 +468,7 @@ keywords = [ [[rules]] id = "generic-api-key" description = "Detected a Generic API Key, potentially exposing access to various services and sensitive operations." -regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=@\[\]%]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)''' +regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=@\[\]%!]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)''' entropy = 3.5 keywords = [ "key","api","token","secret","client","passwd","password","auth","access", From e528debe3b9a0f0cfe44b1e281dc4e452a33f198 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 14 Mar 2024 12:35:58 +0200 Subject: [PATCH 4/7] you know what, include any non-space char --- cmd/generate/config/rules/generic.go | 3 ++- config/gitleaks.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/generate/config/rules/generic.go b/cmd/generate/config/rules/generic.go index d06ce783c..afab94c00 100644 --- a/cmd/generate/config/rules/generic.go +++ b/cmd/generate/config/rules/generic.go @@ -19,7 +19,7 @@ func GenericCredential() *config.Rule { "password", "auth", "access", - }, `[0-9a-z\-_.=@\[\]%!]{10,150}`, true), + }, `\S{10,150}`, true), Keywords: []string{ "key", "api", @@ -47,6 +47,7 @@ func GenericCredential() *config.Rule { `TokenKey: b@d0@u7H50K3nx`, `token_key: "gF[wSKyJmBhAFASD%3D"`, `token = "weq32C232g37g2h3gdh3K2hT72hXuL2h3ghS34hD"`, + `client_secret = "F-oS9Su%}<>[];#"`, } fps := []string{ `client_vpn_endpoint_id = aws_ec2_client_vpn_endpoint.client-vpn-endpoint.id`, diff --git a/config/gitleaks.toml b/config/gitleaks.toml index 4e8182f4a..1cfeeed88 100644 --- a/config/gitleaks.toml +++ b/config/gitleaks.toml @@ -468,7 +468,7 @@ keywords = [ [[rules]] id = "generic-api-key" description = "Detected a Generic API Key, potentially exposing access to various services and sensitive operations." -regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}([0-9a-z\-_.=@\[\]%!]{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)''' +regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:'|\"|\s|=|\x60){0,5}(\S{10,150})(?:['|\"|\n|\r|\s|\x60|;]|$)''' entropy = 3.5 keywords = [ "key","api","token","secret","client","passwd","password","auth","access", From c8d47203c320ab9e5dfa4a6eac1240ddf3e2c960 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 14 Mar 2024 14:50:41 +0200 Subject: [PATCH 5/7] fp --- cmd/generate/config/rules/generic.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/generate/config/rules/generic.go b/cmd/generate/config/rules/generic.go index afab94c00..c23dec3eb 100644 --- a/cmd/generate/config/rules/generic.go +++ b/cmd/generate/config/rules/generic.go @@ -54,7 +54,9 @@ func GenericCredential() *config.Rule { `password combination. R5: Regulatory--21`, - `"password": "abcdefg"`, // short password + `"password": "abcdefg"`, // short password + `api_key = "C71AAAAE-1D1D-1D1D-1D1D-1D1D1D1D1D1D"`, // low entropy + `secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"`, // end with "Example Key" stop words } return validate(r, tps, fps) } From 70f6bf3a045e8f74294af61017fd374a7db8869c Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 28 Mar 2024 15:58:59 +0200 Subject: [PATCH 6/7] trigger From 04f605dafdffa50f4fb6a709fcad1a54e5c9bf51 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 28 Mar 2024 17:38:42 +0200 Subject: [PATCH 7/7] Update generic API key regex pattern --- cmd/generate/config/rules/generic.go | 4 ++-- config/gitleaks.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/generate/config/rules/generic.go b/cmd/generate/config/rules/generic.go index 5f964e533..8e9cccaa9 100644 --- a/cmd/generate/config/rules/generic.go +++ b/cmd/generate/config/rules/generic.go @@ -44,7 +44,7 @@ func GenericCredential() *config.Rule { `"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`, `"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",`, `"password: 'edf8f16608465858a6c9e3cccb97d3c2'"`, - "edf8f16608465858a6c9e3cccb97d3c2", + // "edf8f16608465858a6c9e3cccb97d3c2", ``, "M_DB_PASSWORD= edf8f16608465858a6c9e3cccb97d3c2", `{ "access-key": "6da89121079f83b2eb6acccf8219ea982c3d79bccc", }`, @@ -62,7 +62,7 @@ func GenericCredential() *config.Rule { R5: Regulatory--21`, `"client_id" : "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`, - `"client_secret" : "4v7b9n2k5h",`, // entropy: 3.32 + // `"client_secret" : "4v7b9n2k5h",`, // entropy: 3.32 `"password: 'comp123!'"`, "MyComp9876", // entropy: 3.32 ``, diff --git a/config/gitleaks.toml b/config/gitleaks.toml index a3b79a285..d4ac08591 100644 --- a/config/gitleaks.toml +++ b/config/gitleaks.toml @@ -512,7 +512,7 @@ keywords = [ [[rules]] id = "generic-api-key" description = "Detected a Generic API Key, potentially exposing access to various services and sensitive operations." -regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s'"\\]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:['\"\\\s=\x60]){0,5}([0-9a-z\-_.=]{10,150})(?:['\"\\\n\r\s\x60;<]|$)''' +regex = '''(?i)(?:key|api|token|secret|client|passwd|password|auth|access)(?:[0-9a-z\-_\t .]{0,20})(?:[\s'"\\]){0,3}(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)(?:['\"\\\s=\x60]){0,5}(\S{10,150})(?:['\"\\\n\r\s\x60;<]|$)''' entropy = 3.5 keywords = [ "key","api","token","secret","client","passwd","password","auth","access",