From ce42947cae32cda8d5d8813c1a8ce82eb06f018e Mon Sep 17 00:00:00 2001 From: Zachary Rice Date: Tue, 7 Dec 2021 19:11:37 -0600 Subject: [PATCH] fix deduplication issue caused by clobbered findings (#742) * fix deduplication issue caused by clobbered findings * fix index * remove indexing, slow is better than wrong --- cmd/detect.go | 2 +- config/gitleaks.toml | 11 ++++-- detect/detect.go | 1 + detect/files.go | 7 ++-- detect/files_test.go | 6 ++-- detect/git.go | 6 ++-- detect/git_test.go | 56 +++++++++++++++--------------- report/csv.go | 2 +- report/csv_test.go | 6 ++-- report/finding.go | 12 ++++++- report/json.go | 2 +- report/json_test.go | 6 ++-- report/report.go | 2 +- report/report_test.go | 14 ++++---- report/sarif.go | 10 +++--- report/sarif_test.go | 4 +-- testdata/repos/small/dotGit/index | Bin 317 -> 317 bytes 17 files changed, 81 insertions(+), 66 deletions(-) diff --git a/cmd/detect.go b/cmd/detect.go index d936dee14..833918be7 100644 --- a/cmd/detect.go +++ b/cmd/detect.go @@ -31,7 +31,7 @@ func runDetect(cmd *cobra.Command, args []string) { initConfig() var ( vc config.ViperConfig - findings []*report.Finding + findings []report.Finding err error ) diff --git a/config/gitleaks.toml b/config/gitleaks.toml index db88ee309..dae3c26fc 100644 --- a/config/gitleaks.toml +++ b/config/gitleaks.toml @@ -273,14 +273,19 @@ description = "Dropbox API secret/key" regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]''' [[rules]] -id = "dropbox-sl-api-token" +id = "dropbox--api-key" +description = "Dropbox API secret/key" +regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-z0-9]{15})['\"]''' + +[[rules]] +id = "dropbox-short-lived-api-token" description = "Dropbox short lived API token" regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](sl\.[a-z0-9\-=_]{135})['\"]''' [[rules]] -id = "dropbox-ll-api-token" +id = "dropbox-long-lived-api-token" description = "Dropbox long lived API token" -regex = '''(?i)(dropbox)(.{0,20})['\"](?i)[a-z0-9]{11}(AAAAAAAAAA)[a-z0-9-_=]{43}['\"]''' +regex = '''(?i)(dropbox[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"][a-z0-9]{11}(AAAAAAAAAA)[a-z0-9\-_=]{43}['\"]''' [[rules]] id = "duffel-api-token" diff --git a/detect/detect.go b/detect/detect.go index d5c5b47d6..84b2332b5 100644 --- a/detect/detect.go +++ b/detect/detect.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/rs/zerolog/log" + "github.com/zricethezav/gitleaks/v8/config" "github.com/zricethezav/gitleaks/v8/report" ) diff --git a/detect/files.go b/detect/files.go index 1d45d56c2..580850d4e 100644 --- a/detect/files.go +++ b/detect/files.go @@ -15,9 +15,9 @@ import ( // FromFiles opens the directory or file specified in source and checks each file against the rules // from the configuration. If any secrets are found, they are added to the list of findings. -func FromFiles(source string, cfg config.Config, outputOptions Options) ([]*report.Finding, error) { +func FromFiles(source string, cfg config.Config, outputOptions Options) ([]report.Finding, error) { var ( - findings []*report.Finding + findings []report.Finding mu sync.Mutex ) g, _ := errgroup.WithContext(context.Background()) @@ -51,7 +51,6 @@ func FromFiles(source string, cfg config.Config, outputOptions Options) ([]*repo } fis := DetectFindings(cfg, b, p, "") for _, fi := range fis { - fi.File = p if outputOptions.Redact { fi.Redact() } @@ -59,7 +58,7 @@ func FromFiles(source string, cfg config.Config, outputOptions Options) ([]*repo printFinding(fi) } mu.Lock() - findings = append(findings, &fi) + findings = append(findings, fi) mu.Unlock() } return nil diff --git a/detect/files_test.go b/detect/files_test.go index e7148092c..7f9f7706f 100644 --- a/detect/files_test.go +++ b/detect/files_test.go @@ -17,12 +17,12 @@ func TestFromFiles(t *testing.T) { cfgName string opts Options source string - expectedFindings []*report.Finding + expectedFindings []report.Finding }{ { source: filepath.Join(repoBasePath, "nogit"), cfgName: "simple", - expectedFindings: []*report.Finding{ + expectedFindings: []report.Finding{ { Description: "AWS Access Key", StartLine: 19, @@ -40,7 +40,7 @@ func TestFromFiles(t *testing.T) { { source: filepath.Join(repoBasePath, "nogit", "main.go"), cfgName: "simple", - expectedFindings: []*report.Finding{ + expectedFindings: []report.Finding{ { Description: "AWS Access Key", StartLine: 19, diff --git a/detect/git.go b/detect/git.go index 0060c522a..b06f4970b 100644 --- a/detect/git.go +++ b/detect/git.go @@ -15,8 +15,8 @@ import ( // FromGit accepts a gitdiff.File channel (structure output from `git log -p`) and a configuration // struct. Files from the gitdiff.File channel are then checked against each rule in the configuration to // check for secrets. If any secrets are found, they are added to the list of findings. -func FromGit(files <-chan *gitdiff.File, cfg config.Config, outputOptions Options) []*report.Finding { - var findings []*report.Finding +func FromGit(files <-chan *gitdiff.File, cfg config.Config, outputOptions Options) []report.Finding { + var findings []report.Finding mu := sync.Mutex{} wg := sync.WaitGroup{} commitMap := make(map[string]bool) @@ -82,7 +82,7 @@ func FromGit(files <-chan *gitdiff.File, cfg config.Config, outputOptions Option printFinding(fi) } mu.Lock() - findings = append(findings, &fi) + findings = append(findings, fi) mu.Unlock() } diff --git a/detect/git_test.go b/detect/git_test.go index 9b91f1614..6f191ac95 100644 --- a/detect/git_test.go +++ b/detect/git_test.go @@ -26,13 +26,13 @@ func TestFromGit(t *testing.T) { source string logOpts string expected string - expectedFindings []*report.Finding + expectedFindings []report.Finding }{ { source: filepath.Join(repoBasePath, "small"), expected: filepath.Join(expectPath, "git", "small.txt"), cfgName: "simple", - expectedFindings: []*report.Finding{ + expectedFindings: []report.Finding{ { Description: "AWS Access Key", StartLine: 20, @@ -40,15 +40,15 @@ func TestFromGit(t *testing.T) { StartColumn: 19, EndColumn: 38, Secret: "AKIALALEMEL33243OLIA", + Match: "AKIALALEMEL33243OLIA", File: "main.go", - // Line: "\tawsToken := \"AKIALALEMEL33243OLIA\"", - Date: "2021-11-02T23:37:53Z", - Commit: "1b6da43b82b22e4eaa10bcf8ee591e91abbfc587", - Author: "Zachary Rice", - Email: "zricer@protonmail.com", - Message: "Accidentally add a secret", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, + Date: "2021-11-02T23:37:53Z", + Commit: "1b6da43b82b22e4eaa10bcf8ee591e91abbfc587", + Author: "Zachary Rice", + Email: "zricer@protonmail.com", + Message: "Accidentally add a secret", + RuleID: "aws-access-key", + Tags: []string{"key", "AWS"}, }, { Description: "AWS Access Key", @@ -57,15 +57,15 @@ func TestFromGit(t *testing.T) { StartColumn: 17, EndColumn: 36, Secret: "AKIALALEMEL33243OLIA", + Match: "AKIALALEMEL33243OLIA", File: "foo/foo.go", - // Line: "\taws_token := \"AKIALALEMEL33243OLIA\"", - Date: "2021-11-02T23:48:06Z", - Commit: "491504d5a31946ce75e22554cc34203d8e5ff3ca", - Author: "Zach Rice", - Email: "zricer@protonmail.com", - Message: "adding foo package with secret", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, + Date: "2021-11-02T23:48:06Z", + Commit: "491504d5a31946ce75e22554cc34203d8e5ff3ca", + Author: "Zach Rice", + Email: "zricer@protonmail.com", + Message: "adding foo package with secret", + RuleID: "aws-access-key", + Tags: []string{"key", "AWS"}, }, }, }, @@ -74,7 +74,7 @@ func TestFromGit(t *testing.T) { expected: filepath.Join(expectPath, "git", "small-branch-foo.txt"), logOpts: "--all foo...", cfgName: "simple", - expectedFindings: []*report.Finding{ + expectedFindings: []report.Finding{ { Description: "AWS Access Key", StartLine: 9, @@ -82,15 +82,15 @@ func TestFromGit(t *testing.T) { StartColumn: 17, EndColumn: 36, Secret: "AKIALALEMEL33243OLIA", - // Line: "\taws_token := \"AKIALALEMEL33243OLIA\"", - Date: "2021-11-02T23:48:06Z", - File: "foo/foo.go", - Commit: "491504d5a31946ce75e22554cc34203d8e5ff3ca", - Author: "Zach Rice", - Email: "zricer@protonmail.com", - Message: "adding foo package with secret", - RuleID: "aws-access-key", - Tags: []string{"key", "AWS"}, + Match: "AKIALALEMEL33243OLIA", + Date: "2021-11-02T23:48:06Z", + File: "foo/foo.go", + Commit: "491504d5a31946ce75e22554cc34203d8e5ff3ca", + Author: "Zach Rice", + Email: "zricer@protonmail.com", + Message: "adding foo package with secret", + RuleID: "aws-access-key", + Tags: []string{"key", "AWS"}, }, }, }, diff --git a/report/csv.go b/report/csv.go index 67bf057c1..7cbbd8366 100644 --- a/report/csv.go +++ b/report/csv.go @@ -7,7 +7,7 @@ import ( ) // writeCsv writes the list of findings to a writeCloser. -func writeCsv(f []*Finding, w io.WriteCloser) error { +func writeCsv(f []Finding, w io.WriteCloser) error { if len(f) == 0 { return nil } diff --git a/report/csv_test.go b/report/csv_test.go index da31e8b30..9f6f64533 100644 --- a/report/csv_test.go +++ b/report/csv_test.go @@ -9,7 +9,7 @@ import ( func TestWriteCSV(t *testing.T) { tests := []struct { - findings []*Finding + findings []Finding testReportName string expected string wantEmpty bool @@ -17,7 +17,7 @@ func TestWriteCSV(t *testing.T) { { testReportName: "simple", expected: filepath.Join(expectPath, "report", "csv_simple.csv"), - findings: []*Finding{ + findings: []Finding{ { RuleID: "test-rule", Match: "line containing secret", @@ -39,7 +39,7 @@ func TestWriteCSV(t *testing.T) { wantEmpty: true, testReportName: "empty", expected: filepath.Join(expectPath, "report", "this_should_not_exist.csv"), - findings: []*Finding{}}, + findings: []Finding{}}, } for _, test := range tests { diff --git a/report/finding.go b/report/finding.go index e2c2d9e70..6a4d7f341 100644 --- a/report/finding.go +++ b/report/finding.go @@ -1,6 +1,9 @@ package report -import "strings" +import ( + "strconv" + "strings" +) // Finding contains information about strings that // have been captured by a tree-sitter query. @@ -40,3 +43,10 @@ func (f *Finding) Redact() { f.Match = strings.Replace(f.Match, f.Secret, "REDACTED", -1) f.Secret = "REDACT" } + +func (f *Finding) Hash() string { + return f.Secret + f.Commit + + strconv.Itoa(f.EndLine) + + strconv.Itoa(f.StartLine) + +} diff --git a/report/json.go b/report/json.go index a6b5f35cb..eb3966e3e 100644 --- a/report/json.go +++ b/report/json.go @@ -5,7 +5,7 @@ import ( "io" ) -func writeJson(findings []*Finding, w io.WriteCloser) error { +func writeJson(findings []Finding, w io.WriteCloser) error { if len(findings) == 0 { return nil } diff --git a/report/json_test.go b/report/json_test.go index dd0a08380..79a0229b6 100644 --- a/report/json_test.go +++ b/report/json_test.go @@ -9,7 +9,7 @@ import ( func TestWriteJSON(t *testing.T) { tests := []struct { - findings []*Finding + findings []Finding testReportName string expected string wantEmpty bool @@ -17,7 +17,7 @@ func TestWriteJSON(t *testing.T) { { testReportName: "simple", expected: filepath.Join(expectPath, "report", "json_simple.json"), - findings: []*Finding{ + findings: []Finding{ { Description: "", @@ -42,7 +42,7 @@ func TestWriteJSON(t *testing.T) { wantEmpty: true, testReportName: "empty", expected: filepath.Join(expectPath, "report", "this_should_not_exist.json"), - findings: []*Finding{}}, + findings: []Finding{}}, } for _, test := range tests { diff --git a/report/report.go b/report/report.go index 5a89b7d8c..8393c8149 100644 --- a/report/report.go +++ b/report/report.go @@ -13,7 +13,7 @@ const ( CWE_DESCRIPTION = "Use of Hard-coded Credentials" ) -func Write(findings []*Finding, cfg config.Config, ext string, reportPath string) error { +func Write(findings []Finding, cfg config.Config, ext string, reportPath string) error { if len(findings) == 0 { return nil } diff --git a/report/report_test.go b/report/report_test.go index 91eec01ec..ae0b84db2 100644 --- a/report/report_test.go +++ b/report/report_test.go @@ -16,13 +16,13 @@ const ( func TestReport(t *testing.T) { tests := []struct { - findings []*Finding + findings []Finding ext string wantEmpty bool }{ { ext: "json", - findings: []*Finding{ + findings: []Finding{ { RuleID: "test-rule", }, @@ -30,7 +30,7 @@ func TestReport(t *testing.T) { }, { ext: ".json", - findings: []*Finding{ + findings: []Finding{ { RuleID: "test-rule", }, @@ -38,7 +38,7 @@ func TestReport(t *testing.T) { }, { ext: ".jsonj", - findings: []*Finding{ + findings: []Finding{ { RuleID: "test-rule", }, @@ -47,7 +47,7 @@ func TestReport(t *testing.T) { }, { ext: ".csv", - findings: []*Finding{ + findings: []Finding{ { RuleID: "test-rule", }, @@ -55,7 +55,7 @@ func TestReport(t *testing.T) { }, { ext: "csv", - findings: []*Finding{ + findings: []Finding{ { RuleID: "test-rule", }, @@ -63,7 +63,7 @@ func TestReport(t *testing.T) { }, { ext: "CSV", - findings: []*Finding{ + findings: []Finding{ { RuleID: "test-rule", }, diff --git a/report/sarif.go b/report/sarif.go index 7e5bc2f43..7ab14ef7e 100644 --- a/report/sarif.go +++ b/report/sarif.go @@ -8,7 +8,7 @@ import ( "github.com/zricethezav/gitleaks/v8/config" ) -func writeSarif(cfg config.Config, findings []*Finding, w io.WriteCloser) error { +func writeSarif(cfg config.Config, findings []Finding, w io.WriteCloser) error { sarif := Sarif{ Schema: "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", Version: "2.1.0", @@ -20,7 +20,7 @@ func writeSarif(cfg config.Config, findings []*Finding, w io.WriteCloser) error return encoder.Encode(sarif) } -func getRuns(cfg config.Config, findings []*Finding) []Runs { +func getRuns(cfg config.Config, findings []Finding) []Runs { return []Runs{ { Tool: getTool(cfg), @@ -54,7 +54,7 @@ func getRules(cfg config.Config) []Rules { return rules } -func messageText(f *Finding) string { +func messageText(f Finding) string { if f.Commit == "" { return fmt.Sprintf("%s has detected secret for file %s.", f.RuleID, f.File) } @@ -63,7 +63,7 @@ func messageText(f *Finding) string { } -func getResults(findings []*Finding) []Results { +func getResults(findings []Finding) []Results { var results []Results for _, f := range findings { r := Results{ @@ -87,7 +87,7 @@ func getResults(findings []*Finding) []Results { return results } -func getLocation(f *Finding) []Locations { +func getLocation(f Finding) []Locations { return []Locations{ { PhysicalLocation: PhysicalLocation{ diff --git a/report/sarif_test.go b/report/sarif_test.go index 1fa849f12..f2fb158f4 100644 --- a/report/sarif_test.go +++ b/report/sarif_test.go @@ -14,7 +14,7 @@ const configPath = "../testdata/config/" func TestWriteSarif(t *testing.T) { tests := []struct { - findings []*Finding + findings []Finding testReportName string expected string wantEmpty bool @@ -24,7 +24,7 @@ func TestWriteSarif(t *testing.T) { cfgName: "simple", testReportName: "simple", expected: filepath.Join(expectPath, "report", "sarif_simple.sarif"), - findings: []*Finding{ + findings: []Finding{ { Description: "", diff --git a/testdata/repos/small/dotGit/index b/testdata/repos/small/dotGit/index index 208f3d0e5f1f28ec06287f1a3345245ec45a54d2..936b241166d7e3a9aa7ff91955fad2d3fb8ac26c 100644 GIT binary patch delta 92 zcmdnXw3kW6#WTp6fq{Vuh?x`D*~VDtG_b>HMg|5JX6-qTCaM^UgQSdG5*7>XB?J?ZxUZ%ohOxJRU@8p3;;@d8lV6G delta 92 zcmdnXw3kW6#WTp6fq{Vuh?x^-pZLpS?D+#mGcqu+Fw5_bpQvIe4wAAue+VoEq@hxY d6La>8yxx<_c)3wdDN8Fs@(lYW%QJb3X8@|5A^iXV