From 3b5ad71b4ba9dea39fb28728ac9a717183b99a40 Mon Sep 17 00:00:00 2001 From: cxMiguelSilva Date: Thu, 4 May 2023 11:25:13 +0100 Subject: [PATCH 1/2] add preview secrets mask behavior --- pkg/engine/secrets/inspector.go | 8 +- pkg/scan/post_scan.go | 9 ++ pkg/scan/preview_secrets_mask.go | 150 +++++++++++++++++++++++++ pkg/scan/preview_secrets_mask_test.go | 151 ++++++++++++++++++++++++++ 4 files changed, 314 insertions(+), 4 deletions(-) create mode 100644 pkg/scan/preview_secrets_mask.go create mode 100644 pkg/scan/preview_secrets_mask_test.go diff --git a/pkg/engine/secrets/inspector.go b/pkg/engine/secrets/inspector.go index fec2072d207..385c5d69329 100644 --- a/pkg/engine/secrets/inspector.go +++ b/pkg/engine/secrets/inspector.go @@ -156,7 +156,7 @@ func NewInspector( return nil, err } - allowRules, err := compileRegex(allRegexQueries.AllowRules) + allowRules, err := CompileRegex(allRegexQueries.AllowRules) if err != nil { return nil, err } @@ -282,7 +282,7 @@ func compileRegexQueries( return regexQueries, nil } -func compileRegex(allowRules []AllowRule) ([]AllowRule, error) { +func CompileRegex(allowRules []AllowRule) ([]AllowRule, error) { for j := range allowRules { compiledRegex, err := regexp.Compile(allowRules[j].RegexStr) if err != nil { @@ -307,7 +307,7 @@ func isValueInArray(value string, array []string) bool { } func (c *Inspector) isSecret(s string, query *RegexQuery) (isSecretRet bool, groups [][]string) { - if isAllowRule(s, query.AllowRules) || isAllowRule(s, c.allowRules) { + if IsAllowRule(s, query.AllowRules) || IsAllowRule(s, c.allowRules) { return false, [][]string{} } @@ -339,7 +339,7 @@ func (c *Inspector) isSecret(s string, query *RegexQuery) (isSecretRet bool, gro return false, [][]string{} } -func isAllowRule(s string, allowRules []AllowRule) bool { +func IsAllowRule(s string, allowRules []AllowRule) bool { for i := range allowRules { if allowRules[i].Regex.MatchString(s) { return true diff --git a/pkg/scan/post_scan.go b/pkg/scan/post_scan.go index 272bc66bd37..f58632c5668 100644 --- a/pkg/scan/post_scan.go +++ b/pkg/scan/post_scan.go @@ -104,6 +104,15 @@ func (c *Client) postScan(scanResults *Results) error { } } + // mask results preview if Secrets Scan is disabled + if c.ScanParams.DisableSecrets { + err := maskPreviewLines(c.ScanParams.SecretsRegexesPath, scanResults) + if err != nil { + log.Err(err) + return err + } + } + summary := c.getSummary(scanResults.Results, time.Now(), model.PathParameters{ ScannedPaths: c.ScanParams.Path, PathExtractionMap: scanResults.ExtractedPaths.ExtractionMap, diff --git a/pkg/scan/preview_secrets_mask.go b/pkg/scan/preview_secrets_mask.go new file mode 100644 index 00000000000..95d96d9fba9 --- /dev/null +++ b/pkg/scan/preview_secrets_mask.go @@ -0,0 +1,150 @@ +package scan + +import ( + "encoding/json" + "regexp" + "strings" + + "github.com/Checkmarx/kics/pkg/engine/secrets" + "github.com/Checkmarx/kics/pkg/model" +) + +func maskPreviewLines(secretsPath string, scanResults *Results) error { + secretsRegexRulesContent, err := getSecretsRegexRules(secretsPath) + if err != nil { + return err + } + + var allRegexQueries secrets.RegexRuleStruct + + err = json.Unmarshal([]byte(secretsRegexRulesContent), &allRegexQueries) + if err != nil { + return err + } + + allowRules, err := secrets.CompileRegex(allRegexQueries.AllowRules) + if err != nil { + return err + } + + rules, err := compileRegexQueries(allRegexQueries.Rules) + if err != nil { + return err + } + + for i := range scanResults.Results { + item := scanResults.Results[i] + hideSecret(item.VulnLines, &allowRules, &rules) + } + return nil +} + +func compileRegexQueries(allRegexQueries []secrets.RegexQuery) ([]secrets.RegexQuery, error) { + for i := range allRegexQueries { + compiledRegexp, err := regexp.Compile(allRegexQueries[i].RegexStr) + if err != nil { + return allRegexQueries, err + } + allRegexQueries[i].Regex = compiledRegexp + + for j := range allRegexQueries[i].AllowRules { + allRegexQueries[i].AllowRules[j].Regex = regexp.MustCompile(allRegexQueries[i].AllowRules[j].RegexStr) + } + } + return allRegexQueries, nil +} + +func hideSecret(lines *[]model.CodeLine, allowRules *[]secrets.AllowRule, rules *[]secrets.RegexQuery) { + for idx, line := range *lines { + for i := range *rules { + rule := (*rules)[i] + + isSecret, groups := isSecret(line.Line, &rule, allowRules) + // if not a secret skip to next line + if !isSecret { + continue + } + + if len(rule.Entropies) == 0 { + maskSecret(&rule, lines, idx) + } + + if len(groups[0]) > 0 { + for _, entropy := range rule.Entropies { + // if matched group does not exist continue + if len(groups[0]) <= entropy.Group { + return + } + isMatch, _ := secrets.CheckEntropyInterval( + entropy, + groups[0][entropy.Group], + ) + if isMatch { + maskSecret(&rule, lines, idx) + } + } + } + } + } +} + +func maskSecret(rule *secrets.RegexQuery, lines *[]model.CodeLine, idx int) { + if rule.SpecialMask == "all" { + (*lines)[idx].Line = "" + return + } + + regex := rule.RegexStr + line := (*lines)[idx] + + if len(rule.SpecialMask) > 0 { + regex = "(.+)" + rule.SpecialMask + } + + var re = regexp.MustCompile(regex) + match := re.FindString(line.Line) + + if len(rule.SpecialMask) > 0 { + match = line.Line[len(match):] + } + + if match != "" { + (*lines)[idx].Line = strings.Replace(line.Line, match, "", 1) + } else { + (*lines)[idx].Line = "" + } +} + +// repurposed isSecret from inspector +func isSecret(line string, rule *secrets.RegexQuery, allowRules *[]secrets.AllowRule) (isSecretRet bool, groups [][]string) { + if secrets.IsAllowRule(line, *allowRules) { + return false, [][]string{} + } + + groups = rule.Regex.FindAllStringSubmatch(line, -1) + + for _, group := range groups { + splitedText := strings.Split(line, "\n") + max := -1 + for i, splited := range splitedText { + if len(groups) < rule.Multiline.DetectLineGroup { + if strings.Contains(splited, group[rule.Multiline.DetectLineGroup]) && i > max { + max = i + } + } + } + if max == -1 { + continue + } + secret, newGroups := isSecret(strings.Join(append(splitedText[:max], splitedText[max+1:]...), "\n"), rule, allowRules) + if !secret { + continue + } + groups = append(groups, newGroups...) + } + + if len(groups) > 0 { + return true, groups + } + return false, [][]string{} +} diff --git a/pkg/scan/preview_secrets_mask_test.go b/pkg/scan/preview_secrets_mask_test.go new file mode 100644 index 00000000000..c39437d45b9 --- /dev/null +++ b/pkg/scan/preview_secrets_mask_test.go @@ -0,0 +1,151 @@ +package scan + +import ( + "github.com/Checkmarx/kics/internal/tracker" + "github.com/Checkmarx/kics/pkg/model" + "github.com/Checkmarx/kics/pkg/printer" + "github.com/Checkmarx/kics/pkg/progress" + "github.com/stretchr/testify/require" + "strings" + "testing" +) + +func Test_maskSecrets(t *testing.T) { + tests := []struct { + name string + filename string + scanParameters Parameters + tracker tracker.CITracker + scanResults *Results + }{ + { + name: "print with masked secrets", + filename: "results", + scanParameters: Parameters{ + DisableSecrets: true, + }, + tracker: tracker.CITracker{ + FoundFiles: 1, + FoundCountLines: 9, + ParsedCountLines: 9, + ParsedFiles: 1, + LoadedQueries: 146, + ExecutingQueries: 146, + ExecutedQueries: 146, + FailedSimilarityID: 0, + Version: model.Version{ + Latest: true, + LatestVersionTag: "Dev", + }, + }, + scanResults: &Results{ + Results: []model.Vulnerability{ + { + VulnLines: &[]model.CodeLine{ + { + Position: 4, + Line: " metadata:", + }, + { + Position: 5, + Line: " name: secret-basic-auth:", + }, { + Position: 6, + Line: " password: \"abcd\"", + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := Client{} + c.Tracker = &tt.tracker + c.ScanParams = &tt.scanParameters + c.ProBarBuilder = progress.InitializePbBuilder(true, false, true) + c.Printer = printer.NewPrinter(true) + + err := c.postScan(tt.scanResults) + require.NoError(t, err) + + for _, line := range (*tt.scanResults).Results { + for _, vulnLine := range *line.VulnLines { + if strings.Contains(vulnLine.Line, "password") { + require.Contains(t, vulnLine.Line, "") + } + } + } + + }) + + } +} + +func Test_maskSecretsEntropies(t *testing.T) { + tests := []struct { + name string + filename string + scanParameters Parameters + tracker tracker.CITracker + scanResults *Results + }{ + { + name: "not print with masked secrets with invalid entropies", + filename: "results", + scanParameters: Parameters{ + DisableSecrets: true, + }, + tracker: tracker.CITracker{ + FoundFiles: 1, + FoundCountLines: 9, + ParsedCountLines: 9, + ParsedFiles: 1, + LoadedQueries: 146, + ExecutingQueries: 146, + ExecutedQueries: 146, + FailedSimilarityID: 0, + Version: model.Version{ + Latest: true, + LatestVersionTag: "Dev", + }, + }, + scanResults: &Results{ + Results: []model.Vulnerability{ + { + VulnLines: &[]model.CodeLine{ + { + Position: 4, + Line: "secret = \"eeeeee\"", + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := Client{} + c.Tracker = &tt.tracker + c.ScanParams = &tt.scanParameters + c.ProBarBuilder = progress.InitializePbBuilder(true, false, true) + c.Printer = printer.NewPrinter(true) + + err := c.postScan(tt.scanResults) + require.NoError(t, err) + + for _, line := range (*tt.scanResults).Results { + for _, vulnLine := range *line.VulnLines { + require.NotContains(t, vulnLine.Line, "") + + } + } + + }) + + } +} From 27f3dcc4d568074f6d3c7955aafa704e88f4685f Mon Sep 17 00:00:00 2001 From: cxMiguelSilva Date: Thu, 4 May 2023 11:45:19 +0100 Subject: [PATCH 2/2] fix codacy --- pkg/engine/secrets/inspector.go | 2 ++ pkg/scan/preview_secrets_mask.go | 1 + 2 files changed, 3 insertions(+) diff --git a/pkg/engine/secrets/inspector.go b/pkg/engine/secrets/inspector.go index 385c5d69329..bc5b07996ca 100644 --- a/pkg/engine/secrets/inspector.go +++ b/pkg/engine/secrets/inspector.go @@ -282,6 +282,7 @@ func compileRegexQueries( return regexQueries, nil } +// CompileRegex compiles the regex allow rules func CompileRegex(allowRules []AllowRule) ([]AllowRule, error) { for j := range allowRules { compiledRegex, err := regexp.Compile(allowRules[j].RegexStr) @@ -339,6 +340,7 @@ func (c *Inspector) isSecret(s string, query *RegexQuery) (isSecretRet bool, gro return false, [][]string{} } +// IsAllowRule check if string matches any of the allow rules for the secret queries func IsAllowRule(s string, allowRules []AllowRule) bool { for i := range allowRules { if allowRules[i].Regex.MatchString(s) { diff --git a/pkg/scan/preview_secrets_mask.go b/pkg/scan/preview_secrets_mask.go index 95d96d9fba9..c192f503f2c 100644 --- a/pkg/scan/preview_secrets_mask.go +++ b/pkg/scan/preview_secrets_mask.go @@ -1,3 +1,4 @@ +// Package scan implements functions and helpers to ensure the proper scan of the specified files package scan import (