diff --git a/detect/detect.go b/detect/detect.go index 84b2332b5..aad771c10 100644 --- a/detect/detect.go +++ b/detect/detect.go @@ -17,6 +17,8 @@ type Options struct { Redact bool } +const MAXGOROUTINES = 4 + func DetectFindings(cfg config.Config, b []byte, filePath string, commit string) []report.Finding { var findings []report.Finding linePairs := regexp.MustCompile("\n").FindAllIndex(b, -1) diff --git a/detect/files.go b/detect/files.go index b449a9c99..cd51c2439 100644 --- a/detect/files.go +++ b/detect/files.go @@ -20,7 +20,7 @@ func FromFiles(source string, cfg config.Config, outputOptions Options) ([]repor findings []report.Finding mu sync.Mutex ) - concurrentGoroutines := make(chan struct{}, 4) + concurrentGoroutines := make(chan struct{}, MAXGOROUTINES) g, _ := errgroup.WithContext(context.Background()) paths := make(chan string) g.Go(func() error { @@ -41,16 +41,17 @@ func FromFiles(source string, cfg config.Config, outputOptions Options) ([]repor }) for pa := range paths { p := pa + concurrentGoroutines <- struct{}{} g.Go(func() error { - concurrentGoroutines <- struct{}{} + defer func() { + <-concurrentGoroutines + }() b, err := os.ReadFile(p) if err != nil { - <-concurrentGoroutines return err } if !godocutil.IsText(b) { - <-concurrentGoroutines return nil } fis := DetectFindings(cfg, b, p, "") @@ -69,7 +70,6 @@ func FromFiles(source string, cfg config.Config, outputOptions Options) ([]repor findings = append(findings, fi) mu.Unlock() } - <-concurrentGoroutines return nil }) } diff --git a/detect/git.go b/detect/git.go index b06f4970b..754e8014c 100644 --- a/detect/git.go +++ b/detect/git.go @@ -19,16 +19,20 @@ func FromGit(files <-chan *gitdiff.File, cfg config.Config, outputOptions Option var findings []report.Finding mu := sync.Mutex{} wg := sync.WaitGroup{} + concurrentGoroutines := make(chan struct{}, MAXGOROUTINES) commitMap := make(map[string]bool) for f := range files { // keep track of commits for logging if f.PatchHeader != nil { commitMap[f.PatchHeader.SHA] = true } - wg.Add(1) + concurrentGoroutines <- struct{}{} go func(f *gitdiff.File) { - defer wg.Done() + defer func() { + wg.Done() + <-concurrentGoroutines + }() if f.IsBinary { return }