Skip to content

Commit

Permalink
convert Signed Releases to probes (#3610)
Browse files Browse the repository at this point in the history 
* convert Signed Releases to probes

Signed-off-by: AdamKorcz <[email protected]>

* Specify that probe is for Github and Gitlab only

Signed-off-by: AdamKorcz <[email protected]>

* use  in loop instead of

Signed-off-by: AdamKorcz <[email protected]>

* fix linter issues

Signed-off-by: AdamKorcz <[email protected]>

* fix more linter issues

Signed-off-by: AdamKorcz <[email protected]>

* specify Github and Gitlab in provenance def.yml

Signed-off-by: AdamKorcz <[email protected]>

* Add link to slsa-github-generator

Signed-off-by: AdamKorcz <[email protected]>

* Add instructions on signing with Cosign

Signed-off-by: AdamKorcz <[email protected]>

* refactor evaluation

Signed-off-by: Adam Korczynski <[email protected]>

* debug failing integration test

Signed-off-by: Adam Korczynski <[email protected]>

* remove unused nolints

Signed-off-by: Adam Korczynski <[email protected]>

* expose release name asset names in finding values

Signed-off-by: Adam Korczynski <[email protected]>

* fix failed integration test

Signed-off-by: Adam Korczynski <[email protected]>

* remove 'totalReleases' value from findings

Signed-off-by: Adam Korczynski <[email protected]>

* remove left-over cases of "totalReleases" values in findings

Signed-off-by: Adam Korczynski <[email protected]>

* remove remaining totalReleases values

Signed-off-by: Adam Korczynski <[email protected]>

* use const probe names instead of hard-coded strings

Signed-off-by: Adam Korczynski <[email protected]>

* remove totalReleases from test helper arguments

Signed-off-by: Adam Korczynski <[email protected]>

* merge test helpers

Signed-off-by: Adam Korczynski <[email protected]>

---------

Signed-off-by: AdamKorcz <[email protected]>
Signed-off-by: Adam Korczynski <[email protected]>
  • Loading branch information
@AdamKorcz
AdamKorcz authored Dec 13, 2023
1 parent d03c8cb commit 2c20be0
Show file tree
Hide file tree
Showing 11 changed files with 1,208 additions and 153 deletions.
191 changes: 105 additions & 86 deletions checks/evaluation/signed_releases.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,118 +17,137 @@ package evaluation
import (
"fmt"
"math"
"strings"

"github.com/ossf/scorecard/v4/checker"
sce "github.com/ossf/scorecard/v4/errors"
"github.com/ossf/scorecard/v4/finding"
"github.com/ossf/scorecard/v4/probes/releasesAreSigned"
"github.com/ossf/scorecard/v4/probes/releasesHaveProvenance"
)

var (
signatureExtensions = []string{".asc", ".minisig", ".sig", ".sign"}
provenanceExtensions = []string{".intoto.jsonl"}
)

const releaseLookBack = 5

// SignedReleases applies the score policy for the Signed-Releases check.
//
//nolint:gocognit
func SignedReleases(name string, dl checker.DetailLogger, r *checker.SignedReleasesData) checker.CheckResult {
if r == nil {
e := sce.WithMessage(sce.ErrScorecardInternal, "empty raw data")
func SignedReleases(name string,
findings []finding.Finding, dl checker.DetailLogger,
) checker.CheckResult {
expectedProbes := []string{
releasesAreSigned.Probe,
releasesHaveProvenance.Probe,
}

if !finding.UniqueProbesEqual(findings, expectedProbes) {
e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results")
return checker.CreateRuntimeErrorResult(name, e)
}

totalReleases := 0
total := 0
score := 0
for _, release := range r.Releases {
if len(release.Assets) == 0 {
continue
// Debug all releases and check for OutcomeNotApplicable
// All probes have OutcomeNotApplicable in case the project has no
// releases. Therefore, check for any finding with OutcomeNotApplicable.
loggedReleases := make([]string, 0)
for i := range findings {
f := &findings[i]

// Debug release name
releaseName, err := getReleaseName(f)
if err != nil {
e := sce.WithMessage(sce.ErrScorecardInternal, "could not get release name")
return checker.CreateRuntimeErrorResult(name, e)
}
if !contains(loggedReleases, releaseName) {
dl.Debug(&checker.LogMessage{
Text: fmt.Sprintf("GitHub release found: %s", releaseName),
})
loggedReleases = append(loggedReleases, releaseName)
}

dl.Debug(&checker.LogMessage{
Text: fmt.Sprintf("GitHub release found: %s", release.TagName),
})

totalReleases++
signed := false
hasProvenance := false

// Check for provenance.
for _, asset := range release.Assets {
for _, suffix := range provenanceExtensions {
if strings.HasSuffix(asset.Name, suffix) {
dl.Info(&checker.LogMessage{
Path: asset.URL,
Type: finding.FileTypeURL,
Text: fmt.Sprintf("provenance for release artifact: %s", asset.Name),
})
hasProvenance = true
total++
break
}
}
if hasProvenance {
// Assign maximum points.
score += 10
break
}
// Check if outcome is NotApplicable
if f.Outcome == finding.OutcomeNotApplicable {
dl.Warn(&checker.LogMessage{
Text: "no GitHub releases found",
})
// Generic summary.
return checker.CreateInconclusiveResult(name, "no releases found")
}
}

totalPositive := 0
releaseMap := make(map[string]int)
uniqueReleaseTags := make([]string, 0)
checker.LogFindings(findings, dl)

for i := range findings {
f := &findings[i]

releaseName, err := getReleaseName(f)
if err != nil {
return checker.CreateRuntimeErrorResult(name, err)
}

if hasProvenance {
continue
if !contains(uniqueReleaseTags, releaseName) {
uniqueReleaseTags = append(uniqueReleaseTags, releaseName)
}

dl.Warn(&checker.LogMessage{
Path: release.URL,
Type: finding.FileTypeURL,
Text: fmt.Sprintf("release artifact %s does not have provenance", release.TagName),
})

// No provenance. Try signatures.
for _, asset := range release.Assets {
for _, suffix := range signatureExtensions {
if strings.HasSuffix(asset.Name, suffix) {
dl.Info(&checker.LogMessage{
Path: asset.URL,
Type: finding.FileTypeURL,
Text: fmt.Sprintf("signed release artifact: %s", asset.Name),
})
signed = true
total++
break
if f.Outcome == finding.OutcomePositive {
totalPositive++

switch f.Probe {
case releasesAreSigned.Probe:
if _, ok := releaseMap[releaseName]; !ok {
releaseMap[releaseName] = 8
}
}
if signed {
// Assign 8 points.
score += 8
break
case releasesHaveProvenance.Probe:
releaseMap[releaseName] = 10
}
}
}

if !signed {
dl.Warn(&checker.LogMessage{
Path: release.URL,
Type: finding.FileTypeURL,
Text: fmt.Sprintf("release artifact %s not signed", release.TagName),
})
}
if totalReleases >= releaseLookBack {
break
}
if totalPositive == 0 {
return checker.CreateMinScoreResult(name, "Project has not signed or included provenance with any releases.")
}

totalReleases := len(uniqueReleaseTags)

if totalReleases > 5 {
totalReleases = 5
}
if totalReleases == 0 {
dl.Warn(&checker.LogMessage{
Text: "no GitHub releases found",
})
// Generic summary.
// This should not happen in production, but it is useful to have
// for testing.
return checker.CreateInconclusiveResult(name, "no releases found")
}

score := 0
for _, s := range releaseMap {
score += s
}

score = int(math.Floor(float64(score) / float64(totalReleases)))
reason := fmt.Sprintf("%d out of %d artifacts are signed or have provenance", total, totalReleases)
reason := fmt.Sprintf("%d out of the last %d releases have a total of %d signed artifacts.",
len(releaseMap), totalReleases, totalPositive)
return checker.CreateResultWithScore(name, reason, score)
}

func getReleaseName(f *finding.Finding) (string, error) {
m := f.Values
for k, v := range m {
var value int
switch f.Probe {
case releasesAreSigned.Probe:
value = int(releasesAreSigned.ValueTypeRelease)
case releasesHaveProvenance.Probe:
value = int(releasesHaveProvenance.ValueTypeRelease)
}
if v == value {
return k, nil
}
}
return "", sce.WithMessage(sce.ErrScorecardInternal, "could not get release tag")
}

func contains(releases []string, release string) bool {
for _, r := range releases {
if r == release {
return true
}
}
return false
}
Loading

0 comments on commit 2c20be0

Please sign in to comment.