diff --git a/checks/evaluation/signed_releases.go b/checks/evaluation/signed_releases.go index 5feb11b680f..9f3673ffe21 100644 --- a/checks/evaluation/signed_releases.go +++ b/checks/evaluation/signed_releases.go @@ -17,118 +17,137 @@ package evaluation import ( "fmt" "math" - "strings" "github.com/ossf/scorecard/v4/checker" sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/releasesAreSigned" + "github.com/ossf/scorecard/v4/probes/releasesHaveProvenance" ) -var ( - signatureExtensions = []string{".asc", ".minisig", ".sig", ".sign"} - provenanceExtensions = []string{".intoto.jsonl"} -) - -const releaseLookBack = 5 - // SignedReleases applies the score policy for the Signed-Releases check. -// -//nolint:gocognit -func SignedReleases(name string, dl checker.DetailLogger, r *checker.SignedReleasesData) checker.CheckResult { - if r == nil { - e := sce.WithMessage(sce.ErrScorecardInternal, "empty raw data") +func SignedReleases(name string, + findings []finding.Finding, dl checker.DetailLogger, +) checker.CheckResult { + expectedProbes := []string{ + releasesAreSigned.Probe, + releasesHaveProvenance.Probe, + } + + if !finding.UniqueProbesEqual(findings, expectedProbes) { + e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results") return checker.CreateRuntimeErrorResult(name, e) } - totalReleases := 0 - total := 0 - score := 0 - for _, release := range r.Releases { - if len(release.Assets) == 0 { - continue + // Debug all releases and check for OutcomeNotApplicable + // All probes have OutcomeNotApplicable in case the project has no + // releases. Therefore, check for any finding with OutcomeNotApplicable. + loggedReleases := make([]string, 0) + for i := range findings { + f := &findings[i] + + // Debug release name + releaseName, err := getReleaseName(f) + if err != nil { + e := sce.WithMessage(sce.ErrScorecardInternal, "could not get release name") + return checker.CreateRuntimeErrorResult(name, e) + } + if !contains(loggedReleases, releaseName) { + dl.Debug(&checker.LogMessage{ + Text: fmt.Sprintf("GitHub release found: %s", releaseName), + }) + loggedReleases = append(loggedReleases, releaseName) } - dl.Debug(&checker.LogMessage{ - Text: fmt.Sprintf("GitHub release found: %s", release.TagName), - }) - - totalReleases++ - signed := false - hasProvenance := false - - // Check for provenance. - for _, asset := range release.Assets { - for _, suffix := range provenanceExtensions { - if strings.HasSuffix(asset.Name, suffix) { - dl.Info(&checker.LogMessage{ - Path: asset.URL, - Type: finding.FileTypeURL, - Text: fmt.Sprintf("provenance for release artifact: %s", asset.Name), - }) - hasProvenance = true - total++ - break - } - } - if hasProvenance { - // Assign maximum points. - score += 10 - break - } + // Check if outcome is NotApplicable + if f.Outcome == finding.OutcomeNotApplicable { + dl.Warn(&checker.LogMessage{ + Text: "no GitHub releases found", + }) + // Generic summary. + return checker.CreateInconclusiveResult(name, "no releases found") + } + } + + totalPositive := 0 + releaseMap := make(map[string]int) + uniqueReleaseTags := make([]string, 0) + checker.LogFindings(findings, dl) + + for i := range findings { + f := &findings[i] + + releaseName, err := getReleaseName(f) + if err != nil { + return checker.CreateRuntimeErrorResult(name, err) } - if hasProvenance { - continue + if !contains(uniqueReleaseTags, releaseName) { + uniqueReleaseTags = append(uniqueReleaseTags, releaseName) } - dl.Warn(&checker.LogMessage{ - Path: release.URL, - Type: finding.FileTypeURL, - Text: fmt.Sprintf("release artifact %s does not have provenance", release.TagName), - }) - - // No provenance. Try signatures. - for _, asset := range release.Assets { - for _, suffix := range signatureExtensions { - if strings.HasSuffix(asset.Name, suffix) { - dl.Info(&checker.LogMessage{ - Path: asset.URL, - Type: finding.FileTypeURL, - Text: fmt.Sprintf("signed release artifact: %s", asset.Name), - }) - signed = true - total++ - break + if f.Outcome == finding.OutcomePositive { + totalPositive++ + + switch f.Probe { + case releasesAreSigned.Probe: + if _, ok := releaseMap[releaseName]; !ok { + releaseMap[releaseName] = 8 } - } - if signed { - // Assign 8 points. - score += 8 - break + case releasesHaveProvenance.Probe: + releaseMap[releaseName] = 10 } } + } - if !signed { - dl.Warn(&checker.LogMessage{ - Path: release.URL, - Type: finding.FileTypeURL, - Text: fmt.Sprintf("release artifact %s not signed", release.TagName), - }) - } - if totalReleases >= releaseLookBack { - break - } + if totalPositive == 0 { + return checker.CreateMinScoreResult(name, "Project has not signed or included provenance with any releases.") } + totalReleases := len(uniqueReleaseTags) + + if totalReleases > 5 { + totalReleases = 5 + } if totalReleases == 0 { - dl.Warn(&checker.LogMessage{ - Text: "no GitHub releases found", - }) - // Generic summary. + // This should not happen in production, but it is useful to have + // for testing. return checker.CreateInconclusiveResult(name, "no releases found") } + score := 0 + for _, s := range releaseMap { + score += s + } + score = int(math.Floor(float64(score) / float64(totalReleases))) - reason := fmt.Sprintf("%d out of %d artifacts are signed or have provenance", total, totalReleases) + reason := fmt.Sprintf("%d out of the last %d releases have a total of %d signed artifacts.", + len(releaseMap), totalReleases, totalPositive) return checker.CreateResultWithScore(name, reason, score) } + +func getReleaseName(f *finding.Finding) (string, error) { + m := f.Values + for k, v := range m { + var value int + switch f.Probe { + case releasesAreSigned.Probe: + value = int(releasesAreSigned.ValueTypeRelease) + case releasesHaveProvenance.Probe: + value = int(releasesHaveProvenance.ValueTypeRelease) + } + if v == value { + return k, nil + } + } + return "", sce.WithMessage(sce.ErrScorecardInternal, "could not get release tag") +} + +func contains(releases []string, release string) bool { + for _, r := range releases { + if r == release { + return true + } + } + return false +} diff --git a/checks/evaluation/signed_releases_test.go b/checks/evaluation/signed_releases_test.go index 53a8ee362e2..8aa8c8fe5f4 100644 --- a/checks/evaluation/signed_releases_test.go +++ b/checks/evaluation/signed_releases_test.go @@ -15,96 +15,269 @@ package evaluation import ( + "fmt" "testing" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/ossf/scorecard/v4/checker" - "github.com/ossf/scorecard/v4/clients" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/releasesAreSigned" + "github.com/ossf/scorecard/v4/probes/releasesHaveProvenance" scut "github.com/ossf/scorecard/v4/utests" ) +const ( + release0 = 0 + release1 = 1 + release2 = 2 + release3 = 3 + release4 = 4 +) + +const ( + asset0 = 0 + asset1 = 1 + asset2 = 2 + asset3 = 3 + asset4 = 4 + asset5 = 5 + asset6 = 6 + asset7 = 7 + asset8 = 8 + asset9 = 9 +) + +func signedProbe(release, asset int, outcome finding.Outcome) finding.Finding { + return finding.Finding{ + Probe: releasesAreSigned.Probe, + Outcome: outcome, + Values: map[string]int{ + fmt.Sprintf("v%d", release): int(releasesAreSigned.ValueTypeRelease), + fmt.Sprintf("artifact-%d", asset): int(releasesAreSigned.ValueTypeReleaseAsset), + }, + } +} + +func provenanceProbe(release, asset int, outcome finding.Outcome) finding.Finding { + return finding.Finding{ + Probe: releasesHaveProvenance.Probe, + Outcome: outcome, + Values: map[string]int{ + fmt.Sprintf("v%d", release): int(releasesHaveProvenance.ValueTypeRelease), + fmt.Sprintf("artifact-%d", asset): int(releasesHaveProvenance.ValueTypeReleaseAsset), + }, + } +} + func TestSignedReleases(t *testing.T) { t.Parallel() tests := []struct { - name string - releases []clients.Release - expectedResult checker.CheckResult + name string + findings []finding.Finding + result scut.TestReturn }{ { - name: "Full score", - releases: []clients.Release{ - { - TagName: "v1.0", - Assets: []clients.ReleaseAsset{ - {Name: "binary.tar.gz"}, - {Name: "binary.tar.gz.sig"}, - {Name: "binary.tar.gz.intoto.jsonl"}, - }, - }, + name: "Has one release that is signed but no provenance", + findings: []finding.Finding{ + signedProbe(0, 0, finding.OutcomePositive), + provenanceProbe(0, 0, finding.OutcomeNegative), }, - expectedResult: checker.CheckResult{ - Name: "Signed-Releases", - Version: 2, - Score: 10, - Reason: "1 out of 1 artifacts are signed or have provenance", + result: scut.TestReturn{ + Score: 8, + NumberOfInfo: 1, + NumberOfWarn: 1, + NumberOfDebug: 1, }, }, { - name: "Partial score", - releases: []clients.Release{ - { - TagName: "v1.0", - Assets: []clients.ReleaseAsset{ - {Name: "binary.tar.gz"}, - {Name: "binary.tar.gz.sig"}, - }, - }, + name: "Has one release that is signed and has provenance", + findings: []finding.Finding{ + signedProbe(0, 0, finding.OutcomePositive), + provenanceProbe(0, 0, finding.OutcomePositive), }, - expectedResult: checker.CheckResult{ - Name: "Signed-Releases", - Version: 2, - Score: 8, - Reason: "1 out of 1 artifacts are signed or have provenance", + result: scut.TestReturn{ + Score: 10, + NumberOfInfo: 2, + NumberOfDebug: 1, }, }, { - name: "No score", - releases: []clients.Release{ - { - TagName: "v1.0", - Assets: []clients.ReleaseAsset{ - {Name: "binary.tar.gz"}, - }, - }, + name: "Has one release that is not signed but has provenance", + findings: []finding.Finding{ + signedProbe(0, 0, finding.OutcomeNegative), + provenanceProbe(0, 0, finding.OutcomePositive), }, - expectedResult: checker.CheckResult{ - Name: "Signed-Releases", - Version: 2, - Score: 0, - Reason: "0 out of 1 artifacts are signed or have provenance", + result: scut.TestReturn{ + Score: checker.MaxResultScore, + NumberOfInfo: 1, + NumberOfWarn: 1, + NumberOfDebug: 1, + }, + }, + + { + name: "3 releases. One release has one signed, and one release has two provenance.", + findings: []finding.Finding{ + // Release 1: + // Asset 1: + signedProbe(release0, asset0, finding.OutcomeNegative), + provenanceProbe(release0, asset0, finding.OutcomeNegative), + // Asset 2: + signedProbe(release0, asset1, finding.OutcomePositive), + provenanceProbe(release0, asset1, finding.OutcomeNegative), + // Release 2 + // Asset 1: + signedProbe(release1, asset0, finding.OutcomeNegative), + provenanceProbe(release1, asset0, finding.OutcomeNegative), + // Release 2 + // Asset 2: + signedProbe(release1, asset1, finding.OutcomeNegative), + provenanceProbe(release1, asset1, finding.OutcomeNegative), + // Release 2 + // Asset 3: + signedProbe(release1, asset2, finding.OutcomeNegative), + provenanceProbe(release1, asset2, finding.OutcomeNegative), + // Release 3 + // Asset 1: + signedProbe(release2, asset0, finding.OutcomeNegative), + provenanceProbe(release2, asset0, finding.OutcomePositive), + // Asset 2: + signedProbe(release2, asset1, finding.OutcomeNegative), + provenanceProbe(release2, asset1, finding.OutcomePositive), + // Asset 3: + signedProbe(release2, asset2, finding.OutcomeNegative), + provenanceProbe(release2, asset2, finding.OutcomeNegative), + }, + result: scut.TestReturn{ + Score: 6, + NumberOfInfo: 3, + NumberOfWarn: 13, + NumberOfDebug: 3, + }, + }, + { + name: "5 releases. Two releases have one signed each, and two releases have one provenance each.", + findings: []finding.Finding{ + // Release 1: + // Release 1, Asset 1: + signedProbe(release0, asset0, finding.OutcomeNegative), + provenanceProbe(release0, asset0, finding.OutcomeNegative), + signedProbe(release0, asset1, finding.OutcomePositive), + provenanceProbe(release0, asset1, finding.OutcomeNegative), + // Release 2: + // Release 2, Asset 1: + signedProbe(release1, asset1, finding.OutcomePositive), + provenanceProbe(release1, asset0, finding.OutcomeNegative), + // Release 2, Asset 2: + signedProbe(release1, asset1, finding.OutcomeNegative), + provenanceProbe(release1, asset1, finding.OutcomeNegative), + // Release 2, Asset 3: + signedProbe(release1, asset2, finding.OutcomeNegative), + provenanceProbe(release1, asset2, finding.OutcomeNegative), + // Release 3, Asset 1: + signedProbe(release2, asset0, finding.OutcomeNegative), + provenanceProbe(release2, asset0, finding.OutcomePositive), + // Release 3, Asset 2: + signedProbe(release2, asset1, finding.OutcomeNegative), + provenanceProbe(release2, asset1, finding.OutcomeNegative), + // Release 3, Asset 3: + signedProbe(release2, asset2, finding.OutcomeNegative), + provenanceProbe(release2, asset2, finding.OutcomeNegative), + // Release 4, Asset 1: + signedProbe(release3, asset0, finding.OutcomeNegative), + provenanceProbe(release3, asset0, finding.OutcomePositive), + // Release 4, Asset 2: + signedProbe(release3, asset1, finding.OutcomeNegative), + provenanceProbe(release3, asset1, finding.OutcomeNegative), + // Release 4, Asset 3: + signedProbe(release3, asset2, finding.OutcomeNegative), + provenanceProbe(release3, asset2, finding.OutcomeNegative), + // Release 5, Asset 1: + signedProbe(release4, asset0, finding.OutcomeNegative), + provenanceProbe(release4, asset0, finding.OutcomeNegative), + // Release 5, Asset 2: + signedProbe(release4, asset1, finding.OutcomeNegative), + provenanceProbe(release4, asset1, finding.OutcomeNegative), + // Release 5, Asset 3: + signedProbe(release4, asset2, finding.OutcomeNegative), + provenanceProbe(release4, asset2, finding.OutcomeNegative), + // Release 5, Asset 4: + signedProbe(release4, asset3, finding.OutcomeNegative), + provenanceProbe(release4, asset3, finding.OutcomeNegative), + }, + result: scut.TestReturn{ + Score: 7, + NumberOfInfo: 4, + NumberOfWarn: 26, + NumberOfDebug: 5, }, }, { - name: "No releases", - releases: []clients.Release{}, - expectedResult: checker.CreateInconclusiveResult("Signed-Releases", "no releases found"), + name: "5 releases. All have one signed artifact.", + findings: []finding.Finding{ + // Release 1: + // Release 1, Asset 1: + signedProbe(release0, asset0, finding.OutcomeNegative), + provenanceProbe(release0, asset0, finding.OutcomeNegative), + signedProbe(release0, asset1, finding.OutcomePositive), + provenanceProbe(release0, asset1, finding.OutcomeNegative), + // Release 2: + // Release 2, Asset 1: + signedProbe(release1, asset0, finding.OutcomePositive), + provenanceProbe(release1, asset0, finding.OutcomeNegative), + // Release 2, Asset 2: + signedProbe(release1, asset1, finding.OutcomeNegative), + provenanceProbe(release1, asset1, finding.OutcomeNegative), + // Release 2, Asset 3: + signedProbe(release1, asset2, finding.OutcomeNegative), + provenanceProbe(release1, asset2, finding.OutcomeNegative), + // Release 3, Asset 1: + signedProbe(release2, asset0, finding.OutcomePositive), + provenanceProbe(release2, asset0, finding.OutcomePositive), + // Release 3, Asset 2: + signedProbe(release2, asset1, finding.OutcomeNegative), + provenanceProbe(release2, asset1, finding.OutcomeNegative), + // Release 3, Asset 3: + signedProbe(release2, asset2, finding.OutcomeNegative), + provenanceProbe(release2, asset2, finding.OutcomeNegative), + // Release 4, Asset 1: + signedProbe(release3, asset0, finding.OutcomePositive), + provenanceProbe(release3, asset0, finding.OutcomePositive), + // Release 4, Asset 2: + signedProbe(release3, asset1, finding.OutcomeNegative), + provenanceProbe(release3, asset1, finding.OutcomeNegative), + // Release 4, Asset 3: + signedProbe(release3, asset2, finding.OutcomeNegative), + provenanceProbe(release3, asset2, finding.OutcomeNegative), + // Release 5, Asset 1: + signedProbe(release4, asset0, finding.OutcomePositive), + provenanceProbe(release4, asset0, finding.OutcomeNegative), + // Release 5, Asset 2: + signedProbe(release4, asset1, finding.OutcomeNegative), + provenanceProbe(release4, asset1, finding.OutcomeNegative), + // Release 5, Asset 3: + signedProbe(release4, asset2, finding.OutcomeNegative), + provenanceProbe(release4, asset2, finding.OutcomeNegative), + // Release 5, Asset 4: + signedProbe(release4, asset3, finding.OutcomeNegative), + provenanceProbe(release4, asset3, finding.OutcomeNegative), + }, + result: scut.TestReturn{ + Score: 8, + NumberOfInfo: 7, + NumberOfWarn: 23, + NumberOfDebug: 5, + }, }, } - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { t.Parallel() - dl := &scut.TestDetailLogger{} - data := &checker.SignedReleasesData{Releases: tc.releases} - actualResult := SignedReleases("Signed-Releases", dl, data) - - if !cmp.Equal(tc.expectedResult, actualResult, - cmpopts.IgnoreFields(checker.CheckResult{}, "Error")) { - t.Errorf("SignedReleases() mismatch (-want +got):\n%s", cmp.Diff(tc.expectedResult, actualResult, - cmpopts.IgnoreFields(checker.CheckResult{}, "Error"))) + dl := scut.TestDetailLogger{} + got := SignedReleases(tt.name, tt.findings, &dl) + if !scut.ValidateTestReturn(t, tt.name, &tt.result, &got, &dl) { + t.Errorf("got %v, expected %v", got, tt.result) } }) } diff --git a/checks/signed_releases.go b/checks/signed_releases.go index be2dfe98a61..522672431f7 100644 --- a/checks/signed_releases.go +++ b/checks/signed_releases.go @@ -19,6 +19,8 @@ import ( "github.com/ossf/scorecard/v4/checks/evaluation" "github.com/ossf/scorecard/v4/checks/raw" sce "github.com/ossf/scorecard/v4/errors" + "github.com/ossf/scorecard/v4/probes" + "github.com/ossf/scorecard/v4/probes/zrunner" ) // CheckSignedReleases is the registered name for SignedReleases. @@ -40,11 +42,17 @@ func SignedReleases(c *checker.CheckRequest) checker.CheckResult { return checker.CreateRuntimeErrorResult(CheckSignedReleases, e) } - // Return raw results. - if c.RawResults != nil { - c.RawResults.SignedReleasesResults = rawData + // Set the raw results. + pRawResults := getRawResults(c) + pRawResults.SignedReleasesResults = rawData + + // Evaluate the probes. + findings, err := zrunner.Run(pRawResults, probes.SignedReleases) + if err != nil { + e := sce.WithMessage(sce.ErrScorecardInternal, err.Error()) + return checker.CreateRuntimeErrorResult(CheckFuzzing, e) } // Return the score evaluation. - return evaluation.SignedReleases(CheckSignedReleases, c.Dlogger, &rawData) + return evaluation.SignedReleases(CheckSignedReleases, findings, c.Dlogger) } diff --git a/checks/signed_releases_test.go b/checks/signed_releases_test.go index 39bdc017fb7..b12268f3155 100644 --- a/checks/signed_releases_test.go +++ b/checks/signed_releases_test.go @@ -16,6 +16,7 @@ package checks import ( "errors" + "fmt" "testing" "github.com/golang/mock/gomock" @@ -362,6 +363,42 @@ func TestSignedRelease(t *testing.T) { }, }, + { + name: "Error getting releases", + err: errors.New("Error getting releases"), + expected: checker.CheckResult{ + Score: -1, + Error: errors.New("Error getting releases"), + }, + }, + { + name: "9 Releases with assests with signed artifacts", + releases: []clients.Release{ + release("v0.8.5"), + release("v0.8.4"), + release("v0.8.3"), + release("v0.8.2"), + release("v0.8.1"), + release("v0.8.0"), + release("v0.7.0"), + release("v0.6.0"), + release("v0.5.0"), + release("v0.4.0"), + release("v0.3.0"), + release("v0.2.0"), + release("v0.1.0"), + release("v0.0.6"), + release("v0.0.5"), + release("v0.0.4"), + release("v0.0.3"), + release("v0.0.2"), + release("v0.0.1"), + }, + expected: checker.CheckResult{ + Score: 8, + }, + }, + { name: "Error getting releases", err: errors.New("Error getting releases"), @@ -410,3 +447,45 @@ func TestSignedRelease(t *testing.T) { }) } } + +func release(version string) clients.Release { + return clients.Release{ + TagName: version, + URL: fmt.Sprintf("https://github.com/test/test_artifact/releases/tag/%s", version), + TargetCommitish: "master", + Assets: []clients.ReleaseAsset{ + { + Name: fmt.Sprintf("%s_checksums.txt", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_checksums.txt", version, version), + }, + { + Name: fmt.Sprintf("%s_checksums.txt.sig", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_checksums.txt.sig", version, version), + }, + { + Name: fmt.Sprintf("%s_darwin_x86_64.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_darwin_x86_64.tar.gz", version, version), + }, + { + Name: fmt.Sprintf("%s_Linux_arm64.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_Linux_arm64.tar.gz", version, version), + }, + { + Name: fmt.Sprintf("%s_Linux_i386.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_Linux_i386.tar.gz", version, version), + }, + { + Name: fmt.Sprintf("%s_Linux_x86_64.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_Linux_x86_64.tar.gz", version, version), + }, + { + Name: fmt.Sprintf("%s_windows_i386.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_windows_i386.tar.gz", version, version), + }, + { + Name: fmt.Sprintf("%s_windows_x86_64.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_windows_x86_64.tar.gz", version, version), + }, + }, + } +} diff --git a/probes/entries.go b/probes/entries.go index 6414d224e21..3ae6dcae2ad 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -45,6 +45,8 @@ import ( "github.com/ossf/scorecard/v4/probes/notArchived" "github.com/ossf/scorecard/v4/probes/notCreatedRecently" "github.com/ossf/scorecard/v4/probes/packagedWithAutomatedWorkflow" + "github.com/ossf/scorecard/v4/probes/releasesAreSigned" + "github.com/ossf/scorecard/v4/probes/releasesHaveProvenance" "github.com/ossf/scorecard/v4/probes/sastToolCodeQLInstalled" "github.com/ossf/scorecard/v4/probes/sastToolRunsOnAllCommits" "github.com/ossf/scorecard/v4/probes/sastToolSonarInstalled" @@ -117,6 +119,7 @@ var ( hasDangerousWorkflowScriptInjection.Run, hasDangerousWorkflowUntrustedCheckout.Run, } + Maintained = []ProbeImpl{ notArchived.Run, hasRecentCommits.Run, @@ -135,6 +138,10 @@ var ( CITests = []ProbeImpl{ testsRunInCI.Run, } + SignedReleases = []ProbeImpl{ + releasesAreSigned.Run, + releasesHaveProvenance.Run, + } probeRunners = map[string]func(*checker.RawResults) ([]finding.Finding, string, error){ securityPolicyPresent.Probe: securityPolicyPresent.Run, diff --git a/probes/releasesAreSigned/def.yml b/probes/releasesAreSigned/def.yml new file mode 100644 index 00000000000..5c77599a067 --- /dev/null +++ b/probes/releasesAreSigned/def.yml @@ -0,0 +1,30 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: releasesAreSigned +short: Check that the projects Github and Gitlab releases are signed. +motivation: > + Signed releases allow consumers to verify their artifacts before consuming them. +implementation: > + The implementation checks whether a signature file is present in release assets. The probe checks the last 5 releases on Github and Gitlab. +outcome: + - For each of the last 5 releases, the probe returns OutcomePositive, if the release has a signature file in the release assets. + - For each of the last 5 releases, the probe returns OutcomeNegative, if the the release does not have a signature file in the release assets. + - If the project has no releases, the probe returns OutcomeNotApplicable. +remediation: + effort: Medium + text: + - Install Cosign by following https://docs.sigstore.dev/system_config/installation + - Sign your release artifacts using `cosign sign $YOUR_ARTIFACT`. See more at https://docs.sigstore.dev/signing/quickstart + - Publish your release and add the certificate and signature produced by Cosign. diff --git a/probes/releasesAreSigned/impl.go b/probes/releasesAreSigned/impl.go new file mode 100644 index 00000000000..735cc141df3 --- /dev/null +++ b/probes/releasesAreSigned/impl.go @@ -0,0 +1,133 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package releasesAreSigned + +import ( + "embed" + "fmt" + "strings" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const ( + Probe = "releasesAreSigned" + releaseLookBack = 5 +) + +// ValueType is the type of a value in the finding values. +type ValueType int + +const ( + // ValueTypeRelease represents a release name. + ValueTypeRelease ValueType = iota + // ValueTypeReleaseAsset represents a release asset name. + ValueTypeReleaseAsset +) + +var signatureExtensions = []string{".asc", ".minisig", ".sig", ".sign"} + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + + var findings []finding.Finding + + releases := raw.SignedReleasesResults.Releases + + totalReleases := 0 + for releaseIndex, release := range releases { + if len(release.Assets) == 0 { + continue + } + + if releaseIndex == releaseLookBack { + break + } + + totalReleases++ + signed := false + for j := range release.Assets { + asset := release.Assets[j] + for _, suffix := range signatureExtensions { + if !strings.HasSuffix(asset.Name, suffix) { + continue + } + // Create Positive Finding + // with file info + loc := &finding.Location{ + Type: finding.FileTypeURL, + Path: asset.URL, + } + f, err := finding.NewWith(fs, Probe, + fmt.Sprintf("signed release artifact: %s", asset.Name), + loc, + finding.OutcomePositive) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + f.Values = map[string]int{ + release.TagName: int(ValueTypeRelease), + asset.Name: int(ValueTypeReleaseAsset), + } + findings = append(findings, *f) + signed = true + break + } + if signed { + break + } + } + if signed { + continue + } + + // Release is not signed + loc := &finding.Location{ + Type: finding.FileTypeURL, + Path: release.URL, + } + f, err := finding.NewWith(fs, Probe, + fmt.Sprintf("release artifact %s not signed", release.TagName), + loc, + finding.OutcomeNegative) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + f.Values = map[string]int{ + release.TagName: int(ValueTypeRelease), + } + findings = append(findings, *f) + } + + if len(findings) == 0 { + f, err := finding.NewWith(fs, Probe, + "no GitHub/GitLab releases found", + nil, + finding.OutcomeNotApplicable) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + return findings, Probe, nil +} diff --git a/probes/releasesAreSigned/impl_test.go b/probes/releasesAreSigned/impl_test.go new file mode 100644 index 00000000000..71b8e4444a0 --- /dev/null +++ b/probes/releasesAreSigned/impl_test.go @@ -0,0 +1,261 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package releasesAreSigned + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/clients" + "github.com/ossf/scorecard/v4/finding" +) + +func Test_Run(t *testing.T) { + t.Parallel() + //nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "Has one signed release.", + raw: &checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Releases: []clients.Release{ + { + TagName: "v1.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.sig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "Has two signed releases.", + raw: &checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Releases: []clients.Release{ + { + TagName: "v1.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.sig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + { + TagName: "v2.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.sig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "Has two unsigned releases.", + raw: &checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Releases: []clients.Release{ + { + TagName: "v1.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.notSig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + { + TagName: "v2.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.notSig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + finding.OutcomeNegative, + }, + }, + { + name: "Has two unsigned releases and one signed release.", + raw: &checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Releases: []clients.Release{ + { + TagName: "v1.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.notSig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + { + TagName: "v2.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.sig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + + { + TagName: "v3.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.notSig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + finding.OutcomePositive, + finding.OutcomeNegative, + }, + }, + { + name: "Many releases.", + raw: &checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Releases: []clients.Release{ + release("v0.8.5"), + release("v0.8.4"), + release("v0.8.3"), + release("v0.8.2"), + release("v0.8.1"), + release("v0.8.0"), + release("v0.7.0"), + release("v0.6.0"), + release("v0.5.0"), + release("v0.4.0"), + release("v0.3.0"), + release("v0.2.0"), + release("v0.1.0"), + release("v0.0.6"), + release("v0.0.5"), + release("v0.0.4"), + release("v0.0.3"), + release("v0.0.2"), + release("v0.0.1"), + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + finding.OutcomePositive, + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} + +func release(version string) clients.Release { + return clients.Release{ + TagName: version, + URL: fmt.Sprintf("https://github.com/test/test_artifact/releases/tag/%s", version), + TargetCommitish: "master", + Assets: []clients.ReleaseAsset{ + { + Name: fmt.Sprintf("%s_checksums.txt", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_checksums.txt", version, version), + }, + { + Name: fmt.Sprintf("%s_checksums.txt.sig", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_checksums.txt.sig", version, version), + }, + { + Name: fmt.Sprintf("%s_darwin_x86_64.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_darwin_x86_64.tar.gz", version, version), + }, + { + Name: fmt.Sprintf("%s_Linux_arm64.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_Linux_arm64.tar.gz", version, version), + }, + { + Name: fmt.Sprintf("%s_Linux_i386.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_Linux_i386.tar.gz", version, version), + }, + { + Name: fmt.Sprintf("%s_Linux_x86_64.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_Linux_x86_64.tar.gz", version, version), + }, + { + Name: fmt.Sprintf("%s_windows_i386.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_windows_i386.tar.gz", version, version), + }, + { + Name: fmt.Sprintf("%s_windows_x86_64.tar.gz", version), + URL: fmt.Sprintf("https://github.com/test/repo/releases/%s/%s_windows_x86_64.tar.gz", version, version), + }, + }, + } +} diff --git a/probes/releasesHaveProvenance/def.yml b/probes/releasesHaveProvenance/def.yml new file mode 100644 index 00000000000..4cb0dc70b74 --- /dev/null +++ b/probes/releasesHaveProvenance/def.yml @@ -0,0 +1,28 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: releasesHaveProvenance +short: Check that the projects releases on Github and Gitlab have provenance. +motivation: > + Provenance give users security-critical, verifiable information so that consumers can verify their artifacts before consuming them. +implementation: > + The probe checks whether any of the assets in any of the last five releases on Github or Gitlab have a provenance file. +outcome: + - For each of the last 5 releases, the probe returns OutcomePositive, if the release has a provenance file in the release assets. + - For each of the last 5 releases, the probe returns OutcomeNegative, if the the release does not have a provenance file in the release assets. + - If the project has no releases, the probe returns OutcomeNotApplicable. +remediation: + effort: Medium + text: + - Generate provenance and add it alongside the release assets. Use the [slsa-github-generator](https://github.com/slsa-framework/slsa-github-generator) to add SLSA level 3 provenance to releases. diff --git a/probes/releasesHaveProvenance/impl.go b/probes/releasesHaveProvenance/impl.go new file mode 100644 index 00000000000..02ba874e091 --- /dev/null +++ b/probes/releasesHaveProvenance/impl.go @@ -0,0 +1,134 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package releasesHaveProvenance + +import ( + "embed" + "fmt" + "strings" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const ( + Probe = "releasesHaveProvenance" + releaseLookBack = 5 +) + +// ValueType is the type of a value in the finding values. +type ValueType int + +const ( + // ValueTypeRelease represents a release name. + ValueTypeRelease ValueType = iota + // ValueTypeReleaseAsset represents a release asset name. + ValueTypeReleaseAsset +) + +var provenanceExtensions = []string{".intoto.jsonl"} + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + + var findings []finding.Finding + + releases := raw.SignedReleasesResults.Releases + + totalReleases := 0 + + for i := range releases { + release := releases[i] + if len(release.Assets) == 0 { + continue + } + totalReleases++ + hasProvenance := false + for j := range release.Assets { + asset := release.Assets[j] + for _, suffix := range provenanceExtensions { + if !strings.HasSuffix(asset.Name, suffix) { + continue + } + // Create Positive Finding + // with file info + loc := &finding.Location{ + Type: finding.FileTypeURL, + Path: asset.URL, + } + f, err := finding.NewWith(fs, Probe, + fmt.Sprintf("provenance for release artifact: %s", asset.Name), + loc, + finding.OutcomePositive) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + f.Values = map[string]int{ + release.TagName: int(ValueTypeRelease), + asset.Name: int(ValueTypeReleaseAsset), + } + findings = append(findings, *f) + hasProvenance = true + break + } + if hasProvenance { + break + } + } + if hasProvenance { + continue + } + + // Release does not have provenance + loc := &finding.Location{ + Type: finding.FileTypeURL, + Path: release.URL, + } + f, err := finding.NewWith(fs, Probe, + fmt.Sprintf("release artifact %s does not have provenance", release.TagName), + loc, + finding.OutcomeNegative) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + f.Values = map[string]int{ + release.TagName: int(ValueTypeRelease), + } + findings = append(findings, *f) + + if totalReleases >= releaseLookBack { + break + } + } + + if len(findings) == 0 { + f, err := finding.NewWith(fs, Probe, + "no GitHub releases found", + nil, + finding.OutcomeNotApplicable) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + findings = append(findings, *f) + } + return findings, Probe, nil +} diff --git a/probes/releasesHaveProvenance/impl_test.go b/probes/releasesHaveProvenance/impl_test.go new file mode 100644 index 00000000000..bc8e148c590 --- /dev/null +++ b/probes/releasesHaveProvenance/impl_test.go @@ -0,0 +1,183 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:stylecheck +package releasesHaveProvenance + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/clients" + "github.com/ossf/scorecard/v4/finding" +) + +func Test_Run(t *testing.T) { + t.Parallel() + //nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "Has one release with provenance.", + raw: &checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Releases: []clients.Release{ + { + TagName: "v1.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.sig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "Has two releases with provenance.", + raw: &checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Releases: []clients.Release{ + { + TagName: "v1.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.sig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + { + TagName: "v2.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.sig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + finding.OutcomePositive, + }, + }, + { + name: "Has two releases without provenance.", + raw: &checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Releases: []clients.Release{ + { + TagName: "v1.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.sig"}, + {Name: "binary.tar.gz.intoto.notJsonl"}, + }, + }, + { + TagName: "v2.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.sig"}, + {Name: "binary.tar.gz.intoto.notJsonl"}, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + finding.OutcomeNegative, + }, + }, + { + name: "Has two releases without provenace and one with.", + raw: &checker.RawResults{ + SignedReleasesResults: checker.SignedReleasesData{ + Releases: []clients.Release{ + { + TagName: "v1.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.notSig"}, + {Name: "binary.tar.gz.intoto.notJsonl"}, + }, + }, + { + TagName: "v2.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.sig"}, + {Name: "binary.tar.gz.intoto.jsonl"}, + }, + }, + + { + TagName: "v3.0", + Assets: []clients.ReleaseAsset{ + {Name: "binary.tar.gz"}, + {Name: "binary.tar.gz.sig"}, + {Name: "binary.tar.gz.intoto.notJsonl"}, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + finding.OutcomePositive, + finding.OutcomeNegative, + }, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +}