From 3eec4633315b3af1e5f49d6bda166efc9be00b74 Mon Sep 17 00:00:00 2001 From: attiasas Date: Sun, 12 Jan 2025 15:19:23 +0100 Subject: [PATCH] fix tests and add more --- commands/git/audit/gitaudit.go | 5 +- git_test.go | 72 +++++++++++-------- .../integration/test_integrationutils.go | 50 +++++++------ tests/utils/test_utils.go | 9 ++- xsc_test.go | 5 +- 5 files changed, 85 insertions(+), 56 deletions(-) diff --git a/commands/git/audit/gitaudit.go b/commands/git/audit/gitaudit.go index c1dc469c..6adbebfb 100644 --- a/commands/git/audit/gitaudit.go +++ b/commands/git/audit/gitaudit.go @@ -102,11 +102,14 @@ func toAuditParams(params GitAuditParams) *sourceAudit.AuditParams { func RunGitAudit(params GitAuditParams) (scanResults *results.SecurityCommandResults) { // Send scan started event + event := xsc.CreateAnalyticsEvent(services.CliProduct, services.CliEventType, params.serverDetails) + event.GitInfo = ¶ms.source + event.IsGitInfoFlow = true multiScanId, startTime := xsc.SendNewScanEvent( params.xrayVersion, params.xscVersion, params.serverDetails, - xsc.CreateAnalyticsEvent(services.CliProduct, services.CliEventType, params.serverDetails), + event, ) params.multiScanId = multiScanId params.startTime = startTime diff --git a/git_test.go b/git_test.go index abb5eb9c..d296cca9 100644 --- a/git_test.go +++ b/git_test.go @@ -13,15 +13,16 @@ import ( "github.com/jfrog/jfrog-cli-security/tests/utils/integration" "github.com/jfrog/jfrog-cli-security/utils/results" "github.com/jfrog/jfrog-cli-security/utils/validations" + "github.com/jfrog/jfrog-cli-security/utils/xray/scangraph" "github.com/jfrog/jfrog-client-go/utils/tests" "github.com/jfrog/jfrog-client-go/xray/services" "github.com/jfrog/jfrog-client-go/xray/services/utils" - xscservices "github.com/jfrog/jfrog-client-go/xsc/services" + // xscservices "github.com/jfrog/jfrog-client-go/xsc/services" xscutils "github.com/jfrog/jfrog-client-go/xsc/services/utils" ) func TestCountContributorsFlags(t *testing.T) { - testCleanUp := integration.InitGitTest(t, "") + _, _, testCleanUp := integration.InitGitTest(t, "") defer testCleanUp() err := securityTests.PlatformCli.WithoutCredentials().Exec("git", "count-contributors", "--token", "token", "--owner", "owner", "--scm-api-url", "url") @@ -52,13 +53,8 @@ func TestCountContributorsFlags(t *testing.T) { assert.ErrorContains(t, err, "Unsupported SCM type") } -type gitAuditCommandTestParams struct { - auditCommandTestParams - gitInfoContext *xscservices.XscGitInfoContext -} - -func testGitAuditCommand(t *testing.T, params gitAuditCommandTestParams) (string, error) { - return securityTests.PlatformCli.RunCliCmdWithOutputs(t, append([]string{"git", "audit"}, getAuditCmdArgs(params.auditCommandTestParams)...)...) +func testGitAuditCommand(t *testing.T, params auditCommandTestParams) (string, error) { + return securityTests.PlatformCli.RunCliCmdWithOutputs(t, append([]string{"git", "audit"}, getAuditCmdArgs(params)...)...) } // TODO: replace with 'Git Audit' command when it will be available. @@ -94,9 +90,9 @@ func testGitAuditCommand(t *testing.T, params gitAuditCommandTestParams) (string // } // } -func createTestProjectRunGitAuditAndValidate(t *testing.T, gitAuditParams gitAuditCommandTestParams, expectError string, validationParams validations.ValidationParams) { +func createTestProjectRunGitAuditAndValidate(t *testing.T, projectPath string, gitAuditParams auditCommandTestParams, xrayVersion, xscVersion, expectError string, validationParams validations.ValidationParams) { // Create the project to scan - _, cleanUpProject := securityTestUtils.CreateTestProjectFromZipAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "git", "projects", "issues")) + _, cleanUpProject := securityTestUtils.CreateTestProjectFromZipAndChdir(t, projectPath) defer cleanUpProject() // Run the audit command with git repo and verify violations are reported to the platform. output, err := testGitAuditCommand(t, gitAuditParams) @@ -106,16 +102,33 @@ func createTestProjectRunGitAuditAndValidate(t *testing.T, gitAuditParams gitAud assert.NoError(t, err) } validations.VerifySimpleJsonResults(t, output, validationParams) + validateAnalyticsBasicEvent(t, xrayVersion, xscVersion, output) +} + +func TestGitAuditSimpleJson(t *testing.T) { + xrayVersion, xscVersion, testCleanUp := integration.InitGitTest(t, scangraph.GraphScanMinXrayVersion) + defer testCleanUp() + createTestProjectRunGitAuditAndValidate(t, + filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "git", "projects", "gitlab"), + auditCommandTestParams{Format: string(format.SimpleJson), WithLicense: true, WithVuln: true}, + xrayVersion, xscVersion, "", + validations.ValidationParams{ + Total: &validations.TotalCount{Licenses: 3, Vulnerabilities: 2}, + Vulnerabilities: &validations.VulnerabilityCount{ValidateScan: &validations.ScanCount{Sca: 2}}, + }, + ) } func TestGitAuditViolationsWithIgnoreRule(t *testing.T) { - testCleanUp := integration.InitGitTest(t, services.MinXrayVersionGitRepoKey) + xrayVersion, xscVersion, testCleanUp := integration.InitGitTest(t, services.MinXrayVersionGitRepoKey) defer testCleanUp() // // Create the project to scan // _, cleanUpProject := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "jas", "jas")) // defer cleanUpProject() + projectPath := filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "git", "projects", "issues") + // Create policy and watch for the git repo so we will also get violations (unknown = all vulnerabilities will be reported as violations) policyName, cleanUpPolicy := securityTestUtils.CreateTestSecurityPolicy(t, "git-repo-ignore-rule-policy", utils.Unknown, true, false) defer cleanUpPolicy() @@ -123,9 +136,9 @@ func TestGitAuditViolationsWithIgnoreRule(t *testing.T) { defer cleanUpWatch() // Run the audit command with git repo and verify violations are reported to the platform. - createTestProjectRunGitAuditAndValidate(t, - gitAuditCommandTestParams{gitInfoContext: &validations.TestMockGitInfo, auditCommandTestParams: auditCommandTestParams{Format: string(format.SimpleJson), WithLicense: true, WithVuln: true}}, - "", + createTestProjectRunGitAuditAndValidate(t, projectPath, + auditCommandTestParams{Format: string(format.SimpleJson), WithLicense: true, WithVuln: true}, + xrayVersion, xscVersion, "", validations.ValidationParams{ Total: &validations.TotalCount{Licenses: 3, Violations: 16, Vulnerabilities: 16}, // Check that we have at least one violation for each scan type. (IAC is not supported yet) @@ -150,9 +163,9 @@ func TestGitAuditViolationsWithIgnoreRule(t *testing.T) { }) defer cleanSastUpIgnoreRule() - createTestProjectRunGitAuditAndValidate(t, - gitAuditCommandTestParams{gitInfoContext: &validations.TestMockGitInfo, auditCommandTestParams: auditCommandTestParams{Format: string(format.SimpleJson)}}, - "", + createTestProjectRunGitAuditAndValidate(t, projectPath, + auditCommandTestParams{Format: string(format.SimpleJson)}, + xrayVersion, xscVersion, "", // No Violations should be reported since all violations are ignored. validations.ValidationParams{ExactResultsMatch: true, Total: &validations.TotalCount{}, Violations: &validations.ViolationCount{ValidateScan: &validations.ScanCount{}}}, ) @@ -164,7 +177,7 @@ func TestGitAuditViolationsWithIgnoreRule(t *testing.T) { } func TestGitAuditJasViolationsProjectKeySimpleJson(t *testing.T) { - testCleanUp := integration.InitGitTest(t, services.MinXrayVersionGitRepoKey) + xrayVersion, xscVersion, testCleanUp := integration.InitGitTest(t, services.MinXrayVersionGitRepoKey) defer testCleanUp() if *securityTests.JfrogTestProjectKey == "" { @@ -183,8 +196,9 @@ func TestGitAuditJasViolationsProjectKeySimpleJson(t *testing.T) { // Run the audit command with git repo and verify violations are reported to the platform. createTestProjectRunGitAuditAndValidate(t, - gitAuditCommandTestParams{gitInfoContext: &validations.TestMockGitInfo, auditCommandTestParams: auditCommandTestParams{Format: string(format.SimpleJson), ProjectKey: *securityTests.JfrogTestProjectKey}}, - results.NewFailBuildError().Error(), + filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "git", "projects", "issues"), + auditCommandTestParams{Format: string(format.SimpleJson), ProjectKey: *securityTests.JfrogTestProjectKey}, + xrayVersion, xscVersion, results.NewFailBuildError().Error(), validations.ValidationParams{ Total: &validations.TotalCount{Violations: 16}, // Check that we have at least one violation for each scan type. (IAC is not supported yet) @@ -204,13 +218,15 @@ func TestGitAuditJasViolationsProjectKeySimpleJson(t *testing.T) { } func TestXrayAuditJasSkipNotApplicableCvesViolations(t *testing.T) { - testCleanUp := integration.InitGitTest(t, services.MinXrayVersionGitRepoKey) + xrayVersion, xscVersion, testCleanUp := integration.InitGitTest(t, services.MinXrayVersionGitRepoKey) defer testCleanUp() // // Create the project to scan // _, cleanUpProject := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "jas", "jas")) // defer cleanUpProject() + projectPath := filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "git", "projects", "issues") + // Create policy and watch for the git repo so we will also get violations - This watch DO NOT skip not-applicable results var firstPolicyCleaned, firstWatchCleaned bool policyName, cleanUpPolicy := securityTestUtils.CreateTestSecurityPolicy(t, "without-skip-non-applicable-policy", utils.Low, false, false) @@ -227,9 +243,9 @@ func TestXrayAuditJasSkipNotApplicableCvesViolations(t *testing.T) { }() // Run the git audit command and verify violations are reported to the platform. - createTestProjectRunGitAuditAndValidate(t, - gitAuditCommandTestParams{gitInfoContext: &validations.TestMockGitInfo, auditCommandTestParams: auditCommandTestParams{Format: string(format.SimpleJson), Watches: []string{watchName}, DisableFailOnFailedBuildFlag: true}}, - "", + createTestProjectRunGitAuditAndValidate(t, projectPath, + auditCommandTestParams{Format: string(format.SimpleJson), Watches: []string{watchName}, DisableFailOnFailedBuildFlag: true}, + xrayVersion, xscVersion, "", validations.ValidationParams{ Violations: &validations.ViolationCount{ ValidateScan: &validations.ScanCount{Sca: 8, Sast: 2, Secrets: 2}, @@ -265,9 +281,9 @@ func TestXrayAuditJasSkipNotApplicableCvesViolations(t *testing.T) { defer skipCleanUpWatch() // Run the audit command with git repo and verify violations are reported to the platform and not applicable issues are skipped. - createTestProjectRunGitAuditAndValidate(t, - gitAuditCommandTestParams{gitInfoContext: &validations.TestMockGitInfo, auditCommandTestParams: auditCommandTestParams{Format: string(format.SimpleJson), Watches: []string{skipWatchName}, DisableFailOnFailedBuildFlag: true}}, - "", + createTestProjectRunGitAuditAndValidate(t, projectPath, + auditCommandTestParams{Format: string(format.SimpleJson), Watches: []string{skipWatchName}, DisableFailOnFailedBuildFlag: true}, + xrayVersion, xscVersion, "", validations.ValidationParams{ Violations: &validations.ViolationCount{ ValidateScan: &validations.ScanCount{Sca: 7, Sast: 2, Secrets: 2}, diff --git a/tests/utils/integration/test_integrationutils.go b/tests/utils/integration/test_integrationutils.go index 43112a46..60bffc61 100644 --- a/tests/utils/integration/test_integrationutils.go +++ b/tests/utils/integration/test_integrationutils.go @@ -54,7 +54,7 @@ func InitXrayTest(t *testing.T, minVersion string) { if !*configTests.TestXray { t.Skip(getSkipTestMsg("Xray commands", "--test.xray")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } func GetTestServerDetails() *config.ServerDetails { @@ -65,12 +65,7 @@ func InitXscTest(t *testing.T, validations ...func()) (string, string, func()) { if !*configTests.TestXsc { t.Skip(getSkipTestMsg("XSC integration", "--test.xsc")) } - xrayVersion, err := testUtils.GetTestsXrayVersion() - assert.NoError(t, err) - // validate XSC is enabled at the given server - xscService, err := xsc.CreateXscServiceBackwardCompatible(xrayVersion.GetVersion(), configTests.XscDetails) - assert.NoError(t, err) - xscVersion, err := xscService.GetVersion() + xrayVersion, xscVersion, err := getXrayAndXscTestVersions(t) if err != nil { t.Skip("Skipping XSC integration tests. XSC is not enabled at the given server.") } @@ -79,79 +74,88 @@ func InitXscTest(t *testing.T, validations ...func()) (string, string, func()) { } // Make sure the audit request will work with xsc and not xray assert.NoError(t, os.Setenv(coreutils.ReportUsage, "true")) - return xrayVersion.GetVersion(), xscVersion, func() { + return xrayVersion, xscVersion, func() { assert.NoError(t, os.Setenv(coreutils.ReportUsage, "false")) } } +func getXrayAndXscTestVersions(t *testing.T) (string, string, error) { + xrayVersion, err := testUtils.GetTestsXrayVersion() + assert.NoError(t, err) + xscService, err := xsc.CreateXscServiceBackwardCompatible(xrayVersion.GetVersion(), configTests.XscDetails) + assert.NoError(t, err) + xscVersion, err := xscService.GetVersion() + return xrayVersion.GetVersion(), xscVersion, err +} + func InitAuditGeneralTests(t *testing.T, minVersion string) { if !*configTests.TestAuditGeneral { t.Skip(getSkipTestMsg("Audit command general integration", "--test.audit")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } func InitAuditJasTest(t *testing.T, minVersion string) { if !*configTests.TestAuditJas { t.Skip(getSkipTestMsg("Audit command JFrog Artifactory Security integration", "--test.audit.Jas")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } func InitAuditJavaScriptTest(t *testing.T, minVersion string) { if !*configTests.TestAuditJavaScript { t.Skip(getSkipTestMsg("Audit command JavaScript technologies (Npm, Pnpm, Yarn) integration", "--test.audit.JavaScript")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } func InitAuditJavaTest(t *testing.T, minVersion string) { if !*configTests.TestAuditJava { t.Skip(getSkipTestMsg("Audit command Java technologies (Maven, Gradle) integration", "--test.audit.Java")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } func InitAuditCTest(t *testing.T, minVersion string) { if !*configTests.TestAuditCTypes { t.Skip(getSkipTestMsg("Audit command C/C++/C# technologies (Nuget/DotNet, Conan) integration", "--test.audit.C")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } func InitAuditGoTest(t *testing.T, minVersion string) { if !*configTests.TestAuditGo { t.Skip(getSkipTestMsg("Audit command Go technologies (GoLang) integration", "--test.audit.Go")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } func InitAuditCocoapodsTest(t *testing.T, minVersion string) { if !*configTests.TestAuditCocoapods { t.Skip(getSkipTestMsg("Audit command Cocoapods technologies integration", "--test.audit.Cocoapods")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } func InitAuditSwiftTest(t *testing.T, minVersion string) { if !*configTests.TestAuditSwift { t.Skip(getSkipTestMsg("Audit command Swift technologies integration", "--test.audit.Swift")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } func InitAuditPythonTest(t *testing.T, minVersion string) { if !*configTests.TestAuditPython { t.Skip(getSkipTestMsg("Audit command Python technologies (Pip, PipEnv, Poetry) integration", "--test.audit.Python")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } func InitScanTest(t *testing.T, minVersion string) { if !*configTests.TestScan { t.Skip(getSkipTestMsg("Other scan commands integration", "--test.scan")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } func InitNativeDockerTest(t *testing.T) (mockCli *coreTests.JfrogCli, cleanUp func()) { @@ -171,19 +175,21 @@ func InitEnrichTest(t *testing.T, minVersion string) { if !*configTests.TestEnrich { t.Skip(getSkipTestMsg("Enrich command integration", "--test.enrich")) } - testUtils.ValidateXrayVersion(t, minVersion) + testUtils.GetAndValidateXrayVersion(t, minVersion) } -func InitGitTest(t *testing.T, minXrayVersion string) func() { +func InitGitTest(t *testing.T, minXrayVersion string) (string, string, func()) { if !*configTests.TestGit { t.Skip(getSkipTestMsg("Git commands integration", "--test.git")) } + xrayVersion, xscVersion, err := getXrayAndXscTestVersions(t) + assert.NoError(t, err) if minXrayVersion != "" { - testUtils.ValidateXrayVersion(t, minXrayVersion) + testUtils.ValidateXrayVersion(t, xrayVersion, minXrayVersion) } // Make sure the request will work with xsc and not xray assert.NoError(t, os.Setenv(coreutils.ReportUsage, "true")) - return func() { + return xrayVersion, xscVersion, func() { assert.NoError(t, os.Setenv(coreutils.ReportUsage, "false")) } } diff --git a/tests/utils/test_utils.go b/tests/utils/test_utils.go index 9974f277..1f87790e 100644 --- a/tests/utils/test_utils.go +++ b/tests/utils/test_utils.go @@ -48,14 +48,17 @@ func UnmarshalXML(t *testing.T, output string) formats.Bom { return xmlMap } -func ValidateXrayVersion(t *testing.T, minVersion string) { +func GetAndValidateXrayVersion(t *testing.T, minVersion string) { xrayVersion, err := GetTestsXrayVersion() if err != nil { assert.NoError(t, err) return } - err = clientUtils.ValidateMinimumVersion(clientUtils.Xray, xrayVersion.GetVersion(), minVersion) - if err != nil { + ValidateXrayVersion(t, xrayVersion.GetVersion(), minVersion) +} + +func ValidateXrayVersion(t *testing.T, xrayVersion, minVersion string) { + if err := clientUtils.ValidateMinimumVersion(clientUtils.Xray, xrayVersion, minVersion); err != nil { t.Skip(err) } } diff --git a/xsc_test.go b/xsc_test.go index 71fe5487..eda89b5e 100644 --- a/xsc_test.go +++ b/xsc_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/jfrog/jfrog-cli-core/v2/common/format" @@ -77,11 +78,11 @@ func validateAnalyticsBasicEvent(t *testing.T, xrayVersion, xscVersion, output s // Get MSI. var results formats.SimpleJsonResults err := json.Unmarshal([]byte(output), &results) - assert.NoError(t, err) + require.NoError(t, err) // Verify analytics metrics. event, err := xsc.GetScanEvent(xrayVersion, xscVersion, results.MultiScanId, tests.XscDetails) - assert.NoError(t, err) + require.NoError(t, err) assert.NotNil(t, event) assert.NotEmpty(t, results.MultiScanId)