From 41100a441f9091b6153d20201da3d6720d80724c Mon Sep 17 00:00:00 2001 From: Kishan Bhat Date: Sat, 30 Mar 2024 22:21:36 -0700 Subject: [PATCH 01/11] Expose dry run mode value of lifecycle manager (#931) --- lifecycle/manager.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lifecycle/manager.go b/lifecycle/manager.go index 1b5af7103..72d76dd74 100644 --- a/lifecycle/manager.go +++ b/lifecycle/manager.go @@ -37,6 +37,10 @@ func (lcs *LifecycleServicesManager) Client() *jfroghttpclient.JfrogHttpClient { return lcs.client } +func (lcs *LifecycleServicesManager) IsDryRun() bool { + return lcs.config.IsDryRun() +} + func (lcs *LifecycleServicesManager) CreateReleaseBundleFromAql(rbDetails lifecycle.ReleaseBundleDetails, queryParams lifecycle.CommonOptionalQueryParams, signingKeyName string, aqlQuery string) error { rbService := lifecycle.NewReleaseBundlesService(lcs.config.GetServiceDetails(), lcs.client) From 8292671b7cc4525898218232e6424c293005b94f Mon Sep 17 00:00:00 2001 From: Gai Lazar Date: Wed, 3 Apr 2024 13:03:35 +0300 Subject: [PATCH 02/11] New XSC analytics metrics capabilities. (#928) --- .github/workflows/tests.yml | 4 +- README.md | 98 +++++++++++++++++++--- tests/utils_test.go | 12 +++ tests/xsc_test.go | 36 ++++++++ tests/xscanalyticsevent_test.go | 79 ++++++++++++++++++ utils/utils.go | 16 ++++ utils/utils_test.go | 46 +++++++++++ xray/manager.go | 13 --- xsc/auth/xscdetails.go | 45 ++++++++++ xsc/manager.go | 71 ++++++++++++++++ xsc/services/analytics.go | 140 ++++++++++++++++++++++++++++++++ xsc/services/version.go | 51 ++++++++++++ 12 files changed, 586 insertions(+), 25 deletions(-) create mode 100644 tests/xsc_test.go create mode 100644 tests/xscanalyticsevent_test.go create mode 100644 xsc/auth/xscdetails.go create mode 100644 xsc/manager.go create mode 100644 xsc/services/analytics.go create mode 100644 xsc/services/version.go diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1329c7992..b0302c2ec 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -91,7 +91,7 @@ jobs: strategy: fail-fast: false matrix: - suite: [ distribution, xray, mpu ] + suite: [ distribution, xray, xsc, mpu ] os: [ ubuntu, windows, macos ] runs-on: ${{ matrix.os }}-latest steps: @@ -114,7 +114,7 @@ jobs: restore-keys: ${{ runner.os }}-go- - name: ${{ matrix.suite }} tests - run: go test -v github.com/jfrog/jfrog-client-go/tests --timeout 0 --test.${{ matrix.suite }} --rt.url=${{ secrets.PLATFORM_URL }}/artifactory --ds.url=${{ secrets.PLATFORM_URL }}/distribution --xr.url=${{ secrets.PLATFORM_URL }}/xray --access.url=${{ secrets.PLATFORM_URL }}/access --rt.user=${{ secrets.PLATFORM_USER }} --rt.password=${{ secrets.PLATFORM_PASSWORD }} --access.token=${{ secrets.PLATFORM_ADMIN_TOKEN }} --ci.runId=${{ runner.os }}-${{ matrix.suite }} + run: go test -v github.com/jfrog/jfrog-client-go/tests --timeout 0 --test.${{ matrix.suite }} --rt.url=${{ secrets.PLATFORM_URL }}/artifactory --ds.url=${{ secrets.PLATFORM_URL }}/distribution --xr.url=${{ secrets.PLATFORM_URL }}/xray --xsc.url=${{ secrets.PLATFORM_URL }}/xsc --access.url=${{ secrets.PLATFORM_URL }}/access --rt.user=${{ secrets.PLATFORM_USER }} --rt.password=${{ secrets.PLATFORM_PASSWORD }} --access.token=${{ secrets.PLATFORM_ADMIN_TOKEN }} --ci.runId=${{ runner.os }}-${{ matrix.suite }} JFrog-Client-Go-Pipelines-Tests: needs: Pretest diff --git a/README.md b/README.md index 545377094..80fe1d640 100644 --- a/README.md +++ b/README.md @@ -170,9 +170,17 @@ - [Delete Violations Report](#delete-violations-report) - [Get Artifact Summary](#get-artifact-summary) - [Get Entitlement info](#get-entitlement-info) - - [Using XSC Service](#using-xsc-service) - - [Check if xsc is enabled](#check-if-xsc-is-enabled) - - [Send git info details to xsc](#send-git-info-details-to-xsc) + - [XSC APIs](#xsc-apis) + - [Creating XSC Service Manager](#creating-xray-service-manager) + - [Creating XSC Details](#creating-xsc-details) + - [Creating XSC Service Config](#creating-xsc-service-config) + - [Creating New XSC Service Manager](#creating-new-xsc-service-manager) + - [Using XSC Services](#using-xsc-services) + - [Fetching XSC's Version](#fetching-xscs-version) + - [Report XSC analytics metrics](#report-xsc-analytics-metrics) + - [Add analytics general event](#add-analytics-general-event) + - [Update analytics general event](#update-analytics-general-event) + - [Get analytics general event](#get-analytics-general-event) - [Pipelines APIs](#pipelines-apis) - [Creating Pipelines Service Manager](#creating-pipelines-service-manager) - [Creating Pipelines Details](#creating-pipelines-details) @@ -265,6 +273,7 @@ content of this repository is deleted. | `-test.artifactory` | Artifactory tests | Artifactory Pro | | `-test.distribution` | Distribution tests | Artifactory with Distribution | | `-test.xray` | Xray tests | Artifactory with Xray | +| `-test.xsc` | Xsc tests | Xray with Xsc | | `-test.pipelines` | Pipelines tests | JFrog Pipelines | | `-test.access` | Access tests | Artifactory Pro | | `-test.repositories` | Repositories tests | Artifactory Pro | @@ -277,6 +286,7 @@ content of this repository is deleted. | `-rt.url` | [Default: http://localhost:8081/artifactory] Artifactory URL. | | `-ds.url` | [Optional] JFrog Distribution URL. | | `-xr.url` | [Optional] JFrog Xray URL. | +| `-xsc.url` | [Optional] JFrog Xsc URL. | | `-pipe.url` | [Optional] JFrog Pipelines URL. | | `-access.url` | [Optional] JFrog Access URL. | | `-rt.user` | [Default: admin] Artifactory username. | @@ -2233,20 +2243,88 @@ artifactSummary, err := xrayManager.ArtifactSummary(artifactSummaryRequest) isEntitled, err := xrayManager.IsEntitled(featureId) ``` -### Using XSC Service -#### Check if xsc is enabled +## XSC APIs + +### Creating XSC Service Manager + +#### Creating XSC Details + +```go +xscDetails := auth.NewXscDetails() +xscDetails.SetUrl("http://localhost:8081/xsc") +xscDetails.SetSshKeyPath("path/to/.ssh/") +xscDetails.SetApiKey("apikey") +xscDetails.SetUser("user") +xscDetails.SetPassword("password") +xscDetails.SetAccessToken("accesstoken") +// if client certificates are required +xscDetails.SetClientCertPath("path/to/.cer") +xscDetails.SetClientCertKeyPath("path/to/.key") +``` + +#### Creating XSC Service Config + +```go +serviceConfig, err := config.NewConfigBuilder(). + SetServiceDetails(xscDetails). + SetCertificatesPath(certPath). + // Optionally overwrite the default HTTP retries, which is set to 3. + SetHttpRetries(8). + Build() +``` + +#### Creating New XSC Service Manager ```go -// Will try to get XSC version. If route is not available, user is not entitled for XSC. -xscVersion, err := scanService.IsXscEnabled() +xscManager, err := xsc.New(serviceConfig) ``` -#### Send git info details to xsc +### Using XSC Services + +#### Fetching XSC's Version + +```go +version, err := xscManager.GetVersion() +``` + +#### Report XSC analytics metrics +##### Add analytics general event +Sent XSC a new event which contains analytics data, and get multi-scan id back from XSC. +```go +event := services.XscAnalyticsGeneralEvent{ + XscAnalyticsBasicGeneralEvent: services.XscAnalyticsBasicGeneralEvent{ + EventType: services.CliEventType, + Product: services.CliProduct, + ProductVersion: "2.53.1", + IsDefaultConfig: false, + JfrogUser: "gail", + OsPlatform: "mac", + OsArchitecture: "arm64", + AnalyzerManagerVersion: "1.1.1", + EventStatus: services.Started, +}} +msi, err := xscManager.AddAnalyticsGeneralEvent(event) +``` +##### Update analytics general event +Sent XSC a finalized analytics metrics event with information matching an existing event's msi. +```go +finalizeEvent := services.XscAnalyticsGeneralEventFinalize{ + MultiScanId: msi, + XscAnalyticsBasicGeneralEvent: services.XscAnalyticsBasicGeneralEvent{ + EventStatus: services.Completed, + TotalFindings: 10, + TotalIgnoredFindings: 5, + TotalScanDuration: "15s", + }, +} +err := xscManager.UpdateAnalyticsGeneralEvent(finalizeEvent) +``` +##### Get analytics general event +Get a general event from XSC matching the provided msi. ```go -// Details are the git info details (gitRepoUrl, branchName, commitHash are required fields). Returns multi scan id. -multiScanId, err := scanService.SendScanGitInfoContext(details) +event, err := xscManager.GetAnalyticsGeneralEvent(msi) ``` ## Pipelines APIs diff --git a/tests/utils_test.go b/tests/utils_test.go index 7a0ff6829..16fd2010b 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -42,6 +42,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils/log" xrayAuth "github.com/jfrog/jfrog-client-go/xray/auth" xrayServices "github.com/jfrog/jfrog-client-go/xray/services" + xscAuth "github.com/jfrog/jfrog-client-go/xsc/auth" "github.com/stretchr/testify/assert" ) @@ -49,6 +50,7 @@ var ( TestArtifactory *bool TestDistribution *bool TestXray *bool + TestXsc *bool TestPipelines *bool TestAccess *bool TestRepositories *bool @@ -56,6 +58,7 @@ var ( RtUrl *string DistUrl *string XrayUrl *string + XscUrl *string PipelinesUrl *string RtUser *string RtPassword *string @@ -145,6 +148,7 @@ func init() { TestArtifactory = flag.Bool("test.artifactory", false, "Test Artifactory") TestDistribution = flag.Bool("test.distribution", false, "Test distribution") TestXray = flag.Bool("test.xray", false, "Test xray") + TestXsc = flag.Bool("test.xsc", false, "Test xsc") TestPipelines = flag.Bool("test.pipelines", false, "Test pipelines") TestAccess = flag.Bool("test.access", false, "Test access") TestRepositories = flag.Bool("test.repositories", false, "Test repositories in Artifactory") @@ -152,6 +156,7 @@ func init() { RtUrl = flag.String("rt.url", "http://localhost:8081/artifactory", "Artifactory url") DistUrl = flag.String("ds.url", "", "Distribution url") XrayUrl = flag.String("xr.url", "", "Xray url") + XscUrl = flag.String("xsc.url", "", "Xsc url") PipelinesUrl = flag.String("pipe.url", "", "Pipelines url") AccessUrl = flag.String("access.url", "http://localhost:8081/access", "Access url") RtUser = flag.String("rt.user", "admin", "Artifactory username") @@ -563,6 +568,13 @@ func GetXrayDetails() auth.ServiceDetails { return xrayDetails } +func GetXscDetails() auth.ServiceDetails { + xscDetails := xscAuth.NewXscDetails() + xscDetails.SetUrl(clientutils.AddTrailingSlashIfNeeded(*XscUrl)) + setAuthenticationDetail(xscDetails) + return xscDetails +} + func GetPipelinesDetails() auth.ServiceDetails { pDetails := pipelinesAuth.NewPipelinesDetails() pDetails.SetUrl(clientutils.AddTrailingSlashIfNeeded(*PipelinesUrl)) diff --git a/tests/xsc_test.go b/tests/xsc_test.go new file mode 100644 index 000000000..bc2178a31 --- /dev/null +++ b/tests/xsc_test.go @@ -0,0 +1,36 @@ +package tests + +import ( + clientUtils "github.com/jfrog/jfrog-client-go/utils" + "testing" +) + +func TestXscVersion(t *testing.T) { + initXscTest(t, "") + version, err := GetXscDetails().GetVersion() + if err != nil { + t.Error(err) + } + if version == "" { + t.Error("Expected a version, got empty string") + } +} + +func initXscTest(t *testing.T, minVersion string) { + if !*TestXsc { + t.Skip("Skipping xsc test. To run xsc test add the '-test.xsc=true' option.") + } + validateXscVersion(t, minVersion) +} +func validateXscVersion(t *testing.T, minVersion string) { + // Validate active XSC server. + version, err := GetXscDetails().GetVersion() + if err != nil { + t.Skip(err) + } + // Validate minimum XSC version. + err = clientUtils.ValidateMinimumVersion(clientUtils.Xsc, version, minVersion) + if err != nil { + t.Skip(err) + } +} diff --git a/tests/xscanalyticsevent_test.go b/tests/xscanalyticsevent_test.go new file mode 100644 index 000000000..bfe83792d --- /dev/null +++ b/tests/xscanalyticsevent_test.go @@ -0,0 +1,79 @@ +package tests + +import ( + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/xsc/services" + "github.com/stretchr/testify/assert" + "regexp" + "testing" +) + +var testsEventService *services.AnalyticsEventService + +func TestXscAddAndUpdateGeneralEvent(t *testing.T) { + xscDetails, client := initXscEventTest(t) + testsEventService = services.NewAnalyticsEventService(client) + testsEventService.XscDetails = xscDetails + + event := services.XscAnalyticsGeneralEvent{XscAnalyticsBasicGeneralEvent: services.XscAnalyticsBasicGeneralEvent{ + EventType: services.CliEventType, + EventStatus: services.Started, + Product: services.CliProduct, + ProductVersion: "2.53.1", + IsDefaultConfig: false, + JfrogUser: "gail", + OsPlatform: "mac", + OsArchitecture: "arm64", + MachineId: "id", + AnalyzerManagerVersion: "1.1.1", + }} + msi, err := testsEventService.AddGeneralEvent(event) + assert.NoError(t, err) + assert.True(t, isValidUUID(msi)) + + // Validate that the event sent and saved properly in XSC. + resp, err := testsEventService.GetGeneralEvent(msi) + assert.NoError(t, err) + assert.Equal(t, event, *resp) + + finalizeEvent := services.XscAnalyticsGeneralEventFinalize{ + MultiScanId: msi, + XscAnalyticsBasicGeneralEvent: services.XscAnalyticsBasicGeneralEvent{ + EventStatus: services.Completed, + TotalFindings: 10, + TotalIgnoredFindings: 5, + TotalScanDuration: "15s", + }, + } + + err = testsEventService.UpdateGeneralEvent(finalizeEvent) + assert.NoError(t, err) + + // Validate that the event's update sent and saved properly in XSC. + resp, err = testsEventService.GetGeneralEvent(msi) + assert.NoError(t, err) + event.EventStatus = services.Completed + event.TotalFindings = 10 + event.TotalIgnoredFindings = 5 + event.TotalScanDuration = "15s" + assert.Equal(t, event, *resp) +} + +func isValidUUID(str string) bool { + uuidRegex := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-1[0-9a-f]{3}-[0-9a-f]{4}-[0-9a-f]{12}$`) + return uuidRegex.MatchString(str) +} + +func initXscEventTest(t *testing.T) (xscDetails auth.ServiceDetails, client *jfroghttpclient.JfrogHttpClient) { + var err error + initXscTest(t, services.AnalyticsMetricsMinXscVersion) + xscDetails = GetXscDetails() + client, err = jfroghttpclient.JfrogClientBuilder(). + SetClientCertPath(xscDetails.GetClientCertPath()). + SetClientCertKeyPath(xscDetails.GetClientCertKeyPath()). + AppendPreRequestInterceptor(xscDetails.RunPreRequestFunctions). + Build() + assert.NoError(t, err) + return +} diff --git a/utils/utils.go b/utils/utils.go index 8c232e616..3b9e7d55e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -36,6 +36,7 @@ type MinVersionProduct string const ( Artifactory MinVersionProduct = "JFrog Artifactory" Xray MinVersionProduct = "JFrog Xray" + Xsc MinVersionProduct = "JFrog Xsc" DataTransfer MinVersionProduct = "Data Transfer" DockerApi MinVersionProduct = "Docker API" Projects MinVersionProduct = "JFrog Projects" @@ -601,3 +602,18 @@ func ExtractSha256FromResponseBody(body []byte) (string, error) { func Pointer[K any](val K) *K { return &val } + +func SetEnvWithResetCallback(key, value string) (func() error, error) { + oldValue, exist := os.LookupEnv(key) + if err := os.Setenv(key, value); err != nil { + return func() error { return nil }, errorutils.CheckError(err) + } + if exist { + return func() error { + return errorutils.CheckError(os.Setenv(key, oldValue)) + }, nil + } + return func() error { + return errorutils.CheckError(os.Unsetenv(key)) + }, nil +} diff --git a/utils/utils_test.go b/utils/utils_test.go index ac61ad82c..ee58e29b4 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "os" "reflect" "sort" "testing" @@ -287,3 +288,48 @@ func TestValidateMinimumVersion(t *testing.T) { }) } } + +func TestSetEnvWithResetCallback(t *testing.T) { + type args struct { + key string + value string + } + tests := []struct { + name string + args args + init func() + finish func() + }{ + { + name: "existing environment variable", + args: args{key: "TEST_KEY", value: "test_value"}, + init: func() { + assert.NoError(t, os.Setenv("TEST_KEY", "test-init-value")) + }, + finish: func() { + assert.Equal(t, os.Getenv("TEST_KEY"), "test-init-value") + }, + }, + { + name: "non-existing environment variable", + args: args{key: "NEW_TEST_KEY", value: "test_value"}, + init: func() { + + }, + finish: func() { + _, exist := os.LookupEnv("NEW_TEST_KEY") + assert.False(t, exist) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.init() + resetCallback, err := SetEnvWithResetCallback(tt.args.key, tt.args.value) + assert.NoError(t, err) + assert.Equal(t, tt.args.value, os.Getenv(tt.args.key)) + assert.NoError(t, resetCallback()) + tt.finish() + }) + } +} diff --git a/xray/manager.go b/xray/manager.go index 33fa30c66..d1eb6eca8 100644 --- a/xray/manager.go +++ b/xray/manager.go @@ -198,16 +198,3 @@ func (sm *XrayServicesManager) IsEntitled(featureId string) (bool, error) { entitlementsService.XrayDetails = sm.config.GetServiceDetails() return entitlementsService.IsEntitled(featureId) } - -func (sm *XrayServicesManager) XscEnabled() (string, error) { - scanService := services.NewScanService(sm.client) - scanService.XrayDetails = sm.config.GetServiceDetails() - return scanService.IsXscEnabled() -} - -// SendXscGitInfoRequest sends git info details to xsc service and gets multi scan id -func (sm *XrayServicesManager) SendXscGitInfoRequest(details *services.XscGitInfoContext) (multiScanId string, err error) { - scanService := services.NewScanService(sm.client) - scanService.XrayDetails = sm.config.GetServiceDetails() - return scanService.SendScanGitInfoContext(details) -} diff --git a/xsc/auth/xscdetails.go b/xsc/auth/xscdetails.go new file mode 100644 index 000000000..a41a80d80 --- /dev/null +++ b/xsc/auth/xscdetails.go @@ -0,0 +1,45 @@ +package auth + +import ( + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/config" + "github.com/jfrog/jfrog-client-go/utils/log" + "github.com/jfrog/jfrog-client-go/xsc" +) + +// NewXscDetails creates a struct of the Xsc details +func NewXscDetails() *XscDetails { + return &XscDetails{} +} + +type XscDetails struct { + auth.CommonConfigFields +} + +func (ds *XscDetails) GetVersion() (string, error) { + var err error + if ds.Version == "" { + ds.Version, err = ds.getXscVersion() + if err != nil { + return "", err + } + log.Debug("The Xsc version is:", ds.Version) + } + return ds.Version, nil +} + +func (ds *XscDetails) getXscVersion() (string, error) { + cd := auth.ServiceDetails(ds) + serviceConfig, err := config.NewConfigBuilder(). + SetServiceDetails(cd). + SetCertificatesPath(cd.GetClientCertPath()). + Build() + if err != nil { + return "", err + } + sm, err := xsc.New(serviceConfig) + if err != nil { + return "", err + } + return sm.GetVersion() +} diff --git a/xsc/manager.go b/xsc/manager.go new file mode 100644 index 000000000..7985ac2ba --- /dev/null +++ b/xsc/manager.go @@ -0,0 +1,71 @@ +package xsc + +import ( + "github.com/jfrog/jfrog-client-go/config" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/xsc/services" +) + +// XscServicesManager defines the http client and general configuration +type XscServicesManager struct { + client *jfroghttpclient.JfrogHttpClient + config config.Config +} + +// New creates a service manager to interact with Xsc +func New(config config.Config) (*XscServicesManager, error) { + details := config.GetServiceDetails() + var err error + manager := &XscServicesManager{config: config} + manager.client, err = jfroghttpclient.JfrogClientBuilder(). + SetCertificatesPath(config.GetCertificatesPath()). + SetInsecureTls(config.IsInsecureTls()). + SetContext(config.GetContext()). + SetDialTimeout(config.GetDialTimeout()). + SetOverallRequestTimeout(config.GetOverallRequestTimeout()). + SetClientCertPath(details.GetClientCertPath()). + SetClientCertKeyPath(details.GetClientCertKeyPath()). + AppendPreRequestInterceptor(details.RunPreRequestFunctions). + SetRetries(config.GetHttpRetries()). + SetRetryWaitMilliSecs(config.GetHttpRetryWaitMilliSecs()). + Build() + return manager, err +} + +// Client will return the http client +func (sm *XscServicesManager) Client() *jfroghttpclient.JfrogHttpClient { + return sm.client +} + +func (sm *XscServicesManager) Config() config.Config { + return sm.config +} + +// GetVersion will return the Xsc version +func (sm *XscServicesManager) GetVersion() (string, error) { + versionService := services.NewVersionService(sm.client) + versionService.XscDetails = sm.config.GetServiceDetails() + return versionService.GetVersion() +} + +// AddAnalyticsGeneralEvent will send an analytics metrics general event to Xsc and return MSI (multi scan id) generated by Xsc. +func (sm *XscServicesManager) AddAnalyticsGeneralEvent(event services.XscAnalyticsGeneralEvent) (string, error) { + eventService := services.NewAnalyticsEventService(sm.client) + eventService.XscDetails = sm.config.GetServiceDetails() + return eventService.AddGeneralEvent(event) +} + +// UpdateAnalyticsGeneralEvent upon completion of the scan and we have all the results to report on, +// we send a finalized analytics metrics event with information matching an existing event's msi. +func (sm *XscServicesManager) UpdateAnalyticsGeneralEvent(event services.XscAnalyticsGeneralEventFinalize) error { + eventService := services.NewAnalyticsEventService(sm.client) + eventService.XscDetails = sm.config.GetServiceDetails() + return eventService.UpdateGeneralEvent(event) +} + +// GetAnalyticsGeneralEvent returns general event that match the msi provided. +func (sm *XscServicesManager) GetAnalyticsGeneralEvent(msi string) (*services.XscAnalyticsGeneralEvent, error) { + eventService := services.NewAnalyticsEventService(sm.client) + eventService.XscDetails = sm.config.GetServiceDetails() + return eventService.GetGeneralEvent(msi) +} diff --git a/xsc/services/analytics.go b/xsc/services/analytics.go new file mode 100644 index 000000000..4162a30f5 --- /dev/null +++ b/xsc/services/analytics.go @@ -0,0 +1,140 @@ +package services + +import ( + "encoding/json" + "fmt" + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/xray/services" + "net/http" +) + +const ( + AnalyticsMetricsMinXscVersion = "1.7.1" + xscEventApi = "api/v1/event" +) + +type AnalyticsEventService struct { + client *jfroghttpclient.JfrogHttpClient + XscDetails auth.ServiceDetails +} + +func NewAnalyticsEventService(client *jfroghttpclient.JfrogHttpClient) *AnalyticsEventService { + return &AnalyticsEventService{client: client} +} + +// GetXscDetails returns the Xsc details +func (vs *AnalyticsEventService) GetXscDetails() auth.ServiceDetails { + return vs.XscDetails +} + +// AddGeneralEvent add general event in Xsc and returns msi generated by Xsc. +func (vs *AnalyticsEventService) AddGeneralEvent(event XscAnalyticsGeneralEvent) (string, error) { + httpDetails := vs.XscDetails.CreateHttpClientDetails() + requestContent, err := json.Marshal(event) + if err != nil { + return "", errorutils.CheckError(err) + } + resp, body, err := vs.client.SendPost(vs.XscDetails.GetUrl()+xscEventApi, requestContent, &httpDetails) + if err != nil { + return "", err + } + if err = errorutils.CheckResponseStatus(resp, http.StatusCreated); err != nil { + return "", errorutils.CheckError(errorutils.GenerateResponseError(resp.Status, utils.IndentJson(body))) + } + var response XscAnalyticsGeneralEventResponse + err = json.Unmarshal(body, &response) + return response.MultiScanId, errorutils.CheckError(err) +} + +// UpdateGeneralEvent update finalized analytics metrics info of an existing event. +func (vs *AnalyticsEventService) UpdateGeneralEvent(event XscAnalyticsGeneralEventFinalize) error { + httpDetails := vs.XscDetails.CreateHttpClientDetails() + requestContent, err := json.Marshal(event) + if err != nil { + return errorutils.CheckError(err) + } + resp, body, err := vs.client.SendPut(vs.XscDetails.GetUrl()+xscEventApi, requestContent, &httpDetails) + if err != nil { + return err + } + if err = errorutils.CheckResponseStatus(resp, http.StatusOK); err != nil { + return errorutils.CheckError(errorutils.GenerateResponseError(resp.Status, utils.IndentJson(body))) + } + return nil +} + +// GetGeneralEvent returns event's data matching the provided multi scan id. +func (vs *AnalyticsEventService) GetGeneralEvent(msi string) (*XscAnalyticsGeneralEvent, error) { + httpDetails := vs.XscDetails.CreateHttpClientDetails() + resp, body, _, err := vs.client.SendGet(fmt.Sprintf("%s%s/%s", vs.XscDetails.GetUrl(), xscEventApi, msi), true, &httpDetails) + if err != nil { + return nil, err + } + if err = errorutils.CheckResponseStatus(resp, http.StatusOK); err != nil { + return nil, errorutils.CheckError(errorutils.GenerateResponseError(resp.Status, utils.IndentJson(body))) + } + var response XscAnalyticsGeneralEvent + err = json.Unmarshal(body, &response) + return &response, errorutils.CheckError(err) +} + +// XscAnalyticsGeneralEvent extend the basic struct with Frogbot related info. +type XscAnalyticsGeneralEvent struct { + XscAnalyticsBasicGeneralEvent + GitInfo *services.XscGitInfoContext `json:"gitinfo,omitempty"` + IsGitInfoFlow bool `json:"is_gitinfo_flow,omitempty"` +} + +type XscAnalyticsGeneralEventFinalize struct { + XscAnalyticsBasicGeneralEvent + MultiScanId string `json:"multi_scan_id,omitempty"` +} + +type XscAnalyticsBasicGeneralEvent struct { + EventType EventType `json:"event_type,omitempty"` + EventStatus EventStatus `json:"event_status,omitempty"` + Product ProductName `json:"product,omitempty"` + ProductVersion string `json:"product_version,omitempty"` + TotalFindings int `json:"total_findings,omitempty"` + TotalIgnoredFindings int `json:"total_ignored_findings,omitempty"` + IsDefaultConfig bool `json:"is_default_config,omitempty"` + JfrogUser string `json:"jfrog_user,omitempty"` + OsPlatform string `json:"os_platform,omitempty"` + OsArchitecture string `json:"os_architecture,omitempty"` + MachineId string `json:"machine_id,omitempty"` + AnalyzerManagerVersion string `json:"analyzer_manager_version,omitempty"` + JpdVersion string `json:"jpd_version,omitempty"` + TotalScanDuration string `json:"total_scan_duration,omitempty"` + FrogbotScanType string `json:"frogbot_scan_type,omitempty"` + FrogbotCiProvider string `json:"frogbot_ci_provider,omitempty"` +} + +type XscAnalyticsGeneralEventResponse struct { + MultiScanId string `json:"multi_scan_id,omitempty"` +} + +type EventStatus string + +const ( + Started EventStatus = "started" + Completed EventStatus = "completed" + Cancelled EventStatus = "cancelled" + Failed EventStatus = "failed" +) + +type ProductName string + +const ( + CliProduct ProductName = "cli" + FrogbotProduct ProductName = "frogbot" +) + +type EventType int + +const ( + CliEventType EventType = 1 + FrogbotType EventType = 8 +) diff --git a/xsc/services/version.go b/xsc/services/version.go new file mode 100644 index 000000000..1f4a62f1f --- /dev/null +++ b/xsc/services/version.go @@ -0,0 +1,51 @@ +package services + +import ( + "encoding/json" + "net/http" + "strings" + + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" +) + +// VersionService returns the https client and Xsc details +type VersionService struct { + client *jfroghttpclient.JfrogHttpClient + XscDetails auth.ServiceDetails +} + +// NewVersionService creates a new service to retrieve the version of Xsc +func NewVersionService(client *jfroghttpclient.JfrogHttpClient) *VersionService { + return &VersionService{client: client} +} + +// GetXscDetails returns the Xsc details +func (vs *VersionService) GetXscDetails() auth.ServiceDetails { + return vs.XscDetails +} + +// GetVersion returns the version of Xsc +func (vs *VersionService) GetVersion() (string, error) { + httpDetails := vs.XscDetails.CreateHttpClientDetails() + resp, body, _, err := vs.client.SendGet(vs.XscDetails.GetUrl()+"api/v1/system/version", true, &httpDetails) + if err != nil { + return "", err + } + if err = errorutils.CheckResponseStatus(resp, http.StatusOK); err != nil { + return "", errorutils.CheckError(errorutils.GenerateResponseError(resp.Status, utils.IndentJson(body))) + } + var version xscVersion + err = json.Unmarshal(body, &version) + if err != nil { + return "", errorutils.CheckError(err) + } + return strings.TrimSpace(version.Version), nil +} + +type xscVersion struct { + Version string `json:"xsc_version,omitempty"` + Revision string `json:"xray_version,omitempty"` +} From 27387881399711159d9bbc95497e50b0c18bd694 Mon Sep 17 00:00:00 2001 From: Eran Turgeman <81029514+eranturgeman@users.noreply.github.com> Date: Sun, 7 Apr 2024 11:03:13 +0300 Subject: [PATCH 03/11] New XSC log errors capabilities (#932) --- tests/xsclogerrorevent_test.go | 58 ++++++++++++++++++++++++++++++++++ xsc/manager.go | 6 ++++ xsc/services/logerrorevent.go | 45 ++++++++++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 tests/xsclogerrorevent_test.go create mode 100644 xsc/services/logerrorevent.go diff --git a/tests/xsclogerrorevent_test.go b/tests/xsclogerrorevent_test.go new file mode 100644 index 000000000..13b834428 --- /dev/null +++ b/tests/xsclogerrorevent_test.go @@ -0,0 +1,58 @@ +package tests + +import ( + "encoding/json" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/xsc/services" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "testing" +) + +const errorMessageContentForTest = "THIS IS NOT A REAL ERROR! This Error is posted as part of TestXscSendLogErrorEvent test" + +func TestXscSendLogErrorEvent(t *testing.T) { + initXscTest(t, services.LogErrorMinXscVersion) + mockServer, logErrorService := createXscMockServer(t) + defer mockServer.Close() + + event := &services.ExternalErrorLog{ + Log_level: "error", + Source: "cli", + Message: errorMessageContentForTest, + } + + assert.NoError(t, logErrorService.SendLogErrorEvent(event)) +} + +func createXscMockServer(t *testing.T) (mockServer *httptest.Server, logErrorService *services.LogErrorEventService) { + mockServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/xsc/api/v1/event/logMessage" && r.Method == http.MethodPost { + var reqBody services.ExternalErrorLog + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(&reqBody) + assert.NoError(t, err, "Invalid JSON request body") + if err != nil { + return + } + + assert.Equal(t, "error", reqBody.Log_level) + assert.Equal(t, "cli", reqBody.Source) + assert.Equal(t, errorMessageContentForTest, reqBody.Message) + w.WriteHeader(http.StatusCreated) + return + } + assert.Fail(t, "received an unexpected request") + })) + + xscDetails := GetXscDetails() + xscDetails.SetUrl(mockServer.URL + "/xsc") + + client, err := jfroghttpclient.JfrogClientBuilder().Build() + assert.NoError(t, err) + + logErrorService = services.NewLogErrorEventService(client) + logErrorService.XscDetails = xscDetails + return +} diff --git a/xsc/manager.go b/xsc/manager.go index 7985ac2ba..010dac24e 100644 --- a/xsc/manager.go +++ b/xsc/manager.go @@ -55,6 +55,12 @@ func (sm *XscServicesManager) AddAnalyticsGeneralEvent(event services.XscAnalyti return eventService.AddGeneralEvent(event) } +func (sm *XscServicesManager) SendXscLogErrorRequest(errorLog *services.ExternalErrorLog) error { + logErrorService := services.NewLogErrorEventService(sm.client) + logErrorService.XscDetails = sm.config.GetServiceDetails() + return logErrorService.SendLogErrorEvent(errorLog) +} + // UpdateAnalyticsGeneralEvent upon completion of the scan and we have all the results to report on, // we send a finalized analytics metrics event with information matching an existing event's msi. func (sm *XscServicesManager) UpdateAnalyticsGeneralEvent(event services.XscAnalyticsGeneralEventFinalize) error { diff --git a/xsc/services/logerrorevent.go b/xsc/services/logerrorevent.go new file mode 100644 index 000000000..54bd7946e --- /dev/null +++ b/xsc/services/logerrorevent.go @@ -0,0 +1,45 @@ +package services + +import ( + "encoding/json" + "fmt" + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "net/http" +) + +const ( + postLogErrorAPI = "api/v1/event/logMessage" + LogErrorMinXscVersion = AnalyticsMetricsMinXscVersion +) + +type LogErrorEventService struct { + client *jfroghttpclient.JfrogHttpClient + XscDetails auth.ServiceDetails +} + +type ExternalErrorLog struct { + Log_level string `json:"log_level"` + Source string `json:"source"` + Message string `json:"message"` +} + +func NewLogErrorEventService(client *jfroghttpclient.JfrogHttpClient) *LogErrorEventService { + return &LogErrorEventService{client: client} +} + +func (les *LogErrorEventService) SendLogErrorEvent(errorLog *ExternalErrorLog) error { + httpClientDetails := les.XscDetails.CreateHttpClientDetails() + requestContent, err := json.Marshal(errorLog) + if err != nil { + return fmt.Errorf("failed to convert POST request body's struct into JSON: %q", err) + } + url := utils.AddTrailingSlashIfNeeded(les.XscDetails.GetUrl()) + postLogErrorAPI + response, body, err := les.client.SendPost(url, requestContent, &httpClientDetails) + if err != nil { + return fmt.Errorf("failed to send POST query to '%s': %s", url, err.Error()) + } + return errorutils.CheckResponseStatusWithBody(response, body, http.StatusCreated) +} From 62ee0279ac589facb4fa45902a40259ba356f547 Mon Sep 17 00:00:00 2001 From: Eyal Delarea Date: Mon, 8 Apr 2024 10:14:30 +0300 Subject: [PATCH 04/11] Artifactory Release Lifecycle Management - Add Import bundle function (#921) --- README.md | 8 +++ artifactory/emptymanager.go | 5 ++ artifactory/manager.go | 5 ++ artifactory/services/releasebundle.go | 78 +++++++++++++++++++++++++++ tests/releasebundle_test.go | 52 ++++++++++++++++++ 5 files changed, 148 insertions(+) create mode 100644 artifactory/services/releasebundle.go create mode 100644 tests/releasebundle_test.go diff --git a/README.md b/README.md index 80fe1d640..44c68e0a2 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,7 @@ - [Delete Release Bundle Version](#delete-release-bundle-version) - [Delete Release Bundle Version Promotion](#delete-release-bundle-version-promotion) - [Export Release Bundle](#export-release-bundle) + - [Import Release Bundle](#import-release-bundle) - [Remote Delete Release Bundle](#remote-delete-release-bundle) ## General @@ -2705,6 +2706,13 @@ modifictions:= []utils.PathMapping{{ res,err:= serviceManager.ExportReleaseBundle(rbDetails, modifications, queryParams) ``` +#### Import Release Bundle Archive + +```go +// Imports an exported release bundle archive +res,err:= serviceManager.releaseService.ImportReleaseBundle(filePath) +``` + #### Delete Release Bundle Version ```go diff --git a/artifactory/emptymanager.go b/artifactory/emptymanager.go index 1e9387150..464534e34 100644 --- a/artifactory/emptymanager.go +++ b/artifactory/emptymanager.go @@ -105,6 +105,7 @@ type ArtifactoryServicesManager interface { FileList(relativePath string, optionalParams utils.FileListParams) (*utils.FileListResponse, error) GetStorageInfo() (*utils.StorageInfo, error) CalculateStorageInfo() error + ImportReleaseBundle(string) error } // By using this struct, you have the option of overriding only some of the ArtifactoryServicesManager @@ -465,6 +466,10 @@ func (esm *EmptyArtifactoryServicesManager) CalculateStorageInfo() error { panic("Failed: Method is not implemented") } +func (esm *EmptyArtifactoryServicesManager) ImportReleaseBundle(string) error { + panic("Failed: Method is not implemented") +} + // Compile time check of interface implementation. // Since EmptyArtifactoryServicesManager can be used by tests external to this project, we want this project's tests to fail, // if EmptyArtifactoryServicesManager stops implementing the ArtifactoryServicesManager interface. diff --git a/artifactory/manager.go b/artifactory/manager.go index 2d19c9871..80c1cb152 100644 --- a/artifactory/manager.go +++ b/artifactory/manager.go @@ -605,3 +605,8 @@ func (sm *ArtifactoryServicesManagerImp) CalculateStorageInfo() error { storageService := services.NewStorageService(sm.config.GetServiceDetails(), sm.client) return storageService.StorageInfoRefresh() } + +func (sm *ArtifactoryServicesManagerImp) ImportReleaseBundle(filePath string) error { + releaseService := services.NewReleaseService(sm.config.GetServiceDetails(), sm.client) + return releaseService.ImportReleaseBundle(filePath) +} diff --git a/artifactory/services/releasebundle.go b/artifactory/services/releasebundle.go new file mode 100644 index 000000000..b9c5513da --- /dev/null +++ b/artifactory/services/releasebundle.go @@ -0,0 +1,78 @@ +package services + +import ( + "encoding/json" + "github.com/jfrog/jfrog-client-go/artifactory/services/utils" + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + utils2 "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/io/fileutils" + "github.com/jfrog/jfrog-client-go/utils/log" + "net/http" +) + +const ( + conflictErrorMessage = "Bundle already exists" + ReleaseBundleImportRestApiEndpoint = "api/release/import/" + octetStream = "application/octet-stream" +) + +type releaseService struct { + client *jfroghttpclient.JfrogHttpClient + ArtDetails auth.ServiceDetails +} + +type ErrorResponseWithMessage struct { + Errors []ErrorDetail `json:"errors"` +} + +type ErrorDetail struct { + Status int `json:"status"` + Message string `json:"message"` +} + +func NewReleaseService(artDetails auth.ServiceDetails, client *jfroghttpclient.JfrogHttpClient) *releaseService { + return &releaseService{client: client, ArtDetails: artDetails} +} + +func (rs *releaseService) GetJfrogHttpClient() *jfroghttpclient.JfrogHttpClient { + return rs.client +} + +func (rs *releaseService) ImportReleaseBundle(filePath string) (err error) { + // Load desired file + content, err := fileutils.ReadFile(filePath) + if err != nil { + return + } + // Upload file + httpClientsDetails := rs.ArtDetails.CreateHttpClientDetails() + + url := utils2.AddTrailingSlashIfNeeded(rs.ArtDetails.GetUrl() + ReleaseBundleImportRestApiEndpoint) + + utils.SetContentType(octetStream, &httpClientsDetails.Headers) + var resp *http.Response + var body []byte + log.Info("Uploading archive...") + if resp, body, err = rs.client.SendPost(url, content, &httpClientsDetails); err != nil { + return + } + // When a release bundle already exists, the API returns 400. + // Check the error message, and if it's a conflict, don't fail the operation. + if resp.StatusCode == http.StatusBadRequest { + response := ErrorResponseWithMessage{} + if err = json.Unmarshal(body, &response); err != nil { + return + } + if response.Errors[0].Message == conflictErrorMessage { + log.Warn("Bundle already exists, did not upload a new bundle") + return + } + } + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusAccepted); err != nil { + return + } + log.Info("Release Bundle Imported Successfully") + return +} diff --git a/tests/releasebundle_test.go b/tests/releasebundle_test.go new file mode 100644 index 000000000..4e2543503 --- /dev/null +++ b/tests/releasebundle_test.go @@ -0,0 +1,52 @@ +package tests + +import ( + "github.com/jfrog/jfrog-client-go/artifactory" + artifactoryAuth "github.com/jfrog/jfrog-client-go/artifactory/auth" + "github.com/jfrog/jfrog-client-go/artifactory/services" + "github.com/jfrog/jfrog-client-go/config" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "testing" +) + +func TestImportReleaseBundle(t *testing.T) { + mockServer, rbService := createMockServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI == "/"+services.ReleaseBundleImportRestApiEndpoint { + w.WriteHeader(http.StatusBadRequest) + _, err := w.Write([]byte(` +{ + "errors" : [ { + "status" : 400, + "message" : "Bundle already exists" + } ] +} +`)) + assert.NoError(t, err) + } + }) + defer mockServer.Close() + err := rbService.ImportReleaseBundle("releasebundle_test.go") + assert.NoError(t, err) +} + +func createMockServer(t *testing.T, testHandler http.HandlerFunc) (*httptest.Server, artifactory.ArtifactoryServicesManager) { + testServer := httptest.NewServer(testHandler) + + rtDetails := artifactoryAuth.NewArtifactoryDetails() + rtDetails.SetUrl(testServer.URL + "/") + + serviceConfig, err := config.NewConfigBuilder(). + SetServiceDetails(rtDetails). + SetDryRun(false). + Build() + + if err != nil { + t.Error(err) + } + + artService, err := artifactory.New(serviceConfig) + assert.NoError(t, err) + return testServer, artService +} From 4e96d77edd64c79ab4bfbcf74488a5fbb7a28a54 Mon Sep 17 00:00:00 2001 From: Angshu Mukherjee Date: Wed, 10 Apr 2024 00:44:34 +0530 Subject: [PATCH 05/11] Add CWE and CWE Details in Vulnerabilities model (#935) --- xray/services/scan.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/xray/services/scan.go b/xray/services/scan.go index bf3d485f1..e89a6fa11 100644 --- a/xray/services/scan.go +++ b/xray/services/scan.go @@ -2,12 +2,13 @@ package services import ( "encoding/json" - "github.com/jfrog/jfrog-client-go/utils/log" - xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" "net/http" "strings" "time" + "github.com/jfrog/jfrog-client-go/utils/log" + xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" + "github.com/jfrog/jfrog-client-go/artifactory/services/utils" "github.com/jfrog/jfrog-client-go/auth" "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" @@ -349,11 +350,24 @@ type ImpactPathNode struct { } type Cve struct { - Id string `json:"cve,omitempty"` - CvssV2Score string `json:"cvss_v2_score,omitempty"` - CvssV2Vector string `json:"cvss_v2_vector,omitempty"` - CvssV3Score string `json:"cvss_v3_score,omitempty"` - CvssV3Vector string `json:"cvss_v3_vector,omitempty"` + Id string `json:"cve,omitempty"` + CvssV2Score string `json:"cvss_v2_score,omitempty"` + CvssV2Vector string `json:"cvss_v2_vector,omitempty"` + CvssV3Score string `json:"cvss_v3_score,omitempty"` + CvssV3Vector string `json:"cvss_v3_vector,omitempty"` + Cwe []string `json:"cwe,omitempty"` + CweDetails map[string]Cwe `json:"cwe_details,omitempty"` +} + +type Cwe struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Categories []CweCategory `json:"categories,omitempty"` +} + +type CweCategory struct { + Category string `json:"category,omitempty"` + Rank string `json:"rank,omitempty"` } type ExtendedInformation struct { From 84428abe3540b260bf14943e28dc0bf06c40d219 Mon Sep 17 00:00:00 2001 From: Eyal Delarea Date: Mon, 15 Apr 2024 15:20:08 +0300 Subject: [PATCH 06/11] Promote version to v1.40.0 (#938) --- go.mod | 3 ++- go.sum | 5 +++-- utils/utils.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 823982c9e..7ef48da54 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/gookit/color v1.5.4 github.com/jfrog/archiver/v3 v3.6.0 github.com/jfrog/build-info-go v1.9.25 - github.com/jfrog/gofrog v1.6.3 + github.com/jfrog/gofrog v1.7.1 github.com/stretchr/testify v1.9.0 github.com/xanzy/ssh-agent v0.3.3 golang.org/x/crypto v0.21.0 @@ -51,6 +51,7 @@ require ( github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.22.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/tools v0.19.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 8ef8084b6..92a88aa21 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI= github.com/jfrog/build-info-go v1.9.25 h1:IkjydGQA/HjOWjRaoKq1hOEgCCyBEJwQgXJSo4WVBSA= github.com/jfrog/build-info-go v1.9.25/go.mod h1:doFB4bFDVHeGulD6GF9LzsrRaIOrSoklV9DgIAEqHgc= -github.com/jfrog/gofrog v1.6.3 h1:F7He0+75HcgCe6SGTSHLFCBDxiE2Ja0tekvvcktW6wc= -github.com/jfrog/gofrog v1.6.3/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg= +github.com/jfrog/gofrog v1.7.1 h1:ME1Meg4hukAT/7X6HUQCVSe4DNjMZACCP8aCY37EW/w= +github.com/jfrog/gofrog v1.7.1/go.mod h1:X7bjfWoQDN0Z4FQGbE91j3gbPP7Urwzm4Z8tkvrlbRI= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -139,6 +139,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/utils/utils.go b/utils/utils.go index 3b9e7d55e..09dcecbe7 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -28,7 +28,7 @@ import ( const ( Development = "development" Agent = "jfrog-client-go" - Version = "1.39.0" + Version = "1.40.0" ) type MinVersionProduct string From 2bf625f8e99c208888f3fb606f62ba66d98019ce Mon Sep 17 00:00:00 2001 From: Eyal Delarea Date: Mon, 15 Apr 2024 16:41:04 +0300 Subject: [PATCH 07/11] Promote version to v1.40.1 (#939) --- go.mod | 2 +- go.sum | 4 ++-- utils/utils.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 7ef48da54..47e55ed4f 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gookit/color v1.5.4 github.com/jfrog/archiver/v3 v3.6.0 - github.com/jfrog/build-info-go v1.9.25 + github.com/jfrog/build-info-go v1.9.26 github.com/jfrog/gofrog v1.7.1 github.com/stretchr/testify v1.9.0 github.com/xanzy/ssh-agent v0.3.3 diff --git a/go.sum b/go.sum index 92a88aa21..e8d6cd45e 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F1w= github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI= -github.com/jfrog/build-info-go v1.9.25 h1:IkjydGQA/HjOWjRaoKq1hOEgCCyBEJwQgXJSo4WVBSA= -github.com/jfrog/build-info-go v1.9.25/go.mod h1:doFB4bFDVHeGulD6GF9LzsrRaIOrSoklV9DgIAEqHgc= +github.com/jfrog/build-info-go v1.9.26 h1:1Ddc6+Ecvhc+UMnKhRVG1jGM6fYNwA49207azTBGBc8= +github.com/jfrog/build-info-go v1.9.26/go.mod h1:8T7/ajM9aGshvgpwCtXwIFpyF/R6CEn4W+/FLryNXWw= github.com/jfrog/gofrog v1.7.1 h1:ME1Meg4hukAT/7X6HUQCVSe4DNjMZACCP8aCY37EW/w= github.com/jfrog/gofrog v1.7.1/go.mod h1:X7bjfWoQDN0Z4FQGbE91j3gbPP7Urwzm4Z8tkvrlbRI= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= diff --git a/utils/utils.go b/utils/utils.go index 09dcecbe7..8156f25a5 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -28,7 +28,7 @@ import ( const ( Development = "development" Agent = "jfrog-client-go" - Version = "1.40.0" + Version = "1.40.1" ) type MinVersionProduct string From 5bf715f66eacd1285699c9ef555fefd58708947d Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Wed, 24 Apr 2024 16:36:43 +0300 Subject: [PATCH 08/11] Support providing chunk size in multi-part upload (#936) --- README.md | 2 + artifactory/services/upload.go | 7 ++- artifactory/services/utils/multipartupload.go | 46 ++++++++++--------- .../services/utils/multipartupload_test.go | 32 ++++++------- artifactory/services/utils/storageutils.go | 22 +++++++++ .../services/utils/storageutils_test.go | 18 ++++++++ 6 files changed, 87 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 44c68e0a2..be7c192f9 100644 --- a/README.md +++ b/README.md @@ -424,6 +424,8 @@ params.SplitCount = 10 // The minimum file size in MiB required to attempt a multi-part upload. // MinSplitSize default value: 200 params.MinSplitSize = 100 +// The upload chunk size in MiB that can be concurrently uploaded during a multi-part upload. +params.ChunkSize = 5 // The min file size in bytes for "checksum deploy". // "Checksum deploy" is the action of calculating the file checksum locally, before // the upload, and skipping the actual file transfer if the file already diff --git a/artifactory/services/upload.go b/artifactory/services/upload.go index 0fbd9a077..e7379638f 100644 --- a/artifactory/services/upload.go +++ b/artifactory/services/upload.go @@ -619,7 +619,8 @@ func (us *UploadService) doUpload(artifact UploadData, targetUrlWithProps, logMs return } if shouldTryMultipart { - if err = us.MultipartUpload.UploadFileConcurrently(artifact.Artifact.LocalPath, artifact.Artifact.TargetPath, fileInfo.Size(), details.Checksum.Sha1, us.Progress, uploadParams.SplitCount); err != nil { + if err = us.MultipartUpload.UploadFileConcurrently(artifact.Artifact.LocalPath, artifact.Artifact.TargetPath, + fileInfo.Size(), details.Checksum.Sha1, us.Progress, uploadParams.SplitCount, uploadParams.ChunkSize); err != nil { return } // Once the file is uploaded to the storage, we finalize the multipart upload by performing a checksum deployment to save the file in Artifactory. @@ -709,6 +710,7 @@ type UploadParams struct { MinChecksumDeploy int64 MinSplitSize int64 SplitCount int + ChunkSize int64 ChecksumsCalcEnabled bool Archive string // When using the 'archive' option for upload, we can control the target path inside the uploaded archive using placeholders. This operation determines the TargetPathInArchive value. @@ -716,7 +718,8 @@ type UploadParams struct { } func NewUploadParams() UploadParams { - return UploadParams{CommonParams: &utils.CommonParams{}, MinChecksumDeploy: DefaultMinChecksumDeploy, ChecksumsCalcEnabled: true, MinSplitSize: defaultUploadMinSplit, SplitCount: defaultUploadSplitCount} + return UploadParams{CommonParams: &utils.CommonParams{}, MinChecksumDeploy: DefaultMinChecksumDeploy, + ChecksumsCalcEnabled: true, MinSplitSize: defaultUploadMinSplit, SplitCount: defaultUploadSplitCount, ChunkSize: utils.DefaultUploadChunkSize} } func DeepCopyUploadParams(params *UploadParams) UploadParams { diff --git a/artifactory/services/utils/multipartupload.go b/artifactory/services/utils/multipartupload.go index b212ea845..b9d7b9880 100644 --- a/artifactory/services/utils/multipartupload.go +++ b/artifactory/services/utils/multipartupload.go @@ -54,7 +54,7 @@ const ( // Sizes and limits constants MaxMultipartUploadFileSize = SizeTiB * 5 - uploadPartSize int64 = SizeMiB * 20 + DefaultUploadChunkSize int64 = SizeMiB * 20 // Retries and polling constants retriesInterval = time.Second * 5 @@ -122,13 +122,14 @@ type getConfigResponse struct { Supported bool `json:"supported,omitempty"` } -func (mu *MultipartUpload) UploadFileConcurrently(localPath, targetPath string, fileSize int64, sha1 string, progress ioutils.ProgressMgr, splitCount int) (err error) { +func (mu *MultipartUpload) UploadFileConcurrently(localPath, targetPath string, fileSize int64, sha1 string, + progress ioutils.ProgressMgr, splitCount int, chunkSize int64) (err error) { repoAndPath := strings.SplitN(targetPath, "/", 2) repoKey := repoAndPath[0] repoPath := repoAndPath[1] logMsgPrefix := fmt.Sprintf("[Multipart upload %s] ", repoPath) - token, err := mu.createMultipartUpload(repoKey, repoPath, calculatePartSize(fileSize, 0)) + token, err := mu.createMultipartUpload(repoKey, repoPath, calculatePartSize(fileSize, 0, chunkSize)) if err != nil { return } @@ -154,7 +155,7 @@ func (mu *MultipartUpload) UploadFileConcurrently(localPath, targetPath string, } }() - if err = mu.uploadPartsConcurrently(logMsgPrefix, fileSize, splitCount, localPath, progressReader, multipartUploadClient); err != nil { + if err = mu.uploadPartsConcurrently(logMsgPrefix, fileSize, chunkSize, splitCount, localPath, progressReader, multipartUploadClient); err != nil { return } @@ -175,9 +176,9 @@ func (mu *MultipartUpload) UploadFileConcurrently(localPath, targetPath string, return mu.completeAndPollForStatus(logMsgPrefix, uint(mu.client.GetHttpClient().GetRetries())+1, sha1, multipartUploadClient, progressReader) } -func (mu *MultipartUpload) uploadPartsConcurrently(logMsgPrefix string, fileSize int64, splitCount int, localPath string, progressReader ioutils.Progress, multipartUploadClient *httputils.HttpClientDetails) (err error) { - numberOfParts := calculateNumberOfParts(fileSize) - log.Info(fmt.Sprintf("%sSplitting file to %d parts, using %d working threads for uploading...", logMsgPrefix, numberOfParts, splitCount)) +func (mu *MultipartUpload) uploadPartsConcurrently(logMsgPrefix string, fileSize, chunkSize int64, splitCount int, localPath string, progressReader ioutils.Progress, multipartUploadClient *httputils.HttpClientDetails) (err error) { + numberOfParts := calculateNumberOfParts(fileSize, chunkSize) + log.Info(fmt.Sprintf("%sSplitting file to %d parts of %s each, using %d working threads for uploading...", logMsgPrefix, numberOfParts, ConvertIntToStorageSizeString(chunkSize), splitCount)) producerConsumer := parallel.NewRunner(splitCount, uint(numberOfParts), false) wg := new(sync.WaitGroup) @@ -186,7 +187,7 @@ func (mu *MultipartUpload) uploadPartsConcurrently(logMsgPrefix string, fileSize attemptsAllowed.Add(uint64(numberOfParts) * uint64(mu.client.GetHttpClient().GetRetries())) go func() { for i := 0; i < int(numberOfParts); i++ { - if err = mu.produceUploadTask(producerConsumer, logMsgPrefix, localPath, fileSize, numberOfParts, int64(i), progressReader, multipartUploadClient, attemptsAllowed, wg); err != nil { + if err = mu.produceUploadTask(producerConsumer, logMsgPrefix, localPath, fileSize, numberOfParts, int64(i), chunkSize, progressReader, multipartUploadClient, attemptsAllowed, wg); err != nil { return } } @@ -202,9 +203,9 @@ func (mu *MultipartUpload) uploadPartsConcurrently(logMsgPrefix string, fileSize return } -func (mu *MultipartUpload) produceUploadTask(producerConsumer parallel.Runner, logMsgPrefix, localPath string, fileSize, numberOfParts, partId int64, progressReader ioutils.Progress, multipartUploadClient *httputils.HttpClientDetails, attemptsAllowed *atomic.Uint64, wg *sync.WaitGroup) (retErr error) { +func (mu *MultipartUpload) produceUploadTask(producerConsumer parallel.Runner, logMsgPrefix, localPath string, fileSize, numberOfParts, partId, chunkSize int64, progressReader ioutils.Progress, multipartUploadClient *httputils.HttpClientDetails, attemptsAllowed *atomic.Uint64, wg *sync.WaitGroup) (retErr error) { _, retErr = producerConsumer.AddTaskWithError(func(int) error { - uploadErr := mu.uploadPart(logMsgPrefix, localPath, fileSize, partId, progressReader, multipartUploadClient) + uploadErr := mu.uploadPart(logMsgPrefix, localPath, fileSize, partId, chunkSize, progressReader, multipartUploadClient) if uploadErr == nil { log.Info(fmt.Sprintf("%sCompleted uploading part %d/%d", logMsgPrefix, partId+1, numberOfParts)) wg.Done() @@ -220,14 +221,14 @@ func (mu *MultipartUpload) produceUploadTask(producerConsumer parallel.Runner, l // Sleep before trying again time.Sleep(retriesInterval) - if err := mu.produceUploadTask(producerConsumer, logMsgPrefix, localPath, fileSize, numberOfParts, partId, progressReader, multipartUploadClient, attemptsAllowed, wg); err != nil { + if err := mu.produceUploadTask(producerConsumer, logMsgPrefix, localPath, fileSize, numberOfParts, partId, chunkSize, progressReader, multipartUploadClient, attemptsAllowed, wg); err != nil { retErr = err } }) return } -func (mu *MultipartUpload) uploadPart(logMsgPrefix, localPath string, fileSize, partId int64, progressReader ioutils.Progress, multipartUploadClient *httputils.HttpClientDetails) (err error) { +func (mu *MultipartUpload) uploadPart(logMsgPrefix, localPath string, fileSize, partId, chunkSize int64, progressReader ioutils.Progress, multipartUploadClient *httputils.HttpClientDetails) (err error) { file, err := os.Open(localPath) if err != nil { return errorutils.CheckError(err) @@ -235,10 +236,10 @@ func (mu *MultipartUpload) uploadPart(logMsgPrefix, localPath string, fileSize, defer func() { err = errors.Join(err, errorutils.CheckError(file.Close())) }() - if _, err = file.Seek(partId*uploadPartSize, io.SeekStart); err != nil { + if _, err = file.Seek(partId*chunkSize, io.SeekStart); err != nil { return errorutils.CheckError(err) } - partSize := calculatePartSize(fileSize, partId) + partSize := calculatePartSize(fileSize, partId, chunkSize) limitReader := io.LimitReader(file, partSize) limitReader = bufio.NewReader(limitReader) @@ -402,21 +403,22 @@ func (mu *MultipartUpload) abort(logMsgPrefix string, multipartUploadClient *htt return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusNoContent) } -// Calculates the part size based on the file size and the part number. +// Calculates the part size based on the file size, the part number and the requested chunk size. // fileSize - the file size // partNumber - the current part number -func calculatePartSize(fileSize int64, partNumber int64) int64 { - partOffset := partNumber * uploadPartSize - if partOffset+uploadPartSize > fileSize { +// requestedChunkSize - chunk size requested by the user, or default. +func calculatePartSize(fileSize, partNumber, requestedChunkSize int64) int64 { + partOffset := partNumber * requestedChunkSize + if partOffset+requestedChunkSize > fileSize { return fileSize - partOffset } - return uploadPartSize + return requestedChunkSize } -// Calculates the number of parts based on the file size and the default part size. +// Calculates the number of parts based on the file size and the requested chunks size. // fileSize - the file size -func calculateNumberOfParts(fileSize int64) int64 { - return (fileSize + uploadPartSize - 1) / uploadPartSize +func calculateNumberOfParts(fileSize, chunkSize int64) int64 { + return (fileSize + chunkSize - 1) / chunkSize } func parseMultipartUploadStatus(status statusResponse) (shouldKeepPolling, shouldRerunComplete bool, err error) { diff --git a/artifactory/services/utils/multipartupload_test.go b/artifactory/services/utils/multipartupload_test.go index 6f8fd2cc9..fd43b0a98 100644 --- a/artifactory/services/utils/multipartupload_test.go +++ b/artifactory/services/utils/multipartupload_test.go @@ -109,7 +109,7 @@ func TestUploadPartsConcurrentlyTooManyAttempts(t *testing.T) { defer cleanUp() // Write something to the file - buf := make([]byte, uploadPartSize*3) + buf := make([]byte, DefaultUploadChunkSize*3) _, err := rand.Read(buf) assert.NoError(t, err) _, err = tempFile.Write(buf) @@ -146,7 +146,7 @@ func TestUploadPartsConcurrentlyTooManyAttempts(t *testing.T) { // Execute uploadPartsConcurrently fileSize := int64(len(buf)) - err = multipartUpload.uploadPartsConcurrently("", fileSize, splitCount, tempFile.Name(), nil, &httputils.HttpClientDetails{}) + err = multipartUpload.uploadPartsConcurrently("", fileSize, DefaultUploadChunkSize, splitCount, tempFile.Name(), nil, &httputils.HttpClientDetails{}) assert.ErrorIs(t, err, errTooManyAttempts) } @@ -285,19 +285,19 @@ var calculatePartSizeProvider = []struct { partNumber int64 expectedPartSize int64 }{ - {uploadPartSize - 1, 0, uploadPartSize - 1}, - {uploadPartSize, 0, uploadPartSize}, - {uploadPartSize + 1, 0, uploadPartSize}, + {DefaultUploadChunkSize - 1, 0, DefaultUploadChunkSize - 1}, + {DefaultUploadChunkSize, 0, DefaultUploadChunkSize}, + {DefaultUploadChunkSize + 1, 0, DefaultUploadChunkSize}, - {uploadPartSize*2 - 1, 1, uploadPartSize - 1}, - {uploadPartSize * 2, 1, uploadPartSize}, - {uploadPartSize*2 + 1, 1, uploadPartSize}, + {DefaultUploadChunkSize*2 - 1, 1, DefaultUploadChunkSize - 1}, + {DefaultUploadChunkSize * 2, 1, DefaultUploadChunkSize}, + {DefaultUploadChunkSize*2 + 1, 1, DefaultUploadChunkSize}, } func TestCalculatePartSize(t *testing.T) { for _, testCase := range calculatePartSizeProvider { t.Run(fmt.Sprintf("fileSize: %d partNumber: %d", testCase.fileSize, testCase.partNumber), func(t *testing.T) { - assert.Equal(t, testCase.expectedPartSize, calculatePartSize(testCase.fileSize, testCase.partNumber)) + assert.Equal(t, testCase.expectedPartSize, calculatePartSize(testCase.fileSize, testCase.partNumber, DefaultUploadChunkSize)) }) } } @@ -308,19 +308,19 @@ var calculateNumberOfPartsProvider = []struct { }{ {0, 0}, {1, 1}, - {uploadPartSize - 1, 1}, - {uploadPartSize, 1}, - {uploadPartSize + 1, 2}, + {DefaultUploadChunkSize - 1, 1}, + {DefaultUploadChunkSize, 1}, + {DefaultUploadChunkSize + 1, 2}, - {uploadPartSize*2 - 1, 2}, - {uploadPartSize * 2, 2}, - {uploadPartSize*2 + 1, 3}, + {DefaultUploadChunkSize*2 - 1, 2}, + {DefaultUploadChunkSize * 2, 2}, + {DefaultUploadChunkSize*2 + 1, 3}, } func TestCalculateNumberOfParts(t *testing.T) { for _, testCase := range calculateNumberOfPartsProvider { t.Run(fmt.Sprintf("fileSize: %d", testCase.fileSize), func(t *testing.T) { - assert.Equal(t, testCase.expectedNumberOfParts, calculateNumberOfParts(testCase.fileSize)) + assert.Equal(t, testCase.expectedNumberOfParts, calculateNumberOfParts(testCase.fileSize, DefaultUploadChunkSize)) }) } } diff --git a/artifactory/services/utils/storageutils.go b/artifactory/services/utils/storageutils.go index 8d29d1b7d..3cf22e799 100644 --- a/artifactory/services/utils/storageutils.go +++ b/artifactory/services/utils/storageutils.go @@ -3,6 +3,7 @@ package utils import ( "encoding/json" "errors" + "fmt" ) const ( @@ -126,3 +127,24 @@ type FileStoreSummary struct { UsedSpace string `json:"usedSpace,omitempty"` FreeSpace string `json:"freeSpace,omitempty"` } + +func ConvertIntToStorageSizeString(num int64) string { + if num > SizeTiB { + newNum := float64(num) / float64(SizeTiB) + stringNum := fmt.Sprintf("%.1f", newNum) + return stringNum + "TB" + } + if num > SizeGiB { + newNum := float64(num) / float64(SizeGiB) + stringNum := fmt.Sprintf("%.1f", newNum) + return stringNum + "GB" + } + if num > SizeMiB { + newNum := float64(num) / float64(SizeMiB) + stringNum := fmt.Sprintf("%.1f", newNum) + return stringNum + "MB" + } + newNum := float64(num) / float64(SizeKib) + stringNum := fmt.Sprintf("%.1f", newNum) + return stringNum + "KB" +} diff --git a/artifactory/services/utils/storageutils_test.go b/artifactory/services/utils/storageutils_test.go index 22d7f4e4b..35599077a 100644 --- a/artifactory/services/utils/storageutils_test.go +++ b/artifactory/services/utils/storageutils_test.go @@ -34,3 +34,21 @@ func buildFakeStorageInfo() StorageInfo { FileStoreSummary: FileStoreSummary{}, } } + +func TestConvertIntToStorageSizeString(t *testing.T) { + tests := []struct { + num int + output string + }{ + {12546, "12.3KB"}, + {148576, "145.1KB"}, + {2587985, "2.5MB"}, + {12896547, "12.3MB"}, + {12896547785, "12.0GB"}, + {5248965785422365, "4773.9TB"}, + } + + for _, test := range tests { + assert.Equal(t, test.output, ConvertIntToStorageSizeString(int64(test.num))) + } +} From 9d4dfaf050413deffe959c471921e00bf6f82641 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Tue, 30 Apr 2024 16:29:02 +0300 Subject: [PATCH 09/11] Support attaching Uber Trace ID to every request (#937) --- http/httpclient/client.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/http/httpclient/client.go b/http/httpclient/client.go index 658bd7f6e..b101be52c 100644 --- a/http/httpclient/client.go +++ b/http/httpclient/client.go @@ -36,8 +36,14 @@ type HttpClient struct { const ( apiKeyPrefix = "AKCp8" apiKeyMinimalLength = 73 + uberTraceIdHeader = "uber-trace-id" ) +// If set, the Uber Trace ID header will be attached to every request. +// This allows users to easily identify which logs on the server side are related to requests sent from this client. +// Should be set using SetUberTraceIdToken. +var uberTraceIdToken string + func IsApiKey(key string) bool { return strings.HasPrefix(key, apiKeyPrefix) && len(key) >= apiKeyMinimalLength } @@ -171,6 +177,7 @@ func (jc *HttpClient) doRequest(req *http.Request, content []byte, followRedirec setAuthentication(req, httpClientsDetails) addUserAgentHeader(req) copyHeaders(httpClientsDetails, req) + addUberTraceIdHeaderIfSet(req) if !followRedirect || (followRedirect && req.Method == http.MethodPost) { jc.client.CheckRedirect = func(req *http.Request, via []*http.Request) error { @@ -220,6 +227,21 @@ func copyHeaders(httpClientsDetails httputils.HttpClientDetails, req *http.Reque } } +// Generate an Uber Trace ID token that will be attached to every request. +// Format of the header: {trace-id}:{span-id}:{parent-span-id}:{flags} +// We set the trace-id and span-id to the same value, and the rest to 0. +func SetUberTraceIdToken(traceIdToken string) { + uberTraceIdToken = fmt.Sprintf("%s:%s:0:0", traceIdToken, traceIdToken) +} + +// If a trace ID is set, this function will attach the Uber Trace ID header to every request. +func addUberTraceIdHeaderIfSet(req *http.Request) { + if uberTraceIdToken == "" { + return + } + req.Header.Set(uberTraceIdHeader, uberTraceIdToken) +} + func setRequestHeaders(httpClientsDetails httputils.HttpClientDetails, size int64, req *http.Request) { copyHeaders(httpClientsDetails, req) length := strconv.FormatInt(size, 10) From 7974b79b45d8b4a0fd7767185fed63f0dd816464 Mon Sep 17 00:00:00 2001 From: Asaf Ambar Date: Thu, 2 May 2024 05:58:47 +0300 Subject: [PATCH 10/11] add classifier to graph (#947) --- xray/services/utils/graph.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xray/services/utils/graph.go b/xray/services/utils/graph.go index f4d09a485..58d7640b0 100644 --- a/xray/services/utils/graph.go +++ b/xray/services/utils/graph.go @@ -32,6 +32,8 @@ type OtherComponentIds struct { type GraphNode struct { // Node parent (for internal use) Parent *GraphNode `json:"-"` + // The "classifier" attribute in a Maven pom.xml specifies an additional qualifier for a dependency + Classifier *string `json:"-"` // Node file types (tar, jar, zip, pom) Types *[]string `json:"-"` Id string `json:"component_id,omitempty"` From d12abb9f140e755444e332d97ca3d3ee983de674 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Sun, 5 May 2024 19:43:07 +0300 Subject: [PATCH 11/11] Support STARTED Status for Release Bundle Creation (#948) --- lifecycle/services/status.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lifecycle/services/status.go b/lifecycle/services/status.go index a763da5c8..9aeed788e 100644 --- a/lifecycle/services/status.go +++ b/lifecycle/services/status.go @@ -34,6 +34,7 @@ const ( Failed RbStatus = "FAILED" Rejected RbStatus = "REJECTED" Deleting RbStatus = "DELETING" + Started RbStatus = "STARTED" ) func (rbs *ReleaseBundlesService) GetReleaseBundleCreationStatus(rbDetails ReleaseBundleDetails, projectKey string, sync bool) (ReleaseBundleStatusResponse, error) { @@ -128,7 +129,7 @@ func (rbs *ReleaseBundlesService) waitForRbOperationCompletion(restApi, projectK return true, nil, err } switch rbStatusResponse.Status { - case Pending, Processing: + case Pending, Processing, Started: return false, nil, nil case Completed, Rejected, Failed, Deleting: return true, responseBody, nil