From 12e0c2af73c297d33c6aa6e014ff0092af53ab6e Mon Sep 17 00:00:00 2001 From: Or Geva Date: Wed, 22 Mar 2023 08:16:17 -0700 Subject: [PATCH 01/15] Support for pipelines workspaces --- go.mod | 4 +- go.sum | 3 + pipelines/manager.go | 58 ++++++ pipelines/services/types.go | 68 +++++++ pipelines/services/validate.go | 124 ++++++++++++ pipelines/services/workspace.go | 318 ++++++++++++++++++++++++++++++ tests/jfrogclient_test.go | 1 + tests/pipelinesworkspaces_test.go | 210 ++++++++++++++++++++ tests/testdata/pip.yaml | 8 + tests/testdata/pipelines.yml | 22 +++ utils/antUtils.go | 77 ++++++++ utils/antUtils_test.go | 156 +++++++++++++++ utils/utils.go | 52 +---- utils/utils_test.go | 102 +--------- 14 files changed, 1055 insertions(+), 148 deletions(-) create mode 100644 pipelines/services/validate.go create mode 100644 pipelines/services/workspace.go create mode 100644 tests/pipelinesworkspaces_test.go create mode 100644 tests/testdata/pip.yaml create mode 100644 tests/testdata/pipelines.yml create mode 100644 utils/antUtils.go create mode 100644 utils/antUtils_test.go diff --git a/go.mod b/go.mod index 4123d7d3f..468bb57e8 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/buger/jsonparser v1.1.1 github.com/forPelevin/gomoji v1.1.8 + github.com/ghodss/yaml v1.0.0 github.com/go-git/go-git/v5 v5.6.1 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gookit/color v1.5.2 @@ -52,9 +53,10 @@ require ( golang.org/x/net v0.8.0 // indirect golang.org/x/sys v0.6.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.2.4 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) // replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.5-0.20230102085833-456b7aea6098 -// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.5-0.20221107113836-a4c9225c690e \ No newline at end of file +// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.5-0.20221107113836-a4c9225c690e diff --git a/go.sum b/go.sum index fb6b2b8fa..8aff6aab6 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/forPelevin/gomoji v1.1.8 h1:JElzDdt0TyiUlecy6PfITDL6eGvIaxqYH1V52zrd0qQ= github.com/forPelevin/gomoji v1.1.8/go.mod h1:8+Z3KNGkdslmeGZBC3tCrwMrcPy5GRzAD+gL9NAwMXg= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= @@ -195,6 +197,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pipelines/manager.go b/pipelines/manager.go index 9b9097363..184d6414b 100644 --- a/pipelines/manager.go +++ b/pipelines/manager.go @@ -134,3 +134,61 @@ func (sm *PipelinesServicesManager) CancelRun(runID int) error { runService.ServiceDetails = sm.config.GetServiceDetails() return runService.CancelRun(runID) } + +func (sm *PipelinesServicesManager) ValidatePipelineSources(data []byte) (string, error) { + validateService := services.NewValidateService(sm.client) + validateService.ServiceDetails = sm.config.GetServiceDetails() + return validateService.ValidatePipeline(data) +} + +func (sm *PipelinesServicesManager) WorkspacePollSyncStatus() ([]services.WorkspacesResponse, error) { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + return workspaceService.WorkspacePollSyncStatus() +} + +func (sm *PipelinesServicesManager) WorkspacePipelines() (map[string]string, error) { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + workspacesResponseStatus, err := sm.WorkspacePollSyncStatus() + if err != nil { + return nil, err + } + return workspaceService.GetWorkspacePipelines(workspacesResponseStatus) +} + +func (sm *PipelinesServicesManager) WorkspaceTriggerPipelines(branch, pipName string, isMultiBranch bool) error { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + return sm.TriggerPipelineRun(branch, pipName, isMultiBranch) +} + +func (sm *PipelinesServicesManager) WorkspaceRunIDs(pipelineNames []string) ([]services.PipelinesRunID, error) { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + return workspaceService.WorkspaceRunIDs(pipelineNames) +} + +func (sm *PipelinesServicesManager) WorkspaceRunStatus(pipeRunID int) ([]byte, error) { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + return workspaceService.WorkspaceRunStatus(pipeRunID) +} + +func (sm *PipelinesServicesManager) WorkspaceStepStatus(pipeRunID int) ([]byte, error) { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + return workspaceService.WorkspaceStepStatus(pipeRunID) +} + +func (sm *PipelinesServicesManager) ValidateWorkspace(data []byte) error { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + return workspaceService.ValidateWorkspace(data) +} + +func (sm *PipelinesServicesManager) GetStepConsoles(stepID string) (map[string][]services.Console, error) { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + return workspaceService.GetStepLogsUsingStepID(stepID) +} diff --git a/pipelines/services/types.go b/pipelines/services/types.go index 9e9b332ef..01dc2e52c 100644 --- a/pipelines/services/types.go +++ b/pipelines/services/types.go @@ -87,3 +87,71 @@ type PipelineResources struct { CreatedAt time.Time `json:"createdAt,omitempty"` UpdatedAt time.Time `json:"updatedAt,omitempty"` } + +type WorkspacesResponse struct { + ValuesYmlPropertyBag interface{} `json:"valuesYmlPropertyBag,omitempty"` + PipelinesYmlPropertyBag PipelinesYmlPropertyBag `json:"pipelinesYmlPropertyBag,omitempty"` + ID int `json:"id,omitempty"` + Name string `json:"name,omitempty"` + ProjectID int `json:"projectId,omitempty"` + IsSyncing *bool `json:"isSyncing,omitempty"` + LastSyncStatusCode int `json:"lastSyncStatusCode,omitempty"` + LastSyncStartedAt time.Time `json:"lastSyncStartedAt,omitempty"` + LastSyncEndedAt interface{} `json:"lastSyncEndedAt,omitempty"` + LastSyncLogs string `json:"lastSyncLogs,omitempty"` + CreatedBy int `json:"createdBy,omitempty"` +} + +type PipelinesYmlPropertyBag struct { + Resources []Resources `json:"resources,omitempty"` + Pipelines []Pipelines `json:"pipelines,omitempty"` +} + +type Resources struct { + Configuration Configuration `json:"configuration,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` +} + +type Configuration struct { + Branches Branches `json:"branches,omitempty"` + GitProvider string `json:"gitProvider,omitempty"` + Path string `json:"path,omitempty"` +} + +type Branches struct { + Include string `json:"include,omitempty"` +} + +type PipelinesRunID struct { + LatestRunID int `json:"latestRunId,omitempty"` + Name string `json:"name,omitempty"` + SyntaxVersion string `json:"syntaxVersion,omitempty"` +} + +type Console struct { + ConsoleID string `json:"consoleId"` + IsSuccess interface{} `json:"isSuccess"` + IsShown bool `json:"isShown"` + ParentConsoleID string `json:"parentConsoleId"` + StepletID int `json:"stepletId"` + PipelineID int `json:"pipelineId"` + Timestamp int64 `json:"timestamp"` + TimestampEndedAt interface{} `json:"timestampEndedAt"` + Type string `json:"type"` + Message string `json:"message"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// Validation Types + +type ValidationResponse struct { + IsValid *bool `json:"isValid,omitempty"` + Errors []ValidationErrors `json:"errors,omitempty"` +} + +type ValidationErrors struct { + Text string `json:"text,omitempty"` + LineNumber int `json:"lineNumber,omitempty"` +} diff --git a/pipelines/services/validate.go b/pipelines/services/validate.go new file mode 100644 index 000000000..758706523 --- /dev/null +++ b/pipelines/services/validate.go @@ -0,0 +1,124 @@ +package services + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/gookit/color" + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/io/httputils" + "github.com/jfrog/jfrog-client-go/utils/log" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +const ( + validatePipResourcePath = "api/v1/validateYaml/json" +) + +type ValidateService struct { + client *jfroghttpclient.JfrogHttpClient + auth.ServiceDetails +} + +func NewValidateService(client *jfroghttpclient.JfrogHttpClient) *ValidateService { + return &ValidateService{client: client} +} + +func (rs *ValidateService) getHttpDetails() httputils.HttpClientDetails { + httpDetails := rs.ServiceDetails.CreateHttpClientDetails() + return httpDetails +} + +func (vs *ValidateService) ValidatePipeline(data []byte) (string, error) { + var opMsg string + var err error + httpDetails := vs.getHttpDetails() + + // query params + m := make(map[string]string, 0) + + // URL Construction + uri := vs.constructValidateAPIURL(m, validatePipResourcePath) + + // headers + headers := make(map[string]string, 0) + headers["Content-Type"] = "application/json" + httpDetails.Headers = headers + + // send post request + resp, body, err := vs.client.SendPost(uri, data, &httpDetails) + if err != nil { + return "", err + } + if err := errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return "", err + } + + // process response + if nil != resp && resp.StatusCode == http.StatusOK { + log.Info("Processing resources yaml response ") + time.Sleep(2 * time.Second) + log.Info(string(body)) + opMsg, err = processValidatePipResourceResponse(body, "") + if err != nil { + log.Error("Failed to process resources validation response") + } + } + return opMsg, nil +} + +// processValidatePipResourceResponse process validate pipeline resource response +func processValidatePipResourceResponse(resp []byte, userName string) (string, error) { + fmt.Println("unfurling response") + rsc := make(map[string]ValidationResponse) + err := json.Unmarshal(resp, &rsc) + if err != nil { + return "", err + } + if v, ok := rsc["pipelines.yml"]; ok { + if v.IsValid != nil && *v.IsValid { + userName = "@bhanur" + log.Info("validation of pipeline resources completed successfully ") + log.Info("workspace updated with latest resource files for user: ", userName) + msg := color.Green.Sprintf("validation completed ") + time.Sleep(2 * time.Second) + log.Info(msg) + return msg, nil + } else { + log.Error("pipeline resources validation FAILED!! check below errors and try again") + msg := v.Errors[0].Text + lnNum := v.Errors[0].LineNumber + msgs := strings.Split(msg, ":") + opMsg := color.Red.Sprintf("%s", msgs[0]+":"+strconv.Itoa(lnNum)+":\n"+" "+msgs[1]) + //log.PrintMessage("Please refer pipelines documentation " + coreutils.PrintLink("https://www.jfrog.com/confluence/display/JFROG/Managing+Pipeline+Sources#ManagingPipelineSources-ValidatingYAML") + "") + log.Error(opMsg) + return opMsg, nil + } + } + return "", errors.New("pipelines.yml not found") +} + +/* +constructPipelinesURL creates URL with all required details to make api call +like headers, queryParams, apiPath +*/ +func (rs *ValidateService) constructValidateAPIURL(qParams map[string]string, apiPath string) string { + uri, err := url.Parse(rs.ServiceDetails.GetUrl() + apiPath) + if err != nil { + log.Error("Failed to parse pipelines fetch run status url") + } + + queryString := uri.Query() + for k, v := range qParams { + queryString.Set(k, v) + } + uri.RawQuery = queryString.Encode() + + return uri.String() +} diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go new file mode 100644 index 000000000..5af80a15e --- /dev/null +++ b/pipelines/services/workspace.go @@ -0,0 +1,318 @@ +package services + +import ( + "encoding/json" + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/io/httputils" + "github.com/jfrog/jfrog-client-go/utils/log" + "net/http" + "strconv" + "strings" + "time" +) + +const ( + workspaces = "api/v1/workspaces" + deleteWorkspace = "api/v1/workspaces/:id" + validateWorkspace = "api/v1/validateWorkspace" + workspaceSync = "api/v1/syncWorkspace" + workspacePipelines = "api/v1/pipelines" + workspaceRuns = "api/v1/runs" + workspaceSteps = "api/v1/steps" + stepConsoles = "api/v1/steplets/:stepID/consoles" +) + +type WorkspaceService struct { + client *jfroghttpclient.JfrogHttpClient + auth.ServiceDetails +} + +func NewWorkspaceService(client *jfroghttpclient.JfrogHttpClient) *WorkspaceService { + return &WorkspaceService{client: client} +} + +func (ws *WorkspaceService) getHttpDetails() httputils.HttpClientDetails { + return ws.ServiceDetails.CreateHttpClientDetails() +} + +func (ws *WorkspaceService) GetWorkspace() ([]WorkspacesResponse, error) { + httpDetails := ws.getHttpDetails() + + // Query params + queryParams := make(map[string]string, 0) + + // URL construction + uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaces) + if err != nil { + return nil, err + } + + // Prepare request + resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) + if err != nil { + return nil, err + } + + // Response Analysis + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return nil, err + } + wsStatusResp := make([]WorkspacesResponse, 0) + err = json.Unmarshal(body, &wsStatusResp) + return wsStatusResp, nil +} + +func (ws *WorkspaceService) DeleteWorkspace(wsID string) error { + httpDetails := ws.getHttpDetails() + deleteWorkspaceAPI := strings.Replace(deleteWorkspace, ":id", wsID, 1) + + // Query params + queryParams := make(map[string]string, 0) + + // URL construction + uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), deleteWorkspaceAPI) + if err != nil { + return err + } + + // Prepare request + resp, body, err := ws.client.SendDelete(uri, nil, &httpDetails) + if err != nil { + return err + } + + // Response analysis + err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) + return err +} + +func (ws *WorkspaceService) ValidateWorkspace(data []byte) error { + httpDetails := ws.getHttpDetails() + + // Query params + queryParams := make(map[string]string, 0) + + // URL construction + uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), validateWorkspace) + if err != nil { + return err + } + + // Headers + headers := make(map[string]string, 0) + headers["Content-Type"] = "application/json" + httpDetails.Headers = headers + + // Prepare request + resp, body, err := ws.client.SendPost(uri, data, &httpDetails) + if err != nil { + return err + } + + // Response analysis + return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) +} + +func (ws *WorkspaceService) WorkspaceSync() error { + httpDetails := ws.getHttpDetails() + + // Query params + queryParams := make(map[string]string, 0) + + // URL construction + uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaceSync) + if err != nil { + return err + } + + // Prepare request + resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) + if err != nil { + return err + } + + // Response analysis + return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) +} + +func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunID, error) { + httpDetails := ws.getHttpDetails() + pipelineFilter := strings.Join(pipelines, ",") + + // Query params + // TODO ADD include in query param if needed + queryParams := map[string]string{ + "names": pipelineFilter, + "limit": "1", + "include": "latestRunId,name", + } + + // URL construction + uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspacePipelines) + if err != nil { + return nil, err + } + + // Prepare request + resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) + if err != nil { + return nil, err + } + // Response analysis + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return nil, err + } + pipeRunIDs := make([]PipelinesRunID, 0) + err = json.Unmarshal(body, &pipeRunIDs) + return pipeRunIDs, err +} + +func (ws *WorkspaceService) WorkspaceRunStatus(pipelinesRunID int) ([]byte, error) { + httpDetails := ws.getHttpDetails() + + // Query params + // TODO ADD include in query param if needed + queryParams := map[string]string{} + + // URL construction + uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaceRuns+"/"+strconv.Itoa(pipelinesRunID)) + if err != nil { + return nil, err + } + + // Prepare request + resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) + if err != nil { + return nil, err + } + // Response analysis + err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) + return body, err +} + +func (ws *WorkspaceService) WorkspaceStepStatus(pipelinesRunID int) ([]byte, error) { + httpDetails := ws.getHttpDetails() + + // Query params + queryParams := map[string]string{ + "runIds": strconv.Itoa(pipelinesRunID), + "limit": "15", + //"include": "name,statusCode,triggeredAt,externalBuildUrl", + } + + // URL construction + uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaceSteps) + if err != nil { + return nil, err + } + + // Prepare request + resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) + if err != nil { + return nil, err + } + + // Response analysis + err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) + return body, err +} + +func (ws *WorkspaceService) GetWorkspacePipelines(workspaces []WorkspacesResponse) (map[string]string, error) { + pipelineNames := make(map[string]string, 1) + + log.Info("Collecting pipeline names configured") + // Validate and return pipeline names as slice + if len(workspaces) > 0 && !(*workspaces[0].IsSyncing) { + pipelines := workspaces[0].PipelinesYmlPropertyBag.Pipelines + for _, pi := range pipelines { + pipelineNames[pi.Name] = pi.PipelineSourceBranch + } + } + return pipelineNames, nil +} + +func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, error) { + httpDetails := ws.getHttpDetails() + + // Query params + queryParams := make(map[string]string, 0) + + // URL construction + uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaces) + if err != nil { + return nil, err + } + + pollingAction := func() (shouldStop bool, responseBody []byte, err error) { + log.Info("Polling for pipeline resource sync") + // Prepare request + resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) + if err != nil { + return false, body, err + } + + // Response Analysis + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return false, body, err + } + log.Debug("response received ", string(body)) + wsStatusResp := make([]WorkspacesResponse, 0) + err = json.Unmarshal(body, &wsStatusResp) + if err != nil { + log.Error("failed to unmarshal validation response") + return true, body, err + } + if len(wsStatusResp) > 0 && *wsStatusResp[0].IsSyncing { + return false, body, err + } else if len(wsStatusResp) > 0 && !(*wsStatusResp[0].IsSyncing) { + log.Info("Sync is completed") + return true, body, err + } + return true, body, err + } + pollingExecutor := &httputils.PollingExecutor{ + Timeout: 10 * time.Minute, + PollingInterval: 5 * time.Second, + PollingAction: pollingAction, + MsgPrefix: "Get pipeline workspace sync status...", + } + // Polling execution + body, err := pollingExecutor.Execute() + if err != nil { + return nil, err + } + workspaceStatusResponse := make([]WorkspacesResponse, 0) + err = json.Unmarshal(body, &workspaceStatusResponse) + return workspaceStatusResponse, err +} + +// GetStepLogsUsingStepID retrieve steps logs using step id +func (ws *WorkspaceService) GetStepLogsUsingStepID(stepID string) (map[string][]Console, error) { + httpDetails := ws.getHttpDetails() + stepConsolesAPI := strings.Replace(stepConsoles, ":stepID", stepID, 1) + + // Query params + queryParams := make(map[string]string, 0) + + // URL construction + uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), stepConsolesAPI) + if err != nil { + return nil, err + } + + // Prepare request + resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) + if err != nil { + return nil, err + } + + // Response Analysis + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return nil, err + } + consoles := make(map[string][]Console) + err = json.Unmarshal(body, &consoles) + return consoles, err +} diff --git a/tests/jfrogclient_test.go b/tests/jfrogclient_test.go index f48602460..20a70c71d 100644 --- a/tests/jfrogclient_test.go +++ b/tests/jfrogclient_test.go @@ -71,6 +71,7 @@ func setupIntegrationTests() { createPipelinesRunManager() createPipelinesSyncManager() createPipelinesSyncStatusManager() + createPipelinesWorkspaceServiceManager() } if *TestAccess { createAccessPingManager() diff --git a/tests/pipelinesworkspaces_test.go b/tests/pipelinesworkspaces_test.go new file mode 100644 index 000000000..ad98e6e14 --- /dev/null +++ b/tests/pipelinesworkspaces_test.go @@ -0,0 +1,210 @@ +package tests + +import ( + "bytes" + "encoding/json" + "fmt" + "github.com/ghodss/yaml" + "github.com/jfrog/jfrog-client-go/utils/log" + "github.com/stretchr/testify/assert" + "io" + "os" + "strings" + "testing" + "time" +) + +func testWorkspaceValidationWhenPipelinesResourcesAreNotValid(t *testing.T) { + + if !assert.NotEmpty(t, *PipelinesAccessToken, "cannot run pipelines tests without access token configured") { + return + } + filePath := "/Users/bhanur/go/src/pipeline-cli/jfrog-client-go/tests/testdata/pipelines.yml" + + // get pipelines.yaml content + pipelineRes, err := getPipelinesResourceToValidate(filePath) + fmt.Printf("Bytes received : %s \n", pipelineRes) + if !assert.NoError(t, err) { + return + } + err = testPipelinesWorkspaceService.ValidateWorkspace(pipelineRes) + assert.NoError(t, err) +} + +func testWorkspaceValidationWhenPipelinesResourcesAreValid(t *testing.T) { + if !assert.NotEmpty(t, *PipelinesAccessToken, "cannot run pipelines tests without access token configured") { + return + } + filePath := "/Users/bhanur/go/src/pipeline-cli/jfrog-client-go/tests/testdata/pipelines.yml" + + integrationName := "int_gh_pipe_cli" + _, err := testsPipelinesIntegrationsService.CreateGithubIntegration(integrationName, *PipelinesVcsToken) + assert.NoError(t, err) + //defer deleteIntegrationAndAssert(t, integrationId) + + // get pipelines.yaml content + pipelineRes, err := getPipelinesResourceToValidate(filePath) + if !assert.NoError(t, err) { + return + } + err = testPipelinesWorkspaceService.ValidateWorkspace(pipelineRes) + assert.NoError(t, err) + + workspaces, wsErr := testPipelinesWorkspaceService.WorkspacePollSyncStatus() + assert.NoError(t, wsErr) + pipelineBranch, err := testPipelinesWorkspaceService.GetWorkspacePipelines(workspaces) + assert.NoError(t, err) + for pipName, branch := range pipelineBranch { + err = testPipelinesRunService.TriggerPipelineRun(branch, pipName, false) + assert.NoError(t, err) + } + pipelineNames := make([]string, len(pipelineBranch)) + + i := 0 + for k := range pipelineBranch { + pipelineNames[i] = k + i++ + } + pipeRunIDs, wsRunErr := testPipelinesWorkspaceService.WorkspaceRunIDs(pipelineNames) + assert.NoError(t, wsRunErr) + + for _, runId := range pipeRunIDs { + _, err2 := testPipelinesWorkspaceService.WorkspaceRunStatus(runId.LatestRunID) + assert.NoError(t, err2) + _, err3 := testPipelinesWorkspaceService.WorkspaceStepStatus(runId.LatestRunID) + assert.NoError(t, err3) + } + +} + +func getPipelinesResourceToValidate(filePath string) ([]byte, error) { + + ymlType := "" + readFile, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + fInfo, err := os.Stat(filePath) + if err != nil { + return nil, err + } + + toJSON, err3 := convertYAMLToJSON(err, readFile) + if err3 != nil { + return nil, err3 + } + vsc, err4 := convertJSONDataToMap(fInfo, toJSON) + if err4 != nil { + return nil, err4 + } + var marErr error + + resMap, err5, done := splitDataToPipelinesAndResourcesMap(vsc, marErr, err, ymlType) + if done { + return nil, err5 + } + + data, resErr := getPayloadToValidatePipelineResource(resMap) + if resErr != nil { + return nil, resErr + } + + return data.Bytes(), nil +} + +func convertJSONDataToMap(file os.FileInfo, toJSON []byte) (map[string][]interface{}, error) { + log.Info("validating pipeline resources ", file.Name()) + time.Sleep(1 * time.Second) + vsc := make(map[string][]interface{}) + convErr := yaml.Unmarshal(toJSON, &vsc) + if convErr != nil { + return nil, convErr + } + return vsc, nil +} + +func convertYAMLToJSON(err error, readFile []byte) ([]byte, error) { + toJSON, err := yaml.YAMLToJSON(readFile) + if err != nil { + log.Error("Failed to convert to json") + return nil, err + } + return toJSON, nil +} + +func splitDataToPipelinesAndResourcesMap(vsc map[string][]interface{}, marErr error, err error, ymlType string) (map[string]string, error, bool) { + resMap := make(map[string]string) + var data []byte + if v, ok := vsc["resources"]; ok { + log.Info("resources found preparing to validate") + data, marErr = json.Marshal(v) + if marErr != nil { + log.Error("failed to marshal to json") + return nil, err, true + } + + ymlType = "resources" + resMap[ymlType] = string(data) + + } + if vp, ok := vsc["pipelines"]; ok { + log.Info("pipelines found preparing to validate") + data, marErr = json.Marshal(vp) + if marErr != nil { + log.Error("failed to marshal to json") + return nil, err, true + } + ymlType = "pipelines" + fmt.Println(string(data)) + resMap[ymlType] = string(data) + } + //fmt.Printf("%+v \n", resMap) + return resMap, nil, false +} + +func getPayloadToValidatePipelineResource(resMap map[string]string) (*bytes.Buffer, error) { + payload := getPayloadBasedOnYmlType(resMap) + buf := new(bytes.Buffer) + _, err := buf.ReadFrom(payload) + if err != nil { + log.Error("Failed to read stream to send payload to trigger pipelines") + return nil, err + } + return buf, err +} + +func getPayloadBasedOnYmlType(m map[string]string) *strings.Reader { + var resReader, pipReader, valReader *strings.Reader + for ymlType, _ := range m { + if ymlType == "resources" { + resReader = strings.NewReader(`{"fileName":"` + ymlType + `.yml","content": ` + m[ymlType] + `,"ymlType":"` + ymlType + `"}`) + } else if ymlType == "pipelines" { + //fmt.Printf("data : %+v \n", m[ymlType]) + pipReader = strings.NewReader(`{"fileName":"` + ymlType + `.yml","content":` + m[ymlType] + `,"ymlType":"` + ymlType + `"}`) + } + } + if resReader != nil && pipReader != nil { + resAll, err := io.ReadAll(resReader) + if err != nil { + return nil + } + pipAll, err := io.ReadAll(pipReader) + if err != nil { + return nil + } + valReader = strings.NewReader(`{"files":[` + string(resAll) + `,` + string(pipAll) + `]}`) + } else if resReader != nil { + all, err := io.ReadAll(resReader) + if err != nil { + return nil + } + valReader = strings.NewReader(`{"files":[` + string(all) + `]}`) + } else if pipReader != nil { + all, err := io.ReadAll(pipReader) + if err != nil { + return nil + } + valReader = strings.NewReader(`{"files":[` + string(all) + `]}`) + } + return valReader +} diff --git a/tests/testdata/pip.yaml b/tests/testdata/pip.yaml new file mode 100644 index 000000000..c1ac9a8ff --- /dev/null +++ b/tests/testdata/pip.yaml @@ -0,0 +1,8 @@ +pipelines: +- name: test + steps: + - name: test1 + type: bash + execution: + onSuccess: + - echo "test" diff --git a/tests/testdata/pipelines.yml b/tests/testdata/pipelines.yml new file mode 100644 index 000000000..8c3d7f432 --- /dev/null +++ b/tests/testdata/pipelines.yml @@ -0,0 +1,22 @@ +pipelines: + - name: pipelines_run_int_test + steps: + - name: prepare_env + type: Bash + execution: + onExecute: + - cd /tmp + - mkdir -p "jfrog-cli" + - touch test-pilines.yaml + - echo "test" >> test-pipelines.yaml + - name: build_binary + type: Bash + configuration: + inputSteps: + - name: prepare_env + execution: + onExecute: + - cd /tmp + - mkdir -p "target/pipelines" + - cd target/pipelines + - touch make_data \ No newline at end of file diff --git a/utils/antUtils.go b/utils/antUtils.go new file mode 100644 index 000000000..bc11d9ca9 --- /dev/null +++ b/utils/antUtils.go @@ -0,0 +1,77 @@ +package utils + +import ( + "fmt" + "os" + "regexp" + "strings" + + "github.com/jfrog/gofrog/stringutils" + "github.com/jfrog/jfrog-client-go/utils/io" +) + +var ( + // Replace ** with a special string. + doubleStartSpecialString = "__JFROG_DOUBLE_STAR__" + + // Match **/ ('__JFROG_DOUBLE_STAR__\/...') + prefixDoubleStarRegex = regexp.MustCompile(fmt.Sprintf("%s%s", doubleStartSpecialString, getFileSeparatorForDoubleStart())) + + // match /** ('\/__JFROG_DOUBLE_STAR__...') + postfixDoubleStarRegex = regexp.MustCompile(fmt.Sprintf("%s%s", getFileSeparatorForDoubleStart(), doubleStartSpecialString)) + + // match ** ('...__JFROG_DOUBLE_STAR__...') + middleDoubleStarNoSeparateRegex = regexp.MustCompile(doubleStartSpecialString) +) + +func getFileSeparatorForDoubleStart() string { + if io.IsWindows() { + return `\\\\` + } + return `\/` +} + +func AntToRegex(antPattern string) string { + antPattern = stringutils.EscapeSpecialChars(antPattern) + antPattern = antQuestionMarkToRegex(antPattern) + return "^" + antStarsToRegex(antPattern) + "$" +} + +func getFileSeparatorForAntToRegex() string { + if io.IsWindows() { + return `\\` + } + return `/` +} + +func antStarsToRegex(antPattern string) string { + separator := getFileSeparatorForAntToRegex() + antPattern = addMissingShorthand(antPattern) + + // Replace ** with a special string, so it doesn't get mixed up with single * + antPattern = strings.ReplaceAll(antPattern, "**", doubleStartSpecialString) + + // ant `*` => regexp `([^/]*)` : `*` matches zero or more characters except from `/`. + antPattern = strings.ReplaceAll(antPattern, `*`, "([^"+separator+"]*)") + + // ant `**/` => regexp `(.*/)*` : Matches zero or more 'directories' at the beginning of the path. + antPattern = prefixDoubleStarRegex.ReplaceAllString(antPattern, "(.*"+separator+")*") + + // ant `/**` => regexp `(/.*)*` : Matches zero or more 'directories' at the end of the path. + antPattern = postfixDoubleStarRegex.ReplaceAllString(antPattern, "("+separator+".*)*") + + // ant `**` => regexp `(.*)*` : Matches zero or more 'directories'. + return middleDoubleStarNoSeparateRegex.ReplaceAllString(antPattern, "(.*)") +} + +func antQuestionMarkToRegex(antPattern string) string { + return strings.ReplaceAll(antPattern, "?", ".") +} + +func addMissingShorthand(antPattern string) string { + // There is one "shorthand": if a pattern ends with / or \, then ** is appended. For example, mypackage/test/ is interpreted as if it were mypackage/test/**. + if strings.HasSuffix(antPattern, string(os.PathSeparator)) { + return antPattern + "**" + } + return antPattern +} diff --git a/utils/antUtils_test.go b/utils/antUtils_test.go new file mode 100644 index 000000000..9b05e1e6c --- /dev/null +++ b/utils/antUtils_test.go @@ -0,0 +1,156 @@ +package utils + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +var separator = string(os.PathSeparator) + +var paths = getFileSystemsPathsForTestingAntPattern(separator) + +var testAntPathToRegExpDataProvider = []struct { + description string + antPattern string + paths []string + expectedMatchingPaths []string +}{ + {"check '?' in file's name", filepath.Join("dev", "a", "b?.txt"), paths, []string{filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt")}}, + {"check '?' in directory's name", filepath.Join("dev", "a?", "b.txt"), paths, []string{filepath.Join("dev", "aa", "b.txt")}}, + {"check '*' in file's name", filepath.Join("dev", "a", "b*.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt")}}, + {"check '*' in directory's name", filepath.Join("dev", "*", "b.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "aa", "b.txt")}}, + {"check '*' in directory's name", filepath.Join("dev", "*", "a", "b.txt"), paths, nil}, + {"check '**' in directory path", filepath.Join("**", "b.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "aa", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("test", "a", "b.txt"), filepath.Join("test", "aa", "b.txt")}}, + {"check '**' in the beginning and the end of path", filepath.Join("**", "a2", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt"), "a2"}}, + {"check '**' in the beginning and the end of path", filepath.Join("**a2**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt"), "a2"}}, + {"check double '**'", filepath.Join("**", "a2", "**", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"check '**' in the beginning and the end of file", filepath.Join("**", "b.zip", "**"), paths, []string{filepath.Join("dev", "aa", "b.zip"), filepath.Join("test", "aa", "b.zip"), filepath.Join("b.zip"), filepath.Join("test2", "b.zip")}}, + {"combine '**' and '*'", filepath.Join("**", "a2", "*"), paths, []string{filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"combine '**' and '*'", filepath.Join("**", "a2", "*", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"combine all signs", filepath.Join("**", "b?.*"), paths, []string{filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt"), filepath.Join("dev", "aa", "bb.txt"), filepath.Join("dev", "aa", "bc.txt"), filepath.Join("dev", "aa", "bc.zip"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt"), filepath.Join("test", "a", "bb.txt"), filepath.Join("test", "a", "bc.txt"), filepath.Join("test", "aa", "bb.txt"), filepath.Join("test", "aa", "bc.txt"), filepath.Join("test", "aa", "bc.zip")}}, + {"'**' all files", filepath.Join("**"), paths, paths}, + {"test2/**/b/**", filepath.Join("test2", "**", "b", "**"), paths, []string{filepath.Join("test2", "a", "b", "c.zip")}}, + {"*/b.zip", filepath.Join("*", "b.zip"), paths, []string{filepath.Join("test2", "b.zip")}}, + {"**/dev/**/a3/*c*", filepath.Join("dev", "**", "a3", "*c*"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "bc.txt")}}, + {"**/dev/**/a3/**", filepath.Join("dev", "**", "a3", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "a3", "b.txt")}}, + {"exclude 'temp/foo5/a'", filepath.Join("**", "foo", "**"), paths, []string{filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo")}}, + {"include dirs", filepath.Join("tmp", "*", "**"), paths, []string{"tmp" + separator, filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo5", "a"), filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5")}}, + {"include dirs", filepath.Join("tmp", "**"), paths, []string{"tmp" + separator, filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo5", "a"), filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5")}}, + {"double and single wildcard", filepath.Join("**", "tmp*", "**"), paths, []string{"tmp" + separator, filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo5", "a"), filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5"), filepath.Join("Wrapper", "tmp", "boo"), filepath.Join("Wrapper", "tmp12", "boo")}}, + {"exclude only sub dir", filepath.Join("**", "loo", "**", "bar", "**"), paths, []string{filepath.Join("kmp", "loo", "bar"), filepath.Join("kmp", "loo", "bar", "b"), filepath.Join("kmp", "loo", "bar", "a")}}, + {"**/", "**" + separator, paths, paths}, + {"xxx/x*", filepath.Join("tmp", "f*"), paths, []string{filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5")}}, + {"xxx/x*x", filepath.Join("tmp", "f*5"), paths, []string{filepath.Join("tmp", "foo5")}}, + {"xxx/x*", filepath.Join("dev", "a1", "a2", "b*"), paths, []string{filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"xxx/*x*", filepath.Join("dev", "a1", "a2", "*c*"), paths, []string{filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"*", filepath.Join("*"), paths, []string{"b.zip", "a2"}}, + {"*", filepath.Join("*"), []string{"a", "a" + separator, filepath.Join("a", "b")}, []string{"a"}}, +} + +// In each case, we take an array of paths, simulating a filesystem hierarchy, and an ANT pattern expression and +// check if the conversion to regular expression worked. +func TestAntPathToRegExp(t *testing.T) { + for _, test := range testAntPathToRegExpDataProvider { + t.Run(test.description, func(t *testing.T) { + regExpStr := AntToRegex(cleanPath(test.antPattern)) + var matches []string + for _, checkedPath := range test.paths { + match, _ := regexp.MatchString(regExpStr, checkedPath) + if match { + matches = append(matches, checkedPath) + } + } + if !equalSlicesIgnoreOrder(matches, test.expectedMatchingPaths) { + t.Error("Unmatched! : ant pattern `" + test.antPattern + "` matches paths:\n[" + strings.Join(test.expectedMatchingPaths, ",") + "]\nbut got:\n[" + strings.Join(matches, ",") + "]") + } + }) + } +} + +var testAntToRegexProvider = []struct { + ant string + expectedRegex string +}{ + {"a.zip", "^a\\.zip$"}, + {"ab", "^ab$"}, + {"**", "^(.*)$"}, + {"**/", "^(.*/)*(.*)$"}, + {"**/*", "^(.*/)*([^/]*)$"}, + {"/**", "^(/.*)*$"}, + {"*/**", "^([^/]*)(/.*)*$"}, + {"/**/ab", "^/(.*/)*ab$"}, + {"/**/ab*", "^/(.*/)*ab([^/]*)$"}, + {"/**/ab/", "^/(.*/)*ab(/.*)*$"}, + {"/**/ab/*", "^/(.*/)*ab/([^/]*)$"}, + {"/**/ab*/", "^/(.*/)*ab([^/]*)(/.*)*$"}, + {"ab/**/", "^ab/(.*/)*(.*)$"}, + {"*ab/**/", "^([^/]*)ab/(.*/)*(.*)$"}, + {"/ab/**/", "^/ab/(.*/)*(.*)$"}, + {"/ab*/**/", "^/ab([^/]*)/(.*/)*(.*)$"}, + {"/**/ab/**/", "^/(.*/)*ab/(.*/)*(.*)$"}, + {"/**/a*b/**/", "^/(.*/)*a([^/]*)b/(.*/)*(.*)$"}, + {"/**/ab/**/cd/**/ef/", "^/(.*/)*ab/(.*/)*cd/(.*/)*ef(/.*)*$"}, +} + +func TestAntToRegex(t *testing.T) { + for _, test := range testAntToRegexProvider { + t.Run("'"+test.ant+"'", func(t *testing.T) { + regExpStr := AntToRegex(cleanPath(strings.ReplaceAll(test.ant, "/", separator))) + expectedRegExpStr := strings.ReplaceAll(test.expectedRegex, "/", getFileSeparatorForAntToRegex()) + if regExpStr != expectedRegExpStr { + t.Error("Unmatched! : ant pattern `" + test.ant + "` translated to:\n" + regExpStr + "\nbut expect it to be:\n" + expectedRegExpStr + "") + } + }) + } +} + +func getFileSystemsPathsForTestingAntPattern(separator string) []string { + return []string{ + filepath.Join("dev", "a", "b.txt"), + filepath.Join("dev", "a", "bb.txt"), + filepath.Join("dev", "a", "bc.txt"), + filepath.Join("dev", "aa", "b.txt"), + filepath.Join("dev", "aa", "bb.txt"), + filepath.Join("dev", "aa", "bc.txt"), + filepath.Join("dev", "aa", "b.zip"), + filepath.Join("dev", "aa", "bc.zip"), + filepath.Join("dev", "a1", "a2", "a3", "b.txt"), + filepath.Join("dev", "a1", "a2", "b.txt"), + filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), + filepath.Join("dev", "a1", "a2", "bc.txt"), + "a2", + filepath.Join("test", "a", "b.txt"), + filepath.Join("test", "a", "bb.txt"), + filepath.Join("test", "a", "bc.txt"), + filepath.Join("test", "aa", "b.txt"), + filepath.Join("test", "aa", "bb.txt"), + filepath.Join("test", "aa", "bc.txt"), + filepath.Join("test", "aa", "b.zip"), + filepath.Join("test", "aa", "bc.zip"), + + filepath.Join("test2", "a", "b", "c.zip"), + filepath.Join("test2", "a", "bb", "c.zip"), + filepath.Join("test2", "b.zip"), + filepath.Join("b.zip"), + "tmp" + separator, + filepath.Join("tmp", "foo"), + filepath.Join("Wrapper", "tmp", "boo"), + filepath.Join("Wrapper", "tmp12", "boo"), + filepath.Join("tmp", "foo", "a"), + filepath.Join("tmp", "foo5"), + filepath.Join("tmp", "foo5", "a"), + + filepath.Join("kmp", "loo"), + filepath.Join("kmp", "loo", "bar", "a"), + filepath.Join("kmp", "loo", "bar", "b"), + filepath.Join("kmp", "loo", "bar"), + filepath.Join("kmp", "loo", "lar"), + filepath.Join("kmp", "loo", "lar", "a"), + filepath.Join("kmp", "loo", "lar"), + filepath.Join("kmp", "loo", "kar", "a"), + filepath.Join("kmp", "loo", "kar"), + } +} diff --git a/utils/utils.go b/utils/utils.go index 177f497ac..e407dd133 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -185,9 +185,10 @@ func ConvertLocalPatternToRegexp(localPath string, patternType PatternType) stri localPath = strings.TrimPrefix(localPath, "./") localPath = strings.TrimPrefix(localPath, ".\\") - if patternType == AntPattern { - localPath = antPatternToRegExp(cleanPath(localPath)) - } else if patternType == WildCardPattern { + switch patternType { + case AntPattern: + localPath = AntToRegex(cleanPath(localPath)) + case WildCardPattern: localPath = stringutils.WildcardPatternToRegExp(cleanPath(localPath)) } @@ -208,51 +209,6 @@ func cleanPath(path string) string { return path } -func antPatternToRegExp(localPath string) string { - localPath = stringutils.EscapeSpecialChars(localPath) - separator := getFileSeparator() - // 'xxx/' => 'xxx/**' - if strings.HasSuffix(localPath, separator) { - localPath += "**" - } - var wildcard = ".*" - // ant `*` ~ regexp `([^/]*)` : `*` matches zero or more characters except from `/`. - var regAsterisk = "([^" + separator + "]*)" - // ant `\*` ~ regexp `([^/]+)` : `\*` matches one or more characters (except from `/`) with a `/` prefix. - var regAsteriskWithSeparatorPrefix = "([^" + separator + "]+)" - // ant `**` ~ regexp `(.*)?` : `**` matches zero or more 'directories' in a path. - var doubleRegAsterisk = "(" + wildcard + ")?" - var doubleRegAsteriskWithSeparatorPrefix = "(" + wildcard + separator + ")?" - var doubleRegAsteriskWithSeparatorSuffix = "(" + separator + wildcard + ")?" - - // `?` => `.{1}` : `?` matches one character. - localPath = strings.Replace(localPath, `?`, ".{1}", -1) - // `*` => `([^/]*)` - localPath = strings.Replace(localPath, `*`, regAsterisk, -1) - // `**` => `(.*)?` - localPath = strings.Replace(localPath, regAsterisk+regAsterisk, doubleRegAsterisk, -1) - - // `\([^/]*)` => `\([^/]+)` : there are 2 cases with '*': - // 1. xxx/x* : * will represent 0 or more characters. - // 2. xxx/* : * will represent 1 or more characters. - // This "replace" handles the second option. - localPath = strings.Replace(localPath, separator+regAsterisk, separator+regAsteriskWithSeparatorPrefix, -1) - // `(.*)?/` => `(.*/)?` - localPath = strings.Replace(localPath, doubleRegAsterisk+separator, doubleRegAsteriskWithSeparatorPrefix, -1) - // Convert the last '/**' in the expression if exists : `/(.*)?` => `(/.*)?` - if strings.HasSuffix(localPath, separator+doubleRegAsterisk) { - localPath = strings.TrimSuffix(localPath, separator+doubleRegAsterisk) + doubleRegAsteriskWithSeparatorSuffix - } - return "^" + localPath + "$" -} - -func getFileSeparator() string { - if io.IsWindows() { - return "\\\\" - } - return "/" -} - // BuildTargetPath Replaces matched regular expression from path to corresponding placeholder {i} at target. // Example 1: // diff --git a/utils/utils_test.go b/utils/utils_test.go index 5ea49db96..cf9df8b18 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -1,14 +1,12 @@ package utils import ( - "github.com/jfrog/jfrog-client-go/utils/io" - "github.com/stretchr/testify/assert" - "path/filepath" "reflect" - "regexp" "sort" - "strings" "testing" + + "github.com/jfrog/jfrog-client-go/utils/io" + "github.com/stretchr/testify/assert" ) func TestRemoveRepoFromPath(t *testing.T) { @@ -186,100 +184,6 @@ func TestIsWildcardParentheses(t *testing.T) { } } -// TestAntPathToRegExp check the functionality of antPatternToRegExp function. -// Each time we take an array of paths, simulating files hierarchy on a filesystem, and an ANT pattern expression - -// -// and see if the conversion to regular expression worked as expected. -func TestAntPathToRegExp(t *testing.T) { - separator := getFileSeparator() - var paths = getFileSystemsPathsForTestingAntPattern(separator) - tests := []struct { - description string - antPattern string - paths []string - expectedMatchingPaths []string - }{ - {"check '?' in file's name", filepath.Join("dev", "a", "b?.txt"), paths, []string{filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt")}}, - {"check '?' in directory's name", filepath.Join("dev", "a?", "b.txt"), paths, []string{filepath.Join("dev", "aa", "b.txt")}}, - {"check '*' in file's name", filepath.Join("dev", "a", "b*.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt")}}, - {"check '*' in directory's name", filepath.Join("dev", "*", "b.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "aa", "b.txt")}}, - {"check '*' in directory's name", filepath.Join("dev", "*", "a", "b.txt"), paths, nil}, - {"check '**' in directory path", filepath.Join("**", "b.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "aa", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("test", "a", "b.txt"), filepath.Join("test", "aa", "b.txt")}}, - {"check '**' in the beginning and the end of path", filepath.Join("**", "a2", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"check '**' in the beginning and the end of path", filepath.Join("**a2**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"check double '**'", filepath.Join("**", "a2", "**", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"check '**' in the beginning and the end of file", filepath.Join("**", "b.zip", "**"), paths, []string{filepath.Join("dev", "aa", "b.zip"), filepath.Join("test", "aa", "b.zip"), filepath.Join("b.zip"), filepath.Join("test2", "b.zip")}}, - {"combine '**' and '*'", filepath.Join("**", "a2", "*"), paths, []string{filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"combine '**' and '*'", filepath.Join("**", "a2", "*", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"combine all signs", filepath.Join("**", "b?.*"), paths, []string{filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt"), filepath.Join("dev", "aa", "bb.txt"), filepath.Join("dev", "aa", "bc.txt"), filepath.Join("dev", "aa", "bc.zip"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt"), filepath.Join("test", "a", "bb.txt"), filepath.Join("test", "a", "bc.txt"), filepath.Join("test", "aa", "bb.txt"), filepath.Join("test", "aa", "bc.txt"), filepath.Join("test", "aa", "bc.zip")}}, - {"'**' all files", filepath.Join("**"), paths, paths}, - {"test2/**/b/**", filepath.Join("test2", "**", "b", "**"), paths, []string{filepath.Join("test2", "a", "b", "c.zip")}}, - {"*/b.zip", filepath.Join("*", "b.zip"), paths, []string{filepath.Join("test2", "b.zip")}}, - {"**/dev/**/a3/*c*", filepath.Join("dev", "**", "a3", "*c*"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "bc.txt")}}, - {"**/dev/**/a3/**", filepath.Join("dev", "**", "a3", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "a3", "b.txt")}}, - {"exclude 'temp/foo5/a'", filepath.Join("**", "foo", "**"), paths, []string{filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo")}}, - {"include dirs", filepath.Join("tmp", "*", "**"), paths, []string{filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo5", "a"), filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5")}}, - {"include dirs", filepath.Join("tmp", "**"), paths, []string{filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo5", "a"), filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5"), "tmp" + separator}}, - {"**/", "**" + separator, paths, paths}, - {"xxx/x*", filepath.Join("tmp", "f*"), paths, []string{filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5")}}, - {"xxx/x*x", filepath.Join("tmp", "f*5"), paths, []string{filepath.Join("tmp", "foo5")}}, - {"xxx/x*", filepath.Join("dev", "a1", "a2", "b*"), paths, []string{filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"xxx/*x*", filepath.Join("dev", "a1", "a2", "*c*"), paths, []string{filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"*", filepath.Join("*"), []string{"a", "a" + separator, filepath.Join("a", "b")}, []string{"a"}}, - } - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - regExpStr := antPatternToRegExp(cleanPath(test.antPattern)) - var matches []string - for _, checkedPath := range test.paths { - match, _ := regexp.MatchString(regExpStr, checkedPath) - if match { - matches = append(matches, checkedPath) - } - } - if !equalSlicesIgnoreOrder(matches, test.expectedMatchingPaths) { - t.Error("Unmatched! : ant pattern `" + test.antPattern + "` matches paths:\n[" + strings.Join(test.expectedMatchingPaths, ",") + "]\nbut got:\n[" + strings.Join(matches, ",") + "]") - } - }) - } -} - -func getFileSystemsPathsForTestingAntPattern(separator string) []string { - return []string{ - filepath.Join("dev", "a", "b.txt"), - filepath.Join("dev", "a", "bb.txt"), - filepath.Join("dev", "a", "bc.txt"), - filepath.Join("dev", "aa", "b.txt"), - filepath.Join("dev", "aa", "bb.txt"), - filepath.Join("dev", "aa", "bc.txt"), - filepath.Join("dev", "aa", "b.zip"), - filepath.Join("dev", "aa", "bc.zip"), - filepath.Join("dev", "a1", "a2", "a3", "b.txt"), - filepath.Join("dev", "a1", "a2", "b.txt"), - filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), - filepath.Join("dev", "a1", "a2", "bc.txt"), - - filepath.Join("test", "a", "b.txt"), - filepath.Join("test", "a", "bb.txt"), - filepath.Join("test", "a", "bc.txt"), - filepath.Join("test", "aa", "b.txt"), - filepath.Join("test", "aa", "bb.txt"), - filepath.Join("test", "aa", "bc.txt"), - filepath.Join("test", "aa", "b.zip"), - filepath.Join("test", "aa", "bc.zip"), - - filepath.Join("test2", "a", "b", "c.zip"), - filepath.Join("test2", "a", "bb", "c.zip"), - filepath.Join("test2", "b.zip"), - filepath.Join("b.zip"), - "tmp" + separator, - filepath.Join("tmp", "foo"), - filepath.Join("tmp", "foo", "a"), - filepath.Join("tmp", "foo5"), - filepath.Join("tmp", "foo5", "a"), - } -} - func equalSlicesIgnoreOrder(s1, s2 []string) bool { if len(s1) != len(s2) { return false From f3cf6a88a4541ebbc4c58c7251a54db2bfc1a888 Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Fri, 24 Mar 2023 15:51:43 +0530 Subject: [PATCH 02/15] Revert "Refactor ANT pattern logic (#716)" This reverts commit 3ac03a5ba76abefea926eceea3ff51c12b2f487a. --- utils/antUtils.go | 77 -------------------- utils/antUtils_test.go | 156 ----------------------------------------- utils/utils.go | 52 ++++++++++++-- utils/utils_test.go | 102 ++++++++++++++++++++++++++- 4 files changed, 147 insertions(+), 240 deletions(-) delete mode 100644 utils/antUtils.go delete mode 100644 utils/antUtils_test.go diff --git a/utils/antUtils.go b/utils/antUtils.go deleted file mode 100644 index bc11d9ca9..000000000 --- a/utils/antUtils.go +++ /dev/null @@ -1,77 +0,0 @@ -package utils - -import ( - "fmt" - "os" - "regexp" - "strings" - - "github.com/jfrog/gofrog/stringutils" - "github.com/jfrog/jfrog-client-go/utils/io" -) - -var ( - // Replace ** with a special string. - doubleStartSpecialString = "__JFROG_DOUBLE_STAR__" - - // Match **/ ('__JFROG_DOUBLE_STAR__\/...') - prefixDoubleStarRegex = regexp.MustCompile(fmt.Sprintf("%s%s", doubleStartSpecialString, getFileSeparatorForDoubleStart())) - - // match /** ('\/__JFROG_DOUBLE_STAR__...') - postfixDoubleStarRegex = regexp.MustCompile(fmt.Sprintf("%s%s", getFileSeparatorForDoubleStart(), doubleStartSpecialString)) - - // match ** ('...__JFROG_DOUBLE_STAR__...') - middleDoubleStarNoSeparateRegex = regexp.MustCompile(doubleStartSpecialString) -) - -func getFileSeparatorForDoubleStart() string { - if io.IsWindows() { - return `\\\\` - } - return `\/` -} - -func AntToRegex(antPattern string) string { - antPattern = stringutils.EscapeSpecialChars(antPattern) - antPattern = antQuestionMarkToRegex(antPattern) - return "^" + antStarsToRegex(antPattern) + "$" -} - -func getFileSeparatorForAntToRegex() string { - if io.IsWindows() { - return `\\` - } - return `/` -} - -func antStarsToRegex(antPattern string) string { - separator := getFileSeparatorForAntToRegex() - antPattern = addMissingShorthand(antPattern) - - // Replace ** with a special string, so it doesn't get mixed up with single * - antPattern = strings.ReplaceAll(antPattern, "**", doubleStartSpecialString) - - // ant `*` => regexp `([^/]*)` : `*` matches zero or more characters except from `/`. - antPattern = strings.ReplaceAll(antPattern, `*`, "([^"+separator+"]*)") - - // ant `**/` => regexp `(.*/)*` : Matches zero or more 'directories' at the beginning of the path. - antPattern = prefixDoubleStarRegex.ReplaceAllString(antPattern, "(.*"+separator+")*") - - // ant `/**` => regexp `(/.*)*` : Matches zero or more 'directories' at the end of the path. - antPattern = postfixDoubleStarRegex.ReplaceAllString(antPattern, "("+separator+".*)*") - - // ant `**` => regexp `(.*)*` : Matches zero or more 'directories'. - return middleDoubleStarNoSeparateRegex.ReplaceAllString(antPattern, "(.*)") -} - -func antQuestionMarkToRegex(antPattern string) string { - return strings.ReplaceAll(antPattern, "?", ".") -} - -func addMissingShorthand(antPattern string) string { - // There is one "shorthand": if a pattern ends with / or \, then ** is appended. For example, mypackage/test/ is interpreted as if it were mypackage/test/**. - if strings.HasSuffix(antPattern, string(os.PathSeparator)) { - return antPattern + "**" - } - return antPattern -} diff --git a/utils/antUtils_test.go b/utils/antUtils_test.go deleted file mode 100644 index 9b05e1e6c..000000000 --- a/utils/antUtils_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package utils - -import ( - "os" - "path/filepath" - "regexp" - "strings" - "testing" -) - -var separator = string(os.PathSeparator) - -var paths = getFileSystemsPathsForTestingAntPattern(separator) - -var testAntPathToRegExpDataProvider = []struct { - description string - antPattern string - paths []string - expectedMatchingPaths []string -}{ - {"check '?' in file's name", filepath.Join("dev", "a", "b?.txt"), paths, []string{filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt")}}, - {"check '?' in directory's name", filepath.Join("dev", "a?", "b.txt"), paths, []string{filepath.Join("dev", "aa", "b.txt")}}, - {"check '*' in file's name", filepath.Join("dev", "a", "b*.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt")}}, - {"check '*' in directory's name", filepath.Join("dev", "*", "b.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "aa", "b.txt")}}, - {"check '*' in directory's name", filepath.Join("dev", "*", "a", "b.txt"), paths, nil}, - {"check '**' in directory path", filepath.Join("**", "b.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "aa", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("test", "a", "b.txt"), filepath.Join("test", "aa", "b.txt")}}, - {"check '**' in the beginning and the end of path", filepath.Join("**", "a2", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt"), "a2"}}, - {"check '**' in the beginning and the end of path", filepath.Join("**a2**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt"), "a2"}}, - {"check double '**'", filepath.Join("**", "a2", "**", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"check '**' in the beginning and the end of file", filepath.Join("**", "b.zip", "**"), paths, []string{filepath.Join("dev", "aa", "b.zip"), filepath.Join("test", "aa", "b.zip"), filepath.Join("b.zip"), filepath.Join("test2", "b.zip")}}, - {"combine '**' and '*'", filepath.Join("**", "a2", "*"), paths, []string{filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"combine '**' and '*'", filepath.Join("**", "a2", "*", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"combine all signs", filepath.Join("**", "b?.*"), paths, []string{filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt"), filepath.Join("dev", "aa", "bb.txt"), filepath.Join("dev", "aa", "bc.txt"), filepath.Join("dev", "aa", "bc.zip"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt"), filepath.Join("test", "a", "bb.txt"), filepath.Join("test", "a", "bc.txt"), filepath.Join("test", "aa", "bb.txt"), filepath.Join("test", "aa", "bc.txt"), filepath.Join("test", "aa", "bc.zip")}}, - {"'**' all files", filepath.Join("**"), paths, paths}, - {"test2/**/b/**", filepath.Join("test2", "**", "b", "**"), paths, []string{filepath.Join("test2", "a", "b", "c.zip")}}, - {"*/b.zip", filepath.Join("*", "b.zip"), paths, []string{filepath.Join("test2", "b.zip")}}, - {"**/dev/**/a3/*c*", filepath.Join("dev", "**", "a3", "*c*"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "bc.txt")}}, - {"**/dev/**/a3/**", filepath.Join("dev", "**", "a3", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "a3", "b.txt")}}, - {"exclude 'temp/foo5/a'", filepath.Join("**", "foo", "**"), paths, []string{filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo")}}, - {"include dirs", filepath.Join("tmp", "*", "**"), paths, []string{"tmp" + separator, filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo5", "a"), filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5")}}, - {"include dirs", filepath.Join("tmp", "**"), paths, []string{"tmp" + separator, filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo5", "a"), filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5")}}, - {"double and single wildcard", filepath.Join("**", "tmp*", "**"), paths, []string{"tmp" + separator, filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo5", "a"), filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5"), filepath.Join("Wrapper", "tmp", "boo"), filepath.Join("Wrapper", "tmp12", "boo")}}, - {"exclude only sub dir", filepath.Join("**", "loo", "**", "bar", "**"), paths, []string{filepath.Join("kmp", "loo", "bar"), filepath.Join("kmp", "loo", "bar", "b"), filepath.Join("kmp", "loo", "bar", "a")}}, - {"**/", "**" + separator, paths, paths}, - {"xxx/x*", filepath.Join("tmp", "f*"), paths, []string{filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5")}}, - {"xxx/x*x", filepath.Join("tmp", "f*5"), paths, []string{filepath.Join("tmp", "foo5")}}, - {"xxx/x*", filepath.Join("dev", "a1", "a2", "b*"), paths, []string{filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"xxx/*x*", filepath.Join("dev", "a1", "a2", "*c*"), paths, []string{filepath.Join("dev", "a1", "a2", "bc.txt")}}, - {"*", filepath.Join("*"), paths, []string{"b.zip", "a2"}}, - {"*", filepath.Join("*"), []string{"a", "a" + separator, filepath.Join("a", "b")}, []string{"a"}}, -} - -// In each case, we take an array of paths, simulating a filesystem hierarchy, and an ANT pattern expression and -// check if the conversion to regular expression worked. -func TestAntPathToRegExp(t *testing.T) { - for _, test := range testAntPathToRegExpDataProvider { - t.Run(test.description, func(t *testing.T) { - regExpStr := AntToRegex(cleanPath(test.antPattern)) - var matches []string - for _, checkedPath := range test.paths { - match, _ := regexp.MatchString(regExpStr, checkedPath) - if match { - matches = append(matches, checkedPath) - } - } - if !equalSlicesIgnoreOrder(matches, test.expectedMatchingPaths) { - t.Error("Unmatched! : ant pattern `" + test.antPattern + "` matches paths:\n[" + strings.Join(test.expectedMatchingPaths, ",") + "]\nbut got:\n[" + strings.Join(matches, ",") + "]") - } - }) - } -} - -var testAntToRegexProvider = []struct { - ant string - expectedRegex string -}{ - {"a.zip", "^a\\.zip$"}, - {"ab", "^ab$"}, - {"**", "^(.*)$"}, - {"**/", "^(.*/)*(.*)$"}, - {"**/*", "^(.*/)*([^/]*)$"}, - {"/**", "^(/.*)*$"}, - {"*/**", "^([^/]*)(/.*)*$"}, - {"/**/ab", "^/(.*/)*ab$"}, - {"/**/ab*", "^/(.*/)*ab([^/]*)$"}, - {"/**/ab/", "^/(.*/)*ab(/.*)*$"}, - {"/**/ab/*", "^/(.*/)*ab/([^/]*)$"}, - {"/**/ab*/", "^/(.*/)*ab([^/]*)(/.*)*$"}, - {"ab/**/", "^ab/(.*/)*(.*)$"}, - {"*ab/**/", "^([^/]*)ab/(.*/)*(.*)$"}, - {"/ab/**/", "^/ab/(.*/)*(.*)$"}, - {"/ab*/**/", "^/ab([^/]*)/(.*/)*(.*)$"}, - {"/**/ab/**/", "^/(.*/)*ab/(.*/)*(.*)$"}, - {"/**/a*b/**/", "^/(.*/)*a([^/]*)b/(.*/)*(.*)$"}, - {"/**/ab/**/cd/**/ef/", "^/(.*/)*ab/(.*/)*cd/(.*/)*ef(/.*)*$"}, -} - -func TestAntToRegex(t *testing.T) { - for _, test := range testAntToRegexProvider { - t.Run("'"+test.ant+"'", func(t *testing.T) { - regExpStr := AntToRegex(cleanPath(strings.ReplaceAll(test.ant, "/", separator))) - expectedRegExpStr := strings.ReplaceAll(test.expectedRegex, "/", getFileSeparatorForAntToRegex()) - if regExpStr != expectedRegExpStr { - t.Error("Unmatched! : ant pattern `" + test.ant + "` translated to:\n" + regExpStr + "\nbut expect it to be:\n" + expectedRegExpStr + "") - } - }) - } -} - -func getFileSystemsPathsForTestingAntPattern(separator string) []string { - return []string{ - filepath.Join("dev", "a", "b.txt"), - filepath.Join("dev", "a", "bb.txt"), - filepath.Join("dev", "a", "bc.txt"), - filepath.Join("dev", "aa", "b.txt"), - filepath.Join("dev", "aa", "bb.txt"), - filepath.Join("dev", "aa", "bc.txt"), - filepath.Join("dev", "aa", "b.zip"), - filepath.Join("dev", "aa", "bc.zip"), - filepath.Join("dev", "a1", "a2", "a3", "b.txt"), - filepath.Join("dev", "a1", "a2", "b.txt"), - filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), - filepath.Join("dev", "a1", "a2", "bc.txt"), - "a2", - filepath.Join("test", "a", "b.txt"), - filepath.Join("test", "a", "bb.txt"), - filepath.Join("test", "a", "bc.txt"), - filepath.Join("test", "aa", "b.txt"), - filepath.Join("test", "aa", "bb.txt"), - filepath.Join("test", "aa", "bc.txt"), - filepath.Join("test", "aa", "b.zip"), - filepath.Join("test", "aa", "bc.zip"), - - filepath.Join("test2", "a", "b", "c.zip"), - filepath.Join("test2", "a", "bb", "c.zip"), - filepath.Join("test2", "b.zip"), - filepath.Join("b.zip"), - "tmp" + separator, - filepath.Join("tmp", "foo"), - filepath.Join("Wrapper", "tmp", "boo"), - filepath.Join("Wrapper", "tmp12", "boo"), - filepath.Join("tmp", "foo", "a"), - filepath.Join("tmp", "foo5"), - filepath.Join("tmp", "foo5", "a"), - - filepath.Join("kmp", "loo"), - filepath.Join("kmp", "loo", "bar", "a"), - filepath.Join("kmp", "loo", "bar", "b"), - filepath.Join("kmp", "loo", "bar"), - filepath.Join("kmp", "loo", "lar"), - filepath.Join("kmp", "loo", "lar", "a"), - filepath.Join("kmp", "loo", "lar"), - filepath.Join("kmp", "loo", "kar", "a"), - filepath.Join("kmp", "loo", "kar"), - } -} diff --git a/utils/utils.go b/utils/utils.go index e407dd133..177f497ac 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -185,10 +185,9 @@ func ConvertLocalPatternToRegexp(localPath string, patternType PatternType) stri localPath = strings.TrimPrefix(localPath, "./") localPath = strings.TrimPrefix(localPath, ".\\") - switch patternType { - case AntPattern: - localPath = AntToRegex(cleanPath(localPath)) - case WildCardPattern: + if patternType == AntPattern { + localPath = antPatternToRegExp(cleanPath(localPath)) + } else if patternType == WildCardPattern { localPath = stringutils.WildcardPatternToRegExp(cleanPath(localPath)) } @@ -209,6 +208,51 @@ func cleanPath(path string) string { return path } +func antPatternToRegExp(localPath string) string { + localPath = stringutils.EscapeSpecialChars(localPath) + separator := getFileSeparator() + // 'xxx/' => 'xxx/**' + if strings.HasSuffix(localPath, separator) { + localPath += "**" + } + var wildcard = ".*" + // ant `*` ~ regexp `([^/]*)` : `*` matches zero or more characters except from `/`. + var regAsterisk = "([^" + separator + "]*)" + // ant `\*` ~ regexp `([^/]+)` : `\*` matches one or more characters (except from `/`) with a `/` prefix. + var regAsteriskWithSeparatorPrefix = "([^" + separator + "]+)" + // ant `**` ~ regexp `(.*)?` : `**` matches zero or more 'directories' in a path. + var doubleRegAsterisk = "(" + wildcard + ")?" + var doubleRegAsteriskWithSeparatorPrefix = "(" + wildcard + separator + ")?" + var doubleRegAsteriskWithSeparatorSuffix = "(" + separator + wildcard + ")?" + + // `?` => `.{1}` : `?` matches one character. + localPath = strings.Replace(localPath, `?`, ".{1}", -1) + // `*` => `([^/]*)` + localPath = strings.Replace(localPath, `*`, regAsterisk, -1) + // `**` => `(.*)?` + localPath = strings.Replace(localPath, regAsterisk+regAsterisk, doubleRegAsterisk, -1) + + // `\([^/]*)` => `\([^/]+)` : there are 2 cases with '*': + // 1. xxx/x* : * will represent 0 or more characters. + // 2. xxx/* : * will represent 1 or more characters. + // This "replace" handles the second option. + localPath = strings.Replace(localPath, separator+regAsterisk, separator+regAsteriskWithSeparatorPrefix, -1) + // `(.*)?/` => `(.*/)?` + localPath = strings.Replace(localPath, doubleRegAsterisk+separator, doubleRegAsteriskWithSeparatorPrefix, -1) + // Convert the last '/**' in the expression if exists : `/(.*)?` => `(/.*)?` + if strings.HasSuffix(localPath, separator+doubleRegAsterisk) { + localPath = strings.TrimSuffix(localPath, separator+doubleRegAsterisk) + doubleRegAsteriskWithSeparatorSuffix + } + return "^" + localPath + "$" +} + +func getFileSeparator() string { + if io.IsWindows() { + return "\\\\" + } + return "/" +} + // BuildTargetPath Replaces matched regular expression from path to corresponding placeholder {i} at target. // Example 1: // diff --git a/utils/utils_test.go b/utils/utils_test.go index cf9df8b18..5ea49db96 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -1,12 +1,14 @@ package utils import ( + "github.com/jfrog/jfrog-client-go/utils/io" + "github.com/stretchr/testify/assert" + "path/filepath" "reflect" + "regexp" "sort" + "strings" "testing" - - "github.com/jfrog/jfrog-client-go/utils/io" - "github.com/stretchr/testify/assert" ) func TestRemoveRepoFromPath(t *testing.T) { @@ -184,6 +186,100 @@ func TestIsWildcardParentheses(t *testing.T) { } } +// TestAntPathToRegExp check the functionality of antPatternToRegExp function. +// Each time we take an array of paths, simulating files hierarchy on a filesystem, and an ANT pattern expression - +// +// and see if the conversion to regular expression worked as expected. +func TestAntPathToRegExp(t *testing.T) { + separator := getFileSeparator() + var paths = getFileSystemsPathsForTestingAntPattern(separator) + tests := []struct { + description string + antPattern string + paths []string + expectedMatchingPaths []string + }{ + {"check '?' in file's name", filepath.Join("dev", "a", "b?.txt"), paths, []string{filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt")}}, + {"check '?' in directory's name", filepath.Join("dev", "a?", "b.txt"), paths, []string{filepath.Join("dev", "aa", "b.txt")}}, + {"check '*' in file's name", filepath.Join("dev", "a", "b*.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt")}}, + {"check '*' in directory's name", filepath.Join("dev", "*", "b.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "aa", "b.txt")}}, + {"check '*' in directory's name", filepath.Join("dev", "*", "a", "b.txt"), paths, nil}, + {"check '**' in directory path", filepath.Join("**", "b.txt"), paths, []string{filepath.Join("dev", "a", "b.txt"), filepath.Join("dev", "aa", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("test", "a", "b.txt"), filepath.Join("test", "aa", "b.txt")}}, + {"check '**' in the beginning and the end of path", filepath.Join("**", "a2", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"check '**' in the beginning and the end of path", filepath.Join("**a2**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"check double '**'", filepath.Join("**", "a2", "**", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"check '**' in the beginning and the end of file", filepath.Join("**", "b.zip", "**"), paths, []string{filepath.Join("dev", "aa", "b.zip"), filepath.Join("test", "aa", "b.zip"), filepath.Join("b.zip"), filepath.Join("test2", "b.zip")}}, + {"combine '**' and '*'", filepath.Join("**", "a2", "*"), paths, []string{filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"combine '**' and '*'", filepath.Join("**", "a2", "*", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "b.txt"), filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"combine all signs", filepath.Join("**", "b?.*"), paths, []string{filepath.Join("dev", "a", "bb.txt"), filepath.Join("dev", "a", "bc.txt"), filepath.Join("dev", "aa", "bb.txt"), filepath.Join("dev", "aa", "bc.txt"), filepath.Join("dev", "aa", "bc.zip"), filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "bc.txt"), filepath.Join("test", "a", "bb.txt"), filepath.Join("test", "a", "bc.txt"), filepath.Join("test", "aa", "bb.txt"), filepath.Join("test", "aa", "bc.txt"), filepath.Join("test", "aa", "bc.zip")}}, + {"'**' all files", filepath.Join("**"), paths, paths}, + {"test2/**/b/**", filepath.Join("test2", "**", "b", "**"), paths, []string{filepath.Join("test2", "a", "b", "c.zip")}}, + {"*/b.zip", filepath.Join("*", "b.zip"), paths, []string{filepath.Join("test2", "b.zip")}}, + {"**/dev/**/a3/*c*", filepath.Join("dev", "**", "a3", "*c*"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "bc.txt")}}, + {"**/dev/**/a3/**", filepath.Join("dev", "**", "a3", "**"), paths, []string{filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), filepath.Join("dev", "a1", "a2", "a3", "b.txt")}}, + {"exclude 'temp/foo5/a'", filepath.Join("**", "foo", "**"), paths, []string{filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo")}}, + {"include dirs", filepath.Join("tmp", "*", "**"), paths, []string{filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo5", "a"), filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5")}}, + {"include dirs", filepath.Join("tmp", "**"), paths, []string{filepath.Join("tmp", "foo", "a"), filepath.Join("tmp", "foo5", "a"), filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5"), "tmp" + separator}}, + {"**/", "**" + separator, paths, paths}, + {"xxx/x*", filepath.Join("tmp", "f*"), paths, []string{filepath.Join("tmp", "foo"), filepath.Join("tmp", "foo5")}}, + {"xxx/x*x", filepath.Join("tmp", "f*5"), paths, []string{filepath.Join("tmp", "foo5")}}, + {"xxx/x*", filepath.Join("dev", "a1", "a2", "b*"), paths, []string{filepath.Join("dev", "a1", "a2", "b.txt"), filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"xxx/*x*", filepath.Join("dev", "a1", "a2", "*c*"), paths, []string{filepath.Join("dev", "a1", "a2", "bc.txt")}}, + {"*", filepath.Join("*"), []string{"a", "a" + separator, filepath.Join("a", "b")}, []string{"a"}}, + } + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + regExpStr := antPatternToRegExp(cleanPath(test.antPattern)) + var matches []string + for _, checkedPath := range test.paths { + match, _ := regexp.MatchString(regExpStr, checkedPath) + if match { + matches = append(matches, checkedPath) + } + } + if !equalSlicesIgnoreOrder(matches, test.expectedMatchingPaths) { + t.Error("Unmatched! : ant pattern `" + test.antPattern + "` matches paths:\n[" + strings.Join(test.expectedMatchingPaths, ",") + "]\nbut got:\n[" + strings.Join(matches, ",") + "]") + } + }) + } +} + +func getFileSystemsPathsForTestingAntPattern(separator string) []string { + return []string{ + filepath.Join("dev", "a", "b.txt"), + filepath.Join("dev", "a", "bb.txt"), + filepath.Join("dev", "a", "bc.txt"), + filepath.Join("dev", "aa", "b.txt"), + filepath.Join("dev", "aa", "bb.txt"), + filepath.Join("dev", "aa", "bc.txt"), + filepath.Join("dev", "aa", "b.zip"), + filepath.Join("dev", "aa", "bc.zip"), + filepath.Join("dev", "a1", "a2", "a3", "b.txt"), + filepath.Join("dev", "a1", "a2", "b.txt"), + filepath.Join("dev", "a1", "a2", "a3", "bc.txt"), + filepath.Join("dev", "a1", "a2", "bc.txt"), + + filepath.Join("test", "a", "b.txt"), + filepath.Join("test", "a", "bb.txt"), + filepath.Join("test", "a", "bc.txt"), + filepath.Join("test", "aa", "b.txt"), + filepath.Join("test", "aa", "bb.txt"), + filepath.Join("test", "aa", "bc.txt"), + filepath.Join("test", "aa", "b.zip"), + filepath.Join("test", "aa", "bc.zip"), + + filepath.Join("test2", "a", "b", "c.zip"), + filepath.Join("test2", "a", "bb", "c.zip"), + filepath.Join("test2", "b.zip"), + filepath.Join("b.zip"), + "tmp" + separator, + filepath.Join("tmp", "foo"), + filepath.Join("tmp", "foo", "a"), + filepath.Join("tmp", "foo5"), + filepath.Join("tmp", "foo5", "a"), + } +} + func equalSlicesIgnoreOrder(s1, s2 []string) bool { if len(s1) != len(s2) { return false From 6791fdd4a01cd60498a7823f4831a6964e01728d Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Fri, 24 Mar 2023 18:31:05 +0530 Subject: [PATCH 03/15] Fixed Compileation failures --- tests/pipelinesworkspaces_test.go | 4 ++-- tests/utils_test.go | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/pipelinesworkspaces_test.go b/tests/pipelinesworkspaces_test.go index ad98e6e14..8ef630488 100644 --- a/tests/pipelinesworkspaces_test.go +++ b/tests/pipelinesworkspaces_test.go @@ -14,7 +14,7 @@ import ( "time" ) -func testWorkspaceValidationWhenPipelinesResourcesAreNotValid(t *testing.T) { +func TestWorkspaceValidationWhenPipelinesResourcesAreNotValid(t *testing.T) { if !assert.NotEmpty(t, *PipelinesAccessToken, "cannot run pipelines tests without access token configured") { return @@ -31,7 +31,7 @@ func testWorkspaceValidationWhenPipelinesResourcesAreNotValid(t *testing.T) { assert.NoError(t, err) } -func testWorkspaceValidationWhenPipelinesResourcesAreValid(t *testing.T) { +func TestWorkspaceValidationWhenPipelinesResourcesAreValid(t *testing.T) { if !assert.NotEmpty(t, *PipelinesAccessToken, "cannot run pipelines tests without access token configured") { return } diff --git a/tests/utils_test.go b/tests/utils_test.go index 837dd5649..9aadb5250 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -117,6 +117,7 @@ var ( testPipelinesRunService *pipelinesServices.RunService testPipelinesSyncService *pipelinesServices.SyncService testPipelinesSyncStatusService *pipelinesServices.SyncStatusService + testPipelinesWorkspaceService *pipelinesServices.WorkspaceService // Access Services testsAccessPingService *accessServices.PingService @@ -519,6 +520,18 @@ func createPipelinesSyncStatusManager() { testPipelinesSyncStatusService.ServiceDetails = pipelinesDetails } +func createPipelinesWorkspaceServiceManager() { + pipelinesDetails := GetPipelinesDetails() + client, err := jfroghttpclient.JfrogClientBuilder(). + SetClientCertPath(pipelinesDetails.GetClientCertPath()). + SetClientCertKeyPath(pipelinesDetails.GetClientCertKeyPath()). + AppendPreRequestInterceptor(pipelinesDetails.RunPreRequestFunctions). + Build() + failOnHttpClientCreation(err) + testPipelinesWorkspaceService = pipelinesServices.NewWorkspaceService(client) + testPipelinesWorkspaceService.ServiceDetails = pipelinesDetails +} + func failOnHttpClientCreation(err error) { if err != nil { log.Error(fmt.Sprintf(HttpClientCreationFailureMessage, err.Error())) From 58275f395b4b91b7eb78e31a26ba18e57756a6ab Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Sat, 1 Apr 2023 21:13:09 +0530 Subject: [PATCH 04/15] Added syncStatus runStatus to workspace pipelines command --- pipelines/manager.go | 18 ++++++++++++++ pipelines/services/workspace.go | 44 +++------------------------------ 2 files changed, 21 insertions(+), 41 deletions(-) diff --git a/pipelines/manager.go b/pipelines/manager.go index 184d6414b..2cbf4443e 100644 --- a/pipelines/manager.go +++ b/pipelines/manager.go @@ -141,6 +141,12 @@ func (sm *PipelinesServicesManager) ValidatePipelineSources(data []byte) (string return validateService.ValidatePipeline(data) } +func (sm *PipelinesServicesManager) WorkspaceSync() error { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + return workspaceService.WorkspaceSync() +} + func (sm *PipelinesServicesManager) WorkspacePollSyncStatus() ([]services.WorkspacesResponse, error) { workspaceService := services.NewWorkspaceService(sm.client) workspaceService.ServiceDetails = sm.config.GetServiceDetails() @@ -192,3 +198,15 @@ func (sm *PipelinesServicesManager) GetStepConsoles(stepID string) (map[string][ workspaceService.ServiceDetails = sm.config.GetServiceDetails() return workspaceService.GetStepLogsUsingStepID(stepID) } + +func (sm *PipelinesServicesManager) GetWorkspace() ([]services.WorkspacesResponse, error) { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + return workspaceService.GetWorkspace() +} + +func (sm *PipelinesServicesManager) DeleteWorkspace(projectKey string) error { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + return workspaceService.DeleteWorkspace("2") +} \ No newline at end of file diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go index 5af80a15e..c8a2550f3 100644 --- a/pipelines/services/workspace.go +++ b/pipelines/services/workspace.go @@ -39,22 +39,18 @@ func (ws *WorkspaceService) getHttpDetails() httputils.HttpClientDetails { func (ws *WorkspaceService) GetWorkspace() ([]WorkspacesResponse, error) { httpDetails := ws.getHttpDetails() - // Query params queryParams := make(map[string]string, 0) - // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaces) if err != nil { return nil, err } - // Prepare request resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) if err != nil { return nil, err } - // Response Analysis if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { return nil, err @@ -64,25 +60,21 @@ func (ws *WorkspaceService) GetWorkspace() ([]WorkspacesResponse, error) { return wsStatusResp, nil } -func (ws *WorkspaceService) DeleteWorkspace(wsID string) error { +func (ws *WorkspaceService) DeleteWorkspace(projectKey string) error { httpDetails := ws.getHttpDetails() - deleteWorkspaceAPI := strings.Replace(deleteWorkspace, ":id", wsID, 1) - + deleteWorkspaceAPI := strings.Replace(deleteWorkspace, ":id", projectKey, 1) // Query params queryParams := make(map[string]string, 0) - // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), deleteWorkspaceAPI) if err != nil { return err } - // Prepare request resp, body, err := ws.client.SendDelete(uri, nil, &httpDetails) if err != nil { return err } - // Response analysis err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) return err @@ -90,49 +82,41 @@ func (ws *WorkspaceService) DeleteWorkspace(wsID string) error { func (ws *WorkspaceService) ValidateWorkspace(data []byte) error { httpDetails := ws.getHttpDetails() - // Query params queryParams := make(map[string]string, 0) - // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), validateWorkspace) if err != nil { return err } - // Headers headers := make(map[string]string, 0) headers["Content-Type"] = "application/json" + headers["User-Agent"] = "jfrog-client-go/1.24.3" httpDetails.Headers = headers - // Prepare request resp, body, err := ws.client.SendPost(uri, data, &httpDetails) if err != nil { return err } - // Response analysis return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) } func (ws *WorkspaceService) WorkspaceSync() error { httpDetails := ws.getHttpDetails() - // Query params queryParams := make(map[string]string, 0) - // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaceSync) if err != nil { return err } - // Prepare request resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) if err != nil { return err } - // Response analysis return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) } @@ -140,7 +124,6 @@ func (ws *WorkspaceService) WorkspaceSync() error { func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunID, error) { httpDetails := ws.getHttpDetails() pipelineFilter := strings.Join(pipelines, ",") - // Query params // TODO ADD include in query param if needed queryParams := map[string]string{ @@ -148,13 +131,11 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI "limit": "1", "include": "latestRunId,name", } - // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspacePipelines) if err != nil { return nil, err } - // Prepare request resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) if err != nil { @@ -171,17 +152,14 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI func (ws *WorkspaceService) WorkspaceRunStatus(pipelinesRunID int) ([]byte, error) { httpDetails := ws.getHttpDetails() - // Query params // TODO ADD include in query param if needed queryParams := map[string]string{} - // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaceRuns+"/"+strconv.Itoa(pipelinesRunID)) if err != nil { return nil, err } - // Prepare request resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) if err != nil { @@ -194,26 +172,22 @@ func (ws *WorkspaceService) WorkspaceRunStatus(pipelinesRunID int) ([]byte, erro func (ws *WorkspaceService) WorkspaceStepStatus(pipelinesRunID int) ([]byte, error) { httpDetails := ws.getHttpDetails() - // Query params queryParams := map[string]string{ "runIds": strconv.Itoa(pipelinesRunID), "limit": "15", //"include": "name,statusCode,triggeredAt,externalBuildUrl", } - // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaceSteps) if err != nil { return nil, err } - // Prepare request resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) if err != nil { return nil, err } - // Response analysis err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) return body, err @@ -221,7 +195,6 @@ func (ws *WorkspaceService) WorkspaceStepStatus(pipelinesRunID int) ([]byte, err func (ws *WorkspaceService) GetWorkspacePipelines(workspaces []WorkspacesResponse) (map[string]string, error) { pipelineNames := make(map[string]string, 1) - log.Info("Collecting pipeline names configured") // Validate and return pipeline names as slice if len(workspaces) > 0 && !(*workspaces[0].IsSyncing) { @@ -235,16 +208,13 @@ func (ws *WorkspaceService) GetWorkspacePipelines(workspaces []WorkspacesRespons func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, error) { httpDetails := ws.getHttpDetails() - // Query params queryParams := make(map[string]string, 0) - // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaces) if err != nil { return nil, err } - pollingAction := func() (shouldStop bool, responseBody []byte, err error) { log.Info("Polling for pipeline resource sync") // Prepare request @@ -252,7 +222,6 @@ func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, err if err != nil { return false, body, err } - // Response Analysis if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { return false, body, err @@ -266,9 +235,6 @@ func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, err } if len(wsStatusResp) > 0 && *wsStatusResp[0].IsSyncing { return false, body, err - } else if len(wsStatusResp) > 0 && !(*wsStatusResp[0].IsSyncing) { - log.Info("Sync is completed") - return true, body, err } return true, body, err } @@ -292,22 +258,18 @@ func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, err func (ws *WorkspaceService) GetStepLogsUsingStepID(stepID string) (map[string][]Console, error) { httpDetails := ws.getHttpDetails() stepConsolesAPI := strings.Replace(stepConsoles, ":stepID", stepID, 1) - // Query params queryParams := make(map[string]string, 0) - // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), stepConsolesAPI) if err != nil { return nil, err } - // Prepare request resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) if err != nil { return nil, err } - // Response Analysis if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { return nil, err From 0c5f7c20d8a0448ccd24b07cfabfcadbff5fa317 Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Sun, 25 Jun 2023 20:43:28 +0530 Subject: [PATCH 05/15] Added pipelines workspaces support --- go.mod | 2 - go.sum | 3 - pipelines/manager.go | 14 +- pipelines/services/run.go | 31 +++-- pipelines/services/types.go | 24 ++-- pipelines/services/workspace.go | 81 +++++++++--- tests/pipelinesworkspaces_test.go | 206 ++++++++---------------------- 7 files changed, 160 insertions(+), 201 deletions(-) diff --git a/go.mod b/go.mod index 468bb57e8..2e6edf5d0 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.19 require ( github.com/buger/jsonparser v1.1.1 github.com/forPelevin/gomoji v1.1.8 - github.com/ghodss/yaml v1.0.0 github.com/go-git/go-git/v5 v5.6.1 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gookit/color v1.5.2 @@ -53,7 +52,6 @@ require ( golang.org/x/net v0.8.0 // indirect golang.org/x/sys v0.6.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.2.4 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8aff6aab6..fb6b2b8fa 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,6 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/forPelevin/gomoji v1.1.8 h1:JElzDdt0TyiUlecy6PfITDL6eGvIaxqYH1V52zrd0qQ= github.com/forPelevin/gomoji v1.1.8/go.mod h1:8+Z3KNGkdslmeGZBC3tCrwMrcPy5GRzAD+gL9NAwMXg= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= @@ -197,7 +195,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pipelines/manager.go b/pipelines/manager.go index 2cbf4443e..0aef7e43f 100644 --- a/pipelines/manager.go +++ b/pipelines/manager.go @@ -141,10 +141,10 @@ func (sm *PipelinesServicesManager) ValidatePipelineSources(data []byte) (string return validateService.ValidatePipeline(data) } -func (sm *PipelinesServicesManager) WorkspaceSync() error { +func (sm *PipelinesServicesManager) WorkspaceSync(projectKey string) error { workspaceService := services.NewWorkspaceService(sm.client) workspaceService.ServiceDetails = sm.config.GetServiceDetails() - return workspaceService.WorkspaceSync() + return workspaceService.WorkspaceSync(projectKey) } func (sm *PipelinesServicesManager) WorkspacePollSyncStatus() ([]services.WorkspacesResponse, error) { @@ -199,6 +199,12 @@ func (sm *PipelinesServicesManager) GetStepConsoles(stepID string) (map[string][ return workspaceService.GetStepLogsUsingStepID(stepID) } +func (sm *PipelinesServicesManager) GetStepletConsoles(stepID string) (map[string][]services.Console, error) { + workspaceService := services.NewWorkspaceService(sm.client) + workspaceService.ServiceDetails = sm.config.GetServiceDetails() + return workspaceService.GetStepletLogsUsingStepID(stepID) +} + func (sm *PipelinesServicesManager) GetWorkspace() ([]services.WorkspacesResponse, error) { workspaceService := services.NewWorkspaceService(sm.client) workspaceService.ServiceDetails = sm.config.GetServiceDetails() @@ -208,5 +214,5 @@ func (sm *PipelinesServicesManager) GetWorkspace() ([]services.WorkspacesRespons func (sm *PipelinesServicesManager) DeleteWorkspace(projectKey string) error { workspaceService := services.NewWorkspaceService(sm.client) workspaceService.ServiceDetails = sm.config.GetServiceDetails() - return workspaceService.DeleteWorkspace("2") -} \ No newline at end of file + return workspaceService.DeleteWorkspace(projectKey) +} diff --git a/pipelines/services/run.go b/pipelines/services/run.go index 8eae4c2d7..f8bbab403 100644 --- a/pipelines/services/run.go +++ b/pipelines/services/run.go @@ -13,6 +13,7 @@ import ( "net/http" "strconv" "strings" + "time" ) type RunService struct { @@ -101,18 +102,28 @@ func (rs *RunService) TriggerPipelineRun(branch, pipeline string, isMultiBranch return err } - // Prepare Request - resp, body, err := rs.client.SendPost(uri, buf.Bytes(), &httpDetails) - if err != nil { - return err + pollingAction := func() (shouldStop bool, responseBody []byte, err error) { + // Prepare Request + resp, body, err := rs.client.SendPost(uri, buf.Bytes(), &httpDetails) + if err != nil { + return true, body, err + } + + // Response Analysis + if err := errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return false, body, err + } + log.Info(fmt.Sprintf("Triggered successfully\n%s %s \n%14s %s", "PipelineName :", pipeline, "Branch :", branch)) + return true, body, err } - - // Response Analysis - if err := errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { - return err + pollingExecutor := &httputils.PollingExecutor{ + Timeout: 2 * time.Minute, + PollingInterval: 5 * time.Second, + PollingAction: pollingAction, + MsgPrefix: "Get pipeline workspace sync status...", } - log.Info(fmt.Sprintf("Triggered successfully\n%s %s \n%14s %s", "PipelineName :", pipeline, "Branch :", branch)) - + // Polling execution + _, err = pollingExecutor.Execute() return nil } diff --git a/pipelines/services/types.go b/pipelines/services/types.go index 01dc2e52c..c60d3556c 100644 --- a/pipelines/services/types.go +++ b/pipelines/services/types.go @@ -130,18 +130,18 @@ type PipelinesRunID struct { } type Console struct { - ConsoleID string `json:"consoleId"` - IsSuccess interface{} `json:"isSuccess"` - IsShown bool `json:"isShown"` - ParentConsoleID string `json:"parentConsoleId"` - StepletID int `json:"stepletId"` - PipelineID int `json:"pipelineId"` - Timestamp int64 `json:"timestamp"` - TimestampEndedAt interface{} `json:"timestampEndedAt"` - Type string `json:"type"` - Message string `json:"message"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` + ConsoleID string `json:"consoleId,omitempty"` + IsSuccess *bool `json:"isSuccess,omitempty"` + IsShown interface{} `json:"isShown,omitempty"` + ParentConsoleID string `json:"parentConsoleId,omitempty"` + StepletID int `json:"stepletId,omitempty"` + PipelineID int `json:"pipelineId,omitempty"` + Timestamp int64 `json:"timestamp,omitempty"` + TimestampEndedAt interface{} `json:"timestampEndedAt,omitempty"` + Type string `json:"type,omitempty"` + Message string `json:"message,omitempty"` + CreatedAt time.Time `json:"createdAt,omitempty"` + UpdatedAt time.Time `json:"updatedAt,omitempty"` } // Validation Types diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go index c8a2550f3..e04992657 100644 --- a/pipelines/services/workspace.go +++ b/pipelines/services/workspace.go @@ -2,6 +2,8 @@ package services import ( "encoding/json" + "errors" + "fmt" "github.com/jfrog/jfrog-client-go/auth" "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" "github.com/jfrog/jfrog-client-go/utils/errorutils" @@ -15,13 +17,14 @@ import ( const ( workspaces = "api/v1/workspaces" - deleteWorkspace = "api/v1/workspaces/:id" + deleteWorkspace = "api/v1/workspaces/:project" validateWorkspace = "api/v1/validateWorkspace" - workspaceSync = "api/v1/syncWorkspace" + workspaceSync = "api/v1/syncWorkspace/:project" workspacePipelines = "api/v1/pipelines" workspaceRuns = "api/v1/runs" workspaceSteps = "api/v1/steps" - stepConsoles = "api/v1/steplets/:stepID/consoles" + stepConsoles = "api/v1/steps/:stepID/consoles" + stepletConsoles = "api/v1/steplets/:stepID/consoles" ) type WorkspaceService struct { @@ -60,9 +63,9 @@ func (ws *WorkspaceService) GetWorkspace() ([]WorkspacesResponse, error) { return wsStatusResp, nil } -func (ws *WorkspaceService) DeleteWorkspace(projectKey string) error { +func (ws *WorkspaceService) DeleteWorkspace(workspaceID string) error { httpDetails := ws.getHttpDetails() - deleteWorkspaceAPI := strings.Replace(deleteWorkspace, ":id", projectKey, 1) + deleteWorkspaceAPI := strings.Replace(deleteWorkspace, ":project", workspaceID, 1) // Query params queryParams := make(map[string]string, 0) // URL construction @@ -103,12 +106,13 @@ func (ws *WorkspaceService) ValidateWorkspace(data []byte) error { return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) } -func (ws *WorkspaceService) WorkspaceSync() error { +func (ws *WorkspaceService) WorkspaceSync(project string) error { httpDetails := ws.getHttpDetails() // Query params - queryParams := make(map[string]string, 0) + queryParams := make(map[string]string, 0) // Query params + syncWorkspaceAPI := strings.Replace(workspaceSync, ":project", project, 1) // URL construction - uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaceSync) + uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), syncWorkspaceAPI) if err != nil { return err } @@ -125,7 +129,6 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI httpDetails := ws.getHttpDetails() pipelineFilter := strings.Join(pipelines, ",") // Query params - // TODO ADD include in query param if needed queryParams := map[string]string{ "names": pipelineFilter, "limit": "1", @@ -136,15 +139,31 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI if err != nil { return nil, err } - // Prepare request - resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) - if err != nil { - return nil, err + pollingAction := func() (shouldStop bool, responseBody []byte, err error) { + // Prepare request + resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) + if err != nil { + return true, body, err + } + // Response analysis + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return false, body, err + } + pipeRunIDs := make([]PipelinesRunID, 0) + err = json.Unmarshal(body, &pipeRunIDs) + if pipeRunIDs[0].LatestRunID == 0 { + return false, body, errors.New("Pipeline didnt start running yet") + } + return true, body, err } - // Response analysis - if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { - return nil, err + pollingExecutor := &httputils.PollingExecutor{ + Timeout: 2 * time.Minute, + PollingInterval: 5 * time.Second, + PollingAction: pollingAction, + MsgPrefix: "Get pipeline workspace sync status...", } + // Polling execution + body, err := pollingExecutor.Execute() pipeRunIDs := make([]PipelinesRunID, 0) err = json.Unmarshal(body, &pipeRunIDs) return pipeRunIDs, err @@ -234,12 +253,15 @@ func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, err return true, body, err } if len(wsStatusResp) > 0 && *wsStatusResp[0].IsSyncing { + fmt.Printf("%+v \n", wsStatusResp) return false, body, err + } else if wsStatusResp[0].LastSyncStatusCode == 4003 || wsStatusResp[0].LastSyncStatusCode == 4004 { + return true, body, err } return true, body, err } pollingExecutor := &httputils.PollingExecutor{ - Timeout: 10 * time.Minute, + Timeout: 2 * time.Minute, PollingInterval: 5 * time.Second, PollingAction: pollingAction, MsgPrefix: "Get pipeline workspace sync status...", @@ -278,3 +300,28 @@ func (ws *WorkspaceService) GetStepLogsUsingStepID(stepID string) (map[string][] err = json.Unmarshal(body, &consoles) return consoles, err } + +// GetStepletLogsUsingStepID retrieve steps logs using step id +func (ws *WorkspaceService) GetStepletLogsUsingStepID(stepID string) (map[string][]Console, error) { + httpDetails := ws.getHttpDetails() + stepConsolesAPI := strings.Replace(stepletConsoles, ":stepID", stepID, 1) + // Query params + queryParams := make(map[string]string, 0) + // URL construction + uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), stepConsolesAPI) + if err != nil { + return nil, err + } + // Prepare request + resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) + if err != nil { + return nil, err + } + // Response Analysis + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return nil, err + } + consoles := make(map[string][]Console) + err = json.Unmarshal(body, &consoles) + return consoles, err +} diff --git a/tests/pipelinesworkspaces_test.go b/tests/pipelinesworkspaces_test.go index 8ef630488..c21245882 100644 --- a/tests/pipelinesworkspaces_test.go +++ b/tests/pipelinesworkspaces_test.go @@ -1,19 +1,19 @@ package tests import ( - "bytes" "encoding/json" "fmt" - "github.com/ghodss/yaml" - "github.com/jfrog/jfrog-client-go/utils/log" "github.com/stretchr/testify/assert" - "io" "os" - "strings" + "strconv" "testing" - "time" ) +func TestPipelinesWorkspaceService(t *testing.T) { + t.Run("test trigger pipelines workspace when resources are not valid", TestWorkspaceValidationWhenPipelinesResourcesAreNotValid) + t.Run("test trigger pipelines workspace when resources are valid", TestWorkspaceValidationWhenPipelinesResourcesAreValid) +} + func TestWorkspaceValidationWhenPipelinesResourcesAreNotValid(t *testing.T) { if !assert.NotEmpty(t, *PipelinesAccessToken, "cannot run pipelines tests without access token configured") { @@ -22,7 +22,10 @@ func TestWorkspaceValidationWhenPipelinesResourcesAreNotValid(t *testing.T) { filePath := "/Users/bhanur/go/src/pipeline-cli/jfrog-client-go/tests/testdata/pipelines.yml" // get pipelines.yaml content - pipelineRes, err := getPipelinesResourceToValidate(filePath) + pipelineRes, err := getWorkspaceRunPayload([]string{filePath}) + if !assert.NoError(t, err) { + return + } fmt.Printf("Bytes received : %s \n", pipelineRes) if !assert.NoError(t, err) { return @@ -37,174 +40,71 @@ func TestWorkspaceValidationWhenPipelinesResourcesAreValid(t *testing.T) { } filePath := "/Users/bhanur/go/src/pipeline-cli/jfrog-client-go/tests/testdata/pipelines.yml" - integrationName := "int_gh_pipe_cli" - _, err := testsPipelinesIntegrationsService.CreateGithubIntegration(integrationName, *PipelinesVcsToken) - assert.NoError(t, err) - //defer deleteIntegrationAndAssert(t, integrationId) - // get pipelines.yaml content - pipelineRes, err := getPipelinesResourceToValidate(filePath) + pipelineRes, err := getWorkspaceRunPayload([]string{filePath}) if !assert.NoError(t, err) { return } err = testPipelinesWorkspaceService.ValidateWorkspace(pipelineRes) assert.NoError(t, err) - workspaces, wsErr := testPipelinesWorkspaceService.WorkspacePollSyncStatus() - assert.NoError(t, wsErr) - pipelineBranch, err := testPipelinesWorkspaceService.GetWorkspacePipelines(workspaces) + wsResp, err := testPipelinesWorkspaceService.GetWorkspace() assert.NoError(t, err) - for pipName, branch := range pipelineBranch { - err = testPipelinesRunService.TriggerPipelineRun(branch, pipName, false) - assert.NoError(t, err) - } - pipelineNames := make([]string, len(pipelineBranch)) - - i := 0 - for k := range pipelineBranch { - pipelineNames[i] = k - i++ - } - pipeRunIDs, wsRunErr := testPipelinesWorkspaceService.WorkspaceRunIDs(pipelineNames) - assert.NoError(t, wsRunErr) - - for _, runId := range pipeRunIDs { - _, err2 := testPipelinesWorkspaceService.WorkspaceRunStatus(runId.LatestRunID) - assert.NoError(t, err2) - _, err3 := testPipelinesWorkspaceService.WorkspaceStepStatus(runId.LatestRunID) - assert.NoError(t, err3) - } - -} - -func getPipelinesResourceToValidate(filePath string) ([]byte, error) { - - ymlType := "" - readFile, err := os.ReadFile(filePath) - if err != nil { - return nil, err - } - fInfo, err := os.Stat(filePath) - if err != nil { - return nil, err - } - - toJSON, err3 := convertYAMLToJSON(err, readFile) - if err3 != nil { - return nil, err3 - } - vsc, err4 := convertJSONDataToMap(fInfo, toJSON) - if err4 != nil { - return nil, err4 - } - var marErr error - - resMap, err5, done := splitDataToPipelinesAndResourcesMap(vsc, marErr, err, ymlType) - if done { - return nil, err5 + if len(wsResp) < 1 { + assert.Fail(t, "No workspace created") } - - data, resErr := getPayloadToValidatePipelineResource(resMap) - if resErr != nil { - return nil, resErr + for _, ws := range wsResp { + fmt.Printf("%+v \n", ws) + err = testPipelinesWorkspaceService.DeleteWorkspace(strconv.Itoa(ws.ID)) } - - return data.Bytes(), nil + assert.NoError(t, err) } -func convertJSONDataToMap(file os.FileInfo, toJSON []byte) (map[string][]interface{}, error) { - log.Info("validating pipeline resources ", file.Name()) - time.Sleep(1 * time.Second) - vsc := make(map[string][]interface{}) - convErr := yaml.Unmarshal(toJSON, &vsc) - if convErr != nil { - return nil, convErr - } - return vsc, nil +type PipelineDefinition struct { + FileName string `json:"fileName,omitempty"` + Content string `json:"content,omitempty"` + YmlType string `json:"ymlType,omitempty"` } -func convertYAMLToJSON(err error, readFile []byte) ([]byte, error) { - toJSON, err := yaml.YAMLToJSON(readFile) - if err != nil { - log.Error("Failed to convert to json") - return nil, err - } - return toJSON, nil +type WorkSpaceValidation struct { + ProjectId string `json:"-"` + Files []PipelineDefinition `json:"files,omitempty"` + ProjectName string `json:"projectName,omitempty"` + Name string `json:"name,omitempty"` } -func splitDataToPipelinesAndResourcesMap(vsc map[string][]interface{}, marErr error, err error, ymlType string) (map[string]string, error, bool) { - resMap := make(map[string]string) - var data []byte - if v, ok := vsc["resources"]; ok { - log.Info("resources found preparing to validate") - data, marErr = json.Marshal(v) - if marErr != nil { - log.Error("failed to marshal to json") - return nil, err, true +func getWorkspaceRunPayload(resources []string) ([]byte, error) { + var pipelineDefinitions []PipelineDefinition + for _, pathToFile := range resources { + fileContent, fileInfo, err := getFileContentAndBaseName(pathToFile) + if err != nil { + return nil, err } - - ymlType = "resources" - resMap[ymlType] = string(data) - - } - if vp, ok := vsc["pipelines"]; ok { - log.Info("pipelines found preparing to validate") - data, marErr = json.Marshal(vp) - if marErr != nil { - log.Error("failed to marshal to json") - return nil, err, true + pipeDefinition := PipelineDefinition{ + FileName: fileInfo.Name(), + Content: string(fileContent), + YmlType: "pipelines", } - ymlType = "pipelines" - fmt.Println(string(data)) - resMap[ymlType] = string(data) + pipelineDefinitions = append(pipelineDefinitions, pipeDefinition) } - //fmt.Printf("%+v \n", resMap) - return resMap, nil, false -} - -func getPayloadToValidatePipelineResource(resMap map[string]string) (*bytes.Buffer, error) { - payload := getPayloadBasedOnYmlType(resMap) - buf := new(bytes.Buffer) - _, err := buf.ReadFrom(payload) - if err != nil { - log.Error("Failed to read stream to send payload to trigger pipelines") - return nil, err + // need to define handling values + workSpaceValidation := WorkSpaceValidation{ + ProjectId: "1", + Files: pipelineDefinitions, + ProjectName: "default", + Name: "bhanu", } - return buf, err + return json.Marshal(workSpaceValidation) } -func getPayloadBasedOnYmlType(m map[string]string) *strings.Reader { - var resReader, pipReader, valReader *strings.Reader - for ymlType, _ := range m { - if ymlType == "resources" { - resReader = strings.NewReader(`{"fileName":"` + ymlType + `.yml","content": ` + m[ymlType] + `,"ymlType":"` + ymlType + `"}`) - } else if ymlType == "pipelines" { - //fmt.Printf("data : %+v \n", m[ymlType]) - pipReader = strings.NewReader(`{"fileName":"` + ymlType + `.yml","content":` + m[ymlType] + `,"ymlType":"` + ymlType + `"}`) - } +func getFileContentAndBaseName(pathToFile string) ([]byte, os.FileInfo, error) { + fileContent, err := os.ReadFile(pathToFile) + if err != nil { + return nil, nil, err } - if resReader != nil && pipReader != nil { - resAll, err := io.ReadAll(resReader) - if err != nil { - return nil - } - pipAll, err := io.ReadAll(pipReader) - if err != nil { - return nil - } - valReader = strings.NewReader(`{"files":[` + string(resAll) + `,` + string(pipAll) + `]}`) - } else if resReader != nil { - all, err := io.ReadAll(resReader) - if err != nil { - return nil - } - valReader = strings.NewReader(`{"files":[` + string(all) + `]}`) - } else if pipReader != nil { - all, err := io.ReadAll(pipReader) - if err != nil { - return nil - } - valReader = strings.NewReader(`{"files":[` + string(all) + `]}`) + fileInfo, err := os.Stat(pathToFile) + if err != nil { + return nil, nil, err } - return valReader + return fileContent, fileInfo, nil } From 1bb516fea29399562c6e7662bf6b0e21b82da282 Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Tue, 27 Jun 2023 20:37:12 +0530 Subject: [PATCH 06/15] Added workspace test cases --- pipelines/services/validate.go | 16 +++--- pipelines/services/workspace.go | 5 +- tests/pipelinesworkspaces_test.go | 66 +++++++++++++++++++++--- tests/testdata/pipelines-integration.yml | 24 +++++++++ 4 files changed, 90 insertions(+), 21 deletions(-) create mode 100644 tests/testdata/pipelines-integration.yml diff --git a/pipelines/services/validate.go b/pipelines/services/validate.go index 758706523..095a13d61 100644 --- a/pipelines/services/validate.go +++ b/pipelines/services/validate.go @@ -30,8 +30,8 @@ func NewValidateService(client *jfroghttpclient.JfrogHttpClient) *ValidateServic return &ValidateService{client: client} } -func (rs *ValidateService) getHttpDetails() httputils.HttpClientDetails { - httpDetails := rs.ServiceDetails.CreateHttpClientDetails() +func (vs *ValidateService) getHttpDetails() httputils.HttpClientDetails { + httpDetails := vs.ServiceDetails.CreateHttpClientDetails() return httpDetails } @@ -83,9 +83,7 @@ func processValidatePipResourceResponse(resp []byte, userName string) (string, e } if v, ok := rsc["pipelines.yml"]; ok { if v.IsValid != nil && *v.IsValid { - userName = "@bhanur" log.Info("validation of pipeline resources completed successfully ") - log.Info("workspace updated with latest resource files for user: ", userName) msg := color.Green.Sprintf("validation completed ") time.Sleep(2 * time.Second) log.Info(msg) @@ -104,12 +102,10 @@ func processValidatePipResourceResponse(resp []byte, userName string) (string, e return "", errors.New("pipelines.yml not found") } -/* -constructPipelinesURL creates URL with all required details to make api call -like headers, queryParams, apiPath -*/ -func (rs *ValidateService) constructValidateAPIURL(qParams map[string]string, apiPath string) string { - uri, err := url.Parse(rs.ServiceDetails.GetUrl() + apiPath) +// constructPipelinesURL creates URL with all required details to make api call +// like headers, queryParams, apiPath +func (vs *ValidateService) constructValidateAPIURL(qParams map[string]string, apiPath string) string { + uri, err := url.Parse(vs.ServiceDetails.GetUrl() + apiPath) if err != nil { log.Error("Failed to parse pipelines fetch run status url") } diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go index e04992657..48e31182a 100644 --- a/pipelines/services/workspace.go +++ b/pipelines/services/workspace.go @@ -60,7 +60,7 @@ func (ws *WorkspaceService) GetWorkspace() ([]WorkspacesResponse, error) { } wsStatusResp := make([]WorkspacesResponse, 0) err = json.Unmarshal(body, &wsStatusResp) - return wsStatusResp, nil + return wsStatusResp, err } func (ws *WorkspaceService) DeleteWorkspace(workspaceID string) error { @@ -195,7 +195,6 @@ func (ws *WorkspaceService) WorkspaceStepStatus(pipelinesRunID int) ([]byte, err queryParams := map[string]string{ "runIds": strconv.Itoa(pipelinesRunID), "limit": "15", - //"include": "name,statusCode,triggeredAt,externalBuildUrl", } // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaceSteps) @@ -215,7 +214,7 @@ func (ws *WorkspaceService) WorkspaceStepStatus(pipelinesRunID int) ([]byte, err func (ws *WorkspaceService) GetWorkspacePipelines(workspaces []WorkspacesResponse) (map[string]string, error) { pipelineNames := make(map[string]string, 1) log.Info("Collecting pipeline names configured") - // Validate and return pipeline names as slice + // Validate and return pipeline names and branch as map if len(workspaces) > 0 && !(*workspaces[0].IsSyncing) { pipelines := workspaces[0].PipelinesYmlPropertyBag.Pipelines for _, pi := range pipelines { diff --git a/tests/pipelinesworkspaces_test.go b/tests/pipelinesworkspaces_test.go index c21245882..9d4bb1b58 100644 --- a/tests/pipelinesworkspaces_test.go +++ b/tests/pipelinesworkspaces_test.go @@ -3,15 +3,16 @@ package tests import ( "encoding/json" "fmt" + "github.com/jfrog/jfrog-client-go/pipelines/services" "github.com/stretchr/testify/assert" "os" - "strconv" "testing" ) func TestPipelinesWorkspaceService(t *testing.T) { t.Run("test trigger pipelines workspace when resources are not valid", TestWorkspaceValidationWhenPipelinesResourcesAreNotValid) t.Run("test trigger pipelines workspace when resources are valid", TestWorkspaceValidationWhenPipelinesResourcesAreValid) + t.Run("test workspace sync when it fails for the first time and succeeds later", TestWorkspaceValidationFailureAndSucceedsWhenValidIntegrationCreated) } func TestWorkspaceValidationWhenPipelinesResourcesAreNotValid(t *testing.T) { @@ -19,7 +20,7 @@ func TestWorkspaceValidationWhenPipelinesResourcesAreNotValid(t *testing.T) { if !assert.NotEmpty(t, *PipelinesAccessToken, "cannot run pipelines tests without access token configured") { return } - filePath := "/Users/bhanur/go/src/pipeline-cli/jfrog-client-go/tests/testdata/pipelines.yml" + filePath := "testdata/pipelines.yml" // get pipelines.yaml content pipelineRes, err := getWorkspaceRunPayload([]string{filePath}) @@ -38,7 +39,7 @@ func TestWorkspaceValidationWhenPipelinesResourcesAreValid(t *testing.T) { if !assert.NotEmpty(t, *PipelinesAccessToken, "cannot run pipelines tests without access token configured") { return } - filePath := "/Users/bhanur/go/src/pipeline-cli/jfrog-client-go/tests/testdata/pipelines.yml" + filePath := "testdata/pipelines.yml" // get pipelines.yaml content pipelineRes, err := getWorkspaceRunPayload([]string{filePath}) @@ -53,9 +54,60 @@ func TestWorkspaceValidationWhenPipelinesResourcesAreValid(t *testing.T) { if len(wsResp) < 1 { assert.Fail(t, "No workspace created") } - for _, ws := range wsResp { + syncStatusResp, err := testPipelinesWorkspaceService.WorkspacePollSyncStatus() + assert.NoError(t, err) + for _, ws := range syncStatusResp { + fmt.Printf("%+v \n", ws) + err = testPipelinesWorkspaceService.DeleteWorkspace("default") + } + assert.NoError(t, err) +} + +func TestWorkspaceValidationFailureAndSucceedsWhenValidIntegrationCreated(t *testing.T) { + if !assert.NotEmpty(t, *PipelinesAccessToken, "cannot run pipelines tests without access token configured") { + return + } + filePath := "testdata/pipelines-integration.yml" + + // Get pipelines-integration.yaml content + pipelineRes, err := getWorkspaceRunPayload([]string{filePath}) + if !assert.NoError(t, err) { + return + } + // Call workspace validation + err = testPipelinesWorkspaceService.ValidateWorkspace(pipelineRes) + assert.Error(t, err) + + // Create valid integration required for pipelines + id, err := testsPipelinesIntegrationsService.CreateArtifactoryIntegration("int_workspace_artifactory", testsDummyRtUrl, testsDummyUser, testsDummyApiKey) + if !assert.NoError(t, err) { + return + } + defer deleteIntegrationAndAssert(t, id) + getIntegrationAndAssert(t, id, "int_workspace_artifactory", services.ArtifactoryName) + + // Validate workspace pipelines should be successful here + err = testPipelinesWorkspaceService.ValidateWorkspace(pipelineRes) + assert.NoError(t, err) + + wsResp, err := testPipelinesWorkspaceService.GetWorkspace() + assert.NoError(t, err) + if len(wsResp) < 1 { + assert.Fail(t, "No workspace created") + } + var syncStatusResp []services.WorkspacesResponse + syncStatusResp, err = testPipelinesWorkspaceService.WorkspacePollSyncStatus() + if !assert.NoError(t, err) { + return + } + + err = testPipelinesWorkspaceService.WorkspaceSync("default") + assert.NoError(t, err) + syncStatusResp, err = testPipelinesWorkspaceService.WorkspacePollSyncStatus() + assert.NoError(t, err) + for _, ws := range syncStatusResp { fmt.Printf("%+v \n", ws) - err = testPipelinesWorkspaceService.DeleteWorkspace(strconv.Itoa(ws.ID)) + err = testPipelinesWorkspaceService.DeleteWorkspace("default") } assert.NoError(t, err) } @@ -87,12 +139,10 @@ func getWorkspaceRunPayload(resources []string) ([]byte, error) { } pipelineDefinitions = append(pipelineDefinitions, pipeDefinition) } - // need to define handling values workSpaceValidation := WorkSpaceValidation{ - ProjectId: "1", Files: pipelineDefinitions, ProjectName: "default", - Name: "bhanu", + Name: "", } return json.Marshal(workSpaceValidation) } diff --git a/tests/testdata/pipelines-integration.yml b/tests/testdata/pipelines-integration.yml new file mode 100644 index 000000000..09fe954b7 --- /dev/null +++ b/tests/testdata/pipelines-integration.yml @@ -0,0 +1,24 @@ +pipelines: + - name: pipelines_run_int_test + steps: + - name: prepare_env + type: Bash + execution: + onExecute: + - cd /tmp + - mkdir -p "jfrog-cli" + - touch test-pilines.yaml + - echo "test" >> test-pipelines.yaml + - name: build_binary + type: Bash + configuration: + inputSteps: + - name: prepare_env + integrations: + - name: int_workspace_artifactory + execution: + onExecute: + - cd /tmp + - mkdir -p "target/pipelines" + - cd target/pipelines + - touch make_data \ No newline at end of file From 07f96500daac996075d01d97b6ca62f08317fc45 Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Sun, 2 Jul 2023 10:55:03 +0530 Subject: [PATCH 07/15] Refactored code --- pipelines/manager.go | 2 +- pipelines/services/validate.go | 66 ++++++++++++++++----------------- pipelines/services/workspace.go | 6 +-- 3 files changed, 36 insertions(+), 38 deletions(-) diff --git a/pipelines/manager.go b/pipelines/manager.go index 0aef7e43f..ee8a99a43 100644 --- a/pipelines/manager.go +++ b/pipelines/manager.go @@ -135,7 +135,7 @@ func (sm *PipelinesServicesManager) CancelRun(runID int) error { return runService.CancelRun(runID) } -func (sm *PipelinesServicesManager) ValidatePipelineSources(data []byte) (string, error) { +func (sm *PipelinesServicesManager) ValidatePipelineSources(data []byte) error { validateService := services.NewValidateService(sm.client) validateService.ServiceDetails = sm.config.GetServiceDetails() return validateService.ValidatePipeline(data) diff --git a/pipelines/services/validate.go b/pipelines/services/validate.go index 095a13d61..9238327e0 100644 --- a/pipelines/services/validate.go +++ b/pipelines/services/validate.go @@ -3,7 +3,6 @@ package services import ( "encoding/json" "errors" - "fmt" "github.com/gookit/color" "github.com/jfrog/jfrog-client-go/auth" "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" @@ -13,12 +12,10 @@ import ( "net/http" "net/url" "strconv" - "strings" - "time" ) const ( - validatePipResourcePath = "api/v1/validateYaml/json" + validatePipResourcePath = "api/v1/validateYaml" ) type ValidateService struct { @@ -35,71 +32,72 @@ func (vs *ValidateService) getHttpDetails() httputils.HttpClientDetails { return httpDetails } -func (vs *ValidateService) ValidatePipeline(data []byte) (string, error) { - var opMsg string +func (vs *ValidateService) ValidatePipeline(data []byte) error { var err error httpDetails := vs.getHttpDetails() - // query params + // Query params m := make(map[string]string, 0) // URL Construction uri := vs.constructValidateAPIURL(m, validatePipResourcePath) - // headers + // Headers headers := make(map[string]string, 0) headers["Content-Type"] = "application/json" httpDetails.Headers = headers + log.Debug(string(data)) - // send post request + // Send post request resp, body, err := vs.client.SendPost(uri, data, &httpDetails) if err != nil { - return "", err + return err } if err := errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { - return "", err + return err } - // process response + // Process response if nil != resp && resp.StatusCode == http.StatusOK { log.Info("Processing resources yaml response ") - time.Sleep(2 * time.Second) log.Info(string(body)) - opMsg, err = processValidatePipResourceResponse(body, "") + err = processValidatePipResourceResponse(body) if err != nil { log.Error("Failed to process resources validation response") } } - return opMsg, nil + return nil } // processValidatePipResourceResponse process validate pipeline resource response -func processValidatePipResourceResponse(resp []byte, userName string) (string, error) { - fmt.Println("unfurling response") - rsc := make(map[string]ValidationResponse) - err := json.Unmarshal(resp, &rsc) +func processValidatePipResourceResponse(resp []byte) error { + validationResponse := make(map[string]ValidationResponse) + err := json.Unmarshal(resp, &validationResponse) if err != nil { - return "", err + return err } - if v, ok := rsc["pipelines.yml"]; ok { + if len(validationResponse) == 0 { + return errors.New("pipelines not found") + } + for k, v := range validationResponse { if v.IsValid != nil && *v.IsValid { - log.Info("validation of pipeline resources completed successfully ") - msg := color.Green.Sprintf("validation completed ") - time.Sleep(2 * time.Second) + fileName := color.Green.Sprintf("%s", k) + log.Error(fileName) + log.Info("Validation of pipeline resources completed successfully ") + msg := color.Green.Sprintf("Validation completed") log.Info(msg) - return msg, nil } else { - log.Error("pipeline resources validation FAILED!! check below errors and try again") - msg := v.Errors[0].Text - lnNum := v.Errors[0].LineNumber - msgs := strings.Split(msg, ":") - opMsg := color.Red.Sprintf("%s", msgs[0]+":"+strconv.Itoa(lnNum)+":\n"+" "+msgs[1]) - //log.PrintMessage("Please refer pipelines documentation " + coreutils.PrintLink("https://www.jfrog.com/confluence/display/JFROG/Managing+Pipeline+Sources#ManagingPipelineSources-ValidatingYAML") + "") - log.Error(opMsg) - return opMsg, nil + fileName := color.Red.Sprintf("%s", k) + log.Error(fileName) + validationErrors := v.Errors + for _, validationError := range validationErrors { + lineNum := validationError.LineNumber + errorMessage := color.Red.Sprintf("%s", validationError.Text+":"+strconv.Itoa(lineNum)) + log.Error(errorMessage) + } } } - return "", errors.New("pipelines.yml not found") + return nil } // constructPipelinesURL creates URL with all required details to make api call diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go index 48e31182a..457d5a576 100644 --- a/pipelines/services/workspace.go +++ b/pipelines/services/workspace.go @@ -63,9 +63,9 @@ func (ws *WorkspaceService) GetWorkspace() ([]WorkspacesResponse, error) { return wsStatusResp, err } -func (ws *WorkspaceService) DeleteWorkspace(workspaceID string) error { +func (ws *WorkspaceService) DeleteWorkspace(projectName string) error { httpDetails := ws.getHttpDetails() - deleteWorkspaceAPI := strings.Replace(deleteWorkspace, ":project", workspaceID, 1) + deleteWorkspaceAPI := strings.Replace(deleteWorkspace, ":project", projectName, 1) // Query params queryParams := make(map[string]string, 0) // URL construction @@ -109,7 +109,7 @@ func (ws *WorkspaceService) ValidateWorkspace(data []byte) error { func (ws *WorkspaceService) WorkspaceSync(project string) error { httpDetails := ws.getHttpDetails() // Query params - queryParams := make(map[string]string, 0) // Query params + queryParams := make(map[string]string, 0) syncWorkspaceAPI := strings.Replace(workspaceSync, ":project", project, 1) // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), syncWorkspaceAPI) From 9d51140a98af96fa56ddaf969b74fccda523b552 Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Mon, 28 Aug 2023 13:47:51 +0530 Subject: [PATCH 08/15] Updated syncWorkspace --- pipelines/services/workspace.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go index 457d5a576..e363cc4d5 100644 --- a/pipelines/services/workspace.go +++ b/pipelines/services/workspace.go @@ -19,7 +19,7 @@ const ( workspaces = "api/v1/workspaces" deleteWorkspace = "api/v1/workspaces/:project" validateWorkspace = "api/v1/validateWorkspace" - workspaceSync = "api/v1/syncWorkspace/:project" + workspaceSync = "api/v1/syncWorkspace" workspacePipelines = "api/v1/pipelines" workspaceRuns = "api/v1/runs" workspaceSteps = "api/v1/steps" @@ -110,6 +110,7 @@ func (ws *WorkspaceService) WorkspaceSync(project string) error { httpDetails := ws.getHttpDetails() // Query params queryParams := make(map[string]string, 0) + queryParams["projectName"] = project syncWorkspaceAPI := strings.Replace(workspaceSync, ":project", project, 1) // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), syncWorkspaceAPI) From a77a8f057297123ad2389f80f692f7bd255c3c2a Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Mon, 28 Aug 2023 17:13:47 +0530 Subject: [PATCH 09/15] Updated pipelines workspace service --- pipelines/services/run.go | 2 +- pipelines/services/workspace.go | 5 ++++- tests/pipelinesworkspaces_test.go | 7 +++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pipelines/services/run.go b/pipelines/services/run.go index c3e99d570..5de4f6bf6 100644 --- a/pipelines/services/run.go +++ b/pipelines/services/run.go @@ -122,7 +122,7 @@ func (rs *RunService) TriggerPipelineRun(branch, pipeline string, isMultiBranch } // Polling execution _, err = pollingExecutor.Execute() - return nil + return err } func (rs *RunService) CancelRun(runID int) error { diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go index e363cc4d5..83ed52ff5 100644 --- a/pipelines/services/workspace.go +++ b/pipelines/services/workspace.go @@ -163,9 +163,12 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI PollingAction: pollingAction, MsgPrefix: "Get pipeline workspace sync status...", } + pipeRunIDs := make([]PipelinesRunID, 0) // Polling execution body, err := pollingExecutor.Execute() - pipeRunIDs := make([]PipelinesRunID, 0) + if err != nil { + return pipeRunIDs, err + } err = json.Unmarshal(body, &pipeRunIDs) return pipeRunIDs, err } diff --git a/tests/pipelinesworkspaces_test.go b/tests/pipelinesworkspaces_test.go index 9d4bb1b58..b6edde3b3 100644 --- a/tests/pipelinesworkspaces_test.go +++ b/tests/pipelinesworkspaces_test.go @@ -95,17 +95,16 @@ func TestWorkspaceValidationFailureAndSucceedsWhenValidIntegrationCreated(t *tes if len(wsResp) < 1 { assert.Fail(t, "No workspace created") } - var syncStatusResp []services.WorkspacesResponse - syncStatusResp, err = testPipelinesWorkspaceService.WorkspacePollSyncStatus() + _, err = testPipelinesWorkspaceService.WorkspacePollSyncStatus() if !assert.NoError(t, err) { return } err = testPipelinesWorkspaceService.WorkspaceSync("default") assert.NoError(t, err) - syncStatusResp, err = testPipelinesWorkspaceService.WorkspacePollSyncStatus() + syncStatusRespSuccess, err := testPipelinesWorkspaceService.WorkspacePollSyncStatus() assert.NoError(t, err) - for _, ws := range syncStatusResp { + for _, ws := range syncStatusRespSuccess { fmt.Printf("%+v \n", ws) err = testPipelinesWorkspaceService.DeleteWorkspace("default") } From a2208e2cfd8c7dae42a6023b2bd12bd75eba0733 Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Mon, 28 Aug 2023 17:25:22 +0530 Subject: [PATCH 10/15] Added static analysis fixes --- pipelines/services/validate.go | 2 +- pipelines/services/workspace.go | 22 ++++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pipelines/services/validate.go b/pipelines/services/validate.go index 9238327e0..9c0d16d2f 100644 --- a/pipelines/services/validate.go +++ b/pipelines/services/validate.go @@ -43,7 +43,7 @@ func (vs *ValidateService) ValidatePipeline(data []byte) error { uri := vs.constructValidateAPIURL(m, validatePipResourcePath) // Headers - headers := make(map[string]string, 0) + headers := make(map[string]string) headers["Content-Type"] = "application/json" httpDetails.Headers = headers log.Debug(string(data)) diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go index 83ed52ff5..cf604e27a 100644 --- a/pipelines/services/workspace.go +++ b/pipelines/services/workspace.go @@ -43,7 +43,7 @@ func (ws *WorkspaceService) getHttpDetails() httputils.HttpClientDetails { func (ws *WorkspaceService) GetWorkspace() ([]WorkspacesResponse, error) { httpDetails := ws.getHttpDetails() // Query params - queryParams := make(map[string]string, 0) + queryParams := make(map[string]string) // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaces) if err != nil { @@ -86,14 +86,14 @@ func (ws *WorkspaceService) DeleteWorkspace(projectName string) error { func (ws *WorkspaceService) ValidateWorkspace(data []byte) error { httpDetails := ws.getHttpDetails() // Query params - queryParams := make(map[string]string, 0) + queryParams := make(map[string]string) // URL construction uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), validateWorkspace) if err != nil { return err } // Headers - headers := make(map[string]string, 0) + headers := make(map[string]string) headers["Content-Type"] = "application/json" headers["User-Agent"] = "jfrog-client-go/1.24.3" httpDetails.Headers = headers @@ -109,7 +109,7 @@ func (ws *WorkspaceService) ValidateWorkspace(data []byte) error { func (ws *WorkspaceService) WorkspaceSync(project string) error { httpDetails := ws.getHttpDetails() // Query params - queryParams := make(map[string]string, 0) + queryParams := make(map[string]string) queryParams["projectName"] = project syncWorkspaceAPI := strings.Replace(workspaceSync, ":project", project, 1) // URL construction @@ -152,7 +152,7 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI } pipeRunIDs := make([]PipelinesRunID, 0) err = json.Unmarshal(body, &pipeRunIDs) - if pipeRunIDs[0].LatestRunID == 0 { + if len(pipeRunIDs) > 0 && pipeRunIDs[0].LatestRunID == 0 { return false, body, errors.New("Pipeline didnt start running yet") } return true, body, err @@ -255,11 +255,13 @@ func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, err log.Error("failed to unmarshal validation response") return true, body, err } - if len(wsStatusResp) > 0 && *wsStatusResp[0].IsSyncing { - fmt.Printf("%+v \n", wsStatusResp) - return false, body, err - } else if wsStatusResp[0].LastSyncStatusCode == 4003 || wsStatusResp[0].LastSyncStatusCode == 4004 { - return true, body, err + if len(wsStatusResp) > 0 { + if *wsStatusResp[0].IsSyncing { + fmt.Printf("%+v \n", wsStatusResp) + return false, body, err + } else if wsStatusResp[0].LastSyncStatusCode == 4003 || wsStatusResp[0].LastSyncStatusCode == 4004 { + return true, body, err + } } return true, body, err } From 0a8b07d89cfbd772790a5d214ef9278324338caa Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Mon, 28 Aug 2023 17:35:43 +0530 Subject: [PATCH 11/15] Added static analysis fixes --- pipelines/services/workspace.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go index cf604e27a..782fb2d58 100644 --- a/pipelines/services/workspace.go +++ b/pipelines/services/workspace.go @@ -152,8 +152,10 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI } pipeRunIDs := make([]PipelinesRunID, 0) err = json.Unmarshal(body, &pipeRunIDs) - if len(pipeRunIDs) > 0 && pipeRunIDs[0].LatestRunID == 0 { - return false, body, errors.New("Pipeline didnt start running yet") + for range pipeRunIDs { + if pipeRunIDs[0].LatestRunID == 0 { + return false, body, errors.New("Pipeline didnt start running yet") + } } return true, body, err } @@ -255,7 +257,7 @@ func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, err log.Error("failed to unmarshal validation response") return true, body, err } - if len(wsStatusResp) > 0 { + for range wsStatusResp { if *wsStatusResp[0].IsSyncing { fmt.Printf("%+v \n", wsStatusResp) return false, body, err From 9e0fb7af9b6db9651e7e1cd1b46318e7d4993eac Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Mon, 28 Aug 2023 17:41:15 +0530 Subject: [PATCH 12/15] Added static analysis fixes --- pipelines/services/workspace.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go index 782fb2d58..6a4a3de64 100644 --- a/pipelines/services/workspace.go +++ b/pipelines/services/workspace.go @@ -152,10 +152,11 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI } pipeRunIDs := make([]PipelinesRunID, 0) err = json.Unmarshal(body, &pipeRunIDs) - for range pipeRunIDs { - if pipeRunIDs[0].LatestRunID == 0 { + for i := range pipeRunIDs { + if pipeRunIDs[i].LatestRunID == 0 { return false, body, errors.New("Pipeline didnt start running yet") } + break } return true, body, err } @@ -257,13 +258,14 @@ func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, err log.Error("failed to unmarshal validation response") return true, body, err } - for range wsStatusResp { - if *wsStatusResp[0].IsSyncing { + for i := range wsStatusResp { + if *wsStatusResp[i].IsSyncing { fmt.Printf("%+v \n", wsStatusResp) return false, body, err - } else if wsStatusResp[0].LastSyncStatusCode == 4003 || wsStatusResp[0].LastSyncStatusCode == 4004 { + } else if wsStatusResp[i].LastSyncStatusCode == 4003 || wsStatusResp[i].LastSyncStatusCode == 4004 { return true, body, err } + break } return true, body, err } From 7f1065ad0dbb5dde95bc510cd50552e97ccd8550 Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Mon, 28 Aug 2023 17:48:23 +0530 Subject: [PATCH 13/15] Added static analysis fixes --- pipelines/services/workspace.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go index 6a4a3de64..973731f6d 100644 --- a/pipelines/services/workspace.go +++ b/pipelines/services/workspace.go @@ -152,11 +152,16 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI } pipeRunIDs := make([]PipelinesRunID, 0) err = json.Unmarshal(body, &pipeRunIDs) + if err != nil { + return false, body, errors.New("Failed to parse response to get run status") + } for i := range pipeRunIDs { + if i > 0 { + break + } if pipeRunIDs[i].LatestRunID == 0 { return false, body, errors.New("Pipeline didnt start running yet") } - break } return true, body, err } @@ -259,13 +264,15 @@ func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, err return true, body, err } for i := range wsStatusResp { + if i > 0 { + break + } if *wsStatusResp[i].IsSyncing { fmt.Printf("%+v \n", wsStatusResp) return false, body, err } else if wsStatusResp[i].LastSyncStatusCode == 4003 || wsStatusResp[i].LastSyncStatusCode == 4004 { return true, body, err } - break } return true, body, err } From 9f2dfce04bb6e84a1c6cf0b45f9a834c66047029 Mon Sep 17 00:00:00 2001 From: Bhanu Reddy Date: Thu, 21 Sep 2023 23:05:50 +0530 Subject: [PATCH 14/15] Removed unnecessary log message --- pipelines/services/validate.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pipelines/services/validate.go b/pipelines/services/validate.go index 9c0d16d2f..a761772ea 100644 --- a/pipelines/services/validate.go +++ b/pipelines/services/validate.go @@ -81,8 +81,6 @@ func processValidatePipResourceResponse(resp []byte) error { } for k, v := range validationResponse { if v.IsValid != nil && *v.IsValid { - fileName := color.Green.Sprintf("%s", k) - log.Error(fileName) log.Info("Validation of pipeline resources completed successfully ") msg := color.Green.Sprintf("Validation completed") log.Info(msg) From 64296920828c776743b591a532158f4e8b9a77a1 Mon Sep 17 00:00:00 2001 From: Bhanu Reddy <117445796+bhanurp@users.noreply.github.com> Date: Thu, 14 Dec 2023 12:15:27 +0530 Subject: [PATCH 15/15] Added error checks and suggestions Co-authored-by: Yahav Itzhak --- pipelines/manager.go | 8 +++--- pipelines/services/run.go | 2 +- pipelines/services/types.go | 1 - pipelines/services/validate.go | 11 ++++---- pipelines/services/workspace.go | 42 +++++++++++++++---------------- tests/pipelinesworkspaces_test.go | 1 - 6 files changed, 30 insertions(+), 35 deletions(-) diff --git a/pipelines/manager.go b/pipelines/manager.go index 66cc39f1f..9e68826f7 100644 --- a/pipelines/manager.go +++ b/pipelines/manager.go @@ -155,7 +155,7 @@ func (sm *PipelinesServicesManager) WorkspacePollSyncStatus() ([]services.Worksp return workspaceService.WorkspacePollSyncStatus() } -func (sm *PipelinesServicesManager) WorkspacePipelines() (map[string]string, error) { +func (sm *PipelinesServicesManager) GetWorkspacePipelines() (map[string]string, error) { workspaceService := services.NewWorkspaceService(sm.client) workspaceService.ServiceDetails = sm.config.GetServiceDetails() workspacesResponseStatus, err := sm.WorkspacePollSyncStatus() @@ -171,19 +171,19 @@ func (sm *PipelinesServicesManager) WorkspaceTriggerPipelines(branch, pipName st return sm.TriggerPipelineRun(branch, pipName, isMultiBranch) } -func (sm *PipelinesServicesManager) WorkspaceRunIDs(pipelineNames []string) ([]services.PipelinesRunID, error) { +func (sm *PipelinesServicesManager) GetWorkspaceRunIDs(pipelineNames []string) ([]services.PipelinesRunID, error) { workspaceService := services.NewWorkspaceService(sm.client) workspaceService.ServiceDetails = sm.config.GetServiceDetails() return workspaceService.WorkspaceRunIDs(pipelineNames) } -func (sm *PipelinesServicesManager) WorkspaceRunStatus(pipeRunID int) ([]byte, error) { +func (sm *PipelinesServicesManager) GetWorkspaceRunStatus(pipeRunID int) ([]byte, error) { workspaceService := services.NewWorkspaceService(sm.client) workspaceService.ServiceDetails = sm.config.GetServiceDetails() return workspaceService.WorkspaceRunStatus(pipeRunID) } -func (sm *PipelinesServicesManager) WorkspaceStepStatus(pipeRunID int) ([]byte, error) { +func (sm *PipelinesServicesManager) GetWorkspaceStepStatus(pipeRunID int) ([]byte, error) { workspaceService := services.NewWorkspaceService(sm.client) workspaceService.ServiceDetails = sm.config.GetServiceDetails() return workspaceService.WorkspaceStepStatus(pipeRunID) diff --git a/pipelines/services/run.go b/pipelines/services/run.go index 5de4f6bf6..395a211cf 100644 --- a/pipelines/services/run.go +++ b/pipelines/services/run.go @@ -111,7 +111,7 @@ func (rs *RunService) TriggerPipelineRun(branch, pipeline string, isMultiBranch if err := errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { return false, body, err } - log.Info(fmt.Sprintf("Triggered successfully\n%s %s \n%14s %s", "PipelineName :", pipeline, "Branch :", branch)) + log.Info(fmt.Sprintf("Triggered successfully\n%s %s \n%14s %s", "PipelineName:", pipeline, "Branch:", branch)) return true, body, err } pollingExecutor := &httputils.PollingExecutor{ diff --git a/pipelines/services/types.go b/pipelines/services/types.go index c60d3556c..ca2bd1a2f 100644 --- a/pipelines/services/types.go +++ b/pipelines/services/types.go @@ -145,7 +145,6 @@ type Console struct { } // Validation Types - type ValidationResponse struct { IsValid *bool `json:"isValid,omitempty"` Errors []ValidationErrors `json:"errors,omitempty"` diff --git a/pipelines/services/validate.go b/pipelines/services/validate.go index a761772ea..74afff134 100644 --- a/pipelines/services/validate.go +++ b/pipelines/services/validate.go @@ -28,7 +28,7 @@ func NewValidateService(client *jfroghttpclient.JfrogHttpClient) *ValidateServic } func (vs *ValidateService) getHttpDetails() httputils.HttpClientDetails { - httpDetails := vs.ServiceDetails.CreateHttpClientDetails() + httpDetails := vs.CreateHttpClientDetails() return httpDetails } @@ -44,9 +44,8 @@ func (vs *ValidateService) ValidatePipeline(data []byte) error { // Headers headers := make(map[string]string) - headers["Content-Type"] = "application/json" + utils.SetContentType("application/json", &headers) httpDetails.Headers = headers - log.Debug(string(data)) // Send post request resp, body, err := vs.client.SendPost(uri, data, &httpDetails) @@ -74,14 +73,14 @@ func processValidatePipResourceResponse(resp []byte) error { validationResponse := make(map[string]ValidationResponse) err := json.Unmarshal(resp, &validationResponse) if err != nil { - return err + return errorutils.CheckError(err) } if len(validationResponse) == 0 { - return errors.New("pipelines not found") + return errorutils.CheckErrorf("pipelines not found") } for k, v := range validationResponse { if v.IsValid != nil && *v.IsValid { - log.Info("Validation of pipeline resources completed successfully ") + log.Info("Validation of pipeline resources completed successfully") msg := color.Green.Sprintf("Validation completed") log.Info(msg) } else { diff --git a/pipelines/services/workspace.go b/pipelines/services/workspace.go index 973731f6d..4b88ff423 100644 --- a/pipelines/services/workspace.go +++ b/pipelines/services/workspace.go @@ -37,7 +37,7 @@ func NewWorkspaceService(client *jfroghttpclient.JfrogHttpClient) *WorkspaceServ } func (ws *WorkspaceService) getHttpDetails() httputils.HttpClientDetails { - return ws.ServiceDetails.CreateHttpClientDetails() + return ws.CreateHttpClientDetails() } func (ws *WorkspaceService) GetWorkspace() ([]WorkspacesResponse, error) { @@ -60,7 +60,7 @@ func (ws *WorkspaceService) GetWorkspace() ([]WorkspacesResponse, error) { } wsStatusResp := make([]WorkspacesResponse, 0) err = json.Unmarshal(body, &wsStatusResp) - return wsStatusResp, err + return wsStatusResp, errorutils.CheckError(err) } func (ws *WorkspaceService) DeleteWorkspace(projectName string) error { @@ -69,7 +69,7 @@ func (ws *WorkspaceService) DeleteWorkspace(projectName string) error { // Query params queryParams := make(map[string]string, 0) // URL construction - uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), deleteWorkspaceAPI) + uri, err := constructPipelinesURL(queryParams, ws.GetUrl(), deleteWorkspaceAPI) if err != nil { return err } @@ -88,13 +88,13 @@ func (ws *WorkspaceService) ValidateWorkspace(data []byte) error { // Query params queryParams := make(map[string]string) // URL construction - uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), validateWorkspace) + uri, err := constructPipelinesURL(queryParams, ws.GetUrl(), validateWorkspace) if err != nil { return err } // Headers headers := make(map[string]string) - headers["Content-Type"] = "application/json" + utils.SetContentType("application/json", &headers) headers["User-Agent"] = "jfrog-client-go/1.24.3" httpDetails.Headers = headers // Prepare request @@ -113,7 +113,7 @@ func (ws *WorkspaceService) WorkspaceSync(project string) error { queryParams["projectName"] = project syncWorkspaceAPI := strings.Replace(workspaceSync, ":project", project, 1) // URL construction - uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), syncWorkspaceAPI) + uri, err := constructPipelinesURL(queryParams, ws.GetUrl(), syncWorkspaceAPI) if err != nil { return err } @@ -126,7 +126,7 @@ func (ws *WorkspaceService) WorkspaceSync(project string) error { return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK) } -func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunID, error) { +func (ws *WorkspaceService) GetWorkspaceRunIDs(pipelines []string) ([]PipelinesRunID, error) { httpDetails := ws.getHttpDetails() pipelineFilter := strings.Join(pipelines, ",") // Query params @@ -136,7 +136,7 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI "include": "latestRunId,name", } // URL construction - uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspacePipelines) + uri, err := constructPipelinesURL(queryParams, ws.GetUrl(), workspacePipelines) if err != nil { return nil, err } @@ -153,14 +153,14 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI pipeRunIDs := make([]PipelinesRunID, 0) err = json.Unmarshal(body, &pipeRunIDs) if err != nil { - return false, body, errors.New("Failed to parse response to get run status") + return false, body, errorutils.CheckErrorf("failed to parse response to get run status") } for i := range pipeRunIDs { if i > 0 { break } if pipeRunIDs[i].LatestRunID == 0 { - return false, body, errors.New("Pipeline didnt start running yet") + return false, body, errorutils.CheckErrorf("pipeline didn't start running yet") } } return true, body, err @@ -178,10 +178,10 @@ func (ws *WorkspaceService) WorkspaceRunIDs(pipelines []string) ([]PipelinesRunI return pipeRunIDs, err } err = json.Unmarshal(body, &pipeRunIDs) - return pipeRunIDs, err + return pipeRunIDs, errorutils.CheckError(err) } -func (ws *WorkspaceService) WorkspaceRunStatus(pipelinesRunID int) ([]byte, error) { +func (ws *WorkspaceService) GetWorkspaceRunStatus(pipelinesRunID int) ([]byte, error) { httpDetails := ws.getHttpDetails() // Query params // TODO ADD include in query param if needed @@ -201,7 +201,7 @@ func (ws *WorkspaceService) WorkspaceRunStatus(pipelinesRunID int) ([]byte, erro return body, err } -func (ws *WorkspaceService) WorkspaceStepStatus(pipelinesRunID int) ([]byte, error) { +func (ws *WorkspaceService) GetWorkspaceStepStatus(pipelinesRunID int) ([]byte, error) { httpDetails := ws.getHttpDetails() // Query params queryParams := map[string]string{ @@ -223,7 +223,7 @@ func (ws *WorkspaceService) WorkspaceStepStatus(pipelinesRunID int) ([]byte, err return body, err } -func (ws *WorkspaceService) GetWorkspacePipelines(workspaces []WorkspacesResponse) (map[string]string, error) { +func (ws *WorkspaceService) GetWorkspacePipelines(workspaces []WorkspacesResponse) map[string]string { pipelineNames := make(map[string]string, 1) log.Info("Collecting pipeline names configured") // Validate and return pipeline names and branch as map @@ -241,12 +241,11 @@ func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, err // Query params queryParams := make(map[string]string, 0) // URL construction - uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), workspaces) + uri, err := constructPipelinesURL(queryParams, ws.GetUrl(), workspaces) if err != nil { return nil, err } pollingAction := func() (shouldStop bool, responseBody []byte, err error) { - log.Info("Polling for pipeline resource sync") // Prepare request resp, body, _, err := ws.client.SendGet(uri, true, &httpDetails) if err != nil { @@ -260,8 +259,7 @@ func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, err wsStatusResp := make([]WorkspacesResponse, 0) err = json.Unmarshal(body, &wsStatusResp) if err != nil { - log.Error("failed to unmarshal validation response") - return true, body, err + return true, body, errorutils.CheckError(err) } for i := range wsStatusResp { if i > 0 { @@ -289,7 +287,7 @@ func (ws *WorkspaceService) WorkspacePollSyncStatus() ([]WorkspacesResponse, err } workspaceStatusResponse := make([]WorkspacesResponse, 0) err = json.Unmarshal(body, &workspaceStatusResponse) - return workspaceStatusResponse, err + return workspaceStatusResponse, errorutils.CheckError(err) } // GetStepLogsUsingStepID retrieve steps logs using step id @@ -299,7 +297,7 @@ func (ws *WorkspaceService) GetStepLogsUsingStepID(stepID string) (map[string][] // Query params queryParams := make(map[string]string, 0) // URL construction - uri, err := constructPipelinesURL(queryParams, ws.ServiceDetails.GetUrl(), stepConsolesAPI) + uri, err := constructPipelinesURL(queryParams, ws.GetUrl(), stepConsolesAPI) if err != nil { return nil, err } @@ -314,7 +312,7 @@ func (ws *WorkspaceService) GetStepLogsUsingStepID(stepID string) (map[string][] } consoles := make(map[string][]Console) err = json.Unmarshal(body, &consoles) - return consoles, err + return consoles, errorutils.CheckError(err) } // GetStepletLogsUsingStepID retrieve steps logs using step id @@ -339,5 +337,5 @@ func (ws *WorkspaceService) GetStepletLogsUsingStepID(stepID string) (map[string } consoles := make(map[string][]Console) err = json.Unmarshal(body, &consoles) - return consoles, err + return consoles, errorutils.CheckError(err) } diff --git a/tests/pipelinesworkspaces_test.go b/tests/pipelinesworkspaces_test.go index b6edde3b3..efe6cf403 100644 --- a/tests/pipelinesworkspaces_test.go +++ b/tests/pipelinesworkspaces_test.go @@ -27,7 +27,6 @@ func TestWorkspaceValidationWhenPipelinesResourcesAreNotValid(t *testing.T) { if !assert.NoError(t, err) { return } - fmt.Printf("Bytes received : %s \n", pipelineRes) if !assert.NoError(t, err) { return }