Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introducing Workspaces into pipelines #793

Closed
wants to merge 20 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 82 additions & 0 deletions pipelines/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import (
"github.com/jfrog/jfrog-client-go/config"
"github.com/jfrog/jfrog-client-go/http/jfroghttpclient"
"github.com/jfrog/jfrog-client-go/pipelines/services"

Check failure on line 6 in pipelines/manager.go

View workflow job for this annotation

GitHub Actions / Static-Check

could not import github.com/jfrog/jfrog-client-go/pipelines/services (-: # github.com/jfrog/jfrog-client-go/pipelines/services
)

type PipelinesServicesManager struct {
Expand Down Expand Up @@ -136,3 +136,85 @@
runService.ServiceDetails = sm.config.GetServiceDetails()
return runService.CancelRun(runID)
}

func (sm *PipelinesServicesManager) ValidatePipelineSources(data []byte) error {
validateService := services.NewValidateService(sm.client)
validateService.ServiceDetails = sm.config.GetServiceDetails()
return validateService.ValidatePipeline(data)
}

func (sm *PipelinesServicesManager) WorkspaceSync(projectKey string) error {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
return workspaceService.WorkspaceSync(projectKey)
}

func (sm *PipelinesServicesManager) WorkspacePollSyncStatus() ([]services.WorkspacesResponse, error) {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
return workspaceService.WorkspacePollSyncStatus()
}

func (sm *PipelinesServicesManager) GetWorkspacePipelines() (map[string]string, error) {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
workspacesResponseStatus, err := sm.WorkspacePollSyncStatus()
if err != nil {
return nil, err
}
Comment on lines +161 to +164
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this login be inside workspaceService.GetWorkspacePipelines? Let's try to keep the manager clean.

return workspaceService.GetWorkspacePipelines(workspacesResponseStatus)
}

func (sm *PipelinesServicesManager) WorkspaceTriggerPipelines(branch, pipName string, isMultiBranch bool) error {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
return sm.TriggerPipelineRun(branch, pipName, isMultiBranch)
}

func (sm *PipelinesServicesManager) GetWorkspaceRunIDs(pipelineNames []string) ([]services.PipelinesRunID, error) {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
return workspaceService.WorkspaceRunIDs(pipelineNames)
}

func (sm *PipelinesServicesManager) GetWorkspaceRunStatus(pipeRunID int) ([]byte, error) {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
return workspaceService.WorkspaceRunStatus(pipeRunID)
}

func (sm *PipelinesServicesManager) GetWorkspaceStepStatus(pipeRunID int) ([]byte, error) {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
return workspaceService.WorkspaceStepStatus(pipeRunID)
}

func (sm *PipelinesServicesManager) ValidateWorkspace(data []byte) error {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
return workspaceService.ValidateWorkspace(data)
}

func (sm *PipelinesServicesManager) GetStepConsoles(stepID string) (map[string][]services.Console, error) {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
return workspaceService.GetStepLogsUsingStepID(stepID)
}

func (sm *PipelinesServicesManager) GetStepletConsoles(stepID string) (map[string][]services.Console, error) {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
return workspaceService.GetStepletLogsUsingStepID(stepID)
}

func (sm *PipelinesServicesManager) GetWorkspace() ([]services.WorkspacesResponse, error) {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
return workspaceService.GetWorkspace()
}

func (sm *PipelinesServicesManager) DeleteWorkspace(projectKey string) error {
workspaceService := services.NewWorkspaceService(sm.client)
workspaceService.ServiceDetails = sm.config.GetServiceDetails()
return workspaceService.DeleteWorkspace(projectKey)
}
33 changes: 22 additions & 11 deletions pipelines/services/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"net/http"
"strconv"
"strings"
"time"
)

type RunService struct {
Expand Down Expand Up @@ -99,19 +100,29 @@ func (rs *RunService) TriggerPipelineRun(branch, pipeline string, isMultiBranch
return err
}

// Prepare Request
resp, body, err := rs.client.SendPost(uri, buf.Bytes(), &httpDetails)
if err != nil {
return err
pollingAction := func() (shouldStop bool, responseBody []byte, err error) {
// Prepare Request
resp, body, err := rs.client.SendPost(uri, buf.Bytes(), &httpDetails)
if err != nil {
return true, body, err
}

// Response Analysis
if err := errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil {
return false, body, err
}
log.Info(fmt.Sprintf("Triggered successfully\n%s %s \n%14s %s", "PipelineName:", pipeline, "Branch:", branch))
return true, body, err
}

// Response Analysis
if err := errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil {
return err
pollingExecutor := &httputils.PollingExecutor{
Timeout: 2 * time.Minute,
PollingInterval: 5 * time.Second,
PollingAction: pollingAction,
MsgPrefix: "Get pipeline workspace sync status...",
}
log.Info(fmt.Sprintf("Triggered successfully\n%s %s \n%14s %s", "PipelineName :", pipeline, "Branch :", branch))

return nil
// Polling execution
_, err = pollingExecutor.Execute()
return err
}

func (rs *RunService) CancelRun(runID int) error {
Expand Down
67 changes: 67 additions & 0 deletions pipelines/services/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,3 +87,70 @@ type PipelineResources struct {
CreatedAt time.Time `json:"createdAt,omitempty"`
UpdatedAt time.Time `json:"updatedAt,omitempty"`
}

type WorkspacesResponse struct {
ValuesYmlPropertyBag interface{} `json:"valuesYmlPropertyBag,omitempty"`
PipelinesYmlPropertyBag PipelinesYmlPropertyBag `json:"pipelinesYmlPropertyBag,omitempty"`
ID int `json:"id,omitempty"`
Name string `json:"name,omitempty"`
ProjectID int `json:"projectId,omitempty"`
IsSyncing *bool `json:"isSyncing,omitempty"`
LastSyncStatusCode int `json:"lastSyncStatusCode,omitempty"`
LastSyncStartedAt time.Time `json:"lastSyncStartedAt,omitempty"`
LastSyncEndedAt interface{} `json:"lastSyncEndedAt,omitempty"`
LastSyncLogs string `json:"lastSyncLogs,omitempty"`
CreatedBy int `json:"createdBy,omitempty"`
}

type PipelinesYmlPropertyBag struct {
Resources []Resources `json:"resources,omitempty"`
Pipelines []Pipelines `json:"pipelines,omitempty"`
}

type Resources struct {
Configuration Configuration `json:"configuration,omitempty"`
Name string `json:"name,omitempty"`
Type string `json:"type,omitempty"`
}

type Configuration struct {
Branches Branches `json:"branches,omitempty"`
GitProvider string `json:"gitProvider,omitempty"`
Path string `json:"path,omitempty"`
}

type Branches struct {
Include string `json:"include,omitempty"`
}

type PipelinesRunID struct {
LatestRunID int `json:"latestRunId,omitempty"`
Name string `json:"name,omitempty"`
SyntaxVersion string `json:"syntaxVersion,omitempty"`
}

type Console struct {
ConsoleID string `json:"consoleId,omitempty"`
IsSuccess *bool `json:"isSuccess,omitempty"`
IsShown interface{} `json:"isShown,omitempty"`
ParentConsoleID string `json:"parentConsoleId,omitempty"`
StepletID int `json:"stepletId,omitempty"`
PipelineID int `json:"pipelineId,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
TimestampEndedAt interface{} `json:"timestampEndedAt,omitempty"`
Type string `json:"type,omitempty"`
Message string `json:"message,omitempty"`
CreatedAt time.Time `json:"createdAt,omitempty"`
UpdatedAt time.Time `json:"updatedAt,omitempty"`
}

// Validation Types
type ValidationResponse struct {
IsValid *bool `json:"isValid,omitempty"`
Errors []ValidationErrors `json:"errors,omitempty"`
}

type ValidationErrors struct {
Text string `json:"text,omitempty"`
LineNumber int `json:"lineNumber,omitempty"`
}
115 changes: 115 additions & 0 deletions pipelines/services/validate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package services

import (
"encoding/json"
"errors"

Check failure on line 5 in pipelines/services/validate.go

View workflow job for this annotation

GitHub Actions / Go-Sec

"errors" imported and not used

Check failure on line 5 in pipelines/services/validate.go

View workflow job for this annotation

GitHub Actions / Static-Check

"errors" imported and not used

Check failure on line 5 in pipelines/services/validate.go

View workflow job for this annotation

GitHub Actions / Static-Check

"errors" imported and not used
"github.com/gookit/color"
"github.com/jfrog/jfrog-client-go/auth"
"github.com/jfrog/jfrog-client-go/http/jfroghttpclient"
"github.com/jfrog/jfrog-client-go/utils/errorutils"
"github.com/jfrog/jfrog-client-go/utils/io/httputils"
"github.com/jfrog/jfrog-client-go/utils/log"
"net/http"
"net/url"
"strconv"
)

const (
validatePipResourcePath = "api/v1/validateYaml"
)

type ValidateService struct {
client *jfroghttpclient.JfrogHttpClient
auth.ServiceDetails
}

func NewValidateService(client *jfroghttpclient.JfrogHttpClient) *ValidateService {
return &ValidateService{client: client}
}

func (vs *ValidateService) getHttpDetails() httputils.HttpClientDetails {
httpDetails := vs.CreateHttpClientDetails()
return httpDetails
}

func (vs *ValidateService) ValidatePipeline(data []byte) error {
var err error
httpDetails := vs.getHttpDetails()

// Query params
m := make(map[string]string, 0)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is "m"?


// URL Construction
uri := vs.constructValidateAPIURL(m, validatePipResourcePath)

// Headers
headers := make(map[string]string)
utils.SetContentType("application/json", &headers)

Check failure on line 47 in pipelines/services/validate.go

View workflow job for this annotation

GitHub Actions / Go-Sec

undefined: utils

Check failure on line 47 in pipelines/services/validate.go

View workflow job for this annotation

GitHub Actions / Static-Check

undefined: utils

Check failure on line 47 in pipelines/services/validate.go

View workflow job for this annotation

GitHub Actions / Static-Check

undefined: utils
httpDetails.Headers = headers

// Send post request
resp, body, err := vs.client.SendPost(uri, data, &httpDetails)
if err != nil {
return err
}
if err := errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil {
return err
}

// Process response
if nil != resp && resp.StatusCode == http.StatusOK {
log.Info("Processing resources yaml response ")
log.Info(string(body))
err = processValidatePipResourceResponse(body)
if err != nil {
log.Error("Failed to process resources validation response")
}
}
Comment on lines +60 to +67
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You already check the status code in CheckResponseStatusWithBody. Please also consider nil before "CheckResponseStatusWithBody".

return nil
}

// processValidatePipResourceResponse process validate pipeline resource response
func processValidatePipResourceResponse(resp []byte) error {
validationResponse := make(map[string]ValidationResponse)
err := json.Unmarshal(resp, &validationResponse)
if err != nil {
return errorutils.CheckError(err)
}
if len(validationResponse) == 0 {
return errorutils.CheckErrorf("pipelines not found")
}
for k, v := range validationResponse {
if v.IsValid != nil && *v.IsValid {
log.Info("Validation of pipeline resources completed successfully")
msg := color.Green.Sprintf("Validation completed")
log.Info(msg)
} else {
fileName := color.Red.Sprintf("%s", k)
log.Error(fileName)
validationErrors := v.Errors
for _, validationError := range validationErrors {
lineNum := validationError.LineNumber
errorMessage := color.Red.Sprintf("%s", validationError.Text+":"+strconv.Itoa(lineNum))
log.Error(errorMessage)
}
}
}
return nil
}

// constructPipelinesURL creates URL with all required details to make api call
// like headers, queryParams, apiPath
func (vs *ValidateService) constructValidateAPIURL(qParams map[string]string, apiPath string) string {
uri, err := url.Parse(vs.ServiceDetails.GetUrl() + apiPath)
if err != nil {
log.Error("Failed to parse pipelines fetch run status url")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please return an error here

}

queryString := uri.Query()
for k, v := range qParams {
queryString.Set(k, v)
}
uri.RawQuery = queryString.Encode()

return uri.String()
}
Loading
Loading