diff --git a/Makefile b/Makefile index b0261d3..6047b2d 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,8 @@ mocks: bootstrap mockgen -source ./api/v2/handler.go -destination ./api/v2/mock/handler_mock.go -package mock mockgen -source ./api/v1/jobs/job_handler.go -destination ./api/v1/jobs/mock/job_mock.go -package mock mockgen -source ./api/v1/batches/batch_handler.go -destination ./api/v1/batches/mock/batch_mock.go -package mock - mockgen -source ./models/notifications/notifier.go -destination ./models/notifications/notifier_mock.go -package notifications + mockgen -source ./pkg/notifications/notifier.go -destination ./pkg/notifications/notifier_mock.go -package notifications + mockgen -source ./pkg/batch/history.go -destination ./pkg/batch/history_mock.go -package batch .PHONY: generate generate: swagger mocks diff --git a/api/v1/batches/batch_handler.go b/api/v1/batches/batch_handler.go index e5b5cac..d89ba9f 100644 --- a/api/v1/batches/batch_handler.go +++ b/api/v1/batches/batch_handler.go @@ -3,13 +3,16 @@ package batchesv1 import ( "context" + "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/slice" apiv1 "github.com/equinor/radix-job-scheduler/api/v1" apiv2 "github.com/equinor/radix-job-scheduler/api/v2" + "github.com/equinor/radix-job-scheduler/internal" "github.com/equinor/radix-job-scheduler/models" "github.com/equinor/radix-job-scheduler/models/common" modelsv1 "github.com/equinor/radix-job-scheduler/models/v1" modelsv2 "github.com/equinor/radix-job-scheduler/models/v2" + "github.com/equinor/radix-job-scheduler/pkg/batch" "github.com/equinor/radix-job-scheduler/utils/radix/jobs" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" @@ -32,8 +35,6 @@ type BatchHandler interface { CreateBatch(ctx context.Context, batchScheduleDescription *common.BatchScheduleDescription) (*modelsv1.BatchStatus, error) // CopyBatch creates a copy of an existing batch with deploymentName as value for radixDeploymentJobRef.name CopyBatch(ctx context.Context, batchName string, deploymentName string) (*modelsv1.BatchStatus, error) - // MaintainHistoryLimit Delete outdated batches - MaintainHistoryLimit(ctx context.Context) error // DeleteBatch Delete a batch DeleteBatch(ctx context.Context, batchName string) error // StopBatch Stop a batch @@ -130,11 +131,11 @@ func (handler *batchHandler) CopyBatch(ctx context.Context, batchName string, de func (handler *batchHandler) DeleteBatch(ctx context.Context, batchName string) error { logger := log.Ctx(ctx) logger.Debug().Msgf("delete batch %s for namespace: %s", batchName, handler.common.Env.RadixDeploymentNamespace) - err := handler.common.HandlerApiV2.DeleteRadixBatch(ctx, batchName) + err := batch.DeleteRadixBatchByName(ctx, handler.common.Kube.RadixClient(), handler.common.Env.RadixDeploymentNamespace, batchName) if err != nil { return err } - return handler.common.HandlerApiV2.GarbageCollectPayloadSecrets(ctx) + return internal.GarbageCollectPayloadSecrets(ctx, handler.common.Kube, handler.common.Env.RadixDeploymentNamespace, handler.common.Env.RadixComponentName) } // StopBatch Stop a batch @@ -151,11 +152,6 @@ func (handler *batchHandler) StopBatchJob(ctx context.Context, batchName string, return apiv1.StopJob(ctx, handler.common.HandlerApiV2, jobName) } -// MaintainHistoryLimit Delete outdated batches -func (handler *batchHandler) MaintainHistoryLimit(ctx context.Context) error { - return handler.common.HandlerApiV2.MaintainHistoryLimit(ctx) -} - func setBatchJobEventMessages(radixBatchStatus *modelsv1.BatchStatus, batchJobPodsMap map[string]corev1.Pod, eventMessageForPods map[string]string) { for i := 0; i < len(radixBatchStatus.JobStatuses); i++ { apiv1.SetBatchJobEventMessageToBatchJobStatus(&radixBatchStatus.JobStatuses[i], batchJobPodsMap, eventMessageForPods) @@ -166,6 +162,7 @@ func (handler *batchHandler) getBatchStatusFromRadixBatch(radixBatch *modelsv2.R return &modelsv1.BatchStatus{ JobStatus: modelsv1.JobStatus{ Name: radixBatch.Name, + BatchId: getBatchId(radixBatch), Created: radixBatch.CreationTime, Started: radixBatch.Started, Ended: radixBatch.Ended, @@ -192,3 +189,7 @@ func (handler *batchHandler) getBatchStatus(radixBatch *modelsv2.RadixBatch) rad }) return jobs.GetStatusFromStatusRules(jobStatusPhases, handler.common.RadixDeployJobComponent, radixBatch.Status) } + +func getBatchId(radixBatch *modelsv2.RadixBatch) string { + return utils.TernaryString(radixBatch.BatchType == string(kube.RadixBatchTypeJob), "", radixBatch.BatchId) +} diff --git a/api/v1/batches/mock/batch_mock.go b/api/v1/batches/mock/batch_mock.go index bd00c7c..139207d 100644 --- a/api/v1/batches/mock/batch_mock.go +++ b/api/v1/batches/mock/batch_mock.go @@ -125,20 +125,6 @@ func (mr *MockBatchHandlerMockRecorder) GetBatches(ctx interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBatches", reflect.TypeOf((*MockBatchHandler)(nil).GetBatches), ctx) } -// MaintainHistoryLimit mocks base method. -func (m *MockBatchHandler) MaintainHistoryLimit(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MaintainHistoryLimit", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// MaintainHistoryLimit indicates an expected call of MaintainHistoryLimit. -func (mr *MockBatchHandlerMockRecorder) MaintainHistoryLimit(ctx interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaintainHistoryLimit", reflect.TypeOf((*MockBatchHandler)(nil).MaintainHistoryLimit), ctx) -} - // StopBatch mocks base method. func (m *MockBatchHandler) StopBatch(ctx context.Context, batchName string) error { m.ctrl.T.Helper() diff --git a/api/v1/controllers/batches/controller.go b/api/v1/controllers/batches/controller.go index cec5ffd..90d7358 100644 --- a/api/v1/controllers/batches/controller.go +++ b/api/v1/controllers/batches/controller.go @@ -126,10 +126,6 @@ func (controller *batchController) CreateBatch(c *gin.Context) { return } logger.Info().Msgf("Batch %s has been created", batchState.Name) - err = controller.handler.MaintainHistoryLimit(c.Request.Context()) - if err != nil { - logger.Warn().Err(err).Msg("failed to maintain batch history") - } c.JSON(http.StatusOK, batchState) } diff --git a/api/v1/controllers/batches/controller_test.go b/api/v1/controllers/batches/controller_test.go index f5644e0..df35c9b 100644 --- a/api/v1/controllers/batches/controller_test.go +++ b/api/v1/controllers/batches/controller_test.go @@ -218,11 +218,6 @@ func TestCreateBatch(t *testing.T) { CreateBatch(test.RequestContextMatcher{}, &batchScheduleDescription). Return(&createdBatch, nil). Times(1) - batchHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Return(nil). - Times(1) controllerTestUtils := setupTest(batchHandler) responseChannel := controllerTestUtils.ExecuteRequestWithBody(ctx, http.MethodPost, "/api/v1/batches", nil) response := <-responseChannel @@ -283,58 +278,6 @@ func TestCreateBatch(t *testing.T) { CreateBatch(test.RequestContextMatcher{}, &batchScheduleDescription). Return(&createdBatch, nil). Times(1) - batchHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Return(nil). - Times(1) - controllerTestUtils := setupTest(batchHandler) - responseChannel := controllerTestUtils.ExecuteRequestWithBody(ctx, http.MethodPost, "/api/v1/batches", batchScheduleDescription) - response := <-responseChannel - assert.NotNil(t, response) - - if response != nil { - assert.Equal(t, http.StatusOK, response.StatusCode) - var returnedBatch modelsV1.BatchStatus - err := test.GetResponseBody(response, &returnedBatch) - require.NoError(t, err) - assert.Equal(t, createdBatch.Name, returnedBatch.Name) - assert.Equal(t, createdBatch.Started, returnedBatch.Started) - assert.Equal(t, createdBatch.Ended, returnedBatch.Ended) - assert.Equal(t, createdBatch.Status, returnedBatch.Status) - } - }) - - t.Run("valid payload body - error from MaintainHistoryLimit should not fail request", func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - defer ctrl.Finish() - batchScheduleDescription := models.BatchScheduleDescription{ - JobScheduleDescriptions: []models.JobScheduleDescription{ - {Payload: "a_payload"}, - }, - } - createdBatch := modelsV1.BatchStatus{ - JobStatus: modelsV1.JobStatus{ - Name: "newbatch", - Started: commonUtils.FormatTimestamp(time.Now()), - Ended: commonUtils.FormatTimestamp(time.Now().Add(1 * time.Minute)), - Status: "batchstatus", - }, - BatchType: string(kube.RadixBatchTypeBatch), - } - batchHandler := mock.NewMockBatchHandler(ctrl) - ctx := context.Background() - batchHandler. - EXPECT(). - CreateBatch(test.RequestContextMatcher{}, &batchScheduleDescription). - Return(&createdBatch, nil). - Times(1) - batchHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Return(errors.New("an error")). - Times(1) controllerTestUtils := setupTest(batchHandler) responseChannel := controllerTestUtils.ExecuteRequestWithBody(ctx, http.MethodPost, "/api/v1/batches", batchScheduleDescription) response := <-responseChannel @@ -363,10 +306,6 @@ func TestCreateBatch(t *testing.T) { EXPECT(). CreateBatch(test.RequestContextMatcher{}, gomock.Any()). Times(0) - batchHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Times(0) controllerTestUtils := setupTest(batchHandler) responseChannel := controllerTestUtils.ExecuteRequestWithBody(ctx, http.MethodPost, "/api/v1/batches", struct{ JobScheduleDescriptions interface{} }{JobScheduleDescriptions: struct{}{}}) response := <-responseChannel @@ -397,10 +336,6 @@ func TestCreateBatch(t *testing.T) { CreateBatch(test.RequestContextMatcher{}, &batchScheduleDescription). Return(nil, apiErrors.NewNotFound(anyKind, anyName)). Times(1) - batchHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Times(0) controllerTestUtils := setupTest(batchHandler) responseChannel := controllerTestUtils.ExecuteRequest(ctx, http.MethodPost, "/api/v1/batches") response := <-responseChannel @@ -430,10 +365,6 @@ func TestCreateBatch(t *testing.T) { CreateBatch(test.RequestContextMatcher{}, &batchScheduleDescription). Return(nil, errors.New("any error")). Times(1) - batchHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Times(0) controllerTestUtils := setupTest(batchHandler) responseChannel := controllerTestUtils.ExecuteRequest(ctx, http.MethodPost, "/api/v1/batches") response := <-responseChannel @@ -639,11 +570,6 @@ func TestStopBatchJob(t *testing.T) { StopBatchJob(test.RequestContextMatcher{}, batchName, jobName). Return(nil). Times(1) - batchHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Return(nil). - AnyTimes() controllerTestUtils := setupTest(batchHandler) responseChannel := controllerTestUtils.ExecuteRequest(ctx, http.MethodPost, fmt.Sprintf("/api/v1/batches/%s/jobs/%s/stop", batchName, jobName)) response := <-responseChannel diff --git a/api/v1/controllers/jobs/controller.go b/api/v1/controllers/jobs/controller.go index 87e439d..d07d626 100644 --- a/api/v1/controllers/jobs/controller.go +++ b/api/v1/controllers/jobs/controller.go @@ -115,10 +115,6 @@ func (controller *jobController) CreateJob(c *gin.Context) { } logger.Info().Msgf("Job %s has been created", jobState.Name) - err = controller.handler.MaintainHistoryLimit(c.Request.Context()) - if err != nil { - logger.Warn().Err(err).Msg("failed to maintain job history") - } c.JSON(http.StatusOK, jobState) } diff --git a/api/v1/controllers/jobs/controller_test.go b/api/v1/controllers/jobs/controller_test.go index 8ee2db7..ac84fbd 100644 --- a/api/v1/controllers/jobs/controller_test.go +++ b/api/v1/controllers/jobs/controller_test.go @@ -210,11 +210,6 @@ func TestCreateJob(t *testing.T) { CreateJob(test.RequestContextMatcher{}, &jobScheduleDescription). Return(&createdJob, nil). Times(1) - jobHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Return(nil). - Times(1) controllerTestUtils := setupTest(jobHandler) responseChannel := controllerTestUtils.ExecuteRequestWithBody(ctx, http.MethodPost, "/api/v1/jobs", nil) response := <-responseChannel @@ -269,54 +264,6 @@ func TestCreateJob(t *testing.T) { CreateJob(test.RequestContextMatcher{}, &jobScheduleDescription). Return(&createdJob, nil). Times(1) - jobHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Return(nil). - Times(1) - controllerTestUtils := setupTest(jobHandler) - responseChannel := controllerTestUtils.ExecuteRequestWithBody(ctx, http.MethodPost, "/api/v1/jobs", jobScheduleDescription) - response := <-responseChannel - assert.NotNil(t, response) - - if response != nil { - assert.Equal(t, http.StatusOK, response.StatusCode) - var returnedJob modelsV1.JobStatus - err := test.GetResponseBody(response, &returnedJob) - require.NoError(t, err) - assert.Equal(t, createdJob.Name, returnedJob.Name) - assert.Equal(t, "", returnedJob.BatchName) - assert.Equal(t, createdJob.Started, returnedJob.Started) - assert.Equal(t, createdJob.Ended, returnedJob.Ended) - assert.Equal(t, createdJob.Status, returnedJob.Status) - } - }) - - t.Run("valid payload body - error from MaintainHistoryLimit should not fail request", func(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - defer ctrl.Finish() - jobScheduleDescription := models.JobScheduleDescription{ - Payload: "a_payload", - } - createdJob := modelsV1.JobStatus{ - Name: "newjob", - Started: utils.FormatTimestamp(time.Now()), - Ended: utils.FormatTimestamp(time.Now().Add(1 * time.Minute)), - Status: "jobstatus", - } - jobHandler := mock.NewMockJobHandler(ctrl) - ctx := context.Background() - jobHandler. - EXPECT(). - CreateJob(test.RequestContextMatcher{}, &jobScheduleDescription). - Return(&createdJob, nil). - Times(1) - jobHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Return(errors.New("an error")). - Times(1) controllerTestUtils := setupTest(jobHandler) responseChannel := controllerTestUtils.ExecuteRequestWithBody(ctx, http.MethodPost, "/api/v1/jobs", jobScheduleDescription) response := <-responseChannel @@ -346,10 +293,6 @@ func TestCreateJob(t *testing.T) { EXPECT(). CreateJob(test.RequestContextMatcher{}, gomock.Any()). Times(0) - jobHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Times(0) controllerTestUtils := setupTest(jobHandler) responseChannel := controllerTestUtils.ExecuteRequestWithBody(ctx, http.MethodPost, "/api/v1/jobs", struct{ Payload interface{} }{Payload: struct{}{}}) response := <-responseChannel @@ -380,10 +323,6 @@ func TestCreateJob(t *testing.T) { CreateJob(test.RequestContextMatcher{}, &jobScheduleDescription). Return(nil, apiErrors.NewNotFound(anyKind, anyName)). Times(1) - jobHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Times(0) controllerTestUtils := setupTest(jobHandler) responseChannel := controllerTestUtils.ExecuteRequest(ctx, http.MethodPost, "/api/v1/jobs") response := <-responseChannel @@ -413,10 +352,6 @@ func TestCreateJob(t *testing.T) { CreateJob(test.RequestContextMatcher{}, &jobScheduleDescription). Return(nil, errors.New("any error")). Times(1) - jobHandler. - EXPECT(). - MaintainHistoryLimit(test.RequestContextMatcher{}). - Times(0) controllerTestUtils := setupTest(jobHandler) responseChannel := controllerTestUtils.ExecuteRequest(ctx, http.MethodPost, "/api/v1/jobs") response := <-responseChannel diff --git a/api/v1/events.go b/api/v1/events.go index 80854bf..0521961 100644 --- a/api/v1/events.go +++ b/api/v1/events.go @@ -8,11 +8,15 @@ import ( "github.com/equinor/radix-common/utils/slice" modelsv1 "github.com/equinor/radix-job-scheduler/models/v1" - defaultsv1 "github.com/equinor/radix-job-scheduler/models/v1/defaults" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + // k8sJobNameLabel A label that k8s automatically adds to a Pod created by a Job + k8sJobNameLabel = "job-name" +) + // GetLastEventMessageForPods returns the last event message for pods func (handler *Handler) GetLastEventMessageForPods(ctx context.Context, pods []corev1.Pod) (map[string]string, error) { podNamesMap := slice.Reduce(pods, make(map[string]struct{}), func(acc map[string]struct{}, pod corev1.Pod) map[string]struct{} { @@ -59,7 +63,7 @@ func (handler *Handler) GetRadixBatchJobMessagesAndPodMaps(ctx context.Context, return nil, nil, err } batchJobPodsMap := slice.Reduce(radixBatchesPods, make(map[string]corev1.Pod), func(acc map[string]corev1.Pod, pod corev1.Pod) map[string]corev1.Pod { - if batchJobName, ok := pod.GetLabels()[defaultsv1.K8sJobNameLabel]; ok { + if batchJobName, ok := pod.GetLabels()[k8sJobNameLabel]; ok { acc[batchJobName] = pod } return acc diff --git a/api/v1/jobs.go b/api/v1/jobs.go index b15d36e..9be9f71 100644 --- a/api/v1/jobs.go +++ b/api/v1/jobs.go @@ -35,7 +35,7 @@ func GetJobStatusFromRadixBatchJobsStatus(batchName string, jobStatus modelsv2.R func GetJobStatusFromRadixBatchJobsStatuses(radixBatches ...modelsv2.RadixBatch) []modelsv1.JobStatus { jobStatuses := make([]modelsv1.JobStatus, 0, len(radixBatches)) for _, radixBatch := range radixBatches { - jobStatusBatchName := getJobStatusBatchName(&radixBatch) + jobStatusBatchName := getBatchName(&radixBatch) for _, jobStatus := range radixBatch.JobStatuses { jobStatuses = append(jobStatuses, GetJobStatusFromRadixBatchJobsStatus(jobStatusBatchName, jobStatus)) } @@ -62,7 +62,7 @@ func GetBatchJob(ctx context.Context, handlerApiV2 apiv2.Handler, batchName, job if err != nil { return nil, err } - jobStatusBatchName := getJobStatusBatchName(radixBatch) + jobStatusBatchName := getBatchName(radixBatch) for _, jobStatus := range radixBatch.JobStatuses { if !strings.EqualFold(jobStatus.Name, jobName) { continue @@ -94,6 +94,6 @@ func GetPodStatus(podStatuses []modelsv2.RadixBatchJobPodStatus) []modelsv1.PodS }) } -func getJobStatusBatchName(radixBatch *modelsv2.RadixBatch) string { +func getBatchName(radixBatch *modelsv2.RadixBatch) string { return utils.TernaryString(radixBatch.BatchType == string(kube.RadixBatchTypeJob), "", radixBatch.Name) } diff --git a/api/v1/jobs/job_handler.go b/api/v1/jobs/job_handler.go index 9dca0a3..6e043a0 100644 --- a/api/v1/jobs/job_handler.go +++ b/api/v1/jobs/job_handler.go @@ -12,6 +12,7 @@ import ( "github.com/equinor/radix-job-scheduler/models/common" modelsv1 "github.com/equinor/radix-job-scheduler/models/v1" modelsv2 "github.com/equinor/radix-job-scheduler/models/v2" + "github.com/equinor/radix-job-scheduler/pkg/batch" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/rs/zerolog/log" @@ -31,8 +32,6 @@ type JobHandler interface { CreateJob(ctx context.Context, jobScheduleDescription *common.JobScheduleDescription) (*modelsv1.JobStatus, error) // CopyJob creates a copy of an existing job with deploymentName as value for radixDeploymentJobRef.name CopyJob(ctx context.Context, jobName string, deploymentName string) (*modelsv1.JobStatus, error) - // MaintainHistoryLimit Delete outdated jobs - MaintainHistoryLimit(ctx context.Context) error // DeleteJob Delete a job DeleteJob(ctx context.Context, jobName string) error // StopJob Stop a job @@ -148,14 +147,14 @@ func (handler *jobHandler) DeleteJob(ctx context.Context, jobName string) error if !jobExistInBatch(radixBatchStatus, jobName) { return apiErrors.NewNotFound("batch job", jobName) } - err = handler.common.HandlerApiV2.DeleteRadixBatch(ctx, batchName) + err = batch.DeleteRadixBatchByName(ctx, handler.common.Kube.RadixClient(), handler.common.Env.RadixDeploymentNamespace, batchName) if err != nil { if errors.IsNotFound(err) { return apiErrors.NewNotFound("batch job", jobName) } return apiErrors.NewFromError(err) } - return handler.common.HandlerApiV2.GarbageCollectPayloadSecrets(ctx) + return internal.GarbageCollectPayloadSecrets(ctx, handler.common.Kube, handler.common.Env.RadixDeploymentNamespace, handler.common.Env.RadixComponentName) } func jobExistInBatch(radixBatch *modelsv2.RadixBatch, jobName string) bool { @@ -174,11 +173,6 @@ func (handler *jobHandler) StopJob(ctx context.Context, jobName string) error { return apiv1.StopJob(ctx, handler.common.HandlerApiV2, jobName) } -// MaintainHistoryLimit Delete outdated jobs -func (handler *jobHandler) MaintainHistoryLimit(ctx context.Context) error { - return handler.common.HandlerApiV2.MaintainHistoryLimit(ctx) -} - func getSingleJobStatusFromRadixBatchJob(radixBatch *modelsv2.RadixBatch) (*modelsv1.JobStatus, error) { if len(radixBatch.JobStatuses) != 1 { return nil, fmt.Errorf("batch should have only one job") diff --git a/api/v1/jobs/mock/job_mock.go b/api/v1/jobs/mock/job_mock.go index ef3e2bd..deb80e8 100644 --- a/api/v1/jobs/mock/job_mock.go +++ b/api/v1/jobs/mock/job_mock.go @@ -110,20 +110,6 @@ func (mr *MockJobHandlerMockRecorder) GetJobs(ctx interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetJobs", reflect.TypeOf((*MockJobHandler)(nil).GetJobs), ctx) } -// MaintainHistoryLimit mocks base method. -func (m *MockJobHandler) MaintainHistoryLimit(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MaintainHistoryLimit", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// MaintainHistoryLimit indicates an expected call of MaintainHistoryLimit. -func (mr *MockJobHandlerMockRecorder) MaintainHistoryLimit(ctx interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaintainHistoryLimit", reflect.TypeOf((*MockJobHandler)(nil).MaintainHistoryLimit), ctx) -} - // StopJob mocks base method. func (m *MockJobHandler) StopJob(ctx context.Context, jobName string) error { m.ctrl.T.Helper() diff --git a/api/v2/handler.go b/api/v2/handler.go index 56b39e4..1f5326a 100644 --- a/api/v2/handler.go +++ b/api/v2/handler.go @@ -5,14 +5,11 @@ import ( "encoding/base64" "errors" "fmt" - "sort" "strings" - "time" "dario.cat/mergo" mergoutils "github.com/equinor/radix-common/utils/mergo" "github.com/equinor/radix-common/utils/pointers" - "github.com/equinor/radix-common/utils/slice" apiErrors "github.com/equinor/radix-job-scheduler/api/errors" "github.com/equinor/radix-job-scheduler/internal" "github.com/equinor/radix-job-scheduler/models" @@ -45,6 +42,7 @@ type handler struct { kubeUtil *kube.Kube env *models.Env radixDeployJobComponent *radixv1.RadixDeployJobComponent + jobHistory batch.History } type Handler interface { @@ -62,12 +60,6 @@ type Handler interface { CreateRadixBatchSingleJob(ctx context.Context, jobScheduleDescription *common.JobScheduleDescription) (*modelsv2.RadixBatch, error) // CopyRadixBatchJob Copy a batch job parameter CopyRadixBatchJob(ctx context.Context, jobName, deploymentName string) (*modelsv2.RadixBatch, error) - // MaintainHistoryLimit Delete outdated batches - MaintainHistoryLimit(ctx context.Context) error - // GarbageCollectPayloadSecrets Delete orphaned payload secrets - GarbageCollectPayloadSecrets(ctx context.Context) error - // DeleteRadixBatch Delete a batch - DeleteRadixBatch(ctx context.Context, batchName string) error // DeleteRadixBatchJob Delete a single job DeleteRadixBatchJob(ctx context.Context, jobName string) error // StopRadixBatch Stop a batch @@ -80,20 +72,13 @@ type Handler interface { RestartRadixBatchJob(ctx context.Context, batchName, jobName string) error } -// CompletedRadixBatches Completed RadixBatch lists -type CompletedRadixBatches struct { - SucceededRadixBatches []*modelsv2.RadixBatch - NotSucceededRadixBatches []*modelsv2.RadixBatch - SucceededSingleJobs []*modelsv2.RadixBatch - NotSucceededSingleJobs []*modelsv2.RadixBatch -} - // New Constructor of the batch handler -func New(kube *kube.Kube, env *models.Env, radixDeployJobComponent *radixv1.RadixDeployJobComponent) Handler { +func New(kubeUtil *kube.Kube, env *models.Env, radixDeployJobComponent *radixv1.RadixDeployJobComponent) Handler { return &handler{ - kubeUtil: kube, + kubeUtil: kubeUtil, env: env, radixDeployJobComponent: radixDeployJobComponent, + jobHistory: batch.NewHistory(kubeUtil, env, radixDeployJobComponent), } } @@ -221,15 +206,6 @@ func (h *handler) CopyRadixBatchJob(ctx context.Context, sourceJobName, deployme return batch.CopyRadixBatchOrJob(ctx, h.kubeUtil.RadixClient(), sourceRadixBatch, jobName, h.radixDeployJobComponent, deploymentName) } -// DeleteRadixBatch Delete a batch -func (h *handler) DeleteRadixBatch(ctx context.Context, batchName string) error { - radixBatch, err := internal.GetRadixBatch(ctx, h.kubeUtil.RadixClient(), h.env.RadixDeploymentNamespace, batchName) - if err != nil { - return apiErrors.NewFromError(err) - } - return batch.DeleteRadixBatch(ctx, h.kubeUtil.RadixClient(), radixBatch) -} - // DeleteRadixBatchJob Delete a batch job func (h *handler) DeleteRadixBatchJob(ctx context.Context, jobName string) error { batchName, _, ok := internal.ParseBatchAndJobNameFromScheduledJobName(jobName) @@ -282,131 +258,6 @@ func (h *handler) RestartRadixBatchJob(ctx context.Context, batchName, jobName s return batch.RestartRadixBatchJob(ctx, h.kubeUtil.RadixClient(), radixBatch, jobName) } -// MaintainHistoryLimit Delete outdated batches -func (h *handler) MaintainHistoryLimit(ctx context.Context) error { - logger := log.Ctx(ctx) - const minimumAge = 3600 // TODO add as default env-var and/or job-component property - completedBefore := time.Now().Add(-time.Second * minimumAge) - completedRadixBatches, err := h.getCompletedRadixBatchesSortedByCompletionTimeAsc(ctx, completedBefore) - if err != nil { - return err - } - - historyLimit := h.env.RadixJobSchedulersPerEnvironmentHistoryLimit - logger.Debug().Msg("maintain history limit for succeeded batches") - var errs []error - if err := h.maintainHistoryLimitForBatches(ctx, completedRadixBatches.SucceededRadixBatches, historyLimit); err != nil { - errs = append(errs, err) - } - logger.Debug().Msg("maintain history limit for not succeeded batches") - if err := h.maintainHistoryLimitForBatches(ctx, completedRadixBatches.NotSucceededRadixBatches, historyLimit); err != nil { - errs = append(errs, err) - } - logger.Debug().Msg("maintain history limit for succeeded single jobs") - if err := h.maintainHistoryLimitForBatches(ctx, completedRadixBatches.SucceededSingleJobs, historyLimit); err != nil { - errs = append(errs, err) - } - logger.Debug().Msg("maintain history limit for not succeeded single jobs") - if err := h.maintainHistoryLimitForBatches(ctx, completedRadixBatches.NotSucceededSingleJobs, historyLimit); err != nil { - errs = append(errs, err) - } - logger.Debug().Msg("delete orphaned payload secrets") - err = h.GarbageCollectPayloadSecrets(ctx) - if err != nil { - errs = append(errs, err) - } - return errors.Join(errs...) -} - -func (h *handler) getCompletedRadixBatchesSortedByCompletionTimeAsc(ctx context.Context, completedBefore time.Time) (*CompletedRadixBatches, error) { - radixBatches, err := internal.GetRadixBatches(ctx, h.env.RadixDeploymentNamespace, h.kubeUtil.RadixClient(), radixLabels.ForComponentName(h.env.RadixComponentName)) - if err != nil { - return nil, err - } - radixBatches = sortRJSchByCompletionTimeAsc(radixBatches) - return &CompletedRadixBatches{ - SucceededRadixBatches: h.getSucceededRadixBatches(radixBatches, completedBefore), - NotSucceededRadixBatches: h.getNotSucceededRadixBatches(radixBatches, completedBefore), - SucceededSingleJobs: h.getSucceededSingleJobs(radixBatches, completedBefore), - NotSucceededSingleJobs: h.getNotSucceededSingleJobs(radixBatches, completedBefore), - }, nil -} - -func (h *handler) getNotSucceededRadixBatches(radixBatches []*radixv1.RadixBatch, completedBefore time.Time) []*modelsv2.RadixBatch { - return convertToRadixBatchStatuses(slice.FindAll(radixBatches, func(radixBatch *radixv1.RadixBatch) bool { - return radixBatchHasType(radixBatch, kube.RadixBatchTypeBatch) && internal.IsRadixBatchNotSucceeded(radixBatch) && radixBatchIsCompletedBefore(completedBefore, radixBatch) - }), h.radixDeployJobComponent) -} - -func (h *handler) getSucceededRadixBatches(radixBatches []*radixv1.RadixBatch, completedBefore time.Time) []*modelsv2.RadixBatch { - radixBatches = slice.FindAll(radixBatches, func(radixBatch *radixv1.RadixBatch) bool { - return radixBatchHasType(radixBatch, kube.RadixBatchTypeBatch) && internal.IsRadixBatchSucceeded(radixBatch) && radixBatchIsCompletedBefore(completedBefore, radixBatch) - }) - return convertToRadixBatchStatuses(radixBatches, h.radixDeployJobComponent) -} - -func radixBatchIsCompletedBefore(completedBefore time.Time, radixBatch *radixv1.RadixBatch) bool { - return radixBatch.Status.Condition.CompletionTime != nil && (*radixBatch.Status.Condition.CompletionTime).Before(&metav1.Time{Time: completedBefore}) -} - -func (h *handler) getNotSucceededSingleJobs(radixBatches []*radixv1.RadixBatch, completedBefore time.Time) []*modelsv2.RadixBatch { - return convertToRadixBatchStatuses(slice.FindAll(radixBatches, func(radixBatch *radixv1.RadixBatch) bool { - return radixBatchHasType(radixBatch, kube.RadixBatchTypeJob) && internal.IsRadixBatchNotSucceeded(radixBatch) && radixBatchIsCompletedBefore(completedBefore, radixBatch) - }), h.radixDeployJobComponent) -} - -func (h *handler) getSucceededSingleJobs(radixBatches []*radixv1.RadixBatch, completedBefore time.Time) []*modelsv2.RadixBatch { - return convertToRadixBatchStatuses(slice.FindAll(radixBatches, func(radixBatch *radixv1.RadixBatch) bool { - return radixBatchHasType(radixBatch, kube.RadixBatchTypeJob) && internal.IsRadixBatchSucceeded(radixBatch) && radixBatchIsCompletedBefore(completedBefore, radixBatch) - }), h.radixDeployJobComponent) -} - -func radixBatchHasType(radixBatch *radixv1.RadixBatch, radixBatchType kube.RadixBatchType) bool { - return radixBatch.GetLabels()[kube.RadixBatchTypeLabel] == string(radixBatchType) -} - -func (h *handler) maintainHistoryLimitForBatches(ctx context.Context, radixBatchesSortedByCompletionTimeAsc []*modelsv2.RadixBatch, historyLimit int) error { - logger := log.Ctx(ctx) - numToDelete := len(radixBatchesSortedByCompletionTimeAsc) - historyLimit - if numToDelete <= 0 { - logger.Debug().Msgf("no history batches to delete: %d batches, %d history limit", len(radixBatchesSortedByCompletionTimeAsc), historyLimit) - return nil - } - logger.Debug().Msgf("history batches to delete: %v", numToDelete) - - for i := 0; i < numToDelete; i++ { - radixBatch := radixBatchesSortedByCompletionTimeAsc[i] - logger.Debug().Msgf("deleting batch %s", radixBatch.Name) - if err := h.DeleteRadixBatch(ctx, radixBatch.Name); err != nil { - return err - } - } - return nil -} - -func sortRJSchByCompletionTimeAsc(batches []*radixv1.RadixBatch) []*radixv1.RadixBatch { - sort.Slice(batches, func(i, j int) bool { - batch1 := (batches)[i] - batch2 := (batches)[j] - return isRJS1CompletedBeforeRJS2(batch1, batch2) - }) - return batches -} - -func isRJS1CompletedBeforeRJS2(batch1 *radixv1.RadixBatch, batch2 *radixv1.RadixBatch) bool { - rd1ActiveFrom := getCompletionTimeFrom(batch1) - rd2ActiveFrom := getCompletionTimeFrom(batch2) - - return rd1ActiveFrom.Before(rd2ActiveFrom) -} - -func getCompletionTimeFrom(radixBatch *radixv1.RadixBatch) *metav1.Time { - if radixBatch.Status.Condition.CompletionTime.IsZero() { - return pointers.Ptr(radixBatch.GetCreationTimestamp()) - } - return radixBatch.Status.Condition.CompletionTime -} - func (h *handler) createRadixBatch(ctx context.Context, namespace, appName, radixDeploymentName string, radixJobComponent radixv1.RadixDeployJobComponent, batchScheduleDescription common.BatchScheduleDescription, radixBatchType kube.RadixBatchType) (*radixv1.RadixBatch, error) { logger := log.Ctx(ctx) batchName := internal.GenerateBatchName(radixJobComponent.GetName()) @@ -427,6 +278,7 @@ func (h *handler) createRadixBatch(ctx context.Context, namespace, appName, radi ), }, Spec: radixv1.RadixBatchSpec{ + BatchId: batchScheduleDescription.BatchId, RadixDeploymentJobRef: radixv1.RadixDeploymentJobComponentSelector{ LocalObjectReference: radixv1.LocalObjectReference{Name: radixDeploymentName}, Job: radixJobComponentName, @@ -534,7 +386,7 @@ func (h *handler) createSecrets(ctx context.Context, namespace string, secrets [ logger := log.Ctx(ctx) logger.Debug().Msgf("Create %d secrets", len(secrets)) for _, secret := range secrets { - if secret.Data == nil || len(secret.Data) == 0 { + if len(secret.Data) == 0 { logger.Debug().Msgf("Do not create a secret %s - Data is empty, the secret is not used in any jobs", secret.GetName()) continue } @@ -582,49 +434,6 @@ func buildRadixBatchJob(jobScheduleDescription *common.JobScheduleDescription, d }, nil } -// GarbageCollectPayloadSecrets Delete orphaned payload secrets -func (h *handler) GarbageCollectPayloadSecrets(ctx context.Context) error { - logger := log.Ctx(ctx) - logger.Debug().Msgf("Garbage collecting payload secrets") - payloadSecretRefNames, _ := h.getJobComponentPayloadSecretRefNames(ctx) - payloadSecrets, err := h.kubeUtil.ListSecretsWithSelector(ctx, h.env.RadixDeploymentNamespace, radixLabels.GetRadixBatchDescendantsSelector(h.env.RadixComponentName).String()) - if err != nil { - return apiErrors.NewFromError(err) - } - logger.Debug().Msgf("%d payload secrets, %d secret reference unique names", len(payloadSecrets), len(payloadSecretRefNames)) - yesterday := time.Now().Add(time.Hour * -24) - for _, payloadSecret := range payloadSecrets { - if _, ok := payloadSecretRefNames[payloadSecret.GetName()]; !ok { - if payloadSecret.GetCreationTimestamp().After(yesterday) { - logger.Debug().Msgf("skipping deletion of an orphaned payload secret %s, created within 24 hours", payloadSecret.GetName()) - continue - } - err := h.DeleteSecret(ctx, payloadSecret) - if err != nil { - logger.Error().Err(err).Msgf("failed deleting of an orphaned payload secret %s", payloadSecret.GetName()) - } - logger.Debug().Msgf("deleted an orphaned payload secret %s", payloadSecret.GetName()) - } - } - return nil -} - -func (h *handler) getJobComponentPayloadSecretRefNames(ctx context.Context) (map[string]bool, error) { - radixBatches, err := internal.GetRadixBatches(ctx, h.env.RadixDeploymentNamespace, h.kubeUtil.RadixClient(), radixLabels.ForComponentName(h.env.RadixComponentName)) - if err != nil { - return nil, err - } - payloadSecretRefNames := make(map[string]bool) - for _, radixBatch := range radixBatches { - for _, job := range radixBatch.Spec.Jobs { - if job.PayloadSecretRef != nil { - payloadSecretRefNames[job.PayloadSecretRef.Name] = true - } - } - } - return payloadSecretRefNames, nil -} - func (h *handler) getRadixBatchStatuses(ctx context.Context, radixBatchType kube.RadixBatchType) ([]modelsv2.RadixBatch, error) { logger := log.Ctx(ctx) radixBatches, err := internal.GetRadixBatches(ctx, h.env.RadixDeploymentNamespace, h.kubeUtil.RadixClient(), @@ -645,11 +454,3 @@ func applyDefaultJobDescriptionProperties(jobScheduleDescription *common.JobSche } return mergo.Merge(&jobScheduleDescription.RadixJobComponentConfig, defaultRadixJobComponentConfig, mergo.WithTransformers(authTransformer)) } - -func convertToRadixBatchStatuses(radixBatches []*radixv1.RadixBatch, radixDeployJobComponent *radixv1.RadixDeployJobComponent) []*modelsv2.RadixBatch { - batches := make([]*modelsv2.RadixBatch, 0, len(radixBatches)) - for _, radixBatch := range radixBatches { - batches = append(batches, pointers.Ptr(batch.GetRadixBatchStatus(radixBatch, radixDeployJobComponent))) - } - return batches -} diff --git a/api/v2/mock/handler_mock.go b/api/v2/mock/handler_mock.go index 8f45e73..e5435a7 100644 --- a/api/v2/mock/handler_mock.go +++ b/api/v2/mock/handler_mock.go @@ -96,20 +96,6 @@ func (mr *MockHandlerMockRecorder) CreateRadixBatchSingleJob(ctx, jobScheduleDes return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateRadixBatchSingleJob", reflect.TypeOf((*MockHandler)(nil).CreateRadixBatchSingleJob), ctx, jobScheduleDescription) } -// DeleteRadixBatch mocks base method. -func (m *MockHandler) DeleteRadixBatch(ctx context.Context, batchName string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteRadixBatch", ctx, batchName) - ret0, _ := ret[0].(error) - return ret0 -} - -// DeleteRadixBatch indicates an expected call of DeleteRadixBatch. -func (mr *MockHandlerMockRecorder) DeleteRadixBatch(ctx, batchName interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRadixBatch", reflect.TypeOf((*MockHandler)(nil).DeleteRadixBatch), ctx, batchName) -} - // DeleteRadixBatchJob mocks base method. func (m *MockHandler) DeleteRadixBatchJob(ctx context.Context, jobName string) error { m.ctrl.T.Helper() @@ -124,20 +110,6 @@ func (mr *MockHandlerMockRecorder) DeleteRadixBatchJob(ctx, jobName interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRadixBatchJob", reflect.TypeOf((*MockHandler)(nil).DeleteRadixBatchJob), ctx, jobName) } -// GarbageCollectPayloadSecrets mocks base method. -func (m *MockHandler) GarbageCollectPayloadSecrets(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GarbageCollectPayloadSecrets", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// GarbageCollectPayloadSecrets indicates an expected call of GarbageCollectPayloadSecrets. -func (mr *MockHandlerMockRecorder) GarbageCollectPayloadSecrets(ctx interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GarbageCollectPayloadSecrets", reflect.TypeOf((*MockHandler)(nil).GarbageCollectPayloadSecrets), ctx) -} - // GetRadixBatch mocks base method. func (m *MockHandler) GetRadixBatch(ctx context.Context, batchName string) (*v2.RadixBatch, error) { m.ctrl.T.Helper() @@ -183,20 +155,6 @@ func (mr *MockHandlerMockRecorder) GetRadixBatches(ctx interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRadixBatches", reflect.TypeOf((*MockHandler)(nil).GetRadixBatches), ctx) } -// MaintainHistoryLimit mocks base method. -func (m *MockHandler) MaintainHistoryLimit(ctx context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MaintainHistoryLimit", ctx) - ret0, _ := ret[0].(error) - return ret0 -} - -// MaintainHistoryLimit indicates an expected call of MaintainHistoryLimit. -func (mr *MockHandlerMockRecorder) MaintainHistoryLimit(ctx interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaintainHistoryLimit", reflect.TypeOf((*MockHandler)(nil).MaintainHistoryLimit), ctx) -} - // RestartRadixBatch mocks base method. func (m *MockHandler) RestartRadixBatch(ctx context.Context, batchName string) error { m.ctrl.T.Helper() diff --git a/api/v2/secrets.go b/api/v2/secrets.go index e5d54bd..ffc900c 100644 --- a/api/v2/secrets.go +++ b/api/v2/secrets.go @@ -18,15 +18,6 @@ func (h *handler) GetSecretsForRadixBatch(ctx context.Context, batchName string) return selector, nil } -// DeleteSecret Delete the service -func (h *handler) DeleteSecret(ctx context.Context, secret *corev1.Secret) error { - err := h.kubeUtil.DeleteSecret(ctx, secret.Namespace, secret.Name) - if err != nil { - return apiErrors.NewFromError(err) - } - return nil -} - func getLabelSelectorForRadixBatchSecret(batchName string) string { return kubeLabels.SelectorFromSet(labels.ForBatchName(batchName)).String() } diff --git a/go.mod b/go.mod index 9709807..aa5fcbe 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.22.5 require ( dario.cat/mergo v1.0.0 github.com/equinor/radix-common v1.9.3 - github.com/equinor/radix-operator v1.57.4 + github.com/equinor/radix-operator v1.58.1 github.com/gin-gonic/gin v1.10.0 github.com/go-swagger/go-swagger v0.31.0 github.com/golang/mock v1.6.0 @@ -76,14 +76,14 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect - golang.org/x/time v0.5.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/oauth2 v0.22.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/term v0.23.0 // indirect + golang.org/x/text v0.17.0 // indirect + golang.org/x/time v0.6.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -91,10 +91,10 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.30.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b // indirect - k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect + k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect knative.dev/pkg v0.0.0-20240116073220-b488e7be5902 // indirect - sigs.k8s.io/controller-runtime v0.18.4 // indirect + sigs.k8s.io/controller-runtime v0.18.5 // indirect sigs.k8s.io/gateway-api v1.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 842582a..dcbccb5 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,8 @@ github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtz github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/equinor/radix-common v1.9.3 h1:dLKFzYy8/XyEG9Zygi0rMWIYGCddai/ILwUqjBiGYxQ= github.com/equinor/radix-common v1.9.3/go.mod h1:+g0Wj0D40zz29DjNkYKVmCVeYy4OsFWKI7Qi9rA6kpY= -github.com/equinor/radix-operator v1.57.4 h1:Rr/bjxU+ABWyk1UM9QknMmOHQxiwHi/HHjeA+50fUNU= -github.com/equinor/radix-operator v1.57.4/go.mod h1:zCdAiP/wxyvlUO4qGoJuLW3O+ZSt9kTyHMnjmsR3fCU= +github.com/equinor/radix-operator v1.58.1 h1:Wb/UOP1m4wUdWCL/gynPcnf6axz01Z24fBvK2DRL5m0= +github.com/equinor/radix-operator v1.58.1/go.mod h1:zCdAiP/wxyvlUO4qGoJuLW3O+ZSt9kTyHMnjmsR3fCU= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= @@ -81,8 +81,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= @@ -121,10 +121,10 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.17.2 h1:7eMhcy3GimbsA3hEnVKdw/PQM9XN9krpKVXsZdph0/g= -github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw= +github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -183,30 +183,30 @@ golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -217,24 +217,24 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -264,16 +264,16 @@ k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b h1:Q9xmGWBvOGd8UJyccgpYlLosk/JlfP3xQLNkQlHJeXw= -k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= -k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= -k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 h1:1Wof1cGQgA5pqgo8MxKPtf+qN6Sh/0JzznmeGPm1HnE= +k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8/go.mod h1:Os6V6dZwLNii3vxFpxcNaTmH8LJJBkOTg1N0tOA0fvA= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= knative.dev/pkg v0.0.0-20240116073220-b488e7be5902 h1:H6+JJN23fhwYWCHY1339sY6uhIyoUwDy1a8dN233fdk= knative.dev/pkg v0.0.0-20240116073220-b488e7be5902/go.mod h1:NYk8mMYoLkO7CQWnNkti4YGGnvLxN6MIDbUvtgeo0C0= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= -sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= +sigs.k8s.io/controller-runtime v0.18.5 h1:nTHio/W+Q4aBlQMgbnC5hZb4IjIidyrizMai9P6n4Rk= +sigs.k8s.io/controller-runtime v0.18.5/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/gateway-api v1.0.0 h1:iPTStSv41+d9p0xFydll6d7f7MOBGuqXM6p2/zVYMAs= sigs.k8s.io/gateway-api v1.0.0/go.mod h1:4cUgr0Lnp5FZ0Cdq8FdRwCvpiWws7LVhLHGIudLlf4c= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/internal/batch.go b/internal/batch.go index 2fdc8f1..cd857c8 100644 --- a/internal/batch.go +++ b/internal/batch.go @@ -31,7 +31,7 @@ func GetRadixBatches(ctx context.Context, namespace string, radixClient radixcli ) if err != nil { - return nil, errors.NewFromError(err) + return nil, err } return slice.PointersOf(radixBatchList.Items).([]*radixv1.RadixBatch), nil @@ -77,20 +77,24 @@ func ParseBatchAndJobNameFromScheduledJobName(scheduleJobName string) (batchName return } +// IsRadixBatchSucceeded Check if Radix batch is succeeded func IsRadixBatchSucceeded(batch *radixv1.RadixBatch) bool { return batch.Status.Condition.Type == radixv1.BatchConditionTypeCompleted && slice.All(batch.Status.JobStatuses, func(jobStatus radixv1.RadixBatchJobStatus) bool { return IsRadixBatchJobSucceeded(jobStatus) }) } +// IsRadixBatchJobSucceeded Check if Radix batch job is succeeded func IsRadixBatchJobSucceeded(jobStatus radixv1.RadixBatchJobStatus) bool { return jobStatus.Phase == radixv1.BatchJobPhaseSucceeded || jobStatus.Phase == radixv1.BatchJobPhaseStopped } +// IsRadixBatchJobFailed Check if Radix batch job is failed func IsRadixBatchJobFailed(jobStatus radixv1.RadixBatchJobStatus) bool { return jobStatus.Phase == radixv1.BatchJobPhaseFailed } +// IsRadixBatchNotSucceeded Check if Radix batch is not succeeded func IsRadixBatchNotSucceeded(batch *radixv1.RadixBatch) bool { return batch.Status.Condition.Type == radixv1.BatchConditionTypeCompleted && slice.Any(batch.Status.JobStatuses, func(jobStatus radixv1.RadixBatchJobStatus) bool { return !IsRadixBatchJobSucceeded(jobStatus) diff --git a/internal/payload.go b/internal/payload.go new file mode 100644 index 0000000..b8050cf --- /dev/null +++ b/internal/payload.go @@ -0,0 +1,59 @@ +package internal + +import ( + "context" + "time" + + "github.com/equinor/radix-operator/pkg/apis/kube" + "github.com/equinor/radix-operator/pkg/apis/utils/labels" + radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" + "github.com/rs/zerolog/log" + "k8s.io/apimachinery/pkg/api/errors" +) + +// GetJobComponentPayloadSecretRefNames Get the payload secret ref names for the job components +func GetJobComponentPayloadSecretRefNames(ctx context.Context, radixClient radixclient.Interface, namespace, radixComponentName string) (map[string]bool, error) { + radixBatches, err := GetRadixBatches(ctx, namespace, radixClient, labels.ForComponentName(radixComponentName)) + if err != nil { + return nil, err + } + payloadSecretRefNames := make(map[string]bool) + for _, radixBatch := range radixBatches { + for _, job := range radixBatch.Spec.Jobs { + if job.PayloadSecretRef != nil { + payloadSecretRefNames[job.PayloadSecretRef.Name] = true + } + } + } + return payloadSecretRefNames, nil +} + +// GarbageCollectPayloadSecrets Delete orphaned payload secrets +func GarbageCollectPayloadSecrets(ctx context.Context, kubeUtil *kube.Kube, namespace, radixComponentName string) error { + logger := log.Ctx(ctx) + logger.Debug().Msgf("Garbage collecting payload secrets") + payloadSecretRefNames, err := GetJobComponentPayloadSecretRefNames(ctx, kubeUtil.RadixClient(), namespace, radixComponentName) + if err != nil { + return err + } + payloadSecrets, err := kubeUtil.ListSecretsWithSelector(ctx, namespace, labels.GetRadixBatchDescendantsSelector(radixComponentName).String()) + if err != nil { + return err + } + logger.Debug().Msgf("%d payload secrets, %d secret reference unique names", len(payloadSecrets), len(payloadSecretRefNames)) + yesterday := time.Now().Add(time.Hour * -24) + for _, payloadSecret := range payloadSecrets { + if _, ok := payloadSecretRefNames[payloadSecret.GetName()]; !ok { + if payloadSecret.GetCreationTimestamp().After(yesterday) { + logger.Debug().Msgf("skipping deletion of an orphaned payload secret %s, created within 24 hours", payloadSecret.GetName()) + continue + } + if err := kubeUtil.DeleteSecret(ctx, payloadSecret.GetNamespace(), payloadSecret.GetName()); err != nil && !errors.IsNotFound(err) { + logger.Error().Err(err).Msgf("failed deleting of an orphaned payload secret %s", payloadSecret.GetName()) + } else { + logger.Debug().Msgf("deleted an orphaned payload secret %s", payloadSecret.GetName()) + } + } + } + return nil +} diff --git a/main.go b/main.go index d46e2a3..c124459 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,9 @@ import ( jobControllers "github.com/equinor/radix-job-scheduler/api/v1/controllers/jobs" jobApi "github.com/equinor/radix-job-scheduler/api/v1/jobs" "github.com/equinor/radix-job-scheduler/models" - "github.com/equinor/radix-job-scheduler/models/notifications" + "github.com/equinor/radix-job-scheduler/pkg/batch" + "github.com/equinor/radix-job-scheduler/pkg/notifications" + "github.com/equinor/radix-job-scheduler/pkg/watcher" "github.com/equinor/radix-job-scheduler/router" _ "github.com/equinor/radix-job-scheduler/swaggerui" "github.com/equinor/radix-job-scheduler/utils/radix" @@ -41,11 +43,13 @@ func main() { log.Fatal().Err(err).Msg("failed to get job specification") } - radixBatchWatcher, err := getRadixBatchWatcher(kubeUtil, radixDeployJobComponent, env) + jobHistory := batch.NewHistory(kubeUtil, env, radixDeployJobComponent) + notifier := notifications.NewWebhookNotifier(radixDeployJobComponent) + radixBatchWatcher, err := watcher.NewRadixBatchWatcher(ctx, kubeUtil.RadixClient(), env.RadixDeploymentNamespace, jobHistory, notifier) if err != nil { - log.Fatal().Err(err).Msg("failed to inititialize job watcher") + log.Fatal().Err(err).Msg("failed to initialize job watcher") } - defer close(radixBatchWatcher.Stop) + defer radixBatchWatcher.Stop() runApiServer(ctx, kubeUtil, env, radixDeployJobComponent) } @@ -95,17 +99,6 @@ func runApiServer(ctx context.Context, kubeUtil *kube.Kube, env *models.Env, rad } } -func getRadixBatchWatcher(kubeUtil *kube.Kube, radixDeployJobComponent *radixv1.RadixDeployJobComponent, env *models.Env) (*notifications.Watcher, error) { - notifier := notifications.NewWebhookNotifier(radixDeployJobComponent) - log.Info().Msgf("Created notifier: %s", notifier.String()) - if !notifier.Enabled() { - log.Info().Msg("Notifiers are not enabled, RadixBatch event and changes watcher is skipped.") - return notifications.NullRadixBatchWatcher(), nil - } - - return notifications.NewRadixBatchWatcher(kubeUtil.RadixClient(), env.RadixDeploymentNamespace, notifier) -} - func getKubeUtil(ctx context.Context) *kube.Kube { kubeClient, radixClient, kedaClient, _, secretProviderClient, _ := utils.GetKubernetesClient(ctx) kubeUtil, _ := kube.New(kubeClient, radixClient, kedaClient, secretProviderClient) diff --git a/models/common/batch_description.go b/models/common/batch_description.go index 6c6eba9..01781a4 100644 --- a/models/common/batch_description.go +++ b/models/common/batch_description.go @@ -3,6 +3,12 @@ package common // BatchScheduleDescription holds description about batch scheduling job // swagger:model BatchScheduleDescription type BatchScheduleDescription struct { + // Defines a user defined ID of the batch. + // + // required: false + // example: 'batch-id-1' + BatchId string `json:"batchId,omitempty"` + // JobScheduleDescriptions descriptions of jobs to schedule within the batch // // required: true diff --git a/models/notifications/test/test.go b/models/notifications/test/test.go deleted file mode 100644 index 7a73c7e..0000000 --- a/models/notifications/test/test.go +++ /dev/null @@ -1,25 +0,0 @@ -package test - -import ( - "github.com/equinor/radix-common/utils/numbers" - radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" - "github.com/equinor/radix-operator/pkg/apis/utils" -) - -func GetRadixDeploymentWithRadixJobComponent(appName, environment, componentName string, port int32) *radixv1.RadixDeployment { - return utils.NewDeploymentBuilder(). - WithAppName(appName). - WithImageTag("image-tag"). - WithEnvironment(environment). - WithJobComponent(utils.NewDeployJobComponentBuilder(). - WithImage("radixdev.azurecr.io/some-image:image-tag"). - WithName(componentName). - WithPort("http", port). - WithSchedulerPort(numbers.Int32Ptr(9090))).BuildRD() -} - -func GetRadixApplicationWithRadixJobComponent(appName, environment, envBranch, jobComponentName string, port int32, notifications *radixv1.Notifications) *radixv1.RadixApplication { - return utils.NewRadixApplicationBuilder().WithAppName(appName).WithEnvironment(environment, envBranch). - WithJobComponents(utils.NewApplicationJobComponentBuilder().WithName(jobComponentName).WithPort("http", port).WithNotifications(notifications)). - BuildRA() -} diff --git a/models/notifications/webhook_notifier.go b/models/notifications/webhook_notifier.go deleted file mode 100644 index c618c00..0000000 --- a/models/notifications/webhook_notifier.go +++ /dev/null @@ -1,71 +0,0 @@ -package notifications - -import ( - "bytes" - "encoding/json" - "fmt" - "net/http" - - "github.com/equinor/radix-job-scheduler/models/v1/events" - - radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" -) - -type webhookNotifier struct { - webhookURL string - jobComponentName string - radixDeployJobComponent *radixv1.RadixDeployJobComponent -} - -func NewWebhookNotifier(radixDeployJobComponent *radixv1.RadixDeployJobComponent) Notifier { - notifier := webhookNotifier{ - jobComponentName: radixDeployJobComponent.Name, - radixDeployJobComponent: radixDeployJobComponent, - } - if radixDeployJobComponent.Notifications != nil && webhookIsNotEmpty(radixDeployJobComponent.Notifications.Webhook) { - notifier.webhookURL = *radixDeployJobComponent.Notifications.Webhook - } - return ¬ifier -} - -func (notifier *webhookNotifier) Enabled() bool { - return len(notifier.webhookURL) > 0 -} - -func (notifier *webhookNotifier) String() string { - if notifier.Enabled() { - return fmt.Sprintf("Webhook notifier is enabled. Webhook: %s", notifier.webhookURL) - } - return "Webhook notifier is disabled" -} - -func (notifier *webhookNotifier) Notify(event events.Event, radixBatch *radixv1.RadixBatch, updatedJobStatuses []radixv1.RadixBatchJobStatus, errChan chan error) (done chan struct{}) { - done = make(chan struct{}) - go func() { - if !notifier.Enabled() || len(notifier.webhookURL) == 0 || radixBatch.Spec.RadixDeploymentJobRef.Job != notifier.jobComponentName { - done <- struct{}{} - close(done) - return - } - // RadixBatch status and only changed job statuses - batchStatus := notifier.getRadixBatchEventFromRadixBatch(event, radixBatch, updatedJobStatuses) - statusesJson, err := json.Marshal(batchStatus) - if err != nil { - errChan <- fmt.Errorf("failed serialize updated JobStatuses %v", err) - return - } - buf := bytes.NewReader(statusesJson) - _, err = http.Post(notifier.webhookURL, "application/json", buf) - if err != nil { - errChan <- fmt.Errorf("failed to notify on RadixBatch object create or change %s: %v", radixBatch.GetName(), err) - return - } - done <- struct{}{} - close(done) - }() - return done -} - -func webhookIsNotEmpty(webhook *string) bool { - return webhook != nil && len(*webhook) > 0 -} diff --git a/models/v1/defaults/job.go b/models/v1/defaults/job.go deleted file mode 100644 index f0c29a3..0000000 --- a/models/v1/defaults/job.go +++ /dev/null @@ -1,7 +0,0 @@ -package defaultsv1 - -const ( - //K8sJobNameLabel A label that k8s automatically adds to a Pod created by a Job - K8sJobNameLabel = "job-name" - RadixJobIdLabel = "radix-job-id" -) diff --git a/models/v1/job_status.go b/models/v1/job_status.go index 3a71525..9297f26 100644 --- a/models/v1/job_status.go +++ b/models/v1/job_status.go @@ -8,12 +8,19 @@ type JobStatus struct { // required: false // example: 'job1' JobId string `json:"jobId,omitempty"` + // BatchName Optional Batch ID of a job // // required: false // example: 'batch1' BatchName string `json:"batchName,omitempty"` + // Defines a user defined ID of the batch. + // + // required: false + // example: 'batch-id-1' + BatchId string `json:"batchId,omitempty"` + // Name of the job // required: true // example: calculator diff --git a/models/v2/batch_status.go b/models/v2/batch_status.go index 7c98bf5..1dbaea4 100644 --- a/models/v2/batch_status.go +++ b/models/v2/batch_status.go @@ -10,6 +10,12 @@ type RadixBatch struct { // required: true Name string `json:"name"` + // Defines a user defined ID of the batch. + // + // required: false + // example: 'batch-id-1' + BatchId string `json:"batchId,omitempty"` + // Radix batch creation timestamp // // required: true diff --git a/pkg/batch/batch.go b/pkg/batch/batch.go index 622ed37..bcf52d5 100644 --- a/pkg/batch/batch.go +++ b/pkg/batch/batch.go @@ -9,7 +9,7 @@ import ( "github.com/equinor/radix-common/utils" "github.com/equinor/radix-common/utils/pointers" "github.com/equinor/radix-common/utils/slice" - "github.com/equinor/radix-job-scheduler/api/errors" + apiErrors "github.com/equinor/radix-job-scheduler/api/errors" "github.com/equinor/radix-job-scheduler/internal" modelsv2 "github.com/equinor/radix-job-scheduler/models/v2" "github.com/equinor/radix-job-scheduler/utils/radix/jobs" @@ -17,14 +17,24 @@ import ( radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/client/clientset/versioned" "github.com/rs/zerolog/log" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" ) +// CompletedRadixBatches Completed RadixBatch lists +type CompletedRadixBatches struct { + SucceededRadixBatches []*modelsv2.RadixBatch + NotSucceededRadixBatches []*modelsv2.RadixBatch + SucceededSingleJobs []*modelsv2.RadixBatch + NotSucceededSingleJobs []*modelsv2.RadixBatch +} + // GetRadixBatchStatus Get radix batch func GetRadixBatchStatus(radixBatch *radixv1.RadixBatch, radixDeployJobComponent *radixv1.RadixDeployJobComponent) modelsv2.RadixBatch { return modelsv2.RadixBatch{ Name: radixBatch.GetName(), + BatchId: radixBatch.Spec.BatchId, BatchType: radixBatch.Labels[kube.RadixBatchTypeLabel], CreationTime: utils.FormatTime(pointers.Ptr(radixBatch.GetCreationTimestamp())), Started: utils.FormatTime(radixBatch.Status.Condition.ActiveTime), @@ -36,6 +46,18 @@ func GetRadixBatchStatus(radixBatch *radixv1.RadixBatch, radixDeployJobComponent } } +// DeleteRadixBatchByName Delete a batch by name +func DeleteRadixBatchByName(ctx context.Context, radixClient versioned.Interface, namespace, batchName string) error { + radixBatch, err := internal.GetRadixBatch(ctx, radixClient, namespace, batchName) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return DeleteRadixBatch(ctx, radixClient, radixBatch) +} + func getRadixBatchJobStatusesFromRadixBatch(radixBatch *radixv1.RadixBatch, radixBatchJobStatuses []radixv1.RadixBatchJobStatus) []modelsv2.RadixBatchJobStatus { radixBatchJobsStatuses := internal.GetRadixBatchJobsStatusesMap(radixBatchJobStatuses) jobStatuses := make([]modelsv2.RadixBatchJobStatus, 0, len(radixBatch.Spec.Jobs)) @@ -101,6 +123,7 @@ func CopyRadixBatchOrJob(ctx context.Context, radixClient versioned.Interface, s Labels: sourceRadixBatch.GetLabels(), }, Spec: radixv1.RadixBatchSpec{ + BatchId: sourceRadixBatch.Spec.BatchId, RadixDeploymentJobRef: radixv1.RadixDeploymentJobComponentSelector{ LocalObjectReference: radixv1.LocalObjectReference{Name: radixDeploymentName}, Job: radixComponentName, @@ -112,7 +135,7 @@ func CopyRadixBatchOrJob(ctx context.Context, radixClient versioned.Interface, s logger.Debug().Msgf("Create the copied Radix Batch %s with %d jobs in the cluster", radixBatch.GetName(), len(radixBatch.Spec.Jobs)) createdRadixBatch, err := radixClient.RadixV1().RadixBatches(sourceRadixBatch.GetNamespace()).Create(ctx, &radixBatch, metav1.CreateOptions{}) if err != nil { - return nil, errors.NewFromError(err) + return nil, err } logger.Debug().Msgf("copied the batch %s for the component %s", radixBatch.GetName(), radixComponentName) @@ -124,7 +147,7 @@ func StopRadixBatch(ctx context.Context, radixClient versioned.Interface, radixB logger := log.Ctx(ctx) logger.Info().Msgf("stop batch %s for namespace: %s", radixBatch.GetName(), radixBatch.GetNamespace()) if !isBatchStoppable(radixBatch.Status.Condition) { - return errors.NewBadRequest(fmt.Sprintf("cannot stop completed batch %s", radixBatch.GetName())) + return apiErrors.NewBadRequest(fmt.Sprintf("cannot stop completed batch %s", radixBatch.GetName())) } newRadixBatch := radixBatch.DeepCopy() @@ -144,7 +167,7 @@ func StopRadixBatchJob(ctx context.Context, radixClient versioned.Interface, rad logger := log.Ctx(ctx) logger.Info().Msgf("stop a job %s in the batch %s", jobName, radixBatch.GetName()) if !isBatchStoppable(radixBatch.Status.Condition) { - return errors.NewBadRequest(fmt.Sprintf("cannot stop the job %s in the completed batch %s", jobName, radixBatch.GetName())) + return apiErrors.NewBadRequest(fmt.Sprintf("cannot stop the job %s in the completed batch %s", jobName, radixBatch.GetName())) } newRadixBatch := radixBatch.DeepCopy() @@ -155,12 +178,12 @@ func StopRadixBatchJob(ctx context.Context, radixClient versioned.Interface, rad } if jobStatus, ok := radixBatchJobsStatusMap[radixBatchJob.Name]; ok && (internal.IsRadixBatchJobSucceeded(jobStatus) || internal.IsRadixBatchJobFailed(jobStatus)) { - return errors.NewBadRequest(fmt.Sprintf("cannot stop the job %s with the status %s in the batch %s", jobName, string(jobStatus.Phase), radixBatch.GetName())) + return apiErrors.NewBadRequest(fmt.Sprintf("cannot stop the job %s with the status %s in the batch %s", jobName, string(jobStatus.Phase), radixBatch.GetName())) } newRadixBatch.Spec.Jobs[jobIndex].Stop = pointers.Ptr(true) return updateRadixBatch(ctx, radixClient, radixBatch.GetNamespace(), newRadixBatch) } - return errors.NewNotFound("batch job", jobName) + return apiErrors.NewNotFound("batch job", jobName) } // RestartRadixBatch Restart a batch @@ -172,7 +195,7 @@ func RestartRadixBatch(ctx context.Context, radixClient versioned.Interface, rad setRestartJobTimeout(radixBatch, jobIdx, restartTimestamp) } if _, err := radixClient.RadixV1().RadixBatches(radixBatch.GetNamespace()).Update(ctx, radixBatch, metav1.UpdateOptions{}); err != nil { - return errors.NewFromError(err) + return err } return nil } @@ -187,7 +210,7 @@ func RestartRadixBatchJob(ctx context.Context, radixClient versioned.Interface, } setRestartJobTimeout(radixBatch, jobIdx, utils.FormatTimestamp(time.Now())) if _, err := radixClient.RadixV1().RadixBatches(radixBatch.GetNamespace()).Update(ctx, radixBatch, metav1.UpdateOptions{}); err != nil { - return errors.NewFromError(err) + return err } return nil } @@ -196,8 +219,8 @@ func RestartRadixBatchJob(ctx context.Context, radixClient versioned.Interface, func DeleteRadixBatch(ctx context.Context, radixClient versioned.Interface, radixBatch *radixv1.RadixBatch) error { logger := log.Ctx(ctx) logger.Debug().Msgf("delete batch %s", radixBatch.GetName()) - if err := radixClient.RadixV1().RadixBatches(radixBatch.GetNamespace()).Delete(ctx, radixBatch.GetName(), metav1.DeleteOptions{PropagationPolicy: pointers.Ptr(metav1.DeletePropagationBackground)}); err != nil { - return errors.NewFromError(err) + if err := radixClient.RadixV1().RadixBatches(radixBatch.GetNamespace()).Delete(ctx, radixBatch.GetName(), metav1.DeleteOptions{PropagationPolicy: pointers.Ptr(metav1.DeletePropagationBackground)}); err != nil && !errors.IsNotFound(err) { + return err } return nil } @@ -231,7 +254,7 @@ func updateRadixBatch(ctx context.Context, radixClient versioned.Interface, name return err }) if err != nil { - return errors.NewFromError(fmt.Errorf("failed to patch RadixBatch object: %w", err)) + return apiErrors.NewFromError(fmt.Errorf("failed to patch RadixBatch object: %w", err)) } logger.Debug().Msgf("Patched RadixBatch: %s in namespace %s", radixBatch.GetName(), namespace) return nil diff --git a/pkg/batch/batch_test.go b/pkg/batch/batch_test.go index bd72ba0..cfa1c40 100644 --- a/pkg/batch/batch_test.go +++ b/pkg/batch/batch_test.go @@ -37,8 +37,11 @@ type testArgs struct { const ( batchName1 = "batch1" + batchId1 = "batchId1" batchName2 = "batch2" + batchId2 = "batchId2" batchName3 = "batch3" + batchId3 = "batchId3" jobName1 = "job1" jobName2 = "job2" jobName3 = "job3" @@ -68,7 +71,7 @@ func TestCopyRadixBatchOrJob(t *testing.T) { { name: "only deployment has no rules, no job statuses", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, nil), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props), expectedBatchStatus: radixv1.RadixBatchJobApiStatusWaiting, @@ -77,7 +80,7 @@ func TestCopyRadixBatchOrJob(t *testing.T) { { name: "only deployment has no rules, job statuses waiting, active", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseActive}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props), expectedBatchStatus: radixv1.RadixBatchJobApiStatusWaiting, @@ -86,7 +89,7 @@ func TestCopyRadixBatchOrJob(t *testing.T) { { name: "only deployment has no rules, job statuses failed, succeeded", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseFailed, jobName2: radixv1.BatchJobPhaseSucceeded}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props), expectedBatchStatus: radixv1.RadixBatchJobApiStatusWaiting, @@ -95,7 +98,7 @@ func TestCopyRadixBatchOrJob(t *testing.T) { { name: "only deployment, with rules, job statuses failed, succeeded", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseFailed, jobName2: radixv1.BatchJobPhaseSucceeded}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props, createBatchStatusRule(radixv1.RadixBatchJobApiStatusFailed, radixv1.ConditionAny, radixv1.OperatorIn, radixv1.BatchJobPhaseFailed)), @@ -150,7 +153,7 @@ func TestGetRadixBatchStatus(t *testing.T) { { name: "only deployment has no rules, no job statuses", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props), expectedBatchStatus: radixv1.RadixBatchJobApiStatusWaiting, @@ -159,7 +162,7 @@ func TestGetRadixBatchStatus(t *testing.T) { { name: "only deployment has no rules, job statuses waiting, active", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseActive}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props), expectedBatchStatus: radixv1.RadixBatchJobApiStatusActive, @@ -168,7 +171,7 @@ func TestGetRadixBatchStatus(t *testing.T) { { name: "only deployment has no rules, job statuses failed, succeeded", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseFailed, jobName2: radixv1.BatchJobPhaseSucceeded}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props), expectedBatchStatus: radixv1.RadixBatchJobApiStatusActive, @@ -177,7 +180,7 @@ func TestGetRadixBatchStatus(t *testing.T) { { name: "only deployment, with only rule does not match, job statuses failed, succeeded", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseSucceeded}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props, createBatchStatusRule(radixv1.RadixBatchJobApiStatusFailed, radixv1.ConditionAny, radixv1.OperatorIn, radixv1.BatchJobPhaseFailed)), @@ -187,7 +190,7 @@ func TestGetRadixBatchStatus(t *testing.T) { { name: "only deployment, second rule matches", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseStopped, jobName2: radixv1.BatchJobPhaseSucceeded}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props, createBatchStatusRule(radixv1.RadixBatchJobApiStatusFailed, radixv1.ConditionAny, radixv1.OperatorIn, radixv1.BatchJobPhaseFailed), @@ -198,7 +201,7 @@ func TestGetRadixBatchStatus(t *testing.T) { { name: "only deployment, with only rule any in matches, job statuses failed, succeeded", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseFailed, jobName2: radixv1.BatchJobPhaseSucceeded}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props, createBatchStatusRule(radixv1.RadixBatchJobApiStatusFailed, radixv1.ConditionAny, radixv1.OperatorIn, radixv1.BatchJobPhaseFailed)), @@ -208,7 +211,7 @@ func TestGetRadixBatchStatus(t *testing.T) { { name: "only deployment, with rule all not-in matches", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2, jobName3}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2, jobName3}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseRunning, jobName2: radixv1.BatchJobPhaseActive}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props, createBatchStatusRule(radixv1.RadixBatchJobApiStatusRunning, radixv1.ConditionAll, radixv1.OperatorNotIn, radixv1.BatchJobPhaseWaiting, radixv1.BatchJobPhaseStopped, radixv1.BatchJobPhaseSucceeded, radixv1.BatchJobPhaseFailed), @@ -220,7 +223,7 @@ func TestGetRadixBatchStatus(t *testing.T) { { name: "only deployment, with second rule all not-in matches", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2, jobName3}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2, jobName3}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseRunning, jobName2: radixv1.BatchJobPhaseActive}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props, createBatchStatusRule(radixv1.RadixBatchJobApiStatusWaiting, radixv1.ConditionAll, radixv1.OperatorNotIn, radixv1.BatchJobPhaseActive, radixv1.BatchJobPhaseRunning, radixv1.BatchJobPhaseStopped, radixv1.BatchJobPhaseSucceeded, radixv1.BatchJobPhaseFailed), @@ -232,7 +235,7 @@ func TestGetRadixBatchStatus(t *testing.T) { { name: "only deployment, with none of rules all not-in matches", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2, jobName3}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2, jobName3}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseRunning, jobName2: radixv1.BatchJobPhaseActive, jobName3: radixv1.BatchJobPhaseFailed}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props, createBatchStatusRule(radixv1.RadixBatchJobApiStatusWaiting, radixv1.ConditionAll, radixv1.OperatorNotIn, radixv1.BatchJobPhaseActive, radixv1.BatchJobPhaseRunning, radixv1.BatchJobPhaseStopped, radixv1.BatchJobPhaseSucceeded, radixv1.BatchJobPhaseFailed), @@ -244,7 +247,7 @@ func TestGetRadixBatchStatus(t *testing.T) { { name: "two deployments, with rule from active applied", args: testArgs{ - radixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2, jobName3}, + radixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2, jobName3}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseRunning, jobName2: radixv1.BatchJobPhaseActive, jobName3: radixv1.BatchJobPhaseFailed, jobName4: radixv1.BatchJobPhaseSucceeded}), batchRadixDeploy: createRadixDeployJobComponent(radixDeploymentName1, props, createBatchStatusRule(radixv1.RadixBatchJobApiStatusFailed, radixv1.ConditionAny, radixv1.OperatorIn, radixv1.BatchJobPhaseFailed)), @@ -304,9 +307,9 @@ func TestGetRadixBatchStatuses(t *testing.T) { name: "only deployment has no rules, no job statuses", batchesArgs: multiBatchArgs{ radixBatches: []*radixv1.RadixBatch{ - createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil), - createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil), }, batchRadixDeploymentBuilders: map[string]operatorUtils.DeploymentBuilder{radixDeploymentName1: createRadixDeployJobComponent(radixDeploymentName1, props)}, @@ -320,11 +323,11 @@ func TestGetRadixBatchStatuses(t *testing.T) { name: "only deployment has no rules, batch status is default", batchesArgs: multiBatchArgs{ radixBatches: []*radixv1.RadixBatch{ - createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, jobStatusPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseActive}), - createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseSucceeded, jobName2: radixv1.BatchJobPhaseRunning}), - createRadixBatch(batchName3, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName3, batchId3, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeCompleted, jobStatusPhase{jobName1: radixv1.BatchJobPhaseSucceeded, jobName2: radixv1.BatchJobPhaseFailed}), }, batchRadixDeploymentBuilders: map[string]operatorUtils.DeploymentBuilder{radixDeploymentName1: createRadixDeployJobComponent(radixDeploymentName1, props)}, @@ -340,11 +343,11 @@ func TestGetRadixBatchStatuses(t *testing.T) { name: "only deployment, with only rule does not match, batch status is default", batchesArgs: multiBatchArgs{ radixBatches: []*radixv1.RadixBatch{ - createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, jobStatusPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseWaiting}), - createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseSucceeded, jobName2: radixv1.BatchJobPhaseRunning}), - createRadixBatch(batchName3, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName3, batchId3, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeCompleted, jobStatusPhase{jobName1: radixv1.BatchJobPhaseSucceeded, jobName2: radixv1.BatchJobPhaseStopped}), }, batchRadixDeploymentBuilders: map[string]operatorUtils.DeploymentBuilder{radixDeploymentName1: createRadixDeployJobComponent(radixDeploymentName1, props, @@ -361,11 +364,11 @@ func TestGetRadixBatchStatuses(t *testing.T) { name: "only deployment, with only rule and it matches on two batches, third batch status is default", batchesArgs: multiBatchArgs{ radixBatches: []*radixv1.RadixBatch{ - createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseRunning, jobName3: radixv1.BatchJobPhaseSucceeded}), - createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseStopped, jobName2: radixv1.BatchJobPhaseRunning}), - createRadixBatch(batchName3, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName3, batchId3, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeCompleted, jobStatusPhase{jobName1: radixv1.BatchJobPhaseSucceeded, jobName2: radixv1.BatchJobPhaseFailed}), }, batchRadixDeploymentBuilders: map[string]operatorUtils.DeploymentBuilder{radixDeploymentName1: createRadixDeployJobComponent(radixDeploymentName1, props, @@ -382,11 +385,11 @@ func TestGetRadixBatchStatuses(t *testing.T) { name: "only deployment, multiple rules, first matching rule applied on two batches, third batch status is default", batchesArgs: multiBatchArgs{ radixBatches: []*radixv1.RadixBatch{ - createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseStopped, jobName2: radixv1.BatchJobPhaseRunning, jobName3: radixv1.BatchJobPhaseSucceeded}), - createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseStopped, jobName2: radixv1.BatchJobPhaseWaiting, jobName3: radixv1.BatchJobPhaseFailed}), - createRadixBatch(batchName3, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName3, batchId3, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeCompleted, jobStatusPhase{jobName1: radixv1.BatchJobPhaseSucceeded, jobName2: radixv1.BatchJobPhaseFailed}), }, batchRadixDeploymentBuilders: map[string]operatorUtils.DeploymentBuilder{radixDeploymentName1: createRadixDeployJobComponent(radixDeploymentName1, props, @@ -405,11 +408,11 @@ func TestGetRadixBatchStatuses(t *testing.T) { name: "only deployment, multiple rules, only first matching rule not-in applied on two batches, third batch status is default", batchesArgs: multiBatchArgs{ radixBatches: []*radixv1.RadixBatch{ - createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseRunning, jobName3: radixv1.BatchJobPhaseActive}), - createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeCompleted, jobStatusPhase{jobName1: radixv1.BatchJobPhaseStopped, jobName2: radixv1.BatchJobPhaseSucceeded, jobName3: radixv1.BatchJobPhaseRunning, jobName4: radixv1.BatchJobPhaseWaiting}), - createRadixBatch(batchName3, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName3, batchId3, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeCompleted, jobStatusPhase{jobName1: radixv1.BatchJobPhaseFailed, jobName2: radixv1.BatchJobPhaseFailed}), }, batchRadixDeploymentBuilders: map[string]operatorUtils.DeploymentBuilder{radixDeploymentName1: createRadixDeployJobComponent(radixDeploymentName1, props, @@ -428,11 +431,11 @@ func TestGetRadixBatchStatuses(t *testing.T) { name: "multiple deployments, used rules from active deployment", batchesArgs: multiBatchArgs{ radixBatches: []*radixv1.RadixBatch{ - createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, + createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseRunning, jobName3: radixv1.BatchJobPhaseActive}), - createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName2, []string{jobName1, jobName2}, + createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName2, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseRunning, jobName3: radixv1.BatchJobPhaseActive}), - createRadixBatch(batchName3, props, kube.RadixBatchTypeBatch, radixDeploymentName3, []string{jobName1, jobName2}, + createRadixBatch(batchName3, batchId3, props, kube.RadixBatchTypeBatch, radixDeploymentName3, []string{jobName1, jobName2}, radixv1.BatchConditionTypeActive, jobStatusPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseRunning, jobName3: radixv1.BatchJobPhaseActive}), }, batchRadixDeploymentBuilders: map[string]operatorUtils.DeploymentBuilder{ @@ -488,8 +491,13 @@ func TestGetRadixBatchStatuses(t *testing.T) { acc[batchStatus.Name] = batchStatus return acc }) + radixBatchMap := slice.Reduce(tt.batchesArgs.radixBatches, make(map[string]*radixv1.RadixBatch), func(acc map[string]*radixv1.RadixBatch, batch *radixv1.RadixBatch) map[string]*radixv1.RadixBatch { + acc[batch.Name] = batch + return acc + }) for batchName, actualBatchStatus := range batchStatusesMap { assert.Equal(t, tt.batchesArgs.expectedBatchStatuses[batchName], actualBatchStatus.Status, "Status is not as expected for the batch %s", batchName) + assert.Equal(t, radixBatchMap[batchName].Spec.BatchId, actualBatchStatus.BatchId, "Invalid or missing BatchId in the batch %s status", batchName) } }) } @@ -505,14 +513,13 @@ func TestDeleteRadixBatch(t *testing.T) { tests := []deletionTestArgs{ { name: "Radix batch exists", - existingRadixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil), + existingRadixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil), radixBatchToDelete: batchName1, }, { - name: "Radix batch does not exist", - existingRadixBatch: createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil), + name: "Radix batch does not exist, no error", + existingRadixBatch: createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil), radixBatchToDelete: batchName2, - expectedError: errors.New("radixbatches batch2 not found"), }, } for _, tt := range tests { @@ -537,8 +544,8 @@ func TestRestartRadixBatch(t *testing.T) { radixBatchToRestart *radixv1.RadixBatch expectedError error } - radixBatch1 := createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) - radixBatch2 := createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) + radixBatch1 := createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) + radixBatch2 := createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) tests := []deletionTestArgs{ { name: "Radix batch exists", @@ -546,10 +553,10 @@ func TestRestartRadixBatch(t *testing.T) { radixBatchToRestart: radixBatch1, }, { - name: "Radix batch does not exist", + name: "Radix batch does not exist, no error", existingRadixBatch: radixBatch1, radixBatchToRestart: radixBatch2, - expectedError: errors.New("radixbatches batch2 not found"), + expectedError: errors.New("radixbatches.radix.equinor.com \"batch2\" not found"), }, } for _, tt := range tests { @@ -575,8 +582,8 @@ func TestRestartRadixBatchJob(t *testing.T) { expectedError error radixBatchJobToRestart string } - radixBatch1 := createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) - radixBatch2 := createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) + radixBatch1 := createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) + radixBatch2 := createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) tests := []deletionTestArgs{ { name: "Radix batch exists", @@ -589,7 +596,7 @@ func TestRestartRadixBatchJob(t *testing.T) { existingRadixBatch: radixBatch1, radixBatchToRestart: radixBatch2, radixBatchJobToRestart: jobName1, - expectedError: errors.New("radixbatches batch2 not found"), + expectedError: errors.New("radixbatches.radix.equinor.com \"batch2\" not found"), }, { name: "Radix batch job does not exists", @@ -621,8 +628,8 @@ func TestStopRadixBatch(t *testing.T) { radixBatchToStop *radixv1.RadixBatch expectedError error } - radixBatch1 := createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) - radixBatch2 := createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) + radixBatch1 := createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) + radixBatch2 := createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) tests := []deletionTestArgs{ { name: "Radix batch exists", @@ -659,11 +666,11 @@ func TestStopRadixBatchJob(t *testing.T) { expectedError error radixBatchJobToStop string } - radixBatch1 := createRadixBatch(batchName1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, map[string]radixv1.RadixBatchJobPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseRunning}) - radixBatch2 := createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) - radixBatch3 := createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, map[string]radixv1.RadixBatchJobPhase{jobName1: radixv1.BatchJobPhaseStopped, jobName2: radixv1.BatchJobPhaseRunning}) - radixBatch4 := createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, map[string]radixv1.RadixBatchJobPhase{jobName1: radixv1.BatchJobPhaseSucceeded, jobName2: radixv1.BatchJobPhaseRunning}) - radixBatch5 := createRadixBatch(batchName2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, map[string]radixv1.RadixBatchJobPhase{jobName1: radixv1.BatchJobPhaseFailed, jobName2: radixv1.BatchJobPhaseRunning}) + radixBatch1 := createRadixBatch(batchName1, batchId1, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, map[string]radixv1.RadixBatchJobPhase{jobName1: radixv1.BatchJobPhaseWaiting, jobName2: radixv1.BatchJobPhaseRunning}) + radixBatch2 := createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, nil) + radixBatch3 := createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, map[string]radixv1.RadixBatchJobPhase{jobName1: radixv1.BatchJobPhaseStopped, jobName2: radixv1.BatchJobPhaseRunning}) + radixBatch4 := createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, map[string]radixv1.RadixBatchJobPhase{jobName1: radixv1.BatchJobPhaseSucceeded, jobName2: radixv1.BatchJobPhaseRunning}) + radixBatch5 := createRadixBatch(batchName2, batchId2, props, kube.RadixBatchTypeBatch, radixDeploymentName1, []string{jobName1, jobName2}, radixv1.BatchConditionTypeWaiting, map[string]radixv1.RadixBatchJobPhase{jobName1: radixv1.BatchJobPhaseFailed, jobName2: radixv1.BatchJobPhaseRunning}) tests := []deletionTestArgs{ { name: "Radix batch exists", @@ -745,7 +752,7 @@ func createRadixDeployJobComponent(radixDeploymentName string, props testProps, }) } -func createRadixBatch(batchName string, props testProps, radixBatchType kube.RadixBatchType, radixDeploymentName string, jobNames []string, batchStatus radixv1.RadixBatchConditionType, jobStatuses jobStatusPhase) *radixv1.RadixBatch { +func createRadixBatch(batchName, batchId string, props testProps, radixBatchType kube.RadixBatchType, radixDeploymentName string, jobNames []string, batchStatus radixv1.RadixBatchConditionType, jobStatuses jobStatusPhase) *radixv1.RadixBatch { radixBatch := radixv1.RadixBatch{ ObjectMeta: metav1.ObjectMeta{ Name: batchName, @@ -757,6 +764,7 @@ func createRadixBatch(batchName string, props testProps, radixBatchType kube.Rad ), }, Spec: radixv1.RadixBatchSpec{ + BatchId: batchId, RadixDeploymentJobRef: radixv1.RadixDeploymentJobComponentSelector{ LocalObjectReference: radixv1.LocalObjectReference{Name: radixDeploymentName}, Job: props.radixJobComponentName, diff --git a/pkg/batch/history.go b/pkg/batch/history.go new file mode 100644 index 0000000..36636be --- /dev/null +++ b/pkg/batch/history.go @@ -0,0 +1,172 @@ +package batch + +import ( + "context" + "errors" + "sort" + "time" + + "github.com/equinor/radix-common/utils/pointers" + "github.com/equinor/radix-common/utils/slice" + "github.com/equinor/radix-job-scheduler/internal" + "github.com/equinor/radix-job-scheduler/models" + modelsv2 "github.com/equinor/radix-job-scheduler/models/v2" + "github.com/equinor/radix-operator/pkg/apis/kube" + radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" + radixLabels "github.com/equinor/radix-operator/pkg/apis/utils/labels" + "github.com/rs/zerolog/log" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// History Interface for job History +type History interface { + // Cleanup the pipeline job history for the Radix application + Cleanup(ctx context.Context) error +} + +type history struct { + kubeUtil *kube.Kube + env *models.Env + radixDeployJobComponent *radixv1.RadixDeployJobComponent +} + +// NewHistory Constructor for job History +func NewHistory(kubeUtil *kube.Kube, env *models.Env, radixDeployJobComponent *radixv1.RadixDeployJobComponent) History { + return &history{ + kubeUtil: kubeUtil, + env: env, + radixDeployJobComponent: radixDeployJobComponent, + } +} + +// Cleanup the pipeline job history +func (h *history) Cleanup(ctx context.Context) error { + logger := log.Ctx(ctx) + const minimumAge = 3600 // TODO add as default env-var and/or job-component property + completedBefore := time.Now().Add(-time.Second * minimumAge) + completedRadixBatches, err := h.getCompletedRadixBatchesSortedByCompletionTimeAsc(ctx, completedBefore) + if err != nil { + return err + } + + logger.Debug().Msg("cleanup RadixBatch history for succeeded batches") + var errs []error + historyLimit := h.env.RadixJobSchedulersPerEnvironmentHistoryLimit + if err := h.cleanupRadixBatchHistory(ctx, completedRadixBatches.SucceededRadixBatches, historyLimit); err != nil { + errs = append(errs, err) + } + logger.Debug().Msg("cleanup RadixBatch history for not succeeded batches") + if err := h.cleanupRadixBatchHistory(ctx, completedRadixBatches.NotSucceededRadixBatches, historyLimit); err != nil { + errs = append(errs, err) + } + logger.Debug().Msg("cleanup RadixBatch history for succeeded single jobs") + if err := h.cleanupRadixBatchHistory(ctx, completedRadixBatches.SucceededSingleJobs, historyLimit); err != nil { + errs = append(errs, err) + } + logger.Debug().Msg("cleanup RadixBatch history for not succeeded single jobs") + if err := h.cleanupRadixBatchHistory(ctx, completedRadixBatches.NotSucceededSingleJobs, historyLimit); err != nil { + errs = append(errs, err) + } + logger.Debug().Msg("delete orphaned payload secrets") + if err = internal.GarbageCollectPayloadSecrets(ctx, h.kubeUtil, h.env.RadixDeploymentNamespace, h.env.RadixComponentName); err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) +} + +func (h *history) getCompletedRadixBatchesSortedByCompletionTimeAsc(ctx context.Context, completedBefore time.Time) (*CompletedRadixBatches, error) { + radixBatches, err := internal.GetRadixBatches(ctx, h.env.RadixDeploymentNamespace, h.kubeUtil.RadixClient(), radixLabels.ForComponentName(h.env.RadixComponentName)) + if err != nil { + return nil, err + } + radixBatches = sortRJSchByCompletionTimeAsc(radixBatches) + return &CompletedRadixBatches{ + SucceededRadixBatches: h.getSucceededRadixBatches(radixBatches, completedBefore), + NotSucceededRadixBatches: h.getNotSucceededRadixBatches(radixBatches, completedBefore), + SucceededSingleJobs: h.getSucceededSingleJobs(radixBatches, completedBefore), + NotSucceededSingleJobs: h.getNotSucceededSingleJobs(radixBatches, completedBefore), + }, nil +} + +func (h *history) getNotSucceededRadixBatches(radixBatches []*radixv1.RadixBatch, completedBefore time.Time) []*modelsv2.RadixBatch { + return convertToRadixBatchStatuses(slice.FindAll(radixBatches, func(radixBatch *radixv1.RadixBatch) bool { + return radixBatchHasType(radixBatch, kube.RadixBatchTypeBatch) && internal.IsRadixBatchNotSucceeded(radixBatch) && radixBatchIsCompletedBefore(completedBefore, radixBatch) + }), h.radixDeployJobComponent) +} + +func (h *history) getSucceededRadixBatches(radixBatches []*radixv1.RadixBatch, completedBefore time.Time) []*modelsv2.RadixBatch { + radixBatches = slice.FindAll(radixBatches, func(radixBatch *radixv1.RadixBatch) bool { + return radixBatchHasType(radixBatch, kube.RadixBatchTypeBatch) && internal.IsRadixBatchSucceeded(radixBatch) && radixBatchIsCompletedBefore(completedBefore, radixBatch) + }) + return convertToRadixBatchStatuses(radixBatches, h.radixDeployJobComponent) +} + +func radixBatchIsCompletedBefore(completedBefore time.Time, radixBatch *radixv1.RadixBatch) bool { + return radixBatch.Status.Condition.CompletionTime != nil && (*radixBatch.Status.Condition.CompletionTime).Before(&metav1.Time{Time: completedBefore}) +} + +func (h *history) getNotSucceededSingleJobs(radixBatches []*radixv1.RadixBatch, completedBefore time.Time) []*modelsv2.RadixBatch { + return convertToRadixBatchStatuses(slice.FindAll(radixBatches, func(radixBatch *radixv1.RadixBatch) bool { + return radixBatchHasType(radixBatch, kube.RadixBatchTypeJob) && internal.IsRadixBatchNotSucceeded(radixBatch) && radixBatchIsCompletedBefore(completedBefore, radixBatch) + }), h.radixDeployJobComponent) +} + +func (h *history) getSucceededSingleJobs(radixBatches []*radixv1.RadixBatch, completedBefore time.Time) []*modelsv2.RadixBatch { + return convertToRadixBatchStatuses(slice.FindAll(radixBatches, func(radixBatch *radixv1.RadixBatch) bool { + return radixBatchHasType(radixBatch, kube.RadixBatchTypeJob) && internal.IsRadixBatchSucceeded(radixBatch) && radixBatchIsCompletedBefore(completedBefore, radixBatch) + }), h.radixDeployJobComponent) +} + +func radixBatchHasType(radixBatch *radixv1.RadixBatch, radixBatchType kube.RadixBatchType) bool { + return radixBatch.GetLabels()[kube.RadixBatchTypeLabel] == string(radixBatchType) +} + +func (h *history) cleanupRadixBatchHistory(ctx context.Context, radixBatchesSortedByCompletionTimeAsc []*modelsv2.RadixBatch, historyLimit int) error { + logger := log.Ctx(ctx) + numToDelete := len(radixBatchesSortedByCompletionTimeAsc) - historyLimit + if numToDelete <= 0 { + logger.Debug().Msgf("no history batches to delete: %d batches, %d history limit", len(radixBatchesSortedByCompletionTimeAsc), historyLimit) + return nil + } + logger.Debug().Msgf("history batches to delete: %v", numToDelete) + + for i := 0; i < numToDelete; i++ { + radixBatch := radixBatchesSortedByCompletionTimeAsc[i] + logger.Debug().Msgf("deleting batch %s", radixBatch.Name) + if err := DeleteRadixBatchByName(ctx, h.kubeUtil.RadixClient(), h.env.RadixDeploymentNamespace, radixBatch.Name); err != nil { + return err + } + } + return nil +} + +func sortRJSchByCompletionTimeAsc(batches []*radixv1.RadixBatch) []*radixv1.RadixBatch { + sort.Slice(batches, func(i, j int) bool { + batch1 := (batches)[i] + batch2 := (batches)[j] + return isRJS1CompletedBeforeRJS2(batch1, batch2) + }) + return batches +} + +func isRJS1CompletedBeforeRJS2(batch1 *radixv1.RadixBatch, batch2 *radixv1.RadixBatch) bool { + rd1ActiveFrom := getCompletionTimeFrom(batch1) + rd2ActiveFrom := getCompletionTimeFrom(batch2) + + return rd1ActiveFrom.Before(rd2ActiveFrom) +} + +func getCompletionTimeFrom(radixBatch *radixv1.RadixBatch) *metav1.Time { + if radixBatch.Status.Condition.CompletionTime.IsZero() { + return pointers.Ptr(radixBatch.GetCreationTimestamp()) + } + return radixBatch.Status.Condition.CompletionTime +} + +func convertToRadixBatchStatuses(radixBatches []*radixv1.RadixBatch, radixDeployJobComponent *radixv1.RadixDeployJobComponent) []*modelsv2.RadixBatch { + batches := make([]*modelsv2.RadixBatch, 0, len(radixBatches)) + for _, radixBatch := range radixBatches { + batches = append(batches, pointers.Ptr(GetRadixBatchStatus(radixBatch, radixDeployJobComponent))) + } + return batches +} diff --git a/pkg/batch/history_mock.go b/pkg/batch/history_mock.go new file mode 100644 index 0000000..3121fa2 --- /dev/null +++ b/pkg/batch/history_mock.go @@ -0,0 +1,49 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./pkg/batch/history.go + +// Package batch is a generated GoMock package. +package batch + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockHistory is a mock of History interface. +type MockHistory struct { + ctrl *gomock.Controller + recorder *MockHistoryMockRecorder +} + +// MockHistoryMockRecorder is the mock recorder for MockHistory. +type MockHistoryMockRecorder struct { + mock *MockHistory +} + +// NewMockHistory creates a new mock instance. +func NewMockHistory(ctrl *gomock.Controller) *MockHistory { + mock := &MockHistory{ctrl: ctrl} + mock.recorder = &MockHistoryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockHistory) EXPECT() *MockHistoryMockRecorder { + return m.recorder +} + +// Cleanup mocks base method. +func (m *MockHistory) Cleanup(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Cleanup", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Cleanup indicates an expected call of Cleanup. +func (mr *MockHistoryMockRecorder) Cleanup(ctx interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Cleanup", reflect.TypeOf((*MockHistory)(nil).Cleanup), ctx) +} diff --git a/models/notifications/notifier.go b/pkg/notifications/notifier.go similarity index 100% rename from models/notifications/notifier.go rename to pkg/notifications/notifier.go diff --git a/models/notifications/notifier_mock.go b/pkg/notifications/notifier_mock.go similarity index 98% rename from models/notifications/notifier_mock.go rename to pkg/notifications/notifier_mock.go index e1e520b..578b8de 100644 --- a/models/notifications/notifier_mock.go +++ b/pkg/notifications/notifier_mock.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: ./models/notifications/notifier.go +// Source: ./pkg/notifications/notifier.go // Package notifications is a generated GoMock package. package notifications diff --git a/pkg/notifications/webhook_notifier.go b/pkg/notifications/webhook_notifier.go new file mode 100644 index 0000000..80f95fe --- /dev/null +++ b/pkg/notifications/webhook_notifier.go @@ -0,0 +1,168 @@ +package notifications + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-common/utils/pointers" + "github.com/equinor/radix-common/utils/slice" + "github.com/equinor/radix-job-scheduler/models/v1" + "github.com/equinor/radix-job-scheduler/models/v1/events" + "github.com/equinor/radix-job-scheduler/utils/radix/jobs" + "github.com/equinor/radix-operator/pkg/apis/kube" + v2 "k8s.io/apimachinery/pkg/apis/meta/v1" + + radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" +) + +type webhookNotifier struct { + webhookURL string + jobComponentName string + radixDeployJobComponent *radixv1.RadixDeployJobComponent +} + +func NewWebhookNotifier(radixDeployJobComponent *radixv1.RadixDeployJobComponent) Notifier { + notifier := webhookNotifier{ + jobComponentName: radixDeployJobComponent.Name, + radixDeployJobComponent: radixDeployJobComponent, + } + if radixDeployJobComponent.Notifications != nil && webhookIsNotEmpty(radixDeployJobComponent.Notifications.Webhook) { + notifier.webhookURL = *radixDeployJobComponent.Notifications.Webhook + } + return ¬ifier +} + +func (notifier *webhookNotifier) Enabled() bool { + return len(notifier.webhookURL) > 0 +} + +func (notifier *webhookNotifier) String() string { + if notifier.Enabled() { + return fmt.Sprintf("Webhook notifier is enabled. Webhook: %s", notifier.webhookURL) + } + return "Webhook notifier is disabled" +} + +func (notifier *webhookNotifier) Notify(event events.Event, radixBatch *radixv1.RadixBatch, updatedJobStatuses []radixv1.RadixBatchJobStatus, errChan chan error) (done chan struct{}) { + done = make(chan struct{}) + go func() { + defer close(done) + if !notifier.Enabled() || len(notifier.webhookURL) == 0 || radixBatch.Spec.RadixDeploymentJobRef.Job != notifier.jobComponentName { + done <- struct{}{} + return + } + // RadixBatch status and only changed job statuses + batchStatus := getRadixBatchEventFromRadixBatch(event, radixBatch, updatedJobStatuses, notifier.radixDeployJobComponent) + statusesJson, err := json.Marshal(batchStatus) + if err != nil { + errChan <- fmt.Errorf("failed serialize updated JobStatuses %v", err) + return + } + buf := bytes.NewReader(statusesJson) + _, err = http.Post(notifier.webhookURL, "application/json", buf) + if err != nil { + errChan <- fmt.Errorf("failed to notify on RadixBatch object create or change %s: %v", radixBatch.GetName(), err) + return + } + done <- struct{}{} + }() + return done +} + +func webhookIsNotEmpty(webhook *string) bool { + return webhook != nil && len(*webhook) > 0 +} + +func getRadixBatchEventFromRadixBatch(event events.Event, radixBatch *radixv1.RadixBatch, radixBatchJobStatuses []radixv1.RadixBatchJobStatus, radixDeployJobComponent *radixv1.RadixDeployJobComponent) events.BatchEvent { + batchType := radixBatch.Labels[kube.RadixBatchTypeLabel] + startedTime := utils.FormatTime(radixBatch.Status.Condition.ActiveTime) + endedTime := utils.FormatTime(radixBatch.Status.Condition.CompletionTime) + batchStatus := v1.JobStatus{ + Name: radixBatch.GetName(), + BatchId: getBatchId(radixBatch), + Created: utils.FormatTime(pointers.Ptr(radixBatch.GetCreationTimestamp())), + Started: startedTime, + Ended: endedTime, + Status: string(jobs.GetRadixBatchStatus(radixBatch, radixDeployJobComponent)), + Message: radixBatch.Status.Condition.Message, + Updated: utils.FormatTime(pointers.Ptr(v2.Now())), + } + jobStatuses := getRadixBatchJobStatusesFromRadixBatch(radixBatch, radixBatchJobStatuses) + return events.BatchEvent{ + Event: event, + BatchStatus: v1.BatchStatus{ + JobStatus: batchStatus, + JobStatuses: jobStatuses, + BatchType: batchType, + }, + } +} + +func getRadixBatchJobStatusesFromRadixBatch(radixBatch *radixv1.RadixBatch, radixBatchJobStatuses []radixv1.RadixBatchJobStatus) []v1.JobStatus { + batchName := getBatchName(radixBatch) + radixBatchJobsMap := getRadixBatchJobsMap(radixBatch.Spec.Jobs) + jobStatuses := make([]v1.JobStatus, 0, len(radixBatchJobStatuses)) + for _, radixBatchJobStatus := range radixBatchJobStatuses { + radixBatchJob, ok := radixBatchJobsMap[radixBatchJobStatus.Name] + if !ok { + continue + } + stopJob := radixBatchJob.Stop != nil && *radixBatchJob.Stop + jobName := fmt.Sprintf("%s-%s", radixBatch.Name, radixBatchJobStatus.Name) // composed name in models are always consist of a batchName and original jobName + jobStatus := v1.JobStatus{ + BatchName: batchName, + Name: jobName, + JobId: radixBatchJob.JobId, + Created: utils.FormatTime(radixBatchJobStatus.CreationTime), + Started: utils.FormatTime(radixBatchJobStatus.StartTime), + Ended: utils.FormatTime(radixBatchJobStatus.EndTime), + Status: string(jobs.GetScheduledJobStatus(radixBatchJobStatus, stopJob)), + Failed: radixBatchJobStatus.Failed, + Restart: radixBatchJobStatus.Restart, + Message: radixBatchJobStatus.Message, + Updated: utils.FormatTime(pointers.Ptr(v2.Now())), + PodStatuses: getPodStatusByRadixBatchJobPodStatus(radixBatchJobStatus.RadixBatchJobPodStatuses), + } + jobStatuses = append(jobStatuses, jobStatus) + } + return jobStatuses +} + +func getRadixBatchJobsMap(radixBatchJobs []radixv1.RadixBatchJob) map[string]radixv1.RadixBatchJob { + jobMap := make(map[string]radixv1.RadixBatchJob, len(radixBatchJobs)) + for _, radixBatchJob := range radixBatchJobs { + jobMap[radixBatchJob.Name] = radixBatchJob + } + return jobMap +} + +func getPodStatusByRadixBatchJobPodStatus(podStatuses []radixv1.RadixBatchJobPodStatus) []v1.PodStatus { + return slice.Map(podStatuses, func(status radixv1.RadixBatchJobPodStatus) v1.PodStatus { + return v1.PodStatus{ + Name: status.Name, + Created: utils.FormatTime(status.CreationTime), + StartTime: utils.FormatTime(status.StartTime), + EndTime: utils.FormatTime(status.EndTime), + ContainerStarted: utils.FormatTime(status.StartTime), + Status: v1.ReplicaStatus{Status: string(status.Phase)}, + StatusMessage: status.Message, + RestartCount: status.RestartCount, + Image: status.Image, + ImageId: status.ImageID, + PodIndex: status.PodIndex, + ExitCode: status.ExitCode, + Reason: status.Reason, + } + }) +} + +func getBatchName(radixBatch *radixv1.RadixBatch) string { + return utils.TernaryString(radixBatch.GetLabels()[kube.RadixBatchTypeLabel] == string(kube.RadixBatchTypeJob), "", radixBatch.GetName()) +} + +func getBatchId(radixBatch *radixv1.RadixBatch) string { + return utils.TernaryString(radixBatch.GetLabels()[kube.RadixBatchTypeLabel] == string(kube.RadixBatchTypeJob), "", radixBatch.Spec.BatchId) +} diff --git a/models/notifications/webhook_notifier_test.go b/pkg/notifications/webhook_notifier_test.go similarity index 100% rename from models/notifications/webhook_notifier_test.go rename to pkg/notifications/webhook_notifier_test.go diff --git a/models/notifications/radix-batch-watcher.go b/pkg/watcher/watcher.go similarity index 51% rename from models/notifications/radix-batch-watcher.go rename to pkg/watcher/watcher.go index 3954f18..ceadabe 100644 --- a/models/notifications/radix-batch-watcher.go +++ b/pkg/watcher/watcher.go @@ -1,16 +1,13 @@ -package notifications +package watcher import ( "context" "fmt" + "time" - "github.com/equinor/radix-common/utils" - "github.com/equinor/radix-common/utils/pointers" - "github.com/equinor/radix-common/utils/slice" - modelsv1 "github.com/equinor/radix-job-scheduler/models/v1" "github.com/equinor/radix-job-scheduler/models/v1/events" - "github.com/equinor/radix-job-scheduler/utils/radix/jobs" - "github.com/equinor/radix-operator/pkg/apis/kube" + "github.com/equinor/radix-job-scheduler/pkg/batch" + "github.com/equinor/radix-job-scheduler/pkg/notifications" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned" radixinformers "github.com/equinor/radix-operator/pkg/client/informers/externalversions" @@ -25,26 +22,31 @@ const ( resyncPeriod = 0 ) -type Watcher struct { +// Watcher Watcher interface +type Watcher interface { + Stop() +} + +type watcher struct { radixInformerFactory radixinformers.SharedInformerFactory batchInformer v1.RadixBatchInformer - Stop chan struct{} + stop chan struct{} logger zerolog.Logger + jobHistory batch.History } -// NullRadixBatchWatcher The void watcher -func NullRadixBatchWatcher() *Watcher { - return &Watcher{ - Stop: make(chan struct{}), - } +// Stop Stops the watcher +func (w *watcher) Stop() { + w.stop <- struct{}{} } // NewRadixBatchWatcher New RadixBatch watcher, notifying on adding and changing of RadixBatches and their jobs -func NewRadixBatchWatcher(radixClient radixclient.Interface, namespace string, notifier Notifier) (*Watcher, error) { - watcher := Watcher{ - Stop: make(chan struct{}), +func NewRadixBatchWatcher(ctx context.Context, radixClient radixclient.Interface, namespace string, jobHistory batch.History, notifier notifications.Notifier) (Watcher, error) { + watcher := watcher{ + stop: make(chan struct{}), radixInformerFactory: radixinformers.NewSharedInformerFactoryWithOptions(radixClient, resyncPeriod, radixinformers.WithNamespace(namespace)), logger: log.Logger.With().Str("pkg", "radix-batch-watcher").Logger(), + jobHistory: jobHistory, } existingRadixBatchMap, err := getRadixBatchMap(radixClient, namespace) @@ -58,17 +60,25 @@ func NewRadixBatchWatcher(radixClient radixclient.Interface, namespace string, n errChan := make(chan error) _, err = watcher.batchInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(cur interface{}) { - newRadixBatch := cur.(*radixv1.RadixBatch) - if _, ok := existingRadixBatchMap[newRadixBatch.GetName()]; ok { - watcher.logger.Debug().Msgf("skip existing RadixBatch object %s", newRadixBatch.GetName()) + radixBatch, converted := cur.(*radixv1.RadixBatch) + if !converted { + log.Error().Msg("Failed to cast RadixBatch object") return } - watcher.logger.Debug().Msgf("RadixBatch object was added %s", newRadixBatch.GetName()) - jobStatuses := newRadixBatch.Status.JobStatuses + if radixBatch.Status.Condition.Type != "" { + return // skip existing batch added to the cache + } + if _, ok := existingRadixBatchMap[radixBatch.GetName()]; ok { + watcher.logger.Debug().Msgf("skip existing RadixBatch object %s", radixBatch.GetName()) + return + } + watcher.logger.Debug().Msgf("RadixBatch object was added %s", radixBatch.GetName()) + jobStatuses := radixBatch.Status.JobStatuses if len(jobStatuses) == 0 { jobStatuses = make([]radixv1.RadixBatchJobStatus, 0) } - notifier.Notify(events.Create, newRadixBatch, jobStatuses, errChan) + notifier.Notify(events.Create, radixBatch, jobStatuses, errChan) + watcher.cleanupJobHistory(ctx) }, UpdateFunc: func(old, cur interface{}) { oldRadixBatch := old.(*radixv1.RadixBatch) @@ -102,14 +112,17 @@ func NewRadixBatchWatcher(radixClient radixclient.Interface, namespace string, n return nil, err } - watcher.radixInformerFactory.Start(watcher.Stop) + watcher.radixInformerFactory.Start(watcher.stop) + log.Info().Msg("Waiting for Radix objects caches to sync") + watcher.radixInformerFactory.WaitForCacheSync(ctx.Done()) + log.Info().Msg("Completed syncing informer caches") go func() { for { select { case err := <-errChan: watcher.logger.Error().Err(err).Msg("Notification failed") - case <-watcher.Stop: + case <-watcher.stop: return } } @@ -117,6 +130,16 @@ func NewRadixBatchWatcher(radixClient radixclient.Interface, namespace string, n return &watcher, nil } +func (w *watcher) cleanupJobHistory(ctx context.Context) { + ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Minute*5) + go func() { + defer cancel() + if err := w.jobHistory.Cleanup(ctxWithTimeout); err != nil { + log.Ctx(ctx).Error().Err(err).Msg("Failed to cleanup job history") + } + }() +} + func getUpdatedJobStatuses(oldRadixBatch *radixv1.RadixBatch, newRadixBatch *radixv1.RadixBatch) []radixv1.RadixBatchJobStatus { oldJobStatuses := make(map[string]radixv1.RadixBatchJobStatus) for _, jobStatus := range oldRadixBatch.Status.JobStatuses { @@ -158,85 +181,3 @@ func getRadixBatchMap(radixClient radixclient.Interface, namespace string) (map[ } return radixBatchMap, nil } - -func (notifier *webhookNotifier) getRadixBatchEventFromRadixBatch(event events.Event, radixBatch *radixv1.RadixBatch, radixBatchJobStatuses []radixv1.RadixBatchJobStatus) events.BatchEvent { - batchType := radixBatch.Labels[kube.RadixBatchTypeLabel] - jobStatusBatchName := utils.TernaryString(batchType == string(kube.RadixBatchTypeJob), "", radixBatch.GetName()) - startedTime := utils.FormatTime(radixBatch.Status.Condition.ActiveTime) - endedTime := utils.FormatTime(radixBatch.Status.Condition.CompletionTime) - batchStatus := modelsv1.JobStatus{ - Name: radixBatch.GetName(), - Created: utils.FormatTime(pointers.Ptr(radixBatch.GetCreationTimestamp())), - Started: startedTime, - Ended: endedTime, - Status: string(jobs.GetRadixBatchStatus(radixBatch, notifier.radixDeployJobComponent)), - Message: radixBatch.Status.Condition.Message, - Updated: utils.FormatTime(pointers.Ptr(metav1.Now())), - } - jobStatuses := getRadixBatchJobStatusesFromRadixBatch(radixBatch, radixBatchJobStatuses, jobStatusBatchName) - return events.BatchEvent{ - Event: event, - BatchStatus: modelsv1.BatchStatus{ - JobStatus: batchStatus, - JobStatuses: jobStatuses, - BatchType: batchType, - }, - } -} - -func getRadixBatchJobStatusesFromRadixBatch(radixBatch *radixv1.RadixBatch, radixBatchJobStatuses []radixv1.RadixBatchJobStatus, jobStatusBatchName string) []modelsv1.JobStatus { - radixBatchJobsMap := getRadixBatchJobsMap(radixBatch.Spec.Jobs) - jobStatuses := make([]modelsv1.JobStatus, 0, len(radixBatchJobStatuses)) - for _, radixBatchJobStatus := range radixBatchJobStatuses { - radixBatchJob, ok := radixBatchJobsMap[radixBatchJobStatus.Name] - if !ok { - continue - } - stopJob := radixBatchJob.Stop != nil && *radixBatchJob.Stop - jobName := fmt.Sprintf("%s-%s", radixBatch.Name, radixBatchJobStatus.Name) // composed name in models are always consist of a batchName and original jobName - jobStatus := modelsv1.JobStatus{ - BatchName: jobStatusBatchName, - Name: jobName, - JobId: radixBatchJob.JobId, - Created: utils.FormatTime(radixBatchJobStatus.CreationTime), - Started: utils.FormatTime(radixBatchJobStatus.StartTime), - Ended: utils.FormatTime(radixBatchJobStatus.EndTime), - Status: string(jobs.GetScheduledJobStatus(radixBatchJobStatus, stopJob)), - Failed: radixBatchJobStatus.Failed, - Restart: radixBatchJobStatus.Restart, - Message: radixBatchJobStatus.Message, - Updated: utils.FormatTime(pointers.Ptr(metav1.Now())), - PodStatuses: getPodStatusByRadixBatchJobPodStatus(radixBatchJobStatus.RadixBatchJobPodStatuses), - } - jobStatuses = append(jobStatuses, jobStatus) - } - return jobStatuses -} - -func getRadixBatchJobsMap(radixBatchJobs []radixv1.RadixBatchJob) map[string]radixv1.RadixBatchJob { - jobMap := make(map[string]radixv1.RadixBatchJob, len(radixBatchJobs)) - for _, radixBatchJob := range radixBatchJobs { - jobMap[radixBatchJob.Name] = radixBatchJob - } - return jobMap -} - -func getPodStatusByRadixBatchJobPodStatus(podStatuses []radixv1.RadixBatchJobPodStatus) []modelsv1.PodStatus { - return slice.Map(podStatuses, func(status radixv1.RadixBatchJobPodStatus) modelsv1.PodStatus { - return modelsv1.PodStatus{ - Name: status.Name, - Created: utils.FormatTime(status.CreationTime), - StartTime: utils.FormatTime(status.StartTime), - EndTime: utils.FormatTime(status.EndTime), - ContainerStarted: utils.FormatTime(status.StartTime), - Status: modelsv1.ReplicaStatus{Status: string(status.Phase)}, - StatusMessage: status.Message, - RestartCount: status.RestartCount, - Image: status.Image, - ImageId: status.ImageID, - PodIndex: status.PodIndex, - ExitCode: status.ExitCode, - Reason: status.Reason, - } - }) -} diff --git a/models/notifications/radix-batch-watcher_test.go b/pkg/watcher/watcher_test.go similarity index 88% rename from models/notifications/radix-batch-watcher_test.go rename to pkg/watcher/watcher_test.go index 99cadfd..c373a00 100644 --- a/models/notifications/radix-batch-watcher_test.go +++ b/pkg/watcher/watcher_test.go @@ -1,12 +1,14 @@ -package notifications +package watcher import ( "context" - "github.com/equinor/radix-job-scheduler/models/v1/events" "testing" "time" commonUtils "github.com/equinor/radix-common/utils" + "github.com/equinor/radix-job-scheduler/models/v1/events" + "github.com/equinor/radix-job-scheduler/pkg/batch" + notifications2 "github.com/equinor/radix-job-scheduler/pkg/notifications" "github.com/equinor/radix-operator/pkg/apis/kube" radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1" "github.com/equinor/radix-operator/pkg/apis/utils/labels" @@ -29,7 +31,7 @@ func Test_RadixBatchWatcher(t *testing.T) { event events.Event } type args struct { - getNotifier func(*gomock.Controller) Notifier + getNotifier func(*gomock.Controller) notifications2.Notifier } tests := []struct { name string @@ -42,8 +44,8 @@ func Test_RadixBatchWatcher(t *testing.T) { newRadixBatch: nil, }, args: args{ - getNotifier: func(ctrl *gomock.Controller) Notifier { - return NewMockNotifier(ctrl) + getNotifier: func(ctrl *gomock.Controller) notifications2.Notifier { + return notifications2.NewMockNotifier(ctrl) }, }, }, @@ -60,8 +62,8 @@ func Test_RadixBatchWatcher(t *testing.T) { event: events.Create, }, args: args{ - getNotifier: func(ctrl *gomock.Controller) Notifier { - notifier := NewMockNotifier(ctrl) + getNotifier: func(ctrl *gomock.Controller) notifications2.Notifier { + notifier := notifications2.NewMockNotifier(ctrl) rbMatcher := newRadixBatchMatcher(func(radixBatch *radixv1.RadixBatch) bool { return radixBatch.Name == "batch1" && radixBatch.Status.Condition == radixv1.RadixBatchCondition{} }) @@ -87,8 +89,8 @@ func Test_RadixBatchWatcher(t *testing.T) { event: events.Update, }, args: args{ - getNotifier: func(ctrl *gomock.Controller) Notifier { - notifier := NewMockNotifier(ctrl) + getNotifier: func(ctrl *gomock.Controller) notifications2.Notifier { + notifier := notifications2.NewMockNotifier(ctrl) rbMatcher := newRadixBatchMatcher(func(radixBatch *radixv1.RadixBatch) bool { return radixBatch.Name == "batch1" && radixBatch.Status.Condition.Type == radixv1.BatchConditionTypeWaiting @@ -141,8 +143,8 @@ func Test_RadixBatchWatcher(t *testing.T) { event: events.Update, }, args: args{ - getNotifier: func(ctrl *gomock.Controller) Notifier { - notifier := NewMockNotifier(ctrl) + getNotifier: func(ctrl *gomock.Controller) notifications2.Notifier { + notifier := notifications2.NewMockNotifier(ctrl) rbMatcher := newRadixBatchMatcher(func(radixBatch *radixv1.RadixBatch) bool { return radixBatch.Name == "batch1" && *radixBatch.Status.Condition.ActiveTime == activeTime }) @@ -173,8 +175,8 @@ func Test_RadixBatchWatcher(t *testing.T) { event: events.Delete, }, args: args{ - getNotifier: func(ctrl *gomock.Controller) Notifier { - notifier := NewMockNotifier(ctrl) + getNotifier: func(ctrl *gomock.Controller) notifications2.Notifier { + notifier := notifications2.NewMockNotifier(ctrl) rbMatcher := newRadixBatchMatcher(func(radixBatch *radixv1.RadixBatch) bool { return radixBatch.Name == "batch1" }) @@ -204,19 +206,17 @@ func Test_RadixBatchWatcher(t *testing.T) { } ctrl := gomock.NewController(t) - watcher, err := NewRadixBatchWatcher(radixClient, namespace, tt.args.getNotifier(ctrl)) + history := batch.NewMockHistory(ctrl) + batchWatcher, err := NewRadixBatchWatcher(context.Background(), radixClient, namespace, history, tt.args.getNotifier(ctrl)) + defer batchWatcher.Stop() if err != nil { assert.Fail(t, err.Error()) - watcher.Stop <- struct{}{} return } - assert.False(t, commonUtils.IsNil(watcher)) - for !watcher.batchInformer.Informer().HasSynced() { - t.Log("wait for sync informer") - time.Sleep(time.Millisecond * 100) - } + assert.False(t, commonUtils.IsNil(batchWatcher)) if tt.fields.newRadixBatch != nil && tt.fields.event == events.Create { + history.EXPECT().Cleanup(gomock.Any()).Times(1) // when radix batch exists and during test it will be updated _, err := radixClient.RadixV1().RadixBatches(namespace).Create(context.TODO(), tt.fields.newRadixBatch, metav1.CreateOptions{}) if err != nil { @@ -239,13 +239,8 @@ func Test_RadixBatchWatcher(t *testing.T) { } } } - for !watcher.batchInformer.Informer().HasSynced() { - t.Log("wait for sync informer") - time.Sleep(time.Millisecond * 100) - } time.Sleep(time.Second * 1) // wait to possible fail due to a missed expected call ctrl.Finish() - watcher.Stop <- struct{}{} }) } } diff --git a/swaggerui/html/swagger.json b/swaggerui/html/swagger.json index f88397d..9df3f24 100644 --- a/swaggerui/html/swagger.json +++ b/swaggerui/html/swagger.json @@ -523,6 +523,12 @@ "description": "DeploymentName for this batch", "type": "string" }, + "batchId": { + "description": "Defines a user defined ID of the batch.", + "type": "string", + "x-go-name": "BatchId", + "example": "'batch-id-1'" + }, "batchName": { "description": "BatchName Optional Batch ID of a job", "type": "string", @@ -633,6 +639,12 @@ "jobScheduleDescriptions" ], "properties": { + "batchId": { + "description": "Defines a user defined ID of the batch.", + "type": "string", + "x-go-name": "BatchId", + "example": "'batch-id-1'" + }, "defaultRadixJobComponentConfig": { "$ref": "#/definitions/RadixJobComponentConfig" }, @@ -659,6 +671,12 @@ "description": "DeploymentName for this batch", "type": "string" }, + "batchId": { + "description": "Defines a user defined ID of the batch.", + "type": "string", + "x-go-name": "BatchId", + "example": "'batch-id-1'" + }, "batchName": { "description": "BatchName Optional Batch ID of a job", "type": "string", @@ -819,6 +837,12 @@ "description": "DeploymentName for this batch", "type": "string" }, + "batchId": { + "description": "Defines a user defined ID of the batch.", + "type": "string", + "x-go-name": "BatchId", + "example": "'batch-id-1'" + }, "batchName": { "description": "BatchName Optional Batch ID of a job", "type": "string",