From 967b7024fd3826267242c13676bbd1f852ebf31f Mon Sep 17 00:00:00 2001 From: Joseph Oladeji Date: Mon, 26 Aug 2024 02:14:03 -0400 Subject: [PATCH] feat: Adds e2e tests for chain/schedule policy and bump ScheduledAutoscaler to Alpha (#3946) * setup e2e tests for chain/schedule policies * add default fixtures for schedule & chain fas * Adds e2e tests for chain and schedule policies * Merge main * Revert "Merge main" This reverts commit 2131907626c8d22596818314059fc9918ccf1c80. * Bump ScheduledAutoscaler feature from Dev to Alpha * Make feature gate inverse in cloudbuild.yaml * Fix feature gates ordering and address duration error * Remove den-install feature gate Co-authored-by: igooch * Remove den-install feature gate Co-authored-by: igooch * Update test/e2e/fleetautoscaler_test.go Co-authored-by: igooch * Regenerate install.yaml * Move ScheduledAutoscaler feature stage into 1.43 version --------- Co-authored-by: Mengye (Max) Gong <8364575+gongmax@users.noreply.github.com> Co-authored-by: igooch --- build/Makefile | 2 +- cloudbuild.yaml | 2 +- install/helm/agones/defaultfeaturegates.yaml | 2 +- pkg/fleetautoscalers/fleetautoscalers_test.go | 2 +- pkg/util/runtime/features.go | 12 +- site/content/en/docs/Guides/feature-stages.md | 1 + test/e2e/fleetautoscaler_test.go | 252 ++++++++++++++++++ 7 files changed, 263 insertions(+), 10 deletions(-) diff --git a/build/Makefile b/build/Makefile index 59448844af..0575639e75 100644 --- a/build/Makefile +++ b/build/Makefile @@ -70,7 +70,7 @@ GS_TEST_IMAGE ?= us-docker.pkg.dev/agones-images/examples/simple-game-server:0.3 BETA_FEATURE_GATES ?= "AutopilotPassthroughPort=true&CountsAndLists=true&DisableResyncOnSDKServer=true" # Enable all alpha feature gates. Keep in sync with `false` (alpha) entries in pkg/util/runtime/features.go:featureDefaults -ALPHA_FEATURE_GATES ?= "PlayerAllocationFilter=true&PlayerTracking=true&RollingUpdateFix=true&PortRanges=true&PortPolicyNone=true&Example=true" +ALPHA_FEATURE_GATES ?= "PlayerAllocationFilter=true&PlayerTracking=true&RollingUpdateFix=true&PortRanges=true&PortPolicyNone=true&ScheduledAutoscaler=true&Example=true" # Build with Windows support WITH_WINDOWS=1 diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 73fe392ea8..472dab5552 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -274,7 +274,7 @@ steps: declare -A versionsAndRegions=( [1.28]=us-west1 [1.29]=europe-west1 [1.30]=asia-east1 ) # Keep in sync with (the inverse of) pkg/util/runtime/features.go:featureDefaults - featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&CountsAndLists=false&RollingUpdateFix=true&PortRanges=true&PortPolicyNone=true&DisableResyncOnSDKServer=false&AutopilotPassthroughPort=false&Example=true" + featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&CountsAndLists=false&RollingUpdateFix=true&PortRanges=true&PortPolicyNone=true&ScheduledAutoscaler=true&DisableResyncOnSDKServer=false&AutopilotPassthroughPort=false&Example=true" featureWithoutGate="" # Use this if specific feature gates can only be supported on specific Kubernetes versions. diff --git a/install/helm/agones/defaultfeaturegates.yaml b/install/helm/agones/defaultfeaturegates.yaml index 38ab7703ad..bb7fe64485 100644 --- a/install/helm/agones/defaultfeaturegates.yaml +++ b/install/helm/agones/defaultfeaturegates.yaml @@ -26,9 +26,9 @@ PlayerTracking: false RollingUpdateFix: false PortRanges: false PortPolicyNone: false +ScheduledAutoscaler: false # Dev features -ScheduledAutoscaler: false # Example feature Example: false diff --git a/pkg/fleetautoscalers/fleetautoscalers_test.go b/pkg/fleetautoscalers/fleetautoscalers_test.go index 2104d2fc27..cdb07f9d7c 100644 --- a/pkg/fleetautoscalers/fleetautoscalers_test.go +++ b/pkg/fleetautoscalers/fleetautoscalers_test.go @@ -2538,7 +2538,7 @@ func TestApplyChainPolicy(t *testing.T) { // Parse a time string and return a metav1.Time func mustParseMetav1Time(timeStr string) metav1.Time { t, _ := time.Parse(time.RFC3339, timeStr) - return metav1.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), t.Location()) + return metav1.NewTime(t) } // Parse a time string and return a time.Time diff --git a/pkg/util/runtime/features.go b/pkg/util/runtime/features.go index b32551b74a..473e55a871 100644 --- a/pkg/util/runtime/features.go +++ b/pkg/util/runtime/features.go @@ -55,21 +55,21 @@ const ( // FeaturePlayerTracking is a feature flag to enable/disable player tracking features. FeaturePlayerTracking Feature = "PlayerTracking" - // FeatureRollingUpdateFix is a feature flag to enable/disable fleet controller fixes. - FeatureRollingUpdateFix Feature = "RollingUpdateFix" - // FeaturePortRanges is a feature flag to enable/disable specific port ranges. FeaturePortRanges Feature = "PortRanges" // FeaturePortPolicyNone is a feature flag to allow setting Port Policy to None. FeaturePortPolicyNone Feature = "PortPolicyNone" - //////////////// - // Dev features + // FeatureRollingUpdateFix is a feature flag to enable/disable fleet controller fixes. + FeatureRollingUpdateFix Feature = "RollingUpdateFix" // FeatureScheduledAutoscaler is a feature flag to enable/disable scheduled fleet autoscaling. FeatureScheduledAutoscaler Feature = "ScheduledAutoscaler" + //////////////// + // Dev features + //////////////// // Example feature @@ -139,9 +139,9 @@ var ( FeatureRollingUpdateFix: false, FeaturePortRanges: false, FeaturePortPolicyNone: false, + FeatureScheduledAutoscaler: false, // Dev features - FeatureScheduledAutoscaler: false, // Example feature FeatureExample: false, diff --git a/site/content/en/docs/Guides/feature-stages.md b/site/content/en/docs/Guides/feature-stages.md index e08823cef9..f5ae27fca7 100644 --- a/site/content/en/docs/Guides/feature-stages.md +++ b/site/content/en/docs/Guides/feature-stages.md @@ -54,6 +54,7 @@ The current set of `alpha` and `beta` feature gates: | [Rolling Update Fixes](https://github.com/googleforgames/agones/issues/3688) | `RollingUpdateFix` | Disabled | `Alpha` | 1.41.0 | | [Multiple dynamic port ranges](https://github.com/googleforgames/agones/issues/1911) | `PortRanges` | Disabled | `Alpha` | 1.41.0 | | [Port Policy None](https://github.com/googleforgames/agones/issues/3804) | `PortPolicyNone` | Disabled | `Alpha` | 1.41.0 | +| [Scheduled Fleet Autoscaling](https://github.com/googleforgames/agones/issues/3008) | `ScheduledAutoscaler` | Disabled | `Alpha` | 1.43.0 | | Example Gate (not in use) | `Example` | Disabled | None | 0.13.0 | {{% /feature %}} diff --git a/test/e2e/fleetautoscaler_test.go b/test/e2e/fleetautoscaler_test.go index 85a4338efd..78ea0fa2cd 100644 --- a/test/e2e/fleetautoscaler_test.go +++ b/test/e2e/fleetautoscaler_test.go @@ -1445,3 +1445,255 @@ func TestListAutoscalerWithSDKMethods(t *testing.T) { }) } } + +func TestScheduleAutoscaler(t *testing.T) { + if !runtime.FeatureEnabled(runtime.FeatureScheduledAutoscaler) { + t.SkipNow() + } + t.Parallel() + ctx := context.Background() + log := e2e.TestLogger(t) + + stable := framework.AgonesClient.AgonesV1() + fleets := stable.Fleets(framework.Namespace) + flt, err := fleets.Create(ctx, defaultFleet(framework.Namespace), metav1.CreateOptions{}) + if assert.NoError(t, err) { + defer fleets.Delete(context.Background(), flt.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck + } + + framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(flt.Spec.Replicas)) + + fleetautoscalers := framework.AgonesClient.AutoscalingV1().FleetAutoscalers(framework.Namespace) + + // Active Cron Schedule (e.g. run after 1 * * * *, which is the after the first minute of the hour) + scheduleAutoscaler := defaultAutoscalerSchedule(t, flt) + scheduleAutoscaler.Spec.Policy.Schedule.ActivePeriod.StartCron = nextCronMinute(time.Now()) + fas, err := fleetautoscalers.Create(ctx, scheduleAutoscaler, metav1.CreateOptions{}) + assert.NoError(t, err) + + framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(5)) + fleetautoscalers.Delete(ctx, fas.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck + + // Return to starting 3 replicas + framework.ScaleFleet(t, log, flt, 3) + framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(3)) + + // Between Active Period Cron Schedule (e.g. run between 1-2 * * * *, which is between the first minute and second minute of the hour) + scheduleAutoscaler = defaultAutoscalerSchedule(t, flt) + scheduleAutoscaler.Spec.Policy.Schedule.ActivePeriod.StartCron = nextCronMinuteBetween(time.Now()) + fas, err = fleetautoscalers.Create(ctx, scheduleAutoscaler, metav1.CreateOptions{}) + assert.NoError(t, err) + + framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(5)) + fleetautoscalers.Delete(ctx, fas.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck +} + +func TestChainAutoscaler(t *testing.T) { + if !runtime.FeatureEnabled(runtime.FeatureScheduledAutoscaler) { + t.SkipNow() + } + t.Parallel() + ctx := context.Background() + log := e2e.TestLogger(t) + + stable := framework.AgonesClient.AgonesV1() + fleets := stable.Fleets(framework.Namespace) + flt, err := fleets.Create(ctx, defaultFleet(framework.Namespace), metav1.CreateOptions{}) + if assert.NoError(t, err) { + defer fleets.Delete(context.Background(), flt.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck + } + + framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(flt.Spec.Replicas)) + + fleetautoscalers := framework.AgonesClient.AutoscalingV1().FleetAutoscalers(framework.Namespace) + + // 1st Schedule Inactive, 2nd Schedule Active - 30 seconds (Fallthrough) + chainAutoscaler := defaultAutoscalerChain(t, flt) + fas, err := fleetautoscalers.Create(ctx, chainAutoscaler, metav1.CreateOptions{}) + assert.NoError(t, err) + + // Verify only the second schedule ran + framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(4)) + fleetautoscalers.Delete(ctx, fas.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck + + // Return to starting 3 replicas + framework.ScaleFleet(t, log, flt, 3) + framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(3)) + + // 2 Active Schedules back to back - 1 minute (Fallthrough) + chainAutoscaler = defaultAutoscalerChain(t, flt) + currentTime := time.Now() + + // First schedule runs for 1 minute + chainAutoscaler.Spec.Policy.Chain[0].Schedule.ActivePeriod.StartCron = nextCronMinute(currentTime) + chainAutoscaler.Spec.Policy.Chain[0].Schedule.ActivePeriod.Duration = "1m" + + // Second schedule runs 1 minute after the first schedule + oneMinute := mustParseDuration(t, "1m") + chainAutoscaler.Spec.Policy.Chain[0].Schedule.ActivePeriod.StartCron = nextCronMinute(currentTime.Add(oneMinute)) + chainAutoscaler.Spec.Policy.Chain[1].Schedule.ActivePeriod.Duration = "5m" + + fas, err = fleetautoscalers.Create(ctx, chainAutoscaler, metav1.CreateOptions{}) + assert.NoError(t, err) + + // Verify the first schedule has been applied + framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(10)) + // Verify the second schedule has been applied + framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(4)) + + fleetautoscalers.Delete(ctx, fas.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck +} + +// defaultAutoscalerSchedule returns a default scheduled autoscaler for testing. +func defaultAutoscalerSchedule(t *testing.T, f *agonesv1.Fleet) *autoscalingv1.FleetAutoscaler { + return &autoscalingv1.FleetAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.ObjectMeta.Name + "-scheduled-autoscaler", + Namespace: framework.Namespace, + }, + Spec: autoscalingv1.FleetAutoscalerSpec{ + FleetName: f.ObjectMeta.Name, + Policy: autoscalingv1.FleetAutoscalerPolicy{ + Type: autoscalingv1.SchedulePolicyType, + Schedule: &autoscalingv1.SchedulePolicy{ + Between: autoscalingv1.Between{ + Start: currentTimePlusDuration(t, "1s"), + End: currentTimePlusDuration(t, "1m"), + }, + ActivePeriod: autoscalingv1.ActivePeriod{ + Timezone: "UTC", + StartCron: "* * * * *", + Duration: "", + }, + Policy: autoscalingv1.FleetAutoscalerPolicy{ + Type: autoscalingv1.BufferPolicyType, + Buffer: &autoscalingv1.BufferPolicy{ + BufferSize: intstr.FromInt(5), + MinReplicas: 5, + MaxReplicas: 12, + }, + }, + }, + }, + Sync: &autoscalingv1.FleetAutoscalerSync{ + Type: autoscalingv1.FixedIntervalSyncType, + FixedInterval: autoscalingv1.FixedIntervalSync{ + Seconds: 5, + }, + }, + }, + } +} + +// defaultAutoscalerChain returns a default chain autoscaler for testing. +func defaultAutoscalerChain(t *testing.T, f *agonesv1.Fleet) *autoscalingv1.FleetAutoscaler { + return &autoscalingv1.FleetAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.ObjectMeta.Name + "-chain-autoscaler", + Namespace: framework.Namespace, + }, + Spec: autoscalingv1.FleetAutoscalerSpec{ + FleetName: f.ObjectMeta.Name, + Policy: autoscalingv1.FleetAutoscalerPolicy{ + Type: autoscalingv1.ChainPolicyType, + Chain: autoscalingv1.ChainPolicy{ + { + ID: "schedule-1", + FleetAutoscalerPolicy: autoscalingv1.FleetAutoscalerPolicy{ + Type: autoscalingv1.SchedulePolicyType, + Schedule: &autoscalingv1.SchedulePolicy{ + Between: autoscalingv1.Between{ + Start: currentTimePlusDuration(t, "1s"), + End: currentTimePlusDuration(t, "2m"), + }, + ActivePeriod: autoscalingv1.ActivePeriod{ + Timezone: "", + StartCron: inactiveCronSchedule(time.Now()), + Duration: "1m", + }, + Policy: autoscalingv1.FleetAutoscalerPolicy{ + Type: autoscalingv1.BufferPolicyType, + Buffer: &autoscalingv1.BufferPolicy{ + BufferSize: intstr.FromInt(10), + MinReplicas: 10, + MaxReplicas: 20, + }, + }, + }, + }, + }, + { + ID: "schedule-2", + FleetAutoscalerPolicy: autoscalingv1.FleetAutoscalerPolicy{ + Type: autoscalingv1.SchedulePolicyType, + Schedule: &autoscalingv1.SchedulePolicy{ + Between: autoscalingv1.Between{ + Start: currentTimePlusDuration(t, "1s"), + End: currentTimePlusDuration(t, "5m"), + }, + ActivePeriod: autoscalingv1.ActivePeriod{ + Timezone: "", + StartCron: nextCronMinute(time.Now()), + Duration: "", + }, + Policy: autoscalingv1.FleetAutoscalerPolicy{ + Type: autoscalingv1.BufferPolicyType, + Buffer: &autoscalingv1.BufferPolicy{ + BufferSize: intstr.FromInt(4), + MinReplicas: 3, + MaxReplicas: 7, + }, + }, + }, + }, + }, + }, + }, + Sync: &autoscalingv1.FleetAutoscalerSync{ + Type: autoscalingv1.FixedIntervalSyncType, + FixedInterval: autoscalingv1.FixedIntervalSync{ + Seconds: 5, + }, + }, + }, + } +} + +// inactiveCronSchedule returns the time 3 minutes ago +// e.g. if the current time is 12:00, this method will return "57 * * * *" +// meaning 3 minutes before 12:00 +func inactiveCronSchedule(currentTime time.Time) string { + prevMinute := currentTime.Add(time.Minute * -3).Minute() + return fmt.Sprintf("%d * * * *", prevMinute) +} + +// nextCronMinute returns the very next minute in +// e.g. if the current time is 12:00, this method will return "1 * * * *" +// meaning after 12:01 +func nextCronMinute(currentTime time.Time) string { + nextMinute := currentTime.Add(time.Minute).Minute() + return fmt.Sprintf("%d * * * *", nextMinute) +} + +// nextCronMinuteBetween returns the minute between the very next minute +// e.g. if the current time is 12:00, this method will return "1-2 * * * *" +// meaning between 12:01 - 12:02 +func nextCronMinuteBetween(currentTime time.Time) string { + nextMinute := currentTime.Add(time.Minute).Minute() + secondMinute := currentTime.Add(2 * time.Minute).Minute() + return fmt.Sprintf("%d-%d * * * *", nextMinute, secondMinute) +} + +// Parse a duration string and return a duration struct +func mustParseDuration(t *testing.T, duration string) time.Duration { + d, err := time.ParseDuration(duration) + assert.Nil(t, err) + return d +} + +// Parse a time string and return a metav1.Time +func currentTimePlusDuration(t *testing.T, duration string) metav1.Time { + d := mustParseDuration(t, duration) + currentTimePlusDuration := time.Now().Add(d) + return metav1.NewTime(currentTimePlusDuration) +}