feat: Adds e2e tests for chain/schedule policy and bump ScheduledAuto…

…scaler to Alpha (#3946) * setup e2e tests for chain/schedule policies * add default fixtures for schedule & chain fas * Adds e2e tests for chain and schedule policies * Merge main * Revert "Merge main" This reverts commit 2131907. * Bump ScheduledAutoscaler feature from Dev to Alpha * Make feature gate inverse in cloudbuild.yaml * Fix feature gates ordering and address duration error * Remove den-install feature gate Co-authored-by: igooch <[email protected]> * Remove den-install feature gate Co-authored-by: igooch <[email protected]> * Update test/e2e/fleetautoscaler_test.go Co-authored-by: igooch <[email protected]> * Regenerate install.yaml * Move ScheduledAutoscaler feature stage into 1.43 version --------- Co-authored-by: Mengye (Max) Gong <[email protected]> Co-authored-by: igooch <[email protected]>
googleforgames · Aug 26, 2024 · 967b702 · 967b702
1 parent a78b18b
commit 967b702
Show file tree

Hide file tree

Showing 7 changed files with 263 additions and 10 deletions.
diff --git a/build/Makefile b/build/Makefile
@@ -70,7 +70,7 @@ GS_TEST_IMAGE ?= us-docker.pkg.dev/agones-images/examples/simple-game-server:0.3
 BETA_FEATURE_GATES ?= "AutopilotPassthroughPort=true&CountsAndLists=true&DisableResyncOnSDKServer=true"
 
 # Enable all alpha feature gates. Keep in sync with `false` (alpha) entries in pkg/util/runtime/features.go:featureDefaults
-ALPHA_FEATURE_GATES ?= "PlayerAllocationFilter=true&PlayerTracking=true&RollingUpdateFix=true&PortRanges=true&PortPolicyNone=true&Example=true"
+ALPHA_FEATURE_GATES ?= "PlayerAllocationFilter=true&PlayerTracking=true&RollingUpdateFix=true&PortRanges=true&PortPolicyNone=true&ScheduledAutoscaler=true&Example=true"
 
 # Build with Windows support
 WITH_WINDOWS=1

diff --git a/cloudbuild.yaml b/cloudbuild.yaml
@@ -274,7 +274,7 @@ steps:
         declare -A versionsAndRegions=( [1.28]=us-west1 [1.29]=europe-west1 [1.30]=asia-east1 )
 
         # Keep in sync with (the inverse of) pkg/util/runtime/features.go:featureDefaults
-        featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&CountsAndLists=false&RollingUpdateFix=true&PortRanges=true&PortPolicyNone=true&DisableResyncOnSDKServer=false&AutopilotPassthroughPort=false&Example=true"
+        featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&CountsAndLists=false&RollingUpdateFix=true&PortRanges=true&PortPolicyNone=true&ScheduledAutoscaler=true&DisableResyncOnSDKServer=false&AutopilotPassthroughPort=false&Example=true"
         featureWithoutGate=""
 
         # Use this if specific feature gates can only be supported on specific Kubernetes versions.

diff --git a/install/helm/agones/defaultfeaturegates.yaml b/install/helm/agones/defaultfeaturegates.yaml
@@ -26,9 +26,9 @@ PlayerTracking: false
 RollingUpdateFix: false
 PortRanges: false
 PortPolicyNone: false
+ScheduledAutoscaler: false
 
 # Dev features
-ScheduledAutoscaler: false
 
 # Example feature
 Example: false
diff --git a/pkg/fleetautoscalers/fleetautoscalers_test.go b/pkg/fleetautoscalers/fleetautoscalers_test.go
@@ -2538,7 +2538,7 @@ func TestApplyChainPolicy(t *testing.T) {
 // Parse a time string and return a metav1.Time
 func mustParseMetav1Time(timeStr string) metav1.Time {
 	t, _ := time.Parse(time.RFC3339, timeStr)
-	return metav1.Date(t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), t.Location())
+	return metav1.NewTime(t)
 }
 
 // Parse a time string and return a time.Time

diff --git a/pkg/util/runtime/features.go b/pkg/util/runtime/features.go
@@ -55,21 +55,21 @@ const (
 	// FeaturePlayerTracking is a feature flag to enable/disable player tracking features.
 	FeaturePlayerTracking Feature = "PlayerTracking"
 
-	// FeatureRollingUpdateFix is a feature flag to enable/disable fleet controller fixes.
-	FeatureRollingUpdateFix Feature = "RollingUpdateFix"
-
 	// FeaturePortRanges is a feature flag to enable/disable specific port ranges.
 	FeaturePortRanges Feature = "PortRanges"
 
 	// FeaturePortPolicyNone is a feature flag to allow setting Port Policy to None.
 	FeaturePortPolicyNone Feature = "PortPolicyNone"
 
-	////////////////
-	// Dev features
+	// FeatureRollingUpdateFix is a feature flag to enable/disable fleet controller fixes.
+	FeatureRollingUpdateFix Feature = "RollingUpdateFix"
 
 	// FeatureScheduledAutoscaler is a feature flag to enable/disable scheduled fleet autoscaling.
 	FeatureScheduledAutoscaler Feature = "ScheduledAutoscaler"
 
+	////////////////
+	// Dev features
+
 	////////////////
 	// Example feature
 
@@ -139,9 +139,9 @@ var (
 		FeatureRollingUpdateFix:                 false,
 		FeaturePortRanges:                       false,
 		FeaturePortPolicyNone:                   false,
+		FeatureScheduledAutoscaler:              false,
 
 		// Dev features
-		FeatureScheduledAutoscaler: false,
 
 		// Example feature
 		FeatureExample: false,

diff --git a/site/content/en/docs/Guides/feature-stages.md b/site/content/en/docs/Guides/feature-stages.md
@@ -54,6 +54,7 @@ The current set of `alpha` and `beta` feature gates:
 | [Rolling Update Fixes](https://github.com/googleforgames/agones/issues/3688)                                                | `RollingUpdateFix`                 | Disabled | `Alpha` | 1.41.0 |
 | [Multiple dynamic port ranges](https://github.com/googleforgames/agones/issues/1911)                                        | `PortRanges`                       | Disabled | `Alpha` | 1.41.0 |
 | [Port Policy None](https://github.com/googleforgames/agones/issues/3804)                                                    | `PortPolicyNone`                   | Disabled | `Alpha` | 1.41.0 |
+| [Scheduled Fleet Autoscaling](https://github.com/googleforgames/agones/issues/3008)                                         | `ScheduledAutoscaler`              | Disabled | `Alpha` | 1.43.0 |
 | Example Gate (not in use)                                                                                                   | `Example`                          | Disabled | None    | 0.13.0 |
 
 {{% /feature %}}

diff --git a/test/e2e/fleetautoscaler_test.go b/test/e2e/fleetautoscaler_test.go
@@ -1445,3 +1445,255 @@ func TestListAutoscalerWithSDKMethods(t *testing.T) {
 		})
 	}
 }
+
+func TestScheduleAutoscaler(t *testing.T) {
+	if !runtime.FeatureEnabled(runtime.FeatureScheduledAutoscaler) {
+		t.SkipNow()
+	}
+	t.Parallel()
+	ctx := context.Background()
+	log := e2e.TestLogger(t)
+
+	stable := framework.AgonesClient.AgonesV1()
+	fleets := stable.Fleets(framework.Namespace)
+	flt, err := fleets.Create(ctx, defaultFleet(framework.Namespace), metav1.CreateOptions{})
+	if assert.NoError(t, err) {
+		defer fleets.Delete(context.Background(), flt.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck
+	}
+
+	framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(flt.Spec.Replicas))
+
+	fleetautoscalers := framework.AgonesClient.AutoscalingV1().FleetAutoscalers(framework.Namespace)
+
+	// Active Cron Schedule (e.g. run after 1 * * * *, which is the after the first minute of the hour)
+	scheduleAutoscaler := defaultAutoscalerSchedule(t, flt)
+	scheduleAutoscaler.Spec.Policy.Schedule.ActivePeriod.StartCron = nextCronMinute(time.Now())
+	fas, err := fleetautoscalers.Create(ctx, scheduleAutoscaler, metav1.CreateOptions{})
+	assert.NoError(t, err)
+
+	framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(5))
+	fleetautoscalers.Delete(ctx, fas.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck
+
+	// Return to starting 3 replicas
+	framework.ScaleFleet(t, log, flt, 3)
+	framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(3))
+
+	// Between Active Period Cron Schedule (e.g. run between 1-2 * * * *, which is between the first minute and second minute of the hour)
+	scheduleAutoscaler = defaultAutoscalerSchedule(t, flt)
+	scheduleAutoscaler.Spec.Policy.Schedule.ActivePeriod.StartCron = nextCronMinuteBetween(time.Now())
+	fas, err = fleetautoscalers.Create(ctx, scheduleAutoscaler, metav1.CreateOptions{})
+	assert.NoError(t, err)
+
+	framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(5))
+	fleetautoscalers.Delete(ctx, fas.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck
+}
+
+func TestChainAutoscaler(t *testing.T) {
+	if !runtime.FeatureEnabled(runtime.FeatureScheduledAutoscaler) {
+		t.SkipNow()
+	}
+	t.Parallel()
+	ctx := context.Background()
+	log := e2e.TestLogger(t)
+
+	stable := framework.AgonesClient.AgonesV1()
+	fleets := stable.Fleets(framework.Namespace)
+	flt, err := fleets.Create(ctx, defaultFleet(framework.Namespace), metav1.CreateOptions{})
+	if assert.NoError(t, err) {
+		defer fleets.Delete(context.Background(), flt.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck
+	}
+
+	framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(flt.Spec.Replicas))
+
+	fleetautoscalers := framework.AgonesClient.AutoscalingV1().FleetAutoscalers(framework.Namespace)
+
+	// 1st Schedule Inactive, 2nd Schedule Active - 30 seconds (Fallthrough)
+	chainAutoscaler := defaultAutoscalerChain(t, flt)
+	fas, err := fleetautoscalers.Create(ctx, chainAutoscaler, metav1.CreateOptions{})
+	assert.NoError(t, err)
+
+	// Verify only the second schedule ran
+	framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(4))
+	fleetautoscalers.Delete(ctx, fas.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck
+
+	// Return to starting 3 replicas
+	framework.ScaleFleet(t, log, flt, 3)
+	framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(3))
+
+	// 2 Active Schedules back to back - 1 minute (Fallthrough)
+	chainAutoscaler = defaultAutoscalerChain(t, flt)
+	currentTime := time.Now()
+
+	// First schedule runs for 1 minute
+	chainAutoscaler.Spec.Policy.Chain[0].Schedule.ActivePeriod.StartCron = nextCronMinute(currentTime)
+	chainAutoscaler.Spec.Policy.Chain[0].Schedule.ActivePeriod.Duration = "1m"
+
+	// Second schedule runs 1 minute after the first schedule
+	oneMinute := mustParseDuration(t, "1m")
+	chainAutoscaler.Spec.Policy.Chain[0].Schedule.ActivePeriod.StartCron = nextCronMinute(currentTime.Add(oneMinute))
+	chainAutoscaler.Spec.Policy.Chain[1].Schedule.ActivePeriod.Duration = "5m"
+
+	fas, err = fleetautoscalers.Create(ctx, chainAutoscaler, metav1.CreateOptions{})
+	assert.NoError(t, err)
+
+	// Verify the first schedule has been applied
+	framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(10))
+	// Verify the second schedule has been applied
+	framework.AssertFleetCondition(t, flt, e2e.FleetReadyCount(4))
+
+	fleetautoscalers.Delete(ctx, fas.ObjectMeta.Name, metav1.DeleteOptions{}) // nolint:errcheck
+}
+
+// defaultAutoscalerSchedule returns a default scheduled autoscaler for testing.
+func defaultAutoscalerSchedule(t *testing.T, f *agonesv1.Fleet) *autoscalingv1.FleetAutoscaler {
+	return &autoscalingv1.FleetAutoscaler{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      f.ObjectMeta.Name + "-scheduled-autoscaler",
+			Namespace: framework.Namespace,
+		},
+		Spec: autoscalingv1.FleetAutoscalerSpec{
+			FleetName: f.ObjectMeta.Name,
+			Policy: autoscalingv1.FleetAutoscalerPolicy{
+				Type: autoscalingv1.SchedulePolicyType,
+				Schedule: &autoscalingv1.SchedulePolicy{
+					Between: autoscalingv1.Between{
+						Start: currentTimePlusDuration(t, "1s"),
+						End:   currentTimePlusDuration(t, "1m"),
+					},
+					ActivePeriod: autoscalingv1.ActivePeriod{
+						Timezone:  "UTC",
+						StartCron: "* * * * *",
+						Duration:  "",
+					},
+					Policy: autoscalingv1.FleetAutoscalerPolicy{
+						Type: autoscalingv1.BufferPolicyType,
+						Buffer: &autoscalingv1.BufferPolicy{
+							BufferSize:  intstr.FromInt(5),
+							MinReplicas: 5,
+							MaxReplicas: 12,
+						},
+					},
+				},
+			},
+			Sync: &autoscalingv1.FleetAutoscalerSync{
+				Type: autoscalingv1.FixedIntervalSyncType,
+				FixedInterval: autoscalingv1.FixedIntervalSync{
+					Seconds: 5,
+				},
+			},
+		},
+	}
+}
+
+// defaultAutoscalerChain returns a default chain autoscaler for testing.
+func defaultAutoscalerChain(t *testing.T, f *agonesv1.Fleet) *autoscalingv1.FleetAutoscaler {
+	return &autoscalingv1.FleetAutoscaler{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      f.ObjectMeta.Name + "-chain-autoscaler",
+			Namespace: framework.Namespace,
+		},
+		Spec: autoscalingv1.FleetAutoscalerSpec{
+			FleetName: f.ObjectMeta.Name,
+			Policy: autoscalingv1.FleetAutoscalerPolicy{
+				Type: autoscalingv1.ChainPolicyType,
+				Chain: autoscalingv1.ChainPolicy{
+					{
+						ID: "schedule-1",
+						FleetAutoscalerPolicy: autoscalingv1.FleetAutoscalerPolicy{
+							Type: autoscalingv1.SchedulePolicyType,
+							Schedule: &autoscalingv1.SchedulePolicy{
+								Between: autoscalingv1.Between{
+									Start: currentTimePlusDuration(t, "1s"),
+									End:   currentTimePlusDuration(t, "2m"),
+								},
+								ActivePeriod: autoscalingv1.ActivePeriod{
+									Timezone:  "",
+									StartCron: inactiveCronSchedule(time.Now()),
+									Duration:  "1m",
+								},
+								Policy: autoscalingv1.FleetAutoscalerPolicy{
+									Type: autoscalingv1.BufferPolicyType,
+									Buffer: &autoscalingv1.BufferPolicy{
+										BufferSize:  intstr.FromInt(10),
+										MinReplicas: 10,
+										MaxReplicas: 20,
+									},
+								},
+							},
+						},
+					},
+					{
+						ID: "schedule-2",
+						FleetAutoscalerPolicy: autoscalingv1.FleetAutoscalerPolicy{
+							Type: autoscalingv1.SchedulePolicyType,
+							Schedule: &autoscalingv1.SchedulePolicy{
+								Between: autoscalingv1.Between{
+									Start: currentTimePlusDuration(t, "1s"),
+									End:   currentTimePlusDuration(t, "5m"),
+								},
+								ActivePeriod: autoscalingv1.ActivePeriod{
+									Timezone:  "",
+									StartCron: nextCronMinute(time.Now()),
+									Duration:  "",
+								},
+								Policy: autoscalingv1.FleetAutoscalerPolicy{
+									Type: autoscalingv1.BufferPolicyType,
+									Buffer: &autoscalingv1.BufferPolicy{
+										BufferSize:  intstr.FromInt(4),
+										MinReplicas: 3,
+										MaxReplicas: 7,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			Sync: &autoscalingv1.FleetAutoscalerSync{
+				Type: autoscalingv1.FixedIntervalSyncType,
+				FixedInterval: autoscalingv1.FixedIntervalSync{
+					Seconds: 5,
+				},
+			},
+		},
+	}
+}
+
+// inactiveCronSchedule returns the time 3 minutes ago
+// e.g. if the current time is 12:00, this method will return "57 * * * *"
+// meaning 3 minutes before 12:00
+func inactiveCronSchedule(currentTime time.Time) string {
+	prevMinute := currentTime.Add(time.Minute * -3).Minute()
+	return fmt.Sprintf("%d * * * *", prevMinute)
+}
+
+// nextCronMinute returns the very next minute in
+// e.g. if the current time is 12:00, this method will return "1 * * * *"
+// meaning after 12:01
+func nextCronMinute(currentTime time.Time) string {
+	nextMinute := currentTime.Add(time.Minute).Minute()
+	return fmt.Sprintf("%d * * * *", nextMinute)
+}
+
+// nextCronMinuteBetween returns the minute between the very next minute
+// e.g. if the current time is 12:00, this method will return "1-2 * * * *"
+// meaning between 12:01 - 12:02
+func nextCronMinuteBetween(currentTime time.Time) string {
+	nextMinute := currentTime.Add(time.Minute).Minute()
+	secondMinute := currentTime.Add(2 * time.Minute).Minute()
+	return fmt.Sprintf("%d-%d * * * *", nextMinute, secondMinute)
+}
+
+// Parse a duration string and return a duration struct
+func mustParseDuration(t *testing.T, duration string) time.Duration {
+	d, err := time.ParseDuration(duration)
+	assert.Nil(t, err)
+	return d
+}
+
+// Parse a time string and return a metav1.Time
+func currentTimePlusDuration(t *testing.T, duration string) metav1.Time {
+	d := mustParseDuration(t, duration)
+	currentTimePlusDuration := time.Now().Add(d)
+	return metav1.NewTime(currentTimePlusDuration)
+}