From 195a9b00735e81aab824f900fa015572389e5c61 Mon Sep 17 00:00:00 2001 From: Ryan Zhang Date: Sun, 4 Aug 2024 23:29:03 -0700 Subject: [PATCH] add stage rollout api --- apis/cluster/v1beta1/zz_generated.deepcopy.go | 2 +- apis/placement/v1alpha1/stagerollout_types.go | 135 ++++++++++ .../v1alpha1/zz_generated.deepcopy.go | 158 +++++++++++- .../v1beta1/clusterresourceplacement_types.go | 4 + .../v1beta1/zz_generated.deepcopy.go | 2 +- apis/v1alpha1/zz_generated.deepcopy.go | 2 +- ...ent.kubernetes-fleet.io_stagerollouts.yaml | 235 ++++++++++++++++++ test/apis/v1alpha1/zz_generated.deepcopy.go | 2 +- 8 files changed, 534 insertions(+), 6 deletions(-) create mode 100644 apis/placement/v1alpha1/stagerollout_types.go create mode 100644 config/crd/bases/placement.kubernetes-fleet.io_stagerollouts.yaml diff --git a/apis/cluster/v1beta1/zz_generated.deepcopy.go b/apis/cluster/v1beta1/zz_generated.deepcopy.go index 6c25f0189..6d06cb15b 100644 --- a/apis/cluster/v1beta1/zz_generated.deepcopy.go +++ b/apis/cluster/v1beta1/zz_generated.deepcopy.go @@ -10,7 +10,7 @@ Licensed under the MIT license. package v1beta1 import ( - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/apis/placement/v1alpha1/stagerollout_types.go b/apis/placement/v1alpha1/stagerollout_types.go new file mode 100644 index 000000000..1c3937d63 --- /dev/null +++ b/apis/placement/v1alpha1/stagerollout_types.go @@ -0,0 +1,135 @@ +/* +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// +genclient +// +genclient:nonNamespaced +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope="Cluster",categories={fleet,fleet-placement} +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// StageRollout defines a group of override policies about how to override the selected cluster scope resources +// to target clusters. +type StageRollout struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // The desired state of StageRollout. + // +required + Spec StageRolloutSpec `json:"spec"` + + // The observed status of StageRollout. + // +required + Status StageRolloutStatus `json:"status"` +} + +// StageRolloutSpec defines the desired the rollout sequence stage by stage. +type StageRolloutSpec struct { + // Stage rollout configurations for each rollout stage. + // +required + StageRollout []StageRolloutConfig `json:"stages"` + + // The wait time between each stage is completed. + // +optional + WaitTimeBetweenStage *metav1.Duration `json:"waitTimeBetweenStage,omitempty"` + + // TODO: Add alerting configuration. + + // TODO: Add health-check configuration. +} + +// StageRolloutConfig describes a single rollout stage group configuration. +type StageRolloutConfig struct { + // The name of the stage. This should be unique within the StageRollout. + // +required + Name string `json:"name"` + + // LabelSelector is a label query over all the joined member clusters. Clusters matching the query are selected. + // We don't check if there are overlap between clusters selected by different stages. + LabelSelector *metav1.LabelSelector `json:"labelSelector"` + + // The label key used to sort the selected clusters. + // The clusters within the stage are updated following the rule below: + // - primary: Ascending order based on the value of the label key, interpreted as integers if present. + // - secondary: Ascending order based on the name of the cluster if the label key is absent. + // +optional + SortingLabelKey *string `json:"sortingLabelKey,omitempty"` + + // The maximum number of clusters that can be updated simultaneously within this stage + // default is 1. + // +optional + MaxParallel *intstr.IntOrString `json:"maxParallel,omitempty"` + + // The wait time after all the clusters in this stage are updated. + // This will override the global waitTimeBetweenStage for this specific stage + // +optional + WaitTime *metav1.Duration `json:"waitTime,omitempty"` +} + +// StageRolloutStatus defines the observed state of the StageRollout. +type StageRolloutStatus struct { + // The stage that is in the middle of the rollout. + CurrentStage int `json:"stage"` + + // the clusters that are in the current stage. + ClustersInCurrentStage []string `json:"clustersInCurrentStage"` + + // the clusters that are actively updating. + // +optional + CurrentUpdatingClusters []string `json:"currentUpdatingClusters,omitempty"` + + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + // + // Conditions is an array of current observed conditions for ClusterResourceBinding. + // +optional + Conditions []metav1.Condition `json:"conditions"` +} + +// StageRolloutConditionType identifies a specific condition of the StageRollout. +type StageRolloutConditionType string + +const ( + // StageRollingOut indicates whether the stage is rolling out normally. + // Its condition status can be one of the following: + // - "True" means the stage is rolling out. + // - "False" means the stage rolling out is not progressing. + // - "Unknown" means it is unknown. + StageRollingOut StageRolloutConditionType = "StageRollingOut" + + // StageRolloutWaiting indicates whether the stage is waiting to be rolled out. + // Its condition status can be one of the following: + // - "True" means the staging is waiting to start to be rolled out. + // - "False" means the staging is not waiting to start rollout. + // - "Unknown" means it is unknown. + StageRolloutWaiting StageRolloutConditionType = "RolloutWaiting" + + // TODO: add verification condition + + // TODO: add alerting condition +) + +// StageRolloutList contains a list of StageRollout. +// +kubebuilder:resource:scope="Cluster" +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type StageRolloutList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []StageRollout `json:"items"` +} + +func init() { + SchemeBuilder.Register( + &StageRollout{}, &StageRolloutList{}, + ) +} diff --git a/apis/placement/v1alpha1/zz_generated.deepcopy.go b/apis/placement/v1alpha1/zz_generated.deepcopy.go index 53850a824..06a187f39 100644 --- a/apis/placement/v1alpha1/zz_generated.deepcopy.go +++ b/apis/placement/v1alpha1/zz_generated.deepcopy.go @@ -10,9 +10,10 @@ Licensed under the MIT license. package v1alpha1 import ( - runtime "k8s.io/apimachinery/pkg/runtime" - "go.goms.io/fleet/apis/placement/v1beta1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -420,3 +421,156 @@ func (in *ResourceSelector) DeepCopy() *ResourceSelector { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageRollout) DeepCopyInto(out *StageRollout) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageRollout. +func (in *StageRollout) DeepCopy() *StageRollout { + if in == nil { + return nil + } + out := new(StageRollout) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StageRollout) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageRolloutConfig) DeepCopyInto(out *StageRolloutConfig) { + *out = *in + if in.LabelSelector != nil { + in, out := &in.LabelSelector, &out.LabelSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.SortingLabelKey != nil { + in, out := &in.SortingLabelKey, &out.SortingLabelKey + *out = new(string) + **out = **in + } + if in.MaxParallel != nil { + in, out := &in.MaxParallel, &out.MaxParallel + *out = new(intstr.IntOrString) + **out = **in + } + if in.WaitTime != nil { + in, out := &in.WaitTime, &out.WaitTime + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageRolloutConfig. +func (in *StageRolloutConfig) DeepCopy() *StageRolloutConfig { + if in == nil { + return nil + } + out := new(StageRolloutConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageRolloutList) DeepCopyInto(out *StageRolloutList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]StageRollout, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageRolloutList. +func (in *StageRolloutList) DeepCopy() *StageRolloutList { + if in == nil { + return nil + } + out := new(StageRolloutList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StageRolloutList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageRolloutSpec) DeepCopyInto(out *StageRolloutSpec) { + *out = *in + if in.StageRollout != nil { + in, out := &in.StageRollout, &out.StageRollout + *out = make([]StageRolloutConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.WaitTimeBetweenStage != nil { + in, out := &in.WaitTimeBetweenStage, &out.WaitTimeBetweenStage + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageRolloutSpec. +func (in *StageRolloutSpec) DeepCopy() *StageRolloutSpec { + if in == nil { + return nil + } + out := new(StageRolloutSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageRolloutStatus) DeepCopyInto(out *StageRolloutStatus) { + *out = *in + if in.ClustersInCurrentStage != nil { + in, out := &in.ClustersInCurrentStage, &out.ClustersInCurrentStage + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.CurrentUpdatingClusters != nil { + in, out := &in.CurrentUpdatingClusters, &out.CurrentUpdatingClusters + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageRolloutStatus. +func (in *StageRolloutStatus) DeepCopy() *StageRolloutStatus { + if in == nil { + return nil + } + out := new(StageRolloutStatus) + in.DeepCopyInto(out) + return out +} diff --git a/apis/placement/v1beta1/clusterresourceplacement_types.go b/apis/placement/v1beta1/clusterresourceplacement_types.go index 6e1f22b70..6b45cc346 100644 --- a/apis/placement/v1beta1/clusterresourceplacement_types.go +++ b/apis/placement/v1beta1/clusterresourceplacement_types.go @@ -488,6 +488,10 @@ const ( // RollingUpdateRolloutStrategyType replaces the old placed resource using rolling update // i.e. gradually create the new one while replace the old ones. RollingUpdateRolloutStrategyType RolloutStrategyType = "RollingUpdate" + + // StageRollingUpdateRolloutStrategyType replaces the old placed resource stage by stage + // i.e. update the resources in the testing clusters first, then the staging clusters, and finally the production clusters. + StageRollingUpdateRolloutStrategyType RolloutStrategyType = "StageRollingUpdate" ) // RollingUpdateConfig contains the config to control the desired behavior of rolling update. diff --git a/apis/placement/v1beta1/zz_generated.deepcopy.go b/apis/placement/v1beta1/zz_generated.deepcopy.go index 4c48828e9..dcfc83ab5 100644 --- a/apis/placement/v1beta1/zz_generated.deepcopy.go +++ b/apis/placement/v1beta1/zz_generated.deepcopy.go @@ -10,7 +10,7 @@ Licensed under the MIT license. package v1beta1 import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" ) diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index ac4844274..0d4061551 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -11,7 +11,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/config/crd/bases/placement.kubernetes-fleet.io_stagerollouts.yaml b/config/crd/bases/placement.kubernetes-fleet.io_stagerollouts.yaml new file mode 100644 index 000000000..305b63a86 --- /dev/null +++ b/config/crd/bases/placement.kubernetes-fleet.io_stagerollouts.yaml @@ -0,0 +1,235 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + name: stagerollouts.placement.kubernetes-fleet.io +spec: + group: placement.kubernetes-fleet.io + names: + categories: + - fleet + - fleet-placement + kind: StageRollout + listKind: StageRolloutList + plural: stagerollouts + singular: stagerollout + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + StageRollout defines a group of override policies about how to override the selected cluster scope resources + to target clusters. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: The desired state of StageRollout. + properties: + stages: + description: Stage rollout configurations for each rollout stage. + items: + description: StageRolloutConfig describes a single rollout stage + group configuration. + properties: + labelSelector: + description: |- + LabelSelector is a label query over all the joined member clusters. Clusters matching the query are selected. + We don't check if there are overlap between clusters selected by different stages. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + maxParallel: + anyOf: + - type: integer + - type: string + description: |- + The maximum number of clusters that can be updated simultaneously within this stage + default is 1. + x-kubernetes-int-or-string: true + name: + description: The name of the stage. This should be unique within + the StageRollout. + type: string + sortingLabelKey: + description: |- + The label key used to sort the selected clusters. + The clusters within the stage are updated following the rule below: + - primary: Ascending order based on the value of the label key, interpreted as integers if present. + - secondary: Ascending order based on the name of the cluster if the label key is absent. + type: string + waitTime: + description: |- + The wait time after all the clusters in this stage are updated. + This will override the global waitTimeBetweenStage for this specific stage + type: string + required: + - labelSelector + - name + type: object + type: array + waitTimeBetweenStage: + description: The wait time between each stage is completed. + type: string + required: + - stages + type: object + status: + description: The observed status of StageRollout. + properties: + clustersInCurrentStage: + description: the clusters that are in the current stage. + items: + type: string + type: array + conditions: + description: Conditions is an array of current observed conditions + for ClusterResourceBinding. + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + currentUpdatingClusters: + description: the clusters that are actively updating. + items: + type: string + type: array + stage: + description: The stage that is in the middle of the rollout. + type: integer + required: + - clustersInCurrentStage + - stage + type: object + required: + - spec + - status + type: object + served: true + storage: true diff --git a/test/apis/v1alpha1/zz_generated.deepcopy.go b/test/apis/v1alpha1/zz_generated.deepcopy.go index ef7e4433a..0b5d2e30b 100644 --- a/test/apis/v1alpha1/zz_generated.deepcopy.go +++ b/test/apis/v1alpha1/zz_generated.deepcopy.go @@ -10,7 +10,7 @@ Licensed under the MIT license. package v1alpha1 import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" )