diff --git a/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller_integration_test.go b/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller_integration_test.go index ed94e9022..8fc341fca 100644 --- a/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller_integration_test.go +++ b/operators/endpointmetrics/controllers/observabilityendpoint/observabilityaddon_controller_integration_test.go @@ -4,27 +4,33 @@ //go:build integration -package observabilityendpoint +package observabilityendpoint_test import ( "context" "fmt" "os" "path/filepath" + "strings" "testing" "time" + ocinfrav1 "github.com/openshift/api/config/v1" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1alpha1" promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/controllers/observabilityendpoint" "github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/hypershift" "github.com/stolostron/multicluster-observability-operator/operators/endpointmetrics/pkg/util" observabilityshared "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/shared" oav1beta1 "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/v1beta1" mcov1beta2 "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/v1beta2" + operatorconfig "github.com/stolostron/multicluster-observability-operator/operators/pkg/config" "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -47,7 +53,231 @@ var ( restCfgHub *rest.Config ) +// TestIntegrationReconcileAddon ensures that addon directives are applied to the resources in the cluster. +// This includes enabled/disabled, interval, and resources requirements. +func TestIntegrationReconcileAddon(t *testing.T) { + scheme := createBaseScheme() + assert.NoError(t, ocinfrav1.AddToScheme(scheme)) + assert.NoError(t, oav1beta1.AddToScheme(scheme)) + + // Setup spoke client and resources + k8sSpokeClient, err := client.New(restCfgSpoke, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + + testNamespace := "spoke-ns" + addon := newObservabilityAddon("observability-addon", testNamespace) + addon.Spec.EnableMetrics = true + addon.Spec.Interval = 60 + addon.Spec.Resources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("20Mi"), + }, + } + endpointOperatorDeploy := &appsv1.Deployment{ // needed by the metrics-collector to copy some settings + ObjectMeta: metav1.ObjectMeta{ + Name: "endpoint-observability-operator", + Namespace: testNamespace, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "endpoint-observability-operator", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "endpoint-observability-operator", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "endpoint-observability-operator", + Image: "test", + }, + }, + }, + }, + }, + } + resources := []client.Object{} + resources = append(resources, newAcmResources(testNamespace)...) + resources = append(resources, newOcpSpokeResources(testNamespace)...) + resources = append(resources, addon, endpointOperatorDeploy) + resources = append(resources, newPrometheusUwlResources()...) // to add uwl collector and check config + resources = append(resources, newUwlMetrics("userwl", []string{"dummy_metric"})...) + if err := createResources(k8sSpokeClient, resources...); err != nil { + t.Fatalf("Failed to create resources: %v", err) + } + + // Setup hub client and resources + k8sHubClient, err := client.New(restCfgHub, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + hubClientWithReload, err := util.NewReloadableHubClientWithReloadFunc(func() (client.Client, error) { + return k8sHubClient, nil + }) + assert.NoError(t, err) + hubNamespace := "hub-ns" + hubAddon := addon.DeepCopy() + hubAddon.ResourceVersion = "" + hubAddon.Namespace = hubNamespace + resources = []client.Object{ + makeNamespace(hubNamespace), + hubAddon, + } + if err := createResources(k8sHubClient, resources...); err != nil { + t.Fatalf("Failed to create resources: %v", err) + } + + // Setup the controller + reconciler := observabilityendpoint.ObservabilityAddonReconciler{ + Client: k8sSpokeClient, + HubClient: hubClientWithReload, + IsHubMetricsCollector: false, + Scheme: scheme, + Namespace: testNamespace, + HubNamespace: hubNamespace, + ServiceAccountName: "endpoint-monitoring-operator", + InstallPrometheus: false, + } + mgr, err := ctrl.NewManager(testEnvSpoke.Config, ctrl.Options{ + Scheme: k8sSpokeClient.Scheme(), + Metrics: metricsserver.Options{BindAddress: "0"}, // Avoids port conflict with the default port 8080 + }) + assert.NoError(t, err) + err = reconciler.SetupWithManager(mgr) + assert.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + err = mgr.Start(ctx) + assert.NoError(t, err) + }() + + // Check that addon resources request are applied to the metrics-collector pod + deployment := &appsv1.Deployment{} + for _, resourceName := range []string{"metrics-collector-deployment", "uwl-metrics-collector-deployment"} { + err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 5*time.Second, false, func(ctx context.Context) (bool, error) { + err := k8sSpokeClient.Get(ctx, types.NamespacedName{Name: resourceName, Namespace: testNamespace}, deployment) + if err != nil { + return false, err + } + return true, nil + }) + if err != nil { + t.Fatalf("Failed to get metrics-collector pod: %v", err) + } + assert.EqualValues(t, 1, *deployment.Spec.Replicas) + pod := deployment.Spec.Template + assert.Equal(t, addon.Spec.Resources.Limits.Cpu().String(), pod.Spec.Containers[0].Resources.Limits.Cpu().String()) + assert.Equal(t, addon.Spec.Resources.Limits.Memory().String(), pod.Spec.Containers[0].Resources.Limits.Memory().String()) + assert.Equal(t, addon.Spec.Resources.Requests.Cpu().String(), pod.Spec.Containers[0].Resources.Requests.Cpu().String()) + assert.Equal(t, addon.Spec.Resources.Requests.Memory().String(), pod.Spec.Containers[0].Resources.Requests.Memory().String()) + var intervalArg string + for _, arg := range pod.Spec.Containers[0].Command { + if strings.HasPrefix(arg, "--interval=") { + intervalArg = arg + break + } + } + assert.Equal(t, fmt.Sprintf("--interval=%ds", addon.Spec.Interval), intervalArg) + } + + currentAddon := &oav1beta1.ObservabilityAddon{} + err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 5*time.Second, false, func(ctx context.Context) (bool, error) { + if err := k8sSpokeClient.Get(ctx, types.NamespacedName{Name: "observability-addon", Namespace: testNamespace}, currentAddon); err != nil { + return false, err + } + return true, nil + }) + if err != nil { + t.Fatalf("Failed to get observability addon: %v", err) + } + var progressingCondition *oav1beta1.StatusCondition + for _, condition := range currentAddon.Status.Conditions { + if condition.Type == "Progressing" { + progressingCondition = &condition + break + } + } + assert.NotNil(t, progressingCondition) + assert.Equal(t, metav1.ConditionTrue, progressingCondition.Status) + + // Check that disabled addon removes metrics collector pods and sets the status to disabled + if err := k8sSpokeClient.Get(ctx, types.NamespacedName{Name: "observability-addon", Namespace: testNamespace}, currentAddon); err != nil { + t.Fatalf("Failed to get observability addon: %v", err) + } + disabledAddon := currentAddon.DeepCopy() + disabledAddon.Spec.EnableMetrics = false + if err := k8sSpokeClient.Update(ctx, disabledAddon); err != nil { + t.Fatalf("Failed to update observability addon: %v", err) + } + + for _, resourceName := range []string{"metrics-collector-deployment", "uwl-metrics-collector-deployment"} { + err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 5*time.Second, false, func(ctx context.Context) (bool, error) { + err := k8sSpokeClient.Get(ctx, types.NamespacedName{Name: resourceName, Namespace: testNamespace}, deployment) + if err != nil { + return false, nil + } + return true, nil + }) + if err != nil { + t.Fatalf("Failed to wait for metrics-collector replicas to be zero: %v", err) + } + assert.Equal(t, int32(0), *deployment.Spec.Replicas) + } + + err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 5*time.Second, false, func(ctx context.Context) (bool, error) { + if err := k8sSpokeClient.Get(ctx, types.NamespacedName{Name: "observability-addon", Namespace: testNamespace}, currentAddon); err != nil { + return false, err + } + return true, nil + }) + if err != nil { + t.Fatalf("Failed to get observability addon: %v", err) + } + var disabledCondition *oav1beta1.StatusCondition + for _, condition := range currentAddon.Status.Conditions { + if condition.Type == "Disabled" { + disabledCondition = &condition + break + } + } + assert.NotNil(t, disabledCondition) + assert.Equal(t, metav1.ConditionTrue, disabledCondition.Status) + + // check invalid interval values are refused + if err := k8sSpokeClient.Get(ctx, types.NamespacedName{Name: "observability-addon", Namespace: testNamespace}, currentAddon); err != nil { + t.Fatalf("Failed to get observability addon: %v", err) + } + invalidAddon := currentAddon.DeepCopy() + invalidAddon.Spec.Interval = 1 + if err := k8sSpokeClient.Update(ctx, invalidAddon); err == nil { + t.Fatalf("Expected error when updating observability addon with invalid interval value") + } + + invalidAddon.Spec.Interval = 10e6 + if err := k8sSpokeClient.Update(ctx, invalidAddon); err == nil { + t.Fatalf("Expected error when updating observability addon with invalid interval value") + } + + // delete the addon and check that resources are removed +} + // TestIntegrationReconcileHypershift tests the reconcile function for hypershift CRDs. +// It ensures that the service monitors are created for the hypershift resources. func TestIntegrationReconcileHypershift(t *testing.T) { testNamespace := "test-ns" @@ -88,7 +318,7 @@ func TestIntegrationReconcileHypershift(t *testing.T) { return k8sClient, nil }) assert.NoError(t, err) - reconciler := ObservabilityAddonReconciler{ + reconciler := observabilityendpoint.ObservabilityAddonReconciler{ Client: k8sClient, HubClient: hubClientWithReload, IsHubMetricsCollector: true, @@ -132,9 +362,11 @@ func TestMain(m *testing.M) { rootPath := filepath.Join("..", "..", "..") spokeCrds := readCRDFiles( filepath.Join(rootPath, "multiclusterobservability", "config", "crd", "bases", "observability.open-cluster-management.io_observabilityaddons.yaml"), + filepath.Join(rootPath, "endpointmetrics", "manifests", "prometheus", "crd", "servicemonitor_crd_0_53_1.yaml"), + filepath.Join(rootPath, "endpointmetrics", "manifests", "prometheus", "crd", "prometheusrule_crd_0_53_1.yaml"), ) testEnvSpoke = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("testdata", "crd"), filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("testdata", "crd")}, CRDs: spokeCrds, ControlPlaneStopTimeout: 5 * time.Minute, } @@ -146,13 +378,13 @@ func TestMain(m *testing.M) { } hubCRDs := readCRDFiles( + filepath.Join(rootPath, "multiclusterobservability", "config", "crd", "bases", "observability.open-cluster-management.io_observabilityaddons.yaml"), filepath.Join(rootPath, "multiclusterobservability", "config", "crd", "bases", "observability.open-cluster-management.io_multiclusterobservabilities.yaml"), filepath.Join(rootPath, "endpointmetrics", "manifests", "prometheus", "crd", "servicemonitor_crd_0_53_1.yaml"), ) - hubCRDs = append(hubCRDs, spokeCrds...) testEnvHub = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("testdata", "crd"), filepath.Join("..", "..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{filepath.Join("testdata", "crd")}, CRDs: hubCRDs, ControlPlaneStopTimeout: 5 * time.Minute, } @@ -198,34 +430,55 @@ func setupCommonHubResources(t *testing.T, k8sClient client.Client, ns string) { } } -func tearDownCommonHubResources(t *testing.T, k8sClient client.Client, ns string) { - // Delete resources required for the observability addon controller - resourcesDeps := []client.Object{ +func newAcmResources(ns string) []client.Object { + return []client.Object{ makeNamespace(ns), - } - for _, resource := range resourcesDeps { - if err := k8sClient.Delete(context.Background(), resource); err != nil { - t.Fatalf("Failed to delete resource: %v", err) - } + newHubInfoSecret([]byte{}, ns), + newImagesCM(ns), } } -func setupCommonSpokeResources(t *testing.T, k8sClient client.Client) { - // Create resources required for the observability addon controller - resourcesDeps := []client.Object{ - makeNamespace("open-cluster-management-addon-observability"), - newHubInfoSecret([]byte{}, "open-cluster-management-addon-observability"), - newImagesCM("open-cluster-management-addon-observability"), - } - if err := createResources(k8sClient, resourcesDeps...); err != nil { - t.Fatalf("Failed to create resources: %v", err) +func newOcpSpokeResources(addonNs string) []client.Object { + return []client.Object{ + makeNamespace("openshift-monitoring"), + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "prometheus-k8s", + Namespace: "openshift-monitoring", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "web", + Port: 9090, + }, + }, + }, + }, + &ocinfrav1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "version", + }, + Spec: ocinfrav1.ClusterVersionSpec{ + ClusterID: "b551a7ec-e6c1-4132-a95b-935d726a9766", + }, + }, + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "observability-alertmanager-accessor", + Namespace: addonNs, + }, + Data: map[string][]byte{ + "token": []byte("am token"), + }, + }, } } -func tearDownCommonSpokeResources(t *testing.T, k8sClient client.Client) { +func tearDownCommonHubResources(t *testing.T, k8sClient client.Client, ns string) { // Delete resources required for the observability addon controller resourcesDeps := []client.Object{ - makeNamespace("open-cluster-management-addon-observability"), + makeNamespace(ns), } for _, resource := range resourcesDeps { if err := k8sClient.Delete(context.Background(), resource); err != nil { @@ -240,14 +493,14 @@ func readCRDFiles(crdPaths ...string) []*apiextensionsv1.CustomResourceDefinitio for _, crdPath := range crdPaths { crdYamlData, err := os.ReadFile(crdPath) if err != nil { - panic(fmt.Sprintf("Failed to read CRD file: %v", err)) + panic(fmt.Sprintf("Failed to read CRD file at path %s: %v", crdPath, err)) } dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) var crd apiextensionsv1.CustomResourceDefinition _, _, err = dec.Decode(crdYamlData, nil, &crd) if err != nil { - panic(fmt.Sprintf("Failed to decode CRD: %v", err)) + panic(fmt.Sprintf("Failed to decode CRD at path %s: %v", crdPath, err)) } ret = append(ret, &crd) @@ -274,7 +527,7 @@ func createResources(client client.Client, resources ...client.Object) error { return nil } -func newObservabilityAddonBis(name, ns string) *oav1beta1.ObservabilityAddon { +func newObservabilityAddon(name, ns string) *oav1beta1.ObservabilityAddon { return &oav1beta1.ObservabilityAddon{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -356,3 +609,87 @@ names: }, } } + +func newPrometheusUwlResources() []client.Object { + return []client.Object{ + makeNamespace("openshift-user-workload-monitoring"), + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "prometheus-user-workload", + Namespace: "openshift-user-workload-monitoring", + }, + Spec: appsv1.StatefulSetSpec{ + // Replicas: util.Int32Ptr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "prometheus-user-workload", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "prometheus-user-workload", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "prometheus-user-workload", + Image: "test", + }, + }, + }, + }, + }, + }, + } +} + +func newUwlMetrics(ns string, metrics []string) []client.Object { + var data strings.Builder + data.WriteString("names:\n") + for _, metric := range metrics { + data.WriteString(fmt.Sprintf(" - %s\n", metric)) + } + return []client.Object{ + makeNamespace(ns), + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "observability-metrics-custom-allowlist", + Namespace: ns, + }, + Data: map[string]string{ + "uwl_metrics_list.yaml": data.String(), + }, + }, + } +} + +func newImagesCM(ns string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: operatorconfig.ImageConfigMap, + Namespace: ns, + }, + Data: map[string]string{ + operatorconfig.MetricsCollectorKey: "metrics-collector-image", + operatorconfig.NodeExporterKey: "node-exporter-image", + operatorconfig.KubeStateMetricsKey: "kube-state-metrics-image", + operatorconfig.KubeRbacProxyKey: "kube-rbac-proxy-image", + operatorconfig.PrometheusOperatorKey: "prometheus-operator-image", + }, + } +} + +func newHubInfoSecret(data []byte, ns string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: operatorconfig.HubInfoSecretName, + Namespace: ns, + }, + Data: map[string][]byte{ + operatorconfig.HubInfoSecretKey: data, + operatorconfig.ClusterNameKey: []byte("test-cluster"), + }, + } +} diff --git a/operators/endpointmetrics/pkg/collector/metrics_collector.go b/operators/endpointmetrics/pkg/collector/metrics_collector.go index 5080adf60..069f890e1 100644 --- a/operators/endpointmetrics/pkg/collector/metrics_collector.go +++ b/operators/endpointmetrics/pkg/collector/metrics_collector.go @@ -908,7 +908,10 @@ func (m *MetricsCollector) getMetricsAllowlist(ctx context.Context) (*operatorco err := m.Client.Get(ctx, types.NamespacedName{Name: operatorconfig.AllowlistConfigMapName, Namespace: m.Namespace}, cm) if err != nil { - m.Log.Error(err, "Failed to get configmap", "name", operatorconfig.AllowlistConfigMapName, "namespace", m.Namespace) + if !errors.IsNotFound(err) { + return allowList, userAllowList, fmt.Errorf("failed to get configmap %s/%s: %w", m.Namespace, operatorconfig.AllowlistConfigMapName, err) + } + m.Log.Info(fmt.Sprintf("AllowList configmap not found %s/%s: %v", m.Namespace, operatorconfig.AllowlistConfigMapName, err)) } if cm.Data != nil { diff --git a/operators/endpointmetrics/pkg/collector/metrics_collector_test.go b/operators/endpointmetrics/pkg/collector/metrics_collector_test.go index 011622d48..39e54c547 100644 --- a/operators/endpointmetrics/pkg/collector/metrics_collector_test.go +++ b/operators/endpointmetrics/pkg/collector/metrics_collector_test.go @@ -29,6 +29,7 @@ import ( oashared "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/shared" oav1beta1 "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/v1beta1" operatorconfig "github.com/stolostron/multicluster-observability-operator/operators/pkg/config" + "github.com/stretchr/testify/assert" ) const ( @@ -76,23 +77,23 @@ func TestMetricsCollectorResourcesUpdate(t *testing.T) { newMetricsCollector: func() *collector.MetricsCollector { return baseMetricsCollector() }, - clientObjects: func() []runtime.Object { return []runtime.Object{getEndpointOperatorDeployment()} }, + clientObjects: func() []runtime.Object { return []runtime.Object{getEndpointOperatorDeployment(namespace)} }, expects: func(t *testing.T, deployment, uwlDeployment *appsv1.Deployment) { // Check env vars - operatorEnv := getEndpointOperatorDeployment().Spec.Template.Spec.Containers[0].Env + operatorEnv := getEndpointOperatorDeployment(namespace).Spec.Template.Spec.Containers[0].Env collectorEnv := deployment.Spec.Template.Spec.Containers[0].Env if err := checkProxyEnvVars(operatorEnv, collectorEnv); err != nil { t.Fatalf("Failed to ensure proxy env vars: %v", err) } // Check toleration and node selector - if !slices.Equal(deployment.Spec.Template.Spec.Tolerations, getEndpointOperatorDeployment().Spec.Template.Spec.Tolerations) { + if !slices.Equal(deployment.Spec.Template.Spec.Tolerations, getEndpointOperatorDeployment(namespace).Spec.Template.Spec.Tolerations) { t.Fatalf("Tolerations are not set correctly: expected %v, got %v", - getEndpointOperatorDeployment().Spec.Template.Spec.Tolerations, deployment.Spec.Template.Spec.Tolerations) + getEndpointOperatorDeployment(namespace).Spec.Template.Spec.Tolerations, deployment.Spec.Template.Spec.Tolerations) } - if !maps.Equal(deployment.Spec.Template.Spec.NodeSelector, getEndpointOperatorDeployment().Spec.Template.Spec.NodeSelector) { + if !maps.Equal(deployment.Spec.Template.Spec.NodeSelector, getEndpointOperatorDeployment(namespace).Spec.Template.Spec.NodeSelector) { t.Fatalf("NodeSelector is not set correctly: expected %v, got %v", - getEndpointOperatorDeployment().Spec.Template.Spec.NodeSelector, deployment.Spec.Template.Spec.NodeSelector) + getEndpointOperatorDeployment(namespace).Spec.Template.Spec.NodeSelector, deployment.Spec.Template.Spec.NodeSelector) } // Check annotations @@ -113,7 +114,7 @@ func TestMetricsCollectorResourcesUpdate(t *testing.T) { ret.ClusterInfo.IsHubMetricsCollector = false return ret }, - clientObjects: func() []runtime.Object { return []runtime.Object{getEndpointOperatorDeployment()} }, + clientObjects: func() []runtime.Object { return []runtime.Object{getEndpointOperatorDeployment(namespace)} }, expects: func(t *testing.T, deployment *appsv1.Deployment, uwlDeployment *appsv1.Deployment) { if *deployment.Spec.Replicas != 0 { t.Fatalf("Replicas should be 0 when metrics is disabled and is not hub collector") @@ -127,7 +128,7 @@ func TestMetricsCollectorResourcesUpdate(t *testing.T) { ret.ClusterInfo.IsHubMetricsCollector = true return ret }, - clientObjects: func() []runtime.Object { return []runtime.Object{getEndpointOperatorDeployment()} }, + clientObjects: func() []runtime.Object { return []runtime.Object{getEndpointOperatorDeployment(namespace)} }, expects: func(t *testing.T, deployment *appsv1.Deployment, uwlDeployment *appsv1.Deployment) { if *deployment.Spec.Replicas != 1 { t.Fatalf("Hub metrics collector should have 1 replica even if metrics is disabled") @@ -139,7 +140,7 @@ func TestMetricsCollectorResourcesUpdate(t *testing.T) { return baseMetricsCollector() }, clientObjects: func() []runtime.Object { - ret := []runtime.Object{getEndpointOperatorDeployment()} + ret := []runtime.Object{getEndpointOperatorDeployment(namespace)} metricsCollector := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: metricsCollectorName, @@ -169,7 +170,7 @@ func TestMetricsCollectorResourcesUpdate(t *testing.T) { }, } uwlAllowlistCM := newAllowListCm(operatorconfig.AllowlistCustomConfigMapName, "default", data) - ret := []runtime.Object{getEndpointOperatorDeployment(), newUwlPrometheus(), uwlAllowlistCM} + ret := []runtime.Object{getEndpointOperatorDeployment(namespace), newUwlPrometheus(), uwlAllowlistCM} return ret }, expects: func(t *testing.T, deployment *appsv1.Deployment, uwlDeployment *appsv1.Deployment) { @@ -188,7 +189,7 @@ func TestMetricsCollectorResourcesUpdate(t *testing.T) { return baseMetricsCollector() }, clientObjects: func() []runtime.Object { - ret := []runtime.Object{getEndpointOperatorDeployment(), newUwlPrometheus()} + ret := []runtime.Object{getEndpointOperatorDeployment(namespace), newUwlPrometheus()} return ret }, expects: func(t *testing.T, deployment *appsv1.Deployment, uwlDeployment *appsv1.Deployment) { @@ -214,7 +215,7 @@ func TestMetricsCollectorResourcesUpdate(t *testing.T) { }, } uwlAllowlistCM := newAllowListCm(operatorconfig.AllowlistCustomConfigMapName, "default", data) - ret := []runtime.Object{getEndpointOperatorDeployment(), uwlAllowlistCM, uwlDeploy} + ret := []runtime.Object{getEndpointOperatorDeployment(namespace), uwlAllowlistCM, uwlDeploy} return ret }, expects: func(t *testing.T, deployment *appsv1.Deployment, uwlDeployment *appsv1.Deployment) { @@ -237,19 +238,284 @@ func TestMetricsCollectorResourcesUpdate(t *testing.T) { t.Fatalf("Failed to update metrics collector: %v", err) } - deployment := getMetricsCollectorDeployment(t, context.Background(), c, metricsCollectorName) - uwlDeployment := getMetricsCollectorDeployment(t, context.Background(), c, uwlMetricsCollectorName) + deployment := getMetricsCollectorDeployment(t, context.Background(), c, metricsCollectorName, namespace) + uwlDeployment := getMetricsCollectorDeployment(t, context.Background(), c, uwlMetricsCollectorName, namespace) tc.expects(t, deployment, uwlDeployment) }) } } -func getEndpointOperatorDeployment() *appsv1.Deployment { +// TestMetricsLists ensures that configured metrics are collected appropriately. +func TestMetricsLists(t *testing.T) { + addonNamespace := "test-namespace" + endpointOperatorDeploy := getEndpointOperatorDeployment(addonNamespace) + s := scheme.Scheme + promv1.AddToScheme(s) + testCases := map[string]struct { + resources []runtime.Object + expectMetricsCommand func(*testing.T, []string) + expectUwlMetricsCommand func(*testing.T, []string) + }{ + "Should collect platform metrics": { + resources: []runtime.Object{ + newAllowListCm(operatorconfig.AllowlistConfigMapName, addonNamespace, map[string]operatorconfig.MetricsAllowlist{ + operatorconfig.MetricsConfigMapKey: { + NameList: []string{"a"}, + MatchList: []string{"b"}, + }, + }), + }, + expectMetricsCommand: func(t *testing.T, commands []string) { + assert.Contains(t, commands, `--match={__name__="a"}`) + }, + expectUwlMetricsCommand: func(t *testing.T, commands []string) { + assert.Len(t, commands, 0) + }, + }, + "Should collect uwl metrics": { + resources: []runtime.Object{ + newAllowListCm(operatorconfig.AllowlistConfigMapName, addonNamespace, map[string]operatorconfig.MetricsAllowlist{ + operatorconfig.UwlMetricsConfigMapKey: { + NameList: []string{"c"}, + }, + }), + newAllowListCm(operatorconfig.AllowlistCustomConfigMapName, addonNamespace, map[string]operatorconfig.MetricsAllowlist{ + operatorconfig.UwlMetricsConfigMapKey: { + NameList: []string{"b"}, + }, + }), + }, + expectMetricsCommand: func(t *testing.T, commands []string) { + assert.NotContains(t, commands, `--match={__name__="b"}`) + assert.NotContains(t, commands, `--match={__name__="c"}`) + }, + expectUwlMetricsCommand: func(t *testing.T, commands []string) { + assert.Contains(t, commands, `--match={__name__="b"}`) + assert.Contains(t, commands, `--match={__name__="c"}`) + }, + }, + "Should set recording rules": { + resources: []runtime.Object{ + newAllowListCm(operatorconfig.AllowlistConfigMapName, addonNamespace, map[string]operatorconfig.MetricsAllowlist{ + operatorconfig.MetricsConfigMapKey: { + RecordingRuleList: []operatorconfig.RecordingRule{ + { + Record: "test_record", + Expr: "test_expr", + }, + }, + }, + }), + newAllowListCm(operatorconfig.AllowlistCustomConfigMapName, addonNamespace, map[string]operatorconfig.MetricsAllowlist{ + operatorconfig.UwlMetricsConfigMapKey: { + RecordingRuleList: []operatorconfig.RecordingRule{ + { + Record: "test_record_b", + Expr: "test_expr_b", + }, + }, + }, + }), + }, + expectMetricsCommand: func(t *testing.T, commands []string) { + assert.Contains(t, commands, `--recordingrule={"name":"test_record","query":"test_expr"}`) + assert.NotContains(t, commands, `--recordingrule={"name":"test_record_b","query":"test_expr_b"}`) + }, + // expectUwlMetricsCommand: func(t *testing.T, commands []string) { + // assert.NotContains(t, commands, `--recordingrule={"name":"test_record","query":"test_expr"}`) + // assert.Contains(t, commands, `--recordingrule={"name":"test_record_b","query":"test_expr_b"}`) + // }, + }, + "Should ignore deprecated rules list field": { + resources: []runtime.Object{ + newAllowListCm(operatorconfig.AllowlistConfigMapName, addonNamespace, map[string]operatorconfig.MetricsAllowlist{ + operatorconfig.MetricsConfigMapKey: { + RuleList: []operatorconfig.RecordingRule{ + { + Record: "test_record", + Expr: "test_expr", + }, + }, + }, + }), + newAllowListCm(operatorconfig.AllowlistCustomConfigMapName, addonNamespace, map[string]operatorconfig.MetricsAllowlist{ + operatorconfig.UwlMetricsConfigMapKey: { + RuleList: []operatorconfig.RecordingRule{ + { + Record: "test_record_b", + Expr: "test_expr_b", + }, + }, + }, + }), + }, + expectMetricsCommand: func(t *testing.T, commands []string) { + assert.NotContains(t, commands, `--recordingrule={"name":"test_record","query":"test_expr"}`) + assert.NotContains(t, commands, `--recordingrule={"name":"test_record_b","query":"test_expr_b"}`) + }, + expectUwlMetricsCommand: func(t *testing.T, commands []string) { + assert.NotContains(t, commands, `--recordingrule={"name":"test_record","query":"test_expr"}`) + assert.NotContains(t, commands, `--recordingrule={"name":"test_record_b","query":"test_expr_b"}`) + }, + }, + "Should set rename map": { + resources: []runtime.Object{ + newAllowListCm(operatorconfig.AllowlistConfigMapName, addonNamespace, map[string]operatorconfig.MetricsAllowlist{ + operatorconfig.MetricsConfigMapKey: { + RenameMap: map[string]string{ + "old_name": "new_name", + }, + }, + }), + newAllowListCm(operatorconfig.AllowlistCustomConfigMapName, addonNamespace, map[string]operatorconfig.MetricsAllowlist{ + operatorconfig.UwlMetricsConfigMapKey: { + RenameMap: map[string]string{ + "old_name_b": "new_name_b", + }, + }, + }), + }, + expectMetricsCommand: func(t *testing.T, commands []string) { + assert.Contains(t, commands, `--rename="old_name=new_name"`) + assert.NotContains(t, commands, `--rename="old_name_b=new_name_b"`) + }, + // expectUwlMetricsCommand: func(t *testing.T, commands []string) { + // assert.NotContains(t, commands, `--rename={"old_name":"new_name"}`) + // assert.Contains(t, commands, `--rename={"old_name_b":"new_name_b"}`) + // }, + }, + "Should set collect rule groups": { + resources: []runtime.Object{ + newAllowListCm(operatorconfig.AllowlistConfigMapName, addonNamespace, map[string]operatorconfig.MetricsAllowlist{ + operatorconfig.MetricsConfigMapKey: { + CollectRuleGroupList: []operatorconfig.CollectRuleGroup{ + { + Name: "test_job", + Annotations: map[string]string{ + "test": "test", + }, + Selector: operatorconfig.CollectRuleSelector{ + MatchExpression: []metav1.LabelSelectorRequirement{ + { + Key: "clusterType", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"sno"}, + }, + }, + }, + CollectRuleList: []operatorconfig.CollectRule{ + { + Collect: "test_rule", + Expr: "test_expr", + For: "5m", + Annotations: map[string]string{ + "test": "test", + }, + Metrics: operatorconfig.DynamicMetrics{ + NameList: []string{"a"}, + MatchList: []string{"b"}, + }, + }, + }, + }, + { + Name: "test_job", + Annotations: map[string]string{ + "test": "test", + }, + Selector: operatorconfig.CollectRuleSelector{ + MatchExpression: []metav1.LabelSelectorRequirement{ + { + Key: "clusterType", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"not-sno"}, + }, + }, + }, + CollectRuleList: []operatorconfig.CollectRule{ + { + Collect: "not_sno_rule", + Expr: "test_expr", + For: "5m", + Annotations: map[string]string{ + "test": "test", + }, + Metrics: operatorconfig.DynamicMetrics{ + NameList: []string{"a"}, + MatchList: []string{"b"}, + }, + }, + }, + }, + }, + }, + }), + }, + expectMetricsCommand: func(t *testing.T, commands []string) { + assert.Contains(t, commands, `--collectrule={"name":"test_rule","expr":"test_expr","for":"5m","names":["a"],"matches":["b"]}`) + assert.NotContains(t, commands, `--collectrule={"name":"not_sno_rule","expr":"test_expr","for":"5m","names":["a"],"matches":["b"]}`) + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + resources := []runtime.Object{endpointOperatorDeploy, newPrometheusUwlResources()} + resources = append(resources, tc.resources...) + c := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(resources...).Build() + + metricsCollector := &collector.MetricsCollector{ + Client: c, + ClusterInfo: collector.ClusterInfo{ + ClusterID: "test-cluster", + ClusterType: "sno", + }, + HubInfo: &operatorconfig.HubInfo{ + ClusterName: "mycluster", + ObservatoriumAPIEndpoint: "http://test-endpoint", + }, + Log: logr.Logger{}, + Namespace: addonNamespace, + ObsAddon: &oav1beta1.ObservabilityAddon{ + ObjectMeta: metav1.ObjectMeta{ + Name: "observability-addon", + Namespace: addonNamespace, + }, + Spec: oashared.ObservabilityAddonSpec{}, + }, + ServiceAccountName: "test-sa", + } + if err := metricsCollector.Update(context.Background(), ctrl.Request{}); err != nil { + t.Fatalf("Failed to update metrics collector: %v", err) + } + + // Get metrics-collector and uwl metrics-collector deployments + metricsColDeploy := getMetricsCollectorDeployment(t, context.Background(), c, metricsCollectorName, addonNamespace) + metricsColCommands := metricsColDeploy.Spec.Template.Spec.Containers[0].Command + uwlMetricsColDeploy := getMetricsCollectorDeployment(t, context.Background(), c, uwlMetricsCollectorName, addonNamespace) + uwlMetricsColCommands := []string{} + if uwlMetricsColDeploy != nil { + uwlMetricsColCommands = uwlMetricsColDeploy.Spec.Template.Spec.Containers[0].Command + } + + if tc.expectMetricsCommand != nil { + tc.expectMetricsCommand(t, metricsColCommands) + } + + if tc.expectUwlMetricsCommand != nil { + tc.expectUwlMetricsCommand(t, uwlMetricsColCommands) + } + }) + } + +} + +func getEndpointOperatorDeployment(ns string) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "endpoint-observability-operator", - Namespace: namespace, + Namespace: ns, }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ @@ -283,9 +549,9 @@ func getEndpointOperatorDeployment() *appsv1.Deployment { } } -func getMetricsCollectorDeployment(t *testing.T, ctx context.Context, c client.Client, name string) *appsv1.Deployment { +func getMetricsCollectorDeployment(t *testing.T, ctx context.Context, c client.Client, name, ns string) *appsv1.Deployment { deployment := &appsv1.Deployment{} - err := c.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, deployment) + err := c.Get(ctx, types.NamespacedName{Name: name, Namespace: ns}, deployment) if err != nil { if errors.IsNotFound(err) { return nil @@ -322,6 +588,38 @@ func newAllowListCm(name, namespace string, data map[string]operatorconfig.Metri } } +func newPrometheusUwlResources() client.Object { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "prometheus-user-workload", + Namespace: "openshift-user-workload-monitoring", + }, + Spec: appsv1.StatefulSetSpec{ + // Replicas: util.Int32Ptr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "prometheus-user-workload", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "prometheus-user-workload", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "prometheus-user-workload", + Image: "test", + }, + }, + }, + }, + }, + } +} + func checkProxyEnvVars(expect, has []corev1.EnvVar) error { toCompare := map[string]string{"HTTP_PROXY": "", "HTTPS_PROXY": "", "NO_PROXY": "", "HTTPS_PROXY_CA_BUNDLE": ""} expectMap := make(map[string]string, len(toCompare)) diff --git a/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_observabilityaddons.yaml b/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_observabilityaddons.yaml index e08ba5f1f..130810b5f 100644 --- a/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_observabilityaddons.yaml +++ b/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_observabilityaddons.yaml @@ -22,29 +22,23 @@ spec: description: ObservabilityAddon is the Schema for the observabilityaddon API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object spec: - description: ObservabilityAddonSpec is the spec of observability addon. + description: ObservabilityAddonSpec is the spec of observability addon properties: enableMetrics: default: true - description: EnableMetrics indicates the observability addon push - metrics to hub server. + description: EnableMetrics indicates the observability addon push metrics to hub server. type: boolean interval: default: 300 - description: Interval for the observability addon push metrics to - hub server. + description: Interval for the observability addon push metrics to hub server. format: int32 maximum: 3600 minimum: 15 @@ -52,27 +46,6 @@ spec: resources: description: Resource requirement for metrics-collector properties: - claims: - description: "Claims lists the names of resources, defined in - spec.resourceClaims, that are used by this container. \n This - is an alpha field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable. It can only be set - for containers." - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name of one entry in pod.spec.resourceClaims - of the Pod where this field is used. It makes that resource - available inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -80,8 +53,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources - allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object requests: additionalProperties: @@ -90,11 +62,7 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute - resources required. If Requests is omitted for a container, - it defaults to Limits if that is explicitly specified, otherwise - to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object type: object @@ -103,8 +71,7 @@ spec: properties: conditions: items: - description: StatusCondition contains condition information for - an observability addon + description: StatusCondition contains condition information for an observability addon properties: lastTransitionTime: format: date-time @@ -133,3 +100,9 @@ spec: storage: true subresources: status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_integration_test.go b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_integration_test.go new file mode 100644 index 000000000..3dc7bccc6 --- /dev/null +++ b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_integration_test.go @@ -0,0 +1,269 @@ +// Copyright (c) Red Hat, Inc. +// Copyright Contributors to the Open Cluster Management project +// Licensed under the Apache License 2.0 + +//go:build integration + +package multiclusterobservability_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + ocinfrav1 "github.com/openshift/api/config/v1" + promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + observabilityshared "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/shared" + oav1beta1 "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/v1beta1" + mcov1beta2 "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/v1beta2" + "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/controllers/multiclusterobservability" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + kubescheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + clusterv1 "open-cluster-management.io/api/cluster/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +var ( + testEnvSpoke *envtest.Environment + restCfgSpoke *rest.Config + testEnvHub *envtest.Environment + restCfgHub *rest.Config +) + +// TestIntegrationReconcileAddon ensures that addon directives are applied to the resources in the cluster. +// This includes enabled/disabled, interval, and resources requirements. +func TestIntegrationReconcileMCO(t *testing.T) { + scheme := createBaseScheme() + assert.NoError(t, ocinfrav1.AddToScheme(scheme)) + assert.NoError(t, oav1beta1.AddToScheme(scheme)) + assert.NoError(t, mcov1beta2.AddToScheme(scheme)) + assert.NoError(t, clusterv1.AddToScheme(scheme)) + + // Setup spoke client and resources + k8sSpokeClient, err := client.New(restCfgSpoke, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + + spokeNamespace := "spoke-ns" + resources := []client.Object{ + newNamespace(spokeNamespace), + } + if err := createResources(k8sSpokeClient, resources...); err != nil { + t.Fatalf("Failed to create resources: %v", err) + } + + // Setup hub client and resources + k8sHubClient, err := client.New(restCfgHub, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + hubNamespace := "hub-ns" + resources = []client.Object{ + newNamespace(hubNamespace), + // add MCO + &mcov1beta2.MultiClusterObservability{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mco", + Namespace: hubNamespace, + }, + Spec: mcov1beta2.MultiClusterObservabilitySpec{ + ObservabilityAddonSpec: &observabilityshared.ObservabilityAddonSpec{ + Interval: 44, + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + }, + StorageConfig: &mcov1beta2.StorageConfig{ + MetricObjectStorage: &observabilityshared.PreConfiguredStorage{ + Key: "key", + Name: "name", + }, + }, + }, + }, + // add managedCluster + &clusterv1.ManagedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "managedcluster", + Namespace: "spoke-a", + }, + Spec: clusterv1.ManagedClusterSpec{}, + }, + } + if err := createResources(k8sHubClient, resources...); err != nil { + t.Fatalf("Failed to create resources: %v", err) + } + + // Setup the controller + mgr, err := ctrl.NewManager(testEnvSpoke.Config, ctrl.Options{ + Scheme: k8sSpokeClient.Scheme(), + Metrics: ctrlmetrics.Options{BindAddress: "0"}, // Avoids port conflict with the default port 8080 + }) + assert.NoError(t, err) + // type MultiClusterObservabilityReconciler struct { + // Manager manager.Manager + // Client client.Client + // Log logr.Logger + // Scheme *runtime.Scheme + // CRDMap map[string]bool + // APIReader client.Reader + // RESTMapper meta.RESTMapper + // } + // Manager: mgr, + // Client: mgr.GetClient(), + // Log: ctrl.Log.WithName("controllers").WithName("MultiClusterObservability"), + // Scheme: mgr.GetScheme(), + // CRDMap: crdMaps, + // APIReader: mgr.GetAPIReader(), + // RESTMapper: mgr.GetRESTMapper(), + reconciler := multiclusterobservability.MultiClusterObservabilityReconciler{ + Client: k8sHubClient, + Manager: mgr, + Log: ctrl.Log.WithName("controllers").WithName("MultiClusterObservability"), + Scheme: scheme, + CRDMap: nil, + APIReader: nil, + RESTMapper: mgr.GetRESTMapper(), + } + err = reconciler.SetupWithManager(mgr) + assert.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + err = mgr.Start(ctx) + assert.NoError(t, err) + }() + + // ensure the MCO fields are replicated to the observability addon + +} + +func TestMain(m *testing.M) { + opts := zap.Options{ + Development: true, + } + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + rootPath := filepath.Join("..", "..", "..") + // spokeCrds := readCRDFiles( + // filepath.Join(rootPath, "multiclusterobservability", "config", "crd", "bases", "observability.open-cluster-management.io_observabilityaddons.yaml"), + // filepath.Join(rootPath, "endpointmetrics", "manifests", "prometheus", "crd", "servicemonitor_crd_0_53_1.yaml"), + // filepath.Join(rootPath, "endpointmetrics", "manifests", "prometheus", "crd", "prometheusrule_crd_0_53_1.yaml"), + // ) + testEnvSpoke = &envtest.Environment{ + // CRDDirectoryPaths: []string{filepath.Join("testdata", "crd")}, + // CRDs: spokeCrds, + ControlPlaneStopTimeout: 5 * time.Minute, + } + + var err error + restCfgSpoke, err = testEnvSpoke.Start() + if err != nil { + panic(fmt.Sprintf("Failed to start spoke test environment: %v", err)) + } + + hubCRDs := readCRDFiles( + // filepath.Join(rootPath, "multiclusterobservability", "config", "crd", "bases", "observability.open-cluster-management.io_observabilityaddons.yaml"), + filepath.Join(rootPath, "multiclusterobservability", "config", "crd", "bases", "observability.open-cluster-management.io_multiclusterobservabilities.yaml"), + // filepath.Join(rootPath, "endpointmetrics", "manifests", "prometheus", "crd", "servicemonitor_crd_0_53_1.yaml"), + ) + + testEnvHub = &envtest.Environment{ + // CRDDirectoryPaths: []string{filepath.Join("testdata", "crd")}, + CRDs: hubCRDs, + ControlPlaneStopTimeout: 5 * time.Minute, + } + + restCfgHub, err = testEnvHub.Start() + if err != nil { + panic(fmt.Sprintf("Failed to start hub test environment: %v", err)) + } + + code := m.Run() + + err = testEnvSpoke.Stop() + if err != nil { + panic(fmt.Sprintf("Failed to stop spoke test environment: %v", err)) + } + + err = testEnvHub.Stop() + if err != nil { + panic(fmt.Sprintf("Failed to stop hub test environment: %v", err)) + } + + os.Exit(code) +} + +func createBaseScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + kubescheme.AddToScheme(scheme) + promv1.AddToScheme(scheme) + oav1beta1.AddToScheme(scheme) + mcov1beta2.AddToScheme(scheme) + return scheme +} + +// createResources creates the given resources in the cluster. +func createResources(client client.Client, resources ...client.Object) error { + for _, resource := range resources { + if err := client.Create(context.Background(), resource); err != nil { + return err + } + } + return nil +} + +func readCRDFiles(crdPaths ...string) []*apiextensionsv1.CustomResourceDefinition { + ret := []*apiextensionsv1.CustomResourceDefinition{} + + for _, crdPath := range crdPaths { + crdYamlData, err := os.ReadFile(crdPath) + if err != nil { + panic(fmt.Sprintf("Failed to read CRD file: %v", err)) + } + + dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + var crd apiextensionsv1.CustomResourceDefinition + _, _, err = dec.Decode(crdYamlData, nil, &crd) + if err != nil { + panic(fmt.Sprintf("Failed to decode CRD: %v", err)) + } + + ret = append(ret, &crd) + } + + return ret +} + +func newNamespace(name string) *corev1.Namespace { + return &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } +} diff --git a/operators/pkg/util/allowlist.go b/operators/pkg/util/allowlist.go index a3d844b9d..48558cff8 100644 --- a/operators/pkg/util/allowlist.go +++ b/operators/pkg/util/allowlist.go @@ -70,6 +70,9 @@ func MergeAllowlist(allowlist, customAllowlist, ocp3Allowlist, uwlAllowlist, //check if rules are specified for backward compatibility allowlist.RecordingRuleList = append(allowlist.RecordingRuleList, customAllowlist.RuleList...) } + if customAllowlist.RenameMap == nil { + customAllowlist.RenameMap = make(map[string]string) + } for k, v := range customAllowlist.RenameMap { allowlist.RenameMap[k] = v } @@ -77,6 +80,9 @@ func MergeAllowlist(allowlist, customAllowlist, ocp3Allowlist, uwlAllowlist, ocp3Allowlist.NameList = mergeMetrics(ocp3Allowlist.NameList, customAllowlist.NameList) ocp3Allowlist.MatchList = mergeMetrics(ocp3Allowlist.MatchList, customAllowlist.MatchList) ocp3Allowlist.RuleList = append(ocp3Allowlist.RuleList, customAllowlist.RuleList...) + if ocp3Allowlist.RenameMap == nil { + ocp3Allowlist.RenameMap = make(map[string]string) + } for k, v := range customAllowlist.RenameMap { ocp3Allowlist.RenameMap[k] = v } @@ -84,6 +90,9 @@ func MergeAllowlist(allowlist, customAllowlist, ocp3Allowlist, uwlAllowlist, uwlAllowlist.NameList = mergeMetrics(uwlAllowlist.NameList, customUwlAllowlist.NameList) uwlAllowlist.MatchList = mergeMetrics(uwlAllowlist.MatchList, customUwlAllowlist.MatchList) uwlAllowlist.RuleList = append(uwlAllowlist.RuleList, customUwlAllowlist.RuleList...) + if uwlAllowlist.RenameMap == nil { + uwlAllowlist.RenameMap = make(map[string]string) + } for k, v := range customUwlAllowlist.RenameMap { uwlAllowlist.RenameMap[k] = v } diff --git a/tests/pkg/tests/observability_addon_test.go b/tests/pkg/tests/observability_addon_test.go index f9ce37591..4ed66eb72 100644 --- a/tests/pkg/tests/observability_addon_test.go +++ b/tests/pkg/tests/observability_addon_test.go @@ -6,13 +6,11 @@ package tests import ( "fmt" - "strings" "errors" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "k8s.io/klog" "github.com/stolostron/multicluster-observability-operator/tests/pkg/utils" ) @@ -56,14 +54,14 @@ var _ = Describe("Observability:", func() { Expect(requests["memory"]).To(Equal("100Mi")) }) - It("[Stable] Should have resource requirement in metrics-collector", func() { - By("Check metrics-collector resource requirement") - Eventually(func() error { - return utils.CheckMCOAddonResources(testOptions) - }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) - }) + // It("[Stable] Should have resource requirement in metrics-collector", func() { // move to endpoint ocp unit tests: resources req are applied to the metrics-collector pod + // By("Check metrics-collector resource requirement") + // Eventually(func() error { + // return utils.CheckMCOAddonResources(testOptions) + // }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(Succeed()) + // }) - It("[Stable] Should not have the expected MCO addon pods when disable observabilityaddon", func() { + It("[Stable] Should not have the expected MCO addon pods when disable observabilityaddon", func() { // move to endpoint ocp unit tests: if disabled, no pod Eventually(func() error { return utils.ModifyMCOAddonSpecMetrics(testOptions, false) }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) @@ -102,7 +100,7 @@ var _ = Describe("Observability:", func() { // the corret way is use timestamp, for example: // timestamp(node_memory_MemAvailable_bytes{cluster="local-cluster"}) - // timestamp(node_memory_MemAvailable_bytes{cluster="local-cluster"} offset 1m) > 59 - It("[Stable] Waiting for check no metric data in grafana console", func() { + It("[Stable] Waiting for check no metric data in grafana console", func() { // TODO Eventually(func() error { for _, cluster := range clusters { res, err := utils.QueryGrafana( @@ -120,7 +118,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*2, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[Stable] Modifying MCO cr to enable observabilityaddon", func() { + It("[Stable] Modifying MCO cr to enable observabilityaddon", func() { // To DELETE? Eventually(func() error { return utils.ModifyMCOAddonSpecMetrics(testOptions, true) }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) @@ -140,42 +138,18 @@ var _ = Describe("Observability:", func() { return false }, EventuallyTimeoutMinute*6, EventuallyIntervalSecond*5).Should(BeTrue()) - By("Checking the status in managedclusteraddon reflects the endpoint operator status correctly") - Eventually(func() error { - err = utils.CheckAllOBAsEnabled(testOptions) - if err != nil { - return err - } - return nil - }, EventuallyTimeoutMinute*15, EventuallyIntervalSecond*5).Should(Succeed()) + // By("Checking the status in managedclusteraddon reflects the endpoint operator status correctly") // Make sure it is covered in unit tests (enpoint-op, collector) and e2e tests + // Eventually(func() error { + // err = utils.CheckAllOBAsEnabled(testOptions) + // if err != nil { + // return err + // } + // return nil + // }, EventuallyTimeoutMinute*15, EventuallyIntervalSecond*5).Should(Succeed()) }) }) - It("[P3][Sev3][observability][Stable] Should not set interval to values beyond scope (addon/g0)", func() { - By("Set interval to 14") - Eventually(func() bool { - err := utils.ModifyMCOAddonSpecInterval(testOptions, int64(14)) - if strings.Contains(err.Error(), "Invalid value") && - strings.Contains(err.Error(), "15") { - return true - } - klog.V(1).Infof("error message: <%s>\n", err.Error()) - return false - }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(BeTrue()) - - By("Set interval to 3601") - Eventually(func() bool { - err := utils.ModifyMCOAddonSpecInterval(testOptions, int64(3601)) - if strings.Contains(err.Error(), "Invalid value") && - strings.Contains(err.Error(), "3600") { - return true - } - klog.V(1).Infof("error message: <%s>\n", err.Error()) - return false - }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(BeTrue()) - }) - - Context("[P2][Sev2][observability] Should not have the expected MCO addon pods when disable observability from managedcluster (addon/g0) -", func() { + Context("[P2][Sev2][observability] Should not have the expected MCO addon pods when disable observability from managedcluster (addon/g0) -", func() { // To delete? It("[Stable] Modifying managedcluster cr to disable observability", func() { Eventually(func() error { return utils.UpdateObservabilityFromManagedCluster(testOptions, false) diff --git a/tests/pkg/tests/observability_alert_test.go b/tests/pkg/tests/observability_alert_test.go index 0d21d1aa1..e7f2d5af9 100644 --- a/tests/pkg/tests/observability_alert_test.go +++ b/tests/pkg/tests/observability_alert_test.go @@ -55,7 +55,7 @@ var _ = Describe("Observability:", func() { } secret := "alertmanager-config" - It("@BVT - [P1][Sev1][observability][Stable] Should have the expected statefulsets (alert/g0)", func() { + It("@BVT - [P1][Sev1][observability][Stable] Should have the expected statefulsets (alert/g0)", func() { // move all to integration By("Checking if STS: Alertmanager and observability-thanos-rule exist") for _, label := range statefulsetLabels { sts, err := hubClient.AppsV1(). diff --git a/tests/pkg/tests/observability_certrenew_test.go b/tests/pkg/tests/observability_certrenew_test.go index cdebc9b8c..f164fd615 100644 --- a/tests/pkg/tests/observability_certrenew_test.go +++ b/tests/pkg/tests/observability_certrenew_test.go @@ -33,7 +33,7 @@ var _ = Describe("Observability:", func() { } }) - It("[P1][Sev1][observability][Integration] Should have metrics collector pod restart if cert secret re-generated (certrenew/g0)", func() { + It("[P1][Sev1][observability][Integration] Should have metrics collector pod restart if cert secret re-generated (certrenew/g0)", func() { // unit tests By("Waiting for pods ready: observability-observatorium-api, observability-rbac-query-proxy, metrics-collector-deployment") // sleep 30s to wait for installation is ready time.Sleep(30 * time.Second) @@ -141,40 +141,6 @@ var _ = Describe("Observability:", func() { return false }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(BeTrue()) - - By(fmt.Sprintf("Waiting for old pod <%s> removed and new pod created", collectorPodName)) - Eventually(func() bool { - err, podList := utils.GetPodList( - testOptions, - isHub, - namespace, - "component=metrics-collector", - ) - if err != nil { - klog.V(1).Infof("Failed to get pod list: %v", err) - } - for _, pod := range podList.Items { - if pod.Name != collectorPodName { - if pod.Status.Phase != "Running" { - klog.V(1).Infof("<%s> not in Running status yet", pod.Name) - return false - } - return true - } - } - - // debug code to check label "cert/time-restarted" - deployment, err := utils.GetDeployment( - testOptions, - isHub, - "metrics-collector-deployment", - namespace, - ) - if err == nil { - klog.V(1).Infof("labels: <%v>", deployment.Spec.Template.ObjectMeta.Labels) - } - return false - }, EventuallyTimeoutMinute*5, EventuallyIntervalSecond*5).Should(BeTrue()) }) JustAfterEach(func() { diff --git a/tests/pkg/tests/observability_config_test.go b/tests/pkg/tests/observability_config_test.go index 10336a7ed..9da3a83c8 100644 --- a/tests/pkg/tests/observability_config_test.go +++ b/tests/pkg/tests/observability_config_test.go @@ -153,7 +153,7 @@ var _ = Describe("Observability:", func() { }, } - It("@BVT - [P1][Sev1][observability][Integration] Checking replicas in advanced config for each component (config/g0)", func() { + It("@BVT - [P1][Sev1][observability][Integration] Checking replicas in advanced config for each component (config/g0)", func() { // move to unit tests or integration ALL mcoRes, err := dynClient.Resource(utils.NewMCOGVRV1BETA2()). Get(context.TODO(), MCO_CR_NAME, metav1.GetOptions{}) diff --git a/tests/pkg/tests/observability_dashboard_test.go b/tests/pkg/tests/observability_dashboard_test.go index e43ee580a..1ba2dfdaf 100644 --- a/tests/pkg/tests/observability_dashboard_test.go +++ b/tests/pkg/tests/observability_dashboard_test.go @@ -33,7 +33,7 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.KubeContext) }) - It("[P2][Sev2][observability][Stable] Should have custom dashboard which defined in configmap (dashboard/g0)", func() { + It("[P2][Sev2][observability][Stable] Should have custom dashboard which defined in configmap (dashboard/g0)", func() { // move to integration By("Creating custom dashboard configmap") yamlB, _ := kustomize.Render( kustomize.Options{KustomizationPath: "../../../examples/dashboards/sample_custom_dashboard"}, diff --git a/tests/pkg/tests/observability_endpoint_preserve_test.go b/tests/pkg/tests/observability_endpoint_preserve_test.go index c79d31332..49ae75da7 100644 --- a/tests/pkg/tests/observability_endpoint_preserve_test.go +++ b/tests/pkg/tests/observability_endpoint_preserve_test.go @@ -77,7 +77,7 @@ var _ = Describe("Observability:", func() { return false }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(BeTrue()) }) - It("[Stable] Updating metrics-collector deployment", func() { + It("[Stable] Updating metrics-collector deployment", func() { // TO DELETE or move to integration updateSaName := "test-serviceaccount" Eventually(func() error { newDep, err = utils.GetDeployment( @@ -118,7 +118,7 @@ var _ = Describe("Observability:", func() { }) }) - It("[P2][Sev2][observability][Stable] Should revert any manual changes on metrics-collector-view clusterolebinding (endpoint_preserve/g0)", func() { + It("[P2][Sev2][observability][Stable] Should revert any manual changes on metrics-collector-view clusterolebinding (endpoint_preserve/g0)", func() { // TO DELETE or move to integration if os.Getenv("IS_KIND_ENV") == trueStr { Skip("Skip the case due to run in KinD") } @@ -155,7 +155,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*1).Should(BeTrue()) }) - It("[P2][Sev2][observability][Stable] Should recreate on metrics-collector-serving-certs-ca-bundle configmap if deleted (endpoint_preserve/g0)", func() { + It("[P2][Sev2][observability][Stable] Should recreate on metrics-collector-serving-certs-ca-bundle configmap if deleted (endpoint_preserve/g0)", func() { // TO DELETE or move to integration if os.Getenv("IS_KIND_ENV") == trueStr { Skip("Skip the case due to run in KinD") } diff --git a/tests/pkg/tests/observability_export_test.go b/tests/pkg/tests/observability_export_test.go index 508ca1159..e49974fab 100644 --- a/tests/pkg/tests/observability_export_test.go +++ b/tests/pkg/tests/observability_export_test.go @@ -38,7 +38,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*6, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Integration] Should have acm_remote_write_requests_total metrics with correct labels/value (export/g0)", func() { + It("[P2][Sev2][observability][Integration] Should have acm_remote_write_requests_total metrics with correct labels/value (export/g0)", func() { // merge By("Adding victoriametrics deployment/service/secret") yamlB, err := kustomize.Render(kustomize.Options{KustomizationPath: "../../../examples/export"}) Expect(err).ToNot(HaveOccurred()) diff --git a/tests/pkg/tests/observability_grafana_test.go b/tests/pkg/tests/observability_grafana_test.go index 908351401..326251458 100644 --- a/tests/pkg/tests/observability_grafana_test.go +++ b/tests/pkg/tests/observability_grafana_test.go @@ -26,7 +26,7 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.KubeContext) }) - It("@BVT - [P1][Sev1][observability][Stable] Should have metric data in grafana console (grafana/g0)", func() { + It("@BVT - [P1][Sev1][observability][Stable] Should have metric data in grafana console (grafana/g0)", func() { // redundant with metrics forwarded correctly check Eventually(func() error { clusters, err := utils.ListManagedClusters(testOptions) if err != nil { diff --git a/tests/pkg/tests/observability_manifestwork_test.go b/tests/pkg/tests/observability_manifestwork_test.go index 193abd615..bc66e3fba 100644 --- a/tests/pkg/tests/observability_manifestwork_test.go +++ b/tests/pkg/tests/observability_manifestwork_test.go @@ -32,7 +32,7 @@ var _ = Describe("Observability:", func() { } }) - Context("[P2][Sev2][observability][Stable] Should be automatically created within 1 minute when delete manifestwork (manifestwork/g0) -", func() { + Context("[P2][Sev2][observability][Stable] Should be automatically created within 1 minute when delete manifestwork (manifestwork/g0) -", func() { // move to unit tests or integration ALL manifestWorkName := "endpoint-observability-work" clientDynamic := utils.GetKubeClientDynamic(testOptions, true) clusterName := utils.GetManagedClusterName(testOptions) diff --git a/tests/pkg/tests/observability_metrics_test.go b/tests/pkg/tests/observability_metrics_test.go index 1b8224b3a..4fc04f680 100644 --- a/tests/pkg/tests/observability_metrics_test.go +++ b/tests/pkg/tests/observability_metrics_test.go @@ -50,7 +50,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*6, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Integration] Should have metrics which defined in custom metrics allowlist (metrics/g0)", func() { + It("[P2][Sev2][observability][Integration] Should have metrics which defined in custom metrics allowlist (metrics/g0)", func() { // TO KEEP By("Adding custom metrics allowlist configmap") yamlB, err := kustomize.Render(kustomize.Options{KustomizationPath: "../../../examples/metrics/allowlist"}) Expect(err).ToNot(HaveOccurred()) diff --git a/tests/pkg/tests/observability_observatorium_preserve_test.go b/tests/pkg/tests/observability_observatorium_preserve_test.go index b1bbd2aab..03ef311df 100644 --- a/tests/pkg/tests/observability_observatorium_preserve_test.go +++ b/tests/pkg/tests/observability_observatorium_preserve_test.go @@ -29,7 +29,7 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.KubeContext) }) - Context("[P1][Sev1][observability] Should revert any manual changes on observatorium cr (observatorium_preserve/g0) -", func() { + Context("[P1][Sev1][observability] Should revert any manual changes on observatorium cr (observatorium_preserve/g0) -", func() { // move to unit tests or integration ALL It("[Stable] Updating observatorium cr (spec.thanos.compact.retentionResolution1h) should be automatically reverted", func() { oldCRResourceVersion := "" updateRetention := "10d" diff --git a/tests/pkg/tests/observability_reconcile_test.go b/tests/pkg/tests/observability_reconcile_test.go index c37470a64..7cb706c3c 100644 --- a/tests/pkg/tests/observability_reconcile_test.go +++ b/tests/pkg/tests/observability_reconcile_test.go @@ -41,7 +41,7 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.KubeContext) }) - It("[P2][Sev2][observability][Stable] Modifying MCO CR for reconciling (reconcile/g0)", func() { + It("[P2][Sev2][observability][Stable] Modifying MCO CR for reconciling (reconcile/g0)", func() { // move to unit tests or integration ALL By("Modifying MCO CR for reconciling") err := utils.ModifyMCOCR(testOptions) Expect(err).ToNot(HaveOccurred()) diff --git a/tests/pkg/tests/observability_retention_test.go b/tests/pkg/tests/observability_retention_test.go index acd9ea220..b0e714cb5 100644 --- a/tests/pkg/tests/observability_retention_test.go +++ b/tests/pkg/tests/observability_retention_test.go @@ -64,7 +64,7 @@ var _ = Describe("Observability:", func() { } }) - It("[P2][Sev2][observability][Stable] Check compact args (retention/g0):", func() { + It("[P2][Sev2][observability][Stable] Check compact args (retention/g0):", func() { // move to unit tests or integration ALL By("--delete-delay=" + deleteDelay) Eventually(func() error { compacts, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ @@ -83,7 +83,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Check store args (retention/g0):", func() { + It("[P2][Sev2][observability][Stable] Check store args (retention/g0):", func() { // move to unit tests or integration By("--ignore-deletion-marks-delay=" + ignoreDeletionMarksDelay) Eventually(func() error { stores, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ @@ -105,7 +105,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Check receive args (retention/g0):", func() { + It("[P2][Sev2][observability][Stable] Check receive args (retention/g0):", func() { // move to unit tests or integration By("--tsdb.retention=" + retentionInLocal) Eventually(func() error { receives, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ @@ -127,7 +127,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Check rule args (retention/g0):", func() { + It("[P2][Sev2][observability][Stable] Check rule args (retention/g0):", func() { // move to unit tests or integration By("--tsdb.retention=" + retentionInLocal) Eventually(func() error { rules, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ @@ -149,7 +149,7 @@ var _ = Describe("Observability:", func() { }, EventuallyTimeoutMinute*1, EventuallyIntervalSecond*5).Should(Succeed()) }) - It("[P2][Sev2][observability][Stable] Check rule args (retention/g0):", func() { + It("[P2][Sev2][observability][Stable] Check rule args (retention/g0):", func() { // move to unit tests or integration By("--tsdb.block-duration=" + blockDuration) Eventually(func() error { rules, err := hubClient.AppsV1().StatefulSets(MCO_NAMESPACE).List(context.TODO(), metav1.ListOptions{ diff --git a/tests/pkg/tests/observability_route_test.go b/tests/pkg/tests/observability_route_test.go index 9b01930e9..d98acba27 100644 --- a/tests/pkg/tests/observability_route_test.go +++ b/tests/pkg/tests/observability_route_test.go @@ -40,7 +40,7 @@ var _ = Describe("Observability:", func() { testOptions.HubCluster.KubeContext) }) - It("@BVT - [P1][Sev1][observability][Integration] Should access metrics via rbac-query-proxy route (route/g0)", func() { + It("@BVT - [P1][Sev1][observability][Integration] Should access metrics via rbac-query-proxy route (route/g0)", func() { // to keep somewhat Eventually(func() error { query := "/api/v1/query?query=cluster_version" url := "https://rbac-query-proxy-open-cluster-management-observability.apps." + testOptions.HubCluster.BaseDomain + query