diff --git a/environment/eksdeploymenttype/eks_deployment_type.go b/environment/eksdeploymenttype/eks_deployment_type.go index 44c98998f..0bd1ad301 100644 --- a/environment/eksdeploymenttype/eks_deployment_type.go +++ b/environment/eksdeploymenttype/eks_deployment_type.go @@ -11,17 +11,19 @@ const ( DAEMON EKSDeploymentType = "DAEMON" REPLICA EKSDeploymentType = "REPLICA" SIDECAR EKSDeploymentType = "SIDECAR" + STATEFUL EKSDeploymentType = "STATEFUL" ) var ( - ecsDeploymentTypes = map[string]EKSDeploymentType{ + eksDeploymentTypes = map[string]EKSDeploymentType{ "DAEMON": DAEMON, "REPLICA": REPLICA, "SIDECAR": SIDECAR, + "STATEFUL": STATEFUL, } ) func FromString(str string) (EKSDeploymentType, bool) { - c, ok := ecsDeploymentTypes[strings.ToUpper(str)] + c, ok := eksDeploymentTypes[strings.ToUpper(str)] return c, ok } diff --git a/environment/metadata.go b/environment/metadata.go index f7c03fc77..ae7aa3174 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -88,7 +88,7 @@ type MetaDataStrings struct { AmpWorkspaceId string Region string K8sVersion string - Destroy string + Destroy bool HelmChartsBranch string CloudwatchAgentRepository string CloudwatchAgentTag string @@ -138,7 +138,7 @@ func registerEKSData(d *MetaDataStrings) { func registerEKSE2ETestData(dataString *MetaDataStrings) { flag.StringVar(&(dataString.Region), "region", "", "AWS region") flag.StringVar(&(dataString.K8sVersion), "k8s_version", "", "Kubernetes version") - flag.StringVar(&(dataString.Destroy), "destroy", "false", "Whether to run in destroy mode (true/false)") + flag.BoolVar(&(dataString.Destroy), "destroy", false, "Whether to run in destroy mode (true/false)") flag.StringVar(&(dataString.HelmChartsBranch), "helm_charts_branch", "", "Helm charts branch") flag.StringVar(&(dataString.CloudwatchAgentRepository), "cloudwatch_agent_repository", "", "CloudWatch Agent repository") flag.StringVar(&(dataString.CloudwatchAgentTag), "cloudwatch_agent_tag", "", "CloudWatch Agent tag") @@ -321,7 +321,7 @@ func GetEnvironmentMetaData() *MetaData { metaDataStorage.AmpWorkspaceId = registeredMetaDataStrings.AmpWorkspaceId metaDataStorage.Region = registeredMetaDataStrings.Region metaDataStorage.K8sVersion = registeredMetaDataStrings.K8sVersion - metaDataStorage.Destroy = registeredMetaDataStrings.Destroy == "true" + metaDataStorage.Destroy = registeredMetaDataStrings.Destroy metaDataStorage.HelmChartsBranch = registeredMetaDataStrings.HelmChartsBranch metaDataStorage.CloudwatchAgentRepository = registeredMetaDataStrings.CloudwatchAgentRepository metaDataStorage.CloudwatchAgentTag = registeredMetaDataStrings.CloudwatchAgentTag diff --git a/generator/resources/eks_e2e_jmx_test_matrix.json b/generator/resources/eks_e2e_jmx_test_matrix.json index d484997f7..cf999e588 100644 --- a/generator/resources/eks_e2e_jmx_test_matrix.json +++ b/generator/resources/eks_e2e_jmx_test_matrix.json @@ -1,5 +1,5 @@ [ { - "nodes": 1 + "nodes": 2 } ] \ No newline at end of file diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index bfd0c8dfe..4364696ab 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -39,7 +39,7 @@ resource "aws_eks_node_group" "this" { min_size = var.nodes } - ami_type = var.ami_type + ami_type = "AL2_x86_64" capacity_type = "ON_DEMAND" disk_size = 20 instance_types = [var.instance_type] @@ -155,7 +155,7 @@ resource "null_resource" "validator" { command = <<-EOT echo "Running cleanup for K8s resources" go test -timeout 30m -v ${self.triggers.test_dir} \ - -destroy=true \ + -destroy \ -region=${self.triggers.region} \ -eksClusterName=${self.triggers.cluster_name} \ -computeType=EKS \ diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index 016bbb318..67af48c2e 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -18,12 +18,7 @@ variable "cluster_name" { variable "nodes" { type = number - default = 1 -} - -variable "ami_type" { - type = string - default = "AL2_x86_64" + default = 2 } variable "instance_type" { diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index a2d5ccb2b..f2af0771b 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -4,36 +4,28 @@ package main import ( - "context" "flag" "fmt" - "net/http" "os" - "os/exec" "path/filepath" "strings" "testing" "time" "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/e2e" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" "github.com/aws/amazon-cloudwatch-agent-test/util/common" ) //------------------------------------------------------------------------------ -// Constants and Variables +// Variables //------------------------------------------------------------------------------ -const ( - wait = 5 * time.Minute - interval = 30 * time.Second -) - var ( nodeNames []string env *environment.MetaData @@ -145,7 +137,7 @@ func testMetrics(t *testing.T) { tests := testMetricsRegistry[configFile] fmt.Println("waiting for metrics to propagate...") - time.Sleep(wait) + time.Sleep(e2e.Wait) for _, testFunc := range tests { testFunc(t) @@ -158,37 +150,13 @@ func testMetrics(t *testing.T) { func testAgentResources(t *testing.T, clientset *kubernetes.Clientset) { t.Run("verify_agent_resources", func(t *testing.T) { - daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent DaemonSet") - require.NotNil(t, daemonSet, "CloudWatch Agent DaemonSet not found") - - configMap, err := clientset.CoreV1().ConfigMaps("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent ConfigMap") - require.NotNil(t, configMap, "CloudWatch Agent ConfigMap not found") - - cwConfig, exists := configMap.Data["cwagentconfig.json"] - require.True(t, exists, "cwagentconfig.json not found in ConfigMap") - require.Contains(t, cwConfig, `jmx`, "JMX configuration not found in cwagentconfig.json") - - service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent Service") - require.NotNil(t, service, "CloudWatch Agent Service not found") - - serviceAccount, err := clientset.CoreV1().ServiceAccounts("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent Service Account") - require.NotNil(t, serviceAccount, "CloudWatch Agent Service Account not found") + e2e.VerifyAgentResources(t, clientset, "jmx") }) } func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) { t.Run("verify_jmx_resources", func(t *testing.T) { deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml") - pods, err := clientset.CoreV1().Pods("test").List(context.TODO(), metav1.ListOptions{ - LabelSelector: fmt.Sprintf("app=%s", deploymentName), - FieldSelector: "status.phase=Running", - }) - require.NoError(t, err, "Error getting pods for deployment") - require.NotEmpty(t, pods.Items, "No pods found for deployment") var jmxTargetSystem string switch filepath.Base(env.AgentConfig) { @@ -208,16 +176,7 @@ func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) { "JAVA_TOOL_OPTIONS": " -javaagent:/otel-auto-instrumentation-java/javaagent.jar", } - for _, container := range pods.Items[0].Spec.Containers { - for _, envVar := range container.Env { - if expectedValue, exists := requiredEnvVars[envVar.Name]; exists { - require.Equal(t, expectedValue, envVar.Value, fmt.Sprintf("Unexpected value for environment variable %s in container %s", envVar.Name, container.Name)) - delete(requiredEnvVars, envVar.Name) - } - } - } - - require.Empty(t, requiredEnvVars, "Not all required environment variables were found in the pod") + e2e.VerifyPodEnvironment(t, clientset, deploymentName, requiredEnvVars) }) } @@ -227,7 +186,7 @@ func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) { func testTomcatMetrics(t *testing.T) { t.Run("verify_jvm_tomcat_metrics", func(t *testing.T) { - validateMetrics(t, []string{ + e2e.ValidateMetrics(t, []string{ "jvm.classes.loaded", "jvm.gc.collections.count", "jvm.gc.collections.elapsed", @@ -257,15 +216,15 @@ func testTomcatMetrics(t *testing.T) { func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { - generateTraffic(t) - time.Sleep(wait) + e2e.GenerateTraffic(t) + time.Sleep(e2e.Wait) verifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E", false) }) } func testKafkaMetrics(t *testing.T) { t.Run("verify_kafka_metrics", func(t *testing.T) { - validateMetrics(t, []string{ + e2e.ValidateMetrics(t, []string{ "kafka.message.count", "kafka.request.count", "kafka.request.failed", @@ -285,7 +244,7 @@ func testKafkaMetrics(t *testing.T) { func testContainerInsightsMetrics(t *testing.T) { t.Run("verify_containerinsights_metrics", func(t *testing.T) { - validateMetrics(t, []string{ + e2e.ValidateMetrics(t, []string{ "jvm_classes_loaded", "jvm_threads_current", "jvm_threads_daemon", @@ -310,8 +269,8 @@ func testContainerInsightsMetrics(t *testing.T) { func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { - generateTraffic(t) - time.Sleep(wait) + e2e.GenerateTraffic(t) + time.Sleep(e2e.Wait) verifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus", true) }) } @@ -320,34 +279,8 @@ func testTomcatRejectedSessions(t *testing.T) { // Helper Functions //------------------------------------------------------------------------------ -func validateMetrics(t *testing.T, metrics []string, namespace string) { - for _, metric := range metrics { - t.Run(metric, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric, namespace, nil, 5, interval) - }) - } -} - -func generateTraffic(t *testing.T) { - cmd := exec.Command("kubectl", "get", "nodes", "-o", "jsonpath='{.items[0].status.addresses[?(@.type==\"ExternalIP\")].address}'") - output, err := cmd.CombinedOutput() - require.NoError(t, err, "Error getting node external IP") - - nodeIP := strings.Trim(string(output), "'") - require.NotEmpty(t, nodeIP, "Node IP failed to format") - - for i := 0; i < 5; i++ { - resp, err := http.Get(fmt.Sprintf("http://%s:30080/webapp/index.jsp", nodeIP)) - if err != nil { - t.Logf("Request attempt %d failed: %v", i+1, err) - continue - } - require.NoError(t, resp.Body.Close(), "Failed to close response body") - } -} - func verifyMetricAboveZero(t *testing.T, metricName, namespace string, containerInsights bool) { - startTime := time.Now().Add(-wait) + startTime := time.Now().Add(-e2e.Wait) endTime := time.Now() aboveZero, err := awsservice.CheckMetricAboveZero( @@ -361,4 +294,4 @@ func verifyMetricAboveZero(t *testing.T, metricName, namespace string, container ) require.NoError(t, err, "Failed to check metric above zero") require.True(t, aboveZero, fmt.Sprintf("Expected non-zero %s after applying traffic", metricName)) -} +} \ No newline at end of file diff --git a/test/e2e/testutils.go b/test/e2e/testutils.go new file mode 100644 index 000000000..c44d2189d --- /dev/null +++ b/test/e2e/testutils.go @@ -0,0 +1,107 @@ +package e2e + +import ( + "context" + "fmt" + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "net/http" + "os/exec" + "strings" + "testing" + "time" +) + +//------------------------------------------------------------------------------ +// Constants +//------------------------------------------------------------------------------ + +const ( + Wait = 5 * time.Minute + interval = 30 * time.Second +) + +//------------------------------------------------------------------------------ +// Resource Functions +//------------------------------------------------------------------------------ + +func VerifyPodEnvironment(t *testing.T, clientset *kubernetes.Clientset, deploymentName string, requiredEnvVars map[string]string) { + pods, err := clientset.CoreV1().Pods("test").List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", deploymentName), + FieldSelector: "status.phase=Running", + }) + require.NoError(t, err, "Error getting pods for deployment") + require.NotEmpty(t, pods.Items, "No pods found for deployment") + + remainingEnvVars := make(map[string]string) + for k, v := range requiredEnvVars { + remainingEnvVars[k] = v + } + + for _, container := range pods.Items[0].Spec.Containers { + for _, envVar := range container.Env { + if expectedValue, exists := remainingEnvVars[envVar.Name]; exists { + require.Equal(t, expectedValue, envVar.Value, + fmt.Sprintf("Unexpected value for environment variable %s in container %s", + envVar.Name, container.Name)) + delete(remainingEnvVars, envVar.Name) + } + } + } + + require.Empty(t, remainingEnvVars, "Not all required environment variables were found in the pod") +} + +func VerifyAgentResources(t *testing.T, clientset *kubernetes.Clientset, configKeyword string) { + daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent DaemonSet") + require.NotNil(t, daemonSet, "CloudWatch Agent DaemonSet not found") + + configMap, err := clientset.CoreV1().ConfigMaps("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent ConfigMap") + require.NotNil(t, configMap, "CloudWatch Agent ConfigMap not found") + + cwConfig, exists := configMap.Data["cwagentconfig.json"] + require.True(t, exists, "cwagentconfig.json not found in ConfigMap") + require.Contains(t, cwConfig, configKeyword, fmt.Sprintf("%s configuration not found in cwagentconfig.json", configKeyword)) + + service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent Service") + require.NotNil(t, service, "CloudWatch Agent Service not found") + + serviceAccount, err := clientset.CoreV1().ServiceAccounts("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent Service Account") + require.NotNil(t, serviceAccount, "CloudWatch Agent Service Account not found") +} + +//------------------------------------------------------------------------------ +// Metric Functions +//------------------------------------------------------------------------------ + +func ValidateMetrics(t *testing.T, metrics []string, namespace string) { + for _, metric := range metrics { + t.Run(metric, func(t *testing.T) { + awsservice.ValidateMetricWithTest(t, metric, namespace, nil, 5, interval) + }) + } +} + +func GenerateTraffic(t *testing.T) { + cmd := exec.Command("kubectl", "get", "nodes", "-o", "jsonpath='{.items[0].status.addresses[?(@.type==\"ExternalIP\")].address}'") + output, err := cmd.CombinedOutput() + require.NoError(t, err, "Error getting node external IP") + + nodeIP := strings.Trim(string(output), "'") + require.NotEmpty(t, nodeIP, "Node IP failed to format") + + for i := 0; i < 5; i++ { + resp, err := http.Get(fmt.Sprintf("http://%s:30080/webapp/index.jsp", nodeIP)) + if err != nil { + t.Logf("Request attempt %d failed: %v", i+1, err) + continue + } + require.NoError(t, resp.Body.Close(), "Failed to close response body") + } +}