Skip to content

Commit

Permalink
revision 3
Browse files Browse the repository at this point in the history
  • Loading branch information
musa-asad committed Dec 27, 2024
1 parent 8cb81ce commit bf73309
Show file tree
Hide file tree
Showing 7 changed files with 132 additions and 95 deletions.
6 changes: 4 additions & 2 deletions environment/eksdeploymenttype/eks_deployment_type.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,19 @@ const (
DAEMON EKSDeploymentType = "DAEMON"
REPLICA EKSDeploymentType = "REPLICA"
SIDECAR EKSDeploymentType = "SIDECAR"
STATEFUL EKSDeploymentType = "STATEFUL"
)

var (
ecsDeploymentTypes = map[string]EKSDeploymentType{
eksDeploymentTypes = map[string]EKSDeploymentType{
"DAEMON": DAEMON,
"REPLICA": REPLICA,
"SIDECAR": SIDECAR,
"STATEFUL": STATEFUL,
}
)

func FromString(str string) (EKSDeploymentType, bool) {
c, ok := ecsDeploymentTypes[strings.ToUpper(str)]
c, ok := eksDeploymentTypes[strings.ToUpper(str)]
return c, ok
}
6 changes: 3 additions & 3 deletions environment/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ type MetaDataStrings struct {
AmpWorkspaceId string
Region string
K8sVersion string
Destroy string
Destroy bool
HelmChartsBranch string
CloudwatchAgentRepository string
CloudwatchAgentTag string
Expand Down Expand Up @@ -138,7 +138,7 @@ func registerEKSData(d *MetaDataStrings) {
func registerEKSE2ETestData(dataString *MetaDataStrings) {
flag.StringVar(&(dataString.Region), "region", "", "AWS region")
flag.StringVar(&(dataString.K8sVersion), "k8s_version", "", "Kubernetes version")
flag.StringVar(&(dataString.Destroy), "destroy", "false", "Whether to run in destroy mode (true/false)")
flag.BoolVar(&(dataString.Destroy), "destroy", false, "Whether to run in destroy mode (true/false)")
flag.StringVar(&(dataString.HelmChartsBranch), "helm_charts_branch", "", "Helm charts branch")
flag.StringVar(&(dataString.CloudwatchAgentRepository), "cloudwatch_agent_repository", "", "CloudWatch Agent repository")
flag.StringVar(&(dataString.CloudwatchAgentTag), "cloudwatch_agent_tag", "", "CloudWatch Agent tag")
Expand Down Expand Up @@ -321,7 +321,7 @@ func GetEnvironmentMetaData() *MetaData {
metaDataStorage.AmpWorkspaceId = registeredMetaDataStrings.AmpWorkspaceId
metaDataStorage.Region = registeredMetaDataStrings.Region
metaDataStorage.K8sVersion = registeredMetaDataStrings.K8sVersion
metaDataStorage.Destroy = registeredMetaDataStrings.Destroy == "true"
metaDataStorage.Destroy = registeredMetaDataStrings.Destroy
metaDataStorage.HelmChartsBranch = registeredMetaDataStrings.HelmChartsBranch
metaDataStorage.CloudwatchAgentRepository = registeredMetaDataStrings.CloudwatchAgentRepository
metaDataStorage.CloudwatchAgentTag = registeredMetaDataStrings.CloudwatchAgentTag
Expand Down
2 changes: 1 addition & 1 deletion generator/resources/eks_e2e_jmx_test_matrix.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[
{
"nodes": 1
"nodes": 2
}
]
4 changes: 2 additions & 2 deletions terraform/eks/e2e/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ resource "aws_eks_node_group" "this" {
min_size = var.nodes
}

ami_type = var.ami_type
ami_type = "AL2_x86_64"
capacity_type = "ON_DEMAND"
disk_size = 20
instance_types = [var.instance_type]
Expand Down Expand Up @@ -155,7 +155,7 @@ resource "null_resource" "validator" {
command = <<-EOT
echo "Running cleanup for K8s resources"
go test -timeout 30m -v ${self.triggers.test_dir} \
-destroy=true \
-destroy \
-region=${self.triggers.region} \
-eksClusterName=${self.triggers.cluster_name} \
-computeType=EKS \
Expand Down
7 changes: 1 addition & 6 deletions terraform/eks/e2e/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,7 @@ variable "cluster_name" {

variable "nodes" {
type = number
default = 1
}

variable "ami_type" {
type = string
default = "AL2_x86_64"
default = 2
}

variable "instance_type" {
Expand Down
95 changes: 14 additions & 81 deletions test/e2e/jmx/jmx_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,36 +4,28 @@
package main

import (
"context"
"flag"
"fmt"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"

"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"

"github.com/aws/amazon-cloudwatch-agent-test/environment"
"github.com/aws/amazon-cloudwatch-agent-test/test/e2e"
"github.com/aws/amazon-cloudwatch-agent-test/util/awsservice"
"github.com/aws/amazon-cloudwatch-agent-test/util/common"
)

//------------------------------------------------------------------------------
// Constants and Variables
// Variables
//------------------------------------------------------------------------------

const (
wait = 5 * time.Minute
interval = 30 * time.Second
)

var (
nodeNames []string
env *environment.MetaData
Expand Down Expand Up @@ -145,7 +137,7 @@ func testMetrics(t *testing.T) {
tests := testMetricsRegistry[configFile]

fmt.Println("waiting for metrics to propagate...")
time.Sleep(wait)
time.Sleep(e2e.Wait)

for _, testFunc := range tests {
testFunc(t)
Expand All @@ -158,37 +150,13 @@ func testMetrics(t *testing.T) {

func testAgentResources(t *testing.T, clientset *kubernetes.Clientset) {
t.Run("verify_agent_resources", func(t *testing.T) {
daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{})
require.NoError(t, err, "Error getting CloudWatch Agent DaemonSet")
require.NotNil(t, daemonSet, "CloudWatch Agent DaemonSet not found")

configMap, err := clientset.CoreV1().ConfigMaps("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{})
require.NoError(t, err, "Error getting CloudWatch Agent ConfigMap")
require.NotNil(t, configMap, "CloudWatch Agent ConfigMap not found")

cwConfig, exists := configMap.Data["cwagentconfig.json"]
require.True(t, exists, "cwagentconfig.json not found in ConfigMap")
require.Contains(t, cwConfig, `jmx`, "JMX configuration not found in cwagentconfig.json")

service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{})
require.NoError(t, err, "Error getting CloudWatch Agent Service")
require.NotNil(t, service, "CloudWatch Agent Service not found")

serviceAccount, err := clientset.CoreV1().ServiceAccounts("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{})
require.NoError(t, err, "Error getting CloudWatch Agent Service Account")
require.NotNil(t, serviceAccount, "CloudWatch Agent Service Account not found")
e2e.VerifyAgentResources(t, clientset, "jmx")
})
}

func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) {
t.Run("verify_jmx_resources", func(t *testing.T) {
deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml")
pods, err := clientset.CoreV1().Pods("test").List(context.TODO(), metav1.ListOptions{
LabelSelector: fmt.Sprintf("app=%s", deploymentName),
FieldSelector: "status.phase=Running",
})
require.NoError(t, err, "Error getting pods for deployment")
require.NotEmpty(t, pods.Items, "No pods found for deployment")

var jmxTargetSystem string
switch filepath.Base(env.AgentConfig) {
Expand All @@ -208,16 +176,7 @@ func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) {
"JAVA_TOOL_OPTIONS": " -javaagent:/otel-auto-instrumentation-java/javaagent.jar",
}

for _, container := range pods.Items[0].Spec.Containers {
for _, envVar := range container.Env {
if expectedValue, exists := requiredEnvVars[envVar.Name]; exists {
require.Equal(t, expectedValue, envVar.Value, fmt.Sprintf("Unexpected value for environment variable %s in container %s", envVar.Name, container.Name))
delete(requiredEnvVars, envVar.Name)
}
}
}

require.Empty(t, requiredEnvVars, "Not all required environment variables were found in the pod")
e2e.VerifyPodEnvironment(t, clientset, deploymentName, requiredEnvVars)
})
}

Expand All @@ -227,7 +186,7 @@ func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) {

func testTomcatMetrics(t *testing.T) {
t.Run("verify_jvm_tomcat_metrics", func(t *testing.T) {
validateMetrics(t, []string{
e2e.ValidateMetrics(t, []string{
"jvm.classes.loaded",
"jvm.gc.collections.count",
"jvm.gc.collections.elapsed",
Expand Down Expand Up @@ -257,15 +216,15 @@ func testTomcatMetrics(t *testing.T) {

func testTomcatSessions(t *testing.T) {
t.Run("verify_tomcat_sessions", func(t *testing.T) {
generateTraffic(t)
time.Sleep(wait)
e2e.GenerateTraffic(t)
time.Sleep(e2e.Wait)
verifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E", false)
})
}

func testKafkaMetrics(t *testing.T) {
t.Run("verify_kafka_metrics", func(t *testing.T) {
validateMetrics(t, []string{
e2e.ValidateMetrics(t, []string{
"kafka.message.count",
"kafka.request.count",
"kafka.request.failed",
Expand All @@ -285,7 +244,7 @@ func testKafkaMetrics(t *testing.T) {

func testContainerInsightsMetrics(t *testing.T) {
t.Run("verify_containerinsights_metrics", func(t *testing.T) {
validateMetrics(t, []string{
e2e.ValidateMetrics(t, []string{
"jvm_classes_loaded",
"jvm_threads_current",
"jvm_threads_daemon",
Expand All @@ -310,8 +269,8 @@ func testContainerInsightsMetrics(t *testing.T) {

func testTomcatRejectedSessions(t *testing.T) {
t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) {
generateTraffic(t)
time.Sleep(wait)
e2e.GenerateTraffic(t)
time.Sleep(e2e.Wait)
verifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus", true)
})
}
Expand All @@ -320,34 +279,8 @@ func testTomcatRejectedSessions(t *testing.T) {
// Helper Functions
//------------------------------------------------------------------------------

func validateMetrics(t *testing.T, metrics []string, namespace string) {
for _, metric := range metrics {
t.Run(metric, func(t *testing.T) {
awsservice.ValidateMetricWithTest(t, metric, namespace, nil, 5, interval)
})
}
}

func generateTraffic(t *testing.T) {
cmd := exec.Command("kubectl", "get", "nodes", "-o", "jsonpath='{.items[0].status.addresses[?(@.type==\"ExternalIP\")].address}'")
output, err := cmd.CombinedOutput()
require.NoError(t, err, "Error getting node external IP")

nodeIP := strings.Trim(string(output), "'")
require.NotEmpty(t, nodeIP, "Node IP failed to format")

for i := 0; i < 5; i++ {
resp, err := http.Get(fmt.Sprintf("http://%s:30080/webapp/index.jsp", nodeIP))
if err != nil {
t.Logf("Request attempt %d failed: %v", i+1, err)
continue
}
require.NoError(t, resp.Body.Close(), "Failed to close response body")
}
}

func verifyMetricAboveZero(t *testing.T, metricName, namespace string, containerInsights bool) {
startTime := time.Now().Add(-wait)
startTime := time.Now().Add(-e2e.Wait)
endTime := time.Now()

aboveZero, err := awsservice.CheckMetricAboveZero(
Expand All @@ -361,4 +294,4 @@ func verifyMetricAboveZero(t *testing.T, metricName, namespace string, container
)
require.NoError(t, err, "Failed to check metric above zero")
require.True(t, aboveZero, fmt.Sprintf("Expected non-zero %s after applying traffic", metricName))
}
}
107 changes: 107 additions & 0 deletions test/e2e/testutils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package e2e

import (
"context"
"fmt"
"github.com/aws/amazon-cloudwatch-agent-test/util/awsservice"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"net/http"
"os/exec"
"strings"
"testing"
"time"
)

//------------------------------------------------------------------------------
// Constants
//------------------------------------------------------------------------------

const (
Wait = 5 * time.Minute
interval = 30 * time.Second
)

//------------------------------------------------------------------------------
// Resource Functions
//------------------------------------------------------------------------------

func VerifyPodEnvironment(t *testing.T, clientset *kubernetes.Clientset, deploymentName string, requiredEnvVars map[string]string) {
pods, err := clientset.CoreV1().Pods("test").List(context.TODO(), metav1.ListOptions{
LabelSelector: fmt.Sprintf("app=%s", deploymentName),
FieldSelector: "status.phase=Running",
})
require.NoError(t, err, "Error getting pods for deployment")
require.NotEmpty(t, pods.Items, "No pods found for deployment")

remainingEnvVars := make(map[string]string)
for k, v := range requiredEnvVars {
remainingEnvVars[k] = v
}

for _, container := range pods.Items[0].Spec.Containers {
for _, envVar := range container.Env {
if expectedValue, exists := remainingEnvVars[envVar.Name]; exists {
require.Equal(t, expectedValue, envVar.Value,
fmt.Sprintf("Unexpected value for environment variable %s in container %s",
envVar.Name, container.Name))
delete(remainingEnvVars, envVar.Name)
}
}
}

require.Empty(t, remainingEnvVars, "Not all required environment variables were found in the pod")
}

func VerifyAgentResources(t *testing.T, clientset *kubernetes.Clientset, configKeyword string) {
daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{})
require.NoError(t, err, "Error getting CloudWatch Agent DaemonSet")
require.NotNil(t, daemonSet, "CloudWatch Agent DaemonSet not found")

configMap, err := clientset.CoreV1().ConfigMaps("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{})
require.NoError(t, err, "Error getting CloudWatch Agent ConfigMap")
require.NotNil(t, configMap, "CloudWatch Agent ConfigMap not found")

cwConfig, exists := configMap.Data["cwagentconfig.json"]
require.True(t, exists, "cwagentconfig.json not found in ConfigMap")
require.Contains(t, cwConfig, configKeyword, fmt.Sprintf("%s configuration not found in cwagentconfig.json", configKeyword))

service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{})
require.NoError(t, err, "Error getting CloudWatch Agent Service")
require.NotNil(t, service, "CloudWatch Agent Service not found")

serviceAccount, err := clientset.CoreV1().ServiceAccounts("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{})
require.NoError(t, err, "Error getting CloudWatch Agent Service Account")
require.NotNil(t, serviceAccount, "CloudWatch Agent Service Account not found")
}

//------------------------------------------------------------------------------
// Metric Functions
//------------------------------------------------------------------------------

func ValidateMetrics(t *testing.T, metrics []string, namespace string) {
for _, metric := range metrics {
t.Run(metric, func(t *testing.T) {
awsservice.ValidateMetricWithTest(t, metric, namespace, nil, 5, interval)
})
}
}

func GenerateTraffic(t *testing.T) {
cmd := exec.Command("kubectl", "get", "nodes", "-o", "jsonpath='{.items[0].status.addresses[?(@.type==\"ExternalIP\")].address}'")
output, err := cmd.CombinedOutput()
require.NoError(t, err, "Error getting node external IP")

nodeIP := strings.Trim(string(output), "'")
require.NotEmpty(t, nodeIP, "Node IP failed to format")

for i := 0; i < 5; i++ {
resp, err := http.Get(fmt.Sprintf("http://%s:30080/webapp/index.jsp", nodeIP))
if err != nil {
t.Logf("Request attempt %d failed: %v", i+1, err)
continue
}
require.NoError(t, resp.Body.Close(), "Failed to close response body")
}
}

0 comments on commit bf73309

Please sign in to comment.