Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement EKS E2E Terraform Framework #434

Merged
merged 22 commits into from
Dec 16, 2024
Merged

Implement EKS E2E Terraform Framework #434

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
6f320b0
Preliminary testing.
musa-asad Nov 29, 2024
5727809
Support environment.
musa-asad Dec 1, 2024
912d98e
Fix file positioning.
musa-asad Dec 1, 2024
5d0362f
Fix file positioning.
musa-asad Dec 1, 2024
6740715
Fix values.
musa-asad Dec 1, 2024
c401c0e
Fix otel and prometheus config set-up.
musa-asad Dec 2, 2024
e8d4901
Remove jmx files.
musa-asad Dec 2, 2024
7a02bc2
Merge branch 'main' into e2e
musa-asad Dec 4, 2024
b5b2fb9
Allow to choose e2e or regular tests.
musa-asad Dec 8, 2024
dfd94e0
Add TA.
musa-asad Dec 9, 2024
da21e5a
Merge branch 'main' into e2e
musa-asad Dec 9, 2024
ba53b14
Make 2 hours
musa-asad Dec 12, 2024
f5d2733
Fix providers.tf
musa-asad Dec 12, 2024
e6b5faa
Fix formatting.
musa-asad Dec 13, 2024
17657f0
Remove unnecessary resources.
musa-asad Dec 13, 2024
e0fbd7e
Remove redundant logic.
musa-asad Dec 13, 2024
537cb6f
Revert "Remove redundant logic."
musa-asad Dec 13, 2024
43b7dae
Remove redundant logic.
musa-asad Dec 13, 2024
dbea081
Update nodes.
musa-asad Dec 16, 2024
ce35b43
Merge branch 'main' into e2e
musa-asad Dec 16, 2024
7e4f988
Allow configurable nodes.
musa-asad Dec 16, 2024
2ad98f5
Fix fmt.
musa-asad Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 111 additions & 43 deletions environment/metadata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,52 +23,84 @@ var metaDataStorage *MetaData = nil
var registeredMetaDataStrings = &(MetaDataStrings{})

type MetaData struct {
ComputeType computetype.ComputeType
musa-asad marked this conversation as resolved.
Show resolved Hide resolved
EcsLaunchType ecslaunchtype.ECSLaunchType
EcsDeploymentStrategy ecsdeploymenttype.ECSDeploymentType
EksDeploymentStrategy eksdeploymenttype.EKSDeploymentType
EcsClusterArn string
EcsClusterName string
CwagentConfigSsmParamName string
EcsServiceName string
EC2PluginTests map[string]struct{} // set of EC2 plugin names
ExcludedTests map[string]struct{} // set of excluded names
Bucket string
S3Key string
CwaCommitSha string
CaCertPath string
EKSClusterName string
ProxyUrl string
AssumeRoleArn string
InstanceId string
InstancePlatform string
AgentStartCommand string
EksGpuType string
AmpWorkspaceId string
ComputeType computetype.ComputeType
EcsLaunchType ecslaunchtype.ECSLaunchType
EcsDeploymentStrategy ecsdeploymenttype.ECSDeploymentType
EksDeploymentStrategy eksdeploymenttype.EKSDeploymentType
EcsClusterArn string
EcsClusterName string
CwagentConfigSsmParamName string
EcsServiceName string
EC2PluginTests map[string]struct{} // set of EC2 plugin names
ExcludedTests map[string]struct{} // set of excluded names
Bucket string
S3Key string
CwaCommitSha string
CaCertPath string
EKSClusterName string
ProxyUrl string
AssumeRoleArn string
InstanceId string
InstancePlatform string
AgentStartCommand string
EksGpuType string
AmpWorkspaceId string
Region string
K8sVersion string
HelmChartsBranch string
CloudwatchAgentRepository string
CloudwatchAgentTag string
CloudwatchAgentRepositoryURL string
CloudwatchAgentOperatorRepository string
CloudwatchAgentOperatorTag string
CloudwatchAgentOperatorRepositoryURL string
CloudwatchAgentTargetAllocatorRepository string
musa-asad marked this conversation as resolved.
Show resolved Hide resolved
CloudwatchAgentTargetAllocatorTag string
CloudwatchAgentTargetAllocatorRepositoryURL string
AgentConfig string
PrometheusConfig string
OtelConfig string
SampleApp string
}

type MetaDataStrings struct {
ComputeType string
EcsLaunchType string
EcsDeploymentStrategy string
EksDeploymentStrategy string
EcsClusterArn string
CwagentConfigSsmParamName string
EcsServiceName string
EC2PluginTests string // input comma delimited list of plugin names
ExcludedTests string // Exclude specific tests from OS
Bucket string
S3Key string
CwaCommitSha string
CaCertPath string
EKSClusterName string
ProxyUrl string
AssumeRoleArn string
InstanceId string
InstancePlatform string
AgentStartCommand string
EksGpuType string
AmpWorkspaceId string
ComputeType string
EcsLaunchType string
EcsDeploymentStrategy string
EksDeploymentStrategy string
EcsClusterArn string
CwagentConfigSsmParamName string
EcsServiceName string
EC2PluginTests string // input comma delimited list of plugin names
ExcludedTests string // Exclude specific tests from OS
Bucket string
S3Key string
CwaCommitSha string
CaCertPath string
EKSClusterName string
ProxyUrl string
AssumeRoleArn string
InstanceId string
InstancePlatform string
AgentStartCommand string
EksGpuType string
AmpWorkspaceId string
Region string
K8sVersion string
HelmChartsBranch string
CloudwatchAgentRepository string
CloudwatchAgentTag string
CloudwatchAgentRepositoryURL string
CloudwatchAgentOperatorRepository string
CloudwatchAgentOperatorTag string
CloudwatchAgentOperatorRepositoryURL string
CloudwatchAgentTargetAllocatorRepository string
CloudwatchAgentTargetAllocatorTag string
CloudwatchAgentTargetAllocatorRepositoryURL string
AgentConfig string
PrometheusConfig string
OtelConfig string
SampleApp string
}

func registerComputeType(dataString *MetaDataStrings) {
Expand Down Expand Up @@ -101,6 +133,25 @@ func registerEKSData(d *MetaDataStrings) {
flag.StringVar(&(d.EksGpuType), "eksGpuType", "", "nvidia/inferentia")
}

func registerEKSE2ETestData(dataString *MetaDataStrings) {
flag.StringVar(&(dataString.Region), "region", "", "AWS region")
flag.StringVar(&(dataString.K8sVersion), "k8s_version", "", "Kubernetes version")
flag.StringVar(&(dataString.HelmChartsBranch), "helm_charts_branch", "", "Helm charts branch")
flag.StringVar(&(dataString.CloudwatchAgentRepository), "cloudwatch_agent_repository", "", "CloudWatch Agent repository")
flag.StringVar(&(dataString.CloudwatchAgentTag), "cloudwatch_agent_tag", "", "CloudWatch Agent tag")
flag.StringVar(&(dataString.CloudwatchAgentRepositoryURL), "cloudwatch_agent_repository_url", "", "CloudWatch Agent repository URL")
musa-asad marked this conversation as resolved.
Show resolved Hide resolved
flag.StringVar(&(dataString.CloudwatchAgentOperatorRepository), "cloudwatch_agent_operator_repository", "", "CloudWatch Agent Operator repository")
flag.StringVar(&(dataString.CloudwatchAgentOperatorTag), "cloudwatch_agent_operator_tag", "", "CloudWatch Agent Operator tag")
flag.StringVar(&(dataString.CloudwatchAgentOperatorRepositoryURL), "cloudwatch_agent_operator_repository_url", "", "CloudWatch Agent Operator repository URL")
flag.StringVar(&(dataString.CloudwatchAgentTargetAllocatorRepository), "cloudwatch_agent_target_allocator_repository", "", "CloudWatch Agent Target Allocator repository")
flag.StringVar(&(dataString.CloudwatchAgentTargetAllocatorTag), "cloudwatch_agent_target_allocator_tag", "", "CloudWatch Agent Target Allocator tag")
flag.StringVar(&(dataString.CloudwatchAgentTargetAllocatorRepositoryURL), "cloudwatch_agent_target_allocator_repository_url", "", "CloudWatch Agent Target Allocator repository URL")
flag.StringVar(&(dataString.AgentConfig), "agent_config", "", "Agent configuration file path")
flag.StringVar(&(dataString.PrometheusConfig), "prometheus_config", "", "Prometheus configuration file path")
flag.StringVar(&(dataString.OtelConfig), "otel_config", "", "OpenTelemetry configuration file path")
flag.StringVar(&(dataString.SampleApp), "sample_app", "", "Sample application manifest file path")
}

func registerPluginTestsToExecute(dataString *MetaDataStrings) {
flag.StringVar(&(dataString.EC2PluginTests), "plugins", "", "Comma-delimited list of plugins to test. Default is empty, which tests all")
}
Expand Down Expand Up @@ -226,6 +277,7 @@ func RegisterEnvironmentMetaDataFlags() *MetaDataStrings {
registerComputeType(registeredMetaDataStrings)
registerECSData(registeredMetaDataStrings)
registerEKSData(registeredMetaDataStrings)
registerEKSE2ETestData(registeredMetaDataStrings)
registerBucket(registeredMetaDataStrings)
registerS3Key(registeredMetaDataStrings)
registerCwaCommitSha(registeredMetaDataStrings)
Expand Down Expand Up @@ -264,6 +316,22 @@ func GetEnvironmentMetaData() *MetaData {
metaDataStorage.AgentStartCommand = registeredMetaDataStrings.AgentStartCommand
metaDataStorage.EksGpuType = registeredMetaDataStrings.EksGpuType
metaDataStorage.AmpWorkspaceId = registeredMetaDataStrings.AmpWorkspaceId
metaDataStorage.Region = registeredMetaDataStrings.Region
metaDataStorage.K8sVersion = registeredMetaDataStrings.K8sVersion
metaDataStorage.HelmChartsBranch = registeredMetaDataStrings.HelmChartsBranch
metaDataStorage.CloudwatchAgentRepository = registeredMetaDataStrings.CloudwatchAgentRepository
metaDataStorage.CloudwatchAgentTag = registeredMetaDataStrings.CloudwatchAgentTag
metaDataStorage.CloudwatchAgentRepositoryURL = registeredMetaDataStrings.CloudwatchAgentRepositoryURL
metaDataStorage.CloudwatchAgentOperatorRepository = registeredMetaDataStrings.CloudwatchAgentOperatorRepository
metaDataStorage.CloudwatchAgentOperatorTag = registeredMetaDataStrings.CloudwatchAgentOperatorTag
metaDataStorage.CloudwatchAgentOperatorRepositoryURL = registeredMetaDataStrings.CloudwatchAgentOperatorRepositoryURL
metaDataStorage.CloudwatchAgentTargetAllocatorRepository = registeredMetaDataStrings.CloudwatchAgentTargetAllocatorRepository
metaDataStorage.CloudwatchAgentTargetAllocatorTag = registeredMetaDataStrings.CloudwatchAgentTargetAllocatorTag
metaDataStorage.CloudwatchAgentTargetAllocatorRepositoryURL = registeredMetaDataStrings.CloudwatchAgentTargetAllocatorRepositoryURL
metaDataStorage.AgentConfig = registeredMetaDataStrings.AgentConfig
metaDataStorage.PrometheusConfig = registeredMetaDataStrings.PrometheusConfig
metaDataStorage.OtelConfig = registeredMetaDataStrings.OtelConfig
metaDataStorage.SampleApp = registeredMetaDataStrings.SampleApp

return metaDataStorage
}
5 changes: 5 additions & 0 deletions generator/resources/eks_e2e_test_matrix.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[
{

}
]
musa-asad marked this conversation as resolved.
Show resolved Hide resolved
19 changes: 17 additions & 2 deletions generator/test_case_generator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package main

import (
"encoding/json"
"flag"
"fmt"
"io"
"log"
Expand Down Expand Up @@ -245,6 +246,12 @@ var testTypeToTestConfig = map[string][]testConfig{
},
}

var testTypeToTestConfigE2E = map[string][]testConfig{
"eks_e2e": {
{testDir: "../../../test/e2e/jmx"},
},
}

type partition struct {
configName string
tests []string
Expand Down Expand Up @@ -281,9 +288,17 @@ func copyAllEC2LinuxTestForOnpremTesting() {
}

func main() {
copyAllEC2LinuxTestForOnpremTesting()
useE2E := flag.Bool("e2e", false, "Use e2e test matrix generation")
flag.Parse()

configMap := testTypeToTestConfig
if !*useE2E {
copyAllEC2LinuxTestForOnpremTesting()
} else {
configMap = testTypeToTestConfigE2E
}

for testType, testConfigs := range testTypeToTestConfig {
for testType, testConfigs := range configMap {
for _, partition := range partitionTests {
if len(partition.tests) != 0 && !slices.Contains(partition.tests, testType) {
continue
Expand Down
24 changes: 24 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ require (
golang.org/x/exp v0.0.0-20231127185646-65229373498e
golang.org/x/sys v0.15.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/apimachinery v0.23.0
k8s.io/client-go v0.23.0
)

require (
Expand All @@ -65,16 +67,25 @@ require (
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 // indirect
github.com/imdario/mergo v0.3.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.4 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/qri-io/jsonpointer v0.1.1 // indirect
github.com/shoenig/go-m1cpu v0.1.4 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
Expand All @@ -85,11 +96,24 @@ require (
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/oauth2 v0.13.0 // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/tools v0.16.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect
google.golang.org/grpc v1.59.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/api v0.23.0 // indirect
k8s.io/klog/v2 v2.30.0 // indirect
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
sigs.k8s.io/yaml v1.2.0 // indirect
)
Loading
Loading