From 6f320b0ecea95939af7f5ce9c5bece35c831be8b Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 29 Nov 2024 03:14:39 -0500 Subject: [PATCH 01/92] Preliminary testing. --- generator/resources/eks_e2e_test_matrix.json | 5 + generator/test_case_generator.go | 3 + go.mod | 24 + go.sum | 595 +++++++++++++++++- terraform/eks/e2e/main.tf | 132 ++++ terraform/eks/e2e/providers.tf | 15 + terraform/eks/e2e/variables.tf | 77 +++ test/e2e/jmx/jmx_test.go | 171 +++++ .../resources/cwagent_configs/jvm_tomcat.json | 40 ++ .../sample_apps/tomcat-deployment.yaml | 37 ++ util/awsservice/constant.go | 26 + 11 files changed, 1124 insertions(+), 1 deletion(-) create mode 100644 generator/resources/eks_e2e_test_matrix.json create mode 100644 terraform/eks/e2e/main.tf create mode 100644 terraform/eks/e2e/providers.tf create mode 100644 terraform/eks/e2e/variables.tf create mode 100644 test/e2e/jmx/jmx_test.go create mode 100644 test/e2e/jmx/resources/cwagent_configs/jvm_tomcat.json create mode 100644 test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml diff --git a/generator/resources/eks_e2e_test_matrix.json b/generator/resources/eks_e2e_test_matrix.json new file mode 100644 index 000000000..81ed61090 --- /dev/null +++ b/generator/resources/eks_e2e_test_matrix.json @@ -0,0 +1,5 @@ +[ + { + + } +] \ No newline at end of file diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index a811e7c5c..6f555c256 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -243,6 +243,9 @@ var testTypeToTestConfig = map[string][]testConfig{ "eks_deployment": { {testDir: "./test/metric_value_benchmark"}, }, + "eks_e2e": { + {testDir: "./test/e2e/jmx"}, + }, } type partition struct { diff --git a/go.mod b/go.mod index 56f743d5a..c5ceaa247 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,8 @@ require ( golang.org/x/exp v0.0.0-20231127185646-65229373498e golang.org/x/sys v0.15.0 gopkg.in/yaml.v3 v3.0.1 + k8s.io/apimachinery v0.23.0 + k8s.io/client-go v0.23.0 ) require ( @@ -65,16 +67,25 @@ require ( github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/googleapis/gnostic v0.5.5 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 // indirect + github.com/imdario/mergo v0.3.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.4 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/qri-io/jsonpointer v0.1.1 // indirect github.com/shoenig/go-m1cpu v0.1.4 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -85,11 +96,24 @@ require ( go.opentelemetry.io/proto/otlp v1.0.0 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.19.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/tools v0.16.0 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/api v0.23.0 // indirect + k8s.io/klog/v2 v2.30.0 // indirect + k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect + k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect + sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/go.sum b/go.sum index f6bc59b8a..af09a3b5d 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,61 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1MRDJM= github.com/DataDog/datadog-go v4.8.3+incompatible h1:fNGaYSuObuQb5nzeTQqowRAd9bpDIRRV4/gUtIBjh8Q= github.com/DataDog/datadog-go v4.8.3+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.48.12 h1:n+eGzflzzvYubu2cOjqpVll7lF+Ci0ThyCpg5kzfzbo= github.com/aws/aws-sdk-go v1.48.12/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.23.5 h1:xK6C4udTyDMd82RFvNkDQxtAd00xlzFUtX4fF2nMZyg= @@ -71,11 +122,43 @@ github.com/aws/smithy-go v1.18.1 h1:pOdBTUfXNazOlxLrgeYalVnuTpKreACHtc62xLwIB3c= github.com/aws/smithy-go v1.18.1/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/collectd/go-collectd v0.5.0 h1:dDXnD+hkw7f3XPNtQm1feda+dGZtzTNWXZID+w0Npxk= github.com/collectd/go-collectd v0.5.0/go.mod h1:xQe/Em/Q9IrN5ifWOvK5ZP2vMIB4KlsIA550yYzhkpY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -83,44 +166,166 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kinbiko/jsonassert v1.0.1 h1:8gdLmUaPWuxk2TzQSofKRqatFH6zwTF6AsUH4bugJYY= github.com/kinbiko/jsonassert v1.0.1/go.mod h1:QRwBwiAsrcJpjw+L+Q4WS8psLxuUY+HylVZS/4j74TM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prozz/aws-embedded-metrics-golang v1.2.0 h1:b/LFb8J9LbgANow/9nYZE3M3bkb457/dj0zAB3hPyvo= github.com/prozz/aws-embedded-metrics-golang v1.2.0/go.mod h1:MXOqF9cJCEHjj77LWq7NWK44/AOyaFzwmcAYqR3057M= github.com/qri-io/jsonpointer v0.1.1 h1:prVZBZLL6TW5vsSB9fFHFAMBLI4b0ri5vribQlTJiBA= github.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64= github.com/qri-io/jsonschema v0.2.1 h1:NNFoKms+kut6ABPf6xiKNM5214jzxAhDBrPHCJ97Wg0= github.com/qri-io/jsonschema v0.2.1/go.mod h1:g7DPkiOsK1xv6T/Ao5scXRkd+yTFygcANPBaaqW+VrI= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -130,11 +335,19 @@ github.com/shoenig/go-m1cpu v0.1.4 h1:SZPIgRM2sEF9NJy50mRHu9PKGwxyyTTJIWvCtgVboz github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c= github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -148,8 +361,20 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib/propagators/aws v1.21.1 h1:uQIQIDWb0gzyvon2ICnghpLAf9w7ADOCUiIiwCQgR2o= go.opentelemetry.io/contrib/propagators/aws v1.21.1/go.mod h1:kCcto3ACQxm+VrkQX/NK/TkDmAd99MQhvffzyTKhzL4= go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= @@ -169,40 +394,408 @@ go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v8 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4 h1:W12Pwm4urIbRdGhMEg2NM9O3TWKjNcxQhs46V0ypf/k= google.golang.org/genproto v0.0.0-20231127180814-3a041ad873d4/go.mod h1:5RBcpGRxr25RbDzY5w+dmaqpSEvl8Gwl1x2CICf60ic= google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4 h1:ZcOkrmX74HbKFYnpPY8Qsw93fC29TbJXspYKaBkSXDQ= google.golang.org/genproto/googleapis/api v0.0.0-20231127180814-3a041ad873d4/go.mod h1:k2dtGpRrbsSyKcNPKKI5sstZkrNCZwpU/ns96JoHbGg= google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE= google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.23.0 h1:WrL1gb73VSC8obi8cuYETJGXEoFNEh3LU0Pt+Sokgro= +k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= +k8s.io/apimachinery v0.23.0 h1:mIfWRMjBuMdolAWJ3Fd+aPTMv3X9z+waiARMpvvb0HQ= +k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= +k8s.io/client-go v0.23.0 h1:vcsOqyPq7XV3QmQRCBH/t9BICJM9Q1M18qahjv+rebY= +k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf new file mode 100644 index 000000000..956b4d1dd --- /dev/null +++ b/terraform/eks/e2e/main.tf @@ -0,0 +1,132 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +module "common" { + source = "../../common" +} + +module "basic_components" { + source = "../../basic_components" +} + +locals { + cluster_name = var.cluster_name != "" ? var.cluster_name : "cwagent-monitoring-config-e2e-eks" +} + +data "aws_eks_cluster_auth" "this" { + name = aws_eks_cluster.this.name +} + +resource "aws_eks_cluster" "this" { + name = "${local.cluster_name}-${module.common.testing_id}" + role_arn = module.basic_components.role_arn + version = var.k8s_version + vpc_config { + subnet_ids = module.basic_components.public_subnet_ids + security_group_ids = [module.basic_components.security_group] + } +} + +resource "aws_eks_node_group" "this" { + cluster_name = aws_eks_cluster.this.name + node_group_name = "${local.cluster_name}-node" + node_role_arn = aws_iam_role.node_role.arn + subnet_ids = module.basic_components.public_subnet_ids + + scaling_config { + desired_size = 2 + max_size = 2 + min_size = 2 + } + + ami_type = "AL2_x86_64" + capacity_type = "ON_DEMAND" + disk_size = 20 + instance_types = ["t3a.medium"] + + depends_on = [ + aws_iam_role_policy_attachment.node_CloudWatchAgentServerPolicy, + aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly, + aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy + ] +} + +resource "aws_iam_role" "node_role" { + name = "${local.cluster_name}-Worker-Role-${module.common.testing_id}" + + assume_role_policy = < Date: Sun, 1 Dec 2024 04:11:49 -0500 Subject: [PATCH 02/92] Support environment. --- environment/metadata.go | 142 ++++++++++++++++++++++++++------------ terraform/eks/e2e/main.tf | 3 +- test/e2e/jmx/jmx_test.go | 64 ++++++----------- 3 files changed, 121 insertions(+), 88 deletions(-) diff --git a/environment/metadata.go b/environment/metadata.go index d40b9fa8c..46fbd408d 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -23,52 +23,78 @@ var metaDataStorage *MetaData = nil var registeredMetaDataStrings = &(MetaDataStrings{}) type MetaData struct { - ComputeType computetype.ComputeType - EcsLaunchType ecslaunchtype.ECSLaunchType - EcsDeploymentStrategy ecsdeploymenttype.ECSDeploymentType - EksDeploymentStrategy eksdeploymenttype.EKSDeploymentType - EcsClusterArn string - EcsClusterName string - CwagentConfigSsmParamName string - EcsServiceName string - EC2PluginTests map[string]struct{} // set of EC2 plugin names - ExcludedTests map[string]struct{} // set of excluded names - Bucket string - S3Key string - CwaCommitSha string - CaCertPath string - EKSClusterName string - ProxyUrl string - AssumeRoleArn string - InstanceId string - InstancePlatform string - AgentStartCommand string - EksGpuType string - AmpWorkspaceId string + ComputeType computetype.ComputeType + EcsLaunchType ecslaunchtype.ECSLaunchType + EcsDeploymentStrategy ecsdeploymenttype.ECSDeploymentType + EksDeploymentStrategy eksdeploymenttype.EKSDeploymentType + EcsClusterArn string + EcsClusterName string + CwagentConfigSsmParamName string + EcsServiceName string + EC2PluginTests map[string]struct{} // set of EC2 plugin names + ExcludedTests map[string]struct{} // set of excluded names + Bucket string + S3Key string + CwaCommitSha string + CaCertPath string + EKSClusterName string + ProxyUrl string + AssumeRoleArn string + InstanceId string + InstancePlatform string + AgentStartCommand string + EksGpuType string + AmpWorkspaceId string + Region string + K8sVersion string + HelmChartsBranch string + CloudwatchAgentRepository string + CloudwatchAgentTag string + CloudwatchAgentRepositoryURL string + CloudwatchAgentOperatorRepository string + CloudwatchAgentOperatorTag string + CloudwatchAgentOperatorRepositoryURL string + AgentConfig string + PrometheusConfig string + OtelConfig string + SampleApp string } type MetaDataStrings struct { - ComputeType string - EcsLaunchType string - EcsDeploymentStrategy string - EksDeploymentStrategy string - EcsClusterArn string - CwagentConfigSsmParamName string - EcsServiceName string - EC2PluginTests string // input comma delimited list of plugin names - ExcludedTests string // Exclude specific tests from OS - Bucket string - S3Key string - CwaCommitSha string - CaCertPath string - EKSClusterName string - ProxyUrl string - AssumeRoleArn string - InstanceId string - InstancePlatform string - AgentStartCommand string - EksGpuType string - AmpWorkspaceId string + ComputeType string + EcsLaunchType string + EcsDeploymentStrategy string + EksDeploymentStrategy string + EcsClusterArn string + CwagentConfigSsmParamName string + EcsServiceName string + EC2PluginTests string // input comma delimited list of plugin names + ExcludedTests string // Exclude specific tests from OS + Bucket string + S3Key string + CwaCommitSha string + CaCertPath string + EKSClusterName string + ProxyUrl string + AssumeRoleArn string + InstanceId string + InstancePlatform string + AgentStartCommand string + EksGpuType string + AmpWorkspaceId string + Region string + K8sVersion string + HelmChartsBranch string + CloudwatchAgentRepository string + CloudwatchAgentTag string + CloudwatchAgentRepositoryURL string + CloudwatchAgentOperatorRepository string + CloudwatchAgentOperatorTag string + CloudwatchAgentOperatorRepositoryURL string + AgentConfig string + PrometheusConfig string + OtelConfig string + SampleApp string } func registerComputeType(dataString *MetaDataStrings) { @@ -101,6 +127,22 @@ func registerEKSData(d *MetaDataStrings) { flag.StringVar(&(d.EksGpuType), "eksGpuType", "", "nvidia/inferentia") } +func registerEKSE2ETestData(dataString *MetaDataStrings) { + flag.StringVar(&(dataString.Region), "region", "", "AWS region") + flag.StringVar(&(dataString.K8sVersion), "k8s_version", "", "Kubernetes version") + flag.StringVar(&(dataString.HelmChartsBranch), "helm_charts_branch", "", "Helm charts branch") + flag.StringVar(&(dataString.CloudwatchAgentRepository), "cloudwatch_agent_repository", "", "CloudWatch Agent repository") + flag.StringVar(&(dataString.CloudwatchAgentTag), "cloudwatch_agent_tag", "", "CloudWatch Agent tag") + flag.StringVar(&(dataString.CloudwatchAgentRepositoryURL), "cloudwatch_agent_repository_url", "", "CloudWatch Agent repository URL") + flag.StringVar(&(dataString.CloudwatchAgentOperatorRepository), "cloudwatch_agent_operator_repository", "", "CloudWatch Agent Operator repository") + flag.StringVar(&(dataString.CloudwatchAgentOperatorTag), "cloudwatch_agent_operator_tag", "", "CloudWatch Agent Operator tag") + flag.StringVar(&(dataString.CloudwatchAgentOperatorRepositoryURL), "cloudwatch_agent_operator_repository_url", "", "CloudWatch Agent Operator repository URL") + flag.StringVar(&(dataString.AgentConfig), "agent_config", "", "Agent configuration file path") + flag.StringVar(&(dataString.AgentConfig), "prometheus_config", "", "Prometheus configuration file path") + flag.StringVar(&(dataString.AgentConfig), "otel_config", "", "OpenTelemetry configuration file path") + flag.StringVar(&(dataString.SampleApp), "sample_app", "", "Sample application manifest file path") +} + func registerPluginTestsToExecute(dataString *MetaDataStrings) { flag.StringVar(&(dataString.EC2PluginTests), "plugins", "", "Comma-delimited list of plugins to test. Default is empty, which tests all") } @@ -226,6 +268,7 @@ func RegisterEnvironmentMetaDataFlags() *MetaDataStrings { registerComputeType(registeredMetaDataStrings) registerECSData(registeredMetaDataStrings) registerEKSData(registeredMetaDataStrings) + registerEKSE2ETestData(registeredMetaDataStrings) registerBucket(registeredMetaDataStrings) registerS3Key(registeredMetaDataStrings) registerCwaCommitSha(registeredMetaDataStrings) @@ -264,6 +307,19 @@ func GetEnvironmentMetaData() *MetaData { metaDataStorage.AgentStartCommand = registeredMetaDataStrings.AgentStartCommand metaDataStorage.EksGpuType = registeredMetaDataStrings.EksGpuType metaDataStorage.AmpWorkspaceId = registeredMetaDataStrings.AmpWorkspaceId + metaDataStorage.Region = registeredMetaDataStrings.Region + metaDataStorage.K8sVersion = registeredMetaDataStrings.K8sVersion + metaDataStorage.HelmChartsBranch = registeredMetaDataStrings.HelmChartsBranch + metaDataStorage.CloudwatchAgentRepository = registeredMetaDataStrings.CloudwatchAgentRepository + metaDataStorage.CloudwatchAgentTag = registeredMetaDataStrings.CloudwatchAgentTag + metaDataStorage.CloudwatchAgentRepositoryURL = registeredMetaDataStrings.CloudwatchAgentRepositoryURL + metaDataStorage.CloudwatchAgentOperatorRepository = registeredMetaDataStrings.CloudwatchAgentOperatorRepository + metaDataStorage.CloudwatchAgentOperatorTag = registeredMetaDataStrings.CloudwatchAgentOperatorTag + metaDataStorage.CloudwatchAgentOperatorRepositoryURL = registeredMetaDataStrings.CloudwatchAgentOperatorRepositoryURL + metaDataStorage.AgentConfig = registeredMetaDataStrings.AgentConfig + metaDataStorage.PrometheusConfig = registeredMetaDataStrings.PrometheusConfig + metaDataStorage.OtelConfig = registeredMetaDataStrings.OtelConfig + metaDataStorage.SampleApp = registeredMetaDataStrings.SampleApp return metaDataStorage } diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 956b4d1dd..a8aa9917a 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -115,7 +115,8 @@ resource "null_resource" "validator" { go test -v ${var.test_dir} \ -region=${var.region} \ -k8s_version=${var.k8s_version} \ - -cluster_name=${aws_eks_cluster.this.name} \ + -eksClusterName=${aws_eks_cluster.this.name} \ + -computeType=EKS \ -helm_charts_branch=${var.helm_charts_branch} \ -cloudwatch_agent_repository=${var.cloudwatch_agent_repository} \ -cloudwatch_agent_tag=${var.cloudwatch_agent_tag} \ diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index a48b5737b..198751e0e 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -4,6 +4,7 @@ import ( "context" "flag" "fmt" + "github.com/aws/amazon-cloudwatch-agent-test/environment" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" "os" "os/exec" @@ -16,51 +17,26 @@ import ( "k8s.io/client-go/tools/clientcmd" ) -var ( - clusterName string - region string - k8sVersion string - helmChartsBranch string - cloudwatchAgentRepository string - cloudwatchAgentTag string - cloudwatchAgentRepositoryURL string - cloudwatchAgentOperatorRepository string - cloudwatchAgentOperatorTag string - cloudwatchAgentOperatorRepositoryURL string - agentConfig string - sampleApp string -) - func init() { - flag.StringVar(&clusterName, "cluster_name", "", "EKS cluster name") // Didn't use environment.RegisterEnvironmentMetaDataFlags() since we only need this one flag. - flag.StringVar(®ion, "region", "", "AWS region") - flag.StringVar(&k8sVersion, "k8s_version", "", "Kubernetes version") - flag.StringVar(&helmChartsBranch, "helm_charts_branch", "", "Helm charts branch") - flag.StringVar(&cloudwatchAgentRepository, "cloudwatch_agent_repository", "", "CloudWatch Agent repository") - flag.StringVar(&cloudwatchAgentTag, "cloudwatch_agent_tag", "", "CloudWatch Agent tag") - flag.StringVar(&cloudwatchAgentRepositoryURL, "cloudwatch_agent_repository_url", "", "CloudWatch Agent repository URL") - flag.StringVar(&cloudwatchAgentOperatorRepository, "cloudwatch_agent_operator_repository", "", "CloudWatch Agent Operator repository") - flag.StringVar(&cloudwatchAgentOperatorTag, "cloudwatch_agent_operator_tag", "", "CloudWatch Agent Operator tag") - flag.StringVar(&cloudwatchAgentOperatorRepositoryURL, "cloudwatch_agent_operator_repository_url", "", "CloudWatch Agent Operator repository URL") - flag.StringVar(&agentConfig, "agent_config", "", "Agent configuration file path") - flag.StringVar(&sampleApp, "sample_app", "", "Sample application manifest file path") + environment.RegisterEnvironmentMetaDataFlags() } func TestMain(m *testing.M) { flag.Parse() + env := environment.GetEnvironmentMetaData() - if region != "us-west-2" { - if err := awsservice.ReconfigureAWSClients(region); err != nil { + if env.Region != "us-west-2" { + if err := awsservice.ReconfigureAWSClients(env.Region); err != nil { fmt.Printf("Failed to reconfigure AWS clients: %v\n", err) os.Exit(1) } - fmt.Printf("AWS clients reconfigured to use region: %s\n", region) + fmt.Printf("AWS clients reconfigured to use region: %s\n", env.Region) } else { fmt.Printf("Using default testing region: us-west-2\n") } fmt.Println("Starting Helm installation...") - if err := ApplyHelm(); err != nil { + if err := ApplyHelm(env); err != nil { fmt.Printf("Failed to apply Helm: %v\n", err) os.Exit(1) } @@ -71,8 +47,8 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func ApplyHelm() error { - updateKubeconfigCmd := exec.Command("aws", "eks", "update-kubeconfig", "--name", clusterName) +func ApplyHelm(env *environment.MetaData) error { + updateKubeconfigCmd := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName) output, err := updateKubeconfigCmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to update kubeconfig: %w\nOutput: %s", err, output) @@ -82,20 +58,20 @@ func ApplyHelm() error { "helm", "upgrade", "--install", "amazon-cloudwatch-observability", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability"), "--values", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability", "values.yaml"), - "--set", fmt.Sprintf("clusterName=%s", clusterName), - "--set", fmt.Sprintf("region=%s", region), - "--set", fmt.Sprintf("agent.image.repository=%s", cloudwatchAgentRepository), - "--set", fmt.Sprintf("agent.image.tag=%s", cloudwatchAgentTag), - "--set", fmt.Sprintf("agent.image.repositoryDomainMap.public=%s", cloudwatchAgentRepositoryURL), - "--set", fmt.Sprintf("manager.image.repository=%s", cloudwatchAgentOperatorRepository), - "--set", fmt.Sprintf("manager.image.tag=%s", cloudwatchAgentOperatorTag), - "--set", fmt.Sprintf("manager.image.repositoryDomainMap.public=%s", cloudwatchAgentOperatorRepositoryURL), + "--set", fmt.Sprintf("clusterName=%s", env.EKSClusterName), + "--set", fmt.Sprintf("region=%s", env.Region), + "--set", fmt.Sprintf("agent.image.repository=%s", env.CloudwatchAgentRepository), + "--set", fmt.Sprintf("agent.image.tag=%s", env.CloudwatchAgentTag), + "--set", fmt.Sprintf("agent.image.repositoryDomainMap.public=%s", env.CloudwatchAgentRepositoryURL), + "--set", fmt.Sprintf("manager.image.repository=%s", env.CloudwatchAgentOperatorRepository), + "--set", fmt.Sprintf("manager.image.tag=%s", env.CloudwatchAgentOperatorTag), + "--set", fmt.Sprintf("manager.image.repositoryDomainMap.public=%s", env.CloudwatchAgentOperatorRepositoryURL), "--namespace", "amazon-cloudwatch", "--create-namespace", } - if agentConfig != "" { - agentConfigContent, err := os.ReadFile(agentConfig) + if env.AgentConfig != "" { + agentConfigContent, err := os.ReadFile(env.AgentConfig) if err != nil { return fmt.Errorf("failed to read agent config file: %w", err) } @@ -112,7 +88,7 @@ func ApplyHelm() error { fmt.Println("Waiting for CloudWatch Agent Operator to initialize...") time.Sleep(300 * time.Second) - applyCmd := exec.Command("kubectl", "apply", "-f", sampleApp) + applyCmd := exec.Command("kubectl", "apply", "-f", env.SampleApp) output, err = applyCmd.CombinedOutput() if err != nil { return fmt.Errorf("failed to apply sample app: %w\nOutput: %s", err, output) From 912d98e2ebb3e11c8b7603d353e633bb664498cc Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 1 Dec 2024 17:29:17 -0500 Subject: [PATCH 03/92] Fix file positioning. --- terraform/eks/e2e/main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index a8aa9917a..2e4470aa4 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -124,10 +124,10 @@ resource "null_resource" "validator" { -cloudwatch_agent_operator_repository=${var.cloudwatch_agent_operator_repository} \ -cloudwatch_agent_operator_tag=${var.cloudwatch_agent_operator_tag} \ -cloudwatch_agent_operator_repository_url=${var.cloudwatch_agent_operator_repository_url} \ - -agent_config=${var.agent_config} \ - -otel_config=${var.otel_config} \ - -prometheus_config=${var.prometheus_config} \ - -sample_app=${var.sample_app} + -agent_config=${var.test_dir}/${var.agent_config} \ + -otel_config=${var.test_dir}/${var.otel_config} \ + -prometheus_config=${var.test_dir}/${var.prometheus_config} \ + -sample_app=${var.test_dir}/${var.sample_app} EOT } } From 5d0362fa5d03b7fc25d13e3b9086676e3306c5cb Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 1 Dec 2024 17:30:59 -0500 Subject: [PATCH 04/92] Fix file positioning. --- generator/test_case_generator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 6f555c256..5b2a5c402 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -244,7 +244,7 @@ var testTypeToTestConfig = map[string][]testConfig{ {testDir: "./test/metric_value_benchmark"}, }, "eks_e2e": { - {testDir: "./test/e2e/jmx"}, + {testDir: "../../../test/e2e/jmx"}, }, } From 67407158f7fa1526b87706f79d706cc86616bebb Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 1 Dec 2024 18:48:40 -0500 Subject: [PATCH 05/92] Fix values. --- terraform/eks/e2e/main.tf | 8 ++++---- test/e2e/jmx/jmx_test.go | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 2e4470aa4..ea20f0c8f 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -124,10 +124,10 @@ resource "null_resource" "validator" { -cloudwatch_agent_operator_repository=${var.cloudwatch_agent_operator_repository} \ -cloudwatch_agent_operator_tag=${var.cloudwatch_agent_operator_tag} \ -cloudwatch_agent_operator_repository_url=${var.cloudwatch_agent_operator_repository_url} \ - -agent_config=${var.test_dir}/${var.agent_config} \ - -otel_config=${var.test_dir}/${var.otel_config} \ - -prometheus_config=${var.test_dir}/${var.prometheus_config} \ - -sample_app=${var.test_dir}/${var.sample_app} + -agent_config="${var.test_dir}/${var.agent_config}" \ + -otel_config="${var.test_dir}/${var.otel_config}" \ + -prometheus_config="${var.test_dir}/${var.prometheus_config}" \ + -sample_app="${var.test_dir}/${var.sample_app}" EOT } } diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 198751e0e..7cbbe4d10 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -136,7 +136,7 @@ func TestMetrics(t *testing.T) { name string namespace string }{ - {"tomcat.traffic", "JVM_TOMCAT_E2E"}, + {"tomcat.traffic", "JVM_TOMCAT_E2E"}, // Add more metrics. } for _, metric := range metricsToCheck { @@ -144,4 +144,6 @@ func TestMetrics(t *testing.T) { awsservice.ValidateMetricWithTest(t, metric.name, metric.namespace, nil, 5, 1*time.Minute) }) } + + // Add more testing here from cloudwatchmetrics.go. } From c401c0e863c34871f0a062dd59bccd8bd9ffec14 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 1 Dec 2024 19:59:25 -0500 Subject: [PATCH 06/92] Fix otel and prometheus config set-up. --- environment/metadata.go | 4 ++-- terraform/eks/e2e/main.tf | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/environment/metadata.go b/environment/metadata.go index 46fbd408d..a77eeb12d 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -138,8 +138,8 @@ func registerEKSE2ETestData(dataString *MetaDataStrings) { flag.StringVar(&(dataString.CloudwatchAgentOperatorTag), "cloudwatch_agent_operator_tag", "", "CloudWatch Agent Operator tag") flag.StringVar(&(dataString.CloudwatchAgentOperatorRepositoryURL), "cloudwatch_agent_operator_repository_url", "", "CloudWatch Agent Operator repository URL") flag.StringVar(&(dataString.AgentConfig), "agent_config", "", "Agent configuration file path") - flag.StringVar(&(dataString.AgentConfig), "prometheus_config", "", "Prometheus configuration file path") - flag.StringVar(&(dataString.AgentConfig), "otel_config", "", "OpenTelemetry configuration file path") + flag.StringVar(&(dataString.PrometheusConfig), "prometheus_config", "", "Prometheus configuration file path") + flag.StringVar(&(dataString.OtelConfig), "otel_config", "", "OpenTelemetry configuration file path") flag.StringVar(&(dataString.SampleApp), "sample_app", "", "Sample application manifest file path") } diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index ea20f0c8f..a3477358e 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -117,6 +117,7 @@ resource "null_resource" "validator" { -k8s_version=${var.k8s_version} \ -eksClusterName=${aws_eks_cluster.this.name} \ -computeType=EKS \ + -eksDeploymentStrategy=DAEMON \ -helm_charts_branch=${var.helm_charts_branch} \ -cloudwatch_agent_repository=${var.cloudwatch_agent_repository} \ -cloudwatch_agent_tag=${var.cloudwatch_agent_tag} \ @@ -125,8 +126,8 @@ resource "null_resource" "validator" { -cloudwatch_agent_operator_tag=${var.cloudwatch_agent_operator_tag} \ -cloudwatch_agent_operator_repository_url=${var.cloudwatch_agent_operator_repository_url} \ -agent_config="${var.test_dir}/${var.agent_config}" \ - -otel_config="${var.test_dir}/${var.otel_config}" \ - -prometheus_config="${var.test_dir}/${var.prometheus_config}" \ + ${var.otel_config != "" ? "-otel_config=\"${var.test_dir}/${var.otel_config}\"" : ""} \ + ${var.prometheus_config != "" ? "-prometheus_config=\"${var.test_dir}/${var.prometheus_config}\"" : ""} \ -sample_app="${var.test_dir}/${var.sample_app}" EOT } From e8d49018c5208c37dd9edfbc9c7b148620e968cd Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 1 Dec 2024 23:06:40 -0500 Subject: [PATCH 07/92] Remove jmx files. --- test/e2e/jmx/jmx_test.go | 149 ------------------ .../resources/cwagent_configs/jvm_tomcat.json | 40 ----- .../sample_apps/tomcat-deployment.yaml | 37 ----- 3 files changed, 226 deletions(-) delete mode 100644 test/e2e/jmx/jmx_test.go delete mode 100644 test/e2e/jmx/resources/cwagent_configs/jvm_tomcat.json delete mode 100644 test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go deleted file mode 100644 index 7cbbe4d10..000000000 --- a/test/e2e/jmx/jmx_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package main - -import ( - "context" - "flag" - "fmt" - "github.com/aws/amazon-cloudwatch-agent-test/environment" - "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" - "os" - "os/exec" - "path/filepath" - "testing" - "time" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" -) - -func init() { - environment.RegisterEnvironmentMetaDataFlags() -} - -func TestMain(m *testing.M) { - flag.Parse() - env := environment.GetEnvironmentMetaData() - - if env.Region != "us-west-2" { - if err := awsservice.ReconfigureAWSClients(env.Region); err != nil { - fmt.Printf("Failed to reconfigure AWS clients: %v\n", err) - os.Exit(1) - } - fmt.Printf("AWS clients reconfigured to use region: %s\n", env.Region) - } else { - fmt.Printf("Using default testing region: us-west-2\n") - } - - fmt.Println("Starting Helm installation...") - if err := ApplyHelm(env); err != nil { - fmt.Printf("Failed to apply Helm: %v\n", err) - os.Exit(1) - } - - fmt.Println("Waiting for metrics to propagate...") - time.Sleep(5 * time.Minute) - - os.Exit(m.Run()) -} - -func ApplyHelm(env *environment.MetaData) error { - updateKubeconfigCmd := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName) - output, err := updateKubeconfigCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to update kubeconfig: %w\nOutput: %s", err, output) - } - - helmCmd := []string{ - "helm", "upgrade", "--install", "amazon-cloudwatch-observability", - filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability"), - "--values", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability", "values.yaml"), - "--set", fmt.Sprintf("clusterName=%s", env.EKSClusterName), - "--set", fmt.Sprintf("region=%s", env.Region), - "--set", fmt.Sprintf("agent.image.repository=%s", env.CloudwatchAgentRepository), - "--set", fmt.Sprintf("agent.image.tag=%s", env.CloudwatchAgentTag), - "--set", fmt.Sprintf("agent.image.repositoryDomainMap.public=%s", env.CloudwatchAgentRepositoryURL), - "--set", fmt.Sprintf("manager.image.repository=%s", env.CloudwatchAgentOperatorRepository), - "--set", fmt.Sprintf("manager.image.tag=%s", env.CloudwatchAgentOperatorTag), - "--set", fmt.Sprintf("manager.image.repositoryDomainMap.public=%s", env.CloudwatchAgentOperatorRepositoryURL), - "--namespace", "amazon-cloudwatch", - "--create-namespace", - } - - if env.AgentConfig != "" { - agentConfigContent, err := os.ReadFile(env.AgentConfig) - if err != nil { - return fmt.Errorf("failed to read agent config file: %w", err) - } - helmCmd = append(helmCmd, "--set-json", fmt.Sprintf("agent.config=%s", string(agentConfigContent))) - } - - helmUpgradeCmd := exec.Command(helmCmd[0], helmCmd[1:]...) - helmUpgradeCmd.Stdout = os.Stdout - helmUpgradeCmd.Stderr = os.Stderr - if err := helmUpgradeCmd.Run(); err != nil { - return fmt.Errorf("failed to install Helm release: %w", err) - } - - fmt.Println("Waiting for CloudWatch Agent Operator to initialize...") - time.Sleep(300 * time.Second) - - applyCmd := exec.Command("kubectl", "apply", "-f", env.SampleApp) - output, err = applyCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to apply sample app: %w\nOutput: %s", err, output) - } - - waitCmd := exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=300s", "deployment", "--all", "-n", "default") - output, err = waitCmd.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to wait for deployments: %w\nOutput: %s", err, output) - } - - return nil -} - -func TestResources(t *testing.T) { - config, err := clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config")) - if err != nil { - t.Fatalf("Error building kubeconfig: %v", err) - } - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - t.Fatalf("Error creating Kubernetes client: %v", err) - } - - daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - if err != nil { - t.Errorf("Error getting CloudWatch Agent DaemonSet: %v", err) - } - if daemonSet == nil { - t.Error("CloudWatch Agent DaemonSet not found") - } - - configMap, err := clientset.CoreV1().ConfigMaps("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - if err != nil { - t.Errorf("Error getting CloudWatch Agent ConfigMap: %v", err) - } - if configMap == nil { - t.Error("CloudWatch Agent ConfigMap not found") - } -} - -func TestMetrics(t *testing.T) { - metricsToCheck := []struct { - name string - namespace string - }{ - {"tomcat.traffic", "JVM_TOMCAT_E2E"}, // Add more metrics. - } - - for _, metric := range metricsToCheck { - t.Run(metric.name, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric.name, metric.namespace, nil, 5, 1*time.Minute) - }) - } - - // Add more testing here from cloudwatchmetrics.go. -} diff --git a/test/e2e/jmx/resources/cwagent_configs/jvm_tomcat.json b/test/e2e/jmx/resources/cwagent_configs/jvm_tomcat.json deleted file mode 100644 index ec03d2641..000000000 --- a/test/e2e/jmx/resources/cwagent_configs/jvm_tomcat.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "metrics": { - "namespace": "JVM_TOMCAT_E2E", - "metrics_collected": { - "jmx": { - "jvm": { - "measurement": [ - "jvm.classes.loaded", - "jvm.gc.collections.count", - "jvm.gc.collections.elapsed", - "jvm.memory.heap.init", - "jvm.memory.heap.max", - "jvm.memory.heap.used", - "jvm.memory.heap.committed", - "jvm.memory.nonheap.init", - "jvm.memory.nonheap.max", - "jvm.memory.nonheap.used", - "jvm.memory.nonheap.committed", - "jvm.memory.pool.init", - "jvm.memory.pool.max", - "jvm.memory.pool.used", - "jvm.memory.pool.committed", - "jvm.threads.count" - ] - }, - "tomcat": { - "measurement": [ - "tomcat.sessions", - "tomcat.errors", - "tomcat.request_count", - "tomcat.max_time", - "tomcat.processing_time", - "tomcat.traffic", - "tomcat.threads" - ] - } - } - } - } -} \ No newline at end of file diff --git a/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml b/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml deleted file mode 100644 index 0a966a3a9..000000000 --- a/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: tomcat-service -spec: - type: NodePort - selector: - app: tomcat - ports: - - port: 80 - protocol: TCP - targetPort: 8080 - nodePort: 32227 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tomcat-deployment -spec: - replicas: 1 - selector: - matchLabels: - app: tomcat - template: - metadata: - labels: - app: tomcat - annotations: - instrumentation.opentelemetry.io/inject-java: "true" - cloudwatch.aws.amazon.com/inject-jmx-jvm: "true" - cloudwatch.aws.amazon.com/inject-jmx-tomcat: "true" - spec: - containers: - - name: tomcat-container - image: public.ecr.aws/o5o6y1v9/tomcat:latest - ports: - - containerPort: 8080 \ No newline at end of file From 22186ed7e568b48e569d0d6b8f6973fc92468f97 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 1 Dec 2024 23:13:56 -0500 Subject: [PATCH 08/92] Add jmx testing. --- test/e2e/jmx/jmx_test.go | 149 ++++++++++++++++++ .../resources/cwagent_configs/jvm_tomcat.json | 40 +++++ .../sample_apps/tomcat-deployment.yaml | 37 +++++ 3 files changed, 226 insertions(+) create mode 100644 test/e2e/jmx/jmx_test.go create mode 100644 test/e2e/jmx/resources/cwagent_configs/jvm_tomcat.json create mode 100644 test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go new file mode 100644 index 000000000..7cbbe4d10 --- /dev/null +++ b/test/e2e/jmx/jmx_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "flag" + "fmt" + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +func TestMain(m *testing.M) { + flag.Parse() + env := environment.GetEnvironmentMetaData() + + if env.Region != "us-west-2" { + if err := awsservice.ReconfigureAWSClients(env.Region); err != nil { + fmt.Printf("Failed to reconfigure AWS clients: %v\n", err) + os.Exit(1) + } + fmt.Printf("AWS clients reconfigured to use region: %s\n", env.Region) + } else { + fmt.Printf("Using default testing region: us-west-2\n") + } + + fmt.Println("Starting Helm installation...") + if err := ApplyHelm(env); err != nil { + fmt.Printf("Failed to apply Helm: %v\n", err) + os.Exit(1) + } + + fmt.Println("Waiting for metrics to propagate...") + time.Sleep(5 * time.Minute) + + os.Exit(m.Run()) +} + +func ApplyHelm(env *environment.MetaData) error { + updateKubeconfigCmd := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName) + output, err := updateKubeconfigCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to update kubeconfig: %w\nOutput: %s", err, output) + } + + helmCmd := []string{ + "helm", "upgrade", "--install", "amazon-cloudwatch-observability", + filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability"), + "--values", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability", "values.yaml"), + "--set", fmt.Sprintf("clusterName=%s", env.EKSClusterName), + "--set", fmt.Sprintf("region=%s", env.Region), + "--set", fmt.Sprintf("agent.image.repository=%s", env.CloudwatchAgentRepository), + "--set", fmt.Sprintf("agent.image.tag=%s", env.CloudwatchAgentTag), + "--set", fmt.Sprintf("agent.image.repositoryDomainMap.public=%s", env.CloudwatchAgentRepositoryURL), + "--set", fmt.Sprintf("manager.image.repository=%s", env.CloudwatchAgentOperatorRepository), + "--set", fmt.Sprintf("manager.image.tag=%s", env.CloudwatchAgentOperatorTag), + "--set", fmt.Sprintf("manager.image.repositoryDomainMap.public=%s", env.CloudwatchAgentOperatorRepositoryURL), + "--namespace", "amazon-cloudwatch", + "--create-namespace", + } + + if env.AgentConfig != "" { + agentConfigContent, err := os.ReadFile(env.AgentConfig) + if err != nil { + return fmt.Errorf("failed to read agent config file: %w", err) + } + helmCmd = append(helmCmd, "--set-json", fmt.Sprintf("agent.config=%s", string(agentConfigContent))) + } + + helmUpgradeCmd := exec.Command(helmCmd[0], helmCmd[1:]...) + helmUpgradeCmd.Stdout = os.Stdout + helmUpgradeCmd.Stderr = os.Stderr + if err := helmUpgradeCmd.Run(); err != nil { + return fmt.Errorf("failed to install Helm release: %w", err) + } + + fmt.Println("Waiting for CloudWatch Agent Operator to initialize...") + time.Sleep(300 * time.Second) + + applyCmd := exec.Command("kubectl", "apply", "-f", env.SampleApp) + output, err = applyCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to apply sample app: %w\nOutput: %s", err, output) + } + + waitCmd := exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=300s", "deployment", "--all", "-n", "default") + output, err = waitCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to wait for deployments: %w\nOutput: %s", err, output) + } + + return nil +} + +func TestResources(t *testing.T) { + config, err := clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config")) + if err != nil { + t.Fatalf("Error building kubeconfig: %v", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + t.Fatalf("Error creating Kubernetes client: %v", err) + } + + daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + if err != nil { + t.Errorf("Error getting CloudWatch Agent DaemonSet: %v", err) + } + if daemonSet == nil { + t.Error("CloudWatch Agent DaemonSet not found") + } + + configMap, err := clientset.CoreV1().ConfigMaps("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + if err != nil { + t.Errorf("Error getting CloudWatch Agent ConfigMap: %v", err) + } + if configMap == nil { + t.Error("CloudWatch Agent ConfigMap not found") + } +} + +func TestMetrics(t *testing.T) { + metricsToCheck := []struct { + name string + namespace string + }{ + {"tomcat.traffic", "JVM_TOMCAT_E2E"}, // Add more metrics. + } + + for _, metric := range metricsToCheck { + t.Run(metric.name, func(t *testing.T) { + awsservice.ValidateMetricWithTest(t, metric.name, metric.namespace, nil, 5, 1*time.Minute) + }) + } + + // Add more testing here from cloudwatchmetrics.go. +} diff --git a/test/e2e/jmx/resources/cwagent_configs/jvm_tomcat.json b/test/e2e/jmx/resources/cwagent_configs/jvm_tomcat.json new file mode 100644 index 000000000..ec03d2641 --- /dev/null +++ b/test/e2e/jmx/resources/cwagent_configs/jvm_tomcat.json @@ -0,0 +1,40 @@ +{ + "metrics": { + "namespace": "JVM_TOMCAT_E2E", + "metrics_collected": { + "jmx": { + "jvm": { + "measurement": [ + "jvm.classes.loaded", + "jvm.gc.collections.count", + "jvm.gc.collections.elapsed", + "jvm.memory.heap.init", + "jvm.memory.heap.max", + "jvm.memory.heap.used", + "jvm.memory.heap.committed", + "jvm.memory.nonheap.init", + "jvm.memory.nonheap.max", + "jvm.memory.nonheap.used", + "jvm.memory.nonheap.committed", + "jvm.memory.pool.init", + "jvm.memory.pool.max", + "jvm.memory.pool.used", + "jvm.memory.pool.committed", + "jvm.threads.count" + ] + }, + "tomcat": { + "measurement": [ + "tomcat.sessions", + "tomcat.errors", + "tomcat.request_count", + "tomcat.max_time", + "tomcat.processing_time", + "tomcat.traffic", + "tomcat.threads" + ] + } + } + } + } +} \ No newline at end of file diff --git a/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml b/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml new file mode 100644 index 000000000..0a966a3a9 --- /dev/null +++ b/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: tomcat-service +spec: + type: NodePort + selector: + app: tomcat + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + nodePort: 32227 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tomcat-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: tomcat + template: + metadata: + labels: + app: tomcat + annotations: + instrumentation.opentelemetry.io/inject-java: "true" + cloudwatch.aws.amazon.com/inject-jmx-jvm: "true" + cloudwatch.aws.amazon.com/inject-jmx-tomcat: "true" + spec: + containers: + - name: tomcat-container + image: public.ecr.aws/o5o6y1v9/tomcat:latest + ports: + - containerPort: 8080 \ No newline at end of file From b5b2fb96edb26aaa834382fd87316a63e0c5c9ea Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 8 Dec 2024 16:42:42 -0500 Subject: [PATCH 09/92] Allow to choose e2e or regular tests. --- generator/test_case_generator.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index e74386706..dc9a5d308 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -5,6 +5,7 @@ package main import ( "encoding/json" + "flag" "fmt" "io" "log" @@ -243,6 +244,9 @@ var testTypeToTestConfig = map[string][]testConfig{ "eks_deployment": { {testDir: "./test/metric_value_benchmark"}, }, +} + +var testTypeToTestConfigE2E = map[string][]testConfig{ "eks_e2e": { {testDir: "../../../test/e2e/jmx"}, }, @@ -284,9 +288,17 @@ func copyAllEC2LinuxTestForOnpremTesting() { } func main() { - copyAllEC2LinuxTestForOnpremTesting() + useE2E := flag.Bool("e2e", false, "Use e2e test matrix generation") + flag.Parse() + + configMap := testTypeToTestConfig + if !*useE2E { + copyAllEC2LinuxTestForOnpremTesting() + } else { + configMap = testTypeToTestConfigE2E + } - for testType, testConfigs := range testTypeToTestConfig { + for testType, testConfigs := range configMap { for _, partition := range partitionTests { if len(partition.tests) != 0 && !slices.Contains(partition.tests, testType) { continue From 9a830a0c22adafbf8c4b873cb871e2038ef3588b Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 8 Dec 2024 17:32:15 -0500 Subject: [PATCH 10/92] Add tomcat sessions check. --- test/e2e/jmx/jmx_test.go | 82 ++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 7cbbe4d10..d46d8af3f 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/aws/amazon-cloudwatch-agent-test/environment" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "net/http" "os" "os/exec" "path/filepath" @@ -132,18 +133,75 @@ func TestResources(t *testing.T) { } func TestMetrics(t *testing.T) { - metricsToCheck := []struct { - name string - namespace string - }{ - {"tomcat.traffic", "JVM_TOMCAT_E2E"}, // Add more metrics. - } + t.Run("verify_jvm_tomcat_metrics", func(t *testing.T) { + metricsToCheck := []struct { + name string + namespace string + }{ + {"tomcat.traffic", "JVM_TOMCAT_E2E"}, + {"jvm.classes.loaded", "JVM_TOMCAT_E2E"}, + {"jvm.gc.collections.count", "JVM_TOMCAT_E2E"}, + {"jvm.gc.collections.elapsed", "JVM_TOMCAT_E2E"}, + {"jvm.memory.heap.init", "JVM_TOMCAT_E2E"}, + {"jvm.memory.heap.max", "JVM_TOMCAT_E2E"}, + {"jvm.memory.heap.used", "JVM_TOMCAT_E2E"}, + {"jvm.memory.heap.committed", "JVM_TOMCAT_E2E"}, + {"jvm.memory.nonheap.init", "JVM_TOMCAT_E2E"}, + {"jvm.memory.nonheap.max", "JVM_TOMCAT_E2E"}, + {"jvm.memory.nonheap.used", "JVM_TOMCAT_E2E"}, + {"jvm.memory.nonheap.committed", "JVM_TOMCAT_E2E"}, + {"jvm.memory.pool.init", "JVM_TOMCAT_E2E"}, + {"jvm.memory.pool.max", "JVM_TOMCAT_E2E"}, + {"jvm.memory.pool.used", "JVM_TOMCAT_E2E"}, + {"jvm.memory.pool.committed", "JVM_TOMCAT_E2E"}, + {"jvm.threads.count", "JVM_TOMCAT_E2E"}, + {"tomcat.sessions", "JVM_TOMCAT_E2E"}, + {"tomcat.errors", "JVM_TOMCAT_E2E"}, + {"tomcat.request_count", "JVM_TOMCAT_E2E"}, + {"tomcat.max_time", "JVM_TOMCAT_E2E"}, + {"tomcat.processing_time", "JVM_TOMCAT_E2E"}, + {"tomcat.threads", "JVM_TOMCAT_E2E"}, + } - for _, metric := range metricsToCheck { - t.Run(metric.name, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric.name, metric.namespace, nil, 5, 1*time.Minute) - }) - } + for _, metric := range metricsToCheck { + t.Run(metric.name, func(t *testing.T) { + awsservice.ValidateMetricWithTest(t, metric.name, metric.namespace, nil, 5, 1*time.Minute) + }) + } + }) + + t.Run("verify_tomcat_sessions", func(t *testing.T) { + for i := 0; i < 5; i++ { + resp, err := http.Get("http://localhost:8080/webapp/index.jsp") + if err != nil { + t.Logf("Request attempt %d failed: %v", i+1, err) + continue + } + err = resp.Body.Close() + if err != nil { + t.Errorf("Failed to close response body: %v", err) + return + } + } - // Add more testing here from cloudwatchmetrics.go. + time.Sleep(30 * time.Second) + + startTime := time.Now().Add(-5 * time.Minute) + endTime := time.Now() + + hasActiveSessions := awsservice.ValidateSampleCount( + "tomcat.sessions", + "JVM_TOMCAT_E2E", + nil, + startTime, + endTime, + 1, + 1000, + 60, + ) + + if !hasActiveSessions { + t.Error("Expected non-zero tomcat.sessions after applying traffic") + } + }) } From 579510311b3e1f835e0b3c1637b414635ed2d326 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 8 Dec 2024 19:32:03 -0500 Subject: [PATCH 11/92] Add TA. --- environment/metadata.go | 12 ++++++++++++ terraform/eks/e2e/main.tf | 3 +++ terraform/eks/e2e/variables.tf | 15 +++++++++++++++ .../cwagent_configs/containerinsights.json | 9 +++++++++ test/e2e/jmx/resources/cwagent_configs/kafka.json | 0 .../resources/cwagent_configs/missing_jvm.json | 0 .../resources/sample_apps/kafka-deployment.yaml | 0 7 files changed, 39 insertions(+) create mode 100644 test/e2e/jmx/resources/cwagent_configs/containerinsights.json create mode 100644 test/e2e/jmx/resources/cwagent_configs/kafka.json create mode 100644 test/e2e/jmx/resources/cwagent_configs/missing_jvm.json create mode 100644 test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml diff --git a/environment/metadata.go b/environment/metadata.go index a77eeb12d..07c8055e6 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -54,6 +54,9 @@ type MetaData struct { CloudwatchAgentOperatorRepository string CloudwatchAgentOperatorTag string CloudwatchAgentOperatorRepositoryURL string + CloudwatchAgentTargetAllocatorRepository string + CloudwatchAgentTargetAllocatorTag string + CloudwatchAgentTargetAllocatorRepositoryURL string AgentConfig string PrometheusConfig string OtelConfig string @@ -91,6 +94,9 @@ type MetaDataStrings struct { CloudwatchAgentOperatorRepository string CloudwatchAgentOperatorTag string CloudwatchAgentOperatorRepositoryURL string + CloudwatchAgentTargetAllocatorRepository string + CloudwatchAgentTargetAllocatorTag string + CloudwatchAgentTargetAllocatorRepositoryURL string AgentConfig string PrometheusConfig string OtelConfig string @@ -137,6 +143,9 @@ func registerEKSE2ETestData(dataString *MetaDataStrings) { flag.StringVar(&(dataString.CloudwatchAgentOperatorRepository), "cloudwatch_agent_operator_repository", "", "CloudWatch Agent Operator repository") flag.StringVar(&(dataString.CloudwatchAgentOperatorTag), "cloudwatch_agent_operator_tag", "", "CloudWatch Agent Operator tag") flag.StringVar(&(dataString.CloudwatchAgentOperatorRepositoryURL), "cloudwatch_agent_operator_repository_url", "", "CloudWatch Agent Operator repository URL") + flag.StringVar(&(dataString.CloudwatchAgentTargetAllocatorRepository), "cloudwatch_agent_target_allocator_repository", "", "CloudWatch Agent Target Allocator repository") + flag.StringVar(&(dataString.CloudwatchAgentTargetAllocatorTag), "cloudwatch_agent_target_allocator_tag", "", "CloudWatch Agent Target Allocator tag") + flag.StringVar(&(dataString.CloudwatchAgentTargetAllocatorRepositoryURL), "cloudwatch_agent_target_allocator_repository_url", "", "CloudWatch Agent Target Allocator repository URL") flag.StringVar(&(dataString.AgentConfig), "agent_config", "", "Agent configuration file path") flag.StringVar(&(dataString.PrometheusConfig), "prometheus_config", "", "Prometheus configuration file path") flag.StringVar(&(dataString.OtelConfig), "otel_config", "", "OpenTelemetry configuration file path") @@ -316,6 +325,9 @@ func GetEnvironmentMetaData() *MetaData { metaDataStorage.CloudwatchAgentOperatorRepository = registeredMetaDataStrings.CloudwatchAgentOperatorRepository metaDataStorage.CloudwatchAgentOperatorTag = registeredMetaDataStrings.CloudwatchAgentOperatorTag metaDataStorage.CloudwatchAgentOperatorRepositoryURL = registeredMetaDataStrings.CloudwatchAgentOperatorRepositoryURL + metaDataStorage.CloudwatchAgentTargetAllocatorRepository = registeredMetaDataStrings.CloudwatchAgentTargetAllocatorRepository + metaDataStorage.CloudwatchAgentTargetAllocatorTag = registeredMetaDataStrings.CloudwatchAgentTargetAllocatorTag + metaDataStorage.CloudwatchAgentTargetAllocatorRepositoryURL = registeredMetaDataStrings.CloudwatchAgentTargetAllocatorRepositoryURL metaDataStorage.AgentConfig = registeredMetaDataStrings.AgentConfig metaDataStorage.PrometheusConfig = registeredMetaDataStrings.PrometheusConfig metaDataStorage.OtelConfig = registeredMetaDataStrings.OtelConfig diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index a3477358e..f9f1bb6a6 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -125,6 +125,9 @@ resource "null_resource" "validator" { -cloudwatch_agent_operator_repository=${var.cloudwatch_agent_operator_repository} \ -cloudwatch_agent_operator_tag=${var.cloudwatch_agent_operator_tag} \ -cloudwatch_agent_operator_repository_url=${var.cloudwatch_agent_operator_repository_url} \ + -cloudwatch_agent_target_allocator_repository=${var.cloudwatch_agent_target_allocator_repository} \ + -cloudwatch_agent_target_allocator_tag=${var.cloudwatch_agent_target_allocator_tag} \ + -cloudwatch_agent_target_allocator_repository_url=${var.cloudwatch_agent_target_allocator_repository_url} \ -agent_config="${var.test_dir}/${var.agent_config}" \ ${var.otel_config != "" ? "-otel_config=\"${var.test_dir}/${var.otel_config}\"" : ""} \ ${var.prometheus_config != "" ? "-prometheus_config=\"${var.test_dir}/${var.prometheus_config}\"" : ""} \ diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index 00c8e4166..7c39f462d 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -51,6 +51,21 @@ variable "cloudwatch_agent_operator_repository_url" { default = "public.ecr.aws/cloudwatch-agent" } +variable "cloudwatch_agent_target_allocator_repository" { + type = string + default = "cloudwatch-agent-target-allocator" +} + +variable "cloudwatch_agent_target_allocator_tag" { + type = string + default = "latest" +} + +variable "cloudwatch_agent_target_allocator_repository_url" { + type = string + default = "public.ecr.aws/cloudwatch-agent" +} + variable "test_dir" { type = string default = "" diff --git a/test/e2e/jmx/resources/cwagent_configs/containerinsights.json b/test/e2e/jmx/resources/cwagent_configs/containerinsights.json new file mode 100644 index 000000000..6eaa8f2b6 --- /dev/null +++ b/test/e2e/jmx/resources/cwagent_configs/containerinsights.json @@ -0,0 +1,9 @@ +{ + "logs": { + "metrics_collected": { + "kubernetes": { + "jmx_container_insights": true + } + } + } +} \ No newline at end of file diff --git a/test/e2e/jmx/resources/cwagent_configs/kafka.json b/test/e2e/jmx/resources/cwagent_configs/kafka.json new file mode 100644 index 000000000..e69de29bb diff --git a/test/e2e/jmx/resources/cwagent_configs/missing_jvm.json b/test/e2e/jmx/resources/cwagent_configs/missing_jvm.json new file mode 100644 index 000000000..e69de29bb diff --git a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml new file mode 100644 index 000000000..e69de29bb From 6a5f26e507c2a27d04732889cedcbb95e84d8061 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 8 Dec 2024 19:41:03 -0500 Subject: [PATCH 12/92] Update jmx_test.go. --- test/e2e/jmx/jmx_test.go | 120 ++++++++++++++++-- .../jmx/resources/cwagent_configs/kafka.json | 58 +++++++++ .../sample_apps/kafka-deployment.yaml | 24 ++++ 3 files changed, 190 insertions(+), 12 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index d46d8af3f..7f21d50f5 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -22,6 +22,27 @@ func init() { environment.RegisterEnvironmentMetaDataFlags() } +type TestConfig struct { + metricTests []func(*testing.T) +} + +var testRegistry = map[string]TestConfig{ + "jvm_tomcat.json": { + metricTests: []func(*testing.T){ + testTomcatMetrics, + testTomcatSessions, + }, + }, + "kafka.json": { + metricTests: []func(*testing.T){ + testKafkaMetrics, + }, + }, + "containerinsights.json": { + metricTests: []func(*testing.T){}, + }, +} + func TestMain(m *testing.M) { flag.Parse() env := environment.GetEnvironmentMetaData() @@ -48,14 +69,15 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } +// Make this a utility function. func ApplyHelm(env *environment.MetaData) error { - updateKubeconfigCmd := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName) - output, err := updateKubeconfigCmd.CombinedOutput() + updateKubeconfig := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName) + output, err := updateKubeconfig.CombinedOutput() if err != nil { return fmt.Errorf("failed to update kubeconfig: %w\nOutput: %s", err, output) } - helmCmd := []string{ + helm := []string{ "helm", "upgrade", "--install", "amazon-cloudwatch-observability", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability"), "--values", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability", "values.yaml"), @@ -76,27 +98,27 @@ func ApplyHelm(env *environment.MetaData) error { if err != nil { return fmt.Errorf("failed to read agent config file: %w", err) } - helmCmd = append(helmCmd, "--set-json", fmt.Sprintf("agent.config=%s", string(agentConfigContent))) + helm = append(helm, "--set-json", fmt.Sprintf("agent.config=%s", string(agentConfigContent))) } - helmUpgradeCmd := exec.Command(helmCmd[0], helmCmd[1:]...) - helmUpgradeCmd.Stdout = os.Stdout - helmUpgradeCmd.Stderr = os.Stderr - if err := helmUpgradeCmd.Run(); err != nil { + helmUpgrade := exec.Command(helm[0], helm[1:]...) + helmUpgrade.Stdout = os.Stdout + helmUpgrade.Stderr = os.Stderr + if err := helmUpgrade.Run(); err != nil { return fmt.Errorf("failed to install Helm release: %w", err) } fmt.Println("Waiting for CloudWatch Agent Operator to initialize...") time.Sleep(300 * time.Second) - applyCmd := exec.Command("kubectl", "apply", "-f", env.SampleApp) - output, err = applyCmd.CombinedOutput() + apply := exec.Command("kubectl", "apply", "-f", env.SampleApp) + output, err = apply.CombinedOutput() if err != nil { return fmt.Errorf("failed to apply sample app: %w\nOutput: %s", err, output) } - waitCmd := exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=300s", "deployment", "--all", "-n", "default") - output, err = waitCmd.CombinedOutput() + wait := exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=300s", "deployment", "--all", "-n", "default") + output, err = wait.CombinedOutput() if err != nil { return fmt.Errorf("failed to wait for deployments: %w\nOutput: %s", err, output) } @@ -133,6 +155,21 @@ func TestResources(t *testing.T) { } func TestMetrics(t *testing.T) { + env := environment.GetEnvironmentMetaData() + configFile := filepath.Base(env.AgentConfig) + + config, exists := testRegistry[configFile] + if !exists { + t.Skipf("No tests registered for config file: %s", configFile) + return + } + + for _, testFunc := range config.metricTests { + testFunc(t) + } +} + +func testTomcatMetrics(t *testing.T) { t.Run("verify_jvm_tomcat_metrics", func(t *testing.T) { metricsToCheck := []struct { name string @@ -169,7 +206,9 @@ func TestMetrics(t *testing.T) { }) } }) +} +func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { for i := 0; i < 5; i++ { resp, err := http.Get("http://localhost:8080/webapp/index.jsp") @@ -205,3 +244,60 @@ func TestMetrics(t *testing.T) { } }) } + +func testKafkaMetrics(t *testing.T) { + t.Run("verify_kafka_metrics", func(t *testing.T) { + metricsToCheck := []struct { + name string + namespace string + }{ + {"kafka.message.count", "KAFKA_E2E"}, + {"kafka.request.count", "KAFKA_E2E"}, + {"kafka.request.failed", "KAFKA_E2E"}, + {"kafka.request.time.total", "KAFKA_E2E"}, + {"kafka.request.time.50p", "KAFKA_E2E"}, + {"kafka.request.time.99p", "KAFKA_E2E"}, + {"kafka.request.time.avg", "KAFKA_E2E"}, + {"kafka.network.io", "KAFKA_E2E"}, + {"kafka.purgatory.size", "KAFKA_E2E"}, + {"kafka.partition.count", "KAFKA_E2E"}, + {"kafka.partition.offline", "KAFKA_E2E"}, + {"kafka.partition.under_replicated", "KAFKA_E2E"}, + {"kafka.isr.operation.count", "KAFKA_E2E"}, + {"kafka.max.lag", "KAFKA_E2E"}, + {"kafka.controller.active.count", "KAFKA_E2E"}, + {"kafka.leader.election.rate", "KAFKA_E2E"}, + {"kafka.unclean.election.rate", "KAFKA_E2E"}, + {"kafka.request.queue", "KAFKA_E2E"}, + {"kafka.logs.flush.time.count", "KAFKA_E2E"}, + {"kafka.logs.flush.time.median", "KAFKA_E2E"}, + {"kafka.logs.flush.time.99p", "KAFKA_E2E"}, + {"kafka.consumer.fetch-rate", "KAFKA_E2E"}, + {"kafka.consumer.records-lag-max", "KAFKA_E2E"}, + {"kafka.consumer.total.bytes-consumed-rate", "KAFKA_E2E"}, + {"kafka.consumer.total.fetch-size-avg", "KAFKA_E2E"}, + {"kafka.consumer.total.records-consumed-rate", "KAFKA_E2E"}, + {"kafka.consumer.bytes-consumed-rate", "KAFKA_E2E"}, + {"kafka.consumer.fetch-size-avg", "KAFKA_E2E"}, + {"kafka.consumer.records-consumed-rate", "KAFKA_E2E"}, + {"kafka.producer.io-wait-time-ns-avg", "KAFKA_E2E"}, + {"kafka.producer.outgoing-byte-rate", "KAFKA_E2E"}, + {"kafka.producer.request-latency-avg", "KAFKA_E2E"}, + {"kafka.producer.request-rate", "KAFKA_E2E"}, + {"kafka.producer.response-rate", "KAFKA_E2E"}, + {"kafka.producer.byte-rate", "KAFKA_E2E"}, + {"kafka.producer.compression-rate", "KAFKA_E2E"}, + {"kafka.producer.record-error-rate", "KAFKA_E2E"}, + {"kafka.producer.record-retry-rate", "KAFKA_E2E"}, + {"kafka.producer.record-send-rate", "KAFKA_E2E"}, + } + + for _, metric := range metricsToCheck { + t.Run(metric.name, func(t *testing.T) { + awsservice.ValidateMetricWithTest(t, metric.name, metric.namespace, nil, 5, 1*time.Minute) + }) + } + }) +} + +// Implement order and also helm upgrade + metric test. diff --git a/test/e2e/jmx/resources/cwagent_configs/kafka.json b/test/e2e/jmx/resources/cwagent_configs/kafka.json index e69de29bb..33d50e946 100644 --- a/test/e2e/jmx/resources/cwagent_configs/kafka.json +++ b/test/e2e/jmx/resources/cwagent_configs/kafka.json @@ -0,0 +1,58 @@ +{ + "metrics": { + "namespace": "KAFKA_E2E", + "metrics_collected": { + "kafka": { + "measurement": [ + "kafka.message.count", + "kafka.request.count", + "kafka.request.failed", + "kafka.request.time.total", + "kafka.request.time.50p", + "kafka.request.time.99p", + "kafka.request.time.avg", + "kafka.network.io", + "kafka.purgatory.size", + "kafka.partition.count", + "kafka.partition.offline", + "kafka.partition.under_replicated", + "kafka.isr.operation.count", + "kafka.max.lag", + "kafka.controller.active.count", + "kafka.leader.election.rate", + "kafka.unclean.election.rate", + "kafka.request.queue", + "kafka.logs.flush.time.count", + "kafka.logs.flush.time.median", + "kafka.logs.flush.time.99p" + ] + }, + "kafka-consumer": { + "measurement": [ + "kafka.consumer.fetch-rate", + "kafka.consumer.records-lag-max", + "kafka.consumer.total.bytes-consumed-rate", + "kafka.consumer.total.fetch-size-avg", + "kafka.consumer.total.records-consumed-rate", + "kafka.consumer.bytes-consumed-rate", + "kafka.consumer.fetch-size-avg", + "kafka.consumer.records-consumed-rate" + ] + }, + "kakfa-producer": { + "measurement": [ + "kafka.producer.io-wait-time-ns-avg", + "kafka.producer.outgoing-byte-rate", + "kafka.producer.request-latency-avg", + "kafka.producer.request-rate", + "kafka.producer.response-rate", + "kafka.producer.byte-rate", + "kafka.producer.compression-rate", + "kafka.producer.record-error-rate", + "kafka.producer.record-retry-rate", + "kafka.producer.record-send-rate" + ] + } + } + } +} \ No newline at end of file diff --git a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml index e69de29bb..00935e162 100644 --- a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml +++ b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: kafka + template: + metadata: + labels: + app: kafka + annotations: + instrumentation.opentelemetry.io/inject-java: "true" + cloudwatch.aws.amazon.com/inject-jmx-kafka: "true" + cloudwatch.aws.amazon.com/inject-jmx-kakfa-consumer: "true" + cloudwatch.aws.amazon.com/inject-jmx-kafka-producer: "true" + spec: + containers: + - name: kafka-container + image: apache/kafka:latest + ports: + - containerPort: 8080 \ No newline at end of file From c967e3d7e4d71d6f6e5fa3ff1b77e74915096013 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 8 Dec 2024 20:03:34 -0500 Subject: [PATCH 13/92] Port foward and fix kafka. --- test/e2e/jmx/jmx_test.go | 23 ++++++- .../sample_apps/kafka-deployment.yaml | 63 ++++++++++++++++--- 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 7f21d50f5..b1bb8f214 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" "time" @@ -111,16 +112,34 @@ func ApplyHelm(env *environment.MetaData) error { fmt.Println("Waiting for CloudWatch Agent Operator to initialize...") time.Sleep(300 * time.Second) + deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yml") + apply := exec.Command("kubectl", "apply", "-f", env.SampleApp) output, err = apply.CombinedOutput() if err != nil { return fmt.Errorf("failed to apply sample app: %w\nOutput: %s", err, output) } - wait := exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=300s", "deployment", "--all", "-n", "default") + wait := exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=300s", fmt.Sprintf("deployment/%s", deploymentName), "-n", "default") output, err = wait.CombinedOutput() if err != nil { - return fmt.Errorf("failed to wait for deployments: %w\nOutput: %s", err, output) + return fmt.Errorf("failed to wait for deployment %s: %w\nOutput: %s", deploymentName, err, output) + } + + var ports string + if deploymentName == "tomcat-deployment" { + ports = "8080:8080" + } else if deploymentName == "kafka-deployment" { + ports = "9092:9092" + } else { + return fmt.Errorf("unknown deployment type: %s", deploymentName) + } + + portForward := exec.Command("kubectl", "port-forward", fmt.Sprintf("deployment/%s", deploymentName), ports) + portForward.Stdout = os.Stdout + portForward.Stderr = os.Stderr + if err := portForward.Run(); err != nil { + fmt.Printf("Port forwarding failed: %v\n", err) } return nil diff --git a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml index 00935e162..2d7cf5ffd 100644 --- a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml +++ b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml @@ -1,3 +1,48 @@ +apiVersion: v1 +kind: Service +metadata: + name: zookeeper-service +spec: + selector: + app: zookeeper + ports: + - port: 2181 + targetPort: 2181 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zookeeper-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: zookeeper + template: + metadata: + labels: + app: zookeeper + spec: + containers: + - name: zookeeper + image: bitnami/zookeeper:latest + ports: + - containerPort: 2181 + env: + - name: ZOOKEEPER_CLIENT_PORT + value: "2181" +--- +apiVersion: v1 +kind: Service +metadata: + name: kafka-service +spec: + selector: + app: kafka + ports: + - port: 9092 + targetPort: 9092 +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -11,14 +56,16 @@ spec: metadata: labels: app: kafka - annotations: - instrumentation.opentelemetry.io/inject-java: "true" - cloudwatch.aws.amazon.com/inject-jmx-kafka: "true" - cloudwatch.aws.amazon.com/inject-jmx-kakfa-consumer: "true" - cloudwatch.aws.amazon.com/inject-jmx-kafka-producer: "true" spec: containers: - - name: kafka-container - image: apache/kafka:latest + - name: kafka + image: bitnami/kafka:latest ports: - - containerPort: 8080 \ No newline at end of file + - containerPort: 9092 + env: + - name: KAFKA_ZOOKEEPER_CONNECT + value: "zookeeper-service:2181" + - name: KAFKA_ADVERTISED_LISTENERS + value: "PLAINTEXT://:9092" + - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR + value: "1" \ No newline at end of file From 63b6ba83d3d2a64a1df9d00f4201b52e317fbd99 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 8 Dec 2024 23:40:47 -0500 Subject: [PATCH 14/92] Add CI. --- terraform/eks/e2e/main.tf | 2 +- test/e2e/jmx/jmx_test.go | 96 ++++++++++++++++++- .../cwagent_configs/missing_jvm.json | 0 3 files changed, 94 insertions(+), 4 deletions(-) delete mode 100644 test/e2e/jmx/resources/cwagent_configs/missing_jvm.json diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index f9f1bb6a6..b6edecca2 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -112,7 +112,7 @@ resource "null_resource" "validator" { provisioner "local-exec" { command = <<-EOT echo "Validating K8s resources and metrics" - go test -v ${var.test_dir} \ + go test -timeout 30m -v ${var.test_dir} \ -region=${var.region} \ -k8s_version=${var.k8s_version} \ -eksClusterName=${aws_eks_cluster.this.name} \ diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index b1bb8f214..b0e826044 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -40,7 +40,10 @@ var testRegistry = map[string]TestConfig{ }, }, "containerinsights.json": { - metricTests: []func(*testing.T){}, + metricTests: []func(*testing.T){ + testContainerInsightsMetrics, + testTomcatRejectedSessions, + }, }, } @@ -112,7 +115,7 @@ func ApplyHelm(env *environment.MetaData) error { fmt.Println("Waiting for CloudWatch Agent Operator to initialize...") time.Sleep(300 * time.Second) - deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yml") + deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml") apply := exec.Command("kubectl", "apply", "-f", env.SampleApp) output, err = apply.CombinedOutput() @@ -171,6 +174,22 @@ func TestResources(t *testing.T) { if configMap == nil { t.Error("CloudWatch Agent ConfigMap not found") } + + service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + if err != nil { + t.Errorf("Error getting CloudWatch Agent Service: %v", err) + } + if service == nil { + t.Error("CloudWatch Agent Service not found") + } + + serviceAccount, err := clientset.CoreV1().ServiceAccounts("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + if err != nil { + t.Errorf("Error getting CloudWatch Agent Service Account: %v", err) + } + if serviceAccount == nil { + t.Error("CloudWatch Agent Service Account not found") + } } func TestMetrics(t *testing.T) { @@ -319,4 +338,75 @@ func testKafkaMetrics(t *testing.T) { }) } -// Implement order and also helm upgrade + metric test. +func testContainerInsightsMetrics(t *testing.T) { + t.Run("verify_containerinsights_metrics", func(t *testing.T) { + metricsToCheck := []struct { + name string + namespace string + }{ + {"jvm_classes_loaded", "ContainerInsights/Prometheus"}, + {"jvm_threads_current", "ContainerInsights/Prometheus"}, + {"jvm_threads_daemon", "ContainerInsights/Prometheus"}, + {"java_lang_operatingsystem_totalswapspacesize", "ContainerInsights/Prometheus"}, + {"java_lang_operatingsystem_systemcpuload", "ContainerInsights/Prometheus"}, + {"java_lang_operatingsystem_processcpuload", "ContainerInsights/Prometheus"}, + {"java_lang_operatingsystem_freeswapspacesize", "ContainerInsights/Prometheus"}, + {"java_lang_operatingsystem_totalphysicalmemorysize", "ContainerInsights/Prometheus"}, + {"java_lang_operatingsystem_freephysicalmemorysize", "ContainerInsights/Prometheus"}, + {"java_lang_operatingsystem_openfiledescriptorcount", "ContainerInsights/Prometheus"}, + {"java_lang_operatingsystem_availableprocessors", "ContainerInsights/Prometheus"}, + {"jvm_memory_bytes_used", "ContainerInsights/Prometheus"}, + {"jvm_memory_pool_bytes_used", "ContainerInsights/Prometheus"}, + {"catalina_manager_activesessions", "ContainerInsights/Prometheus"}, + {"catalina_manager_rejectedsessions", "ContainerInsights/Prometheus"}, + {"catalina_globalrequestprocessor_bytesreceived", "ContainerInsights/Prometheus"}, + {"catalina_globalrequestprocessor_bytessent", "ContainerInsights/Prometheus"}, + {"catalina_globalrequestprocessor_requestcount", "ContainerInsights/Prometheus"}, + {"catalina_globalrequestprocessor_errorcount", "ContainerInsights/Prometheus"}, + {"catalina_globalrequestprocessor_processingtime", "ContainerInsights/Prometheus"}, + } + + for _, metric := range metricsToCheck { + t.Run(metric.name, func(t *testing.T) { + awsservice.ValidateMetricWithTest(t, metric.name, metric.namespace, nil, 5, 1*time.Minute) + }) + } + }) +} + +func testTomcatRejectedSessions(t *testing.T) { + t.Run("verify_tomcat_sessions", func(t *testing.T) { + for i := 0; i < 5; i++ { + resp, err := http.Get("http://localhost:8080/webapp/index.jsp") + if err != nil { + t.Logf("Request attempt %d failed: %v", i+1, err) + continue + } + err = resp.Body.Close() + if err != nil { + t.Errorf("Failed to close response body: %v", err) + return + } + } + + time.Sleep(30 * time.Second) + + startTime := time.Now().Add(-5 * time.Minute) + endTime := time.Now() + + hasActiveSessions := awsservice.ValidateSampleCount( + "catalina_manager_rejectedsessions", + "ContainerInsights/Prometheus", + nil, + startTime, + endTime, + 1, + 1000, + 60, + ) + + if !hasActiveSessions { + t.Error("Expected non-zero catalina_manager_rejectedsessions after applying traffic") + } + }) +} diff --git a/test/e2e/jmx/resources/cwagent_configs/missing_jvm.json b/test/e2e/jmx/resources/cwagent_configs/missing_jvm.json deleted file mode 100644 index e69de29bb..000000000 From dfd94e097d44150f3fb322af027f523adc4b249f Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 8 Dec 2024 23:42:55 -0500 Subject: [PATCH 15/92] Add TA. --- environment/metadata.go | 12 ++++++++++++ terraform/eks/e2e/main.tf | 5 ++++- terraform/eks/e2e/variables.tf | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/environment/metadata.go b/environment/metadata.go index a77eeb12d..07c8055e6 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -54,6 +54,9 @@ type MetaData struct { CloudwatchAgentOperatorRepository string CloudwatchAgentOperatorTag string CloudwatchAgentOperatorRepositoryURL string + CloudwatchAgentTargetAllocatorRepository string + CloudwatchAgentTargetAllocatorTag string + CloudwatchAgentTargetAllocatorRepositoryURL string AgentConfig string PrometheusConfig string OtelConfig string @@ -91,6 +94,9 @@ type MetaDataStrings struct { CloudwatchAgentOperatorRepository string CloudwatchAgentOperatorTag string CloudwatchAgentOperatorRepositoryURL string + CloudwatchAgentTargetAllocatorRepository string + CloudwatchAgentTargetAllocatorTag string + CloudwatchAgentTargetAllocatorRepositoryURL string AgentConfig string PrometheusConfig string OtelConfig string @@ -137,6 +143,9 @@ func registerEKSE2ETestData(dataString *MetaDataStrings) { flag.StringVar(&(dataString.CloudwatchAgentOperatorRepository), "cloudwatch_agent_operator_repository", "", "CloudWatch Agent Operator repository") flag.StringVar(&(dataString.CloudwatchAgentOperatorTag), "cloudwatch_agent_operator_tag", "", "CloudWatch Agent Operator tag") flag.StringVar(&(dataString.CloudwatchAgentOperatorRepositoryURL), "cloudwatch_agent_operator_repository_url", "", "CloudWatch Agent Operator repository URL") + flag.StringVar(&(dataString.CloudwatchAgentTargetAllocatorRepository), "cloudwatch_agent_target_allocator_repository", "", "CloudWatch Agent Target Allocator repository") + flag.StringVar(&(dataString.CloudwatchAgentTargetAllocatorTag), "cloudwatch_agent_target_allocator_tag", "", "CloudWatch Agent Target Allocator tag") + flag.StringVar(&(dataString.CloudwatchAgentTargetAllocatorRepositoryURL), "cloudwatch_agent_target_allocator_repository_url", "", "CloudWatch Agent Target Allocator repository URL") flag.StringVar(&(dataString.AgentConfig), "agent_config", "", "Agent configuration file path") flag.StringVar(&(dataString.PrometheusConfig), "prometheus_config", "", "Prometheus configuration file path") flag.StringVar(&(dataString.OtelConfig), "otel_config", "", "OpenTelemetry configuration file path") @@ -316,6 +325,9 @@ func GetEnvironmentMetaData() *MetaData { metaDataStorage.CloudwatchAgentOperatorRepository = registeredMetaDataStrings.CloudwatchAgentOperatorRepository metaDataStorage.CloudwatchAgentOperatorTag = registeredMetaDataStrings.CloudwatchAgentOperatorTag metaDataStorage.CloudwatchAgentOperatorRepositoryURL = registeredMetaDataStrings.CloudwatchAgentOperatorRepositoryURL + metaDataStorage.CloudwatchAgentTargetAllocatorRepository = registeredMetaDataStrings.CloudwatchAgentTargetAllocatorRepository + metaDataStorage.CloudwatchAgentTargetAllocatorTag = registeredMetaDataStrings.CloudwatchAgentTargetAllocatorTag + metaDataStorage.CloudwatchAgentTargetAllocatorRepositoryURL = registeredMetaDataStrings.CloudwatchAgentTargetAllocatorRepositoryURL metaDataStorage.AgentConfig = registeredMetaDataStrings.AgentConfig metaDataStorage.PrometheusConfig = registeredMetaDataStrings.PrometheusConfig metaDataStorage.OtelConfig = registeredMetaDataStrings.OtelConfig diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index a3477358e..b6edecca2 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -112,7 +112,7 @@ resource "null_resource" "validator" { provisioner "local-exec" { command = <<-EOT echo "Validating K8s resources and metrics" - go test -v ${var.test_dir} \ + go test -timeout 30m -v ${var.test_dir} \ -region=${var.region} \ -k8s_version=${var.k8s_version} \ -eksClusterName=${aws_eks_cluster.this.name} \ @@ -125,6 +125,9 @@ resource "null_resource" "validator" { -cloudwatch_agent_operator_repository=${var.cloudwatch_agent_operator_repository} \ -cloudwatch_agent_operator_tag=${var.cloudwatch_agent_operator_tag} \ -cloudwatch_agent_operator_repository_url=${var.cloudwatch_agent_operator_repository_url} \ + -cloudwatch_agent_target_allocator_repository=${var.cloudwatch_agent_target_allocator_repository} \ + -cloudwatch_agent_target_allocator_tag=${var.cloudwatch_agent_target_allocator_tag} \ + -cloudwatch_agent_target_allocator_repository_url=${var.cloudwatch_agent_target_allocator_repository_url} \ -agent_config="${var.test_dir}/${var.agent_config}" \ ${var.otel_config != "" ? "-otel_config=\"${var.test_dir}/${var.otel_config}\"" : ""} \ ${var.prometheus_config != "" ? "-prometheus_config=\"${var.test_dir}/${var.prometheus_config}\"" : ""} \ diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index 00c8e4166..7c39f462d 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -51,6 +51,21 @@ variable "cloudwatch_agent_operator_repository_url" { default = "public.ecr.aws/cloudwatch-agent" } +variable "cloudwatch_agent_target_allocator_repository" { + type = string + default = "cloudwatch-agent-target-allocator" +} + +variable "cloudwatch_agent_target_allocator_tag" { + type = string + default = "latest" +} + +variable "cloudwatch_agent_target_allocator_repository_url" { + type = string + default = "public.ecr.aws/cloudwatch-agent" +} + variable "test_dir" { type = string default = "" From ad76db485e75c60b008a7e88730ce41e3d4d16fa Mon Sep 17 00:00:00 2001 From: musa-asad Date: Tue, 10 Dec 2024 18:37:35 -0500 Subject: [PATCH 16/92] Update to 2h. --- terraform/eks/e2e/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index b6edecca2..062fdb1ba 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -112,7 +112,7 @@ resource "null_resource" "validator" { provisioner "local-exec" { command = <<-EOT echo "Validating K8s resources and metrics" - go test -timeout 30m -v ${var.test_dir} \ + go test -timeout 2h -v ${var.test_dir} \ -region=${var.region} \ -k8s_version=${var.k8s_version} \ -eksClusterName=${aws_eks_cluster.this.name} \ From d918ffbf86fa116d80bc74eb032254145090930a Mon Sep 17 00:00:00 2001 From: musa-asad Date: Tue, 10 Dec 2024 23:12:36 -0500 Subject: [PATCH 17/92] Remove port forwarding. --- test/e2e/jmx/jmx_test.go | 18 +----------------- .../sample_apps/tomcat-deployment.yaml | 6 ++---- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index b0e826044..390fdcab8 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -129,22 +129,6 @@ func ApplyHelm(env *environment.MetaData) error { return fmt.Errorf("failed to wait for deployment %s: %w\nOutput: %s", deploymentName, err, output) } - var ports string - if deploymentName == "tomcat-deployment" { - ports = "8080:8080" - } else if deploymentName == "kafka-deployment" { - ports = "9092:9092" - } else { - return fmt.Errorf("unknown deployment type: %s", deploymentName) - } - - portForward := exec.Command("kubectl", "port-forward", fmt.Sprintf("deployment/%s", deploymentName), ports) - portForward.Stdout = os.Stdout - portForward.Stderr = os.Stderr - if err := portForward.Run(); err != nil { - fmt.Printf("Port forwarding failed: %v\n", err) - } - return nil } @@ -249,7 +233,7 @@ func testTomcatMetrics(t *testing.T) { func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { for i := 0; i < 5; i++ { - resp, err := http.Get("http://localhost:8080/webapp/index.jsp") + resp, err := http.Get("http://tomcat-service:8080/webapp/index.jsp") if err != nil { t.Logf("Request attempt %d failed: %v", i+1, err) continue diff --git a/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml b/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml index 0a966a3a9..0a558bbc9 100644 --- a/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml +++ b/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml @@ -3,14 +3,12 @@ kind: Service metadata: name: tomcat-service spec: - type: NodePort + type: ClusterIP selector: app: tomcat ports: - - port: 80 - protocol: TCP + - port: 8080 targetPort: 8080 - nodePort: 32227 --- apiVersion: apps/v1 kind: Deployment From 987b0055bc1c4a36d1f79272f3c171b9f2e0918c Mon Sep 17 00:00:00 2001 From: musa-asad Date: Wed, 11 Dec 2024 01:13:23 -0500 Subject: [PATCH 18/92] Fix providers. --- terraform/eks/e2e/providers.tf | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/terraform/eks/e2e/providers.tf b/terraform/eks/e2e/providers.tf index 1c2b6fff6..9bd2885f5 100644 --- a/terraform/eks/e2e/providers.tf +++ b/terraform/eks/e2e/providers.tf @@ -1,15 +1,17 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -terraform { - required_providers { - docker = { - source = "kreuzwerker/docker" - version = "3.0.2" - } - } -} - provider "aws" { region = var.region +} + +provider "kubernetes" { + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] + } + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.this.token } \ No newline at end of file From 9ea9eb250bd4227ca5b3e5928a876895b05ae52f Mon Sep 17 00:00:00 2001 From: musa-asad Date: Wed, 11 Dec 2024 03:20:08 -0500 Subject: [PATCH 19/92] Use loadbalancer. --- test/e2e/jmx/jmx_test.go | 31 ++++++++++++++++--- .../sample_apps/tomcat-deployment.yaml | 4 +-- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 390fdcab8..97ba4c797 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -73,7 +73,6 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -// Make this a utility function. func ApplyHelm(env *environment.MetaData) error { updateKubeconfig := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName) output, err := updateKubeconfig.CombinedOutput() @@ -232,8 +231,19 @@ func testTomcatMetrics(t *testing.T) { func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { + cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Error getting LoadBalancer URL: %v", err) + } + + lbURL := strings.Trim(string(output), "'") + if lbURL == "" { + t.Fatal("LoadBalancer URL failed to format") + } + for i := 0; i < 5; i++ { - resp, err := http.Get("http://tomcat-service:8080/webapp/index.jsp") + resp, err := http.Get(fmt.Sprintf("http://%s/webapp/index.jsp", lbURL)) if err != nil { t.Logf("Request attempt %d failed: %v", i+1, err) continue @@ -245,7 +255,7 @@ func testTomcatSessions(t *testing.T) { } } - time.Sleep(30 * time.Second) + time.Sleep(5 * time.Minute) startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() @@ -360,8 +370,19 @@ func testContainerInsightsMetrics(t *testing.T) { func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { + cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Error getting LoadBalancer URL: %v", err) + } + + lbURL := strings.Trim(string(output), "'") + if lbURL == "" { + t.Fatal("LoadBalancer URL failed to format") + } + for i := 0; i < 5; i++ { - resp, err := http.Get("http://localhost:8080/webapp/index.jsp") + resp, err := http.Get(fmt.Sprintf("http://%s/webapp/index.jsp", lbURL)) if err != nil { t.Logf("Request attempt %d failed: %v", i+1, err) continue @@ -373,7 +394,7 @@ func testTomcatRejectedSessions(t *testing.T) { } } - time.Sleep(30 * time.Second) + time.Sleep(5 * time.Minute) startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() diff --git a/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml b/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml index 0a558bbc9..9fa072137 100644 --- a/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml +++ b/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml @@ -3,11 +3,11 @@ kind: Service metadata: name: tomcat-service spec: - type: ClusterIP + type: LoadBalancer selector: app: tomcat ports: - - port: 8080 + - port: 80 targetPort: 8080 --- apiVersion: apps/v1 From 6f8d7d1551a52c3927343375f1789a14197f280c Mon Sep 17 00:00:00 2001 From: musa-asad Date: Wed, 11 Dec 2024 19:37:19 -0500 Subject: [PATCH 20/92] Fix json. --- .../jmx/resources/cwagent_configs/kafka.json | 102 +++++++++--------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/test/e2e/jmx/resources/cwagent_configs/kafka.json b/test/e2e/jmx/resources/cwagent_configs/kafka.json index 33d50e946..1337f7d82 100644 --- a/test/e2e/jmx/resources/cwagent_configs/kafka.json +++ b/test/e2e/jmx/resources/cwagent_configs/kafka.json @@ -2,56 +2,58 @@ "metrics": { "namespace": "KAFKA_E2E", "metrics_collected": { - "kafka": { - "measurement": [ - "kafka.message.count", - "kafka.request.count", - "kafka.request.failed", - "kafka.request.time.total", - "kafka.request.time.50p", - "kafka.request.time.99p", - "kafka.request.time.avg", - "kafka.network.io", - "kafka.purgatory.size", - "kafka.partition.count", - "kafka.partition.offline", - "kafka.partition.under_replicated", - "kafka.isr.operation.count", - "kafka.max.lag", - "kafka.controller.active.count", - "kafka.leader.election.rate", - "kafka.unclean.election.rate", - "kafka.request.queue", - "kafka.logs.flush.time.count", - "kafka.logs.flush.time.median", - "kafka.logs.flush.time.99p" - ] - }, - "kafka-consumer": { - "measurement": [ - "kafka.consumer.fetch-rate", - "kafka.consumer.records-lag-max", - "kafka.consumer.total.bytes-consumed-rate", - "kafka.consumer.total.fetch-size-avg", - "kafka.consumer.total.records-consumed-rate", - "kafka.consumer.bytes-consumed-rate", - "kafka.consumer.fetch-size-avg", - "kafka.consumer.records-consumed-rate" - ] - }, - "kakfa-producer": { - "measurement": [ - "kafka.producer.io-wait-time-ns-avg", - "kafka.producer.outgoing-byte-rate", - "kafka.producer.request-latency-avg", - "kafka.producer.request-rate", - "kafka.producer.response-rate", - "kafka.producer.byte-rate", - "kafka.producer.compression-rate", - "kafka.producer.record-error-rate", - "kafka.producer.record-retry-rate", - "kafka.producer.record-send-rate" - ] + "jmx": { + "kafka": { + "measurement": [ + "kafka.message.count", + "kafka.request.count", + "kafka.request.failed", + "kafka.request.time.total", + "kafka.request.time.50p", + "kafka.request.time.99p", + "kafka.request.time.avg", + "kafka.network.io", + "kafka.purgatory.size", + "kafka.partition.count", + "kafka.partition.offline", + "kafka.partition.under_replicated", + "kafka.isr.operation.count", + "kafka.max.lag", + "kafka.controller.active.count", + "kafka.leader.election.rate", + "kafka.unclean.election.rate", + "kafka.request.queue", + "kafka.logs.flush.time.count", + "kafka.logs.flush.time.median", + "kafka.logs.flush.time.99p" + ] + }, + "kafka-consumer": { + "measurement": [ + "kafka.consumer.fetch-rate", + "kafka.consumer.records-lag-max", + "kafka.consumer.total.bytes-consumed-rate", + "kafka.consumer.total.fetch-size-avg", + "kafka.consumer.total.records-consumed-rate", + "kafka.consumer.bytes-consumed-rate", + "kafka.consumer.fetch-size-avg", + "kafka.consumer.records-consumed-rate" + ] + }, + "kakfa-producer": { + "measurement": [ + "kafka.producer.io-wait-time-ns-avg", + "kafka.producer.outgoing-byte-rate", + "kafka.producer.request-latency-avg", + "kafka.producer.request-rate", + "kafka.producer.response-rate", + "kafka.producer.byte-rate", + "kafka.producer.compression-rate", + "kafka.producer.record-error-rate", + "kafka.producer.record-retry-rate", + "kafka.producer.record-send-rate" + ] + } } } } From ba53b1494a76dc138b83b8d4af29714f2fb87ec4 Mon Sep 17 00:00:00 2001 From: Musa Date: Thu, 12 Dec 2024 11:42:13 -0500 Subject: [PATCH 21/92] Make 2 hours --- terraform/eks/e2e/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index b6edecca2..062fdb1ba 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -112,7 +112,7 @@ resource "null_resource" "validator" { provisioner "local-exec" { command = <<-EOT echo "Validating K8s resources and metrics" - go test -timeout 30m -v ${var.test_dir} \ + go test -timeout 2h -v ${var.test_dir} \ -region=${var.region} \ -k8s_version=${var.k8s_version} \ -eksClusterName=${aws_eks_cluster.this.name} \ From f5d2733a133da25f407383a0dc14756f159ddef9 Mon Sep 17 00:00:00 2001 From: Musa Date: Thu, 12 Dec 2024 11:43:40 -0500 Subject: [PATCH 22/92] Fix providers.tf --- terraform/eks/e2e/providers.tf | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/terraform/eks/e2e/providers.tf b/terraform/eks/e2e/providers.tf index 1c2b6fff6..9bd2885f5 100644 --- a/terraform/eks/e2e/providers.tf +++ b/terraform/eks/e2e/providers.tf @@ -1,15 +1,17 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -terraform { - required_providers { - docker = { - source = "kreuzwerker/docker" - version = "3.0.2" - } - } -} - provider "aws" { region = var.region +} + +provider "kubernetes" { + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] + } + host = aws_eks_cluster.this.endpoint + cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) + token = data.aws_eks_cluster_auth.this.token } \ No newline at end of file From b6416ae1c379ef6ea41eaf61fac853c89c3a9afc Mon Sep 17 00:00:00 2001 From: musa-asad Date: Thu, 12 Dec 2024 21:50:40 -0500 Subject: [PATCH 23/92] Add auth for IAM role. --- terraform/eks/e2e/main.tf | 29 +++++++++++++++++++++++++++++ terraform/eks/e2e/variables.tf | 5 +++++ 2 files changed, 34 insertions(+) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 062fdb1ba..9b81942f7 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -27,6 +27,35 @@ resource "aws_eks_cluster" "this" { } } +resource "kubernetes_config_map" "aws_auth" { + metadata { + name = "aws-auth" + namespace = "kube-system" + } + + data = { + mapRoles = yamlencode([ + { + rolearn = aws_iam_role.node_role.arn + username = "system:node:{{EC2PrivateDNSName}}" + groups = [ + "system:bootstrappers", + "system:nodes" + ] + }, + { + rolearn = var.terraform_assume_role + username = "github-actions" + groups = [ + "system:masters" + ] + } + ]) + } + + depends_on = [aws_eks_cluster.this] +} + resource "aws_eks_node_group" "this" { cluster_name = aws_eks_cluster.this.name node_group_name = "${local.cluster_name}-node" diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index 7c39f462d..357713e58 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -16,6 +16,11 @@ variable "cluster_name" { default = "cwagent-monitoring-config-e2e-eks" } +variable "terraform_assume_role" { + type = string + default = "" +} + variable "helm_charts_branch" { type = string default = "main" From 5714a7e0e2017838fa6ec2de6e6a42badd83585d Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 00:54:52 -0500 Subject: [PATCH 24/92] Verify cluster access. --- terraform/eks/e2e/main.tf | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 9b81942f7..6d07f3d8a 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -11,6 +11,7 @@ module "basic_components" { locals { cluster_name = var.cluster_name != "" ? var.cluster_name : "cwagent-monitoring-config-e2e-eks" + aws_eks = "aws eks" } data "aws_eks_cluster_auth" "this" { @@ -53,6 +54,8 @@ resource "kubernetes_config_map" "aws_auth" { ]) } + force = true + depends_on = [aws_eks_cluster.this] } @@ -120,6 +123,20 @@ resource "aws_iam_role_policy_attachment" "node_CloudWatchAgentServerPolicy" { role = aws_iam_role.node_role.name } +resource "null_resource" "kubectl" { + depends_on = [ + aws_eks_cluster.this, + aws_eks_node_group.this, + ] + provisioner "local-exec" { + command = <<-EOT + ${local.aws_eks} update-kubeconfig --name ${aws_eks_cluster.this.name} + ${local.aws_eks} list-clusters --output text + ${local.aws_eks} describe-cluster --name ${aws_eks_cluster.this.name} --output text + EOT + } +} + resource "null_resource" "helm_charts" { provisioner "local-exec" { command = <<-EOT From 9a4bb6edbdcfeb22138d0b663534474653931757 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 00:57:25 -0500 Subject: [PATCH 25/92] Remove force. --- terraform/eks/e2e/main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 6d07f3d8a..3bdd7d34f 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -54,8 +54,6 @@ resource "kubernetes_config_map" "aws_auth" { ]) } - force = true - depends_on = [aws_eks_cluster.this] } From 8628c7dded0e01ef2cbf8c37cbb2a5b0c0e9007e Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 02:24:43 -0500 Subject: [PATCH 26/92] Add assume role. --- terraform/eks/e2e/providers.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/eks/e2e/providers.tf b/terraform/eks/e2e/providers.tf index 9bd2885f5..dd85d2ac1 100644 --- a/terraform/eks/e2e/providers.tf +++ b/terraform/eks/e2e/providers.tf @@ -3,6 +3,9 @@ provider "aws" { region = var.region + assume_role { + role_arn = var.terraform_assume_role + } } provider "kubernetes" { From 20fa9beb2b30a494fad418b4b56f795388de171d Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 02:58:05 -0500 Subject: [PATCH 27/92] Update JMX tests. --- test/e2e/jmx/jmx_test.go | 8 +-- .../cwagent_configs/containerinsights.json | 1 + .../jmx/resources/cwagent_configs/kafka.json | 48 +--------------- .../sample_apps/kafka-deployment.yaml | 57 ++++++++++++++++++- 4 files changed, 60 insertions(+), 54 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 97ba4c797..739f91ac2 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -255,7 +255,7 @@ func testTomcatSessions(t *testing.T) { } } - time.Sleep(5 * time.Minute) + time.Sleep(3 * time.Minute) startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() @@ -353,8 +353,6 @@ func testContainerInsightsMetrics(t *testing.T) { {"jvm_memory_pool_bytes_used", "ContainerInsights/Prometheus"}, {"catalina_manager_activesessions", "ContainerInsights/Prometheus"}, {"catalina_manager_rejectedsessions", "ContainerInsights/Prometheus"}, - {"catalina_globalrequestprocessor_bytesreceived", "ContainerInsights/Prometheus"}, - {"catalina_globalrequestprocessor_bytessent", "ContainerInsights/Prometheus"}, {"catalina_globalrequestprocessor_requestcount", "ContainerInsights/Prometheus"}, {"catalina_globalrequestprocessor_errorcount", "ContainerInsights/Prometheus"}, {"catalina_globalrequestprocessor_processingtime", "ContainerInsights/Prometheus"}, @@ -369,7 +367,7 @@ func testContainerInsightsMetrics(t *testing.T) { } func testTomcatRejectedSessions(t *testing.T) { - t.Run("verify_tomcat_sessions", func(t *testing.T) { + t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") output, err := cmd.CombinedOutput() if err != nil { @@ -394,7 +392,7 @@ func testTomcatRejectedSessions(t *testing.T) { } } - time.Sleep(5 * time.Minute) + time.Sleep(3 * time.Minute) startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() diff --git a/test/e2e/jmx/resources/cwagent_configs/containerinsights.json b/test/e2e/jmx/resources/cwagent_configs/containerinsights.json index 6eaa8f2b6..9ee4ef445 100644 --- a/test/e2e/jmx/resources/cwagent_configs/containerinsights.json +++ b/test/e2e/jmx/resources/cwagent_configs/containerinsights.json @@ -2,6 +2,7 @@ "logs": { "metrics_collected": { "kubernetes": { + "cluster_name": "TestCluster", "jmx_container_insights": true } } diff --git a/test/e2e/jmx/resources/cwagent_configs/kafka.json b/test/e2e/jmx/resources/cwagent_configs/kafka.json index 1337f7d82..15dd1634c 100644 --- a/test/e2e/jmx/resources/cwagent_configs/kafka.json +++ b/test/e2e/jmx/resources/cwagent_configs/kafka.json @@ -3,55 +3,11 @@ "namespace": "KAFKA_E2E", "metrics_collected": { "jmx": { - "kafka": { - "measurement": [ - "kafka.message.count", - "kafka.request.count", - "kafka.request.failed", - "kafka.request.time.total", - "kafka.request.time.50p", - "kafka.request.time.99p", - "kafka.request.time.avg", - "kafka.network.io", - "kafka.purgatory.size", - "kafka.partition.count", - "kafka.partition.offline", - "kafka.partition.under_replicated", - "kafka.isr.operation.count", - "kafka.max.lag", - "kafka.controller.active.count", - "kafka.leader.election.rate", - "kafka.unclean.election.rate", - "kafka.request.queue", - "kafka.logs.flush.time.count", - "kafka.logs.flush.time.median", - "kafka.logs.flush.time.99p" - ] - }, "kafka-consumer": { "measurement": [ - "kafka.consumer.fetch-rate", - "kafka.consumer.records-lag-max", - "kafka.consumer.total.bytes-consumed-rate", - "kafka.consumer.total.fetch-size-avg", "kafka.consumer.total.records-consumed-rate", - "kafka.consumer.bytes-consumed-rate", - "kafka.consumer.fetch-size-avg", - "kafka.consumer.records-consumed-rate" - ] - }, - "kakfa-producer": { - "measurement": [ - "kafka.producer.io-wait-time-ns-avg", - "kafka.producer.outgoing-byte-rate", - "kafka.producer.request-latency-avg", - "kafka.producer.request-rate", - "kafka.producer.response-rate", - "kafka.producer.byte-rate", - "kafka.producer.compression-rate", - "kafka.producer.record-error-rate", - "kafka.producer.record-retry-rate", - "kafka.producer.record-send-rate" + "kafka.consumer.total.bytes-consumed-rate", + "kafka.consumer.fetch-rate" ] } } diff --git a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml index 2d7cf5ffd..7e27fd9bb 100644 --- a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml +++ b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: zookeeper - image: bitnami/zookeeper:latest + image: wurstmeister/zookeeper:latest ports: - containerPort: 2181 env: @@ -59,13 +59,64 @@ spec: spec: containers: - name: kafka - image: bitnami/kafka:latest + image: wurstmeister/kafka:latest ports: - containerPort: 9092 env: - name: KAFKA_ZOOKEEPER_CONNECT value: "zookeeper-service:2181" + - name: KAFKA_LISTENERS + value: "PLAINTEXT://:9092" - name: KAFKA_ADVERTISED_LISTENERS value: "PLAINTEXT://:9092" - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR - value: "1" \ No newline at end of file + value: "1" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka-producer +spec: + replicas: 1 + selector: + matchLabels: + app: kafka-producer + template: + metadata: + labels: + app: kafka-producer + spec: + containers: + - name: kafka-producer + image: wurstmeister/kafka:latest + command: ["/bin/sh"] + args: ["-c", "while true; do echo 'Producing message'; kafka-console-producer.sh --broker-list kafka-service:9092 --topic test-topic | 'Test message'; sleep 5; done"] + env: + - name: KAFKA_BROKER_LIST + value: "kafka-service:9092" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka-consumer +spec: + replicas: 1 + selector: + matchLabels: + app: kafka-consumer + template: + metadata: + labels: + app: kafka-consumer + annotations: + instrumentation.opentelemetry.io/inject-java: "true" + cloudwatch.aws.amazon.com/inject-jmx-kafka-consumer: "true" + spec: + containers: + - name: kafka-consumer + image: wurstmeister/kafka:latest + command: ["/bin/sh"] + args: ["-c", "kafka-console-consumer.sh --bootstrap-server kafka-service:9092 --topic test-topic --from-beginning"] + env: + - name: KAFKA_BROKER_LIST + value: "kafka-service:9092" \ No newline at end of file From cfc6fe165493de2393ee6cb9df0140acbc4158e8 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 03:03:10 -0500 Subject: [PATCH 28/92] Consumer only metrics. --- test/e2e/jmx/jmx_test.go | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 739f91ac2..db07645df 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -283,45 +283,9 @@ func testKafkaMetrics(t *testing.T) { name string namespace string }{ - {"kafka.message.count", "KAFKA_E2E"}, - {"kafka.request.count", "KAFKA_E2E"}, - {"kafka.request.failed", "KAFKA_E2E"}, - {"kafka.request.time.total", "KAFKA_E2E"}, - {"kafka.request.time.50p", "KAFKA_E2E"}, - {"kafka.request.time.99p", "KAFKA_E2E"}, - {"kafka.request.time.avg", "KAFKA_E2E"}, - {"kafka.network.io", "KAFKA_E2E"}, - {"kafka.purgatory.size", "KAFKA_E2E"}, - {"kafka.partition.count", "KAFKA_E2E"}, - {"kafka.partition.offline", "KAFKA_E2E"}, - {"kafka.partition.under_replicated", "KAFKA_E2E"}, - {"kafka.isr.operation.count", "KAFKA_E2E"}, - {"kafka.max.lag", "KAFKA_E2E"}, - {"kafka.controller.active.count", "KAFKA_E2E"}, - {"kafka.leader.election.rate", "KAFKA_E2E"}, - {"kafka.unclean.election.rate", "KAFKA_E2E"}, - {"kafka.request.queue", "KAFKA_E2E"}, - {"kafka.logs.flush.time.count", "KAFKA_E2E"}, - {"kafka.logs.flush.time.median", "KAFKA_E2E"}, - {"kafka.logs.flush.time.99p", "KAFKA_E2E"}, {"kafka.consumer.fetch-rate", "KAFKA_E2E"}, - {"kafka.consumer.records-lag-max", "KAFKA_E2E"}, {"kafka.consumer.total.bytes-consumed-rate", "KAFKA_E2E"}, - {"kafka.consumer.total.fetch-size-avg", "KAFKA_E2E"}, {"kafka.consumer.total.records-consumed-rate", "KAFKA_E2E"}, - {"kafka.consumer.bytes-consumed-rate", "KAFKA_E2E"}, - {"kafka.consumer.fetch-size-avg", "KAFKA_E2E"}, - {"kafka.consumer.records-consumed-rate", "KAFKA_E2E"}, - {"kafka.producer.io-wait-time-ns-avg", "KAFKA_E2E"}, - {"kafka.producer.outgoing-byte-rate", "KAFKA_E2E"}, - {"kafka.producer.request-latency-avg", "KAFKA_E2E"}, - {"kafka.producer.request-rate", "KAFKA_E2E"}, - {"kafka.producer.response-rate", "KAFKA_E2E"}, - {"kafka.producer.byte-rate", "KAFKA_E2E"}, - {"kafka.producer.compression-rate", "KAFKA_E2E"}, - {"kafka.producer.record-error-rate", "KAFKA_E2E"}, - {"kafka.producer.record-retry-rate", "KAFKA_E2E"}, - {"kafka.producer.record-send-rate", "KAFKA_E2E"}, } for _, metric := range metricsToCheck { From ff5cb0acd18dfd726dfabab5d2b2045238b33655 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 11:01:49 -0500 Subject: [PATCH 29/92] Fix formatting. --- environment/metadata.go | 138 ++++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/environment/metadata.go b/environment/metadata.go index 07c8055e6..2cf99ad48 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -23,84 +23,84 @@ var metaDataStorage *MetaData = nil var registeredMetaDataStrings = &(MetaDataStrings{}) type MetaData struct { - ComputeType computetype.ComputeType - EcsLaunchType ecslaunchtype.ECSLaunchType - EcsDeploymentStrategy ecsdeploymenttype.ECSDeploymentType - EksDeploymentStrategy eksdeploymenttype.EKSDeploymentType - EcsClusterArn string - EcsClusterName string - CwagentConfigSsmParamName string - EcsServiceName string - EC2PluginTests map[string]struct{} // set of EC2 plugin names - ExcludedTests map[string]struct{} // set of excluded names - Bucket string - S3Key string - CwaCommitSha string - CaCertPath string - EKSClusterName string - ProxyUrl string - AssumeRoleArn string - InstanceId string - InstancePlatform string - AgentStartCommand string - EksGpuType string - AmpWorkspaceId string - Region string - K8sVersion string - HelmChartsBranch string - CloudwatchAgentRepository string - CloudwatchAgentTag string - CloudwatchAgentRepositoryURL string - CloudwatchAgentOperatorRepository string - CloudwatchAgentOperatorTag string - CloudwatchAgentOperatorRepositoryURL string + ComputeType computetype.ComputeType + EcsLaunchType ecslaunchtype.ECSLaunchType + EcsDeploymentStrategy ecsdeploymenttype.ECSDeploymentType + EksDeploymentStrategy eksdeploymenttype.EKSDeploymentType + EcsClusterArn string + EcsClusterName string + CwagentConfigSsmParamName string + EcsServiceName string + EC2PluginTests map[string]struct{} // set of EC2 plugin names + ExcludedTests map[string]struct{} // set of excluded names + Bucket string + S3Key string + CwaCommitSha string + CaCertPath string + EKSClusterName string + ProxyUrl string + AssumeRoleArn string + InstanceId string + InstancePlatform string + AgentStartCommand string + EksGpuType string + AmpWorkspaceId string + Region string + K8sVersion string + HelmChartsBranch string + CloudwatchAgentRepository string + CloudwatchAgentTag string + CloudwatchAgentRepositoryURL string + CloudwatchAgentOperatorRepository string + CloudwatchAgentOperatorTag string + CloudwatchAgentOperatorRepositoryURL string CloudwatchAgentTargetAllocatorRepository string CloudwatchAgentTargetAllocatorTag string CloudwatchAgentTargetAllocatorRepositoryURL string - AgentConfig string - PrometheusConfig string - OtelConfig string - SampleApp string + AgentConfig string + PrometheusConfig string + OtelConfig string + SampleApp string } type MetaDataStrings struct { - ComputeType string - EcsLaunchType string - EcsDeploymentStrategy string - EksDeploymentStrategy string - EcsClusterArn string - CwagentConfigSsmParamName string - EcsServiceName string - EC2PluginTests string // input comma delimited list of plugin names - ExcludedTests string // Exclude specific tests from OS - Bucket string - S3Key string - CwaCommitSha string - CaCertPath string - EKSClusterName string - ProxyUrl string - AssumeRoleArn string - InstanceId string - InstancePlatform string - AgentStartCommand string - EksGpuType string - AmpWorkspaceId string - Region string - K8sVersion string - HelmChartsBranch string - CloudwatchAgentRepository string - CloudwatchAgentTag string - CloudwatchAgentRepositoryURL string - CloudwatchAgentOperatorRepository string - CloudwatchAgentOperatorTag string - CloudwatchAgentOperatorRepositoryURL string + ComputeType string + EcsLaunchType string + EcsDeploymentStrategy string + EksDeploymentStrategy string + EcsClusterArn string + CwagentConfigSsmParamName string + EcsServiceName string + EC2PluginTests string // input comma delimited list of plugin names + ExcludedTests string // Exclude specific tests from OS + Bucket string + S3Key string + CwaCommitSha string + CaCertPath string + EKSClusterName string + ProxyUrl string + AssumeRoleArn string + InstanceId string + InstancePlatform string + AgentStartCommand string + EksGpuType string + AmpWorkspaceId string + Region string + K8sVersion string + HelmChartsBranch string + CloudwatchAgentRepository string + CloudwatchAgentTag string + CloudwatchAgentRepositoryURL string + CloudwatchAgentOperatorRepository string + CloudwatchAgentOperatorTag string + CloudwatchAgentOperatorRepositoryURL string CloudwatchAgentTargetAllocatorRepository string CloudwatchAgentTargetAllocatorTag string CloudwatchAgentTargetAllocatorRepositoryURL string - AgentConfig string - PrometheusConfig string - OtelConfig string - SampleApp string + AgentConfig string + PrometheusConfig string + OtelConfig string + SampleApp string } func registerComputeType(dataString *MetaDataStrings) { From 82f9d34d2c4b7aed0b1b5f2a950ae880356548bb Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 12:13:47 -0500 Subject: [PATCH 30/92] Remove unnecessary resources. --- terraform/eks/e2e/main.tf | 44 ---------------------------------- terraform/eks/e2e/providers.tf | 14 ----------- terraform/eks/e2e/variables.tf | 5 ---- 3 files changed, 63 deletions(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 3bdd7d34f..062fdb1ba 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -11,7 +11,6 @@ module "basic_components" { locals { cluster_name = var.cluster_name != "" ? var.cluster_name : "cwagent-monitoring-config-e2e-eks" - aws_eks = "aws eks" } data "aws_eks_cluster_auth" "this" { @@ -28,35 +27,6 @@ resource "aws_eks_cluster" "this" { } } -resource "kubernetes_config_map" "aws_auth" { - metadata { - name = "aws-auth" - namespace = "kube-system" - } - - data = { - mapRoles = yamlencode([ - { - rolearn = aws_iam_role.node_role.arn - username = "system:node:{{EC2PrivateDNSName}}" - groups = [ - "system:bootstrappers", - "system:nodes" - ] - }, - { - rolearn = var.terraform_assume_role - username = "github-actions" - groups = [ - "system:masters" - ] - } - ]) - } - - depends_on = [aws_eks_cluster.this] -} - resource "aws_eks_node_group" "this" { cluster_name = aws_eks_cluster.this.name node_group_name = "${local.cluster_name}-node" @@ -121,20 +91,6 @@ resource "aws_iam_role_policy_attachment" "node_CloudWatchAgentServerPolicy" { role = aws_iam_role.node_role.name } -resource "null_resource" "kubectl" { - depends_on = [ - aws_eks_cluster.this, - aws_eks_node_group.this, - ] - provisioner "local-exec" { - command = <<-EOT - ${local.aws_eks} update-kubeconfig --name ${aws_eks_cluster.this.name} - ${local.aws_eks} list-clusters --output text - ${local.aws_eks} describe-cluster --name ${aws_eks_cluster.this.name} --output text - EOT - } -} - resource "null_resource" "helm_charts" { provisioner "local-exec" { command = <<-EOT diff --git a/terraform/eks/e2e/providers.tf b/terraform/eks/e2e/providers.tf index dd85d2ac1..d8a1f722b 100644 --- a/terraform/eks/e2e/providers.tf +++ b/terraform/eks/e2e/providers.tf @@ -3,18 +3,4 @@ provider "aws" { region = var.region - assume_role { - role_arn = var.terraform_assume_role - } -} - -provider "kubernetes" { - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] - } - host = aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) - token = data.aws_eks_cluster_auth.this.token } \ No newline at end of file diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index 357713e58..7c39f462d 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -16,11 +16,6 @@ variable "cluster_name" { default = "cwagent-monitoring-config-e2e-eks" } -variable "terraform_assume_role" { - type = string - default = "" -} - variable "helm_charts_branch" { type = string default = "main" From e6b5faaa10ca84b70acf294ee395318d92fe6953 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 11:01:49 -0500 Subject: [PATCH 31/92] Fix formatting. --- environment/metadata.go | 138 ++++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/environment/metadata.go b/environment/metadata.go index 07c8055e6..2cf99ad48 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -23,84 +23,84 @@ var metaDataStorage *MetaData = nil var registeredMetaDataStrings = &(MetaDataStrings{}) type MetaData struct { - ComputeType computetype.ComputeType - EcsLaunchType ecslaunchtype.ECSLaunchType - EcsDeploymentStrategy ecsdeploymenttype.ECSDeploymentType - EksDeploymentStrategy eksdeploymenttype.EKSDeploymentType - EcsClusterArn string - EcsClusterName string - CwagentConfigSsmParamName string - EcsServiceName string - EC2PluginTests map[string]struct{} // set of EC2 plugin names - ExcludedTests map[string]struct{} // set of excluded names - Bucket string - S3Key string - CwaCommitSha string - CaCertPath string - EKSClusterName string - ProxyUrl string - AssumeRoleArn string - InstanceId string - InstancePlatform string - AgentStartCommand string - EksGpuType string - AmpWorkspaceId string - Region string - K8sVersion string - HelmChartsBranch string - CloudwatchAgentRepository string - CloudwatchAgentTag string - CloudwatchAgentRepositoryURL string - CloudwatchAgentOperatorRepository string - CloudwatchAgentOperatorTag string - CloudwatchAgentOperatorRepositoryURL string + ComputeType computetype.ComputeType + EcsLaunchType ecslaunchtype.ECSLaunchType + EcsDeploymentStrategy ecsdeploymenttype.ECSDeploymentType + EksDeploymentStrategy eksdeploymenttype.EKSDeploymentType + EcsClusterArn string + EcsClusterName string + CwagentConfigSsmParamName string + EcsServiceName string + EC2PluginTests map[string]struct{} // set of EC2 plugin names + ExcludedTests map[string]struct{} // set of excluded names + Bucket string + S3Key string + CwaCommitSha string + CaCertPath string + EKSClusterName string + ProxyUrl string + AssumeRoleArn string + InstanceId string + InstancePlatform string + AgentStartCommand string + EksGpuType string + AmpWorkspaceId string + Region string + K8sVersion string + HelmChartsBranch string + CloudwatchAgentRepository string + CloudwatchAgentTag string + CloudwatchAgentRepositoryURL string + CloudwatchAgentOperatorRepository string + CloudwatchAgentOperatorTag string + CloudwatchAgentOperatorRepositoryURL string CloudwatchAgentTargetAllocatorRepository string CloudwatchAgentTargetAllocatorTag string CloudwatchAgentTargetAllocatorRepositoryURL string - AgentConfig string - PrometheusConfig string - OtelConfig string - SampleApp string + AgentConfig string + PrometheusConfig string + OtelConfig string + SampleApp string } type MetaDataStrings struct { - ComputeType string - EcsLaunchType string - EcsDeploymentStrategy string - EksDeploymentStrategy string - EcsClusterArn string - CwagentConfigSsmParamName string - EcsServiceName string - EC2PluginTests string // input comma delimited list of plugin names - ExcludedTests string // Exclude specific tests from OS - Bucket string - S3Key string - CwaCommitSha string - CaCertPath string - EKSClusterName string - ProxyUrl string - AssumeRoleArn string - InstanceId string - InstancePlatform string - AgentStartCommand string - EksGpuType string - AmpWorkspaceId string - Region string - K8sVersion string - HelmChartsBranch string - CloudwatchAgentRepository string - CloudwatchAgentTag string - CloudwatchAgentRepositoryURL string - CloudwatchAgentOperatorRepository string - CloudwatchAgentOperatorTag string - CloudwatchAgentOperatorRepositoryURL string + ComputeType string + EcsLaunchType string + EcsDeploymentStrategy string + EksDeploymentStrategy string + EcsClusterArn string + CwagentConfigSsmParamName string + EcsServiceName string + EC2PluginTests string // input comma delimited list of plugin names + ExcludedTests string // Exclude specific tests from OS + Bucket string + S3Key string + CwaCommitSha string + CaCertPath string + EKSClusterName string + ProxyUrl string + AssumeRoleArn string + InstanceId string + InstancePlatform string + AgentStartCommand string + EksGpuType string + AmpWorkspaceId string + Region string + K8sVersion string + HelmChartsBranch string + CloudwatchAgentRepository string + CloudwatchAgentTag string + CloudwatchAgentRepositoryURL string + CloudwatchAgentOperatorRepository string + CloudwatchAgentOperatorTag string + CloudwatchAgentOperatorRepositoryURL string CloudwatchAgentTargetAllocatorRepository string CloudwatchAgentTargetAllocatorTag string CloudwatchAgentTargetAllocatorRepositoryURL string - AgentConfig string - PrometheusConfig string - OtelConfig string - SampleApp string + AgentConfig string + PrometheusConfig string + OtelConfig string + SampleApp string } func registerComputeType(dataString *MetaDataStrings) { From 17657f0afbe5fb8d3056514e90880d2e77f472fb Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 12:13:47 -0500 Subject: [PATCH 32/92] Remove unnecessary resources. --- terraform/eks/e2e/providers.tf | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/terraform/eks/e2e/providers.tf b/terraform/eks/e2e/providers.tf index 9bd2885f5..d8a1f722b 100644 --- a/terraform/eks/e2e/providers.tf +++ b/terraform/eks/e2e/providers.tf @@ -3,15 +3,4 @@ provider "aws" { region = var.region -} - -provider "kubernetes" { - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - args = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name] - } - host = aws_eks_cluster.this.endpoint - cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data) - token = data.aws_eks_cluster_auth.this.token } \ No newline at end of file From 1969fe04d0a7faecafec66ad0c13e8da89878362 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 14:55:48 -0500 Subject: [PATCH 33/92] 5 minutes. --- test/e2e/jmx/jmx_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index db07645df..d8fcc60e7 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -255,7 +255,7 @@ func testTomcatSessions(t *testing.T) { } } - time.Sleep(3 * time.Minute) + time.Sleep(5 * time.Minute) startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() @@ -356,7 +356,7 @@ func testTomcatRejectedSessions(t *testing.T) { } } - time.Sleep(3 * time.Minute) + time.Sleep(5 * time.Minute) startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() From e0fbd7eafde621ff4b595b01c3a509d3d39c32ba Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 16:59:39 -0500 Subject: [PATCH 34/92] Remove redundant logic. --- terraform/eks/e2e/helm-charts | 1 + 1 file changed, 1 insertion(+) create mode 160000 terraform/eks/e2e/helm-charts diff --git a/terraform/eks/e2e/helm-charts b/terraform/eks/e2e/helm-charts new file mode 160000 index 000000000..ef153a962 --- /dev/null +++ b/terraform/eks/e2e/helm-charts @@ -0,0 +1 @@ +Subproject commit ef153a962c1603b96a3e1d85c3bf3bf7c779b480 From 537cb6f459509ff26770c33ae14a8300b2bf7dc7 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 17:23:56 -0500 Subject: [PATCH 35/92] Revert "Remove redundant logic." This reverts commit e0fbd7eafde621ff4b595b01c3a509d3d39c32ba. --- terraform/eks/e2e/helm-charts | 1 - 1 file changed, 1 deletion(-) delete mode 160000 terraform/eks/e2e/helm-charts diff --git a/terraform/eks/e2e/helm-charts b/terraform/eks/e2e/helm-charts deleted file mode 160000 index ef153a962..000000000 --- a/terraform/eks/e2e/helm-charts +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ef153a962c1603b96a3e1d85c3bf3bf7c779b480 From 43b7dae699e5ff5db856d1fd56758155c4391e5f Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 17:26:55 -0500 Subject: [PATCH 36/92] Remove redundant logic. --- util/awsservice/constant.go | 39 +++++++++++++------------------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/util/awsservice/constant.go b/util/awsservice/constant.go index e9cd80edb..f9a3fd015 100644 --- a/util/awsservice/constant.go +++ b/util/awsservice/constant.go @@ -58,12 +58,23 @@ var ( func init() { ctx = context.Background() - var err error - awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion("us-west-2")) + + err := ConfigureAWSClients("us-west-2") + if err != nil { + fmt.Println("There was an error trying to configure the AWS clients: ", err) + } +} + +// ConfigureAWSClients configures the AWS clients using a set region. +func ConfigureAWSClients(region string) error { + mu.Lock() + defer mu.Unlock() + + awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) if err != nil { // handle error fmt.Println("There was an error trying to load default config: ", err) - return + return err } fmt.Println("This is the aws region: ", awsCfg.Region) @@ -78,28 +89,6 @@ func init() { S3Client = s3.NewFromConfig(awsCfg) CloudformationClient = cloudformation.NewFromConfig(awsCfg) XrayClient = xray.NewFromConfig(awsCfg) -} - -// ReconfigureAWSClients reconfigures the AWS clients using a new region. -func ReconfigureAWSClients(region string) error { - mu.Lock() - defer mu.Unlock() - - newCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) - if err != nil { - return err - } - - Ec2Client = ec2.NewFromConfig(newCfg) - EcsClient = ecs.NewFromConfig(newCfg) - SsmClient = ssm.NewFromConfig(newCfg) - ImdsClient = imds.NewFromConfig(newCfg) - CwmClient = cloudwatch.NewFromConfig(newCfg) - CwlClient = cloudwatchlogs.NewFromConfig(newCfg) - DynamodbClient = dynamodb.NewFromConfig(newCfg) - S3Client = s3.NewFromConfig(newCfg) - CloudformationClient = cloudformation.NewFromConfig(newCfg) - XrayClient = xray.NewFromConfig(newCfg) return nil } From d2b7860438cca3ab545bf261e05b30107821671f Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 13 Dec 2024 17:32:05 -0500 Subject: [PATCH 37/92] Remove redundant logic. --- test/e2e/jmx/jmx_test.go | 2 +- util/awsservice/constant.go | 39 +++++++++++++------------------------ 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index d8fcc60e7..9e1493371 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -52,7 +52,7 @@ func TestMain(m *testing.M) { env := environment.GetEnvironmentMetaData() if env.Region != "us-west-2" { - if err := awsservice.ReconfigureAWSClients(env.Region); err != nil { + if err := awsservice.ConfigureAWSClients(env.Region); err != nil { fmt.Printf("Failed to reconfigure AWS clients: %v\n", err) os.Exit(1) } diff --git a/util/awsservice/constant.go b/util/awsservice/constant.go index e9cd80edb..f9a3fd015 100644 --- a/util/awsservice/constant.go +++ b/util/awsservice/constant.go @@ -58,12 +58,23 @@ var ( func init() { ctx = context.Background() - var err error - awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion("us-west-2")) + + err := ConfigureAWSClients("us-west-2") + if err != nil { + fmt.Println("There was an error trying to configure the AWS clients: ", err) + } +} + +// ConfigureAWSClients configures the AWS clients using a set region. +func ConfigureAWSClients(region string) error { + mu.Lock() + defer mu.Unlock() + + awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) if err != nil { // handle error fmt.Println("There was an error trying to load default config: ", err) - return + return err } fmt.Println("This is the aws region: ", awsCfg.Region) @@ -78,28 +89,6 @@ func init() { S3Client = s3.NewFromConfig(awsCfg) CloudformationClient = cloudformation.NewFromConfig(awsCfg) XrayClient = xray.NewFromConfig(awsCfg) -} - -// ReconfigureAWSClients reconfigures the AWS clients using a new region. -func ReconfigureAWSClients(region string) error { - mu.Lock() - defer mu.Unlock() - - newCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) - if err != nil { - return err - } - - Ec2Client = ec2.NewFromConfig(newCfg) - EcsClient = ecs.NewFromConfig(newCfg) - SsmClient = ssm.NewFromConfig(newCfg) - ImdsClient = imds.NewFromConfig(newCfg) - CwmClient = cloudwatch.NewFromConfig(newCfg) - CwlClient = cloudwatchlogs.NewFromConfig(newCfg) - DynamodbClient = dynamodb.NewFromConfig(newCfg) - S3Client = s3.NewFromConfig(newCfg) - CloudformationClient = cloudformation.NewFromConfig(newCfg) - XrayClient = xray.NewFromConfig(newCfg) return nil } From 18b1257f174170fe275405b15f3af023e329e673 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 15 Dec 2024 18:10:29 -0500 Subject: [PATCH 38/92] Use older version of Kafka and update annotations/config to include Kafka and Kafka Producer metrics. --- .../jmx/resources/cwagent_configs/kafka.json | 34 +++++++++++++++++-- .../sample_apps/kafka-deployment.yaml | 12 +++++-- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/test/e2e/jmx/resources/cwagent_configs/kafka.json b/test/e2e/jmx/resources/cwagent_configs/kafka.json index 15dd1634c..dce338b31 100644 --- a/test/e2e/jmx/resources/cwagent_configs/kafka.json +++ b/test/e2e/jmx/resources/cwagent_configs/kafka.json @@ -7,10 +7,40 @@ "measurement": [ "kafka.consumer.total.records-consumed-rate", "kafka.consumer.total.bytes-consumed-rate", - "kafka.consumer.fetch-rate" + "kafka.consumer.fetch-rate", + "kafka.consumer.records-lag-max", + "kafka.consumer.total.fetch-size-avg", + "kafka.consumer.bytes-consumed-rate", + "kafka.consumer.fetch-size-avg", + "kafka.consumer.records-consumed-rate" + ] + }, + "kafka-producer": { + "measurement": [ + "kafka.producer.io-wait-time-ns-avg", + "kafka.producer.outgoing-byte-rate", + "kafka.producer.request-latency-avg", + "kafka.producer.request-rate", + "kafka.producer.response-rate", + "kafka.producer.byte-rate", + "kafka.producer.compression-rate", + "kafka.producer.record-error-rate", + "kafka.producer.record-retry-rate", + "kafka.producer.record-send-rate" + ] + }, + "kafka": { + "measurement": [ + "kafka.message.count", + "kafka.request.count", + "kafka.request.failed", + "kafka.request.time.total", + "kafka.request.time.50p", + "kafka.request.time.99p", + "kafka.request.time.avg" ] } } } } -} \ No newline at end of file +} diff --git a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml index 7e27fd9bb..e239b969c 100644 --- a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml +++ b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: zookeeper - image: wurstmeister/zookeeper:latest + image: wurstmeister/kafka:2.13-2.8.1 ports: - containerPort: 2181 env: @@ -56,10 +56,13 @@ spec: metadata: labels: app: kafka + annotations: + instrumentation.opentelemetry.io/inject-java: "true" + cloudwatch.aws.amazon.com/inject-jmx-kafka: "true" spec: containers: - name: kafka - image: wurstmeister/kafka:latest + image: wurstmeister/kafka:2.13-2.8.1 ports: - containerPort: 9092 env: @@ -85,6 +88,9 @@ spec: metadata: labels: app: kafka-producer + annotations: + instrumentation.opentelemetry.io/inject-java: "true" + cloudwatch.aws.amazon.com/inject-jmx-kafka-producer: "true" spec: containers: - name: kafka-producer @@ -114,7 +120,7 @@ spec: spec: containers: - name: kafka-consumer - image: wurstmeister/kafka:latest + image: wurstmeister/kafka:2.13-2.8.1 command: ["/bin/sh"] args: ["-c", "kafka-console-consumer.sh --bootstrap-server kafka-service:9092 --topic test-topic --from-beginning"] env: From 67f3247c0652ded1e2425d6ac64e435d7584b1a5 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 15 Dec 2024 18:14:30 -0500 Subject: [PATCH 39/92] Missing. --- test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml index e239b969c..e0ed0a5fc 100644 --- a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml +++ b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml @@ -94,7 +94,7 @@ spec: spec: containers: - name: kafka-producer - image: wurstmeister/kafka:latest + image: wurstmeister/kafka:2.13-2.8.1 command: ["/bin/sh"] args: ["-c", "while true; do echo 'Producing message'; kafka-console-producer.sh --broker-list kafka-service:9092 --topic test-topic | 'Test message'; sleep 5; done"] env: From 497c79b146b3920a19eef97bf9dc829e474bdc21 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 15 Dec 2024 18:29:46 -0500 Subject: [PATCH 40/92] Allow for float metrics. --- test/e2e/jmx/jmx_test.go | 4 ++-- util/awsservice/cloudwatchmetrics.go | 36 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 9e1493371..c0b2ebd2c 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -260,13 +260,13 @@ func testTomcatSessions(t *testing.T) { startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() - hasActiveSessions := awsservice.ValidateSampleCount( + hasActiveSessions := awsservice.ValidateSampleCountFloat( "tomcat.sessions", "JVM_TOMCAT_E2E", nil, startTime, endTime, - 1, + 0.1, 1000, 60, ) diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index 59ef886b2..5a2d3d6dc 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -106,6 +106,42 @@ func ValidateSampleCount(metricName, namespace string, dimensions []types.Dimens return false } +func ValidateSampleCountFloat(metricName, namespace string, dimensions []types.Dimension, + startTime time.Time, endTime time.Time, + lowerBoundInclusive float64, upperBoundInclusive float64, periodInSeconds int32) bool { + + metricStatsInput := cloudwatch.GetMetricStatisticsInput{ + MetricName: aws.String(metricName), + Namespace: aws.String(namespace), + StartTime: aws.Time(startTime), + EndTime: aws.Time(endTime), + Period: aws.Int32(periodInSeconds), + Dimensions: dimensions, + Statistics: []types.Statistic{types.StatisticSampleCount}, + } + data, err := CwmClient.GetMetricStatistics(ctx, &metricStatsInput) + if err != nil { + return false + } + + var dataPoints float64 + log.Printf("These are the data points: %v", data) + log.Printf("These are the data points: %v", data.Datapoints) + + for _, datapoint := range data.Datapoints { + dataPoints += *datapoint.SampleCount + } + log.Printf("Number of datapoints for start time %v with endtime %v and period %d is %.3f is inclusive between %.3f and %.3f", + startTime, endTime, periodInSeconds, dataPoints, lowerBoundInclusive, upperBoundInclusive) + + if lowerBoundInclusive <= dataPoints && dataPoints <= upperBoundInclusive { + return true + } + + return false +} + + func GetMetricStatistics( metricName string, namespace string, From 65272afb14b83ac15a4ed722eb2b7392dd4e2ec3 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 15 Dec 2024 22:29:18 -0500 Subject: [PATCH 41/92] Set to 1 node. --- terraform/eks/e2e/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 062fdb1ba..044d3a7d0 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -34,9 +34,9 @@ resource "aws_eks_node_group" "this" { subnet_ids = module.basic_components.public_subnet_ids scaling_config { - desired_size = 2 - max_size = 2 - min_size = 2 + desired_size = 1 + max_size = 1 + min_size = 1 } ami_type = "AL2_x86_64" From 9f9e9dbaa06e056b10f536994d72dc638ccd1b63 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 15 Dec 2024 22:33:32 -0500 Subject: [PATCH 42/92] Use 0.01. --- test/e2e/jmx/jmx_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index c0b2ebd2c..5403dea5c 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -266,7 +266,7 @@ func testTomcatSessions(t *testing.T) { nil, startTime, endTime, - 0.1, + 0.01, 1000, 60, ) @@ -361,13 +361,13 @@ func testTomcatRejectedSessions(t *testing.T) { startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() - hasActiveSessions := awsservice.ValidateSampleCount( + hasActiveSessions := awsservice.ValidateSampleCountFloat( "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus", nil, startTime, endTime, - 1, + 0.01, 1000, 60, ) From 4ea25c2da027b2252d0a59519b31bf16e1eaaf6a Mon Sep 17 00:00:00 2001 From: musa-asad Date: Mon, 16 Dec 2024 01:53:10 -0500 Subject: [PATCH 43/92] Consumer only metrics. --- test/e2e/jmx/jmx_test.go | 8 ++--- .../jmx/resources/cwagent_configs/kafka.json | 32 +------------------ .../sample_apps/kafka-deployment.yaml | 14 +++----- 3 files changed, 9 insertions(+), 45 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 5403dea5c..c67cdcd1c 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -255,9 +255,9 @@ func testTomcatSessions(t *testing.T) { } } - time.Sleep(5 * time.Minute) + time.Sleep(10 * time.Minute) - startTime := time.Now().Add(-5 * time.Minute) + startTime := time.Now().Add(-15 * time.Minute) endTime := time.Now() hasActiveSessions := awsservice.ValidateSampleCountFloat( @@ -356,9 +356,9 @@ func testTomcatRejectedSessions(t *testing.T) { } } - time.Sleep(5 * time.Minute) + time.Sleep(10 * time.Minute) - startTime := time.Now().Add(-5 * time.Minute) + startTime := time.Now().Add(-15 * time.Minute) endTime := time.Now() hasActiveSessions := awsservice.ValidateSampleCountFloat( diff --git a/test/e2e/jmx/resources/cwagent_configs/kafka.json b/test/e2e/jmx/resources/cwagent_configs/kafka.json index dce338b31..8b16add8c 100644 --- a/test/e2e/jmx/resources/cwagent_configs/kafka.json +++ b/test/e2e/jmx/resources/cwagent_configs/kafka.json @@ -7,37 +7,7 @@ "measurement": [ "kafka.consumer.total.records-consumed-rate", "kafka.consumer.total.bytes-consumed-rate", - "kafka.consumer.fetch-rate", - "kafka.consumer.records-lag-max", - "kafka.consumer.total.fetch-size-avg", - "kafka.consumer.bytes-consumed-rate", - "kafka.consumer.fetch-size-avg", - "kafka.consumer.records-consumed-rate" - ] - }, - "kafka-producer": { - "measurement": [ - "kafka.producer.io-wait-time-ns-avg", - "kafka.producer.outgoing-byte-rate", - "kafka.producer.request-latency-avg", - "kafka.producer.request-rate", - "kafka.producer.response-rate", - "kafka.producer.byte-rate", - "kafka.producer.compression-rate", - "kafka.producer.record-error-rate", - "kafka.producer.record-retry-rate", - "kafka.producer.record-send-rate" - ] - }, - "kafka": { - "measurement": [ - "kafka.message.count", - "kafka.request.count", - "kafka.request.failed", - "kafka.request.time.total", - "kafka.request.time.50p", - "kafka.request.time.99p", - "kafka.request.time.avg" + "kafka.consumer.fetch-rate" ] } } diff --git a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml index e0ed0a5fc..7e27fd9bb 100644 --- a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml +++ b/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: zookeeper - image: wurstmeister/kafka:2.13-2.8.1 + image: wurstmeister/zookeeper:latest ports: - containerPort: 2181 env: @@ -56,13 +56,10 @@ spec: metadata: labels: app: kafka - annotations: - instrumentation.opentelemetry.io/inject-java: "true" - cloudwatch.aws.amazon.com/inject-jmx-kafka: "true" spec: containers: - name: kafka - image: wurstmeister/kafka:2.13-2.8.1 + image: wurstmeister/kafka:latest ports: - containerPort: 9092 env: @@ -88,13 +85,10 @@ spec: metadata: labels: app: kafka-producer - annotations: - instrumentation.opentelemetry.io/inject-java: "true" - cloudwatch.aws.amazon.com/inject-jmx-kafka-producer: "true" spec: containers: - name: kafka-producer - image: wurstmeister/kafka:2.13-2.8.1 + image: wurstmeister/kafka:latest command: ["/bin/sh"] args: ["-c", "while true; do echo 'Producing message'; kafka-console-producer.sh --broker-list kafka-service:9092 --topic test-topic | 'Test message'; sleep 5; done"] env: @@ -120,7 +114,7 @@ spec: spec: containers: - name: kafka-consumer - image: wurstmeister/kafka:2.13-2.8.1 + image: wurstmeister/kafka:latest command: ["/bin/sh"] args: ["-c", "kafka-console-consumer.sh --bootstrap-server kafka-service:9092 --topic test-topic --from-beginning"] env: From dbea081708d3728497dab30fa3dad7092c94a068 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Mon, 16 Dec 2024 02:28:38 -0500 Subject: [PATCH 44/92] Update nodes. --- terraform/eks/e2e/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 062fdb1ba..044d3a7d0 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -34,9 +34,9 @@ resource "aws_eks_node_group" "this" { subnet_ids = module.basic_components.public_subnet_ids scaling_config { - desired_size = 2 - max_size = 2 - min_size = 2 + desired_size = 1 + max_size = 1 + min_size = 1 } ami_type = "AL2_x86_64" From 37dcf0bbb0e810c3c308387c92172642a2cf7b06 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Mon, 16 Dec 2024 18:46:28 -0500 Subject: [PATCH 45/92] Allow node configuration. --- generator/resources/eks_e2e_test_matrix.json | 2 +- generator/test_case_generator.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/generator/resources/eks_e2e_test_matrix.json b/generator/resources/eks_e2e_test_matrix.json index 81ed61090..d484997f7 100644 --- a/generator/resources/eks_e2e_test_matrix.json +++ b/generator/resources/eks_e2e_test_matrix.json @@ -1,5 +1,5 @@ [ { - + "nodes": 1 } ] \ No newline at end of file diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index dc9a5d308..f5887186e 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -30,6 +30,7 @@ type matrixRow struct { CaCertPath string `json:"caCertPath"` ValuesPerMinute int `json:"values_per_minute"` // Number of metrics to be sent or number of log lines to write K8sVersion string `json:"k8sVersion"` + Nodes int `json:"nodes"` TerraformDir string `json:"terraform_dir"` UseSSM bool `json:"useSSM"` ExcludedTests string `json:"excludedTests"` From cf4a54c506a2b5edebf8ec9d817d450847c3f6ac Mon Sep 17 00:00:00 2001 From: musa-asad Date: Mon, 16 Dec 2024 18:48:02 -0500 Subject: [PATCH 46/92] Fix spacing. --- util/awsservice/cloudwatchmetrics.go | 1 - 1 file changed, 1 deletion(-) diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index 5a2d3d6dc..cd6e1c0ab 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -141,7 +141,6 @@ func ValidateSampleCountFloat(metricName, namespace string, dimensions []types.D return false } - func GetMetricStatistics( metricName string, namespace string, From 58101e3c2bb347da1a8234d682ca9d5da55b9131 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Mon, 16 Dec 2024 18:51:57 -0500 Subject: [PATCH 47/92] Add license. --- test/e2e/jmx/jmx_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index c67cdcd1c..e69bd7585 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package main import ( From 0db3337aba75b9971e37a34822103f17796fabbd Mon Sep 17 00:00:00 2001 From: musa-asad Date: Mon, 16 Dec 2024 22:36:10 -0500 Subject: [PATCH 48/92] Check for max metric value. --- test/e2e/jmx/jmx_test.go | 30 +++++---- util/awsservice/cloudwatchmetrics.go | 97 ++++++++++++++++++---------- 2 files changed, 78 insertions(+), 49 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index e69bd7585..d8fe439af 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -258,23 +258,24 @@ func testTomcatSessions(t *testing.T) { } } - time.Sleep(10 * time.Minute) + time.Sleep(5 * time.Minute) - startTime := time.Now().Add(-15 * time.Minute) + startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() - hasActiveSessions := awsservice.ValidateSampleCountFloat( + maxSessions, err := awsservice.GetMetricMaximum( "tomcat.sessions", "JVM_TOMCAT_E2E", - nil, startTime, endTime, - 0.01, - 1000, 60, ) + if err != nil { + t.Errorf("Failed to get metric maximum: %v", err) + return + } - if !hasActiveSessions { + if maxSessions == 0 { t.Error("Expected non-zero tomcat.sessions after applying traffic") } }) @@ -359,23 +360,24 @@ func testTomcatRejectedSessions(t *testing.T) { } } - time.Sleep(10 * time.Minute) + time.Sleep(5 * time.Minute) - startTime := time.Now().Add(-15 * time.Minute) + startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() - hasActiveSessions := awsservice.ValidateSampleCountFloat( + maxRejectedSessions, err := awsservice.GetMetricMaximum( "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus", - nil, startTime, endTime, - 0.01, - 1000, 60, ) + if err != nil { + t.Errorf("Failed to get metric maximum: %v", err) + return + } - if !hasActiveSessions { + if maxRejectedSessions == 0 { t.Error("Expected non-zero catalina_manager_rejectedsessions after applying traffic") } }) diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index cd6e1c0ab..0bc74fb97 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -106,41 +106,6 @@ func ValidateSampleCount(metricName, namespace string, dimensions []types.Dimens return false } -func ValidateSampleCountFloat(metricName, namespace string, dimensions []types.Dimension, - startTime time.Time, endTime time.Time, - lowerBoundInclusive float64, upperBoundInclusive float64, periodInSeconds int32) bool { - - metricStatsInput := cloudwatch.GetMetricStatisticsInput{ - MetricName: aws.String(metricName), - Namespace: aws.String(namespace), - StartTime: aws.Time(startTime), - EndTime: aws.Time(endTime), - Period: aws.Int32(periodInSeconds), - Dimensions: dimensions, - Statistics: []types.Statistic{types.StatisticSampleCount}, - } - data, err := CwmClient.GetMetricStatistics(ctx, &metricStatsInput) - if err != nil { - return false - } - - var dataPoints float64 - log.Printf("These are the data points: %v", data) - log.Printf("These are the data points: %v", data.Datapoints) - - for _, datapoint := range data.Datapoints { - dataPoints += *datapoint.SampleCount - } - log.Printf("Number of datapoints for start time %v with endtime %v and period %d is %.3f is inclusive between %.3f and %.3f", - startTime, endTime, periodInSeconds, dataPoints, lowerBoundInclusive, upperBoundInclusive) - - if lowerBoundInclusive <= dataPoints && dataPoints <= upperBoundInclusive { - return true - } - - return false -} - func GetMetricStatistics( metricName string, namespace string, @@ -169,6 +134,68 @@ func GetMetricStatistics( return CwmClient.GetMetricStatistics(ctx, &metricStatsInput) } +func GetMetricMaximum( + metricName string, + namespace string, + startTime time.Time, + endTime time.Time, + periodInSeconds int32, +) (float64, error) { + // List metrics without any dimension filter to see what's available + listMetricsInput := cloudwatch.ListMetricsInput{ + MetricName: aws.String(metricName), + Namespace: aws.String(namespace), + RecentlyActive: "PT3H", + } + + metrics, err := CwmClient.ListMetrics(ctx, &listMetricsInput) + if err != nil { + return 0, err + } + if len(metrics.Metrics) == 0 { + return 0, fmt.Errorf("no metrics found for %s", metricName) + } + + // Log all available metrics and their dimensions + for _, metric := range metrics.Metrics { + log.Printf("Found metric: %s", *metric.MetricName) + for _, dim := range metric.Dimensions { + log.Printf(" Dimension: %s = %s", *dim.Name, *dim.Value) + } + } + + // Use the dimensions from the first matching metric + dimensions := metrics.Metrics[0].Dimensions + + data, err := GetMetricStatistics( + metricName, + namespace, + dimensions, + startTime, + endTime, + periodInSeconds, + []types.Statistic{types.StatisticMaximum}, + nil, + ) + if err != nil { + return 0, err + } + + if len(data.Datapoints) == 0 { + return 0, fmt.Errorf("no datapoints found for metric %s", metricName) + } + + maxValue := float64(0) + for _, datapoint := range data.Datapoints { + if *datapoint.Maximum > maxValue { + maxValue = *datapoint.Maximum + } + } + log.Printf("Maximum value found: %v", maxValue) + + return maxValue, nil +} + // GetMetricData takes the metric name, metric dimension and metric namespace and return the query metrics func GetMetricData(metricDataQueries []types.MetricDataQuery, startTime, endTime time.Time) (*cloudwatch.GetMetricDataOutput, error) { getMetricDataInput := cloudwatch.GetMetricDataInput{ From 803bad7f46fc4e2e034efc79c974104fe0db1390 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Mon, 16 Dec 2024 23:16:28 -0500 Subject: [PATCH 49/92] Simplify --- test/e2e/jmx/jmx_test.go | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index d8fe439af..d34faf00b 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -26,27 +26,17 @@ func init() { environment.RegisterEnvironmentMetaDataFlags() } -type TestConfig struct { - metricTests []func(*testing.T) -} - -var testRegistry = map[string]TestConfig{ +var testRegistry = map[string][]func(*testing.T){ "jvm_tomcat.json": { - metricTests: []func(*testing.T){ - testTomcatMetrics, - testTomcatSessions, - }, + testTomcatMetrics, + testTomcatSessions, }, "kafka.json": { - metricTests: []func(*testing.T){ - testKafkaMetrics, - }, + testKafkaMetrics, }, "containerinsights.json": { - metricTests: []func(*testing.T){ - testContainerInsightsMetrics, - testTomcatRejectedSessions, - }, + testContainerInsightsMetrics, + testTomcatRejectedSessions, }, } @@ -182,13 +172,13 @@ func TestMetrics(t *testing.T) { env := environment.GetEnvironmentMetaData() configFile := filepath.Base(env.AgentConfig) - config, exists := testRegistry[configFile] + tests, exists := testRegistry[configFile] if !exists { t.Skipf("No tests registered for config file: %s", configFile) return } - for _, testFunc := range config.metricTests { + for _, testFunc := range tests { testFunc(t) } } From 36112264c3e008dc0141e8d58917bd367fd9dc3b Mon Sep 17 00:00:00 2001 From: musa-asad Date: Mon, 16 Dec 2024 23:18:02 -0500 Subject: [PATCH 50/92] Fix imports. --- test/e2e/jmx/jmx_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index d34faf00b..3dd22f509 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -7,8 +7,6 @@ import ( "context" "flag" "fmt" - "github.com/aws/amazon-cloudwatch-agent-test/environment" - "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" "net/http" "os" "os/exec" @@ -20,6 +18,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" ) func init() { From 123f8e550ac107fb1e8c90894585d01c34dfc4e7 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Mon, 16 Dec 2024 23:51:47 -0500 Subject: [PATCH 51/92] Prevent test from running when it doesn't need to. --- test/e2e/jmx/jmx_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 3dd22f509..0963bae5f 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -43,6 +43,9 @@ var testRegistry = map[string][]func(*testing.T){ func TestMain(m *testing.M) { flag.Parse() + if flag.Lookup("test.run").Value.String() == "NO_MATCH" { + os.Exit(0) + } env := environment.GetEnvironmentMetaData() if env.Region != "us-west-2" { From 9591142366442cb21cc2126f76cd0296e5f6123b Mon Sep 17 00:00:00 2001 From: musa-asad Date: Tue, 17 Dec 2024 03:38:38 -0500 Subject: [PATCH 52/92] Update logic for max metric value. --- util/awsservice/cloudwatchmetrics.go | 53 +++++++++++++--------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index 0bc74fb97..0da0c7c29 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -156,43 +156,38 @@ func GetMetricMaximum( return 0, fmt.Errorf("no metrics found for %s", metricName) } - // Log all available metrics and their dimensions + maxValue := float64(0) + // Iterate through all metrics found for _, metric := range metrics.Metrics { log.Printf("Found metric: %s", *metric.MetricName) - for _, dim := range metric.Dimensions { - log.Printf(" Dimension: %s = %s", *dim.Name, *dim.Value) + data, err := GetMetricStatistics( + metricName, + namespace, + metric.Dimensions, + startTime, + endTime, + periodInSeconds, + []types.Statistic{types.StatisticMaximum}, + nil, + ) + if err != nil { + log.Printf("Error getting statistics for metric with dimensions %v: %v", metric.Dimensions, err) + continue // Continue with next metric if this one fails } - } - - // Use the dimensions from the first matching metric - dimensions := metrics.Metrics[0].Dimensions - - data, err := GetMetricStatistics( - metricName, - namespace, - dimensions, - startTime, - endTime, - periodInSeconds, - []types.Statistic{types.StatisticMaximum}, - nil, - ) - if err != nil { - return 0, err - } - if len(data.Datapoints) == 0 { - return 0, fmt.Errorf("no datapoints found for metric %s", metricName) + // Check datapoints for this metric + for _, datapoint := range data.Datapoints { + if *datapoint.Maximum > maxValue { + maxValue = *datapoint.Maximum + } + } } - maxValue := float64(0) - for _, datapoint := range data.Datapoints { - if *datapoint.Maximum > maxValue { - maxValue = *datapoint.Maximum - } + if maxValue == 0 { + return 0, fmt.Errorf("no valid datapoints found for metric %s", metricName) } - log.Printf("Maximum value found: %v", maxValue) + log.Printf("Maximum value found across all metrics: %v", maxValue) return maxValue, nil } From 6fcde7ec7900cd7af5e4cd730b2d77e82f224cc3 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Tue, 17 Dec 2024 04:29:28 -0500 Subject: [PATCH 53/92] Improve logic to check nodes being ran. --- test/e2e/jmx/jmx_test.go | 13 +++++++ util/awsservice/cloudwatchmetrics.go | 57 ++++++++++++++++++---------- 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 0963bae5f..95393331f 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -41,6 +41,8 @@ var testRegistry = map[string][]func(*testing.T){ }, } +var nodeNames []string + func TestMain(m *testing.M) { flag.Parse() if flag.Lookup("test.run").Value.String() == "NO_MATCH" { @@ -139,6 +141,15 @@ func TestResources(t *testing.T) { t.Fatalf("Error creating Kubernetes client: %v", err) } + nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("Error listing nodes: %v", err) + } + + for _, node := range nodes.Items { + nodeNames = append(nodeNames, node.Name) + } + daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) if err != nil { t.Errorf("Error getting CloudWatch Agent DaemonSet: %v", err) @@ -263,6 +274,7 @@ func testTomcatSessions(t *testing.T) { startTime, endTime, 60, + nodeNames, ) if err != nil { t.Errorf("Failed to get metric maximum: %v", err) @@ -365,6 +377,7 @@ func testTomcatRejectedSessions(t *testing.T) { startTime, endTime, 60, + nodeNames, ) if err != nil { t.Errorf("Failed to get metric maximum: %v", err) diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index 0da0c7c29..7592d1e64 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -140,8 +140,8 @@ func GetMetricMaximum( startTime time.Time, endTime time.Time, periodInSeconds int32, + nodeNames []string, ) (float64, error) { - // List metrics without any dimension filter to see what's available listMetricsInput := cloudwatch.ListMetricsInput{ MetricName: aws.String(metricName), Namespace: aws.String(namespace), @@ -157,28 +157,43 @@ func GetMetricMaximum( } maxValue := float64(0) - // Iterate through all metrics found for _, metric := range metrics.Metrics { - log.Printf("Found metric: %s", *metric.MetricName) - data, err := GetMetricStatistics( - metricName, - namespace, - metric.Dimensions, - startTime, - endTime, - periodInSeconds, - []types.Statistic{types.StatisticMaximum}, - nil, - ) - if err != nil { - log.Printf("Error getting statistics for metric with dimensions %v: %v", metric.Dimensions, err) - continue // Continue with next metric if this one fails + var nodeNameMatch bool + var nodeName string + for _, dim := range metric.Dimensions { + if *dim.Name == "k8s.node.name" { + nodeName = *dim.Value + for _, name := range nodeNames { + if nodeName == name { + nodeNameMatch = true + break + } + } + break + } } - // Check datapoints for this metric - for _, datapoint := range data.Datapoints { - if *datapoint.Maximum > maxValue { - maxValue = *datapoint.Maximum + if nodeNameMatch { + log.Printf("Found metric: %s for node: %s", *metric.MetricName, nodeName) + data, err := GetMetricStatistics( + metricName, + namespace, + metric.Dimensions, + startTime, + endTime, + periodInSeconds, + []types.Statistic{types.StatisticMaximum}, + nil, + ) + if err != nil { + log.Printf("Error getting statistics for metric with dimensions %v: %v", metric.Dimensions, err) + continue + } + + for _, datapoint := range data.Datapoints { + if *datapoint.Maximum > maxValue { + maxValue = *datapoint.Maximum + } } } } @@ -187,7 +202,7 @@ func GetMetricMaximum( return 0, fmt.Errorf("no valid datapoints found for metric %s", metricName) } - log.Printf("Maximum value found across all metrics: %v", maxValue) + log.Printf("Maximum value found across specified nodes: %v", maxValue) return maxValue, nil } From 0533bfa666385690d6384b6a912d0067b45642df Mon Sep 17 00:00:00 2001 From: musa-asad Date: Tue, 17 Dec 2024 07:01:42 -0500 Subject: [PATCH 54/92] Delete loadbalancer after complete --- test/e2e/jmx/jmx_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 95393331f..5614a7726 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -284,6 +284,13 @@ func testTomcatSessions(t *testing.T) { if maxSessions == 0 { t.Error("Expected non-zero tomcat.sessions after applying traffic") } + + deleteCmd := exec.Command("kubectl", "delete", "svc", "tomcat-service") + if output, err := deleteCmd.CombinedOutput(); err != nil { + t.Logf("Warning: Failed to delete load balancer service: %v\nOutput: %s", err, output) + } else { + t.Log("Successfully deleted load balancer service") + } }) } @@ -387,5 +394,12 @@ func testTomcatRejectedSessions(t *testing.T) { if maxRejectedSessions == 0 { t.Error("Expected non-zero catalina_manager_rejectedsessions after applying traffic") } + + deleteCmd := exec.Command("kubectl", "delete", "svc", "tomcat-service") + if output, err := deleteCmd.CombinedOutput(); err != nil { + t.Logf("Warning: Failed to delete load balancer service: %v\nOutput: %s", err, output) + } else { + t.Log("Successfully deleted load balancer service") + } }) } From c995fc7fa30c37dd4766b97144809a00d6f63700 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Tue, 17 Dec 2024 16:47:02 -0500 Subject: [PATCH 55/92] constant --- ...trix.json => eks_e2e_jmx_test_matrix.json} | 0 .../eks_e2e_prometheus_test_matrix.json | 5 + test/e2e/jmx/jmx_test.go | 98 ++++++++++--------- test/e2e/prometheus/prometheus_test.go | 1 + .../resources/cwagent_configs/prometheus.json | 0 .../prometheus_configs/prometheus.yaml | 0 .../sample_apps/memcached-sample.yaml | 63 ++++++++++++ 7 files changed, 121 insertions(+), 46 deletions(-) rename generator/resources/{eks_e2e_test_matrix.json => eks_e2e_jmx_test_matrix.json} (100%) create mode 100644 generator/resources/eks_e2e_prometheus_test_matrix.json create mode 100644 test/e2e/prometheus/prometheus_test.go create mode 100644 test/e2e/prometheus/resources/cwagent_configs/prometheus.json create mode 100644 test/e2e/prometheus/resources/prometheus_configs/prometheus.yaml create mode 100644 test/e2e/prometheus/resources/sample_apps/memcached-sample.yaml diff --git a/generator/resources/eks_e2e_test_matrix.json b/generator/resources/eks_e2e_jmx_test_matrix.json similarity index 100% rename from generator/resources/eks_e2e_test_matrix.json rename to generator/resources/eks_e2e_jmx_test_matrix.json diff --git a/generator/resources/eks_e2e_prometheus_test_matrix.json b/generator/resources/eks_e2e_prometheus_test_matrix.json new file mode 100644 index 000000000..d484997f7 --- /dev/null +++ b/generator/resources/eks_e2e_prometheus_test_matrix.json @@ -0,0 +1,5 @@ +[ + { + "nodes": 1 + } +] \ No newline at end of file diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 5614a7726..9a5b29502 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -27,6 +27,12 @@ func init() { environment.RegisterEnvironmentMetaDataFlags() } +const ( + NAMESPACE_JVM_TOMCAT = "JVM_TOMCAT_E2E" + NAMESPACE_KAFKA = "KAFKA_E2E" + NAMESPACE_CONTAINERINSIGHTS = "ContainerInsights/Prometheus" +) + var testRegistry = map[string][]func(*testing.T){ "jvm_tomcat.json": { testTomcatMetrics, @@ -204,29 +210,29 @@ func testTomcatMetrics(t *testing.T) { name string namespace string }{ - {"tomcat.traffic", "JVM_TOMCAT_E2E"}, - {"jvm.classes.loaded", "JVM_TOMCAT_E2E"}, - {"jvm.gc.collections.count", "JVM_TOMCAT_E2E"}, - {"jvm.gc.collections.elapsed", "JVM_TOMCAT_E2E"}, - {"jvm.memory.heap.init", "JVM_TOMCAT_E2E"}, - {"jvm.memory.heap.max", "JVM_TOMCAT_E2E"}, - {"jvm.memory.heap.used", "JVM_TOMCAT_E2E"}, - {"jvm.memory.heap.committed", "JVM_TOMCAT_E2E"}, - {"jvm.memory.nonheap.init", "JVM_TOMCAT_E2E"}, - {"jvm.memory.nonheap.max", "JVM_TOMCAT_E2E"}, - {"jvm.memory.nonheap.used", "JVM_TOMCAT_E2E"}, - {"jvm.memory.nonheap.committed", "JVM_TOMCAT_E2E"}, - {"jvm.memory.pool.init", "JVM_TOMCAT_E2E"}, - {"jvm.memory.pool.max", "JVM_TOMCAT_E2E"}, - {"jvm.memory.pool.used", "JVM_TOMCAT_E2E"}, - {"jvm.memory.pool.committed", "JVM_TOMCAT_E2E"}, - {"jvm.threads.count", "JVM_TOMCAT_E2E"}, - {"tomcat.sessions", "JVM_TOMCAT_E2E"}, - {"tomcat.errors", "JVM_TOMCAT_E2E"}, - {"tomcat.request_count", "JVM_TOMCAT_E2E"}, - {"tomcat.max_time", "JVM_TOMCAT_E2E"}, - {"tomcat.processing_time", "JVM_TOMCAT_E2E"}, - {"tomcat.threads", "JVM_TOMCAT_E2E"}, + {"tomcat.traffic", NAMESPACE_JVM_TOMCAT}, + {"jvm.classes.loaded", NAMESPACE_JVM_TOMCAT}, + {"jvm.gc.collections.count", NAMESPACE_JVM_TOMCAT}, + {"jvm.gc.collections.elapsed", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.heap.init", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.heap.max", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.heap.used", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.heap.committed", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.nonheap.init", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.nonheap.max", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.nonheap.used", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.nonheap.committed", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.pool.init", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.pool.max", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.pool.used", NAMESPACE_JVM_TOMCAT}, + {"jvm.memory.pool.committed", NAMESPACE_JVM_TOMCAT}, + {"jvm.threads.count", NAMESPACE_JVM_TOMCAT}, + {"tomcat.sessions", NAMESPACE_JVM_TOMCAT}, + {"tomcat.errors", NAMESPACE_JVM_TOMCAT}, + {"tomcat.request_count", NAMESPACE_JVM_TOMCAT}, + {"tomcat.max_time", NAMESPACE_JVM_TOMCAT}, + {"tomcat.processing_time", NAMESPACE_JVM_TOMCAT}, + {"tomcat.threads", NAMESPACE_JVM_TOMCAT}, } for _, metric := range metricsToCheck { @@ -270,7 +276,7 @@ func testTomcatSessions(t *testing.T) { maxSessions, err := awsservice.GetMetricMaximum( "tomcat.sessions", - "JVM_TOMCAT_E2E", + NAMESPACE_JVM_TOMCAT, startTime, endTime, 60, @@ -300,9 +306,9 @@ func testKafkaMetrics(t *testing.T) { name string namespace string }{ - {"kafka.consumer.fetch-rate", "KAFKA_E2E"}, - {"kafka.consumer.total.bytes-consumed-rate", "KAFKA_E2E"}, - {"kafka.consumer.total.records-consumed-rate", "KAFKA_E2E"}, + {"kafka.consumer.fetch-rate", NAMESPACE_KAFKA}, + {"kafka.consumer.total.bytes-consumed-rate", NAMESPACE_KAFKA}, + {"kafka.consumer.total.records-consumed-rate", NAMESPACE_KAFKA}, } for _, metric := range metricsToCheck { @@ -319,24 +325,24 @@ func testContainerInsightsMetrics(t *testing.T) { name string namespace string }{ - {"jvm_classes_loaded", "ContainerInsights/Prometheus"}, - {"jvm_threads_current", "ContainerInsights/Prometheus"}, - {"jvm_threads_daemon", "ContainerInsights/Prometheus"}, - {"java_lang_operatingsystem_totalswapspacesize", "ContainerInsights/Prometheus"}, - {"java_lang_operatingsystem_systemcpuload", "ContainerInsights/Prometheus"}, - {"java_lang_operatingsystem_processcpuload", "ContainerInsights/Prometheus"}, - {"java_lang_operatingsystem_freeswapspacesize", "ContainerInsights/Prometheus"}, - {"java_lang_operatingsystem_totalphysicalmemorysize", "ContainerInsights/Prometheus"}, - {"java_lang_operatingsystem_freephysicalmemorysize", "ContainerInsights/Prometheus"}, - {"java_lang_operatingsystem_openfiledescriptorcount", "ContainerInsights/Prometheus"}, - {"java_lang_operatingsystem_availableprocessors", "ContainerInsights/Prometheus"}, - {"jvm_memory_bytes_used", "ContainerInsights/Prometheus"}, - {"jvm_memory_pool_bytes_used", "ContainerInsights/Prometheus"}, - {"catalina_manager_activesessions", "ContainerInsights/Prometheus"}, - {"catalina_manager_rejectedsessions", "ContainerInsights/Prometheus"}, - {"catalina_globalrequestprocessor_requestcount", "ContainerInsights/Prometheus"}, - {"catalina_globalrequestprocessor_errorcount", "ContainerInsights/Prometheus"}, - {"catalina_globalrequestprocessor_processingtime", "ContainerInsights/Prometheus"}, + {"jvm_classes_loaded", NAMESPACE_CONTAINERINSIGHTS}, + {"jvm_threads_current", NAMESPACE_CONTAINERINSIGHTS}, + {"jvm_threads_daemon", NAMESPACE_CONTAINERINSIGHTS}, + {"java_lang_operatingsystem_totalswapspacesize", NAMESPACE_CONTAINERINSIGHTS}, + {"java_lang_operatingsystem_systemcpuload", NAMESPACE_CONTAINERINSIGHTS}, + {"java_lang_operatingsystem_processcpuload", NAMESPACE_CONTAINERINSIGHTS}, + {"java_lang_operatingsystem_freeswapspacesize", NAMESPACE_CONTAINERINSIGHTS}, + {"java_lang_operatingsystem_totalphysicalmemorysize", NAMESPACE_CONTAINERINSIGHTS}, + {"java_lang_operatingsystem_freephysicalmemorysize", NAMESPACE_CONTAINERINSIGHTS}, + {"java_lang_operatingsystem_openfiledescriptorcount", NAMESPACE_CONTAINERINSIGHTS}, + {"java_lang_operatingsystem_availableprocessors", NAMESPACE_CONTAINERINSIGHTS}, + {"jvm_memory_bytes_used", NAMESPACE_CONTAINERINSIGHTS}, + {"jvm_memory_pool_bytes_used", NAMESPACE_CONTAINERINSIGHTS}, + {"catalina_manager_activesessions", NAMESPACE_CONTAINERINSIGHTS}, + {"catalina_manager_rejectedsessions", NAMESPACE_CONTAINERINSIGHTS}, + {"catalina_globalrequestprocessor_requestcount", NAMESPACE_CONTAINERINSIGHTS}, + {"catalina_globalrequestprocessor_errorcount", NAMESPACE_CONTAINERINSIGHTS}, + {"catalina_globalrequestprocessor_processingtime", NAMESPACE_CONTAINERINSIGHTS}, } for _, metric := range metricsToCheck { @@ -380,7 +386,7 @@ func testTomcatRejectedSessions(t *testing.T) { maxRejectedSessions, err := awsservice.GetMetricMaximum( "catalina_manager_rejectedsessions", - "ContainerInsights/Prometheus", + NAMESPACE_CONTAINERINSIGHTS, startTime, endTime, 60, diff --git a/test/e2e/prometheus/prometheus_test.go b/test/e2e/prometheus/prometheus_test.go new file mode 100644 index 000000000..7b1b4c03e --- /dev/null +++ b/test/e2e/prometheus/prometheus_test.go @@ -0,0 +1 @@ +package prometheus diff --git a/test/e2e/prometheus/resources/cwagent_configs/prometheus.json b/test/e2e/prometheus/resources/cwagent_configs/prometheus.json new file mode 100644 index 000000000..e69de29bb diff --git a/test/e2e/prometheus/resources/prometheus_configs/prometheus.yaml b/test/e2e/prometheus/resources/prometheus_configs/prometheus.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/test/e2e/prometheus/resources/sample_apps/memcached-sample.yaml b/test/e2e/prometheus/resources/sample_apps/memcached-sample.yaml new file mode 100644 index 000000000..03826c375 --- /dev/null +++ b/test/e2e/prometheus/resources/sample_apps/memcached-sample.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: memcached-sample + labels: + app: memcached +spec: + replicas: 1 + selector: + matchLabels: + app: memcached + template: + metadata: + labels: + app: memcached + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9150" + spec: + containers: + - name: memcached + image: docker.io/bitnami/memcached:1.6.33 + ports: + - containerPort: 11211 + resources: + limits: + cpu: 150m + memory: 192Mi + requests: + cpu: 100m + memory: 128Mi + - name: metrics + image: docker.io/bitnami/memcached-exporter:0.15.0 + ports: + - containerPort: 9150 + resources: + limits: + cpu: 150m + memory: 192Mi + requests: + cpu: 100m + memory: 128Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: memcached-service + labels: + app: memcached + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9150" +spec: + selector: + app: memcached + ports: + - name: memcache + port: 11211 + targetPort: 11211 + - name: metrics + port: 9150 + targetPort: 9150 From 5efca7c725ddca21601650deaa476953fe20c04e Mon Sep 17 00:00:00 2001 From: musa-asad Date: Tue, 17 Dec 2024 17:17:54 -0500 Subject: [PATCH 56/92] constant --- .../eks_e2e_prometheus_test_matrix.json | 5 -- ...t_matrix.json => eks_e2e_test_matrix.json} | 0 test/e2e/prometheus/prometheus_test.go | 1 - .../resources/cwagent_configs/prometheus.json | 0 .../prometheus_configs/prometheus.yaml | 0 .../sample_apps/memcached-sample.yaml | 63 ------------------- 6 files changed, 69 deletions(-) delete mode 100644 generator/resources/eks_e2e_prometheus_test_matrix.json rename generator/resources/{eks_e2e_jmx_test_matrix.json => eks_e2e_test_matrix.json} (100%) delete mode 100644 test/e2e/prometheus/prometheus_test.go delete mode 100644 test/e2e/prometheus/resources/cwagent_configs/prometheus.json delete mode 100644 test/e2e/prometheus/resources/prometheus_configs/prometheus.yaml delete mode 100644 test/e2e/prometheus/resources/sample_apps/memcached-sample.yaml diff --git a/generator/resources/eks_e2e_prometheus_test_matrix.json b/generator/resources/eks_e2e_prometheus_test_matrix.json deleted file mode 100644 index d484997f7..000000000 --- a/generator/resources/eks_e2e_prometheus_test_matrix.json +++ /dev/null @@ -1,5 +0,0 @@ -[ - { - "nodes": 1 - } -] \ No newline at end of file diff --git a/generator/resources/eks_e2e_jmx_test_matrix.json b/generator/resources/eks_e2e_test_matrix.json similarity index 100% rename from generator/resources/eks_e2e_jmx_test_matrix.json rename to generator/resources/eks_e2e_test_matrix.json diff --git a/test/e2e/prometheus/prometheus_test.go b/test/e2e/prometheus/prometheus_test.go deleted file mode 100644 index 7b1b4c03e..000000000 --- a/test/e2e/prometheus/prometheus_test.go +++ /dev/null @@ -1 +0,0 @@ -package prometheus diff --git a/test/e2e/prometheus/resources/cwagent_configs/prometheus.json b/test/e2e/prometheus/resources/cwagent_configs/prometheus.json deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/e2e/prometheus/resources/prometheus_configs/prometheus.yaml b/test/e2e/prometheus/resources/prometheus_configs/prometheus.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/e2e/prometheus/resources/sample_apps/memcached-sample.yaml b/test/e2e/prometheus/resources/sample_apps/memcached-sample.yaml deleted file mode 100644 index 03826c375..000000000 --- a/test/e2e/prometheus/resources/sample_apps/memcached-sample.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: memcached-sample - labels: - app: memcached -spec: - replicas: 1 - selector: - matchLabels: - app: memcached - template: - metadata: - labels: - app: memcached - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9150" - spec: - containers: - - name: memcached - image: docker.io/bitnami/memcached:1.6.33 - ports: - - containerPort: 11211 - resources: - limits: - cpu: 150m - memory: 192Mi - requests: - cpu: 100m - memory: 128Mi - - name: metrics - image: docker.io/bitnami/memcached-exporter:0.15.0 - ports: - - containerPort: 9150 - resources: - limits: - cpu: 150m - memory: 192Mi - requests: - cpu: 100m - memory: 128Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: memcached-service - labels: - app: memcached - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9150" -spec: - selector: - app: memcached - ports: - - name: memcache - port: 11211 - targetPort: 11211 - - name: metrics - port: 9150 - targetPort: 9150 From 74c44df63584ccaddea31625f5c843eda24fb917 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Wed, 18 Dec 2024 17:20:00 -0500 Subject: [PATCH 57/92] address initial comments --- .gitignore | 3 +- generator/test_case_generator.go | 5 +- test/e2e/jmx/jmx_test.go | 173 +++++++++++---------------- util/awsservice/cloudwatchmetrics.go | 23 ++-- 4 files changed, 86 insertions(+), 118 deletions(-) diff --git a/.gitignore b/.gitignore index ed0aa65ed..214284244 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ terraform.* **/.terraform/* /test/**/final_*.yml coverage.txt -generator/resources/*complete*.json \ No newline at end of file +generator/resources/*complete*.json +terraform/eks/e2e/helm-charts \ No newline at end of file diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index f5887186e..696d4ddd5 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -248,9 +248,12 @@ var testTypeToTestConfig = map[string][]testConfig{ } var testTypeToTestConfigE2E = map[string][]testConfig{ - "eks_e2e": { + "eks_e2e_jmx": { {testDir: "../../../test/e2e/jmx"}, }, + "eks_e2e_prometheus": { + {testDir: "../../../test/e2e/prometheus"}, + }, } type partition struct { diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 9a5b29502..b959077f9 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" @@ -88,7 +89,6 @@ func ApplyHelm(env *environment.MetaData) error { helm := []string{ "helm", "upgrade", "--install", "amazon-cloudwatch-observability", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability"), - "--values", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability", "values.yaml"), "--set", fmt.Sprintf("clusterName=%s", env.EKSClusterName), "--set", fmt.Sprintf("region=%s", env.Region), "--set", fmt.Sprintf("agent.image.repository=%s", env.CloudwatchAgentRepository), @@ -138,55 +138,33 @@ func ApplyHelm(env *environment.MetaData) error { func TestResources(t *testing.T) { config, err := clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config")) - if err != nil { - t.Fatalf("Error building kubeconfig: %v", err) - } + require.NoError(t, err, "Error building kubeconfig") clientset, err := kubernetes.NewForConfig(config) - if err != nil { - t.Fatalf("Error creating Kubernetes client: %v", err) - } + require.NoError(t, err, "Error building kubeconfig") nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - t.Fatalf("Error listing nodes: %v", err) - } + require.NoError(t, err, "Error listing nodes") for _, node := range nodes.Items { nodeNames = append(nodeNames, node.Name) } daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - if err != nil { - t.Errorf("Error getting CloudWatch Agent DaemonSet: %v", err) - } - if daemonSet == nil { - t.Error("CloudWatch Agent DaemonSet not found") - } + require.NoError(t, err, "Error getting CloudWatch Agent DaemonSet") + require.NotNil(t, daemonSet, "CloudWatch Agent DaemonSet not found") configMap, err := clientset.CoreV1().ConfigMaps("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - if err != nil { - t.Errorf("Error getting CloudWatch Agent ConfigMap: %v", err) - } - if configMap == nil { - t.Error("CloudWatch Agent ConfigMap not found") - } + require.NoError(t, err, "Error getting CloudWatch Agent ConfigMap") + require.NotNil(t, configMap, "CloudWatch Agent ConfigMap not found") service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - if err != nil { - t.Errorf("Error getting CloudWatch Agent Service: %v", err) - } - if service == nil { - t.Error("CloudWatch Agent Service not found") - } + require.NoError(t, err, "Error getting CloudWatch Agent Service") + require.NotNil(t, service, "CloudWatch Agent Service not found") serviceAccount, err := clientset.CoreV1().ServiceAccounts("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - if err != nil { - t.Errorf("Error getting CloudWatch Agent Service Account: %v", err) - } - if serviceAccount == nil { - t.Error("CloudWatch Agent Service Account not found") - } + require.NoError(t, err, "Error getting CloudWatch Agent Service Account") + require.NotNil(t, serviceAccount, "CloudWatch Agent Service Account not found") } func TestMetrics(t *testing.T) { @@ -206,38 +184,35 @@ func TestMetrics(t *testing.T) { func testTomcatMetrics(t *testing.T) { t.Run("verify_jvm_tomcat_metrics", func(t *testing.T) { - metricsToCheck := []struct { - name string - namespace string - }{ - {"tomcat.traffic", NAMESPACE_JVM_TOMCAT}, - {"jvm.classes.loaded", NAMESPACE_JVM_TOMCAT}, - {"jvm.gc.collections.count", NAMESPACE_JVM_TOMCAT}, - {"jvm.gc.collections.elapsed", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.heap.init", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.heap.max", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.heap.used", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.heap.committed", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.nonheap.init", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.nonheap.max", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.nonheap.used", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.nonheap.committed", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.pool.init", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.pool.max", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.pool.used", NAMESPACE_JVM_TOMCAT}, - {"jvm.memory.pool.committed", NAMESPACE_JVM_TOMCAT}, - {"jvm.threads.count", NAMESPACE_JVM_TOMCAT}, - {"tomcat.sessions", NAMESPACE_JVM_TOMCAT}, - {"tomcat.errors", NAMESPACE_JVM_TOMCAT}, - {"tomcat.request_count", NAMESPACE_JVM_TOMCAT}, - {"tomcat.max_time", NAMESPACE_JVM_TOMCAT}, - {"tomcat.processing_time", NAMESPACE_JVM_TOMCAT}, - {"tomcat.threads", NAMESPACE_JVM_TOMCAT}, + metricsToCheck := []string{ + "tomcat.traffic", + "jvm.classes.loaded", + "jvm.gc.collections.count", + "jvm.gc.collections.elapsed", + "jvm.memory.heap.init", + "jvm.memory.heap.max", + "jvm.memory.heap.used", + "jvm.memory.heap.committed", + "jvm.memory.nonheap.init", + "jvm.memory.nonheap.max", + "jvm.memory.nonheap.used", + "jvm.memory.nonheap.committed", + "jvm.memory.pool.init", + "jvm.memory.pool.max", + "jvm.memory.pool.used", + "jvm.memory.pool.committed", + "jvm.threads.count", + "tomcat.sessions", + "tomcat.errors", + "tomcat.request_count", + "tomcat.max_time", + "tomcat.processing_time", + "tomcat.threads", } for _, metric := range metricsToCheck { - t.Run(metric.name, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric.name, metric.namespace, nil, 5, 1*time.Minute) + t.Run(metric, func(t *testing.T) { + awsservice.ValidateMetricWithTest(t, metric, metric, nil, 5, 1*time.Minute) }) } }) @@ -274,7 +249,7 @@ func testTomcatSessions(t *testing.T) { startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() - maxSessions, err := awsservice.GetMetricMaximum( + aboveZero, err := awsservice.CheckMetricAboveZero( "tomcat.sessions", NAMESPACE_JVM_TOMCAT, startTime, @@ -283,11 +258,11 @@ func testTomcatSessions(t *testing.T) { nodeNames, ) if err != nil { - t.Errorf("Failed to get metric maximum: %v", err) + t.Errorf("Failed to check metric above zero: %v", err) return } - if maxSessions == 0 { + if !aboveZero { t.Error("Expected non-zero tomcat.sessions after applying traffic") } @@ -302,18 +277,15 @@ func testTomcatSessions(t *testing.T) { func testKafkaMetrics(t *testing.T) { t.Run("verify_kafka_metrics", func(t *testing.T) { - metricsToCheck := []struct { - name string - namespace string - }{ - {"kafka.consumer.fetch-rate", NAMESPACE_KAFKA}, - {"kafka.consumer.total.bytes-consumed-rate", NAMESPACE_KAFKA}, - {"kafka.consumer.total.records-consumed-rate", NAMESPACE_KAFKA}, + metricsToCheck := []string{ + "kafka.consumer.fetch-rate", + "kafka.consumer.total.bytes-consumed-rate", + "kafka.consumer.total.records-consumed-rate", } for _, metric := range metricsToCheck { - t.Run(metric.name, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric.name, metric.namespace, nil, 5, 1*time.Minute) + t.Run(metric, func(t *testing.T) { + awsservice.ValidateMetricWithTest(t, metric, NAMESPACE_KAFKA, nil, 5, 1*time.Minute) }) } }) @@ -321,33 +293,30 @@ func testKafkaMetrics(t *testing.T) { func testContainerInsightsMetrics(t *testing.T) { t.Run("verify_containerinsights_metrics", func(t *testing.T) { - metricsToCheck := []struct { - name string - namespace string - }{ - {"jvm_classes_loaded", NAMESPACE_CONTAINERINSIGHTS}, - {"jvm_threads_current", NAMESPACE_CONTAINERINSIGHTS}, - {"jvm_threads_daemon", NAMESPACE_CONTAINERINSIGHTS}, - {"java_lang_operatingsystem_totalswapspacesize", NAMESPACE_CONTAINERINSIGHTS}, - {"java_lang_operatingsystem_systemcpuload", NAMESPACE_CONTAINERINSIGHTS}, - {"java_lang_operatingsystem_processcpuload", NAMESPACE_CONTAINERINSIGHTS}, - {"java_lang_operatingsystem_freeswapspacesize", NAMESPACE_CONTAINERINSIGHTS}, - {"java_lang_operatingsystem_totalphysicalmemorysize", NAMESPACE_CONTAINERINSIGHTS}, - {"java_lang_operatingsystem_freephysicalmemorysize", NAMESPACE_CONTAINERINSIGHTS}, - {"java_lang_operatingsystem_openfiledescriptorcount", NAMESPACE_CONTAINERINSIGHTS}, - {"java_lang_operatingsystem_availableprocessors", NAMESPACE_CONTAINERINSIGHTS}, - {"jvm_memory_bytes_used", NAMESPACE_CONTAINERINSIGHTS}, - {"jvm_memory_pool_bytes_used", NAMESPACE_CONTAINERINSIGHTS}, - {"catalina_manager_activesessions", NAMESPACE_CONTAINERINSIGHTS}, - {"catalina_manager_rejectedsessions", NAMESPACE_CONTAINERINSIGHTS}, - {"catalina_globalrequestprocessor_requestcount", NAMESPACE_CONTAINERINSIGHTS}, - {"catalina_globalrequestprocessor_errorcount", NAMESPACE_CONTAINERINSIGHTS}, - {"catalina_globalrequestprocessor_processingtime", NAMESPACE_CONTAINERINSIGHTS}, + metricsToCheck := []string{ + "jvm_classes_loaded", + "jvm_threads_current", + "jvm_threads_daemon", + "java_lang_operatingsystem_totalswapspacesize", + "java_lang_operatingsystem_systemcpuload", + "java_lang_operatingsystem_processcpuload", + "java_lang_operatingsystem_freeswapspacesize", + "java_lang_operatingsystem_totalphysicalmemorysize", + "java_lang_operatingsystem_freephysicalmemorysize", + "java_lang_operatingsystem_openfiledescriptorcount", + "java_lang_operatingsystem_availableprocessors", + "jvm_memory_bytes_used", + "jvm_memory_pool_bytes_used", + "catalina_manager_activesessions", + "catalina_manager_rejectedsessions", + "catalina_globalrequestprocessor_requestcount", + "catalina_globalrequestprocessor_errorcount", + "catalina_globalrequestprocessor_processingtime", } for _, metric := range metricsToCheck { - t.Run(metric.name, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric.name, metric.namespace, nil, 5, 1*time.Minute) + t.Run(metric, func(t *testing.T) { + awsservice.ValidateMetricWithTest(t, metric, NAMESPACE_CONTAINERINSIGHTS, nil, 5, 1*time.Minute) }) } }) @@ -384,7 +353,7 @@ func testTomcatRejectedSessions(t *testing.T) { startTime := time.Now().Add(-5 * time.Minute) endTime := time.Now() - maxRejectedSessions, err := awsservice.GetMetricMaximum( + aboveZero, err := awsservice.CheckMetricAboveZero( "catalina_manager_rejectedsessions", NAMESPACE_CONTAINERINSIGHTS, startTime, @@ -393,11 +362,11 @@ func testTomcatRejectedSessions(t *testing.T) { nodeNames, ) if err != nil { - t.Errorf("Failed to get metric maximum: %v", err) + t.Errorf("Failed to check metric above zero: %v", err) return } - if maxRejectedSessions == 0 { + if !aboveZero { t.Error("Expected non-zero catalina_manager_rejectedsessions after applying traffic") } diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index 7592d1e64..14304961e 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -134,14 +134,14 @@ func GetMetricStatistics( return CwmClient.GetMetricStatistics(ctx, &metricStatsInput) } -func GetMetricMaximum( +func CheckMetricAboveZero( metricName string, namespace string, startTime time.Time, endTime time.Time, periodInSeconds int32, nodeNames []string, -) (float64, error) { +) (bool, error) { listMetricsInput := cloudwatch.ListMetricsInput{ MetricName: aws.String(metricName), Namespace: aws.String(namespace), @@ -150,13 +150,12 @@ func GetMetricMaximum( metrics, err := CwmClient.ListMetrics(ctx, &listMetricsInput) if err != nil { - return 0, err + return false, err } if len(metrics.Metrics) == 0 { - return 0, fmt.Errorf("no metrics found for %s", metricName) + return false, fmt.Errorf("no metrics found for %s", metricName) } - maxValue := float64(0) for _, metric := range metrics.Metrics { var nodeNameMatch bool var nodeName string @@ -174,7 +173,7 @@ func GetMetricMaximum( } if nodeNameMatch { - log.Printf("Found metric: %s for node: %s", *metric.MetricName, nodeName) + log.Printf("Checking metric: %s for node: %s", *metric.MetricName, nodeName) data, err := GetMetricStatistics( metricName, namespace, @@ -191,19 +190,15 @@ func GetMetricMaximum( } for _, datapoint := range data.Datapoints { - if *datapoint.Maximum > maxValue { - maxValue = *datapoint.Maximum + if *datapoint.Maximum > 0 { + log.Printf("Found value above zero for node: %s", nodeName) + return true, nil } } } } - if maxValue == 0 { - return 0, fmt.Errorf("no valid datapoints found for metric %s", metricName) - } - - log.Printf("Maximum value found across specified nodes: %v", maxValue) - return maxValue, nil + return false, nil } // GetMetricData takes the metric name, metric dimension and metric namespace and return the query metrics From 5d0af54264aa99754bcb075cafa58c17de9b33ac Mon Sep 17 00:00:00 2001 From: musa-asad Date: Thu, 19 Dec 2024 03:24:29 -0500 Subject: [PATCH 58/92] addressed comments --- environment/metadata.go | 4 + terraform/eks/e2e/main.tf | 23 ++- terraform/eks/e2e/variables.tf | 10 ++ test/e2e/jmx/jmx_test.go | 169 ++++-------------- .../jmx/resources/cwagent_configs/kafka.json | 32 +++- .../{kafka-deployment.yaml => kafka.yaml} | 29 ++- .../{tomcat-deployment.yaml => tomcat.yaml} | 11 +- util/common/kubernetes.go | 131 ++++++++++++++ 8 files changed, 263 insertions(+), 146 deletions(-) rename test/e2e/jmx/resources/sample_apps/{kafka-deployment.yaml => kafka.yaml} (78%) rename test/e2e/jmx/resources/sample_apps/{tomcat-deployment.yaml => tomcat.yaml} (79%) create mode 100644 util/common/kubernetes.go diff --git a/environment/metadata.go b/environment/metadata.go index 2cf99ad48..0bf122349 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -47,6 +47,7 @@ type MetaData struct { AmpWorkspaceId string Region string K8sVersion string + Destroy bool HelmChartsBranch string CloudwatchAgentRepository string CloudwatchAgentTag string @@ -87,6 +88,7 @@ type MetaDataStrings struct { AmpWorkspaceId string Region string K8sVersion string + Destroy string HelmChartsBranch string CloudwatchAgentRepository string CloudwatchAgentTag string @@ -136,6 +138,7 @@ func registerEKSData(d *MetaDataStrings) { func registerEKSE2ETestData(dataString *MetaDataStrings) { flag.StringVar(&(dataString.Region), "region", "", "AWS region") flag.StringVar(&(dataString.K8sVersion), "k8s_version", "", "Kubernetes version") + flag.StringVar(&(dataString.Destroy), "destroy", "false", "Whether to run in destroy mode (true/false)") flag.StringVar(&(dataString.HelmChartsBranch), "helm_charts_branch", "", "Helm charts branch") flag.StringVar(&(dataString.CloudwatchAgentRepository), "cloudwatch_agent_repository", "", "CloudWatch Agent repository") flag.StringVar(&(dataString.CloudwatchAgentTag), "cloudwatch_agent_tag", "", "CloudWatch Agent tag") @@ -318,6 +321,7 @@ func GetEnvironmentMetaData() *MetaData { metaDataStorage.AmpWorkspaceId = registeredMetaDataStrings.AmpWorkspaceId metaDataStorage.Region = registeredMetaDataStrings.Region metaDataStorage.K8sVersion = registeredMetaDataStrings.K8sVersion + metaDataStorage.Destroy = registeredMetaDataStrings.Destroy == "true" metaDataStorage.HelmChartsBranch = registeredMetaDataStrings.HelmChartsBranch metaDataStorage.CloudwatchAgentRepository = registeredMetaDataStrings.CloudwatchAgentRepository metaDataStorage.CloudwatchAgentTag = registeredMetaDataStrings.CloudwatchAgentTag diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 064e0b21b..470dc0b65 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -39,10 +39,10 @@ resource "aws_eks_node_group" "this" { min_size = var.nodes } - ami_type = "AL2_x86_64" + ami_type = var.ami_type capacity_type = "ON_DEMAND" disk_size = 20 - instance_types = ["t3a.medium"] + instance_types = [var.instance_type] depends_on = [ aws_iam_role_policy_attachment.node_CloudWatchAgentServerPolicy, @@ -109,6 +109,12 @@ resource "null_resource" "helm_charts" { resource "null_resource" "validator" { depends_on = [aws_eks_cluster.this, aws_eks_node_group.this, null_resource.helm_charts] + triggers = { + cluster_name = aws_eks_cluster.this.name + region = var.region + test_dir = var.test_dir + } + provisioner "local-exec" { command = <<-EOT echo "Validating K8s resources and metrics" @@ -134,4 +140,17 @@ resource "null_resource" "validator" { -sample_app="${var.test_dir}/${var.sample_app}" EOT } + + provisioner "local-exec" { + when = destroy + command = <<-EOT + echo "Running cleanup before destruction" + go test -timeout 15m -v ${self.triggers.test_dir} \ + -destroy=true \ + -region=${self.triggers.region} \ + -eksClusterName=${self.triggers.cluster_name} \ + -computeType=EKS \ + -eksDeploymentStrategy=DAEMON + EOT + } } diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index ac0d38ebc..f3f26f61e 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -21,6 +21,16 @@ variable "nodes" { default = 1 } +variable "ami_type" { + type = string + default = "AL2_x86_64" +} + +variable "instance_type" { + type = string + default = "t3a.medium" +} + variable "helm_charts_branch" { type = string default = "main" diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index b959077f9..3c070c36b 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -22,18 +22,13 @@ import ( "github.com/aws/amazon-cloudwatch-agent-test/environment" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" ) func init() { environment.RegisterEnvironmentMetaDataFlags() } -const ( - NAMESPACE_JVM_TOMCAT = "JVM_TOMCAT_E2E" - NAMESPACE_KAFKA = "KAFKA_E2E" - NAMESPACE_CONTAINERINSIGHTS = "ContainerInsights/Prometheus" -) - var testRegistry = map[string][]func(*testing.T){ "jvm_tomcat.json": { testTomcatMetrics, @@ -57,83 +52,32 @@ func TestMain(m *testing.M) { } env := environment.GetEnvironmentMetaData() - if env.Region != "us-west-2" { - if err := awsservice.ConfigureAWSClients(env.Region); err != nil { - fmt.Printf("Failed to reconfigure AWS clients: %v\n", err) + if env.Destroy { + if err := common.DestroyResources(env); err != nil { + fmt.Printf("Failed to delete helm resources: %v\n", err) os.Exit(1) } - fmt.Printf("AWS clients reconfigured to use region: %s\n", env.Region) - } else { - fmt.Printf("Using default testing region: us-west-2\n") + os.Exit(0) } - fmt.Println("Starting Helm installation...") - if err := ApplyHelm(env); err != nil { - fmt.Printf("Failed to apply Helm: %v\n", err) + if err := common.InitializeEnvironment(env); err != nil { + fmt.Printf("Failed to initialize environment: %v\n", err) os.Exit(1) } - fmt.Println("Waiting for metrics to propagate...") - time.Sleep(5 * time.Minute) - - os.Exit(m.Run()) -} - -func ApplyHelm(env *environment.MetaData) error { - updateKubeconfig := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName) - output, err := updateKubeconfig.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to update kubeconfig: %w\nOutput: %s", err, output) - } - - helm := []string{ - "helm", "upgrade", "--install", "amazon-cloudwatch-observability", - filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability"), - "--set", fmt.Sprintf("clusterName=%s", env.EKSClusterName), - "--set", fmt.Sprintf("region=%s", env.Region), - "--set", fmt.Sprintf("agent.image.repository=%s", env.CloudwatchAgentRepository), - "--set", fmt.Sprintf("agent.image.tag=%s", env.CloudwatchAgentTag), - "--set", fmt.Sprintf("agent.image.repositoryDomainMap.public=%s", env.CloudwatchAgentRepositoryURL), - "--set", fmt.Sprintf("manager.image.repository=%s", env.CloudwatchAgentOperatorRepository), - "--set", fmt.Sprintf("manager.image.tag=%s", env.CloudwatchAgentOperatorTag), - "--set", fmt.Sprintf("manager.image.repositoryDomainMap.public=%s", env.CloudwatchAgentOperatorRepositoryURL), - "--namespace", "amazon-cloudwatch", - "--create-namespace", + eksInstances, err := awsservice.GetEKSInstances(env.EKSClusterName) + if err != nil || len(eksInstances) == 0 { + fmt.Printf("Failed to get EKS instances: %v", err) + os.Exit(1) } - if env.AgentConfig != "" { - agentConfigContent, err := os.ReadFile(env.AgentConfig) - if err != nil { - return fmt.Errorf("failed to read agent config file: %w", err) + for _, instance := range eksInstances { + if instance.InstanceName != nil { + nodeNames = append(nodeNames, *instance.InstanceName) } - helm = append(helm, "--set-json", fmt.Sprintf("agent.config=%s", string(agentConfigContent))) } - helmUpgrade := exec.Command(helm[0], helm[1:]...) - helmUpgrade.Stdout = os.Stdout - helmUpgrade.Stderr = os.Stderr - if err := helmUpgrade.Run(); err != nil { - return fmt.Errorf("failed to install Helm release: %w", err) - } - - fmt.Println("Waiting for CloudWatch Agent Operator to initialize...") - time.Sleep(300 * time.Second) - - deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml") - - apply := exec.Command("kubectl", "apply", "-f", env.SampleApp) - output, err = apply.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to apply sample app: %w\nOutput: %s", err, output) - } - - wait := exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=300s", fmt.Sprintf("deployment/%s", deploymentName), "-n", "default") - output, err = wait.CombinedOutput() - if err != nil { - return fmt.Errorf("failed to wait for deployment %s: %w\nOutput: %s", deploymentName, err, output) - } - - return nil + os.Exit(m.Run()) } func TestResources(t *testing.T) { @@ -141,14 +85,7 @@ func TestResources(t *testing.T) { require.NoError(t, err, "Error building kubeconfig") clientset, err := kubernetes.NewForConfig(config) - require.NoError(t, err, "Error building kubeconfig") - - nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - require.NoError(t, err, "Error listing nodes") - - for _, node := range nodes.Items { - nodeNames = append(nodeNames, node.Name) - } + require.NoError(t, err, "Error creating clientset") daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) require.NoError(t, err, "Error getting CloudWatch Agent DaemonSet") @@ -158,6 +95,10 @@ func TestResources(t *testing.T) { require.NoError(t, err, "Error getting CloudWatch Agent ConfigMap") require.NotNil(t, configMap, "CloudWatch Agent ConfigMap not found") + cwConfig, exists := configMap.Data["cwagentconfig.json"] + require.True(t, exists, "cwagentconfig.json not found in ConfigMap") + require.Contains(t, cwConfig, `"jmx"`, "JMX configuration not found in cwagentconfig.json") + service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) require.NoError(t, err, "Error getting CloudWatch Agent Service") require.NotNil(t, service, "CloudWatch Agent Service not found") @@ -222,14 +163,10 @@ func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("Error getting LoadBalancer URL: %v", err) - } + require.NoError(t, err, "Error getting LoadBalancer URL") lbURL := strings.Trim(string(output), "'") - if lbURL == "" { - t.Fatal("LoadBalancer URL failed to format") - } + require.NotEmpty(t, lbURL, "LoadBalancer URL failed to format") for i := 0; i < 5; i++ { resp, err := http.Get(fmt.Sprintf("http://%s/webapp/index.jsp", lbURL)) @@ -237,11 +174,7 @@ func testTomcatSessions(t *testing.T) { t.Logf("Request attempt %d failed: %v", i+1, err) continue } - err = resp.Body.Close() - if err != nil { - t.Errorf("Failed to close response body: %v", err) - return - } + require.NoError(t, resp.Body.Close(), "Failed to close response body") } time.Sleep(5 * time.Minute) @@ -251,27 +184,14 @@ func testTomcatSessions(t *testing.T) { aboveZero, err := awsservice.CheckMetricAboveZero( "tomcat.sessions", - NAMESPACE_JVM_TOMCAT, + "JVM_TOMCAT_E2E", startTime, endTime, 60, nodeNames, ) - if err != nil { - t.Errorf("Failed to check metric above zero: %v", err) - return - } - - if !aboveZero { - t.Error("Expected non-zero tomcat.sessions after applying traffic") - } - - deleteCmd := exec.Command("kubectl", "delete", "svc", "tomcat-service") - if output, err := deleteCmd.CombinedOutput(); err != nil { - t.Logf("Warning: Failed to delete load balancer service: %v\nOutput: %s", err, output) - } else { - t.Log("Successfully deleted load balancer service") - } + require.NoError(t, err, "Failed to check metric above zero") + require.True(t, aboveZero, "Expected non-zero tomcat.sessions after applying traffic") }) } @@ -285,7 +205,7 @@ func testKafkaMetrics(t *testing.T) { for _, metric := range metricsToCheck { t.Run(metric, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric, NAMESPACE_KAFKA, nil, 5, 1*time.Minute) + awsservice.ValidateMetricWithTest(t, metric, "KAFKA_E2E", nil, 5, 1*time.Minute) }) } }) @@ -316,7 +236,7 @@ func testContainerInsightsMetrics(t *testing.T) { for _, metric := range metricsToCheck { t.Run(metric, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric, NAMESPACE_CONTAINERINSIGHTS, nil, 5, 1*time.Minute) + awsservice.ValidateMetricWithTest(t, metric, "ContainerInsights/Prometheus", nil, 5, 1*time.Minute) }) } }) @@ -326,14 +246,10 @@ func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("Error getting LoadBalancer URL: %v", err) - } + require.NoError(t, err, "Error getting LoadBalancer URL") lbURL := strings.Trim(string(output), "'") - if lbURL == "" { - t.Fatal("LoadBalancer URL failed to format") - } + require.NotEmpty(t, lbURL, "LoadBalancer URL failed to format") for i := 0; i < 5; i++ { resp, err := http.Get(fmt.Sprintf("http://%s/webapp/index.jsp", lbURL)) @@ -341,11 +257,7 @@ func testTomcatRejectedSessions(t *testing.T) { t.Logf("Request attempt %d failed: %v", i+1, err) continue } - err = resp.Body.Close() - if err != nil { - t.Errorf("Failed to close response body: %v", err) - return - } + require.NoError(t, resp.Body.Close(), "Failed to close response body") } time.Sleep(5 * time.Minute) @@ -355,26 +267,13 @@ func testTomcatRejectedSessions(t *testing.T) { aboveZero, err := awsservice.CheckMetricAboveZero( "catalina_manager_rejectedsessions", - NAMESPACE_CONTAINERINSIGHTS, + "ContainerInsights/Prometheus", startTime, endTime, 60, nodeNames, ) - if err != nil { - t.Errorf("Failed to check metric above zero: %v", err) - return - } - - if !aboveZero { - t.Error("Expected non-zero catalina_manager_rejectedsessions after applying traffic") - } - - deleteCmd := exec.Command("kubectl", "delete", "svc", "tomcat-service") - if output, err := deleteCmd.CombinedOutput(); err != nil { - t.Logf("Warning: Failed to delete load balancer service: %v\nOutput: %s", err, output) - } else { - t.Log("Successfully deleted load balancer service") - } + require.NoError(t, err, "Failed to check metric above zero") + require.True(t, aboveZero, "Expected non-zero catalina_manager_rejectedsessions after applying traffic") }) } diff --git a/test/e2e/jmx/resources/cwagent_configs/kafka.json b/test/e2e/jmx/resources/cwagent_configs/kafka.json index 8b16add8c..dce338b31 100644 --- a/test/e2e/jmx/resources/cwagent_configs/kafka.json +++ b/test/e2e/jmx/resources/cwagent_configs/kafka.json @@ -7,7 +7,37 @@ "measurement": [ "kafka.consumer.total.records-consumed-rate", "kafka.consumer.total.bytes-consumed-rate", - "kafka.consumer.fetch-rate" + "kafka.consumer.fetch-rate", + "kafka.consumer.records-lag-max", + "kafka.consumer.total.fetch-size-avg", + "kafka.consumer.bytes-consumed-rate", + "kafka.consumer.fetch-size-avg", + "kafka.consumer.records-consumed-rate" + ] + }, + "kafka-producer": { + "measurement": [ + "kafka.producer.io-wait-time-ns-avg", + "kafka.producer.outgoing-byte-rate", + "kafka.producer.request-latency-avg", + "kafka.producer.request-rate", + "kafka.producer.response-rate", + "kafka.producer.byte-rate", + "kafka.producer.compression-rate", + "kafka.producer.record-error-rate", + "kafka.producer.record-retry-rate", + "kafka.producer.record-send-rate" + ] + }, + "kafka": { + "measurement": [ + "kafka.message.count", + "kafka.request.count", + "kafka.request.failed", + "kafka.request.time.total", + "kafka.request.time.50p", + "kafka.request.time.99p", + "kafka.request.time.avg" ] } } diff --git a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml b/test/e2e/jmx/resources/sample_apps/kafka.yaml similarity index 78% rename from test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml rename to test/e2e/jmx/resources/sample_apps/kafka.yaml index 7e27fd9bb..8c3588613 100644 --- a/test/e2e/jmx/resources/sample_apps/kafka-deployment.yaml +++ b/test/e2e/jmx/resources/sample_apps/kafka.yaml @@ -1,7 +1,13 @@ apiVersion: v1 +kind: Namespace +metadata: + name: test +--- +apiVersion: v1 kind: Service metadata: name: zookeeper-service + namespace: test spec: selector: app: zookeeper @@ -12,7 +18,8 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: zookeeper-deployment + name: zookeeper + namespace: test spec: replicas: 1 selector: @@ -25,7 +32,7 @@ spec: spec: containers: - name: zookeeper - image: wurstmeister/zookeeper:latest + image: public.ecr.aws/l9b8e0i6/zookeeper:latest ports: - containerPort: 2181 env: @@ -36,6 +43,7 @@ apiVersion: v1 kind: Service metadata: name: kafka-service + namespace: test spec: selector: app: kafka @@ -46,7 +54,8 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: kafka-deployment + name: kafka + namespace: test spec: replicas: 1 selector: @@ -56,10 +65,13 @@ spec: metadata: labels: app: kafka + annotations: + instrumentation.opentelemetry.io/inject-java: "true" + cloudwatch.aws.amazon.com/inject-jmx-kafka: "true" spec: containers: - name: kafka - image: wurstmeister/kafka:latest + image: public.ecr.aws/l9b8e0i6/kafka:latest ports: - containerPort: 9092 env: @@ -76,6 +88,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: kafka-producer + namespace: test spec: replicas: 1 selector: @@ -85,10 +98,13 @@ spec: metadata: labels: app: kafka-producer + annotations: + instrumentation.opentelemetry.io/inject-java: "true" + cloudwatch.aws.amazon.com/inject-jmx-kafka-producer: "true" spec: containers: - name: kafka-producer - image: wurstmeister/kafka:latest + image: public.ecr.aws/l9b8e0i6/kafka:latest command: ["/bin/sh"] args: ["-c", "while true; do echo 'Producing message'; kafka-console-producer.sh --broker-list kafka-service:9092 --topic test-topic | 'Test message'; sleep 5; done"] env: @@ -99,6 +115,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: kafka-consumer + namespace: test spec: replicas: 1 selector: @@ -114,7 +131,7 @@ spec: spec: containers: - name: kafka-consumer - image: wurstmeister/kafka:latest + image: public.ecr.aws/l9b8e0i6/kafka:latest command: ["/bin/sh"] args: ["-c", "kafka-console-consumer.sh --bootstrap-server kafka-service:9092 --topic test-topic --from-beginning"] env: diff --git a/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml b/test/e2e/jmx/resources/sample_apps/tomcat.yaml similarity index 79% rename from test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml rename to test/e2e/jmx/resources/sample_apps/tomcat.yaml index 9fa072137..d152f00ca 100644 --- a/test/e2e/jmx/resources/sample_apps/tomcat-deployment.yaml +++ b/test/e2e/jmx/resources/sample_apps/tomcat.yaml @@ -1,7 +1,13 @@ apiVersion: v1 +kind: Namespace +metadata: + name: test +--- +apiVersion: v1 kind: Service metadata: name: tomcat-service + namespace: test spec: type: LoadBalancer selector: @@ -13,7 +19,8 @@ spec: apiVersion: apps/v1 kind: Deployment metadata: - name: tomcat-deployment + name: tomcat + namespace: test spec: replicas: 1 selector: @@ -30,6 +37,6 @@ spec: spec: containers: - name: tomcat-container - image: public.ecr.aws/o5o6y1v9/tomcat:latest + image: public.ecr.aws/l9b8e0i6/tomcat:latest ports: - containerPort: 8080 \ No newline at end of file diff --git a/util/common/kubernetes.go b/util/common/kubernetes.go new file mode 100644 index 000000000..2d8ae0267 --- /dev/null +++ b/util/common/kubernetes.go @@ -0,0 +1,131 @@ +package common + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" +) + +func InitializeEnvironment(env *environment.MetaData) error { + if env.Region != "us-west-2" { + if err := awsservice.ConfigureAWSClients(env.Region); err != nil { + return fmt.Errorf("failed to reconfigure AWS clients: %v", err) + } + fmt.Printf("AWS clients reconfigured to use region: %s\n", env.Region) + } else { + fmt.Printf("Using default testing region: us-west-2\n") + } + + fmt.Println("Starting Helm installation...") + if err := ApplyResources(env); err != nil { + return fmt.Errorf("failed to apply Helm: %v", err) + } + + fmt.Println("Waiting for metrics to propagate...") + time.Sleep(5 * time.Minute) + + return nil +} + +func ApplyResources(env *environment.MetaData) error { + updateKubeconfig := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName) + output, err := updateKubeconfig.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to update kubeconfig: %w\nOutput: %s", err, output) + } + + helm := []string{ + "helm", "upgrade", "--install", "amazon-cloudwatch-observability", + filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability"), + "--values", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability", "values.yaml"), + "--set", fmt.Sprintf("clusterName=%s", env.EKSClusterName), + "--set", fmt.Sprintf("region=%s", env.Region), + "--set", fmt.Sprintf("agent.image.repository=%s", env.CloudwatchAgentRepository), + "--set", fmt.Sprintf("agent.image.tag=%s", env.CloudwatchAgentTag), + "--set", fmt.Sprintf("agent.image.repositoryDomainMap.public=%s", env.CloudwatchAgentRepositoryURL), + "--set", fmt.Sprintf("manager.image.repository=%s", env.CloudwatchAgentOperatorRepository), + "--set", fmt.Sprintf("manager.image.tag=%s", env.CloudwatchAgentOperatorTag), + "--set", fmt.Sprintf("manager.image.repositoryDomainMap.public=%s", env.CloudwatchAgentOperatorRepositoryURL), + "--namespace", "amazon-cloudwatch", + "--create-namespace", + } + + if env.AgentConfig != "" { + agentConfigContent, err := os.ReadFile(env.AgentConfig) + if err != nil { + return fmt.Errorf("failed to read agent config file: %w", err) + } + helm = append(helm, "--set-json", fmt.Sprintf("agent.config=%s", string(agentConfigContent))) + } + + helmUpgrade := exec.Command(helm[0], helm[1:]...) + helmUpgrade.Stdout = os.Stdout + helmUpgrade.Stderr = os.Stderr + if err := helmUpgrade.Run(); err != nil { + return fmt.Errorf("failed to install Helm release: %w", err) + } + + fmt.Println("Waiting for CloudWatch Agent Operator to initialize...") + wait := exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=60s", "deployment/amazon-cloudwatch-observability-controller-manager", "-n", "amazon-cloudwatch") + output, err = wait.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to wait for operator deployment: %w\nOutput: %s", err, output) + } + + deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml") + + apply := exec.Command("kubectl", "apply", "-f", env.SampleApp) + output, err = apply.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to apply sample app: %w\nOutput: %s", err, output) + } + + fmt.Println("Waiting for Sample Application to initialize...") + wait = exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=60s", fmt.Sprintf("deployment/%s", deploymentName), "-n", "test") + output, err = wait.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to wait for deployment %s: %w\nOutput: %s", deploymentName, err, output) + } + + return nil +} + +func DestroyResources(env *environment.MetaData) error { + updateKubeconfig := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName) + output, err := updateKubeconfig.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to update kubeconfig: %w\nOutput: %s", err, output) + } + + var errors []error + + deleteCmd := exec.Command("kubectl", "delete", "namespace", "test", "--timeout=60s") + output, err = deleteCmd.CombinedOutput() + // Note that we don't want to consider not finding the namespace to be an error since that's the outcome we want. + if err != nil && !strings.Contains(string(output), "not found") { + errors = append(errors, fmt.Errorf("failed to delete test namespace: %w\nOutput: %s", err, output)) + } + + helm := []string{ + "helm", "uninstall", "amazon-cloudwatch-observability", "--namespace", "amazon-cloudwatch", + } + + helmUninstall := exec.Command(helm[0], helm[1:]...) + helmUninstall.Stdout = os.Stdout + helmUninstall.Stderr = os.Stderr + if err := helmUninstall.Run(); err != nil { + errors = append(errors, fmt.Errorf("failed to uninstall Helm release: %w", err)) + } + + if len(errors) > 0 { + return fmt.Errorf("cleanup errors: %v", errors) + } + + return nil +} From 917a6067f183e181198b6b01f4e482ee5eebade2 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Thu, 19 Dec 2024 03:28:39 -0500 Subject: [PATCH 59/92] change generator --- generator/test_case_generator.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 696d4ddd5..f5887186e 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -248,12 +248,9 @@ var testTypeToTestConfig = map[string][]testConfig{ } var testTypeToTestConfigE2E = map[string][]testConfig{ - "eks_e2e_jmx": { + "eks_e2e": { {testDir: "../../../test/e2e/jmx"}, }, - "eks_e2e_prometheus": { - {testDir: "../../../test/e2e/prometheus"}, - }, } type partition struct { From 975168c026fb41625b1b3cff942b189271ad0294 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Thu, 19 Dec 2024 08:30:59 -0500 Subject: [PATCH 60/92] clean --- ...trix.json => eks_e2e_jmx_test_matrix.json} | 0 generator/test_case_generator.go | 2 +- terraform/eks/e2e/main.tf | 2 +- test/e2e/jmx/jmx_test.go | 39 +++++++++++++++---- .../jmx/resources/cwagent_configs/kafka.json | 37 +++++++----------- test/e2e/jmx/resources/sample_apps/kafka.yaml | 12 ++---- 6 files changed, 49 insertions(+), 43 deletions(-) rename generator/resources/{eks_e2e_test_matrix.json => eks_e2e_jmx_test_matrix.json} (100%) diff --git a/generator/resources/eks_e2e_test_matrix.json b/generator/resources/eks_e2e_jmx_test_matrix.json similarity index 100% rename from generator/resources/eks_e2e_test_matrix.json rename to generator/resources/eks_e2e_jmx_test_matrix.json diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index f5887186e..4daaae1d2 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -248,7 +248,7 @@ var testTypeToTestConfig = map[string][]testConfig{ } var testTypeToTestConfigE2E = map[string][]testConfig{ - "eks_e2e": { + "eks_e2e_jmx": { {testDir: "../../../test/e2e/jmx"}, }, } diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 470dc0b65..f9b91713c 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -144,7 +144,7 @@ resource "null_resource" "validator" { provisioner "local-exec" { when = destroy command = <<-EOT - echo "Running cleanup before destruction" + echo "Running cleanup for K8s resources" go test -timeout 15m -v ${self.triggers.test_dir} \ -destroy=true \ -region=${self.triggers.region} \ diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 3c070c36b..b78ea2f9b 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -80,7 +80,19 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestResources(t *testing.T) { +func TestAll(t *testing.T) { + t.Run("Resources", func(t *testing.T) { + testResources(t) + }) + + if !t.Failed() { + t.Run("Metrics", func(t *testing.T) { + testMetrics(t) + }) + } +} + +func testResources(t *testing.T) { config, err := clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config")) require.NoError(t, err, "Error building kubeconfig") @@ -108,7 +120,7 @@ func TestResources(t *testing.T) { require.NotNil(t, serviceAccount, "CloudWatch Agent Service Account not found") } -func TestMetrics(t *testing.T) { +func testMetrics(t *testing.T) { env := environment.GetEnvironmentMetaData() configFile := filepath.Base(env.AgentConfig) @@ -126,7 +138,6 @@ func TestMetrics(t *testing.T) { func testTomcatMetrics(t *testing.T) { t.Run("verify_jvm_tomcat_metrics", func(t *testing.T) { metricsToCheck := []string{ - "tomcat.traffic", "jvm.classes.loaded", "jvm.gc.collections.count", "jvm.gc.collections.elapsed", @@ -143,6 +154,7 @@ func testTomcatMetrics(t *testing.T) { "jvm.memory.pool.used", "jvm.memory.pool.committed", "jvm.threads.count", + "tomcat.traffic", "tomcat.sessions", "tomcat.errors", "tomcat.request_count", @@ -153,7 +165,7 @@ func testTomcatMetrics(t *testing.T) { for _, metric := range metricsToCheck { t.Run(metric, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric, metric, nil, 5, 1*time.Minute) + awsservice.ValidateMetricWithTest(t, metric, "JVM_TOMCAT_E2E", nil, 5, 30*time.Second) }) } }) @@ -161,7 +173,7 @@ func testTomcatMetrics(t *testing.T) { func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { - cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") + cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-n", "test", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") output, err := cmd.CombinedOutput() require.NoError(t, err, "Error getting LoadBalancer URL") @@ -198,6 +210,17 @@ func testTomcatSessions(t *testing.T) { func testKafkaMetrics(t *testing.T) { t.Run("verify_kafka_metrics", func(t *testing.T) { metricsToCheck := []string{ + "kafka.message.count", + "kafka.request.count", + "kafka.request.failed", + "kafka.request.time.total", + "kafka.request.time.50p", + "kafka.request.time.99p", + "kafka.request.time.avg", + "kafka.producer.io-wait-time-ns-avg", + "kafka.producer.outgoing-byte-rate", + "kafka.producer.request-rate", + "kafka.producer.response-rate", "kafka.consumer.fetch-rate", "kafka.consumer.total.bytes-consumed-rate", "kafka.consumer.total.records-consumed-rate", @@ -205,7 +228,7 @@ func testKafkaMetrics(t *testing.T) { for _, metric := range metricsToCheck { t.Run(metric, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric, "KAFKA_E2E", nil, 5, 1*time.Minute) + awsservice.ValidateMetricWithTest(t, metric, "KAFKA_E2E", nil, 5, 30*time.Second) }) } }) @@ -236,7 +259,7 @@ func testContainerInsightsMetrics(t *testing.T) { for _, metric := range metricsToCheck { t.Run(metric, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric, "ContainerInsights/Prometheus", nil, 5, 1*time.Minute) + awsservice.ValidateMetricWithTest(t, metric, "ContainerInsights/Prometheus", nil, 5, 30*time.Second) }) } }) @@ -244,7 +267,7 @@ func testContainerInsightsMetrics(t *testing.T) { func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { - cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") + cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-n", "test", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") output, err := cmd.CombinedOutput() require.NoError(t, err, "Error getting LoadBalancer URL") diff --git a/test/e2e/jmx/resources/cwagent_configs/kafka.json b/test/e2e/jmx/resources/cwagent_configs/kafka.json index dce338b31..4fc5c2a82 100644 --- a/test/e2e/jmx/resources/cwagent_configs/kafka.json +++ b/test/e2e/jmx/resources/cwagent_configs/kafka.json @@ -3,41 +3,30 @@ "namespace": "KAFKA_E2E", "metrics_collected": { "jmx": { - "kafka-consumer": { + "kafka": { "measurement": [ - "kafka.consumer.total.records-consumed-rate", - "kafka.consumer.total.bytes-consumed-rate", - "kafka.consumer.fetch-rate", - "kafka.consumer.records-lag-max", - "kafka.consumer.total.fetch-size-avg", - "kafka.consumer.bytes-consumed-rate", - "kafka.consumer.fetch-size-avg", - "kafka.consumer.records-consumed-rate" + "kafka.message.count", + "kafka.request.count", + "kafka.request.failed", + "kafka.request.time.total", + "kafka.request.time.50p", + "kafka.request.time.99p", + "kafka.request.time.avg" ] }, "kafka-producer": { "measurement": [ "kafka.producer.io-wait-time-ns-avg", "kafka.producer.outgoing-byte-rate", - "kafka.producer.request-latency-avg", "kafka.producer.request-rate", - "kafka.producer.response-rate", - "kafka.producer.byte-rate", - "kafka.producer.compression-rate", - "kafka.producer.record-error-rate", - "kafka.producer.record-retry-rate", - "kafka.producer.record-send-rate" + "kafka.producer.response-rate" ] }, - "kafka": { + "kafka-consumer": { "measurement": [ - "kafka.message.count", - "kafka.request.count", - "kafka.request.failed", - "kafka.request.time.total", - "kafka.request.time.50p", - "kafka.request.time.99p", - "kafka.request.time.avg" + "kafka.consumer.total.records-consumed-rate", + "kafka.consumer.total.bytes-consumed-rate", + "kafka.consumer.fetch-rate" ] } } diff --git a/test/e2e/jmx/resources/sample_apps/kafka.yaml b/test/e2e/jmx/resources/sample_apps/kafka.yaml index 8c3588613..8e1680f1d 100644 --- a/test/e2e/jmx/resources/sample_apps/kafka.yaml +++ b/test/e2e/jmx/resources/sample_apps/kafka.yaml @@ -80,7 +80,7 @@ spec: - name: KAFKA_LISTENERS value: "PLAINTEXT://:9092" - name: KAFKA_ADVERTISED_LISTENERS - value: "PLAINTEXT://:9092" + value: "PLAINTEXT://kafka-service:9092" - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR value: "1" --- @@ -106,10 +106,7 @@ spec: - name: kafka-producer image: public.ecr.aws/l9b8e0i6/kafka:latest command: ["/bin/sh"] - args: ["-c", "while true; do echo 'Producing message'; kafka-console-producer.sh --broker-list kafka-service:9092 --topic test-topic | 'Test message'; sleep 5; done"] - env: - - name: KAFKA_BROKER_LIST - value: "kafka-service:9092" + args: ["-c", "while true; do echo 'Test message' | kafka-console-producer.sh --broker-list kafka-service:9092 --topic test-topic; sleep 5; done"] --- apiVersion: apps/v1 kind: Deployment @@ -133,7 +130,4 @@ spec: - name: kafka-consumer image: public.ecr.aws/l9b8e0i6/kafka:latest command: ["/bin/sh"] - args: ["-c", "kafka-console-consumer.sh --bootstrap-server kafka-service:9092 --topic test-topic --from-beginning"] - env: - - name: KAFKA_BROKER_LIST - value: "kafka-service:9092" \ No newline at end of file + args: ["-c", "kafka-console-consumer.sh --bootstrap-server kafka-service:9092 --topic test-topic --from-beginning"] \ No newline at end of file From 28e5c5521cd60ff5e18667b129c794ecb20007de Mon Sep 17 00:00:00 2001 From: musa-asad Date: Thu, 19 Dec 2024 20:57:59 -0500 Subject: [PATCH 61/92] update sample --- test/e2e/jmx/resources/sample_apps/kafka.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/jmx/resources/sample_apps/kafka.yaml b/test/e2e/jmx/resources/sample_apps/kafka.yaml index 8e1680f1d..35de6cbff 100644 --- a/test/e2e/jmx/resources/sample_apps/kafka.yaml +++ b/test/e2e/jmx/resources/sample_apps/kafka.yaml @@ -106,7 +106,7 @@ spec: - name: kafka-producer image: public.ecr.aws/l9b8e0i6/kafka:latest command: ["/bin/sh"] - args: ["-c", "while true; do echo 'Test message' | kafka-console-producer.sh --broker-list kafka-service:9092 --topic test-topic; sleep 5; done"] + args: ["-c", "while true; do echo 'message' | kafka-console-producer.sh --topic quickstart-events --bootstrap-server kafka-service:9092; sleep 5; done"] --- apiVersion: apps/v1 kind: Deployment @@ -130,4 +130,4 @@ spec: - name: kafka-consumer image: public.ecr.aws/l9b8e0i6/kafka:latest command: ["/bin/sh"] - args: ["-c", "kafka-console-consumer.sh --bootstrap-server kafka-service:9092 --topic test-topic --from-beginning"] \ No newline at end of file + args: ["-c", "kafka-console-consumer.sh --topic quickstart-events --from-beginning --bootstrap-server kafka-service:9092"] \ No newline at end of file From 8bf45174914eb1d4ce64d30f4fe7b7b386a0bd55 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Thu, 19 Dec 2024 21:42:48 -0500 Subject: [PATCH 62/92] fix kafka --- environment/metadata.go | 2 +- terraform/eks/e2e/main.tf | 6 +- terraform/eks/e2e/variables.tf | 4 +- test/e2e/jmx/jmx_test.go | 61 ++++++++++++------- test/e2e/jmx/resources/sample_apps/kafka.yaml | 37 ++++++++++- util/common/kubernetes.go | 14 ++--- 6 files changed, 87 insertions(+), 37 deletions(-) diff --git a/environment/metadata.go b/environment/metadata.go index 0bf122349..f7c03fc77 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -47,7 +47,7 @@ type MetaData struct { AmpWorkspaceId string Region string K8sVersion string - Destroy bool + Destroy bool HelmChartsBranch string CloudwatchAgentRepository string CloudwatchAgentTag string diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index f9b91713c..8ef9751d3 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -111,8 +111,8 @@ resource "null_resource" "validator" { triggers = { cluster_name = aws_eks_cluster.this.name - region = var.region - test_dir = var.test_dir + region = var.region + test_dir = var.test_dir } provisioner "local-exec" { @@ -142,7 +142,7 @@ resource "null_resource" "validator" { } provisioner "local-exec" { - when = destroy + when = destroy command = <<-EOT echo "Running cleanup for K8s resources" go test -timeout 15m -v ${self.triggers.test_dir} \ diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index f3f26f61e..016bbb318 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -22,12 +22,12 @@ variable "nodes" { } variable "ami_type" { - type = string + type = string default = "AL2_x86_64" } variable "instance_type" { - type = string + type = string default = "t3a.medium" } diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index b78ea2f9b..966dead1d 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -29,7 +29,7 @@ func init() { environment.RegisterEnvironmentMetaDataFlags() } -var testRegistry = map[string][]func(*testing.T){ +var testMetricsRegistry = map[string][]func(*testing.T){ "jvm_tomcat.json": { testTomcatMetrics, testTomcatSessions, @@ -43,6 +43,10 @@ var testRegistry = map[string][]func(*testing.T){ }, } +var testResourcesRegistry = []func(*testing.T){ + testJMXResources, +} + var nodeNames []string func TestMain(m *testing.M) { @@ -54,7 +58,7 @@ func TestMain(m *testing.M) { if env.Destroy { if err := common.DestroyResources(env); err != nil { - fmt.Printf("Failed to delete helm resources: %v\n", err) + fmt.Printf("Failed to delete kubernetes resources: %v\n", err) os.Exit(1) } os.Exit(0) @@ -93,43 +97,56 @@ func TestAll(t *testing.T) { } func testResources(t *testing.T) { - config, err := clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config")) - require.NoError(t, err, "Error building kubeconfig") + tests := testResourcesRegistry + + for _, testFunc := range tests { + testFunc(t) + } +} + +func testJMXResources(t *testing.T) { + t.Run("verify_jmx_resources", func(t *testing.T) { + config, err := clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config")) + require.NoError(t, err, "Error building kubeconfig") - clientset, err := kubernetes.NewForConfig(config) - require.NoError(t, err, "Error creating clientset") + clientset, err := kubernetes.NewForConfig(config) + require.NoError(t, err, "Error creating clientset") - daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent DaemonSet") - require.NotNil(t, daemonSet, "CloudWatch Agent DaemonSet not found") + daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent DaemonSet") + require.NotNil(t, daemonSet, "CloudWatch Agent DaemonSet not found") - configMap, err := clientset.CoreV1().ConfigMaps("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent ConfigMap") - require.NotNil(t, configMap, "CloudWatch Agent ConfigMap not found") + configMap, err := clientset.CoreV1().ConfigMaps("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent ConfigMap") + require.NotNil(t, configMap, "CloudWatch Agent ConfigMap not found") - cwConfig, exists := configMap.Data["cwagentconfig.json"] - require.True(t, exists, "cwagentconfig.json not found in ConfigMap") - require.Contains(t, cwConfig, `"jmx"`, "JMX configuration not found in cwagentconfig.json") + cwConfig, exists := configMap.Data["cwagentconfig.json"] + require.True(t, exists, "cwagentconfig.json not found in ConfigMap") + require.Contains(t, cwConfig, `"jmx"`, "JMX configuration not found in cwagentconfig.json") - service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent Service") - require.NotNil(t, service, "CloudWatch Agent Service not found") + service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent Service") + require.NotNil(t, service, "CloudWatch Agent Service not found") - serviceAccount, err := clientset.CoreV1().ServiceAccounts("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent Service Account") - require.NotNil(t, serviceAccount, "CloudWatch Agent Service Account not found") + serviceAccount, err := clientset.CoreV1().ServiceAccounts("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent Service Account") + require.NotNil(t, serviceAccount, "CloudWatch Agent Service Account not found") + }) } func testMetrics(t *testing.T) { env := environment.GetEnvironmentMetaData() configFile := filepath.Base(env.AgentConfig) - tests, exists := testRegistry[configFile] + tests, exists := testMetricsRegistry[configFile] if !exists { t.Skipf("No tests registered for config file: %s", configFile) return } + fmt.Println("Waiting for metrics to propagate...") + time.Sleep(10 * time.Minute) + for _, testFunc := range tests { testFunc(t) } diff --git a/test/e2e/jmx/resources/sample_apps/kafka.yaml b/test/e2e/jmx/resources/sample_apps/kafka.yaml index 35de6cbff..ba8d57ed1 100644 --- a/test/e2e/jmx/resources/sample_apps/kafka.yaml +++ b/test/e2e/jmx/resources/sample_apps/kafka.yaml @@ -38,6 +38,16 @@ spec: env: - name: ZOOKEEPER_CLIENT_PORT value: "2181" + readinessProbe: + tcpSocket: + port: 2181 + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 2181 + initialDelaySeconds: 10 + periodSeconds: 10 --- apiVersion: v1 kind: Service @@ -69,6 +79,11 @@ spec: instrumentation.opentelemetry.io/inject-java: "true" cloudwatch.aws.amazon.com/inject-jmx-kafka: "true" spec: + initContainers: + - name: wait-for-zookeeper + image: busybox + command: ["/bin/sh"] + args: ["-c", "until nc -z zookeeper-service 2181; do echo waiting for zookeeper; sleep 2; done"] containers: - name: kafka image: public.ecr.aws/l9b8e0i6/kafka:latest @@ -83,6 +98,16 @@ spec: value: "PLAINTEXT://kafka-service:9092" - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR value: "1" + readinessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 30 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 9092 + initialDelaySeconds: 30 + periodSeconds: 10 --- apiVersion: apps/v1 kind: Deployment @@ -102,11 +127,16 @@ spec: instrumentation.opentelemetry.io/inject-java: "true" cloudwatch.aws.amazon.com/inject-jmx-kafka-producer: "true" spec: + initContainers: + - name: wait-for-kafka + image: busybox + command: ["/bin/sh"] + args: ["-c", "until nc -z kafka-service 9092; do echo waiting for kafka; sleep 2; done"] containers: - name: kafka-producer image: public.ecr.aws/l9b8e0i6/kafka:latest command: ["/bin/sh"] - args: ["-c", "while true; do echo 'message' | kafka-console-producer.sh --topic quickstart-events --bootstrap-server kafka-service:9092; sleep 5; done"] + args: ["-c", "while true; do echo message | kafka-console-producer.sh --topic quickstart-events --bootstrap-server kafka-service:9092; sleep 5; done"] --- apiVersion: apps/v1 kind: Deployment @@ -126,6 +156,11 @@ spec: instrumentation.opentelemetry.io/inject-java: "true" cloudwatch.aws.amazon.com/inject-jmx-kafka-consumer: "true" spec: + initContainers: + - name: wait-for-kafka + image: busybox + command: ["/bin/sh"] + args: ["-c", "until nc -z kafka-service 9092; do echo waiting for kafka; sleep 2; done"] containers: - name: kafka-consumer image: public.ecr.aws/l9b8e0i6/kafka:latest diff --git a/util/common/kubernetes.go b/util/common/kubernetes.go index 2d8ae0267..ab3da983f 100644 --- a/util/common/kubernetes.go +++ b/util/common/kubernetes.go @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package common import ( @@ -6,7 +9,6 @@ import ( "os/exec" "path/filepath" "strings" - "time" "github.com/aws/amazon-cloudwatch-agent-test/environment" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" @@ -22,14 +24,11 @@ func InitializeEnvironment(env *environment.MetaData) error { fmt.Printf("Using default testing region: us-west-2\n") } - fmt.Println("Starting Helm installation...") + fmt.Println("Applying K8s resources...") if err := ApplyResources(env); err != nil { - return fmt.Errorf("failed to apply Helm: %v", err) + return fmt.Errorf("failed to apply K8s resources: %v", err) } - fmt.Println("Waiting for metrics to propagate...") - time.Sleep(5 * time.Minute) - return nil } @@ -43,7 +42,6 @@ func ApplyResources(env *environment.MetaData) error { helm := []string{ "helm", "upgrade", "--install", "amazon-cloudwatch-observability", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability"), - "--values", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability", "values.yaml"), "--set", fmt.Sprintf("clusterName=%s", env.EKSClusterName), "--set", fmt.Sprintf("region=%s", env.Region), "--set", fmt.Sprintf("agent.image.repository=%s", env.CloudwatchAgentRepository), @@ -87,7 +85,7 @@ func ApplyResources(env *environment.MetaData) error { } fmt.Println("Waiting for Sample Application to initialize...") - wait = exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=60s", fmt.Sprintf("deployment/%s", deploymentName), "-n", "test") + wait = exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=300s", fmt.Sprintf("deployment/%s", deploymentName), "-n", "test") output, err = wait.CombinedOutput() if err != nil { return fmt.Errorf("failed to wait for deployment %s: %w\nOutput: %s", deploymentName, err, output) From 8e94f7454068b25889f2abcdb62be46d0e9f1a7c Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 01:50:14 -0500 Subject: [PATCH 63/92] minor adjustments --- test/e2e/jmx/jmx_test.go | 11 +++++------ test/e2e/jmx/resources/cwagent_configs/kafka.json | 1 - test/e2e/jmx/resources/sample_apps/kafka.yaml | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 966dead1d..3e8501888 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -48,13 +48,14 @@ var testResourcesRegistry = []func(*testing.T){ } var nodeNames []string +var env *environment.MetaData func TestMain(m *testing.M) { flag.Parse() if flag.Lookup("test.run").Value.String() == "NO_MATCH" { os.Exit(0) } - env := environment.GetEnvironmentMetaData() + env = environment.GetEnvironmentMetaData() if env.Destroy { if err := common.DestroyResources(env); err != nil { @@ -135,7 +136,6 @@ func testJMXResources(t *testing.T) { } func testMetrics(t *testing.T) { - env := environment.GetEnvironmentMetaData() configFile := filepath.Base(env.AgentConfig) tests, exists := testMetricsRegistry[configFile] @@ -234,13 +234,12 @@ func testKafkaMetrics(t *testing.T) { "kafka.request.time.50p", "kafka.request.time.99p", "kafka.request.time.avg", - "kafka.producer.io-wait-time-ns-avg", - "kafka.producer.outgoing-byte-rate", - "kafka.producer.request-rate", - "kafka.producer.response-rate", "kafka.consumer.fetch-rate", "kafka.consumer.total.bytes-consumed-rate", "kafka.consumer.total.records-consumed-rate", + "kafka.producer.io-wait-time-ns-avg", + "kafka.producer.outgoing-byte-rate", + "kafka.producer.response-rate", } for _, metric := range metricsToCheck { diff --git a/test/e2e/jmx/resources/cwagent_configs/kafka.json b/test/e2e/jmx/resources/cwagent_configs/kafka.json index 4fc5c2a82..d70fb47f0 100644 --- a/test/e2e/jmx/resources/cwagent_configs/kafka.json +++ b/test/e2e/jmx/resources/cwagent_configs/kafka.json @@ -18,7 +18,6 @@ "measurement": [ "kafka.producer.io-wait-time-ns-avg", "kafka.producer.outgoing-byte-rate", - "kafka.producer.request-rate", "kafka.producer.response-rate" ] }, diff --git a/test/e2e/jmx/resources/sample_apps/kafka.yaml b/test/e2e/jmx/resources/sample_apps/kafka.yaml index ba8d57ed1..1ef7fcaaa 100644 --- a/test/e2e/jmx/resources/sample_apps/kafka.yaml +++ b/test/e2e/jmx/resources/sample_apps/kafka.yaml @@ -136,7 +136,7 @@ spec: - name: kafka-producer image: public.ecr.aws/l9b8e0i6/kafka:latest command: ["/bin/sh"] - args: ["-c", "while true; do echo message | kafka-console-producer.sh --topic quickstart-events --bootstrap-server kafka-service:9092; sleep 5; done"] + args: ["-c", "yes | nohup kafka-console-producer.sh --topic quickstart-events --bootstrap-server kafka-service:9092"] --- apiVersion: apps/v1 kind: Deployment From 2fbb559f2c362a3ea1e12dc5e020aac17c756ca6 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 02:39:03 -0500 Subject: [PATCH 64/92] check annotations --- test/e2e/jmx/jmx_test.go | 69 ++++++++++++++++++++++++++++++--------- util/common/kubernetes.go | 1 - 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 3e8501888..ee5ef23ac 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -43,7 +43,8 @@ var testMetricsRegistry = map[string][]func(*testing.T){ }, } -var testResourcesRegistry = []func(*testing.T){ +var testResourcesRegistry = []func(*testing.T, *kubernetes.Clientset){ + testAgentResources, testJMXResources, } @@ -100,19 +101,19 @@ func TestAll(t *testing.T) { func testResources(t *testing.T) { tests := testResourcesRegistry + config, err := clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config")) + require.NoError(t, err, "Error building kubeconfig") + + clientset, err := kubernetes.NewForConfig(config) + require.NoError(t, err, "Error creating clientset") + for _, testFunc := range tests { - testFunc(t) + testFunc(t, clientset) } } -func testJMXResources(t *testing.T) { - t.Run("verify_jmx_resources", func(t *testing.T) { - config, err := clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config")) - require.NoError(t, err, "Error building kubeconfig") - - clientset, err := kubernetes.NewForConfig(config) - require.NoError(t, err, "Error creating clientset") - +func testAgentResources(t *testing.T, clientset *kubernetes.Clientset) { + t.Run("verify_agent_resources", func(t *testing.T) { daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) require.NoError(t, err, "Error getting CloudWatch Agent DaemonSet") require.NotNil(t, daemonSet, "CloudWatch Agent DaemonSet not found") @@ -135,14 +136,50 @@ func testJMXResources(t *testing.T) { }) } +func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) { + t.Run("verify_jmx_resources", func(t *testing.T) { + deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml") + pods, err := clientset.CoreV1().Pods("test").List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", deploymentName), + FieldSelector: "status.phase=Running", + }) + require.NoError(t, err, "Error getting pods for deployment") + require.NotEmpty(t, pods.Items, "No pods found for deployment") + + var jmxTargetSystem string + switch filepath.Base(env.AgentConfig) { + case "jvm_tomcat.json", "containerinsights.json": + jmxTargetSystem = "jvm,tomcat" + case "kafka.json": + jmxTargetSystem = "kafka" + } + + requiredEnvVars := map[string]string{ + "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf", + "OTEL_METRICS_EXPORTER": "none", + "OTEL_LOGS_EXPORTER": "none", + "OTEL_TRACES_EXPORTER": "none", + "OTEL_AWS_JMX_EXPORTER_METRICS_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4314/v1/metrics", + "OTEL_JMX_TARGET_SYSTEM": jmxTargetSystem, + "JAVA_TOOL_OPTIONS": "-javaagent:/otel-auto-instrumentation-java/javaagent.jar", + } + + for _, container := range pods.Items[0].Spec.Containers { + for _, envVar := range container.Env { + if expectedValue, exists := requiredEnvVars[envVar.Name]; exists { + require.Equal(t, expectedValue, envVar.Value, fmt.Sprintf("Unexpected value for environment variable %s in container %s", envVar.Name, container.Name)) + delete(requiredEnvVars, envVar.Name) + } + } + } + + require.Empty(t, requiredEnvVars, "Not all required environment variables were found in the pod") + }) +} + func testMetrics(t *testing.T) { configFile := filepath.Base(env.AgentConfig) - - tests, exists := testMetricsRegistry[configFile] - if !exists { - t.Skipf("No tests registered for config file: %s", configFile) - return - } + tests := testMetricsRegistry[configFile] fmt.Println("Waiting for metrics to propagate...") time.Sleep(10 * time.Minute) diff --git a/util/common/kubernetes.go b/util/common/kubernetes.go index ab3da983f..4274ee082 100644 --- a/util/common/kubernetes.go +++ b/util/common/kubernetes.go @@ -77,7 +77,6 @@ func ApplyResources(env *environment.MetaData) error { } deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml") - apply := exec.Command("kubectl", "apply", "-f", env.SampleApp) output, err = apply.CombinedOutput() if err != nil { From 4360691135bfec0c5d9707b4a30e5ab9f387d99d Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 03:40:30 -0500 Subject: [PATCH 65/92] clean up --- test/e2e/jmx/jmx_test.go | 209 ++++++++++++++++++++------------------ util/common/kubernetes.go | 14 ++- 2 files changed, 122 insertions(+), 101 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index ee5ef23ac..7a36d2ddb 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -25,9 +25,23 @@ import ( "github.com/aws/amazon-cloudwatch-agent-test/util/common" ) -func init() { - environment.RegisterEnvironmentMetaDataFlags() -} +//------------------------------------------------------------------------------ +// Constants and Variables +//------------------------------------------------------------------------------ + +const ( + wait = 5 * time.Minute + interval = 30 * time.Second +) + +var ( + nodeNames []string + env *environment.MetaData +) + +//------------------------------------------------------------------------------ +// Test Registry Maps +//------------------------------------------------------------------------------ var testMetricsRegistry = map[string][]func(*testing.T){ "jvm_tomcat.json": { @@ -48,16 +62,24 @@ var testResourcesRegistry = []func(*testing.T, *kubernetes.Clientset){ testJMXResources, } -var nodeNames []string -var env *environment.MetaData +//------------------------------------------------------------------------------ +// Test Setup +//------------------------------------------------------------------------------ + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} func TestMain(m *testing.M) { flag.Parse() + + // Added this to prevent running tests when we pass in "NO_MATCH" if flag.Lookup("test.run").Value.String() == "NO_MATCH" { os.Exit(0) } env = environment.GetEnvironmentMetaData() + // Destroy K8s resources if terraform destroy if env.Destroy { if err := common.DestroyResources(env); err != nil { fmt.Printf("Failed to delete kubernetes resources: %v\n", err) @@ -66,11 +88,13 @@ func TestMain(m *testing.M) { os.Exit(0) } + // Configure AWS clients and create K8s resources if err := common.InitializeEnvironment(env); err != nil { fmt.Printf("Failed to initialize environment: %v\n", err) os.Exit(1) } + // Get names of nodes so they can be used as dimensions to check for metrics eksInstances, err := awsservice.GetEKSInstances(env.EKSClusterName) if err != nil || len(eksInstances) == 0 { fmt.Printf("Failed to get EKS instances: %v", err) @@ -86,11 +110,16 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } +//------------------------------------------------------------------------------ +// Main Test Functions +//------------------------------------------------------------------------------ + func TestAll(t *testing.T) { t.Run("Resources", func(t *testing.T) { testResources(t) }) + // Don't run metric tests if resource tests fail if !t.Failed() { t.Run("Metrics", func(t *testing.T) { testMetrics(t) @@ -112,6 +141,22 @@ func testResources(t *testing.T) { } } +func testMetrics(t *testing.T) { + configFile := filepath.Base(env.AgentConfig) + tests := testMetricsRegistry[configFile] + + fmt.Println("waiting for metrics to propagate...") + time.Sleep(wait) + + for _, testFunc := range tests { + testFunc(t) + } +} + +//------------------------------------------------------------------------------ +// Resource Test Functions +//------------------------------------------------------------------------------ + func testAgentResources(t *testing.T, clientset *kubernetes.Clientset) { t.Run("verify_agent_resources", func(t *testing.T) { daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) @@ -177,21 +222,13 @@ func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) { }) } -func testMetrics(t *testing.T) { - configFile := filepath.Base(env.AgentConfig) - tests := testMetricsRegistry[configFile] - - fmt.Println("Waiting for metrics to propagate...") - time.Sleep(10 * time.Minute) - - for _, testFunc := range tests { - testFunc(t) - } -} +//------------------------------------------------------------------------------ +// Metric Test Functions +//------------------------------------------------------------------------------ func testTomcatMetrics(t *testing.T) { t.Run("verify_jvm_tomcat_metrics", func(t *testing.T) { - metricsToCheck := []string{ + validateMetrics(t, []string{ "jvm.classes.loaded", "jvm.gc.collections.count", "jvm.gc.collections.elapsed", @@ -215,55 +252,21 @@ func testTomcatMetrics(t *testing.T) { "tomcat.max_time", "tomcat.processing_time", "tomcat.threads", - } - - for _, metric := range metricsToCheck { - t.Run(metric, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric, "JVM_TOMCAT_E2E", nil, 5, 30*time.Second) - }) - } + }, "JVM_TOMCAT_E2E") }) } func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { - cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-n", "test", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") - output, err := cmd.CombinedOutput() - require.NoError(t, err, "Error getting LoadBalancer URL") - - lbURL := strings.Trim(string(output), "'") - require.NotEmpty(t, lbURL, "LoadBalancer URL failed to format") - - for i := 0; i < 5; i++ { - resp, err := http.Get(fmt.Sprintf("http://%s/webapp/index.jsp", lbURL)) - if err != nil { - t.Logf("Request attempt %d failed: %v", i+1, err) - continue - } - require.NoError(t, resp.Body.Close(), "Failed to close response body") - } - - time.Sleep(5 * time.Minute) - - startTime := time.Now().Add(-5 * time.Minute) - endTime := time.Now() - - aboveZero, err := awsservice.CheckMetricAboveZero( - "tomcat.sessions", - "JVM_TOMCAT_E2E", - startTime, - endTime, - 60, - nodeNames, - ) - require.NoError(t, err, "Failed to check metric above zero") - require.True(t, aboveZero, "Expected non-zero tomcat.sessions after applying traffic") + generateTraffic(t) + time.Sleep(wait) + verifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E") }) } func testKafkaMetrics(t *testing.T) { t.Run("verify_kafka_metrics", func(t *testing.T) { - metricsToCheck := []string{ + validateMetrics(t, []string{ "kafka.message.count", "kafka.request.count", "kafka.request.failed", @@ -277,19 +280,13 @@ func testKafkaMetrics(t *testing.T) { "kafka.producer.io-wait-time-ns-avg", "kafka.producer.outgoing-byte-rate", "kafka.producer.response-rate", - } - - for _, metric := range metricsToCheck { - t.Run(metric, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric, "KAFKA_E2E", nil, 5, 30*time.Second) - }) - } + }, "KAFKA_E2E") }) } func testContainerInsightsMetrics(t *testing.T) { t.Run("verify_containerinsights_metrics", func(t *testing.T) { - metricsToCheck := []string{ + validateMetrics(t, []string{ "jvm_classes_loaded", "jvm_threads_current", "jvm_threads_daemon", @@ -308,48 +305,60 @@ func testContainerInsightsMetrics(t *testing.T) { "catalina_globalrequestprocessor_requestcount", "catalina_globalrequestprocessor_errorcount", "catalina_globalrequestprocessor_processingtime", - } - - for _, metric := range metricsToCheck { - t.Run(metric, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric, "ContainerInsights/Prometheus", nil, 5, 30*time.Second) - }) - } + }, "ContainerInsights/Prometheus") }) } func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { - cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-n", "test", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") - output, err := cmd.CombinedOutput() - require.NoError(t, err, "Error getting LoadBalancer URL") - - lbURL := strings.Trim(string(output), "'") - require.NotEmpty(t, lbURL, "LoadBalancer URL failed to format") - - for i := 0; i < 5; i++ { - resp, err := http.Get(fmt.Sprintf("http://%s/webapp/index.jsp", lbURL)) - if err != nil { - t.Logf("Request attempt %d failed: %v", i+1, err) - continue - } - require.NoError(t, resp.Body.Close(), "Failed to close response body") - } + generateTraffic(t) + time.Sleep(wait) + verifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus") + }) +} + +//------------------------------------------------------------------------------ +// Helper Functions +//------------------------------------------------------------------------------ + +func validateMetrics(t *testing.T, metrics []string, namespace string) { + for _, metric := range metrics { + t.Run(metric, func(t *testing.T) { + awsservice.ValidateMetricWithTest(t, metric, namespace, nil, 5, interval) + }) + } +} - time.Sleep(5 * time.Minute) +func generateTraffic(t *testing.T) { + cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-n", "test", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") + output, err := cmd.CombinedOutput() + require.NoError(t, err, "Error getting LoadBalancer URL") - startTime := time.Now().Add(-5 * time.Minute) - endTime := time.Now() + lbURL := strings.Trim(string(output), "'") + require.NotEmpty(t, lbURL, "LoadBalancer URL failed to format") - aboveZero, err := awsservice.CheckMetricAboveZero( - "catalina_manager_rejectedsessions", - "ContainerInsights/Prometheus", - startTime, - endTime, - 60, - nodeNames, - ) - require.NoError(t, err, "Failed to check metric above zero") - require.True(t, aboveZero, "Expected non-zero catalina_manager_rejectedsessions after applying traffic") - }) + for i := 0; i < 5; i++ { + resp, err := http.Get(fmt.Sprintf("http://%s/webapp/index.jsp", lbURL)) + if err != nil { + t.Logf("Request attempt %d failed: %v", i+1, err) + continue + } + require.NoError(t, resp.Body.Close(), "Failed to close response body") + } +} + +func verifyMetricAboveZero(t *testing.T, metricName, namespace string) { + startTime := time.Now().Add(-wait) + endTime := time.Now() + + aboveZero, err := awsservice.CheckMetricAboveZero( + metricName, + namespace, + startTime, + endTime, + 60, + nodeNames, + ) + require.NoError(t, err, "Failed to check metric above zero") + require.True(t, aboveZero, fmt.Sprintf("Expected non-zero %s after applying traffic", metricName)) } diff --git a/util/common/kubernetes.go b/util/common/kubernetes.go index 4274ee082..52b71ca41 100644 --- a/util/common/kubernetes.go +++ b/util/common/kubernetes.go @@ -14,6 +14,10 @@ import ( "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" ) +//------------------------------------------------------------------------------ +// Environment Setup +//------------------------------------------------------------------------------ + func InitializeEnvironment(env *environment.MetaData) error { if env.Region != "us-west-2" { if err := awsservice.ConfigureAWSClients(env.Region); err != nil { @@ -32,6 +36,10 @@ func InitializeEnvironment(env *environment.MetaData) error { return nil } +//------------------------------------------------------------------------------ +// K8s Resource Management Functions +//------------------------------------------------------------------------------ + func ApplyResources(env *environment.MetaData) error { updateKubeconfig := exec.Command("aws", "eks", "update-kubeconfig", "--name", env.EKSClusterName) output, err := updateKubeconfig.CombinedOutput() @@ -39,6 +47,7 @@ func ApplyResources(env *environment.MetaData) error { return fmt.Errorf("failed to update kubeconfig: %w\nOutput: %s", err, output) } + fmt.Println("Installing Helm release...") helm := []string{ "helm", "upgrade", "--install", "amazon-cloudwatch-observability", filepath.Join("..", "..", "..", "terraform", "eks", "e2e", "helm-charts", "charts", "amazon-cloudwatch-observability"), @@ -102,13 +111,16 @@ func DestroyResources(env *environment.MetaData) error { var errors []error + fmt.Println("Deleting test namespace...") deleteCmd := exec.Command("kubectl", "delete", "namespace", "test", "--timeout=60s") output, err = deleteCmd.CombinedOutput() - // Note that we don't want to consider not finding the namespace to be an error since that's the outcome we want. + + // We don't want to consider not finding the namespace to be an error since that's the outcome we want if err != nil && !strings.Contains(string(output), "not found") { errors = append(errors, fmt.Errorf("failed to delete test namespace: %w\nOutput: %s", err, output)) } + fmt.Println("Uninstalling Helm release...") helm := []string{ "helm", "uninstall", "amazon-cloudwatch-observability", "--namespace", "amazon-cloudwatch", } From 7ab4f90e1519410e7300835290c2a7598ed84b64 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 03:59:42 -0500 Subject: [PATCH 66/92] 30m --- terraform/eks/e2e/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 8ef9751d3..2c0348301 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -145,7 +145,7 @@ resource "null_resource" "validator" { when = destroy command = <<-EOT echo "Running cleanup for K8s resources" - go test -timeout 15m -v ${self.triggers.test_dir} \ + go test -timeout 30m -v ${self.triggers.test_dir} \ -destroy=true \ -region=${self.triggers.region} \ -eksClusterName=${self.triggers.cluster_name} \ From 91aa2ad03185817a669506cc5c7dfe8617103b9d Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 04:09:45 -0500 Subject: [PATCH 67/92] clean function --- util/awsservice/cloudwatchmetrics.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index 14304961e..fd4c11afa 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -142,20 +142,21 @@ func CheckMetricAboveZero( periodInSeconds int32, nodeNames []string, ) (bool, error) { - listMetricsInput := cloudwatch.ListMetricsInput{ + metrics, err := CwmClient.ListMetrics(ctx, &cloudwatch.ListMetricsInput{ MetricName: aws.String(metricName), Namespace: aws.String(namespace), RecentlyActive: "PT3H", - } + }) - metrics, err := CwmClient.ListMetrics(ctx, &listMetricsInput) if err != nil { return false, err } + if len(metrics.Metrics) == 0 { return false, fmt.Errorf("no metrics found for %s", metricName) } + // Make sure to only check metrics that match node names for _, metric := range metrics.Metrics { var nodeNameMatch bool var nodeName string @@ -184,6 +185,7 @@ func CheckMetricAboveZero( []types.Statistic{types.StatisticMaximum}, nil, ) + if err != nil { log.Printf("Error getting statistics for metric with dimensions %v: %v", metric.Dimensions, err) continue From 953272dceeabe812c25d79104de4dedff0011014 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 04:41:31 -0500 Subject: [PATCH 68/92] spacing --- test/e2e/jmx/jmx_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 7a36d2ddb..90ae8e86e 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -206,7 +206,7 @@ func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) { "OTEL_TRACES_EXPORTER": "none", "OTEL_AWS_JMX_EXPORTER_METRICS_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4314/v1/metrics", "OTEL_JMX_TARGET_SYSTEM": jmxTargetSystem, - "JAVA_TOOL_OPTIONS": "-javaagent:/otel-auto-instrumentation-java/javaagent.jar", + "JAVA_TOOL_OPTIONS": " -javaagent:/otel-auto-instrumentation-java/javaagent.jar", } for _, container := range pods.Items[0].Spec.Containers { From 545a256e7d762ef8deb73e890342773a0a40991a Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 04:53:31 -0500 Subject: [PATCH 69/92] move variable --- util/common/kubernetes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/common/kubernetes.go b/util/common/kubernetes.go index 52b71ca41..526f38ced 100644 --- a/util/common/kubernetes.go +++ b/util/common/kubernetes.go @@ -85,7 +85,6 @@ func ApplyResources(env *environment.MetaData) error { return fmt.Errorf("failed to wait for operator deployment: %w\nOutput: %s", err, output) } - deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml") apply := exec.Command("kubectl", "apply", "-f", env.SampleApp) output, err = apply.CombinedOutput() if err != nil { @@ -93,6 +92,7 @@ func ApplyResources(env *environment.MetaData) error { } fmt.Println("Waiting for Sample Application to initialize...") + deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml") wait = exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=300s", fmt.Sprintf("deployment/%s", deploymentName), "-n", "test") output, err = wait.CombinedOutput() if err != nil { From 67c6371a85797490c49c9b530ad3252186848a16 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 04:54:21 -0500 Subject: [PATCH 70/92] move variable --- util/common/kubernetes.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/common/kubernetes.go b/util/common/kubernetes.go index 526f38ced..5645894e2 100644 --- a/util/common/kubernetes.go +++ b/util/common/kubernetes.go @@ -85,6 +85,8 @@ func ApplyResources(env *environment.MetaData) error { return fmt.Errorf("failed to wait for operator deployment: %w\nOutput: %s", err, output) } + deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml") + apply := exec.Command("kubectl", "apply", "-f", env.SampleApp) output, err = apply.CombinedOutput() if err != nil { @@ -92,7 +94,6 @@ func ApplyResources(env *environment.MetaData) error { } fmt.Println("Waiting for Sample Application to initialize...") - deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml") wait = exec.Command("kubectl", "wait", "--for=condition=available", "--timeout=300s", fmt.Sprintf("deployment/%s", deploymentName), "-n", "test") output, err = wait.CombinedOutput() if err != nil { From 78287c6bb23a8c68d7ab69e090af2cd11df572aa Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 05:23:01 -0500 Subject: [PATCH 71/92] remove bad exit --- test/e2e/jmx/jmx_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 90ae8e86e..1394ae8ca 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -83,7 +83,6 @@ func TestMain(m *testing.M) { if env.Destroy { if err := common.DestroyResources(env); err != nil { fmt.Printf("Failed to delete kubernetes resources: %v\n", err) - os.Exit(1) } os.Exit(0) } From bf1c276d17803a9874f6734ca061eec888470e24 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 11:15:28 -0500 Subject: [PATCH 72/92] remove quotes --- test/e2e/jmx/jmx_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 1394ae8ca..a3064fa3f 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -168,7 +168,7 @@ func testAgentResources(t *testing.T, clientset *kubernetes.Clientset) { cwConfig, exists := configMap.Data["cwagentconfig.json"] require.True(t, exists, "cwagentconfig.json not found in ConfigMap") - require.Contains(t, cwConfig, `"jmx"`, "JMX configuration not found in cwagentconfig.json") + require.Contains(t, cwConfig, `jmx`, "JMX configuration not found in cwagentconfig.json") service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) require.NoError(t, err, "Error getting CloudWatch Agent Service") From 9799dc9b65a9254c9302792294866567bf314f34 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 14:03:42 -0500 Subject: [PATCH 73/92] adjust waits --- test/e2e/jmx/jmx_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index a3064fa3f..d488a375c 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -30,8 +30,9 @@ import ( //------------------------------------------------------------------------------ const ( - wait = 5 * time.Minute - interval = 30 * time.Second + shortWait = 5 * time.Minute + longWait = 10 * time.Minute + interval = 30 * time.Second ) var ( @@ -145,7 +146,7 @@ func testMetrics(t *testing.T) { tests := testMetricsRegistry[configFile] fmt.Println("waiting for metrics to propagate...") - time.Sleep(wait) + time.Sleep(longWait) for _, testFunc := range tests { testFunc(t) @@ -258,7 +259,7 @@ func testTomcatMetrics(t *testing.T) { func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { generateTraffic(t) - time.Sleep(wait) + time.Sleep(shortWait) verifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E") }) } @@ -311,7 +312,7 @@ func testContainerInsightsMetrics(t *testing.T) { func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { generateTraffic(t) - time.Sleep(wait) + time.Sleep(shortWait) verifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus") }) } @@ -347,7 +348,7 @@ func generateTraffic(t *testing.T) { } func verifyMetricAboveZero(t *testing.T, metricName, namespace string) { - startTime := time.Now().Add(-wait) + startTime := time.Now().Add(-shortWait) endTime := time.Now() aboveZero, err := awsservice.CheckMetricAboveZero( From 13a6985af8f609c388e37543008e91ce4e0c4df1 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 18:12:03 -0500 Subject: [PATCH 74/92] fix for ci dimensions --- util/awsservice/cloudwatchmetrics.go | 68 +++++++++++++++------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index fd4c11afa..06278d445 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -141,6 +141,7 @@ func CheckMetricAboveZero( endTime time.Time, periodInSeconds int32, nodeNames []string, + containerInsights bool, ) (bool, error) { metrics, err := CwmClient.ListMetrics(ctx, &cloudwatch.ListMetricsInput{ MetricName: aws.String(metricName), @@ -156,46 +157,51 @@ func CheckMetricAboveZero( return false, fmt.Errorf("no metrics found for %s", metricName) } - // Make sure to only check metrics that match node names for _, metric := range metrics.Metrics { - var nodeNameMatch bool - var nodeName string - for _, dim := range metric.Dimensions { - if *dim.Name == "k8s.node.name" { - nodeName = *dim.Value - for _, name := range nodeNames { - if nodeName == name { - nodeNameMatch = true - break + // Skip node name check if containerInsights is true + if !containerInsights { + var nodeNameMatch bool + var nodeName string + for _, dim := range metric.Dimensions { + if *dim.Name == "k8s.node.name" { + nodeName = *dim.Value + for _, name := range nodeNames { + if nodeName == name { + nodeNameMatch = true + break + } } + break } - break } + if !nodeNameMatch { + continue + } + log.Printf("Checking metric: %s for node: %s", *metric.MetricName, nodeName) } - if nodeNameMatch { - log.Printf("Checking metric: %s for node: %s", *metric.MetricName, nodeName) - data, err := GetMetricStatistics( - metricName, - namespace, - metric.Dimensions, - startTime, - endTime, - periodInSeconds, - []types.Statistic{types.StatisticMaximum}, - nil, - ) + data, err := GetMetricStatistics( + metricName, + namespace, + metric.Dimensions, + startTime, + endTime, + periodInSeconds, + []types.Statistic{types.StatisticMaximum}, + nil, + ) - if err != nil { - log.Printf("Error getting statistics for metric with dimensions %v: %v", metric.Dimensions, err) - continue - } + if err != nil { + log.Printf("Error getting statistics for metric with dimensions %v: %v", metric.Dimensions, err) + continue + } - for _, datapoint := range data.Datapoints { - if *datapoint.Maximum > 0 { - log.Printf("Found value above zero for node: %s", nodeName) - return true, nil + for _, datapoint := range data.Datapoints { + if *datapoint.Maximum > 0 { + if !containerInsights { + log.Printf("Found value above zero for node: %s", metric.Dimensions[0].Value) } + return true, nil } } } From e1256783c8829d68783c581887092c7c18fac660 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 18:37:25 -0500 Subject: [PATCH 75/92] fix for ci dimensions --- test/e2e/jmx/jmx_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index d488a375c..7c38bad58 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -260,7 +260,7 @@ func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { generateTraffic(t) time.Sleep(shortWait) - verifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E") + verifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E", false) }) } @@ -313,7 +313,7 @@ func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { generateTraffic(t) time.Sleep(shortWait) - verifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus") + verifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus", true) }) } @@ -347,7 +347,7 @@ func generateTraffic(t *testing.T) { } } -func verifyMetricAboveZero(t *testing.T, metricName, namespace string) { +func verifyMetricAboveZero(t *testing.T, metricName, namespace string, containerinsights bool) { startTime := time.Now().Add(-shortWait) endTime := time.Now() @@ -358,6 +358,7 @@ func verifyMetricAboveZero(t *testing.T, metricName, namespace string) { endTime, 60, nodeNames, + containerinsights, ) require.NoError(t, err, "Failed to check metric above zero") require.True(t, aboveZero, fmt.Sprintf("Expected non-zero %s after applying traffic", metricName)) From 1e2944394fcc837ded335f465ed758d3666a3c2b Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 18:38:26 -0500 Subject: [PATCH 76/92] fix for ci dimensions --- test/e2e/jmx/jmx_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 7c38bad58..9dd990fef 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -347,7 +347,7 @@ func generateTraffic(t *testing.T) { } } -func verifyMetricAboveZero(t *testing.T, metricName, namespace string, containerinsights bool) { +func verifyMetricAboveZero(t *testing.T, metricName, namespace string, containerInsights bool) { startTime := time.Now().Add(-shortWait) endTime := time.Now() @@ -358,7 +358,7 @@ func verifyMetricAboveZero(t *testing.T, metricName, namespace string, container endTime, 60, nodeNames, - containerinsights, + containerInsights, ) require.NoError(t, err, "Failed to check metric above zero") require.True(t, aboveZero, fmt.Sprintf("Expected non-zero %s after applying traffic", metricName)) From a8f40e240cf623f761341b2efad9ce2fd4156881 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 18:50:00 -0500 Subject: [PATCH 77/92] typo --- util/awsservice/cloudwatchmetrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index 06278d445..89d594eb6 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -199,7 +199,7 @@ func CheckMetricAboveZero( for _, datapoint := range data.Datapoints { if *datapoint.Maximum > 0 { if !containerInsights { - log.Printf("Found value above zero for node: %s", metric.Dimensions[0].Value) + log.Printf("Found value above zero for node: %s", *metric.Dimensions[0].Value) } return true, nil } From 2fb3b53cbdf79a351f94914595a376d05e60da1b Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 20 Dec 2024 18:52:33 -0500 Subject: [PATCH 78/92] remove var --- util/awsservice/cloudwatchmetrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index 89d594eb6..2b94e2696 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -199,7 +199,7 @@ func CheckMetricAboveZero( for _, datapoint := range data.Datapoints { if *datapoint.Maximum > 0 { if !containerInsights { - log.Printf("Found value above zero for node: %s", *metric.Dimensions[0].Value) + log.Printf("Found value above zero") } return true, nil } From bef2af6a74c5014d36adf806f04eaf9e91c91a57 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 22 Dec 2024 01:29:32 -0500 Subject: [PATCH 79/92] switch to nodeport --- terraform/eks/e2e/main.tf | 9 +++++++++ test/e2e/jmx/jmx_test.go | 10 +++++----- test/e2e/jmx/resources/sample_apps/tomcat.yaml | 3 ++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 2c0348301..bfd0c8dfe 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -52,6 +52,15 @@ resource "aws_eks_node_group" "this" { ] } +resource "aws_security_group_rule" "nodeport_inbound" { + type = "ingress" + from_port = 30080 + to_port = 30080 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + security_group_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id +} + resource "aws_iam_role" "node_role" { name = "${local.cluster_name}-Worker-Role-${module.common.testing_id}" diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 9dd990fef..8b0489ecf 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -330,15 +330,15 @@ func validateMetrics(t *testing.T, metrics []string, namespace string) { } func generateTraffic(t *testing.T) { - cmd := exec.Command("kubectl", "get", "svc", "tomcat-service", "-n", "test", "-o", "jsonpath='{.status.loadBalancer.ingress[0].hostname}'") + cmd := exec.Command("kubectl", "get", "nodes", "-o", "jsonpath='{.items[0].status.addresses[?(@.type==\"ExternalIP\")].address}'") output, err := cmd.CombinedOutput() - require.NoError(t, err, "Error getting LoadBalancer URL") + require.NoError(t, err, "Error getting node external IP") - lbURL := strings.Trim(string(output), "'") - require.NotEmpty(t, lbURL, "LoadBalancer URL failed to format") + nodeIP := strings.Trim(string(output), "'") + require.NotEmpty(t, nodeIP, "Node IP failed to format") for i := 0; i < 5; i++ { - resp, err := http.Get(fmt.Sprintf("http://%s/webapp/index.jsp", lbURL)) + resp, err := http.Get(fmt.Sprintf("http://%s:30080/webapp/index.jsp", nodeIP)) if err != nil { t.Logf("Request attempt %d failed: %v", i+1, err) continue diff --git a/test/e2e/jmx/resources/sample_apps/tomcat.yaml b/test/e2e/jmx/resources/sample_apps/tomcat.yaml index d152f00ca..1ddb8dc4a 100644 --- a/test/e2e/jmx/resources/sample_apps/tomcat.yaml +++ b/test/e2e/jmx/resources/sample_apps/tomcat.yaml @@ -9,12 +9,13 @@ metadata: name: tomcat-service namespace: test spec: - type: LoadBalancer + type: NodePort selector: app: tomcat ports: - port: 80 targetPort: 8080 + nodePort: 30080 --- apiVersion: apps/v1 kind: Deployment From 8cb81ce05f18b584b05c7317295b6379bcd780f6 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 22 Dec 2024 02:34:13 -0500 Subject: [PATCH 80/92] 5 min --- test/e2e/jmx/jmx_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 8b0489ecf..a2d5ccb2b 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -30,9 +30,8 @@ import ( //------------------------------------------------------------------------------ const ( - shortWait = 5 * time.Minute - longWait = 10 * time.Minute - interval = 30 * time.Second + wait = 5 * time.Minute + interval = 30 * time.Second ) var ( @@ -146,7 +145,7 @@ func testMetrics(t *testing.T) { tests := testMetricsRegistry[configFile] fmt.Println("waiting for metrics to propagate...") - time.Sleep(longWait) + time.Sleep(wait) for _, testFunc := range tests { testFunc(t) @@ -259,7 +258,7 @@ func testTomcatMetrics(t *testing.T) { func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { generateTraffic(t) - time.Sleep(shortWait) + time.Sleep(wait) verifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E", false) }) } @@ -312,7 +311,7 @@ func testContainerInsightsMetrics(t *testing.T) { func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { generateTraffic(t) - time.Sleep(shortWait) + time.Sleep(wait) verifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus", true) }) } @@ -348,7 +347,7 @@ func generateTraffic(t *testing.T) { } func verifyMetricAboveZero(t *testing.T, metricName, namespace string, containerInsights bool) { - startTime := time.Now().Add(-shortWait) + startTime := time.Now().Add(-wait) endTime := time.Now() aboveZero, err := awsservice.CheckMetricAboveZero( From bf7330934452d1f8e096a8ee6d5cd062cfeea028 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 27 Dec 2024 14:19:03 -0500 Subject: [PATCH 81/92] revision 3 --- .../eksdeploymenttype/eks_deployment_type.go | 6 +- environment/metadata.go | 6 +- .../resources/eks_e2e_jmx_test_matrix.json | 2 +- terraform/eks/e2e/main.tf | 4 +- terraform/eks/e2e/variables.tf | 7 +- test/e2e/jmx/jmx_test.go | 95 +++------------- test/e2e/testutils.go | 107 ++++++++++++++++++ 7 files changed, 132 insertions(+), 95 deletions(-) create mode 100644 test/e2e/testutils.go diff --git a/environment/eksdeploymenttype/eks_deployment_type.go b/environment/eksdeploymenttype/eks_deployment_type.go index 44c98998f..0bd1ad301 100644 --- a/environment/eksdeploymenttype/eks_deployment_type.go +++ b/environment/eksdeploymenttype/eks_deployment_type.go @@ -11,17 +11,19 @@ const ( DAEMON EKSDeploymentType = "DAEMON" REPLICA EKSDeploymentType = "REPLICA" SIDECAR EKSDeploymentType = "SIDECAR" + STATEFUL EKSDeploymentType = "STATEFUL" ) var ( - ecsDeploymentTypes = map[string]EKSDeploymentType{ + eksDeploymentTypes = map[string]EKSDeploymentType{ "DAEMON": DAEMON, "REPLICA": REPLICA, "SIDECAR": SIDECAR, + "STATEFUL": STATEFUL, } ) func FromString(str string) (EKSDeploymentType, bool) { - c, ok := ecsDeploymentTypes[strings.ToUpper(str)] + c, ok := eksDeploymentTypes[strings.ToUpper(str)] return c, ok } diff --git a/environment/metadata.go b/environment/metadata.go index f7c03fc77..ae7aa3174 100644 --- a/environment/metadata.go +++ b/environment/metadata.go @@ -88,7 +88,7 @@ type MetaDataStrings struct { AmpWorkspaceId string Region string K8sVersion string - Destroy string + Destroy bool HelmChartsBranch string CloudwatchAgentRepository string CloudwatchAgentTag string @@ -138,7 +138,7 @@ func registerEKSData(d *MetaDataStrings) { func registerEKSE2ETestData(dataString *MetaDataStrings) { flag.StringVar(&(dataString.Region), "region", "", "AWS region") flag.StringVar(&(dataString.K8sVersion), "k8s_version", "", "Kubernetes version") - flag.StringVar(&(dataString.Destroy), "destroy", "false", "Whether to run in destroy mode (true/false)") + flag.BoolVar(&(dataString.Destroy), "destroy", false, "Whether to run in destroy mode (true/false)") flag.StringVar(&(dataString.HelmChartsBranch), "helm_charts_branch", "", "Helm charts branch") flag.StringVar(&(dataString.CloudwatchAgentRepository), "cloudwatch_agent_repository", "", "CloudWatch Agent repository") flag.StringVar(&(dataString.CloudwatchAgentTag), "cloudwatch_agent_tag", "", "CloudWatch Agent tag") @@ -321,7 +321,7 @@ func GetEnvironmentMetaData() *MetaData { metaDataStorage.AmpWorkspaceId = registeredMetaDataStrings.AmpWorkspaceId metaDataStorage.Region = registeredMetaDataStrings.Region metaDataStorage.K8sVersion = registeredMetaDataStrings.K8sVersion - metaDataStorage.Destroy = registeredMetaDataStrings.Destroy == "true" + metaDataStorage.Destroy = registeredMetaDataStrings.Destroy metaDataStorage.HelmChartsBranch = registeredMetaDataStrings.HelmChartsBranch metaDataStorage.CloudwatchAgentRepository = registeredMetaDataStrings.CloudwatchAgentRepository metaDataStorage.CloudwatchAgentTag = registeredMetaDataStrings.CloudwatchAgentTag diff --git a/generator/resources/eks_e2e_jmx_test_matrix.json b/generator/resources/eks_e2e_jmx_test_matrix.json index d484997f7..cf999e588 100644 --- a/generator/resources/eks_e2e_jmx_test_matrix.json +++ b/generator/resources/eks_e2e_jmx_test_matrix.json @@ -1,5 +1,5 @@ [ { - "nodes": 1 + "nodes": 2 } ] \ No newline at end of file diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index bfd0c8dfe..4364696ab 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -39,7 +39,7 @@ resource "aws_eks_node_group" "this" { min_size = var.nodes } - ami_type = var.ami_type + ami_type = "AL2_x86_64" capacity_type = "ON_DEMAND" disk_size = 20 instance_types = [var.instance_type] @@ -155,7 +155,7 @@ resource "null_resource" "validator" { command = <<-EOT echo "Running cleanup for K8s resources" go test -timeout 30m -v ${self.triggers.test_dir} \ - -destroy=true \ + -destroy \ -region=${self.triggers.region} \ -eksClusterName=${self.triggers.cluster_name} \ -computeType=EKS \ diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index 016bbb318..67af48c2e 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -18,12 +18,7 @@ variable "cluster_name" { variable "nodes" { type = number - default = 1 -} - -variable "ami_type" { - type = string - default = "AL2_x86_64" + default = 2 } variable "instance_type" { diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index a2d5ccb2b..f2af0771b 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -4,36 +4,28 @@ package main import ( - "context" "flag" "fmt" - "net/http" "os" - "os/exec" "path/filepath" "strings" "testing" "time" "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/e2e" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" "github.com/aws/amazon-cloudwatch-agent-test/util/common" ) //------------------------------------------------------------------------------ -// Constants and Variables +// Variables //------------------------------------------------------------------------------ -const ( - wait = 5 * time.Minute - interval = 30 * time.Second -) - var ( nodeNames []string env *environment.MetaData @@ -145,7 +137,7 @@ func testMetrics(t *testing.T) { tests := testMetricsRegistry[configFile] fmt.Println("waiting for metrics to propagate...") - time.Sleep(wait) + time.Sleep(e2e.Wait) for _, testFunc := range tests { testFunc(t) @@ -158,37 +150,13 @@ func testMetrics(t *testing.T) { func testAgentResources(t *testing.T, clientset *kubernetes.Clientset) { t.Run("verify_agent_resources", func(t *testing.T) { - daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent DaemonSet") - require.NotNil(t, daemonSet, "CloudWatch Agent DaemonSet not found") - - configMap, err := clientset.CoreV1().ConfigMaps("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent ConfigMap") - require.NotNil(t, configMap, "CloudWatch Agent ConfigMap not found") - - cwConfig, exists := configMap.Data["cwagentconfig.json"] - require.True(t, exists, "cwagentconfig.json not found in ConfigMap") - require.Contains(t, cwConfig, `jmx`, "JMX configuration not found in cwagentconfig.json") - - service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent Service") - require.NotNil(t, service, "CloudWatch Agent Service not found") - - serviceAccount, err := clientset.CoreV1().ServiceAccounts("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) - require.NoError(t, err, "Error getting CloudWatch Agent Service Account") - require.NotNil(t, serviceAccount, "CloudWatch Agent Service Account not found") + e2e.VerifyAgentResources(t, clientset, "jmx") }) } func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) { t.Run("verify_jmx_resources", func(t *testing.T) { deploymentName := strings.TrimSuffix(filepath.Base(env.SampleApp), ".yaml") - pods, err := clientset.CoreV1().Pods("test").List(context.TODO(), metav1.ListOptions{ - LabelSelector: fmt.Sprintf("app=%s", deploymentName), - FieldSelector: "status.phase=Running", - }) - require.NoError(t, err, "Error getting pods for deployment") - require.NotEmpty(t, pods.Items, "No pods found for deployment") var jmxTargetSystem string switch filepath.Base(env.AgentConfig) { @@ -208,16 +176,7 @@ func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) { "JAVA_TOOL_OPTIONS": " -javaagent:/otel-auto-instrumentation-java/javaagent.jar", } - for _, container := range pods.Items[0].Spec.Containers { - for _, envVar := range container.Env { - if expectedValue, exists := requiredEnvVars[envVar.Name]; exists { - require.Equal(t, expectedValue, envVar.Value, fmt.Sprintf("Unexpected value for environment variable %s in container %s", envVar.Name, container.Name)) - delete(requiredEnvVars, envVar.Name) - } - } - } - - require.Empty(t, requiredEnvVars, "Not all required environment variables were found in the pod") + e2e.VerifyPodEnvironment(t, clientset, deploymentName, requiredEnvVars) }) } @@ -227,7 +186,7 @@ func testJMXResources(t *testing.T, clientset *kubernetes.Clientset) { func testTomcatMetrics(t *testing.T) { t.Run("verify_jvm_tomcat_metrics", func(t *testing.T) { - validateMetrics(t, []string{ + e2e.ValidateMetrics(t, []string{ "jvm.classes.loaded", "jvm.gc.collections.count", "jvm.gc.collections.elapsed", @@ -257,15 +216,15 @@ func testTomcatMetrics(t *testing.T) { func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { - generateTraffic(t) - time.Sleep(wait) + e2e.GenerateTraffic(t) + time.Sleep(e2e.Wait) verifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E", false) }) } func testKafkaMetrics(t *testing.T) { t.Run("verify_kafka_metrics", func(t *testing.T) { - validateMetrics(t, []string{ + e2e.ValidateMetrics(t, []string{ "kafka.message.count", "kafka.request.count", "kafka.request.failed", @@ -285,7 +244,7 @@ func testKafkaMetrics(t *testing.T) { func testContainerInsightsMetrics(t *testing.T) { t.Run("verify_containerinsights_metrics", func(t *testing.T) { - validateMetrics(t, []string{ + e2e.ValidateMetrics(t, []string{ "jvm_classes_loaded", "jvm_threads_current", "jvm_threads_daemon", @@ -310,8 +269,8 @@ func testContainerInsightsMetrics(t *testing.T) { func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { - generateTraffic(t) - time.Sleep(wait) + e2e.GenerateTraffic(t) + time.Sleep(e2e.Wait) verifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus", true) }) } @@ -320,34 +279,8 @@ func testTomcatRejectedSessions(t *testing.T) { // Helper Functions //------------------------------------------------------------------------------ -func validateMetrics(t *testing.T, metrics []string, namespace string) { - for _, metric := range metrics { - t.Run(metric, func(t *testing.T) { - awsservice.ValidateMetricWithTest(t, metric, namespace, nil, 5, interval) - }) - } -} - -func generateTraffic(t *testing.T) { - cmd := exec.Command("kubectl", "get", "nodes", "-o", "jsonpath='{.items[0].status.addresses[?(@.type==\"ExternalIP\")].address}'") - output, err := cmd.CombinedOutput() - require.NoError(t, err, "Error getting node external IP") - - nodeIP := strings.Trim(string(output), "'") - require.NotEmpty(t, nodeIP, "Node IP failed to format") - - for i := 0; i < 5; i++ { - resp, err := http.Get(fmt.Sprintf("http://%s:30080/webapp/index.jsp", nodeIP)) - if err != nil { - t.Logf("Request attempt %d failed: %v", i+1, err) - continue - } - require.NoError(t, resp.Body.Close(), "Failed to close response body") - } -} - func verifyMetricAboveZero(t *testing.T, metricName, namespace string, containerInsights bool) { - startTime := time.Now().Add(-wait) + startTime := time.Now().Add(-e2e.Wait) endTime := time.Now() aboveZero, err := awsservice.CheckMetricAboveZero( @@ -361,4 +294,4 @@ func verifyMetricAboveZero(t *testing.T, metricName, namespace string, container ) require.NoError(t, err, "Failed to check metric above zero") require.True(t, aboveZero, fmt.Sprintf("Expected non-zero %s after applying traffic", metricName)) -} +} \ No newline at end of file diff --git a/test/e2e/testutils.go b/test/e2e/testutils.go new file mode 100644 index 000000000..c44d2189d --- /dev/null +++ b/test/e2e/testutils.go @@ -0,0 +1,107 @@ +package e2e + +import ( + "context" + "fmt" + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "net/http" + "os/exec" + "strings" + "testing" + "time" +) + +//------------------------------------------------------------------------------ +// Constants +//------------------------------------------------------------------------------ + +const ( + Wait = 5 * time.Minute + interval = 30 * time.Second +) + +//------------------------------------------------------------------------------ +// Resource Functions +//------------------------------------------------------------------------------ + +func VerifyPodEnvironment(t *testing.T, clientset *kubernetes.Clientset, deploymentName string, requiredEnvVars map[string]string) { + pods, err := clientset.CoreV1().Pods("test").List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", deploymentName), + FieldSelector: "status.phase=Running", + }) + require.NoError(t, err, "Error getting pods for deployment") + require.NotEmpty(t, pods.Items, "No pods found for deployment") + + remainingEnvVars := make(map[string]string) + for k, v := range requiredEnvVars { + remainingEnvVars[k] = v + } + + for _, container := range pods.Items[0].Spec.Containers { + for _, envVar := range container.Env { + if expectedValue, exists := remainingEnvVars[envVar.Name]; exists { + require.Equal(t, expectedValue, envVar.Value, + fmt.Sprintf("Unexpected value for environment variable %s in container %s", + envVar.Name, container.Name)) + delete(remainingEnvVars, envVar.Name) + } + } + } + + require.Empty(t, remainingEnvVars, "Not all required environment variables were found in the pod") +} + +func VerifyAgentResources(t *testing.T, clientset *kubernetes.Clientset, configKeyword string) { + daemonSet, err := clientset.AppsV1().DaemonSets("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent DaemonSet") + require.NotNil(t, daemonSet, "CloudWatch Agent DaemonSet not found") + + configMap, err := clientset.CoreV1().ConfigMaps("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent ConfigMap") + require.NotNil(t, configMap, "CloudWatch Agent ConfigMap not found") + + cwConfig, exists := configMap.Data["cwagentconfig.json"] + require.True(t, exists, "cwagentconfig.json not found in ConfigMap") + require.Contains(t, cwConfig, configKeyword, fmt.Sprintf("%s configuration not found in cwagentconfig.json", configKeyword)) + + service, err := clientset.CoreV1().Services("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent Service") + require.NotNil(t, service, "CloudWatch Agent Service not found") + + serviceAccount, err := clientset.CoreV1().ServiceAccounts("amazon-cloudwatch").Get(context.TODO(), "cloudwatch-agent", metav1.GetOptions{}) + require.NoError(t, err, "Error getting CloudWatch Agent Service Account") + require.NotNil(t, serviceAccount, "CloudWatch Agent Service Account not found") +} + +//------------------------------------------------------------------------------ +// Metric Functions +//------------------------------------------------------------------------------ + +func ValidateMetrics(t *testing.T, metrics []string, namespace string) { + for _, metric := range metrics { + t.Run(metric, func(t *testing.T) { + awsservice.ValidateMetricWithTest(t, metric, namespace, nil, 5, interval) + }) + } +} + +func GenerateTraffic(t *testing.T) { + cmd := exec.Command("kubectl", "get", "nodes", "-o", "jsonpath='{.items[0].status.addresses[?(@.type==\"ExternalIP\")].address}'") + output, err := cmd.CombinedOutput() + require.NoError(t, err, "Error getting node external IP") + + nodeIP := strings.Trim(string(output), "'") + require.NotEmpty(t, nodeIP, "Node IP failed to format") + + for i := 0; i < 5; i++ { + resp, err := http.Get(fmt.Sprintf("http://%s:30080/webapp/index.jsp", nodeIP)) + if err != nil { + t.Logf("Request attempt %d failed: %v", i+1, err) + continue + } + require.NoError(t, resp.Body.Close(), "Failed to close response body") + } +} From 8fe5e559b204acaaa6ad032b52c96aa61c9421fd Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 27 Dec 2024 14:39:03 -0500 Subject: [PATCH 82/92] move and rename kubernetes.go --- util/common/kubernetes.go => test/e2e/envutils.go | 2 +- test/e2e/jmx/jmx_test.go | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) rename util/common/kubernetes.go => test/e2e/envutils.go (99%) diff --git a/util/common/kubernetes.go b/test/e2e/envutils.go similarity index 99% rename from util/common/kubernetes.go rename to test/e2e/envutils.go index 5645894e2..35d25f9d0 100644 --- a/util/common/kubernetes.go +++ b/test/e2e/envutils.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package common +package e2e import ( "fmt" diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index f2af0771b..b5c5e89a2 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -19,7 +19,6 @@ import ( "github.com/aws/amazon-cloudwatch-agent-test/environment" "github.com/aws/amazon-cloudwatch-agent-test/test/e2e" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" - "github.com/aws/amazon-cloudwatch-agent-test/util/common" ) //------------------------------------------------------------------------------ @@ -73,14 +72,14 @@ func TestMain(m *testing.M) { // Destroy K8s resources if terraform destroy if env.Destroy { - if err := common.DestroyResources(env); err != nil { + if err := e2e.DestroyResources(env); err != nil { fmt.Printf("Failed to delete kubernetes resources: %v\n", err) } os.Exit(0) } // Configure AWS clients and create K8s resources - if err := common.InitializeEnvironment(env); err != nil { + if err := e2e.InitializeEnvironment(env); err != nil { fmt.Printf("Failed to initialize environment: %v\n", err) os.Exit(1) } @@ -294,4 +293,4 @@ func verifyMetricAboveZero(t *testing.T, metricName, namespace string, container ) require.NoError(t, err, "Failed to check metric above zero") require.True(t, aboveZero, fmt.Sprintf("Expected non-zero %s after applying traffic", metricName)) -} \ No newline at end of file +} From fadc7857fbf6d0ca585c4e7e5967d4f5ef3142fb Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 27 Dec 2024 15:08:47 -0500 Subject: [PATCH 83/92] added var for deployment strategy --- terraform/eks/e2e/main.tf | 5 +++-- terraform/eks/e2e/variables.tf | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 4364696ab..992669044 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -122,6 +122,7 @@ resource "null_resource" "validator" { cluster_name = aws_eks_cluster.this.name region = var.region test_dir = var.test_dir + eks_deployment_strategy = var.eks_deployment_strategy } provisioner "local-exec" { @@ -132,7 +133,7 @@ resource "null_resource" "validator" { -k8s_version=${var.k8s_version} \ -eksClusterName=${aws_eks_cluster.this.name} \ -computeType=EKS \ - -eksDeploymentStrategy=DAEMON \ + -eksDeploymentStrategy=${var.eks_deployment_strategy} \ -helm_charts_branch=${var.helm_charts_branch} \ -cloudwatch_agent_repository=${var.cloudwatch_agent_repository} \ -cloudwatch_agent_tag=${var.cloudwatch_agent_tag} \ @@ -159,7 +160,7 @@ resource "null_resource" "validator" { -region=${self.triggers.region} \ -eksClusterName=${self.triggers.cluster_name} \ -computeType=EKS \ - -eksDeploymentStrategy=DAEMON + -eksDeploymentStrategy=${self.triggers.eks_deployment_strategy} EOT } } diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index 67af48c2e..490fd1063 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -100,3 +100,8 @@ variable "sample_app" { type = string default = "" } + +variable "eks_deployment_strategy" { + type = string + default = "DAEMON" +} From 9dbaa09e23b4acb1777dbfa81119017080f146c4 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 27 Dec 2024 15:18:10 -0500 Subject: [PATCH 84/92] increase wait --- test/e2e/jmx/jmx_test.go | 6 +++--- test/e2e/testutils.go | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index b5c5e89a2..7d8282588 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -216,7 +216,7 @@ func testTomcatMetrics(t *testing.T) { func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { e2e.GenerateTraffic(t) - time.Sleep(e2e.Wait) + time.Sleep(e2e.LongWait) verifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E", false) }) } @@ -269,7 +269,7 @@ func testContainerInsightsMetrics(t *testing.T) { func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { e2e.GenerateTraffic(t) - time.Sleep(e2e.Wait) + time.Sleep(e2e.LongWait) verifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus", true) }) } @@ -279,7 +279,7 @@ func testTomcatRejectedSessions(t *testing.T) { //------------------------------------------------------------------------------ func verifyMetricAboveZero(t *testing.T, metricName, namespace string, containerInsights bool) { - startTime := time.Now().Add(-e2e.Wait) + startTime := time.Now().Add(-e2e.LongWait) endTime := time.Now() aboveZero, err := awsservice.CheckMetricAboveZero( diff --git a/test/e2e/testutils.go b/test/e2e/testutils.go index c44d2189d..5980f7f35 100644 --- a/test/e2e/testutils.go +++ b/test/e2e/testutils.go @@ -20,6 +20,7 @@ import ( const ( Wait = 5 * time.Minute + LongWait = 10 * time.Minute interval = 30 * time.Second ) From 369da5c8792804ad2fb36717a785f6d972fd00c9 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 27 Dec 2024 15:21:51 -0500 Subject: [PATCH 85/92] Revert "increase wait" This reverts commit 9dbaa09e23b4acb1777dbfa81119017080f146c4. --- test/e2e/jmx/jmx_test.go | 6 +++--- test/e2e/testutils.go | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 7d8282588..b5c5e89a2 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -216,7 +216,7 @@ func testTomcatMetrics(t *testing.T) { func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { e2e.GenerateTraffic(t) - time.Sleep(e2e.LongWait) + time.Sleep(e2e.Wait) verifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E", false) }) } @@ -269,7 +269,7 @@ func testContainerInsightsMetrics(t *testing.T) { func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { e2e.GenerateTraffic(t) - time.Sleep(e2e.LongWait) + time.Sleep(e2e.Wait) verifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus", true) }) } @@ -279,7 +279,7 @@ func testTomcatRejectedSessions(t *testing.T) { //------------------------------------------------------------------------------ func verifyMetricAboveZero(t *testing.T, metricName, namespace string, containerInsights bool) { - startTime := time.Now().Add(-e2e.LongWait) + startTime := time.Now().Add(-e2e.Wait) endTime := time.Now() aboveZero, err := awsservice.CheckMetricAboveZero( diff --git a/test/e2e/testutils.go b/test/e2e/testutils.go index 5980f7f35..c44d2189d 100644 --- a/test/e2e/testutils.go +++ b/test/e2e/testutils.go @@ -20,7 +20,6 @@ import ( const ( Wait = 5 * time.Minute - LongWait = 10 * time.Minute interval = 30 * time.Second ) From 94b3d9c56d33fd6bfd08885661e462ac7ecff365 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 27 Dec 2024 15:34:14 -0500 Subject: [PATCH 86/92] added ami_type --- terraform/eks/e2e/main.tf | 2 +- terraform/eks/e2e/variables.tf | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 992669044..4320ef8e3 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -39,7 +39,7 @@ resource "aws_eks_node_group" "this" { min_size = var.nodes } - ami_type = "AL2_x86_64" + ami_type = var.ami_type capacity_type = "ON_DEMAND" disk_size = 20 instance_types = [var.instance_type] diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index 490fd1063..2dfac30fb 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -21,6 +21,11 @@ variable "nodes" { default = 2 } +variable "ami_type" { + type = string + default = "AL2_x86_64" +} + variable "instance_type" { type = string default = "t3a.medium" From 21c2eb137878ef84b4dc290198cecb83e3e25689 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 27 Dec 2024 15:42:13 -0500 Subject: [PATCH 87/92] add deployment strategy --- generator/test_case_generator.go | 1 + 1 file changed, 1 insertion(+) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 4daaae1d2..26c245623 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -31,6 +31,7 @@ type matrixRow struct { ValuesPerMinute int `json:"values_per_minute"` // Number of metrics to be sent or number of log lines to write K8sVersion string `json:"k8sVersion"` Nodes int `json:"nodes"` + DeploymentStrategy string `json:"deploymentStrategy"` TerraformDir string `json:"terraform_dir"` UseSSM bool `json:"useSSM"` ExcludedTests string `json:"excludedTests"` From e620f14c0bc87e7aa67c8e76752e0a1b4f392184 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 27 Dec 2024 15:43:15 -0500 Subject: [PATCH 88/92] fmt --- .../eksdeploymenttype/eks_deployment_type.go | 12 ++++++------ terraform/eks/e2e/main.tf | 6 +++--- terraform/eks/e2e/variables.tf | 2 +- test/e2e/testutils.go | 13 +++++++++---- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/environment/eksdeploymenttype/eks_deployment_type.go b/environment/eksdeploymenttype/eks_deployment_type.go index 0bd1ad301..856a2db00 100644 --- a/environment/eksdeploymenttype/eks_deployment_type.go +++ b/environment/eksdeploymenttype/eks_deployment_type.go @@ -8,17 +8,17 @@ import "strings" type EKSDeploymentType string const ( - DAEMON EKSDeploymentType = "DAEMON" - REPLICA EKSDeploymentType = "REPLICA" - SIDECAR EKSDeploymentType = "SIDECAR" + DAEMON EKSDeploymentType = "DAEMON" + REPLICA EKSDeploymentType = "REPLICA" + SIDECAR EKSDeploymentType = "SIDECAR" STATEFUL EKSDeploymentType = "STATEFUL" ) var ( eksDeploymentTypes = map[string]EKSDeploymentType{ - "DAEMON": DAEMON, - "REPLICA": REPLICA, - "SIDECAR": SIDECAR, + "DAEMON": DAEMON, + "REPLICA": REPLICA, + "SIDECAR": SIDECAR, "STATEFUL": STATEFUL, } ) diff --git a/terraform/eks/e2e/main.tf b/terraform/eks/e2e/main.tf index 4320ef8e3..2d23eda90 100644 --- a/terraform/eks/e2e/main.tf +++ b/terraform/eks/e2e/main.tf @@ -119,9 +119,9 @@ resource "null_resource" "validator" { depends_on = [aws_eks_cluster.this, aws_eks_node_group.this, null_resource.helm_charts] triggers = { - cluster_name = aws_eks_cluster.this.name - region = var.region - test_dir = var.test_dir + cluster_name = aws_eks_cluster.this.name + region = var.region + test_dir = var.test_dir eks_deployment_strategy = var.eks_deployment_strategy } diff --git a/terraform/eks/e2e/variables.tf b/terraform/eks/e2e/variables.tf index 2dfac30fb..ad8313e37 100644 --- a/terraform/eks/e2e/variables.tf +++ b/terraform/eks/e2e/variables.tf @@ -107,6 +107,6 @@ variable "sample_app" { } variable "eks_deployment_strategy" { - type = string + type = string default = "DAEMON" } diff --git a/test/e2e/testutils.go b/test/e2e/testutils.go index c44d2189d..b9849125b 100644 --- a/test/e2e/testutils.go +++ b/test/e2e/testutils.go @@ -1,17 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package e2e import ( "context" "fmt" - "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" "net/http" "os/exec" "strings" "testing" "time" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" ) //------------------------------------------------------------------------------ From f65adc64f4d74f74c153bf496a26694a975eb867 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 27 Dec 2024 16:42:00 -0500 Subject: [PATCH 89/92] move function to utils --- test/e2e/jmx/jmx_test.go | 25 ++----------------------- test/e2e/testutils.go | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index b5c5e89a2..d55cbc7bd 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -217,7 +217,7 @@ func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { e2e.GenerateTraffic(t) time.Sleep(e2e.Wait) - verifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E", false) + e2e.VerifyMetricAboveZero(t, "tomcat.sessions", nodeNames, "JVM_TOMCAT_E2E", false) }) } @@ -270,27 +270,6 @@ func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { e2e.GenerateTraffic(t) time.Sleep(e2e.Wait) - verifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus", true) + e2e.VerifyMetricAboveZero(t, "catalina_manager_rejectedsessions", nodeNames,"ContainerInsights/Prometheus", true) }) } - -//------------------------------------------------------------------------------ -// Helper Functions -//------------------------------------------------------------------------------ - -func verifyMetricAboveZero(t *testing.T, metricName, namespace string, containerInsights bool) { - startTime := time.Now().Add(-e2e.Wait) - endTime := time.Now() - - aboveZero, err := awsservice.CheckMetricAboveZero( - metricName, - namespace, - startTime, - endTime, - 60, - nodeNames, - containerInsights, - ) - require.NoError(t, err, "Failed to check metric above zero") - require.True(t, aboveZero, fmt.Sprintf("Expected non-zero %s after applying traffic", metricName)) -} diff --git a/test/e2e/testutils.go b/test/e2e/testutils.go index b9849125b..3cd6f7ff3 100644 --- a/test/e2e/testutils.go +++ b/test/e2e/testutils.go @@ -110,3 +110,20 @@ func GenerateTraffic(t *testing.T) { require.NoError(t, resp.Body.Close(), "Failed to close response body") } } + +func VerifyMetricAboveZero(t *testing.T, metricName string, nodeNames []string, namespace string, containerInsights bool) { + startTime := time.Now().Add(-Wait) + endTime := time.Now() + + aboveZero, err := awsservice.CheckMetricAboveZero( + metricName, + namespace, + startTime, + endTime, + 60, + nodeNames, + containerInsights, + ) + require.NoError(t, err, "Failed to check metric above zero") + require.True(t, aboveZero, fmt.Sprintf("Expected non-zero %s after applying traffic", metricName)) +} From fc9f53b9b30fc7042eba31da1702a360c57105f3 Mon Sep 17 00:00:00 2001 From: musa-asad Date: Fri, 27 Dec 2024 16:43:48 -0500 Subject: [PATCH 90/92] spacing --- test/e2e/jmx/jmx_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index d55cbc7bd..4fb77f2de 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -270,6 +270,6 @@ func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { e2e.GenerateTraffic(t) time.Sleep(e2e.Wait) - e2e.VerifyMetricAboveZero(t, "catalina_manager_rejectedsessions", nodeNames,"ContainerInsights/Prometheus", true) + e2e.VerifyMetricAboveZero(t, "catalina_manager_rejectedsessions", nodeNames, "ContainerInsights/Prometheus", true) }) } From f1dc36f56162cdc75ccf5c8d72af64304e3e956b Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 29 Dec 2024 21:42:20 -0500 Subject: [PATCH 91/92] remove flaky node names logic --- test/e2e/jmx/jmx_test.go | 19 ++---------------- test/e2e/testutils.go | 4 +--- util/awsservice/cloudwatchmetrics.go | 30 ++-------------------------- 3 files changed, 5 insertions(+), 48 deletions(-) diff --git a/test/e2e/jmx/jmx_test.go b/test/e2e/jmx/jmx_test.go index 4fb77f2de..4c8ba57f7 100644 --- a/test/e2e/jmx/jmx_test.go +++ b/test/e2e/jmx/jmx_test.go @@ -18,7 +18,6 @@ import ( "github.com/aws/amazon-cloudwatch-agent-test/environment" "github.com/aws/amazon-cloudwatch-agent-test/test/e2e" - "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" ) //------------------------------------------------------------------------------ @@ -26,7 +25,6 @@ import ( //------------------------------------------------------------------------------ var ( - nodeNames []string env *environment.MetaData ) @@ -84,19 +82,6 @@ func TestMain(m *testing.M) { os.Exit(1) } - // Get names of nodes so they can be used as dimensions to check for metrics - eksInstances, err := awsservice.GetEKSInstances(env.EKSClusterName) - if err != nil || len(eksInstances) == 0 { - fmt.Printf("Failed to get EKS instances: %v", err) - os.Exit(1) - } - - for _, instance := range eksInstances { - if instance.InstanceName != nil { - nodeNames = append(nodeNames, *instance.InstanceName) - } - } - os.Exit(m.Run()) } @@ -217,7 +202,7 @@ func testTomcatSessions(t *testing.T) { t.Run("verify_tomcat_sessions", func(t *testing.T) { e2e.GenerateTraffic(t) time.Sleep(e2e.Wait) - e2e.VerifyMetricAboveZero(t, "tomcat.sessions", nodeNames, "JVM_TOMCAT_E2E", false) + e2e.VerifyMetricAboveZero(t, "tomcat.sessions", "JVM_TOMCAT_E2E") }) } @@ -270,6 +255,6 @@ func testTomcatRejectedSessions(t *testing.T) { t.Run("verify_catalina_manager_rejectedsessions", func(t *testing.T) { e2e.GenerateTraffic(t) time.Sleep(e2e.Wait) - e2e.VerifyMetricAboveZero(t, "catalina_manager_rejectedsessions", nodeNames, "ContainerInsights/Prometheus", true) + e2e.VerifyMetricAboveZero(t, "catalina_manager_rejectedsessions", "ContainerInsights/Prometheus") }) } diff --git a/test/e2e/testutils.go b/test/e2e/testutils.go index 3cd6f7ff3..3e299a2a5 100644 --- a/test/e2e/testutils.go +++ b/test/e2e/testutils.go @@ -111,7 +111,7 @@ func GenerateTraffic(t *testing.T) { } } -func VerifyMetricAboveZero(t *testing.T, metricName string, nodeNames []string, namespace string, containerInsights bool) { +func VerifyMetricAboveZero(t *testing.T, metricName string, namespace string) { startTime := time.Now().Add(-Wait) endTime := time.Now() @@ -121,8 +121,6 @@ func VerifyMetricAboveZero(t *testing.T, metricName string, nodeNames []string, startTime, endTime, 60, - nodeNames, - containerInsights, ) require.NoError(t, err, "Failed to check metric above zero") require.True(t, aboveZero, fmt.Sprintf("Expected non-zero %s after applying traffic", metricName)) diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index 2b94e2696..d0ecfe7dc 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -140,13 +140,11 @@ func CheckMetricAboveZero( startTime time.Time, endTime time.Time, periodInSeconds int32, - nodeNames []string, - containerInsights bool, ) (bool, error) { metrics, err := CwmClient.ListMetrics(ctx, &cloudwatch.ListMetricsInput{ MetricName: aws.String(metricName), Namespace: aws.String(namespace), - RecentlyActive: "PT3H", + RecentlyActive: "PT5M", }) if err != nil { @@ -158,28 +156,6 @@ func CheckMetricAboveZero( } for _, metric := range metrics.Metrics { - // Skip node name check if containerInsights is true - if !containerInsights { - var nodeNameMatch bool - var nodeName string - for _, dim := range metric.Dimensions { - if *dim.Name == "k8s.node.name" { - nodeName = *dim.Value - for _, name := range nodeNames { - if nodeName == name { - nodeNameMatch = true - break - } - } - break - } - } - if !nodeNameMatch { - continue - } - log.Printf("Checking metric: %s for node: %s", *metric.MetricName, nodeName) - } - data, err := GetMetricStatistics( metricName, namespace, @@ -198,9 +174,7 @@ func CheckMetricAboveZero( for _, datapoint := range data.Datapoints { if *datapoint.Maximum > 0 { - if !containerInsights { - log.Printf("Found value above zero") - } + log.Printf("Found value above zero") return true, nil } } From 52f3a8639c9f8a1c6090c41971a39dcb08d83ddf Mon Sep 17 00:00:00 2001 From: musa-asad Date: Sun, 29 Dec 2024 21:53:39 -0500 Subject: [PATCH 92/92] set back to 3h --- util/awsservice/cloudwatchmetrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/awsservice/cloudwatchmetrics.go b/util/awsservice/cloudwatchmetrics.go index d0ecfe7dc..66c43875c 100644 --- a/util/awsservice/cloudwatchmetrics.go +++ b/util/awsservice/cloudwatchmetrics.go @@ -144,7 +144,7 @@ func CheckMetricAboveZero( metrics, err := CwmClient.ListMetrics(ctx, &cloudwatch.ListMetricsInput{ MetricName: aws.String(metricName), Namespace: aws.String(namespace), - RecentlyActive: "PT5M", + RecentlyActive: "PT3H", }) if err != nil {