feat: improve e2e tests (#105)

feat: improve e2e tests Signed-off-by: Bence Csati <[email protected]>
bank-vaults · Aug 8, 2024 · cf1646b · cf1646b
1 parent f0101d7
commit cf1646b
Show file tree

Hide file tree

Showing 5 changed files with 92 additions and 27 deletions.
diff --git a/e2e/main_test.go b/e2e/main_test.go
@@ -64,8 +64,7 @@ func TestMain(m *testing.M) {
 
 	// Set up cluster
 	if useRealCluster {
-		path := conf.ResolveKubeConfigFile()
-		cfg := envconf.NewWithKubeConfig(path)
+		cfg := envconf.NewWithKubeConfig(conf.ResolveKubeConfigFile())
 
 		if context := os.Getenv("USE_CONTEXT"); context != "" {
 			cfg.WithKubeContext(context)
@@ -233,8 +232,6 @@ func (e *reverseFinishEnvironment) Run(m *testing.M) int {
 	return e.Environment.Run(m)
 }
 
-// ======== VAULT ========
-
 func installVault(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
 	r, err := resources.New(cfg.Client().RESTConfig())
 	if err != nil {

diff --git a/e2e/test/deployment-init-seccontext-vault.yaml b/e2e/test/deployment-init-seccontext-vault.yaml
@@ -25,7 +25,7 @@ spec:
       containers:
         - name: alpine
           image: alpine
-          command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
+          command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
           env:
           - name: AWS_SECRET_ACCESS_KEY
             value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY

diff --git a/e2e/test/deployment-seccontext-vault.yaml b/e2e/test/deployment-seccontext-vault.yaml
@@ -25,7 +25,7 @@ spec:
       initContainers:
       - name: init-ubuntu
         image: ubuntu
-        command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
+        command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
         env:
         - name: AWS_SECRET_ACCESS_KEY
           value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY
@@ -36,7 +36,7 @@ spec:
       containers:
       - name: alpine
         image: alpine
-        command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
+        command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
         env:
         - name: AWS_SECRET_ACCESS_KEY
           value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY

diff --git a/e2e/test/deployment-vault.yaml b/e2e/test/deployment-vault.yaml
@@ -22,7 +22,7 @@ spec:
       initContainers:
       - name: init-ubuntu
         image: ubuntu
-        command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
+        command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo initContainers ready"]
         env:
         - name: AWS_SECRET_ACCESS_KEY
           value: vault:secret/data/accounts/aws#${.AWS_SECRET_ACCESS_KEY} # Go templates are also supported with ${} delimiters
@@ -33,7 +33,7 @@ spec:
       containers:
       - name: alpine
         image: alpine
-        command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
+        command: ["sh", "-c", "echo AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
         env:
         - name: AWS_SECRET_ACCESS_KEY
           value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY

diff --git a/e2e/webhook_test.go b/e2e/webhook_test.go
@@ -30,6 +30,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/e2e-framework/klient/decoder"
 	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
 	"sigs.k8s.io/e2e-framework/klient/wait"
@@ -172,24 +173,41 @@ func TestPodMutation(t *testing.T) {
 		}).
 		Assess("security context defaults are correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			r := cfg.Client().Resources()
-
 			pods := &v1.PodList{}
 
 			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault"))
 			require.NoError(t, err)
 
-			if len(pods.Items) == 0 {
-				t.Fatal("no pods found")
-			}
+			assert.NotEmpty(t, pods.Items, "no pods found")
 
 			securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext
-
 			assert.Nil(t, securityContext.RunAsNonRoot)
 			assert.Nil(t, securityContext.RunAsUser)
 			assert.Nil(t, securityContext.RunAsGroup)
 
 			return ctx
 		}).
+		Assess("secret values are injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			r := cfg.Client().Resources()
+			pods := &v1.PodList{}
+
+			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault"))
+			require.NoError(t, err)
+
+			assert.NotEmpty(t, pods.Items, "no pods found")
+
+			// wait for the container to become available
+			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
+			require.NoError(t, err)
+
+			initContainerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.InitContainers[1].Name)
+			assert.Contains(t, initContainerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")
+
+			containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name)
+			assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")
+
+			return ctx
+		}).
 		Feature()
 
 	deploymentSeccontextVault := applyResource(features.New("deployment-seccontext-vault"), "deployment-seccontext-vault.yaml").
@@ -204,6 +222,27 @@ func TestPodMutation(t *testing.T) {
 
 			return ctx
 		}).
+		Assess("secret values are injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			r := cfg.Client().Resources()
+			pods := &v1.PodList{}
+
+			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-vault"))
+			require.NoError(t, err)
+
+			assert.NotEmpty(t, pods.Items, "no pods found")
+
+			// wait for the container to become available
+			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
+			require.NoError(t, err)
+
+			initContainerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.InitContainers[1].Name)
+			assert.Contains(t, initContainerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")
+
+			containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name)
+			assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")
+
+			return ctx
+		}).
 		Feature()
 
 	deploymentTemplatingVault := applyResource(features.New("deployment-template-vault"), "deployment-template-vault.yaml").
@@ -222,29 +261,23 @@ func TestPodMutation(t *testing.T) {
 		}).
 		Assess("config template", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			r := cfg.Client().Resources()
-
 			pods := &v1.PodList{}
 
 			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-template-vault"))
 			require.NoError(t, err)
 
-			if len(pods.Items) == 0 {
-				t.Fatal("no pods found")
-			}
+			assert.NotEmpty(t, pods.Items, "no pods found")
 
 			// wait for the container to become available
 			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
 			require.NoError(t, err)
 
 			var stdout, stderr bytes.Buffer
-			podName := pods.Items[0].Name
 			command := []string{"cat", "/vault/secrets/config.yaml"}
-
-			if err := r.ExecInPod(context.TODO(), cfg.Namespace(), podName, "alpine", command, &stdout, &stderr); err != nil {
+			if err := r.ExecInPod(ctx, cfg.Namespace(), pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name, command, &stdout, &stderr); err != nil {
 				t.Log(stderr.String())
 				t.Fatal(err)
 			}
-
 			assert.Equal(t, "\n    {\n      \"id\": \"secretId\",\n      \"key\": \"s3cr3t\"\n    }\n    \n  ", stdout.String())
 
 			return ctx
@@ -265,22 +298,18 @@ func TestPodMutation(t *testing.T) {
 		}).
 		Assess("security context is correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 			r := cfg.Client().Resources()
-
 			pods := &v1.PodList{}
 
 			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-init-seccontext-vault"))
 			require.NoError(t, err)
 
-			if len(pods.Items) == 0 {
-				t.Fatal("no pods found")
-			}
+			assert.NotEmpty(t, pods.Items, "no pods found")
 
 			// wait for the container to become available
 			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
 			require.NoError(t, err)
 
 			securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext
-
 			require.NotNil(t, securityContext.RunAsNonRoot)
 			assert.Equal(t, true, *securityContext.RunAsNonRoot)
 			require.NotNil(t, securityContext.RunAsUser)
@@ -290,6 +319,24 @@ func TestPodMutation(t *testing.T) {
 
 			return ctx
 		}).
+		Assess("secret value is injected", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			r := cfg.Client().Resources()
+			pods := &v1.PodList{}
+
+			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-init-seccontext-vault"))
+			require.NoError(t, err)
+
+			assert.NotEmpty(t, pods.Items, "no pods found")
+
+			// wait for the container to become available
+			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(defaultTimeout))
+			require.NoError(t, err)
+
+			containerLogs := getLogsFromContainer(t, ctx, cfg, pods.Items[0].Name, pods.Items[0].Spec.Containers[0].Name)
+			assert.Contains(t, containerLogs, "AWS_SECRET_ACCESS_KEY=s3cr3t")
+
+			return ctx
+		}).
 		Feature()
 
 	testenv.Test(t, deploymentVault, deploymentSeccontextVault, deploymentTemplatingVault, deploymentInitSeccontextVault)
@@ -318,3 +365,24 @@ func applyResource(builder *features.FeatureBuilder, file string) *features.Feat
 			return ctx
 		})
 }
+
+func getLogsFromContainer(t *testing.T, ctx context.Context, cfg *envconf.Config, podName string, containerName string) string {
+	clientset, err := kubernetes.NewForConfig(cfg.Client().RESTConfig())
+	require.NoError(t, err)
+
+	req := clientset.CoreV1().Pods(cfg.Namespace()).GetLogs(
+		podName,
+		&v1.PodLogOptions{
+			Container: containerName,
+		})
+
+	podLogs, err := req.Stream(ctx)
+	require.NoError(t, err)
+	defer podLogs.Close()
+
+	var buf bytes.Buffer
+	_, err = buf.ReadFrom(podLogs)
+	require.NoError(t, err)
+
+	return buf.String()
+}