From 2c29e0b123455e6187351428b3113cac0b0555c6 Mon Sep 17 00:00:00 2001
From: David Cheung <cheungdavid@google.com>
Date: Thu, 19 Oct 2023 20:58:27 +0000
Subject: [PATCH] Add test for ingress-custom-grpc-health-check

* Update SSL policy name in the recipe README and yaml to avoid naming
  collision.
* Add a helper function to check if a string has the expected pattern
  count.
* Update instance image version since the offical docker install script
  doesn't work for debian-11.
* Add test for ingress-grpc recipe. The traffic is validated by running
  a grpc-client image on a docker and send grpc request to the load
  balancer IP.
---
 .../cleanup.sh                                | 51 +++++++++++
 .../example/README.md                         | 10 +--
 .../example/fe-ingress.yaml                   |  2 +-
 .../example/fe-secret.yaml                    |  1 -
 .../run-test.sh                               | 89 +++++++++++++++++++
 .../ingress-custom-grpc-health-check/setup.sh | 39 ++++++++
 test/helper.sh                                | 22 ++++-
 7 files changed, 205 insertions(+), 9 deletions(-)
 create mode 100755 ingress/single-cluster/ingress-custom-grpc-health-check/cleanup.sh
 create mode 100755 ingress/single-cluster/ingress-custom-grpc-health-check/run-test.sh
 create mode 100755 ingress/single-cluster/ingress-custom-grpc-health-check/setup.sh

diff --git a/ingress/single-cluster/ingress-custom-grpc-health-check/cleanup.sh b/ingress/single-cluster/ingress-custom-grpc-health-check/cleanup.sh
new file mode 100755
index 00000000..ef90d006
--- /dev/null
+++ b/ingress/single-cluster/ingress-custom-grpc-health-check/cleanup.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit;
+set -o nounset;
+set -o pipefail;
+set -o xtrace;
+
+source ./test/helper.sh
+source ./test/test.conf
+test_name="ingress-custom-grpc-health-check"
+suffix=$(get_hash "${test_name}")
+context=$(kubectl config view -o json | jq -r ".contexts[] | select(.name | test(\"-${suffix}\")).name" || true)
+
+if [[ ! -z "${context}" ]]; then
+    ilb_ingress_name="fe-ilb-ingress"
+    ilb_fr=$(get_forwarding_rule "${ilb_ingress_name}" "${test_name}" "${context}")
+    ilb_thp=$(get_target_http_proxy "${ilb_ingress_name}" "${test_name}" "${context}")
+    ilb_thsp=$(get_target_https_proxy "${ilb_ingress_name}" "${test_name}" "${context}")
+    ilb_um=$(get_url_map "${ilb_ingress_name}" "${test_name}" "${context}")
+    ilb_backends=$(get_backends "${ilb_ingress_name}" "${test_name}" "${context}")
+    negs=$(get_negs "${context}")
+
+    xlb_ingress_name="fe-ingress"
+    xlb_fr=$(get_forwarding_rule "${xlb_ingress_name}" "${test_name}" "${context}")
+    xlb_thp=$(get_target_http_proxy "${xlb_ingress_name}" "${test_name}" "${context}")
+    xlb_thsp=$(get_target_https_proxy "${xlb_ingress_name}" "${test_name}" "${context}")
+    xlb_um=$(get_url_map "${xlb_ingress_name}" "${test_name}" "${context}")
+    xlb_backends=$(get_backends "${xlb_ingress_name}" "${test_name}" "${context}")
+
+    kubectl --context "${context}" delete -f ingress/single-cluster/ingress-custom-grpc-health-check/example/ -n "${test_name}" || true
+    wait_for_glbc_deletion "${ilb_fr}" "${ilb_thp}" "${ilb_thsp}" "${ilb_um}" "${ilb_backends}" "${negs}"
+    wait_for_glbc_deletion "${xlb_fr}" "${xlb_thp}" "${xlb_thsp}" "${xlb_um}" "${xlb_backends}" "${negs}"
+    kubectl --context "${context}" delete namespace "${test_name}" || true
+fi
+
+gcloud compute ssl-policies delete gke-ingress-ssl-policy-grpc --quiet || true
+cleanup_gke_basic "${test_name}" "${zone}" "${subnet_region}"
diff --git a/ingress/single-cluster/ingress-custom-grpc-health-check/example/README.md b/ingress/single-cluster/ingress-custom-grpc-health-check/example/README.md
index a0c2c744..89d70405 100644
--- a/ingress/single-cluster/ingress-custom-grpc-health-check/example/README.md
+++ b/ingress/single-cluster/ingress-custom-grpc-health-check/example/README.md
@@ -10,10 +10,9 @@ $ gcloud container  clusters create cluster-1 \
   --zone us-central1-a  --num-nodes 2 --enable-ip-alias  -q
 ```
 
-Configure a custom SSL Policy (this is optional and simply added to demonstrate custom TLS policies using `networking.gke.io/v1beta1.FrontEndConfig`)
-
+Configure a custom SSL Policy.
 ```
-gcloud compute ssl-policies create gke-ingress-ssl-policy \
+gcloud compute ssl-policies create gke-ingress-ssl-policy-grpc \
     --profile MODERN \
     --min-tls-version 1.2 
 ```
@@ -116,10 +115,10 @@ $ docker run --add-host grpc.domain.com:$XLB_IP \
 
 #### Test Internal
 
-To test the internal loadbalancer, you must configure a VM from within an [allocated network](https://cloud.google.com/load-balancing/docs/l7-internal/setting-up-l7-internal#configuring_the_proxy-only_subnet) and export the environment variable `$XLB_IP` locally
+To test the internal loadbalancer, you must configure a VM from within an [allocated network](https://cloud.google.com/load-balancing/docs/l7-internal/setting-up-l7-internal#configuring_the_proxy-only_subnet) and export the environment variable `$ILB_IP` locally
 
 ```log
- $ docker run --add-host grpc.domain.com:$XLB_IP \
+ $ docker run --add-host grpc.domain.com:$ILB_IP \
      -t docker.io/salrashid123/grpc_app /grpc_client \
      --host=grpc.domain.com:443 --tlsCert /certs/CA_crt.pem  \
      --servername grpc.domain.com --repeat 10 -skipHealthCheck
@@ -150,7 +149,6 @@ data:
 kind: Secret
 metadata:
   name: hc-secret
-  namespace: default
 type: Opaque
 ---
 ```
diff --git a/ingress/single-cluster/ingress-custom-grpc-health-check/example/fe-ingress.yaml b/ingress/single-cluster/ingress-custom-grpc-health-check/example/fe-ingress.yaml
index f39d5911..e79ed333 100644
--- a/ingress/single-cluster/ingress-custom-grpc-health-check/example/fe-ingress.yaml
+++ b/ingress/single-cluster/ingress-custom-grpc-health-check/example/fe-ingress.yaml
@@ -42,4 +42,4 @@ kind: FrontendConfig
 metadata:
   name: fe-frontend-config
 spec:
-  sslPolicy: gke-ingress-ssl-policy
\ No newline at end of file
+  sslPolicy: gke-ingress-ssl-policy-grpc
\ No newline at end of file
diff --git a/ingress/single-cluster/ingress-custom-grpc-health-check/example/fe-secret.yaml b/ingress/single-cluster/ingress-custom-grpc-health-check/example/fe-secret.yaml
index 135b25aa..b2d89760 100644
--- a/ingress/single-cluster/ingress-custom-grpc-health-check/example/fe-secret.yaml
+++ b/ingress/single-cluster/ingress-custom-grpc-health-check/example/fe-secret.yaml
@@ -19,7 +19,6 @@ data:
 kind: Secret
 metadata:
   name: fe-secret
-  namespace: default
 type: Opaque
 ---
 
diff --git a/ingress/single-cluster/ingress-custom-grpc-health-check/run-test.sh b/ingress/single-cluster/ingress-custom-grpc-health-check/run-test.sh
new file mode 100755
index 00000000..9a99e06a
--- /dev/null
+++ b/ingress/single-cluster/ingress-custom-grpc-health-check/run-test.sh
@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit;
+set -o nounset;
+set -o pipefail;
+set -o xtrace;
+
+source ./test/helper.sh
+source ./test/test.conf
+test_name="ingress-custom-grpc-health-check"
+suffix=$(get_hash "${test_name}")
+context=$(kubectl config view -o json | jq -r ".contexts[] | select(.name | test(\"-${suffix}\")).name" || true)
+
+if [[ -z "${context}" ]]; then
+    exit 1
+fi
+
+xlb_ip=$(wait_for_ingress_ip "fe-ingress" "${test_name}" "${context}")
+ilb_ip=$(wait_for_ingress_ip "fe-ilb-ingress" "${test_name}" "${context}")
+
+resource_name="gke-net-recipes-${suffix}"
+network="${resource_name}"
+instance="${resource_name}"
+
+repeating=10  # Number of RPC requests we are sending.
+pattern="fe-deployment" # Deployment name, inclueded in the echo.EchoServer SayHello function response.
+
+# Check ingress-grpc traffic by sending RPC request to load balancer IP,
+# and look for the pattern in the response.
+# Arguments:
+#   Load balancer IP.
+#   Number of RPC requests to send.
+#   Pattern to look for in the response.
+#   vm instance name. If provided, request will be sent via ssh into the instance.
+check_ingress_grpc_response() {
+    local vip repeating pattern instance eval_cmd ATTEMPT
+    vip="$1"
+    repeating="$2"
+    pattern="$3"
+    instance="${4:-}"
+    eval_cmd="docker run \
+                --add-host grpc.domain.com:${vip} \
+                -t \
+                docker.io/salrashid123/grpc_app /grpc_client \
+                --host=grpc.domain.com:443 \
+                --tlsCert /certs/CA_crt.pem  \
+                --servername grpc.domain.com --repeat ${repeating} -skipHealthCheck"
+
+    if [[ ! -z "${instance}" ]]; then
+        eval_cmd="gcloud compute ssh "${instance}" --zone="${zone}" -- \
+                    '{ docker --version || \
+                        (curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh); } > /dev/null && \
+                     sudo ${eval_cmd}'"
+    fi
+
+    for ATTEMPT in $(seq 60); do
+        local response
+        response=$(eval ${eval_cmd} || true)
+        if [[ -z "${response}" ]]; then
+            sleep 5
+            continue
+        fi
+
+        # Wait for server and SSL certificate to be ready.
+        if ! check_pattern_count "${response}" "${pattern}" "${repeating}"; then
+            sleep 5
+            continue
+        fi
+        return 0
+    done
+    return 1
+}
+
+check_ingress_grpc_response "${xlb_ip}" "${repeating}" "${pattern}"
+check_ingress_grpc_response "${ilb_ip}" "${repeating}" "${pattern}" "${instance}"
diff --git a/ingress/single-cluster/ingress-custom-grpc-health-check/setup.sh b/ingress/single-cluster/ingress-custom-grpc-health-check/setup.sh
new file mode 100755
index 00000000..ded18419
--- /dev/null
+++ b/ingress/single-cluster/ingress-custom-grpc-health-check/setup.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit;
+set -o nounset;
+set -o pipefail;
+set -o xtrace;
+
+source ./test/helper.sh
+source ./test/test.conf
+test_name="ingress-custom-grpc-health-check"
+setup_gke_basic "${test_name}" "${zone}" "${subnet_region}"
+setup_ilb "${test_name}" "${subnet_region}"
+suffix=$(get_hash "${test_name}")
+context=$(kubectl config view -o json | jq -r ".contexts[] | select(.name | test(\"-${suffix}\")).name")
+
+if [[ -z "${context}" ]]; then
+    exit 1
+fi
+
+gcloud compute ssl-policies create gke-ingress-ssl-policy-grpc \
+    --profile MODERN \
+    --min-tls-version 1.2
+
+kubectl --context "${context}" create namespace "${test_name}"
+kubectl --context "${context}" apply -f ingress/single-cluster/ingress-custom-grpc-health-check/example/ -n "${test_name}"
diff --git a/test/helper.sh b/test/helper.sh
index 2de9acdd..905e251b 100644
--- a/test/helper.sh
+++ b/test/helper.sh
@@ -199,7 +199,7 @@ setup_gke_basic() {
         --zone="${zone}" \
         --network="${network}" \
         --subnet="${subnet}" \
-        --image-family="debian-11" \
+        --image-family="debian-10" \
         --image-project="debian-cloud" \
         --tags="allow-ssh"
     gcloud container clusters create "${cluster}" \
@@ -336,3 +336,23 @@ check_http_status() {
     done
     return 1
 }
+
+# Check if the input string has the expected count of the given pattern.
+# Arguments:
+#   String to be counted.
+#   Pattern to look for.
+#   Expected count of pattern within the string.
+# Returns:
+#   0 if the pattern count matches the expected, 1 if not.
+check_pattern_count() {
+    local string pattern expect_count got_count
+    string="$1"
+    pattern="$2"
+    expect_count="$3"
+
+    got_count=$(echo "${string}" | grep -o "${pattern}" | wc -l)
+    if [[ "${expect_count}" != "${got_count}" ]]; then
+        return 1
+    fi
+    return 0
+}