Add test for ingress-custom-grpc-health-check

ingress/single-cluster/ingress-custom-grpc-health-check/cleanup.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit;
+set -o nounset;
+set -o pipefail;
+set -o xtrace;
+
+source ./test/helper.sh
+source ./test/test.conf
+test_name="ingress-custom-grpc-health-check"
+suffix=$(get_hash "${test_name}")
+context=$(kubectl config view -o json | jq -r ".contexts[] | select(.name | test(\"-${suffix}\")).name" || true)
+
+if [[ ! -z "${context}" ]]; then
+    ilb_ingress_name="fe-ilb-ingress"
+    ilb_fr=$(get_forwarding_rule "${ilb_ingress_name}" "${test_name}" "${context}")
+    ilb_thp=$(get_target_http_proxy "${ilb_ingress_name}" "${test_name}" "${context}")
+    ilb_thsp=$(get_target_https_proxy "${ilb_ingress_name}" "${test_name}" "${context}")
+    ilb_um=$(get_url_map "${ilb_ingress_name}" "${test_name}" "${context}")
+    ilb_backends=$(get_backends "${ilb_ingress_name}" "${test_name}" "${context}")
+    negs=$(get_negs "${context}")
+
+    xlb_ingress_name="fe-ingress"
+    xlb_fr=$(get_forwarding_rule "${xlb_ingress_name}" "${test_name}" "${context}")
+    xlb_thp=$(get_target_http_proxy "${xlb_ingress_name}" "${test_name}" "${context}")
+    xlb_thsp=$(get_target_https_proxy "${xlb_ingress_name}" "${test_name}" "${context}")
+    xlb_um=$(get_url_map "${xlb_ingress_name}" "${test_name}" "${context}")
+    xlb_backends=$(get_backends "${xlb_ingress_name}" "${test_name}" "${context}")
+
+    kubectl --context "${context}" delete -f ingress/single-cluster/ingress-custom-grpc-health-check/example/ -n "${test_name}" || true
+    wait_for_glbc_deletion "${ilb_fr}" "${ilb_thp}" "${ilb_thsp}" "${ilb_um}" "${ilb_backends}" "${negs}"
+    wait_for_glbc_deletion "${xlb_fr}" "${xlb_thp}" "${xlb_thsp}" "${xlb_um}" "${xlb_backends}" "${negs}"
+    kubectl --context "${context}" delete namespace "${test_name}" || true
+fi
+
+gcloud compute ssl-policies delete gke-ingress-ssl-policy-grpc --quiet || true
+cleanup_gke_basic "${test_name}" "${zone}" "${subnet_region}"

ingress/single-cluster/ingress-custom-grpc-health-check/example/README.md

@@ -10,10 +10,9 @@ $ gcloud container  clusters create cluster-1 \
   --zone us-central1-a  --num-nodes 2 --enable-ip-alias  -q
 ```
 
-Configure a custom SSL Policy (this is optional and simply added to demonstrate custom TLS policies using `networking.gke.io/v1beta1.FrontEndConfig`)
-
+Configure a custom SSL Policy.
 ```
-gcloud compute ssl-policies create gke-ingress-ssl-policy \
+gcloud compute ssl-policies create gke-ingress-ssl-policy-grpc \
     --profile MODERN \
     --min-tls-version 1.2 
 ```
@@ -116,10 +115,10 @@ $ docker run --add-host grpc.domain.com:$XLB_IP \
 
 #### Test Internal
 
-To test the internal loadbalancer, you must configure a VM from within an [allocated network](https://cloud.google.com/load-balancing/docs/l7-internal/setting-up-l7-internal#configuring_the_proxy-only_subnet) and export the environment variable `$XLB_IP` locally
+To test the internal loadbalancer, you must configure a VM from within an [allocated network](https://cloud.google.com/load-balancing/docs/l7-internal/setting-up-l7-internal#configuring_the_proxy-only_subnet) and export the environment variable `$ILB_IP` locally
 
 ```log
- $ docker run --add-host grpc.domain.com:$XLB_IP \
+ $ docker run --add-host grpc.domain.com:$ILB_IP \
      -t docker.io/salrashid123/grpc_app /grpc_client \
      --host=grpc.domain.com:443 --tlsCert /certs/CA_crt.pem  \
      --servername grpc.domain.com --repeat 10 -skipHealthCheck
@@ -150,7 +149,6 @@ data:
 kind: Secret
 metadata:
   name: hc-secret
-  namespace: default
 type: Opaque
 ---
 ```

ingress/single-cluster/ingress-custom-grpc-health-check/example/fe-ingress.yaml

@@ -42,4 +42,4 @@ kind: FrontendConfig
 metadata:
   name: fe-frontend-config
 spec:
-  sslPolicy: gke-ingress-ssl-policy
+  sslPolicy: gke-ingress-ssl-policy-grpc

ingress/single-cluster/ingress-custom-grpc-health-check/example/fe-secret.yaml

@@ -19,7 +19,6 @@ data:
 kind: Secret
 metadata:
   name: fe-secret
-  namespace: default
 type: Opaque
 ---
 

ingress/single-cluster/ingress-custom-grpc-health-check/run-test.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit;
+set -o nounset;
+set -o pipefail;
+set -o xtrace;
+
+source ./test/helper.sh
+source ./test/test.conf
+test_name="ingress-custom-grpc-health-check"
+suffix=$(get_hash "${test_name}")
+context=$(kubectl config view -o json | jq -r ".contexts[] | select(.name | test(\"-${suffix}\")).name" || true)
+
+if [[ -z "${context}" ]]; then
+    exit 1
+fi
+
+xlb_ip=$(wait_for_ingress_ip "fe-ingress" "${test_name}" "${context}")
+ilb_ip=$(wait_for_ingress_ip "fe-ilb-ingress" "${test_name}" "${context}")
+
+resource_name="gke-net-recipes-${suffix}"
+network="${resource_name}"
+instance="${resource_name}"
+
+repeating=10  # Number of RPC requests we are sending.
+pattern="fe-deployment" # Deployment name, inclueded in the echo.EchoServer SayHello function response.
+
+# Check ingress-grpc traffic by sending RPC request to load balancer IP,
+# and look for the pattern in the response.
+# Arguments:
+#   Load balancer IP.
+#   Number of RPC requests to send.
+#   Pattern to look for in the response.
+#   vm instance name. If provided, request will be sent via ssh into the instance.
+check_ingress_grpc_response() {
+    local vip repeating pattern instance eval_cmd ATTEMPT
+    vip="$1"
+    repeating="$2"
+    pattern="$3"
+    instance="${4:-}"
+    eval_cmd="docker run \
+                --add-host grpc.domain.com:${vip} \
+                -t \
+                docker.io/salrashid123/grpc_app /grpc_client \
+                --host=grpc.domain.com:443 \
+                --tlsCert /certs/CA_crt.pem  \
+                --servername grpc.domain.com --repeat ${repeating} -skipHealthCheck"
+
+    if [[ ! -z "${instance}" ]]; then
+        eval_cmd="gcloud compute ssh "${instance}" --zone="${zone}" -- \
+                    '{ docker --version || \
+                        (curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh); } > /dev/null && \
+                     sudo ${eval_cmd}'"
+    fi
+
+    for ATTEMPT in $(seq 60); do
+        local response
+        response=$(eval ${eval_cmd} || true)
+        if [[ -z "${response}" ]]; then
+            sleep 5
+            continue
+        fi
+
+        # Wait for server and SSL certificate to be ready.
+        if ! check_pattern_count "${response}" "${pattern}" "${repeating}"; then
+            sleep 5
+            continue
+        fi
+        return 0
+    done
+    return 1
+}
+
+check_ingress_grpc_response "${xlb_ip}" "${repeating}" "${pattern}"
+check_ingress_grpc_response "${ilb_ip}" "${repeating}" "${pattern}" "${instance}"

ingress/single-cluster/ingress-custom-grpc-health-check/setup.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit;
+set -o nounset;
+set -o pipefail;
+set -o xtrace;
+
+source ./test/helper.sh
+source ./test/test.conf
+test_name="ingress-custom-grpc-health-check"
+setup_gke_basic "${test_name}" "${zone}" "${subnet_region}"
+setup_ilb "${test_name}" "${subnet_region}"
+suffix=$(get_hash "${test_name}")
+context=$(kubectl config view -o json | jq -r ".contexts[] | select(.name | test(\"-${suffix}\")).name")
+
+if [[ -z "${context}" ]]; then
+    exit 1
+fi
+
+gcloud compute ssl-policies create gke-ingress-ssl-policy-grpc \
+    --profile MODERN \
+    --min-tls-version 1.2
+
+kubectl --context "${context}" create namespace "${test_name}"
+kubectl --context "${context}" apply -f ingress/single-cluster/ingress-custom-grpc-health-check/example/ -n "${test_name}"

test/helper.sh

@@ -199,7 +199,7 @@ setup_gke_basic() {
         --zone="${zone}" \
         --network="${network}" \
         --subnet="${subnet}" \
-        --image-family="debian-11" \
+        --image-family="debian-10" \
         --image-project="debian-cloud" \
         --tags="allow-ssh"
     gcloud container clusters create "${cluster}" \
@@ -336,3 +336,23 @@ check_http_status() {
     done
     return 1
 }
+
+# Check if the input string has the expected count of the given pattern.
+# Arguments:
+#   String to be counted.
+#   Pattern to look for.
+#   Expected count of pattern within the string.
+# Returns:
+#   0 if the pattern count matches the expected, 1 if not.
+check_pattern_count() {
+    local string pattern expect_count got_count
+    string="$1"
+    pattern="$2"
+    expect_count="$3"
+
+    got_count=$(echo "${string}" | grep -o "${pattern}" | wc -l)
+    if [[ "${expect_count}" != "${got_count}" ]]; then
+        return 1
+    fi
+    return 0
+}