actions.yml

name: Crystal Specs
on:
  push:
    paths:
      - '**'
      - '!**.md'
  pull_request:
    paths:
      - '**'
      - '!**.md'
jobs:
  tests:
    name: Fetch Matrix Tests
    runs-on: [ubuntu-latest]
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - id: set-matrix
      run: |
        JSON="{\"include\":["
        TEST_ARRAY=$(grep -roP --no-filename 'tags: \K(\[|")(.*)(\]|")' spec/ | tr -d '[],' | tr -s '\n' ' ' | xargs -n1 | sort -u | xargs)
        TEST_ARRAY=("${TEST_ARRAY[@]/testsuite-config-lifecycle/}")
        TEST_ARRAY=("${TEST_ARRAY[@]/testsuite-microservice/}")
        TEST_ARRAY=("${TEST_ARRAY[@]/testsuite-all/}")
        TEST_ARRAY=("${TEST_ARRAY[@]/disk_fill/}")
        TEST_ARRAY=("${TEST_ARRAY[@]/chaos_container_kill/}")
        TEST_ARRAY=("${TEST_ARRAY[@]/chaos_cpu_hog/}")
        TEST_ARRAY=("${TEST_ARRAY[@]/pod_delete/}")
        TEST_ARRAY=("${TEST_ARRAY[@]/pod_io_stress/}")
        TEST_ARRAY=("${TEST_ARRAY[@]/pod_memory_hog/}")
        TEST_ARRAY=("${TEST_ARRAY[@]/pod_network_latency/}")
        TEST_ARRAY=("${TEST_ARRAY[@]/zombie/}")
        TEST_ARRAY=("${TEST_ARRAY[@]/oran/}")
        TEST_LIST=$(for i in ${TEST_ARRAY[@]}
        do
                 echo "{\"spec\":\"$i\"}," | tr -d '\n'
        done)
        TEST_LIST="${TEST_LIST%?}"
        JSON="$JSON$TEST_LIST"
        JSON="$JSON]}"

        echo "TESTS: $JSON"
        echo "matrix=$JSON" >> $GITHUB_OUTPUT

  spec:
    name: Crystal Specs
    needs: [tests]
    runs-on: [v1.0.0]
    strategy:
      fail-fast: false
      matrix: ${{fromJson(needs.tests.outputs.matrix)}}
    steps:
    - name: Cleanup Tmp DIR
      run: |
        sudo rm -rf /tmp/*
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Mirror Setup
      run: |
        cat << EOF > /tmp/cluster.yml
        kind: Cluster
        apiVersion: kind.x-k8s.io/v1alpha4
        nodes:
        - role: control-plane
          image: kindest/node:v1.23.13@sha256:e7968cda1b4ff790d5b0b5b0c29bda0404cdb825fd939fe50fd5accc43e3a730
        containerdConfigPatches:
        - |-
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
            endpoint = ["http://139.178.70.81:80"]
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:5000"]
            endpoint = ["http://localhost:5000"]
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.default.svc.cluster.local:5000"]
            endpoint = ["http://localhost:5000"]
        EOF
    - name: sysctls specs kind config override
      if: matrix.spec == 'sysctls'
      run: |
        cat << EOF > /tmp/cluster.yml
        kind: Cluster
        apiVersion: kind.x-k8s.io/v1alpha4

        # Enabled additional unsafe sysctls to support the negative spec test for sysctls
        nodes:
        - role: control-plane
          image: kindest/node:v1.23.13@sha256:e7968cda1b4ff790d5b0b5b0c29bda0404cdb825fd939fe50fd5accc43e3a730
          kubeadmConfigPatches:
          - |
            kind: KubeletConfiguration
            allowedUnsafeSysctls: ["kernel.msg*"]
        EOF
    - name: Mirror Override
      if: matrix.spec == 'private_registry_image'
      run: |
        cat << EOF > /tmp/cluster.yml
        kind: Cluster
        apiVersion: kind.x-k8s.io/v1alpha4
        nodes:
        - role: control-plane
          image: kindest/node:v1.23.13@sha256:e7968cda1b4ff790d5b0b5b0c29bda0404cdb825fd939fe50fd5accc43e3a730
        containerdConfigPatches:
          - |-
            [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:5000"]
              endpoint = ["http://localhost:5000"]
            [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.default.svc.cluster.local:5000"]
              endpoint = ["http://localhost:5000"]
    - name: Install Latest Kind
      env:
        KIND_VERSION: v0.17.0
        KIND_URL: https://kind.sigs.k8s.io/dl
      run: |
        echo "Existing kind binary path: $(which kind)"
        if [[ -s $(which kind) ]]; then sudo rm $(which kind); fi
        wget -O kind "$KIND_URL/$KIND_VERSION/kind-linux-amd64" --progress=dot:giga; 
        chmod +x ./kind
        sudo mv ./kind /usr/local/bin/kind
        kind --version
    - name: Create Kind Cluster 
      run: |
        cat /tmp/cluster.yml
        export CLUSTER=$(uuidgen)
        echo "export CLUSTER=$CLUSTER" > cluster.env
        echo kind create cluster --name $CLUSTER --config=/tmp/cluster.yml --kubeconfig ./$CLUSTER.conf
        kind --version
        kind create cluster --name $CLUSTER --config=/tmp/cluster.yml --kubeconfig ./$CLUSTER.conf
        export KUBECONFIG=$(pwd)/$CLUSTER.conf
        kubectl get nodes 
    - name: Cache crystal shards
      uses: actions/cache@v3
      env:
        cache-name: cache-crystal-shards
      with:
        path: ./lib
        key: lib-${{ hashFiles('**/shard.lock') }}
        restore-keys: |
          lib-
    - name: Setup CNF-Conformance
      run: |
        helm repo add stable https://cncf.gitlab.io/stable
        git fetch --all --tags --force
        shards install  
        echo "RUNNER: $RUNNER_NAME"
    - name: Run Crystal Spec
      env:
        FALCO_ENV: CI
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        DOCKERHUB_USERNAMES: ${{ secrets.DOCKERHUB_USERNAMES }}
        DOCKERHUB_PASSWORDS: ${{ secrets.DOCKERHUB_PASSWORDS }}
        DOCKERHUB_EMAIL: ${{ secrets.DOCKERHUB_EMAIL }}
        IMAGE_REPO: ${{ secrets.IMAGE_REPO }}
      run: |
        USERNAME_ARRAY=($DOCKERHUB_USERNAMES)
        PASSWORD_ARRAY=($DOCKERHUB_PASSWORDS)
        EMAIL_ARRAY=($DOCKERHUB_EMAIL)
        IMAGE_ARRAY=($IMAGE_REPO)
        
        RANDOMIZER=$(( 0 + $RANDOM % 3 ))

        export DOCKERHUB_USERNAME=${USERNAME_ARRAY[$RANDOMIZER]}
        export DOCKERHUB_PASSWORD=${PASSWORD_ARRAY[$RANDOMIZER]}
        
        export PROTECTED_DOCKERHUB_USERNAME=$DOCKERHUB_USERNAME
        export PROTECTED_DOCKERHUB_PASSWORD=$DOCKERHUB_PASSWORD
        export PROTECTED_DOCKERHUB_EMAIL=${EMAIL_ARRAY[$RANDOMIZER]}
        export PROTECTED_IMAGE_REPO=${IMAGE_ARRAY[$RANDOMIZER]}

        docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD

        source cluster.env
        export KUBECONFIG=$(pwd)/$CLUSTER.conf
        until [[ $(kubectl get pods -l app=local-path-provisioner --namespace=local-path-storage -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') == "True" ]]; do
            echo "Waiting for local-path-storage"
            sleep 1
        done
        LOCAL_PATH_STORAGE_POD=$(kubectl get pods -l app=local-path-provisioner --namespace=local-path-storage -o jsonpath='{range .items[*]}{.metadata.name}')
        #        until [[ $(kubectl exec -ti $LOCAL_PATH_STORAGE_POD --namespace=local-path-storage -- apk add curl jq) ]]; do
        #   echo "Failed to install packages, retrying"
        #   sleep 1
        #done
        CLUSTER_RATE_LIMIT=$(kubectl exec -ti $LOCAL_PATH_STORAGE_POD --namespace=local-path-storage -- curl --head -H "Authorization: Bearer $(curl "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token)" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep ratelimit-remaining || true)
        TOKEN=$(curl "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token || true)
        ANONYMOUS_RUNNER_RATE_LIMIT=$(curl --head -H "Authorization: Bearer $TOKEN" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep ratelimit-remaining || echo "Runner Rate Limit Exceeded: $RUNNER_NAME")
        TOKEN=$(curl --user "$DOCKERHUB_USERNAME:$DOCKERHUB_PASSWORD" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token)
        AUTH_RATE_LIMIT=$(curl --head -H "Authorization: Bearer $TOKEN" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep ratelimit-remaining || echo "Authenticated Rate Limit Exceeded")
        echo "RUNNER RATE LIMIT: $ANONYMOUS_RUNNER_RATE_LIMIT"
        echo "CLUSTER RATE LIMIT: $CLUSTER_RATE_LIMIT" 
        echo "DOCKER USER RATE LIMIT: $AUTH_RATE_LIMIT" 

        LOG_LEVEL=info crystal spec --warnings none --tag ${{ matrix.spec }} -v


    - name: Delete Cluster
      if: ${{ always() }}
      run: |
        source cluster.env
        kind delete cluster --name $CLUSTER
        docker container prune -f || true
        docker volume prune -f || true
        
  chaos:
    name: Chaos & Oran Tests
    needs: [tests]
    runs-on: [ubuntu-22.04]
    strategy:
      fail-fast: false
      matrix:
        tag: ["pod_delete", "pod_io_stress", "pod_memory_hog", "pod_network_latency", "disk_fill", "pod_network_corruption", "pod_network_duplication", "zombie", "oran"]
    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Install Latest Kind
      env:
        KIND_VERSION: v0.17.0
        KIND_URL: https://kind.sigs.k8s.io/dl
      run: |
        echo "Existing kind binary path: $(which kind)"
        if [[ -s $(which kind) ]]; then sudo rm $(which kind); fi
        wget -O kind "$KIND_URL/$KIND_VERSION/kind-linux-amd64" --progress=dot:giga; 
        chmod +x ./kind
        sudo mv ./kind /usr/local/bin/kind
        kind --version
    - name: Install kubectl
      run: |
        wget -O kubectl "https://dl.k8s.io/release/v1.28.3/bin/linux/amd64/kubectl"
        ls -la
        sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl

    - name: Create Kind Cluster 
      run: |
        cat << EOF > /tmp/cluster.yml
        kind: Cluster
        apiVersion: kind.x-k8s.io/v1alpha4
        nodes:
        - role: control-plane
          image: kindest/node:v1.23.13@sha256:e7968cda1b4ff790d5b0b5b0c29bda0404cdb825fd939fe50fd5accc43e3a730
        containerdConfigPatches:
        - |-
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:5000"]
            endpoint = ["http://localhost:5000"]
        EOF
        export CLUSTER=$(uuidgen)
        echo "export CLUSTER=$CLUSTER" > cluster.env
        echo kind create cluster --name $CLUSTER --config=/tmp/cluster.yml --kubeconfig ./$CLUSTER.conf
        kind --version
        kind create cluster --name $CLUSTER --config=/tmp/cluster.yml --kubeconfig ./$CLUSTER.conf
        export KUBECONFIG=$(pwd)/$CLUSTER.conf
        kubectl get nodes 

    - name: Cache crystal shards
      uses: actions/cache@v3
      env:
        cache-name: cache-crystal-shards
      with:
        path: ./lib
        key: lib-${{ hashFiles('**/shard.lock') }}
        restore-keys: |
          lib-
    - name: Install Crystal
      uses: crystal-lang/install-crystal@v1
      with:
        crystal: 1.6.2
    - name: Setup CNF-Conformance
      run: |
        helm repo add stable https://cncf.gitlab.io/stable
        git fetch --all --tags --force
        shards install  
        echo "RUNNER: $RUNNER_NAME"
    - name: Run Crystal Spec
      run: |
        echo "Current path: $(echo pwd)"
        source cluster.env
        export KUBECONFIG=$(pwd)/$CLUSTER.conf
        until [[ $(kubectl get pods -l app=kindnet --namespace=kube-system -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') == "True" ]]; do
            echo "Waiting for kindnet"
            sleep 1
        done

        CLUSTER_RATE_LIMIT=$(kubectl run -i tmp-shell --restart=Never --rm --image ubuntu -- /bin/bash -c "apt update && apt install -y curl jq; curl --head -H \"Authorization: Bearer $(curl "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token)\" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest" | grep ratelimit-remaining || true)

        TOKEN=$(curl "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token || true)
        ANONYMOUS_RUNNER_RATE_LIMIT=$(curl --head -H "Authorization: Bearer $TOKEN" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep ratelimit-remaining || echo "Runner Rate Limit Exceeded: $RUNNER_NAME")
        TOKEN=$(curl --user "$DOCKERHUB_USERNAME:$DOCKERHUB_PASSWORD" "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token)
        AUTH_RATE_LIMIT=$(curl --head -H "Authorization: Bearer $TOKEN" https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep ratelimit-remaining || echo "Authenticated Rate Limit Exceeded")
        echo "RUNNER RATE LIMIT: $ANONYMOUS_RUNNER_RATE_LIMIT"
        echo "CLUSTER RATE LIMIT: $CLUSTER_RATE_LIMIT"
        echo "DOCKER USER RATE LIMIT: $AUTH_RATE_LIMIT"
        LOG_LEVEL=info crystal spec --warnings none --tag ${{ matrix.tag }} -v

  # chaos-airgapped:
  #   name: Chaos Tests Airgapped
  #   needs: [tests]
  #   runs-on: [ubuntu-latest]
  #   strategy:
  #     fail-fast: false
  #     matrix:
  #       test: ["pod_delete", "pod_io_stress", "pod_memory_hog", "pod_network_latency", "disk_fill", "pod_network_corruption", "pod_network_duplication"]
  #   env:
  #     OFFLINE_IMAGE: "conformance/offline:crystal-1.0.0-cache"
  #   steps: 
  #   - name: Maximize build space
  #     uses: easimon/maximize-build-space@master
  #     with:
  #       root-reserve-mb: 35000
  #       swap-size-mb: 100
  #       remove-dotnet: 'true'
  #       remove-android: 'true'
  #       remove-haskell: 'true'
  #   - name: Checkout code
  #     uses: actions/checkout@v3
  #     with:
  #       fetch-depth: 0
  #   - name: Cache crystal shards
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: cache-crystal-shards
  #     with:
  #       path: ./lib
  #       key: lib-${{ hashFiles('**/shard.lock') }}
  #       restore-keys: |
  #         lib-
  #   - name: Install Crystal
  #     env:
  #       CRYSTAL_VERSION: 1.0.0
  #       CRYSTAL_URL: https://github.com/crystal-lang/crystal/releases/download
  #     run: |
  #       wget -O crystal.deb "$CRYSTAL_URL/$CRYSTAL_VERSION/crystal_$CRYSTAL_VERSION-1_amd64.deb" --progress=dot:giga; 
  #       sudo apt install -y --no-install-recommends \
  #       git \
  #       libssl-dev \
  #       libxml2-dev \
  #       libyaml-dev \
  #       libgmp-dev \
  #       libz-dev \
  #       ./crystal.deb \
  #       && sudo rm -rf /var/lib/apt/lists/*
  #   - name: Cache airgapped.tar.gz
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: cache-airgapped-tar
  #     with:
  #       path: /tmp/airgapped.tar.gz
  #       key: airgapped-${{ hashFiles('**/utils/airgap/airgap.cr', '**/src/tasks/utils/cnf_manager.cr') }}
  #   - name: Create airgapped.tar.gz if one is not found in cache
  #     run: |
  #       sudo mv /tmp/airgapped.tar.gz $(pwd) || true ; sudo rm -rf /tmp/ ; sudo mkdir /tmp ; sudo mount /dev/buildvg/buildlv /tmp; sudo chmod 777 /tmp -R
  #       shards install
  #       crystal src/cnf-testsuite.cr setup 
  #       helm repo add stable https://cncf.gitlab.io/stable
  #       if ! [ -f "/tmp/airgapped.tar.gz" ]; then 
  #             echo "Cached airgapped.tar.gz not found, re-creating airgapped.tar.gz"
  #             LOG_LEVEL=info crystal src/cnf-testsuite.cr airgapped output-file=/tmp/airgapped.tar.gz
  #             LOG_LEVEL=info crystal src/cnf-testsuite.cr cnf_setup cnf-config=example-cnfs/coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz
  #       else
  #             echo "Cached airgapped.tar.gz file found, using cache for JOB_ID: $GITHUB_JOB"
  #       fi
  #       echo "Checking for CoreDNS"
  #       tar -C /tmp/ -xvf /tmp/airgapped.tar.gz
  #       if [ -f "/tmp/images/coredns_1.6.7.tar" ]; then
  #          echo "CoreDNS found, CNF Setup Ran."
  #       else
  #          echo "CoreDNS not found, Re-Running CNF Setup."
  #          LOG_LEVEL=debug crystal src/cnf-testsuite.cr cnf_setup cnf-config=example-cnfs/coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz
  #       fi
  #   - name: Create Cluster & Run Tests.
  #     run: |
  #       sed -i "/${{ matrix.test }}/a \ \ required: true" $(pwd)/embedded_files/points.yml
  #       docker run --entrypoint=/bin/bash --name shards -v $(pwd):/cnf-testsuite -t $OFFLINE_IMAGE -c "shards install"
  #       docker run --name cluster --network none --privileged -e KUBECONFIG=/root/.kube/config -v $(pwd):/cnf-testsuite -v /tmp/airgapped.tar.gz:/airgapped/airgapped.tar.gz -v $(pwd)/tmpdata:/tmp -t $OFFLINE_IMAGE /bin/bash -c "LOG_LEVEL=info crystal src/cnf-testsuite.cr setup offline=/airgapped/airgapped.tar.gz && LOG_LEVEL=info crystal src/cnf-testsuite.cr install_litmus offline=true && LOG_LEVEL=info crystal src/cnf-testsuite.cr cnf_setup cnf-config=./example-cnfs/coredns/cnf-testsuite.yml input-file=/airgapped/airgapped.tar.gz && ./.github/workflows/check_litmus.sh && LOG_LEVEL=info crystal src/cnf-testsuite.cr ${{ matrix.test }} offline=true verbose strict"

  build:
    name: Build Release
    runs-on: ubuntu-latest
    env:
      CRYSTAL_IMAGE: "conformance/crystal:1.6.2-alpine"
    steps: 
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Cache crystal shards
      uses: actions/cache@v3
      env:
        cache-name: cache-crystal-shards
      with:
        path: ./lib
        key: lib-${{ hashFiles('**/shard.lock') }}
        restore-keys: |
          lib-
    - name: Build Release
      run: |
        docker pull $CRYSTAL_IMAGE
        docker run --rm -v $PWD:/workspace -w /workspace $CRYSTAL_IMAGE shards install
        docker run --rm -v $PWD:/workspace -w /workspace $CRYSTAL_IMAGE crystal build --warnings none src/cnf-testsuite.cr --release --static --link-flags '-lxml2 -llzma'
    - name: upload artifact
      uses: actions/upload-artifact@v3
      with:
        name: release
        path: cnf-testsuite
        

  #TODO Split into a new repo #1160
  # setup_airgapped_env:
  #   name: Test Setup Command In An Airgapped Env.
  #   runs-on: [v1.0.0]
  #   env:
  #     OFFLINE_IMAGE: "conformance/offline:crystal-1.0.0-cache"
  #   steps: 
  #   - name: Cleanup Tmp DIR
  #     run: |
  #       sudo rm -rf /tmp/*
  #   - name: Checkout code
  #     uses: actions/checkout@v3
  #     with:
  #       fetch-depth: 0
  #   - name: Print Runner Name
  #     run: |
  #       echo "RUNNER: $RUNNER_NAME"
  #   - name: Cache crystal shards
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: cache-crystal-shards
  #     with:
  #       path: ./lib
  #       key: lib-${{ hashFiles('**/shard.lock') }}
  #       restore-keys: |
  #         lib-
  #   - name: Cache airgapped.tar.gz
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: setup-airgapped-tar
  #     with:
  #       path: /tmp/airgapped.tar.gz
  #       key: setup-${{ hashFiles('**/utils/airgap/airgap.cr', '**/src/tasks/utils/cnf_manager.cr') }}
  #   - name: Create environment.tar if one is not found in cache
  #     run: |
  #       if ! [ -f "/tmp/airgapped.tar.gz" ]; then 
  #             echo "Cached airgapped.tar.gz not found, re-creating airgapped.tar.gz"
  #             shards install
  #             export DIR=$(uuidgen)
  #             mkdir /shared/$DIR
  #             echo "export DIR=$DIR" > /tmp/environment.env
  #             cp -a $(pwd) /shared/$DIR/cnf-testsuite
  #             pushd /shared/$DIR/cnf-testsuite
  #             LOG_LEVEL=info crystal src/cnf-testsuite.cr setup 
  #             LOG_LEVEL=info crystal src/cnf-testsuite.cr airgapped output-file=/shared/$DIR/airgapped.tar.gz
  #             cp /shared/$DIR/airgapped.tar.gz /tmp/airgapped.tar.gz
  #             popd
  #             docker run --entrypoint=/bin/bash --name $DIR-shards -v /shared/$DIR/cnf-testsuite:/cnf-testsuite -t $OFFLINE_IMAGE -c "shards install"
  #           else
  #             echo "Cached airgapped.tar.gz file found, using cache for JOB_ID: $GITHUB_JOB"
  #             shards install
  #             export DIR=$(uuidgen)
  #             mkdir /shared/$DIR
  #             echo "export DIR=$DIR" > /tmp/environment.env
  #             cp -a $(pwd) /shared/$DIR/cnf-testsuite
  #             docker run --entrypoint=/bin/bash --name $DIR-shards -v /shared/$DIR/cnf-testsuite:/cnf-testsuite -t $OFFLINE_IMAGE -c "shards install"
  #             cp /tmp/airgapped.tar.gz /shared/$DIR/airgapped.tar.gz
  #       fi
  #   - name: Create Cluster & Run Tests.
  #     run: |
  #       source /tmp/environment.env
  #       docker run --name $DIR --network none --privileged -v /shared/$DIR/cnf-testsuite:/cnf-testsuite -v /shared/$DIR/airgapped.tar.gz:/airgapped/airgapped.tar.gz -v /shared/$DIR/cnf-testsuite/tmpdata:/tmp -t $OFFLINE_IMAGE /bin/bash -c "LOG_LEVEL=info crystal src/cnf-testsuite.cr setup offline=/airgapped/airgapped.tar.gz && LOG_LEVEL=info crystal src/cnf-testsuite.cr install_litmus offline=true && ./.github/workflows/check_litmus.sh"
  #   - name: Delete Cluster
  #     if: ${{ always() }}
  #     run: |
  #       source /tmp/environment.env
  #       docker rm -f $DIR
  #       docker rm -f $DIR-shards
  #       docker rm -f $DIR-cache
  #       sudo rm -rf /shared/$DIR
  #       docker container prune -f || true
  #       docker volume prune -f || true
  #       docker network prune -f || true

  # opa-airgapped:
  #   name: Run OPA tests in airgapped env
  #   runs-on: [v1.0.0]
  #   env:
  #     OFFLINE_IMAGE: "conformance/offline:crystal-1.0.0-r1"
  #   steps:
  #   - name: Cleanup tmp DIR
  #     run: |
  #       sudo rm -rf /tmp/*
  #   - name: Checkout code
  #     uses: actions/checkout@v3
  #     with:
  #       fetch-depth: 0
  #   - name: Cache crystal shards
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: cache-crystal-shards
  #     with:
  #       path: ./lib
  #       key: lib-${{ hashFiles('**/shard.lock') }}
  #       restore-keys: |
  #         lib-
  #   - name: Cache airgapped.tar.gz
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: cache-airgapped-tar
  #     with:
  #       path: /tmp/airgapped.tar.gz
  #       key: airgapped-$GITHUB_JOB-reset122-${{ hashFiles('**/src/tasks/airgap.cr', '**/src/tasks/utils/airgap.cr', '**/src/tasks/utils/airgap_utils.cr', '**/src/tasks/utils/tar.cr', '**/src/tasks/utils/cnf_manager.cr') }}
  #   - name: Create airgapped.tar.gz if one is not found in cache
  #     run: |
  #       shards install
  #       crystal src/cnf-testsuite.cr setup
  #       helm repo add stable https://cncf.gitlab.io/stable
  #       if ! [ -f "/tmp/airgapped.tar.gz" ]; then
  #             echo "Cached airgapped.tar.gz not found, re-creating airgapped.tar.gz"
  #             crystal src/cnf-testsuite.cr airgapped output-file=/tmp/airgapped.tar.gz
  #             LOG_LEVEL=debug crystal src/cnf-testsuite.cr cnf_setup cnf-config=example-cnfs/coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz
  #       else
  #             echo "Cached airgapped.tar.gz file found, using cache for JOB_ID: $GITHUB_JOB"
  #       fi
  #       echo "Checking for CoreDNS"
  #       tar -C /tmp/ -xvf /tmp/airgapped.tar.gz
  #       if [ -f "/tmp/images/coredns_1.6.7.tar" ]; then
  #          echo "CoreDNS found, CNF Setup Ran."
  #       else
  #          echo "CoreDNS not found, Re-Running CNF Setup."
  #          LOG_LEVEL=debug crystal src/cnf-testsuite.cr cnf_setup cnf-config=example-cnfs/coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz
  #       fi
  #   - name: Create Cluster & Run Tests.
  #     env:
  #       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  #     run: |
  #       export DIR=$(uuidgen)
  #       echo "export DIR=$DIR" > dir.env
  #       mkdir /shared/$DIR
  #       # Create Airgapped Tar
  #       #DOTO Use pre-created airgapped.tar.gz
  #       # wget --auth-no-challenge --header='Accept:application/octet-stream' -H "Authorization: Bearer $GITHUB_TOKEN" https://api.github.com/repos/cncf/cnf-testsuite/releases/assets/38092818 -O airgapped.tar.gz
  #       cp -a $(pwd) /shared/$DIR/cnf-testsuite
  #       cp /tmp/airgapped.tar.gz /shared/$DIR/
  #       sed -i "/versioned_tag/a \ \ required: true" embedded_files/points.yml
  #       docker run --entrypoint=/bin/bash --name $DIR-shards -v /shared/$DIR/cnf-testsuite:/cnf-testsuite -t $OFFLINE_IMAGE -c "shards install"
  #       docker run --name $DIR --network none --privileged -v /shared/$DIR/cnf-testsuite:/cnf-testsuite -v /shared/$DIR/airgapped.tar.gz:/airgapped/airgapped.tar.gz -v /shared/$DIR/tmpdata:/tmp -t $OFFLINE_IMAGE /bin/bash -c "LOG_LEVEL=info crystal src/cnf-testsuite.cr cnf_setup cnf-config=./example-cnfs/coredns/cnf-testsuite.yml input-file=/airgapped/airgapped.tar.gz && LOG_LEVEL=info crystal src/cnf-testsuite.cr versioned_tag verbose strict offline=true"
  #   - name: Delete Cluster
  #     if: ${{ always() }}
  #     run: |
  #       source dir.env
  #       docker rm -f $DIR
  #       docker rm -f $DIR-shards
  #       sudo rm -rf /shared/$DIR
  #       docker container prune -f || true
  #       docker volume prune -f || true
  #       docker network prune -f || true
        
  # test_helm_chart_in_airgapped_env:
  #   name: Test helm chart In An Airgapped Env.
  #   runs-on: [v1.0.0]
  #   env:
  #     OFFLINE_IMAGE: "conformance/offline:crystal-1.0.0-r1"
  #   steps: 
  #   - name: Cleanup Tmp DIR
  #     run: |
  #       sudo rm -rf /tmp/*
  #   - name: Checkout code
  #     uses: actions/checkout@v3
  #     with:
  #       fetch-depth: 0
  #   - name: Cache crystal shards
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: cache-crystal-shards
  #     with:
  #       path: ./lib
  #       key: lib-${{ hashFiles('**/shard.lock') }}
  #       restore-keys: |
  #         lib-
  #   - name: Cache airgapped.tar.gz
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: cache-airgapped-tar
  #     with:
  #       path: /tmp/airgapped.tar.gz
  #       key: airgapped-$GITHUB_JOB-reset122-${{ hashFiles('**/src/tasks/airgap.cr', '**/src/tasks/utils/airgap.cr', '**/src/tasks/utils/airgap_utils.cr', '**/src/tasks/utils/tar.cr', '**/src/tasks/utils/cnf_manager.cr') }}
  #   - name: Create airgapped.tar.gz if one is not found in cache
  #     run: |
  #       shards install
  #       crystal src/cnf-testsuite.cr setup 
  #       helm repo add stable https://cncf.gitlab.io/stable
  #       if ! [ -f "/tmp/airgapped.tar.gz" ]; then 
  #             echo "Cached airgapped.tar.gz not found, re-creating airgapped.tar.gz"
  #             crystal src/cnf-testsuite.cr airgapped output-file=/tmp/airgapped.tar.gz
  #             LOG_LEVEL=debug crystal src/cnf-testsuite.cr cnf_setup cnf-config=example-cnfs/coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz
  #       else
  #             echo "Cached airgapped.tar.gz file found, using cache for JOB_ID: $GITHUB_JOB"
  #       fi
  #       echo "Checking for CoreDNS"
  #       tar -C /tmp/ -xvf /tmp/airgapped.tar.gz
  #       if [ -f "/tmp/images/coredns_1.6.7.tar" ]; then
  #          echo "CoreDNS found, CNF Setup Ran."
  #       else
  #          echo "CoreDNS not found, Re-Running CNF Setup."
  #          LOG_LEVEL=debug crystal src/cnf-testsuite.cr cnf_setup cnf-config=example-cnfs/coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz
  #       fi
  #   - name: Create Cluster & Run Tests.
  #     env:
  #       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  #     run: |
  #       export DIR=$(uuidgen)
  #       echo "export DIR=$DIR" > dir.env
  #       mkdir /shared/$DIR
  #       # Create Airgapped Tar
  #       #DOTO Use pre-created airgapped.tar.gz
  #       # wget --auth-no-challenge --header='Accept:application/octet-stream' -H "Authorization: Bearer $GITHUB_TOKEN" https://api.github.com/repos/cncf/cnf-testsuite/releases/assets/38092818 -O airgapped.tar.gz
  #       cp -a $(pwd) /shared/$DIR/cnf-testsuite
  #       cp /tmp/airgapped.tar.gz /shared/$DIR/
  #       docker run --entrypoint=/bin/bash --name $DIR-shards -v /shared/$DIR/cnf-testsuite:/cnf-testsuite -t $OFFLINE_IMAGE -c "shards install"
  #       docker run --name $DIR --network none --privileged -e KUBECONFIG=/root/.kube/config -v /shared/$DIR/cnf-testsuite:/cnf-testsuite -v /shared/$DIR/airgapped.tar.gz:/airgapped/airgapped.tar.gz -v /shared/$DIR/tmpdata:/tmp -t $OFFLINE_IMAGE /bin/bash -c "LOG_LEVEL=info crystal src/cnf-testsuite.cr cnf_setup cnf-config=./example-cnfs/coredns/cnf-testsuite.yml input-file=/airgapped/airgapped.tar.gz && LOG_LEVEL=info crystal src/cnf-testsuite.cr workload offline=true ~microservice ~resilience ~volume_hostpath_not_found ~non_root_user ~privileged ~increase_capacity ~decrease_capacity ~install_script_helm ~helm_chart_valid ~helm_chart_published verbose"
  #   - name: Delete Cluster
  #     if: ${{ always() }}
  #     run: |
  #       source dir.env
  #       docker rm -f $DIR
  #       docker rm -f $DIR-shards
  #       sudo rm -rf /shared/$DIR
  #       docker container prune -f || true
  #       docker volume prune -f || true
  #       docker network prune -f || true


  # test_helm_directory_in_airgapped_env:
  #   name: Test helm directory In An Airgapped Env.
  #   runs-on: [v1.0.0]
  #   env:
  #     OFFLINE_IMAGE: "conformance/offline:crystal-1.0.0-r1"
  #   steps: 
  #   - name: Cleanup Tmp DIR
  #     run: |
  #       sudo rm -rf /tmp/*
  #   - name: Checkout code
  #     uses: actions/checkout@v3
  #     with:
  #       fetch-depth: 0
  #   - name: Cache crystal shards
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: cache-crystal-shards
  #     with:
  #       path: ./lib
  #       key: lib-${{ hashFiles('**/shard.lock') }}
  #       restore-keys: |
  #         lib-
  #   - name: Cache airgapped.tar.gz
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: cache-airgapped-tar
  #     with:
  #       path: /tmp/airgapped.tar.gz
  #       key: airgapped-$GITHUB_JOB-reset122-${{ hashFiles('**/src/tasks/airgap.cr', '**/src/tasks/utils/airgap.cr', '**/src/tasks/utils/airgap_utils.cr', '**/src/tasks/utils/tar.cr', '**/src/tasks/utils/cnf_manager.cr') }}
  #   - name: Create airgapped.tar.gz if one is not found in cache
  #     run: |
  #       shards install
  #       crystal src/cnf-testsuite.cr setup 
  #       helm repo add stable https://cncf.gitlab.io/stable
  #       if ! [ -f "/tmp/airgapped.tar.gz" ]; then 
  #             echo "Cached airgapped.tar.gz not found, re-creating airgapped.tar.gz"
  #             crystal src/cnf-testsuite.cr airgapped output-file=/tmp/airgapped.tar.gz
  #             LOG_LEVEL=debug crystal src/cnf-testsuite.cr cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz
  #       else
  #             echo "Cached airgapped.tar.gz file found, using cache for JOB_ID: $GITHUB_JOB"
  #       fi
  #       echo "Checking for CoreDNS"
  #       tar -C /tmp/ -xvf /tmp/airgapped.tar.gz
  #       if [ -f "/tmp/images/coredns_1.6.7.tar" ]; then
  #          echo "CoreDNS found, CNF Setup Ran."
  #       else
  #          echo "CoreDNS not found, Re-Running CNF Setup."
  #          LOG_LEVEL=debug crystal src/cnf-testsuite.cr cnf_setup cnf-config=sample-cnfs/sample_coredns/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz
  #       fi
  #   - name: Create Cluster & Run Tests.
  #     env:
  #       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  #     run: |
  #       export DIR=$(uuidgen)
  #       echo "export DIR=$DIR" > dir.env
  #       mkdir /shared/$DIR
  #       #DOTO Use pre-created airgapped.tar.gz
  #       cp -a $(pwd) /shared/$DIR/cnf-testsuite
  #       cp /tmp/airgapped.tar.gz /shared/$DIR/
  #       docker run --entrypoint=/bin/bash --name $DIR-shards -v /shared/$DIR/cnf-testsuite:/cnf-testsuite -t $OFFLINE_IMAGE -c "shards install"
  #       docker run --name $DIR --network none --privileged -e KUBECONFIG=/root/.kube/config -v /shared/$DIR/cnf-testsuite:/cnf-testsuite -v /shared/$DIR/airgapped.tar.gz:/airgapped/airgapped.tar.gz -v /shared/$DIR/tmpdata:/tmp -t $OFFLINE_IMAGE /bin/bash -c "LOG_LEVEL=info crystal src/cnf-testsuite.cr cnf_setup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml input-file=/airgapped/airgapped.tar.gz && LOG_LEVEL=info crystal src/cnf-testsuite.cr workload offline=true ~microservice ~resilience ~volume_hostpath_not_found ~non_root_user ~privileged ~increase_capacity ~decrease_capacity ~install_script_helm ~helm_chart_valid ~helm_chart_published verbose"
  #   - name: Delete Cluster
  #     if: ${{ always() }}
  #     run: |
  #       source dir.env
  #       docker rm -f $DIR
  #       docker rm -f $DIR-shards
  #       sudo rm -rf /shared/$DIR
  #       docker container prune -f || true
  #       docker volume prune -f || true
  #       docker network prune -f || true


  # test_manifest_directory_in_airgapped_env:
  #   name: Test manifest directory In An Airgapped Env.
  #   runs-on: [v1.0.0]
  #   env:
  #     OFFLINE_IMAGE: "conformance/offline:crystal-1.0.0-r1"
  #   steps: 
  #   - name: Cleanup Tmp DIR
  #     run: |
  #       sudo rm -rf /tmp/*
  #   - name: Checkout code
  #     uses: actions/checkout@v3
  #     with:
  #       fetch-depth: 0
  #   - name: Cache crystal shards
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: cache-crystal-shards
  #     with:
  #       path: ./lib
  #       key: lib-${{ hashFiles('**/shard.lock') }}
  #       restore-keys: |
  #         lib-
  #   - name: Cache airgapped.tar.gz
  #     uses: actions/cache@v3
  #     env:
  #       cache-name: cache-airgapped-tar
  #     with:
  #       path: /tmp/airgapped.tar.gz
  #       key: airgapped-$GITHUB_JOB-reset122-${{ hashFiles('**/src/tasks/airgap.cr', '**/src/tasks/utils/airgap.cr', '**/src/tasks/utils/airgap_utils.cr', '**/src/tasks/utils/tar.cr', '**/src/tasks/utils/cnf_manager.cr') }}
  #   - name: Create airgapped.tar.gz if one is not found in cache
  #     run: |
  #       shards install
  #       crystal src/cnf-testsuite.cr setup 
  #       helm repo add stable https://cncf.gitlab.io/stable
  #       if ! [ -f "/tmp/airgapped.tar.gz" ]; then 
  #             echo "Cached airgapped.tar.gz not found, re-creating airgapped.tar.gz"
  #             crystal src/cnf-testsuite.cr airgapped output-file=/tmp/airgapped.tar.gz
  #             LOG_LEVEL=debug crystal src/cnf-testsuite.cr cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz
  #       else
  #             echo "Cached airgapped.tar.gz file found, using cache for JOB_ID: $GITHUB_JOB"
  #       fi
  #       echo "Checking for Nginx"
  #       tar -C /tmp/ -xvf /tmp/airgapped.tar.gz
  #       if [ -f "/tmp/images/nginx_1.21.0.tar" ]; then
  #          echo "Nginx found, CNF Setup Ran."
  #       else
  #          echo "Nginx not found, Re-Running CNF Setup."
  #          LOG_LEVEL=debug crystal src/cnf-testsuite.cr cnf_setup cnf-config=sample-cnfs/k8s-non-helm/cnf-testsuite.yml airgapped=/tmp/airgapped.tar.gz
  #       fi
  #   - name: Create Cluster & Run Tests.
  #     env:
  #       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  #     run: |
  #       export DIR=$(uuidgen)
  #       echo "export DIR=$DIR" > dir.env
  #       mkdir /shared/$DIR
  #        #Create Airgapped Tar
  #       #DOTO Use pre-created airgapped.tar.gz
  #       cp -a $(pwd) /shared/$DIR/cnf-testsuite
  #       cp /tmp/airgapped.tar.gz /shared/$DIR/
  #       docker run --entrypoint=/bin/bash --name $DIR-shards -v /shared/$DIR/cnf-testsuite:/cnf-testsuite -t $OFFLINE_IMAGE -c "shards install"
  #       docker run --name $DIR --network none --privileged -e KUBECONFIG=/root/.kube/config -v /shared/$DIR/cnf-testsuite:/cnf-testsuite -v /shared/$DIR/airgapped.tar.gz:/airgapped/airgapped.tar.gz -v /shared/$DIR/tmpdata:/tmp -t $OFFLINE_IMAGE /bin/bash -c "LOG_LEVEL=info crystal src/cnf-testsuite.cr cnf_setup cnf-config=./sample-cnfs/k8s-non-helm/cnf-testsuite.yml input-file=/airgapped/airgapped.tar.gz && LOG_LEVEL=info crystal src/cnf-testsuite.cr workload offline=true ~microservice ~resilience ~volume_hostpath_not_found ~non_root_user ~privileged ~increase_capacity ~decrease_capacity ~install_script_helm ~helm_chart_valid ~helm_chart_published verbose"
  #   - name: Delete Cluster
  #     if: ${{ always() }}
  #     run: |
  #       source dir.env
  #       docker rm -f $DIR
  #       docker rm -f $DIR-shards
  #       sudo rm -rf /shared/$DIR
  #       docker container prune -f || true
  #       docker volume prune -f || true
  #       docker network prune -f || true

  test_binary_configuration_lifecycle:
    name: Test Binary Without Source(config_lifecycle)
    runs-on: [v1.0.0]
    steps: 
    - name: Cleanup Tmp DIR
      run: |
        sudo rm -rf /tmp/*
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Cache crystal shards
      uses: actions/cache@v3
      env:
        cache-name: cache-crystal-shards
      with:
        path: ./lib
        key: lib-${{ hashFiles('**/shard.lock') }}
        restore-keys: |
          lib-
    - name: Install Latest Kind
      env:
        KIND_VERSION: v0.17.0
        KIND_URL: https://kind.sigs.k8s.io/dl
      run: |
        echo "Existing kind binary path: $(which kind)"
        if [[ -s $(which kind) ]]; then sudo rm $(which kind); fi
        wget -O kind "$KIND_URL/$KIND_VERSION/kind-linux-amd64" --progress=dot:giga; 
        chmod +x ./kind
        sudo mv ./kind /usr/local/bin/kind
        kind --version
    - name: Build cnf-testsuite & Create Kind Cluster 
      run: |
        shards install
        crystal build src/cnf-testsuite.cr --warnings none &
        cat << EOF > /tmp/cluster.yml
        kind: Cluster
        apiVersion: kind.x-k8s.io/v1alpha4
        nodes:
        - role: control-plane
          image: kindest/node:v1.23.13@sha256:e7968cda1b4ff790d5b0b5b0c29bda0404cdb825fd939fe50fd5accc43e3a730
        containerdConfigPatches:
        - |-
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:5000"]
            endpoint = ["http://localhost:5000"]
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.default.svc.cluster.local:5000"]
            endpoint = ["http://localhost:5000"]
        EOF
        export CLUSTER=$(uuidgen)
        echo "export CLUSTER=$CLUSTER"
        echo "export CLUSTER=$CLUSTER" > cluster.env
        kind --version
        kind create cluster --name $CLUSTER --config=/tmp/cluster.yml --kubeconfig /tmp/$CLUSTER.conf
        export KUBECONFIG=/tmp/$CLUSTER.conf
        kubectl get nodes 
    - name: Run Test Suite without source(config_lifecycle)
      run: |
        source cluster.env
        echo "SHARDS_INSTALL_PATH: $SHARDS_INSTALL_PATH"
        export KUBECONFIG=/tmp/$CLUSTER.conf
        helm repo add stable https://cncf.gitlab.io/stable
        export DIR=$(uuidgen)
        echo "Shared DIR: /shared/$DIR"
        mkdir /shared/$DIR
        mv cnf-testsuite /shared/$DIR 
        cd /shared/$DIR
        # ./cnf-testsuite setup
        # wget -O cnf-testsuite.yml https://raw.githubusercontent.com/cncf/cnf-testsuite/main/example-cnfs/coredns/cnf-testsuite.yml
        # ./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml
        # export FALCO_ENV="CI"
    #    LOG_LEVEL=info ./cnf-testsuite all ~compatibility ~resilience ~reasonable_startup_time ~reasonable_image_size ~platform ~volume_hostpath_not_found ~privileged ~increase_capacity ~decrease_capacity ~install_script_helm ~helm_chart_valid ~helm_chart_published verbose
    - name: Delete Cluster
      if: ${{ always() }}
      run: |
        source cluster.env
        kind delete cluster --name $CLUSTER
        docker container prune -f || true
        docker volume prune -f || true
        docker network prune -f || true

  test_binary_microservice:
    name: Test Binary Without Source(microservice)
    runs-on: [v1.0.0]
    steps: 
    - name: Cleanup Tmp DIR
      run: |
        sudo rm -rf /tmp/*
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Cache crystal shards
      uses: actions/cache@v3
      env:
        cache-name: cache-crystal-shards
      with:
        path: ./lib
        key: lib-${{ hashFiles('**/shard.lock') }}
        restore-keys: |
          lib-
    - name: Install Latest Kind
      env:
        KIND_VERSION: v0.17.0
        KIND_URL: https://kind.sigs.k8s.io/dl
      run: |
        echo "Existing kind binary path: $(which kind)"
        if [[ -s $(which kind) ]]; then sudo rm $(which kind); fi
        wget -O kind "$KIND_URL/$KIND_VERSION/kind-linux-amd64" --progress=dot:giga; 
        chmod +x ./kind
        sudo mv ./kind /usr/local/bin/kind
        kind --version
    - name: Build cnf-testsuite & Create Kind Cluster 
      run: |
        shards install
        crystal build src/cnf-testsuite.cr --warnings none &
        cat << EOF > /tmp/cluster.yml
        kind: Cluster
        apiVersion: kind.x-k8s.io/v1alpha4
        containerdConfigPatches:
        - |-
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:5000"]
            endpoint = ["http://localhost:5000"]
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.default.svc.cluster.local:5000"]
            endpoint = ["http://localhost:5000"]
        EOF
        export CLUSTER=$(uuidgen)
        echo "export CLUSTER=$CLUSTER" > cluster.env
        kind --version
        kind create cluster --name $CLUSTER --config=/tmp/cluster.yml --kubeconfig /tmp/$CLUSTER.conf
        export KUBECONFIG=/tmp/$CLUSTER.conf
        kubectl get nodes 
    - name: Run Test Suite without source(microservice)
      run: |
        source cluster.env
        export KUBECONFIG=/tmp/$CLUSTER.conf
        helm repo add stable https://cncf.gitlab.io/stable
        ./cnf-testsuite setup
        wget -O cnf-testsuite.yml https://raw.githubusercontent.com/cncf/cnf-testsuite/main/example-cnfs/coredns/cnf-testsuite.yml
        ./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml
        export FALCO_ENV="CI"
        LOG_LEVEL=info ./cnf-testsuite all ~resilience ~compatibility ~pod_network_latency ~platform ~volume_hostpath_not_found ~privileged ~increase_capacity ~decrease_capacity ~ip_addresses ~liveness ~readiness ~rolling_update ~rolling_downgrade ~rolling_version_change ~nodeport_not_used ~hostport_not_used ~hardcoded_ip_addresses_in_k8s_runtime_configuration ~install_script_helm ~helm_chart_valid ~helm_chart_published ~rollback ~secrets_used ~immutable_configmap verbose
    - name: Delete Cluster
      if: ${{ always() }}
      run: |
        source cluster.env
        kind delete cluster --name $CLUSTER
        docker container prune -f || true
        docker volume prune -f || true
        docker network prune -f || true

  test_binary_all:
    name: Test Binary Without Source(all)
    runs-on: [v1.0.0]
    steps: 
    - name: Cleanup Tmp DIR
      run: |
        sudo rm -rf /tmp/*
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Cache crystal shards
      uses: actions/cache@v3
      env:
        cache-name: cache-crystal-shards
      with:
        path: ./lib
        key: lib-${{ hashFiles('**/shard.lock') }}
        restore-keys: |
          lib-
    - name: Install Latest Kind
      env:
        KIND_VERSION: v0.17.0
        KIND_URL: https://kind.sigs.k8s.io/dl
      run: |
        echo "Existing kind binary path: $(which kind)"
        if [[ -s $(which kind) ]]; then sudo rm $(which kind); fi
        wget -O kind "$KIND_URL/$KIND_VERSION/kind-linux-amd64" --progress=dot:giga; 
        chmod +x ./kind
        sudo mv ./kind /usr/local/bin/kind
        kind --version
    - name: Build cnf-testsuite & Create Kind Cluster 
      run: |
        shards install
        crystal build src/cnf-testsuite.cr --warnings none &
        cat << EOF > /tmp/cluster.yml
        kind: Cluster
        apiVersion: kind.x-k8s.io/v1alpha4
        containerdConfigPatches:
        - |-
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:5000"]
            endpoint = ["http://localhost:5000"]
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.default.svc.cluster.local:5000"]
            endpoint = ["http://localhost:5000"]
        EOF
        export CLUSTER=$(uuidgen)
        echo "export CLUSTER=$CLUSTER" > cluster.env
        kind --version
        kind create cluster --name $CLUSTER --config=/tmp/cluster.yml --kubeconfig /tmp/$CLUSTER.conf
        export KUBECONFIG=/tmp/$CLUSTER.conf
        kubectl get nodes 
    - name: Run Test Suite without source(all)
      run: |
        source cluster.env
        export KUBECONFIG=/tmp/$CLUSTER.conf
        helm repo add stable https://cncf.gitlab.io/stable
        export DIR=$(uuidgen)
        ./cnf-testsuite setup
        wget -O cnf-testsuite.yml https://raw.githubusercontent.com/cncf/cnf-testsuite/main/example-cnfs/coredns/cnf-testsuite.yml
        ./cnf-testsuite cnf_setup cnf-config=./cnf-testsuite.yml
        export FALCO_ENV="CI"
        LOG_LEVEL=info ./cnf-testsuite all ~resilience ~platform ~ip_addresses ~liveness ~readiness ~rolling_update ~rolling_downgrade ~rolling_version_change ~nodeport_not_used ~hostport_not_used ~hardcoded_ip_addresses_in_k8s_runtime_configuration ~rollback ~secrets_used ~immutable_configmap ~reasonable_startup_time ~reasonable_image_size verbose
    - name: Delete Cluster
      if: ${{ always() }}
      run: |
        source cluster.env
        kind delete cluster --name $CLUSTER
        docker container prune -f || true
        docker volume prune -f || true
        docker network prune -f || true

  release:
    name: Publish Release
    needs: [spec, build]
    runs-on: ubuntu-latest
    steps: 
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Download artifact
      uses: actions/download-artifact@v3
      with:
        name: release
    - name: Make release executable
      run: chmod +x ./cnf-testsuite
    - name: Publish Release
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        if [ -z "${GITHUB_TOKEN+x}" ]; then
          exit 0
        else
          ./cnf-testsuite upsert_release
        fi