utils.yml

# tasks that let you interact with the indexer for operational requirements
version: '3'

tasks:
  get-recent-oom:
    desc: Get recent OOM events from the cluster
    cmds:
    - |
      set - {{ .CLI_ARGS }}
      set -euo pipefail

      echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] Fetching pod data from the cluster..."
      PODS_JSON=$(kubectl get pods --all-namespaces -o json)

      echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] Processing OOMKilled pod information..."

      # Use jq to:
      # - Extract pods that have OOMKilled containers.
      # - For each pod, gather OOMKilled containers and their timestamps.
      # - Gather other containers in the same pod that are currently running.
      PODS_OOM=$(echo "$PODS_JSON" | jq '[.items[]
        | {
            name: .metadata.name,
            namespace: .metadata.namespace,
            nodeName: .spec.nodeName,

            # OOMKilled containers
            oomContainers: [
              .status.containerStatuses[]?
              | select(.lastState.terminated?.reason == "OOMKilled")
              | {
                  containerName: .name,
                  reason: .lastState.terminated.reason,
                  finishedAt: .lastState.terminated.finishedAt
                }
            ],

            # Other currently running containers in the same pod
            otherContainers: [
              .status.containerStatuses[]?
              | select((.lastState.terminated?.reason != "OOMKilled") and (.state.running?))
              | {
                  containerName: .name,
                  startedAt: .state.running.startedAt
                }
            ],

            # Container specs for reference
            containerSpecs: [
              .spec.containers[]
              | {
                  containerName: .name,
                  memoryRequest: (.resources.requests.memory // "None"),
                  memoryLimit: (.resources.limits.memory // "None")
                }
            ]
          }
        | select((.oomContainers | length) > 0)
      ] | sort_by(.oomContainers[].finishedAt)')

      COUNT=$(echo "$PODS_OOM" | jq 'length')

      echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] Found $COUNT pods with OOMKilled containers."

      if [ "$COUNT" -eq 0 ]; then
          echo "No pods found with OOMKilled containers."
          exit 0
      fi

      echo ""
      echo "========== OOMKilled Pods Detailed Report =========="
      echo ""

      # JQ filter to add relative hours
      # Given a timestamp in RFC3339 format (e.g. "2024-12-11T11:21:39Z"),
      # calculate how many hours ago it was from now.
      jq_filter='
        def hours_ago($t):
          if $t == null then
            "N/A"
          else
            ($t | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) as $ftime
            | (now - $ftime) as $diff
            | if $diff < 0 then
                ($t + " (in the future?)")
              else
                ($t + " (" + ( ($diff / 3600) | floor | tostring ) + " hours ago)")
              end
          end;

        .[] 
        | (
            "Pod Name: " + .name 
            + "\nNamespace: " + .namespace 
            + "\nNode: " + (.nodeName // "N/A")
          )
          + "\n\nContainers OOMKilled:" 
          + (
            if (.oomContainers | length) > 0 then
              (.oomContainers
              | map(
                  "\n  - Container: " + .containerName 
                  + "\n    Reason: " + (.reason // "N/A")
                  + "\n    FinishedAt: " + (hours_ago(.finishedAt))
                )
              | join("")
              )
            else
              "\n  None"
            end
          )
          + "\n\nOther Running Containers (same pod):"
          + (
            if (.otherContainers | length) > 0 then
              (.otherContainers
              | map(
                  "\n  - Container: " + .containerName
                  + "\n    StartedAt: " + (.startedAt // "N/A")
                )
              | join("")
              )
            else
              "\n  None"
            end
          )
          + "\n\nContainer Specs (Memory):"
          + (
            if (.containerSpecs | length) > 0 then
              (.containerSpecs
              | map(
                  "\n  - Container: " + .containerName
                  + "\n    Request: " + .memoryRequest
                  + "\n    Limit: " + .memoryLimit
                )
              | join("")
              )
            else
              "\n  None"
            end
          )
          + "\n------------------------------------------\n"
      '

      echo "$PODS_OOM" | jq -r "$jq_filter"


  seal-secret:
    desc: "Util to encrypt secret values and create in cluster only k8s secrets"
    interactive: true
    silent: true
    cmds:
      - |
        set - {{ .CLI_ARGS }}
        set -eu

        usage() {
          echo "Usage: seal-secret -- [-n namespace] [-s secret-name] [-k key] [-v value] [-c context-name]"
          echo "All arguments are optional and will be prompted for if not provided:"
          echo "[-n namespace] The k8s namespace intended for the secret"
          echo "[-s secret-name] The k8s secret name the controller will create" 
          echo "[-k key] The k8s secret key name"
          echo "[-v value] The secret value being encrypted"
          echo "[-c context-name] Will default to current-context unless provided"
        }

        # Print usage first
        usage

        NAMESPACE="${NAMESPACE:-}"
        SECRET_NAME="${NAME:-}"
        KEY="${KEY:-}"
        VALUE="${VALUE:-}"
        CLUSTER_CONTEXT="${CLUSTER_CONTEXT:-}"

        while getopts 'n:s:k:v:c:' opt; do
          case "$opt" in
            n) NAMESPACE="$OPTARG";;
            s) SECRET_NAME="$OPTARG";;
            k) KEY="$OPTARG";;
            v) VALUE="$OPTARG";;
            c) CLUSTER_CONTEXT="$OPTARG";;
            *) usage
              exit 1 ;;
          esac
        done

        # Print status of variables before interactive prompts
        echo -e "\nCurrent variable status:"
        [ -n "$NAMESPACE" ] && echo "✓ Namespace is set to: $NAMESPACE" || echo "⨯ Namespace needs to be set interactively"
        [ -n "$SECRET_NAME" ] && echo "✓ Secret name is set to: $SECRET_NAME" || echo "⨯ Secret name needs to be set interactively"
        [ -n "$KEY" ] && echo "✓ Key is set to: $KEY" || echo "⨯ Key needs to be set interactively"
        [ -n "$VALUE" ] && echo "✓ Value is set" || echo "⨯ Value needs to be set interactively"
        [ -n "$CLUSTER_CONTEXT" ] && echo "✓ Cluster context is set to: $CLUSTER_CONTEXT" || echo "⨯ Cluster context will use current context"
        echo ""

        # Interactive prompts for missing values
        if [ -z "$NAMESPACE" ]; then
          read -p "Enter namespace: " NAMESPACE
        fi

        if [ -z "$SECRET_NAME" ]; then
          read -p "Enter secret name: " SECRET_NAME
        fi

        if [ -z "$KEY" ]; then
          read -p "Enter secret key: " KEY
        fi

        if [ -z "$VALUE" ]; then
          stty -echo
          read -p "Enter secret value (hidden): " VALUE
          stty echo
          echo "" # Add newline after hidden input
        fi

        if [ -z "$CLUSTER_CONTEXT" ]; then
          CURRENT_CONTEXT=$(kubectl config current-context)
          echo "Cluster context not set, using currently set context $CURRENT_CONTEXT"
          echo "If you're creating a secret for a different cluster rerun the script and pass the context name for the cluster"
          echo ""
          CLUSTER_CONTEXT=$CURRENT_CONTEXT
        else 
          echo "Context name set - creating secret $SECRET_NAME against $CLUSTER_CONTEXT"
          kubectl config use-context "$CLUSTER_CONTEXT"
        fi

        # Validate that we have all required values
        if [ -z "$NAMESPACE" ] || [ -z "$SECRET_NAME" ] || [ -z "$KEY" ] || [ -z "$VALUE" ]; then
          echo "Error: All values must be provided either via arguments or interactive input"
          usage
          exit 1
        fi

        secretManifest=$(echo -n "${VALUE}" | kubectl create secret generic "${SECRET_NAME}" --dry-run=client --namespace "${NAMESPACE}"  --from-literal="${KEY}"="${VALUE}" -o json 2>&1)
        sealedManifest=`echo $secretManifest | kubeseal --namespace "${NAMESPACE}" --controller-namespace sealed-secrets --controller-name sealed-secrets  2>&1`
        sealedValue=`echo $sealedManifest | jq --arg 'key' "${KEY}" '.spec.encryptedData[$key]' -r`

        echo "${KEY}: '${sealedValue}'"

  octant:
    desc: Run octant to navigate your cluster in your web browser
    cmds:
    - octant {{.CLI_ARGS}}

  k9s:
    desc: Run k9s to navigate your cluster in your terminal
    cmds:
    - k9s {{.CLI_ARGS}}