breeze-legacy

#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail


AIRFLOW_SOURCES="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

if [[ ${BREEZE_REDIRECT=} == "" ]]; then
    set +u
    mkdir -p "${AIRFLOW_SOURCES}"/logs
    export BREEZE_REDIRECT="true"
    if [[ "$(uname)" == "Darwin" ]]; then
      exec script -q "${AIRFLOW_SOURCES}"/logs/breeze.out "$(command -v bash)" -c "$(printf "%q " "${0}" "${@}")"
    else
      exec script --return --quiet "${AIRFLOW_SOURCES}"/logs/breeze.out -c "$(printf "%q " "${0}" "${@}")"
    fi
    set -u
fi

export AIRFLOW_SOURCES
readonly AIRFLOW_SOURCES


# Bash arrays need to be defined outside of functions unfortunately :(
# Because on Mac OS Bash 3.4 defining arrays inside functions does not work
# Array with extra options for Docker compose
declare -a EXTRA_DC_OPTIONS
export EXTRA_DC_OPTIONS

# Array with selected integrations
declare -a INTEGRATIONS
export INTEGRATIONS

# This is where remaining args are passed
declare -a REMAINING_ARGS
export REMAINING_ARGS

#######################################################################################################
# Sets up all the default variables for Breeze. They are needed by all other functions
# All those variables are exported. They are not set to read-only because those
# defaults can be modified later on when command line arguments are parsed
# and variables stored in .build directory (stored in the previous run) are read
#
# Used globals:
#    FORCE_SCREEN_WIDTH
#
# Modified globals (constants or candidates for constants after we override them via appropriate flags):
#
#    BREEZE
#    SUPPRESS_CHEATSHEET_FILE
#    SUPPRESS_ASCIIART_FILE
#    MAX_SCREEN_WIDTH
#    SCREEN_WIDTH
#    MOUNT_SELECTED_LOCAL_SOURCES
#    FORWARD_CREDENTIALS
#    DB_RESET
#    START_AIRFLOW
#    INSTALL_AIRFLOW_VERSION
#    INSTALL_AIRFLOW_REFERENCE
#    FORCE_BUILD_IMAGES
#    PRODUCTION_IMAGE
#    PYTHON_MAJOR_MINOR_VERSION
#
# Global variables:
#
#    command_to_run
#    second_command_to_run
#    docker_compose_command
#
# Also it sets the variables and globals set by common initialization functions from
# scripts/ci/libraries/_initialization.sh and breeze-complete script (which sets-up auto-complete).
#
#######################################################################################################
function breeze::setup_default_breeze_constants() {
    # Indicates that we are inside Breeze environment
    export BREEZE=true
    readonly BREEZE

    # If those files are present, the ASCII-art/cheat-sheet are suppressed
    SUPPRESS_CHEATSHEET_FILE="${AIRFLOW_SOURCES}/.build/.suppress_cheatsheet"
    readonly SUPPRESS_CHEATSHEET_FILE

    SUPPRESS_ASCIIART_FILE="${AIRFLOW_SOURCES}/.build/.suppress_asciiart"
    readonly SUPPRESS_ASCIIART_FILE

    # Maximum screen indented_screen_width to print the lines spanning the whole terminal indented_screen_width
    export MAX_SCREEN_WIDTH=100
    readonly MAX_SCREEN_WIDTH

    # By default we mount selected local Airflow sources
    export MOUNT_SELECTED_LOCAL_SOURCES="true"

    # By default we do not mount all local Airflow sources
    export MOUNT_ALL_LOCAL_SOURCES="false"

    # Forward common host credentials to docker (gcloud, aws etc.).
    export FORWARD_CREDENTIALS="false"

    # If set to true, the database will be reset at entry. Works for Postgres and MySQL
    export DB_RESET="false"

    # If set to true, the database will be initialized, a user created and webserver and scheduler started
    export START_AIRFLOW="false"

    # If set to true, the test connections will be created
    export LOAD_DEFAULT_CONNECTIONS="false"

    # If set to true, the sample dags will be used
    export LOAD_EXAMPLES="false"

    # If set to true, Breeze db volumes will be preserved when breeze is stopped and reused next time
    # Which means that you do not have to start from scratch
    export PRESERVE_VOLUMES="false"

    # Sources by default are installed from local sources when using breeze
    AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM:="."}
    export AIRFLOW_SOURCES_FROM

    # They are copied to /opt/airflow by default in breeze
    AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO:="/opt/airflow"}
    export AIRFLOW_SOURCES_TO

    # Sources by default are installed from local sources when using breeze
    AIRFLOW_SOURCES_WWW_FROM=${AIRFLOW_SOURCES_WWW_FROM:="./airflow/www"}
    export AIRFLOW_SOURCES_WWW_FROM

    # They are copied to /opt/airflow by default in breeze
    AIRFLOW_SOURCES_WWW_TO=${AIRFLOW_SOURCES_WWW_TO:="/opt/airflow/airflow/www"}
    export AIRFLOW_SOURCES_WWW_TO

    # Unlike in CI scripts, in breeze by default production image is installed from sources
    export AIRFLOW_INSTALLATION_METHOD="."

    # If it set is set to specified version, then the source version of Airflow
    # is removed and the specified version of Airflow is installed from PyPi
    export INSTALL_AIRFLOW_VERSION=""

    # If it is set to specified reference (tag/branch), then the source version
    # of Airflow is removed and the specified version of Airflow is installed from GitHub
    export INSTALL_AIRFLOW_REFERENCE=""

    # Determines whether to force build without checking if it is needed
    # Can be overridden by '--force-build-images' flag.
    export FORCE_BUILD_IMAGES="false"

    # Determines whether to reinstall airflow at entering the image.
    export USE_AIRFLOW_VERSION=""
    # if set to true, the ci image will look for wheel packages in dist folder and will install them
    # during entering the container
    export USE_PACKAGES_FROM_DIST="false"

    # load all the common functions here - those are the functions that are shared between Breeze
    # and CI scripts. The CI scripts do not use Breeze as driving script - they read all configuration
    # from the environment variables. That's why we keep all the common initialization in those libs
    # shellcheck source=scripts/ci/libraries/_all_libs.sh
    . "${AIRFLOW_SOURCES}/scripts/ci/libraries/_all_libs.sh"

    # When we generate documentation for README files, we want to force the indented_screen_width of terminal so that
    # No matter who is running the documentation generation gets the same output
    if [[ ${FORCE_SCREEN_WIDTH:="false"} != "true" ]]; then
        # Sets indented_screen_width of the screen from terminal
        SCREEN_WIDTH="$(tput cols)"
        if [[ -z ${SCREEN_WIDTH=} ]]; then
            SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
        fi
        if ((SCREEN_WIDTH > MAX_SCREEN_WIDTH)); then
            SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
        fi
    else
        SCREEN_WIDTH=${MAX_SCREEN_WIDTH}
    fi

    export SCREEN_WIDTH
    readonly SCREEN_WIDTH

    # for Breeze default tests executed are "All"
    export TEST_TYPE=${TEST_TYPE:="All"}

    # Update short and long options in the breeze-complete script
    # This way autocomplete will work automatically with all options available
    # shellcheck source=breeze-complete
    . "${AIRFLOW_SOURCES}/breeze-complete"

    # Default command to run - entering breeze environment
    command_to_run="enter_breeze"

    # In some cases we also want to run two commands in a row (for example when we restart the environment)
    second_command_to_run=""

    # Determines if help should be run (set to true by --help flag)
    run_help="false"

    # Holds docker compose command if the `docker-compose` command is used.
    docker_compose_command=""
}

#######################################################################################################
#
# Initializes development-friendly virtualenv if you are already in such env. It installs all the necessary
# packages from PyPI and it case of problems it provides useful hints on what prerequisites should be
# installed. It also removes and resets the existing AIRFLOW_HOME installation to make sure that you
# have it synchronized with the version of airflow installed. It resets the airflow's sqlite database to
# a clean state. You can use this function if your virtualenv is broken, to clean it up
#
# Used globals:
#    PYTHON_MAJOR_MINOR_VERSION
#    AIRFLOW_HOME_DIR
#    AIRFLOW_SOURCES
#    VIRTUALENV_EXTRAS
#    DEFAULT_CONSTRAINTS_BRANCH
#    OSTYPE
#
#######################################################################################################
function breeze::initialize_virtualenv() {
    # Check if we are inside virtualenv
    set +e
    echo -e "import sys\nif not hasattr(sys,'base_prefix'):\n  sys.exit(1)" |
        python"${PYTHON_MAJOR_MINOR_VERSION}"
    local res=$?
    set -e
    if [[ ${res} != "0" ]]; then
        echo
        echo  "${COLOR_RED}ERROR: Initializing local virtualenv only works when you have virtualenv activated  ${COLOR_RESET}"
        echo
        echo "Please enter your local virtualenv before (for example using 'pyenv activate' or 'workon') "
        echo
        exit 1
    else
        echo
        echo "Initializing the virtualenv: $(command -v python)!"
        echo
        echo "Extras to be installed: ${VIRTUALENV_EXTRAS}"
        echo
        echo "This will wipe out ${AIRFLOW_HOME_DIR} and reset all the databases!"
        echo
        "${AIRFLOW_SOURCES}/scripts/tools/confirm" "Proceeding with the initialization"
        echo
        pushd "${AIRFLOW_SOURCES}" >/dev/null 2>&1 || exit 1
        set +e
        pip install -e ".[${VIRTUALENV_EXTRAS}]" \
            --constraint "https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-source-providers-${PYTHON_MAJOR_MINOR_VERSION}.txt"
        res=$?
        set -e
        popd
        if [[ ${res} != "0" ]]; then
            echo "#######################################################################"
            echo "  You had some troubles installing the venv !!!!!"
            echo "  Try running the command below and rerun virtualenv installation"
            echo
            if [[ ${OSTYPE} == "darwin"* ]]; then
                echo "  brew install sqlite mysql postgresql openssl"
                echo "  export LDFLAGS=\"-L/usr/local/opt/openssl/lib\""
                echo "  export CPPFLAGS=\"-I/usr/local/opt/openssl/include\""
            else
                echo "  sudo apt install build-essential python3-dev libsqlite3-dev openssl \\"
                echo "              sqlite default-libmysqlclient-dev libmysqlclient-dev postgresql"
            fi
            echo
            echo "#######################################################################"
            exit ${res}
        fi
        echo
        echo "Wiping and recreating ${AIRFLOW_HOME_DIR}"
        echo
        if [[ "${AIRFLOW_SOURCES}" == "${AIRFLOW_HOME_DIR}" ]]; then
            echo "AIRFLOW_HOME and Source code for Apache Airflow resides in the same path ${AIRFLOW_HOME_DIR}"
            echo "When running this command it will delete all the files in the path ${AIRFLOW_HOME_DIR} to clear dynamic files like config/logs/db"
            echo "Move your source code for Apache Airflow to different folder to avoid deletion"
            exit 1
        fi
        rm -rvf "${AIRFLOW_HOME_DIR}"
        mkdir -p "${AIRFLOW_HOME_DIR}"
        echo
        echo "Resetting AIRFLOW sqlite database"
        echo
        AIRFLOW__CORE__LOAD_EXAMPLES="False" \
            AIRFLOW__CORE__UNIT_TEST_MODE="False" \
            AIRFLOW__CORE__SQL_ALCHEMY_POOL_ENABLED="False" \
            AIRFLOW__CORE__DAGS_FOLDER="${AIRFLOW_SOURCES}/empty" \
            AIRFLOW__CORE__PLUGINS_FOLDER="${AIRFLOW_SOURCES}/empty" \
            airflow db reset -y
        echo
        echo "Resetting AIRFLOW sqlite unit test database"
        echo
        AIRFLOW__CORE__LOAD_EXAMPLES="False" \
            AIRFLOW__CORE__UNIT_TEST_MODE="True" \
            AIRFLOW__CORE__SQL_ALCHEMY_POOL_ENABLED="False" \
            AIRFLOW__CORE__DAGS_FOLDER="${AIRFLOW_SOURCES}/empty" \
            AIRFLOW__CORE__PLUGINS_FOLDER="${AIRFLOW_SOURCES}/empty" \
            airflow db reset -y
        echo
        echo "Initialization of virtualenv was successful! Go ahead and develop Airflow!"
        echo
        exit 0
    fi
}

#######################################################################################################
#
# Sets up autocomplete for Breeze for both - bash and zsh
#
# Used globals:
#
#    AIRFLOW_SOURCES
#    HOME
#    OSTYPE
#
#######################################################################################################
function breeze::setup_autocomplete() {
    echo "Installing bash/zsh completion for local user"
    echo
    "${AIRFLOW_SOURCES}/scripts/tools/confirm" "This will create ~/.bash_completion.d/ directory and modify ~/.*rc files"
    echo
    echo
    mkdir -pv ~/.bash_completion.d
    ln -sf "${AIRFLOW_SOURCES}/breeze-complete" "${HOME}/.bash_completion.d/"
    echo
    echo "Breeze Bash completion is now linked to: ${AIRFLOW_SOURCES}/breeze-complete"
    echo
    local breeze_comment="Added by Airflow Breeze autocomplete setup"
    if ! grep "${breeze_comment}" "${HOME}/.bashrc" >/dev/null 2>&1; then
        touch ~/.bashrc
        # shellcheck disable=SC2129
        echo "# START: ${breeze_comment}" >>~/.bashrc
        cat <<"EOF" >>~/.bashrc
for bcfile in ~/.bash_completion.d/* ; do
    . ${bcfile}
done
EOF
        echo "# END: ${breeze_comment}" >>~/.bashrc
        echo
        echo "The ${HOME}/.bashrc has been modified"
        echo
    else
        echo
        echo "The ${HOME}/.bashrc was already modified before. Not changing it."
        echo
    fi
    if ! grep "${breeze_comment}" "${HOME}/.zshrc" >/dev/null 2>&1; then
        # shellcheck disable=SC2129
        echo "# START: ${breeze_comment}" >>~/.zshrc
        cat <<"EOF" >>~/.zshrc
autoload compinit && compinit
autoload bashcompinit && bashcompinit
source ~/.bash_completion.d/breeze-complete
EOF
        echo "# END: ${breeze_comment}" >>~/.zshrc
        echo
        echo "The ${HOME}/.zshrc has been modified"
        echo
    else
        echo
        echo "The ${HOME}/.zshrc was already modified before. Not changing it."
        echo
    fi
    if [[ "${OSTYPE}" == "darwin"* ]]; then
        #  For MacOS we have to handle the special case where terminal app DOES NOT run .bashrc by default
        #  But re-runs .bash_profile :(
        #  See https://scriptingosx.com/2017/04/about-bash_profile-and-bashrc-on-macos/
        if ! grep "${breeze_comment}" "${HOME}/.bash_profile"; then
            # shellcheck disable=SC2129
            echo "# START: ${breeze_comment}" >>~/.bash_profile
            cat <<"EOF" >>~/.bash_profile
if [ -r ~/.bashrc ]; then
    source ~/.bashrc
fi
EOF
            echo "# END: ${breeze_comment}" >>~/.bash_profile
            echo
            echo "The ${HOME}/.bash_profile has been modified"
            echo
        else
            echo
            echo "The ${HOME}/.bash_profile was already modified before. Not changing it."
            echo
        fi
    fi
    echo
    echo
    echo "Breeze completion is installed to ~/.bash_completion.d/breeze-complete"
    echo
    echo "Please exit and re-enter your shell or run:"
    echo
    if [[ "${OSTYPE}" == "darwin"* ]]; then
        if grep "${breeze_comment}" "${HOME}/.zshrc" >/dev/null 2>&1; then
            echo "    source ~/.zshrc"
            echo
            echo "    source ~/.bash_completion.d/breeze-complete"
            echo
            exec zsh
            exit 0
        fi
    fi
    echo "    source ~/.bash_completion.d/breeze-complete"
    echo
    exit 0
}

#######################################################################################################
#
# Prints information about the current configuration of Breeze - if you enter breeze interactively
# and you did not suppress cheatsheet or asciiart, it also prints those. It also prints values
# of constants set by breeze::read_saved_environment_variables() function and other initialization functions.
#
# Used globals:
#
#    BACKEND
#    POSTGRES_VERSION
#    MYSQL_VERSION
#    SUPPRESS_CHEATSHEET_FILE
#    SUPPRESS_ASCIIART_FILE
#    PRODUCTION_IMAGE
#    BRANCH_NAME
#    AIRFLOW_CI_IMAGE
#    AIRFLOW_PROD_IMAGE
#    AIRFLOW_VERSION
#    INSTALL_AIRFLOW_VERSION
#    INSTALL_AIRFLOW_REFERENCE
#
# Outputs:
#    Prints the information about the build to stdout.
#
#######################################################################################################
function breeze::print_badge() {
    local backend_version=""
    if [[ ${BACKEND} == "postgres" ]]; then
        backend_version="${POSTGRES_VERSION}"
    elif [[ ${BACKEND} == "mysql" ]]; then
        backend_version="${MYSQL_VERSION}"
    fi
    if [[ ! -f "${SUPPRESS_ASCIIART_FILE}" && ${command_to_run} == "enter_breeze" ]]; then
        cat <<EOF




                                  @&&&&&&@
                                 @&&&&&&&&&&&@
                                &&&&&&&&&&&&&&&&
                                        &&&&&&&&&&
                                            &&&&&&&
                                             &&&&&&&
                           @@@@@@@@@@@@@@@@   &&&&&&
                          @&&&&&&&&&&&&&&&&&&&&&&&&&&
                         &&&&&&&&&&&&&&&&&&&&&&&&&&&&
                                         &&&&&&&&&&&&
                                             &&&&&&&&&
                                           &&&&&&&&&&&&
                                      @@&&&&&&&&&&&&&&&@
                   @&&&&&&&&&&&&&&&&&&&&&&&&&&&&  &&&&&&
                  &&&&&&&&&&&&&&&&&&&&&&&&&&&&    &&&&&&
                 &&&&&&&&&&&&&&&&&&&&&&&&         &&&&&&
                                                 &&&&&&
                                               &&&&&&&
                                            @&&&&&&&&
            @&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
           &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&
          &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&



     @&&&@       &&  @&&&&&&&&&&&   &&&&&&&&&&&&  &&            &&&&&&&&&&  &&&     &&&     &&&
    &&& &&&      &&  @&&       &&&  &&            &&          &&&       &&&@ &&&   &&&&&   &&&
   &&&   &&&     &&  @&&&&&&&&&&&&  &&&&&&&&&&&   &&          &&         &&&  &&& &&& &&@ &&&
  &&&&&&&&&&&    &&  @&&&&&&&&&     &&            &&          &&@        &&&   &&@&&   &&@&&
 &&&       &&&   &&  @&&     &&&@   &&            &&&&&&&&&&&  &&&&&&&&&&&&     &&&&   &&&&

&&&&&&&&&&&&   &&&&&&&&&&&&   &&&&&&&&&&&@  &&&&&&&&&&&&   &&&&&&&&&&&   &&&&&&&&&&&
&&&       &&&  &&        &&&  &&            &&&                  &&&&    &&
&&&&&&&&&&&&@  &&&&&&&&&&&&   &&&&&&&&&&&   &&&&&&&&&&&       &&&&       &&&&&&&&&&
&&&        &&  &&   &&&&      &&            &&&             &&&&         &&
&&&&&&&&&&&&&  &&     &&&&@   &&&&&&&&&&&@  &&&&&&&&&&&&  @&&&&&&&&&&&   &&&&&&&&&&&

EOF
        if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
            cat <<EOF

                               Use production image.

                               Branch name:            ${BRANCH_NAME}
                               Docker image:           ${AIRFLOW_PROD_IMAGE}
                               Platform:               ${PLATFORM}
EOF
        else
            cat <<EOF

                               Use CI image.

                               Branch name:            ${BRANCH_NAME}
                               Docker image:           ${AIRFLOW_CI_IMAGE_WITH_TAG}
                               Platform:               ${PLATFORM}
                               Airflow source version: ${AIRFLOW_VERSION}
EOF
        fi
        cat <<EOF
                               Python version:         ${PYTHON_MAJOR_MINOR_VERSION}
                               Debian version:         ${DEBIAN_VERSION}
                               Backend:                ${BACKEND} ${backend_version}
EOF
        if [[ -n ${USE_AIRFLOW_VERSION=} ]]; then
            cat <<EOF

                               Airflow used at runtime: ${USE_AIRFLOW_VERSION=}

EOF
        fi
    else
        if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
            cat <<EOF

   Production image.

   Branch name:             ${BRANCH_NAME}
   Docker image:            ${AIRFLOW_PROD_IMAGE}
   Platform:                ${PLATFORM}
EOF
        else
            cat <<EOF

   CI image.

   Branch name:             ${BRANCH_NAME}
   Docker image:            ${AIRFLOW_CI_IMAGE}
   Platform:                ${PLATFORM}

EOF
        fi
        if [[ -n ${INSTALL_AIRFLOW_VERSION=} || -n ${INSTALL_AIRFLOW_REFERENCE=} ]]; then
            cat <<EOF
   Airflow version installed: ${INSTALL_AIRFLOW_VERSION=}${INSTALL_AIRFLOW_REFERENCE=}

EOF
        fi
        cat <<EOF

   Airflow source version:  ${AIRFLOW_VERSION}
   Python version:          ${PYTHON_MAJOR_MINOR_VERSION}
   Backend:                 ${BACKEND} ${backend_version}
EOF
    fi
    if [[ ${VERBOSE} == "true" ]]; then
        initialization::summarize_build_environment
    fi

}

#######################################################################################################
#
# Prepares command file that can be used to easily run the docker commands outside of Breeze.
#
# The command file generated in cache ./build directory is a standalone script that contains
# All the environment variables and docker-compose configuration to run the command.
# This is because depending on configuration of Breeze we might have different compose files
# used and different env variables set.
#
# Those are a convenience scripts that you might use to debug command execution although
# In most cases they are used internally by Breeze.
#
# Arguments:
#
#   file to prepare
#   command to run
#   compose_file to use
#   airflow_image to use
#
# Outputs:
#   Creates the convenience command file that can be run to use the docker command.
#
#######################################################################################################
function breeze::prepare_command_file() {
    local file="${1}"
    local command="${2}"
    local compose_file="${3}"
    cat <<EOF >"${file}"
#!/usr/bin/env bash
docker_compose_version=\$(docker-compose --version || true)
if [[ \${docker_compose_version} =~ .*([0-9]+)\.([0-9]+)\.([0-9]+).* ]]; then
    major=\${BASH_REMATCH[1]}
    minor=\${BASH_REMATCH[2]}
    patch=\${BASH_REMATCH[3]}
    if [[ \${major} == "1" ]]; then
        if (( minor < 29 )); then
            echo
            echo "${COLOR_RED}You have too old version of docker-compose: \${major}.\${minor}.\${patch}! At least 1.29 is needed! Please upgrade!${COLOR_RESET}"
            echo "${COLOR_RED}See https://docs.docker.com/compose/install/ for instructions. Make sure docker-compose you install is first on the PATH variable of yours.${COLOR_RESET}"
            echo
            exit 1
        fi
    fi
    echo
    echo "${COLOR_GREEN}Good version of docker-compose: \${major}.\${minor}.\${patch}${COLOR_RESET}"
    echo
else
    echo
    echo "${COLOR_YELLOW}Unknown docker-compose version. At least 1.29 is needed! If Breeze fails - upgrade to latest available docker-compose version${COLOR_RESET}"
    echo
fi

if [[ \${VERBOSE} == "true" ]]; then
  echo
  echo "Executing script:"
  echo
  echo "${COLOR_CYAN}${file} \${@}${COLOR_RESET}"
  echo
  set -x
fi
cd "\$( dirname "\${BASH_SOURCE[0]}" )" || exit
export HOST_USER_ID=${HOST_USER_ID}
export HOST_GROUP_ID=${HOST_GROUP_ID}
export COMPOSE_FILE="${compose_file}"
export PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}"
export DEBIAN_VERSION="${DEBIAN_VERSION}"
export BACKEND="${BACKEND}"
export AIRFLOW_VERSION="${AIRFLOW_VERSION}"
export INSTALL_AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION}"
export PLATFORM="${PLATFORM}"
export SSH_PORT="${SSH_PORT}"
export WEBSERVER_HOST_PORT="${WEBSERVER_HOST_PORT}"
export FLOWER_HOST_PORT="${FLOWER_HOST_PORT}"
export REDIS_HOST_PORT="${REDIS_HOST_PORT}"
export POSTGRES_HOST_PORT="${POSTGRES_HOST_PORT}"
export POSTGRES_VERSION="${POSTGRES_VERSION}"
export MYSQL_HOST_PORT="${MYSQL_HOST_PORT}"
export MYSQL_VERSION="${MYSQL_VERSION}"
export MSSQL_HOST_PORT="${MSSQL_HOST_PORT}"
export MSSQL_VERSION="${MSSQL_VERSION}"
export AIRFLOW_SOURCES="${AIRFLOW_SOURCES}"
export AIRFLOW_CI_IMAGE="${AIRFLOW_CI_IMAGE}"
export AIRFLOW_CI_IMAGE_WITH_TAG="${AIRFLOW_CI_IMAGE_WITH_TAG}"
export AIRFLOW_PROD_IMAGE="${AIRFLOW_PROD_IMAGE}"
export AIRFLOW_IMAGE_KUBERNETES="${AIRFLOW_IMAGE_KUBERNETES}"
export SQLITE_URL="${SQLITE_URL}"
export USE_AIRFLOW_VERSION="${USE_AIRFLOW_VERSION}"
export USE_PACKAGES_FROM_DIST="${USE_PACKAGES_FROM_DIST}"
export EXECUTOR="${EXECUTOR}"
export START_AIRFLOW="${START_AIRFLOW}"
export ENABLED_INTEGRATIONS="${ENABLED_INTEGRATIONS}"
export ENABLED_SYSTEMS="${ENABLED_SYSTEMS}"
export GITHUB_ACTIONS="${GITHUB_ACTIONS}"
export ISSUE_ID="${ISSUE_ID}"
export NUM_RUNS="${NUM_RUNS}"
docker-compose ${command}
EOF
    chmod u+x "${file}"
}

#######################################################################################################
#
# Prepare all command files that we are using. We use dc_ci - to run docker compose command for CI image
#
# Global constants set:
#
#     PYTHON_BASE_IMAGE
#     AIRFLOW_CI_IMAGE
#     BUILT_CI_IMAGE_FLAG_FILE
#
#######################################################################################################
function breeze::prepare_command_files() {
    local main_ci_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/base.yml
    if [[ ${BACKEND} == "mssql" ]]; then
        local backend_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}-${DEBIAN_VERSION}.yml
    else
        local backend_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml
    fi

    local backend_port_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}-port.yml
    local local_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/local.yml
    local local_all_sources_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/local-all-sources.yml
    local files_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/files.yml
    local remove_sources_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml
    local forward_credentials_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml


    if [[ ${BACKEND} == "mssql" ]]; then
        local docker_filesystem
        docker_filesystem=$(stat "-f" "-c" "%T" /var/lib/docker 2>/dev/null || echo "unknown")
        if [[ ${docker_filesystem} == "tmpfs" ]]; then
            # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
            # O_DIRECT parameter for direct writing to the filesystem
            # https://github.com/microsoft/mssql-docker/issues/13
            # so we need to mount an external volume for its db location
            # specified by MSSQL_DATA_VOLUME
            backend_docker_compose_file="${backend_docker_compose_file}:${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-bind-volume.yml"
        else
            backend_docker_compose_file="${backend_docker_compose_file}:${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-docker-volume.yml"
        fi
    fi


    local compose_ci_file=${main_ci_docker_compose_file}:${backend_docker_compose_file}:${files_docker_compose_file}

    if [[ "${MOUNT_SELECTED_LOCAL_SOURCES}" != "false" ]]; then
        compose_ci_file=${compose_ci_file}:${local_docker_compose_file}:${backend_port_docker_compose_file}
    fi

    if [[ "${MOUNT_ALL_LOCAL_SOURCES}" != "false" ]]; then
        compose_ci_file=${compose_ci_file}:${local_all_sources_docker_compose_file}:${backend_port_docker_compose_file}
    fi

    if [[ ${FORWARD_CREDENTIALS} == "true" ]]; then
        compose_ci_file=${compose_ci_file}:${forward_credentials_docker_compose_file}
    fi

    if [[ -n ${INSTALL_AIRFLOW_VERSION=} || -n ${INSTALL_AIRFLOW_REFERENCE} || -n ${USE_AIRFLOW_VERSION=} ]]; then
        compose_ci_file=${compose_ci_file}:${remove_sources_docker_compose_file}
    fi

    set +u
    local unique_integrations
    # shellcheck disable=SC2207
    unique_integrations=($(echo "${INTEGRATIONS[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '))

    local integration
    for integration in "${unique_integrations[@]}"; do
        compose_ci_file=${compose_ci_file}:${SCRIPTS_CI_DIR}/docker-compose/integration-${integration}.yml
    done
    set -u

    export DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI="dc_ci"
    readonly DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI

    breeze::prepare_command_file "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}" \
        "\"\${@}\"" "${compose_ci_file}"
}

#######################################################################################################
#
# Prints detailed help for all commands and flags. Used to generate documentation added to BREEZE.rst
# automatically.
#
# Used global variables:
#    _breeze_all_commands
#
# Outputs:
#   Prints detailed help for all commands to stdout.
#
#######################################################################################################
function breeze::do_help_all() {
    echo
    breeze::print_line
    breeze::usage
    breeze::print_line
    echo
    echo
    echo "Detailed usage"
    echo
    breeze::print_line
    echo
    local subcommand
    # shellcheck disable=SC2154
    for subcommand in ${_breeze_all_commands}; do
        breeze::detailed_usage "${subcommand}"
        breeze::print_line
        echo
    done
    echo
    breeze::flags
}

#######################################################################################################
#
# Parses all arguments that can be passed to Breeze command - that includes command to run and flags.
#
# Used global variables:
#   _breeze_getopt_short_options
#   _breeze_getopt_long_options
#   _breeze_allowed_integrations
#
# Updated global constants:
#      By the end of this function, all the constants from `initialization::make_constants_read_only`
#      function are set and they are set as read-only.
#
#######################################################################################################
function breeze::parse_arguments() {
    set -u
    local params
    if ! params=$(getopt \
        -o "${_breeze_getopt_short_options:=}" \
        -l "${_breeze_getopt_long_options:=}" \
        --name "$CMDNAME" -- "$@"); then
        breeze::flags
        exit 1
    fi

    eval set -- "${params}"
    unset params

    # Parse Flags.
    # Please update short and long options in the breeze-complete script
    # This way autocomplete will work out-of-the-box
    while true; do
        case "${1}" in
        -h | --help)
            run_help="true"
            shift
            ;;
        -p | --python)
            export PYTHON_MAJOR_MINOR_VERSION="${2}"
            echo "Python version: ${PYTHON_MAJOR_MINOR_VERSION}"
            echo
            shift 2
            ;;
        -b | --backend)
            export BACKEND="${2}"
            echo "Backend: ${BACKEND}"
            echo
            shift 2
            ;;
        -i | --integration)
            local INTEGRATION=${2}
            parameters::check_and_save_allowed_param "INTEGRATION" "integration" "--integration"
            echo "Integration: ${INTEGRATION}"
            if [[ ${INTEGRATION} == "all" ]]; then
                # shellcheck disable=SC2154
                for INTEGRATION in ${_breeze_allowed_integrations}; do
                    if [[ ${INTEGRATION} != "all" ]]; then
                        echo "${INTEGRATION}"
                        INTEGRATIONS+=("${INTEGRATION}")
                    fi
                done
            else
                INTEGRATIONS+=("${INTEGRATION}")
            fi
            if [[ " ${INTEGRATIONS[*]} " =~ " trino " ]]; then
              INTEGRATIONS+=("kerberos");
            fi
            echo
            shift 2
            ;;
        -K | --kubernetes-mode)
            export KUBERNETES_MODE="${2}"
            echo "Kubernetes mode: ${KUBERNETES_MODE}"
            echo
            shift 2
            ;;
        -V | --kubernetes-version)
            export KUBERNETES_VERSION="${2}"
            echo "Kubernetes version: ${KUBERNETES_VERSION}"
            echo
            shift 2
            ;;
        --kind-version)
            export KIND_VERSION="${2}"
            echo "Kind version: ${KIND_VERSION}"
            echo
            shift 2
            ;;
        --executor)
            export EXECUTOR="${2}"
            echo "Using ${EXECUTOR} in cluster"
            echo
            shift 2
            ;;
        --helm-version)
            export HELM_VERSION="${2}"
            echo "Helm version: ${HELM_VERSION}"
            echo
            shift 2
            ;;
        --postgres-version)
            export POSTGRES_VERSION="${2}"
            echo "Postgres version: ${POSTGRES_VERSION}"
            echo
            shift 2
            ;;
        --mysql-version)
            export MYSQL_VERSION="${2}"
            echo "MySQL version: ${MYSQL_VERSION}"
            echo
            shift 2
            ;;
        --mssql-version)
            export MSSQL_VERSION="${2}"
            echo "MSSQL version: ${MSSQL_VERSION}"
            echo
            shift 2
            ;;
        -l | --skip-mounting-local-sources)
            MOUNT_SELECTED_LOCAL_SOURCES="false"
            MOUNT_ALL_LOCAL_SOURCES="false"
            echo "Mount selected local sources: ${MOUNT_SELECTED_LOCAL_SOURCES}"
            echo "Mount all local sources: ${MOUNT_ALL_LOCAL_SOURCES}"
            echo
            shift
            ;;
        --mount-all-local-sources)
            MOUNT_ALL_LOCAL_SOURCES="true"
            MOUNT_SELECTED_LOCAL_SOURCES="false"
            echo "Mount selected local sources: ${MOUNT_SELECTED_LOCAL_SOURCES}"
            echo "Mount all local sources: ${MOUNT_ALL_LOCAL_SOURCES}"
            echo
            shift
            ;;
        -t | --install-airflow-reference)
            INSTALL_AIRFLOW_REFERENCE="${2}"
            # Reference is mutually exclusive with version
            INSTALL_AIRFLOW_VERSION=""
            # Skip mounting local sources when airflow is installed from remote
            INSTALL_PROVIDERS_FROM_SOURCES="false"
            echo "Installs Airflow from reference: ${INSTALL_AIRFLOW_REFERENCE}"
            echo
            shift 2
            ;;
        --use-airflow-version)
            USE_AIRFLOW_VERSION="${2}"
            MOUNT_SELECTED_LOCAL_SOURCES="false"
            MOUNT_ALL_LOCAL_SOURCES="false"
            DB_RESET="true"
            echo "Uses Airflow at runtime from: ${USE_AIRFLOW_VERSION}, reset DB and skip mounting local sources"
            echo
            shift 2
            ;;
        --db-reset)
            echo "Resetting the DB!"
            echo
            export DB_RESET="true"
            shift
            ;;
        -v | --verbose)
            export VERBOSE="true"
            echo "Verbose output"
            echo
            shift
            ;;
        -y | --assume-yes)
            export ANSWER="yes"
            echo "Assuming 'yes' answer to all questions."
            echo
            shift
            ;;
        -n | --assume-no)
            export ANSWER="no"
            echo "Assuming 'no' answer to all questions."
            echo
            shift
            ;;
        -q | --assume-quit)
            export ANSWER="quit"
            echo "Assuming 'quit' answer to all questions."
            echo
            shift
            ;;
        -f | --forward-credentials)
            echo "Forwarding credentials. Be careful as your credentials are available in the container!"
            echo
            export FORWARD_CREDENTIALS="true"
            shift
            ;;
        -g | --github-repository)
            echo
            echo "Using GitHub registry."
            echo "GitHub repository: ${2}"
            echo
            export GITHUB_REPOSITORY="${2}"
            shift 2
            ;;
        --init-script)
            export INIT_SCRIPT_FILE="${2}"
            echo "The initialization file is in ${INIT_SCRIPT_FILE}"
            echo
            shift 2
            ;;
        --load-example-dags)
            export LOAD_EXAMPLES="true"
            echo "Include Airflow sample dags"
            echo
            shift
            ;;
        --load-default-connections)
            export LOAD_DEFAULT_CONNECTIONS="true"
            echo "Include Airflow default connections"
            echo
            shift
            ;;
        --preserve-volumes)
            export PRESERVE_VOLUMES="true"
            echo "Preserves data volumes when stopping airflow"
            echo
            shift
            ;;
        --installation-method)
            export AIRFLOW_INSTALLATION_METHOD="${2}"
            echo "Airflow installation method: ${AIRFLOW_INSTALLATION_METHOD}"
            echo
            shift 2
            ;;
        --test-type)
            export TEST_TYPE="${2}"
            echo "Selected test type: ${TEST_TYPE}"
            echo
            shift 2
            ;;
        --dry-run-docker)
            export DRY_RUN_DOCKER="true"
            echo "Dry run mode"
            echo
            shift
            ;;
        --)
            shift
            break
            ;;
        *)
            breeze::flags
            echo
            echo  "${COLOR_RED}ERROR: Unknown flag  ${COLOR_RESET}"
            echo
            exit 1
            ;;
        esac
    done
    local last_subcommand=""
    # Parse commands
    if [[ "$#" -ne 0 ]]; then
        case "${1}" in
        shell)
            last_subcommand="${1}"
            shift
            ;;
        exec)
            last_subcommand="${1}"
            command_to_run="run_exec"
            shift
            ;;
        docker-compose)
            last_subcommand="${1}"
            if [[ $# -lt 2 ]]; then
                echo "You should specify docker compose command to run"
                shift
                run_help="true"
            else
                docker_compose_command="${2}"
                shift 2
            fi
            command_to_run="run_docker_compose"
            ;;
        initialize-local-virtualenv)
            last_subcommand="${1}"
            echo "Initializing local virtualenv"
            echo
            command_to_run="perform_initialize_local_virtualenv"
            export INSTALL_PROVIDERS_FROM_SOURCES="true"
            shift
            ;;
        kind-cluster)
            last_subcommand="${1}"
            # Switch to production image for all kind operations
            export PRODUCTION_IMAGE="true"
            command_to_run="manage_kind_cluster"
            export KIND_CLUSTER_OPERATION="${2:-}"
            if [[ -n ${KIND_CLUSTER_OPERATION=} ]]; then
                shift 2
            else
                shift
            fi
            ;;
        setup-autocomplete)
            last_subcommand="${1}"
            echo "Setting up autocomplete"
            echo
            command_to_run="perform_setup_autocomplete"
            shift
            ;;
        start-airflow)
            last_subcommand="${1}"
            export START_AIRFLOW="true"
            shift
            ;;
        stop)
            last_subcommand="${1}"
            command_to_run="run_docker_compose"
            docker_compose_command="down"
            EXTRA_DC_OPTIONS+=("--remove-orphans")
            for INTEGRATION in ${_breeze_allowed_integrations}; do
                if [[ ${INTEGRATION} != "all" ]]; then
                    INTEGRATIONS+=("${INTEGRATION}")
                fi
            done
            # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
            # O_DIRECT parameter for direct writing to the filesystem
            # https://github.com/microsoft/mssql-docker/issues/13
            # so we need to mount an external volume for its db location
            # the external db must allow for parallel testing so external volume is mapped
            # to the data volume. Stop should also clean the volume
            rm -rf "${MSSQL_DATA_VOLUME:?"MSSQL_DATA_VOLUME should never be empty!"}"/*
            shift
            ;;
        restart)
            last_subcommand="${1}"
            command_to_run="run_docker_compose"
            docker_compose_command="down"
            EXTRA_DC_OPTIONS+=("--remove-orphans")
            second_command_to_run="enter_breeze"
            echo "Restarts the environment. Includes emptying the databases."
            shift
            ;;
        tests)
            last_subcommand="${1}"
            command_to_run="run_tests"
            shift
            ;;
        toggle-suppress-cheatsheet)
            last_subcommand="${1}"
            if [[ -f "${SUPPRESS_CHEATSHEET_FILE}" ]]; then
                rm -f "${SUPPRESS_CHEATSHEET_FILE}"
            else
                touch "${SUPPRESS_CHEATSHEET_FILE}"
            fi
            echo "Toggle suppress cheatsheet"
            echo
            shift
            command_to_run="toggle_suppress_cheatsheet"
            ;;
        toggle-suppress-asciiart)
            last_subcommand="${1}"
            if [[ -f "${SUPPRESS_ASCIIART_FILE}" ]]; then
                rm -f "${SUPPRESS_ASCIIART_FILE}"
            else
                touch "${SUPPRESS_ASCIIART_FILE}"
            fi
            echo "Toggle suppress asciiart"
            echo
            shift
            command_to_run="toggle_suppress_asciiart"
            ;;
        flags)
            breeze::flags
            exit 0
            ;;
        help)
            breeze::usage
            exit 0
            ;;
        help-all)
            breeze::do_help_all
            exit 0
            ;;
        *)
            breeze::usage
            echo
            echo  "${COLOR_RED}ERROR: Unknown command  ${COLOR_RESET}"
            echo
            exit 1
            ;;
        esac
    else
        :
        # By default, start interactive terminal
    fi

    if [[ ${run_help} == "true" ]]; then
        if [[ ${last_subcommand} == "" ]]; then
            breeze::usage
            breeze::flag_footer
        else
            breeze::detailed_usage "${last_subcommand}"
        fi
        exit 0
    fi

    if [[ ${PRESERVE_VOLUMES} != "true" ]]; then
        EXTRA_DC_OPTIONS+=("--volumes")
    fi
    # EXTRA_DC_OPTIONS is only used by Breeze. It's value is set here as well.
    readonly EXTRA_DC_OPTIONS

    # Also Remaining args are set here and set as read-only - no more changes to it.
    REMAINING_ARGS+=("$@")
    export REMAINING_ARGS
    readonly REMAINING_ARGS
}

#######################################################################################################
#
# Prepares nicely formatted versions of list of allowed and default values defined in Breeze.
# It is used in help command to print the lists in a readable format and fold the lists
# so that they fit the screen indented_screen_width.
#
# Used global variables:
#    _breeze_allowed_*
#
# Updated global constants:
#       FORMATTED_* constant variables that can be used in Breeze Help output
#
#######################################################################################################
function breeze::prepare_formatted_versions() {
    if [[ -n "${_breeze_formatted_versions_prepared:-}" ]]; then
        return
    fi

    _breeze_formatted_versions_prepared=1

    local indent=15
    local list_prefix
    list_prefix=$(printf "%-${indent}s" " ")
    local indented_screen_width=$((SCREEN_WIDTH - indent))

    FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS=$(echo "${_breeze_allowed_python_major_minor_versions=""}" | tr '\n' ' ' |
        fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS

    FORMATTED_BACKENDS=$(echo "${_breeze_allowed_backends=""}" | tr '\n' ' ' |
        fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_BACKENDS

    FORMATTED_INTEGRATIONS=$(echo "${_breeze_allowed_integrations=""}" | tr '\n' ' ' |
        fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_INTEGRATIONS

    FORMATTED_KUBERNETES_MODES=$(echo "${_breeze_allowed_kubernetes_modes=""}" | tr '\n' ' ' |
        fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_KUBERNETES_MODES

    FORMATTED_KUBERNETES_VERSIONS=$(echo "${_breeze_allowed_kubernetes_versions=""}" | tr '\n' ' ' |
        fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_KUBERNETES_VERSIONS

    FORMATTED_KIND_VERSIONS=$(echo "${_breeze_allowed_kind_versions=""}" | tr '\n' ' ' |
        fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_KIND_VERSIONS

    FORMATTED_HELM_VERSIONS=$(echo "${_breeze_allowed_helm_versions=""}" | tr '\n' ' ' |
        fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_HELM_VERSIONS

    FORMATTED_EXECUTORS=$(echo "${_breeze_allowed_executors=""}" | tr '\n' ' ' |
        fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_EXECUTORS

    FORMATTED_KIND_OPERATIONS=$(echo "${_breeze_allowed_kind_operations=""}" | tr '\n' ' ' |
        fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_KIND_OPERATIONS

    FORMATTED_POSTGRES_VERSIONS=$(echo "${_breeze_allowed_postgres_versions=""}" |
        tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_POSTGRES_VERSIONS

    FORMATTED_MYSQL_VERSIONS=$(echo "${_breeze_allowed_mysql_versions=""}" |
        tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_MYSQL_VERSIONS

    FORMATTED_MSSQL_VERSIONS=$(echo "${_breeze_allowed_mssql_versions=""}" |
        tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_MSSQL_VERSIONS

    FORMATTED_TEST_TYPES=$(echo "${_breeze_allowed_test_types=""}" |
        fold -w "${indented_screen_width}" -s | sed "s/ /,/g; s/^/${list_prefix}/")
    readonly FORMATTED_TEST_TYPES

    FORMATTED_PACKAGE_FORMATS=$(echo "${_breeze_allowed_package_formats=""}" |
        fold -w "${indented_screen_width}" -s | sed "s/ /,/g; s/^/${list_prefix}/")
    readonly FORMATTED_PACKAGE_FORMATS

    FORMATTED_USE_AIRFLOW_VERSION=$(echo "${_breeze_allowed_USE_AIRFLOW_VERSION=""}" |
        tr '\n' ' ' | fold -w "${indented_screen_width}" -s | sed "s/^/${list_prefix}/")
    readonly FORMATTED_USE_AIRFLOW_VERSION

}

#######################################################################################################
#
# Prepares usage information for all the commands in Breeze.
# Those usage commands are stored in appropriate environment variables.
#
# Created constants:
#    USAGE_*
#    DETAILED_USAGE_*
#
#######################################################################################################

# shellcheck disable=SC2034,SC2090,SC2089,SC2155

function breeze::prepare_usage() {
    if [[ -n "${_breeze_usage_prepared:-}" ]]; then
        return
    fi

    _breeze_usage_prepared=1

    breeze::prepare_formatted_versions

    # Note that MacOS uses Bash 3.* and we cannot use associative arrays
    export USAGE_SHELL="[Default] Enters interactive shell in the container"
    readonly USAGE_SHELL
    export USAGE_EXEC="Execs into running breeze container in new terminal"
    readonly USAGE_EXEC
    export USAGE_BUILD_DOCS="Builds documentation in the container"
    readonly USAGE_BUILD_DOCS
    export USAGE_DOCKER_COMPOSE="Executes specified docker-compose command"
    readonly USAGE_DOCKER_COMPOSE
    export USAGE_FLAGS="Shows all breeze's flags"
    readonly USAGE_FLAGS
    export USAGE_INITIALIZE_LOCAL_VIRTUALENV="Initializes local virtualenv"
    readonly USAGE_INITIALIZE_LOCAL_VIRTUALENV
    export USAGE_PUSH_IMAGE="Pushes images to registry"
    readonly USAGE_PUSH_IMAGE
    export USAGE_KIND_CLUSTER="Manages KinD cluster on the host"
    readonly USAGE_KIND_CLUSTER
    export USAGE_SETUP_AUTOCOMPLETE="Sets up autocomplete for breeze"
    readonly USAGE_SETUP_AUTOCOMPLETE
    export USAGE_START_AIRFLOW="Starts Scheduler and Webserver and enters the shell"
    readonly USAGE_START_AIRFLOW
    export USAGE_STOP="Stops the docker-compose environment"
    readonly USAGE_STOP
    export USAGE_RESTART="Stops the docker-compose environment including DB cleanup"
    readonly USAGE_RESTART
    export USAGE_TOGGLE_SUPPRESS_CHEATSHEET="Toggles on/off cheatsheet"
    readonly USAGE_TOGGLE_SUPPRESS_CHEATSHEET
    export USAGE_TOGGLE_SUPPRESS_ASCIIART="Toggles on/off asciiart"
    readonly USAGE_TOGGLE_SUPPRESS_ASCIIART
    export USAGE_TESTS="Runs selected tests in the container"
    readonly USAGE_TESTS
    export USAGE_HELP="Shows this help message"
    readonly USAGE_HELP
    export USAGE_HELP_ALL="Shows detailed help for all commands and flags"
    readonly USAGE_HELP_ALL

    export DETAILED_USAGE_SHELL="
${CMDNAME} shell [FLAGS] [-- <EXTRA_ARGS>]

      This is default subcommand if no subcommand is used.

      Enters interactive shell where you can run all tests, start Airflow webserver, scheduler,
      workers, interact with the database, run DAGs etc. It is the default command if no command
      is selected. The shell is executed in the container and in case integrations are chosen,
      the integrations will be started as separated docker containers - under the docker-compose
      supervision. Local sources are by default mounted to within the container so you can edit
      them locally and run tests immediately in the container. Several folders ('files', 'dist')
      are also mounted so that you can exchange files between the host and container.

      The 'files/airflow-breeze-config/variables.env' file can contain additional variables
      and setup. This file is automatically sourced when you enter the container. Database
      and webserver ports are forwarded to appropriate database/webserver so that you can
      connect to it from your host environment.

      You can also pass <EXTRA_ARGS> after -- they will be passed as bash parameters, this is
      especially useful to pass bash options, for example -c to execute command:

      '${CMDNAME} shell -- -c \"ls -la\"'
      '${CMDNAME} -- -c \"ls -la\"'

"
    readonly DETAILED_USAGE_SHELL
    export DETAILED_USAGE_EXEC="
${CMDNAME} exec [-- <EXTRA_ARGS>]

      Execs into interactive shell to an already running container. The container must be started
      already by breeze shell command. If you are not familiar with tmux, this is the best
      way to run multiple processes in the same container at the same time for example scheduler,
      webserver, workers, database console and interactive terminal.
"
    export DETAILED_USAGE_DOCKER_COMPOSE="
${CMDNAME} docker-compose [FLAGS] COMMAND [-- <EXTRA_ARGS>]

      Run docker-compose command instead of entering the environment. Use 'help' as command
      to see available commands. The <EXTRA_ARGS> passed after -- are treated
      as additional options passed to docker-compose. For example

      '${CMDNAME} docker-compose pull -- --ignore-pull-failures'

Flags:
$(breeze::flag_airflow_variants)
$(breeze::flag_backend_variants)
$(breeze::flag_verbosity)
"
    readonly DETAILED_USAGE_DOCKER_COMPOSE
    export DETAILED_USAGE_FLAGS="
      Explains in detail all the flags that can be used with breeze.
"
    readonly DETAILED_USAGE_FLAGS
    export DETAILED_USAGE_KIND_CLUSTER="
${CMDNAME} kind-cluster [FLAGS] OPERATION

      Manages host-side Kind Kubernetes cluster that is used to run Kubernetes integration tests.
      It allows to start/stop/restart/status the Kind Kubernetes cluster and deploy Airflow to it.
      This enables you to run tests inside the breeze environment with latest airflow images.
      Note that in case of deploying airflow, the first step is to rebuild the image and loading it
      to the cluster so you can also pass appropriate build image flags that will influence
      rebuilding the production image. Operation is one of:

${FORMATTED_KIND_OPERATIONS}

      The last two operations - shell and k9s allow you to perform interactive testing with
      kubernetes tests. You can enter the shell from which you can run kubernetes tests and in
      another terminal you can start the k9s CLI to debug kubernetes instance. It is an easy
      way to debug the kubernetes deployments.

      You can read more about k9s at https://k9scli.io/

Flags:
$(breeze::flag_airflow_variants)
"
    readonly DETAILED_USAGE_KIND_CLUSTER
    export DETAILED_USAGE_SETUP_AUTOCOMPLETE="
${CMDNAME} setup-autocomplete

      Sets up autocomplete for breeze commands. Once you do it you need to re-enter the bash
      shell and when typing breeze command <TAB> will provide autocomplete for
      parameters and values.
"
    readonly DETAILED_USAGE_SETUP_AUTOCOMPLETE
    export DETAILED_USAGE_START_AIRFLOW="
${CMDNAME} start-airflow

      Like the Shell command this will enter the interactive shell, but it will also start
      automatically the Scheduler and the Webserver. It will leave you in a tmux session where you
      can also observe what is happening in your Airflow.

      This is a convenient way to setup a development environment. Your dags will be loaded from the
      folder 'files/dags' on your host machine (it could take some times).

      If you want to load default connections and example dags you can use the dedicated flags.

Flags:
$(breeze::flag_use_different_airflow_version)
$(breeze::flag_start_airflow)
"
    readonly DETAILED_USAGE_START_AIRFLOW
    export DETAILED_USAGE_STOP="
${CMDNAME} stop

      Brings down running docker compose environment. When you start the environment, the docker
      containers will continue running so that startup time is shorter. But they take quite a lot of
      memory and CPU. This command stops all running containers from the environment.

Flags:
$(breeze::flag_stop_airflow)
"
    readonly DETAILED_USAGE_STOP
    export DETAILED_USAGE_RESTART="
${CMDNAME} restart [FLAGS]

      Restarts running docker compose environment. When you restart the environment, the docker
      containers will be restarted. That includes cleaning up the databases. This is
      especially useful if you switch between different versions of Airflow.

Flags:
$(breeze::flag_stop_airflow)
"
    readonly DETAILED_USAGE_RESTART
    export DETAILED_USAGE_TESTS="
${CMDNAME} tests [FLAGS] [TEST_TARGET ..] [-- <EXTRA_ARGS>]

      Run the specified unit test target. There might be multiple
      targets specified separated with comas. The <EXTRA_ARGS> passed after -- are treated
      as additional options passed to pytest. You can pass 'tests' as target to
      run all tests. For example:

      '${CMDNAME} tests tests/core/test_core.py -- --logging-level=DEBUG'
      '${CMDNAME} tests tests

Flags:
$(breeze::flag_tests)
"
    readonly DETAILED_USAGE_TESTS
    export DETAILED_USAGE_TOGGLE_SUPPRESS_CHEATSHEET="
${CMDNAME} toggle-suppress-cheatsheet

      Toggles on/off cheatsheet displayed before starting bash shell.
"
    readonly DETAILED_USAGE_TOGGLE_SUPPRESS_CHEATSHEET
    export DETAILED_USAGE_TOGGLE_SUPPRESS_ASCIIART="
${CMDNAME} toggle-suppress-asciiart

      Toggles on/off asciiart displayed before starting bash shell.
"
    readonly DETAILED_USAGE_TOGGLE_SUPPRESS_ASCIIART
    export DETAILED_USAGE_HELP="
${CMDNAME} help

      Shows general help message for all commands.
"
    readonly DETAILED_USAGE_HELP
    export DETAILED_USAGE_HELP_ALL="
${CMDNAME} help-all

      Shows detailed help for all commands and flags.
"
    readonly DETAILED_USAGE_HELP_ALL
}

# shellcheck enable=all

#######################################################################################################
#
# Gets environment variable value converting the lowercase name of command into variable name
# Arguments:
#     prefix for the variable to add
#     name of the variable
# Outputs:
#     Writes the capitalized name of the variable to stdout
#######################################################################################################
function breeze::get_variable_from_lowercase_name() {
    local prefix="${1}"
    local name="${2}"
    local suffix
    suffix="$(echo "${name}" | tr "[:lower:]-" "[:upper:]_")"
    local variable_name="${prefix}_${suffix}"
    echo "${!variable_name}"
}

#######################################################################################################
#
# Gets usage information from lowercase command
# Arguments:
#     lowercase command name
# Outputs:
#    usage information for the command.
#######################################################################################################
function breeze::get_usage() {
    breeze::prepare_usage
    breeze::get_variable_from_lowercase_name "USAGE" "${1}"
}

#######################################################################################################
#
# Gets detailed usage information from lowercase command
# Arguments:
#     lowercase command name
# Outputs:
#    Detailed usage information for the command.
#######################################################################################################
function breeze::get_detailed_usage() {
    breeze::prepare_usage
    breeze::get_variable_from_lowercase_name "DETAILED_USAGE" "${1}"
}

#######################################################################################################
#
# Prints general usage information for all commands.
#
# Globals used:
#    CMDNAME
#    _breeze_commands
#    _breeze_extra_arg_commands
#    _breeze_help_commands
#
# Outputs:
#    General usage information for all commands.
#######################################################################################################
function breeze::usage() {
    breeze::prepare_usage
    echo "

usage: ${CMDNAME} [FLAGS] [COMMAND] -- <EXTRA_ARGS>

By default the script enters the  CI container and drops you to bash shell, but you can choose
one of the commands to run specific actions instead.

Add --help after each command to see details:

Commands without arguments:
"
    for subcommand in ${_breeze_commands}; do
        printf "  %-40s %s\n" "${subcommand}" "$(breeze::get_usage "${subcommand}")"
    done
    echo "
Commands with arguments:
"
    # shellcheck disable=SC2154
    for subcommand in ${_breeze_extra_arg_commands}; do
        printf "  %-35s%-10s %s\n" "${subcommand}" "<ARG>" "$(breeze::get_usage "${subcommand}")"
    done
    echo "
Help commands:
"
    # shellcheck disable=SC2154
    for subcommand in ${_breeze_help_commands}; do
        printf "  %-40s %s\n" "${subcommand}" "$(breeze::get_usage "${subcommand}")"
    done
    echo
}

#######################################################################################################
#
# Prints detailed usage for command specified
#
# Argument:
#    Command name.
#
# Outputs:
#    Detailed  usage information for the command
#######################################################################################################
# Prints detailed usage for command specified
function breeze::detailed_usage() {
    subcommand=${1}
    echo "

Detailed usage for command: ${subcommand}

$(breeze::get_detailed_usage "${subcommand}")

"
}

#######################################################################################################
#
# Prints flag footer - common to all commands..
#
# Outputs:
#    Footer common for all commands.
#######################################################################################################
function breeze::flag_footer() {
    echo "
Run '${CMDNAME} flags' to see all applicable flags.
"
}

#######################################################################################################
#
# Prints flags for different variants of airflow to use
#
# Global constants used:
#       FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_airflow_variants() {
    echo "
-p, --python PYTHON_MAJOR_MINOR_VERSION
        Python version used for the image. This is always major/minor version.

        One of:

${FORMATTED_PYTHON_MAJOR_MINOR_VERSIONS}
"
}

#######################################################################################################
#
# Prints flags for different backend to use
#
# Global constants used:
#       FORMATTED_BACKENDS
#       FORMATTED_POSTGRES_VERSIONS
#       FORMATTED_MYSQL_VERSIONS
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_backend_variants() {
    echo "
-b, --backend BACKEND
        Backend to use for tests - it determines which database is used.
        One of:

${FORMATTED_BACKENDS}

        Default: ${_breeze_default_backend:=}

--postgres-version POSTGRES_VERSION
        Postgres version used. One of:

${FORMATTED_POSTGRES_VERSIONS}

--mysql-version MYSQL_VERSION
        MySql version used. One of:

${FORMATTED_MYSQL_VERSIONS}

--mssql-version MSSQL_VERSION
        MSSql version used. One of:

${FORMATTED_MSSQL_VERSIONS}

"
}

#######################################################################################################
#
# Prints production image flags
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_production_image() {
    echo "
-I, --production-image
        Use production image for entering the environment and builds (not for tests).
"
}

#######################################################################################################
#
# Prints additional breeze action flags
#
# Global constants used:
#       FORMATTED_INTEGRATIONS
#
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_breeze_actions() {
    echo "
-d, --db-reset
        Resets the database at entry to the environment. It will drop all the tables
        and data and recreate the DB from scratch even if 'restart' command was not used.
        Combined with 'restart' command it enters the environment in the state that is
        ready to start Airflow webserver/scheduler/worker. Without the switch, the database
        does not have any tables and you need to run reset db manually.

-i, --integration INTEGRATION
        Integration to start during tests - it determines which integrations are started
        for integration tests. There can be more than one integration started, or all to
        start all integrations. Selected integrations are not saved for future execution.
        One of:

${FORMATTED_INTEGRATIONS}

--init-script INIT_SCRIPT_FILE
        Initialization script name - Sourced from files/airflow-breeze-config. Default value
        init.sh. It will be executed after the environment is configured and started.

"
}

#######################################################################################################
#
# Prints Kubernetes action flags
#
# Global constants used:
#       FORMATTED_KUBERNETES_MODE
#       FORMATTED_KUBERNETES_VERSIONS
#       FORMATTED_KIND_VERSIONS
#       FORMATTED_HELM_VERSIONS
#       FORMATTED_EXECUTORS
#
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_kubernetes_configuration() {
    echo "
Configuration for the KinD Kubernetes cluster and tests:

-K, --kubernetes-mode KUBERNETES_MODE
        Kubernetes mode - only used in case one of kind-cluster commands is used.
        One of:

${FORMATTED_KUBERNETES_MODES}

        Default: ${_breeze_default_kubernetes_mode:=}

-V, --kubernetes-version KUBERNETES_VERSION
        Kubernetes version - only used in case one of kind-cluster commands is used.
        One of:

${FORMATTED_KUBERNETES_VERSIONS}

        Default: ${_breeze_default_kubernetes_version:=}

--kind-version KIND_VERSION
        Kind version - only used in case one of kind-cluster commands is used.
        One of:

${FORMATTED_KIND_VERSIONS}

        Default: ${_breeze_default_kind_version:=}

--helm-version HELM_VERSION
        Helm version - only used in case one of kind-cluster commands is used.
        One of:

${FORMATTED_HELM_VERSIONS}

        Default: ${_breeze_default_helm_version:=}

--executor EXECUTOR
        Executor to use in a kubernetes cluster.
        One of:

${FORMATTED_EXECUTORS}

        Default: ${_breeze_default_executor:=}

"
}

#######################################################################################################
#
# Prints flags that determine what is the source mounting scheme
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_local_file_mounting() {
    echo "
-l, --skip-mounting-local-sources
        Skips mounting local volume with sources - you get exactly what is in the
        docker image rather than your current local sources of Airflow.

"
}

#######################################################################################################
#
# Prints flags that allow to use different airflow variants
#
# Global constants used:
#       FORMATTED_INSTALL_AIRFLOW_VERSIONS
#
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_use_different_airflow_version() {
    echo "
--use-airflow-version AIRFLOW_SPECIFICATION
        In CI image, installs Airflow at runtime from PIP released version or using
        the installation method specified (sdist, wheel, none). When 'none' is used,
        airflow is just removed. In this case airflow package should be added to dist folder
        and --use-packages-from-dist flag should be used.

${FORMATTED_USE_AIRFLOW_VERSION}

--use-packages-from-dist
        In CI image, if specified it will look for packages placed in dist folder and
        it will install the packages after entering the image.
        This is useful for testing provider packages.

"
}

#######################################################################################################
#
# Prints flags that allow to set assumed answers to questions
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_assume_answers_to_questions() {
    echo "
-y, --assume-yes
        Assume 'yes' answer to all questions.

-n, --assume-no
        Assume 'no' answer to all questions.

-q, --assume-quit
        Assume 'quit' answer to all questions.
"
}

#######################################################################################################
#
# Prints flags that are used for credential forwarding
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_credentials() {
    echo "
-f, --forward-credentials
        Forwards host credentials to docker container. Use with care as it will make
        your credentials available to everything you install in Docker.
"
}

#######################################################################################################
#
# Prints flags that control package preparation
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_packages() {
    echo "
--package-format PACKAGE_FORMAT

        Chooses format of packages to prepare.

        One of:

${FORMATTED_PACKAGE_FORMATS}

        Default: ${_breeze_default_package_format:=}

"
}

#######################################################################################################
#
# Prints flags that control verbosity
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_verbosity() {
    echo "
-v, --verbose
        Show verbose information about executed docker, kind, kubectl, helm commands. Useful for
        debugging - when you run breeze with --verbose flags you will be able to see the commands
        executed under the hood and copy&paste them to your terminal to debug them more easily.

        Note that you can further increase verbosity and see all the commands executed by breeze
        by running 'export VERBOSE_COMMANDS=\"true\"' before running breeze.

--dry-run-docker
        Only show docker commands to execute instead of actually executing them. The docker
        commands are printed in yellow color.

"
}

#######################################################################################################
#
# Prints information about help flag
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_help() {
    echo "
-h, --help
        Shows detailed help message for the command specified.
"
}

#####################################################################################################
#
# Prints flags that control how Airflow should be populated with the command start-airflow
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_start_airflow() {
    echo "
--load-example-dags
        Include Airflow example dags.

--load-default-connections
        Include Airflow Default Connections.
"
}

#####################################################################################################
#
# Prints flags that control how Airflow should be populated with the command stop-airflow
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_stop_airflow() {
    echo "
--preserve-volumes
        Use this flag if you would like to preserve data volumes from the databases used
        by the integrations. By default, those volumes are deleted, so when you run 'stop'
        or 'restart' commands you start from scratch, but by using this flag you can
        preserve them. If you want to delete those volumes after stopping Breeze, just
        run the 'breeze stop' again without this flag.
"
}


#####################################################################################################
#
# Prints flags that control tests
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flag_tests() {
    echo "
--test-type TEST_TYPE
        Type of the test to run. One of:

${FORMATTED_TEST_TYPES}

        Default: ${_breeze_default_test_type:=}

"
}

#######################################################################################################
#
# Prints all flags
#
# Outputs:
#    Flag information.
#######################################################################################################
function breeze::flags() {
    breeze::prepare_formatted_versions

    echo "
$(breeze::print_line)

Summary of all flags supported by Breeze:

$(breeze::print_star_line)
 Choose Airflow variant
$(breeze::flag_airflow_variants)

$(breeze::print_star_line)
 Choose backend to run for Airflow
$(breeze::flag_backend_variants)

$(breeze::print_star_line)
 Enable production image
$(breeze::flag_production_image)

$(breeze::print_star_line)
 Additional actions executed while entering breeze
$(breeze::flag_breeze_actions)

$(breeze::print_star_line)
 Additional actions executed while starting Airflow
$(breeze::flag_start_airflow)

$(breeze::print_star_line)
 Cleanup options when stopping Airflow
$(breeze::flag_stop_airflow)

$(breeze::print_star_line)
 Kind kubernetes and Kubernetes tests configuration(optional)
$(breeze::flag_kubernetes_configuration)

$(breeze::print_star_line)
 Manage mounting local files
$(breeze::flag_local_file_mounting)

$(breeze::print_star_line)
 Assume answers to questions
$(breeze::flag_assume_answers_to_questions)

$(breeze::print_star_line)
 Use different Airflow version at runtime in CI image
$(breeze::flag_use_different_airflow_version)

$(breeze::print_star_line)
 Credentials
$(breeze::flag_credentials)

$(breeze::print_star_line)
 Flags for running tests
$(breeze::flag_tests)

$(breeze::print_star_line)
 Increase verbosity of the scripts
$(breeze::flag_verbosity)

$(breeze::print_star_line)
 Print detailed help message
$(breeze::flag_help)
"
}

#######################################################################################################
#
# Prints header line filling screen indented_screen_width - only when VERBOSE is set
#
# Outputs:
#    Prints header line.
#######################################################################################################
function breeze::print_header_line() {
    if [ ${VERBOSE:="false"} == "true" ]; then
        echo
        printf '=%.0s' $(seq "${SCREEN_WIDTH}")
        echo
    fi
}

#######################################################################################################
#
# Prints separation line filling screen indented_screen_width
#
# Outputs:
#    Prints line.
#######################################################################################################
function breeze::print_line() {
    printf '#%.0s' $(seq "${SCREEN_WIDTH}")
}

# Prints star line filling screen indented_screen_width
function breeze::print_star_line() {
    printf '*%.0s' $(seq "${SCREEN_WIDTH}")
}

#######################################################################################################
#
# Reads saved environment variables. Some of the variables are stored across session so that once
# you use them you do not have to use it next time. That makes those flags persistent.
#
# An example of it is `--backend` or '--kubernetes-mode' flags.
#
# Note that PYTHON_MAJOR_MINOR_VERSION is not read here - it is read at the
# `setup_default_breeze_variables` method because it is needed
# to determine the right images to use and set several variables that depend on the Python version
#
# Uses:
#   _breeze_default_* corresponding defaults for variables it reads
#
# Modified and used global constants:
#
#   BACKEND
#   KUBERNETES_MODE
#   KUBERNETES_VERSION
#   KIND_VERSION
#   HELM_VERSION
#   EXECUTOR
#   POSTGRES_VERSION
#   MYSQL_VERSION
#
#######################################################################################################

function breeze::read_saved_environment_variables() {
    BACKEND="${BACKEND:=$(parameters::read_from_file BACKEND)}"
    BACKEND=${BACKEND:-${_breeze_default_backend}}

    KUBERNETES_MODE="${KUBERNETES_MODE:=$(parameters::read_from_file KUBERNETES_MODE)}"
    KUBERNETES_MODE=${KUBERNETES_MODE:=${_breeze_default_kubernetes_mode}}

    KUBERNETES_VERSION="${KUBERNETES_VERSION:=$(parameters::read_from_file KUBERNETES_VERSION)}"
    KUBERNETES_VERSION=${KUBERNETES_VERSION:=${_breeze_default_kubernetes_version}}

    KIND_VERSION="${KIND_VERSION:=$(parameters::read_from_file KIND_VERSION)}"
    KIND_VERSION=${KIND_VERSION:=${_breeze_default_kind_version}}

    HELM_VERSION="${HELM_VERSION:=$(parameters::read_from_file HELM_VERSION)}"
    HELM_VERSION=${HELM_VERSION:=${_breeze_default_helm_version}}

    EXECUTOR="${EXECUTOR:=$(parameters::read_from_file EXECUTOR)}"
    EXECUTOR=${EXECUTOR:-${_breeze_default_executor}}

    POSTGRES_VERSION="${POSTGRES_VERSION:=$(parameters::read_from_file POSTGRES_VERSION)}"
    POSTGRES_VERSION=${POSTGRES_VERSION:=${_breeze_default_postgres_version}}

    MYSQL_VERSION="${MYSQL_VERSION:=$(parameters::read_from_file MYSQL_VERSION)}"
    MYSQL_VERSION=${MYSQL_VERSION:=${_breeze_default_mysql_version}}

    MSSQL_VERSION="${MSSQL_VERSION:=$(parameters::read_from_file MSSQL_VERSION)}"
    MSSQL_VERSION=${MSSQL_VERSION:=${_breeze_default_mssql_version}}
}

#######################################################################################################
#
# Checks if variables are correctly set and if they are - saves them so that they can be used across
# sessions.
#
# In case the variables are matching expected values they are saved in ".build/VARIABLE_NAME" for
# later reuse. If not, error is printed and the saved file is cleaned, so that next time
# default value can be used.
#
# Used Global constants:
#     INSTALL_AIRFLOW_REFERENCE
#     INSTALL_AIRFLOW_VERSION
#     BACKEND
#     KUBERNETES_MODE
#     KUBERNETES_VERSION
#     KIND_VERSION
#     HELM_VERSION
#     EXECUTOR
#     POSTGRES_VERSION
#     MYSQL_VERSION
#     MSSQL_VERSION
#
# Updated Global constants:
#      BRANCH_NAME
#
# Output: saved variable files in .build,
#######################################################################################################
function breeze::check_and_save_all_params() {
    parameters::check_and_save_allowed_param "PYTHON_MAJOR_MINOR_VERSION" "Python version" "--python"
    # TODO: Generalize me (make it automatically use matching versions)
    if [[ -n "${INSTALL_AIRFLOW_REFERENCE=}" ]]; then
        if [[ ${INSTALL_AIRFLOW_REFERENCE=} == *2_0* ]]; then
            export BRANCH_NAME="v2-0-test"
        elif [[ ${INSTALL_AIRFLOW_REFERENCE=} == *2_1* ]]; then
            export BRANCH_NAME="v2-1-test"
        elif [[ ${INSTALL_AIRFLOW_REFERENCE=} == *2_2* ]]; then
            export BRANCH_NAME="v2-2-test"
        fi
    elif [[ -n "${INSTALL_AIRFLOW_VERSION=}" ]]; then
        if [[ ${INSTALL_AIRFLOW_VERSION=} == *2.0* ]]; then
            export BRANCH_NAME="v2-0-test"
        elif [[ ${INSTALL_AIRFLOW_VERSION=} == *2.1* ]]; then
            export BRANCH_NAME="v2-1-test"
        elif [[ ${INSTALL_AIRFLOW_VERSION=} == *2.2* ]]; then
            export BRANCH_NAME="v2-2-test"
        fi
    fi

    parameters::check_and_save_allowed_param "BACKEND" "backend" "--backend"
    parameters::check_and_save_allowed_param "KUBERNETES_MODE" "Kubernetes mode" "--kubernetes-mode"
    parameters::check_and_save_allowed_param "KUBERNETES_VERSION" "Kubernetes version" "--kubernetes-version"
    parameters::check_and_save_allowed_param "KIND_VERSION" "KinD version" "--kind-version"
    parameters::check_and_save_allowed_param "HELM_VERSION" "Helm version" "--helm-version"
    parameters::check_and_save_allowed_param "EXECUTOR" "Executors" "--executor"
    parameters::check_and_save_allowed_param "POSTGRES_VERSION" "Postgres version" "--postgres-version"
    parameters::check_and_save_allowed_param "MYSQL_VERSION" "Mysql version" "--mysql-version"
    parameters::check_and_save_allowed_param "MSSQL_VERSION" "MSSql version" "--mssql-version"

    parameters::check_allowed_param TEST_TYPE "Type of tests" "--test-type"
}

#######################################################################################################
#
# Prints cheatsheet if it is not suppressed
#
# Used global constants:
#
#     AIRFLOW_SOURCES
#     SSH_PORT
#     WEBSERVER_HOST_PORT
#     POSTGRES_HOST_PORT
#     MYSQL_HOST_PORT
#     MSSQL_HOST_PORT
#
#######################################################################################################
function breeze::print_cheatsheet() {
    if [[ ! -f ${SUPPRESS_CHEATSHEET_FILE} && ${command_to_run} == "enter_breeze" ]]; then

        echo
        breeze::print_line
        echo
        echo "                                  Airflow Breeze CHEATSHEET"
        echo
        set +e
        if ! command -v breeze; then
            breeze::print_line
            echo
            echo " Adding breeze to your path:"
            echo "    When you exit the environment, you can add sources of Airflow to the path - you can"
            echo "    run breeze or the scripts above from any directory by calling 'breeze' commands directly"
            echo
            echo "     export PATH=\${PATH}:\"${AIRFLOW_SOURCES}\""
            echo
        fi
        set -e
        breeze::print_line

        echo
        echo " Port forwarding:"
        echo
        echo "   Ports are forwarded to the running docker containers for webserver and database"
        echo "     * ${SSH_PORT} -> forwarded to Airflow ssh server -> airflow:22"
        echo "     * ${WEBSERVER_HOST_PORT} -> forwarded to Airflow webserver -> airflow:8080"
        echo "     * ${FLOWER_HOST_PORT} -> forwarded to Flower dashboard -> airflow:5555"
        echo "     * ${POSTGRES_HOST_PORT} -> forwarded to Postgres database -> postgres:5432"
        echo "     * ${MYSQL_HOST_PORT} -> forwarded to MySQL database  -> mysql:3306"
        echo "     * ${MSSQL_HOST_PORT} -> forwarded to MSSQL database  -> mssql:1443"
        echo "     * ${REDIS_HOST_PORT} -> forwarded to Redis broker -> redis:6379"
        echo
        echo "   Here are links to those services that you can use on host:"
        echo "     * ssh connection for remote debugging: ssh -p ${SSH_PORT} airflow@127.0.0.1 pw: airflow"
        echo "     * Webserver: http://127.0.0.1:${WEBSERVER_HOST_PORT}"
        echo "     * Flower:    http://127.0.0.1:${FLOWER_HOST_PORT}"
        echo "     * Postgres:  jdbc:postgresql://127.0.0.1:${POSTGRES_HOST_PORT}/airflow?user=postgres&password=airflow"
        echo "     * Mysql:     jdbc:mysql://127.0.0.1:${MYSQL_HOST_PORT}/airflow?user=root"
        echo "     * MSSQL:     jdbc:sqlserver://127.0.0.1:${MSSQL_HOST_PORT};databaseName=airflow;user=sa;password=Airflow123"
        echo "     * Redis:     redis://127.0.0.1:${REDIS_HOST_PORT}/0"
    else
        echo
    fi
}

#######################################################################################################
#
# Prints setup instruction in case we find that autocomplete is not set
# also prints how to toggle asciiart/cheatsheet
#
# Used global constants:
#     CMDNAME
#######################################################################################################
function breeze::print_setup_instructions() {
    if [[ ${command_to_run} == "enter_breeze" ]]; then
        # shellcheck disable=SC2034  # Unused variables left for comp_breeze usage
        if ! typeset -f "_comp_breeze" >/dev/null; then
            breeze::print_line
            echo
            echo "  You can setup autocomplete by running '${CMDNAME} setup-autocomplete'"
            echo
            echo
        fi
        breeze::print_line
        echo
        echo "  You can toggle ascii/cheatsheet by running:"
        echo "      * ${CMDNAME} toggle-suppress-cheatsheet"
        echo "      * ${CMDNAME} toggle-suppress-asciiart"
        echo
        breeze::print_line
        echo
        echo
        echo
        echo
    fi
}

#######################################################################################################
#
# Runs Build before a command if needed - the right build command will be determined and used
# depending on which command you are trying ton
#
# Used global constants:
#   PRODUCTION_IMAGE
#   KIND_CLUSTER_OPERATION
#   FORMATTED_KIND_OPERATIONS
#
# Used variables:
#   command_to_run
#
#######################################################################################################
function breeze::run_build_command() {
    case "${command_to_run}" in
    run_tests | run_docker_compose)
        ;;
    enter_breeze)
        ;;
    build_docs )
        ;;
    perform_initialize_local_virtualenv | perform_setup_autocomplete | \
        toggle_suppress_cheatsheet | toggle_suppress_asciiart )
        ;;
    manage_kind_cluster)
        if [[ ${KIND_CLUSTER_OPERATION} == "start" ]]; then
            echo "Starts KinD cluster"
        elif [[ ${KIND_CLUSTER_OPERATION} == "stop" ]]; then
            echo "Stops KinD cluster"
        elif [[ ${KIND_CLUSTER_OPERATION} == "restart" ]]; then
            echo "Restarts KinD cluster"
        elif [[ ${KIND_CLUSTER_OPERATION} == "recreate" ]]; then
            echo "Recreates KinD cluster"
        elif [[ ${KIND_CLUSTER_OPERATION} == "status" ]]; then
            echo "Checks status of KinD cluster"
        elif [[ ${KIND_CLUSTER_OPERATION} == "deploy" ]]; then
            echo "Deploys Airflow to KinD cluster"
        elif [[ ${KIND_CLUSTER_OPERATION} == "test" ]]; then
            echo "Run Kubernetes tests with the KinD cluster "
        elif [[ ${KIND_CLUSTER_OPERATION} == "shell" ]]; then
            echo "Enter an interactive shell for kubernetes testing"
        elif [[ ${KIND_CLUSTER_OPERATION} == "k9s" ]]; then
            echo "Run k9s cli to debug in style"
        elif [[ -z ${KIND_CLUSTER_OPERATION=} ]]; then
            echo
            echo "Please provide an operation to run"
            echo
            echo "Should be one of:"
            echo "${FORMATTED_KIND_OPERATIONS}"
            echo
            exit 1
        else
            echo
            echo "ERROR: Unknown Kind Kubernetes cluster operation: '${KIND_CLUSTER_OPERATION}'"
            echo
            echo "Should be one of:"
            echo "${FORMATTED_KIND_OPERATIONS}"
            echo
            exit 1
        fi
        ;;
    *)
        echo
        echo  "${COLOR_RED}ERROR: Unknown command to run ${command_to_run}  ${COLOR_RESET}"
        echo
        exit 1
        ;;
    esac
}

# executes command
function breeze::run_command() {
    "${@}"
}


# print command instead of executing
function breeze::print_command() {
    echo
    echo "${COLOR_YELLOW}" "${@}" "${COLOR_RESET}"
    echo
}


#######################################################################################################
#
# Runs the actual command - depending on the command chosen it will use the right
# Convenient script and run the right command for the script.
#
# Used variables:
#   command_to_run
#
# Used global constants:
#     PRODUCTION_IMAGE
#     SCRIPTS_CI_DIR
#     BUILD_CACHE_DIR
#     KIND_CLUSTER_OPERATION
#     EXTRA_DC_OPTIONS
#
# Set Global variables:
#     RUN_TESTS
#######################################################################################################
function breeze::run_breeze_command() {
    set +u
    local dc_run_file
    local run_command="breeze::run_command"
    if [[ ${DRY_RUN_DOCKER=} != "false" ]]; then
        run_command="breeze::print_command"
    fi
    if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
        dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_PROD}"
    else
        dc_run_file="${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}"
    fi

    case "${command_to_run}" in
    enter_breeze)
        docker_engine_resources::check_all_resources
        if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then
            if [[ ${BACKEND} == "mysql" || ${BACKEND} == "mssql" ]]; then
                echo "${COLOR_RED}MacOS with ARM processor is not supported for ${BACKEND} backend. Exiting.${COLOR_RESET}"
                exit 1
            fi
        fi
        if [[ ${PRODUCTION_IMAGE} == "true" ]]; then
            echo "${COLOR_RED}ERROR: Entering production image via breeze is not supported${COLOR_RESET}"
            echo
            echo "Please run ${COLOR_BLUE}docker run -it apache/airflow:latest bash${COLOR_RESET} instead"
            echo
            echo "See https://airflow.apache.org/docs/docker-stack/entrypoint.html#executing-commands for details"
            exit 1
        else
            ${run_command} "${dc_run_file}" run --service-ports --rm airflow "${@}"
        fi
        ;;
    run_exec)
        docker_engine_resources::check_all_resources
        # Unfortunately `docker-compose exec` does not support exec'ing into containers started with run :(
        # so we have to find it manually
        set +e
        local airflow_testing_container
        airflow_testing_container=$("${dc_run_file}" ps | grep airflow | awk '{print $1}' 2>/dev/null)
        : "${airflow_testing_container:?"ERROR! Breeze must be running in order to exec into running container"}"
        set -e
        docker exec -it "${airflow_testing_container}" \
            "/opt/airflow/scripts/docker/entrypoint_exec.sh" "${@}"
        ;;
    run_tests)
        docker_engine_resources::check_all_resources
        export RUN_TESTS="true"
        readonly RUN_TESTS
        export ENABLED_INTEGRATIONS="${INTEGRATIONS[*]}"
        export LIST_OF_INTEGRATION_TESTS_TO_RUN="${INTEGRATIONS[*]}"
        ${run_command} "${BUILD_CACHE_DIR}/${DOCKER_COMPOSE_RUN_SCRIPT_FOR_CI}" run --service-ports --rm airflow "$@"
        ;;
    run_docker_compose)
        docker_engine_resources::check_all_resources
        set +u
        ${run_command} "${dc_run_file}" "${docker_compose_command}" "${EXTRA_DC_OPTIONS[@]}" "$@"
        set -u
        ;;
    perform_initialize_local_virtualenv)
        breeze::initialize_virtualenv
        ;;
    perform_setup_autocomplete)
        breeze::setup_autocomplete
        ;;
    manage_kind_cluster)
        docker_engine_resources::check_all_resources
        kind::make_sure_kubernetes_tools_are_installed
        kind::get_kind_cluster_name
        kind::perform_kind_cluster_operation "${KIND_CLUSTER_OPERATION}"
        ;;
    build_docs)
        docker_engine_resources::check_all_resources
        runs::run_docs "${@}"
        ;;
    toggle_suppress_cheatsheet)
        if [[ -f "${SUPPRESS_CHEATSHEET_FILE}" ]]; then
            echo
            echo "Cheatsheet disabled"
            echo
        else
            echo
            echo "Cheatsheet enabled"
            echo
        fi
        ;;
    toggle_suppress_asciiart)
        if [[ -f "${SUPPRESS_ASCIIART_FILE}" ]]; then
            echo
            echo "ASCIIart disabled"
            echo
        else
            echo
            echo "ASCIIart enabled"
            echo
        fi
        ;;
    *)
        echo
        echo  "${COLOR_RED}ERROR: Unknown command to run ${command_to_run}  ${COLOR_RESET}"
        echo
        ;;
    esac
    set -u
}

#######################################################################################################
#
# We have different versions of images depending on the python version used. We keep up with the
# Latest patch-level changes in Python (this is done automatically during CI test runs) so we have
# To only take into account MAJOR and MINOR version of python. This variable keeps the major/minor
# version of python in X.Y format (3.7, 3.8, 3.9, 3.10).
#
# In Breeze the precedence of setting the version is as follows:
#      1. --python flag (if set, it will explicitly override it in the next step)
#      2. PYTHON_MAJOR_MINOR_VERSION exported from outside
#      3. last used version stored in ./build/PYTHON_MAJOR_MINOR_VERSION
#      4. DEFAULT_PYTHON_MAJOR_MINOR_VERSION from scripts/ci/libraries/_initialization.sh
#
# Here points 2. and 3. are realized. If result is empty string , the 4. will be set in
#      the next step (sanity_checks::basic_sanity_checks() is called and the version is still not set by then)
#      finally, if  --python flag is specified, it will override whatever is set above.
#
# We need to run after initialization::initialize_common_environment (so that parameters::read_from_file function is present)
# But before we set the default value for Python
#
# Used and modified global constants:
#     PYTHON_MAJOR_MINOR_VERSION
#######################################################################################################
function breeze::determine_python_version_to_use_in_breeze() {
    PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION:=$(parameters::read_from_file PYTHON_MAJOR_MINOR_VERSION)}"
    export PYTHON_MAJOR_MINOR_VERSION
}

breeze::setup_default_breeze_constants

initialization::create_directories

breeze::read_saved_environment_variables

initialization::initialize_common_environment

initialization::check_docker_version

initialization::get_environment_for_builds_on_ci

breeze::determine_python_version_to_use_in_breeze

initialization::get_docker_cache_image_names

sanity_checks::basic_sanity_checks

start_end::script_start

traps::add_trap start_end::script_end EXIT

set +u
breeze::parse_arguments "${@}"

breeze::print_header_line

breeze::check_and_save_all_params

initialization::make_constants_read_only

breeze::prepare_command_files

breeze::run_build_command

breeze::print_header_line

breeze::print_badge

breeze::print_cheatsheet

breeze::print_setup_instructions

set +u # Account for an empty array
breeze::run_breeze_command "${REMAINING_ARGS[@]}"

set +u # Account for an empty array
if [[ -n ${second_command_to_run} ]]; then
    command_to_run=${second_command_to_run}
    breeze::run_breeze_command "${REMAINING_ARGS[@]}"
fi