Skip to content

Commit

Permalink
fix:fail early for any required script failure
Browse files Browse the repository at this point in the history
When greenboot encounters an error in required scripts,it braks out of
the healthcheck loop and moves to redscript excution skipping the other
required and wanted scripts.Logs are added to notify users of the
skipped required scripts due to critical failure.Test modified to
validate only one error is reported.Also cleasning code base of redundant
codes.

Signed-off-by: saypaul <[email protected]>
  • Loading branch information
say-paul committed Feb 7, 2025
1 parent 275985c commit b943b39
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 36 deletions.
14 changes: 9 additions & 5 deletions tests/greenboot_check_fail_required.bats
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,16 @@ function setup() {
[ "$status" -ne 0 ]
}

@test "Test greenboot runs all required scripts even if one fails" {
@test "Test greenboot exits when one required script fails" {
run $GREENBOOT_BIN_PATH check
[[ "$output" == *"10_failing_check"* ]]
[[ "$output" == *"20_failing_check"* ]]
[[ "$output" == *"30_failing_check"* ]]
[[ "$output" == *"40_failing_check"* ]]
[ "$status" -ne 0 ] # Ensure greenboot exits with a non-zero status

# Count occurrences of failing scripts in the output
fail_count=$(echo "$output" | grep -E "FAILURE" | \
grep -c -E "10_failing_check.sh|20_failing_check.sh|30_failing_check.sh|40_failing_check.sh")

# Ensure only one failing script is reported
[ "$fail_count" -eq 1 ]
}

function teardown() {
Expand Down
46 changes: 15 additions & 31 deletions usr/libexec/greenboot/greenboot
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ LC_ALL=C
SCRIPTS_CHECK_PATHS=("/usr/lib/greenboot/check" "/etc/greenboot/check")
SCRIPTS_GREEN_PATHS=("/usr/lib/greenboot/green.d" "/etc/greenboot/green.d")
SCRIPTS_RED_PATHS=("/usr/lib/greenboot/red.d" "/etc/greenboot/red.d")
declare -A DIS_CHECK_STATUS=()

source_configuration_file() {
greenboot_configuration_file=/etc/greenboot/greenboot.conf
Expand All @@ -16,51 +15,34 @@ source_configuration_file() {
fi
}

source_configuration_file
function init_disabled_map {
for disabled_healthcheck in "${DISABLED_HEALTHCHECKS[@]}"; do
DIS_CHECK_STATUS["$disabled_healthcheck"]=1;
done
}

source_configuration_file
function print_unexecuted_checks {
for disabled_healthcheck in "${DISABLED_HEALTHCHECKS[@]}"; do
if [[ "${DIS_CHECK_STATUS[$disabled_healthcheck]}" == 1 ]]; then
echo "WARNING: $disabled_healthcheck wasn't executed because it's disabled"
fi
done
}

source_configuration_file
function is_disabled {
healthcheck=$1
for disabled_healthcheck in "${DISABLED_HEALTHCHECKS[@]}"; do
if [ "${healthcheck}" == "${disabled_healthcheck}" ]; then
DIS_CHECK_STATUS["${disabled_healthcheck}"]=0
if [ "${healthcheck}" == "${disabled_healthcheck}" ]; then
return 0
fi
done
return 1
}

init_disabled_map

script_runner () {
local scripts_dir=$1; shift
local mode=$1; shift
local start_msg=$1; shift
local required_hc_failed=false
echo "$start_msg"
for script in $(find "$scripts_dir" -name '*.sh' | sort); do
if is_disabled "$(basename "$script")"; then
echo "'$(basename "$script")' was skipped, as specified in config"
if [[ "$required_hc_failed" == true && "$mode" == "strict" ]]; then
echo "<5>'$(basename "$script")' was skipped due to a previous failure"
elif is_disabled "$(basename "$script")"; then
echo "<5>'$(basename "$script")' was skipped, as specified in config"
else
local rc=0
systemd-cat -t "$(basename "$script")" bash "$script" || rc=$?
if [ $rc -ne 0 ]; then
local return_code=0
systemd-cat -t "$(basename "$script")" bash "$script" || return_code=$?
if [ $return_code -ne 0 ]; then
local failure_msg
failure_msg="Script '$(basename "$script")' FAILURE (exit code '$rc')"
failure_msg="Script '$(basename "$script")' FAILURE (exit code '$return_code')"
case "$mode" in
"relaxed")
echo "<2>$failure_msg. Continuing..." >&2
Expand All @@ -80,13 +62,15 @@ script_runner () {

case "$1" in
"check")
rc=0
return_code=0
for health_check_path in "${SCRIPTS_CHECK_PATHS[@]}"; do
script_runner "$health_check_path/required.d" "strict" "Running Required Health Check Scripts..." || rc=1
script_runner "$health_check_path/required.d" "strict" "Running Required Health Check Scripts..." || {
return_code=1
break
}
script_runner "$health_check_path/wanted.d" "relaxed" "Running Wanted Health Check Scripts..."
done
print_unexecuted_checks
exit $rc
exit $return_code
;;
"green")
echo "<5>Boot Status is GREEN - Health Check SUCCESS"
Expand Down

0 comments on commit b943b39

Please sign in to comment.