Skip to content

Commit

Permalink
Merge pull request red-hat-storage#3787 from sunilkumarn417/master
Browse files Browse the repository at this point in the history
fix syntax in server utils
  • Loading branch information
mergify[bot] authored Jun 17, 2024
2 parents a77b1b2 + f771490 commit d403f16
Show file tree
Hide file tree
Showing 3 changed files with 111 additions and 93 deletions.
2 changes: 2 additions & 0 deletions pipeline/scripts/ci/reimage-octo-node.sh
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ for node in ${NODES}; do
initial_setup "${node}"
set_hostnames_repos "${node}"
wipe_drives "${node}"
cleanup_node "${node}"
reboot_node "${node}"
done

Expand All @@ -100,6 +101,7 @@ while [ ${#failed_nodes[@]} -gt 0 ] && [ ${retry_count} -lt 3 ]; do
new_failed_nodes=()
for node in "${failed_nodes[@]}"; do
wipe_drives "${node}"
cleanup_node "${node}"
reboot_node "${node}"
done

Expand Down
198 changes: 106 additions & 92 deletions pipeline/scripts/ci/server_setup_utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,20 @@
# Arguments:
# node: The node to perform setup on
# password: The password to set for the root user (default: "passwd")

function cleanup_node {
# This function is to cleanup any ceph residuals.
# node: FQDN or IP address of the node
# passwd: Password of the root user

local node="$1"
local username="${2:-root}"
local password="${3:-passwd}"
echo "Clean-up ceph residuals on ${node}"
output=$(sshpass -p ${password} ssh ${username}@${node} 'sudo rm -rf /etc/ceph /var/lib/ceph /var/log/ceph')
echo "Status : ${output}"
}

function initial_setup {
# This function performs initial setup on a given node. It takes two arguments:
# node: The name or IP address of the node to perform setup on
Expand Down Expand Up @@ -48,123 +62,123 @@ function initial_setup_dsal {
# node: The node to wipe disks on
# password: The password to use to access the node (default: "passwd")
function wipe_drives {
# This function wipes all data disks clean on a given node. It takes two arguments:
# node: The name or IP address of the node to wipe disks on
# password: The password to use to access the node (default: "passwd")
# The function first retrieves the list of data disks on the node using lsblk.
# It then identifies the root disk by looking for the disk that has the / mount point.
# The function then iterates through all disks except the root disk and uses the wipefs command to wipe them clean.
# If the root disk cannot be found or more than one root disk is found, an error message is printed
# and the function exits.
local node="$1"
local username="${2:-root}"
local password="${3:-passwd}"
echo "Wipe all data disks clean."

# Find all disks and its type
disks=$(sshpass -p ${password} ssh -q ${username}@${node} "lsblk -ndo name,type | awk '{print \$1\",\"\$2}'")

# Find root disk device
root_disk=$(sshpass -p ${password} ssh -q ${username}@${node} "sudo findmnt -v -n -T /boot -o SOURCE"
if [ -z "${root_disk}" ]; then
echo "ERR: Unable to find root disk on ${node}"
exit 2
fi
# This function wipes all data disks clean on a given node. It takes two arguments:
# node: The name or IP address of the node to wipe disks on
# password: The password to use to access the node (default: "passwd")
# The function first retrieves the list of data disks on the node using lsblk.
# It then identifies the root disk by looking for the disk that has the / mount point.
# The function then iterates through all disks except the root disk and uses the wipefs command to wipe them clean.
# If the root disk cannot be found or more than one root disk is found, an error message is printed
# and the function exits.
local node="$1"
local username="${2:-root}"
local password="${3:-passwd}"
echo "Wipe all data disks clean."

# Find all disks and its type
disks=$(sshpass -p ${password} ssh -q ${username}@${node} "lsblk -ndo name,type | awk '{print \$1\",\"\$2}'")

# Find root disk device
root_disk=$(sshpass -p ${password} ssh -q ${username}@${node} "sudo findmnt -v -n -T /boot -o SOURCE")
if [ -z "${root_disk}" ]; then
echo "ERR: Unable to find root disk on ${node}"
exit 2
fi

if [ "$(echo ${root_disk} | wc -l)" != "1" ]; then
echo "ERR: More than one root disk found on ${node}"
exit 2
fi
if [ "$(echo ${root_disk} | wc -l)" != "1" ]; then
echo "ERR: More than one root disk found on ${node}"
exit 2
fi

mapfile -t arr < <(printf "%s\n" "$disks")
mapfile -t arr < <(printf "%s\n" "$disks")

for disk in "${arr[@]}" ; do
IFS=',' read -r name type <<< "$disk"
echo "name: $name, type: $type"
for disk in "${arr[@]}" ; do
IFS=',' read -r name type <<< "$disk"
echo "name: $name, type: $type"

# shellcheck disable=SC2076
# Skip wipefs for root, cd-rom.,etc disks
if [[ "${root_disk}" =~ "/dev/${name}" || "${type}" != "disk" ]]; then
echo "Skipping this disk : disk - ${name} , type - ${type}"
continue
else
sshpass -p ${password} ssh ${username}@${node} "wipefs -a --force /dev/${name}"
fi
done
# shellcheck disable=SC2076
# Skip wipefs for root, cd-rom.,etc disks
if [[ "${root_disk}" =~ "/dev/${name}" || "${type}" != "disk" ]]; then
echo "Skipping this disk : disk - ${name} , type - ${type}"
continue
else
sshpass -p ${password} ssh ${username}@${node} "wipefs -a --force /dev/${name}"
fi
done
}

# This function sets the hostname of a node to its shortname and cleans the default repo files to avoid conflicts.
# Arguments:
# node: The node to perform setup on
# password: The password to set for the root user (default: "passwd")
function set_hostnames_repos {
local node="$1"
local username="${2:-root}"
local password="${3:-passwd}"
echo 'Setting the systems to use shortnames'
sshpass -p ${password} ssh ${username}@${node} 'sudo hostnamectl set-hostname $(hostname -s)'
sshpass -p ${password} ssh ${username}@${node} 'sudo sed -i "s/$(hostname)/$(hostname -s)/g" /etc/hosts'
echo 'Cleaning default repo files to avoid conflicts'
sshpass -p ${password} ssh ${username}@${node} 'sudo rm -f /etc/yum.repos.d/*; sudo yum clean all'
local node="$1"
local username="${2:-root}"
local password="${3:-passwd}"
echo 'Setting the systems to use shortnames'
sshpass -p ${password} ssh ${username}@${node} 'sudo hostnamectl set-hostname $(hostname -s)'
sshpass -p ${password} ssh ${username}@${node} 'sudo sed -i "s/$(hostname)/$(hostname -s)/g" /etc/hosts'

echo 'Cleaning default repo files to avoid conflicts'
sshpass -p ${password} ssh ${username}@${node} 'sudo rm -f /etc/yum.repos.d/*; sudo yum clean all'
}

function remove_repos {
local node="$1"
local username="${2:-root}"
local password="${3:-passwd}"
local username="${2:-root}"
local password="${3:-passwd}"

echo 'Cleaning default repo files to avoid conflicts'
sshpass -p ${password} ssh ${username}@${node} 'sudo rm -f /etc/yum.repos.d/*; sudo yum clean all'
echo 'Cleaning default repo files to avoid conflicts'
sshpass -p ${password} ssh ${username}@${node} 'sudo rm -f /etc/yum.repos.d/*; sudo yum clean all'

}
}

function reboot_node {
local node="$1"
local username="${2:-root}"
local password="${3:-passwd}"
local username="${2:-root}"
local password="${3:-passwd}"

echo 'Cleaning default repo files to avoid conflicts'
sshpass -p ${password} ssh ${username}@${node} 'sudo reboot'
echo 'Cleaning default repo files to avoid conflicts'
sshpass -p ${password} ssh ${username}@${node} 'sudo reboot'

}
}

function wait_for_node {
local node="$1"
local username="${2:-root}"
local password="${3:-passwd}"
local max_attempts=10
local wait_time=60
echo "Waiting for node ${node} to come back online..."
for ((attempt=1; attempt<=max_attempts; attempt++)); do
if ping -c 1 ${node} &> /dev/null; then
echo "Node ${node} is online"
return 0
else
echo "Node ${node} is still offline. Attempt ${attempt}/${max_attempts}"
sleep ${wait_time}
fi
done
echo "Node ${node} did not come back online after ${max_attempts} attempts."
return 1
}
local username="${2:-root}"
local password="${3:-passwd}"
local max_attempts=10
local wait_time=60

echo "Waiting for node ${node} to come back online..."

for ((attempt=1; attempt<=max_attempts; attempt++)); do
if ping -c 1 ${node} &> /dev/null; then
echo "Node ${node} is online"
return 0
else
echo "Node ${node} is still offline. Attempt ${attempt}/${max_attempts}"
sleep ${wait_time}
fi
done

echo "Node ${node} did not come back online after ${max_attempts} attempts."
return 1
}

function verify_disk() {
local node="$1"
local username="${2:-root}"
local password="${3:-passwd}"
echo "Verifying wipefs on ${node}"
output=$(sshpass -p ${password} ssh ${username}@${node} 'sudo lsblk -o name,fstype')
echo "ouput of lsblk ${output}"
# Check if any block device has a non-empty fstype
while read -r line; do
if [[ ! -z $(echo "${line}" | awk '{print $2}') ]]; then
echo true
return
fi
done <<< "$output"
echo false
local node="$1"
local username="${2:-root}"
local password="${3:-passwd}"
echo "Verifying wipefs on ${node}"
output=$(sshpass -p ${password} ssh ${username}@${node} 'sudo lsblk -o name,fstype')
echo "ouput of lsblk ${output}"
# Check if any block device has a non-empty fstype
while read -r line; do
if [[ ! -z $(echo "${line}" | awk '{print $2}') ]]; then
echo true
return
fi
done <<< "$output"
echo false
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ tests:
orphan-initial-daemons: true
registry-url: registry.redhat.io
allow-fqdn-hostname: true
allow-overwrite: true
base_cmd_args:
verbose: true
- config:
Expand Down Expand Up @@ -78,8 +79,9 @@ tests:
spec:
data_devices:
model: "PERC H755 Front"
dedicated_devices:
db_devices:
# model: "Dell Ent NVMe CM6 MU 1.6TB"
paths:
- /dev/nvme0n1
encrypted: "true" # boolean as string

Expand Down

0 comments on commit d403f16

Please sign in to comment.