Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix Hadoop JAR Download Timeouts in Behave Tests #1503

Merged
merged 2 commits into from
Jan 15, 2025
Merged

Fix Hadoop JAR Download Timeouts in Behave Tests #1503

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c2d0825
Fix hadoop download url
parthosa Jan 14, 2025
5a1cb64
Using archive links
parthosa Jan 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions user_tools/tests/spark_rapids_tools_e2e/resources/scripts/hdfs/setup_hdfs.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

# Copyright (c) 2024, NVIDIA CORPORATION.
# Copyright (c) 2024-2025, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -67,6 +67,7 @@ readonly CURRENT_FILE_PATH=$(realpath "${0}")
readonly HDFS_SCRIPTS_DIR=$(dirname "${CURRENT_FILE_PATH}")
readonly VERIFY_HDFS_SERVICES_MAX_RETRY=3
readonly VERIFY_HDFS_SERVICES_SLEEP_SEC=5
readonly WGET_TIMEOUT_SEC=1800 # 30 minutes

load_common_scripts() {
local scripts_dir=$(dirname "${HDFS_SCRIPTS_DIR}")
Expand Down Expand Up @@ -107,19 +108,19 @@ verify_checksum() {
# Function to download and extract Hadoop
download_and_extract_hadoop() {
echo "Downloading and extracting Hadoop..."
local hadoop_url="https://dlcdn.apache.org/hadoop/common/hadoop-${E2E_TEST_HADOOP_VERSION}/hadoop-${E2E_TEST_HADOOP_VERSION}.tar.gz"
local hadoop_url="https://archive.apache.org/dist/hadoop/common/hadoop-${E2E_TEST_HADOOP_VERSION}/hadoop-${E2E_TEST_HADOOP_VERSION}.tar.gz"
local hadoop_tar_file="${E2E_TEST_TMP_DIR}/hadoop-${E2E_TEST_HADOOP_VERSION}.tar.gz"
local checksum_url="${hadoop_url}.sha512"
local checksum_file="${hadoop_tar_file}.sha512"

if [ ! -f "${hadoop_tar_file}" ]; then
wget -O "${hadoop_tar_file}" "${hadoop_url}" || err "Failed to download Hadoop tarball."
wget --timeout=${WGET_TIMEOUT_SEC} -O"${hadoop_tar_file}" "${hadoop_url}" || err "Failed to download Hadoop tarball."
fi

# Verify checksum and re-download if needed
wget -O "${checksum_file}" "${checksum_url}" || err "Failed to download checksum file."
wget --timeout=${WGET_TIMEOUT_SEC} -O"${checksum_file}" "${checksum_url}" || err "Failed to download checksum file."
if ! verify_checksum "${hadoop_tar_file}" "${checksum_file}"; then
wget -O "${hadoop_tar_file}" "${hadoop_url}" || err "Failed to download Hadoop tarball."
wget --timeout=${WGET_TIMEOUT_SEC} -O"${hadoop_tar_file}" "${hadoop_url}" || err "Failed to download Hadoop tarball."
if ! verify_checksum "${hadoop_tar_file}" "${checksum_file}"; then
err "Checksum verification failed after re-downloading. Exiting..."
fi
Expand Down
Loading