From c084b8a050678af49a7d9e3997d3f0520089708f Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 27 May 2024 16:41:41 +0800 Subject: [PATCH 1/7] Fix docker start datanode failed --- catalogs/catalog-hive/build.gradle.kts | 1 + dev/docker/kerberos-hive/start.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index bf01ee5fc0a..ee98d2ed5c8 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -165,6 +165,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.12") + environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.2") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit diff --git a/dev/docker/kerberos-hive/start.sh b/dev/docker/kerberos-hive/start.sh index e937a12309b..d011d6f62c7 100644 --- a/dev/docker/kerberos-hive/start.sh +++ b/dev/docker/kerberos-hive/start.sh @@ -7,8 +7,9 @@ # start ssh HOSTNAME=`hostname` service ssh start -ssh-keyscan localhost > /root/.ssh/known_hosts +ssh-keyscan localhost >> /root/.ssh/known_hosts ssh-keyscan 0.0.0.0 >> /root/.ssh/known_hosts +ssh-keyscan 127.0.0.1 >> /root/.ssh/known_hosts # init the Kerberos database echo -e "${PASS}\n${PASS}" | kdb5_util create -s From 73d17268c7a5cf22898628e06566fbfe1c9fc36b Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 27 May 2024 17:29:11 +0800 Subject: [PATCH 2/7] Change kerberos version to 0.1.2 --- catalogs/catalog-hadoop/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 8afcf6a23e3..ccdc0519683 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -99,7 +99,7 @@ tasks.test { doFirst { environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-hive:0.1.12") - environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.0") + environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE", "datastrato/gravitino-ci-kerberos-hive:0.1.2") } val init = project.extra.get("initIntegrationTest") as (Test) -> Unit From cd25ea259c6036ba87885d28a331e96fbb24740e Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 27 May 2024 18:15:53 +0800 Subject: [PATCH 3/7] Fix --- dev/docker/kerberos-hive/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker/kerberos-hive/start.sh b/dev/docker/kerberos-hive/start.sh index d011d6f62c7..6e06603b130 100644 --- a/dev/docker/kerberos-hive/start.sh +++ b/dev/docker/kerberos-hive/start.sh @@ -7,7 +7,7 @@ # start ssh HOSTNAME=`hostname` service ssh start -ssh-keyscan localhost >> /root/.ssh/known_hosts +ssh-keyscan ${HOSTNAME} >> /root/.ssh/known_hosts ssh-keyscan 0.0.0.0 >> /root/.ssh/known_hosts ssh-keyscan 127.0.0.1 >> /root/.ssh/known_hosts From 833881e6249b898101faa0cd2c73c192d3cf2d2e Mon Sep 17 00:00:00 2001 From: yuqi Date: Mon, 27 May 2024 19:24:45 +0800 Subject: [PATCH 4/7] Fix --- dev/docker/kerberos-hive/start.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dev/docker/kerberos-hive/start.sh b/dev/docker/kerberos-hive/start.sh index 6e06603b130..c8dfb572dd5 100644 --- a/dev/docker/kerberos-hive/start.sh +++ b/dev/docker/kerberos-hive/start.sh @@ -8,6 +8,7 @@ HOSTNAME=`hostname` service ssh start ssh-keyscan ${HOSTNAME} >> /root/.ssh/known_hosts +ssh-keyscan localhost >> /root/.ssh/known_hosts ssh-keyscan 0.0.0.0 >> /root/.ssh/known_hosts ssh-keyscan 127.0.0.1 >> /root/.ssh/known_hosts @@ -66,6 +67,13 @@ ${HADOOP_HOME}/sbin/hadoop-daemon.sh start namenode echo "Starting DataNode..." ${HADOOP_HOME}/sbin/start-secure-dns.sh +sleep 1 + +ps -ef | grep DataNode | grep -v "color=auto" +if [[ $? -ne 0 ]]; then + echo "DataNode failed to start, please check the logs" + exit 1 +fi # start mysql and create databases/users for hive chown -R mysql:mysql /var/lib/mysql From 3c62d1de6b4f3055873cf9477b90e869622f7277 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 28 May 2024 19:36:16 +0800 Subject: [PATCH 5/7] Fix --- dev/docker/build-docker.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/docker/build-docker.sh b/dev/docker/build-docker.sh index 067dbd214fa..6f0d621019b 100755 --- a/dev/docker/build-docker.sh +++ b/dev/docker/build-docker.sh @@ -105,14 +105,14 @@ fi cd ${script_dir}/${component_type} if [[ "${platform_type}" == "all" ]]; then if [ ${build_latest} -eq 1 ]; then - docker buildx build --no-cache --pull --platform=linux/amd64,linux/arm64 ${build_args} --push --progress plain -f Dockerfile -t ${image_name}:latest -t ${image_name}:${tag_name} . + docker buildx build --platform=linux/amd64,linux/arm64 ${build_args} --push --progress plain -f Dockerfile -t ${image_name}:latest -t ${image_name}:${tag_name} . else - docker buildx build --no-cache --pull --platform=linux/amd64,linux/arm64 ${build_args} --push --progress plain -f Dockerfile -t ${image_name}:${tag_name} . + docker buildx build --platform=linux/amd64,linux/arm64 ${build_args} --push --progress plain -f Dockerfile -t ${image_name}:${tag_name} . fi else if [ ${build_latest} -eq 1 ]; then - docker buildx build --no-cache --pull --platform=${platform_type} ${build_args} --output type=docker --progress plain -f Dockerfile -t ${image_name}:latest -t ${image_name}:${tag_name} . + docker buildx build --platform=${platform_type} ${build_args} --output type=docker --progress plain -f Dockerfile -t ${image_name}:latest -t ${image_name}:${tag_name} . else - docker buildx build --no-cache --pull --platform=${platform_type} ${build_args} --output type=docker --progress plain -f Dockerfile -t ${image_name}:${tag_name} . + docker buildx build --platform=${platform_type} ${build_args} --output type=docker --progress plain -f Dockerfile -t ${image_name}:${tag_name} . fi fi From bc19cc68adff523879560825d7357a35520f0588 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 28 May 2024 19:41:27 +0800 Subject: [PATCH 6/7] Fix --- dev/docker/build-docker.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/docker/build-docker.sh b/dev/docker/build-docker.sh index 6f0d621019b..067dbd214fa 100755 --- a/dev/docker/build-docker.sh +++ b/dev/docker/build-docker.sh @@ -105,14 +105,14 @@ fi cd ${script_dir}/${component_type} if [[ "${platform_type}" == "all" ]]; then if [ ${build_latest} -eq 1 ]; then - docker buildx build --platform=linux/amd64,linux/arm64 ${build_args} --push --progress plain -f Dockerfile -t ${image_name}:latest -t ${image_name}:${tag_name} . + docker buildx build --no-cache --pull --platform=linux/amd64,linux/arm64 ${build_args} --push --progress plain -f Dockerfile -t ${image_name}:latest -t ${image_name}:${tag_name} . else - docker buildx build --platform=linux/amd64,linux/arm64 ${build_args} --push --progress plain -f Dockerfile -t ${image_name}:${tag_name} . + docker buildx build --no-cache --pull --platform=linux/amd64,linux/arm64 ${build_args} --push --progress plain -f Dockerfile -t ${image_name}:${tag_name} . fi else if [ ${build_latest} -eq 1 ]; then - docker buildx build --platform=${platform_type} ${build_args} --output type=docker --progress plain -f Dockerfile -t ${image_name}:latest -t ${image_name}:${tag_name} . + docker buildx build --no-cache --pull --platform=${platform_type} ${build_args} --output type=docker --progress plain -f Dockerfile -t ${image_name}:latest -t ${image_name}:${tag_name} . else - docker buildx build --platform=${platform_type} ${build_args} --output type=docker --progress plain -f Dockerfile -t ${image_name}:${tag_name} . + docker buildx build --no-cache --pull --platform=${platform_type} ${build_args} --output type=docker --progress plain -f Dockerfile -t ${image_name}:${tag_name} . fi fi From 4e4c57e6f15ab9a84de66ececc36b62db44e7af7 Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 28 May 2024 19:41:39 +0800 Subject: [PATCH 7/7] Fix --- dev/docker/kerberos-hive/start.sh | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/dev/docker/kerberos-hive/start.sh b/dev/docker/kerberos-hive/start.sh index c8dfb572dd5..869d8603913 100644 --- a/dev/docker/kerberos-hive/start.sh +++ b/dev/docker/kerberos-hive/start.sh @@ -67,14 +67,34 @@ ${HADOOP_HOME}/sbin/hadoop-daemon.sh start namenode echo "Starting DataNode..." ${HADOOP_HOME}/sbin/start-secure-dns.sh -sleep 1 +sleep 5 +# Check if the DataNode is running ps -ef | grep DataNode | grep -v "color=auto" if [[ $? -ne 0 ]]; then echo "DataNode failed to start, please check the logs" exit 1 fi +retry_times=0 +ready=0 +while [[ ${retry_times} -lt 10 ]]; do + hdfs_ready=$(hdfs dfsadmin -report | grep "Live datanodes" | awk '{print $3}') + if [[ ${hdfs_ready} == "(1):" ]]; then + echo "HDFS is ready, retry_times = ${retry_times}" + let "ready=0" + break + fi + retry_times=$((retry_times+1)) +done + +if [[ ${ready} -ne 0 ]]; then + echo "HDFS is not ready" + cat ${HADOOP_HOME}/bin/logs/hadoop-root-datanode-*.log + exit 1 +fi + + # start mysql and create databases/users for hive chown -R mysql:mysql /var/lib/mysql usermod -d /var/lib/mysql/ mysql