ADBDEV-6920: Changes related to renaming

arenadata/Dockerfile

@@ -59,7 +59,7 @@ ENV JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8
 SHELL ["bash", "-c"]
 RUN source gpdb_src/concourse/scripts/common.bash && \
     install_gpdb && \
-    source '/usr/local/greenplum-db-devel/greenplum_path.sh' && \
+    if [ -d /usr/local/greengage-db-devel ]; then source '/usr/local/greengage-db-devel/greengage_path.sh'; else source '/usr/local/greenplum-db-devel/greenplum_path.sh'; fi && \
     mkdir ${OUTPUT_ARTIFACT_DIR} && \
     export SKIP_FDW_BUILD_REASON=0 && \
     pxf_src/concourse/scripts/compile_pxf.bash

arenadata/test_in_docker.sh

@@ -1,26 +1,37 @@
 #!/usr/bin/env bash
 # This script depends on hub.adsw.io/library/gpdb6_pxf_regress
-set -exo pipefail
+set -exo
+
+export GPHOME=/usr/local/greenplum-db-devel
+export GP_PATH_FILE=greenplum_path.sh
+
+if [[ -f bin_gpdb/bin_gpdb.tar.gz ]]; then
+  count=$(tar -tvf bin_gpdb/bin_gpdb.tar.gz | grep greengage_path.sh | wc -l)
+  if [ $count -ge 1 ]; then
+    export GPHOME=/usr/local/greengage-db-devel
+    export GP_PATH_FILE=greengage_path.sh
+  fi
+fi
 
 # manually prepare gpadmin user; test_pxf.bash doesn't tweak gpadmin folder permissions and ssh keys
 ./gpdb_src/concourse/scripts/setup_gpadmin_user.bash
 # unpack gpdb and pxf; run gpdb cluster and pxf server
-/tmp/build/pxf_src/concourse/scripts/test_pxf.bash
+GPHOME=$GPHOME GP_PATH_FILE=$GP_PATH_FILE /tmp/build/pxf_src/concourse/scripts/test_pxf.bash
 # tweak necessary folders to run regression tests later
-chown gpadmin:gpadmin -R /usr/local/greenplum-db-devel
+chown gpadmin:gpadmin -R ${GPHOME}
 chown gpadmin:gpadmin -R /tmp/build/pxf_src
 
 # Display the diff if we fail
 trap "cat /tmp/build/pxf_src/fdw/regression.diffs /tmp/build/pxf_src/external-table/regression.diffs" ERR
 
 # test fdw and external-table
-su - gpadmin -c "
-    source '/usr/local/greenplum-db-devel/greenplum_path.sh';
-    source '/home/gpadmin/gpdb_src/gpAux/gpdemo/gpdemo-env.sh';
+test_command='source '${GPHOME}'/'${GP_PATH_FILE}';
+    source /home/gpadmin/gpdb_src/gpAux/gpdemo/gpdemo-env.sh;
     cd /tmp/build/pxf_src/fdw &&
     make install &&
     make installcheck &&
     cd ../external-table/ &&
     make install &&
-    make installcheck;
-"
+    make installcheck;'
+
+su - gpadmin -c "$test_command"

automation/arenadata/Dockerfile

@@ -1,4 +1,4 @@
-ARG GPDB_IMAGE=cloud-hub.adsw.io/library/gpdb6_u22:adb-6.x-dev
+ARG GPDB_IMAGE=hub.adsw.io/library/gg6_u22:it
 FROM $GPDB_IMAGE
 
 SHELL ["bash", "-c"]
@@ -62,14 +62,16 @@ RUN mkdir -p /data1/master /data1/primary /data1/mirror && chmod -R 755 /data1 &
     chown -R gpadmin:gpadmin /home/gpadmin && \
     install_gpdb
 
+# Create symlinks if needed
+RUN if [ -d /usr/local/greengage-db-devel ]; then ln -s /usr/local/greengage-db-devel /usr/local/greenplum-db-devel; fi;
+RUN if [ -e /usr/local/greengage-db-devel/greengage_path.sh ]; then ln -s /usr/local/greengage-db-devel/greengage_path.sh /usr/local/greenplum-db-devel/greenplum_path.sh; fi;
+
 # Install PXF
 ENV JAVA_TOOL_OPTIONS=-Dfile.encoding=UTF8
 COPY . /home/gpadmin/pxf_src
 COPY ./automation/arenadata/scripts/compile_pxf_without_test.sh ./pxf_src/concourse/scripts/compile_pxf_without_test.sh
 RUN chmod +x ./pxf_src/concourse/scripts/compile_pxf_without_test.sh
 RUN source gpdb_src/concourse/scripts/common.bash && \
-    mkdir pxf_tarball && \
-    install_gpdb && \
     source '/usr/local/greenplum-db-devel/greenplum_path.sh' && \
     export SKIP_FDW_BUILD_REASON=0 && \
     pxf_src/concourse/scripts/compile_pxf_without_test.sh && \
@@ -106,5 +108,5 @@ RUN set -eux; \
 RUN cp /tmp/libs/* ${PXF_HOME}/lib/
 
 RUN chmod a+x ./pxf_src/automation/arenadata/scripts/start_adb_cluster.sh
-RUN chown -R gpadmin:gpadmin /usr/local/greenplum-db-devel
+RUN if [ -d /usr/local/greengage-db-devel ]; then chown -R gpadmin:gpadmin /usr/local/greengage-db-devel; else chown -R gpadmin:gpadmin /usr/local/greenplum-db-devel; fi;
 ENTRYPOINT ["/home/gpadmin/pxf_src/automation/arenadata/scripts/start_adb_cluster.sh"]

automation/arenadata/build-images.sh

@@ -24,5 +24,5 @@ echo "=============================="
 # default image is ubuntu
 # To build centos image - run the command: GPDB_IMAGE=hub.adsw.io/library/gpdb6_regress:adb-6.x-dev bash build-images.sh
 pushd ../..
-docker build -t gpdb6_pxf_automation:it --build-arg "GPDB_IMAGE=${GPDB_IMAGE:-cloud-hub.adsw.io/library/gpdb6_u22:adb-6.x-dev}" -f automation/arenadata/Dockerfile .
+docker build -t gpdb6_pxf_automation:it --build-arg "GPDB_IMAGE=${GPDB_IMAGE:-hub.adsw.io/library/gg6_u22:it}" -f automation/arenadata/Dockerfile .
 popd

automation/arenadata/docker-compose.yaml

@@ -157,7 +157,7 @@ services:
       retries: 3
 
   oracle:
-    image: "absolutapps/oracle-12c-ee"
+    image: "haoyu233/oracle-12c-ee"
     restart: unless-stopped
     container_name: oracle
     hostname: oracle

automation/arenadata/scripts/compile_pxf_without_test.sh

@@ -2,7 +2,11 @@
 
 set -eoux pipefail
 
-GPHOME=/usr/local/greenplum-db-devel
+if [ -d /usr/local/greengage-db-devel ]; then
+  GPHOME=/usr/local/greengage-db-devel
+else
+  GPHOME=/usr/local/greenplum-db-devel
+fi
 
 # use a login shell for setting environment
 bash --login -c "

automation/arenadata/scripts/start_adb_cluster.sh

@@ -169,22 +169,49 @@ if [ "$HOSTNAME" == "$DOCKER_GP_MASTER_SERVER" ]; then
     echo "----------------------------------"
     echo "Run Greenplum cluster installation"
     echo "----------------------------------"
-    sudo -H -u gpadmin bash -c "source /home/gpadmin/.bash_profile &&
-        /usr/local/greenplum-db-devel/bin/gpinitsystem -a -I /home/gpadmin/gpdb_src/gpAux/gpdemo/create_cluster.conf -l /home/gpadmin/gpAdminLogs"
-    sudo -H -u gpadmin bash -c "source /home/gpadmin/.bash_profile &&
-        psql -d postgres -Atc 'CREATE EXTENSION IF NOT EXISTS pxf;' &&
-        psql -d postgres -Atc 'CREATE EXTENSION IF NOT EXISTS pxf_fdw;' &&
-        echo 'local all testuser trust' >> /data1/master/gpseg-1/pg_hba.conf &&
-        echo 'host all gpadmin 0.0.0.0/0 trust' >> /data1/master/gpseg-1/pg_hba.conf &&
-        echo 'host all all 0.0.0.0/0 md5' >> /data1/master/gpseg-1/pg_hba.conf &&
-        gpconfig -c gp_resource_manager -v group &&
-        gpstop -aM fast && gpstart -a"
-
-    # Check cluster
+
+    echo "---------------------------------"
+    echo "Check ssh connection to the hosts"
+    echo "---------------------------------"
+    max_iterations=20
+    wait_seconds=5
+    iterations=0
+    while true
+    do
+      ((iterations++))
+      echo "Check SSH connection. Attempt $iterations"
+      status=0
+      for server in $DOCKER_GP_CLUSTER_HOSTS
+      do
+          echo "Check SSH connection to the $server"
+          sudo -H -u gpadmin bash -c -l "ssh -o PasswordAuthentication=no $server 'exit'"
+          if ! [ $? -eq 0 ]; then
+            echo "Server $server is not available for ssh connection. We will try again..."
+            status=1
+            break
+          fi
+      done
+      if [ $status -eq 0 ]; then
+        echo "All Greenplum servers are available for SSH connection"
+        break
+      elif [ "$iterations" -ge "$max_iterations" ]; then
+        echo "Error to connect to some Greenplum server via SSH after $max_iterations tries. Exit from script!"
+        exit 1
+      else
+        echo "Wait $wait_seconds seconds and try again to connect to the servers"
+        sleep $wait_seconds
+      fi
+    done
+
+    echo "-------------------------"
+    echo "Install Greenplum cluster"
+    echo "-------------------------"
+    sudo -H -u gpadmin bash -c -l "/usr/local/greenplum-db-devel/bin/gpinitsystem -a -I /home/gpadmin/gpdb_src/gpAux/gpdemo/create_cluster.conf -l /home/gpadmin/gpAdminLogs"
+
     echo "-------------------------------------"
     echo "Check connection to Greenplum cluster"
     echo "-------------------------------------"
-    result="$( sudo -H -u gpadmin bash -c "source /home/gpadmin/.bash_profile && /usr/local/greenplum-db-devel/bin/psql -d postgres -Atc 'SELECT 1;'" )"
+    result="$( sudo -H -u gpadmin bash -c -l "psql -d postgres -Atc 'SELECT 1;'" )"
     if [ "${result}" == "1" ]; then
       echo "--------------------------------------------"
       echo "Fantastic!!! Greenplum cluster is available!"
@@ -194,20 +221,35 @@ if [ "$HOSTNAME" == "$DOCKER_GP_MASTER_SERVER" ]; then
             echo "------------------------------"
             echo "Activate standby master server"
             echo "------------------------------"
-            sudo -H -u gpadmin bash -c "source /home/gpadmin/.bash_profile && /usr/local/greenplum-db-devel/bin/gpinitstandby -a -s $DOCKER_GP_STANDBY_SERVER"
+            sudo -H -u gpadmin bash -c -l "/usr/local/greenplum-db-devel/bin/gpinitstandby -a -s $DOCKER_GP_STANDBY_SERVER"
       fi
     else
       echo "-------------------------------------"
       echo "Error to connect to Greenplum cluster"
       echo "-------------------------------------"
       exit 1;
     fi
+
+   echo "-------------------"
+   echo "Configure Greenplum"
+   echo "-------------------"
+   sudo -H -u gpadmin bash -c -l "psql -d postgres -Atc 'CREATE EXTENSION IF NOT EXISTS pxf;' &&
+       psql -d postgres -Atc 'CREATE EXTENSION IF NOT EXISTS pxf_fdw;' &&
+       echo 'local all testuser trust' >> /data1/master/gpseg-1/pg_hba.conf &&
+       echo 'host all gpadmin 0.0.0.0/0 trust' >> /data1/master/gpseg-1/pg_hba.conf &&
+       echo 'host all all 0.0.0.0/0 md5' >> /data1/master/gpseg-1/pg_hba.conf &&
+       gpconfig -c gp_resource_manager -v group"
+
+   echo "*******************"
+   echo "Restart ADB cluster"
+   echo "*******************"
+   sudo -H -u gpadmin bash -c -l "gpstop -aM fast && gpstart -a"
   else
     echo "-------------------------"
     echo "Starting Greenplum server"
     echo "-------------------------"
-    sudo -H -u gpadmin bash -c "source /home/gpadmin/.bash_profile && gpstart -a"
-    result="$( sudo -H -u gpadmin bash -c "source /home/gpadmin/.bash_profile && /usr/local/greenplum-db-devel/bin/psql -d postgres -Atc 'SELECT 1;'" )"
+    sudo -H -u gpadmin bash -c -l "gpstart -a"
+    result="$( sudo -H -u gpadmin bash -c -l "psql -d postgres -Atc 'SELECT 1;'" )"
     if [ "${result}" == "1" ]; then
       echo "--------------------------------------------"
       echo "Fantastic!!! Greenplum cluster is available!"
@@ -233,9 +275,26 @@ do
   echo "---------"
   echo "Start PXF"
   echo "---------"
+  # Init Vault environment
+  if [[ "$PXF_VAULT_ENABLED" = true ]]; then
+      echo "----------------------------------------"
+      echo "Init Vault env variables for PXF service"
+      echo "----------------------------------------"
+      ksh -c env | grep -E 'PXF_VAULT' | sed 's/^/export /' >> /home/gpadmin/.bash_profile
+      ksh -c env | grep -E 'PXF_VAULT' | sed 's/^/export /' >> /home/gpadmin/.bashrc
+  fi
+  # Init SSL environment
+  if [[ "$PXF_PROTOCOL" = "https" ]]; then
+    echo "--------------------------------------"
+    echo "Init SSL env variables for PXF service"
+    echo "--------------------------------------"
+    ksh -c env | grep -E 'PXF_SSL|PXF_HOST|PXF_PROTOCOL' | sed 's/^/export /' >> /home/gpadmin/.bash_profile
+    ksh -c env | grep -E 'PXF_SSL|PXF_HOST|PXF_PROTOCOL' | sed 's/^/export /' >> /home/gpadmin/.bashrc
+  fi
+
   if [ "$HOSTNAME" == "$DOCKER_GP_MASTER_SERVER" ]; then
     sudo -H -u gpadmin bash -c -l "pxf start && tail -f /data1/master/gpseg-1/pg_log/gpdb-*.csv"
   else
     sudo -H -u gpadmin bash -c -l "pxf start && tail -f /usr/local/greenplum-db-devel/pxf/logs/pxf-service.log"
   fi
-done
+done

automation/sqlrepo/arenadata/jdbc-close-session/expected/query01.ans

@@ -1,6 +1,12 @@
--- start_ignore
--- end_ignore
 -- @description query01 for PXF test to check that the session is closed correctly
+
+-- start_matchsubs
+--
+-- m/.*File '\/usr\/local\/.*\/pxf\/servers\/named\/wrong_file_name.sql.*/
+-- s/.*File '\/usr\/local\/.*\/pxf\/servers\/named\/wrong_file_name.sql.*/ERROR:  PXF server error : Failed to read text of query wrong_file_name : File '\/usr\/local\/greenplum-db-devel\/pxf\/servers\/named\/wrong_file_name.sql' does not exist/
+--
+-- end_matchsubs
+
 SELECT * FROM named_query_read_ext_table ORDER BY id;
  id | name
 ----+-------

automation/sqlrepo/arenadata/jdbc-close-session/sql/query01.sql

@@ -1,4 +1,12 @@
 -- @description query01 for PXF test to check that the session is closed correctly
+
+-- start_matchsubs
+--
+-- m/.*File '\/usr\/local\/.*\/pxf\/servers\/named\/wrong_file_name.sql.*/
+-- s/.*File '\/usr\/local\/.*\/pxf\/servers\/named\/wrong_file_name.sql.*/ERROR:  PXF server error : Failed to read text of query wrong_file_name : File '\/usr\/local\/greenplum-db-devel\/pxf\/servers\/named\/wrong_file_name.sql' does not exist/
+--
+-- end_matchsubs
+
 SELECT * FROM named_query_read_ext_table ORDER BY id;
 
 SELECT * FROM named_query_wrong_read_ext_table ORDER BY id;

automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java

@@ -23,7 +23,7 @@
 public class Gpdb extends DbSystemObject {
 
 	private static final String DEFAULT_PORT = "5432";
-	private static final String GREENPLUM_DATABASE_PREFIX = "Greenplum Database ";
+	private static final String GREENPLUM_DATABASE_PREFIX = "Database ";
 	private static final String IF_NOT_EXISTS_OPTION = "IF NOT EXISTS";
 
 	private String sshUserName;

concourse/scripts/pxf_common.bash

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 GPHOME=${GPHOME:=/usr/local/greenplum-db-devel}
+GP_PATH_FILE=${GP_PATH_FILE:-greenplum_path.sh}
 PXF_HOME=${PXF_HOME:=${GPHOME}/pxf}
 CDD_VALUE=/data/gpdata/coordinator/gpseg-1
 PXF_COMMON_SRC_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
@@ -97,7 +98,7 @@ function run_pxf_automation() {
 	fi
 
 	su gpadmin -c "
-		source '${GPHOME}/greenplum_path.sh' &&
+		source '${GPHOME}/${GP_PATH_FILE}' &&
 		psql -p ${PGPORT} -d template1 -c 'CREATE EXTENSION IF NOT EXISTS ${extension_name}'
 	"
 	# prepare certification output directory
@@ -120,7 +121,7 @@ function run_pxf_automation() {
 		time make GROUP=${GROUP} test
 
 		# if the test is successful, create certification file
-		gpdb_build_from_sql=\$(source \$GPHOME/greenplum_path.sh && psql -c 'select version()' | grep Greenplum | cut -d ' ' -f 6,8)
+		gpdb_build_from_sql=\$(source \$GPHOME/\$GP_PATH_FILE && psql -c 'select version()' | grep Greenplum | cut -d ' ' -f 6,8)
 		gpdb_build_clean=\${gpdb_build_from_sql%)}
 		pxf_version=\$(< ${PXF_HOME}/version)
 		echo "GPDB-\${gpdb_build_clean/ commit:/-}-PXF-\${pxf_version}" > "${PWD}/certification/certification.txt"
@@ -146,7 +147,7 @@ function run_regression_test() {
 	cat > ~gpadmin/run_regression_test.sh <<-EOF
 		#!/bin/bash
 		source /opt/gcc_env.sh || true
-		source ${GPHOME}/greenplum_path.sh
+		source ${GPHOME}/${GP_PATH_FILE}
 		source gpdb_src/gpAux/gpdemo/gpdemo-env.sh
 		export PATH=\$PATH:${GPHD_ROOT}/bin
 
@@ -167,7 +168,7 @@ function run_regression_test() {
 function run_load_test() {
 	local extension_name="pxf"
 	su gpadmin -c "
-		source '${GPHOME}/greenplum_path.sh' &&
+		source '${GPHOME}/${GP_PATH_FILE}' &&
 		psql -p ${PGPORT} -d template1 -c 'CREATE EXTENSION IF NOT EXISTS ${extension_name}'
 	"
 
@@ -176,7 +177,7 @@ function run_load_test() {
 		set -exo pipefail
 
 		source ~gpadmin/.pxfrc
-		source ${GPHOME}/greenplum_path.sh
+		source ${GPHOME}/${GP_PATH_FILE}
 		export PGPORT=${PGPORT}
 
 		cd pxf_src/load
@@ -306,7 +307,7 @@ function remote_access_to_gpdb() {
 	cp cluster_env_files/public_key.openssh /home/gpadmin/.ssh/authorized_keys
 	awk '{print "localhost", $1, $2; print "0.0.0.0", $1, $2}' /etc/ssh/ssh_host_rsa_key.pub >> /home/gpadmin/.ssh/known_hosts
 	ssh "${SSH_OPTS[@]}" gpadmin@cdw "
-		source ${GPHOME}/greenplum_path.sh &&
+		source ${GPHOME}/${GP_PATH_FILE} &&
 		export MASTER_DATA_DIRECTORY=${CDD_VALUE} &&
 		echo 'host all all 10.0.0.0/16 trust' >> ${CDD_VALUE}/pg_hba.conf &&
 		psql -d template1 <<-EOF && gpstop -u
@@ -318,7 +319,7 @@ function remote_access_to_gpdb() {
 }
 
 function create_gpdb_cluster() {
-	su gpadmin -c "source ${GPHOME}/greenplum_path.sh && make -C gpdb_src/gpAux/gpdemo create-demo-cluster"
+	su gpadmin -c "source ${GPHOME}/${GP_PATH_FILE} && make -C gpdb_src/gpAux/gpdemo create-demo-cluster"
 }
 
 function add_remote_user_access_for_gpdb() {
@@ -333,7 +334,7 @@ function add_remote_user_access_for_gpdb() {
 		    export MASTER_DATA_DIRECTORY=~gpadmin/data/master/gpseg-1
 		fi
 		echo 'local    all     ${username}     trust' >> \${MASTER_DATA_DIRECTORY}/pg_hba.conf
-		source ${GPHOME}/greenplum_path.sh
+		source ${GPHOME}/${GP_PATH_FILE}
 		gpstop -u
 	"
 }
@@ -375,7 +376,7 @@ function install_pxf_client() {
 	[[ ${TEST_ENV} == dev ]] || return 0
 	# recompile pxf.so file for dev environments only
 	bash -c "
-		source '${GPHOME}/greenplum_path.sh'
+		source '${GPHOME}/${GP_PATH_FILE}'
 		source /opt/gcc_env.sh || true
 		USE_PGXS=1 make -C '${PXF_EXTENSIONS_DIR}' install
 	"