From b547dccee448069be6c91a624012ae0f34b6aa30 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 16 Mar 2024 23:52:33 -0700 Subject: [PATCH 01/10] Update maven.yml --- .github/workflows/maven.yml | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 8e21a946cc..be59602f2e 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -32,16 +32,30 @@ on: jobs: build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up JDK 8 - uses: actions/setup-java@v3 - with: - java-version: '8' - distribution: 'temurin' - cache: maven - - name: Build with Maven - run: mvn clean install --no-transfer-progress -B -V + - uses: actions/checkout@v3 + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'temurin' + cache: maven + - name: Build with Maven + run: mvn clean install --no-transfer-progress -B -V + docker-build: + needs: + - build + runs-on: ubuntu-latest + steps: + - name: Build base image + run: | + cp target/ranger-* dev-support/ranger-docker/dist/ + cp target/version dev-support/ranger-docker/dist/ + cd dev-support/ranger-docker + chmod +x download-archives.sh && ./download-archives.sh + docker-compose -f docker-compose.ranger-base.yml build + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 + export RANGER_DB_TYPE=postgres + docker-compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml build From 049427b85de55eab9730c85f3a1eae477a0cac59 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 17 Mar 2024 00:57:32 -0700 Subject: [PATCH 02/10] Update maven.yml --- .github/workflows/maven.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index be59602f2e..6e563ca4db 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -34,24 +34,36 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up JDK 8 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: '8' distribution: 'temurin' cache: maven - name: Build with Maven run: mvn clean install --no-transfer-progress -B -V + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: target + path: target/* docker-build: needs: - build runs-on: ubuntu-latest steps: + - uses: actions/checkout@v4 + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: target + - name: Copy artifacts for docker build + run: | + cp ranger-*.tar.gz dev-support/ranger-docker/dist + cp version dev-support/ranger-docker/dist - name: Build base image run: | - cp target/ranger-* dev-support/ranger-docker/dist/ - cp target/version dev-support/ranger-docker/dist/ cd dev-support/ranger-docker chmod +x download-archives.sh && ./download-archives.sh docker-compose -f docker-compose.ranger-base.yml build From 0d927dc0a00949c085f0b2a37e9fc346adb671c9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 17 Mar 2024 00:58:13 -0700 Subject: [PATCH 03/10] Update maven.yml --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 6e563ca4db..35d7760f6d 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -42,7 +42,7 @@ jobs: distribution: 'temurin' cache: maven - name: Build with Maven - run: mvn clean install --no-transfer-progress -B -V + run: mvn clean install -DskipTests --no-transfer-progress -B -V - name: Upload artifacts uses: actions/upload-artifact@v4 with: From 019cbf59d4748d0090a416778e6d780326e58668 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 18 Mar 2024 16:31:52 -0700 Subject: [PATCH 04/10] Update maven.yml --- .github/workflows/maven.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 35d7760f6d..4376810b90 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -62,7 +62,7 @@ jobs: run: | cp ranger-*.tar.gz dev-support/ranger-docker/dist cp version dev-support/ranger-docker/dist - - name: Build base image + - name: Build all ranger-service images run: | cd dev-support/ranger-docker chmod +x download-archives.sh && ./download-archives.sh @@ -70,4 +70,14 @@ jobs: export DOCKER_BUILDKIT=1 export COMPOSE_DOCKER_CLI_BUILD=1 export RANGER_DB_TYPE=postgres - docker-compose -f docker-compose.ranger.yml -f docker-compose.ranger-${RANGER_DB_TYPE}.yml build + docker-compose \ + -f docker-compose.ranger-${RANGER_DB_TYPE}.yml \ + -f docker-compose.ranger.yml \ + -f docker-compose.ranger-usersync.yml \ + -f docker-compose.ranger-tagsync.yml \ + -f docker-compose.ranger-kms.yml \ + -f docker-compose.ranger-hadoop.yml \ + -f docker-compose.ranger-hbase.yml \ + -f docker-compose.ranger-kafka.yml \ + -f docker-compose.ranger-hive.yml \ + -f docker-compose.ranger-knox.yml build From a382b79d830d9f2396412770a4dcadbbfb42bebd Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 23 Mar 2024 19:02:15 -0700 Subject: [PATCH 05/10] Update maven.yml --- .github/workflows/maven.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 1cfb9125bc..d7fbd8f914 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -46,7 +46,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: target + name: target-8 path: target/* build-11: @@ -64,7 +64,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: target + name: target-11 path: target/*trino* docker-build: @@ -74,10 +74,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Download build artifacts + - name: Download build-8 artifacts uses: actions/download-artifact@v4 with: - name: target + name: target-8 + - name: Download build-11 artifacts + uses: actions/download-artifact@v4 + with: + name: target-11 - name: Copy artifacts for docker build run: | cp ranger-*.tar.gz dev-support/ranger-docker/dist From 4e3c7de10c2e0a4a474988768c6803ea16ed3dc1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 16 Jul 2024 14:17:35 -0700 Subject: [PATCH 06/10] Bring up containers and check if they are running --- .github/workflows/maven.yml | 39 +++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index d7fbd8f914..5a7b76436f 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -104,5 +104,40 @@ jobs: -f docker-compose.ranger-hbase.yml \ -f docker-compose.ranger-kafka.yml \ -f docker-compose.ranger-hive.yml \ - -f docker-compose.ranger-knox.yml \ - -f docker-compose.ranger-trino.yml build + -f docker-compose.ranger-knox.yml build + - name: Bring up containers + run: | + cd dev-support/ranger-docker + export RANGER_DB_TYPE=postgres + docker-compose \ + -f docker-compose.ranger-${RANGER_DB_TYPE}.yml \ + -f docker-compose.ranger.yml \ + -f docker-compose.ranger-usersync.yml \ + -f docker-compose.ranger-tagsync.yml \ + -f docker-compose.ranger-kms.yml \ + -f docker-compose.ranger-hadoop.yml \ + -f docker-compose.ranger-hbase.yml \ + -f docker-compose.ranger-kafka.yml \ + -f docker-compose.ranger-hive.yml \ + -f docker-compose.ranger-knox.yml up -d + - name: Check status of containers + run: | + sleep 60 + containers=(ranger ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox); + flag=true; + for container in "${containers[@]}"; do + if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then + echo "Container $container is running!"; + else + flag=false; + echo "Container $container is NOT running!"; + fi + done + + if [[ $flag == true ]]; then + echo "All required containers are up and running"; + docker stop $(docker ps -q) && docker rm $(docker ps -aq); + else + docker stop $(docker ps -q) && docker rm $(docker ps -aq); + exit 1; + fi From f19bc2e08f93bb77fbcaad9e0355a911d2c2d274 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 16 Jul 2024 15:36:01 -0700 Subject: [PATCH 07/10] Remove trino --- .github/workflows/maven.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 5a7b76436f..344c656566 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -78,10 +78,6 @@ jobs: uses: actions/download-artifact@v4 with: name: target-8 - - name: Download build-11 artifacts - uses: actions/download-artifact@v4 - with: - name: target-11 - name: Copy artifacts for docker build run: | cp ranger-*.tar.gz dev-support/ranger-docker/dist From 847385d49e850023295137bd5299a4664b120e7a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 18 Jul 2024 19:58:26 -0700 Subject: [PATCH 08/10] Add ranger-zk, ranger-solr and ranger-postgres to monitor list --- .github/workflows/maven.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 344c656566..7ed10d3448 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -60,12 +60,12 @@ jobs: distribution: 'temurin' cache: maven - name: build (11) - run: mvn -T 8 clean install -P ranger-jdk11,!all,!linux --no-transfer-progress -B -V + run: mvn -T 8 clean install --no-transfer-progress -B -V - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: target-11 - path: target/*trino* + path: target/* docker-build: needs: @@ -116,10 +116,10 @@ jobs: -f docker-compose.ranger-kafka.yml \ -f docker-compose.ranger-hive.yml \ -f docker-compose.ranger-knox.yml up -d - - name: Check status of containers + - name: Check status of containers and remove them run: | sleep 60 - containers=(ranger ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox); + containers=(ranger ranger-zk ranger-solr ranger-postgres ranger-usersync ranger-tagsync ranger-kms ranger-hadoop ranger-hbase ranger-kafka ranger-hive ranger-knox); flag=true; for container in "${containers[@]}"; do if [[ $(docker inspect -f '{{.State.Running}}' $container 2>/dev/null) == "true" ]]; then From 06e844417810bf54cf82059b1ab8daefa5e5f739 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 21 Jul 2024 21:07:29 -0700 Subject: [PATCH 09/10] Remove building of knox-agent from jdk11. Broken KnoxRangerTest --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 7ed10d3448..7dce76e115 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -60,7 +60,7 @@ jobs: distribution: 'temurin' cache: maven - name: build (11) - run: mvn -T 8 clean install --no-transfer-progress -B -V + run: mvn -T 8 clean install -DskipTests -pl '!knox-agent' --no-transfer-progress -B -V - name: Upload artifacts uses: actions/upload-artifact@v4 with: From ecdc07ec62d9a0f61f39e484e3fcad0e80d38e10 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 21 Jul 2024 21:08:34 -0700 Subject: [PATCH 10/10] Enable tests for jdk11 --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 7dce76e115..53f551151a 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -60,7 +60,7 @@ jobs: distribution: 'temurin' cache: maven - name: build (11) - run: mvn -T 8 clean install -DskipTests -pl '!knox-agent' --no-transfer-progress -B -V + run: mvn -T 8 clean install -pl '!knox-agent' --no-transfer-progress -B -V - name: Upload artifacts uses: actions/upload-artifact@v4 with: