Rebuilding CI (elastic#227)

* CI: Reintegrate whole CI into the same Dockerfile * CI: Update Makefile with CI improvments Change certificates names Update flavor resolution * CI: Github: Implement CI modifications into Actions * Examples: Update certificates names * Generator: Update go.mod & go.sum * CI: Fix mistake in COPY command * CI: Add mod download for generated tests * CI: Fix typo * CI: Add jenkins jobs for 7.12 * CI: Removing jenkins jobs for 7.10 * CI: Removing jenkins jobs for 5.x
cqy123456 · Feb 22, 2021 · c653ef6 · c653ef6
1 parent 6f546f7
commit c653ef6
Show file tree

Hide file tree

Showing 11 changed files with 117 additions and 146 deletions.
diff --git a/Dockerfile → .ci/Dockerfile b/Dockerfile → .ci/Dockerfile
@@ -8,7 +8,7 @@ FROM golang:${VERSION}
 
 RUN apk add --no-cache --quiet make curl git jq unzip tree && \
     go get -u golang.org/x/lint/golint && \
-    curl -sSL --retry 3 --retry-connrefused https://github.com/gotestyourself/gotestsum/releases/download/v0.5.3/gotestsum_0.5.3_linux_amd64.tar.gz | tar -xz -C /usr/local/bin gotestsum
+    curl -sSL --retry 3 --retry-connrefused https://github.com/gotestyourself/gotestsum/releases/download/v1.6.1/gotestsum_1.6.1_linux_amd64.tar.gz | tar -xz -C /usr/local/bin gotestsum
 
 VOLUME ["/tmp"]
 
@@ -17,10 +17,11 @@ ENV TERM xterm-256color
 
 WORKDIR /go-elasticsearch
 
-COPY go.mod go.sum ./
+COPY . .
+
 RUN go mod download
 
-COPY internal/cmd/generate/go.mod internal/cmd/generate/go.sum
-RUN cd internal/cmd/generate && go mod download
+WORKDIR internal/cmd/generate
+RUN go mod download
 
-COPY . .
+WORKDIR /go-elasticsearch
diff --git a/.ci/jobs/elastic+go-elasticsearch+5.x.yml b/.ci/jobs/elastic+go-elasticsearch+5.x.yml
diff --git a/.ci/jobs/elastic+go-elasticsearch+7.10.yml → .ci/jobs/elastic+go-elasticsearch+7.12.yml b/.ci/jobs/elastic+go-elasticsearch+7.10.yml → .ci/jobs/elastic+go-elasticsearch+7.12.yml
@@ -1,14 +1,15 @@
 ---
 - job:
-    name: elastic+go-elasticsearch+7.10
-    display-name: 'elastic / go-elasticsearch # 7.10'
-    description: Testing the go-elasticsearch 7.10 branch.
+    name: elastic+go-elasticsearch+7.12
+    display-name: 'elastic / go-elasticsearch # 7.12'
+    description: Testing the go-elasticsearch 7.12 branch.
     junit_results: "*-junit.xml"
     parameters:
       - string:
           name: branch_specifier
-          default: refs/heads/7.10
+          default: refs/heads/7.12
           description: The Git branch specifier to build
     triggers:
       - github
       - timed: 'H */12 * * *'
+
diff --git a/.ci/run-repository.sh b/.ci/run-repository.sh
@@ -15,10 +15,9 @@ set -euo pipefail
 echo -e "\033[34;1mINFO:\033[0m VERSION: ${STACK_VERSION}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m TEST_SUITE: ${TEST_SUITE}\033[0m"
 echo -e "\033[34;1mINFO:\033[0m RUNSCRIPTS: ${RUNSCRIPTS}\033[0m"
-echo -e "\033[34;1mINFO:\033[0m elasticsearch_url: ${elasticsearch_url}\033[0m"
-echo -e "\033[34;1mINFO:\033[0m external_elasticsearch_url: ${external_elasticsearch_url}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m URL: ${elasticsearch_url}\033[0m"
 
-echo -e "\033[34;1mINFO:\033[0m Pinging Elasticsearch...\033[0m"
+echo -e "\033[34;1mINFO:\033[0m pinging Elasticsearch ..\033[0m"
 curl --insecure --fail $external_elasticsearch_url/_cluster/health?pretty
 
 if [[ "$RUNSCRIPTS" = *"enterprise-search"* ]]; then
@@ -29,86 +28,41 @@ fi
 
 echo -e "\033[32;1mSUCCESS:\033[0m successfully started the ${STACK_VERSION} stack.\033[0m"
 
-# -----------------------------------------------------------------------------
+echo -e "\033[34;1mINFO:\033[0m Building the [elastic/go-elasticsearch] image... \033[0m"
 
-TIMEFORMAT="Duration: %0lR"
+docker build \
+    --file .ci/Dockerfile \
+    --tag elastic/go-elasticsearch \
+    .
 
-if [[ $TEST_SUITE != "free" && $TEST_SUITE != "platinum" ]]; then
-  echo -e "\033[31;1mERROR:\033[0m Unknown value [$TEST_SUITE] for [TEST_SUITE]\033[0m"
-  exit 1
-fi
-
-echo
-echo -e "\033[1m>>>>> SETUP [$TEST_SUITE] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+echo -e "\033[34;1mINFO:\033[0m Retrieving Elasticsearch Version & Hash from container... \033[0m"
 
-# Store the Elasticsearch version and build hash
 ELASTICSEARCH_BUILD_VERSION=$(curl -sSk $external_elasticsearch_url | jq -r '.version.number')
 ELASTICSEARCH_BUILD_HASH=$(curl -sSk $external_elasticsearch_url | jq -r '.version.build_hash')
 
-# Build the go-elasticsearch image
-echo -e ">>>>> Building the [elastic/go-elasticsearch] image..."
-docker build --file Dockerfile --tag elastic/go-elasticsearch .
+echo -e "\033[34;1mINFO:\033[0m Downloading Elasticsearch repository @ $ELASTICSEARCH_BUILD_HASH... \033[0m"
+mkdir -p $WORKSPACE/tmp
+curl --retry 10 -sSL "https://github.com/elastic/elasticsearch/archive/$ELASTICSEARCH_BUILD_HASH.zip" > "$WORKSPACE/tmp/elasticsearch-$ELASTICSEARCH_BUILD_HASH.zip"
 
-# Download Elasticsearch source code at ELASTICSEARCH_BUILD_HASH and store it in a container
-echo -e ">>>>> Downloading Elasticsearch repository @ $ELASTICSEARCH_BUILD_HASH..."
-time curl --retry 10 -sSL "https://github.com/elastic/elasticsearch/archive/$ELASTICSEARCH_BUILD_HASH.zip" > "/tmp/elasticsearch-$ELASTICSEARCH_BUILD_HASH.zip"
-echo -e ">>>>> Extracting the Elasticsearch source..."
-time docker run --volume=/tmp:/tmp --workdir=/tmp --rm elastic/go-elasticsearch unzip -q -o "elasticsearch-$ELASTICSEARCH_BUILD_HASH.zip" '*.properties' '*.json' '*.y*ml'
-docker run --volume=/tmp:/tmp --workdir=/tmp --rm elastic/go-elasticsearch /bin/sh -c "
+echo -e "\033[34;1mINFO:\033[0m Extracting the Elasticsearch source... \033[0m"
+docker run --volume=$WORKSPACE/tmp:/tmp --workdir=/tmp --rm elastic/go-elasticsearch unzip -q -o "elasticsearch-$ELASTICSEARCH_BUILD_HASH.zip" '*.properties' '*.json' '*.y*ml'
+docker run --volume=$WORKSPACE/tmp:/tmp --workdir=/tmp --rm elastic/go-elasticsearch /bin/sh -c "
   rm -rf /tmp/elasticsearch-$ELASTICSEARCH_BUILD_HASH.zip
   rm -rf /tmp/elasticsearch/
   mv /tmp/elasticsearch-$ELASTICSEARCH_BUILD_HASH* /tmp/elasticsearch/
 "
 
-echo -e ">>>>> Creating the container with Elasticsearch repository..."
-docker container rm -f elasticsearch-source > /dev/null 2>&1 || true
-docker create --name elasticsearch-source --volume /elasticsearch-source alpine /bin/true
-docker cp /tmp/elasticsearch elasticsearch-source:/elasticsearch-source
+echo -e "\033[34;1mINFO:\033[0m Execute [$TEST_SUITE] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
-# Launch the container; actual commands are called with "docker exec"
-docker run \
-  --name go-elasticsearch \
+docker run -t \
+  --volume=$WORKSPACE/tmp:/tmp \
+  --rm \
   --network "$network_name" \
   --env "ELASTICSEARCH_URL=$elasticsearch_url" \
   --env "ELASTICSEARCH_VERSION=$STACK_VERSION" \
   --env "ELASTICSEARCH_BUILD_VERSION=$ELASTICSEARCH_BUILD_VERSION" \
   --env "ELASTICSEARCH_BUILD_HASH=$ELASTICSEARCH_BUILD_HASH" \
   --env "WORKSPACE=${WORKSPACE:-/workspace}" \
-  --volume "/go-elasticsearch" \
   --volume "${WORKSPACE:-workspace}:${WORKSPACE:-/workspace}" \
-  --volumes-from "elasticsearch-source" \
-  --rm \
-  --detach \
-  elastic/go-elasticsearch sleep 3600
-
-echo
-echo -e "\033[1m>>>>> EXECUTE [$TEST_SUITE] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
-
-# Run the tests
-# NOTE: Conditions needed to prevent early exit due to the 'set -e' option
-status=100
-case $TEST_SUITE in
-  "free" )
-    if bash .ci/scripts/tests-free.sh; then
-      status=$?
-    else
-      status=$?
-    fi
-    ;;
-  "platinum" )
-    if bash .ci/scripts/tests-platinum.sh; then
-      status=$?
-    else
-      status=$?
-    fi
-    ;;
-esac
-
-# Report status and exit
-if [[ $status == "0" ]]; then
-  echo -e "\n\033[32;1mSUCCESS [$TEST_SUITE]\033[0m"
-  exit 0
-else
-  echo -e "\n\033[31;1mFAILURE [$TEST_SUITE]\033[0m"
-  exit $status
-fi
+  elastic/go-elasticsearch \
+  .ci/scripts/tests-$TEST_SUITE.sh
diff --git a/.ci/scripts/tests-free.sh b/.ci/scripts/tests-free.sh
@@ -1,26 +1,32 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
-set -euo pipefail
+set -uo pipefail
 
 TIMEFORMAT="(Duration: %0lR)"
 
-echo -e "\033[1m>>>>> Cleaning up test files\033[0m"
+echo -e "\033[34;1mINFO:\033[0m Cleaning up test files\033[0m"
 
-docker exec go-elasticsearch /bin/sh -c 'rm -rf esapi/test/*_test.go'
-docker exec go-elasticsearch /bin/sh -c 'rm -rf esapi/test/xpack'
+rm -rf esapi/test/*_test.go
+rm -rf esapi/test/xpack
 
-echo -e "\033[1m>>>>> Generating the API registry\033[0m"
+echo -e "\033[34;1mINFO:\033[0m Generating the API registry\033[0m"
 
-docker exec --workdir=/go-elasticsearch/internal/cmd/generate go-elasticsearch go get -u golang.org/x/tools/cmd/goimports
-docker exec --workdir=/go-elasticsearch/internal/cmd/generate --env PACKAGE_PATH=/go-elasticsearch/esapi go-elasticsearch go generate ./...
+cd /go-elasticsearch/internal/cmd/generate || exit
+go get -u golang.org/x/tools/cmd/goimports
+PACKAGE_PATH=/go-elasticsearch/esapi go generate ./...
 
-echo -e "\033[1m>>>>> Generating the test files\033[0m"
+echo -e "\033[34;1mINFO:\033[0m Generating the test files\033[0m"
 
-time docker exec --tty --workdir=/go-elasticsearch/internal/cmd/generate go-elasticsearch go run main.go apitests --output '/go-elasticsearch/esapi/test' --input '/elasticsearch-source/elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/test/**/*.y*ml'
+go run main.go apitests --output '/go-elasticsearch/esapi/test' --input '/tmp/elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/test/**/*.y*ml'
 
-echo -e "\033[1m>>>>> Running the tests\033[0m"
+echo -e "\033[34;1mINFO:\033[0m Download tests deps >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
-time docker exec --tty --workdir=/go-elasticsearch/esapi/test go-elasticsearch /bin/sh -c 'gotestsum --format=short-verbose --junitfile=$WORKSPACE/TEST-integration-api-junit.xml -- -tags=integration -timeout=1h *_test.go'
+cd /go-elasticsearch/esapi/test || exit
+go mod download
+
+echo -e "\033[34;1mINFO:\033[0m Running the tests\033[0m"
+
+gotestsum --format=short-verbose --junitfile=$WORKSPACE/TEST-integration-api-junit.xml -- -tags=integration -timeout=1h *_test.go
 status=$?
 
 exit $status
diff --git a/.ci/scripts/tests-platinum.sh b/.ci/scripts/tests-platinum.sh
@@ -1,46 +1,53 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 set -uo pipefail
 
 TIMEFORMAT="(Duration: %0lR)"
 
-echo -e "\033[1m>>>>> Cleaning up test files\033[0m"
+echo -e "\033[34;1mINFO:\033[0m Cleaning up test files\033[0m"
 
-docker exec go-elasticsearch /bin/sh -c 'rm -rf esapi/test/*_test.go'
-docker exec go-elasticsearch /bin/sh -c 'rm -rf rm -rf esapi/test/ml*'
+rm -rf esapi/test/*_test.go
+rm -rf rm -rf esapi/test/ml*
 
-echo -e "\033[1m>>>>> Generating the API registry\033[0m"
+echo -e "\033[34;1mINFO:\033[0m Generating the API registry\033[0m"
 
-docker exec --workdir=/go-elasticsearch/internal/cmd/generate go-elasticsearch go get -u golang.org/x/tools/cmd/goimports
-docker exec --workdir=/go-elasticsearch/internal/cmd/generate --env PACKAGE_PATH=/go-elasticsearch/esapi go-elasticsearch go generate ./...
+cd /go-elasticsearch/internal/cmd/generate || exit
+go get -u golang.org/x/tools/cmd/goimports
+PACKAGE_PATH=/go-elasticsearch/esapi go generate ./...
 
-echo -e "\033[1m>>>>> Generating the test files\033[0m"
+echo -e "\033[34;1mINFO:\033[0m Generating the test files\033[0m"
 
-time docker exec --tty --workdir=/go-elasticsearch/internal/cmd/generate go-elasticsearch go run main.go apitests --output '/go-elasticsearch/esapi/test/xpack' --input '/elasticsearch-source/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/test/**/*.yml'
+go run main.go apitests --output '/go-elasticsearch/esapi/test/xpack' --input '/tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/test/**/*.yml'
+go run main.go apitests --output '/go-elasticsearch/esapi/test/xpack' --input '/tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/test/**/**/*.yml'
 
-time docker exec --tty --workdir=/go-elasticsearch/internal/cmd/generate go-elasticsearch go run main.go apitests --output '/go-elasticsearch/esapi/test/xpack' --input '/elasticsearch-source/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/test/**/**/*.yml'
+cd /go-elasticsearch || exit
 
-docker exec go-elasticsearch mkdir -p esapi/test/xpack/ml
-docker exec go-elasticsearch mkdir -p esapi/test/xpack/ml-crud
+mkdir -p esapi/test/xpack/ml
+mkdir -p esapi/test/xpack/ml-crud
 
-docker exec go-elasticsearch /bin/sh -c 'mv esapi/test/xpack/xpack_ml* esapi/test/xpack/ml/'
-docker exec go-elasticsearch mv esapi/test/xpack/ml/xpack_ml__jobs_crud_test.go esapi/test/xpack/ml-crud/
+mv esapi/test/xpack/xpack_ml* esapi/test/xpack/ml/
+mv esapi/test/xpack/ml/xpack_ml__jobs_crud_test.go esapi/test/xpack/ml-crud/
 
 set +e # Do not fail immediately when a single test suite fails
 
-echo -e "\033[1m>>>>> Running tests: XPACK >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+echo -e "\033[34;1mINFO:\033[0m Download tests deps >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
-time docker exec --tty --workdir=/go-elasticsearch/esapi/test go-elasticsearch /bin/sh -c 'gotestsum --format=short-verbose --junitfile=$WORKSPACE/TEST-integration-api-xpack-junit.xml -- --tags=integration --timeout=1h -v xpack/*_test.go'
+cd /go-elasticsearch/esapi/test || exit
+go mod download
+
+echo -e "\033[34;1mINFO:\033[0m Running tests: XPACK >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+
+gotestsum --format=short-verbose --junitfile=$WORKSPACE/TEST-integration-api-xpack-junit.xml -- --tags=integration --timeout=1h -v xpack/*_test.go
 status1=$?
 
-echo -e "\033[1m>>>>> Running tests: XPACK ML >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+echo -e "\033[34;1mINFO:\033[0m Running tests: XPACK ML >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
-time docker exec --tty --workdir=/go-elasticsearch/esapi/test go-elasticsearch /bin/sh -c 'gotestsum --format=short-verbose --junitfile=$WORKSPACE/TEST-integration-api-xpack-ml-junit.xml -- --tags=integration --timeout=1h -v ./xpack/ml/*_test.go'
+gotestsum --format=short-verbose --junitfile=$WORKSPACE/TEST-integration-api-xpack-ml-junit.xml -- --tags=integration --timeout=1h -v ./xpack/ml/*_test.go
 status2=$?
 
-echo -e "\033[1m>>>>> Running tests: XPACK ML CRUD >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+echo -e "\033[34;1mINFO:\033[0m Running tests: XPACK ML CRUD >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 
-time docker exec --tty --workdir=/go-elasticsearch/esapi/test go-elasticsearch /bin/sh -c 'gotestsum --format=short-verbose --junitfile=$WORKSPACE/TEST-integration-api-xpack-ml-crud-junit.xml -- --tags=integration --timeout=1h -v ./xpack/ml-crud/*_test.go'
+gotestsum --format=short-verbose --junitfile=$WORKSPACE/TEST-integration-api-xpack-ml-crud-junit.xml -- --tags=integration --timeout=1h -v ./xpack/ml-crud/*_test.go
 status3=$?
 
 if [[ $status1 == 0 && $status2 == 0 && $status3 == 0 ]]; then

diff --git a/.github/workflows/test-api.yml b/.github/workflows/test-api.yml
@@ -8,8 +8,8 @@ env:
   GITHUB_ACTIONS: true
 
 jobs:
-  test-core:
-    name: Core
+  test-free:
+    name: Free
     env:
       ELASTICSEARCH_VERSION: elasticsearch:8.0.0-SNAPSHOT
       ELASTICSEARCH_URL: http://localhost:9200
@@ -40,8 +40,8 @@ jobs:
       - run: ELASTICSEARCH_BUILD_HASH=$(cat elasticsearch_build_hash) make gen-tests
       - run: make test-api
 
-  test-xpack:
-      name: X-Pack
+  test-platinum:
+      name: Platinum
       env:
         ELASTICSEARCH_VERSION: elasticsearch:8.0.0-SNAPSHOT
         ELASTICSEARCH_URL: https://elastic:elastic@localhost:9200
@@ -62,7 +62,7 @@ jobs:
           run: |
             docker pull --quiet docker.elastic.co/elasticsearch/${{ env.ELASTICSEARCH_VERSION }}
             docker pull --quiet appropriate/curl
-            make cluster-clean cluster-update cluster detach=true version="${{ env.ELASTICSEARCH_VERSION }}"
+            make cluster-clean cluster-update cluster flavor=platinum detach=true version="${{ env.ELASTICSEARCH_VERSION }}"
         - name: Download Elasticsearch source
           run: |
             curl -sSk ${{ env.ELASTICSEARCH_URL }}