[AIRFLOW-5079] Checklicence test uses own, much smaller image (apache…

…#5692)

Dockerfile-checklicence

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# WARNING: THIS DOCKERFILE IS NOT INTENDED FOR PRODUCTION USE OR DEPLOYMENT.
+#
+FROM debian:stretch-slim
+
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+# Make sure noninteractie debian install is used and language variables set
+ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
+    LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
+
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
+
+# Note missing man directories on debian-stretch
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
+# Install OpenJDK
+RUN mkdir -pv /usr/share/man/man1 \
+    && mkdir -pv /usr/share/man/man7 \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+            sudo \
+            curl \
+            openjdk-8-jdk \
+    && apt-get autoremove -yqq --purge \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG RAT_VERSION="0.12"
+
+ENV RAT_VERSION="${RAT_VERSION}" \
+    RAT_JAR="/tmp/apache-rat-${RAT_VERSION}.jar" \
+    RAT_URL="http://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar"
+
+RUN echo "Downloading RAT from ${RAT_URL} to ${RAT_JAR}" \
+    && curl -sL ${RAT_URL} > ${RAT_JAR}
+
+ARG AIRFLOW_USER=airflow
+ENV AIRFLOW_USER=${AIRFLOW_USER}
+
+ARG HOME=/home/airflow
+ENV HOME=${HOME}
+
+ARG AIRFLOW_HOME=${HOME}/airflow
+ENV AIRFLOW_HOME=${AIRFLOW_HOME}
+
+ARG AIRFLOW_SOURCES=/opt/airflow
+ENV AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+
+RUN adduser airflow \
+    && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \
+    && chmod 0440 /etc/sudoers.d/airflow
+
+COPY --chown=airflow:airflow scripts ${AIRFLOW_SOURCES}/scripts
+
+USER airflow
+
+WORKDIR ${AIRFLOW_SOURCES}
+
+ENTRYPOINT ["/opt/airflow/scripts/ci/in_container/run_check_licence.sh"]

hooks/build

@@ -294,6 +294,10 @@ echo "Skip slim image: ${AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE}"
 export AIRFLOW_CONTAINER_SKIP_CI_IMAGE=${AIRFLOW_CONTAINER_SKIP_CI_IMAGE:="false"}
 echo "Skip CI image: ${AIRFLOW_CONTAINER_SKIP_CI_IMAGE}"
 
+# Skips downloading and building the checklicence image - one used for checking the licences
+export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE=${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE:="false"}
+echo "Skip CHECKLICENCE image: ${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}"
+
 # Skips pulling the airflow images - this will use cache but will build it all from scratch
 export AIRFLOW_CONTAINER_SKIP_PULLING_AIRFLOW_IMAGES=${AIRFLOW_CONTAINER_SKIP_PULLING_AIRFLOW_IMAGES:="false"}
 echo "Skip pulling Airflow images: ${AIRFLOW_CONTAINER_SKIP_PULLING_AIRFLOW_IMAGES}"
@@ -322,12 +326,15 @@ if [[ "${AIRFLOW_RELEASE_BUILD}" == "true" ]]; then
     export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${AIRFLOW_VERSION}-ci"
     export AIRFLOW_CI_IMAGE_LATEST="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-python${PYTHON_VERSION}-${TAG_PREFIX}-ci"
     export AIRFLOW_CI_IMAGE_LATEST_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:latest-${TAG_PREFIX}-ci"
+
+    export AIRFLOW_CHECKLICENCE_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:checklicence"
 else
     export AIRFLOW_SLIM_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-python${PYTHON_VERSION}-ci-slim"
     export AIRFLOW_SLIM_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-ci-slim"
 
     export AIRFLOW_CI_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-python${PYTHON_VERSION}-ci"
     export AIRFLOW_CI_IMAGE_DEFAULT="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:${TAG_PREFIX}-ci"
+    export AIRFLOW_CHECKLICENCE_IMAGE="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:checklicence"
 fi
 # In the future we can enable buildkit.
 # It's experimental now and cache does not work out of the box with buildkit in Docker 18.09.2, buildkit 0.3.3
@@ -350,6 +357,7 @@ if [[ "${AIRFLOW_CONTAINER_CLEANUP_IMAGES}" == "true" ]]; then
     docker rmi "${PYTHON_BASE_IMAGE}" || true
     docker rmi "${AIRFLOW_SLIM_CI_IMAGE}" || true
     docker rmi "${AIRFLOW_CI_IMAGE}" || true
+    docker rmi "${AIRFLOW_CHECKLICENCE_IMAGE}" || true
     echo
     echo "###################################################################"
     echo "NOTE!! Removed Airflow images for Python version ${PYTHON_VERSION}."
@@ -394,6 +402,11 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then
     else
         IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_SLIM_CI_IMAGE}"
     fi
+    if [[ "${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}" == "true" ]]; then
+        echo "Skip downloading the CHECKLICENCE Airflow image"
+    else
+        IMAGES_TO_PULL="${IMAGES_TO_PULL} ${AIRFLOW_CHECKLICENCE_IMAGE}"
+    fi
 
     DOCKER_CACHE_DIRECTIVE_CI=()
     DOCKER_CACHE_DIRECTIVE_CI_SLIM=()
@@ -439,6 +452,8 @@ if [[ "${AIRFLOW_CONTAINER_USE_PULLED_IMAGES_CACHE}" == "true" ]]; then
             DOCKER_CACHE_DIRECTIVE_CI+=("--cache-from" "${IMAGE}")
         elif [[ "${IMAGE}" == "${AIRFLOW_SLIM_CI_IMAGE}" ]]; then
             DOCKER_CACHE_DIRECTIVE_CI_SLIM+=("--cache-from" "${IMAGE}")
+        elif [[ "${IMAGE}" == "${AIRFLOW_CHECKLICENCE_IMAGE}" ]]; then
+            DOCKER_CACHE_DIRECTIVE_CHECKLICENCE+=("--cache-from" "${IMAGE}")
         else
             echo
             echo "Don't know how to set cache directive for ${IMAGE}. Exiting"
@@ -595,6 +610,17 @@ else
     fi
 fi
 
+if [[ "${AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE}" == "true" ]]; then
+    echo "Skip building the CHECKLICENCE image"
+else
+    start_step "Build Airflow CHECKLICENCE image"
+    docker build . -f Dockerfile-checklicence -t ${AIRFLOW_CHECKLICENCE_IMAGE}
+    add_image_to_push "${AIRFLOW_CHECKLICENCE_IMAGE}"
+
+    save_to_file AIRFLOW_CHECKLICENCE_IMAGE
+fi
+
+
 start_step "Pushing images"
 
 if [[ "${AIRFLOW_CONTAINER_PUSH_IMAGES}" != "false" ]]; then

scripts/ci/_utils.sh

@@ -298,6 +298,7 @@ function force_python_3_5() {
 function rebuild_image_if_needed_for_static_checks() {
     export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="false"
     export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true"
+    export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true"
     export AIRFLOW_CONTAINER_PUSH_IMAGES="false"
     export AIRFLOW_CONTAINER_BUILD_NPM="false"  # Skip NPM builds to make them faster !
 
@@ -359,6 +360,7 @@ EOF
 
 function rebuild_image_if_needed_for_tests() {
     export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true"
+    export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="true"
     export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="false"
     PYTHON_VERSION=${PYTHON_VERSION:=$(python -c \
         'import sys; print("%s.%s" % (sys.version_info.major, sys.version_info.minor))')}
@@ -417,6 +419,25 @@ EOF
     export AIRFLOW_CI_IMAGE
 }
 
+function rebuild_image_for_checklicence() {
+    export AIRFLOW_CONTAINER_SKIP_SLIM_CI_IMAGE="true"
+    export AIRFLOW_CONTAINER_SKIP_CHECKLICENCE_IMAGE="false"
+    export AIRFLOW_CONTAINER_SKIP_CI_IMAGE="true"
+    export AIRFLOW_CONTAINER_PUSH_IMAGES="false"
+
+    export THE_IMAGE="CHECKLICENCE"
+    echo
+    echo "Rebuilding image"
+    echo
+    # shellcheck source=../../hooks/build
+    ./hooks/build | tee -a "${OUTPUT_LOG}"
+    update_all_md5_files
+    echo
+    echo "Image rebuilt"
+    echo
+    AIRFLOW_CHECKLICENCE_IMAGE=$(cat "${BUILD_CACHE_DIR}/.AIRFLOW_CHECKLICENCE_IMAGE")
+    export AIRFLOW_CHECKLICENCE_IMAGE
+}
 #
 # Starts the script/ If VERBOSE variable is set to true, it enables verbose output of commands executed
 # Also prints some useful diagnostics information at start of the script

scripts/ci/ci_check_license.sh

@@ -29,15 +29,12 @@ force_python_3_5
 
 script_start
 
-rebuild_image_if_needed_for_tests
+rebuild_image_for_checklicence
 
 docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \
-       --entrypoint /opt/airflow/scripts/ci/in_container/run_check_licence.sh \
-       --env PYTHONDONTWRITEBYTECODE="true" \
-       --env AIRFLOW_CI_VERBOSE=${VERBOSE} \
-       --env AIRFLOW_CI_VERBOSE=${VERBOSE} \
+       --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
        --env HOST_USER_ID="$(id -ur)" \
        --env HOST_GROUP_ID="$(id -gr)" \
-       "${AIRFLOW_CI_IMAGE}" \
+       "${AIRFLOW_CHECKLICENCE_IMAGE}"
 
 script_end