Docker - Airflow - version updated to 2.10.0 and 2.9.3

docker/airflow/2/debian11/2.8/Dockerfile → docker/airflow/2/debian12/2.10/Dockerfile

@@ -1,4 +1,4 @@
-ARG AIRFLOW_VERSION="2.8.2"
+ARG AIRFLOW_VERSION="2.10.0"
 ARG AIRFLOW_EXTRAS="amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,apache-airflow-providers-cncf-kubernetes"
 
 ARG AIRFLOW_HOME=/opt/airflow
@@ -7,13 +7,13 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow
 ARG AIRFLOW_PIP_VERSION="22.3.1"
 
 
-FROM marketplace.gcr.io/google/c2d-debian11 as ospo
+FROM marketplace.gcr.io/google/debian12 as ospo
 
 # Download Licenses and restricted source-code
 COPY components.csv /components.csv
 COPY source_code.txt /source_code.txt
 
-RUN apt update && apt -y install ca-certificates
+RUN apt update && apt -y install ca-certificates curl
 
 RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \
 		&& curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \
@@ -25,7 +25,7 @@ RUN mkdir -p /usr/src/licenses \
 		&& /download-ref-repos.sh /source_code.txt /usr/src
 
 
-FROM marketplace.gcr.io/google/c2d-debian11 as airflow-build-image
+FROM marketplace.gcr.io/google/debian12 as airflow-build-image
 
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
 ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
@@ -59,6 +59,7 @@ ENV DEV_APT_DEPS="\
      python3 \
      python3-pip \
      python3-dev \
+     python3-venv \
      sasl2-bin \
      software-properties-common \
      sqlite3 \
@@ -119,6 +120,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw
 
 USER airflow
 
+RUN python3 -m venv ~/venv
+
+ENV PATH="/home/airflow/venv/bin:$PATH"
+
 COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/
 
 RUN /scripts/docker/install_pip_version.sh
@@ -136,7 +141,7 @@ RUN /scripts/docker/install_airflow.sh \
     && find "${AIRFLOW_USER_HOME_DIR}/.local" -executable -print0 | xargs --null chmod g+x \
     && find "${AIRFLOW_USER_HOME_DIR}/.local" -print0 | xargs --null chmod g+rw
 
-FROM marketplace.gcr.io/google/c2d-debian11 as main
+FROM marketplace.gcr.io/google/debian12 as main
 
 COPY --from=ospo /usr/src /usr/src
 
@@ -162,18 +167,17 @@ ENV RUNTIME_APT_DEPS="\
        freetds-bin \
        krb5-user \
        ldap-utils \
-       libffi7 \
-       libldap-2.4-2 \
+       libffi8 \
+       libldap-common \
        libsasl2-2 \
        libsasl2-modules \
-       libssl1.1 \
+       libssl3 \
        locales  \
        lsb-release \
-       netcat \
+       netcat-traditional \
        openssh-client \
        pkg-config \
        postgresql-client \
-       python2 \
        python3 \
        python3-pip \
        rsync \
@@ -280,7 +284,7 @@ WORKDIR ${AIRFLOW_HOME}
 EXPOSE 8080
 USER ${AIRFLOW_UID}
 
-ENV C2D_RELEASE 2.8.2
+ENV C2D_RELEASE 2.10.0
 
 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
 CMD [""]

docker/airflow/2/debian11/2.8/scripts/docker/install_mysql.sh → docker/airflow/2/debian12/2.10/scripts/docker/install_mysql.sh

@@ -40,7 +40,7 @@ install_mysql_client() {
     gpgconf --kill all
     rm -rf "${GNUPGHOME}"
     unset GNUPGHOME
-    echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
+    echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
     apt-get update
     apt-get install --no-install-recommends -y "${packages[@]}"
     apt-get autoremove -yqq --purge

docker/airflow/2/debian11/2.8/scripts/docker/install_pip_version.sh → docker/airflow/2/debian12/2.10/scripts/docker/install_pip_version.sh

@@ -6,8 +6,8 @@ function install_pip_version() {
     echo
     echo "Installing pip version ${AIRFLOW_PIP_VERSION}"
     echo
-    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
-        mkdir -p ${HOME}/.local/bin
+    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+    mkdir -p ${HOME}/.local/bin
 }
 
 install_pip_version

docker/airflow/2/debian11/2.6/Dockerfile → docker/airflow/2/debian12/2.9/Dockerfile

@@ -1,4 +1,4 @@
-ARG AIRFLOW_VERSION="2.6.3"
+ARG AIRFLOW_VERSION="2.9.3"
 ARG AIRFLOW_EXTRAS="amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,apache-airflow-providers-cncf-kubernetes"
 
 ARG AIRFLOW_HOME=/opt/airflow
@@ -7,13 +7,13 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow
 ARG AIRFLOW_PIP_VERSION="22.3.1"
 
 
-FROM marketplace.gcr.io/google/c2d-debian11 as ospo
+FROM marketplace.gcr.io/google/debian12 as ospo
 
 # Download Licenses and restricted source-code
 COPY components.csv /components.csv
 COPY source_code.txt /source_code.txt
 
-RUN apt update && apt -y install ca-certificates
+RUN apt update && apt -y install ca-certificates curl
 
 RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \
 		&& curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \
@@ -25,7 +25,7 @@ RUN mkdir -p /usr/src/licenses \
 		&& /download-ref-repos.sh /source_code.txt /usr/src
 
 
-FROM marketplace.gcr.io/google/c2d-debian11 as airflow-build-image
+FROM marketplace.gcr.io/google/debian12 as airflow-build-image
 
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
 ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
@@ -59,6 +59,7 @@ ENV DEV_APT_DEPS="\
      python3 \
      python3-pip \
      python3-dev \
+     python3-venv \
      sasl2-bin \
      software-properties-common \
      sqlite3 \
@@ -119,6 +120,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw
 
 USER airflow
 
+RUN python3 -m venv ~/venv
+
+ENV PATH="/home/airflow/venv/bin:$PATH"
+
 COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/
 
 RUN /scripts/docker/install_pip_version.sh
@@ -136,7 +141,7 @@ RUN /scripts/docker/install_airflow.sh \
     && find "${AIRFLOW_USER_HOME_DIR}/.local" -executable -print0 | xargs --null chmod g+x \
     && find "${AIRFLOW_USER_HOME_DIR}/.local" -print0 | xargs --null chmod g+rw
 
-FROM marketplace.gcr.io/google/c2d-debian11 as main
+FROM marketplace.gcr.io/google/debian12 as main
 
 COPY --from=ospo /usr/src /usr/src
 
@@ -162,18 +167,17 @@ ENV RUNTIME_APT_DEPS="\
        freetds-bin \
        krb5-user \
        ldap-utils \
-       libffi7 \
-       libldap-2.4-2 \
+       libffi8 \
+       libldap-common \
        libsasl2-2 \
        libsasl2-modules \
-       libssl1.1 \
+       libssl3 \
        locales  \
        lsb-release \
-       netcat \
+       netcat-traditional \
        openssh-client \
        pkg-config \
        postgresql-client \
-       python2 \
        python3 \
        python3-pip \
        rsync \
@@ -280,7 +284,7 @@ WORKDIR ${AIRFLOW_HOME}
 EXPOSE 8080
 USER ${AIRFLOW_UID}
 
-ENV C2D_RELEASE 2.6.3
+ENV C2D_RELEASE 2.9.3
 
 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
 CMD [""]

docker/airflow/2/debian11/2.6/scripts/docker/install_mysql.sh → docker/airflow/2/debian12/2.9/scripts/docker/install_mysql.sh

@@ -40,7 +40,7 @@ install_mysql_client() {
     gpgconf --kill all
     rm -rf "${GNUPGHOME}"
     unset GNUPGHOME
-    echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
+    echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
     apt-get update
     apt-get install --no-install-recommends -y "${packages[@]}"
     apt-get autoremove -yqq --purge

docker/airflow/2/debian11/2.6/scripts/docker/install_pip_version.sh → docker/airflow/2/debian12/2.9/scripts/docker/install_pip_version.sh

@@ -6,8 +6,8 @@ function install_pip_version() {
     echo
     echo "Installing pip version ${AIRFLOW_PIP_VERSION}"
     echo
-    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
-        mkdir -p ${HOME}/.local/bin
+    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+    mkdir -p ${HOME}/.local/bin
 }
 
 install_pip_version

docker/airflow/templates/Dockerfile.template

@@ -17,7 +17,7 @@ FROM {{ .From }} as ospo
 COPY components.csv /components.csv
 COPY source_code.txt /source_code.txt
 
-RUN apt update && apt -y install ca-certificates
+RUN apt update && apt -y install ca-certificates curl
 
 RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \
 		&& curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \
@@ -63,6 +63,7 @@ ENV DEV_APT_DEPS="\
      python3 \
      python3-pip \
      python3-dev \
+     python3-venv \
      sasl2-bin \
      software-properties-common \
      sqlite3 \
@@ -123,6 +124,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw
 
 USER airflow
 
+RUN python3 -m venv ~/venv
+
+ENV PATH="/home/airflow/venv/bin:$PATH"
+
 COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/
 
 RUN /scripts/docker/install_pip_version.sh
@@ -166,18 +171,17 @@ ENV RUNTIME_APT_DEPS="\
        freetds-bin \
        krb5-user \
        ldap-utils \
-       libffi7 \
-       libldap-2.4-2 \
+       libffi8 \
+       libldap-common \
        libsasl2-2 \
        libsasl2-modules \
-       libssl1.1 \
+       libssl3 \
        locales  \
        lsb-release \
-       netcat \
+       netcat-traditional \
        openssh-client \
        pkg-config \
        postgresql-client \
-       python2 \
        python3 \
        python3-pip \
        rsync \

docker/airflow/templates/scripts/docker/install_mysql.sh

@@ -40,7 +40,7 @@ install_mysql_client() {
     gpgconf --kill all
     rm -rf "${GNUPGHOME}"
     unset GNUPGHOME
-    echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
+    echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
     apt-get update
     apt-get install --no-install-recommends -y "${packages[@]}"
     apt-get autoremove -yqq --purge

docker/airflow/templates/scripts/docker/install_pip_version.sh

@@ -6,8 +6,8 @@ function install_pip_version() {
     echo
     echo "Installing pip version ${AIRFLOW_PIP_VERSION}"
     echo
-    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
-        mkdir -p ${HOME}/.local/bin
+    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+    mkdir -p ${HOME}/.local/bin
 }
 
 install_pip_version

docker/airflow/tests/functional_tests/running_test.yaml

@@ -31,7 +31,7 @@ setup:
   - 'MYSQL_ROOT_PASSWORD=some-password'
   - '-d'
   - 'marketplace.gcr.io/google/postgresql13'
-- command: [sleep, '20s']
+- command: [sleep, '120s']
 - command:
   - 'docker'
   - 'run'
@@ -54,7 +54,7 @@ setup:
   - '-d'
   - '$IMAGE'
   - 'webserver'
-- command: [sleep, '70s']
+- command: [sleep, '120s']
 
 teardown:
 - command: [docker, stop, some-airflow-$UNIQUE-id, some-postgres-$UNIQUE-id]

docker/airflow/versions.yaml

@@ -16,38 +16,38 @@
 cloudbuild:
   enable_parallel: false
 versions:
-- dir: 2/debian11/2.8
-  from: marketplace.gcr.io/google/c2d-debian11
+- dir: 2/debian12/2.10
+  from: marketplace.gcr.io/google/debian12
   packages:
     airflow:
-      version: 2.8.2
+      version: 2.10.0
     pip:
       version: 22.3.1
     gosu:
       gpg: B42F6819007F00F88E364FD4036A9C25BF357DD4
       version: '1.17'
   repo: airflow2
   tags:
-  - 2.8.2-debian11
-  - 2.8-debian11
-  - 2-debian11
-  - 2.8.2
-  - '2.8'
+  - 2.10.0-debian12
+  - 2.10-debian12
+  - 2-debian12
+  - 2.10.0
+  - '2.10'
   - '2'
   - latest
-- dir: 2/debian11/2.6
-  from: marketplace.gcr.io/google/c2d-debian11
+- dir: 2/debian12/2.9
+  from: marketplace.gcr.io/google/debian12
   packages:
     airflow:
-      version: 2.6.3
+      version: 2.9.3
     gosu:
       gpg: B42F6819007F00F88E364FD4036A9C25BF357DD4
       version: '1.17'
     pip:
       version: 22.3.1
   repo: airflow2
   tags:
-  - 2.6.3-debian11
-  - 2.6-debian11
-  - 2.6.3
-  - '2.6'
+  - 2.9.3-debian12
+  - 2.9-debian12
+  - 2.9.3
+  - '2.9'