Docker - Airflow - version updated to 2.10.0 and 2.9.3

GoogleCloudPlatform · Sep 4, 2024 · c3cbf8b · c3cbf8b
1 parent 39e071b
commit c3cbf8b
Show file tree

Hide file tree

Showing 22 changed files with 54 additions and 42 deletions.
diff --git a/docker/airflow/2/debian11/2.10/Dockerfile → docker/airflow/2/debian12/2.10/Dockerfile b/docker/airflow/2/debian11/2.10/Dockerfile → docker/airflow/2/debian12/2.10/Dockerfile
@@ -7,13 +7,13 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow
 ARG AIRFLOW_PIP_VERSION="22.3.1"
 
 
-FROM marketplace.gcr.io/google/c2d-debian11 as ospo
+FROM marketplace.gcr.io/google/debian12 as ospo
 
 # Download Licenses and restricted source-code
 COPY components.csv /components.csv
 COPY source_code.txt /source_code.txt
 
-RUN apt update && apt -y install ca-certificates
+RUN apt update && apt -y install ca-certificates curl
 
 RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \
 		&& curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \
@@ -25,7 +25,7 @@ RUN mkdir -p /usr/src/licenses \
 		&& /download-ref-repos.sh /source_code.txt /usr/src
 
 
-FROM marketplace.gcr.io/google/c2d-debian11 as airflow-build-image
+FROM marketplace.gcr.io/google/debian12 as airflow-build-image
 
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
 ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
@@ -59,6 +59,7 @@ ENV DEV_APT_DEPS="\
      python3 \
      python3-pip \
      python3-dev \
+     python3-venv \
      sasl2-bin \
      software-properties-common \
      sqlite3 \
@@ -119,6 +120,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw
 
 USER airflow
 
+RUN python3 -m venv ~/venv
+
+ENV PATH="/home/airflow/venv/bin:$PATH"
+
 COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/
 
 RUN /scripts/docker/install_pip_version.sh
@@ -136,7 +141,7 @@ RUN /scripts/docker/install_airflow.sh \
     && find "${AIRFLOW_USER_HOME_DIR}/.local" -executable -print0 | xargs --null chmod g+x \
     && find "${AIRFLOW_USER_HOME_DIR}/.local" -print0 | xargs --null chmod g+rw
 
-FROM marketplace.gcr.io/google/c2d-debian11 as main
+FROM marketplace.gcr.io/google/debian12 as main
 
 COPY --from=ospo /usr/src /usr/src
 
@@ -162,18 +167,17 @@ ENV RUNTIME_APT_DEPS="\
        freetds-bin \
        krb5-user \
        ldap-utils \
-       libffi7 \
-       libldap-2.4-2 \
+       libffi8 \
+       libldap-common \
        libsasl2-2 \
        libsasl2-modules \
-       libssl1.1 \
+       libssl3 \
        locales  \
        lsb-release \
-       netcat \
+       netcat-traditional \
        openssh-client \
        pkg-config \
        postgresql-client \
-       python2 \
        python3 \
        python3-pip \
        rsync \

diff --git a/...er/airflow/2/debian11/2.10/components.csv → ...er/airflow/2/debian12/2.10/components.csv b/...er/airflow/2/debian11/2.10/components.csv → ...er/airflow/2/debian12/2.10/components.csv
diff --git a/...11/2.10/scripts/docker/install_airflow.sh → ...12/2.10/scripts/docker/install_airflow.sh b/...11/2.10/scripts/docker/install_airflow.sh → ...12/2.10/scripts/docker/install_airflow.sh
diff --git a/...an11/2.10/scripts/docker/install_mssql.sh → ...an12/2.10/scripts/docker/install_mssql.sh b/...an11/2.10/scripts/docker/install_mssql.sh → ...an12/2.10/scripts/docker/install_mssql.sh
diff --git a/...ian11/2.9/scripts/docker/install_mysql.sh → ...an12/2.10/scripts/docker/install_mysql.sh b/...ian11/2.9/scripts/docker/install_mysql.sh → ...an12/2.10/scripts/docker/install_mysql.sh
@@ -40,7 +40,7 @@ install_mysql_client() {
     gpgconf --kill all
     rm -rf "${GNUPGHOME}"
     unset GNUPGHOME
-    echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
+    echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
     apt-get update
     apt-get install --no-install-recommends -y "${packages[@]}"
     apt-get autoremove -yqq --purge

diff --git a/...2.9/scripts/docker/install_pip_version.sh → ....10/scripts/docker/install_pip_version.sh b/...2.9/scripts/docker/install_pip_version.sh → ....10/scripts/docker/install_pip_version.sh
@@ -6,8 +6,8 @@ function install_pip_version() {
     echo
     echo "Installing pip version ${AIRFLOW_PIP_VERSION}"
     echo
-    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
-        mkdir -p ${HOME}/.local/bin
+    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+    mkdir -p ${HOME}/.local/bin
 }
 
 install_pip_version
diff --git a/.../debian11/2.10/scripts/prod/clean-logs.sh → .../debian12/2.10/scripts/prod/clean-logs.sh b/.../debian11/2.10/scripts/prod/clean-logs.sh → .../debian12/2.10/scripts/prod/clean-logs.sh
diff --git a/...an11/2.10/scripts/prod/entrypoint_prod.sh → ...an12/2.10/scripts/prod/entrypoint_prod.sh b/...an11/2.10/scripts/prod/entrypoint_prod.sh → ...an12/2.10/scripts/prod/entrypoint_prod.sh
diff --git a/...r/airflow/2/debian11/2.10/source_code.txt → ...r/airflow/2/debian12/2.10/source_code.txt b/...r/airflow/2/debian11/2.10/source_code.txt → ...r/airflow/2/debian12/2.10/source_code.txt
diff --git a/docker/airflow/2/debian11/2.9/Dockerfile → docker/airflow/2/debian12/2.9/Dockerfile b/docker/airflow/2/debian11/2.9/Dockerfile → docker/airflow/2/debian12/2.9/Dockerfile
@@ -7,13 +7,13 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow
 ARG AIRFLOW_PIP_VERSION="22.3.1"
 
 
-FROM marketplace.gcr.io/google/c2d-debian11 as ospo
+FROM marketplace.gcr.io/google/debian12 as ospo
 
 # Download Licenses and restricted source-code
 COPY components.csv /components.csv
 COPY source_code.txt /source_code.txt
 
-RUN apt update && apt -y install ca-certificates
+RUN apt update && apt -y install ca-certificates curl
 
 RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \
 		&& curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \
@@ -25,7 +25,7 @@ RUN mkdir -p /usr/src/licenses \
 		&& /download-ref-repos.sh /source_code.txt /usr/src
 
 
-FROM marketplace.gcr.io/google/c2d-debian11 as airflow-build-image
+FROM marketplace.gcr.io/google/debian12 as airflow-build-image
 
 SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
 ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
@@ -59,6 +59,7 @@ ENV DEV_APT_DEPS="\
      python3 \
      python3-pip \
      python3-dev \
+     python3-venv \
      sasl2-bin \
      software-properties-common \
      sqlite3 \
@@ -119,6 +120,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw
 
 USER airflow
 
+RUN python3 -m venv ~/venv
+
+ENV PATH="/home/airflow/venv/bin:$PATH"
+
 COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/
 
 RUN /scripts/docker/install_pip_version.sh
@@ -136,7 +141,7 @@ RUN /scripts/docker/install_airflow.sh \
     && find "${AIRFLOW_USER_HOME_DIR}/.local" -executable -print0 | xargs --null chmod g+x \
     && find "${AIRFLOW_USER_HOME_DIR}/.local" -print0 | xargs --null chmod g+rw
 
-FROM marketplace.gcr.io/google/c2d-debian11 as main
+FROM marketplace.gcr.io/google/debian12 as main
 
 COPY --from=ospo /usr/src /usr/src
 
@@ -162,18 +167,17 @@ ENV RUNTIME_APT_DEPS="\
        freetds-bin \
        krb5-user \
        ldap-utils \
-       libffi7 \
-       libldap-2.4-2 \
+       libffi8 \
+       libldap-common \
        libsasl2-2 \
        libsasl2-modules \
-       libssl1.1 \
+       libssl3 \
        locales  \
        lsb-release \
-       netcat \
+       netcat-traditional \
        openssh-client \
        pkg-config \
        postgresql-client \
-       python2 \
        python3 \
        python3-pip \
        rsync \

diff --git a/docker/airflow/2/debian11/2.9/components.csv → docker/airflow/2/debian12/2.9/components.csv b/docker/airflow/2/debian11/2.9/components.csv → docker/airflow/2/debian12/2.9/components.csv
diff --git a/...n11/2.9/scripts/docker/install_airflow.sh → ...n12/2.9/scripts/docker/install_airflow.sh b/...n11/2.9/scripts/docker/install_airflow.sh → ...n12/2.9/scripts/docker/install_airflow.sh
diff --git a/...ian11/2.9/scripts/docker/install_mssql.sh → ...ian12/2.9/scripts/docker/install_mssql.sh b/...ian11/2.9/scripts/docker/install_mssql.sh → ...ian12/2.9/scripts/docker/install_mssql.sh
diff --git a/...an11/2.10/scripts/docker/install_mysql.sh → ...ian12/2.9/scripts/docker/install_mysql.sh b/...an11/2.10/scripts/docker/install_mysql.sh → ...ian12/2.9/scripts/docker/install_mysql.sh
@@ -40,7 +40,7 @@ install_mysql_client() {
     gpgconf --kill all
     rm -rf "${GNUPGHOME}"
     unset GNUPGHOME
-    echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
+    echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
     apt-get update
     apt-get install --no-install-recommends -y "${packages[@]}"
     apt-get autoremove -yqq --purge

diff --git a/....10/scripts/docker/install_pip_version.sh → ...2.9/scripts/docker/install_pip_version.sh b/....10/scripts/docker/install_pip_version.sh → ...2.9/scripts/docker/install_pip_version.sh
@@ -6,8 +6,8 @@ function install_pip_version() {
     echo
     echo "Installing pip version ${AIRFLOW_PIP_VERSION}"
     echo
-    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
-        mkdir -p ${HOME}/.local/bin
+    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+    mkdir -p ${HOME}/.local/bin
 }
 
 install_pip_version
diff --git a/...2/debian11/2.9/scripts/prod/clean-logs.sh → ...2/debian12/2.9/scripts/prod/clean-logs.sh b/...2/debian11/2.9/scripts/prod/clean-logs.sh → ...2/debian12/2.9/scripts/prod/clean-logs.sh
diff --git a/...ian11/2.9/scripts/prod/entrypoint_prod.sh → ...ian12/2.9/scripts/prod/entrypoint_prod.sh b/...ian11/2.9/scripts/prod/entrypoint_prod.sh → ...ian12/2.9/scripts/prod/entrypoint_prod.sh
diff --git a/...er/airflow/2/debian11/2.9/source_code.txt → ...er/airflow/2/debian12/2.9/source_code.txt b/...er/airflow/2/debian11/2.9/source_code.txt → ...er/airflow/2/debian12/2.9/source_code.txt
diff --git a/docker/airflow/templates/Dockerfile.template b/docker/airflow/templates/Dockerfile.template
@@ -17,7 +17,7 @@ FROM {{ .From }} as ospo
 COPY components.csv /components.csv
 COPY source_code.txt /source_code.txt
 
-RUN apt update && apt -y install ca-certificates
+RUN apt update && apt -y install ca-certificates curl
 
 RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \
 		&& curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \
@@ -63,6 +63,7 @@ ENV DEV_APT_DEPS="\
      python3 \
      python3-pip \
      python3-dev \
+     python3-venv \
      sasl2-bin \
      software-properties-common \
      sqlite3 \
@@ -123,6 +124,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw
 
 USER airflow
 
+RUN python3 -m venv ~/venv
+
+ENV PATH="/home/airflow/venv/bin:$PATH"
+
 COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/
 
 RUN /scripts/docker/install_pip_version.sh
@@ -166,18 +171,17 @@ ENV RUNTIME_APT_DEPS="\
        freetds-bin \
        krb5-user \
        ldap-utils \
-       libffi7 \
-       libldap-2.4-2 \
+       libffi8 \
+       libldap-common \
        libsasl2-2 \
        libsasl2-modules \
-       libssl1.1 \
+       libssl3 \
        locales  \
        lsb-release \
-       netcat \
+       netcat-traditional \
        openssh-client \
        pkg-config \
        postgresql-client \
-       python2 \
        python3 \
        python3-pip \
        rsync \

diff --git a/docker/airflow/templates/scripts/docker/install_mysql.sh b/docker/airflow/templates/scripts/docker/install_mysql.sh
@@ -40,7 +40,7 @@ install_mysql_client() {
     gpgconf --kill all
     rm -rf "${GNUPGHOME}"
     unset GNUPGHOME
-    echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
+    echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
     apt-get update
     apt-get install --no-install-recommends -y "${packages[@]}"
     apt-get autoremove -yqq --purge

diff --git a/docker/airflow/templates/scripts/docker/install_pip_version.sh b/docker/airflow/templates/scripts/docker/install_pip_version.sh
@@ -6,8 +6,8 @@ function install_pip_version() {
     echo
     echo "Installing pip version ${AIRFLOW_PIP_VERSION}"
     echo
-    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
-        mkdir -p ${HOME}/.local/bin
+    pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+    mkdir -p ${HOME}/.local/bin
 }
 
 install_pip_version
diff --git a/docker/airflow/versions.yaml b/docker/airflow/versions.yaml
@@ -16,8 +16,8 @@
 cloudbuild:
   enable_parallel: false
 versions:
-- dir: 2/debian11/2.10
-  from: marketplace.gcr.io/google/c2d-debian11
+- dir: 2/debian12/2.10
+  from: marketplace.gcr.io/google/debian12
   packages:
     airflow:
       version: 2.10.0
@@ -28,15 +28,15 @@ versions:
       version: '1.17'
   repo: airflow2
   tags:
-  - 2.10.0-debian11
-  - 2.10-debian11
-  - 2-debian11
+  - 2.10.0-debian12
+  - 2.10-debian12
+  - 2-debian12
   - 2.10.0
   - '2.10'
   - '2'
   - latest
-- dir: 2/debian11/2.9
-  from: marketplace.gcr.io/google/c2d-debian11
+- dir: 2/debian12/2.9
+  from: marketplace.gcr.io/google/debian12
   packages:
     airflow:
       version: 2.9.3
@@ -47,7 +47,7 @@ versions:
       version: 22.3.1
   repo: airflow2
   tags:
-  - 2.9.3-debian11
-  - 2.9-debian11
+  - 2.9.3-debian12
+  - 2.9-debian12
   - 2.9.3
   - '2.9'