diff --git a/docker/airflow/2/debian11/2.8/Dockerfile b/docker/airflow/2/debian12/2.10/Dockerfile similarity index 95% rename from docker/airflow/2/debian11/2.8/Dockerfile rename to docker/airflow/2/debian12/2.10/Dockerfile index d2a97ff7c5..ed50d79a29 100644 --- a/docker/airflow/2/debian11/2.8/Dockerfile +++ b/docker/airflow/2/debian12/2.10/Dockerfile @@ -1,4 +1,4 @@ -ARG AIRFLOW_VERSION="2.8.2" +ARG AIRFLOW_VERSION="2.10.0" ARG AIRFLOW_EXTRAS="amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,apache-airflow-providers-cncf-kubernetes" ARG AIRFLOW_HOME=/opt/airflow @@ -7,13 +7,13 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow ARG AIRFLOW_PIP_VERSION="22.3.1" -FROM marketplace.gcr.io/google/c2d-debian11 as ospo +FROM marketplace.gcr.io/google/debian12 as ospo # Download Licenses and restricted source-code COPY components.csv /components.csv COPY source_code.txt /source_code.txt -RUN apt update && apt -y install ca-certificates +RUN apt update && apt -y install ca-certificates curl RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \ && curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \ @@ -25,7 +25,7 @@ RUN mkdir -p /usr/src/licenses \ && /download-ref-repos.sh /source_code.txt /usr/src -FROM marketplace.gcr.io/google/c2d-debian11 as airflow-build-image +FROM marketplace.gcr.io/google/debian12 as airflow-build-image SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ @@ -59,6 +59,7 @@ ENV DEV_APT_DEPS="\ python3 \ python3-pip \ python3-dev \ + python3-venv \ sasl2-bin \ software-properties-common \ sqlite3 \ @@ -119,6 +120,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw USER airflow +RUN python3 -m venv ~/venv + +ENV PATH="/home/airflow/venv/bin:$PATH" + COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/ RUN /scripts/docker/install_pip_version.sh @@ -136,7 +141,7 @@ RUN /scripts/docker/install_airflow.sh \ && find "${AIRFLOW_USER_HOME_DIR}/.local" -executable -print0 | xargs --null chmod g+x \ && find "${AIRFLOW_USER_HOME_DIR}/.local" -print0 | xargs --null chmod g+rw -FROM marketplace.gcr.io/google/c2d-debian11 as main +FROM marketplace.gcr.io/google/debian12 as main COPY --from=ospo /usr/src /usr/src @@ -162,18 +167,17 @@ ENV RUNTIME_APT_DEPS="\ freetds-bin \ krb5-user \ ldap-utils \ - libffi7 \ - libldap-2.4-2 \ + libffi8 \ + libldap-common \ libsasl2-2 \ libsasl2-modules \ - libssl1.1 \ + libssl3 \ locales \ lsb-release \ - netcat \ + netcat-traditional \ openssh-client \ pkg-config \ postgresql-client \ - python2 \ python3 \ python3-pip \ rsync \ @@ -280,7 +284,7 @@ WORKDIR ${AIRFLOW_HOME} EXPOSE 8080 USER ${AIRFLOW_UID} -ENV C2D_RELEASE 2.8.2 +ENV C2D_RELEASE 2.10.0 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] CMD [""] diff --git a/docker/airflow/2/debian11/2.6/components.csv b/docker/airflow/2/debian12/2.10/components.csv similarity index 100% rename from docker/airflow/2/debian11/2.6/components.csv rename to docker/airflow/2/debian12/2.10/components.csv diff --git a/docker/airflow/2/debian11/2.6/scripts/docker/install_airflow.sh b/docker/airflow/2/debian12/2.10/scripts/docker/install_airflow.sh similarity index 100% rename from docker/airflow/2/debian11/2.6/scripts/docker/install_airflow.sh rename to docker/airflow/2/debian12/2.10/scripts/docker/install_airflow.sh diff --git a/docker/airflow/2/debian11/2.6/scripts/docker/install_mssql.sh b/docker/airflow/2/debian12/2.10/scripts/docker/install_mssql.sh similarity index 100% rename from docker/airflow/2/debian11/2.6/scripts/docker/install_mssql.sh rename to docker/airflow/2/debian12/2.10/scripts/docker/install_mssql.sh diff --git a/docker/airflow/2/debian11/2.8/scripts/docker/install_mysql.sh b/docker/airflow/2/debian12/2.10/scripts/docker/install_mysql.sh similarity index 91% rename from docker/airflow/2/debian11/2.8/scripts/docker/install_mysql.sh rename to docker/airflow/2/debian12/2.10/scripts/docker/install_mysql.sh index 3a37bc93a0..9581a6e63a 100755 --- a/docker/airflow/2/debian11/2.8/scripts/docker/install_mysql.sh +++ b/docker/airflow/2/debian12/2.10/scripts/docker/install_mysql.sh @@ -40,7 +40,7 @@ install_mysql_client() { gpgconf --kill all rm -rf "${GNUPGHOME}" unset GNUPGHOME - echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list + echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge diff --git a/docker/airflow/2/debian11/2.8/scripts/docker/install_pip_version.sh b/docker/airflow/2/debian12/2.10/scripts/docker/install_pip_version.sh similarity index 77% rename from docker/airflow/2/debian11/2.8/scripts/docker/install_pip_version.sh rename to docker/airflow/2/debian12/2.10/scripts/docker/install_pip_version.sh index ba15c70501..3d2f6e7a5c 100755 --- a/docker/airflow/2/debian11/2.8/scripts/docker/install_pip_version.sh +++ b/docker/airflow/2/debian12/2.10/scripts/docker/install_pip_version.sh @@ -6,8 +6,8 @@ function install_pip_version() { echo echo "Installing pip version ${AIRFLOW_PIP_VERSION}" echo - pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && - mkdir -p ${HOME}/.local/bin + pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" + mkdir -p ${HOME}/.local/bin } install_pip_version diff --git a/docker/airflow/2/debian11/2.6/scripts/prod/clean-logs.sh b/docker/airflow/2/debian12/2.10/scripts/prod/clean-logs.sh similarity index 100% rename from docker/airflow/2/debian11/2.6/scripts/prod/clean-logs.sh rename to docker/airflow/2/debian12/2.10/scripts/prod/clean-logs.sh diff --git a/docker/airflow/2/debian11/2.6/scripts/prod/entrypoint_prod.sh b/docker/airflow/2/debian12/2.10/scripts/prod/entrypoint_prod.sh similarity index 100% rename from docker/airflow/2/debian11/2.6/scripts/prod/entrypoint_prod.sh rename to docker/airflow/2/debian12/2.10/scripts/prod/entrypoint_prod.sh diff --git a/docker/airflow/2/debian11/2.6/source_code.txt b/docker/airflow/2/debian12/2.10/source_code.txt similarity index 100% rename from docker/airflow/2/debian11/2.6/source_code.txt rename to docker/airflow/2/debian12/2.10/source_code.txt diff --git a/docker/airflow/2/debian11/2.6/Dockerfile b/docker/airflow/2/debian12/2.9/Dockerfile similarity index 95% rename from docker/airflow/2/debian11/2.6/Dockerfile rename to docker/airflow/2/debian12/2.9/Dockerfile index ed465e1c5d..63ff55710f 100644 --- a/docker/airflow/2/debian11/2.6/Dockerfile +++ b/docker/airflow/2/debian12/2.9/Dockerfile @@ -1,4 +1,4 @@ -ARG AIRFLOW_VERSION="2.6.3" +ARG AIRFLOW_VERSION="2.9.3" ARG AIRFLOW_EXTRAS="amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,apache-airflow-providers-cncf-kubernetes" ARG AIRFLOW_HOME=/opt/airflow @@ -7,13 +7,13 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow ARG AIRFLOW_PIP_VERSION="22.3.1" -FROM marketplace.gcr.io/google/c2d-debian11 as ospo +FROM marketplace.gcr.io/google/debian12 as ospo # Download Licenses and restricted source-code COPY components.csv /components.csv COPY source_code.txt /source_code.txt -RUN apt update && apt -y install ca-certificates +RUN apt update && apt -y install ca-certificates curl RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \ && curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \ @@ -25,7 +25,7 @@ RUN mkdir -p /usr/src/licenses \ && /download-ref-repos.sh /source_code.txt /usr/src -FROM marketplace.gcr.io/google/c2d-debian11 as airflow-build-image +FROM marketplace.gcr.io/google/debian12 as airflow-build-image SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ @@ -59,6 +59,7 @@ ENV DEV_APT_DEPS="\ python3 \ python3-pip \ python3-dev \ + python3-venv \ sasl2-bin \ software-properties-common \ sqlite3 \ @@ -119,6 +120,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw USER airflow +RUN python3 -m venv ~/venv + +ENV PATH="/home/airflow/venv/bin:$PATH" + COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/ RUN /scripts/docker/install_pip_version.sh @@ -136,7 +141,7 @@ RUN /scripts/docker/install_airflow.sh \ && find "${AIRFLOW_USER_HOME_DIR}/.local" -executable -print0 | xargs --null chmod g+x \ && find "${AIRFLOW_USER_HOME_DIR}/.local" -print0 | xargs --null chmod g+rw -FROM marketplace.gcr.io/google/c2d-debian11 as main +FROM marketplace.gcr.io/google/debian12 as main COPY --from=ospo /usr/src /usr/src @@ -162,18 +167,17 @@ ENV RUNTIME_APT_DEPS="\ freetds-bin \ krb5-user \ ldap-utils \ - libffi7 \ - libldap-2.4-2 \ + libffi8 \ + libldap-common \ libsasl2-2 \ libsasl2-modules \ - libssl1.1 \ + libssl3 \ locales \ lsb-release \ - netcat \ + netcat-traditional \ openssh-client \ pkg-config \ postgresql-client \ - python2 \ python3 \ python3-pip \ rsync \ @@ -280,7 +284,7 @@ WORKDIR ${AIRFLOW_HOME} EXPOSE 8080 USER ${AIRFLOW_UID} -ENV C2D_RELEASE 2.6.3 +ENV C2D_RELEASE 2.9.3 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] CMD [""] diff --git a/docker/airflow/2/debian11/2.8/components.csv b/docker/airflow/2/debian12/2.9/components.csv similarity index 100% rename from docker/airflow/2/debian11/2.8/components.csv rename to docker/airflow/2/debian12/2.9/components.csv diff --git a/docker/airflow/2/debian11/2.8/scripts/docker/install_airflow.sh b/docker/airflow/2/debian12/2.9/scripts/docker/install_airflow.sh similarity index 100% rename from docker/airflow/2/debian11/2.8/scripts/docker/install_airflow.sh rename to docker/airflow/2/debian12/2.9/scripts/docker/install_airflow.sh diff --git a/docker/airflow/2/debian11/2.8/scripts/docker/install_mssql.sh b/docker/airflow/2/debian12/2.9/scripts/docker/install_mssql.sh similarity index 100% rename from docker/airflow/2/debian11/2.8/scripts/docker/install_mssql.sh rename to docker/airflow/2/debian12/2.9/scripts/docker/install_mssql.sh diff --git a/docker/airflow/2/debian11/2.6/scripts/docker/install_mysql.sh b/docker/airflow/2/debian12/2.9/scripts/docker/install_mysql.sh similarity index 91% rename from docker/airflow/2/debian11/2.6/scripts/docker/install_mysql.sh rename to docker/airflow/2/debian12/2.9/scripts/docker/install_mysql.sh index 3a37bc93a0..9581a6e63a 100755 --- a/docker/airflow/2/debian11/2.6/scripts/docker/install_mysql.sh +++ b/docker/airflow/2/debian12/2.9/scripts/docker/install_mysql.sh @@ -40,7 +40,7 @@ install_mysql_client() { gpgconf --kill all rm -rf "${GNUPGHOME}" unset GNUPGHOME - echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list + echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge diff --git a/docker/airflow/2/debian11/2.6/scripts/docker/install_pip_version.sh b/docker/airflow/2/debian12/2.9/scripts/docker/install_pip_version.sh similarity index 77% rename from docker/airflow/2/debian11/2.6/scripts/docker/install_pip_version.sh rename to docker/airflow/2/debian12/2.9/scripts/docker/install_pip_version.sh index ba15c70501..3d2f6e7a5c 100755 --- a/docker/airflow/2/debian11/2.6/scripts/docker/install_pip_version.sh +++ b/docker/airflow/2/debian12/2.9/scripts/docker/install_pip_version.sh @@ -6,8 +6,8 @@ function install_pip_version() { echo echo "Installing pip version ${AIRFLOW_PIP_VERSION}" echo - pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && - mkdir -p ${HOME}/.local/bin + pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" + mkdir -p ${HOME}/.local/bin } install_pip_version diff --git a/docker/airflow/2/debian11/2.8/scripts/prod/clean-logs.sh b/docker/airflow/2/debian12/2.9/scripts/prod/clean-logs.sh similarity index 100% rename from docker/airflow/2/debian11/2.8/scripts/prod/clean-logs.sh rename to docker/airflow/2/debian12/2.9/scripts/prod/clean-logs.sh diff --git a/docker/airflow/2/debian11/2.8/scripts/prod/entrypoint_prod.sh b/docker/airflow/2/debian12/2.9/scripts/prod/entrypoint_prod.sh similarity index 100% rename from docker/airflow/2/debian11/2.8/scripts/prod/entrypoint_prod.sh rename to docker/airflow/2/debian12/2.9/scripts/prod/entrypoint_prod.sh diff --git a/docker/airflow/2/debian11/2.8/source_code.txt b/docker/airflow/2/debian12/2.9/source_code.txt similarity index 100% rename from docker/airflow/2/debian11/2.8/source_code.txt rename to docker/airflow/2/debian12/2.9/source_code.txt diff --git a/docker/airflow/templates/Dockerfile.template b/docker/airflow/templates/Dockerfile.template index 28adde8d0e..a7f33e416b 100644 --- a/docker/airflow/templates/Dockerfile.template +++ b/docker/airflow/templates/Dockerfile.template @@ -17,7 +17,7 @@ FROM {{ .From }} as ospo COPY components.csv /components.csv COPY source_code.txt /source_code.txt -RUN apt update && apt -y install ca-certificates +RUN apt update && apt -y install ca-certificates curl RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \ && curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \ @@ -63,6 +63,7 @@ ENV DEV_APT_DEPS="\ python3 \ python3-pip \ python3-dev \ + python3-venv \ sasl2-bin \ software-properties-common \ sqlite3 \ @@ -123,6 +124,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw USER airflow +RUN python3 -m venv ~/venv + +ENV PATH="/home/airflow/venv/bin:$PATH" + COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/ RUN /scripts/docker/install_pip_version.sh @@ -166,18 +171,17 @@ ENV RUNTIME_APT_DEPS="\ freetds-bin \ krb5-user \ ldap-utils \ - libffi7 \ - libldap-2.4-2 \ + libffi8 \ + libldap-common \ libsasl2-2 \ libsasl2-modules \ - libssl1.1 \ + libssl3 \ locales \ lsb-release \ - netcat \ + netcat-traditional \ openssh-client \ pkg-config \ postgresql-client \ - python2 \ python3 \ python3-pip \ rsync \ diff --git a/docker/airflow/templates/scripts/docker/install_mysql.sh b/docker/airflow/templates/scripts/docker/install_mysql.sh index 3a37bc93a0..9581a6e63a 100755 --- a/docker/airflow/templates/scripts/docker/install_mysql.sh +++ b/docker/airflow/templates/scripts/docker/install_mysql.sh @@ -40,7 +40,7 @@ install_mysql_client() { gpgconf --kill all rm -rf "${GNUPGHOME}" unset GNUPGHOME - echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list + echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge diff --git a/docker/airflow/templates/scripts/docker/install_pip_version.sh b/docker/airflow/templates/scripts/docker/install_pip_version.sh index ba15c70501..3d2f6e7a5c 100755 --- a/docker/airflow/templates/scripts/docker/install_pip_version.sh +++ b/docker/airflow/templates/scripts/docker/install_pip_version.sh @@ -6,8 +6,8 @@ function install_pip_version() { echo echo "Installing pip version ${AIRFLOW_PIP_VERSION}" echo - pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && - mkdir -p ${HOME}/.local/bin + pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" + mkdir -p ${HOME}/.local/bin } install_pip_version diff --git a/docker/airflow/tests/functional_tests/running_test.yaml b/docker/airflow/tests/functional_tests/running_test.yaml index 5cc2a4119d..230ac48232 100644 --- a/docker/airflow/tests/functional_tests/running_test.yaml +++ b/docker/airflow/tests/functional_tests/running_test.yaml @@ -31,7 +31,7 @@ setup: - 'MYSQL_ROOT_PASSWORD=some-password' - '-d' - 'marketplace.gcr.io/google/postgresql13' -- command: [sleep, '20s'] +- command: [sleep, '120s'] - command: - 'docker' - 'run' @@ -54,7 +54,7 @@ setup: - '-d' - '$IMAGE' - 'webserver' -- command: [sleep, '70s'] +- command: [sleep, '120s'] teardown: - command: [docker, stop, some-airflow-$UNIQUE-id, some-postgres-$UNIQUE-id] diff --git a/docker/airflow/versions.yaml b/docker/airflow/versions.yaml index d5df139c0d..7df2daaffe 100644 --- a/docker/airflow/versions.yaml +++ b/docker/airflow/versions.yaml @@ -16,11 +16,11 @@ cloudbuild: enable_parallel: false versions: -- dir: 2/debian11/2.8 - from: marketplace.gcr.io/google/c2d-debian11 +- dir: 2/debian12/2.10 + from: marketplace.gcr.io/google/debian12 packages: airflow: - version: 2.8.2 + version: 2.10.0 pip: version: 22.3.1 gosu: @@ -28,18 +28,18 @@ versions: version: '1.17' repo: airflow2 tags: - - 2.8.2-debian11 - - 2.8-debian11 - - 2-debian11 - - 2.8.2 - - '2.8' + - 2.10.0-debian12 + - 2.10-debian12 + - 2-debian12 + - 2.10.0 + - '2.10' - '2' - latest -- dir: 2/debian11/2.6 - from: marketplace.gcr.io/google/c2d-debian11 +- dir: 2/debian12/2.9 + from: marketplace.gcr.io/google/debian12 packages: airflow: - version: 2.6.3 + version: 2.9.3 gosu: gpg: B42F6819007F00F88E364FD4036A9C25BF357DD4 version: '1.17' @@ -47,7 +47,7 @@ versions: version: 22.3.1 repo: airflow2 tags: - - 2.6.3-debian11 - - 2.6-debian11 - - 2.6.3 - - '2.6' + - 2.9.3-debian12 + - 2.9-debian12 + - 2.9.3 + - '2.9'