From 9c4d8a0beac839629a7b8ec95cb6bf0225e999f5 Mon Sep 17 00:00:00 2001 From: Adrian Manikowski Date: Mon, 19 Aug 2024 08:54:33 +0000 Subject: [PATCH 1/4] Docker - Airflow - version updated to 2.10.0 and 2.9.3 --- .../2/debian11/{2.8 => 2.10}/Dockerfile | 4 ++-- .../2/debian11/{2.6 => 2.10}/components.csv | 0 .../scripts/docker/install_airflow.sh | 0 .../scripts/docker/install_mssql.sh | 0 .../scripts/docker/install_mysql.sh | 0 .../scripts/docker/install_pip_version.sh | 0 .../{2.6 => 2.10}/scripts/prod/clean-logs.sh | 0 .../scripts/prod/entrypoint_prod.sh | 0 .../2/debian11/{2.6 => 2.10}/source_code.txt | 0 .../2/debian11/{2.6 => 2.9}/Dockerfile | 4 ++-- .../2/debian11/{2.8 => 2.9}/components.csv | 0 .../scripts/docker/install_airflow.sh | 0 .../scripts/docker/install_mssql.sh | 0 .../scripts/docker/install_mysql.sh | 0 .../scripts/docker/install_pip_version.sh | 0 .../{2.8 => 2.9}/scripts/prod/clean-logs.sh | 0 .../scripts/prod/entrypoint_prod.sh | 0 .../2/debian11/{2.8 => 2.9}/source_code.txt | 0 docker/airflow/versions.yaml | 24 +++++++++---------- 19 files changed, 16 insertions(+), 16 deletions(-) rename docker/airflow/2/debian11/{2.8 => 2.10}/Dockerfile (99%) rename docker/airflow/2/debian11/{2.6 => 2.10}/components.csv (100%) rename docker/airflow/2/debian11/{2.6 => 2.10}/scripts/docker/install_airflow.sh (100%) rename docker/airflow/2/debian11/{2.6 => 2.10}/scripts/docker/install_mssql.sh (100%) rename docker/airflow/2/debian11/{2.6 => 2.10}/scripts/docker/install_mysql.sh (100%) rename docker/airflow/2/debian11/{2.6 => 2.10}/scripts/docker/install_pip_version.sh (100%) rename docker/airflow/2/debian11/{2.6 => 2.10}/scripts/prod/clean-logs.sh (100%) rename docker/airflow/2/debian11/{2.6 => 2.10}/scripts/prod/entrypoint_prod.sh (100%) rename docker/airflow/2/debian11/{2.6 => 2.10}/source_code.txt (100%) rename docker/airflow/2/debian11/{2.6 => 2.9}/Dockerfile (99%) rename docker/airflow/2/debian11/{2.8 => 2.9}/components.csv (100%) rename docker/airflow/2/debian11/{2.8 => 2.9}/scripts/docker/install_airflow.sh (100%) rename docker/airflow/2/debian11/{2.8 => 2.9}/scripts/docker/install_mssql.sh (100%) rename docker/airflow/2/debian11/{2.8 => 2.9}/scripts/docker/install_mysql.sh (100%) rename docker/airflow/2/debian11/{2.8 => 2.9}/scripts/docker/install_pip_version.sh (100%) rename docker/airflow/2/debian11/{2.8 => 2.9}/scripts/prod/clean-logs.sh (100%) rename docker/airflow/2/debian11/{2.8 => 2.9}/scripts/prod/entrypoint_prod.sh (100%) rename docker/airflow/2/debian11/{2.8 => 2.9}/source_code.txt (100%) diff --git a/docker/airflow/2/debian11/2.8/Dockerfile b/docker/airflow/2/debian11/2.10/Dockerfile similarity index 99% rename from docker/airflow/2/debian11/2.8/Dockerfile rename to docker/airflow/2/debian11/2.10/Dockerfile index d2a97ff7c5..c4b42f145b 100644 --- a/docker/airflow/2/debian11/2.8/Dockerfile +++ b/docker/airflow/2/debian11/2.10/Dockerfile @@ -1,4 +1,4 @@ -ARG AIRFLOW_VERSION="2.8.2" +ARG AIRFLOW_VERSION="2.10.0" ARG AIRFLOW_EXTRAS="amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,apache-airflow-providers-cncf-kubernetes" ARG AIRFLOW_HOME=/opt/airflow @@ -280,7 +280,7 @@ WORKDIR ${AIRFLOW_HOME} EXPOSE 8080 USER ${AIRFLOW_UID} -ENV C2D_RELEASE 2.8.2 +ENV C2D_RELEASE 2.10.0 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] CMD [""] diff --git a/docker/airflow/2/debian11/2.6/components.csv b/docker/airflow/2/debian11/2.10/components.csv similarity index 100% rename from docker/airflow/2/debian11/2.6/components.csv rename to docker/airflow/2/debian11/2.10/components.csv diff --git a/docker/airflow/2/debian11/2.6/scripts/docker/install_airflow.sh b/docker/airflow/2/debian11/2.10/scripts/docker/install_airflow.sh similarity index 100% rename from docker/airflow/2/debian11/2.6/scripts/docker/install_airflow.sh rename to docker/airflow/2/debian11/2.10/scripts/docker/install_airflow.sh diff --git a/docker/airflow/2/debian11/2.6/scripts/docker/install_mssql.sh b/docker/airflow/2/debian11/2.10/scripts/docker/install_mssql.sh similarity index 100% rename from docker/airflow/2/debian11/2.6/scripts/docker/install_mssql.sh rename to docker/airflow/2/debian11/2.10/scripts/docker/install_mssql.sh diff --git a/docker/airflow/2/debian11/2.6/scripts/docker/install_mysql.sh b/docker/airflow/2/debian11/2.10/scripts/docker/install_mysql.sh similarity index 100% rename from docker/airflow/2/debian11/2.6/scripts/docker/install_mysql.sh rename to docker/airflow/2/debian11/2.10/scripts/docker/install_mysql.sh diff --git a/docker/airflow/2/debian11/2.6/scripts/docker/install_pip_version.sh b/docker/airflow/2/debian11/2.10/scripts/docker/install_pip_version.sh similarity index 100% rename from docker/airflow/2/debian11/2.6/scripts/docker/install_pip_version.sh rename to docker/airflow/2/debian11/2.10/scripts/docker/install_pip_version.sh diff --git a/docker/airflow/2/debian11/2.6/scripts/prod/clean-logs.sh b/docker/airflow/2/debian11/2.10/scripts/prod/clean-logs.sh similarity index 100% rename from docker/airflow/2/debian11/2.6/scripts/prod/clean-logs.sh rename to docker/airflow/2/debian11/2.10/scripts/prod/clean-logs.sh diff --git a/docker/airflow/2/debian11/2.6/scripts/prod/entrypoint_prod.sh b/docker/airflow/2/debian11/2.10/scripts/prod/entrypoint_prod.sh similarity index 100% rename from docker/airflow/2/debian11/2.6/scripts/prod/entrypoint_prod.sh rename to docker/airflow/2/debian11/2.10/scripts/prod/entrypoint_prod.sh diff --git a/docker/airflow/2/debian11/2.6/source_code.txt b/docker/airflow/2/debian11/2.10/source_code.txt similarity index 100% rename from docker/airflow/2/debian11/2.6/source_code.txt rename to docker/airflow/2/debian11/2.10/source_code.txt diff --git a/docker/airflow/2/debian11/2.6/Dockerfile b/docker/airflow/2/debian11/2.9/Dockerfile similarity index 99% rename from docker/airflow/2/debian11/2.6/Dockerfile rename to docker/airflow/2/debian11/2.9/Dockerfile index ed465e1c5d..322a1ab94b 100644 --- a/docker/airflow/2/debian11/2.6/Dockerfile +++ b/docker/airflow/2/debian11/2.9/Dockerfile @@ -1,4 +1,4 @@ -ARG AIRFLOW_VERSION="2.6.3" +ARG AIRFLOW_VERSION="2.9.3" ARG AIRFLOW_EXTRAS="amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,apache-airflow-providers-cncf-kubernetes" ARG AIRFLOW_HOME=/opt/airflow @@ -280,7 +280,7 @@ WORKDIR ${AIRFLOW_HOME} EXPOSE 8080 USER ${AIRFLOW_UID} -ENV C2D_RELEASE 2.6.3 +ENV C2D_RELEASE 2.9.3 ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"] CMD [""] diff --git a/docker/airflow/2/debian11/2.8/components.csv b/docker/airflow/2/debian11/2.9/components.csv similarity index 100% rename from docker/airflow/2/debian11/2.8/components.csv rename to docker/airflow/2/debian11/2.9/components.csv diff --git a/docker/airflow/2/debian11/2.8/scripts/docker/install_airflow.sh b/docker/airflow/2/debian11/2.9/scripts/docker/install_airflow.sh similarity index 100% rename from docker/airflow/2/debian11/2.8/scripts/docker/install_airflow.sh rename to docker/airflow/2/debian11/2.9/scripts/docker/install_airflow.sh diff --git a/docker/airflow/2/debian11/2.8/scripts/docker/install_mssql.sh b/docker/airflow/2/debian11/2.9/scripts/docker/install_mssql.sh similarity index 100% rename from docker/airflow/2/debian11/2.8/scripts/docker/install_mssql.sh rename to docker/airflow/2/debian11/2.9/scripts/docker/install_mssql.sh diff --git a/docker/airflow/2/debian11/2.8/scripts/docker/install_mysql.sh b/docker/airflow/2/debian11/2.9/scripts/docker/install_mysql.sh similarity index 100% rename from docker/airflow/2/debian11/2.8/scripts/docker/install_mysql.sh rename to docker/airflow/2/debian11/2.9/scripts/docker/install_mysql.sh diff --git a/docker/airflow/2/debian11/2.8/scripts/docker/install_pip_version.sh b/docker/airflow/2/debian11/2.9/scripts/docker/install_pip_version.sh similarity index 100% rename from docker/airflow/2/debian11/2.8/scripts/docker/install_pip_version.sh rename to docker/airflow/2/debian11/2.9/scripts/docker/install_pip_version.sh diff --git a/docker/airflow/2/debian11/2.8/scripts/prod/clean-logs.sh b/docker/airflow/2/debian11/2.9/scripts/prod/clean-logs.sh similarity index 100% rename from docker/airflow/2/debian11/2.8/scripts/prod/clean-logs.sh rename to docker/airflow/2/debian11/2.9/scripts/prod/clean-logs.sh diff --git a/docker/airflow/2/debian11/2.8/scripts/prod/entrypoint_prod.sh b/docker/airflow/2/debian11/2.9/scripts/prod/entrypoint_prod.sh similarity index 100% rename from docker/airflow/2/debian11/2.8/scripts/prod/entrypoint_prod.sh rename to docker/airflow/2/debian11/2.9/scripts/prod/entrypoint_prod.sh diff --git a/docker/airflow/2/debian11/2.8/source_code.txt b/docker/airflow/2/debian11/2.9/source_code.txt similarity index 100% rename from docker/airflow/2/debian11/2.8/source_code.txt rename to docker/airflow/2/debian11/2.9/source_code.txt diff --git a/docker/airflow/versions.yaml b/docker/airflow/versions.yaml index d5df139c0d..4a734ad76e 100644 --- a/docker/airflow/versions.yaml +++ b/docker/airflow/versions.yaml @@ -16,11 +16,11 @@ cloudbuild: enable_parallel: false versions: -- dir: 2/debian11/2.8 +- dir: 2/debian11/2.10 from: marketplace.gcr.io/google/c2d-debian11 packages: airflow: - version: 2.8.2 + version: 2.10.0 pip: version: 22.3.1 gosu: @@ -28,18 +28,18 @@ versions: version: '1.17' repo: airflow2 tags: - - 2.8.2-debian11 - - 2.8-debian11 + - 2.10.0-debian11 + - 2.10-debian11 - 2-debian11 - - 2.8.2 - - '2.8' + - 2.10.0 + - '2.10' - '2' - latest -- dir: 2/debian11/2.6 +- dir: 2/debian11/2.9 from: marketplace.gcr.io/google/c2d-debian11 packages: airflow: - version: 2.6.3 + version: 2.9.3 gosu: gpg: B42F6819007F00F88E364FD4036A9C25BF357DD4 version: '1.17' @@ -47,7 +47,7 @@ versions: version: 22.3.1 repo: airflow2 tags: - - 2.6.3-debian11 - - 2.6-debian11 - - 2.6.3 - - '2.6' + - 2.9.3-debian11 + - 2.9-debian11 + - 2.9.3 + - '2.9' From 6c4c1c2a3accc1c7c3a9a4f47137230108b2a576 Mon Sep 17 00:00:00 2001 From: Adrian Manikowski Date: Wed, 21 Aug 2024 07:43:45 +0000 Subject: [PATCH 2/4] Docker - Airflow - version updated to 2.10.0 and 2.9.3 --- docker/airflow/tests/functional_tests/running_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/airflow/tests/functional_tests/running_test.yaml b/docker/airflow/tests/functional_tests/running_test.yaml index 5cc2a4119d..639d7819ce 100644 --- a/docker/airflow/tests/functional_tests/running_test.yaml +++ b/docker/airflow/tests/functional_tests/running_test.yaml @@ -31,7 +31,7 @@ setup: - 'MYSQL_ROOT_PASSWORD=some-password' - '-d' - 'marketplace.gcr.io/google/postgresql13' -- command: [sleep, '20s'] +- command: [sleep, '70s'] - command: - 'docker' - 'run' From fce982c0f7efbbbbe9f712386e72dd21b844de9b Mon Sep 17 00:00:00 2001 From: Adrian Manikowski Date: Wed, 21 Aug 2024 08:20:02 +0000 Subject: [PATCH 3/4] Docker - Airflow - version updated to 2.10.0 and 2.9.3 --- docker/airflow/tests/functional_tests/running_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/airflow/tests/functional_tests/running_test.yaml b/docker/airflow/tests/functional_tests/running_test.yaml index 639d7819ce..230ac48232 100644 --- a/docker/airflow/tests/functional_tests/running_test.yaml +++ b/docker/airflow/tests/functional_tests/running_test.yaml @@ -31,7 +31,7 @@ setup: - 'MYSQL_ROOT_PASSWORD=some-password' - '-d' - 'marketplace.gcr.io/google/postgresql13' -- command: [sleep, '70s'] +- command: [sleep, '120s'] - command: - 'docker' - 'run' @@ -54,7 +54,7 @@ setup: - '-d' - '$IMAGE' - 'webserver' -- command: [sleep, '70s'] +- command: [sleep, '120s'] teardown: - command: [docker, stop, some-airflow-$UNIQUE-id, some-postgres-$UNIQUE-id] From c3cbf8b9f54655c38f5021590a0ad5c6ab910b02 Mon Sep 17 00:00:00 2001 From: Adrian Manikowski Date: Wed, 4 Sep 2024 08:12:49 +0000 Subject: [PATCH 4/4] Docker - Airflow - version updated to 2.10.0 and 2.9.3 --- .../2/{debian11 => debian12}/2.10/Dockerfile | 22 +++++++++++-------- .../2.10/components.csv | 0 .../2.10/scripts/docker/install_airflow.sh | 0 .../2.10/scripts/docker/install_mssql.sh | 0 .../2.10}/scripts/docker/install_mysql.sh | 2 +- .../scripts/docker/install_pip_version.sh | 4 ++-- .../2.10/scripts/prod/clean-logs.sh | 0 .../2.10/scripts/prod/entrypoint_prod.sh | 0 .../2.10/source_code.txt | 0 .../2/{debian11 => debian12}/2.9/Dockerfile | 22 +++++++++++-------- .../{debian11 => debian12}/2.9/components.csv | 0 .../2.9/scripts/docker/install_airflow.sh | 0 .../2.9/scripts/docker/install_mssql.sh | 0 .../2.9}/scripts/docker/install_mysql.sh | 2 +- .../scripts/docker/install_pip_version.sh | 4 ++-- .../2.9/scripts/prod/clean-logs.sh | 0 .../2.9/scripts/prod/entrypoint_prod.sh | 0 .../2.9/source_code.txt | 0 docker/airflow/templates/Dockerfile.template | 16 +++++++++----- .../templates/scripts/docker/install_mysql.sh | 2 +- .../scripts/docker/install_pip_version.sh | 4 ++-- docker/airflow/versions.yaml | 18 +++++++-------- 22 files changed, 54 insertions(+), 42 deletions(-) rename docker/airflow/2/{debian11 => debian12}/2.10/Dockerfile (96%) rename docker/airflow/2/{debian11 => debian12}/2.10/components.csv (100%) rename docker/airflow/2/{debian11 => debian12}/2.10/scripts/docker/install_airflow.sh (100%) rename docker/airflow/2/{debian11 => debian12}/2.10/scripts/docker/install_mssql.sh (100%) rename docker/airflow/2/{debian11/2.9 => debian12/2.10}/scripts/docker/install_mysql.sh (91%) rename docker/airflow/2/{debian11/2.9 => debian12/2.10}/scripts/docker/install_pip_version.sh (77%) rename docker/airflow/2/{debian11 => debian12}/2.10/scripts/prod/clean-logs.sh (100%) rename docker/airflow/2/{debian11 => debian12}/2.10/scripts/prod/entrypoint_prod.sh (100%) rename docker/airflow/2/{debian11 => debian12}/2.10/source_code.txt (100%) rename docker/airflow/2/{debian11 => debian12}/2.9/Dockerfile (96%) rename docker/airflow/2/{debian11 => debian12}/2.9/components.csv (100%) rename docker/airflow/2/{debian11 => debian12}/2.9/scripts/docker/install_airflow.sh (100%) rename docker/airflow/2/{debian11 => debian12}/2.9/scripts/docker/install_mssql.sh (100%) rename docker/airflow/2/{debian11/2.10 => debian12/2.9}/scripts/docker/install_mysql.sh (91%) rename docker/airflow/2/{debian11/2.10 => debian12/2.9}/scripts/docker/install_pip_version.sh (77%) rename docker/airflow/2/{debian11 => debian12}/2.9/scripts/prod/clean-logs.sh (100%) rename docker/airflow/2/{debian11 => debian12}/2.9/scripts/prod/entrypoint_prod.sh (100%) rename docker/airflow/2/{debian11 => debian12}/2.9/source_code.txt (100%) diff --git a/docker/airflow/2/debian11/2.10/Dockerfile b/docker/airflow/2/debian12/2.10/Dockerfile similarity index 96% rename from docker/airflow/2/debian11/2.10/Dockerfile rename to docker/airflow/2/debian12/2.10/Dockerfile index c4b42f145b..ed50d79a29 100644 --- a/docker/airflow/2/debian11/2.10/Dockerfile +++ b/docker/airflow/2/debian12/2.10/Dockerfile @@ -7,13 +7,13 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow ARG AIRFLOW_PIP_VERSION="22.3.1" -FROM marketplace.gcr.io/google/c2d-debian11 as ospo +FROM marketplace.gcr.io/google/debian12 as ospo # Download Licenses and restricted source-code COPY components.csv /components.csv COPY source_code.txt /source_code.txt -RUN apt update && apt -y install ca-certificates +RUN apt update && apt -y install ca-certificates curl RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \ && curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \ @@ -25,7 +25,7 @@ RUN mkdir -p /usr/src/licenses \ && /download-ref-repos.sh /source_code.txt /usr/src -FROM marketplace.gcr.io/google/c2d-debian11 as airflow-build-image +FROM marketplace.gcr.io/google/debian12 as airflow-build-image SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ @@ -59,6 +59,7 @@ ENV DEV_APT_DEPS="\ python3 \ python3-pip \ python3-dev \ + python3-venv \ sasl2-bin \ software-properties-common \ sqlite3 \ @@ -119,6 +120,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw USER airflow +RUN python3 -m venv ~/venv + +ENV PATH="/home/airflow/venv/bin:$PATH" + COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/ RUN /scripts/docker/install_pip_version.sh @@ -136,7 +141,7 @@ RUN /scripts/docker/install_airflow.sh \ && find "${AIRFLOW_USER_HOME_DIR}/.local" -executable -print0 | xargs --null chmod g+x \ && find "${AIRFLOW_USER_HOME_DIR}/.local" -print0 | xargs --null chmod g+rw -FROM marketplace.gcr.io/google/c2d-debian11 as main +FROM marketplace.gcr.io/google/debian12 as main COPY --from=ospo /usr/src /usr/src @@ -162,18 +167,17 @@ ENV RUNTIME_APT_DEPS="\ freetds-bin \ krb5-user \ ldap-utils \ - libffi7 \ - libldap-2.4-2 \ + libffi8 \ + libldap-common \ libsasl2-2 \ libsasl2-modules \ - libssl1.1 \ + libssl3 \ locales \ lsb-release \ - netcat \ + netcat-traditional \ openssh-client \ pkg-config \ postgresql-client \ - python2 \ python3 \ python3-pip \ rsync \ diff --git a/docker/airflow/2/debian11/2.10/components.csv b/docker/airflow/2/debian12/2.10/components.csv similarity index 100% rename from docker/airflow/2/debian11/2.10/components.csv rename to docker/airflow/2/debian12/2.10/components.csv diff --git a/docker/airflow/2/debian11/2.10/scripts/docker/install_airflow.sh b/docker/airflow/2/debian12/2.10/scripts/docker/install_airflow.sh similarity index 100% rename from docker/airflow/2/debian11/2.10/scripts/docker/install_airflow.sh rename to docker/airflow/2/debian12/2.10/scripts/docker/install_airflow.sh diff --git a/docker/airflow/2/debian11/2.10/scripts/docker/install_mssql.sh b/docker/airflow/2/debian12/2.10/scripts/docker/install_mssql.sh similarity index 100% rename from docker/airflow/2/debian11/2.10/scripts/docker/install_mssql.sh rename to docker/airflow/2/debian12/2.10/scripts/docker/install_mssql.sh diff --git a/docker/airflow/2/debian11/2.9/scripts/docker/install_mysql.sh b/docker/airflow/2/debian12/2.10/scripts/docker/install_mysql.sh similarity index 91% rename from docker/airflow/2/debian11/2.9/scripts/docker/install_mysql.sh rename to docker/airflow/2/debian12/2.10/scripts/docker/install_mysql.sh index 3a37bc93a0..9581a6e63a 100755 --- a/docker/airflow/2/debian11/2.9/scripts/docker/install_mysql.sh +++ b/docker/airflow/2/debian12/2.10/scripts/docker/install_mysql.sh @@ -40,7 +40,7 @@ install_mysql_client() { gpgconf --kill all rm -rf "${GNUPGHOME}" unset GNUPGHOME - echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list + echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge diff --git a/docker/airflow/2/debian11/2.9/scripts/docker/install_pip_version.sh b/docker/airflow/2/debian12/2.10/scripts/docker/install_pip_version.sh similarity index 77% rename from docker/airflow/2/debian11/2.9/scripts/docker/install_pip_version.sh rename to docker/airflow/2/debian12/2.10/scripts/docker/install_pip_version.sh index ba15c70501..3d2f6e7a5c 100755 --- a/docker/airflow/2/debian11/2.9/scripts/docker/install_pip_version.sh +++ b/docker/airflow/2/debian12/2.10/scripts/docker/install_pip_version.sh @@ -6,8 +6,8 @@ function install_pip_version() { echo echo "Installing pip version ${AIRFLOW_PIP_VERSION}" echo - pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && - mkdir -p ${HOME}/.local/bin + pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" + mkdir -p ${HOME}/.local/bin } install_pip_version diff --git a/docker/airflow/2/debian11/2.10/scripts/prod/clean-logs.sh b/docker/airflow/2/debian12/2.10/scripts/prod/clean-logs.sh similarity index 100% rename from docker/airflow/2/debian11/2.10/scripts/prod/clean-logs.sh rename to docker/airflow/2/debian12/2.10/scripts/prod/clean-logs.sh diff --git a/docker/airflow/2/debian11/2.10/scripts/prod/entrypoint_prod.sh b/docker/airflow/2/debian12/2.10/scripts/prod/entrypoint_prod.sh similarity index 100% rename from docker/airflow/2/debian11/2.10/scripts/prod/entrypoint_prod.sh rename to docker/airflow/2/debian12/2.10/scripts/prod/entrypoint_prod.sh diff --git a/docker/airflow/2/debian11/2.10/source_code.txt b/docker/airflow/2/debian12/2.10/source_code.txt similarity index 100% rename from docker/airflow/2/debian11/2.10/source_code.txt rename to docker/airflow/2/debian12/2.10/source_code.txt diff --git a/docker/airflow/2/debian11/2.9/Dockerfile b/docker/airflow/2/debian12/2.9/Dockerfile similarity index 96% rename from docker/airflow/2/debian11/2.9/Dockerfile rename to docker/airflow/2/debian12/2.9/Dockerfile index 322a1ab94b..63ff55710f 100644 --- a/docker/airflow/2/debian11/2.9/Dockerfile +++ b/docker/airflow/2/debian12/2.9/Dockerfile @@ -7,13 +7,13 @@ ARG AIRFLOW_USER_HOME_DIR=/home/airflow ARG AIRFLOW_PIP_VERSION="22.3.1" -FROM marketplace.gcr.io/google/c2d-debian11 as ospo +FROM marketplace.gcr.io/google/debian12 as ospo # Download Licenses and restricted source-code COPY components.csv /components.csv COPY source_code.txt /source_code.txt -RUN apt update && apt -y install ca-certificates +RUN apt update && apt -y install ca-certificates curl RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \ && curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \ @@ -25,7 +25,7 @@ RUN mkdir -p /usr/src/licenses \ && /download-ref-repos.sh /source_code.txt /usr/src -FROM marketplace.gcr.io/google/c2d-debian11 as airflow-build-image +FROM marketplace.gcr.io/google/debian12 as airflow-build-image SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ @@ -59,6 +59,7 @@ ENV DEV_APT_DEPS="\ python3 \ python3-pip \ python3-dev \ + python3-venv \ sasl2-bin \ software-properties-common \ sqlite3 \ @@ -119,6 +120,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw USER airflow +RUN python3 -m venv ~/venv + +ENV PATH="/home/airflow/venv/bin:$PATH" + COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/ RUN /scripts/docker/install_pip_version.sh @@ -136,7 +141,7 @@ RUN /scripts/docker/install_airflow.sh \ && find "${AIRFLOW_USER_HOME_DIR}/.local" -executable -print0 | xargs --null chmod g+x \ && find "${AIRFLOW_USER_HOME_DIR}/.local" -print0 | xargs --null chmod g+rw -FROM marketplace.gcr.io/google/c2d-debian11 as main +FROM marketplace.gcr.io/google/debian12 as main COPY --from=ospo /usr/src /usr/src @@ -162,18 +167,17 @@ ENV RUNTIME_APT_DEPS="\ freetds-bin \ krb5-user \ ldap-utils \ - libffi7 \ - libldap-2.4-2 \ + libffi8 \ + libldap-common \ libsasl2-2 \ libsasl2-modules \ - libssl1.1 \ + libssl3 \ locales \ lsb-release \ - netcat \ + netcat-traditional \ openssh-client \ pkg-config \ postgresql-client \ - python2 \ python3 \ python3-pip \ rsync \ diff --git a/docker/airflow/2/debian11/2.9/components.csv b/docker/airflow/2/debian12/2.9/components.csv similarity index 100% rename from docker/airflow/2/debian11/2.9/components.csv rename to docker/airflow/2/debian12/2.9/components.csv diff --git a/docker/airflow/2/debian11/2.9/scripts/docker/install_airflow.sh b/docker/airflow/2/debian12/2.9/scripts/docker/install_airflow.sh similarity index 100% rename from docker/airflow/2/debian11/2.9/scripts/docker/install_airflow.sh rename to docker/airflow/2/debian12/2.9/scripts/docker/install_airflow.sh diff --git a/docker/airflow/2/debian11/2.9/scripts/docker/install_mssql.sh b/docker/airflow/2/debian12/2.9/scripts/docker/install_mssql.sh similarity index 100% rename from docker/airflow/2/debian11/2.9/scripts/docker/install_mssql.sh rename to docker/airflow/2/debian12/2.9/scripts/docker/install_mssql.sh diff --git a/docker/airflow/2/debian11/2.10/scripts/docker/install_mysql.sh b/docker/airflow/2/debian12/2.9/scripts/docker/install_mysql.sh similarity index 91% rename from docker/airflow/2/debian11/2.10/scripts/docker/install_mysql.sh rename to docker/airflow/2/debian12/2.9/scripts/docker/install_mysql.sh index 3a37bc93a0..9581a6e63a 100755 --- a/docker/airflow/2/debian11/2.10/scripts/docker/install_mysql.sh +++ b/docker/airflow/2/debian12/2.9/scripts/docker/install_mysql.sh @@ -40,7 +40,7 @@ install_mysql_client() { gpgconf --kill all rm -rf "${GNUPGHOME}" unset GNUPGHOME - echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list + echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge diff --git a/docker/airflow/2/debian11/2.10/scripts/docker/install_pip_version.sh b/docker/airflow/2/debian12/2.9/scripts/docker/install_pip_version.sh similarity index 77% rename from docker/airflow/2/debian11/2.10/scripts/docker/install_pip_version.sh rename to docker/airflow/2/debian12/2.9/scripts/docker/install_pip_version.sh index ba15c70501..3d2f6e7a5c 100755 --- a/docker/airflow/2/debian11/2.10/scripts/docker/install_pip_version.sh +++ b/docker/airflow/2/debian12/2.9/scripts/docker/install_pip_version.sh @@ -6,8 +6,8 @@ function install_pip_version() { echo echo "Installing pip version ${AIRFLOW_PIP_VERSION}" echo - pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && - mkdir -p ${HOME}/.local/bin + pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" + mkdir -p ${HOME}/.local/bin } install_pip_version diff --git a/docker/airflow/2/debian11/2.9/scripts/prod/clean-logs.sh b/docker/airflow/2/debian12/2.9/scripts/prod/clean-logs.sh similarity index 100% rename from docker/airflow/2/debian11/2.9/scripts/prod/clean-logs.sh rename to docker/airflow/2/debian12/2.9/scripts/prod/clean-logs.sh diff --git a/docker/airflow/2/debian11/2.9/scripts/prod/entrypoint_prod.sh b/docker/airflow/2/debian12/2.9/scripts/prod/entrypoint_prod.sh similarity index 100% rename from docker/airflow/2/debian11/2.9/scripts/prod/entrypoint_prod.sh rename to docker/airflow/2/debian12/2.9/scripts/prod/entrypoint_prod.sh diff --git a/docker/airflow/2/debian11/2.9/source_code.txt b/docker/airflow/2/debian12/2.9/source_code.txt similarity index 100% rename from docker/airflow/2/debian11/2.9/source_code.txt rename to docker/airflow/2/debian12/2.9/source_code.txt diff --git a/docker/airflow/templates/Dockerfile.template b/docker/airflow/templates/Dockerfile.template index 28adde8d0e..a7f33e416b 100644 --- a/docker/airflow/templates/Dockerfile.template +++ b/docker/airflow/templates/Dockerfile.template @@ -17,7 +17,7 @@ FROM {{ .From }} as ospo COPY components.csv /components.csv COPY source_code.txt /source_code.txt -RUN apt update && apt -y install ca-certificates +RUN apt update && apt -y install ca-certificates curl RUN curl -o /download-licenses.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-licenses.sh \ && curl -o /download-ref-repos.sh -L https://raw.githubusercontent.com/GoogleCloudPlatform/click-to-deploy/master/scripts/download-ref-repos.sh \ @@ -63,6 +63,7 @@ ENV DEV_APT_DEPS="\ python3 \ python3-pip \ python3-dev \ + python3-venv \ sasl2-bin \ software-properties-common \ sqlite3 \ @@ -123,6 +124,10 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw USER airflow +RUN python3 -m venv ~/venv + +ENV PATH="/home/airflow/venv/bin:$PATH" + COPY --chown=airflow:0 scripts/docker/install_pip_version.sh /scripts/docker/ RUN /scripts/docker/install_pip_version.sh @@ -166,18 +171,17 @@ ENV RUNTIME_APT_DEPS="\ freetds-bin \ krb5-user \ ldap-utils \ - libffi7 \ - libldap-2.4-2 \ + libffi8 \ + libldap-common \ libsasl2-2 \ libsasl2-modules \ - libssl1.1 \ + libssl3 \ locales \ lsb-release \ - netcat \ + netcat-traditional \ openssh-client \ pkg-config \ postgresql-client \ - python2 \ python3 \ python3-pip \ rsync \ diff --git a/docker/airflow/templates/scripts/docker/install_mysql.sh b/docker/airflow/templates/scripts/docker/install_mysql.sh index 3a37bc93a0..9581a6e63a 100755 --- a/docker/airflow/templates/scripts/docker/install_mysql.sh +++ b/docker/airflow/templates/scripts/docker/install_mysql.sh @@ -40,7 +40,7 @@ install_mysql_client() { gpgconf --kill all rm -rf "${GNUPGHOME}" unset GNUPGHOME - echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list + echo "deb http://repo.mysql.com/apt/debian/ bookworm mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list apt-get update apt-get install --no-install-recommends -y "${packages[@]}" apt-get autoremove -yqq --purge diff --git a/docker/airflow/templates/scripts/docker/install_pip_version.sh b/docker/airflow/templates/scripts/docker/install_pip_version.sh index ba15c70501..3d2f6e7a5c 100755 --- a/docker/airflow/templates/scripts/docker/install_pip_version.sh +++ b/docker/airflow/templates/scripts/docker/install_pip_version.sh @@ -6,8 +6,8 @@ function install_pip_version() { echo echo "Installing pip version ${AIRFLOW_PIP_VERSION}" echo - pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && - mkdir -p ${HOME}/.local/bin + pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" + mkdir -p ${HOME}/.local/bin } install_pip_version diff --git a/docker/airflow/versions.yaml b/docker/airflow/versions.yaml index 4a734ad76e..7df2daaffe 100644 --- a/docker/airflow/versions.yaml +++ b/docker/airflow/versions.yaml @@ -16,8 +16,8 @@ cloudbuild: enable_parallel: false versions: -- dir: 2/debian11/2.10 - from: marketplace.gcr.io/google/c2d-debian11 +- dir: 2/debian12/2.10 + from: marketplace.gcr.io/google/debian12 packages: airflow: version: 2.10.0 @@ -28,15 +28,15 @@ versions: version: '1.17' repo: airflow2 tags: - - 2.10.0-debian11 - - 2.10-debian11 - - 2-debian11 + - 2.10.0-debian12 + - 2.10-debian12 + - 2-debian12 - 2.10.0 - '2.10' - '2' - latest -- dir: 2/debian11/2.9 - from: marketplace.gcr.io/google/c2d-debian11 +- dir: 2/debian12/2.9 + from: marketplace.gcr.io/google/debian12 packages: airflow: version: 2.9.3 @@ -47,7 +47,7 @@ versions: version: 22.3.1 repo: airflow2 tags: - - 2.9.3-debian11 - - 2.9-debian11 + - 2.9.3-debian12 + - 2.9-debian12 - 2.9.3 - '2.9'