From 99844f79fa04da616a05073076ef5e6379784a02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thibaut=20Barr=C3=A8re?= Date: Wed, 9 Feb 2022 16:09:32 +0100 Subject: [PATCH 1/3] =?UTF-8?q?Mises=20=C3=A0=20jour=20techniques=20(Elixi?= =?UTF-8?q?r,=20Erlang,=20NVM,=20NodeJS)=20(#33)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Make sure the docker hub link is closer from the image * Update to most recent elixir/erlang couple * Bump NVM to latest https://github.com/nvm-sh/nvm/releases * Bump NodeJS to latest LTS available https://nodejs.org * Try to fix the build * Update test for NodeJS * Update test for Elixir version * Modify script to make it work with newer format * Add readme test to verify OTP version in full (improved) * Update GitHub action to assert on full version * Now use full version specifier * Add explanation about how the hex images are built * Add very useful help to see output of commands locally The CI already does that by default. * Upgrade packages (except kernel) for security See https://github.com/etalab/transport_deploy/issues/46 for context * Bump to latest * Bump Node to latest LTS * Bump tests --- .github/workflows/docker.yml | 10 ++++----- README.md | 7 ++++-- Rakefile | 2 +- transport-site/Dockerfile | 41 +++++++++++++++++++++++++++++------- 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6ee392a..8f1d832 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,9 +17,9 @@ env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} TEST_TAG: ${{ github.repository }}:test - TEST_EXPECTED_NODE_OUTPUT: "v14.16.1" - TEST_EXPECTED_ELIXIR_OUTPUT: "Elixir 1.12.2 (compiled with Erlang/OTP 24)" - TEST_EXPECTED_ERLANG_OUTPUT: "Erlang/OTP 24" + TEST_EXPECTED_NODE_OUTPUT: "v16.14.0" + TEST_EXPECTED_ELIXIR_OUTPUT: "Elixir 1.13.2 (compiled with Erlang/OTP 24)" + TEST_EXPECTED_ERLANG_OUTPUT: "24.2.1" jobs: build-and-push-image: runs-on: ubuntu-latest @@ -66,8 +66,8 @@ jobs: - name: Test that Elixir can start and has expected version run: docker run --rm ${{ env.TEST_TAG }} /bin/bash -c 'elixir --version' | grep '${{ env.TEST_EXPECTED_ELIXIR_OUTPUT }}' - - name: Test that Erlang can start and has (major) expected version - run: docker run --rm ${{ env.TEST_TAG }} /bin/bash -c "erl -noshell -eval 'erlang:display(erlang:system_info(system_version))' -eval 'init:stop()'" | grep '${{ env.TEST_EXPECTED_ERLANG_OUTPUT }}' + - name: Test that Erlang can start and has expected version (major + minor + optional revision number) + run: docker run --rm ${{ env.TEST_TAG }} /bin/bash -c "erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), \"releases\", erlang:system_info(otp_release), \"OTP_VERSION\"])), io:fwrite(Version), halt().' -noshell" | grep '${{ env.TEST_EXPECTED_ERLANG_OUTPUT }}' # If we reach this point, we consider the tested imaged is OK, so we can extract the metadat & publish # https://github.com/docker/metadata-action diff --git a/README.md b/README.md index cf2a43f..fb00f40 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ As a work-around for [#17](https://github.com/etalab/transport-ops/issues/17): ``` IMAGE_VERSION=$(rake get_image_version) IMAGE_NAME=betagouv/transport:$IMAGE_VERSION -docker build transport-site --no-cache -t $IMAGE_NAME +docker build transport-site --no-cache -t $IMAGE_NAME --progress=plain ``` * Carefully verify the versions (this will be translated into a testing script later): @@ -72,7 +72,10 @@ docker build transport-site --no-cache -t $IMAGE_NAME ``` docker run -it --rm $IMAGE_NAME /bin/bash -c 'node --version' docker run -it --rm $IMAGE_NAME /bin/bash -c 'elixir --version' +# only major docker run -it --rm $IMAGE_NAME /bin/bash -c "erl -noshell -eval 'erlang:display(erlang:system_info(system_version))' -eval 'init:stop()'" +# full version (https://stackoverflow.com/a/34326368) +docker run -it --rm $IMAGE_NME /bin/bash -c "erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), \"releases\", erlang:system_info(otp_release), \"OTP_VERSION\"])), io:fwrite(Version), halt().' -noshell" ``` * Read the [docker push documentation](https://docs.docker.com/engine/reference/commandline/push/) @@ -88,7 +91,7 @@ Before creating a tag, the following commands can be used to verify the versions ``` cd transport-site -docker build . -t test:latest +docker build . -t test:latest --progress=plain docker run -it --rm test:latest /bin/bash -c 'node --version' docker run -it --rm test:latest /bin/bash -c 'elixir --version' docker run -it --rm test:latest /bin/bash -c "erl -noshell -eval 'erlang:display(erlang:system_info(system_version))' -eval 'init:stop()'" diff --git a/Rakefile b/Rakefile index 6265732..c03ebe2 100644 --- a/Rakefile +++ b/Rakefile @@ -1,6 +1,6 @@ task :get_image_version do version = IO.read("transport-site/Dockerfile")[/FROM (hexpm\/elixir.*)/, 1] version = version.gsub('hexpm/elixir:','elixir-') - fail "Unexpected FROM format, script must be verified" unless version =~ /\Aelixir\-[^\-]+\-erlang\-[^\-]+\-alpine\-[^\-]+\z/ + fail "Unexpected FROM value (got #{version}), script must be adapted?" unless version =~ /\Aelixir\-[^\-]+\-erlang\-[^\-]+\-ubuntu\-focal\-[^\-]+\z/ puts version end diff --git a/transport-site/Dockerfile b/transport-site/Dockerfile index a2f2c36..ce42515 100644 --- a/transport-site/Dockerfile +++ b/transport-site/Dockerfile @@ -1,7 +1,14 @@ -# We leverage the base images published by hexpm. +# We are interested in the binaries compiled on that container: +FROM ghcr.io/etalab/transport-tools:v1.0.2 as transport-tools + +# We leverage the base images published by hexpm at: # # https://hub.docker.com/r/hexpm/elixir # +# Which are built via: +# +# https://github.com/hexpm/bob#docker-images +# # They provide the ability to decouple Elixir version # and OTP version, which is a nice feature for # incremental/decoupled upgrades. @@ -10,11 +17,11 @@ # - https://github.com/elixir-lang/elixir/releases # - https://github.com/erlang/otp/releases # - -# We are interested in the binaries compiled on that container: -FROM ghcr.io/etalab/transport-tools:v1.0.2 as transport-tools - -FROM hexpm/elixir:1.12.2-erlang-24.0.4-ubuntu-focal-20210325 +# +# So again, to upgrade this, check out : +# +# https://hub.docker.com/r/hexpm/elixir +FROM hexpm/elixir:1.13.2-erlang-24.2.1-ubuntu-focal-20211006 ARG DEBIAN_FRONTEND=noninteractive ENV TZ=Europe/Paris @@ -27,10 +34,28 @@ RUN apt-get update && apt-get install -y \ tzdata \ zip -ENV NVM_VERSION v0.29.0 -ENV NODE_VERSION 14.16.1 +# Helps bump the output of /etc/os-release from says "Ubuntu 24.04.2 LTS" to "... 24.04.3" +# +# The source image (hex) is itself based on a ubuntu image whose +# packages are not necessarily up-to-date all the time. We want to +# upgrade and ensure we are as up-to-date as possible. +# Note that the kernel itself cannot be upgraded here apparently +# (https://stackoverflow.com/a/66413248) +# +# See https://github.com/etalab/transport_deploy/issues/46 for more context +RUN apt-get upgrade -y + +# debugging information +RUN uname --all +RUN cat /etc/os-release +RUN cat /etc/lsb-release + +ENV NVM_VERSION v0.39.1 +ENV NODE_VERSION 16.14.0 ENV NVM_DIR $HOME/.nvm +RUN mkdir $NVM_DIR + # Install NVM RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/${NVM_VERSION}/install.sh | bash From 531f5ffef16e6b8425aa6d950fd8fa50b930f478 Mon Sep 17 00:00:00 2001 From: Antoine Augusti Date: Wed, 23 Feb 2022 16:15:06 +0100 Subject: [PATCH 2/3] Elixir 1.12.2 et transport-tools 1.0.3 --- .github/workflows/docker.yml | 12 ++++++------ transport-site/Dockerfile | 20 ++++++++++++-------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8f1d832..70e80ae 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,9 +17,9 @@ env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} TEST_TAG: ${{ github.repository }}:test - TEST_EXPECTED_NODE_OUTPUT: "v16.14.0" - TEST_EXPECTED_ELIXIR_OUTPUT: "Elixir 1.13.2 (compiled with Erlang/OTP 24)" - TEST_EXPECTED_ERLANG_OUTPUT: "24.2.1" + TEST_EXPECTED_NODE_OUTPUT: "v14.16.1" + TEST_EXPECTED_ELIXIR_OUTPUT: "Elixir 1.12.2 (compiled with Erlang/OTP 24)" + TEST_EXPECTED_ERLANG_OUTPUT: "24.0.4" jobs: build-and-push-image: runs-on: ubuntu-latest @@ -55,14 +55,14 @@ jobs: # --output=type=docker to enforce local work # https://github.com/docker/buildx/blob/master/docs/reference/buildx_build.md#load # https://github.com/docker/buildx/blob/master/docs/reference/buildx_build.md#docker - load: true + load: true push: false # the default, but added for clarity tags: ${{ env.TEST_TAG }} # NOTE: the following tests rely on grep exit code (0 if match found) - name: Test that Node can start and has expected version run: docker run --rm ${{ env.TEST_TAG }} /bin/bash -c 'node --version' | grep '${{ env.TEST_EXPECTED_NODE_OUTPUT }}' - + - name: Test that Elixir can start and has expected version run: docker run --rm ${{ env.TEST_TAG }} /bin/bash -c 'elixir --version' | grep '${{ env.TEST_EXPECTED_ELIXIR_OUTPUT }}' @@ -89,4 +89,4 @@ jobs: labels: ${{ steps.meta.outputs.labels }} # TODO: consider caching - # - https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md \ No newline at end of file + # - https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md diff --git a/transport-site/Dockerfile b/transport-site/Dockerfile index ce42515..08a587a 100644 --- a/transport-site/Dockerfile +++ b/transport-site/Dockerfile @@ -1,5 +1,5 @@ # We are interested in the binaries compiled on that container: -FROM ghcr.io/etalab/transport-tools:v1.0.2 as transport-tools +FROM ghcr.io/etalab/transport-tools:v1.0.3 as transport-tools # We leverage the base images published by hexpm at: # @@ -8,7 +8,7 @@ FROM ghcr.io/etalab/transport-tools:v1.0.2 as transport-tools # Which are built via: # # https://github.com/hexpm/bob#docker-images -# +# # They provide the ability to decouple Elixir version # and OTP version, which is a nice feature for # incremental/decoupled upgrades. @@ -18,10 +18,10 @@ FROM ghcr.io/etalab/transport-tools:v1.0.2 as transport-tools # - https://github.com/erlang/otp/releases # # -# So again, to upgrade this, check out : +# So again, to upgrade this, check out : # # https://hub.docker.com/r/hexpm/elixir -FROM hexpm/elixir:1.13.2-erlang-24.2.1-ubuntu-focal-20211006 +FROM hexpm/elixir:1.12.2-erlang-24.0.4-ubuntu-focal-20210325 ARG DEBIAN_FRONTEND=noninteractive ENV TZ=Europe/Paris @@ -32,7 +32,9 @@ RUN apt-get update && apt-get install -y \ libtool \ git \ tzdata \ - zip + zip \ + default-jre \ + maven # Helps bump the output of /etc/os-release from says "Ubuntu 24.04.2 LTS" to "... 24.04.3" # @@ -41,7 +43,7 @@ RUN apt-get update && apt-get install -y \ # upgrade and ensure we are as up-to-date as possible. # Note that the kernel itself cannot be upgraded here apparently # (https://stackoverflow.com/a/66413248) -# +# # See https://github.com/etalab/transport_deploy/issues/46 for more context RUN apt-get upgrade -y @@ -50,8 +52,8 @@ RUN uname --all RUN cat /etc/os-release RUN cat /etc/lsb-release -ENV NVM_VERSION v0.39.1 -ENV NODE_VERSION 16.14.0 +ENV NVM_VERSION v0.29.0 +ENV NODE_VERSION 14.16.1 ENV NVM_DIR $HOME/.nvm RUN mkdir $NVM_DIR @@ -87,3 +89,5 @@ COPY --from=transport-tools /usr/lib/libproj.* /usr/lib/ RUN mkdir /usr/share/proj/ COPY --from=transport-tools /usr/share/proj/ /usr/share/proj/ RUN /transport-tools/gtfs2netexfr --help +COPY --from=transport-tools /usr/local/bin/gtfs-realtime-validator-lib-1.0.0-SNAPSHOT.jar ./transport-tools +RUN java -jar /transport-tools/gtfs-realtime-validator-lib-1.0.0-SNAPSHOT.jar 2>&1 | grep "For batch mode you must provide a path and file name to GTFS data" From e19ba37754ba07b938ab954040275e09ba0e0eff Mon Sep 17 00:00:00 2001 From: Antoine Augusti Date: Wed, 23 Feb 2022 16:37:12 +0100 Subject: [PATCH 3/3] Update rakefile --- Rakefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Rakefile b/Rakefile index c03ebe2..4569d1e 100644 --- a/Rakefile +++ b/Rakefile @@ -1,6 +1,8 @@ task :get_image_version do - version = IO.read("transport-site/Dockerfile")[/FROM (hexpm\/elixir.*)/, 1] + dockerfile_content = IO.read("transport-site/Dockerfile") + version = dockerfile_content[/FROM (hexpm\/elixir.*)/, 1] version = version.gsub('hexpm/elixir:','elixir-') + tools_version = dockerfile_content[/transport-tools:v(\d+\.\d+\.\d+) as transport-tools/, 1] fail "Unexpected FROM value (got #{version}), script must be adapted?" unless version =~ /\Aelixir\-[^\-]+\-erlang\-[^\-]+\-ubuntu\-focal\-[^\-]+\z/ - puts version + puts version + "-transport-tools-" + tools_version end