From 50177a83ae4713597b9c957472326837ce7530e4 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 2 May 2024 14:37:50 +0900 Subject: [PATCH] Support reproducible builds (except packages) See docker-library/official-images issue 16044 - `ARG SOURCE_DATE_EPOCH` is added. The argument value is consumed by the build scripts to make the binary reproducible. - `/tmp/*` is removed as they contain files created by `memcached` - For Debian, `/var/log/*` is removed as they contain timestamps - For Debian, `/var/cache/ldconfig/aux-cache` is removed as they contain inode numbers, etc. - For Alpine, virtual package versions are pinned to "0" to eliminate the timestamp-based version numbers that appear in `/etc/apk/world` and `/lib/apk/db/installed` > [!NOTE] > The following topics are NOT covered by this commit: > > - To reproduce file timestamps in layers, BuildKit has to be executed with > `--output type=,rewrite-timestamp=true`. > Needs BuildKit v0.13 or later. > > - To reproduce the base image by the hash, reproducers may: > - modify the `FROM` instruction in Dockerfile manually > - or, use the `CONVERT` action of source policies to replace the base image. > > > - To reproduce packages, see the `RUN` instruction hook proposed in > moby/buildkit issue 4576 Signed-off-by: Akihiro Suda --- 1/alpine/Dockerfile | 16 +++++++++++++--- 1/debian/Dockerfile | 20 ++++++++++++++++++-- Dockerfile.template | 24 ++++++++++++++++++++---- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/1/alpine/Dockerfile b/1/alpine/Dockerfile index fdba7a2..e9e4822 100644 --- a/1/alpine/Dockerfile +++ b/1/alpine/Dockerfile @@ -6,7 +6,11 @@ FROM alpine:3.20 +# The global SOURCE_DATE_EPOCH is consumed by commands that are not associated with a source artifact +ARG SOURCE_DATE_EPOCH + # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +# On Debian, useradd recognizes SOURCE_DATE_EPOCH to reproduce the "lastchanged" field in /etc/shadow. RUN set -eux; \ addgroup -g 11211 memcache; \ adduser -D -u 11211 -G memcache memcache @@ -20,7 +24,7 @@ ENV MEMCACHED_SHA1 d7857f9bf51abb2d2106fc6c278682d2f5d7471b RUN set -eux; \ \ - apk add --no-cache --virtual .build-deps \ + apk add --no-cache --virtual .build-deps=0 \ ca-certificates \ coreutils \ cyrus-sasl-dev \ @@ -46,6 +50,10 @@ RUN set -eux; \ cd /usr/src/memcached; \ \ gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + SOURCE_DATE_EPOCH="$(find . -type f -exec stat -c '%Y' {} + | sort -nr | head -n1)"; \ + export SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822; \ ./configure \ --build="$gnuArch" \ --enable-extstore \ @@ -69,10 +77,12 @@ RUN set -eux; \ | sort -u \ | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \ )"; \ - apk add --no-network --virtual .memcached-rundeps $runDeps; \ + apk add --no-network --virtual .memcached-rundeps=0 $runDeps; \ apk del --no-network .build-deps; \ \ - memcached -V + memcached -V ;\ +# clean up for reproducibility + rm -rf /tmp/* COPY docker-entrypoint.sh /usr/local/bin/ RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat diff --git a/1/debian/Dockerfile b/1/debian/Dockerfile index 265faa5..a8bad0a 100644 --- a/1/debian/Dockerfile +++ b/1/debian/Dockerfile @@ -6,7 +6,11 @@ FROM debian:bookworm-slim +# The global SOURCE_DATE_EPOCH is consumed by commands that are not associated with a source artifact +ARG SOURCE_DATE_EPOCH + # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +# On Debian, useradd recognizes SOURCE_DATE_EPOCH to reproduce the "lastchanged" field in /etc/shadow. RUN set -eux; \ groupadd --system --gid 11211 memcache; \ useradd --system --gid memcache --uid 11211 memcache @@ -17,7 +21,9 @@ RUN set -eux; \ apt-get install -y --no-install-recommends \ libsasl2-modules \ ; \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/*; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache ENV MEMCACHED_VERSION 1.6.28 ENV MEMCACHED_URL https://memcached.org/files/memcached-1.6.28.tar.gz @@ -41,6 +47,8 @@ RUN set -eux; \ wget \ ; \ rm -rf /var/lib/apt/lists/*; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache; \ \ wget -O memcached.tar.gz "$MEMCACHED_URL"; \ echo "$MEMCACHED_SHA1 memcached.tar.gz" | sha1sum -c -; \ @@ -51,6 +59,10 @@ RUN set -eux; \ cd /usr/src/memcached; \ \ gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + SOURCE_DATE_EPOCH="$(find . -type f -exec stat -c '%Y' {} + | sort -nr | head -n1)"; \ + export SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822; \ ./configure \ --build="$gnuArch" \ --enable-extstore \ @@ -82,8 +94,12 @@ RUN set -eux; \ | xargs -r apt-mark manual \ ; \ apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache; \ \ - memcached -V + memcached -V ;\ +# clean up for reproducibility + rm -rf /tmp/* COPY docker-entrypoint.sh /usr/local/bin/ RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat diff --git a/Dockerfile.template b/Dockerfile.template index e509d4b..c9a2799 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -4,7 +4,11 @@ FROM alpine:{{ .alpine.version }} FROM debian:{{ .debian.version }}-slim {{ ) end -}} +# The global SOURCE_DATE_EPOCH is consumed by commands that are not associated with a source artifact +ARG SOURCE_DATE_EPOCH + # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +# On Debian, useradd recognizes SOURCE_DATE_EPOCH to reproduce the "lastchanged" field in /etc/shadow. {{ if env.variant == "alpine" then ( -}} RUN set -eux; \ addgroup -g 11211 memcache; \ @@ -24,7 +28,9 @@ RUN set -eux; \ apt-get install -y --no-install-recommends \ libsasl2-modules \ ; \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/*; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache {{ ) end -}} ENV MEMCACHED_VERSION {{ .version }} @@ -34,7 +40,7 @@ ENV MEMCACHED_SHA1 {{ .sha1 }} RUN set -eux; \ \ {{ if env.variant == "alpine" then ( -}} - apk add --no-cache --virtual .build-deps \ + apk add --no-cache --virtual .build-deps=0 \ ca-certificates \ coreutils \ cyrus-sasl-dev \ @@ -67,6 +73,8 @@ RUN set -eux; \ wget \ ; \ rm -rf /var/lib/apt/lists/*; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache; \ {{ ) end -}} \ wget -O memcached.tar.gz "$MEMCACHED_URL"; \ @@ -78,6 +86,10 @@ RUN set -eux; \ cd /usr/src/memcached; \ \ gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + SOURCE_DATE_EPOCH="$(find . -type f -exec stat -c '%Y' {} + | sort -nr | head -n1)"; \ + export SOURCE_DATE_EPOCH; \ +# for logging validation/edification + date --date "@$SOURCE_DATE_EPOCH" --rfc-2822; \ ./configure \ --build="$gnuArch" \ --enable-extstore \ @@ -109,7 +121,7 @@ RUN set -eux; \ | sort -u \ | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \ )"; \ - apk add --no-network --virtual .memcached-rundeps $runDeps; \ + apk add --no-network --virtual .memcached-rundeps=0 $runDeps; \ apk del --no-network .build-deps; \ {{ ) else ( -}} apt-mark auto '.*' > /dev/null; \ @@ -123,9 +135,13 @@ RUN set -eux; \ | xargs -r apt-mark manual \ ; \ apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ +# clean up for reproducibility + rm -rf /var/log/* /var/cache/ldconfig/aux-cache; \ {{ ) end -}} \ - memcached -V + memcached -V ;\ +# clean up for reproducibility + rm -rf /tmp/* COPY docker-entrypoint.sh /usr/local/bin/ RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh # backwards compat