diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml index 8d4c2a9682e9..ed707ca357b2 100644 --- a/.github/workflows/.test.yml +++ b/.github/workflows/.test.yml @@ -30,8 +30,6 @@ env: GO_VERSION: "1.20" SETUP_BUILDX_VERSION: "latest" SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" - TESTFLAGS: "-v --parallel=6 --timeout=30m" - GOTESTSUM_FORMAT: "standard-verbose" jobs: prepare: @@ -102,6 +100,11 @@ jobs: runs-on: ubuntu-22.04 needs: - prepare + env: + TESTFLAGS: "-v --parallel=6 --timeout=30m" + GOTESTSUM_FORMAT: "standard-verbose" + TEST_IMAGE_BUILD: "0" + TEST_IMAGE_ID: "buildkit-tests" strategy: fail-fast: false matrix: @@ -140,6 +143,14 @@ jobs: version: ${{ env.SETUP_BUILDX_VERSION }} driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} buildkitd-flags: --debug + - + name: Build test image + uses: docker/bake-action@v4 + with: + targets: integration-tests + set: | + *.cache-from=type=gha,scope=${{ inputs.cache_scope }} + *.output=type=docker,name=${{ env.TEST_IMAGE_ID }} - name: Test continue-on-error: ${{ matrix.tags == 'nydus' }} diff --git a/Makefile b/Makefile index 99b273865d70..cc5035a9787f 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,14 @@ prefix=/usr/local bindir=$(prefix)/bin +ifneq (, $(BUILDX_BIN)) + export BUILDX_CMD = $(BUILDX_BIN) +else ifneq (, $(shell docker buildx version)) + export BUILDX_CMD = docker buildx +else ifneq (, $(shell command -v buildx)) + export BUILDX_CMD = $(command -v buildx) +endif + export BUILDX_CMD ?= docker buildx .PHONY: binaries diff --git a/docker-bake.hcl b/docker-bake.hcl index 89557afaef6d..f9a6906195a9 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -14,6 +14,30 @@ variable "BUILDKITD_TAGS" { default = null } +variable "HTTP_PROXY" { + default = null +} + +variable "HTTPS_PROXY" { + default = null +} + +variable "NO_PROXY" { + default = null +} + +variable "GOBUILDFLAGS" { + default = null +} + +variable "VERIFYFLAGS" { + default = null +} + +variable "CGO_ENABLED" { + default = null +} + # Defines the output folder variable "DESTDIR" { default = "" @@ -28,6 +52,13 @@ target "_common" { ALPINE_VERSION = ALPINE_VERSION GO_VERSION = GO_VERSION NODE_VERSION = NODE_VERSION + BUILDKITD_TAGS = BUILDKITD_TAGS + HTTP_PROXY = HTTP_PROXY + HTTPS_PROXY = HTTPS_PROXY + NO_PROXY = NO_PROXY + GOBUILDFLAGS = GOBUILDFLAGS + VERIFYFLAGS = VERIFYFLAGS + CGO_ENABLED = CGO_ENABLED BUILDKIT_CONTEXT_KEEP_GIT_DIR = 1 } } @@ -39,9 +70,6 @@ group "default" { target "binaries" { inherits = ["_common"] target = "binaries" - args = { - BUILDKITD_TAGS = BUILDKITD_TAGS - } output = [bindir("build")] } @@ -74,6 +102,11 @@ target "integration-tests-base" { output = ["type=cacheonly"] } +target "integration-tests" { + inherits = ["integration-tests-base"] + target = "integration-tests" +} + group "validate" { targets = ["lint", "validate-vendor", "validate-doctoc", "validate-generated-files", "validate-shfmt", "validate-docs"] } diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile index 9106ba47c623..de9323d89440 100644 --- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile +++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile @@ -35,18 +35,4 @@ LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.f COPY --from=build /dockerfile-frontend /bin/dockerfile-frontend ENTRYPOINT ["/bin/dockerfile-frontend"] - -FROM base AS buildid-check -RUN apt-get update && apt-get --no-install-recommends install -y jq -COPY /frontend/dockerfile/cmd/dockerfile-frontend/hack/check-daily-outdated . -COPY --from=r.j3ss.co/reg /usr/bin/reg /bin -COPY --from=build /dockerfile-frontend . -ARG CHANNEL -ARG REPO -ARG DATE -RUN ./check-daily-outdated $CHANNEL $REPO $DATE /out - -FROM scratch AS buildid -COPY --from=buildid-check /out/ / - FROM release diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/check-daily-outdated b/frontend/dockerfile/cmd/dockerfile-frontend/hack/check-daily-outdated deleted file mode 100755 index ed7f3196d84d..000000000000 --- a/frontend/dockerfile/cmd/dockerfile-frontend/hack/check-daily-outdated +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -usage() { - echo "./check-daily-outdated channel repo date outdir" - exit 1 -} - -if [ "$#" != 4 ]; then - usage -fi - -CHANNEL=$1 -REPO=$2 -DATE=$3 -OUTDIR=$4 - -mkdir -p $OUTDIR - -if [ ! -d "$OUTDIR" ]; then - echo "invalid output directory $OUTDIR" - exit 1 -fi - -set -x - -reg digest "$REPO:$DATE-$CHANNEL" -if [[ $? == 0 ]]; then - exit 0 -fi - -lastTag=$(reg tags $REPO | grep "\-$CHANNEL" | sort -r | head -n 1) - -oldBuildID="" - -if [ ! -z "$lastTag" ]; then - layer=$(reg manifest $REPO:$lastTag | jq -r ".layers[0].digest") - tmpdir=$(mktemp -d -t frontend.XXXXXXXXXX) - reg layer "$REPO@$layer" | tar xvz --strip-components=1 -C $tmpdir - oldBuildID=$(go tool buildid $tmpdir/dockerfile-frontend) - rm $tmpdir/dockerfile-frontend - rm -r $tmpdir -fi - -newBuildID=$(go tool buildid dockerfile-frontend) - -if [ "$oldBuildID" != "$newBuildID" ]; then - echo -n $newBuildID > $OUTDIR/buildid -fi diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release index 7a7909c5fca3..b06e36f4d5af 100755 --- a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release +++ b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release @@ -5,10 +5,9 @@ set -eu : "${RELEASE=false}" : "${PLATFORMS=}" -: "${DAILY_TARGETS=}" usage() { - echo "$0 (master|tag|daily) (tag|channel) [push]" + echo "$0 (master|tag) (tag|channel) [push]" exit 1 } @@ -114,50 +113,4 @@ case $TYP in --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ $currentcontext ;; -"daily") - # if [ -z $DAILY_TARGETS ]; then - # DAILY_TARGETS="" - # fi - - for TAG in $DAILY_TARGETS; do - - tagf=./frontend/dockerfile/release/$TAG/tags - if [ ! -f $tagf ]; then - echo "invalid release $TAG" - exit 1 - fi - buildTags=$(cat $tagf) - - # find the buildID of the last pushed image - # returns a BuildID if rebuild needed - - tmp=$(mktemp -d -t buildid.XXXXXXXXXX) - dt=$(date +%Y%m%d) - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag \ - --target "buildid" \ - --build-arg "CHANNEL=$TAG" \ - --build-arg "BUILDTAGS=$buildTags" \ - --build-arg "REPO=$REPO" \ - --build-arg "DATE=$dt" \ - --output "type=local,dest=$tmp" \ - --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ - $currentcontext - - if [ -f $tmp/buildid ]; then - buildid=$(cat $tmp/buildid) - echo "buildid: $buildid" - - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \ - --build-arg "CHANNEL=$TAG" \ - --build-arg "BUILDTAGS=$buildTags" \ - --output "type=image,name=$REPO:$dt-$TAG,$pushFlag" \ - --file "./frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile" \ - $currentcontext - rm $tmp/buildid - fi - rm -r $tmp - - done - - ;; esac diff --git a/hack/bootstrap-buildx b/hack/bootstrap-buildx deleted file mode 100755 index c254719d8219..000000000000 --- a/hack/bootstrap-buildx +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -eu -o pipefail -if [ "$#" -ne 1 ]; then - echo >&2 "usage: $0 " - exit 1 -fi -NAME="$1" -echo "Bootstrapping buildx builder \"${NAME}\"" - -topdir="$(realpath "$(dirname "$0")/..")" -cd "${topdir}" -buildx="./bin/buildx" -if ! "${buildx}" inspect "${NAME}" >/dev/null 2>&1; then - set -x - "${buildx}" create --driver docker-container --name "${NAME}" -fi - -set -x -"${buildx}" inspect --bootstrap "${NAME}" diff --git a/hack/images b/hack/images index d1315e6a17ef..5f536557f048 100755 --- a/hack/images +++ b/hack/images @@ -7,9 +7,9 @@ PUSH=$3 . $(dirname $0)/util set -eu -o pipefail -: ${RELEASE=false} -: ${PLATFORMS=} -: ${TARGET=} +: "${RELEASE=false}" +: "${PLATFORMS=}" +: "${TARGET=}" versionTag=$(git describe --always --tags --match "v[0-9]*") diff --git a/hack/install-buildx b/hack/install-buildx deleted file mode 100755 index d486dca33d3a..000000000000 --- a/hack/install-buildx +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [ -z "${BINDIR:-}" ] && [ -z "${PREFIX:-}" ]; then - PREFIX="$(realpath "$(dirname "$0")/..")" -fi - -: "${VERSION:=v0.8.2}" -: "${BINDIR:="${PREFIX}/bin"}" -: "${DEST:="${BINDIR}/buildx"}" - -os="" -case "$(uname)" in - Linux) - os="linux" - ;; - Darwin) - os="darwin" - ;; - *) - echo >&2 "unknown OS" - exit 1 - ;; -esac - -arch="$(uname -m)" -case "${arch}" in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - armv7l) - arch="arm-v7" - ;; - armv6l) - arch="arm-v6" - ;; -esac - -url="https://github.com/docker/buildx/releases/download/${VERSION}/buildx-${VERSION}.${os}-${arch}" - -echo "Installing buildx ${VERSION} as ${DEST}" -set -x -mkdir -p "${BINDIR}" -rm -f "${DEST}.download" -curl -f -L -o "${DEST}.download" "${url}" -chmod +x "${DEST}.download" -mv "${DEST}.download" "${DEST}" diff --git a/hack/test b/hack/test index cd2845b5945b..9fd1ea3e28bb 100755 --- a/hack/test +++ b/hack/test @@ -3,23 +3,27 @@ . $(dirname $0)/util set -eu -o pipefail -: ${GO_VERSION=} -: ${HTTP_PROXY=} -: ${HTTPS_PROXY=} -: ${NO_PROXY=} -: ${TEST_INTEGRATION=} -: ${TEST_GATEWAY=} -: ${TEST_DOCKERFILE=} -: ${TEST_DOCKERD=} -: ${TEST_DOCKERD_BINARY=$(which dockerd)} -: ${TEST_REPORT_SUFFIX=} -: ${TEST_KEEP_CACHE=} -: ${GOBUILDFLAGS=} -: ${VERIFYFLAGS=} -: ${CGO_ENABLED=} -: ${DOCKERFILE_RELEASES=} -: ${BUILDKIT_WORKER_RANDOM=} -: ${BUILDKITD_TAGS=} +: "${GO_VERSION=}" +: "${BUILDKITD_TAGS=}" +: "${HTTP_PROXY=}" +: "${HTTPS_PROXY=}" +: "${NO_PROXY=}" +: "${GOBUILDFLAGS=}" +: "${VERIFYFLAGS=}" +: "${CGO_ENABLED=}" + +: "${TEST_IMAGE_BUILD=1}" +: "${TEST_IMAGE_ID=buildkit-tests}" +: "${TEST_INTEGRATION=}" +: "${TEST_GATEWAY=}" +: "${TEST_DOCKERFILE=}" +: "${TEST_DOCKERD=}" +: "${TEST_DOCKERD_BINARY=$(which dockerd)}" +: "${TEST_REPORT_SUFFIX=}" +: "${TEST_KEEP_CACHE=}" + +: "${DOCKERFILE_RELEASES=}" +: "${BUILDKIT_WORKER_RANDOM=}" if [ "$TEST_DOCKERD" == "1" ]; then if [ ! -f "$TEST_DOCKERD_BINARY" ]; then @@ -55,9 +59,6 @@ while test $# -gt 0; do shift done -iid="buildkit-tests" -iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX) - testReportsDir="$(pwd)/bin/testreports" mkdir -p "$testReportsDir" testReportsVol="-v $testReportsDir:/testreports" @@ -74,37 +75,61 @@ if [[ "$GOBUILDFLAGS" == *"-race"* ]]; then export VERIFYFLAGS="" # prevent -static verification fi -buildxCmd build $cacheFromFlags \ - --build-arg GO_VERSION \ - --build-arg BUILDKITD_TAGS \ - --build-arg HTTP_PROXY \ - --build-arg HTTPS_PROXY \ - --build-arg NO_PROXY \ - --build-arg GOBUILDFLAGS \ - --build-arg VERIFYFLAGS \ - --build-arg CGO_ENABLED \ - --target "integration-tests" \ - --output "type=docker,name=$iid" \ - $currentcontext +if [ "$TEST_IMAGE_BUILD" = "1" ]; then + buildxCmd build $cacheFromFlags \ + --build-arg GO_VERSION \ + --build-arg BUILDKITD_TAGS \ + --build-arg HTTP_PROXY \ + --build-arg HTTPS_PROXY \ + --build-arg NO_PROXY \ + --build-arg GOBUILDFLAGS \ + --build-arg VERIFYFLAGS \ + --build-arg CGO_ENABLED \ + --target "integration-tests" \ + --output "type=docker,name=$TEST_IMAGE_ID" \ + $currentcontext +fi cacheVolume="buildkit-test-cache" if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then - docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod --name "$cacheVolume" alpine + dockerCmd create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod --name "$cacheVolume" alpine +fi +if [ "$TEST_KEEP_CACHE" != "1" ]; then + trap 'docker rm -v $cacheVolume' EXIT fi +baseCreateFlags="--rm --privileged \ +-v /tmp $testReportsVol \ +--volumes-from=$cacheVolume \ +-e CGO_ENABLED \ +-e GITHUB_REF \ +-e ACTIONS_RUNTIME_TOKEN \ +-e ACTIONS_CACHE_URL \ +-e TEST_DOCKERD \ +-e BUILDKIT_TEST_ENABLE_FEATURES \ +-e BUILDKIT_TEST_DISABLE_FEATURES \ +-e GOTESTSUM_FORMAT \ +-e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry \ +-e BUILDKIT_INTEGRATION_DOCKERD_FLAGS \ +-e GORACE" + if [ "$TEST_INTEGRATION" == 1 ]; then - cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e CGO_ENABLED -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS -e BUILDKIT_TEST_ENABLE_FEATURES -e BUILDKIT_TEST_DISABLE_FEATURES -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS -e GORACE --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) + cid=$(dockerCmd create $baseCreateFlags \ + ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} \ + -e SKIP_INTEGRATION_TESTS \ + $TEST_IMAGE_ID \ + gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) if [ "$TEST_DOCKERD" = "1" ]; then - docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd + dockerCmd cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd fi - docker start -a $cid + dockerCmd start -a $cid fi if [ "$TEST_GATEWAY" == 1 ]; then # Build-test "github.com/moby/buildkit/frontend/gateway/client", which isn't otherwise built by CI - # It really only needs buildkit-base. We have integration-tests in $iid, which is a direct child of buildkit-base. - cid=$(docker create --rm --volumes-from=$cacheVolume --entrypoint="" $iid go build -v ./frontend/gateway/client) - docker start -a $cid + # It really only needs buildkit-base. We have integration-tests in $TEST_IMAGE_ID, which is a direct child of buildkit-base. + cid=$(dockerCmd create --rm --volumes-from=$cacheVolume --entrypoint="" $TEST_IMAGE_ID go build -v ./frontend/gateway/client) + dockerCmd start -a $cid fi DOCKERFILE_RELEASES_CUSTOM="" @@ -133,18 +158,19 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then if [ -s $tarout ]; then if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$GITHUB_ACTIONS" = "true" ]; then - cid=$(docker create -v /tmp $testReportsVol --rm --privileged --volumes-from=$cacheVolume -e CGO_ENABLED -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e BUILDKIT_TEST_ENABLE_FEATURES -e BUILDKIT_TEST_DISABLE_FEATURES -e GOTESTSUM_FORMAT -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend -e GORACE $iid gotestsum $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v}) - docker cp $tarout $cid:/$release.tar + cid=$(dockerCmd create $baseCreateFlags \ + -e BUILDKIT_WORKER_RANDOM \ + -e FRONTEND_GATEWAY_ONLY=local:/$release.tar \ + -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend \ + $TEST_IMAGE_ID \ + gotestsum $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v}) + dockerCmd cp $tarout $cid:/$release.tar if [ "$TEST_DOCKERD" = "1" ]; then - docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd + dockerCmd cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd fi - docker start -a $cid + dockerCmd start -a $cid fi fi rm $tarout done fi - -if [ "$TEST_KEEP_CACHE" != "1" ]; then - docker rm -v $cacheVolume -fi diff --git a/hack/util b/hack/util index ed25cee3d315..2c917d60405a 100755 --- a/hack/util +++ b/hack/util @@ -1,46 +1,32 @@ #!/usr/bin/env sh -: "${CI=}" +: "${BUILDX_CMD=docker buildx}" +: "${BUILDX_BUILDER=}" + : "${GITHUB_ACTIONS=}" : "${GITHUB_REPOSITORY=}" : "${GITHUB_RUN_ID=}" -: "${BUILDX_BUILDER=}" : "${CONTEXT=}" : "${CACHE_FROM=}" : "${CACHE_TO=}" -progressFlag="" -if [ "$CI" = "true" ]; then - progressFlag="--progress=plain" -fi - -buildxBin="" -builderName="" -if docker buildx version >/dev/null 2>&1; then - buildxBin="docker buildx" -elif buildx version >/dev/null 2>&1; then - buildxBin="buildx" -else - topdir="$(realpath $(dirname "$0")/..)" - if [ ! -x "${topdir}/bin/buildx" ]; then +dockerCmd() { + ( set -x - "${topdir}/hack/install-buildx" - fi - buildxBin="${topdir}/bin/buildx" - builderName="moby-buildkit" - "${topdir}/hack/bootstrap-buildx" "${builderName}" -fi + docker "$@" + ) +} buildxCmd() { ( set -x - BUILDX_NO_DEFAULT_LOAD=true BUILDX_BUILDER="${builderName:-${BUILDX_BUILDER}}" $buildxBin "$@" $progressFlag + BUILDX_NO_DEFAULT_LOAD=true BUILDX_BUILDER="${BUILDX_BUILDER}" ${BUILDX_CMD} "$@" ) } buildAttestFlags() { - if $buildxBin build --help 2>&1 | grep -- '--attest' >/dev/null; then + if ${BUILDX_CMD} build --help 2>&1 | grep -- '--attest' >/dev/null; then prvattrs="mode=max" if [ "$GITHUB_ACTIONS" = "true" ]; then prvattrs="$prvattrs,builder-id=https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"