mergegroup build on fix: benchmark default bytesizes (#4356) #29

Workflow file for this run

.github/workflows/merge_queue.yaml at e02ac38

	# Build all binaries
	name: mergegroup build
	run-name: mergegroup build on ${{ github.event.merge_group.head_commit.message }}

	permissions: read-all

	concurrency:
	group: mergegroup-${{ github.ref }}
	cancel-in-progress: true

	on:
	merge_group:
	# pull_request:
	# branches: [master]



	env:
	CARGO_TERM_COLOR: always
	VERBOSE: ${{ github.events.input.verbose }}
	K3D_VERSION: v5.4.9
	BATS_VERSION: 1.11.0
	MINIKUBE_VERSION: v1.30.1
	K8_VERSION: v1.26.3
	TLS_ARGS: --tls --domain fluvio.local --server-key ./tls/certs/server.key --server-cert ./tls/certs/server.crt --ca-cert ./tls/certs/ca.crt --client-cert ./tls/certs/client-root.crt --client-key ./tls/certs/client-root.key
	AUTH_FILE: crates/fluvio-sc/test-data/auth_config/policy.json
	X509_SCOPE_FILE: crates/fluvio-sc/test-data/auth_config/scopes.json
	FLV_CLUSTER_PROVISION_TIMEOUT: 600
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	jobs:
	# this job set up dynamic configuration shared among jobs
	config:
	name: Set up Config
	runs-on: ubuntu-latest
	env:
	ENABLE_CHECK: true
	TEST_RUNS: "[r1]" # default runs
	outputs:
	check: ${{ steps.check.outputs.check }}
	runs: ${{ steps.runs.outputs.runs }}
	steps:
	- id: check
	run: echo "check=${{ env.ENABLE_CHECK }}" >> "$GITHUB_OUTPUT"
	#- name: Set Staging runs
	## if: github.event_name == 'merge_group'
	# run: echo "TEST_RUNS='[r1,r2,r3,r4,r5]'" \| tee -a $GITHUB_ENV

	# build binaries for linux x86_64/aarch64 musl which is primary OS for testing clusters
	build_primary_binaries:
	name: Primary build ${{ matrix.binary }} for ${{ matrix.rust-target }} on (${{ matrix.os }})
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	rust-target:
	- x86_64-unknown-linux-musl
	- aarch64-unknown-linux-musl
	binary: [fluvio, fluvio-run, fluvio-test, fluvio-channel, smdk, fluvio-benchmark, cdk, fvm]
	os: [ubuntu-latest]
	env:
	RUST_BACKTRACE: full
	TARGET: ${{ matrix.rust-target }}
	RUST_BIN_DIR: target/${{ matrix.rust-target }}/release
	RELEASE_NAME: release
	RELEASE: true
	steps:
	- uses: actions/checkout@v4

	- name: Print env
	run: \|
	echo "RUST_BIN_DIR = ${{ env.RUST_BIN_DIR }} "
	- name: Install Rust Stable
	uses: dtolnay/rust-toolchain@stable
	- name: install rust target
	run: rustup target add ${{ matrix.rust-target }}
	- name: Install zig
	run: ./actions/zig-install.sh
	- uses: Swatinem/rust-cache@v2
	timeout-minutes: 10
	with:
	key: ${{ matrix.os }}-${{ env.RUST_BIN_DIR }}-${{ matrix.binary }}

	- name: Build fluvio
	if: matrix.binary == 'fluvio'
	run: make build-cli

	- name: Build fluvio-run
	if: matrix.binary == 'fluvio-run'
	run: make build-cluster

	- name: Build fluvio-test
	if: matrix.binary == 'fluvio-test'
	run: make build-test

	- name: Build fluvio-channel
	if: matrix.binary == 'fluvio-channel'
	run: make build-channel

	- name: Build smdk
	if: matrix.binary == 'smdk'
	run: make build-smdk

	- name: Build cdk
	timeout-minutes: 40
	if: matrix.binary == 'cdk'
	run: make build-cdk

	- name: Build fluvio-benchmark
	if: matrix.binary == 'fluvio-benchmark'
	run: make build-benchmark

	- name: Build fvm
	if: matrix.binary == 'fvm'
	run: make build-fvm

	# Upload artifacts
	- name: Upload artifact - ${{ matrix.binary }}
	uses: actions/upload-artifact@v4
	with:
	name: ${{ matrix.binary }}-${{ matrix.rust-target }}
	path: ${{ env.RUST_BIN_DIR }}/${{ matrix.binary }}
	retention-days: 1

	# build other binaries which doesn't need test
	build_binaries:
	name: Build ${{ matrix.binary }} for ${{ matrix.rust-target }} on (${{ matrix.os }})
	runs-on: ${{ matrix.os }}
	strategy:
	# fail-fast: false
	matrix:
	include:
	# fluvio
	- os: ubuntu-latest
	rust-target: arm-unknown-linux-gnueabihf
	binary: fluvio
	- os: ubuntu-latest
	rust-target: armv7-unknown-linux-gnueabihf
	binary: fluvio
	- os: ubuntu-latest
	rust-target: armv7-unknown-linux-gnueabihf
	binary: fluvio-run
	- os: ubuntu-24.04
	rust-target: x86_64-pc-windows-gnu
	binary: fluvio.exe
	- os: macos-latest
	rust-target: x86_64-apple-darwin
	binary: fluvio
	- os: macos-latest
	rust-target: aarch64-apple-darwin
	binary: fluvio

	# fluvio-run
	- os: macos-latest
	rust-target: x86_64-apple-darwin
	binary: fluvio-run
	- os: macos-latest
	rust-target: aarch64-apple-darwin
	binary: fluvio-run

	# fluvio-channel
	- os: ubuntu-latest
	rust-target: arm-unknown-linux-gnueabihf
	binary: fluvio-channel
	- os: ubuntu-latest
	rust-target: armv7-unknown-linux-gnueabihf
	binary: fluvio-channel
	- os: ubuntu-24.04
	rust: stable
	rust-target: x86_64-pc-windows-gnu
	binary: fluvio-channel.exe
	- os: macos-latest
	rust-target: x86_64-apple-darwin
	binary: fluvio-channel
	- os: macos-latest
	rust-target: aarch64-apple-darwin
	binary: fluvio-channel

	# smdk
	- os: macos-latest
	rust-target: x86_64-apple-darwin
	binary: smdk
	- os: macos-latest
	rust-target: aarch64-apple-darwin
	binary: smdk

	# cdk
	- os: macos-latest
	rust-target: x86_64-apple-darwin
	binary: cdk
	- os: macos-latest
	rust-target: aarch64-apple-darwin
	binary: cdk

	# fluvio-benchmark
	- os: ubuntu-latest
	rust-target: arm-unknown-linux-gnueabihf
	binary: fluvio-benchmark
	- os: ubuntu-latest
	rust-target: armv7-unknown-linux-gnueabihf
	binary: fluvio-benchmark
	- os: ubuntu-24.04
	rust-target: x86_64-pc-windows-gnu
	binary: fluvio-benchmark.exe
	- os: macos-latest
	rust-target: x86_64-apple-darwin
	binary: fluvio-benchmark
	- os: macos-latest
	rust-target: aarch64-apple-darwin
	binary: fluvio-benchmark

	# fvm
	- os: ubuntu-latest
	rust-target: arm-unknown-linux-gnueabihf
	binary: fvm
	- os: ubuntu-latest
	rust-target: armv7-unknown-linux-gnueabihf
	binary: fvm
	- os: ubuntu-24.04
	rust-target: x86_64-pc-windows-gnu
	binary: fvm.exe
	- os: macos-latest
	rust-target: x86_64-apple-darwin
	binary: fvm
	- os: macos-latest
	rust-target: aarch64-apple-darwin
	binary: fvm

	env:
	RUST_BACKTRACE: full
	TARGET: ${{ matrix.rust-target }}
	RUST_BIN_DIR: target/${{ matrix.rust-target }}/debug
	RELEASE_NAME: debug
	steps:
	- uses: actions/checkout@v4

	# If this job is being run by merge_group,
	# then build and run in release mode
	- name: Set RELEASE mode
	if: github.event_name == 'merge_group'
	shell: bash
	run: \|
	echo "RELEASE=true" \| tee -a $GITHUB_ENV
	echo "RELEASE_NAME=release" \| tee -a $GITHUB_ENV
	echo "RUST_BIN_DIR=target/${{ matrix.rust-target }}/release" \| tee -a $GITHUB_ENV

	- name: Print env
	run: \|
	echo "RUST_BIN_DIR = ${{ env.RUST_BIN_DIR }} "
	- name: Install Rust Stable
	uses: dtolnay/rust-toolchain@stable
	- name: install Rust target
	run: rustup target add ${{ matrix.rust-target }}
	- name: Install zig
	run: ./actions/zig-install.sh ${{ matrix.os }}
	- name: Install gcc-multilib
	run: sudo apt-get install gcc-multilib
	if: matrix.rust-target == 'arm-unknown-linux-gnueabihf' \|\| matrix.rust-target == 'armv7-unknown-linux-gnueabihf'
	- name: Install NASM
	if: matrix.rust-target == 'x86_64-pc-windows-gnu'
	uses: ilammy/setup-nasm@v1
	- uses: Swatinem/rust-cache@v2
	timeout-minutes: 10
	with:
	key: ${{ matrix.os }}-${{ matrix.rust-target }}-${{ matrix.binary }}
	- name: Install Helm for Mac
	run: brew install helm
	if: matrix.binary == 'fluvio' && matrix.os == 'macos-latest'

	- name: Install mingw gcc
	if: matrix.rust-target == 'x86_64-pc-windows-gnu'
	run: \|
	sudo apt-get update -o="APT::Acquire::Retries=3"
	sudo apt-get install -y -V -o="APT::Acquire::Retries=3" gcc-mingw-w64-x86-64

	- name: Install bindgen-cli
	if: matrix.rust-target == 'x86_64-pc-windows-gnu'
	run: cargo install --force --locked bindgen-cli

	- name: Build fluvio
	timeout-minutes: 40
	if: matrix.binary == 'fluvio'
	run: make build-cli

	- name: Build fluvio.exe
	timeout-minutes: 40
	if: matrix.binary == 'fluvio.exe'
	run: make build-cli-minimal

	- name: Build fluvio-run
	timeout-minutes: 40
	if: matrix.binary == 'fluvio-run'
	run: make build-cluster

	- name: Build fluvio-channel
	timeout-minutes: 40
	if: matrix.binary == 'fluvio-channel'
	run: make build-channel

	- name: Build fluvio-channel.exe
	timeout-minutes: 40
	if: matrix.binary == 'fluvio-channel.exe'
	run: make build-channel

	- name: Build smdk
	timeout-minutes: 40
	if: matrix.binary == 'smdk'
	run: make build-smdk

	- name: Build smdk.exe
	timeout-minutes: 40
	if: matrix.binary == 'smdk.exe'
	run: make build-smdk

	- name: Build cdk
	timeout-minutes: 40
	if: matrix.binary == 'cdk'
	run: make build-cdk

	- name: Build fluvio-benchmark
	timeout-minutes: 40
	if: matrix.binary == 'fluvio-benchmark'
	run: make build-benchmark

	- name: Build fluvio-benchmark.exe
	timeout-minutes: 40
	if: matrix.binary == 'fluvio-benchmark.exe'
	run: make build-benchmark

	- name: Build fvm
	timeout-minutes: 40
	if: matrix.binary == 'fvm' \|\| matrix.binary == 'fvm.exe'
	run: make build-fvm

	# Upload artifacts
	- name: Upload artifact - ${{ matrix.binary }}
	uses: actions/upload-artifact@v4
	with:
	name: ${{ matrix.binary }}-${{ matrix.rust-target }}
	path: ${{ env.RUST_BIN_DIR }}/${{ matrix.binary }}
	retention-days: 1



	build_image:
	name: Build Fluvio Docker image
	needs: build_primary_binaries
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ubuntu-latest]
	rust-target:
	- x86_64-unknown-linux-musl
	- aarch64-unknown-linux-musl
	rust: [stable]
	steps:
	# Needed for k8-util/docker/build.sh
	- uses: actions/checkout@v4

	# Download artifacts
	- name: Download fluvio-run
	uses: actions/download-artifact@v4
	with:
	name: fluvio-run-${{ matrix.rust-target }}
	path: .

	- name: Print fluvio-run path
	run: pwd && ls -la . && chmod +x ./fluvio-run && ./fluvio-run -h \|\| true

	# Build and upload docker image
	- name: Build Docker image
	run: k8-util/docker/build.sh ${{ matrix.rust-target }} ${{ github.sha }} "$(pwd)/fluvio-run"
	- name: Export Docker Image to tarball
	run: docker image save infinyon/fluvio:${{ github.sha }}-${{ matrix.rust-target }} --output /tmp/infinyon-fluvio-${{ matrix.rust-target }}.tar
	- name: Upload tarball as artifact
	uses: actions/upload-artifact@v4
	with:
	name: infinyon-fluvio-${{ matrix.rust-target }}.tar
	path: /tmp/infinyon-fluvio-${{ matrix.rust-target }}.tar
	retention-days: 1


	# Ensure all checks, tests are perform and all binaries are built
	# After this, we are committed for release
	docker_push:
	name: Publish Docker Image
	if: github.event_name == 'merge_group'
	needs:
	- build_image
	- build_binaries
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Download x86_64-unknown-linux-musl Docker Image as Artifact
	uses: actions/download-artifact@v4
	with:
	name: infinyon-fluvio-x86_64-unknown-linux-musl.tar
	path: /tmp
	- name: Download aarch64-unknown-linux-musl Docker Image as Artifact
	uses: actions/download-artifact@v4
	with:
	name: infinyon-fluvio-aarch64-unknown-linux-musl.tar
	path: /tmp

	# Load image infinyon/fluvio:SHA-ARCH
	- name: Load Fluvio Docker Image
	run: \|
	ls -la /tmp
	docker image load --input /tmp/infinyon-fluvio-x86_64-unknown-linux-musl.tar
	docker image load --input /tmp/infinyon-fluvio-aarch64-unknown-linux-musl.tar
	docker image ls -a

	- name: Publish Fluvio Docker images
	run: \|
	export TAG="$(cat VERSION)-${{ github.sha }}"
	docker login --username=${{ secrets.DOCKER_USERNAME }} --password=${{ secrets.DOCKER_PASSWORD }}
	docker tag "infinyon/fluvio:${{ github.sha }}-x86_64-unknown-linux-musl" "docker.io/infinyon/fluvio:${TAG}-amd64"
	docker tag "infinyon/fluvio:${{ github.sha }}-aarch64-unknown-linux-musl" "docker.io/infinyon/fluvio:${TAG}-arm64v8"
	docker push "docker.io/infinyon/fluvio:${TAG}-amd64"
	docker push "docker.io/infinyon/fluvio:${TAG}-arm64v8"
	docker manifest create "docker.io/infinyon/fluvio:${TAG}" \
	"docker.io/infinyon/fluvio:${TAG}-amd64" \
	"docker.io/infinyon/fluvio:${TAG}-arm64v8"
	docker manifest push "docker.io/infinyon/fluvio:${TAG}"

	# When all required jobs pass, bump the `dev` GH prerelease to this commit
	bump_github_release:
	name: Bump dev tag
	if: github.event_name == 'merge_group'
	needs:
	- docker_push
	runs-on: ubuntu-latest
	permissions: write-all
	steps:
	- uses: actions/checkout@v4
	#- name: Login GH CLI
	# run: gh auth login --with-token < <(echo ${{ secrets.GITHUB_TOKEN }})
	- name: Delete old release
	run: gh release delete -R infinyon/fluvio dev -y \|\| true
	- name: Bump dev tag
	run: \|
	git tag -f dev
	git push -f origin dev
	- name: Create new release
	run: gh release create -R infinyon/fluvio dev -p --notes "Published artifacts from the latest build"

	# Upload the build artifacts to the `dev` GH release, overwriting old artifacts
	publish_github_helm_pkg:
	name: Publish helm chart to GitHub Releases dev
	if: github.event_name == 'merge_group'
	needs: bump_github_release
	runs-on: ubuntu-latest
	permissions: write-all
	steps:
	- uses: actions/checkout@v4
	#- name: Login GH CLI
	# run: gh auth login --with-token < <(echo ${{ secrets.GITHUB_TOKEN }})
	- name: package helm
	run: make helm_pkg
	- name: Publish artifact
	run: \|
	gh release upload -R infinyon/fluvio --clobber dev k8-util/helm/pkg_app/.
	gh release upload -R infinyon/fluvio --clobber dev k8-util/helm/pkg_sys/.

	publish_github_binaries:
	name: Publish to GitHub Releases dev (${{ matrix.artifact }}-${{ matrix.rust-target }})
	if: github.event_name == 'merge_group'
	needs:
	- bump_github_release
	runs-on: ubuntu-latest
	strategy:
	matrix:
	rust-target:
	- aarch64-unknown-linux-musl
	- aarch64-apple-darwin
	- armv7-unknown-linux-gnueabihf
	- arm-unknown-linux-gnueabihf
	- x86_64-apple-darwin
	- x86_64-unknown-linux-musl
	artifact: [fluvio, fluvio-channel, smdk, fluvio-benchmark, cdk, fvm]
	exclude:
	- rust-target: armv7-unknown-linux-gnueabihf
	artifact: smdk
	- rust-target: arm-unknown-linux-gnueabihf
	artifact: smdk
	- rust-target: x86_64-pc-windows-gnu
	artifact: smdk.exe
	- rust-target: armv7-unknown-linux-gnueabihf
	artifact: cdk
	- rust-target: arm-unknown-linux-gnueabihf
	artifact: cdk
	- rust-target: x86_64-pc-windows-gnu
	artifact: cdk.exe
	include:
	- rust-target: x86_64-unknown-linux-musl
	artifact: fluvio-run
	- rust-target: aarch64-unknown-linux-musl
	artifact: fluvio-run
	- rust-target: x86_64-apple-darwin
	artifact: fluvio-run
	- rust-target: aarch64-apple-darwin
	artifact: fluvio-run
	- rust-target: armv7-unknown-linux-gnueabihf
	artifact: fluvio-run
	- rust-target: x86_64-pc-windows-gnu
	artifact: fluvio.exe
	- rust-target: x86_64-pc-windows-gnu
	artifact: fluvio-channel.exe
	- rust-target: x86_64-unknown-linux-musl
	artifact: fluvio-test
	- rust-target: x86_64-pc-windows-gnu
	artifact: fluvio-benchmark.exe
	permissions: write-all
	steps:
	#- name: Login GH CLI
	# run: gh auth login --with-token < <(echo ${{ secrets.GITHUB_TOKEN }})
	- name: Download artifact
	uses: actions/download-artifact@v4
	with:
	name: ${{ matrix.artifact }}-${{ matrix.rust-target }}
	- name: Publish artifact
	run: \|
	ls -la
	echo "${{ matrix.rust-target }}" > .target
	zip "${{ matrix.artifact }}-${{ matrix.rust-target }}.zip" "${{ matrix.artifact }}" .target
	gh release upload -R infinyon/fluvio --clobber dev "${{ matrix.artifact }}-${{ matrix.rust-target }}.zip"

	# Job that follows the success of all required jobs in this workflow.
	done:
	name: Done
	permissions:
	contents: write
	if: always()
	needs:
	- build_image
	- build_binaries
	- publish_github_binaries
	- publish_github_helm_pkg
	runs-on: ubuntu-latest
	steps:
	- name: Dump needs context
	env:
	CONTEXT: ${{ toJson(needs) }}
	run: \|
	echo -e "\033[33;1;4mDump context\033[0m"
	echo -e "$CONTEXT\n"
	- name: Report failure on cancellation
	if: ${{ contains(needs.*.result, 'cancelled') \|\| cancelled() }}
	run: exit 1
	- name: Failing test and build
	if: ${{ contains(needs.*.result, 'failure') }}
	run: exit 1
	- name: Don't allow skipped
	if: ${{ contains(needs.*.result, 'skipped') && github.event_name == 'merge_group' }}
	run: exit 1
	- name: Successful test and build
	if: ${{ !(contains(needs.*.result, 'failure')) }}
	run: exit 0
	- name: Repo dispatch ci-success
	if: github.event_name == 'merge_group'
	uses: peter-evans/repository-dispatch@v3
	with:
	event-type: ci-success
	client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

mergegroup build on fix: benchmark default bytesizes (#4356) #29

Workflow file

mergegroup build on fix: benchmark default bytesizes (#4356) #29

Jobs

Run details

Workflow file for this run