diff --git a/ci/mkpipeline.py b/ci/mkpipeline.py index 66f66ca254c5e..da47bc37896b7 100644 --- a/ci/mkpipeline.py +++ b/ci/mkpipeline.py @@ -35,6 +35,7 @@ from materialize.ci_util.trim_pipeline import permit_rerunning_successful_steps from materialize.mzcompose.composition import Composition from materialize.ui import UIError +from materialize.xcompile import Arch from .deploy.deploy_util import rust_version @@ -137,6 +138,8 @@ def visit(step: dict[str, Any]) -> None: check_depends_on(pipeline, args.pipeline) + trim_builds(pipeline, args.coverage) + # Remove the Materialize-specific keys from the configuration that are # only used to inform how to trim the pipeline and for coverage runs. def visit(step: dict[str, Any]) -> None: @@ -390,6 +393,34 @@ def visit(step: PipelineStep) -> None: ] +def trim_builds(pipeline: Any, coverage: bool) -> None: + """Trim unnecessary x86-64/aarch64 builds if all artifacts already exist.""" + + def builds_published(arch: Arch) -> bool: + repo = mzbuild.Repository(Path("."), arch=arch, coverage=coverage) + deps_publish = repo.resolve_dependencies( + image for image in repo if image.publish + ) + return deps_publish.check() + + def visit(step: dict[str, Any]) -> None: + if step.get("id") == "build-x86_64": + if builds_published(Arch.X86_64): + step["skip"] = True + if step.get("id") == "build-aarch64": + branch = os.environ["BUILDKITE_BRANCH"] + if ( + branch == "main" or branch.startswith("v") and "." in branch + ) and builds_published(Arch.AARCH64): + step["skip"] = True + + for step in pipeline["steps"]: + visit(step) + if "group" in step: + for inner_step in step.get("steps", []): + visit(inner_step) + + def have_paths_changed(globs: Iterable[str]) -> bool: """Reports whether the specified globs have diverged from origin/main.""" diff = subprocess.run( diff --git a/misc/python/materialize/mzbuild.py b/misc/python/materialize/mzbuild.py index 64cd4eebfcd6b..3c899699c2bfa 100644 --- a/misc/python/materialize/mzbuild.py +++ b/misc/python/materialize/mzbuild.py @@ -780,6 +780,11 @@ def ensure(self, post_build: Callable[[ResolvedImage], None] | None = None): if returncode: raise subprocess.CalledProcessError(returncode, push.args) + def check(self) -> bool: + """Check all publishable images in this dependency set exist on Docker + Hub. Don't try to download or build them.""" + return all(dep.is_published_if_necessary() for dep in self) + def __iter__(self) -> Iterator[ResolvedImage]: return iter(self._dependencies.values())