Merge branch 'main' into work/starflow-migration

docs/explanation/remote-build.rst

@@ -82,22 +82,30 @@ Remote builds can be orchestrated for multiple platforms and architectures.
 Current
 ^^^^^^^
 
-``--platform`` and ``--build-for``
-**********************************
+``--build-for``
+***************
 
 .. note::
-   ``--platform`` and ``--build-for`` behave differently than they do for
+   ``--build-for`` behaves differently for ``remote-build`` than it does for
    :ref:`lifecycle commands<reference-lifecycle-commands>`.
 
-``--platform`` or ``--build-for`` can only be provided when the ``platforms``
-or ``architectures`` keywords are not defined in the project metadata
-(`[12]`_).
+Remote builds are useful for building snaps on different architectures. Due
+to this, the semantics for the ``--build-for`` argument is more complex than
+when building a snap locally.
+
+The argument operates in one of two different ways depending on the presence
+of a ``platforms`` or ``architectures`` key in the project file.
 
-These keywords are mutually exclusive and must be a comma-separated list of
-debian architectures.
+The first mode of operation is when the ``platforms`` or ``architectures``
+key is present in the project file. In this scenario, ``--build-for`` operates
+similar to how it does for lifecycle commands. The difference from its usage in
+lifecycle commands is that ``--build-for`` may be a comma-separated list, which
+allows multiple snaps to be built. For more information about build plans and
+filtering, see :ref:`Build plans <build-plans>`.
 
-``core22`` snaps can only use ``--build-for``. ``core24`` and newer snaps
-can use ``--platform`` or ``--build-for``.
+The second mode of operation is when there isn't a ``platforms`` or
+``architectures`` key in the project file. In this scenario, ``--build-for``
+defines the architectures to build for.
 
 Project platforms and architectures
 ***********************************
@@ -117,8 +125,8 @@ Snapcraft will request a build for each unique ``build-for`` architecture.
     ``build-on`` architecture (`[14]`_).
 
 If the project metadata does not contain a ``platforms`` or ``architectures``
-entry and no ``--build-for`` or ``--platform`` are passed, Snapcraft will
-request a build on, and for, the host's architecture.
+entry and ``--build-for`` is not provided, Snapcraft will request a build on,
+and for, the host's architecture.
 
 The remote builder does not work for ``core20`` snaps because it cannot parse
 the ``run-on`` keyword in a ``core20`` architecture entry (`[2]`_).
@@ -180,6 +188,5 @@ Launchpad is not able to parse this notation (`[9]`_).
 .. _`[8]`: https://bugs.launchpad.net/snapcraft/+bug/2007789
 .. _`[9]`: https://bugs.launchpad.net/snapcraft/+bug/2042167
 .. _`[10]`: https://github.com/canonical/snapcraft/issues/4885
-.. _`[12]`: https://github.com/canonical/snapcraft/issues/4992
 .. _`[13]`: https://github.com/canonical/snapcraft/issues/4996
 .. _`[14]`: https://github.com/canonical/snapcraft/issues/4995

pyproject.toml

@@ -8,9 +8,9 @@ dependencies = [
     "click==8.1.7",
     "craft-application~=4.4",
     "craft-archives~=2.0",
-    "craft-cli~=2.9",
+    "craft-cli>=2.15.0",
     "craft-grammar>=2.0.1,<3.0.0",
-    "craft-parts==2.3.0",
+    "craft-parts==2.4.1",
     "craft-platforms~=0.4",
     "craft-store>=3.0.2,<4.0.0",
     "cryptography==43.0.3", # Pinned due to https://github.com/canonical/snapcraft/issues/5217

snapcraft/application.py

@@ -27,7 +27,7 @@
 import craft_cli
 import craft_parts
 import craft_store
-from craft_application import Application, AppMetadata, remote, util
+from craft_application import Application, AppMetadata, launchpad, remote, util
 from craft_application.commands import get_other_command_group
 from craft_cli import emit
 from craft_parts.plugins.plugins import PluginType
@@ -224,6 +224,14 @@ def _run_inner(self) -> int:
             err.doc_slug = "/explanation/remote-build"
             self._emit_error(err)
             return_code = err.retcode
+        except launchpad.LaunchpadError as err:
+            self._emit_error(
+                craft_cli.errors.CraftError(
+                    f"{err}", doc_slug="/explanation/remote-build"
+                ),
+                cause=err,
+            )
+            return_code = 1
 
         return return_code
 

snapcraft/commands/remote.py

@@ -24,14 +24,17 @@
 from pathlib import Path
 from typing import Any, cast
 
+import craft_application.errors
 import lazr.restfulclient.errors
-from craft_application import errors
+from craft_application import errors, launchpad
+from craft_application.application import filter_plan
 from craft_application.commands import ExtensibleCommand
 from craft_application.errors import RemoteBuildError
 from craft_application.launchpad.models import Build, BuildState
 from craft_application.remote.utils import get_build_id
 from craft_application.util import humanize_list
 from craft_cli import emit
+from craft_platforms import DebianArchitecture
 from overrides import overrides
 
 from snapcraft import models
@@ -57,9 +60,17 @@ class RemoteBuildCommand(ExtensibleCommand):
         architecture are retrieved and will be available in the
         local filesystem.
 
-        If not specified in the snapcraft.yaml file, the list of
-        architectures to build can be set using the --platforms option.
-        If both are specified, an error will occur.
+        If the project contains a ``platforms`` or ``architectures`` key,
+        then the project's build plan is used. The build plan can be filtered
+        using the ``--build-for`` argument.
+
+        If the project doesn't contain a ``platforms`` or ``architectures`` key,
+        then the architectures to build for are defined by the ``--build-for``
+        argument.
+
+        If there are no architectures defined in the project file or with
+        ``--build-for``, then the default behavior is to build for the host
+        architecture of the local machine.
 
         Interrupted remote builds can be resumed using the --recover
         option, followed by the build number informed when the remote
@@ -99,18 +110,7 @@ def _fill_parser(self, parser: argparse.ArgumentParser) -> None:
             "--build-id", metavar="build-id", help="Specific build ID to retrieve"
         )
 
-        group = parser.add_mutually_exclusive_group()
-        group.add_argument(
-            "--platform",
-            type=lambda arg: [arch.strip() for arch in arg.split(",")],
-            metavar="name",
-            default=os.getenv("CRAFT_PLATFORM"),
-            help="Comma-separated list of platforms to build for",
-            # '--platform' needs to be handled differently since remote-build can
-            # build for an architecture that is not in the project metadata
-            dest="remote_build_platforms",
-        )
-        group.add_argument(
+        parser.add_argument(
             "--build-for",
             type=lambda arg: [arch.strip() for arch in arg.split(",")],
             metavar="arch",
@@ -154,57 +154,17 @@ def _validate(self, parsed_args: argparse.Namespace) -> None:
                 retcode=os.EX_NOPERM,
             )
 
-        build_fors = parsed_args.remote_build_build_fors or None
-        platforms = parsed_args.remote_build_platforms or None
-        parameter = "--build-for" if build_fors else "--platform" if platforms else None
-        keyword = (
-            "architectures"
-            if self.project._architectures_in_yaml
-            else "platforms"
-            if self.project.platforms
-            else None
-        )
-
-        if keyword and parameter:
-            raise errors.RemoteBuildError(
-                f"{parameter!r} cannot be used when {keyword!r} is in the snapcraft.yaml.",
-                resolution=f"Remove {parameter!r} from the command line or remove {keyword!r} in the snapcraft.yaml.",
-                doc_slug="/explanation/remote-build.html",
-                retcode=os.EX_CONFIG,
-            )
-
-        if platforms:
-            if self.project.get_effective_base() == "core22":
+        for build_for in parsed_args.remote_build_build_fors or []:
+            if build_for not in [*SUPPORTED_ARCHS, "all"]:
                 raise errors.RemoteBuildError(
-                    "'--platform' cannot be used for core22 snaps.",
-                    resolution="Use '--build-for' instead.",
+                    f"Unsupported build-for architecture {build_for!r}.",
+                    resolution=(
+                        "Use a supported debian architecture. Supported "
+                        f"architectures are: {humanize_list(SUPPORTED_ARCHS, 'and')}"
+                    ),
                     doc_slug="/explanation/remote-build.html",
                     retcode=os.EX_CONFIG,
                 )
-            for platform in platforms:
-                if platform not in SUPPORTED_ARCHS:
-                    raise errors.RemoteBuildError(
-                        f"Unsupported platform {platform!r}.",
-                        resolution=(
-                            "Use a supported debian architecture. Supported "
-                            f"architectures are: {humanize_list(SUPPORTED_ARCHS, 'and')}"
-                        ),
-                        doc_slug="/explanation/remote-build.html",
-                        retcode=os.EX_CONFIG,
-                    )
-
-        if build_fors:
-            for build_for in build_fors:
-                if build_for not in SUPPORTED_ARCHS:
-                    raise errors.RemoteBuildError(
-                        f"Unsupported build-for architecture {build_for!r}.",
-                        resolution=(
-                            "Use a supported debian architecture. Supported "
-                            f"architectures are: {humanize_list(SUPPORTED_ARCHS, 'and')}"
-                        ),
-                        doc_slug="/explanation/remote-build.html",
-                        retcode=os.EX_CONFIG,
-                    )
 
         self._validate_single_artifact_per_build_on()
 
@@ -276,14 +236,7 @@ def _run(  # noqa: PLR0915 [too-many-statements]
         emit.trace(f"Project directory: {project_dir}")
         self._validate(parsed_args)
 
-        if parsed_args.remote_build_build_fors:
-            architectures = parsed_args.remote_build_build_fors
-        elif parsed_args.remote_build_platforms:
-            architectures = parsed_args.remote_build_platforms
-        else:
-            architectures = self._get_project_build_fors()
-
-        emit.debug(f"Architectures to build for: {architectures}")
+        archs = self._get_archs(parsed_args.remote_build_build_fors)
 
         if parsed_args.launchpad_timeout:
             emit.debug(f"Setting timeout to {parsed_args.launchpad_timeout} seconds")
@@ -298,10 +251,8 @@ def _run(  # noqa: PLR0915 [too-many-statements]
                 "Starting new build. It may take a while to upload large projects."
             )
             try:
-                builds = builder.start_builds(
-                    project_dir, architectures=architectures or None
-                )
-            except RemoteBuildError:
+                builds = builder.start_builds(project_dir, architectures=archs)
+            except (RemoteBuildError, launchpad.LaunchpadError):
                 emit.progress("Starting build failed.", permanent=True)
                 emit.progress("Cleaning up")
                 builder.cleanup()
@@ -409,12 +360,56 @@ def _monitor_and_complete(  # noqa: PLR0912, PLR0915
         )
         return return_code
 
-    def _get_project_build_fors(self) -> list[str]:
-        """Get a unique list of build-for architectures from the project.
+    def _get_archs(self, build_fors: list[str]) -> list[str]:
+        """Get the architectures to build for.
+
+        If the project contains a ``platforms`` or ``architectures`` key, then project's
+        build plan is used to determine the architectures to build for. The build plan
+        can be filtered using the ``--build-for`` argument.
+
+        If the project doesn't contain a ``platforms`` or ``architectures`` key, then
+        the architectures to build for are defined by the ``--build-for`` argument.
+
+        If there are no architectures defined in the project or as arguments, then the
+        default behavior is to build for the host architecture of the local machine.
+
+        :param build_fors: A list of build-for entries.
+
+        :raises EmptyBuildPlanError: If the build plan is filtered to an empty list.
+        :raises RemoteBuildError: If an unsupported architecture is provided.
 
         :returns: A list of architectures.
         """
-        build_fors = set({build_info.build_for for build_info in self.build_plan})
-        emit.debug(f"Parsed build-for architectures from build plan: {build_fors}")
+        archs: list[str] = []
+        if self.project.platforms or self.project._architectures_in_yaml:
+            # if the project has platforms, then `--build-for` acts as a filter
+            if build_fors:
+                emit.debug("Filtering the build plan using the '--build-for' argument.")
+                for build_for in build_fors:
+                    filtered_build_plan = filter_plan(
+                        self.build_plan,
+                        platform=None,
+                        build_for=build_for,
+                        host_arch=None,
+                    )
+                    archs.extend([info.build_for for info in filtered_build_plan])
+                    if not archs:
+                        raise craft_application.errors.EmptyBuildPlanError()
+            else:
+                emit.debug("Using the project's build plan")
+                archs = [build_info.build_for for build_info in self.build_plan]
+        # No architectures in the project means '--build-for' no longer acts as a filter.
+        # Instead, it defines the architectures to build for.
+        elif build_fors:
+            emit.debug("Using '--build-for' as the list of architectures to build for")
+            archs = build_fors
+        # default is to build for the host architecture
+        else:
+            archs = [str(DebianArchitecture.from_host())]
+            emit.debug(
+                f"Using host architecture {archs[0]} because no architectures were "
+                "defined in the project or as a command-line argument."
+            )
 
-        return list(build_fors)
+        emit.debug(f"Architectures to build for: {humanize_list(archs, 'and')}")
+        return archs

snapcraft_legacy/internal/repo/_base.py

@@ -346,6 +346,10 @@ def fix_pkg_config(
     - From snaps built locally: `<local-path-to-project>/stage`
     - Built during the build stage: the install directory
 
+    But if the prefix begins with `pcfiledir` variable, it must be kept as-is,
+    because that variable refers to the current location of the .pc file. It
+    allows to create "relocatable" pkgconfig files, so no changes are required.
+
     :param pkg_config_file: pkg-config (.pc) file to modify
     :param prefix_prepend: directory to prepend to the prefix
     :param prefix_trim: directory to remove from prefix
@@ -358,10 +362,17 @@ def fix_pkg_config(
         f"^prefix=(?P<trim>{'|'.join(prefixes_to_trim)})(?P<prefix>.*)"
     )
     pattern = re.compile("^prefix=(?P<prefix>.*)")
+    pattern_pcfiledir = re.compile("^prefix *= *[$]{pcfiledir}.*")
 
     # process .pc file
     with fileinput.input(pkg_config_file, inplace=True) as input_file:
         for line in input_file:
+            # If the prefix begins with ${pcfiledir} statement, this is
+            # a position-independent (thus, "relocatable") .pc file, so
+            # no changes are required.
+            if pattern_pcfiledir.search(line) is not None:
+                print(line, end="")
+                continue
             match = pattern.search(line)
             match_trim = pattern_trim.search(line)
 

tests/legacy/unit/repo/test_base.py

@@ -369,6 +369,22 @@ def test_fix_pkg_config_trim_prefix(
     )
 
 
+def test_fix_pkg_config_with_pcfiledir(
+    tmpdir,
+    pkg_config_file,
+    expected_pkg_config_content,
+):
+    """Verify prefixes that begin with ${pcfiledir} aren't modified."""
+    pc_file = tmpdir / "my-file.pc"
+    pkg_config_file(pc_file, "${pcfiledir}/../../..")
+
+    fix_pkg_config(tmpdir, pc_file)
+
+    assert pc_file.read_text(encoding="utf-8") == expected_pkg_config_content(
+        "${pcfiledir}/../../.."
+    )
+
+
 def test_normalize_fix_pkg_config(tmpdir, pkg_config_file, expected_pkg_config_content):
     """Verify normalization fixes pkg-config files."""
     pc_file = tmpdir / "my-file.pc"

tests/spread/core20/remote-build-legacy/snaps/all/expected-snaps.txt

@@ -1 +1 @@
-test-snap-all-core20_1.0_all.snap
+all-core20_1.0_all.snap

tests/spread/core20/remote-build-legacy/snaps/all/snapcraft.yaml

@@ -1,4 +1,4 @@
-name: test-snap-all-core20
+name: all-core20
 base: core20
 version: "1.0"
 summary: Test snap for remote build

tests/spread/core20/remote-build-legacy/snaps/archs/expected-snaps.txt

@@ -0,0 +1,2 @@
+archs-core20_1.0_amd64.snap
+archs-core20_1.0_arm64.snap

tests/spread/core20/remote-build-legacy/snaps/architectures/snapcraft.yaml → tests/spread/core20/remote-build-legacy/snaps/archs/snapcraft.yaml

@@ -1,4 +1,4 @@
-name: test-snap-architectures-core20
+name: archs-core20
 base: core20
 version: "1.0"
 summary: Test snap for remote build

tests/spread/core20/remote-build-legacy/snaps/no-archs/expected-snaps.txt

@@ -0,0 +1 @@
+no-archs-core20_1.0_amd64.snap

tests/spread/core20/remote-build-legacy/snaps/no-architectures/snapcraft.yaml → tests/spread/core20/remote-build-legacy/snaps/no-archs/snapcraft.yaml

@@ -1,4 +1,4 @@
-name: test-snap-no-architectures-core20
+name: no-archs-core20
 base: core20
 version: "1.0"
 summary: Test snap for remote build