Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement PEP-517/518 build system locking.
Browse files Browse the repository at this point in the history
Currently build systems are locked if requested, but the lock data is
not yet used at lock use time to set up reproducible sdist builds...

Part 1/2.

Fixes pex-tool#2100
jsirois committed Jan 6, 2025
1 parent 19374c4 commit 577b818
Showing 22 changed files with 658 additions and 202 deletions.
22 changes: 22 additions & 0 deletions pex/build_system/__init__.py
Original file line number Diff line number Diff line change
@@ -3,6 +3,16 @@

from __future__ import absolute_import

from pex.typing import TYPE_CHECKING

if TYPE_CHECKING:
from typing import Tuple

import attr # vendor:skip
else:
from pex.third_party import attr


# The split of PEP-517 / PEP-518 is quite awkward. PEP-518 doesn't really work without also
# specifying a build backend or knowing a default value for one, but the concept is not defined
# until PEP-517. As such, we break this historical? strange division and define the default outside
@@ -11,3 +21,15 @@
# See: https://peps.python.org/pep-0517/#source-trees
DEFAULT_BUILD_BACKEND = "setuptools.build_meta:__legacy__"
DEFAULT_BUILD_REQUIRES = ("setuptools",)


@attr.s(frozen=True)
class BuildSystemTable(object):
requires = attr.ib() # type: Tuple[str, ...]
build_backend = attr.ib(default=DEFAULT_BUILD_BACKEND) # type: str
backend_path = attr.ib(default=()) # type: Tuple[str, ...]


DEFAULT_BUILD_SYSTEM_TABLE = BuildSystemTable(
requires=DEFAULT_BUILD_REQUIRES, build_backend=DEFAULT_BUILD_BACKEND
)
9 changes: 3 additions & 6 deletions pex/build_system/pep_517.py
Original file line number Diff line number Diff line change
@@ -10,11 +10,10 @@

from pex import third_party
from pex.build_system import DEFAULT_BUILD_BACKEND
from pex.build_system.pep_518 import BuildSystem, load_build_system, load_build_system_table
from pex.build_system.pep_518 import BuildSystem, load_build_system
from pex.common import safe_mkdtemp
from pex.dist_metadata import DistMetadata, Distribution, MetadataType
from pex.jobs import Job, SpawnedJob
from pex.orderedset import OrderedSet
from pex.pip.version import PipVersion, PipVersionValue
from pex.resolve.resolvers import Resolver
from pex.result import Error, try_
@@ -257,8 +256,6 @@ def get_requires_for_build_wheel(
):
# type: (...) -> Tuple[str, ...]

build_system_table = try_(load_build_system_table(project_directory))
requires = OrderedSet(build_system_table.requires)
spawned_job = try_(
_invoke_build_hook(
project_directory,
@@ -269,11 +266,11 @@ def get_requires_for_build_wheel(
)
)
try:
requires.update(spawned_job.await_result())
return tuple(spawned_job.await_result())
except Job.Error as e:
if e.exitcode != _HOOK_UNAVAILABLE_EXIT_CODE:
raise e
return tuple(requires)
return ()


def spawn_prepare_metadata(
11 changes: 2 additions & 9 deletions pex/build_system/pep_518.py
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@
import subprocess

from pex import toml
from pex.build_system import DEFAULT_BUILD_BACKEND, DEFAULT_BUILD_REQUIRES
from pex.build_system import DEFAULT_BUILD_BACKEND, DEFAULT_BUILD_SYSTEM_TABLE, BuildSystemTable
from pex.common import REPRODUCIBLE_BUILDS_ENV, CopyMode
from pex.dist_metadata import Distribution
from pex.interpreter import PythonInterpreter
@@ -31,13 +31,6 @@
from pex.third_party import attr


@attr.s(frozen=True)
class BuildSystemTable(object):
requires = attr.ib() # type: Tuple[str, ...]
build_backend = attr.ib(default=DEFAULT_BUILD_BACKEND) # type: str
backend_path = attr.ib(default=()) # type: Tuple[str, ...]


def _read_build_system_table(
pyproject_toml, # type: str
):
@@ -175,7 +168,7 @@ def load_build_system_table(project_directory):
maybe_build_system_table_or_error = _maybe_load_build_system_table(project_directory)
if maybe_build_system_table_or_error is not None:
return maybe_build_system_table_or_error
return BuildSystemTable(requires=DEFAULT_BUILD_REQUIRES, build_backend=DEFAULT_BUILD_BACKEND)
return DEFAULT_BUILD_SYSTEM_TABLE


def load_build_system(
91 changes: 45 additions & 46 deletions pex/cli/commands/lock.py
Original file line number Diff line number Diff line change
@@ -542,6 +542,20 @@ def add_create_lock_options(cls, create_parser):
"extras deps that are never activated, but may trim more in the future."
),
)
create_parser.add_argument(
"--lock-build-systems",
"--no-lock-build-systems",
dest="lock_build_systems",
default=False,
action=HandleBoolAction,
type=bool,
help=(
"When creating a lock that includes sdists, VCS requirements or local project "
"directories that will later need to be built into wheels when using the lock, "
"also lock the build system for each of these source tree artifacts to ensure "
"consistent build environments at future times."
),
)
cls._add_lock_options(create_parser)
cls._add_resolve_options(create_parser)
cls.add_json_options(create_parser, entity="lock", include_switch=False)
@@ -818,6 +832,33 @@ def add_extra_arguments(
) as sync_parser:
cls._add_sync_arguments(sync_parser)

def _get_lock_configuration(self, target_configuration):
# type: (TargetConfiguration) -> Union[LockConfiguration, Error]
if self.options.style is LockStyle.UNIVERSAL:
return LockConfiguration(
style=LockStyle.UNIVERSAL,
requires_python=tuple(
str(interpreter_constraint.requires_python)
for interpreter_constraint in target_configuration.interpreter_constraints
),
target_systems=tuple(self.options.target_systems),
elide_unused_requires_dist=self.options.elide_unused_requires_dist,
lock_build_systems=self.options.lock_build_systems,
)

if self.options.target_systems:
return Error(
"The --target-system option only applies to --style {universal} locks.".format(
universal=LockStyle.UNIVERSAL.value
)
)

return LockConfiguration(
style=self.options.style,
elide_unused_requires_dist=self.options.elide_unused_requires_dist,
lock_build_systems=self.options.lock_build_systems,
)

def _resolve_targets(
self,
action, # type: str
@@ -907,28 +948,7 @@ def _create(self):
target_configuration = target_options.configure(
self.options, pip_configuration=pip_configuration
)
if self.options.style == LockStyle.UNIVERSAL:
lock_configuration = LockConfiguration(
style=LockStyle.UNIVERSAL,
requires_python=tuple(
str(interpreter_constraint.requires_python)
for interpreter_constraint in target_configuration.interpreter_constraints
),
target_systems=tuple(self.options.target_systems),
elide_unused_requires_dist=self.options.elide_unused_requires_dist,
)
elif self.options.target_systems:
return Error(
"The --target-system option only applies to --style {universal} locks.".format(
universal=LockStyle.UNIVERSAL.value
)
)
else:
lock_configuration = LockConfiguration(
style=self.options.style,
elide_unused_requires_dist=self.options.elide_unused_requires_dist,
)

lock_configuration = try_(self._get_lock_configuration(target_configuration))
targets = try_(
self._resolve_targets(
action="creating",
@@ -1491,8 +1511,8 @@ def process_req_edits(
lock_file=attr.evolve(
lock_file,
pex_version=__version__,
requirements=SortedTuple(requirements_by_project_name.values(), key=str),
constraints=SortedTuple(constraints_by_project_name.values(), key=str),
requirements=SortedTuple(requirements_by_project_name.values()),
constraints=SortedTuple(constraints_by_project_name.values()),
locked_resolves=SortedTuple(
resolve_update.updated_resolve for resolve_update in lock_update.resolves
),
@@ -1576,28 +1596,7 @@ def _sync(self):
target_configuration = target_options.configure(
self.options, pip_configuration=pip_configuration
)
if self.options.style == LockStyle.UNIVERSAL:
lock_configuration = LockConfiguration(
style=LockStyle.UNIVERSAL,
requires_python=tuple(
str(interpreter_constraint.requires_python)
for interpreter_constraint in target_configuration.interpreter_constraints
),
target_systems=tuple(self.options.target_systems),
elide_unused_requires_dist=self.options.elide_unused_requires_dist,
)
elif self.options.target_systems:
return Error(
"The --target-system option only applies to --style {universal} locks.".format(
universal=LockStyle.UNIVERSAL.value
)
)
else:
lock_configuration = LockConfiguration(
style=self.options.style,
elide_unused_requires_dist=self.options.elide_unused_requires_dist,
)

lock_configuration = try_(self._get_lock_configuration(target_configuration))
lock_file_path = self.options.lock
if os.path.exists(lock_file_path):
build_configuration = pip_configuration.build_configuration
17 changes: 15 additions & 2 deletions pex/dist_metadata.py
Original file line number Diff line number Diff line change
@@ -730,7 +730,8 @@ def __str__(self):
)


@attr.s(frozen=True)
@functools.total_ordering
@attr.s(frozen=True, order=False)
class Constraint(object):
@classmethod
def parse(
@@ -849,8 +850,14 @@ def as_requirement(self):
# type: () -> Requirement
return Requirement(name=self.name, specifier=self.specifier, marker=self.marker)

def __lt__(self, other):
# type: (Any) -> bool
if not isinstance(other, Constraint):
return NotImplemented
return self._str < other._str

@attr.s(frozen=True)

@attr.s(frozen=True, order=False)
class Requirement(Constraint):
@classmethod
def parse(
@@ -899,6 +906,12 @@ def as_constraint(self):
# type: () -> Constraint
return Constraint(name=self.name, specifier=self.specifier, marker=self.marker)

def __lt__(self, other):
# type: (Any) -> bool
if not isinstance(other, Requirement):
return NotImplemented
return self._str < other._str


# N.B.: DistributionMetadata can have an expensive hash when a distribution has many requirements;
# so we cache the hash. See: https://github.com/pex-tool/pex/issues/1928
12 changes: 8 additions & 4 deletions pex/pip/vcs.py
Original file line number Diff line number Diff line change
@@ -7,6 +7,8 @@
import re

from pex import hashing
from pex.build_system import BuildSystemTable
from pex.build_system.pep_518 import load_build_system_table
from pex.common import is_pyc_dir, is_pyc_file, open_zip, temporary_dir
from pex.hashing import Sha256
from pex.pep_440 import Version
@@ -61,24 +63,24 @@ def fingerprint_downloaded_vcs_archive(
version, # type: str
vcs, # type: VCS.Value
):
# type: (...) -> Tuple[Fingerprint, str]
# type: (...) -> Tuple[Fingerprint, BuildSystemTable, str]

archive_path = try_(
_find_built_source_dist(
build_dir=download_dir, project_name=ProjectName(project_name), version=Version(version)
)
)
digest = Sha256()
digest_vcs_archive(archive_path=archive_path, vcs=vcs, digest=digest)
return Fingerprint.from_digest(digest), archive_path
build_system_table = digest_vcs_archive(archive_path=archive_path, vcs=vcs, digest=digest)
return Fingerprint.from_digest(digest), build_system_table, archive_path


def digest_vcs_archive(
archive_path, # type: str
vcs, # type: VCS.Value
digest, # type: HintedDigest
):
# type: (...) -> None
# type: (...) -> BuildSystemTable

# All VCS requirements are prepared as zip archives as encoded in:
# `pip._internal.req.req_install.InstallRequirement.archive`.
@@ -109,3 +111,5 @@ def digest_vcs_archive(
),
file_filter=lambda f: not is_pyc_file(f),
)

return try_(load_build_system_table(chroot))
Loading

0 comments on commit 577b818

Please sign in to comment.