Implement PEP-517/518 build system locking.

Currently build systems are locked if requested, but the lock data is
not yet used at lock use time to set up reproducible sdist builds...

Part 1/2.

Fixes #2100

Author: jsirois

pex/build_system/__init__.py

@@ -3,6 +3,16 @@
 
 from __future__ import absolute_import
 
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Tuple
+
+    import attr  # vendor:skip
+else:
+    from pex.third_party import attr
+
+
 # The split of PEP-517 / PEP-518 is quite awkward. PEP-518 doesn't really work without also
 # specifying a build backend or knowing a default value for one, but the concept is not defined
 # until PEP-517. As such, we break this historical? strange division and define the default outside
@@ -11,3 +21,15 @@
 # See: https://peps.python.org/pep-0517/#source-trees
 DEFAULT_BUILD_BACKEND = "setuptools.build_meta:__legacy__"
 DEFAULT_BUILD_REQUIRES = ("setuptools",)
+
+
+@attr.s(frozen=True)
+class BuildSystemTable(object):
+    requires = attr.ib()  # type: Tuple[str, ...]
+    build_backend = attr.ib(default=DEFAULT_BUILD_BACKEND)  # type: str
+    backend_path = attr.ib(default=())  # type: Tuple[str, ...]
+
+
+DEFAULT_BUILD_SYSTEM_TABLE = BuildSystemTable(
+    requires=DEFAULT_BUILD_REQUIRES, build_backend=DEFAULT_BUILD_BACKEND
+)

pex/build_system/pep_517.py

@@ -10,11 +10,10 @@
 
 from pex import third_party
 from pex.build_system import DEFAULT_BUILD_BACKEND
-from pex.build_system.pep_518 import BuildSystem, load_build_system, load_build_system_table
+from pex.build_system.pep_518 import BuildSystem, load_build_system
 from pex.common import safe_mkdtemp
 from pex.dist_metadata import DistMetadata, Distribution, MetadataType
 from pex.jobs import Job, SpawnedJob
-from pex.orderedset import OrderedSet
 from pex.pip.version import PipVersion, PipVersionValue
 from pex.resolve.resolvers import Resolver
 from pex.result import Error, try_
@@ -257,8 +256,6 @@ def get_requires_for_build_wheel(
 ):
     # type: (...) -> Tuple[str, ...]
 
-    build_system_table = try_(load_build_system_table(project_directory))
-    requires = OrderedSet(build_system_table.requires)
     spawned_job = try_(
         _invoke_build_hook(
             project_directory,
@@ -269,11 +266,11 @@ def get_requires_for_build_wheel(
         )
     )
     try:
-        requires.update(spawned_job.await_result())
+        return tuple(spawned_job.await_result())
     except Job.Error as e:
         if e.exitcode != _HOOK_UNAVAILABLE_EXIT_CODE:
             raise e
-    return tuple(requires)
+    return ()
 
 
 def spawn_prepare_metadata(

pex/build_system/pep_518.py

@@ -7,7 +7,7 @@
 import subprocess
 
 from pex import toml
-from pex.build_system import DEFAULT_BUILD_BACKEND, DEFAULT_BUILD_REQUIRES
+from pex.build_system import DEFAULT_BUILD_BACKEND, DEFAULT_BUILD_SYSTEM_TABLE, BuildSystemTable
 from pex.common import REPRODUCIBLE_BUILDS_ENV, CopyMode
 from pex.dist_metadata import Distribution
 from pex.interpreter import PythonInterpreter
@@ -31,13 +31,6 @@
     from pex.third_party import attr
 
 
-@attr.s(frozen=True)
-class BuildSystemTable(object):
-    requires = attr.ib()  # type: Tuple[str, ...]
-    build_backend = attr.ib(default=DEFAULT_BUILD_BACKEND)  # type: str
-    backend_path = attr.ib(default=())  # type: Tuple[str, ...]
-
-
 def _read_build_system_table(
     pyproject_toml,  # type: str
 ):
@@ -175,7 +168,7 @@ def load_build_system_table(project_directory):
     maybe_build_system_table_or_error = _maybe_load_build_system_table(project_directory)
     if maybe_build_system_table_or_error is not None:
         return maybe_build_system_table_or_error
-    return BuildSystemTable(requires=DEFAULT_BUILD_REQUIRES, build_backend=DEFAULT_BUILD_BACKEND)
+    return DEFAULT_BUILD_SYSTEM_TABLE
 
 
 def load_build_system(

pex/cli/commands/lock.py

@@ -611,6 +611,20 @@ def add_create_lock_options(cls, create_parser):
                 "extras deps that are never activated, but may trim more in the future."
             ),
         )
+        create_parser.add_argument(
+            "--lock-build-systems",
+            "--no-lock-build-systems",
+            dest="lock_build_systems",
+            default=False,
+            action=HandleBoolAction,
+            type=bool,
+            help=(
+                "When creating a lock that includes sdists, VCS requirements or local project "
+                "directories that will later need to be built into wheels when using the lock, "
+                "also lock the build system for each of these source tree artifacts to ensure "
+                "consistent build environments at future times."
+            ),
+        )
         cls._add_lock_options(create_parser)
         cls._add_resolve_options(create_parser)
         cls.add_json_options(create_parser, entity="lock", include_switch=False)
@@ -905,6 +919,33 @@ def add_extra_arguments(
         ) as sync_parser:
             cls._add_sync_arguments(sync_parser)
 
+    def _get_lock_configuration(self, target_configuration):
+        # type: (TargetConfiguration) -> Union[LockConfiguration, Error]
+        if self.options.style is LockStyle.UNIVERSAL:
+            return LockConfiguration(
+                style=LockStyle.UNIVERSAL,
+                requires_python=tuple(
+                    str(interpreter_constraint.requires_python)
+                    for interpreter_constraint in target_configuration.interpreter_constraints
+                ),
+                target_systems=tuple(self.options.target_systems),
+                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
+                lock_build_systems=self.options.lock_build_systems,
+            )
+
+        if self.options.target_systems:
+            return Error(
+                "The --target-system option only applies to --style {universal} locks.".format(
+                    universal=LockStyle.UNIVERSAL.value
+                )
+            )
+
+        return LockConfiguration(
+            style=self.options.style,
+            elide_unused_requires_dist=self.options.elide_unused_requires_dist,
+            lock_build_systems=self.options.lock_build_systems,
+        )
+
     def _resolve_targets(
         self,
         action,  # type: str
@@ -987,6 +1028,7 @@ def _merge_project_requirements(
 
     def _locking_configuration(self, pip_configuration):
         # type: (PipConfiguration) -> Union[LockingConfiguration, Error]
+
         requirement_configuration = requirement_options.configure(self.options)
         target_configuration = target_options.configure(
             self.options, pip_configuration=pip_configuration
@@ -998,27 +1040,7 @@ def _locking_configuration(self, pip_configuration):
             )
             requirement_configuration = script_metadata_application.requirement_configuration
             target_configuration = script_metadata_application.target_configuration
-        if self.options.style == LockStyle.UNIVERSAL:
-            lock_configuration = LockConfiguration(
-                style=LockStyle.UNIVERSAL,
-                requires_python=tuple(
-                    str(interpreter_constraint.requires_python)
-                    for interpreter_constraint in target_configuration.interpreter_constraints
-                ),
-                target_systems=tuple(self.options.target_systems),
-                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
-            )
-        elif self.options.target_systems:
-            return Error(
-                "The --target-system option only applies to --style {universal} locks.".format(
-                    universal=LockStyle.UNIVERSAL.value
-                )
-            )
-        else:
-            lock_configuration = LockConfiguration(
-                style=self.options.style,
-                elide_unused_requires_dist=self.options.elide_unused_requires_dist,
-            )
+        lock_configuration = try_(self._get_lock_configuration(target_configuration))
         return LockingConfiguration(
             requirement_configuration,
             target_configuration,
@@ -1840,8 +1862,8 @@ def process_req_edits(
                 lock_file=attr.evolve(
                     lock_file,
                     pex_version=__version__,
-                    requirements=SortedTuple(requirements_by_project_name.values(), key=str),
-                    constraints=SortedTuple(constraints_by_project_name.values(), key=str),
+                    requirements=SortedTuple(requirements_by_project_name.values()),
+                    constraints=SortedTuple(constraints_by_project_name.values()),
                     locked_resolves=SortedTuple(
                         resolve_update.updated_resolve for resolve_update in lock_update.resolves
                     ),

pex/dist_metadata.py

@@ -730,7 +730,8 @@ def __str__(self):
         )
 
 
-@attr.s(frozen=True)
+@functools.total_ordering
+@attr.s(frozen=True, order=False)
 class Constraint(object):
     @classmethod
     def parse(
@@ -849,8 +850,14 @@ def as_requirement(self):
         # type: () -> Requirement
         return Requirement(name=self.name, specifier=self.specifier, marker=self.marker)
 
+    def __lt__(self, other):
+        # type: (Any) -> bool
+        if not isinstance(other, Constraint):
+            return NotImplemented
+        return self._str < other._str
 
-@attr.s(frozen=True)
+
+@attr.s(frozen=True, order=False)
 class Requirement(Constraint):
     @classmethod
     def parse(
@@ -899,6 +906,12 @@ def as_constraint(self):
         # type: () -> Constraint
         return Constraint(name=self.name, specifier=self.specifier, marker=self.marker)
 
+    def __lt__(self, other):
+        # type: (Any) -> bool
+        if not isinstance(other, Requirement):
+            return NotImplemented
+        return self._str < other._str
+
 
 # N.B.: DistributionMetadata can have an expensive hash when a distribution has many requirements;
 # so we cache the hash. See: https://github.com/pex-tool/pex/issues/1928

pex/pip/vcs.py

@@ -7,6 +7,8 @@
 import re
 
 from pex import hashing
+from pex.build_system import BuildSystemTable
+from pex.build_system.pep_518 import load_build_system_table
 from pex.common import is_pyc_dir, is_pyc_file, open_zip, temporary_dir
 from pex.hashing import Sha256
 from pex.pep_440 import Version
@@ -61,24 +63,24 @@ def fingerprint_downloaded_vcs_archive(
     version,  # type: str
     vcs,  # type: VCS.Value
 ):
-    # type: (...) -> Tuple[Fingerprint, str]
+    # type: (...) -> Tuple[Fingerprint, BuildSystemTable, str]
 
     archive_path = try_(
         _find_built_source_dist(
             build_dir=download_dir, project_name=ProjectName(project_name), version=Version(version)
         )
     )
     digest = Sha256()
-    digest_vcs_archive(archive_path=archive_path, vcs=vcs, digest=digest)
-    return Fingerprint.from_digest(digest), archive_path
+    build_system_table = digest_vcs_archive(archive_path=archive_path, vcs=vcs, digest=digest)
+    return Fingerprint.from_digest(digest), build_system_table, archive_path
 
 
 def digest_vcs_archive(
     archive_path,  # type: str
     vcs,  # type: VCS.Value
     digest,  # type: HintedDigest
 ):
-    # type: (...) -> None
+    # type: (...) -> BuildSystemTable
 
     # All VCS requirements are prepared as zip archives as encoded in:
     # `pip._internal.req.req_install.InstallRequirement.archive`.
@@ -109,3 +111,5 @@ def digest_vcs_archive(
             ),
             file_filter=lambda f: not is_pyc_file(f),
         )
+
+        return try_(load_build_system_table(chroot))