Skip to content

Commit cbc9fb4

Browse files
jsiroisjsirois
committed
Probably mostly working locked build system use.
1 parent 19d04d8 commit cbc9fb4

File tree

9 files changed

+492
-208
lines changed

9 files changed

+492
-208
lines changed

‎pex/build_system/__init__.py‎

Lines changed: 175 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,26 @@
33

44
from __future__ import absolute_import
55

6+
import json
7+
import os
8+
import subprocess
9+
from textwrap import dedent
10+
11+
from pex.common import REPRODUCIBLE_BUILDS_ENV, CopyMode, safe_mkdtemp
12+
from pex.dist_metadata import Distribution
13+
from pex.interpreter import PythonInterpreter
14+
from pex.jobs import Job, SpawnedJob
15+
from pex.pex import PEX
16+
from pex.pex_bootstrapper import VenvPex, ensure_venv
17+
from pex.pex_builder import PEXBuilder
18+
from pex.result import Error
619
from pex.typing import TYPE_CHECKING
20+
from pex.variables import ENV
21+
from pex.venv.bin_path import BinPath
22+
from pex.venv.virtualenv import Virtualenv
723

824
if TYPE_CHECKING:
9-
from typing import Tuple
25+
from typing import Any, Iterable, Mapping, Optional, Tuple, Union
1026

1127
import attr # vendor:skip
1228
else:
@@ -33,3 +49,161 @@ class BuildSystemTable(object):
3349
DEFAULT_BUILD_SYSTEM_TABLE = BuildSystemTable(
3450
requires=DEFAULT_BUILD_REQUIRES, build_backend=DEFAULT_BUILD_BACKEND
3551
)
52+
53+
54+
# Exit code 75 is EX_TEMPFAIL defined in /usr/include/sysexits.h
55+
# this seems an appropriate signal of DNE vs execute and fail.
56+
_HOOK_UNAVAILABLE_EXIT_CODE = 75
57+
58+
59+
@attr.s(frozen=True)
60+
class BuildSystem(object):
61+
@classmethod
62+
def create(
63+
cls,
64+
interpreter, # type: PythonInterpreter
65+
requires, # type: Iterable[str]
66+
resolved, # type: Iterable[Distribution]
67+
build_backend, # type: str
68+
backend_path, # type: Tuple[str, ...]
69+
extra_requirements=None, # type: Optional[Iterable[str]]
70+
use_system_time=False, # type: bool
71+
**extra_env # type: str
72+
):
73+
# type: (...) -> Union[BuildSystem, Error]
74+
pex_builder = PEXBuilder(copy_mode=CopyMode.SYMLINK)
75+
pex_builder.info.venv = True
76+
pex_builder.info.venv_site_packages_copies = True
77+
pex_builder.info.venv_bin_path = BinPath.PREPEND
78+
# Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect.
79+
pex_builder.info.venv_hermetic_scripts = False
80+
for req in requires:
81+
pex_builder.add_requirement(req)
82+
for dist in resolved:
83+
pex_builder.add_distribution(dist)
84+
pex_builder.freeze(bytecode_compile=False)
85+
venv_pex = ensure_venv(PEX(pex_builder.path(), interpreter=interpreter))
86+
if extra_requirements:
87+
# N.B.: We install extra requirements separately instead of having them resolved and
88+
# handed in with the `resolved` above because there are cases in the wild where the
89+
# build system requires (PEP-518) and the results of PEP-517 `get_requires_for_*` can
90+
# return overlapping requirements. Pip will error for overlaps complaining of duplicate
91+
# requirements if we attempt to resolve all the requirements at once; so we instead
92+
# resolve and install in two phases. This obviously has problems! That said, it is, in
93+
# fact, how Pip's internal PEP-517 build frontend works; so we emulate that.
94+
virtualenv = Virtualenv(venv_pex.venv_dir)
95+
# Python 3.5 comes with Pip 9.0.1 which is pretty broken: it doesn't work with our test
96+
# cases; so we upgrade.
97+
# For Python 2.7 we use virtualenv (there is no -m venv built into Python) and that
98+
# comes with Pip 22.0.2, Python 3.6 comes with Pip 18.1 and Python 3.7 comes with
99+
# Pip 22.04 and the default Pips only get newer with newer version of Pythons. These all
100+
# work well enough for our test cases and, in general, they should work well enough with
101+
# the Python they come paired with.
102+
upgrade_pip = virtualenv.interpreter.version[:2] == (3, 5)
103+
virtualenv.ensure_pip(upgrade=upgrade_pip)
104+
with open(os.devnull, "wb") as dev_null:
105+
_, process = virtualenv.interpreter.open_process(
106+
args=[
107+
"-m",
108+
"pip",
109+
"install",
110+
"--ignore-installed",
111+
"--no-user",
112+
"--no-warn-script-location",
113+
]
114+
+ list(extra_requirements),
115+
stdout=dev_null,
116+
stderr=subprocess.PIPE,
117+
)
118+
_, stderr = process.communicate()
119+
if process.returncode != 0:
120+
return Error(
121+
"Failed to install extra requirement in venv at {venv_dir}: "
122+
"{extra_requirements}\nSTDERR:\n{stderr}".format(
123+
venv_dir=venv_pex.venv_dir,
124+
extra_requirements=", ".join(extra_requirements),
125+
stderr=stderr.decode("utf-8"),
126+
)
127+
)
128+
129+
# Ensure all PEX* env vars are stripped except for PEX_ROOT and PEX_VERBOSE. We want folks
130+
# to be able to steer the location of the cache and the logging verbosity, but nothing else.
131+
# We control the entry-point, etc. of the PEP-518 build backend venv for internal use.
132+
with ENV.strip().patch(PEX_ROOT=ENV.PEX_ROOT, PEX_VERBOSE=str(ENV.PEX_VERBOSE)) as env:
133+
if extra_env:
134+
env.update(extra_env)
135+
if backend_path:
136+
env.update(PEX_EXTRA_SYS_PATH=os.pathsep.join(backend_path))
137+
if not use_system_time:
138+
env.update(REPRODUCIBLE_BUILDS_ENV)
139+
return cls(
140+
venv_pex=venv_pex, build_backend=build_backend, requires=tuple(requires), env=env
141+
)
142+
143+
venv_pex = attr.ib() # type: VenvPex
144+
build_backend = attr.ib() # type: str
145+
requires = attr.ib() # type: Tuple[str, ...]
146+
env = attr.ib() # type: Mapping[str, str]
147+
148+
def invoke_build_hook(
149+
self,
150+
project_directory, # type: str
151+
hook_method, # type: str
152+
hook_args=(), # type: Iterable[Any]
153+
hook_kwargs=None, # type: Optional[Mapping[str, Any]]
154+
):
155+
# type: (...) -> Union[SpawnedJob[Any], Error]
156+
157+
# The interfaces are spec'd here: https://peps.python.org/pep-0517
158+
build_backend_module, _, _ = self.build_backend.partition(":")
159+
build_backend_object = self.build_backend.replace(":", ".")
160+
build_hook_result = os.path.join(
161+
safe_mkdtemp(prefix="pex-pep-517."), "build_hook_result.json"
162+
)
163+
args = self.venv_pex.execute_args(
164+
additional_args=(
165+
"-c",
166+
dedent(
167+
"""\
168+
import json
169+
import sys
170+
171+
import {build_backend_module}
172+
173+
174+
if not hasattr({build_backend_object}, {hook_method!r}):
175+
sys.exit({hook_unavailable_exit_code})
176+
177+
result = {build_backend_object}.{hook_method}(*{hook_args!r}, **{hook_kwargs!r})
178+
with open({result_file!r}, "w") as fp:
179+
json.dump(result, fp)
180+
"""
181+
).format(
182+
build_backend_module=build_backend_module,
183+
build_backend_object=build_backend_object,
184+
hook_method=hook_method,
185+
hook_args=tuple(hook_args),
186+
hook_kwargs=dict(hook_kwargs) if hook_kwargs else {},
187+
hook_unavailable_exit_code=_HOOK_UNAVAILABLE_EXIT_CODE,
188+
result_file=build_hook_result,
189+
),
190+
)
191+
)
192+
process = subprocess.Popen(
193+
args=args,
194+
env=self.env,
195+
cwd=project_directory,
196+
stdout=subprocess.PIPE,
197+
stderr=subprocess.PIPE,
198+
)
199+
return SpawnedJob.file(
200+
Job(
201+
command=args,
202+
process=process,
203+
context="PEP-517:{hook_method} at {project_directory}".format(
204+
hook_method=hook_method, project_directory=project_directory
205+
),
206+
),
207+
output_file=build_hook_result,
208+
result_func=lambda file_content: json.loads(file_content.decode("utf-8")),
209+
)

‎pex/build_system/pep_517.py‎

Lines changed: 11 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,11 @@
33

44
from __future__ import absolute_import
55

6-
import json
76
import os
8-
import subprocess
9-
from textwrap import dedent
107

118
from pex import third_party
12-
from pex.build_system import DEFAULT_BUILD_BACKEND
13-
from pex.build_system.pep_518 import BuildSystem, load_build_system
9+
from pex.build_system import DEFAULT_BUILD_BACKEND, BuildSystem
10+
from pex.build_system.pep_518 import load_build_system
1411
from pex.common import safe_mkdtemp
1512
from pex.dist_metadata import DistMetadata, Distribution, MetadataType
1613
from pex.jobs import Job, SpawnedJob
@@ -134,67 +131,21 @@ def _invoke_build_hook(
134131
)
135132
)
136133

137-
build_system_or_error = _get_build_system(
134+
result = _get_build_system(
138135
target,
139136
resolver,
140137
project_directory,
141138
extra_requirements=hook_extra_requirements,
142139
pip_version=pip_version,
143140
)
144-
if isinstance(build_system_or_error, Error):
145-
return build_system_or_error
146-
build_system = build_system_or_error
147-
148-
# The interfaces are spec'd here: https://peps.python.org/pep-0517
149-
build_backend_module, _, _ = build_system.build_backend.partition(":")
150-
build_backend_object = build_system.build_backend.replace(":", ".")
151-
build_hook_result = os.path.join(safe_mkdtemp(prefix="pex-pep-517."), "build_hook_result.json")
152-
args = build_system.venv_pex.execute_args(
153-
additional_args=(
154-
"-c",
155-
dedent(
156-
"""\
157-
import json
158-
import sys
159-
160-
import {build_backend_module}
161-
162-
163-
if not hasattr({build_backend_object}, {hook_method!r}):
164-
sys.exit({hook_unavailable_exit_code})
165-
166-
result = {build_backend_object}.{hook_method}(*{hook_args!r}, **{hook_kwargs!r})
167-
with open({result_file!r}, "w") as fp:
168-
json.dump(result, fp)
169-
"""
170-
).format(
171-
build_backend_module=build_backend_module,
172-
build_backend_object=build_backend_object,
173-
hook_method=hook_method,
174-
hook_args=tuple(hook_args),
175-
hook_kwargs=dict(hook_kwargs) if hook_kwargs else {},
176-
hook_unavailable_exit_code=_HOOK_UNAVAILABLE_EXIT_CODE,
177-
result_file=build_hook_result,
178-
),
179-
)
180-
)
181-
process = subprocess.Popen(
182-
args=args,
183-
env=build_system.env,
184-
cwd=project_directory,
185-
stdout=subprocess.PIPE,
186-
stderr=subprocess.PIPE,
187-
)
188-
return SpawnedJob.file(
189-
Job(
190-
command=args,
191-
process=process,
192-
context="PEP-517:{hook_method} at {project_directory}".format(
193-
hook_method=hook_method, project_directory=project_directory
194-
),
195-
),
196-
output_file=build_hook_result,
197-
result_func=lambda file_content: json.loads(file_content.decode("utf-8")),
141+
if isinstance(result, Error):
142+
return result
143+
144+
return result.invoke_build_hook(
145+
project_directory=project_directory,
146+
hook_method=hook_method,
147+
hook_args=hook_args,
148+
hook_kwargs=hook_kwargs,
198149
)
199150

200151

0 commit comments

Comments
 (0)