Test PEP 517 hooks

.builders/images/runner_dependencies.txt

@@ -3,3 +3,5 @@ urllib3==2.2.0
 auditwheel==6.0.0; sys_platform == 'linux'
 delvewheel==1.5.2; sys_platform == 'win32'
 delocate==0.13.0; sys_platform == 'darwin'
+wheel==0.45.1
+pathspec==0.12.1

.builders/scripts/build_backend.py

@@ -0,0 +1,129 @@
+# datadog_hatch_wrapper/build.py
+from __future__ import annotations
+
+import os
+import shutil
+from functools import lru_cache
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Optional
+
+
+@lru_cache(maxsize=1)
+def _load_excluded_spec():
+    try:
+        import tomllib
+    except ImportError:
+        import tomli as tomllib
+
+    import pathspec
+
+    cfg = Path(__file__).parent / "files_to_remove.toml"
+    with cfg.open("rb") as f:
+        data = tomllib.load(f)
+    patterns = data.get("excluded_paths", [])
+    return pathspec.PathSpec.from_lines("gitignore", patterns)
+
+
+# PEP 517 functions
+def get_requires_for_build_wheel(config_settings=None):
+    # We need hatchling (the real backend) + wheel tools for repacking + pathspec/tomllib
+    return ["hatchling", "wheel", "pathspec"]
+
+
+def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None):
+    # delegate to hatchling if available
+    try:
+        from hatchling.build import prepare_metadata_for_build_wheel as _pmd
+    except Exception:
+        raise
+    return _pmd(metadata_directory, config_settings=config_settings)
+
+
+def build_wheel(
+    wheel_directory: str, config_settings: Optional[dict] = None, metadata_directory: Optional[str] = None
+) -> str:
+    """
+    Build the wheel using hatchling into a temporary directory, post-process it (remove tests),
+    then move the final wheel into `wheel_directory` and return the filename.
+    """
+
+    from wheel.cli.pack import pack
+    from wheel.cli.unpack import unpack
+
+    # 1) use hatchling to build wheel(s) into a temp dir
+    tmpd = TemporaryDirectory()
+    tmp_path = Path(tmpd.name)
+    try:
+        # call hatchling's build_wheel; its signature matches PEP 517:
+        from hatchling.build import build_wheel as hatch_build_wheel
+
+        hatch_build_wheel(str(tmp_path), config_settings=config_settings)
+    except Exception:
+        tmpd.cleanup()
+        raise
+
+    # 2) find the built wheel
+    wheels = list(tmp_path.glob("*.whl"))
+    if not wheels:
+        tmpd.cleanup()
+        raise RuntimeError("hatchling did not produce a wheel")
+
+    # If there is more than one, choose the one you expect (or iterate)
+    wheel_path = wheels[0]
+
+    # 3) strip tests using same logic as in your PR:
+    #    - check against files_to_remove.toml or a spec
+    spec = _load_excluded_spec()
+
+    def _is_excluded(member: str) -> bool:
+        rel = Path(member).as_posix()
+        return spec.match_file(rel) or spec.match_file(rel + "/")
+
+    # quick check: does wheel contain excluded entries?
+    from zipfile import ZipFile
+
+    with ZipFile(wheel_path, "r") as zf:
+        if not any(_is_excluded(name) for name in zf.namelist()):
+            # nothing to do; just move the wheel to final dir
+            final = Path(wheel_directory) / wheel_path.name
+            shutil.move(str(wheel_path), str(final))
+            tmpd.cleanup()
+            return final.name
+
+    # Unpack, remove excluded files/directories, repack
+    tmp_unpack = TemporaryDirectory()
+    try:
+        unpack(wheel_path, dest=tmp_unpack)
+        unpacked_dir = next(Path(tmp_unpack.name).iterdir())
+
+        # walk bottom-up and remove excluded files/folders
+        for root, dirs, files in os.walk(unpacked_dir, topdown=False):
+            rootp = Path(root)
+            for d in list(dirs):
+                full_dir = rootp / d
+                rel = full_dir.relative_to(unpacked_dir).as_posix()
+                if _is_excluded(rel):
+                    shutil.rmtree(full_dir)
+                    dirs.remove(d)
+            for f in files:
+                rel = (rootp / f).relative_to(unpacked_dir).as_posix()
+                if _is_excluded(rel):
+                    (rootp / f).unlink()
+
+        # repack into wheel_directory (pack writes a new wheel file)
+        pack(unpacked_dir, dest_dir=wheel_directory)
+
+        # pack puts a new wheel file in wheel_directory; pick the freshest one
+        # and return its filename
+        final_wheels = sorted(Path(wheel_directory).glob("*.whl"), key=lambda p: p.stat().st_mtime, reverse=True)
+        if not final_wheels:
+            raise RuntimeError("Failed to repack wheel")
+        final = final_wheels[0]
+        return final.name
+    finally:
+        tmpd.cleanup()
+        try:
+            tmp_unpack and shutil.rmtree(tmp_unpack.name)
+        except Exception:
+            pass

.builders/scripts/build_wheels.py

@@ -1,26 +1,38 @@
 from __future__ import annotations
 
 import argparse
+import email
 import json
 import os
+import re
+import shutil
 import subprocess
 import sys
+import time
+import tomllib
+from functools import cache
+from hashlib import sha256
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import TypedDict
 from zipfile import ZipFile
 
+import pathspec
+import urllib3
 from dotenv import dotenv_values
-from utils import extract_metadata, normalize_project_name
+from utils import iter_wheels
 
 INDEX_BASE_URL = 'https://agent-int-packages.datadoghq.com'
 CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external'
 CUSTOM_BUILT_INDEX = f'{INDEX_BASE_URL}/built'
+UNNORMALIZED_PROJECT_NAME_CHARS = re.compile(r'[-_.]+')
+
 
 class WheelSizes(TypedDict):
     compressed: int
     uncompressed: int
 
+
 if sys.platform == 'win32':
     PY3_PATH = Path('C:\\py3\\Scripts\\python.exe')
     PY2_PATH = Path('C:\\py2\\Scripts\\python.exe')
@@ -62,6 +74,83 @@ def check_process(*args, **kwargs) -> subprocess.CompletedProcess:
     return process
 
 
+def extract_metadata(wheel: Path) -> email.Message:
+    with ZipFile(str(wheel)) as zip_archive:
+        for path in zip_archive.namelist():
+            root = path.split('/', 1)[0]
+            if root.endswith('.dist-info'):
+                dist_info_dir = root
+                break
+        else:
+            message = f'Could not find the `.dist-info` directory in wheel: {wheel.name}'
+            raise RuntimeError(message)
+
+        try:
+            with zip_archive.open(f'{dist_info_dir}/METADATA') as zip_file:
+                metadata_file_contents = zip_file.read().decode('utf-8')
+        except KeyError:
+            message = f'Could not find a `METADATA` file in the `{dist_info_dir}` directory'
+            raise RuntimeError(message) from None
+
+    return email.message_from_string(metadata_file_contents)
+
+
+def normalize_project_name(name: str) -> str:
+    # https://peps.python.org/pep-0503/#normalized-names
+    return UNNORMALIZED_PROJECT_NAME_CHARS.sub('-', name).lower()
+
+
+@cache
+def get_wheel_hashes(project) -> dict[str, str]:
+    retry_wait = 2
+    while True:
+        try:
+            response = urllib3.request(
+                'GET',
+                f'https://pypi.org/simple/{project}',
+                headers={"Accept": "application/vnd.pypi.simple.v1+json"},
+            )
+        except urllib3.exceptions.HTTPError as e:
+            err_msg = f'Failed to fetch hashes for `{project}`: {e}'
+        else:
+            if response.status == 200:
+                break
+
+            err_msg = f'Failed to fetch hashes for `{project}`, status code: {response.status}'
+
+        print(err_msg)
+        print(f'Retrying in {retry_wait} seconds')
+        time.sleep(retry_wait)
+        retry_wait *= 2
+        continue
+
+    data = response.json()
+    return {
+        file['filename']: file['hashes']['sha256']
+        for file in data['files']
+        if file['filename'].endswith('.whl') and 'sha256' in file['hashes']
+    }
+
+
+def wheel_was_built(wheel: Path) -> bool:
+    project_metadata = extract_metadata(wheel)
+    project_name = normalize_project_name(project_metadata['Name'])
+    wheel_hashes = get_wheel_hashes(project_name)
+    if wheel.name not in wheel_hashes:
+        return True
+
+    file_hash = sha256(wheel.read_bytes()).hexdigest()
+    return file_hash != wheel_hashes[wheel.name]
+
+
+def add_dependency(dependencies: dict[str, str], sizes: dict[str, WheelSizes], wheel: Path) -> None:
+    project_metadata = extract_metadata(wheel)
+    project_name = normalize_project_name(project_metadata['Name'])
+    project_version = project_metadata['Version']
+    dependencies[project_name] = project_version
+    sizes[project_name] = {'version': project_version, **calculate_wheel_sizes(wheel)}
+
+
 def calculate_wheel_sizes(wheel_path: Path) -> WheelSizes:
     compressed_size = wheel_path.stat(follow_symlinks=True).st_size
     with ZipFile(wheel_path) as zf:
@@ -92,6 +181,13 @@ def main():
 
     with TemporaryDirectory() as d:
         staged_wheel_dir = Path(d).resolve()
+        staged_built_wheels_dir = staged_wheel_dir / 'built'
+        staged_external_wheels_dir = staged_wheel_dir / 'external'
+
+        # Create the directories
+        staged_built_wheels_dir.mkdir(parents=True, exist_ok=True)
+        staged_external_wheels_dir.mkdir(parents=True, exist_ok=True)
+
         env_vars = dict(os.environ)
         env_vars['PATH'] = f'{python_path.parent}{os.pathsep}{env_vars["PATH"]}'
         env_vars['PIP_WHEEL_DIR'] = str(staged_wheel_dir)
@@ -100,7 +196,7 @@ def main():
         env_vars['DD_ENV_FILE'] = str(ENV_FILE)
 
         # Off is on, see: https://github.com/pypa/pip/issues/5735
-        env_vars['PIP_NO_BUILD_ISOLATION'] = '0'
+        # env_vars['PIP_NO_BUILD_ISOLATION'] = '0'
 
         # Spaces are used to separate multiple values which means paths themselves cannot contain spaces, see:
         # https://github.com/pypa/pip/issues/10114#issuecomment-1880125475
@@ -121,35 +217,47 @@ def main():
         if constraints_file := env_vars.get('PIP_CONSTRAINT'):
             env_vars['PIP_CONSTRAINT'] = path_to_uri(constraints_file)
 
+        print("--------------------------------")
+        print("Building wheels")
+        print("--------------------------------")
         # Fetch or build wheels
         command_args = [
             str(python_path),
             '-m',
             'pip',
+            '-vvv',
             'wheel',
+            '--config-settings=build-backend=scripts.build_backend',
+            f'--config-settings=backend-path={MOUNT_DIR / ".builders"}',
             '-r',
             str(MOUNT_DIR / 'requirements.in'),
             '--wheel-dir',
             str(staged_wheel_dir),
-            # Temporarily removing extra index urls. See below.
-            # '--extra-index-url', CUSTOM_EXTERNAL_INDEX,
+            # '--extra-index-url',
+            # CUSTOM_EXTERNAL_INDEX,
         ]
-        # Temporarily disable extra index urls. There are broken wheels in the gcloud bucket
-        # while working on removing tests from them. Adding extra indices causes undefined behavior
-        # and can pull a broken image, preventing the building from running.
-        # if args.use_built_index:
-        #     command_args.extend(['--extra-index-url', CUSTOM_BUILT_INDEX])
 
         check_process(command_args, env=env_vars)
+        print("--------------------------------")
+        print("Finished building wheels")
+        print("--------------------------------")
+        # Classify wheels
+        for wheel in iter_wheels(staged_wheel_dir):
+            if wheel_was_built(wheel):
+                shutil.move(wheel, staged_built_wheels_dir)
+            else:
+                shutil.move(wheel, staged_external_wheels_dir)
 
         # Repair wheels
         check_process(
             [
                 sys.executable,
                 '-u',
                 str(MOUNT_DIR / 'scripts' / 'repair_wheels.py'),
-                '--source-dir',
-                str(staged_wheel_dir),
+                '--source-built-dir',
+                str(staged_built_wheels_dir),
+                '--source-external-dir',
+                str(staged_external_wheels_dir),
                 '--built-dir',
                 str(built_wheels_dir),
                 '--external-dir',
@@ -167,7 +275,6 @@ def main():
             project_version = project_metadata['Version']
             dependencies[project_name] = project_version
 
-
             sizes[project_name] = {'version': project_version, **calculate_wheel_sizes(wheel)}
 
     output_path = MOUNT_DIR / 'sizes.json'

.builders/scripts/files_to_remove.toml

@@ -0,0 +1,45 @@
+excluded_paths = [
+  # --- Type annotation ---
+  "krb5/**/*.pyi",
+  "krb5/**/py.typed",
+
+  "Cryptodome/**/*.pyi",
+  "Cryptodome/**/py.typed",
+
+  "ddtrace/**/*.pyi",
+  "ddtrace/**/py.typed",
+
+  "pyVmomi/**/*.pyi",
+  "pyVmomi/**/py.typed",
+
+  "gssapi/**/*.pyi",
+  "gssapi/**/py.typed",
+
+  # --- Tests ---
+
+  "idlelib/idle_test/",
+  "bs4/tests/",
+  "Cryptodome/SelfTest/",
+  "gssapi/tests/",
+  "keystoneauth1/tests/",
+  "lazy_loader/tests/",
+  "openstack/tests/",
+  "os_service_types/tests/",
+  "pbr/tests/",
+  "pkg_resources/tests/",
+  "pip/_vendor/colorama/tests/",
+  "psutil/tests/",
+  "requests_unixsocket/tests/",
+  "securesystemslib/_vendor/ed25519/test_data/",
+  "setuptools/_distutils/compilers/C/tests/",
+  "setuptools/_vendor/packaging/tests/",
+  "setuptools/_distutils/tests/",
+  "setuptools/tests/",
+  "simplejson/tests/",
+  "stevedore/tests/",
+  "supervisor/tests/",
+  "/test/",
+  "vertica_python/tests/",
+  "websocket/tests/",
+  "win32com/test/",
+]