stop using the json API at pypi

Author: dimbleby

src/poetry/repositories/http_repository.py

@@ -37,6 +37,8 @@
 
 if TYPE_CHECKING:
     from packaging.utils import NormalizedName
+    from poetry.core.constraints.version import Version
+    from poetry.core.packages.package import Package
     from poetry.core.packages.utils.link import Link
 
     from poetry.repositories.link_sources.base import LinkSource
@@ -91,6 +93,36 @@ def certificates(self) -> RepositoryCertificateConfig:
     def authenticated_url(self) -> str:
         return self._authenticator.authenticated_url(url=self.url)
 
+    def find_links_for_package(self, package: Package) -> list[Link]:
+        try:
+            page = self.get_page(package.name)
+        except PackageNotFound:
+            return []
+
+        return list(page.links_for_version(package.name, package.version))
+
+    def _get_release_info(
+        self, name: NormalizedName, version: Version
+    ) -> dict[str, Any]:
+        page = self.get_page(name)
+
+        links = list(page.links_for_version(name, version))
+        yanked = page.yanked(name, version)
+
+        return self._links_to_data(
+            links,
+            PackageInfo(
+                name=name,
+                version=version.text,
+                summary="",
+                requires_dist=[],
+                requires_python=None,
+                files=[],
+                yanked=yanked,
+                cache_version=str(self.CACHE_VERSION),
+            ),
+        )
+
     def _download(
         self, url: str, dest: Path, *, raise_accepts_ranges: bool = False
     ) -> None:

src/poetry/repositories/legacy_repository.py

@@ -3,13 +3,11 @@
 from contextlib import suppress
 from functools import cached_property
 from typing import TYPE_CHECKING
-from typing import Any
 
 import requests.adapters
 
 from poetry.core.packages.package import Package
 
-from poetry.inspection.info import PackageInfo
 from poetry.repositories.exceptions import PackageNotFound
 from poetry.repositories.http_repository import HTTPRepository
 from poetry.repositories.link_sources.html import SimpleRepositoryPage
@@ -20,7 +18,6 @@
     from packaging.utils import NormalizedName
     from poetry.core.constraints.version import Version
     from poetry.core.constraints.version import VersionConstraint
-    from poetry.core.packages.utils.link import Link
 
     from poetry.config.config import Config
 
@@ -65,14 +62,6 @@ def package(
 
             return package
 
-    def find_links_for_package(self, package: Package) -> list[Link]:
-        try:
-            page = self.get_page(package.name)
-        except PackageNotFound:
-            return []
-
-        return list(page.links_for_version(package.name, package.version))
-
     def _find_packages(
         self, name: NormalizedName, constraint: VersionConstraint
     ) -> list[Package]:
@@ -103,28 +92,6 @@ def _find_packages(
             for version, yanked in versions
         ]
 
-    def _get_release_info(
-        self, name: NormalizedName, version: Version
-    ) -> dict[str, Any]:
-        page = self.get_page(name)
-
-        links = list(page.links_for_version(name, version))
-        yanked = page.yanked(name, version)
-
-        return self._links_to_data(
-            links,
-            PackageInfo(
-                name=name,
-                version=version.text,
-                summary="",
-                requires_dist=[],
-                requires_python=None,
-                files=[],
-                yanked=yanked,
-                cache_version=str(self.CACHE_VERSION),
-            ),
-        )
-
     def _get_page(self, name: NormalizedName) -> SimpleRepositoryPage:
         if not (response := self._get_response(f"/{name}/")):
             raise PackageNotFound(f"Package [{name}] not found.")

src/poetry/repositories/link_sources/json.py

@@ -41,8 +41,13 @@ def _link_cache(self) -> LinkCache:
                         metadata = bool(metadata_value)
                     break
 
+            hashes = file.get("hashes")
             link = Link(
-                url, requires_python=requires_python, yanked=yanked, metadata=metadata
+                url,
+                requires_python=requires_python,
+                hashes=hashes,
+                yanked=yanked,
+                metadata=metadata,
             )
 
             if link.ext not in self.SUPPORTED_FORMATS:

src/poetry/repositories/pypi_repository.py

@@ -10,7 +10,6 @@
 
 from cachecontrol.controller import logger as cache_control_logger
 from poetry.core.packages.package import Package
-from poetry.core.packages.utils.link import Link
 from poetry.core.version.exceptions import InvalidVersion
 
 from poetry.repositories.exceptions import PackageNotFound
@@ -26,18 +25,14 @@
 
 if TYPE_CHECKING:
     from packaging.utils import NormalizedName
-    from poetry.core.constraints.version import Version
     from poetry.core.constraints.version import VersionConstraint
 
-SUPPORTED_PACKAGE_TYPES = {"sdist", "bdist_wheel"}
-
 
 class PyPiRepository(HTTPRepository):
     def __init__(
         self,
         url: str = "https://pypi.org/",
         disable_cache: bool = False,
-        fallback: bool = True,
         pool_size: int = requests.adapters.DEFAULT_POOLSIZE,
     ) -> None:
         super().__init__(
@@ -48,7 +43,6 @@ def __init__(
         )
 
         self._base_url = url
-        self._fallback = fallback
 
     def search(self, query: str) -> list[Package]:
         results = []
@@ -110,79 +104,6 @@ def _get_package_info(self, name: NormalizedName) -> dict[str, Any]:
 
         return info
 
-    def find_links_for_package(self, package: Package) -> list[Link]:
-        json_data = self._get(f"pypi/{package.name}/{package.version}/json")
-        if json_data is None:
-            return []
-
-        links = []
-        for url in json_data["urls"]:
-            if url["packagetype"] in SUPPORTED_PACKAGE_TYPES:
-                h = f"sha256={url['digests']['sha256']}"
-                links.append(Link(url["url"] + "#" + h, yanked=self._get_yanked(url)))
-
-        return links
-
-    def _get_release_info(
-        self, name: NormalizedName, version: Version
-    ) -> dict[str, Any]:
-        from poetry.inspection.info import PackageInfo
-
-        self._log(f"Getting info for {name} ({version}) from PyPI", "debug")
-
-        json_data = self._get(f"pypi/{name}/{version}/json")
-        if json_data is None:
-            raise PackageNotFound(f"Package [{name}] not found.")
-
-        info = json_data["info"]
-
-        data = PackageInfo(
-            name=info["name"],
-            version=info["version"],
-            summary=info["summary"],
-            requires_dist=info["requires_dist"],
-            requires_python=info["requires_python"],
-            yanked=self._get_yanked(info),
-            cache_version=str(self.CACHE_VERSION),
-        )
-
-        try:
-            version_info = json_data["urls"]
-        except KeyError:
-            version_info = []
-
-        files = info.get("files", [])
-        for file_info in version_info:
-            if file_info["packagetype"] in SUPPORTED_PACKAGE_TYPES:
-                files.append(
-                    {
-                        "file": file_info["filename"],
-                        "hash": "sha256:" + file_info["digests"]["sha256"],
-                    }
-                )
-        data.files = files
-
-        if self._fallback and data.requires_dist is None:
-            self._log(
-                "No dependencies found, downloading metadata and/or archives",
-                level="debug",
-            )
-            # No dependencies set (along with other information)
-            # This might be due to actually no dependencies
-            # or badly set metadata when uploading.
-            # So, we need to make sure there is actually no
-            # dependencies by introspecting packages.
-            page = self.get_page(name)
-            links = list(page.links_for_version(name, version))
-            info = self._get_info_from_links(links)
-
-            data.requires_dist = info.requires_dist
-
-            if not data.requires_python:
-                data.requires_python = info.requires_python
-
-        return data.asdict()
-
     def _get_page(self, name: NormalizedName) -> SimpleJsonPage:
         source = self._base_url + f"simple/{name}/"
         info = self.get_package_info(name)

tests/installation/fixtures/with-pypi-repository.test

@@ -47,7 +47,7 @@ name = "pluggy"
 version = "0.6.0"
 description = "plugin and hook calling mechanisms for python"
 optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+python-versions = ">=2.7,<3.0.dev0 || >=3.4.dev0"
 files = [
     {file = "pluggy-0.6.0-py2-none-any.whl", hash = "sha256:9b835f86bfe5498c87ace7f4899cb1b0c40e71c9277377f6851c74a307879285"},
     {file = "pluggy-0.6.0-py3-none-any.whl", hash = "sha256:8c646771f5eab7557d1f3924077c55408e86bdfb700f7d86a6d83abeabff4c66"},

tests/installation/test_executor.py

@@ -126,7 +126,6 @@ def io_not_decorated() -> BufferedIO:
 def pool(pypi_repository: PyPiRepository) -> RepositoryPool:
     pool = RepositoryPool()
 
-    pypi_repository._fallback = True
     pool.add_repository(pypi_repository)
 
     return pool

tests/installation/test_installer.py

@@ -28,6 +28,7 @@
 from poetry.repositories import RepositoryPool
 from poetry.repositories.installed_repository import InstalledRepository
 from poetry.toml.file import TOMLFile
+from poetry.utils._compat import WINDOWS
 from poetry.utils.env import MockEnv
 from poetry.utils.env import NullEnv
 from tests.helpers import MOCK_DEFAULT_GIT_REVISION
@@ -1900,8 +1901,6 @@ def test_installer_required_extras_should_not_be_removed_when_updating_single_de
     config: Config,
     pypi_repository: PyPiRepository,
 ) -> None:
-    mocker.patch("sys.platform", "darwin")
-
     pool = RepositoryPool()
     pool.add_repository(pypi_repository)
 
@@ -1954,7 +1953,8 @@ def test_installer_required_extras_should_not_be_removed_when_updating_single_de
     result = installer.run()
     assert result == 0
 
-    assert installer.executor.installations_count == 7
+    installations = 8 if WINDOWS else 7
+    assert installer.executor.installations_count == installations
     assert installer.executor.updates_count == 0
     assert installer.executor.removals_count == 0
 

tests/puzzle/test_solver.py

@@ -3016,25 +3016,6 @@ def test_solver_can_solve_with_legacy_repository_using_proper_python_compatible_
     )
 
 
-def test_solver_skips_invalid_versions(
-    package: ProjectPackage, io: NullIO, pypi_repository: PyPiRepository
-) -> None:
-    package.python_versions = "^3.9"
-
-    pool = RepositoryPool([pypi_repository])
-
-    solver = Solver(package, pool, [], [], io)
-
-    package.add_dependency(Factory.create_dependency("six-unknown-version", "^1.11"))
-
-    transaction = solver.solve()
-
-    check_solver_result(
-        transaction,
-        [{"job": "install", "package": get_package("six-unknown-version", "1.11.0")}],
-    )
-
-
 def test_multiple_constraints_on_root(
     package: ProjectPackage, solver: Solver, repo: Repository
 ) -> None:
@@ -4707,6 +4688,25 @@ def test_solver_resolves_duplicate_dependency_in_extra(
     )
 
 
+def test_solver_skips_invalid_versions(
+    package: ProjectPackage, io: NullIO, pypi_repository: PyPiRepository
+) -> None:
+    package.python_versions = "^3.9"
+
+    pool = RepositoryPool([pypi_repository])
+
+    solver = Solver(package, pool, [], [], io)
+
+    package.add_dependency(Factory.create_dependency("six-unknown-version", "^1.11"))
+
+    transaction = solver.solve()
+
+    check_solver_result(
+        transaction,
+        [{"job": "install", "package": get_package("six-unknown-version", "1.11.0")}],
+    )
+
+
 def test_solver_resolves_duplicate_dependencies_with_restricted_extras(
     package: ProjectPackage,
     pool: RepositoryPool,