diff --git a/src/poetry/repositories/http_repository.py b/src/poetry/repositories/http_repository.py index 0af90e4230b..56437a590a6 100644 --- a/src/poetry/repositories/http_repository.py +++ b/src/poetry/repositories/http_repository.py @@ -26,6 +26,7 @@ from poetry.repositories.exceptions import PackageNotFoundError from poetry.repositories.exceptions import RepositoryError from poetry.repositories.link_sources.html import HTMLPage +from poetry.repositories.link_sources.json import SimpleJsonPage from poetry.utils.authenticator import Authenticator from poetry.utils.constants import REQUESTS_TIMEOUT from poetry.utils.helpers import HTTPRangeRequestSupportedError @@ -417,11 +418,13 @@ def calculate_sha256(self, link: Link) -> str | None: return f"{required_hash.name}:{required_hash.hexdigest()}" return None - def _get_response(self, endpoint: str) -> requests.Response | None: + def _get_response( + self, endpoint: str, *, headers: dict[str, str] | None = None + ) -> requests.Response | None: url = self._url + endpoint try: response: requests.Response = self.session.get( - url, raise_for_status=False, timeout=REQUESTS_TIMEOUT + url, raise_for_status=False, timeout=REQUESTS_TIMEOUT, headers=headers ) if response.status_code in (401, 403): self._log( @@ -442,8 +445,25 @@ def _get_response(self, endpoint: str) -> requests.Response | None: ) return response + def _get_prefer_json_header(self) -> dict[str, str]: + # Prefer json, but accept anything for backwards compatibility. + # Although the more specific value should be preferred to the less specific one + # according to https://developer.mozilla.org/en-US/docs/Glossary/Quality_values, + # we add a quality value because some servers still prefer html without one. + return {"Accept": "application/vnd.pypi.simple.v1+json, */*;q=0.1"} + + def _is_json_response(self, response: requests.Response) -> bool: + return ( + response.headers.get("Content-Type", "").split(";")[0].strip() + == "application/vnd.pypi.simple.v1+json" + ) + def _get_page(self, name: NormalizedName) -> LinkSource: - response = self._get_response(f"/{name}/") + response = self._get_response( + f"/{name}/", headers=self._get_prefer_json_header() + ) if not response: raise PackageNotFoundError(f"Package [{name}] not found.") + if self._is_json_response(response): + return SimpleJsonPage(response.url, response.json()) return HTMLPage(response.url, response.text) diff --git a/src/poetry/repositories/legacy_repository.py b/src/poetry/repositories/legacy_repository.py index f3f83bbfc01..04f82d0aa04 100644 --- a/src/poetry/repositories/legacy_repository.py +++ b/src/poetry/repositories/legacy_repository.py @@ -12,8 +12,9 @@ from poetry.inspection.info import PackageInfo from poetry.repositories.exceptions import PackageNotFoundError from poetry.repositories.http_repository import HTTPRepository -from poetry.repositories.link_sources.html import HTMLPage -from poetry.repositories.link_sources.html import SimpleRepositoryRootPage +from poetry.repositories.link_sources.base import SimpleRepositoryRootPage +from poetry.repositories.link_sources.html import SimpleRepositoryHTMLRootPage +from poetry.repositories.link_sources.json import SimpleRepositoryJsonRootPage if TYPE_CHECKING: @@ -130,21 +131,21 @@ def _get_release_info( ), ) - def _get_page(self, name: NormalizedName) -> HTMLPage: - if not (response := self._get_response(f"/{name}/")): - raise PackageNotFoundError(f"Package [{name}] not found.") - return HTMLPage(response.url, response.text) - @cached_property def root_page(self) -> SimpleRepositoryRootPage: - if not (response := self._get_response("/")): + if not ( + response := self._get_response("/", headers=self._get_prefer_json_header()) + ): self._log( f"Unable to retrieve package listing from package source {self.name}", level="error", ) return SimpleRepositoryRootPage() - return SimpleRepositoryRootPage(response.text) + if self._is_json_response(response): + return SimpleRepositoryJsonRootPage(response.json()) + + return SimpleRepositoryHTMLRootPage(response.text) def search(self, query: str | list[str]) -> list[Package]: results: list[Package] = [] diff --git a/src/poetry/repositories/link_sources/base.py b/src/poetry/repositories/link_sources/base.py index 10ec64f056c..65c1c2c7dd7 100644 --- a/src/poetry/repositories/link_sources/base.py +++ b/src/poetry/repositories/link_sources/base.py @@ -124,3 +124,24 @@ def yanked(self, name: NormalizedName, version: Version) -> str | bool: @cached_property def _link_cache(self) -> LinkCache: raise NotImplementedError() + + +class SimpleRepositoryRootPage: + """ + This class represents the parsed content of a "simple" repository's root page. + """ + + def search(self, query: str | list[str]) -> list[str]: + results: list[str] = [] + tokens = query if isinstance(query, list) else [query] + + for name in self.package_names: + if any(token in name for token in tokens): + results.append(name) + + return results + + @cached_property + def package_names(self) -> list[str]: + # should be overridden in subclasses + return [] diff --git a/src/poetry/repositories/link_sources/html.py b/src/poetry/repositories/link_sources/html.py index ef13a876395..83adf4944da 100644 --- a/src/poetry/repositories/link_sources/html.py +++ b/src/poetry/repositories/link_sources/html.py @@ -10,6 +10,7 @@ from poetry.core.packages.utils.link import Link from poetry.repositories.link_sources.base import LinkSource +from poetry.repositories.link_sources.base import SimpleRepositoryRootPage from poetry.repositories.parsers.html_page_parser import HTMLPageParser @@ -68,10 +69,11 @@ def _link_cache(self) -> LinkCache: return links -class SimpleRepositoryRootPage: +class SimpleRepositoryHTMLRootPage(SimpleRepositoryRootPage): """ - This class represents the parsed content of a "simple" repository's root page. This follows the - specification laid out in PEP 503. + This class represents the parsed content of the HTML version + of a "simple" repository's root page. + This follows the specification laid out in PEP 503. See: https://peps.python.org/pep-0503/ """ @@ -81,17 +83,6 @@ def __init__(self, content: str | None = None) -> None: parser.feed(content or "") self._parsed = parser.anchors - def search(self, query: str | list[str]) -> list[str]: - results: list[str] = [] - tokens = query if isinstance(query, list) else [query] - - for anchor in self._parsed: - href = anchor.get("href") - if href and any(token in href for token in tokens): - results.append(href.rstrip("/")) - - return results - @cached_property def package_names(self) -> list[str]: results: list[str] = [] diff --git a/src/poetry/repositories/link_sources/json.py b/src/poetry/repositories/link_sources/json.py index f33a679ab28..6311453cfb5 100644 --- a/src/poetry/repositories/link_sources/json.py +++ b/src/poetry/repositories/link_sources/json.py @@ -1,5 +1,7 @@ from __future__ import annotations +import urllib.parse + from collections import defaultdict from functools import cached_property from typing import TYPE_CHECKING @@ -8,6 +10,7 @@ from poetry.core.packages.utils.link import Link from poetry.repositories.link_sources.base import LinkSource +from poetry.repositories.link_sources.base import SimpleRepositoryRootPage if TYPE_CHECKING: @@ -25,8 +28,9 @@ def __init__(self, url: str, content: dict[str, Any]) -> None: def _link_cache(self) -> LinkCache: links: LinkCache = defaultdict(lambda: defaultdict(list)) for file in self.content["files"]: - url = file["url"] + url = self.clean_link(urllib.parse.urljoin(self._url, file["url"])) requires_python = file.get("requires-python") + hashes = file.get("hashes", {}) yanked = file.get("yanked", False) # see https://peps.python.org/pep-0714/#clients @@ -42,7 +46,11 @@ def _link_cache(self) -> LinkCache: break link = Link( - url, requires_python=requires_python, yanked=yanked, metadata=metadata + url, + requires_python=requires_python, + hashes=hashes, + yanked=yanked, + metadata=metadata, ) if link.ext not in self.SUPPORTED_FORMATS: @@ -53,3 +61,26 @@ def _link_cache(self) -> LinkCache: links[pkg.name][pkg.version].append(link) return links + + +class SimpleRepositoryJsonRootPage(SimpleRepositoryRootPage): + """ + This class represents the parsed content of the JSON version + of a "simple" repository's root page. + This follows the specification laid out in PEP 691. + + See: https://peps.python.org/pep-0691/ + """ + + def __init__(self, content: dict[str, Any]) -> None: + self._content = content + + @cached_property + def package_names(self) -> list[str]: + results: list[str] = [] + + for project in self._content.get("projects", []): + if name := project.get("name"): + results.append(name) + + return results diff --git a/tests/console/commands/test_search.py b/tests/console/commands/test_search.py index 1696d15fec6..5e52817e00b 100644 --- a/tests/console/commands/test_search.py +++ b/tests/console/commands/test_search.py @@ -113,9 +113,10 @@ def test_search_only_legacy_repository( tester.execute("ipython") expected = """\ - Package Version Source Description - ipython 5.7.0 legacy - ipython 7.5.0 legacy + Package Version Source Description + ipython 4.1.0rc1 legacy + ipython 5.7.0 legacy + ipython 7.5.0 legacy """ output = clean_output(tester.io.fetch_output()) @@ -133,11 +134,12 @@ def test_search_multiple_queries( tester.execute("ipython isort") expected = """\ - Package Version Source Description - ipython 5.7.0 legacy - ipython 7.5.0 legacy - isort 4.3.4 legacy - isort-metadata 4.3.4 legacy + Package Version Source Description + ipython 4.1.0rc1 legacy + ipython 5.7.0 legacy + ipython 7.5.0 legacy + isort 4.3.4 legacy + isort-metadata 4.3.4 legacy """ output = clean_output(tester.io.fetch_output()) diff --git a/tests/installation/conftest.py b/tests/installation/conftest.py index 316a2756d74..c15e3cb5466 100644 --- a/tests/installation/conftest.py +++ b/tests/installation/conftest.py @@ -21,7 +21,7 @@ def env() -> MockEnv: @pytest.fixture() -def pool(legacy_repository: LegacyRepository) -> RepositoryPool: +def pool(legacy_repository_html: LegacyRepository) -> RepositoryPool: pool = RepositoryPool() pool.add_repository(PyPiRepository(disable_cache=True)) diff --git a/tests/puzzle/test_solver.py b/tests/puzzle/test_solver.py index 961ccfe733a..eb61f28706c 100644 --- a/tests/puzzle/test_solver.py +++ b/tests/puzzle/test_solver.py @@ -56,6 +56,16 @@ ) +@pytest.fixture +def legacy_repository(legacy_repository_html: LegacyRepository) -> LegacyRepository: + """ + Override fixture to only test with the html version of the legacy repository + because the json version has the same packages as the PyPI repository and thus + cause different results in the tests that rely on differences. + """ + return legacy_repository_html + + def set_package_python_versions(provider: Provider, python_versions: str) -> None: provider._package.python_versions = python_versions provider._package_python_constraint = provider._package.python_constraint diff --git a/tests/repositories/fixtures/legacy.py b/tests/repositories/fixtures/legacy.py index 1dc1fb758b3..f35a0eabad9 100644 --- a/tests/repositories/fixtures/legacy.py +++ b/tests/repositories/fixtures/legacy.py @@ -1,9 +1,11 @@ from __future__ import annotations +import json import re from pathlib import Path from typing import TYPE_CHECKING +from typing import Any from urllib.parse import urlparse import pytest @@ -13,14 +15,16 @@ from poetry.repositories.legacy_repository import LegacyRepository from tests.helpers import FIXTURE_PATH_REPOSITORIES_LEGACY +from tests.helpers import FIXTURE_PATH_REPOSITORIES_PYPI if TYPE_CHECKING: from packaging.utils import NormalizedName + from pytest import FixtureRequest from pytest_mock import MockerFixture from requests import PreparedRequest - from poetry.repositories.link_sources.html import HTMLPage + from poetry.repositories.link_sources.base import LinkSource from tests.types import HttpRequestCallback from tests.types import HttpResponse from tests.types import NormalizedNameTransformer @@ -36,6 +40,14 @@ def legacy_repository_directory() -> Path: return FIXTURE_PATH_REPOSITORIES_LEGACY +@pytest.fixture +def legacy_package_json_locations() -> list[Path]: + return [ + FIXTURE_PATH_REPOSITORIES_LEGACY / "json", + FIXTURE_PATH_REPOSITORIES_PYPI / "json", + ] + + @pytest.fixture def legacy_repository_package_names(legacy_repository_directory: Path) -> set[str]: return { @@ -65,6 +77,14 @@ def legacy_repository_index_html( """ +@pytest.fixture +def legacy_repository_index_json( + legacy_repository_directory: Path, legacy_repository_package_names: set[str] +) -> dict[str, Any]: + names = [{"name": name} for name in legacy_repository_package_names] + return {"meta": {"api-version": "1.4"}, "projects": names} + + @pytest.fixture def legacy_repository_url() -> str: return "https://legacy.foo.bar" @@ -91,7 +111,32 @@ def html_callback(request: PreparedRequest) -> HttpResponse: @pytest.fixture -def legacy_repository( +def legacy_repository_json_callback( + legacy_package_json_locations: list[Path], + legacy_repository_index_json: dict[str, Any], +) -> HttpRequestCallback: + def json_callback(request: PreparedRequest) -> HttpResponse: + assert request.url + headers = {"Content-Type": "application/vnd.pypi.simple.v1+json"} + if name := Path(urlparse(request.url).path).name: + fixture = Path() + for location in legacy_package_json_locations: + fixture = location / f"{name}.json" + if fixture.exists(): + break + + if not fixture.exists(): + return 404, {}, b"Not Found" + + return 200, headers, fixture.read_bytes() + + return 200, headers, json.dumps(legacy_repository_index_json).encode("utf-8") + + return json_callback + + +@pytest.fixture +def legacy_repository_html( http: responses.RequestsMock, legacy_repository_url: str, legacy_repository_html_callback: HttpRequestCallback, @@ -106,9 +151,30 @@ def legacy_repository( return LegacyRepository("legacy", legacy_repository_url, disable_cache=True) +@pytest.fixture +def legacy_repository_json( + http: responses.RequestsMock, + legacy_repository_url: str, + legacy_repository_json_callback: HttpRequestCallback, + mock_files_python_hosted: None, +) -> LegacyRepository: + http.add_callback( + responses.GET, + re.compile(r"^https://legacy\.(.*)+/?(.*)?$"), + callback=legacy_repository_json_callback, + ) + + return LegacyRepository("legacy", legacy_repository_url, disable_cache=True) + + +@pytest.fixture(params=["legacy_repository_html", "legacy_repository_json"]) +def legacy_repository(request: FixtureRequest) -> LegacyRepository: + return request.getfixturevalue(request.param) # type: ignore[no-any-return] + + @pytest.fixture def specialized_legacy_repository_mocker( - legacy_repository: LegacyRepository, + legacy_repository_html: LegacyRepository, legacy_repository_url: str, mocker: MockerFixture, ) -> SpecializedLegacyRepositoryMocker: @@ -127,7 +193,7 @@ def mock( ) original_get_page = specialized_repository._get_page - def _mocked_get_page(name: NormalizedName) -> HTMLPage: + def _mocked_get_page(name: NormalizedName) -> LinkSource: return original_get_page( canonicalize_name(f"{name}{transformer_or_suffix}") if isinstance(transformer_or_suffix, str) diff --git a/tests/repositories/fixtures/legacy/black.html b/tests/repositories/fixtures/legacy/black.html index 092dea09c36..8157a6a5794 100644 --- a/tests/repositories/fixtures/legacy/black.html +++ b/tests/repositories/fixtures/legacy/black.html @@ -5,7 +5,9 @@

Links for black

black-19.10b0-py36-none-any.whl + black-19.10b0.tar.gz black-21.11b0-py3-none-any.whl + black-21.11b0.tar.gz diff --git a/tests/repositories/fixtures/legacy/ipython.html b/tests/repositories/fixtures/legacy/ipython.html index 555711f6ebb..54903e21059 100644 --- a/tests/repositories/fixtures/legacy/ipython.html +++ b/tests/repositories/fixtures/legacy/ipython.html @@ -5,6 +5,8 @@

Links for ipython

+ ipython-4.1.0rc1-py2.py3-none-any.whl
+ ipython-4.1.0rc1.tar.gz
ipython-5.7.0-py2-none-any.whl
ipython-5.7.0-py3-none-any.whl
ipython-5.7.0.tar.gz
diff --git a/tests/repositories/fixtures/legacy/json/_readme b/tests/repositories/fixtures/legacy/json/_readme new file mode 100644 index 00000000000..6ae9145252d --- /dev/null +++ b/tests/repositories/fixtures/legacy/json/_readme @@ -0,0 +1 @@ +Files from tests/repositories/fixtures/pypi.org/json are used as fallback! diff --git a/tests/repositories/fixtures/legacy/json/absolute.json b/tests/repositories/fixtures/legacy/json/absolute.json new file mode 100644 index 00000000000..c9597d6b356 --- /dev/null +++ b/tests/repositories/fixtures/legacy/json/absolute.json @@ -0,0 +1,36 @@ +{ + "alternate-locations": [], + "files": [ + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "poetry-0.1.0-py3-none-any.whl", + "hashes": { + "sha256": "1d85132efab8ead3c6f69202843da40a03823992091c29f8d65a31af68940163" + }, + "requires-python": ">=3.6.0", + "url": "https://files.pythonhosted.org/packages/e9/df/0ab4afa9c5d9e6b690c5c27c9f50330b98a7ecfe1185ce2dc1b19188b064/poetry-0.1.0-py3-none-any.whl" + }, + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "poetry-0.1.0.tar.gz", + "hashes": { + "sha256": "db33179244321b0b86c6c3645225ff2062ed3495ca16d0d64b3a5df804a82273" + }, + "requires-python": ">=3.6.0", + "url": "https://files.pythonhosted.org/packages/d5/c5/4efe096ce56505435ccbe8aeefcd5c8c3bb0da211ef8fe58934d087daef2/poetry-0.1.0.tar.gz" + } + ], + "meta": { + "_last-serial": 0, + "api-version": "1.0" + }, + "name": "poetry", + "project-status": { + "status": "active" + }, + "versions": [ + "0.1.0" + ] +} diff --git a/tests/repositories/fixtures/legacy/json/demo.json b/tests/repositories/fixtures/legacy/json/demo.json new file mode 100644 index 00000000000..656e919fb86 --- /dev/null +++ b/tests/repositories/fixtures/legacy/json/demo.json @@ -0,0 +1,22 @@ +{ + "alternate-locations": [], + "files": [ + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "demo-0.1.0.tar.gz", + "url": "https://files.pythonhosted.org/distributions/demo-0.1.0.tar.gz" + } + ], + "meta": { + "_last-serial": 0, + "api-version": "1.0" + }, + "name": "demo", + "project-status": { + "status": "active" + }, + "versions": [ + "0.1.0" + ] +} diff --git a/tests/repositories/fixtures/legacy/json/invalid-version.json b/tests/repositories/fixtures/legacy/json/invalid-version.json new file mode 100644 index 00000000000..dd6b906cc43 --- /dev/null +++ b/tests/repositories/fixtures/legacy/json/invalid-version.json @@ -0,0 +1,37 @@ +{ + "alternate-locations": [], + "files": [ + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "poetry-21.07.28.5ffb65e2ff8067c732e2b178d03b707c7fb27855-py3-none-any.whl", + "hashes": { + "sha256": "1d85132efab8ead3c6f69202843da40a03823992091c29f8d65a31af68940163" + }, + "requires-python": ">=3.6.0", + "url": "poetry-21.07.28.5ffb65e2ff8067c732e2b178d03b707c7fb27855-py3-none-any.whl" + }, + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "poetry-0.1.0-py3-none-any.whl", + "hashes": { + "sha256": "1d85132efab8ead3c6f69202843da40a03823992091c29f8d65a31af68940163" + }, + "requires-python": ">=3.6.0", + "url": "poetry-0.1.0-py3-none-any.whl" + } + ], + "meta": { + "_last-serial": 0, + "api-version": "1.0" + }, + "name": "poetry", + "project-status": { + "status": "active" + }, + "versions": [ + "0.1.0", + "21.07.28.5ffb65e2ff8067c732e2b178d03b707c7fb27855" + ] +} diff --git a/tests/repositories/fixtures/legacy/json/isort-metadata.json b/tests/repositories/fixtures/legacy/json/isort-metadata.json new file mode 100644 index 00000000000..76ef60a1a34 --- /dev/null +++ b/tests/repositories/fixtures/legacy/json/isort-metadata.json @@ -0,0 +1,19 @@ +{ + "name": "isort-metadata", + "files": [ + { + "filename": "isort-metadata-4.3.4-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/1f/2c/non-existent/isort-metadata-4.3.4-py3-none-any.whl", + "dist-info-metadata": { + "sha256": "e360bf0ed8a06390513d50dd5b7e9d635c789853a93b84163f9de4ae0647580c" + }, + "hashes": { + "sha256": "1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af" + } + } + ], + "meta": { + "api-version": "1.0", + "_last-serial": 3575149 + } +} diff --git a/tests/repositories/fixtures/legacy/json/jupyter.json b/tests/repositories/fixtures/legacy/json/jupyter.json new file mode 100644 index 00000000000..12b32e5c3d4 --- /dev/null +++ b/tests/repositories/fixtures/legacy/json/jupyter.json @@ -0,0 +1,22 @@ +{ + "alternate-locations": [], + "files": [ + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "jupyter-1.0.0.tar.gz", + "url": "https://files.pythonhosted.org/packages/c9/a9/371d0b8fe37dd231cf4b2cff0a9f0f25e98f3a73c3771742444be27f2944/jupyter-1.0.0.tar.gz" + } + ], + "meta": { + "_last-serial": 0, + "api-version": "1.0" + }, + "name": "jupyter", + "project-status": { + "status": "active" + }, + "versions": [ + "1.0.0" + ] +} diff --git a/tests/repositories/fixtures/legacy/json/poetry-test-py2-py3-metadata-merge.json b/tests/repositories/fixtures/legacy/json/poetry-test-py2-py3-metadata-merge.json new file mode 100644 index 00000000000..ce837f6addd --- /dev/null +++ b/tests/repositories/fixtures/legacy/json/poetry-test-py2-py3-metadata-merge.json @@ -0,0 +1,28 @@ +{ + "alternate-locations": [], + "files": [ + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "poetry_test_py2_py3_metadata_merge-0.1.0-py2-none-any.whl", + "url": "https://files.pythonhosted.org/packages/52/19/poetry_test_py2_py3_metadata_merge-0.1.0-py2-none-any.whl" + }, + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "poetry_test_py2_py3_metadata_merge-0.1.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/c7/b6/poetry_test_py2_py3_metadata_merge-0.1.0-py3-none-any.whl" + } + ], + "meta": { + "_last-serial": 0, + "api-version": "1.0" + }, + "name": "poetry-test-py2-py3-metadata-merge", + "project-status": { + "status": "active" + }, + "versions": [ + "0.1.0" + ] +} diff --git a/tests/repositories/fixtures/legacy/json/relative.json b/tests/repositories/fixtures/legacy/json/relative.json new file mode 100644 index 00000000000..807c063c1a3 --- /dev/null +++ b/tests/repositories/fixtures/legacy/json/relative.json @@ -0,0 +1,47 @@ +{ + "alternate-locations": [], + "files": [ + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "poetry-0.1.0-py3-none-any.whl", + "hashes": { + "sha256": "1d85132efab8ead3c6f69202843da40a03823992091c29f8d65a31af68940163" + }, + "requires-python": ">=3.6.0", + "url": "poetry-0.1.0-py3-none-any.whl" + }, + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "poetry-0.1.0.tar.gz", + "hashes": { + "sha256": "db33179244321b0b86c6c3645225ff2062ed3495ca16d0d64b3a5df804a82273" + }, + "requires-python": ">=3.6.0", + "url": "poetry-0.1.0.tar.gz" + }, + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "poetry-0.1.1.tar.bz2", + "hashes": { + "sha256": "db33179244321b0b86c6c3645225ff2062ed3495ca16d0d64b3a5df804a82273" + }, + "requires-python": ">=3.6.0", + "url": "poetry-0.1.1.tar.bz2" + } + ], + "meta": { + "_last-serial": 0, + "api-version": "1.0" + }, + "name": "poetry", + "project-status": { + "status": "active" + }, + "versions": [ + "0.1.0", + "0.1.1" + ] +} diff --git a/tests/repositories/fixtures/legacy/json/sqlalchemy-legacy.json b/tests/repositories/fixtures/legacy/json/sqlalchemy-legacy.json new file mode 100644 index 00000000000..6e0289bc429 --- /dev/null +++ b/tests/repositories/fixtures/legacy/json/sqlalchemy-legacy.json @@ -0,0 +1,22 @@ +{ + "alternate-locations": [], + "files": [ + { + "core-metadata": false, + "data-dist-info-metadata": false, + "filename": "sqlalchemy-legacy-4.3.4-py2-none-any.whl", + "url": "https://files.pythonhosted.org/packages/41/d8/a945da414f2adc1d9e2f7d6e7445b27f2be42766879062a2e63616ad4199/sqlalchemy-legacy-4.3.4-py2-none-any.whl" + } + ], + "meta": { + "_last-serial": 0, + "api-version": "1.0" + }, + "name": "sqlalchemy-legacy", + "project-status": { + "status": "active" + }, + "versions": [ + "4.3.4" + ] +} diff --git a/tests/repositories/link_sources/test_base.py b/tests/repositories/link_sources/test_base.py index 50664734713..8dfba17ae0e 100644 --- a/tests/repositories/link_sources/test_base.py +++ b/tests/repositories/link_sources/test_base.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections import defaultdict +from functools import cached_property from typing import TYPE_CHECKING from unittest.mock import PropertyMock @@ -12,6 +13,7 @@ from poetry.core.packages.utils.link import Link from poetry.repositories.link_sources.base import LinkSource +from poetry.repositories.link_sources.base import SimpleRepositoryRootPage if TYPE_CHECKING: @@ -20,6 +22,16 @@ from pytest_mock import MockerFixture +@pytest.fixture +def root_page() -> SimpleRepositoryRootPage: + class TestRootPage(SimpleRepositoryRootPage): + @cached_property + def package_names(self) -> list[str]: + return ["poetry", "poetry-core", "requests", "urllib3"] + + return TestRootPage() + + @pytest.fixture def link_source(mocker: MockerFixture) -> LinkSource: url = "https://example.org" @@ -99,3 +111,18 @@ def test_links_for_version( set(link_source.links_for_version(canonicalize_name("demo"), version)) == expected ) + + +@pytest.mark.parametrize( + "query, expected", + [ + ("poetry", ["poetry", "poetry-core"]), + (["requests", "urllib3"], ["requests", "urllib3"]), + ("lib", ["urllib3"]), + ("nonexistent", []), + ], +) +def test_root_page_search( + root_page: SimpleRepositoryRootPage, query: str | list[str], expected: list[str] +) -> None: + assert root_page.search(query) == expected diff --git a/tests/repositories/link_sources/test_html.py b/tests/repositories/link_sources/test_html.py index 14451ea78dc..3ed7054f88b 100644 --- a/tests/repositories/link_sources/test_html.py +++ b/tests/repositories/link_sources/test_html.py @@ -9,12 +9,35 @@ from poetry.core.packages.utils.link import Link from poetry.repositories.link_sources.html import HTMLPage +from poetry.repositories.link_sources.html import SimpleRepositoryHTMLRootPage if TYPE_CHECKING: from tests.types import HTMLPageGetter +@pytest.fixture +def root_page() -> SimpleRepositoryHTMLRootPage: + names = ["poetry", "poetry-core", "requests"] + hrefs = [f'{name}
' for name in names] + + return SimpleRepositoryHTMLRootPage(f"""\ + + + + Legacy Repository + + + {"".join(hrefs)} + + +""") + + +def test_root_page_package_names(root_page: SimpleRepositoryHTMLRootPage) -> None: + assert root_page.package_names == ["poetry", "poetry-core", "requests"] + + @pytest.mark.parametrize( "attributes, expected_link", [ @@ -60,6 +83,19 @@ def test_link_attributes( assert link.yanked_reason == expected_link.yanked_reason +def test_hash_from_url(html_page_content: HTMLPageGetter) -> None: + anchor = ( + 'demo-1.0.0.whl
' + ) + content = html_page_content(anchor) + page = HTMLPage("https://example.org", content) + + assert len(list(page.links)) == 1 + link = next(iter(page.links)) + assert link.hashes == {"sha256": "abcd1234"} + + @pytest.mark.parametrize( "yanked_attrs, expected", [ @@ -146,24 +182,42 @@ def test_metadata( @pytest.mark.parametrize( - "anchor, base_url, expected", + "anchor, base_url, repo_url, expected", ( ( 'demo-0.1.whl', None, + "https://example.org/simple/", "https://example.org/demo-0.1.whl", ), ( - 'demo-0.1.whl', - "https://example.org/", + 'demo-0.1.whl', + "https://example.org/files/", + "https://example.org/simple/", "https://example.org/demo-0.1.whl", ), + ( + 'demo-0.1.whl', + "https://example.org/files/", + "https://example.org/simple/", + "https://example.org/files/demo-0.1.whl", + ), + ( + 'demo-0.1.whl', + None, + "https://example.org/simple/", + "https://example.org/simple/demo-0.1.whl", + ), ), ) def test_base_url( - html_page_content: HTMLPageGetter, anchor: str, base_url: str | None, expected: str + html_page_content: HTMLPageGetter, + anchor: str, + base_url: str | None, + repo_url: str, + expected: str, ) -> None: content = html_page_content(anchor, base_url) - page = HTMLPage("https://example.org", content) + page = HTMLPage(repo_url, content) link = next(iter(page.links)) assert link.url == expected diff --git a/tests/repositories/link_sources/test_json.py b/tests/repositories/link_sources/test_json.py index 1f11d8b42d0..a66c017c6e5 100644 --- a/tests/repositories/link_sources/test_json.py +++ b/tests/repositories/link_sources/test_json.py @@ -2,7 +2,94 @@ import pytest +from packaging.utils import canonicalize_name +from poetry.core.constraints.version.version import Version + from poetry.repositories.link_sources.json import SimpleJsonPage +from poetry.repositories.link_sources.json import SimpleRepositoryJsonRootPage + + +@pytest.fixture +def root_page() -> SimpleRepositoryJsonRootPage: + names = ["poetry", "poetry-core", "requests"] + + return SimpleRepositoryJsonRootPage( + { + "meta": {"api-version": "1.4"}, + "projects": [{"name": name} for name in names], + } + ) + + +def test_root_page_package_names(root_page: SimpleRepositoryJsonRootPage) -> None: + assert root_page.package_names == ["poetry", "poetry-core", "requests"] + + +def test_attributes() -> None: + content = { + "files": [ + # minimal + {"url": "https://example.org/demo-0.1.whl"}, + # all (with non-default values) + { + "url": "https://example.org/demo-0.1.tar.gz", + "requires-python": ">=3.6", + "yanked": True, + "hashes": {"sha256": "abcd1234"}, + "core-metadata": True, + }, + ] + } + page = SimpleJsonPage("https://example.org", content) + + assert page.url == "https://example.org" + links = list(page.links) + assert len(links) == 2 + + assert links[0].url == "https://example.org/demo-0.1.whl" + assert links[0].requires_python is None + assert links[0].yanked is False + assert links[0].hashes == {} + assert links[0].has_metadata is False + + assert links[1].url == "https://example.org/demo-0.1.tar.gz" + assert links[1].requires_python == ">=3.6" + assert links[1].yanked is True + assert links[1].hashes == {"sha256": "abcd1234"} + assert links[1].has_metadata is True + + +@pytest.mark.parametrize( + ("yanked", "expected"), + [ + ((None, None), False), + ((False, False), False), + ((True, False), False), + ((False, True), False), + ((True, True), True), + (("reason", True), "reason"), + ((True, "reason"), "reason"), + (("reason", "reason"), "reason"), + (("reason 1", "reason 2"), "reason 1\nreason 2"), + ], +) +def test_yanked( + yanked: tuple[str | None, str | None], + expected: bool | str, +) -> None: + content = { + "files": [ + {"url": "https://example.org/demo-0.1.tar.gz", "yanked": yanked[0]}, + {"url": "https://example.org/demo-0.1.whl", "yanked": yanked[1]}, + ] + } + if yanked[0] is None: + del content["files"][0]["yanked"] + if yanked[1] is None: + del content["files"][1]["yanked"] + page = SimpleJsonPage("https://example.org", content) + + assert page.yanked(canonicalize_name("demo"), Version.parse("0.1")) == expected @pytest.mark.parametrize( @@ -77,3 +164,24 @@ def test_metadata( link = next(page.links) assert link.has_metadata is expected_has_metadata assert link.metadata_hashes == expected_metadata_hashes + + +@pytest.mark.parametrize( + ("url", "repo_url", "expected"), + ( + ( + "https://example.org/files/demo-0.1.whl", + "https://example.org/simple/", + "https://example.org/files/demo-0.1.whl", + ), + ( + "demo-0.1.whl", + "https://example.org/simple/", + "https://example.org/simple/demo-0.1.whl", + ), + ), +) +def test_base_url(url: str, repo_url: str, expected: str) -> None: + page = SimpleJsonPage(repo_url, {"files": [{"url": url}]}) + link = next(iter(page.links)) + assert link.url == expected diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py index f9e264f2485..bc012f60d04 100644 --- a/tests/repositories/test_legacy_repository.py +++ b/tests/repositories/test_legacy_repository.py @@ -4,6 +4,7 @@ import re from typing import TYPE_CHECKING +from typing import Any import pytest import requests @@ -490,10 +491,10 @@ def test_package_yanked( def test_package_partial_yank( - legacy_repository: LegacyRepository, + legacy_repository_html: LegacyRepository, legacy_repository_partial_yank: LegacyRepository, ) -> None: - repo = legacy_repository + repo = legacy_repository_html package = repo.package("futures", Version.parse("3.2.0")) assert len(package.files) == 2 @@ -522,7 +523,7 @@ def test_find_links_for_package_yanked( package = repo.package(package_name, Version.parse(version)) links = repo.find_links_for_package(package) - assert len(links) == 1 + assert len(links) == 2 for link in links: assert link.yanked == yanked assert link.yanked_reason == yanked_reason @@ -581,9 +582,7 @@ def test_get_redirected_response_url( repo = MockHttpRepository({"/foo/": 200}, http) redirect_url = "http://legacy.redirect.bar" - def get_mock( - url: str, raise_for_status: bool = True, timeout: int = 5 - ) -> requests.Response: + def get_mock(*args: Any, **kwargs: Any) -> requests.Response: response = requests.Response() response.status_code = 200 response.url = redirect_url + "/foo" @@ -595,6 +594,36 @@ def get_mock( assert page._url == "http://legacy.redirect.bar/foo" +def test_get_page_prefers_json(http: responses.RequestsMock) -> None: + repo = MockHttpRepository({"/foo/": 200}, http) + + _ = repo.get_page("foo") + + accepted = [ + item.strip() + for item in http.calls[-1].request.headers.get("Accept", "").split(",") + ] + preferred = [item for item in accepted if "q=0" not in item.split(";")[-1]] + + assert preferred == ["application/vnd.pypi.simple.v1+json"] + assert any("*/*" in item for item in accepted) + + +def test_root_page_prefers_json(http: responses.RequestsMock) -> None: + repo = MockHttpRepository({"/": 200}, http) + + _ = repo.root_page + + accepted = [ + item.strip() + for item in http.calls[-1].request.headers.get("Accept", "").split(",") + ] + preferred = [item for item in accepted if "q=0" not in item.split(";")[-1]] + + assert preferred == ["application/vnd.pypi.simple.v1+json"] + assert any("*/*" in item for item in accepted) + + @pytest.mark.parametrize( ("repositories",), [