Housekeeping

Author: elacuesta

docs/changelog.md

@@ -1,5 +1,12 @@
 # scrapy-playwright changelog
 
+### [v0.0.27](https://github.com/scrapy-plugins/scrapy-playwright/releases/tag/v0.0.27) (2023-07-24)
+
+* Override method only for navigation requests (#177)
+* Pass spider argument to _create_browser_context (#212)
+* await AsyncPlaywright.stop on close (#214)
+
+
 ### [v0.0.26](https://github.com/scrapy-plugins/scrapy-playwright/releases/tag/v0.0.26) (2023-02-01)
 
 * Fix logging (pass extra args instead of updating log record factory)

tests/__init__.py

@@ -1,5 +1,7 @@
 from contextlib import asynccontextmanager
 
+from scrapy import Request
+from scrapy.http.response.html import HtmlResponse
 from scrapy.utils.test import get_crawler
 
 
@@ -19,3 +21,11 @@ async def make_handler(settings_dict: dict):
         yield handler
     finally:
         await handler._close()
+
+
+def assert_correct_response(response: HtmlResponse, request: Request) -> None:
+    assert isinstance(response, HtmlResponse)
+    assert response.request is request
+    assert response.url == request.url
+    assert response.status == 200
+    assert "playwright" in response.flags

tests/conftest.py

@@ -21,7 +21,7 @@ def _is_coroutine(obj):
     return asyncio.iscoroutinefunction(obj) or inspect.isgeneratorfunction(obj)
 
 
-@pytest.mark.tryfirst
+@pytest.hookimpl(tryfirst=True)
 def pytest_pycollect_makeitem(collector, name, obj):
     """A pytest hook to collect asyncio coroutines."""
     if collector.funcnamefilter(name) and _is_coroutine(obj):

tests/test_encoding.py

@@ -9,7 +9,7 @@
 
 from scrapy_playwright.page import PageMethod
 
-from tests import make_handler
+from tests import make_handler, assert_correct_response
 from tests.mockserver import StaticMockServer
 
 
@@ -32,14 +32,6 @@ async def test_page_methods():
     assert str(screenshot) == "<PageMethod for method 'screenshot'>"
 
 
-def assert_correct_response(response: HtmlResponse, request: Request) -> None:
-    assert isinstance(response, HtmlResponse)
-    assert response.request is request
-    assert response.url == request.url
-    assert response.status == 200
-    assert "playwright" in response.flags
-
-
 class MixinPageMethodTestCase:
     @pytest.mark.asyncio
     async def test_page_non_page_method(self, caplog):
@@ -139,11 +131,7 @@ async def test_page_method_infinite_scroll(self):
                 )
                 resp = await handler._download_request(req, Spider("foo"))
 
-            assert isinstance(resp, HtmlResponse)
-            assert resp.request is req
-            assert resp.url == server.urljoin("/scroll.html")
-            assert resp.status == 200
-            assert "playwright" in resp.flags
+            assert_correct_response(resp, req)
             assert len(resp.css("div.quote")) == 30
 
     @pytest.mark.asyncio

tests/test_page_methods.py

@@ -13,12 +13,12 @@
     TimeoutError as PlaywrightTimeoutError,
 )
 from scrapy import Spider, Request, FormRequest
-from scrapy.http import Response, HtmlResponse
+from scrapy.http import Response
 
 from scrapy_playwright.handler import DEFAULT_CONTEXT_NAME
 from scrapy_playwright.page import PageMethod
 
-from tests import make_handler
+from tests import make_handler, assert_correct_response
 from tests.mockserver import MockServer, StaticMockServer
 
 
@@ -44,11 +44,7 @@ async def test_basic_response(self):
                 req = Request(server.urljoin("/index.html"), meta=meta)
                 resp = await handler._download_request(req, Spider("foo"))
 
-            assert isinstance(resp, HtmlResponse)
-            assert resp.request is req
-            assert resp.url == req.url
-            assert resp.status == 200
-            assert "playwright" in resp.flags
+            assert_correct_response(resp, req)
             assert resp.css("a::text").getall() == ["Lorem Ipsum", "Infinite Scroll"]
             assert isinstance(resp.meta["playwright_page"], PlaywrightPage)
             assert resp.meta["playwright_page"].url == resp.url
@@ -64,10 +60,7 @@ async def test_post_request(self):
                 )
                 resp = await handler._download_request(req, Spider("foo"))
 
-            assert resp.request is req
-            assert resp.url == req.url
-            assert resp.status == 200
-            assert "playwright" in resp.flags
+            assert_correct_response(resp, req)
             assert "Request body: foo=bar" in resp.text
 
     @pytest.mark.asyncio
@@ -166,6 +159,7 @@ async def test_route_continue_exception(self, logger):
             route = MagicMock()
             playwright_request = AsyncMock()
             playwright_request.url = scrapy_request.url
+            playwright_request.method = scrapy_request.method
             playwright_request.is_navigation_request = MagicMock(return_value=True)
             playwright_request.all_headers.return_value = {}
 

tests/test_playwright_requests.py

@@ -4,7 +4,12 @@
 import pytest
 from playwright.async_api import Error as PlaywrightError
 from scrapy import Spider
-from scrapy_playwright._utils import _get_page_content, _NAVIGATION_ERROR_MSG
+from scrapy.http.headers import Headers
+from scrapy_playwright._utils import _get_page_content, _NAVIGATION_ERROR_MSG, _encode_body
+
+
+# page content retrieval
+# ======================
 
 
 @pytest.mark.skipif(sys.version_info < (3, 8), reason="AsyncMock was added on Python 3.8")
@@ -67,3 +72,65 @@ async def test_get_page_content_reraise_unknown_exception():
             scrapy_request_url="https://example.org",
             scrapy_request_method="GET",
         )
+
+
+# body encoding
+# =============
+
+
+def body_str(charset: str, content: str = "áéíóú") -> str:
+    return f"""
+        <!doctype html>
+        <html>
+        <head>
+        <meta charset="{charset}">
+        </head>
+        <body>
+        <p>{content}</p>
+        </body>
+        </html>
+    """.strip()
+
+
+@pytest.mark.asyncio
+async def test_encode_from_headers():
+    """Charset declared in headers takes precedence"""
+    text = body_str(charset="gb2312")
+    body, encoding = _encode_body(
+        headers=Headers({"content-type": "text/html; charset=ISO-8859-1"}),
+        text=text,
+    )
+    assert encoding == "cp1252"
+    assert body == text.encode(encoding)
+
+
+@pytest.mark.asyncio
+async def test_encode_from_body():
+    """No charset declared in headers, use the one declared in the body"""
+    text = body_str(charset="gb2312")
+    body, encoding = _encode_body(headers=Headers({}), text=text)
+    assert encoding == "gb18030"
+    assert body == text.encode(encoding)
+
+
+@pytest.mark.asyncio
+async def test_encode_fallback_utf8():
+    """No charset declared, use utf-8 as fallback"""
+    text = "<html>áéíóú</html>"
+    body, encoding = _encode_body(headers=Headers(), text=text)
+    assert encoding == "utf-8"
+    assert body == text.encode(encoding)
+
+
+@pytest.mark.asyncio
+async def test_encode_mismatch():
+    """Charset declared in headers and body do not match, and the headers
+    one fails to encode: use the one in the body (first one that works)
+    """
+    text = body_str(charset="gb2312", content="空手道")
+    body, encoding = _encode_body(
+        headers=Headers({"content-type": "text/html; charset=ISO-8859-1"}),
+        text=text,
+    )
+    assert encoding == "gb18030"
+    assert body == text.encode(encoding)