fix(playwright): scope custom request headers to the navigation request

docs/guides/code_examples/http_headers/browser_page_headers.py

@@ -0,0 +1,27 @@
+import asyncio
+
+from crawlee.crawlers import (
+    PlaywrightCrawler,
+    PlaywrightCrawlingContext,
+    PlaywrightPreNavCrawlingContext,
+)
+
+
+async def main() -> None:
+    crawler = PlaywrightCrawler(max_requests_per_crawl=10)
+
+    # Page headers are attached to every request the page makes, to any origin.
+    @crawler.pre_navigation_hook
+    async def set_page_headers(context: PlaywrightPreNavCrawlingContext) -> None:
+        await context.page.set_extra_http_headers({'X-Custom-Header': 'my-value'})
+
+    @crawler.router.default_handler
+    async def request_handler(context: PlaywrightCrawlingContext) -> None:
+        # `httpbin.org/headers` echoes the received request headers back.
+        context.log.info(await context.response.text())
+
+    await crawler.run(['https://httpbin.org/headers'])
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

docs/guides/http_headers.mdx

@@ -8,6 +8,7 @@ import ApiLink from '@site/src/components/ApiLink';
 import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
 
 import SetHeadersExample from '!!raw-loader!roa-loader!./code_examples/http_headers/set_headers.py';
+import BrowserPageHeadersExample from '!!raw-loader!roa-loader!./code_examples/http_headers/browser_page_headers.py';
 
 Every request a crawler sends includes [HTTP headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers). These headers tell the server who is making the request, what content is acceptable, and in what language. The server reads them and decides what to return. The same URL can return different content, a different status code, or a blocked page depending on the headers it sees. This guide covers the headers that shape a scraping request, like `User-Agent`, `Accept-Language`, and `Content-Type`, what Crawlee sends by default, and how to change them.
 
@@ -109,6 +110,21 @@ The example below sets `X-Api-Key` on the client and `Accept` on one of two requ
 
 Header names are case-insensitive, and <ApiLink to="class/HttpHeaders">`HttpHeaders`</ApiLink> normalizes the casing for you, so `user-agent` and `User-Agent` refer to the same header.
 
+## Custom headers in browser-based crawlers
+
+The `headers` field of a <ApiLink to="class/Request">`Request`</ApiLink> works differently in browser-based crawlers such as <ApiLink to="class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>. The custom headers are sent with the navigation request and any redirects it goes through, merged into the headers the browser sends on its own. They aren't attached to the requests the page then makes by itself, such as images, scripts, or API calls. The scoping keeps sensitive values like `Authorization` from leaking to third-party origins that the crawled page decides to contact.
+
+There are two ways to send a header with every request the browser makes:
+
+- Set it on a single page with `page.set_extra_http_headers()` in a [pre-navigation hook](./request-router#pre-navigation-hooks).
+- Set it on the browser context through `browser_new_context_options={'extra_http_headers': {...}}`. The default fingerprint generator replaces context-level headers with its own, so context headers only take effect with fingerprinting turned off (`fingerprint_generator=None`).
+
+Either way, the header goes to every origin the page contacts, so use these options for values you're willing to send anywhere, and keep credentials on the request. The following example sets a page header in a pre-navigation hook, which works with the default fingerprinting:
+
+<RunnableCodeBlock className="language-python" language="python">
+    {BrowserPageHeadersExample}
+</RunnableCodeBlock>
+
 ## Header order and fingerprinting
 
 Anti-bot systems look at more than header values. They look at which headers are present, their casing, and the order they arrive in. Real browsers send a consistent, recognizable set. A request that has a browser `User-Agent` but the wrong header order, or missing client hints, still looks automated.

src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -33,23 +33,19 @@
 from ._playwright_post_nav_crawling_context import PlaywrightPostNavCrawlingContext
 from ._playwright_pre_nav_crawling_context import PlaywrightPreNavCrawlingContext
 from ._types import BlockRequestsFunction, GotoOptions
-from ._utils import block_requests, infinite_scroll
+from ._utils import NavigationRequestInterceptor, block_requests, infinite_scroll
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping
     from pathlib import Path
 
-    from playwright.async_api import Page, Response, Route
-    from playwright.async_api import Request as PlaywrightRequest
+    from playwright.async_api import Page, Response
     from typing_extensions import Unpack
 
     from crawlee import RequestTransformAction
     from crawlee._types import (
         EnqueueLinksKwargs,
         ExtractLinksFunction,
-        HttpHeaders,
-        HttpMethod,
-        HttpPayload,
         JsonSerializable,
     )
     from crawlee.browsers._types import BrowserType
@@ -350,27 +346,6 @@ async def _open_page(
             # Yield should be inside the browser_page_context.
             yield pre_navigation_context
 
-    def _prepare_request_interceptor(
-        self,
-        method: HttpMethod = 'GET',
-        headers: HttpHeaders | dict[str, str] | None = None,
-        payload: HttpPayload | None = None,
-    ) -> Callable:
-        """Create a request interceptor for Playwright to support non-GET methods with custom parameters.
-
-        The interceptor modifies requests by adding custom headers and payload before they are sent.
-
-        Args:
-            method: HTTP method to use for the request.
-            headers: Custom HTTP headers to send with the request.
-            payload: Request body data for POST/PUT requests.
-        """
-
-        async def route_handler(route: Route, _: PlaywrightRequest) -> None:
-            await route.continue_(method=method, headers=dict(headers) if headers else None, post_data=payload)
-
-        return route_handler
-
     async def _navigate(
         self,
         context: TPreNavContext,
@@ -399,9 +374,6 @@ async def _navigate(
             session_cookies = context.session.cookies.get_cookies_as_playwright_format()
             await self._update_cookies(context.page, session_cookies)
 
-        if context.request.headers:
-            await context.page.set_extra_http_headers(context.request.headers.model_dump())
-        # Navigate to the URL and get response.
         if context.request.method != 'GET':
             # Call the notification only once
             warnings.warn(
@@ -411,14 +383,16 @@ async def _navigate(
                 stacklevel=2,
             )
 
-            route_handler = self._prepare_request_interceptor(
+        # Apply the custom method, headers, and payload to the navigation request only; the details of doing
+        # that correctly live in `NavigationRequestInterceptor`.
+        if context.request.headers or context.request.method != 'GET':
+            interceptor = NavigationRequestInterceptor(
+                context.page,
                 method=context.request.method,
                 headers=context.request.headers,
                 payload=context.request.payload,
             )
-
-            # Set route_handler only for current request
-            await context.page.route(context.request.url, route_handler)
+            await interceptor.register()
 
         try:
             async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout:

src/crawlee/crawlers/_playwright/_utils.py

@@ -5,9 +5,11 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from playwright.async_api import Page
+    from playwright.async_api import Page, Route
     from playwright.async_api import Request as PlaywrightRequest
 
+    from crawlee._types import HttpHeaders, HttpMethod, HttpPayload
+
 _DEFAULT_BLOCK_REQUEST_URL_PATTERNS = [
     '.css',
     '.webp',
@@ -22,6 +24,51 @@
 ]
 
 
+class NavigationRequestInterceptor:
+    """One-shot page route that applies a custom method, headers, and payload to the main-frame navigation request.
+
+    Scoping the overrides to the navigation request is the point: applying headers page-wide via
+    `Page.set_extra_http_headers` would leak sensitive values like `Authorization` to every subresource request
+    the page makes, including cross-origin ones. The navigation request is matched by its role (a main-frame
+    navigation) rather than by its URL, because the browser normalizes URLs (adds the root path, strips
+    fragments), so a URL comparison can silently miss. Redirect hops bypass routing and inherit the overrides.
+
+    Custom headers are merged into the headers the browser would send on its own (e.g. `User-Agent` or
+    fingerprint headers), with the custom ones winning.
+    """
+
+    def __init__(
+        self,
+        page: Page,
+        *,
+        method: HttpMethod = 'GET',
+        headers: HttpHeaders | dict[str, str] | None = None,
+        payload: HttpPayload | None = None,
+    ) -> None:
+        self._page = page
+        self._method = method
+        self._headers = headers
+        self._payload = payload
+        self._applied = False
+
+    async def register(self) -> None:
+        """Start routing the page's requests through this interceptor.
+
+        Once the overrides are applied, the route's predicate stops matching, so any later request
+        (subresources, XHRs, client-side navigations) skips the handler entirely.
+        """
+        await self._page.route(lambda _: not self._applied, self._handle_route)
+
+    async def _handle_route(self, route: Route, request: PlaywrightRequest) -> None:
+        if self._applied or not request.is_navigation_request() or request.frame != self._page.main_frame:
+            await route.fallback()
+            return
+
+        self._applied = True
+        merged_headers = {**request.headers, **dict(self._headers)} if self._headers else None
+        await route.continue_(method=self._method, headers=merged_headers, post_data=self._payload)
+
+
 async def infinite_scroll(page: Page) -> None:
     """Scroll to the bottom of a page, handling loading of additional items."""
     scrolled_distance = 0

tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -47,6 +47,7 @@
 if TYPE_CHECKING:
     from pathlib import Path
 
+    from playwright.async_api import Request as PlaywrightRequest
     from yarl import URL
 
     from crawlee._request import RequestOptions
@@ -316,6 +317,72 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
     assert response_headers.get('my-test-header') == request_headers['My-Test-Header']
 
 
+async def test_custom_headers_not_sent_with_cross_origin_requests(server_url: URL, redirect_server_url: URL) -> None:
+    """Request headers are sent with the main navigation only, not with cross-origin requests made by the page."""
+    crawler = PlaywrightCrawler()
+
+    subresource_url = str(redirect_server_url / 'headers')
+    page_html = f'<html><body><img src="{subresource_url}"></body></html>'
+    start_url = str((server_url / 'echo_content').with_query(content=page_html))
+
+    navigation_headers = dict[str, str]()
+    subresource_headers = dict[str, str]()
+
+    @crawler.pre_navigation_hook
+    async def capture_subresource_headers(context: PlaywrightPreNavCrawlingContext) -> None:
+        def capture(request: PlaywrightRequest) -> None:
+            if request.url == subresource_url:
+                subresource_headers.update(request.headers)
+
+        context.page.on('request', capture)
+
+    @crawler.router.default_handler
+    async def request_handler(context: PlaywrightCrawlingContext) -> None:
+        await context.page.wait_for_load_state()
+        navigation_headers.update(await context.response.request.all_headers())
+
+    await crawler.run([Request.from_url(start_url, headers={'authorization': 'Bearer secret-token'})])
+
+    assert navigation_headers.get('authorization') == 'Bearer secret-token'
+    # Custom headers are merged into the browser-generated ones, not replacing them.
+    assert 'user-agent' in navigation_headers
+    assert subresource_headers
+    assert 'authorization' not in subresource_headers
+
+
+async def test_custom_headers_sent_when_browser_normalizes_url(server_url: URL) -> None:
+    """Custom headers are applied even when the browser normalizes the request URL (e.g. adds the root path)."""
+    crawler = PlaywrightCrawler()
+    navigation_headers = dict[str, str]()
+
+    @crawler.router.default_handler
+    async def request_handler(context: PlaywrightCrawlingContext) -> None:
+        navigation_headers.update(await context.response.request.all_headers())
+
+    # A bare origin without the trailing slash; the browser requests `/`.
+    bare_url = f'http://{server_url.host}:{server_url.port}'
+    await crawler.run([Request.from_url(bare_url, headers={'authorization': 'Bearer secret-token'})])
+
+    assert navigation_headers.get('authorization') == 'Bearer secret-token'
+
+
+async def test_custom_headers_survive_redirect(server_url: URL) -> None:
+    """Custom headers applied to the navigation request are inherited by redirect hops."""
+    crawler = PlaywrightCrawler()
+    received_headers = dict[str, str]()
+
+    @crawler.router.default_handler
+    async def request_handler(context: PlaywrightCrawlingContext) -> None:
+        # The `/headers` endpoint echoes what the server actually received after the redirect hop.
+        received_headers.update(json.loads(await context.response.text()))
+
+    target_url = str(server_url / 'headers')
+    start_url = str((server_url / 'redirect').with_query(url=target_url))
+    await crawler.run([Request.from_url(start_url, headers={'authorization': 'Bearer secret-token'})])
+
+    assert received_headers.get('authorization') == 'Bearer secret-token'
+
+
 async def test_pre_navigation_hook() -> None:
     crawler = PlaywrightCrawler(request_handler=mock.AsyncMock())
     visit = mock.Mock()