From 64e272cf0a4ac4eb387ee988d0a352654b8c405e Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 18 Aug 2026 18:30:03 +0200 Subject: [PATCH 1/5] fix(playwright): scope custom request headers to the navigation request --- .../_playwright/_playwright_crawler.py | 18 +++++------ .../_playwright/test_playwright_crawler.py | 32 +++++++++++++++++++ 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 1235b3675c..f55f9d2d94 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -356,18 +356,18 @@ def _prepare_request_interceptor( headers: HttpHeaders | dict[str, str] | None = None, payload: HttpPayload | None = None, ) -> Callable: - """Create a request interceptor for Playwright to support non-GET methods with custom parameters. - - The interceptor modifies requests by adding custom headers and payload before they are sent. + """Create a request interceptor that applies a custom method, headers, and payload to matching requests. Args: method: HTTP method to use for the request. - headers: Custom HTTP headers to send with the request. + headers: Custom HTTP headers to send with the request. They are merged into the headers the browser + would send on its own (e.g. `User-Agent` or fingerprint headers), with the custom ones winning. payload: Request body data for POST/PUT requests. """ - async def route_handler(route: Route, _: PlaywrightRequest) -> None: - await route.continue_(method=method, headers=dict(headers) if headers else None, post_data=payload) + async def route_handler(route: Route, request: PlaywrightRequest) -> None: + merged_headers = {**request.headers, **dict(headers)} if headers else None + await route.continue_(method=method, headers=merged_headers, post_data=payload) return route_handler @@ -399,9 +399,6 @@ async def _navigate( session_cookies = context.session.cookies.get_cookies_as_playwright_format() await self._update_cookies(context.page, session_cookies) - if context.request.headers: - await context.page.set_extra_http_headers(context.request.headers.model_dump()) - # Navigate to the URL and get response. if context.request.method != 'GET': # Call the notification only once warnings.warn( @@ -411,6 +408,9 @@ async def _navigate( stacklevel=2, ) + # Apply custom headers via a route scoped to the navigation URL; page-wide `set_extra_http_headers` + # would leak sensitive values like `Authorization` to every subresource request the page makes. + if context.request.headers or context.request.method != 'GET': route_handler = self._prepare_request_interceptor( method=context.request.method, headers=context.request.headers, diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index cc2e474e22..5825411224 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -47,6 +47,7 @@ if TYPE_CHECKING: from pathlib import Path + from playwright.async_api import Request as PlaywrightRequest from yarl import URL from crawlee._request import RequestOptions @@ -316,6 +317,37 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: assert response_headers.get('my-test-header') == request_headers['My-Test-Header'] +async def test_custom_headers_not_sent_with_cross_origin_requests(server_url: URL, redirect_server_url: URL) -> None: + """Request headers are sent with the main navigation only, not with cross-origin requests made by the page.""" + crawler = PlaywrightCrawler() + + subresource_url = str(redirect_server_url / 'headers') + page_html = f'' + start_url = str((server_url / 'echo_content').with_query(content=page_html)) + + navigation_headers = dict[str, str]() + subresource_headers = dict[str, str]() + + @crawler.pre_navigation_hook + async def capture_subresource_headers(context: PlaywrightPreNavCrawlingContext) -> None: + def capture(request: PlaywrightRequest) -> None: + if request.url == subresource_url: + subresource_headers.update(request.headers) + + context.page.on('request', capture) + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + await context.page.wait_for_load_state() + navigation_headers.update(await context.response.request.all_headers()) + + await crawler.run([Request.from_url(start_url, headers={'authorization': 'Bearer secret-token'})]) + + assert navigation_headers.get('authorization') == 'Bearer secret-token' + assert subresource_headers + assert 'authorization' not in subresource_headers + + async def test_pre_navigation_hook() -> None: crawler = PlaywrightCrawler(request_handler=mock.AsyncMock()) visit = mock.Mock() From 73c11b6ce6c621cb13a223b19c29ec1f4c6930e6 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 19 Aug 2026 09:10:55 +0200 Subject: [PATCH 2/5] fix(playwright): match interception route by navigation request instead of URL --- .../_playwright/_playwright_crawler.py | 21 ++++++++--- .../_playwright/test_playwright_crawler.py | 35 +++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index f55f9d2d94..9a4e7d35e9 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -408,7 +408,7 @@ async def _navigate( stacklevel=2, ) - # Apply custom headers via a route scoped to the navigation URL; page-wide `set_extra_http_headers` + # Apply custom headers via a route scoped to the navigation request; page-wide `set_extra_http_headers` # would leak sensitive values like `Authorization` to every subresource request the page makes. if context.request.headers or context.request.method != 'GET': route_handler = self._prepare_request_interceptor( @@ -416,9 +416,22 @@ async def _navigate( headers=context.request.headers, payload=context.request.payload, ) - - # Set route_handler only for current request - await context.page.route(context.request.url, route_handler) + applied = False + + async def navigation_route_handler(route: Route, request: PlaywrightRequest) -> None: + nonlocal applied + # Match the main-frame navigation request itself rather than its URL; the browser normalizes + # URLs (adds the root path, strips fragments), so a string comparison with `request.url` can + # silently miss. Redirect hops bypass routing and inherit the header overrides. + if not applied and request.is_navigation_request() and request.frame == context.page.main_frame: + applied = True + await route_handler(route, request) + else: + await route.fallback() + + # Once the overrides are applied, the predicate stops matching, so subresource requests + # skip the handler entirely. + await context.page.route(lambda _: not applied, navigation_route_handler) try: async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout: diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 5825411224..03b6eb309f 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -344,10 +344,45 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: await crawler.run([Request.from_url(start_url, headers={'authorization': 'Bearer secret-token'})]) assert navigation_headers.get('authorization') == 'Bearer secret-token' + # Custom headers are merged into the browser-generated ones, not replacing them. + assert 'user-agent' in navigation_headers assert subresource_headers assert 'authorization' not in subresource_headers +async def test_custom_headers_sent_when_browser_normalizes_url(server_url: URL) -> None: + """Custom headers are applied even when the browser normalizes the request URL (e.g. adds the root path).""" + crawler = PlaywrightCrawler() + navigation_headers = dict[str, str]() + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + navigation_headers.update(await context.response.request.all_headers()) + + # A bare origin without the trailing slash; the browser requests `/`. + bare_url = f'http://{server_url.host}:{server_url.port}' + await crawler.run([Request.from_url(bare_url, headers={'authorization': 'Bearer secret-token'})]) + + assert navigation_headers.get('authorization') == 'Bearer secret-token' + + +async def test_custom_headers_survive_redirect(server_url: URL) -> None: + """Custom headers applied to the navigation request are inherited by redirect hops.""" + crawler = PlaywrightCrawler() + received_headers = dict[str, str]() + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + # The `/headers` endpoint echoes what the server actually received after the redirect hop. + received_headers.update(json.loads(await context.response.text())) + + target_url = str(server_url / 'headers') + start_url = str((server_url / 'redirect').with_query(url=target_url)) + await crawler.run([Request.from_url(start_url, headers={'authorization': 'Bearer secret-token'})]) + + assert received_headers.get('authorization') == 'Bearer secret-token' + + async def test_pre_navigation_hook() -> None: crawler = PlaywrightCrawler(request_handler=mock.AsyncMock()) visit = mock.Mock() From 03c498e802dfc3e0f9a99fa0d28265b88c5fb3ae Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 19 Aug 2026 09:19:51 +0200 Subject: [PATCH 3/5] docs: document request header scoping in browser-based crawlers --- docs/guides/http_headers.mdx | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/guides/http_headers.mdx b/docs/guides/http_headers.mdx index 06f4566efa..4e39cbe85e 100644 --- a/docs/guides/http_headers.mdx +++ b/docs/guides/http_headers.mdx @@ -109,6 +109,27 @@ The example below sets `X-Api-Key` on the client and `Accept` on one of two requ Header names are case-insensitive, and `HttpHeaders` normalizes the casing for you, so `user-agent` and `User-Agent` refer to the same header. +## Custom headers in browser-based crawlers + +The `headers` field of a `Request` works differently in browser-based crawlers such as `PlaywrightCrawler`. The custom headers are sent with the navigation request and any redirects it goes through, merged into the headers the browser sends on its own. They aren't attached to the requests the page then makes by itself, such as images, scripts, or API calls. The scoping keeps sensitive values like `Authorization` from leaking to third-party origins that the crawled page decides to contact. + +There are two ways to send a header with every request the browser makes: + +- Set it on the browser context through `browser_new_context_options`. Context headers apply to all pages and all requests from them, so use them for values you're willing to send to every origin, and keep credentials on the request. +- Set it on a single page with `page.set_extra_http_headers()` in a [pre-navigation hook](./request-router#pre-navigation-hooks). + +```python +from crawlee.crawlers import PlaywrightCrawler + +crawler = PlaywrightCrawler( + browser_new_context_options={ + 'extra_http_headers': { + 'X-Custom-Header': 'my-value', + }, + }, +) +``` + ## Header order and fingerprinting Anti-bot systems look at more than header values. They look at which headers are present, their casing, and the order they arrive in. Real browsers send a consistent, recognizable set. A request that has a browser `User-Agent` but the wrong header order, or missing client hints, still looks automated. From 50ef0580a3bae282d7450a9889ed87c7938923bc Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 19 Aug 2026 11:37:33 +0200 Subject: [PATCH 4/5] refactor(playwright): extract navigation request interception into a helper class --- .../_playwright/_playwright_crawler.py | 53 +++---------------- src/crawlee/crawlers/_playwright/_utils.py | 49 ++++++++++++++++- 2 files changed, 55 insertions(+), 47 deletions(-) diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index 9a4e7d35e9..2e2e403411 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -33,23 +33,19 @@ from ._playwright_post_nav_crawling_context import PlaywrightPostNavCrawlingContext from ._playwright_pre_nav_crawling_context import PlaywrightPreNavCrawlingContext from ._types import BlockRequestsFunction, GotoOptions -from ._utils import block_requests, infinite_scroll +from ._utils import NavigationRequestInterceptor, block_requests, infinite_scroll if TYPE_CHECKING: from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping from pathlib import Path - from playwright.async_api import Page, Response, Route - from playwright.async_api import Request as PlaywrightRequest + from playwright.async_api import Page, Response from typing_extensions import Unpack from crawlee import RequestTransformAction from crawlee._types import ( EnqueueLinksKwargs, ExtractLinksFunction, - HttpHeaders, - HttpMethod, - HttpPayload, JsonSerializable, ) from crawlee.browsers._types import BrowserType @@ -350,27 +346,6 @@ async def _open_page( # Yield should be inside the browser_page_context. yield pre_navigation_context - def _prepare_request_interceptor( - self, - method: HttpMethod = 'GET', - headers: HttpHeaders | dict[str, str] | None = None, - payload: HttpPayload | None = None, - ) -> Callable: - """Create a request interceptor that applies a custom method, headers, and payload to matching requests. - - Args: - method: HTTP method to use for the request. - headers: Custom HTTP headers to send with the request. They are merged into the headers the browser - would send on its own (e.g. `User-Agent` or fingerprint headers), with the custom ones winning. - payload: Request body data for POST/PUT requests. - """ - - async def route_handler(route: Route, request: PlaywrightRequest) -> None: - merged_headers = {**request.headers, **dict(headers)} if headers else None - await route.continue_(method=method, headers=merged_headers, post_data=payload) - - return route_handler - async def _navigate( self, context: TPreNavContext, @@ -408,30 +383,16 @@ async def _navigate( stacklevel=2, ) - # Apply custom headers via a route scoped to the navigation request; page-wide `set_extra_http_headers` - # would leak sensitive values like `Authorization` to every subresource request the page makes. + # Apply the custom method, headers, and payload to the navigation request only; the details of doing + # that correctly live in `NavigationRequestInterceptor`. if context.request.headers or context.request.method != 'GET': - route_handler = self._prepare_request_interceptor( + interceptor = NavigationRequestInterceptor( + context.page, method=context.request.method, headers=context.request.headers, payload=context.request.payload, ) - applied = False - - async def navigation_route_handler(route: Route, request: PlaywrightRequest) -> None: - nonlocal applied - # Match the main-frame navigation request itself rather than its URL; the browser normalizes - # URLs (adds the root path, strips fragments), so a string comparison with `request.url` can - # silently miss. Redirect hops bypass routing and inherit the header overrides. - if not applied and request.is_navigation_request() and request.frame == context.page.main_frame: - applied = True - await route_handler(route, request) - else: - await route.fallback() - - # Once the overrides are applied, the predicate stops matching, so subresource requests - # skip the handler entirely. - await context.page.route(lambda _: not applied, navigation_route_handler) + await interceptor.register() try: async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout: diff --git a/src/crawlee/crawlers/_playwright/_utils.py b/src/crawlee/crawlers/_playwright/_utils.py index 153d50e204..06e91ee60f 100644 --- a/src/crawlee/crawlers/_playwright/_utils.py +++ b/src/crawlee/crawlers/_playwright/_utils.py @@ -5,9 +5,11 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from playwright.async_api import Page + from playwright.async_api import Page, Route from playwright.async_api import Request as PlaywrightRequest + from crawlee._types import HttpHeaders, HttpMethod, HttpPayload + _DEFAULT_BLOCK_REQUEST_URL_PATTERNS = [ '.css', '.webp', @@ -22,6 +24,51 @@ ] +class NavigationRequestInterceptor: + """One-shot page route that applies a custom method, headers, and payload to the main-frame navigation request. + + Scoping the overrides to the navigation request is the point: applying headers page-wide via + `Page.set_extra_http_headers` would leak sensitive values like `Authorization` to every subresource request + the page makes, including cross-origin ones. The navigation request is matched by its role (a main-frame + navigation) rather than by its URL, because the browser normalizes URLs (adds the root path, strips + fragments), so a URL comparison can silently miss. Redirect hops bypass routing and inherit the overrides. + + Custom headers are merged into the headers the browser would send on its own (e.g. `User-Agent` or + fingerprint headers), with the custom ones winning. + """ + + def __init__( + self, + page: Page, + *, + method: HttpMethod = 'GET', + headers: HttpHeaders | dict[str, str] | None = None, + payload: HttpPayload | None = None, + ) -> None: + self._page = page + self._method = method + self._headers = headers + self._payload = payload + self._applied = False + + async def register(self) -> None: + """Start routing the page's requests through this interceptor. + + Once the overrides are applied, the route's predicate stops matching, so any later request + (subresources, XHRs, client-side navigations) skips the handler entirely. + """ + await self._page.route(lambda _: not self._applied, self._handle_route) + + async def _handle_route(self, route: Route, request: PlaywrightRequest) -> None: + if self._applied or not request.is_navigation_request() or request.frame != self._page.main_frame: + await route.fallback() + return + + self._applied = True + merged_headers = {**request.headers, **dict(self._headers)} if self._headers else None + await route.continue_(method=self._method, headers=merged_headers, post_data=self._payload) + + async def infinite_scroll(page: Page) -> None: """Scroll to the bottom of a page, handling loading of additional items.""" scrolled_distance = 0 From 0b9d79bcee05c82a1de871e3595b34641f2bfc5f Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 19 Aug 2026 11:47:04 +0200 Subject: [PATCH 5/5] docs: replace inline browser headers snippet with a runnable example --- .../http_headers/browser_page_headers.py | 27 +++++++++++++++++++ docs/guides/http_headers.mdx | 19 +++++-------- 2 files changed, 34 insertions(+), 12 deletions(-) create mode 100644 docs/guides/code_examples/http_headers/browser_page_headers.py diff --git a/docs/guides/code_examples/http_headers/browser_page_headers.py b/docs/guides/code_examples/http_headers/browser_page_headers.py new file mode 100644 index 0000000000..e8fc7e510d --- /dev/null +++ b/docs/guides/code_examples/http_headers/browser_page_headers.py @@ -0,0 +1,27 @@ +import asyncio + +from crawlee.crawlers import ( + PlaywrightCrawler, + PlaywrightCrawlingContext, + PlaywrightPreNavCrawlingContext, +) + + +async def main() -> None: + crawler = PlaywrightCrawler(max_requests_per_crawl=10) + + # Page headers are attached to every request the page makes, to any origin. + @crawler.pre_navigation_hook + async def set_page_headers(context: PlaywrightPreNavCrawlingContext) -> None: + await context.page.set_extra_http_headers({'X-Custom-Header': 'my-value'}) + + @crawler.router.default_handler + async def request_handler(context: PlaywrightCrawlingContext) -> None: + # `httpbin.org/headers` echoes the received request headers back. + context.log.info(await context.response.text()) + + await crawler.run(['https://httpbin.org/headers']) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/http_headers.mdx b/docs/guides/http_headers.mdx index 4e39cbe85e..bd723e8f47 100644 --- a/docs/guides/http_headers.mdx +++ b/docs/guides/http_headers.mdx @@ -8,6 +8,7 @@ import ApiLink from '@site/src/components/ApiLink'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import SetHeadersExample from '!!raw-loader!roa-loader!./code_examples/http_headers/set_headers.py'; +import BrowserPageHeadersExample from '!!raw-loader!roa-loader!./code_examples/http_headers/browser_page_headers.py'; Every request a crawler sends includes [HTTP headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers). These headers tell the server who is making the request, what content is acceptable, and in what language. The server reads them and decides what to return. The same URL can return different content, a different status code, or a blocked page depending on the headers it sees. This guide covers the headers that shape a scraping request, like `User-Agent`, `Accept-Language`, and `Content-Type`, what Crawlee sends by default, and how to change them. @@ -115,20 +116,14 @@ The `headers` field of a `Request` works d There are two ways to send a header with every request the browser makes: -- Set it on the browser context through `browser_new_context_options`. Context headers apply to all pages and all requests from them, so use them for values you're willing to send to every origin, and keep credentials on the request. - Set it on a single page with `page.set_extra_http_headers()` in a [pre-navigation hook](./request-router#pre-navigation-hooks). +- Set it on the browser context through `browser_new_context_options={'extra_http_headers': {...}}`. The default fingerprint generator replaces context-level headers with its own, so context headers only take effect with fingerprinting turned off (`fingerprint_generator=None`). -```python -from crawlee.crawlers import PlaywrightCrawler - -crawler = PlaywrightCrawler( - browser_new_context_options={ - 'extra_http_headers': { - 'X-Custom-Header': 'my-value', - }, - }, -) -``` +Either way, the header goes to every origin the page contacts, so use these options for values you're willing to send anywhere, and keep credentials on the request. The following example sets a page header in a pre-navigation hook, which works with the default fingerprinting: + + + {BrowserPageHeadersExample} + ## Header order and fingerprinting