diff --git a/README.md b/README.md index 71a306807..a08accc13 100644 --- a/README.md +++ b/README.md @@ -125,13 +125,13 @@ Whatever you build, the Apify SDK doesn't lock you into a particular framework. The examples below show two common setups, but the same `async with Actor:` pattern works with any stack. For more, see the [guides](https://docs.apify.com/sdk/python/docs/guides/beautifulsoup-httpx). -### HTTPX with BeautifulSoup +### HTTPX2 with BeautifulSoup -Scrape pages with [HTTPX](https://www.python-httpx.org/) and [BeautifulSoup](https://pypi.org/project/beautifulsoup4/), using the Actor's request queue to track URLs: +Scrape pages with [HTTPX2](https://pydantic.dev/docs/httpx2/) and [BeautifulSoup](https://pypi.org/project/beautifulsoup4/), using the Actor's request queue to track URLs: ```python from bs4 import BeautifulSoup -from httpx import AsyncClient +from httpx2 import AsyncClient from apify import Actor diff --git a/docs/01_introduction/code/01_introduction.py b/docs/01_introduction/code/01_introduction.py index 3c8751709..a44cd7273 100644 --- a/docs/01_introduction/code/01_introduction.py +++ b/docs/01_introduction/code/01_introduction.py @@ -1,6 +1,6 @@ import asyncio -import httpx +import httpx2 from bs4 import BeautifulSoup from apify import Actor @@ -10,7 +10,7 @@ async def main() -> None: async with Actor: actor_input = await Actor.get_input() or {} url = actor_input.get('url', 'https://apify.com') - async with httpx.AsyncClient() as client: + async with httpx2.AsyncClient() as client: response = await client.get(url) soup = BeautifulSoup(response.content, 'html.parser') data = { diff --git a/docs/01_introduction/quick-start.mdx b/docs/01_introduction/quick-start.mdx index 699ea932e..f94434d52 100644 --- a/docs/01_introduction/quick-start.mdx +++ b/docs/01_introduction/quick-start.mdx @@ -103,7 +103,7 @@ To learn more about the features of the Apify SDK and how to use them, check out To see how you can integrate the Apify SDK with popular scraping libraries and frameworks, check out these guides: -- [Scraping with BeautifulSoup and HTTPX](./guides/beautifulsoup-httpx) +- [Scraping with BeautifulSoup and HTTPX2](./guides/beautifulsoup-httpx) - [Scraping with Parsel and Impit](./guides/parsel-impit) - [Browser automation with Playwright](./guides/playwright) - [Browser automation with Selenium](./guides/selenium) diff --git a/docs/02_concepts/05_proxy_management.mdx b/docs/02_concepts/05_proxy_management.mdx index d3c37fd4a..5397714cd 100644 --- a/docs/02_concepts/05_proxy_management.mdx +++ b/docs/02_concepts/05_proxy_management.mdx @@ -12,7 +12,7 @@ import ProxyRotationExample from '!!raw-loader!roa-loader!./code/05_proxy_rotati import ApifyProxyConfig from '!!raw-loader!roa-loader!./code/05_apify_proxy_config.py'; import CustomProxyFunctionExample from '!!raw-loader!roa-loader!./code/05_custom_proxy_function.py'; import ProxyActorInputExample from '!!raw-loader!roa-loader!./code/05_proxy_actor_input.py'; -import ProxyHttpxExample from '!!raw-loader!roa-loader!./code/05_proxy_httpx.py'; +import ProxyHttpx2Example from '!!raw-loader!roa-loader!./code/05_proxy_httpx2.py'; import TieredProxyExample from '!!raw-loader!roa-loader!./code/05_tiered_proxy.py'; import ApiLink from '@theme/ApiLink'; @@ -108,16 +108,16 @@ You can then use that input to create the proxy configuration: `ProxyConfiguration` only generates proxy URLs. It doesn't make requests. To route requests through the proxy, pass a generated URL to the HTTP client your Actor uses. -### HTTPX +### HTTPX2 -To use the generated proxy URLs with the `httpx` library, use the [`proxy`](https://www.python-httpx.org/advanced/proxies/) argument: +To use the generated proxy URLs with the `httpx2` library, use the [`proxy`](https://pydantic.dev/docs/httpx2/advanced/proxies/) argument: - {ProxyHttpxExample} + {ProxyHttpx2Example} -Make sure you have the `httpx` library installed: +Make sure you have the `httpx2` library installed: ```bash -pip install httpx +pip install httpx2 ``` diff --git a/docs/02_concepts/code/05_proxy_httpx.py b/docs/02_concepts/code/05_proxy_httpx2.py similarity index 77% rename from docs/02_concepts/code/05_proxy_httpx.py rename to docs/02_concepts/code/05_proxy_httpx2.py index dbabacaa9..789766ed6 100644 --- a/docs/02_concepts/code/05_proxy_httpx.py +++ b/docs/02_concepts/code/05_proxy_httpx2.py @@ -1,6 +1,6 @@ import asyncio -import httpx +import httpx2 from apify import Actor @@ -19,8 +19,8 @@ async def main() -> None: proxy_url = await proxy_cfg.new_url() - async with httpx.AsyncClient(proxy=proxy_url) as httpx_client: - response = await httpx_client.get('http://example.com') + async with httpx2.AsyncClient(proxy=proxy_url) as httpx2_client: + response = await httpx2_client.get('http://example.com') Actor.log.info(f'Response: {response}') diff --git a/docs/03_guides/01_beautifulsoup_httpx.mdx b/docs/03_guides/01_beautifulsoup_httpx.mdx deleted file mode 100644 index b11af63ed..000000000 --- a/docs/03_guides/01_beautifulsoup_httpx.mdx +++ /dev/null @@ -1,41 +0,0 @@ ---- -id: beautifulsoup-httpx -title: Scraping with BeautifulSoup and HTTPX -description: Build an Apify Actor that scrapes web pages using BeautifulSoup and HTTPX. ---- - -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import BeautifulSoupHttpxExample from '!!raw-loader!roa-loader!./code/01_beautifulsoup_httpx.py'; - -In this guide, you'll learn how to scrape web pages with the [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) and [HTTPX](https://www.python-httpx.org/) libraries in your Apify Actors. - -## Introduction - -[BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) is a Python library for extracting data from HTML and XML files. It provides simple methods and Pythonic idioms for navigating, searching, and modifying a website's element tree, enabling efficient data extraction. - -[HTTPX](https://www.python-httpx.org/) is a modern, high-level HTTP client library for Python. It provides a simple interface for making HTTP requests and supports both synchronous and asynchronous requests. - -To create an Actor which uses those libraries, start from the [BeautifulSoup & Python](https://apify.com/templates/categories/python) Actor template. This template includes the [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) and [HTTPX](https://www.python-httpx.org/) libraries preinstalled, allowing you to begin development immediately. - -## Example Actor - -The following example is a simple Actor that recursively scrapes data from linked pages on the same site, up to a specified maximum depth, starting from URLs provided in the Actor input. It uses [HTTPX](https://www.python-httpx.org/) for fetching pages through [Apify Proxy](https://docs.apify.com/platform/proxy) and [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) for parsing their content to extract the title, headings, and links to other pages. - - - {BeautifulSoupHttpxExample} - - -## Using Apify Proxy - -Running on the Apify platform gives your scraper access to [Apify Proxy](https://docs.apify.com/platform/proxy), which rotates IP addresses to avoid rate limiting and blocking. The example creates a proxy configuration with `Actor.create_proxy_configuration` and fetches a fresh proxy URL for every request. Each page then goes through a different IP. A new HTTPX client is created per request to apply that URL. To select specific proxy groups or a country, pass the relevant arguments to `Actor.create_proxy_configuration`. For details, see [Proxy management](../concepts/proxy-management). - -## Conclusion - -In this guide, you learned how to use the [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) with the [HTTPX](https://www.python-httpx.org/) in your Apify Actors. By combining these libraries, you can efficiently extract data from HTML or XML files, making it easy to build web scraping tasks in Python. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! - -## Additional resources - -- [Apify templates: BeautifulSoup](https://apify.com/templates/python-beautifulsoup) -- [BeautifulSoup: Official documentation](https://www.crummy.com/software/BeautifulSoup/bs4/doc/) -- [HTTPX: Official documentation](https://www.python-httpx.org/) diff --git a/docs/03_guides/01_beautifulsoup_httpx2.mdx b/docs/03_guides/01_beautifulsoup_httpx2.mdx new file mode 100644 index 000000000..0ef94a7d7 --- /dev/null +++ b/docs/03_guides/01_beautifulsoup_httpx2.mdx @@ -0,0 +1,45 @@ +--- +id: beautifulsoup-httpx +title: Scraping with BeautifulSoup and HTTPX2 +description: Build an Apify Actor that scrapes web pages using BeautifulSoup and HTTPX2. +--- + +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import BeautifulSoupHttpx2Example from '!!raw-loader!roa-loader!./code/01_beautifulsoup_httpx2.py'; + +In this guide, you'll learn how to scrape web pages with the [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) and [HTTPX2](https://pydantic.dev/docs/httpx2/) libraries in your Apify Actors. + +## Introduction + +[BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) is a Python library for extracting data from HTML and XML files. It provides simple methods and Pythonic idioms for navigating, searching, and modifying a website's element tree, enabling efficient data extraction. + +[HTTPX2](https://pydantic.dev/docs/httpx2/) is a modern, high-level HTTP client library for Python, maintained by Pydantic as the continuation of HTTPX. It provides a simple interface for making HTTP requests and supports both synchronous and asynchronous requests. + +To create an Actor which uses those libraries, start from the [BeautifulSoup & Python](https://apify.com/templates/categories/python) Actor template, which includes both libraries preinstalled. To add them to an existing project instead, use: + +```bash +pip install beautifulsoup4 httpx2 +``` + +## Example Actor + +The following example is a simple Actor that recursively scrapes data from linked pages on the same site, up to a specified maximum depth, starting from URLs provided in the Actor input. It uses [HTTPX2](https://pydantic.dev/docs/httpx2/) for fetching pages through [Apify Proxy](https://docs.apify.com/platform/proxy) and [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) for parsing their content to extract the title, headings, and links to other pages. + + + {BeautifulSoupHttpx2Example} + + +## Using Apify Proxy + +Running on the Apify platform gives your scraper access to [Apify Proxy](https://docs.apify.com/platform/proxy), which rotates IP addresses to avoid rate limiting and blocking. The example creates a proxy configuration with `Actor.create_proxy_configuration` and fetches a fresh proxy URL for every request. Each page then goes through a different IP. A new HTTPX2 client is created per request to apply that URL. To select specific proxy groups or a country, pass the relevant arguments to `Actor.create_proxy_configuration`. For details, see [Proxy management](../concepts/proxy-management). + +## Conclusion + +In this guide, you learned how to use the [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/) with the [HTTPX2](https://pydantic.dev/docs/httpx2/) in your Apify Actors. By combining these libraries, you can efficiently extract data from HTML or XML files, making it easy to build web scraping tasks in Python. See the [Actor templates](https://apify.com/templates/categories/python) to get started with your own scraping tasks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/apify-sdk-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! + +## Additional resources + +- [Apify templates: BeautifulSoup](https://apify.com/templates/python-beautifulsoup) +- [BeautifulSoup: Official documentation](https://www.crummy.com/software/BeautifulSoup/bs4/doc/) +- [HTTPX2: Official documentation](https://pydantic.dev/docs/httpx2/) diff --git a/docs/03_guides/10_uv.mdx b/docs/03_guides/10_uv.mdx index a5edbe990..effdb791b 100644 --- a/docs/03_guides/10_uv.mdx +++ b/docs/03_guides/10_uv.mdx @@ -166,13 +166,13 @@ Day-to-day dependency management goes through uv as well: ```bash # Add a dependency (records it in pyproject.toml and updates uv.lock). -uv add httpx +uv add httpx2 # Add a development-only dependency (skipped in the Docker build by --no-dev). uv add --dev ruff # Remove a dependency. -uv remove httpx +uv remove httpx2 # Upgrade all dependencies to the latest versions allowed by pyproject.toml. uv lock --upgrade diff --git a/docs/03_guides/code/01_beautifulsoup_httpx.py b/docs/03_guides/code/01_beautifulsoup_httpx2.py similarity index 96% rename from docs/03_guides/code/01_beautifulsoup_httpx.py rename to docs/03_guides/code/01_beautifulsoup_httpx2.py index 791b7fc91..6b798116d 100644 --- a/docs/03_guides/code/01_beautifulsoup_httpx.py +++ b/docs/03_guides/code/01_beautifulsoup_httpx2.py @@ -2,7 +2,7 @@ from typing import Any from urllib.parse import urljoin, urlsplit -import httpx +import httpx2 from bs4 import BeautifulSoup from apify import Actor, Request @@ -14,9 +14,9 @@ async def scrape_page( *, proxy_url: str | None = None, ) -> tuple[dict[str, Any], list[str]]: - """Fetch a page with HTTPX and return its data and same-site links.""" + """Fetch a page with HTTPX2 and return its data and same-site links.""" # A fresh client per call lets each request use a new proxy URL. - async with httpx.AsyncClient(proxy=proxy_url) as client: + async with httpx2.AsyncClient(proxy=proxy_url) as client: response = await client.get(url, follow_redirects=True) soup = BeautifulSoup(response.content, 'html.parser') diff --git a/src/apify/_actor.py b/src/apify/_actor.py index be798c420..57b2cdf62 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -84,7 +84,7 @@ class _ActorType: ```python import asyncio - import httpx + import httpx2 from apify import Actor from bs4 import BeautifulSoup @@ -92,7 +92,7 @@ class _ActorType: async def main() -> None: async with Actor: actor_input = await Actor.get_input() - async with httpx.AsyncClient() as client: + async with httpx2.AsyncClient() as client: response = await client.get(actor_input['url']) soup = BeautifulSoup(response.content, 'html.parser') data = {