From d03ebc6851f327e4fb2ddf4574ffc7f7c0c80111 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 11 Sep 2026 11:26:52 +0200 Subject: [PATCH 1/9] feat: Stream request bodies from files, iterators, and responses --- README.md | 2 +- docs/02_concepts/09_streaming.mdx | 82 +++- docs/02_concepts/10_custom_http_clients.mdx | 4 +- docs/02_concepts/13_http_compression.mdx | 6 +- docs/02_concepts/code/09_upload_file_async.py | 16 + docs/02_concepts/code/09_upload_file_sync.py | 14 + .../code/09_upload_generator_async.py | 21 + .../code/09_upload_generator_sync.py | 21 + .../code/10_architecture_imports.py | 1 + docs/02_concepts/code/10_plugging_in_async.py | 4 +- docs/02_concepts/code/10_plugging_in_sync.py | 4 +- docs/03_guides/05_custom_http_client.mdx | 2 + docs/03_guides/06_chain_actors.mdx | 52 +++ .../code/05_custom_http_client_async.py | 2 +- .../code/05_custom_http_client_sync.py | 2 +- docs/03_guides/code/06_chain_actors_async.py | 42 ++ docs/03_guides/code/06_chain_actors_sync.py | 38 ++ src/apify_client/_consts.py | 8 + src/apify_client/_resource_clients/actor.py | 16 +- .../_resource_clients/key_value_store.py | 10 +- src/apify_client/_resource_clients/run.py | 8 +- src/apify_client/_utils/encoding.py | 58 ++- src/apify_client/http_clients/__init__.py | 2 + src/apify_client/http_clients/_base.py | 106 +++-- src/apify_client/http_clients/_httpx2.py | 5 +- src/apify_client/http_clients/_impit.py | 5 +- .../http_clients/_streamed_body.py | 219 ++++++++++ src/apify_client/types.py | 30 +- tests/integration/test_key_value_store.py | 73 ++++ tests/unit/test_actor_start_params.py | 50 +++ tests/unit/test_http_clients.py | 220 +++++++++- tests/unit/test_key_value_store.py | 379 +++++++++++++++--- tests/unit/test_pluggable_http_client.py | 26 +- tests/unit/test_streamed_request_body.py | 337 ++++++++++++++++ tests/unit/test_utils.py | 117 ++++-- 35 files changed, 1785 insertions(+), 197 deletions(-) create mode 100644 docs/02_concepts/code/09_upload_file_async.py create mode 100644 docs/02_concepts/code/09_upload_file_sync.py create mode 100644 docs/02_concepts/code/09_upload_generator_async.py create mode 100644 docs/02_concepts/code/09_upload_generator_sync.py create mode 100644 docs/03_guides/06_chain_actors.mdx create mode 100644 docs/03_guides/code/06_chain_actors_async.py create mode 100644 docs/03_guides/code/06_chain_actors_sync.py create mode 100644 src/apify_client/http_clients/_streamed_body.py create mode 100644 tests/unit/test_streamed_request_body.py diff --git a/README.md b/README.md index 6ac6c0b8..c9a6950a 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ For a guided walkthrough — authenticating, running an Actor, and reading its r - **Fully typed responses** — every method returns a [Pydantic](https://docs.pydantic.dev/) model generated from the Apify OpenAPI spec, with IDE autocomplete and runtime validation ([Typed models](https://docs.apify.com/api/client/python/docs/concepts/typed-models)). - **Automatic retries** — exponential backoff for network errors, HTTP 429, and 5xx responses, configurable per client ([Retries](https://docs.apify.com/api/client/python/docs/concepts/retries)). - **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call ([Timeouts](https://docs.apify.com/api/client/python/docs/concepts/timeouts)). -- **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)). +- **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering, and upload large records straight from a file or another stream ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)). - **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks ([Convenience methods](https://docs.apify.com/api/client/python/docs/concepts/convenience-methods)). - **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX2](https://github.com/pydantic/httpx2) client, or plug in any custom implementation ([HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). - **Structured errors** — every API error surfaces as an [`ApifyApiError`](https://docs.apify.com/api/client/python/reference/class/ApifyApiError) with HTTP-specific subclasses for precise handling ([Error handling](https://docs.apify.com/api/client/python/docs/concepts/error-handling)). diff --git a/docs/02_concepts/09_streaming.mdx b/docs/02_concepts/09_streaming.mdx index de0bb896..763b5026 100644 --- a/docs/02_concepts/09_streaming.mdx +++ b/docs/02_concepts/09_streaming.mdx @@ -1,7 +1,7 @@ --- id: streaming-resources title: Streaming resources -description: Stream large datasets, key-value store records, and logs without loading them into memory. +description: Stream large datasets, key-value store records, and logs in both directions without loading them into memory. --- import Tabs from '@theme/Tabs'; @@ -12,16 +12,22 @@ import ApiLink from '@theme/ApiLink'; import StreamingAsyncExample from '!!raw-loader!./code/09_streaming_async.py'; import StreamingSyncExample from '!!raw-loader!./code/09_streaming_sync.py'; +import UploadFileAsyncExample from '!!raw-loader!./code/09_upload_file_async.py'; +import UploadFileSyncExample from '!!raw-loader!./code/09_upload_file_sync.py'; +import UploadGeneratorAsyncExample from '!!raw-loader!./code/09_upload_generator_async.py'; +import UploadGeneratorSyncExample from '!!raw-loader!./code/09_upload_generator_sync.py'; -Certain resources, such as dataset items, key-value store records, and logs, support streaming directly from the Apify API. This allows you to process large resources incrementally without downloading them entirely into memory, making it ideal for handling large or continuously updated data. +Large resources don't have to pass through memory whole. The client streams downloads of dataset items, key-value store records, and logs as they arrive, and it streams uploads of key-value store records and Actor inputs from a file or another stream as they're read. Both directions work with the synchronous and the asynchronous client. A download can also feed an upload directly, which is how you [chain Actors into a pipeline](../03_guides/06_chain_actors.mdx). -Supported streaming methods: +## Streaming downloads + +Dataset items, key-value store records, and logs can be streamed directly from the Apify API, so you process them incrementally instead of downloading them whole. The supported methods are: - `DatasetClient.stream_items` - Stream dataset items incrementally. Yields a raw streaming `HttpResponse`. - `KeyValueStoreClient.stream_record` - Stream a key-value store record as raw data. Yields a `dict` with the `key`, `value`, and `content_type` fields, where `value` holds the raw streaming response, or `None` when the record doesn't exist. - `LogClient.stream` - Stream logs in real time. Yields a raw streaming `HttpResponse`, or `None` when the log doesn't exist. -All three methods are context managers. Consume the streamed data within a `with` block to ensure that the connection is closed automatically, preventing memory leaks or unclosed connections. +All three methods are context managers. Consume the streamed data within a `with` block, so the connection is closed automatically. The following example shows how to stream the logs of an Actor run incrementally: @@ -38,4 +44,70 @@ The following example shows how to stream the logs of an Actor run incrementally -Streaming is ideal for processing large logs, datasets, or files incrementally without downloading them entirely into memory. +## Streaming uploads + +`KeyValueStoreClient.set_record` and the `run_input` of `ActorClient.start`, `ActorClient.call`, and `RunClient.metamorph` accept a value the client streams to the API in chunks: + +- A file-like object, meaning anything with a `read(size)` method: an open file, an `io.BytesIO`, a member of a ZIP archive, or a pipe. The client reads it in 64 KiB chunks from its current position and doesn't close it. +- An iterator of `bytes` or `str` chunks, such as a generator. The iterator decides the chunk sizes. +- A streaming `HttpResponse` from one of the download methods, whose body is forwarded chunk by chunk. + +The asynchronous client also accepts an async iterator and a file-like object with an `async def read`, such as a file opened with `aiofiles`. The synchronous client rejects those with a `TypeError`. A synchronous file or iterator works with both clients. The asynchronous client reads it in a worker thread, so the event loop stays free. + +Only the chunk being sent is held in memory, so the process uploading a file doesn't need memory for the whole file. + +The following example uploads a file from disk. The asynchronous variant opens it with [aiofiles](https://github.com/Tinche/aiofiles), which isn't an `apify-client` dependency, so install it alongside the client: + +```bash +pip install apify-client aiofiles +``` + + + + + {UploadFileAsyncExample} + + + + + {UploadFileSyncExample} + + + + +With a generator, you produce the data while it uploads, for example from pages of a database query: + + + + + {UploadGeneratorAsyncExample} + + + + + {UploadGeneratorSyncExample} + + + + +### Content type + +A file opened in text mode is uploaded as `text/plain; charset=utf-8`, encoded to UTF-8 chunk by chunk. Any other streamed value is uploaded as `application/octet-stream` unless you pass `content_type`. An iterator or a response carries no type of its own, so set `content_type` explicitly, as the examples do. + +### Compression + +A streamed body is never compressed, because the client sees each chunk only as it's sent. To upload compressed data, compress it yourself and declare the encoding with `content_encoding`. The client then forwards the bytes and the header as they are. For details, see [Pre-compressed bodies](./13_http_compression.mdx#pre-compressed-bodies). + +### Retries + +The client retries a failed request only when it can send the body again: + +- A seekable file-like object, such as an open file or an `io.BytesIO`, is sought back to where it started before each retry, so the upload is retried like any other request. +- An iterator, a streaming response, a pipe, or any asynchronous source is consumed by the attempt that sends it. The request gets a single attempt, and a failure raises immediately. +- A failure inside the source itself, such as a disk error while reading the file, is raised as that error and isn't retried, whatever the source. + +If an upload from a source that can't be rewound needs retries, download the data to a temporary file first and upload the file. + +### Custom HTTP clients + +A custom transport receives a streamed body as an iterator of `bytes` chunks, or as an async iterator in the asynchronous client, so any HTTP library that sends iterable bodies can send it. The `StreamedRequestBody` class documents the rest of the contract. For details, see [The transport contract](./10_custom_http_clients.mdx#the-transport-contract). diff --git a/docs/02_concepts/10_custom_http_clients.mdx b/docs/02_concepts/10_custom_http_clients.mdx index 7d4289d6..645588a0 100644 --- a/docs/02_concepts/10_custom_http_clients.mdx +++ b/docs/02_concepts/10_custom_http_clients.mdx @@ -98,7 +98,7 @@ The base classes and the response protocol are available from the `apify_client. The public `call` method provides the shared request pipeline. A concrete transport implements these hooks: -- `send_request(...)` sends one prepared request and returns an `HttpResponse`, error statuses included. It receives the URL with the query parameters already encoded into it, the headers with the client's default headers already merged in, the body already serialized and compressed, and the timeout for this attempt in seconds. The inherited `call` needs it, so every transport adapter has to implement it. Let the HTTP library's exceptions propagate unwrapped, and leave status handling and `ApifyApiError` to `call`. +- `send_request(...)` sends one prepared request and returns an `HttpResponse`, error statuses included. It receives the URL with the query parameters already encoded into it, the headers with the client's default headers already merged in, the body, and the timeout for this attempt in seconds. The body is either `bytes`, already serialized and compressed, or an iterator of `bytes` chunks for a streamed body, which is an async iterator in the asynchronous client. The inherited `call` needs it, so every transport adapter has to implement it. Let the HTTP library's exceptions propagate unwrapped, and leave status handling and `ApifyApiError` to `call`. - `is_retryable_transport_error(exc)` classifies transport failures for the shared retry loop. The default classifies nothing as retryable, so a transport that doesn't override it gives up on the first connection failure. - `is_timeout_error(exc)` identifies transport-specific timeout exceptions for higher-level client features. The default recognizes Python's `TimeoutError`. Timeout classification is independent of retryability, so a timeout the retry loop should retry has to be listed in `is_retryable_transport_error` too. - `close()` or `aclose()` closes resources owned by the transport. The default does nothing, which is correct for a transport that owns no pool or session. @@ -152,6 +152,8 @@ After that, all API calls made through the client will go through your custom HT If you override `call` itself, your implementation becomes responsible for request preparation, retries, timeouts, API error conversion, logging, and statistics. Implementing the transport hooks and inheriting `call` keeps the shared behavior. ::: +If you override `call` and prepare requests with the inherited helpers, a streamed body arrives as a `StreamedRequestBody`. Send its `iter_bytes()` or `aiter_bytes()`, and send the body a second time only when `rewindable` is true, after calling `rewind()`. For what the client streams and how, see [Streaming uploads](./09_streaming.mdx#streaming-uploads). + ## Use cases Custom HTTP clients might be useful when the built-in Impit and HTTPX2 clients don't cover your requirements, for example when you need to: diff --git a/docs/02_concepts/13_http_compression.mdx b/docs/02_concepts/13_http_compression.mdx index a4d39ea6..b35bd84c 100644 --- a/docs/02_concepts/13_http_compression.mdx +++ b/docs/02_concepts/13_http_compression.mdx @@ -17,7 +17,7 @@ The Apify client compresses request bodies before sending them to the API. It re ## How it works -The client compresses request bodies using the compressor configured via the `compression` parameter (default `'gzip'`). The server supports both gzip and brotli and decompresses the request body transparently. A body is compressed only when it's large enough to benefit, its content type isn't already compressed, and the request carries no `Content-Encoding` of its own. For details, see [Minimum body size](#minimum-body-size), [Already-compressed payloads](#already-compressed-payloads), and [Pre-compressed bodies](#pre-compressed-bodies). +The client compresses request bodies using the compressor configured via the `compression` parameter (default `'gzip'`). The server supports both gzip and brotli and decompresses the request body transparently. A body is compressed only when it's large enough to benefit, its content type isn't already compressed, the request carries no `Content-Encoding` of its own, and the body isn't streamed from a file or an iterator. For details, see [Minimum body size](#minimum-body-size), [Already-compressed payloads](#already-compressed-payloads), [Pre-compressed bodies](#pre-compressed-bodies), and [Streaming uploads](./09_streaming.mdx#streaming-uploads). ## Minimum body size @@ -47,7 +47,7 @@ Two kinds of media type are compressed anyway: raw formats such as `image/bmp`, -Without an explicit content type, a `bytes` value is sent as `application/octet-stream`, which the client can't tell apart from uncompressed binary data and therefore still compresses. File-like values are read into memory before they're sent, so they follow the same rules as any other body. +Without an explicit content type, a `bytes` value is sent as `application/octet-stream`, which the client can't tell apart from uncompressed binary data and therefore still compresses. A streamed value, such as an open file or an iterator of chunks, is never compressed. For details, see [Streaming uploads](./09_streaming.mdx#streaming-uploads). ## Pre-compressed bodies @@ -66,7 +66,7 @@ A payload can reach the client already encoded, for example a gzipped file read -The header is forwarded verbatim, so it also covers encodings the client ships no compressor for, such as `deflate`. The API accepts `gzip`, `br`, `deflate`, and `identity`. Passing `identity` turns compression off for a single request without changing how the client is configured. +The header is forwarded verbatim, so it also covers encodings the client ships no compressor for, such as `deflate`. It's forwarded for a streamed value too, so an open gzipped file can be passed in place of its bytes. The API accepts `gzip`, `br`, `deflate`, and `identity`. Passing `identity` turns compression off for a single request without changing how the client is configured. A value that can't be compressed at all - a string, an object serialized to JSON, or a file-like value opened in text mode - is rejected with a `TypeError` when `content_encoding` names a compression. Beyond that the client can't verify that the bytes match the header, so set `Content-Encoding` only when the payload really is encoded that way. Key-value store records are stored exactly as you upload them, which makes the header part of the stored record rather than a transport detail. diff --git a/docs/02_concepts/code/09_upload_file_async.py b/docs/02_concepts/code/09_upload_file_async.py new file mode 100644 index 00000000..04c0e151 --- /dev/null +++ b/docs/02_concepts/code/09_upload_file_async.py @@ -0,0 +1,16 @@ +import aiofiles + +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + kvs_client = apify_client.key_value_store('MY-KVS-ID') + + # The file is read in chunks as it uploads, so its size doesn't matter. + async with aiofiles.open('backup.tar.gz', 'rb') as backup: + await kvs_client.set_record( + 'backup.tar.gz', backup, content_type='application/gzip' + ) diff --git a/docs/02_concepts/code/09_upload_file_sync.py b/docs/02_concepts/code/09_upload_file_sync.py new file mode 100644 index 00000000..537009b1 --- /dev/null +++ b/docs/02_concepts/code/09_upload_file_sync.py @@ -0,0 +1,14 @@ +from pathlib import Path + +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + kvs_client = apify_client.key_value_store('MY-KVS-ID') + + # The file is read in chunks as it uploads, so its size doesn't matter. + with Path('backup.tar.gz').open('rb') as backup: + kvs_client.set_record('backup.tar.gz', backup, content_type='application/gzip') diff --git a/docs/02_concepts/code/09_upload_generator_async.py b/docs/02_concepts/code/09_upload_generator_async.py new file mode 100644 index 00000000..7f809a0c --- /dev/null +++ b/docs/02_concepts/code/09_upload_generator_async.py @@ -0,0 +1,21 @@ +from collections.abc import AsyncIterator + +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def csv_chunks() -> AsyncIterator[str]: + """Build the CSV in pieces, for example from pages of a database query.""" + yield 'id,value\n' + for start in range(0, 1_000_000, 10_000): + yield ''.join(f'{i},{i * i}\n' for i in range(start, start + 10_000)) + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + kvs_client = apify_client.key_value_store('MY-KVS-ID') + + # Each chunk is encoded and sent as it's produced. An iterator can't be + # rewound, so a failed upload isn't retried. + await kvs_client.set_record('report.csv', csv_chunks(), content_type='text/csv') diff --git a/docs/02_concepts/code/09_upload_generator_sync.py b/docs/02_concepts/code/09_upload_generator_sync.py new file mode 100644 index 00000000..2e9464c9 --- /dev/null +++ b/docs/02_concepts/code/09_upload_generator_sync.py @@ -0,0 +1,21 @@ +from collections.abc import Iterator + +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def csv_chunks() -> Iterator[str]: + """Build the CSV in pieces, for example from pages of a database query.""" + yield 'id,value\n' + for start in range(0, 1_000_000, 10_000): + yield ''.join(f'{i},{i * i}\n' for i in range(start, start + 10_000)) + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + kvs_client = apify_client.key_value_store('MY-KVS-ID') + + # Each chunk is encoded and sent as it's produced. An iterator can't be + # rewound, so a failed upload isn't retried. + kvs_client.set_record('report.csv', csv_chunks(), content_type='text/csv') diff --git a/docs/02_concepts/code/10_architecture_imports.py b/docs/02_concepts/code/10_architecture_imports.py index c462b2f9..c0a055a3 100644 --- a/docs/02_concepts/code/10_architecture_imports.py +++ b/docs/02_concepts/code/10_architecture_imports.py @@ -2,4 +2,5 @@ HttpClient, HttpClientAsync, HttpResponse, + StreamedRequestBody, ) diff --git a/docs/02_concepts/code/10_plugging_in_async.py b/docs/02_concepts/code/10_plugging_in_async.py index dad83733..69837312 100644 --- a/docs/02_concepts/code/10_plugging_in_async.py +++ b/docs/02_concepts/code/10_plugging_in_async.py @@ -1,3 +1,5 @@ +from collections.abc import AsyncIterator + from typing_extensions import override from apify_client import ApifyClientAsync @@ -16,7 +18,7 @@ async def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | AsyncIterator[bytes] | None, timeout: float | None, stream: bool, ) -> HttpResponse: diff --git a/docs/02_concepts/code/10_plugging_in_sync.py b/docs/02_concepts/code/10_plugging_in_sync.py index 8f0aa362..1a8ae9c6 100644 --- a/docs/02_concepts/code/10_plugging_in_sync.py +++ b/docs/02_concepts/code/10_plugging_in_sync.py @@ -1,3 +1,5 @@ +from collections.abc import Iterator + from typing_extensions import override from apify_client import ApifyClient @@ -16,7 +18,7 @@ def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | Iterator[bytes] | None, timeout: float | None, stream: bool, ) -> HttpResponse: diff --git a/docs/03_guides/05_custom_http_client.mdx b/docs/03_guides/05_custom_http_client.mdx index 05f1c05e..63f77bb3 100644 --- a/docs/03_guides/05_custom_http_client.mdx +++ b/docs/03_guides/05_custom_http_client.mdx @@ -33,6 +33,8 @@ Each example has three parts: 2. The client, `AiohttpHttpClient` or `RequestsHttpClient`, implements the transport, error-classification, and lifecycle hooks. It inherits request preparation, retry handling, timeout growth, and API error conversion from its base class. Only the requests client also overrides timeout classification. `requests.Timeout` doesn't derive from Python's `TimeoutError`, while aiohttp's timeout errors do, so the inherited default already recognizes them. 3. `with_custom_http_client()` connects the implementation to the resource clients and applies the API token. The context manager closes the session at shutdown. +Both libraries accept an iterator of chunks as the request body, a synchronous one in `requests` and an asynchronous one in aiohttp, so [streamed uploads](../02_concepts/09_streaming.mdx#streaming-uploads) work through these clients as they are. + diff --git a/docs/03_guides/06_chain_actors.mdx b/docs/03_guides/06_chain_actors.mdx new file mode 100644 index 00000000..3799d79f --- /dev/null +++ b/docs/03_guides/06_chain_actors.mdx @@ -0,0 +1,52 @@ +--- +id: chain-actors-into-a-pipeline +title: Chain Actors into a pipeline +description: Pass one Actor's output to another Actor as input, and file dataset exports away, without holding the data in memory. +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; + +import ChainActorsAsyncExample from '!!raw-loader!./code/06_chain_actors_async.py'; +import ChainActorsSyncExample from '!!raw-loader!./code/06_chain_actors_sync.py'; + +A pipeline runs several Actors in sequence, and each stage consumes what the previous one produced. The client moves the data between stages as a stream: a record or a dataset export downloads from one place and uploads to another chunk by chunk, so the process connecting the stages needs no memory for the whole payload. An orchestrating Actor can therefore run with a small memory allocation even when the data it moves is large. + +For the values that can be streamed and how retries and compression apply to them, see [Streaming uploads](../02_concepts/09_streaming.mdx#streaming-uploads). + +## Before you start + +The example uses two placeholder Actors. Replace `my-org/producer-actor` with an Actor that saves its result to the `OUTPUT` record of its default key-value store, and `my-org/consumer-actor` with an Actor that accepts that record as its input. + +## Pipe a record into another Actor's input + +The example runs a two-stage pipeline: + +1. Run the producer Actor and wait for it to finish. +2. Open the `OUTPUT` record of the producer's default key-value store as a stream. +3. Start the consumer Actor with the stream as its input. The record uploads as it downloads, under the content type the producer stored it with. +4. Export the producer's dataset as CSV into a named key-value store, where a later stage or a person can pick it up as one file. + + + + + {ChainActorsAsyncExample} + + + + + {ChainActorsSyncExample} + + + + +A streaming response is consumed by the attempt that sends it, so a failed upload isn't retried. If a stage needs retries, download the data to a temporary file first and upload the file, which the client can rewind. + +## Other data you can pipe + +The same pattern works for anything the client can stream: + +- A dataset export in any format `stream_items` supports, such as `json`, `jsonl`, `csv`, or `xlsx`. Set `content_type` to match the format. +- A run's log from `LogClient.stream`, for example to archive it as a record once the run finishes. +- A local file or a generator, as shown in [Streaming uploads](../02_concepts/09_streaming.mdx#streaming-uploads). diff --git a/docs/03_guides/code/05_custom_http_client_async.py b/docs/03_guides/code/05_custom_http_client_async.py index c3aeb030..944743fa 100644 --- a/docs/03_guides/code/05_custom_http_client_async.py +++ b/docs/03_guides/code/05_custom_http_client_async.py @@ -97,7 +97,7 @@ async def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | AsyncIterator[bytes] | None, timeout: float | None, stream: bool, ) -> HttpResponse: diff --git a/docs/03_guides/code/05_custom_http_client_sync.py b/docs/03_guides/code/05_custom_http_client_sync.py index 75386b63..d88cce0e 100644 --- a/docs/03_guides/code/05_custom_http_client_sync.py +++ b/docs/03_guides/code/05_custom_http_client_sync.py @@ -105,7 +105,7 @@ def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | Iterator[bytes] | None, timeout: float | None, stream: bool, ) -> HttpResponse: diff --git a/docs/03_guides/code/06_chain_actors_async.py b/docs/03_guides/code/06_chain_actors_async.py new file mode 100644 index 00000000..292934ff --- /dev/null +++ b/docs/03_guides/code/06_chain_actors_async.py @@ -0,0 +1,42 @@ +import asyncio + +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(token=TOKEN) + + # Run the first Actor and wait for it to finish. + producer_run = await apify_client.actor('my-org/producer-actor').call( + run_input={'startUrls': [{'url': 'https://example.com'}]}, + ) + if producer_run is None: + raise RuntimeError('The producer run was not found') + + # Stream the producer's OUTPUT record straight into the consumer's input. + # The record uploads as it downloads, so it never has to fit in memory. + producer_store = apify_client.key_value_store(producer_run.default_key_value_store_id) + async with producer_store.stream_record('OUTPUT') as record: + if record is None: + raise RuntimeError('The producer stored no OUTPUT record') + consumer_run = await apify_client.actor('my-org/consumer-actor').call( + run_input=record['value'], + content_type=record['content_type'], + ) + print(consumer_run) + + # Export the producer's dataset as CSV into a named key-value store, where + # the next stage of the pipeline can pick it up as a single file. + reports_store = await apify_client.key_value_stores().get_or_create( + name='pipeline-reports' + ) + reports_client = apify_client.key_value_store(reports_store.id) + producer_dataset = apify_client.dataset(producer_run.default_dataset_id) + async with producer_dataset.stream_items(item_format='csv') as items: + await reports_client.set_record('items.csv', items, content_type='text/csv') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/03_guides/code/06_chain_actors_sync.py b/docs/03_guides/code/06_chain_actors_sync.py new file mode 100644 index 00000000..61b55efd --- /dev/null +++ b/docs/03_guides/code/06_chain_actors_sync.py @@ -0,0 +1,38 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(token=TOKEN) + + # Run the first Actor and wait for it to finish. + producer_run = apify_client.actor('my-org/producer-actor').call( + run_input={'startUrls': [{'url': 'https://example.com'}]}, + ) + if producer_run is None: + raise RuntimeError('The producer run was not found') + + # Stream the producer's OUTPUT record straight into the consumer's input. + # The record uploads as it downloads, so it never has to fit in memory. + producer_store = apify_client.key_value_store(producer_run.default_key_value_store_id) + with producer_store.stream_record('OUTPUT') as record: + if record is None: + raise RuntimeError('The producer stored no OUTPUT record') + consumer_run = apify_client.actor('my-org/consumer-actor').call( + run_input=record['value'], + content_type=record['content_type'], + ) + print(consumer_run) + + # Export the producer's dataset as CSV into a named key-value store, where + # the next stage of the pipeline can pick it up as a single file. + reports_store = apify_client.key_value_stores().get_or_create(name='pipeline-reports') + reports_client = apify_client.key_value_store(reports_store.id) + producer_dataset = apify_client.dataset(producer_run.default_dataset_id) + with producer_dataset.stream_items(item_format='csv') as items: + reports_client.set_record('items.csv', items, content_type='text/csv') + + +if __name__ == '__main__': + main() diff --git a/src/apify_client/_consts.py b/src/apify_client/_consts.py index d64fa1ef..e8875b00 100644 --- a/src/apify_client/_consts.py +++ b/src/apify_client/_consts.py @@ -42,6 +42,14 @@ saving a round trip. """ +STREAMED_BODY_CHUNK_SIZE = 64 * 1024 +"""Size, in bytes, of the chunks a streamed request body reads from a file-like source. + +A chunk is the most of a streamed body that is in memory at once, and in the asynchronous client every chunk costs +one worker-thread hop, so the size balances memory against per-chunk overhead. It matches the buffer size the +standard library and common HTTP libraries use to copy files. +""" + ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES = ('audio/', 'image/', 'video/') """Media type prefixes whose payloads carry their own compression, so compressing the request body is wasted work.""" diff --git a/src/apify_client/_resource_clients/actor.py b/src/apify_client/_resource_clients/actor.py index 705a4c27..37a15fab 100644 --- a/src/apify_client/_resource_clients/actor.py +++ b/src/apify_client/_resource_clients/actor.py @@ -236,7 +236,9 @@ def start( https://docs.apify.com/api/v2#/reference/actors/run-collection/run-actor Args: - run_input: The input to pass to the Actor run. + run_input: The input to pass to the Actor run. Accepts the same values as + `KeyValueStoreClient.set_record`, including a file-like object, an iterator of byte chunks, or a + streamed `HttpResponse`, which are uploaded in chunks without being held in memory. content_type: The content type of the input. build: Specifies the Actor build to run. It can be either a build tag or build number. By default, the run uses the build specified in the default run configuration for the Actor (typically latest). @@ -315,7 +317,9 @@ def call( https://docs.apify.com/api/v2#/reference/actors/run-collection/run-actor Args: - run_input: The input to pass to the Actor run. + run_input: The input to pass to the Actor run. Accepts the same values as + `KeyValueStoreClient.set_record`, including a file-like object, an iterator of byte chunks, or a + streamed `HttpResponse`, which are uploaded in chunks without being held in memory. content_type: The content type of the input. build: Specifies the Actor build to run. It can be either a build tag or build number. By default, the run uses the build specified in the default run configuration for the Actor (typically latest). @@ -738,7 +742,9 @@ async def start( https://docs.apify.com/api/v2#/reference/actors/run-collection/run-actor Args: - run_input: The input to pass to the Actor run. + run_input: The input to pass to the Actor run. Accepts the same values as + `KeyValueStoreClientAsync.set_record`, including a file-like object, an iterator of byte chunks, or a + streamed `HttpResponse`, which are uploaded in chunks without being held in memory. content_type: The content type of the input. build: Specifies the Actor build to run. It can be either a build tag or build number. By default, the run uses the build specified in the default run configuration for the Actor (typically latest). @@ -817,7 +823,9 @@ async def call( https://docs.apify.com/api/v2#/reference/actors/run-collection/run-actor Args: - run_input: The input to pass to the Actor run. + run_input: The input to pass to the Actor run. Accepts the same values as + `KeyValueStoreClientAsync.set_record`, including a file-like object, an iterator of byte chunks, or a + streamed `HttpResponse`, which are uploaded in chunks without being held in memory. content_type: The content type of the input. build: Specifies the Actor build to run. It can be either a build tag or build number. By default, the run uses the build specified in the default run configuration for the Actor (typically latest). diff --git a/src/apify_client/_resource_clients/key_value_store.py b/src/apify_client/_resource_clients/key_value_store.py index 34814055..723f2ffb 100644 --- a/src/apify_client/_resource_clients/key_value_store.py +++ b/src/apify_client/_resource_clients/key_value_store.py @@ -369,7 +369,10 @@ def set_record( Args: key: The key of the record to save the value to. - value: The value to save into the record. + value: The value to save into the record. A file-like object, an iterator of byte chunks, or a streamed + `HttpResponse` is uploaded in chunks as it is read, without being held in memory whole or compressed. + Only a seekable file-like value can be retried, any other streamed value gets a single attempt. See + `StreamedRequestBody` for details. content_type: The content type of the saved value. content_encoding: The encoding already applied to `value`, sent as the `Content-Encoding` header. Pass it to upload a pre-compressed value - the client then forwards the bytes as they are instead of @@ -802,7 +805,10 @@ async def set_record( Args: key: The key of the record to save the value to. - value: The value to save into the record. + value: The value to save into the record. A file-like object, an iterator of byte chunks, or a streamed + `HttpResponse` is uploaded in chunks as it is read, without being held in memory whole or compressed. + Only a seekable file-like value can be retried, any other streamed value gets a single attempt. See + `StreamedRequestBody` for details. content_type: The content type of the saved value. content_encoding: The encoding already applied to `value`, sent as the `Content-Encoding` header. Pass it to upload a pre-compressed value - the client then forwards the bytes as they are instead of diff --git a/src/apify_client/_resource_clients/run.py b/src/apify_client/_resource_clients/run.py index 7ddd6b30..e794d50d 100644 --- a/src/apify_client/_resource_clients/run.py +++ b/src/apify_client/_resource_clients/run.py @@ -180,7 +180,9 @@ def metamorph( target_actor_build: The build of the target Actor. It can be either a build tag or build number. By default, the run uses the build specified in the default run configuration for the target Actor (typically the latest build). - run_input: The input to pass to the new run. + run_input: The input to pass to the new run. Accepts the same values as + `KeyValueStoreClient.set_record`, including a file-like object, an iterator of byte chunks, or a + streamed `HttpResponse`, which are uploaded in chunks without being held in memory. content_type: The content type of the input. timeout: Timeout for the API HTTP request. @@ -608,7 +610,9 @@ async def metamorph( target_actor_build: The build of the target Actor. It can be either a build tag or build number. By default, the run uses the build specified in the default run configuration for the target Actor (typically the latest build). - run_input: The input to pass to the new run. + run_input: The input to pass to the new run. Accepts the same values as + `KeyValueStoreClientAsync.set_record`, including a file-like object, an iterator of byte chunks, or a + streamed `HttpResponse`, which are uploaded in chunks without being held in memory. content_type: The content type of the input. timeout: Timeout for the API HTTP request. diff --git a/src/apify_client/_utils/encoding.py b/src/apify_client/_utils/encoding.py index 39e5496f..ae78dc21 100644 --- a/src/apify_client/_utils/encoding.py +++ b/src/apify_client/_utils/encoding.py @@ -1,32 +1,34 @@ from __future__ import annotations +import io import json from base64 import b64encode from functools import cache -from inspect import isawaitable, iscoroutine from typing import TYPE_CHECKING, Any from apify_client._models import WebhookCreate, WebhookRepresentation +from apify_client.http_clients._streamed_body import StreamedRequestBody if TYPE_CHECKING: - from apify_client.types import WebhooksList + from apify_client.types import StreamedBodySource, WebhooksList def encode_key_value_store_record_value( value: Any, *, content_type: str | None = None, content_encoding: str | None = None -) -> tuple[bytes | bytearray | str, str]: +) -> tuple[bytes | bytearray | str | StreamedBodySource, str]: """Encode a value for storage in a key-value store record. Args: - value: The value to encode. Anything exposing a callable `read` is treated as a file-like object: `read` - is called with no arguments, so the value is consumed from its current position and buffered in - memory whole - the object is neither rewound nor closed, and async file-like objects are rejected. - Any other value is JSON-serialized unless it is already bytes or a string. - content_type: The content type; if None, it's inferred from the value type. + value: The value to encode. A file-like object (anything with a callable `read`), an iterator of byte chunks, + or a streamed `HttpResponse` is returned as it is, to be streamed to the API in chunks from its current + position - the object is neither rewound nor closed. Any other value is JSON-serialized unless it is + already bytes or a string. + content_type: The content type; if None, it's inferred from the value type. A file opened in text mode is + `text/plain; charset=utf-8`, any other streamed value is `application/octet-stream`. content_encoding: The encoding the caller declares the value already carries, if any. Anything other than - `identity` means the value is compressed, which only a bytes-like payload can be, so any other value - is rejected. The check belongs here because a file-like value has to be read before its payload type - is known, and reading it a second time in the caller is not possible. + `identity` means the value is compressed, which only a bytes-like payload can be, so a string, a + JSON-serialized object, or a text-mode file is rejected. Any other streamed value is taken at its word, + since its bytes are only seen as they are sent. Returns: A tuple of (encoded_value, content_type). @@ -35,29 +37,23 @@ def encode_key_value_store_record_value( TypeError: If the value cannot be encoded into a body the transport accepts, or if it cannot be carrying the declared `content_encoding`. """ - # Read file-like values into memory; the transport only accepts bytes-like bodies. Detect them by a - # callable `read` (not `io.IOBase`) so duck-typed file-likes are read, not JSON-serialized. Impit exposes - # no streaming `content=` API, so the value has to be buffered whole. - read = getattr(value, 'read', None) - if callable(read): - value = read() - - if isawaitable(value): - if iscoroutine(value): - value.close() # Prevent a "coroutine was never awaited" warning. + declared_encoding = (content_encoding or '').strip().lower() + declares_compression = declared_encoding not in ('', 'identity') + + if StreamedRequestBody.is_source(value): + is_text = isinstance(value, io.TextIOBase) + if declares_compression and is_text: raise TypeError( - 'Async file-like objects are not supported. Await the read yourself and pass the resulting ' - 'bytes or string.' + f'Cannot upload a file-like value opened in text mode with `Content-Encoding: {content_encoding}`. ' + 'An encoding other than `identity` declares the value is already compressed, so pass the compressed ' + 'bytes, or a binary file-like object that reads them.' ) + return (value, content_type or ('text/plain; charset=utf-8' if is_text else 'application/octet-stream')) - if not isinstance(value, (bytes, bytearray, str)): - raise TypeError(f'Reading the file-like value returned {type(value).__name__}, expected bytes or str.') - - # A declared compression describes bytes the caller compressed. A string, a JSON-serializable object, or a - # text-mode file cannot be carrying one, and would otherwise be stored under a header that misdescribes it - - # the client forwards the header untouched and never inspects the body. - declared_encoding = (content_encoding or '').strip().lower() - if declared_encoding not in ('', 'identity') and not isinstance(value, (bytes, bytearray)): + # A declared compression describes bytes the caller compressed. A string or a JSON-serializable object cannot + # be carrying one, and would otherwise be stored under a header that misdescribes it - the client forwards the + # header untouched and never inspects the body. + if declares_compression and not isinstance(value, (bytes, bytearray)): raise TypeError( f'Cannot upload a {type(value).__name__} value with `Content-Encoding: {content_encoding}`. An encoding ' 'other than `identity` declares the value is already compressed, so pass the compressed bytes, or a ' diff --git a/src/apify_client/http_clients/__init__.py b/src/apify_client/http_clients/__init__.py index 6bfea628..8aaec575 100644 --- a/src/apify_client/http_clients/__init__.py +++ b/src/apify_client/http_clients/__init__.py @@ -2,6 +2,7 @@ from apify_client._utils.try_import import try_import as _try_import from apify_client.http_clients._base import HttpClient, HttpClientAsync, HttpResponse from apify_client.http_clients._impit import ImpitHttpClient, ImpitHttpClientAsync +from apify_client.http_clients._streamed_body import StreamedRequestBody _install_import_hook(__name__) @@ -22,6 +23,7 @@ 'HttpResponse', 'ImpitHttpClient', 'ImpitHttpClientAsync', + 'StreamedRequestBody', ] if _httpx2_import.available: diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index 0e49c6c6..8cac33cc 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -34,6 +34,7 @@ from apify_client._utils.http import is_compressible_content_type from apify_client._utils.time import to_seconds from apify_client.errors import ApifyApiError +from apify_client.http_clients._streamed_body import StreamedRequestBody from apify_client.http_compressors._gzip import GzipHttpCompressor if TYPE_CHECKING: @@ -42,7 +43,7 @@ from typing import Self from apify_client.http_compressors._base import HttpCompressor - from apify_client.types import JsonSerializable, Timeout + from apify_client.types import JsonSerializable, StreamedBodySource, Timeout logger = logging.getLogger(logger_name) logger_once = LoggerOnce(logger) @@ -237,7 +238,7 @@ def _parse_params(params: dict[str, Any] | None) -> dict[str, Any] | None: return parsed_params @staticmethod - def _is_body_worth_compressing(data: str | bytes | bytearray | None) -> bool: + def _is_body_worth_compressing(data: object) -> bool: """Whether the body is large enough that `_prepare_request_call` may compress it. This gate only picks where the preparation runs (worker thread or inline), so it approximates rather @@ -294,9 +295,9 @@ def _prepare_request_call( *, headers: dict[str, str] | None = None, params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, + data: str | bytes | bytearray | StreamedBodySource | None = None, json: JsonSerializable | None = None, - ) -> tuple[dict[str, str], dict[str, Any] | None, bytes | None]: + ) -> tuple[dict[str, str], dict[str, Any] | None, bytes | StreamedRequestBody | None]: """Prepare headers, params, and body for an HTTP request. Merges the client's default headers (including authorization) with per-request headers and serializes a @@ -308,6 +309,10 @@ def _prepare_request_call( `Content-Encoding` is forwarded verbatim, which is how a pre-encoded body is uploaded - including one in an encoding the client ships no compressor for. `Content-Encoding: identity` therefore opts a single request out of compression. + + A body streamed from a file-like object, an iterator of byte chunks, or a streamed response is wrapped in + `StreamedRequestBody` and never compressed: its chunks go out as they are produced, so nothing is buffered. + A caller-supplied `Content-Encoding` is forwarded for it too, which is how pre-compressed data is streamed. """ if json is not None and data is not None: raise ValueError('Cannot pass both "json" and "data" parameters at the same time!') @@ -320,23 +325,24 @@ def _prepare_request_call( if self._get_header(headers, 'content-type') is None: headers['Content-Type'] = 'application/json' + if StreamedRequestBody.is_source(data): + return (headers, self._parse_params(params), StreamedRequestBody(data)) + + content: bytes | None = None if isinstance(data, (str, bytes, bytearray)): - if isinstance(data, str): - data = data.encode('utf-8') - elif isinstance(data, bytearray): - data = bytes(data) + content = data.encode('utf-8') if isinstance(data, str) else bytes(data) # A caller-supplied encoding says the body arrives already encoded, so compressing it here would # both mislabel it and waste the work. if ( self._get_header(headers, 'content-encoding') is None - and len(data) >= MIN_COMPRESSION_SIZE + and len(content) >= MIN_COMPRESSION_SIZE and is_compressible_content_type(self._get_header(headers, 'content-type')) ): - data = self._http_compressor.compress(data) + content = self._http_compressor.compress(content) headers = self._merge_headers(headers, {'Content-Encoding': self._http_compressor.content_encoding}) - return (headers, self._parse_params(params), data) + return (headers, self._parse_params(params), content) def _build_url_with_params(self, url: str, *, params: dict[str, Any] | None = None) -> str: """Build a URL with query parameters appended. List values are expanded into multiple key=value pairs.""" @@ -354,13 +360,53 @@ def _build_url_with_params(self, url: str, *, params: dict[str, Any] | None = No return f'{url}?{query_string}' - def _handle_request_exception(self, exc: Exception, *, stop_retrying: Callable[[], None]) -> None: - """Stop retrying when an exception is not a retryable transport failure.""" + def _handle_request_exception( + self, + exc: Exception, + *, + content: bytes | StreamedRequestBody | None, + stop_retrying: Callable[[], None], + ) -> None: + """Stop retrying when an exception is not a retryable transport failure. + + A failure of a streamed body's source stops retrying too, since sending the body again cannot fix it. The + transport reports such a failure as its own error, which may wrap the cause beyond recognition and pass as + transient, so the source's exception is raised in its place, with the transport error as its cause. + """ logger.debug('Request threw exception', exc_info=exc) + + source_error = content.error if isinstance(content, StreamedRequestBody) else None + if source_error is not None: + logger.debug('Producing the streamed request body failed', exc_info=source_error) + stop_retrying() + if source_error is not exc: + raise source_error from exc + return + if not self.is_retryable_transport_error(exc): logger.debug('Exception is not retryable', exc_info=exc) stop_retrying() + @staticmethod + def _prepare_streamed_body( + content: bytes | StreamedRequestBody | None, + *, + attempt: int, + stop_retrying: Callable[[], None], + ) -> None: + """Get a streamed body ready for a request attempt. + + A rewindable body is sought back to its start before every attempt but the first. Any other streamed body is + consumed by the attempt that sends it, so retrying stops up front and a failure of the attempt is final. + """ + if not isinstance(content, StreamedRequestBody): + return + if not content.rewindable: + logger.debug('The streamed request body cannot be rewound, so a failed attempt is not retried') + stop_retrying() + elif attempt > 1: + content.rewind() + def _handle_response_status( self, response: HttpResponse, @@ -425,7 +471,7 @@ def call( url: str, headers: dict[str, str] | None = None, params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, + data: str | bytes | bytearray | StreamedBodySource | None = None, json: JsonSerializable | None = None, stream: bool | None = None, timeout: Timeout = 'medium', @@ -437,7 +483,8 @@ def call( url: Full URL to make the request to. headers: Additional headers to include. params: Query parameters to append to the URL. - data: Raw request body data. Cannot be used together with json. + data: Raw request body. A file-like object, an iterator of byte chunks, or a streamed `HttpResponse` + is sent in chunks as it is read, see `StreamedRequestBody`. Cannot be used together with json. json: JSON-serializable data for the request body. Cannot be used together with data. stream: Whether to stream the response body. timeout: Timeout for the API HTTP request. Use `short`, `medium`, or `long` tier literals for @@ -491,7 +538,7 @@ def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | Iterator[bytes] | None, timeout: float | None, stream: bool, ) -> HttpResponse: @@ -505,7 +552,8 @@ def send_request( method: HTTP method (GET, POST, PUT, DELETE, etc.). url: Full request URL, with the query parameters already encoded into it. headers: Final request headers, with the client's default headers already merged in. - content: Request body, already serialized and compressed, or None for a request without a body. + content: Request body, already serialized and compressed, an iterator of chunks for a streamed body, + or None for a request without a body. timeout: Timeout for this attempt in seconds, or None for no timeout at all. stream: Whether to return the response with the body unread, so the caller can stream it. @@ -558,7 +606,7 @@ def _make_request( url: str, headers: dict[str, str], params: dict[str, Any] | None, - content: bytes | None, + content: bytes | StreamedRequestBody | None, stream: bool | None, timeout: Timeout, ) -> HttpResponse: @@ -569,16 +617,17 @@ def _make_request( self._statistics.requests += 1 try: + self._prepare_streamed_body(content, attempt=attempt, stop_retrying=stop_retrying) response = self.send_request( method=method, url=self._build_url_with_params(url, params=params), headers=headers, - content=content, + content=content.iter_bytes() if isinstance(content, StreamedRequestBody) else content, timeout=self._compute_timeout(timeout, attempt=attempt), stream=stream or False, ) except Exception as exc: - self._handle_request_exception(exc, stop_retrying=stop_retrying) + self._handle_request_exception(exc, content=content, stop_retrying=stop_retrying) raise if self._handle_response_status(response, attempt=attempt, stop_retrying=stop_retrying): @@ -628,7 +677,7 @@ async def call( url: str, headers: dict[str, str] | None = None, params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, + data: str | bytes | bytearray | StreamedBodySource | None = None, json: JsonSerializable | None = None, stream: bool | None = None, timeout: Timeout = 'medium', @@ -640,7 +689,8 @@ async def call( url: Full URL to make the request to. headers: Additional headers to include. params: Query parameters to append to the URL. - data: Raw request body data. Cannot be used together with json. + data: Raw request body. A file-like object, an iterator of byte chunks, or a streamed `HttpResponse` + is sent in chunks as it is read, see `StreamedRequestBody`. Cannot be used together with json. json: JSON-serializable data for the request body. Cannot be used together with data. stream: Whether to stream the response body. timeout: Timeout for the API HTTP request. Use `short`, `medium`, or `long` tier literals for @@ -707,7 +757,7 @@ async def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | AsyncIterator[bytes] | None, timeout: float | None, stream: bool, ) -> HttpResponse: @@ -721,7 +771,8 @@ async def send_request( method: HTTP method (GET, POST, PUT, DELETE, etc.). url: Full request URL, with the query parameters already encoded into it. headers: Final request headers, with the client's default headers already merged in. - content: Request body, already serialized and compressed, or None for a request without a body. + content: Request body, already serialized and compressed, an iterator of chunks for a streamed body, + or None for a request without a body. timeout: Timeout for this attempt in seconds, or None for no timeout at all. stream: Whether to return the response with the body unread, so the caller can stream it. @@ -774,7 +825,7 @@ async def _make_request( url: str, headers: dict[str, str], params: dict[str, Any] | None, - content: bytes | None, + content: bytes | StreamedRequestBody | None, stream: bool | None, timeout: Timeout, ) -> HttpResponse: @@ -785,16 +836,17 @@ async def _make_request( self._statistics.requests += 1 try: + self._prepare_streamed_body(content, attempt=attempt, stop_retrying=stop_retrying) response = await self.send_request( method=method, url=self._build_url_with_params(url, params=params), headers=headers, - content=content, + content=content.aiter_bytes() if isinstance(content, StreamedRequestBody) else content, timeout=self._compute_timeout(timeout, attempt=attempt), stream=stream or False, ) except Exception as exc: - self._handle_request_exception(exc, stop_retrying=stop_retrying) + self._handle_request_exception(exc, content=content, stop_retrying=stop_retrying) raise if self._handle_response_status(response, attempt=attempt, stop_retrying=stop_retrying): diff --git a/src/apify_client/http_clients/_httpx2.py b/src/apify_client/http_clients/_httpx2.py index 429fda78..48eac9d9 100644 --- a/src/apify_client/http_clients/_httpx2.py +++ b/src/apify_client/http_clients/_httpx2.py @@ -17,6 +17,7 @@ from apify_client.http_clients._base import HttpClient, HttpClientAsync if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterator from datetime import timedelta from apify_client._statistics import ClientStatistics @@ -122,7 +123,7 @@ def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | Iterator[bytes] | None, timeout: float | None, stream: bool, ) -> httpx2.Response: @@ -226,7 +227,7 @@ async def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | AsyncIterator[bytes] | None, timeout: float | None, stream: bool, ) -> httpx2.Response: diff --git a/src/apify_client/http_clients/_impit.py b/src/apify_client/http_clients/_impit.py index e18a420b..5d01b2c2 100644 --- a/src/apify_client/http_clients/_impit.py +++ b/src/apify_client/http_clients/_impit.py @@ -17,6 +17,7 @@ from apify_client.http_clients._base import HttpClient, HttpClientAsync if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterator from datetime import timedelta from apify_client._statistics import ClientStatistics @@ -121,7 +122,7 @@ def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | Iterator[bytes] | None, timeout: float | None, stream: bool, ) -> impit.Response: @@ -219,7 +220,7 @@ async def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | AsyncIterator[bytes] | None, timeout: float | None, stream: bool, ) -> impit.Response: diff --git a/src/apify_client/http_clients/_streamed_body.py b/src/apify_client/http_clients/_streamed_body.py new file mode 100644 index 00000000..87f45126 --- /dev/null +++ b/src/apify_client/http_clients/_streamed_body.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import asyncio +import inspect +from collections.abc import AsyncIterator, Iterator +from typing import TYPE_CHECKING, Any + +from apify_client._consts import STREAMED_BODY_CHUNK_SIZE +from apify_client._docs import docs_group + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable + from typing import TypeGuard + + from apify_client.types import StreamedBodySource + +_DONE = object() +"""Sentinel a worker thread returns once a synchronous iterator is exhausted.""" + + +@docs_group('HTTP clients') +class StreamedRequestBody: + """A request body sent from its source in chunks, so the body is never held in memory whole. + + `HttpClient.call` and `HttpClientAsync.call` wrap a `data` argument that is a file-like object, an iterator of + byte chunks, or a streamed `HttpResponse` in this class. The transport pulls the chunks from `iter_bytes` or + `aiter_bytes` and sends each one as it arrives, and the body is never compressed. + + The shared retry loop can send a body again only when its source is a seekable file-like object, in which case + `rewind` seeks back to where the source was when the body was created. Any other source is consumed by the attempt + that sends it, so the request gets a single attempt. + + A file opened in text mode, or an iterator yielding strings, is UTF-8 encoded chunk by chunk. A file-like object + whose `read` is a coroutine function, as `aiofiles` provides, and an async iterator can only be sent by the + asynchronous client. + """ + + def __init__(self, source: StreamedBodySource, *, chunk_size: int = STREAMED_BODY_CHUNK_SIZE) -> None: + """Initialize the streamed request body. + + Args: + source: The object the chunks come from. See `is_source` for the accepted kinds. + chunk_size: Size, in bytes, of the chunks a file-like source is read in. An iterator or a response + decides its own chunk sizes. + + Raises: + TypeError: If `source` is not an object the body can be streamed from. + """ + self._chunk_size = chunk_size + self._error: Exception | None = None + self._is_async = False + + # Exactly one of these produces the chunks: a file-like `read`, a factory of a synchronous iterable, or a + # factory of an asynchronous one. A response provides both factories. + self._read: Callable[[int], Any] | None = None + self._sync_chunks: Callable[[], Iterable[Any]] | None = None + self._async_chunks: Callable[[], AsyncIterator[Any]] | None = None + + # Set for a seekable file-like source, the only kind that can be sent more than once. + self._seek: Callable[[int], Any] | None = None + self._start: int | None = None + + read = getattr(source, 'read', None) + if _is_response(source): + self._sync_chunks = source.iter_bytes + self._async_chunks = getattr(source, 'aiter_bytes', None) + elif callable(read): + self._read = read + self._is_async = inspect.iscoroutinefunction(read) + # The `seekable` check guards the `tell` call, which a pipe or a socket rejects. An async file-like + # object also seeks asynchronously, so it is treated as a source that cannot be rewound. + seekable = getattr(source, 'seekable', None) + tell = getattr(source, 'tell', None) + seek = getattr(source, 'seek', None) + if not self._is_async and callable(seekable) and callable(tell) and callable(seek) and seekable(): + self._start = tell() + self._seek = seek + elif isinstance(source, Iterator): + self._sync_chunks = lambda: source + elif isinstance(source, AsyncIterator): + self._async_chunks = lambda: source + self._is_async = True + else: + raise TypeError( + f'Cannot stream a request body from a {type(source).__name__}. Pass a file-like object, an iterator ' + 'of byte chunks, or a streamed response.' + ) + + @staticmethod + def is_source(value: object) -> TypeGuard[StreamedBodySource]: + """Return whether a value is an object a request body can be streamed from. + + These are a streamed `HttpResponse` (anything with a callable `iter_bytes`), a file-like object (anything + with a callable `read`), and an iterator or async iterator of byte chunks. A `str`, `bytes`, `bytearray`, or + a container such as a `list` or `dict` is not a source, even though some of them can be iterated. + """ + return ( + _is_response(value) + or callable(getattr(value, 'read', None)) + or isinstance(value, (Iterator, AsyncIterator)) + ) + + @property + def is_async(self) -> bool: + """Whether the chunks can only be produced asynchronously, so only `HttpClientAsync` can send the body.""" + return self._is_async + + @property + def rewindable(self) -> bool: + """Whether the body can be sent again after `rewind`, which only a seekable file-like source allows.""" + return self._seek is not None + + @property + def error(self) -> Exception | None: + """The exception the source raised while the chunks were pulled, if any. + + A transport reports such a failure as its own error, which may wrap the cause beyond recognition. The retry + loop raises this exception instead, since sending the body again cannot fix its source. + """ + return self._error + + def rewind(self) -> None: + """Seek the source back to where it was when the body was created, so the body can be sent again. + + Raises: + RuntimeError: If the source cannot be rewound. Check `rewindable` first. + """ + if self._seek is None or self._start is None: + raise RuntimeError('The source of the streamed request body cannot be rewound.') + self._error = None + self._seek(self._start) + + def iter_bytes(self) -> Iterator[bytes]: + """Yield the body in chunks, reading a file-like source in `chunk_size` pieces. + + Raises: + TypeError: If the source can only produce its chunks asynchronously, see `is_async`. + """ + if self._is_async: + raise TypeError( + 'The request body is streamed from an asynchronous source, which only the asynchronous client can ' + 'send. Use `ApifyClientAsync`, or pass a synchronous file-like object or iterator.' + ) + return self._iter_chunks() + + def aiter_bytes(self) -> AsyncIterator[bytes]: + """Yield the body in chunks asynchronously, pulling a synchronous source in a worker thread. + + A blocking `read` or `__next__` would stall the event loop, so a synchronous file-like object or iterator is + pulled through `asyncio.to_thread`, one chunk at a time. + """ + return self._aiter_chunks() + + def _iter_chunks(self) -> Iterator[bytes]: + try: + if self._read is not None: + # A file-like source signals its end with an empty read. + while data := _to_bytes(self._read(self._chunk_size)): + yield data + elif self._sync_chunks is not None: + for chunk in self._sync_chunks(): + # In chunked transfer encoding an empty chunk terminates the body, so none is passed on. + if data := _to_bytes(chunk): + yield data + except Exception as exc: + self._error = exc + raise + + async def _aiter_chunks(self) -> AsyncIterator[bytes]: + try: + if self._read is not None: + while True: + chunk = ( + await self._read(self._chunk_size) + if self._is_async + else await asyncio.to_thread(self._read, self._chunk_size) + ) + data = _to_bytes(chunk) + if not data: + return + yield data + elif self._async_chunks is not None: + async for chunk in self._async_chunks(): + if data := _to_bytes(chunk): + yield data + elif self._sync_chunks is not None: + iterator = iter(self._sync_chunks()) + while (chunk := await asyncio.to_thread(_next_or_done, iterator)) is not _DONE: + if data := _to_bytes(chunk): + yield data + except Exception as exc: + self._error = exc + raise + + +def _next_or_done(iterator: Iterator[Any]) -> Any: + """Return the next item of a synchronous iterator, or `_DONE` once it is exhausted.""" + return next(iterator, _DONE) + + +def _is_response(value: object) -> TypeGuard[Any]: + """Return whether a value is a streamed response, recognized by a callable `iter_bytes`.""" + return callable(getattr(value, 'iter_bytes', None)) + + +def _to_bytes(chunk: object) -> bytes: + """Convert one chunk to bytes, UTF-8 encoding a string. + + Raises: + TypeError: If the chunk is neither bytes-like nor a string, for example `None` from a non-blocking stream + with no data available, or a coroutine from an async `read` called synchronously. + """ + if isinstance(chunk, bytes): + return chunk + if isinstance(chunk, (bytearray, memoryview)): + return bytes(chunk) + if isinstance(chunk, str): + return chunk.encode('utf-8') + raise TypeError(f'The streamed request body produced a {type(chunk).__name__} chunk, expected bytes or str.') diff --git a/src/apify_client/types.py b/src/apify_client/types.py index 47c5d7b6..c1b43406 100644 --- a/src/apify_client/types.py +++ b/src/apify_client/types.py @@ -1,7 +1,8 @@ from __future__ import annotations +from collections.abc import AsyncIterator, Iterator from datetime import timedelta -from typing import Literal +from typing import TYPE_CHECKING, Literal, Protocol from apify_client._models import WebhookCreate, WebhookRepresentation from apify_client._typeddicts import ( @@ -10,6 +11,10 @@ WebhookRepresentationCamelDict, WebhookRepresentationDict, ) +from apify_client.http_clients import HttpResponse + +if TYPE_CHECKING: + from collections.abc import Awaitable HttpCompressionAlgorithm = Literal['brotli', 'gzip'] """Accepted string literals for the `compression` parameter on `ApifyClient` and `ApifyClientAsync`.""" @@ -38,6 +43,27 @@ matching the Apify API spelling. """ + +class SupportsRead(Protocol): + """A file-like object a request body can be streamed from, for example an open file or an `io.BytesIO`. + + `read` is called with the chunk size until it returns an empty value. A file opened in text mode returns `str` + chunks, which are UTF-8 encoded. An `async def read`, as `aiofiles` provides, is accepted by `ApifyClientAsync`. + """ + + def read(self, size: int = ..., /) -> bytes | str | Awaitable[bytes | str]: + """Read up to `size` bytes or characters, returning an empty value at the end.""" + + +StreamedBodySource = SupportsRead | Iterator[bytes | str] | AsyncIterator[bytes | str] | HttpResponse +"""Type for a request body the client streams to the API in chunks instead of holding it in memory whole. + +A file-like object is read in chunks, an iterator or async iterator yields the chunks itself, and a streamed +`HttpResponse` forwards its body, which chains one API call's output into another's input. Accepted as the `data` of +`HttpClient.call`, as the `value` of `KeyValueStoreClient.set_record`, and as the `run_input` of Actor runs. See +`StreamedRequestBody` for the retry and compression rules that apply. +""" + JsonSerializable = dict[str, 'JsonSerializable'] | list['JsonSerializable'] | str | int | float | bool | None """Recursive type for JSON-serializable values - primitives plus objects and arrays with JSON-serializable contents. @@ -47,6 +73,8 @@ __all__ = [ 'HttpCompressionAlgorithm', 'JsonSerializable', + 'StreamedBodySource', + 'SupportsRead', 'Timeout', 'WebhooksList', ] diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index 5d1d9238..ca240999 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -3,6 +3,7 @@ from __future__ import annotations import gzip +import io import json from collections.abc import AsyncIterator, Iterator from datetime import timedelta @@ -777,3 +778,75 @@ async def get_keys() -> ListOfKeys: assert first_keys.isdisjoint(second_keys) finally: await maybe_await(store_client.delete()) + + +async def test_key_value_store_set_streamed_record(client: ApifyClient | ApifyClientAsync) -> None: + """A file-like value is streamed to the API in chunks and stored whole.""" + store_name = get_random_resource_name('kvs') + created_store = await maybe_await(client.key_value_stores().get_or_create(name=store_name)) + assert isinstance(created_store, KeyValueStore) + store_client = client.key_value_store(created_store.id) + + try: + # Several chunks' worth of non-repeating bytes, so a dropped or reordered chunk would show in the comparison. + data = bytes(range(256)) * (3 * 1024 * 4) + await maybe_await( + store_client.set_record('stream.bin', io.BytesIO(data), content_type='application/octet-stream') + ) + + # Poll until the record is visible (eventual consistency) + async def get_record() -> dict | None: + return await maybe_await(store_client.get_record_as_bytes('stream.bin')) + + record = await poll_until_condition(get_record, lambda record: record is not None) + assert isinstance(record, dict) + assert record['value'] == data + assert record['content_type'] == 'application/octet-stream' + finally: + await maybe_await(store_client.delete()) + + +async def test_key_value_store_pipe_record_between_stores( + client: ApifyClient | ApifyClientAsync, + *, + is_async: bool, +) -> None: + """A record streamed from one store is uploaded to another as it downloads, keeping its content type.""" + source_store = await maybe_await(client.key_value_stores().get_or_create(name=get_random_resource_name('kvs'))) + target_store = await maybe_await(client.key_value_stores().get_or_create(name=get_random_resource_name('kvs'))) + assert isinstance(source_store, KeyValueStore) + assert isinstance(target_store, KeyValueStore) + source_client = client.key_value_store(source_store.id) + target_client = client.key_value_store(target_store.id) + + try: + data = ('id,value\n' + '\n'.join(f'{i},{i * i}' for i in range(50_000))).encode('utf-8') + await maybe_await(source_client.set_record('items.csv', data, content_type='text/csv')) + + async def get_source_record() -> dict | None: + return await maybe_await(source_client.get_record_as_bytes('items.csv')) + + assert await poll_until_condition(get_source_record, lambda record: record is not None) is not None + + # `stream_record` is a context manager, so the sync and async clients cannot share one code path here. + if is_async: + async with source_client.stream_record('items.csv') as record: # ty: ignore[invalid-context-manager] + assert record is not None + await maybe_await( + target_client.set_record('items.csv', record['value'], content_type=record['content_type']) + ) + else: + with source_client.stream_record('items.csv') as record: # ty: ignore[invalid-context-manager] + assert record is not None + target_client.set_record('items.csv', record['value'], content_type=record['content_type']) + + async def get_target_record() -> dict | None: + return await maybe_await(target_client.get_record_as_bytes('items.csv')) + + copied = await poll_until_condition(get_target_record, lambda record: record is not None) + assert isinstance(copied, dict) + assert copied['value'] == data + assert copied['content_type'].startswith('text/csv') + finally: + await maybe_await(source_client.delete()) + await maybe_await(target_client.delete()) diff --git a/tests/unit/test_actor_start_params.py b/tests/unit/test_actor_start_params.py index 20a4ccd6..f35ca19e 100644 --- a/tests/unit/test_actor_start_params.py +++ b/tests/unit/test_actor_start_params.py @@ -1,5 +1,6 @@ from __future__ import annotations +import io import json from datetime import timedelta from typing import TYPE_CHECKING @@ -8,6 +9,7 @@ from werkzeug import Request, Response from apify_client import ApifyClient, ApifyClientAsync +from apify_client._consts import MIN_COMPRESSION_SIZE if TYPE_CHECKING: from pytest_httpserver import HTTPServer @@ -231,3 +233,51 @@ def capture_request(request: Request) -> Response: assert len(captured_requests) == 1 assert captured_requests[0].args['timeout'] == str(timeout_value) + + +# Above the compression threshold, so an uncompressed upload proves the input was streamed rather than buffered. +_STREAMED_RUN_INPUT = json.dumps({'text': 'x' * MIN_COMPRESSION_SIZE}).encode('utf-8') + + +def test_actor_start_streams_file_like_input_sync(httpserver: HTTPServer) -> None: + """A file-like run input is uploaded in chunks as it is read, uncompressed, under the given content type.""" + captured_requests: list[Request] = [] + + def capture_request(request: Request) -> Response: + captured_requests.append(request) + return Response(response=json.dumps(_create_minimal_run_response()), status=200, mimetype='application/json') + + httpserver.expect_request(f'/v2/actors/{_MOCKED_ACTOR_ID}/runs', method='POST').respond_with_handler( + capture_request + ) + client = ApifyClient(token='test_token', api_url=httpserver.url_for('/').removesuffix('/')) + + client.actor(_MOCKED_ACTOR_ID).start(run_input=io.BytesIO(_STREAMED_RUN_INPUT), content_type='application/json') + + assert len(captured_requests) == 1 + assert captured_requests[0].headers['content-type'] == 'application/json' + assert 'content-encoding' not in captured_requests[0].headers + assert captured_requests[0].get_data() == _STREAMED_RUN_INPUT + + +async def test_actor_start_streams_file_like_input_async(httpserver: HTTPServer) -> None: + """A file-like run input is uploaded in chunks as it is read, uncompressed, under the given content type.""" + captured_requests: list[Request] = [] + + def capture_request(request: Request) -> Response: + captured_requests.append(request) + return Response(response=json.dumps(_create_minimal_run_response()), status=200, mimetype='application/json') + + httpserver.expect_request(f'/v2/actors/{_MOCKED_ACTOR_ID}/runs', method='POST').respond_with_handler( + capture_request + ) + client = ApifyClientAsync(token='test_token', api_url=httpserver.url_for('/').removesuffix('/')) + + await client.actor(_MOCKED_ACTOR_ID).start( + run_input=io.BytesIO(_STREAMED_RUN_INPUT), content_type='application/json' + ) + + assert len(captured_requests) == 1 + assert captured_requests[0].headers['content-type'] == 'application/json' + assert 'content-encoding' not in captured_requests[0].headers + assert captured_requests[0].get_data() == _STREAMED_RUN_INPUT diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index 3e441189..4f03e961 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -5,9 +5,10 @@ import json import threading import time +from collections.abc import AsyncIterator, Iterator from datetime import UTC, datetime, timedelta from io import BytesIO -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING from unittest.mock import AsyncMock, Mock import brotli @@ -26,6 +27,7 @@ Httpx2HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, + StreamedRequestBody, ) from apify_client.http_compressors._base import HttpCompressor from apify_client.http_compressors._brotli import BrotliHttpCompressor @@ -851,20 +853,42 @@ def test_prepare_request_call_skips_compression_for_already_compressed_content(c assert headers['User-Agent'] == client._headers['User-Agent'] -def test_prepare_request_call_keeps_caller_content_encoding_for_a_file_like_body() -> None: - """A file-like body skips compression entirely, and its `Content-Encoding` reaches the transport untouched.""" +def test_prepare_request_call_keeps_caller_content_encoding_for_a_streamed_body() -> None: + """A streamed body skips compression entirely, and its `Content-Encoding` reaches the transport untouched.""" client = ConcreteHttpClient(http_compressor=GzipHttpCompressor()) - stream = BytesIO(gzip.compress(b'raw payload')) + compressed = gzip.compress(b'raw payload') headers, _params, data = client._prepare_request_call( headers={'content-encoding': 'gzip'}, - data=cast('bytes', stream), + data=BytesIO(compressed), ) - assert data is stream + assert isinstance(data, StreamedRequestBody) + assert b''.join(data.iter_bytes()) == compressed assert headers['content-encoding'] == 'gzip' +@pytest.mark.parametrize( + 'make_data', + [ + pytest.param(lambda: BytesIO(b'x' * MIN_COMPRESSION_SIZE * 4), id='file-like'), + pytest.param(lambda: iter([b'x' * MIN_COMPRESSION_SIZE * 4]), id='iterator'), + ], +) +def test_prepare_request_call_streams_body_without_compression( + compressor_case: tuple, make_data: Callable[[], Any] +) -> None: + """A streamed body above the size threshold is wrapped for streaming and never compressed.""" + compressor, _content_encoding, _decompress = compressor_case + client = ConcreteHttpClient(http_compressor=compressor) + + headers, _params, data = client._prepare_request_call(data=make_data()) + + assert isinstance(data, StreamedRequestBody) + assert b''.join(data.iter_bytes()) == b'x' * MIN_COMPRESSION_SIZE * 4 + assert not any(key.lower() == 'content-encoding' for key in headers) + + @pytest.mark.parametrize( 'content_type', [ @@ -1141,3 +1165,187 @@ async def test_async_call_skips_thread_offload_for_a_body_it_cannot_compress( await client.call(method='PUT', url='https://api.test.com/endpoint', data=body) spy.assert_not_called() + + +class StreamingTransport(HttpClient): + """A transport that drains every body it is handed and answers from a script of statuses and exceptions.""" + + def __init__(self, outcomes: list[int | Exception]) -> None: + super().__init__(min_delay_between_retries=timedelta(milliseconds=1)) + self._outcomes = outcomes + self.bodies: list[bytes | None] = [] + self.attempts_with_iterator: list[bool] = [] + + def is_retryable_transport_error(self, exc: Exception) -> bool: + return isinstance(exc, ConnectionError) + + def send_request( + self, + *, + method: str, + url: str, + headers: dict[str, str], + content: bytes | Iterator[bytes] | None, + timeout: float | None, + stream: bool, + ) -> HttpResponse: + _ = method, url, headers, timeout, stream + self.attempts_with_iterator.append(not isinstance(content, (bytes, type(None)))) + self.bodies.append(b''.join(content) if isinstance(content, Iterator) else content) + outcome = self._outcomes.pop(0) + if isinstance(outcome, Exception): + raise outcome + return Mock(status_code=outcome) + + +class StreamingTransportAsync(HttpClientAsync): + """Asynchronous counterpart of `StreamingTransport`.""" + + def __init__(self, outcomes: list[int | Exception]) -> None: + super().__init__(min_delay_between_retries=timedelta(milliseconds=1)) + self._outcomes = outcomes + self.bodies: list[bytes | None] = [] + self.attempts_with_iterator: list[bool] = [] + + def is_retryable_transport_error(self, exc: Exception) -> bool: + return isinstance(exc, ConnectionError) + + async def send_request( + self, + *, + method: str, + url: str, + headers: dict[str, str], + content: bytes | AsyncIterator[bytes] | None, + timeout: float | None, + stream: bool, + ) -> HttpResponse: + _ = method, url, headers, timeout, stream + self.attempts_with_iterator.append(not isinstance(content, (bytes, type(None)))) + self.bodies.append( + b''.join([chunk async for chunk in content]) if isinstance(content, AsyncIterator) else content + ) + outcome = self._outcomes.pop(0) + if isinstance(outcome, Exception): + raise outcome + return Mock(status_code=outcome) + + +class WrappingTransport(StreamingTransport): + """A transport that, like Impit, reports a failure while pulling chunks as its own transient error.""" + + def send_request(self, **kwargs: Any) -> HttpResponse: + try: + return super().send_request(**kwargs) + except OSError as exc: + raise ConnectionError('the internal HTTP library has thrown an error') from exc + + +class WrappingTransportAsync(StreamingTransportAsync): + """Asynchronous counterpart of `WrappingTransport`.""" + + async def send_request(self, **kwargs: Any) -> HttpResponse: + try: + return await super().send_request(**kwargs) + except OSError as exc: + raise ConnectionError('the internal HTTP library has thrown an error') from exc + + +def failing_chunks() -> Iterator[bytes]: + yield b'first chunk' + raise OSError('disk on fire') + + +def test_send_request_receives_a_streamed_body_as_an_iterator_of_chunks() -> None: + """The transport gets the chunks, not the source object, so any library that streams iterables can send them.""" + transport = StreamingTransport([200]) + + transport.call(method='PUT', url='https://api.test.com/endpoint', data=BytesIO(b'streamed')) + + assert transport.attempts_with_iterator == [True] + assert transport.bodies == [b'streamed'] + + +async def test_send_request_receives_a_streamed_body_as_an_async_iterator_of_chunks() -> None: + """The asynchronous transport gets an async iterator, which a synchronous source is adapted to.""" + transport = StreamingTransportAsync([200]) + + await transport.call(method='PUT', url='https://api.test.com/endpoint', data=BytesIO(b'streamed')) + + assert transport.attempts_with_iterator == [True] + assert transport.bodies == [b'streamed'] + + +def test_rewindable_streamed_body_is_rewound_between_attempts() -> None: + """A seekable source is sent again from its starting position after a retryable failure.""" + transport = StreamingTransport([ConnectionError('reset'), 200]) + buffer = BytesIO(b'skip-payload') + buffer.read(5) + + transport.call(method='PUT', url='https://api.test.com/endpoint', data=buffer) + + assert transport.bodies == [b'payload', b'payload'] + + +async def test_rewindable_streamed_body_is_rewound_between_attempts_async() -> None: + """A seekable source is sent again from its starting position after a retryable failure.""" + transport = StreamingTransportAsync([ConnectionError('reset'), 200]) + buffer = BytesIO(b'skip-payload') + buffer.read(5) + + await transport.call(method='PUT', url='https://api.test.com/endpoint', data=buffer) + + assert transport.bodies == [b'payload', b'payload'] + + +def test_non_rewindable_streamed_body_gets_a_single_attempt() -> None: + """An iterator is consumed by the attempt that sends it, so even a retryable failure is final.""" + transport = StreamingTransport([ConnectionError('reset'), 200]) + + with pytest.raises(ConnectionError, match='reset'): + transport.call(method='PUT', url='https://api.test.com/endpoint', data=iter([b'payload'])) + + assert transport.bodies == [b'payload'] + + +async def test_non_rewindable_streamed_body_gets_a_single_attempt_async() -> None: + """An iterator is consumed by the attempt that sends it, so even a retryable failure is final.""" + transport = StreamingTransportAsync([ConnectionError('reset'), 200]) + + with pytest.raises(ConnectionError, match='reset'): + await transport.call(method='PUT', url='https://api.test.com/endpoint', data=iter([b'payload'])) + + assert transport.bodies == [b'payload'] + + +def test_streamed_body_source_error_replaces_the_wrapped_transport_error() -> None: + """A source failure is raised as itself, with the transport's report as its cause, and is never retried.""" + transport = WrappingTransport([200, 200]) + + with pytest.raises(OSError, match='disk on fire') as exc_info: + transport.call(method='PUT', url='https://api.test.com/endpoint', data=failing_chunks()) + + assert isinstance(exc_info.value.__cause__, ConnectionError) + assert len(transport.attempts_with_iterator) == 1 + + +async def test_streamed_body_source_error_replaces_the_wrapped_transport_error_async() -> None: + """A source failure is raised as itself, with the transport's report as its cause, and is never retried.""" + transport = WrappingTransportAsync([200, 200]) + + with pytest.raises(OSError, match='disk on fire') as exc_info: + await transport.call(method='PUT', url='https://api.test.com/endpoint', data=failing_chunks()) + + assert isinstance(exc_info.value.__cause__, ConnectionError) + assert len(transport.attempts_with_iterator) == 1 + + +def test_streamed_body_source_error_stops_retrying_when_the_transport_propagates_it() -> None: + """A transport that lets the source error through, like HTTPX2, ends up with the same single attempt.""" + transport = StreamingTransport([200, 200]) + + with pytest.raises(OSError, match='disk on fire') as exc_info: + transport.call(method='PUT', url='https://api.test.com/endpoint', data=failing_chunks()) + + assert exc_info.value.__cause__ is None + assert len(transport.attempts_with_iterator) == 1 diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index cedafaba..b55dc265 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -3,17 +3,20 @@ import gzip import io import zlib +from datetime import timedelta from typing import TYPE_CHECKING, Any +from unittest.mock import Mock import brotli import pytest from werkzeug import Request, Response from apify_client import ApifyClient, ApifyClientAsync -from apify_client._consts import MIN_COMPRESSION_SIZE +from apify_client._consts import MIN_COMPRESSION_SIZE, STREAMED_BODY_CHUNK_SIZE +from apify_client.errors import ApifyApiError if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import AsyncIterator, Callable, Iterator from pytest_httpserver import HTTPServer @@ -33,16 +36,73 @@ class DuckTypedReader: """A file-like object that is not an `io.IOBase`, so only duck-typed detection picks it up.""" - def read(self) -> bytes: - return _BYTES_VALUE + def __init__(self, data: bytes = _BYTES_VALUE) -> None: + self._buffer = io.BytesIO(data) + def read(self, size: int = -1) -> bytes: + return self._buffer.read(size) -# The values are built by a factory because reading consumes them, and each case runs once per compression -# algorithm. Each case is (value factory, expected uploaded body, expected content type). -_FILE_LIKE_VALUE_CASES = [ - pytest.param(lambda: io.BytesIO(_BYTES_VALUE), _BYTES_VALUE, 'application/octet-stream', id='bytes io'), - pytest.param(lambda: io.StringIO(_TEXT_VALUE), _BYTES_VALUE, 'text/plain; charset=utf-8', id='string io'), - pytest.param(DuckTypedReader, _BYTES_VALUE, 'application/octet-stream', id='duck-typed reader'), + +class AsyncDuckTypedReader: + """A file-like object with a coroutine `read`, as `aiofiles` provides.""" + + def __init__(self, data: bytes = _BYTES_VALUE) -> None: + self._buffer = io.BytesIO(data) + + async def read(self, size: int = -1) -> bytes: + return self._buffer.read(size) + + +class RecordingReader: + """A file-like object that records the size of every read it serves.""" + + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + self.read_sizes: list[int] = [] + + def read(self, size: int = -1) -> bytes: + self.read_sizes.append(size) + return self._buffer.read(size) + + +class FailingReader: + """A file-like object whose second read fails, like a disk error halfway through a file.""" + + def __init__(self) -> None: + self._reads = 0 + + def read(self, size: int = -1) -> bytes: + _ = size + self._reads += 1 + if self._reads > 1: + raise OSError('disk on fire') + return b'first chunk' + + +def bytes_chunks() -> Iterator[bytes]: + yield _BYTES_VALUE[:100] + yield _BYTES_VALUE[100:] + + +async def async_bytes_chunks() -> AsyncIterator[bytes]: + yield _BYTES_VALUE[:100] + yield _BYTES_VALUE[100:] + + +# The values are built by a factory because streaming consumes them, and each case runs once per compression +# algorithm and transport. Each case is (value factory, expected content type); every value streams `_BYTES_VALUE`. +_STREAMED_VALUE_CASES = [ + pytest.param(lambda: io.BytesIO(_BYTES_VALUE), 'application/octet-stream', id='binary file-like'), + pytest.param(lambda: io.StringIO(_TEXT_VALUE), 'text/plain; charset=utf-8', id='text-mode file-like'), + pytest.param(DuckTypedReader, 'application/octet-stream', id='duck-typed reader'), + pytest.param(bytes_chunks, 'application/octet-stream', id='bytes iterator'), + pytest.param(lambda: iter([_TEXT_VALUE[:100], _TEXT_VALUE[100:]]), 'application/octet-stream', id='str iterator'), +] + +# Sources only the asynchronous client can consume. +_ASYNC_STREAMED_VALUE_CASES = [ + pytest.param(async_bytes_chunks, id='async bytes iterator'), + pytest.param(AsyncDuckTypedReader, id='async file-like'), ] # Each case is (content encoding passed to `set_record`, the body the caller hands over already encoded that way). @@ -103,50 +163,6 @@ def decode_body(request: Request) -> bytes: return raw -@pytest.mark.parametrize(('make_value', 'expected_body', 'expected_content_type'), _FILE_LIKE_VALUE_CASES) -def test_set_record_reads_file_like_value_sync( - *, - api_url: str, - captured_records: list[Request], - compression_case: tuple[HttpCompressionAlgorithm, str], - make_value: Callable[[], Any], - expected_body: bytes, - expected_content_type: str, -) -> None: - """A file-like value is read and its bytes are uploaded, not passed through unread.""" - algorithm, content_encoding = compression_case - client = ApifyClient(token='test_token', api_url=api_url, compression=algorithm) - - client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value()) - - assert len(captured_records) == 1 - assert captured_records[0].headers['content-encoding'] == content_encoding - assert decode_body(captured_records[0]) == expected_body - assert captured_records[0].headers['content-type'] == expected_content_type - - -@pytest.mark.parametrize(('make_value', 'expected_body', 'expected_content_type'), _FILE_LIKE_VALUE_CASES) -async def test_set_record_reads_file_like_value_async( - *, - api_url: str, - captured_records: list[Request], - compression_case: tuple[HttpCompressionAlgorithm, str], - make_value: Callable[[], Any], - expected_body: bytes, - expected_content_type: str, -) -> None: - """A file-like value is read and its bytes are uploaded, not passed through unread.""" - algorithm, content_encoding = compression_case - client = ApifyClientAsync(token='test_token', api_url=api_url, compression=algorithm) - - await client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value()) - - assert len(captured_records) == 1 - assert captured_records[0].headers['content-encoding'] == content_encoding - assert decode_body(captured_records[0]) == expected_body - assert captured_records[0].headers['content-type'] == expected_content_type - - @pytest.mark.parametrize(('content_encoding', 'value'), _PRE_ENCODED_VALUE_CASES) def test_set_record_uploads_pre_encoded_value_sync( *, @@ -227,3 +243,258 @@ async def test_set_record_rejects_declared_compression_of_non_bytes_value_async( await client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value(), content_encoding='gzip') assert captured_records == [] + + +def assert_streamed_upload(request: Request, expected_content_type: str, body: bytes = _BYTES_VALUE) -> None: + """Check that a captured upload arrived chunked, uncompressed, and complete.""" + assert request.headers.get('transfer-encoding') == 'chunked' + assert 'content-encoding' not in request.headers + assert request.headers['content-type'] == expected_content_type + assert request.get_data() == body + + +@pytest.fixture +def flaky_records(httpserver: HTTPServer) -> list[Request]: + """Fail the first record upload with a 500 and accept the rest, collecting the requests the client sent.""" + requests: list[Request] = [] + + def handle_request(request: Request) -> Response: + requests.append(request) + return Response(status=500 if len(requests) == 1 else 201) + + httpserver.expect_request(_RECORD_PATH, method='PUT').respond_with_handler(handle_request) + return requests + + +@pytest.mark.parametrize(('make_value', 'expected_content_type'), _STREAMED_VALUE_CASES) +def test_set_record_streams_value_sync( + *, + api_url: str, + captured_records: list[Request], + compression_case: tuple[HttpCompressionAlgorithm, str], + make_value: Callable[[], Any], + expected_content_type: str, +) -> None: + """A streamed value is uploaded in chunks as it is read, and never compressed, whichever compressor is set.""" + algorithm, _content_encoding = compression_case + client = ApifyClient(token='test_token', api_url=api_url, compression=algorithm) + + client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value()) + + assert len(captured_records) == 1 + assert_streamed_upload(captured_records[0], expected_content_type) + + +@pytest.mark.parametrize(('make_value', 'expected_content_type'), _STREAMED_VALUE_CASES) +async def test_set_record_streams_value_async( + *, + api_url: str, + captured_records: list[Request], + compression_case: tuple[HttpCompressionAlgorithm, str], + make_value: Callable[[], Any], + expected_content_type: str, +) -> None: + """A streamed value is uploaded in chunks as it is read, and never compressed, whichever compressor is set.""" + algorithm, _content_encoding = compression_case + client = ApifyClientAsync(token='test_token', api_url=api_url, compression=algorithm) + + await client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value()) + + assert len(captured_records) == 1 + assert_streamed_upload(captured_records[0], expected_content_type) + + +@pytest.mark.parametrize('make_value', _ASYNC_STREAMED_VALUE_CASES) +async def test_set_record_streams_async_value_async( + *, + api_url: str, + captured_records: list[Request], + make_value: Callable[[], Any], +) -> None: + """The asynchronous client streams an async iterator or an async file-like value as it produces chunks.""" + client = ApifyClientAsync(token='test_token', api_url=api_url) + + await client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value()) + + assert len(captured_records) == 1 + assert_streamed_upload(captured_records[0], 'application/octet-stream') + + +@pytest.mark.parametrize('make_value', _ASYNC_STREAMED_VALUE_CASES) +def test_set_record_rejects_async_value_sync( + *, + api_url: str, + captured_records: list[Request], + make_value: Callable[[], Any], +) -> None: + """The synchronous client cannot consume an asynchronous source, and the error names the client that can.""" + client = ApifyClient(token='test_token', api_url=api_url) + + with pytest.raises(TypeError, match='Use `ApifyClientAsync`'): + client.key_value_store(_MOCKED_KVS_ID).set_record('f', make_value()) + + assert captured_records == [] + + +def test_set_record_reads_file_like_value_in_chunks_sync(*, api_url: str, captured_records: list[Request]) -> None: + """A file-like value is read in `STREAMED_BODY_CHUNK_SIZE` pieces, never whole.""" + data = b'x' * (STREAMED_BODY_CHUNK_SIZE * 2 + 1) + reader = RecordingReader(data) + client = ApifyClient(token='test_token', api_url=api_url) + + client.key_value_store(_MOCKED_KVS_ID).set_record('f', reader) + + # Three reads deliver the data, and a fourth, empty one ends the body. + assert reader.read_sizes == [STREAMED_BODY_CHUNK_SIZE] * 4 + assert captured_records[0].get_data() == data + + +async def test_set_record_reads_file_like_value_in_chunks_async( + *, api_url: str, captured_records: list[Request] +) -> None: + """A file-like value is read in `STREAMED_BODY_CHUNK_SIZE` pieces, never whole.""" + data = b'x' * (STREAMED_BODY_CHUNK_SIZE * 2 + 1) + reader = RecordingReader(data) + client = ApifyClientAsync(token='test_token', api_url=api_url) + + await client.key_value_store(_MOCKED_KVS_ID).set_record('f', reader) + + assert reader.read_sizes == [STREAMED_BODY_CHUNK_SIZE] * 4 + assert captured_records[0].get_data() == data + + +_SOURCE_RECORD_PATH = '/v2/key-value-stores/source_kvs_id/records/f' + + +def test_set_record_pipes_streamed_record_sync( + *, httpserver: HTTPServer, api_url: str, captured_records: list[Request] +) -> None: + """A record streamed from one store is uploaded to another as it downloads, under its own content type.""" + httpserver.expect_request(_SOURCE_RECORD_PATH, method='GET').respond_with_data( + _BYTES_VALUE, content_type='text/csv' + ) + client = ApifyClient(token='test_token', api_url=api_url) + + with client.key_value_store('source_kvs_id').stream_record('f') as record: + assert record is not None + client.key_value_store(_MOCKED_KVS_ID).set_record('f', record['value'], content_type=record['content_type']) + + assert len(captured_records) == 1 + assert_streamed_upload(captured_records[0], record['content_type']) + + +async def test_set_record_pipes_streamed_record_async( + *, httpserver: HTTPServer, api_url: str, captured_records: list[Request] +) -> None: + """A record streamed from one store is uploaded to another as it downloads, under its own content type.""" + httpserver.expect_request(_SOURCE_RECORD_PATH, method='GET').respond_with_data( + _BYTES_VALUE, content_type='text/csv' + ) + client = ApifyClientAsync(token='test_token', api_url=api_url) + + async with client.key_value_store('source_kvs_id').stream_record('f') as record: + assert record is not None + await client.key_value_store(_MOCKED_KVS_ID).set_record( + 'f', record['value'], content_type=record['content_type'] + ) + + assert len(captured_records) == 1 + assert_streamed_upload(captured_records[0], record['content_type']) + + +def test_set_record_retries_seekable_file_like_value_sync(*, api_url: str, flaky_records: list[Request]) -> None: + """A seekable file-like value is rewound to where it started, so the retry uploads the same bytes again.""" + buffer = io.BytesIO(b'skip' + _BYTES_VALUE) + buffer.read(4) + client = ApifyClient(token='test_token', api_url=api_url, min_delay_between_retries=timedelta(milliseconds=1)) + + client.key_value_store(_MOCKED_KVS_ID).set_record('f', buffer) + + assert [request.get_data() for request in flaky_records] == [_BYTES_VALUE, _BYTES_VALUE] + + +async def test_set_record_retries_seekable_file_like_value_async(*, api_url: str, flaky_records: list[Request]) -> None: + """A seekable file-like value is rewound to where it started, so the retry uploads the same bytes again.""" + buffer = io.BytesIO(b'skip' + _BYTES_VALUE) + buffer.read(4) + client = ApifyClientAsync(token='test_token', api_url=api_url, min_delay_between_retries=timedelta(milliseconds=1)) + + await client.key_value_store(_MOCKED_KVS_ID).set_record('f', buffer) + + assert [request.get_data() for request in flaky_records] == [_BYTES_VALUE, _BYTES_VALUE] + + +def test_set_record_does_not_retry_non_rewindable_value_sync(*, api_url: str, flaky_records: list[Request]) -> None: + """An iterator is consumed by the attempt that sends it, so a failed upload is not retried.""" + client = ApifyClient(token='test_token', api_url=api_url, min_delay_between_retries=timedelta(milliseconds=1)) + + with pytest.raises(ApifyApiError): + client.key_value_store(_MOCKED_KVS_ID).set_record('f', bytes_chunks()) + + assert len(flaky_records) == 1 + + +async def test_set_record_does_not_retry_non_rewindable_value_async( + *, api_url: str, flaky_records: list[Request] +) -> None: + """An iterator is consumed by the attempt that sends it, so a failed upload is not retried.""" + client = ApifyClientAsync(token='test_token', api_url=api_url, min_delay_between_retries=timedelta(milliseconds=1)) + + with pytest.raises(ApifyApiError): + await client.key_value_store(_MOCKED_KVS_ID).set_record('f', bytes_chunks()) + + assert len(flaky_records) == 1 + + +@pytest.mark.usefixtures('captured_records') +def test_set_record_raises_source_error_sync(*, api_url: str, monkeypatch: pytest.MonkeyPatch) -> None: + """A value that fails mid-upload surfaces its own error after a single attempt, not a retried transport error.""" + client = ApifyClient(token='test_token', api_url=api_url, min_delay_between_retries=timedelta(milliseconds=1)) + send_request = Mock(wraps=client.http_client.send_request) + monkeypatch.setattr(client.http_client, 'send_request', send_request) + + with pytest.raises(OSError, match='disk on fire'): + client.key_value_store(_MOCKED_KVS_ID).set_record('f', FailingReader()) + + send_request.assert_called_once() + + +@pytest.mark.usefixtures('captured_records') +async def test_set_record_raises_source_error_async(*, api_url: str, monkeypatch: pytest.MonkeyPatch) -> None: + """A value that fails mid-upload surfaces its own error after a single attempt, not a retried transport error.""" + client = ApifyClientAsync(token='test_token', api_url=api_url, min_delay_between_retries=timedelta(milliseconds=1)) + send_request = Mock(wraps=client.http_client.send_request) + monkeypatch.setattr(client.http_client, 'send_request', send_request) + + with pytest.raises(OSError, match='disk on fire'): + await client.key_value_store(_MOCKED_KVS_ID).set_record('f', FailingReader()) + + send_request.assert_called_once() + + +def test_set_record_streams_pre_encoded_file_like_value_sync(*, api_url: str, captured_records: list[Request]) -> None: + """A pre-compressed file streams under the caller's `Content-Encoding`, with nothing compressed twice.""" + compressed = gzip.compress(_BYTES_VALUE) + client = ApifyClient(token='test_token', api_url=api_url) + + client.key_value_store(_MOCKED_KVS_ID).set_record( + 'f', io.BytesIO(compressed), content_type='text/plain', content_encoding='gzip' + ) + + assert captured_records[0].headers['content-encoding'] == 'gzip' + assert captured_records[0].get_data() == compressed + + +async def test_set_record_streams_pre_encoded_file_like_value_async( + *, api_url: str, captured_records: list[Request] +) -> None: + """A pre-compressed file streams under the caller's `Content-Encoding`, with nothing compressed twice.""" + compressed = gzip.compress(_BYTES_VALUE) + client = ApifyClientAsync(token='test_token', api_url=api_url) + + await client.key_value_store(_MOCKED_KVS_ID).set_record( + 'f', io.BytesIO(compressed), content_type='text/plain', content_encoding='gzip' + ) + + assert captured_records[0].headers['content-encoding'] == 'gzip' + assert captured_records[0].get_data() == compressed diff --git a/tests/unit/test_pluggable_http_client.py b/tests/unit/test_pluggable_http_client.py index 563f65b8..edbc6c0a 100644 --- a/tests/unit/test_pluggable_http_client.py +++ b/tests/unit/test_pluggable_http_client.py @@ -4,6 +4,7 @@ import json as jsonlib import subprocess import sys +from collections.abc import AsyncIterator from dataclasses import dataclass, field from datetime import timedelta from http.client import HTTPConnection @@ -26,14 +27,15 @@ HttpResponse, ImpitHttpClient, ImpitHttpClientAsync, + StreamedRequestBody, ) if TYPE_CHECKING: - from collections.abc import AsyncIterator, Iterator + from collections.abc import Iterator from pytest_httpserver import HTTPServer - from apify_client.types import Timeout + from apify_client.types import StreamedBodySource, Timeout @dataclass @@ -93,7 +95,7 @@ def call( url: str, headers: dict[str, str] | None = None, params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, + data: str | bytes | bytearray | StreamedBodySource | None = None, json: Any = None, stream: bool | None = None, timeout: Timeout = 'medium', @@ -127,7 +129,7 @@ async def call( url: str, headers: dict[str, str] | None = None, params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, + data: str | bytes | bytearray | StreamedBodySource | None = None, json: Any = None, stream: bool | None = None, timeout: Timeout = 'medium', @@ -152,7 +154,7 @@ def _stdlib_fetch( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | Iterator[bytes] | None, timeout: float | None, ) -> FakeResponse: """Send one request over `http.client` and adapt the result to the `HttpResponse` protocol.""" @@ -187,7 +189,7 @@ def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | Iterator[bytes] | None, timeout: float | None, stream: bool, ) -> HttpResponse: @@ -204,11 +206,13 @@ async def send_request( method: str, url: str, headers: dict[str, str], - content: bytes | None, + content: bytes | AsyncIterator[bytes] | None, timeout: float | None, stream: bool, ) -> HttpResponse: _ = stream + if isinstance(content, AsyncIterator): + content = b''.join([chunk async for chunk in content]) return await asyncio.to_thread( _stdlib_fetch, method=method, url=url, headers=headers, content=content, timeout=timeout ) @@ -547,12 +551,14 @@ def call( url: str, headers: dict[str, str] | None = None, params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, + data: str | bytes | bytearray | StreamedBodySource | None = None, json: Any = None, **_kwargs: Any, ) -> HttpResponse: headers, params, content = self._prepare_request_call(headers=headers, params=params, data=data, json=json) url = self._build_url_with_params(url, params=params) + if isinstance(content, StreamedRequestBody): + content = content.iter_bytes() return self._impit_client.request(method=method, url=url, headers=headers, content=content) @@ -570,12 +576,14 @@ async def call( url: str, headers: dict[str, str] | None = None, params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, + data: str | bytes | bytearray | StreamedBodySource | None = None, json: Any = None, **_kwargs: Any, ) -> HttpResponse: headers, params, content = self._prepare_request_call(headers=headers, params=params, data=data, json=json) url = self._build_url_with_params(url, params=params) + if isinstance(content, StreamedRequestBody): + content = content.aiter_bytes() return await self._impit_client.request(method=method, url=url, headers=headers, content=content) diff --git a/tests/unit/test_streamed_request_body.py b/tests/unit/test_streamed_request_body.py new file mode 100644 index 00000000..d752af23 --- /dev/null +++ b/tests/unit/test_streamed_request_body.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +import io +from typing import TYPE_CHECKING, Any, cast + +import pytest + +from apify_client._consts import STREAMED_BODY_CHUNK_SIZE +from apify_client.http_clients import StreamedRequestBody + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterator + + +class Reader: + """A duck-typed file-like object: a callable `read`, no `io.IOBase` ancestry, no seeking.""" + + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + + def read(self, size: int = -1) -> bytes: + return self._buffer.read(size) + + +class AsyncReader: + """A file-like object with a coroutine `read`, as `aiofiles` provides.""" + + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + + async def read(self, size: int = -1) -> bytes: + return self._buffer.read(size) + + +class FakeStreamedResponse: + """The streaming half of the `HttpResponse` protocol, with both a sync and an async chunk iterator.""" + + def __init__(self, chunks: list[bytes]) -> None: + self.chunks = chunks + + def read(self) -> bytes: + return b''.join(self.chunks) + + def iter_bytes(self) -> Iterator[bytes]: + yield from self.chunks + + async def aiter_bytes(self) -> AsyncIterator[bytes]: + for chunk in self.chunks: + yield chunk + + +class SyncOnlyStreamedResponse: + """A response-like object with only a synchronous chunk iterator.""" + + def __init__(self, chunks: list[bytes]) -> None: + self.chunks = chunks + + def iter_bytes(self) -> Iterator[bytes]: + yield from self.chunks + + +def bytes_chunks() -> Iterator[bytes]: + yield b'abc' + yield b'' + yield b'def' + + +async def async_bytes_chunks() -> AsyncIterator[bytes]: + yield b'abc' + yield b'' + yield b'def' + + +@pytest.mark.parametrize( + 'value', + [ + pytest.param(io.BytesIO(b'data'), id='binary file-like'), + pytest.param(io.StringIO('data'), id='text-mode file-like'), + pytest.param(Reader(b'data'), id='duck-typed reader'), + pytest.param(AsyncReader(b'data'), id='async reader'), + pytest.param(iter([b'data']), id='iterator'), + pytest.param(bytes_chunks(), id='generator'), + pytest.param(async_bytes_chunks(), id='async generator'), + pytest.param(FakeStreamedResponse([b'data']), id='streamed response'), + ], +) +def test_is_source(value: Any) -> None: + """A file-like object, an iterator, an async iterator, and a streamed response can all feed a body.""" + assert StreamedRequestBody.is_source(value) + + +@pytest.mark.parametrize( + 'value', + [ + pytest.param(b'data', id='bytes'), + pytest.param(bytearray(b'data'), id='bytearray'), + pytest.param('data', id='str'), + pytest.param([b'data'], id='list of chunks'), + pytest.param({'key': 'value'}, id='dict'), + pytest.param(None, id='none'), + pytest.param(42, id='int'), + ], +) +def test_is_not_source(value: Any) -> None: + """Buffered bodies and plain containers are not sources, even the iterable ones.""" + assert not StreamedRequestBody.is_source(value) + + +def test_rejects_non_source() -> None: + """Building a body from a value that is not a source fails with a clear error.""" + with pytest.raises(TypeError, match='Cannot stream a request body from a list'): + StreamedRequestBody(cast('Any', [b'data'])) + + +def test_file_like_is_read_in_chunks_of_chunk_size() -> None: + """A file-like source is pulled through `read(chunk_size)` until it runs dry.""" + body = StreamedRequestBody(io.BytesIO(b'x' * 10), chunk_size=4) + + assert list(body.iter_bytes()) == [b'xxxx', b'xxxx', b'xx'] + + +def test_default_chunk_size_is_the_constant() -> None: + """Without an explicit chunk size, a file-like source is read in `STREAMED_BODY_CHUNK_SIZE` pieces.""" + body = StreamedRequestBody(io.BytesIO(b'x' * (STREAMED_BODY_CHUNK_SIZE + 1))) + + assert [len(chunk) for chunk in body.iter_bytes()] == [STREAMED_BODY_CHUNK_SIZE, 1] + + +@pytest.mark.parametrize( + ('source', 'expected'), + [ + pytest.param(io.StringIO('héllo'), b'h\xc3\xa9llo', id='text-mode file-like'), + pytest.param(iter(['hé', 'llo']), b'h\xc3\xa9llo', id='str iterator'), + pytest.param(iter([bytearray(b'he'), memoryview(b'llo')]), b'hello', id='bytes-like chunks'), + ], +) +def test_chunks_are_normalized_to_bytes(source: Any, expected: bytes) -> None: + """String chunks are UTF-8 encoded and other bytes-like chunks are converted, whichever the source yields.""" + body = StreamedRequestBody(source) + + assert b''.join(body.iter_bytes()) == expected + + +def test_empty_iterator_chunks_are_skipped() -> None: + """An empty chunk would terminate a chunked transfer, so it never reaches the transport.""" + body = StreamedRequestBody(bytes_chunks()) + + assert list(body.iter_bytes()) == [b'abc', b'def'] + + +def test_streamed_response_is_forwarded() -> None: + """A streamed response feeds the body through its own chunk iterator.""" + body = StreamedRequestBody(cast('Any', FakeStreamedResponse([b'abc', b'def']))) + + assert list(body.iter_bytes()) == [b'abc', b'def'] + assert not body.is_async + assert not body.rewindable + + +def test_seekable_file_like_is_rewindable_to_its_starting_position() -> None: + """A seekable source can be sent again from where it was when the body was created, not from offset zero.""" + buffer = io.BytesIO(b'skip-rest') + buffer.read(5) + body = StreamedRequestBody(buffer) + + assert body.rewindable + assert b''.join(body.iter_bytes()) == b'rest' + assert b''.join(body.iter_bytes()) == b'' + + body.rewind() + + assert b''.join(body.iter_bytes()) == b'rest' + + +def test_text_mode_file_like_is_rewindable() -> None: + """A text file's opaque `tell` cookie works as the rewind position.""" + buffer = io.StringIO('skip-rest') + buffer.read(5) + body = StreamedRequestBody(buffer) + list(body.iter_bytes()) + + body.rewind() + + assert b''.join(body.iter_bytes()) == b'rest' + + +class NonSeekableBytesIO(io.BytesIO): + """A binary stream that reports itself as non-seekable, like a pipe.""" + + def seekable(self) -> bool: + return False + + +@pytest.mark.parametrize( + 'source', + [ + pytest.param(NonSeekableBytesIO(b'data'), id='non-seekable file-like'), + pytest.param(Reader(b'data'), id='reader without seek support'), + pytest.param(AsyncReader(b'data'), id='async reader'), + pytest.param(iter([b'data']), id='iterator'), + pytest.param(async_bytes_chunks(), id='async iterator'), + pytest.param(FakeStreamedResponse([b'data']), id='streamed response'), + ], +) +def test_is_not_rewindable(source: Any) -> None: + """Everything but a seekable file-like source is consumed once, and says so.""" + body = StreamedRequestBody(source) + + assert not body.rewindable + with pytest.raises(RuntimeError, match='cannot be rewound'): + body.rewind() + + +@pytest.mark.parametrize( + 'source', + [ + pytest.param(AsyncReader(b'data'), id='async reader'), + pytest.param(async_bytes_chunks(), id='async iterator'), + ], +) +def test_async_source_cannot_be_iterated_synchronously(source: Any) -> None: + """An asynchronous source is rejected up front, before the transport pulls a single chunk.""" + body = StreamedRequestBody(source) + + assert body.is_async + with pytest.raises(TypeError, match='Use `ApifyClientAsync`'): + body.iter_bytes() + + +@pytest.mark.parametrize( + 'source', + [ + pytest.param(io.BytesIO(b'data'), id='binary file-like'), + pytest.param(Reader(b'data'), id='duck-typed reader'), + pytest.param(iter([b'data']), id='iterator'), + pytest.param(FakeStreamedResponse([b'data']), id='streamed response'), + ], +) +def test_sync_source_is_not_async(source: Any) -> None: + """A synchronous source works with both clients.""" + assert not StreamedRequestBody(source).is_async + + +@pytest.mark.parametrize( + ('source', 'expected'), + [ + pytest.param(io.BytesIO(b'x' * 10), [b'xxxx', b'xxxx', b'xx'], id='binary file-like'), + pytest.param(io.StringIO('héllo'), [b'h\xc3\xa9ll', b'o'], id='text-mode file-like'), + pytest.param(Reader(b'x' * 10), [b'xxxx', b'xxxx', b'xx'], id='duck-typed reader'), + pytest.param(AsyncReader(b'x' * 10), [b'xxxx', b'xxxx', b'xx'], id='async reader'), + pytest.param(bytes_chunks(), [b'abc', b'def'], id='generator'), + pytest.param(async_bytes_chunks(), [b'abc', b'def'], id='async generator'), + pytest.param(FakeStreamedResponse([b'abc', b'def']), [b'abc', b'def'], id='streamed response'), + pytest.param(SyncOnlyStreamedResponse([b'abc', b'def']), [b'abc', b'def'], id='sync-only response'), + ], +) +async def test_aiter_bytes_handles_every_source_kind(source: Any, expected: list[bytes]) -> None: + """The asynchronous iteration reads file-likes in chunks and forwards iterators, sync or async.""" + body = StreamedRequestBody(source, chunk_size=4) + + assert [chunk async for chunk in body.aiter_bytes()] == expected + + +class NoneReader: + """Mimics a non-blocking raw stream with no data available: `read` returns `None`.""" + + def read(self, size: int = -1) -> None: + _ = size + + +def test_bad_chunk_raises_and_is_recorded_as_the_error() -> None: + """A chunk that is neither bytes-like nor text fails the body, and the failure is kept for the retry loop.""" + body = StreamedRequestBody(cast('Any', NoneReader())) + + with pytest.raises(TypeError, match='produced a NoneType chunk') as exc_info: + list(body.iter_bytes()) + + assert body.error is exc_info.value + + +class FailingReader: + """A file-like object whose second read fails, like a disk error halfway through a file.""" + + def __init__(self) -> None: + self._reads = 0 + + def read(self, size: int = -1) -> bytes: + _ = size + self._reads += 1 + if self._reads > 1: + raise OSError('disk on fire') + return b'first chunk' + + +def test_source_failure_is_recorded_as_the_error() -> None: + """An exception from the source propagates and is kept, so the retry loop can raise it instead of a wrapper.""" + body = StreamedRequestBody(FailingReader()) + chunks = body.iter_bytes() + + assert next(chunks) == b'first chunk' + with pytest.raises(OSError, match='disk on fire') as exc_info: + next(chunks) + assert body.error is exc_info.value + + +async def test_source_failure_is_recorded_as_the_error_async() -> None: + """The asynchronous iteration keeps a source failure the same way.""" + body = StreamedRequestBody(FailingReader()) + + with pytest.raises(OSError, match='disk on fire') as exc_info: + _ = [chunk async for chunk in body.aiter_bytes()] + assert body.error is exc_info.value + + +def test_rewind_clears_the_error() -> None: + """A rewound body starts the next attempt with a clean slate.""" + + class FailOnceBuffer(io.BytesIO): + def __init__(self) -> None: + super().__init__(b'data') + self.fail = True + + def read(self, size: int | None = -1) -> bytes: + if self.fail: + self.fail = False + raise OSError('transient') + return super().read(size) + + body = StreamedRequestBody(FailOnceBuffer()) + with pytest.raises(OSError, match='transient'): + list(body.iter_bytes()) + assert body.error is not None + + body.rewind() + + assert body.error is None + assert b''.join(body.iter_bytes()) == b'data' diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index eccf6ef5..1da54990 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -29,6 +29,8 @@ from apify_client.errors import ApifyApiError if TYPE_CHECKING: + from collections.abc import AsyncIterator, Callable + from apify_client._typeddicts import WebhookRepresentationDict from apify_client.types import WebhooksList @@ -283,69 +285,82 @@ def test_encode_key_value_store_record_value( assert content_type == expected_content_type -def test_encode_key_value_store_record_value_bytesio() -> None: - """Test that BytesIO is read into bytes and encoded as octet-stream.""" - buffer = io.BytesIO(b'buffer data') - value, content_type = encode_key_value_store_record_value(buffer) - assert value == b'buffer data' - assert content_type == 'application/octet-stream' +class Reader: + """A duck-typed file-like object: a callable `read`, no `io.IOBase` ancestry.""" + + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + + def read(self, size: int = -1) -> bytes: + return self._buffer.read(size) + + +class AsyncReader: + """A file-like object with a coroutine `read`, as `aiofiles` provides.""" + + def __init__(self, data: bytes) -> None: + self._buffer = io.BytesIO(data) + async def read(self, size: int = -1) -> bytes: + return self._buffer.read(size) -def test_encode_key_value_store_record_value_stringio() -> None: - """Test that StringIO is read into text and encoded as text/plain.""" - buffer = io.StringIO('buffer data') - value, content_type = encode_key_value_store_record_value(buffer) - assert value == 'buffer data' - assert content_type == 'text/plain; charset=utf-8' +async def async_chunks() -> AsyncIterator[bytes]: + yield b'buffer data' -def test_encode_key_value_store_record_value_duck_typed_file_like() -> None: - """Test that a duck-typed file-like value (a callable `read`, not an `io.IOBase`) is read into bytes.""" - class Reader: - def read(self) -> bytes: - return b'buffer data' +@pytest.mark.parametrize( + ('make_value', 'expected_content_type'), + [ + pytest.param(lambda: io.BytesIO(b'buffer data'), 'application/octet-stream', id='binary file-like'), + pytest.param(lambda: io.StringIO('buffer data'), 'text/plain; charset=utf-8', id='text-mode file-like'), + pytest.param(lambda: Reader(b'buffer data'), 'application/octet-stream', id='duck-typed reader'), + pytest.param(lambda: AsyncReader(b'buffer data'), 'application/octet-stream', id='async reader'), + pytest.param(lambda: iter([b'buffer', b' data']), 'application/octet-stream', id='bytes iterator'), + pytest.param(async_chunks, 'application/octet-stream', id='async bytes iterator'), + ], +) +def test_encode_key_value_store_record_value_passes_streamed_value_through( + make_value: Callable[[], Any], expected_content_type: str +) -> None: + """A streamed value is returned unread for the transport to stream, with a content type fitting its kind.""" + value = make_value() - value, content_type = encode_key_value_store_record_value(Reader()) - assert value == b'buffer data' - assert content_type == 'application/octet-stream' + encoded, content_type = encode_key_value_store_record_value(value) + assert encoded is value + assert content_type == expected_content_type -def test_encode_key_value_store_record_value_async_file_like_raises() -> None: - """Test that an async file-like value is rejected instead of storing the repr of an un-awaited coroutine.""" - class AsyncReader: - async def read(self) -> bytes: - return b'buffer data' +def test_encode_key_value_store_record_value_keeps_explicit_content_type_of_streamed_value() -> None: + """An explicit content type is kept for a streamed value, whose bytes are never inspected.""" + value = io.BytesIO(b'{"a": 1}') - with pytest.raises(TypeError, match='Async file-like objects are not supported'): - encode_key_value_store_record_value(AsyncReader()) + encoded, content_type = encode_key_value_store_record_value(value, content_type='application/json') + assert encoded is value + assert content_type == 'application/json' -def test_encode_key_value_store_record_value_non_bytes_read_raises() -> None: - """Test that a `read` returning neither bytes nor str is rejected instead of being JSON-serialized.""" - class EmptyNonBlockingReader: - def read(self) -> None: - """Mimic a non-blocking raw stream with no data available.""" +def test_encode_key_value_store_record_value_serializes_list_as_json() -> None: + """A list is JSON data, not a stream of chunks, even though it can be iterated.""" + value, content_type = encode_key_value_store_record_value([1, 2]) - with pytest.raises(TypeError, match='returned NoneType, expected bytes or str'): - encode_key_value_store_record_value(EmptyNonBlockingReader()) + assert value == b'[1, 2]' + assert content_type == 'application/json; charset=utf-8' @pytest.mark.parametrize( - ('value', 'expected_type_name'), + ('value', 'match'), [ - pytest.param('already gzipped, honest', 'str', id='string'), - pytest.param({'a': 1}, 'dict', id='json-serializable object'), - pytest.param(io.StringIO('buffer data'), 'str', id='text-mode file-like'), + pytest.param('already gzipped, honest', 'Cannot upload a str value', id='string'), + pytest.param({'a': 1}, 'Cannot upload a dict value', id='json-serializable object'), + pytest.param(io.StringIO('buffer data'), 'opened in text mode', id='text-mode file-like'), ], ) -def test_encode_key_value_store_record_value_declared_compression_of_non_bytes_raises( - value: Any, expected_type_name: str -) -> None: +def test_encode_key_value_store_record_value_declared_compression_of_non_bytes_raises(value: Any, match: str) -> None: """A value that cannot be compressed is rejected when the content encoding declares a compression.""" - with pytest.raises(TypeError, match=f'Cannot upload a {expected_type_name} value'): + with pytest.raises(TypeError, match=match): encode_key_value_store_record_value(value, content_encoding='gzip') @@ -354,7 +369,6 @@ def test_encode_key_value_store_record_value_declared_compression_of_non_bytes_r [ pytest.param(_GZIPPED_DATA, 'gzip', _GZIPPED_DATA, id='bytes'), pytest.param(bytearray(_GZIPPED_DATA), 'gzip', bytearray(_GZIPPED_DATA), id='bytearray'), - pytest.param(io.BytesIO(_GZIPPED_DATA), 'GZip', _GZIPPED_DATA, id='binary file-like, mixed-case encoding'), pytest.param('buffer data', 'identity', 'buffer data', id='string under identity'), pytest.param({'a': 1}, ' Identity ', b'{"a": 1}', id='json-serializable object under padded identity'), ], @@ -367,6 +381,23 @@ def test_encode_key_value_store_record_value_accepts_declared_encoding( assert encoded == expected_value +@pytest.mark.parametrize( + 'make_value', + [ + pytest.param(lambda: io.BytesIO(_GZIPPED_DATA), id='binary file-like'), + pytest.param(lambda: Reader(_GZIPPED_DATA), id='duck-typed reader'), + pytest.param(lambda: iter([_GZIPPED_DATA]), id='bytes iterator'), + ], +) +def test_encode_key_value_store_record_value_streams_pre_encoded_value(make_value: Callable[[], Any]) -> None: + """A streamed value passes the compression guard unread, since only its bytes can carry the encoding.""" + value = make_value() + + encoded, _content_type = encode_key_value_store_record_value(value, content_encoding='GZip') + + assert encoded is value + + def test_encode_key_value_store_record_value_non_encodable_with_explicit_content_type_raises() -> None: """Test that a non-bytes-like value with a non-JSON content type is rejected before it reaches the transport.""" with pytest.raises(TypeError, match="Cannot encode a dict value as 'image/png'"): From 10c22d9b76409a285a5639bde0cd614ac550a91a Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 11 Sep 2026 11:43:54 +0200 Subject: [PATCH 2/9] docs: Drop the Actor pipeline guide from the streaming docs --- docs/02_concepts/09_streaming.mdx | 2 +- docs/03_guides/06_chain_actors.mdx | 52 -------------------- docs/03_guides/code/06_chain_actors_async.py | 42 ---------------- docs/03_guides/code/06_chain_actors_sync.py | 38 -------------- 4 files changed, 1 insertion(+), 133 deletions(-) delete mode 100644 docs/03_guides/06_chain_actors.mdx delete mode 100644 docs/03_guides/code/06_chain_actors_async.py delete mode 100644 docs/03_guides/code/06_chain_actors_sync.py diff --git a/docs/02_concepts/09_streaming.mdx b/docs/02_concepts/09_streaming.mdx index 763b5026..ac68df11 100644 --- a/docs/02_concepts/09_streaming.mdx +++ b/docs/02_concepts/09_streaming.mdx @@ -17,7 +17,7 @@ import UploadFileSyncExample from '!!raw-loader!./code/09_upload_file_sync.py'; import UploadGeneratorAsyncExample from '!!raw-loader!./code/09_upload_generator_async.py'; import UploadGeneratorSyncExample from '!!raw-loader!./code/09_upload_generator_sync.py'; -Large resources don't have to pass through memory whole. The client streams downloads of dataset items, key-value store records, and logs as they arrive, and it streams uploads of key-value store records and Actor inputs from a file or another stream as they're read. Both directions work with the synchronous and the asynchronous client. A download can also feed an upload directly, which is how you [chain Actors into a pipeline](../03_guides/06_chain_actors.mdx). +Large resources don't have to pass through memory whole. The client streams downloads of dataset items, key-value store records, and logs as they arrive, and it streams uploads of key-value store records and Actor inputs from a file or another stream as they're read. Both directions work with the synchronous and the asynchronous client. A download can also feed an upload directly, so one Actor's output becomes another Actor's input without passing through memory. ## Streaming downloads diff --git a/docs/03_guides/06_chain_actors.mdx b/docs/03_guides/06_chain_actors.mdx deleted file mode 100644 index 3799d79f..00000000 --- a/docs/03_guides/06_chain_actors.mdx +++ /dev/null @@ -1,52 +0,0 @@ ---- -id: chain-actors-into-a-pipeline -title: Chain Actors into a pipeline -description: Pass one Actor's output to another Actor as input, and file dataset exports away, without holding the data in memory. ---- - -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; - -import ChainActorsAsyncExample from '!!raw-loader!./code/06_chain_actors_async.py'; -import ChainActorsSyncExample from '!!raw-loader!./code/06_chain_actors_sync.py'; - -A pipeline runs several Actors in sequence, and each stage consumes what the previous one produced. The client moves the data between stages as a stream: a record or a dataset export downloads from one place and uploads to another chunk by chunk, so the process connecting the stages needs no memory for the whole payload. An orchestrating Actor can therefore run with a small memory allocation even when the data it moves is large. - -For the values that can be streamed and how retries and compression apply to them, see [Streaming uploads](../02_concepts/09_streaming.mdx#streaming-uploads). - -## Before you start - -The example uses two placeholder Actors. Replace `my-org/producer-actor` with an Actor that saves its result to the `OUTPUT` record of its default key-value store, and `my-org/consumer-actor` with an Actor that accepts that record as its input. - -## Pipe a record into another Actor's input - -The example runs a two-stage pipeline: - -1. Run the producer Actor and wait for it to finish. -2. Open the `OUTPUT` record of the producer's default key-value store as a stream. -3. Start the consumer Actor with the stream as its input. The record uploads as it downloads, under the content type the producer stored it with. -4. Export the producer's dataset as CSV into a named key-value store, where a later stage or a person can pick it up as one file. - - - - - {ChainActorsAsyncExample} - - - - - {ChainActorsSyncExample} - - - - -A streaming response is consumed by the attempt that sends it, so a failed upload isn't retried. If a stage needs retries, download the data to a temporary file first and upload the file, which the client can rewind. - -## Other data you can pipe - -The same pattern works for anything the client can stream: - -- A dataset export in any format `stream_items` supports, such as `json`, `jsonl`, `csv`, or `xlsx`. Set `content_type` to match the format. -- A run's log from `LogClient.stream`, for example to archive it as a record once the run finishes. -- A local file or a generator, as shown in [Streaming uploads](../02_concepts/09_streaming.mdx#streaming-uploads). diff --git a/docs/03_guides/code/06_chain_actors_async.py b/docs/03_guides/code/06_chain_actors_async.py deleted file mode 100644 index 292934ff..00000000 --- a/docs/03_guides/code/06_chain_actors_async.py +++ /dev/null @@ -1,42 +0,0 @@ -import asyncio - -from apify_client import ApifyClientAsync - -TOKEN = 'MY-APIFY-TOKEN' - - -async def main() -> None: - apify_client = ApifyClientAsync(token=TOKEN) - - # Run the first Actor and wait for it to finish. - producer_run = await apify_client.actor('my-org/producer-actor').call( - run_input={'startUrls': [{'url': 'https://example.com'}]}, - ) - if producer_run is None: - raise RuntimeError('The producer run was not found') - - # Stream the producer's OUTPUT record straight into the consumer's input. - # The record uploads as it downloads, so it never has to fit in memory. - producer_store = apify_client.key_value_store(producer_run.default_key_value_store_id) - async with producer_store.stream_record('OUTPUT') as record: - if record is None: - raise RuntimeError('The producer stored no OUTPUT record') - consumer_run = await apify_client.actor('my-org/consumer-actor').call( - run_input=record['value'], - content_type=record['content_type'], - ) - print(consumer_run) - - # Export the producer's dataset as CSV into a named key-value store, where - # the next stage of the pipeline can pick it up as a single file. - reports_store = await apify_client.key_value_stores().get_or_create( - name='pipeline-reports' - ) - reports_client = apify_client.key_value_store(reports_store.id) - producer_dataset = apify_client.dataset(producer_run.default_dataset_id) - async with producer_dataset.stream_items(item_format='csv') as items: - await reports_client.set_record('items.csv', items, content_type='text/csv') - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/03_guides/code/06_chain_actors_sync.py b/docs/03_guides/code/06_chain_actors_sync.py deleted file mode 100644 index 61b55efd..00000000 --- a/docs/03_guides/code/06_chain_actors_sync.py +++ /dev/null @@ -1,38 +0,0 @@ -from apify_client import ApifyClient - -TOKEN = 'MY-APIFY-TOKEN' - - -def main() -> None: - apify_client = ApifyClient(token=TOKEN) - - # Run the first Actor and wait for it to finish. - producer_run = apify_client.actor('my-org/producer-actor').call( - run_input={'startUrls': [{'url': 'https://example.com'}]}, - ) - if producer_run is None: - raise RuntimeError('The producer run was not found') - - # Stream the producer's OUTPUT record straight into the consumer's input. - # The record uploads as it downloads, so it never has to fit in memory. - producer_store = apify_client.key_value_store(producer_run.default_key_value_store_id) - with producer_store.stream_record('OUTPUT') as record: - if record is None: - raise RuntimeError('The producer stored no OUTPUT record') - consumer_run = apify_client.actor('my-org/consumer-actor').call( - run_input=record['value'], - content_type=record['content_type'], - ) - print(consumer_run) - - # Export the producer's dataset as CSV into a named key-value store, where - # the next stage of the pipeline can pick it up as a single file. - reports_store = apify_client.key_value_stores().get_or_create(name='pipeline-reports') - reports_client = apify_client.key_value_store(reports_store.id) - producer_dataset = apify_client.dataset(producer_run.default_dataset_id) - with producer_dataset.stream_items(item_format='csv') as items: - reports_client.set_record('items.csv', items, content_type='text/csv') - - -if __name__ == '__main__': - main() From e9445bf6299949d18e6857998df138557ebbed8c Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 11 Sep 2026 13:06:59 +0200 Subject: [PATCH 3/9] fix: Send a caller-built streamed request body as it is and reject unsupported ones --- src/apify_client/http_clients/_base.py | 14 +++++++++ .../http_clients/_streamed_body.py | 20 ++++++++++--- src/apify_client/types.py | 11 +++++-- tests/unit/test_http_clients.py | 29 +++++++++++++++++++ tests/unit/test_key_value_store.py | 23 +++++++++++++++ tests/unit/test_streamed_request_body.py | 8 +++++ 6 files changed, 98 insertions(+), 7 deletions(-) diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index 8cac33cc..be957e91 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -325,6 +325,11 @@ def _prepare_request_call( if self._get_header(headers, 'content-type') is None: headers['Content-Type'] = 'application/json' + # A body the caller built itself carries its own chunk size and rewind position, and wrapping it again + # would read it as a response and drop both. + if isinstance(data, StreamedRequestBody): + return (headers, self._parse_params(params), data) + if StreamedRequestBody.is_source(data): return (headers, self._parse_params(params), StreamedRequestBody(data)) @@ -342,6 +347,13 @@ def _prepare_request_call( content = self._http_compressor.compress(content) headers = self._merge_headers(headers, {'Content-Encoding': self._http_compressor.content_encoding}) + elif data is not None: + # Without this the request would go out with no body at all, losing the payload without a word. + raise TypeError( + f'Cannot send a {type(data).__name__} value as a request body. Pass bytes, a string, or a value ' + 'the client can stream, such as a file-like object or an iterator of byte chunks.' + ) + return (headers, self._parse_params(params), content) def _build_url_with_params(self, url: str, *, params: dict[str, Any] | None = None) -> str: @@ -497,6 +509,7 @@ def call( Raises: ApifyApiError: If the request fails after all retries or returns a non-retryable error status. ValueError: If both json and data are provided. + TypeError: If data is neither bytes-like nor a value the client can stream. """ log_context.method.set(method) log_context.url.set(url) @@ -703,6 +716,7 @@ async def call( Raises: ApifyApiError: If the request fails after all retries or returns a non-retryable error status. ValueError: If both json and data are provided. + TypeError: If data is neither bytes-like nor a value the client can stream. """ log_context.method.set(method) log_context.url.set(url) diff --git a/src/apify_client/http_clients/_streamed_body.py b/src/apify_client/http_clients/_streamed_body.py index 87f45126..e713df37 100644 --- a/src/apify_client/http_clients/_streamed_body.py +++ b/src/apify_client/http_clients/_streamed_body.py @@ -24,7 +24,8 @@ class StreamedRequestBody: `HttpClient.call` and `HttpClientAsync.call` wrap a `data` argument that is a file-like object, an iterator of byte chunks, or a streamed `HttpResponse` in this class. The transport pulls the chunks from `iter_bytes` or - `aiter_bytes` and sends each one as it arrives, and the body is never compressed. + `aiter_bytes` and sends each one as it arrives, and the body is never compressed. Build one yourself and pass it + as the `data` to choose the `chunk_size`, and it is sent as it is. The shared retry loop can send a body again only when its source is a seekable file-like object, in which case `rewind` seeks back to where the source was when the body was created. Any other source is consumed by the attempt @@ -40,12 +41,19 @@ def __init__(self, source: StreamedBodySource, *, chunk_size: int = STREAMED_BOD Args: source: The object the chunks come from. See `is_source` for the accepted kinds. - chunk_size: Size, in bytes, of the chunks a file-like source is read in. An iterator or a response - decides its own chunk sizes. + chunk_size: Size of the chunks a file-like source is read in - bytes from a binary source, characters + from a text-mode one. An iterator or a response decides its own chunk sizes. Raises: - TypeError: If `source` is not an object the body can be streamed from. + TypeError: If `source` is not an object the body can be streamed from, or is already a body itself, which + would be read as a response and lose its rewind position. """ + if isinstance(source, StreamedRequestBody): + raise TypeError( + 'The source is already a streamed request body. Pass it as the `data` of a request directly, since ' + 'wrapping it again reads it as a response and drops its rewind position.' + ) + self._chunk_size = chunk_size self._error: Exception | None = None self._is_async = False @@ -93,6 +101,10 @@ def is_source(value: object) -> TypeGuard[StreamedBodySource]: These are a streamed `HttpResponse` (anything with a callable `iter_bytes`), a file-like object (anything with a callable `read`), and an iterator or async iterator of byte chunks. A `str`, `bytes`, `bytearray`, or a container such as a `list` or `dict` is not a source, even though some of them can be iterated. + + A `StreamedRequestBody` matches on its own `iter_bytes`, which is how a hand-built body reaches the request + pipeline untouched. The constructor refuses one, so a caller that builds a body from what this accepts has + to check for an existing body first. """ return ( _is_response(value) diff --git a/src/apify_client/types.py b/src/apify_client/types.py index c1b43406..cefd8d7d 100644 --- a/src/apify_client/types.py +++ b/src/apify_client/types.py @@ -11,7 +11,7 @@ WebhookRepresentationCamelDict, WebhookRepresentationDict, ) -from apify_client.http_clients import HttpResponse +from apify_client.http_clients import HttpResponse, StreamedRequestBody if TYPE_CHECKING: from collections.abc import Awaitable @@ -51,17 +51,22 @@ class SupportsRead(Protocol): chunks, which are UTF-8 encoded. An `async def read`, as `aiofiles` provides, is accepted by `ApifyClientAsync`. """ - def read(self, size: int = ..., /) -> bytes | str | Awaitable[bytes | str]: + def read(self, size: int, /) -> bytes | str | Awaitable[bytes | str]: """Read up to `size` bytes or characters, returning an empty value at the end.""" -StreamedBodySource = SupportsRead | Iterator[bytes | str] | AsyncIterator[bytes | str] | HttpResponse +StreamedBodySource = ( + SupportsRead | Iterator[bytes | str] | AsyncIterator[bytes | str] | HttpResponse | StreamedRequestBody +) """Type for a request body the client streams to the API in chunks instead of holding it in memory whole. A file-like object is read in chunks, an iterator or async iterator yields the chunks itself, and a streamed `HttpResponse` forwards its body, which chains one API call's output into another's input. Accepted as the `data` of `HttpClient.call`, as the `value` of `KeyValueStoreClient.set_record`, and as the `run_input` of Actor runs. See `StreamedRequestBody` for the retry and compression rules that apply. + +Pass a `StreamedRequestBody` built by hand to choose the chunk size a file-like source is read in, which no resource +client exposes on its own. """ JsonSerializable = dict[str, 'JsonSerializable'] | list['JsonSerializable'] | str | int | float | bool | None diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index 4f03e961..3ab6c10a 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -889,6 +889,35 @@ def test_prepare_request_call_streams_body_without_compression( assert not any(key.lower() == 'content-encoding' for key in headers) +def test_prepare_request_call_keeps_a_body_the_caller_built() -> None: + """A hand-built body keeps its rewind position and chunk size, rather than being wrapped a second time.""" + client = ConcreteHttpClient() + body = StreamedRequestBody(BytesIO(b'payload'), chunk_size=3) + + _headers, _params, data = client._prepare_request_call(data=body) + + assert data is body + assert body.rewindable + assert list(body.iter_bytes()) == [b'pay', b'loa', b'd'] + + +@pytest.mark.parametrize( + 'data', + [ + pytest.param({'key': 'value'}, id='dict'), + pytest.param([b'chunk'], id='list of chunks'), + pytest.param(memoryview(b'payload'), id='memoryview'), + pytest.param(42, id='int'), + ], +) +def test_prepare_request_call_rejects_a_body_it_can_neither_send_nor_stream(data: Any) -> None: + """A body that is neither bytes-like nor streamable is rejected, so no request goes out without its payload.""" + client = ConcreteHttpClient() + + with pytest.raises(TypeError, match='as a request body'): + client._prepare_request_call(data=data) + + @pytest.mark.parametrize( 'content_type', [ diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index b55dc265..24862305 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -14,6 +14,7 @@ from apify_client import ApifyClient, ApifyClientAsync from apify_client._consts import MIN_COMPRESSION_SIZE, STREAMED_BODY_CHUNK_SIZE from apify_client.errors import ApifyApiError +from apify_client.http_clients import StreamedRequestBody if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable, Iterator @@ -363,6 +364,28 @@ async def test_set_record_reads_file_like_value_in_chunks_async( assert captured_records[0].get_data() == data +def test_set_record_uploads_a_hand_built_body_sync(*, api_url: str, captured_records: list[Request]) -> None: + """A body the caller built keeps the chunk size it was given, which no resource client exposes on its own.""" + reader = RecordingReader(_BYTES_VALUE) + client = ApifyClient(token='test_token', api_url=api_url) + + client.key_value_store(_MOCKED_KVS_ID).set_record('f', StreamedRequestBody(reader, chunk_size=64)) + + assert set(reader.read_sizes) == {64} + assert_streamed_upload(captured_records[0], 'application/octet-stream') + + +async def test_set_record_uploads_a_hand_built_body_async(*, api_url: str, captured_records: list[Request]) -> None: + """A body the caller built keeps the chunk size it was given, which no resource client exposes on its own.""" + reader = RecordingReader(_BYTES_VALUE) + client = ApifyClientAsync(token='test_token', api_url=api_url) + + await client.key_value_store(_MOCKED_KVS_ID).set_record('f', StreamedRequestBody(reader, chunk_size=64)) + + assert set(reader.read_sizes) == {64} + assert_streamed_upload(captured_records[0], 'application/octet-stream') + + _SOURCE_RECORD_PATH = '/v2/key-value-stores/source_kvs_id/records/f' diff --git a/tests/unit/test_streamed_request_body.py b/tests/unit/test_streamed_request_body.py index d752af23..a02ffd59 100644 --- a/tests/unit/test_streamed_request_body.py +++ b/tests/unit/test_streamed_request_body.py @@ -112,6 +112,14 @@ def test_rejects_non_source() -> None: StreamedRequestBody(cast('Any', [b'data'])) +def test_rejects_a_body_that_is_already_streamed() -> None: + """Wrapping a body again would read it as a response and drop its rewind position, so it is refused.""" + body = StreamedRequestBody(io.BytesIO(b'data')) + + with pytest.raises(TypeError, match='already a streamed request body'): + StreamedRequestBody(body) + + def test_file_like_is_read_in_chunks_of_chunk_size() -> None: """A file-like source is pulled through `read(chunk_size)` until it runs dry.""" body = StreamedRequestBody(io.BytesIO(b'x' * 10), chunk_size=4) From d8be115138cf909108fef6757744010cd54fb1d2 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 11 Sep 2026 13:07:34 +0200 Subject: [PATCH 4/9] refactor: Rewind a streamed request body before every request attempt --- src/apify_client/http_clients/_base.py | 16 ++--- .../http_clients/_streamed_body.py | 15 ++-- tests/unit/test_http_clients.py | 69 +++++++++++++++++++ 3 files changed, 84 insertions(+), 16 deletions(-) diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index be957e91..469edd5b 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -403,21 +403,21 @@ def _handle_request_exception( def _prepare_streamed_body( content: bytes | StreamedRequestBody | None, *, - attempt: int, stop_retrying: Callable[[], None], ) -> None: """Get a streamed body ready for a request attempt. - A rewindable body is sought back to its start before every attempt but the first. Any other streamed body is - consumed by the attempt that sends it, so retrying stops up front and a failure of the attempt is final. + A rewindable body is sought back to its start before every attempt, so each one sends the same bytes and + starts with no recorded source error. Any other streamed body is consumed by the attempt that sends it, so + retrying stops up front and a failure of the attempt is final. """ if not isinstance(content, StreamedRequestBody): return - if not content.rewindable: + if content.rewindable: + content.rewind() + else: logger.debug('The streamed request body cannot be rewound, so a failed attempt is not retried') stop_retrying() - elif attempt > 1: - content.rewind() def _handle_response_status( self, @@ -630,7 +630,7 @@ def _make_request( self._statistics.requests += 1 try: - self._prepare_streamed_body(content, attempt=attempt, stop_retrying=stop_retrying) + self._prepare_streamed_body(content, stop_retrying=stop_retrying) response = self.send_request( method=method, url=self._build_url_with_params(url, params=params), @@ -850,7 +850,7 @@ async def _make_request( self._statistics.requests += 1 try: - self._prepare_streamed_body(content, attempt=attempt, stop_retrying=stop_retrying) + self._prepare_streamed_body(content, stop_retrying=stop_retrying) response = await self.send_request( method=method, url=self._build_url_with_params(url, params=params), diff --git a/src/apify_client/http_clients/_streamed_body.py b/src/apify_client/http_clients/_streamed_body.py index e713df37..42373db2 100644 --- a/src/apify_client/http_clients/_streamed_body.py +++ b/src/apify_client/http_clients/_streamed_body.py @@ -68,11 +68,12 @@ def __init__(self, source: StreamedBodySource, *, chunk_size: int = STREAMED_BOD self._seek: Callable[[int], Any] | None = None self._start: int | None = None - read = getattr(source, 'read', None) + # A response is recognized first, so its `read` is never touched - on an unread streaming response that + # either raises or buffers the whole body. if _is_response(source): self._sync_chunks = source.iter_bytes self._async_chunks = getattr(source, 'aiter_bytes', None) - elif callable(read): + elif callable(read := getattr(source, 'read', None)): self._read = read self._is_async = inspect.iscoroutinefunction(read) # The `seekable` check guards the `tell` call, which a pipe or a socket rejects. An async file-like @@ -182,6 +183,9 @@ async def _aiter_chunks(self) -> AsyncIterator[bytes]: try: if self._read is not None: while True: + # A cancelled `to_thread` await abandons the worker thread rather than stopping it, and the + # thread goes on moving a seekable source's position. Reaching a retry from there would need a + # transport that swallows the cancellation and reports something retryable in its place. chunk = ( await self._read(self._chunk_size) if self._is_async @@ -197,7 +201,7 @@ async def _aiter_chunks(self) -> AsyncIterator[bytes]: yield data elif self._sync_chunks is not None: iterator = iter(self._sync_chunks()) - while (chunk := await asyncio.to_thread(_next_or_done, iterator)) is not _DONE: + while (chunk := await asyncio.to_thread(next, iterator, _DONE)) is not _DONE: if data := _to_bytes(chunk): yield data except Exception as exc: @@ -205,11 +209,6 @@ async def _aiter_chunks(self) -> AsyncIterator[bytes]: raise -def _next_or_done(iterator: Iterator[Any]) -> Any: - """Return the next item of a synchronous iterator, or `_DONE` once it is exhausted.""" - return next(iterator, _DONE) - - def _is_response(value: object) -> TypeGuard[Any]: """Return whether a value is a streamed response, recognized by a callable `iter_bytes`.""" return callable(getattr(value, 'iter_bytes', None)) diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index 3ab6c10a..8db67eaa 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -1285,6 +1285,28 @@ def failing_chunks() -> Iterator[bytes]: raise OSError('disk on fire') +class FailingBuffer(BytesIO): + """A seekable source whose reads fail, so a body that could be rewound still hits a source error.""" + + def read(self, size: int | None = -1) -> bytes: + _ = size + raise OSError('disk on fire') + + +class FailOnceBuffer(BytesIO): + """A seekable source whose first read fails, so a body carries a source error into the next request.""" + + def __init__(self) -> None: + super().__init__(b'payload') + self.fail = True + + def read(self, size: int | None = -1) -> bytes: + if self.fail: + self.fail = False + raise OSError('disk on fire') + return super().read(size) + + def test_send_request_receives_a_streamed_body_as_an_iterator_of_chunks() -> None: """The transport gets the chunks, not the source object, so any library that streams iterables can send them.""" transport = StreamingTransport([200]) @@ -1327,6 +1349,31 @@ async def test_rewindable_streamed_body_is_rewound_between_attempts_async() -> N assert transport.bodies == [b'payload', b'payload'] +def test_rewindable_streamed_body_is_rewound_before_every_request() -> None: + """A body handed to a second request is sent again in full, instead of the source arriving already drained.""" + body = StreamedRequestBody(BytesIO(b'payload')) + first = StreamingTransport([200]) + second = StreamingTransport([200]) + + first.call(method='PUT', url='https://api.test.com/endpoint', data=body) + second.call(method='PUT', url='https://api.test.com/endpoint', data=body) + + assert first.bodies == [b'payload'] + assert second.bodies == [b'payload'] + + +def test_rewindable_streamed_body_drops_a_source_error_of_an_earlier_request() -> None: + """A later request is classified by its own outcome, not by the source error an earlier one recorded.""" + body = StreamedRequestBody(FailOnceBuffer()) + with pytest.raises(OSError, match='disk on fire'): + WrappingTransport([200]).call(method='PUT', url='https://api.test.com/endpoint', data=body) + + transport = WrappingTransport([ConnectionError('reset'), 200]) + transport.call(method='PUT', url='https://api.test.com/endpoint', data=body) + + assert transport.bodies == [b'payload', b'payload'] + + def test_non_rewindable_streamed_body_gets_a_single_attempt() -> None: """An iterator is consumed by the attempt that sends it, so even a retryable failure is final.""" transport = StreamingTransport([ConnectionError('reset'), 200]) @@ -1369,6 +1416,28 @@ async def test_streamed_body_source_error_replaces_the_wrapped_transport_error_a assert len(transport.attempts_with_iterator) == 1 +def test_rewindable_streamed_body_source_error_is_not_retried() -> None: + """A rewindable source that fails is raised as itself after one attempt, since rewinding cannot fix it.""" + transport = WrappingTransport([200, 200]) + + with pytest.raises(OSError, match='disk on fire') as exc_info: + transport.call(method='PUT', url='https://api.test.com/endpoint', data=FailingBuffer(b'payload')) + + assert isinstance(exc_info.value.__cause__, ConnectionError) + assert len(transport.attempts_with_iterator) == 1 + + +async def test_rewindable_streamed_body_source_error_is_not_retried_async() -> None: + """A rewindable source that fails is raised as itself after one attempt, since rewinding cannot fix it.""" + transport = WrappingTransportAsync([200, 200]) + + with pytest.raises(OSError, match='disk on fire') as exc_info: + await transport.call(method='PUT', url='https://api.test.com/endpoint', data=FailingBuffer(b'payload')) + + assert isinstance(exc_info.value.__cause__, ConnectionError) + assert len(transport.attempts_with_iterator) == 1 + + def test_streamed_body_source_error_stops_retrying_when_the_transport_propagates_it() -> None: """A transport that lets the source error through, like HTTPX2, ends up with the same single attempt.""" transport = StreamingTransport([200, 200]) From 144b0d0fa6251def8667c790614420e54e461347 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 11 Sep 2026 13:08:09 +0200 Subject: [PATCH 5/9] docs: Correct the wording for streamed values in docs and docstrings --- docs/02_concepts/09_streaming.mdx | 2 +- src/apify_client/_resource_clients/actor.py | 8 ++++++-- .../_resource_clients/key_value_store.py | 12 ++++++------ src/apify_client/_utils/encoding.py | 5 +++-- tests/unit/test_key_value_store.py | 2 +- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/docs/02_concepts/09_streaming.mdx b/docs/02_concepts/09_streaming.mdx index ac68df11..3bb66a0b 100644 --- a/docs/02_concepts/09_streaming.mdx +++ b/docs/02_concepts/09_streaming.mdx @@ -92,7 +92,7 @@ With a generator, you produce the data while it uploads, for example from pages ### Content type -A file opened in text mode is uploaded as `text/plain; charset=utf-8`, encoded to UTF-8 chunk by chunk. Any other streamed value is uploaded as `application/octet-stream` unless you pass `content_type`. An iterator or a response carries no type of its own, so set `content_type` explicitly, as the examples do. +A file the standard library opened in text mode is uploaded as `text/plain; charset=utf-8`, encoded to UTF-8 chunk by chunk. Any other streamed value is uploaded as `application/octet-stream` unless you pass `content_type`, an `aiofiles` text file included - the client can't tell it from a binary one. An iterator or a response carries no type of its own, so set `content_type` explicitly, as the examples do. ### Compression diff --git a/src/apify_client/_resource_clients/actor.py b/src/apify_client/_resource_clients/actor.py index 37a15fab..d8292620 100644 --- a/src/apify_client/_resource_clients/actor.py +++ b/src/apify_client/_resource_clients/actor.py @@ -544,7 +544,9 @@ def validate_input( """Validate an input for the Actor that defines an input schema. Args: - run_input: The input to validate. + run_input: The input to validate. Accepts the same values as `KeyValueStoreClient.set_record`, + including a file-like object, an iterator of byte chunks, or a streamed `HttpResponse`, which + are uploaded in chunks without being held in memory. build_tag: The Actor's build tag. content_type: The content type of the input. timeout: Timeout for the API HTTP request. @@ -1051,7 +1053,9 @@ async def validate_input( """Validate an input for the Actor that defines an input schema. Args: - run_input: The input to validate. + run_input: The input to validate. Accepts the same values as `KeyValueStoreClientAsync.set_record`, + including a file-like object, an iterator of byte chunks, or a streamed `HttpResponse`, which + are uploaded in chunks without being held in memory. build_tag: The Actor's build tag. content_type: The content type of the input. timeout: Timeout for the API HTTP request. diff --git a/src/apify_client/_resource_clients/key_value_store.py b/src/apify_client/_resource_clients/key_value_store.py index 723f2ffb..b85fb2e9 100644 --- a/src/apify_client/_resource_clients/key_value_store.py +++ b/src/apify_client/_resource_clients/key_value_store.py @@ -377,9 +377,9 @@ def set_record( content_encoding: The encoding already applied to `value`, sent as the `Content-Encoding` header. Pass it to upload a pre-compressed value - the client then forwards the bytes as they are instead of compressing them itself. The API accepts `gzip`, `br`, `deflate`, and `identity`, and stores the - record exactly as uploaded, so this also becomes the encoding the record is served with. Only a - bytes-like `value`, or a file-like one that reads into bytes, can carry a compression - anything - else raises `TypeError` instead of being stored under a header that misdescribes it. + record exactly as uploaded, so this also becomes the encoding the record is served with. A `str`, a + JSON-serializable object, or a text-mode file cannot be carrying a compression and raises + `TypeError`; a streamed value is taken at its word, since its bytes are only seen as they are sent. timeout: Timeout for the API HTTP request. """ value, content_type = encode_key_value_store_record_value( @@ -813,9 +813,9 @@ async def set_record( content_encoding: The encoding already applied to `value`, sent as the `Content-Encoding` header. Pass it to upload a pre-compressed value - the client then forwards the bytes as they are instead of compressing them itself. The API accepts `gzip`, `br`, `deflate`, and `identity`, and stores the - record exactly as uploaded, so this also becomes the encoding the record is served with. Only a - bytes-like `value`, or a file-like one that reads into bytes, can carry a compression - anything - else raises `TypeError` instead of being stored under a header that misdescribes it. + record exactly as uploaded, so this also becomes the encoding the record is served with. A `str`, a + JSON-serializable object, or a text-mode file cannot be carrying a compression and raises + `TypeError`; a streamed value is taken at its word, since its bytes are only seen as they are sent. timeout: Timeout for the API HTTP request. """ value, content_type = encode_key_value_store_record_value( diff --git a/src/apify_client/_utils/encoding.py b/src/apify_client/_utils/encoding.py index ae78dc21..e5e78432 100644 --- a/src/apify_client/_utils/encoding.py +++ b/src/apify_client/_utils/encoding.py @@ -23,8 +23,9 @@ def encode_key_value_store_record_value( or a streamed `HttpResponse` is returned as it is, to be streamed to the API in chunks from its current position - the object is neither rewound nor closed. Any other value is JSON-serialized unless it is already bytes or a string. - content_type: The content type; if None, it's inferred from the value type. A file opened in text mode is - `text/plain; charset=utf-8`, any other streamed value is `application/octet-stream`. + content_type: The content type; if None, it's inferred from the value type. An `io.TextIOBase`, which is what + the standard library returns for a file opened in text mode, is `text/plain; charset=utf-8`; any other + streamed value is `application/octet-stream`. content_encoding: The encoding the caller declares the value already carries, if any. Anything other than `identity` means the value is compressed, which only a bytes-like payload can be, so a string, a JSON-serialized object, or a text-mode file is rejected. Any other streamed value is taken at its word, diff --git a/tests/unit/test_key_value_store.py b/tests/unit/test_key_value_store.py index 24862305..319f7c6e 100644 --- a/tests/unit/test_key_value_store.py +++ b/tests/unit/test_key_value_store.py @@ -115,7 +115,7 @@ async def async_bytes_chunks() -> AsyncIterator[bytes]: ] # Values that cannot be carrying the `gzip` encoding the caller declares for them. Built by a factory for the -# same reason as `_FILE_LIKE_VALUE_CASES`, as the sync and async test each consume their own value. +# same reason as `_STREAMED_VALUE_CASES`, as the sync and async test each consume their own value. _INCOMPRESSIBLE_VALUE_CASES = [ pytest.param(lambda: _TEXT_VALUE, id='string'), pytest.param(lambda: {'key': 'value'}, id='json-serializable object'), From a6ef26a69e49f777b2e65d33908dbb592afeefbc Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 16 Sep 2026 12:55:05 +0200 Subject: [PATCH 6/9] docs: Clarify streaming upload retries, size cap, and chunk size units --- docs/02_concepts/09_streaming.mdx | 6 +++--- src/apify_client/_consts.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/02_concepts/09_streaming.mdx b/docs/02_concepts/09_streaming.mdx index 3bb66a0b..000a8af8 100644 --- a/docs/02_concepts/09_streaming.mdx +++ b/docs/02_concepts/09_streaming.mdx @@ -49,12 +49,12 @@ The following example shows how to stream the logs of an Actor run incrementally `KeyValueStoreClient.set_record` and the `run_input` of `ActorClient.start`, `ActorClient.call`, and `RunClient.metamorph` accept a value the client streams to the API in chunks: - A file-like object, meaning anything with a `read(size)` method: an open file, an `io.BytesIO`, a member of a ZIP archive, or a pipe. The client reads it in 64 KiB chunks from its current position and doesn't close it. -- An iterator of `bytes` or `str` chunks, such as a generator. The iterator decides the chunk sizes. +- An iterator of `bytes` or `str` chunks, such as a generator. The iterator decides the chunk sizes, and it's consumed by the upload, so passing the same one to a second upload sends an empty body. - A streaming `HttpResponse` from one of the download methods, whose body is forwarded chunk by chunk. The asynchronous client also accepts an async iterator and a file-like object with an `async def read`, such as a file opened with `aiofiles`. The synchronous client rejects those with a `TypeError`. A synchronous file or iterator works with both clients. The asynchronous client reads it in a worker thread, so the event loop stays free. -Only the chunk being sent is held in memory, so the process uploading a file doesn't need memory for the whole file. +Only the chunk being sent is held in memory, so the process uploading a file doesn't need memory for the whole file. What the API accepts doesn't change: an Actor input is still capped at 9 MB, and a streamed body over that size is rejected with a `413` during the upload. The following example uploads a file from disk. The asynchronous variant opens it with [aiofiles](https://github.com/Tinche/aiofiles), which isn't an `apify-client` dependency, so install it alongside the client: @@ -106,7 +106,7 @@ The client retries a failed request only when it can send the body again: - An iterator, a streaming response, a pipe, or any asynchronous source is consumed by the attempt that sends it. The request gets a single attempt, and a failure raises immediately. - A failure inside the source itself, such as a disk error while reading the file, is raised as that error and isn't retried, whatever the source. -If an upload from a source that can't be rewound needs retries, download the data to a temporary file first and upload the file. +If an upload from a source that can't be rewound needs retries, download the data to a temporary file first and upload the file. In the asynchronous client, prefer a file opened with `open()` over one opened with `aiofiles` when the upload has to survive a retry: the asynchronous client reads it in a worker thread either way, but only the plain file can be sought back. ### Custom HTTP clients diff --git a/src/apify_client/_consts.py b/src/apify_client/_consts.py index e8875b00..08dc81f4 100644 --- a/src/apify_client/_consts.py +++ b/src/apify_client/_consts.py @@ -43,7 +43,8 @@ """ STREAMED_BODY_CHUNK_SIZE = 64 * 1024 -"""Size, in bytes, of the chunks a streamed request body reads from a file-like source. +"""Size of the chunks a streamed request body reads from a file-like source, in bytes from a binary source and in +characters from a text-mode one. A chunk is the most of a streamed body that is in memory at once, and in the asynchronous client every chunk costs one worker-thread hop, so the size balances memory against per-chunk overhead. It matches the buffer size the From 0e036af43ea0ccdb7dca49545e8d5c4b6a4d38ed Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 16 Sep 2026 12:55:07 +0200 Subject: [PATCH 7/9] refactor: Narrow the types of _is_body_worth_compressing and _is_response --- src/apify_client/http_clients/_base.py | 8 ++++---- src/apify_client/http_clients/_streamed_body.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index 469edd5b..664220c2 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -238,14 +238,14 @@ def _parse_params(params: dict[str, Any] | None) -> dict[str, Any] | None: return parsed_params @staticmethod - def _is_body_worth_compressing(data: object) -> bool: + def _is_body_worth_compressing(data: str | bytes | bytearray | StreamedBodySource | None) -> bool: """Whether the body is large enough that `_prepare_request_call` may compress it. This gate only picks where the preparation runs (worker thread or inline), so it approximates rather than replicates the compression conditions: the content type and `Content-Encoding` checks are skipped, - and a `str` is measured in characters instead of encoded bytes. A misjudged body costs either one - needless thread hop or an inline preparation of a body under `MIN_COMPRESSION_SIZE` characters - both - cheap. + and a `str` is measured in characters instead of encoded bytes. A streamed body is never compressed, + so it never earns the hop. A misjudged body costs either one needless thread hop or an inline + preparation of a body under `MIN_COMPRESSION_SIZE` characters - both cheap. """ if isinstance(data, (str, bytes, bytearray)): return len(data) >= MIN_COMPRESSION_SIZE diff --git a/src/apify_client/http_clients/_streamed_body.py b/src/apify_client/http_clients/_streamed_body.py index 42373db2..965b39e6 100644 --- a/src/apify_client/http_clients/_streamed_body.py +++ b/src/apify_client/http_clients/_streamed_body.py @@ -12,6 +12,7 @@ from collections.abc import Callable, Iterable from typing import TypeGuard + from apify_client.http_clients._base import HttpResponse from apify_client.types import StreamedBodySource _DONE = object() @@ -209,7 +210,7 @@ async def _aiter_chunks(self) -> AsyncIterator[bytes]: raise -def _is_response(value: object) -> TypeGuard[Any]: +def _is_response(value: object) -> TypeGuard[HttpResponse]: """Return whether a value is a streamed response, recognized by a callable `iter_bytes`.""" return callable(getattr(value, 'iter_bytes', None)) From 50e58943ac86e45028b7da4e29ee3707467b4158 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 16 Sep 2026 12:55:08 +0200 Subject: [PATCH 8/9] test: Make the streamed record payload detect reordered chunks --- tests/integration/test_key_value_store.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index ca240999..40ee1043 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -788,8 +788,9 @@ async def test_key_value_store_set_streamed_record(client: ApifyClient | ApifyCl store_client = client.key_value_store(created_store.id) try: - # Several chunks' worth of non-repeating bytes, so a dropped or reordered chunk would show in the comparison. - data = bytes(range(256)) * (3 * 1024 * 4) + # Several chunks' worth of data, distinct in every 256-byte block, so a dropped or reordered chunk shows + # up in the comparison. + data = b''.join(index.to_bytes(4, 'big') + bytes(range(252)) for index in range(12 * 1024)) await maybe_await( store_client.set_record('stream.bin', io.BytesIO(data), content_type='application/octet-stream') ) From 3e56d019901dd1699cab450735901af69cfe93d0 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 16 Sep 2026 13:38:08 +0200 Subject: [PATCH 9/9] fix: Accept a file-like read that takes no size argument --- docs/02_concepts/09_streaming.mdx | 2 +- .../http_clients/_streamed_body.py | 36 +++++++++++-- src/apify_client/types.py | 21 +++++++- tests/unit/test_streamed_request_body.py | 51 +++++++++++++++++++ 4 files changed, 103 insertions(+), 7 deletions(-) diff --git a/docs/02_concepts/09_streaming.mdx b/docs/02_concepts/09_streaming.mdx index 000a8af8..28bb1f34 100644 --- a/docs/02_concepts/09_streaming.mdx +++ b/docs/02_concepts/09_streaming.mdx @@ -48,7 +48,7 @@ The following example shows how to stream the logs of an Actor run incrementally `KeyValueStoreClient.set_record` and the `run_input` of `ActorClient.start`, `ActorClient.call`, and `RunClient.metamorph` accept a value the client streams to the API in chunks: -- A file-like object, meaning anything with a `read(size)` method: an open file, an `io.BytesIO`, a member of a ZIP archive, or a pipe. The client reads it in 64 KiB chunks from its current position and doesn't close it. +- A file-like object, meaning anything with a `read` method: an open file, an `io.BytesIO`, a member of a ZIP archive, or a pipe. The client reads it in 64 KiB chunks from its current position and doesn't close it. A `read` that takes no size is called once instead, so that source is held in memory whole. - An iterator of `bytes` or `str` chunks, such as a generator. The iterator decides the chunk sizes, and it's consumed by the upload, so passing the same one to a second upload sends an empty body. - A streaming `HttpResponse` from one of the download methods, whose body is forwarded chunk by chunk. diff --git a/src/apify_client/http_clients/_streamed_body.py b/src/apify_client/http_clients/_streamed_body.py index 965b39e6..72eb3219 100644 --- a/src/apify_client/http_clients/_streamed_body.py +++ b/src/apify_client/http_clients/_streamed_body.py @@ -34,7 +34,8 @@ class StreamedRequestBody: A file opened in text mode, or an iterator yielding strings, is UTF-8 encoded chunk by chunk. A file-like object whose `read` is a coroutine function, as `aiofiles` provides, and an async iterator can only be sent by the - asynchronous client. + asynchronous client. A `read` that takes no size is called once and its result sent as a single chunk, so such + a source is held in memory whole. """ def __init__(self, source: StreamedBodySource, *, chunk_size: int = STREAMED_BODY_CHUNK_SIZE) -> None: @@ -61,7 +62,8 @@ def __init__(self, source: StreamedBodySource, *, chunk_size: int = STREAMED_BOD # Exactly one of these produces the chunks: a file-like `read`, a factory of a synchronous iterable, or a # factory of an asynchronous one. A response provides both factories. - self._read: Callable[[int], Any] | None = None + self._read: Callable[..., Any] | None = None + self._read_takes_size = True self._sync_chunks: Callable[[], Iterable[Any]] | None = None self._async_chunks: Callable[[], AsyncIterator[Any]] | None = None @@ -76,6 +78,7 @@ def __init__(self, source: StreamedBodySource, *, chunk_size: int = STREAMED_BOD self._async_chunks = getattr(source, 'aiter_bytes', None) elif callable(read := getattr(source, 'read', None)): self._read = read + self._read_takes_size = _accepts_chunk_size(read) self._is_async = inspect.iscoroutinefunction(read) # The `seekable` check guards the `tell` call, which a pipe or a socket rejects. An async file-like # object also seeks asynchronously, so it is treated as a source that cannot be rewound. @@ -168,9 +171,14 @@ def aiter_bytes(self) -> AsyncIterator[bytes]: def _iter_chunks(self) -> Iterator[bytes]: try: if self._read is not None: - # A file-like source signals its end with an empty read. - while data := _to_bytes(self._read(self._chunk_size)): - yield data + if not self._read_takes_size: + # A `read` that takes no size hands over the whole source in one call, so it is one chunk. + if data := _to_bytes(self._read()): + yield data + else: + # A file-like source signals its end with an empty read. + while data := _to_bytes(self._read(self._chunk_size)): + yield data elif self._sync_chunks is not None: for chunk in self._sync_chunks(): # In chunked transfer encoding an empty chunk terminates the body, so none is passed on. @@ -183,6 +191,11 @@ def _iter_chunks(self) -> Iterator[bytes]: async def _aiter_chunks(self) -> AsyncIterator[bytes]: try: if self._read is not None: + if not self._read_takes_size: + chunk = await self._read() if self._is_async else await asyncio.to_thread(self._read) + if data := _to_bytes(chunk): + yield data + return while True: # A cancelled `to_thread` await abandons the worker thread rather than stopping it, and the # thread goes on moving a seekable source's position. Reaching a retry from there would need a @@ -210,6 +223,19 @@ async def _aiter_chunks(self) -> AsyncIterator[bytes]: raise +def _accepts_chunk_size(read: Callable[..., Any]) -> bool: + """Whether a file-like object's `read` takes the chunk size, which a duck-typed one may leave out.""" + try: + inspect.signature(read).bind(STREAMED_BODY_CHUNK_SIZE) + except ValueError: + # A `read` implemented in C can hide its signature, and every file object in the standard library takes + # the size, so the chunked call is the safer guess. + return True + except TypeError: + return False + return True + + def _is_response(value: object) -> TypeGuard[HttpResponse]: """Return whether a value is a streamed response, recognized by a callable `iter_bytes`.""" return callable(getattr(value, 'iter_bytes', None)) diff --git a/src/apify_client/types.py b/src/apify_client/types.py index cefd8d7d..bedc0096 100644 --- a/src/apify_client/types.py +++ b/src/apify_client/types.py @@ -49,14 +49,32 @@ class SupportsRead(Protocol): `read` is called with the chunk size until it returns an empty value. A file opened in text mode returns `str` chunks, which are UTF-8 encoded. An `async def read`, as `aiofiles` provides, is accepted by `ApifyClientAsync`. + + A `read` that takes no size is accepted as `SupportsReadAll`. """ def read(self, size: int, /) -> bytes | str | Awaitable[bytes | str]: """Read up to `size` bytes or characters, returning an empty value at the end.""" +class SupportsReadAll(Protocol): + """A file-like object whose `read` takes no size, so one call hands over the whole source. + + It is called once and its result sent as a single chunk, which holds the source in memory whole. `SupportsRead` + is the shape that streams. + """ + + def read(self) -> bytes | str | Awaitable[bytes | str]: + """Read the whole source, returning an empty value once it has been read.""" + + StreamedBodySource = ( - SupportsRead | Iterator[bytes | str] | AsyncIterator[bytes | str] | HttpResponse | StreamedRequestBody + SupportsRead + | SupportsReadAll + | Iterator[bytes | str] + | AsyncIterator[bytes | str] + | HttpResponse + | StreamedRequestBody ) """Type for a request body the client streams to the API in chunks instead of holding it in memory whole. @@ -80,6 +98,7 @@ def read(self, size: int, /) -> bytes | str | Awaitable[bytes | str]: 'JsonSerializable', 'StreamedBodySource', 'SupportsRead', + 'SupportsReadAll', 'Timeout', 'WebhooksList', ] diff --git a/tests/unit/test_streamed_request_body.py b/tests/unit/test_streamed_request_body.py index a02ffd59..32c3dae5 100644 --- a/tests/unit/test_streamed_request_body.py +++ b/tests/unit/test_streamed_request_body.py @@ -32,6 +32,34 @@ async def read(self, size: int = -1) -> bytes: return self._buffer.read(size) +class ZeroArgReader: + """A duck-typed file-like object whose `read` takes no size, so it hands over everything at once.""" + + def __init__(self, data: bytes) -> None: + self._data = data + self._read = False + + def read(self) -> bytes: + if self._read: + return b'' + self._read = True + return self._data + + +class AsyncZeroArgReader: + """A file-like object with a coroutine `read` that takes no size.""" + + def __init__(self, data: bytes) -> None: + self._data = data + self._read = False + + async def read(self) -> bytes: + if self._read: + return b'' + self._read = True + return self._data + + class FakeStreamedResponse: """The streaming half of the `HttpResponse` protocol, with both a sync and an async chunk iterator.""" @@ -127,6 +155,29 @@ def test_file_like_is_read_in_chunks_of_chunk_size() -> None: assert list(body.iter_bytes()) == [b'xxxx', b'xxxx', b'xx'] +def test_file_like_without_a_size_argument_is_read_once() -> None: + """A `read` that takes no size is called without one and its result sent as a single chunk.""" + body = StreamedRequestBody(ZeroArgReader(b'buffer data'), chunk_size=4) + + assert list(body.iter_bytes()) == [b'buffer data'] + + +def test_file_like_without_a_size_argument_yields_nothing_when_empty() -> None: + """A `read` that takes no size and returns nothing sends no chunk, since an empty chunk ends a chunked body.""" + body = StreamedRequestBody(ZeroArgReader(b'')) + + assert list(body.iter_bytes()) == [] + + +async def test_file_like_without_a_size_argument_is_read_once_async() -> None: + """The asynchronous client reads a size-less `read` once, whether it is a coroutine function or not.""" + sync_body = StreamedRequestBody(ZeroArgReader(b'buffer data'), chunk_size=4) + async_body = StreamedRequestBody(AsyncZeroArgReader(b'buffer data'), chunk_size=4) + + assert [chunk async for chunk in sync_body.aiter_bytes()] == [b'buffer data'] + assert [chunk async for chunk in async_body.aiter_bytes()] == [b'buffer data'] + + def test_default_chunk_size_is_the_constant() -> None: """Without an explicit chunk size, a file-like source is read in `STREAMED_BODY_CHUNK_SIZE` pieces.""" body = StreamedRequestBody(io.BytesIO(b'x' * (STREAMED_BODY_CHUNK_SIZE + 1)))