diff --git a/docs/02_concepts/08_pagination.mdx b/docs/02_concepts/08_pagination.mdx
index e68179ee..757f51c2 100644
--- a/docs/02_concepts/08_pagination.mdx
+++ b/docs/02_concepts/08_pagination.mdx
@@ -25,6 +25,8 @@ Most methods named `list` or `list_something` in the Apify client return a page
- `count` - The number of items in the current page.
- `limit` - The maximum number of items per page.
+On `DatasetItemsPage`, `count` reports how many dataset rows the API scanned for the page. Filters drop items from a page and `unwind` multiplies them, so a page can hold fewer or more items than `count`.
+
Some methods paginate differently. For example, `RequestQueueClient.list_requests` returns a cursor-based `ListOfRequests` without the `total`, `offset`, and `count` fields. To fetch the next page, pass its `next_cursor` value back as the `cursor` parameter. Other examples include `list_keys` and `list_head`. Regardless, the primary results are always stored under the `items` field, and the `limit` field can be used to control the number of results returned.
The following example shows how to fetch all items from a dataset using pagination:
diff --git a/src/apify_client/_pagination.py b/src/apify_client/_pagination.py
index 4facbe58..46637719 100644
--- a/src/apify_client/_pagination.py
+++ b/src/apify_client/_pagination.py
@@ -19,9 +19,10 @@
class HasItems(Protocol[T]):
"""Structural contract for a single page of results from a paginated API endpoint.
- Implementations must expose `items`. They may optionally expose `count` - the number of items scanned by the API for
- this page, which can exceed `len(items)` when filters drop items from the response. The iterator helpers consult
- `count` opportunistically via `getattr` for offset bookkeeping and fall back to `len(items)` when it is absent.
+ Implementations must expose `items`. They may optionally expose `count` - the number of rows the API scanned to
+ produce this page, which `len(items)` can land below (filters drop items) or above (`unwind` splits one row into
+ several items). The iterator helpers consult `count` opportunistically via `getattr` for offset bookkeeping and
+ fall back to `len(items)` when it is absent.
"""
items: list[T]
@@ -38,20 +39,19 @@ def get_items_iterator(
The `callback` is invoked lazily to fetch each page from the API. It must accept `limit` and `offset` keyword
arguments and return an object whose `items` attribute is a list. If the object also exposes a `count` attribute, it
- is used for offset bookkeeping (the Apify API's `count` reflects items scanned, which can exceed items returned when
- filters are applied).
+ is used for offset bookkeeping - `_page_scanned_rows` describes how the next offset is derived.
- Iteration stops when a page scans no items (`count` is `0`, or `items` is empty when `count` is absent) or when the
- user-requested `limit` is reached. A page can scan items while returning none - filters like `clean` drop items from
- `items` but still count toward `count` - so terminating on scanned rather than returned items keeps the iterator
- advancing across fully-filtered pages. The `total` field is intentionally not consulted, because it can change
- between calls.
+ Iteration stops when a page scans no rows or when the user-requested `limit` is reached. A page can scan rows while
+ returning no items - filters like `clean` drop items from `items` but still count toward `count` - so terminating on
+ scanned rather than returned rows keeps the iterator advancing across fully-filtered pages. The `total` field is
+ intentionally not consulted, because it can change between calls.
Args:
callback: Function returning a single page of items.
- limit: Maximum total number of items to yield across all pages. `None` or `0` means no limit.
+ limit: Maximum total number of rows scanned across all pages. On the dataset items endpoint `unwind` can
+ turn one row into several items, so more items than this can be yielded. `None` or `0` means no limit.
offset: Starting offset for the first page.
- chunk_size: Maximum number of items requested per API call. `None` or `0` lets the API decide.
+ chunk_size: Per-page cap, sent to the API as its `limit`. `None` or `0` lets the API decide.
"""
effective_chunk = chunk_size or 0
initial_offset = offset or 0
@@ -59,13 +59,14 @@ def get_items_iterator(
fetched_items = 0
while True:
+ page_limit = _next_page_limit(initial_limit, fetched_items, effective_chunk)
current_page = callback(
- limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
+ limit=page_limit,
offset=initial_offset + fetched_items,
)
yield from current_page.items
- page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items))
+ page_scanned = _page_scanned_rows(current_page, page_limit)
fetched_items += page_scanned
if not page_scanned or (initial_limit and fetched_items >= initial_limit):
@@ -89,14 +90,15 @@ async def get_items_iterator_async(
fetched_items = 0
while True:
+ page_limit = _next_page_limit(initial_limit, fetched_items, effective_chunk)
current_page = await callback(
- limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
+ limit=page_limit,
offset=initial_offset + fetched_items,
)
for item in current_page.items:
yield item
- page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items))
+ page_scanned = _page_scanned_rows(current_page, page_limit)
fetched_items += page_scanned
if not page_scanned or (initial_limit and fetched_items >= initial_limit):
@@ -126,20 +128,30 @@ def get_cursor_iterator(
limit: int | None = None,
chunk_size: int | None = None,
) -> Iterator[KeyValueStoreKey] | Iterator[Request]:
- """Yield individual items from cursor-paginated API responses.
+ """Yield individual items from a cursor-paginated API response.
+
+ This iterator supports the two API responses that use cursor pagination. `ListOfKeys` is used for key-value store
+ keys, while `ListOfRequests` is used for request queue requests.
+
+ Pagination continues until either:
+
+ - the API returns no next cursor, or
+ - the requested `limit` is reached.
+
+ An empty page does not explicitly stop the iteration. In practice, both supported endpoints return a next cursor
+ only when the current page contains items, so an empty page always has a `None` cursor and naturally ends the
+ iteration.
- Cursor pagination is restricted to the two API responses that expose it: `ListOfKeys` (for key-value store keys) and
- `ListOfRequests` (for request queue requests). Iteration ends when the next cursor is `None` or the user-requested
- `limit` is reached. Emptiness alone does not stop iteration: server-side filters (such as the request-queue state
- `filter`) can drop every item on a page while a live cursor still points at more data, so termination relies on the
- cursor, not on whether a page returned items. Unlike offset responses, cursor responses expose no scanned-item
- `count`, so `count` cannot be used to detect a fully-filtered page here.
+ The endpoints determine the next cursor differently:
+
+ - For key-value store keys, the cursor is the last key returned on the current page.
+ - For request queue requests, a cursor is returned only when the current page is full.
Args:
- callback: Function returning a single page of items. Receives `cursor` and `limit` kwargs.
- cursor: Value of the cursor for the first request, or `None` to start from the beginning.
+ callback: Function that returns one page of items and accepts `cursor` and `limit` keyword arguments.
+ cursor: Cursor to use for the first request. If `None`, iteration starts from the beginning.
limit: Maximum total number of items to yield across all pages.
- chunk_size: Maximum number of items requested per API call.
+ chunk_size: Maximum number of items to request in a single API call.
"""
effective_chunk = chunk_size or 0
initial_limit = limit or 0
@@ -218,3 +230,19 @@ def _next_page_limit(initial_limit: int, fetched_items: int, effective_chunk: in
if not effective_chunk:
return remaining
return min(remaining, effective_chunk)
+
+
+def _page_scanned_rows(page: HasItems[T], requested_limit: int) -> int:
+ """Compute how far the offset advances past `page`, in dataset rows.
+
+ Neither reported number is right on its own. `count` follows the rows the API scanned, but it is derived from a
+ dataset's item count, which is incremented by a throttled write and so lags a fresh push. `len(items)` counts the
+ items the API shaped out of those rows: filters (`clean`, `skip_empty`, `skip_hidden`) drop some, and `unwind`
+ splits one row into several. The larger of the two absorbs a `count` that lags behind the items returned, and
+ capping it at the rows the call asked for keeps an unwound page from advancing past rows the next call would then
+ never read. The cap is a valid bound because the endpoint applies the `limit` it is sent verbatim; on a page
+ covering fewer rows than that, the advance can still overshoot into rows a concurrent push appends afterwards. A
+ `requested_limit` of `0` means the call sent no limit, leaving the advance unbounded.
+ """
+ scanned_rows = max(getattr(page, 'count', 0), len(page.items))
+ return min(scanned_rows, requested_limit) if requested_limit else scanned_rows
diff --git a/src/apify_client/_resource_clients/dataset.py b/src/apify_client/_resource_clients/dataset.py
index 194e6572..1f772dde 100644
--- a/src/apify_client/_resource_clients/dataset.py
+++ b/src/apify_client/_resource_clients/dataset.py
@@ -40,7 +40,7 @@ class DatasetItemsPage:
"""The offset of the first item in this page."""
count: int
- """Number of items in this page."""
+ """Number of dataset rows the API scanned for this page, or the number of items returned when that is larger."""
limit: int
"""The limit that was used for this request."""
@@ -204,8 +204,8 @@ def list_items(
items=items,
total=int(response.headers['x-apify-pagination-total']),
offset=int(response.headers['x-apify-pagination-offset']),
- # x-apify-pagination-count returns count of processed items, not count of returned items
- # This makes difference when items were filtered using hidden/empty
+ # The header counts the rows the API scanned, which `unwind` and a lagging dataset item count can
+ # both leave below the number of items returned.
count=max(int(response.headers['x-apify-pagination-count']), len(items)),
# API returns 999999999999 when no limit is used
limit=int(response.headers['x-apify-pagination-limit']),
@@ -237,7 +237,8 @@ def iterate_items(
Args:
offset: Number of items that should be skipped at the start. The default value is 0.
- limit: Maximum number of items to return. By default there is no limit.
+ limit: Maximum number of dataset rows to scan. Fewer items are yielded when filters drop some, more
+ when `unwind` splits a row into several. By default there is no limit.
desc: By default, results are returned in the same order as they were stored. To reverse the order,
set this parameter to True.
clean: If True, returns only non-empty items and skips hidden fields (i.e. fields starting with
@@ -260,7 +261,7 @@ def iterate_items(
skip_hidden: If True, then hidden fields are skipped from the output, i.e. fields starting with
the # character.
signature: Signature used to access the items.
- chunk_size: Maximum number of items requested per API call when iterating across pages.
+ chunk_size: Maximum number of dataset rows requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.
Yields:
@@ -763,8 +764,8 @@ async def list_items(
items=items,
total=int(response.headers['x-apify-pagination-total']),
offset=int(response.headers['x-apify-pagination-offset']),
- # x-apify-pagination-count returns count of processed items, not count of returned items
- # This makes difference when items were filtered using hidden/empty
+ # The header counts the rows the API scanned, which `unwind` and a lagging dataset item count can
+ # both leave below the number of items returned.
count=max(int(response.headers['x-apify-pagination-count']), len(items)),
# API returns 999999999999 when no limit is used
limit=int(response.headers['x-apify-pagination-limit']),
@@ -796,7 +797,8 @@ def iterate_items(
Args:
offset: Number of items that should be skipped at the start. The default value is 0.
- limit: Maximum number of items to return. By default there is no limit.
+ limit: Maximum number of dataset rows to scan. Fewer items are yielded when filters drop some, more
+ when `unwind` splits a row into several. By default there is no limit.
desc: By default, results are returned in the same order as they were stored. To reverse the order,
set this parameter to True.
clean: If True, returns only non-empty items and skips hidden fields (i.e. fields starting with
@@ -819,7 +821,7 @@ def iterate_items(
skip_hidden: If True, then hidden fields are skipped from the output, i.e. fields starting with
the # character.
signature: Signature used to access the items.
- chunk_size: Maximum number of items requested per API call when iterating across pages.
+ chunk_size: Maximum number of dataset rows requested per API call when iterating across pages.
timeout: Timeout for the API HTTP request.
Yields:
diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py
index 333c7229..79fc5816 100644
--- a/tests/integration/test_dataset.py
+++ b/tests/integration/test_dataset.py
@@ -596,6 +596,47 @@ async def get_items() -> DatasetItemsPage:
await maybe_await(dataset_client.delete())
+async def test_dataset_iterate_items_unwound(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None:
+ """Test iterate_items with `unwind`, where a page carries more items than the rows it scanned."""
+ dataset_name = get_random_resource_name('dataset')
+ created_dataset = await maybe_await(client.datasets().get_or_create(name=dataset_name))
+ assert isinstance(created_dataset, Dataset)
+ dataset_client = client.dataset(created_dataset.id)
+
+ try:
+ items_to_push = [{'idx': i, 'parts': [{'part': p} for p in range(3)]} for i in range(12)]
+ await maybe_await(dataset_client.push_items(items_to_push))
+
+ # Poll until all 12 rows are visible (eventual consistency) so the chunked iteration sees every page
+ async def get_items() -> DatasetItemsPage:
+ page = await maybe_await(dataset_client.list_items(limit=12))
+ assert isinstance(page, DatasetItemsPage)
+ return page
+
+ await poll_until_condition(get_items, lambda page: len(page.items) == 12)
+
+ # chunk_size=5 caps a page at 5 rows, which `unwind` expands into 15 items
+ iterator = dataset_client.iterate_items(unwind=['parts'], chunk_size=5)
+ collected: list[dict] = []
+ if is_async:
+ assert isinstance(iterator, AsyncIterator)
+ async for item in iterator:
+ assert isinstance(item, dict)
+ collected.append(item)
+ else:
+ assert isinstance(iterator, Iterator)
+ for item in iterator:
+ assert isinstance(item, dict)
+ collected.append(item)
+
+ # Every part of every row arrives exactly once: no page is skipped and none is read twice.
+ assert sorted((item['idx'], item['part']) for item in collected) == [
+ (idx, part) for idx in range(12) for part in range(3)
+ ]
+ finally:
+ await maybe_await(dataset_client.delete())
+
+
async def test_dataset_iterate_items_with_fields(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None:
"""Test iterate_items with `fields` filter."""
dataset_name = get_random_resource_name('dataset')
diff --git a/tests/unit/test_client_pagination.py b/tests/unit/test_client_pagination.py
index 63c9b7c1..4b99ed11 100644
--- a/tests/unit/test_client_pagination.py
+++ b/tests/unit/test_client_pagination.py
@@ -100,6 +100,7 @@
NORMAL_ITEMS = 2500
EXTRA_ITEMS_UNNAMED = 100
MAX_ITEMS_PER_PAGE = 1000
+UNWIND_PARTS = 3
# Inner list models whose `items: list[]` is relaxed to `list[dict]`. Point of these tests is
# pagination mechanism, not internal object validation.
@@ -179,6 +180,11 @@ def create_items(start: int, end: int, step: int | None = None) -> list[dict[str
return [{'id': i} for i in range(start, end, step)]
+def create_unwound_items(start: int, end: int) -> list[dict[str, int]]:
+ """Create the items the simulated `unwind` produces for the given index range."""
+ return [{**item, 'part': part} for item in create_items(start, end) for part in range(UNWIND_PARTS)]
+
+
def _is_true(value: str | None) -> bool:
"""Match the `'true'` wire form produced by the client's bool->string serialization."""
return value == 'true'
@@ -191,9 +197,14 @@ def _parse_int_param(value: str | None) -> int:
def _handle_offset_pagination(request: Request) -> Response:
"""Serve an offset-paginated Apify API response.
- The simulated platform holds 2500 items normally and an additional 100 when `unnamed=true` is requested. Pages are
- capped at 1000 items regardless of the requested limit, mirroring the real API. The dataset items endpoint returns
- items as a raw list; all other endpoints wrap them in `{'data': {...}}`.
+ The simulated platform holds 2500 items normally and an additional 100 when `unnamed=true` is requested. The
+ collection endpoints cap a page at 1000 items regardless of the requested limit, mirroring the real API, while the
+ dataset items endpoint applies the requested limit verbatim and returns its items as a raw list; all other
+ endpoints wrap them in `{'data': {...}}`.
+
+ The `x-apify-pagination-count` header reports the rows the API scanned, which `offset` and `limit` pick before the
+ result is shaped: the filters drop items from the page and `unwind` multiplies them, so `len(items)` lands below or
+ above the header.
"""
params = request.args
@@ -206,15 +217,22 @@ def _handle_offset_pagination(request: Request) -> Response:
desc = _is_true(params.get('desc'))
items = create_items(total_items, 0) if desc else create_items(0, total_items)
+ is_dataset_items = request.path.endswith(f'/datasets/{ID_PLACEHOLDER}/items')
+ page_size = total_items if is_dataset_items else MAX_ITEMS_PER_PAGE
+
lower_index = min(offset, total_items)
upper_index = min(offset + (limit or total_items), total_items)
- count = min(max(upper_index - lower_index, 0), MAX_ITEMS_PER_PAGE)
- selected_items = items[lower_index : min(upper_index, lower_index + MAX_ITEMS_PER_PAGE)]
+ count = min(max(upper_index - lower_index, 0), page_size)
+ selected_items = items[lower_index : min(upper_index, lower_index + page_size)]
# Every second item is filtered out when `skipEmpty=true`, `skipHidden=true`, or `clean=true`.
if _is_true(params.get('skipEmpty')) or _is_true(params.get('skipHidden')) or _is_true(params.get('clean')):
selected_items = selected_items[::2]
+ # `unwind` splits each item into `UNWIND_PARTS` records, so the page carries more items than the rows it scanned.
+ if params.get('unwind'):
+ selected_items = [{**item, 'part': part} for item in selected_items for part in range(UNWIND_PARTS)]
+
headers = {
'x-apify-pagination-count': str(count),
'x-apify-pagination-total': str(total_items),
@@ -224,7 +242,7 @@ def _handle_offset_pagination(request: Request) -> Response:
'content-type': 'application/json',
}
- if request.path.endswith(f'/datasets/{ID_PLACEHOLDER}/items'):
+ if is_dataset_items:
body: Any = selected_items
else:
body = {
@@ -443,6 +461,25 @@ def __hash__(self) -> int:
create_items(0, 1500, 2),
DATASET_CLIENTS,
),
+ _PaginationCase(
+ 'Unwind',
+ {'unwind': ['parts']},
+ create_unwound_items(0, 2500),
+ DATASET_CLIENTS,
+ ),
+ _PaginationCase(
+ 'Unwind, limit, chunk_size',
+ # `limit` counts the rows the API scans, matching a single `list_items` call, so `unwind` yields more items.
+ {'unwind': ['parts'], 'limit': 150, 'chunk_size': 100},
+ create_unwound_items(0, 150),
+ DATASET_CLIENTS,
+ ),
+ _PaginationCase(
+ 'Unwind, chunk_size above 1000',
+ {'unwind': ['parts'], 'chunk_size': 2000},
+ create_unwound_items(0, 2500),
+ DATASET_CLIENTS,
+ ),
_PaginationCase(
'Exclusive start key',
{'exclusive_start_key': '1000'},
@@ -629,7 +666,7 @@ async def test_rq_list_requests_iterable_async(
class FakeOffsetPage:
- """Offset-paginated page whose `count` (items scanned) may exceed `len(items)` when filters drop items."""
+ """Offset-paginated page whose `count` (rows scanned) and `len(items)` can differ in either direction."""
def __init__(self, items: list[dict[str, int]], count: int) -> None:
self.items = items
@@ -662,8 +699,62 @@ async def callback(*, offset: int | None = None, **_kwargs: object) -> FakeOffse
assert [item async for item in get_items_iterator_async(callback, chunk_size=1000)] == [{'id': 1}, {'id': 2}]
-def test_cursor_iterator_continues_past_fully_filtered_page() -> None:
- """A fully-filtered page (`items=[]`) with a live cursor must not stop the cursor iterator."""
+def test_items_iterator_advances_by_items_when_count_lags() -> None:
+ """A `count` lagging behind the items returned (`count=0`) must still advance the offset iterator."""
+ pages = {
+ 0: FakeOffsetPage(items=create_items(0, 1000), count=0),
+ 1000: FakeOffsetPage(items=create_items(1000, 1500), count=0),
+ }
+
+ def callback(*, offset: int | None = None, **_kwargs: object) -> FakeOffsetPage:
+ return pages.get(offset or 0, FakeOffsetPage(items=[], count=0))
+
+ assert list(get_items_iterator(callback, chunk_size=1000)) == create_items(0, 1500)
+
+
+async def test_items_iterator_async_advances_by_items_when_count_lags() -> None:
+ """A `count` lagging behind the items returned (`count=0`) must still advance the async offset iterator."""
+ pages = {
+ 0: FakeOffsetPage(items=create_items(0, 1000), count=0),
+ 1000: FakeOffsetPage(items=create_items(1000, 1500), count=0),
+ }
+
+ async def callback(*, offset: int | None = None, **_kwargs: object) -> FakeOffsetPage:
+ return pages.get(offset or 0, FakeOffsetPage(items=[], count=0))
+
+ collected = [item async for item in get_items_iterator_async(callback, chunk_size=1000)]
+ assert collected == create_items(0, 1500)
+
+
+def test_items_iterator_advances_by_scanned_rows_when_unwind_inflates_items() -> None:
+ """An unwound page holding more items than the rows it scanned must advance the offset by the rows alone."""
+ pages = {
+ 0: FakeOffsetPage(items=create_unwound_items(0, 1000), count=1000),
+ 1000: FakeOffsetPage(items=create_unwound_items(1000, 1500), count=500),
+ }
+
+ def callback(*, offset: int | None = None, **_kwargs: object) -> FakeOffsetPage:
+ return pages.get(offset or 0, FakeOffsetPage(items=[], count=0))
+
+ assert list(get_items_iterator(callback, chunk_size=1000)) == create_unwound_items(0, 1500)
+
+
+async def test_items_iterator_async_advances_by_scanned_rows_when_unwind_inflates_items() -> None:
+ """An unwound page holding more items than the rows it scanned must advance the async offset iterator by rows."""
+ pages = {
+ 0: FakeOffsetPage(items=create_unwound_items(0, 1000), count=1000),
+ 1000: FakeOffsetPage(items=create_unwound_items(1000, 1500), count=500),
+ }
+
+ async def callback(*, offset: int | None = None, **_kwargs: object) -> FakeOffsetPage:
+ return pages.get(offset or 0, FakeOffsetPage(items=[], count=0))
+
+ collected = [item async for item in get_items_iterator_async(callback, chunk_size=1000)]
+ assert collected == create_unwound_items(0, 1500)
+
+
+def test_cursor_iterator_continues_past_empty_page() -> None:
+ """An empty page with a live cursor must not stop the cursor iterator."""
pages = {
None: ListOfRequests(items=[], limit=1000, next_cursor='c1'),
'c1': ListOfRequests(items=[{'id': 1}, {'id': 2}], limit=1000, next_cursor=None),
@@ -675,8 +766,8 @@ def callback(*, cursor: str | None = None, **_kwargs: object) -> ListOfRequests:
assert list(get_cursor_iterator(callback, chunk_size=1000)) == [{'id': 1}, {'id': 2}]
-async def test_cursor_iterator_async_continues_past_fully_filtered_page() -> None:
- """A fully-filtered page (`items=[]`) with a live cursor must not stop the async cursor iterator."""
+async def test_cursor_iterator_async_continues_past_empty_page() -> None:
+ """An empty page with a live cursor must not stop the async cursor iterator."""
pages = {
None: ListOfRequests(items=[], limit=1000, next_cursor='c1'),
'c1': ListOfRequests(items=[{'id': 1}, {'id': 2}], limit=1000, next_cursor=None),