diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index 5cae743313..a50759d3a0 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -65,6 +65,13 @@ def _get_snapshot(self, snapshot_id: int | None = None) -> Snapshot: else: raise ValueError("Cannot get a snapshot as the table does not have any.") + def _get_snapshots_by_id(self) -> dict[int, Snapshot]: + """Index the snapshots by ID, for methods that look up many of them. + + snapshot_by_id is a linear scan, so calling it once per row is quadratic. + """ + return {snapshot.snapshot_id: snapshot for snapshot in self.tbl.metadata.snapshots} + def snapshots(self) -> pa.Table: import pyarrow as pa @@ -310,13 +317,14 @@ def partitions( ) partitions_map: dict[tuple[str, Any], Any] = {} + snapshots_by_id = self._get_snapshots_by_id() for entry in itertools.chain.from_iterable(scan._plan_manifest_entries()): partition = entry.data_file.partition partition_record_dict = { field.name: partition[pos] for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields) } - entry_snapshot = self.tbl.snapshot_by_id(entry.snapshot_id) if entry.snapshot_id is not None else None + entry_snapshot = snapshots_by_id.get(entry.snapshot_id) if entry.snapshot_id is not None else None self._update_partitions_map_from_manifest_entry( partitions_map, entry.data_file, partition_record_dict, entry_snapshot ) @@ -532,9 +540,10 @@ def history(self) -> pa.Table: history = [] metadata = self.tbl.metadata + snapshots_by_id = self._get_snapshots_by_id() for snapshot_entry in metadata.snapshot_log: - snapshot = metadata.snapshot_by_id(snapshot_entry.snapshot_id) + snapshot = snapshots_by_id.get(snapshot_entry.snapshot_id) history.append( {