From b043df07940e1c38dd571fe26ee7c17a235f86b9 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 2 Aug 2026 11:08:03 -0700 Subject: [PATCH 1/4] Default writes to merge append --- mkdocs/docs/api.md | 4 ++-- mkdocs/docs/configuration.md | 6 +++--- pyiceberg/table/__init__.py | 2 +- tests/integration/test_writes/test_writes.py | 6 +++++- tests/table/test_snapshots.py | 19 +++++++++++++++++++ 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index 0a2a72b6f0..e3fb75f5e3 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -311,8 +311,8 @@ Next, write the data to the table. Both `append` and `overwrite` produce the sam -!!! note inline end "Fast append" - PyIceberg defaults to the [fast append](https://iceberg.apache.org/spec/#snapshots) to minimize the amount of data written. This enables fast commit operations, reducing the possibility of conflicts. The downside of the fast append is that it creates more metadata than a merge commit. [Compaction is planned](https://github.com/apache/iceberg-python/issues/270) and will automatically rewrite all the metadata when a threshold is hit, to maintain performant reads. +!!! note inline end "Merge append" + PyIceberg defaults to [merge append](https://iceberg.apache.org/spec/#snapshots), which automatically merges manifests when the configured threshold is reached. To minimize metadata written during a commit, set the table property `commit.manifest-merge.enabled` to `False` to use fast append instead. diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 44b5e395a9..ccc8653693 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -99,12 +99,12 @@ Iceberg tables support table properties to configure table behavior. | ------------------------------------ | ------------------- | ------------- | ----------------------------------------------------------- | | `commit.manifest.target-size-bytes` | Size in bytes | 8388608 (8MB) | Target size when merging manifest files | | `commit.manifest.min-count-to-merge` | Number of manifests | 100 | Minimum number of manifests to accumulate before merging | -| `commit.manifest-merge.enabled` | Boolean | False | Controls whether to automatically merge manifests on writes | +| `commit.manifest-merge.enabled` | Boolean | True | Controls whether to automatically merge manifests on writes | -!!! note "Fast append" - Unlike Java implementation, PyIceberg default to the [fast append](api.md#write-to-a-table) and thus `commit.manifest-merge.enabled` is set to `False` by default. +!!! note "Append mode" + PyIceberg defaults to [merge append](api.md#write-to-a-table). Set `commit.manifest-merge.enabled` to `False` on a table to use fast append instead. diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index c0adce84dc..0e344c22c7 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -198,7 +198,7 @@ class TableProperties: MANIFEST_MIN_MERGE_COUNT_DEFAULT = 100 MANIFEST_MERGE_ENABLED = "commit.manifest-merge.enabled" - MANIFEST_MERGE_ENABLED_DEFAULT = False + MANIFEST_MERGE_ENABLED_DEFAULT = True METADATA_PREVIOUS_VERSIONS_MAX = "write.metadata.previous-versions-max" METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT = 100 diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 30fdd76ab7..09affbdf13 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -1588,7 +1588,11 @@ def test_merge_manifests(session_catalog: Catalog, arrow_table_with_null: pa.Tab tbl_c = _create_table( session_catalog, "default.merge_manifest_c", - {"commit.manifest.min-count-to-merge": "1", "format-version": format_version}, + { + "commit.manifest-merge.enabled": "false", + "commit.manifest.min-count-to-merge": "1", + "format-version": format_version, + }, [], ) diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 5f1680ed59..d813665e2d 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -649,3 +649,22 @@ def summary_calls(n_files: int) -> int: f"_MergeAppendFiles.__init__ made {merge_init - fast_init} extra update_table_metadata " "calls over its superclass; expected 1 (hoisted)" ) + + +def test_append_snapshot_producer_defaults_to_merge_append(table_v2: Table) -> None: + from pyiceberg.table.update.snapshot import _MergeAppendFiles + + append = table_v2.transaction()._append_snapshot_producer({}) + + assert type(append) is _MergeAppendFiles + + +def test_append_snapshot_producer_uses_fast_append_when_manifest_merge_disabled(table_v2: Table) -> None: + from pyiceberg.table.update.snapshot import _FastAppendFiles + + transaction = table_v2.transaction() + transaction.set_properties({"commit.manifest-merge.enabled": "false"}) + + append = transaction._append_snapshot_producer({}) + + assert type(append) is _FastAppendFiles From eb107aadbe6c009db32a19b3537234ddbac9adde Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 2 Aug 2026 11:13:03 -0700 Subject: [PATCH 2/4] Document fast append tradeoffs --- mkdocs/docs/api.md | 4 ++-- mkdocs/docs/configuration.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index e3fb75f5e3..6faf98874c 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -311,8 +311,8 @@ Next, write the data to the table. Both `append` and `overwrite` produce the sam -!!! note inline end "Merge append" - PyIceberg defaults to [merge append](https://iceberg.apache.org/spec/#snapshots), which automatically merges manifests when the configured threshold is reached. To minimize metadata written during a commit, set the table property `commit.manifest-merge.enabled` to `False` to use fast append instead. +!!! note inline end "Merge and fast append" + PyIceberg defaults to [merge append](https://iceberg.apache.org/spec/#snapshots), which automatically merges manifests when the configured threshold is reached to keep metadata compact and reads performant. Fast append remains available by setting the table property `commit.manifest-merge.enabled` to `False`. Fast append minimizes the metadata written during a commit, enabling faster commits and reducing the possibility of conflicts, but accumulates more manifest metadata over time. diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index ccc8653693..98b3bf856c 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -103,8 +103,8 @@ Iceberg tables support table properties to configure table behavior. -!!! note "Append mode" - PyIceberg defaults to [merge append](api.md#write-to-a-table). Set `commit.manifest-merge.enabled` to `False` on a table to use fast append instead. +!!! note "Append modes" + PyIceberg defaults to [merge append](api.md#write-to-a-table) to keep manifest metadata compact. Set `commit.manifest-merge.enabled` to `False` on a table to use fast append, which writes less metadata during each commit but accumulates more manifests over time. From 049389809cd7e4495586f4b2957494d1e827a773 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 2 Aug 2026 12:19:43 -0700 Subject: [PATCH 3/4] Fix order-dependent partition overwrite assertion --- tests/integration/test_writes/test_partitioned_writes.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_writes/test_partitioned_writes.py b/tests/integration/test_writes/test_partitioned_writes.py index 1d1488255f..040c0cfeea 100644 --- a/tests/integration/test_writes/test_partitioned_writes.py +++ b/tests/integration/test_writes/test_partitioned_writes.py @@ -748,8 +748,11 @@ def test_dynamic_partition_overwrite_evolve_partition(spark: SparkSession, sessi tbl.dynamic_partition_overwrite(arrow_table) result = tbl.scan().to_arrow() - assert result["place"].to_pylist() == ["Groningen", "Amsterdam", "Drachten"] - assert result["inhabitants"].to_pylist() == [238147, 921402, 44940] + assert sorted(zip(result["place"].to_pylist(), result["inhabitants"].to_pylist(), strict=True)) == [ + ("Amsterdam", 921402), + ("Drachten", 44940), + ("Groningen", 238147), + ] @pytest.mark.integration From e63ec7d94424c9828ada321643ab6c6cfa586a70 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Sun, 2 Aug 2026 14:24:26 -0700 Subject: [PATCH 4/4] Test merge append default end to end --- tests/integration/test_writes/test_writes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index 09affbdf13..553a9ac574 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -1571,7 +1571,7 @@ def test_merge_manifests(session_catalog: Catalog, arrow_table_with_null: pa.Tab tbl_a = _create_table( session_catalog, "default.merge_manifest_a", - {"commit.manifest-merge.enabled": "true", "commit.manifest.min-count-to-merge": "1", "format-version": format_version}, + {"commit.manifest.min-count-to-merge": "1", "format-version": format_version}, [], ) tbl_b = _create_table(