Skip to content

Commit

Permalink
[FEAT] add pyiceberg 0.8.0 support (#3448)
Browse files Browse the repository at this point in the history
add pyiceberg 0.8.0 check (from 0.8.0 the PropertyUtil class has been
moved to a separate file located at utils/properties.py )
  • Loading branch information
rongfengliang authored Nov 28, 2024
1 parent f2d4f73 commit 794a4fd
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 3 deletions.
13 changes: 11 additions & 2 deletions daft/dataframe/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -728,7 +728,16 @@ def write_iceberg(self, table: "pyiceberg.table.Table", mode: str = "append") ->
partitioning[field].append(getattr(data_file.partition, field, None))

if parse(pyiceberg.__version__) >= parse("0.7.0"):
from pyiceberg.table import ALWAYS_TRUE, PropertyUtil, TableProperties
from pyiceberg.table import ALWAYS_TRUE, TableProperties

if parse(pyiceberg.__version__) >= parse("0.8.0"):
from pyiceberg.utils.properties import property_as_bool

property_as_bool = property_as_bool
else:
from pyiceberg.table import PropertyUtil

property_as_bool = PropertyUtil.property_as_bool

tx = table.transaction()

Expand All @@ -737,7 +746,7 @@ def write_iceberg(self, table: "pyiceberg.table.Table", mode: str = "append") ->

update_snapshot = tx.update_snapshot()

manifest_merge_enabled = mode == "append" and PropertyUtil.property_as_bool(
manifest_merge_enabled = mode == "append" and property_as_bool(
tx.table_metadata.properties,
TableProperties.MANIFEST_MERGE_ENABLED,
TableProperties.MANIFEST_MERGE_ENABLED_DEFAULT,
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ azure = []
deltalake = ["deltalake"]
gcp = []
hudi = ["pyarrow >= 8.0.0"]
iceberg = ["pyiceberg >= 0.4.0, < 0.8", "packaging"]
iceberg = ["pyiceberg >= 0.4.0", "packaging"]
lance = ["lancedb"]
numpy = ["numpy"]
pandas = ["pandas"]
Expand Down

0 comments on commit 794a4fd

Please sign in to comment.