From 794a4fd868ea4d5733f356b741a34bca27442721 Mon Sep 17 00:00:00 2001 From: rong fengliang <1141591465@qq.com> Date: Fri, 29 Nov 2024 04:09:56 +0800 Subject: [PATCH] [FEAT] add pyiceberg 0.8.0 support (#3448) add pyiceberg 0.8.0 check (from 0.8.0 the PropertyUtil class has been moved to a separate file located at utils/properties.py ) --- daft/dataframe/dataframe.py | 13 +++++++++++-- pyproject.toml | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/daft/dataframe/dataframe.py b/daft/dataframe/dataframe.py index 77528da220..1ec5668b2e 100644 --- a/daft/dataframe/dataframe.py +++ b/daft/dataframe/dataframe.py @@ -728,7 +728,16 @@ def write_iceberg(self, table: "pyiceberg.table.Table", mode: str = "append") -> partitioning[field].append(getattr(data_file.partition, field, None)) if parse(pyiceberg.__version__) >= parse("0.7.0"): - from pyiceberg.table import ALWAYS_TRUE, PropertyUtil, TableProperties + from pyiceberg.table import ALWAYS_TRUE, TableProperties + + if parse(pyiceberg.__version__) >= parse("0.8.0"): + from pyiceberg.utils.properties import property_as_bool + + property_as_bool = property_as_bool + else: + from pyiceberg.table import PropertyUtil + + property_as_bool = PropertyUtil.property_as_bool tx = table.transaction() @@ -737,7 +746,7 @@ def write_iceberg(self, table: "pyiceberg.table.Table", mode: str = "append") -> update_snapshot = tx.update_snapshot() - manifest_merge_enabled = mode == "append" and PropertyUtil.property_as_bool( + manifest_merge_enabled = mode == "append" and property_as_bool( tx.table_metadata.properties, TableProperties.MANIFEST_MERGE_ENABLED, TableProperties.MANIFEST_MERGE_ENABLED_DEFAULT, diff --git a/pyproject.toml b/pyproject.toml index 99987e4d37..3759a0c41a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,7 @@ azure = [] deltalake = ["deltalake"] gcp = [] hudi = ["pyarrow >= 8.0.0"] -iceberg = ["pyiceberg >= 0.4.0, < 0.8", "packaging"] +iceberg = ["pyiceberg >= 0.4.0", "packaging"] lance = ["lancedb"] numpy = ["numpy"] pandas = ["pandas"]