Skip to content

Commit

Permalink
raise errors for non supported functions
Browse files Browse the repository at this point in the history
  • Loading branch information
samster25 committed Nov 14, 2023
1 parent 3111951 commit 8ee7e82
Showing 1 changed file with 7 additions and 0 deletions.
7 changes: 7 additions & 0 deletions daft/iceberg/iceberg_scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ def partitioning_keys(self) -> list[PartitionField]:
def _make_scan_tasks(self) -> list[ScanTask]:
iceberg_tasks = self._table.scan().plan_files()
scan_tasks = []
# TODO(sammy): multithreading should be RayRunner config?
storage_config = StorageConfig.native(NativeStorageConfig(True, self._io_config))
for task in iceberg_tasks:
file = task.file
Expand All @@ -96,8 +97,14 @@ def _make_scan_tasks(self) -> list[ScanTask]:
if file_format == "PARQUET":
file_format_config = FileFormatConfig.from_parquet_config(ParquetSourceConfig())
else:
# TODO: Support ORC and AVRO when we can read it
raise NotImplementedError(f"{file_format} for iceberg not implemented!")

if (task.delete_files) > 0:
raise NotImplementedError(f"Iceberg Merge-on-Read currently not supported, please make an issue!")

# TODO: Thread in PartitionSpec to each ScanTask: P1
# TODO: Thread in Statistics to each ScanTask: P2
st = ScanTask.catalog_scan_task(
file=path,
file_format=file_format_config,
Expand Down

0 comments on commit 8ee7e82

Please sign in to comment.