From 9ed9b3f4dc42965310080ae682eb9f7337b22fa7 Mon Sep 17 00:00:00 2001 From: Jay Chia <17691182+jaychia@users.noreply.github.com> Date: Tue, 5 Mar 2024 12:17:22 -0800 Subject: [PATCH] [CHORE] Disables anonymous mode for S3 accesses in DeltaLake (#1975) DeltaLake SDK does not support anonymous mode access (see: issue https://github.com/delta-io/delta-rs/issues/1554) We throw an error if a user attempts to supply `anonymous=True`. --------- Co-authored-by: Jay Chia --- daft/delta_lake/delta_lake_scan.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/daft/delta_lake/delta_lake_scan.py b/daft/delta_lake/delta_lake_scan.py index 90fe2c9215..3a3dd873b4 100644 --- a/daft/delta_lake/delta_lake_scan.py +++ b/daft/delta_lake/delta_lake_scan.py @@ -206,6 +206,10 @@ def _s3_config_to_storage_options(s3_config: S3Config) -> dict[str, str]: storage_options["allow_invalid_certificates"] = "false" if s3_config.verify_ssl else "true" if s3_config.connect_timeout_ms is not None: storage_options["connect_timeout"] = str(s3_config.connect_timeout_ms) + "ms" + if s3_config.anonymous: + raise ValueError( + "Reading from DeltaLake does not support anonymous mode! Please supply credentials via your S3Config." + ) return storage_options