Iceberg write support

splitgraph · Dec 11, 2024 · 9234f7f · 9234f7f
1 parent 3817af1
commit 9234f7f
Show file tree

Hide file tree

Showing 17 changed files with 644 additions and 795 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -21,8 +21,8 @@ datafusion-functions-nested = "43.0.0"
 
 futures = "0.3"
 
-iceberg = { git = "https://github.com/splitgraph/iceberg-rust", rev = "dbe5858039e676b45329a5cfc46cc1e7d9b98402" }
-iceberg-datafusion = { git = "https://github.com/splitgraph/iceberg-rust", rev = "dbe5858039e676b45329a5cfc46cc1e7d9b98402" }
+iceberg = { git = "https://github.com/splitgraph/iceberg-rust", rev = "6e87893e733379558df597aa7d66f026549e3b0c" }
+iceberg-datafusion = { git = "https://github.com/splitgraph/iceberg-rust", rev = "6e87893e733379558df597aa7d66f026549e3b0c" }
 
 itertools = ">=0.10.0"
 object_store = { version = "0.11", features = ["aws", "azure", "gcp"] }
@@ -100,6 +100,7 @@ datafusion-functions-nested = { workspace = true }
 datafusion-remote-tables = { path = "./datafusion_remote_tables", optional = true }
 
 deltalake = { git = "https://github.com/splitgraph/delta-rs", rev = "eff5735698279c12ae4a3aac2afa268d168242b2", features = ["datafusion", "s3"] }
+fastrand = "2.2.0"
 
 futures = "0.3"
 hex = ">=0.4.0"
@@ -115,6 +116,8 @@ metrics-exporter-prometheus = { version = "0.15.3" }
 moka = { version = "0.12.5", default-features = false, features = ["future", "atomic64", "quanta"] }
 object_store = { workspace = true }
 object_store_factory = { path = "object_store_factory" }
+opendal = { version = "0.50" }
+parquet = "53.3.0"
 percent-encoding = "2.2.0"
 prost = { workspace = true }
 
@@ -150,7 +153,7 @@ warp = "0.3.6"
 # For WASM user-defined functions
 wasi-common = "25.0.0"
 wasmtime = "25.0.2"
-wasmtime-wasi = "27.0.0"
+wasmtime-wasi = "26.0.1"
 
 [dev-dependencies]
 assert_cmd = "2"