[CHORE] factor io config into common code (#1335)

* Factors out IOConfig, S3Config, etc into a common io-config crate.
* Reexports them in the daft-io crate for convenience
* sadly `daft-dsl` still depends on `daft-io` for the url_download
function so we still have to recompile quite a bit.
* This should be fixed when we allow for dynamic function registration.

Cargo.toml

@@ -52,6 +52,7 @@ inherits = "dev"
 [workspace]
 members = [
   "src/common/error",
+  "src/common/io-config",
   "src/daft-core",
   "src/daft-io",
   "src/daft-parquet",

src/common/io-config/Cargo.toml

@@ -0,0 +1,14 @@
+[dependencies]
+common-error = {path = "../error", default-features = false}
+pyo3 = {workspace = true, optional = true}
+serde = {workspace = true}
+serde_json = {workspace = true}
+
+[features]
+default = ["python"]
+python = ["dep:pyo3", "common-error/python"]
+
+[package]
+edition = {workspace = true}
+name = "common-io-config"
+version = {workspace = true}

src/common/io-config/src/azure.rs

@@ -0,0 +1,25 @@
+use std::fmt::Display;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
+pub struct AzureConfig {
+    pub storage_account: Option<String>,
+    pub access_key: Option<String>,
+    pub anonymous: bool,
+}
+
+impl Display for AzureConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
+        write!(
+            f,
+            "AzureConfig
+    storage_account: {:?}
+    access_key: {:?}
+    anonymous: {:?}",
+            self.storage_account, self.access_key, self.anonymous
+        )
+    }
+}

src/common/io-config/src/config.rs

@@ -0,0 +1,26 @@
+use std::fmt::Display;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::{AzureConfig, GCSConfig, S3Config};
+#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
+pub struct IOConfig {
+    pub s3: S3Config,
+    pub azure: AzureConfig,
+    pub gcs: GCSConfig,
+}
+
+impl Display for IOConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
+        write!(
+            f,
+            "IOConfig:
+{}
+{}
+{}",
+            self.s3, self.azure, self.gcs
+        )
+    }
+}

src/common/io-config/src/gcs.rs

@@ -0,0 +1,23 @@
+use std::fmt::Display;
+use std::fmt::Formatter;
+
+use serde::Deserialize;
+use serde::Serialize;
+
+#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)]
+pub struct GCSConfig {
+    pub project_id: Option<String>,
+    pub anonymous: bool,
+}
+
+impl Display for GCSConfig {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
+        write!(
+            f,
+            "GCSConfig
+    project_id: {:?}
+    anonymous: {:?}",
+            self.project_id, self.anonymous
+        )
+    }
+}

src/common/io-config/src/lib.rs

@@ -0,0 +1,9 @@
+#[cfg(feature = "python")]
+pub mod python;
+
+mod azure;
+mod config;
+mod gcs;
+mod s3;
+
+pub use crate::{azure::AzureConfig, config::IOConfig, gcs::GCSConfig, s3::S3Config};