push based Vayu Execution Engine with multiple pipelines support (#4)

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "arrow-datafusion"]
+	path = arrow-datafusion
+	url = [email protected]:yashkothari42/arrow-datafusion.git

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "rust-analyzer.linkedProjects": [
+        "./Cargo.toml"
+    ]
+}

Cargo.toml

@@ -1,3 +1,7 @@
+[workspace] 
+members = ["datafusion-integration", "lighting", "vayu"] 
+exclude = ["arrow-datafusion"]
+
 [package]
 name = "ee2"
 version = "0.1.0"
@@ -6,7 +10,5 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-substrait = "0.24.1"
-prost = "0.12"
-prost-types = "0.12"
-protoc-rust = "2.28.0"
+arrow = { version = "50.0.0"}
+tokio = { version = "1.4.0", features = ["rt", "rt-multi-thread", "macros"] }

datafusion-integration/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "datafusion_integration"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+arrow = { version = "50.0.0"}
+tokio = { version = "1.4.0", features = ["rt", "rt-multi-thread", "macros"] }
+vayu = { path = "../vayu", version = "0.1.0"}
+datafusion = { version = "36.0.0", path = "../arrow-datafusion/datafusion/core"}
+datafusion-proto = { version = "36.0.0", path = "../arrow-datafusion/datafusion/proto"}
+ahash = { version = "0.8", default-features = false, features = [
+    "runtime-rng",
+] }