merged

Eventual-Inc · Feb 20, 2024 · dc8c015 · dc8c015
2 parents 552a973 + 9674214
commit dc8c015
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 3 deletions.
diff --git a/src/daft-io/src/http.rs b/src/daft-io/src/http.rs
@@ -152,6 +152,7 @@ impl From<Error> for super::Error {
                     source: source.into(),
                 },
             },
+            UnableToDetermineSize { path } => super::Error::UnableToDetermineSize { path },
             _ => super::Error::Generic {
                 store: super::SourceType::Http,
                 source: error.into(),

diff --git a/src/daft-io/src/lib.rs b/src/daft-io/src/lib.rs
@@ -72,6 +72,9 @@ pub enum Error {
     #[snafu(display("Not a File: \"{}\"", path))]
     NotAFile { path: String },
 
+    #[snafu(display("Unable to determine size of {}", path))]
+    UnableToDetermineSize { path: String },
+
     #[snafu(display("Unable to load Credentials for store: {store}\nDetails:\n{source:?}"))]
     UnableToLoadCredentials { store: SourceType, source: DynError },
 

diff --git a/src/daft-io/src/object_store_glob.rs b/src/daft-io/src/object_store_glob.rs
@@ -355,7 +355,7 @@ pub(crate) async fn glob(
                 let maybe_size = source.get_size(&glob, io_stats.clone()).await;
                 match maybe_size {
                     Ok(size_bytes) => yield Ok(FileMetadata{filepath: glob.clone(), size: Some(size_bytes as u64), filetype: FileType::File  }),
-                    Err(crate::Error::NotAFile {..} | crate::Error::NotFound { .. }) => {attempt_as_dir = true;},
+                    Err(crate::Error::NotAFile {..} | crate::Error::NotFound { .. } | crate::Error::UnableToDetermineSize { .. }) => {attempt_as_dir = true;},
                     Err(err) => yield Err(err),
                 }
             }

diff --git a/tests/integration/io/test_list_files_http.py b/tests/integration/io/test_list_files_http.py
@@ -103,7 +103,7 @@ def test_http_listing_absolute_urls(nginx_config, tmpdir):
     )
 
     with mount_data_nginx(nginx_config, tmpdir):
-        http_path = f"{nginx_http_url}/index.html"
+        http_path = f"{nginx_http_url}/"
         daft_ls_result = io_glob(http_path)
 
         # NOTE: Cannot use fsspec here because they do not correctly find the links
@@ -129,7 +129,7 @@ def test_http_listing_absolute_base_urls(nginx_config, tmpdir):
     )
 
     with mount_data_nginx(nginx_config, tmpdir):
-        http_path = f"{nginx_http_url}/index.html"
+        http_path = f"{nginx_http_url}/"
         daft_ls_result = io_glob(http_path)
 
         # NOTE: Cannot use fsspec here because they do not correctly find the links