size=None for http implementation

Eventual-Inc · Sep 22, 2023 · 4db517b · 4db517b
1 parent 60eaaf3
commit 4db517b
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/src/daft-io/src/http.rs b/src/daft-io/src/http.rs
@@ -112,7 +112,9 @@ fn _get_file_metadata_from_html(path: &str, text: &str) -> super::Result<Vec<Fil
             };
             Ok(Some(FileMetadata {
                 filepath: absolute_path,
-                size: None, // TODO: fire HEAD requests to grab the content-length headers
+                // NOTE: This is consistent with fsspec behavior, but we may choose to HEAD the files to grab Content-Length
+                // for populating `size` if necessary
+                size: None,
                 filetype,
             }))
         })

diff --git a/tests/integration/io/test_list_files_http.py b/tests/integration/io/test_list_files_http.py
@@ -61,7 +61,7 @@ def test_gs_single_file_listing(nginx_http_url):
     # fsspec_result = fs.ls(path, detail=True)
 
     assert len(daft_ls_result) == 1
-    assert daft_ls_result[0] == {"path": path, "size": 24, "type": "File"}
+    assert daft_ls_result[0] == {"path": path, "size": None, "type": "File"}
 
 
 @pytest.mark.integration()