fix: renterd uploads new intermediate directories

.changeset/sweet-fans-suffer.md

@@ -0,0 +1,5 @@
+---
+'renterd': minor
+---
+
+The file explorer now shows new directories right away when uploading nested files.

apps/renterd-e2e/src/fixtures/files.ts

@@ -12,7 +12,10 @@ export async function deleteFile(page: Page, path: string) {
 }
 
 export async function deleteFileIfExists(page: Page, path: string) {
-  const exists = await page.getByRole('table').getByTestId(path).isVisible()
+  const exists = await page
+    .getByTestId('filesTable')
+    .getByTestId(path)
+    .isVisible()
   if (exists) {
     await deleteFile(page, path)
   }
@@ -34,7 +37,10 @@ export async function deleteDirectory(page: Page, path: string) {
 }
 
 export async function deleteDirectoryIfExists(page: Page, path: string) {
-  const exists = await page.getByRole('table').getByTestId(path).isVisible()
+  const exists = await page
+    .getByTestId('filesTable')
+    .getByTestId(path)
+    .isVisible()
   if (exists) {
     await deleteDirectory(page, path)
   }
@@ -56,12 +62,16 @@ export async function openFileContextMenu(page: Page, path: string) {
 }
 
 export async function openDirectory(page: Page, path: string) {
-  await page.getByRole('table').getByTestId(path).click()
+  await page.getByTestId('filesTable').getByTestId(path).click()
   for (const dir of path.split('/').slice(0, -1)) {
     await expect(page.getByTestId('navbar').getByText(dir)).toBeVisible()
   }
 }
 
+export async function navigateToParentDirectory(page: Page) {
+  await page.getByRole('cell', { name: '..' }).click()
+}
+
 export async function crateDirectory(page: Page, name: string) {
   await expect(page.getByLabel('Create directory')).toBeVisible()
   await page.getByLabel('Create directory').click()
@@ -71,18 +81,25 @@ export async function crateDirectory(page: Page, name: string) {
 }
 
 export async function createDirectoryIfNotExists(page: Page, name: string) {
-  const exists = await page.getByRole('table').getByTestId(name).isVisible()
+  const exists = await page
+    .getByTestId('filesTable')
+    .getByTestId(name)
+    .isVisible()
   if (!exists) {
     await crateDirectory(page, name)
   }
 }
 
 export async function fileInList(page: Page, path: string) {
-  await expect(page.getByRole('table').getByTestId(path)).toBeVisible()
+  await expect(page.getByTestId('filesTable').getByTestId(path)).toBeVisible()
 }
 
 export async function fileNotInList(page: Page, path: string) {
-  await expect(page.getByRole('table').getByTestId(path)).toBeHidden()
+  await expect(page.getByTestId('filesTable').getByTestId(path)).toBeHidden()
+}
+
+export async function getFileRowById(page: Page, id: string) {
+  return page.getByTestId('filesTable').getByTestId(id)
 }
 
 export async function dragAndDropFile(

apps/renterd-e2e/src/specs/files.spec.ts

@@ -13,6 +13,8 @@ import {
   dragAndDropFile,
   fileInList,
   fileNotInList,
+  getFileRowById,
+  navigateToParentDirectory,
   openDirectory,
   openFileContextMenu,
 } from '../fixtures/files'
@@ -44,7 +46,7 @@ test('can create directory, upload file, rename file, navigate, delete a file, d
     page.getByText('bucket does not contain any files')
   ).toBeVisible()
 
-  // create directory
+  // Create directory.
   await createDirectoryIfNotExists(page, dirName)
   await fileInList(page, dirPath)
   await openDirectory(page, dirPath)
@@ -53,7 +55,7 @@ test('can create directory, upload file, rename file, navigate, delete a file, d
   ).toBeVisible()
   await clearToasts({ page })
 
-  // upload
+  // Upload.
   await dragAndDropFile(
     page,
     `[data-testid=filesDropzone]`,
@@ -63,20 +65,20 @@ test('can create directory, upload file, rename file, navigate, delete a file, d
   await fileInList(page, originalFilePath)
   await expect(page.getByText('100%')).toBeVisible()
 
-  // rename
+  // Rename file.
   await openFileContextMenu(page, originalFilePath)
   await page.getByRole('menuitem', { name: 'Rename file' }).click()
   await fillTextInputByName(page, 'name', 'renamed.txt')
   await page.locator('input[name=name]').press('Enter')
   await expect(page.getByRole('dialog')).toBeHidden()
   await fileInList(page, newFilePath)
 
-  // delete
+  // Delete file.
   await deleteFileIfExists(page, newFilePath)
   await fileNotInList(page, newFilePath)
   await clearToasts({ page })
 
-  // upload again
+  // Upload the file again.
   await dragAndDropFile(
     page,
     `[data-testid=filesDropzone]`,
@@ -86,15 +88,79 @@ test('can create directory, upload file, rename file, navigate, delete a file, d
   await fileInList(page, originalFilePath)
   await expect(page.getByText('100%')).toBeVisible()
 
-  // navigate back to root
-  await page.getByRole('cell', { name: '..' }).click()
+  // Clean up the directory.
+  await navigateToParentDirectory(page)
   await fileInList(page, dirPath)
-
-  // delete directory
   await deleteDirectoryIfExists(page, dirPath)
   await fileNotInList(page, dirPath)
 
-  // delete bucket
+  // Clean up the bucket.
+  await navigateToBuckets({ page })
+  await deleteBucketIfExists(page, bucketName)
+})
+
+test('shows a new intermediate directory when uploading nested files', async ({
+  page,
+}) => {
+  test.setTimeout(80_000)
+  const bucketName = 'files-test'
+  const containerDir = `test-dir-${Date.now()}`
+  const containerDirPath = `${bucketName}/${containerDir}/`
+  const systemDir = 'nested-sample'
+  const systemFile = 'sample.txt'
+  const systemFilePath = `${systemDir}/${systemFile}`
+  const dirPath = `${bucketName}/${containerDir}/${systemDir}/`
+  const filePath = `${dirPath}${systemFile}`
+
+  await navigateToBuckets({ page })
+  await deleteBucketIfExists(page, bucketName)
+  await createBucket(page, bucketName)
+  await openBucket(page, bucketName)
+  await expect(
+    page.getByText('bucket does not contain any files')
+  ).toBeVisible()
+
+  // Create a container directory for the test.
+  await createDirectoryIfNotExists(page, containerDir)
+  await fileInList(page, containerDirPath)
+  await openDirectory(page, containerDirPath)
+  await expect(
+    page.getByText('The current directory does not contain any files yet')
+  ).toBeVisible()
+  await clearToasts({ page })
+
+  // Upload a nested file.
+  await dragAndDropFile(
+    page,
+    `[data-testid=filesDropzone]`,
+    path.join(__dirname, systemFilePath),
+    '/' + systemFilePath
+  )
+  await fileInList(page, dirPath)
+  const dirRow = await getFileRowById(page, dirPath)
+  // The intermediate directory should show up before the file is finished uploading.
+  await expect(dirRow.getByText('0 B')).toBeVisible()
+  // Check that filtering the current directory works with the upload directory.
+  const filterInput = page.getByLabel('filter files in current directory')
+  await filterInput.fill(systemDir.slice(0, 2))
+  await expect(dirRow).toBeVisible()
+  await filterInput.fill('xxxxx')
+  await expect(dirRow).toBeHidden()
+  await filterInput.clear()
+  await openDirectory(page, dirPath)
+  const fileRow = await getFileRowById(page, filePath)
+  await expect(fileRow.getByText('11 B')).toBeVisible()
+  await navigateToParentDirectory(page)
+  // The intermediate directory is now an actual directory response with the correct size.
+  await expect(dirRow.getByText('11 B')).toBeVisible()
+
+  // Clean up the container directory.
+  await navigateToParentDirectory(page)
+  await fileInList(page, containerDirPath)
+  await deleteDirectoryIfExists(page, containerDirPath)
+  await fileNotInList(page, containerDirPath)
+
+  // Clean up the bucket.
   await navigateToBuckets({ page })
   await deleteBucketIfExists(page, bucketName)
 })

apps/renterd-e2e/src/specs/nested-sample/sample.txt

@@ -0,0 +1 @@
+hello world

apps/renterd/components/Files/FilesFilterDirectoryMenu/index.tsx

@@ -41,6 +41,7 @@ export function FilesFilterDirectoryMenu({ placeholder }: Props) {
   return (
     <div className="flex gap-1 flex-1">
       <TextField
+        aria-label="filter files in current directory"
         variant="ghost"
         focus="none"
         placeholder={placeholder || 'Filter files in current directory'}

apps/renterd/contexts/filesManager/dataset.tsx

@@ -75,11 +75,53 @@ export function useDataset({ id, objects }: Props) {
             type: isDirectory(key) ? 'directory' : 'file',
           }
         })
+        // Find intermediate directories that may not exist yet and add them
+        // so that they show up before the files are fully uploaded.
+        for (const upload of uploadsList) {
+          if (upload.path.startsWith(activeDirectoryPath)) {
+            // Must be a child of the active directory.
+            if (!upload.path.startsWith(activeDirectoryPath)) {
+              continue
+            }
+            const nestedPath = upload.path.slice(activeDirectoryPath.length)
+            const parts = nestedPath.split('/')
+            // Must be a directory with nested children.
+            if (parts.length <= 1) {
+              continue
+            }
+            const newDirName = parts[0]
+            const newDirPath = join(activeDirectoryPath, newDirName) + '/'
+            // Must not already exist.
+            if (dataMap[newDirPath]) {
+              continue
+            }
+            dataMap[newDirPath] = {
+              id: newDirPath,
+              path: newDirPath,
+              bucket: activeBucket,
+              size: 0,
+              health: 0,
+              name: newDirName + '/',
+              onClick: () => {
+                setActiveDirectory((p) => p.concat(newDirName))
+              },
+              type: 'directory',
+            }
+          }
+        }
+        // Add file uploads that are direct children of the active directory.
         uploadsList
-          .filter(({ path, name }) => path === join(activeDirectoryPath, name))
-          .filter(({ path }) =>
-            path.startsWith(join(activeBucketName, fileNamePrefixFilter))
-          )
+          .filter(({ path }) => {
+            if (!path.startsWith(activeDirectoryPath)) {
+              return false
+            }
+            const parts = path.slice(activeDirectoryPath.length).split('/')
+            const isDirectChild = parts.length === 1
+            const prefix = fileNamePrefixFilter
+              ? join(activeDirectoryPath, fileNamePrefixFilter)
+              : activeDirectoryPath
+            return isDirectChild && path.startsWith(prefix)
+          })
           .forEach((upload) => {
             dataMap[upload.path] = upload
           })
@@ -97,10 +139,17 @@ export function useDataset({ id, objects }: Props) {
       keepPreviousData: true,
     }
   )
-  // refetch when the dependent data changes
+  // Refetch when the dependent data changes. Adding these object reference
+  // dependencies to the swr key would cause the swr cache to grow indefinitely.
   useEffect(() => {
     response.mutate()
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [objects.data, uploadsList, allContracts, buckets.data])
+  }, [
+    objects.data,
+    uploadsList,
+    allContracts,
+    buckets.data,
+    fileNamePrefixFilter,
+  ])
   return response
 }