feat: renterd multipart upload manager

SiaFoundation · Feb 23, 2024 · dfbd336 · dfbd336
1 parent 3d93dc6
commit dfbd336
Show file tree

Hide file tree

Showing 9 changed files with 261 additions and 102 deletions.
diff --git a/.changeset/empty-garlics-look.md b/.changeset/empty-garlics-look.md
@@ -0,0 +1,5 @@
+---
+'renterd': minor
+---
+
+File uploads now use multipart uploads.
diff --git a/.changeset/pretty-roses-develop.md b/.changeset/pretty-roses-develop.md
@@ -0,0 +1,5 @@
+---
+'renterd': minor
+---
+
+File uploads now have a max concurrency and get queued.
diff --git a/apps/renterd/components/Files/Columns/FilesHealthColumn/FilesHealthColumnContents.tsx b/apps/renterd/components/Files/Columns/FilesHealthColumn/FilesHealthColumnContents.tsx
@@ -57,6 +57,7 @@ export function FilesHealthColumnContents({
   const slabs = sortBy(
     obj.data.object.slabs.map((s) => ({
       ...s.slab,
+      key: `${s.offset}${s.length}${s.slab.key}`,
       isPartialSlab: !!s.slab.shards,
       contractSetShards: s.slab.shards?.length
         ? computeSlabContractSetShards({

diff --git a/apps/renterd/components/TransfersBar.tsx b/apps/renterd/components/TransfersBar.tsx
@@ -15,15 +15,15 @@ import {
 import { useState } from 'react'
 import { useFilesManager } from '../contexts/filesManager'
 import { useAppSettings } from '@siafoundation/react-core'
+import { upperFirst } from '@technically/lodash'
 
 function getProgress(transfer: { loaded?: number; size?: number }) {
   return transfer.loaded !== undefined ? transfer.loaded / transfer.size : 1
 }
 
 export function TransfersBar() {
   const { isUnlockedAndAuthedRoute } = useAppSettings()
-  const { uploadsList, uploadCancel, downloadsList, downloadCancel } =
-    useFilesManager()
+  const { uploadsList, downloadsList, downloadCancel } = useFilesManager()
   const [maximized, setMaximized] = useState<boolean>(true)
 
   const uploadCount = uploadsList.length
@@ -56,7 +56,7 @@ export function TransfersBar() {
                   const progress = getProgress(upload)
                   return (
                     <div
-                      key={upload.path}
+                      key={upload.id}
                       className="flex flex-col gap-1 border-t first:border-t-0 border-gray-200 dark:border-graydark-300 px-3 py-2"
                     >
                       <div className="flex gap-1">
@@ -67,7 +67,7 @@ export function TransfersBar() {
                           tip="Cancel file upload"
                           variant="ghost"
                           size="none"
-                          onClick={() => uploadCancel(upload)}
+                          onClick={() => upload.uploadAbort?.()}
                         >
                           <Close16 />
                         </Button>
@@ -80,7 +80,7 @@ export function TransfersBar() {
                       />
                       <div className="flex justify-between mt-1">
                         <Text size="12" color="subtle">
-                          {progress === 1 ? 'Processing' : 'Uploading'}
+                          {upperFirst(upload.uploadStatus)}
                         </Text>
                         <Text size="12" color="subtle">
                           {(progress * 100).toFixed(0)}%

diff --git a/apps/renterd/contexts/filesManager/index.tsx b/apps/renterd/contexts/filesManager/index.tsx
@@ -87,7 +87,7 @@ function useFilesManagerMain() {
     [router, activeDirectory]
   )
 
-  const { uploadFiles, uploadsList, uploadCancel } = useUploads({
+  const { uploadFiles, uploadsList } = useUploads({
     activeDirectoryPath,
   })
   const { downloadFiles, downloadsList, getFileUrl, downloadCancel } =
@@ -174,7 +174,6 @@ function useFilesManagerMain() {
     navigateToModeSpecificFiltering,
     uploadFiles,
     uploadsList,
-    uploadCancel,
     downloadFiles,
     downloadsList,
     downloadCancel,

diff --git a/apps/renterd/contexts/filesManager/types.ts b/apps/renterd/contexts/filesManager/types.ts
@@ -1,6 +1,7 @@
 import { Bucket } from '@siafoundation/react-renterd'
 import { FullPath } from '../../lib/paths'
 import { TableColumn } from '@siafoundation/design-system'
+import { MultipartUpload } from '../../lib/multipartUpload'
 
 export type ObjectType = 'bucket' | 'directory' | 'file'
 
@@ -78,3 +79,18 @@ export const sortOptions: { id: SortField; label: string; category: string }[] =
   ]
 
 export type ExplorerMode = 'directory' | 'flat'
+
+export type UploadStatus = 'queued' | 'uploading' | 'processing'
+
+export type ObjectUploadData = ObjectData & {
+  upload: MultipartUpload
+  uploadStatus: UploadStatus
+  uploadAbort?: () => Promise<void>
+  uploadFile?: File
+}
+
+export type ObjectUploadRemoteData = ObjectData & {
+  remote: true
+}
+
+export type UploadsMap = Record<string, ObjectUploadData>