refactor(renterd): migrate objects and multipart apis

SiaFoundation · Sep 16, 2024 · 527ad5e · 527ad5e
1 parent f9576ca
commit 527ad5e
Show file tree

Hide file tree

Showing 19 changed files with 166 additions and 166 deletions.
diff --git a/.changeset/brown-pianos-travel.md b/.changeset/brown-pianos-travel.md
@@ -0,0 +1,7 @@
+---
+'@siafoundation/renterd-js': minor
+'@siafoundation/renterd-react': minor
+'@siafoundation/renterd-types': minor
+---
+
+Removed deprecated object search and object directory APIs.
diff --git a/.changeset/cyan-jokes-prove.md b/.changeset/cyan-jokes-prove.md
@@ -0,0 +1,5 @@
+---
+'renterd': minor
+---
+
+The files directory and global mode explorers now use the new list objects API.
diff --git a/.changeset/dry-elephants-try.md b/.changeset/dry-elephants-try.md
@@ -0,0 +1,7 @@
+---
+'@siafoundation/renterd-js': minor
+'@siafoundation/renterd-react': minor
+'@siafoundation/renterd-types': minor
+---
+
+Updated the multipart upload APIs with the new payloads.
diff --git a/apps/renterd/components/Files/Columns/FilesHealthColumn/FilesHealthColumnContents.tsx b/apps/renterd/components/Files/Columns/FilesHealthColumn/FilesHealthColumnContents.tsx
@@ -3,6 +3,7 @@ import {
   ScrollArea,
   Separator,
   Text,
+  stripPrefix,
 } from '@siafoundation/design-system'
 import { useObject } from '@siafoundation/renterd-react'
 import { cx } from 'class-variance-authority'
@@ -62,7 +63,7 @@ export function FilesHealthColumnContents({
       // multipart uploads might reference the same slab over and over but at
       // different offsets and lengths. So we should not assume that they are
       // always unique.
-      id: `${s.offset}${s.length}${s.slab.key}`,
+      id: `${s.offset}${s.length}${s.slab.encryptionKey}`,
       isPartialSlab: !!s.slab.shards,
       contractSetShards: s.slab.shards?.length
         ? computeSlabContractSetShards({
@@ -91,7 +92,7 @@ export function FilesHealthColumnContents({
             className="flex items-center"
             font="mono"
           >
-            Slab {slab.key.replace('key:', '').slice(0, 4)}:
+            Slab {stripPrefix(slab.encryptionKey).slice(0, 4)}:
           </Text>
           <Text size="12" className="flex items-center">
             {slab.isPartialSlab

diff --git a/apps/renterd/components/Files/FilesCmd/FilesSearchCmd/index.tsx b/apps/renterd/components/Files/FilesCmd/FilesSearchCmd/index.tsx
@@ -1,6 +1,6 @@
 import { CommandGroup, CommandItemSearch } from '../../../CmdRoot/Item'
 import { Page } from '../../../CmdRoot/types'
-import { useObjectSearch } from '@siafoundation/renterd-react'
+import { useObjectList } from '@siafoundation/renterd-react'
 import { isDirectory } from '../../../../lib/paths'
 import { Text } from '@siafoundation/design-system'
 import { Document16, FolderIcon } from '@siafoundation/react-icons'
@@ -31,13 +31,13 @@ export function FilesSearchCmd({
     useFilesManager()
   const onSearchPage = currentPage?.namespace === filesSearchPage.namespace
   const searchBucket = activeBucket || 'default'
-  const results = useObjectSearch({
+  const results = useObjectList({
     disabled: !onSearchPage,
     params: {
       bucket: searchBucket,
-      key: debouncedSearch,
-      offset: 0,
+      prefix: debouncedSearch,
       limit: 10,
+      delimiter: '',
     },
     config: {
       swr: {
@@ -46,34 +46,34 @@ export function FilesSearchCmd({
     },
   })
 
-  if (!onSearchPage || !results.data) {
+  if (!onSearchPage || !results.data?.objects) {
     return null
   }
 
   return (
     <CommandGroup currentPage={currentPage} commandPage={filesSearchPage}>
-      {results.data.map(({ name: path }) => {
-        const compressedPath = compressPath(path, search, 55)
+      {results.data?.objects.map(({ key }) => {
+        const compressedPath = compressPath(key, search, 55)
         const { startIndex, endIndex } = findLastMatch(compressedPath, search)
 
         return (
           <CommandItemSearch
             commandPage={filesSearchPage}
             currentPage={currentPage}
-            key={path}
+            key={key}
             onSelect={() => {
               beforeSelect()
-              navigateToModeSpecificFiltering(searchBucket + path)
+              navigateToModeSpecificFiltering(searchBucket + key)
               afterSelect()
             }}
-            value={path}
+            value={key}
           >
             <div className="flex items-center gap-2 overflow-hidden">
               <Text
                 color="verySubtle"
                 className="group-data-[selected=true]:text-gray-1000 dark:group-data-[selected=true]:text-graydark-1000"
               >
-                {isDirectory(path) ? <FolderIcon size={16} /> : <Document16 />}
+                {isDirectory(key) ? <FolderIcon size={16} /> : <Document16 />}
               </Text>
               <Text className="flex items-center">
                 <Text color="verySubtle" ellipsis>

diff --git a/apps/renterd/components/FilesDirectory/FilesStatsMenu/index.tsx b/apps/renterd/components/FilesDirectory/FilesStatsMenu/index.tsx
@@ -1,12 +1,12 @@
-import { PaginatorUnknownTotal } from '@siafoundation/design-system'
+import { PaginatorMarker } from '@siafoundation/design-system'
 import { useFilesDirectory } from '../../../contexts/filesDirectory'
 import { FilesStatsMenuShared } from '../../Files/FilesStatsMenuShared'
 import { FilesFilterDirectoryMenu } from '../../Files/FilesFilterDirectoryMenu'
 import { useFilesManager } from '../../../contexts/filesManager'
 
 export function FilesStatsMenu() {
   const { isViewingABucket, isViewingBuckets } = useFilesManager()
-  const { limit, offset, pageCount, dataState } = useFilesDirectory()
+  const { limit, marker, isMore, pageCount, dataState } = useFilesDirectory()
   return (
     <div className="flex gap-3 w-full">
       {isViewingBuckets ? (
@@ -16,8 +16,9 @@ export function FilesStatsMenu() {
       )}
       <FilesStatsMenuShared />
       {isViewingABucket && (
-        <PaginatorUnknownTotal
-          offset={offset}
+        <PaginatorMarker
+          isMore={isMore}
+          marker={marker}
           limit={limit}
           pageTotal={pageCount}
           isLoading={dataState === 'loading'}

diff --git a/apps/renterd/contexts/filesDirectory/dataset.tsx b/apps/renterd/contexts/filesDirectory/dataset.tsx
@@ -1,11 +1,11 @@
-import { ObjectDirectoryParams } from '@siafoundation/renterd-types'
-import { useObjectDirectory } from '@siafoundation/renterd-react'
+import { useObjectList } from '@siafoundation/renterd-react'
 import { useDataset as useDatasetGeneric } from '../filesManager/dataset'
 import { bucketAndKeyParamsFromPath } from '../../lib/paths'
 import { useRouter } from 'next/router'
 import { useMemo } from 'react'
 import { useFilesManager } from '../filesManager'
 import { defaultDatasetRefreshInterval } from '../../config/swr'
+import { ObjectListParams } from '@siafoundation/renterd-types'
 
 const defaultLimit = 50
 
@@ -19,32 +19,39 @@ export function useDataset() {
   } = useFilesManager()
   const router = useRouter()
   const limit = Number(router.query.limit || defaultLimit)
-  const offset = Number(router.query.offset || 0)
+  const marker = router.query.marker as string
+
+  const pathParams = bucketAndKeyParamsFromPath(activeDirectoryPath)
 
   const params = useMemo(() => {
-    const p: ObjectDirectoryParams = {
-      ...bucketAndKeyParamsFromPath(activeDirectoryPath),
+    let prefix = pathParams.key
+    if (fileNamePrefixFilter) {
+      prefix += fileNamePrefixFilter.startsWith('/')
+        ? fileNamePrefixFilter.slice(1)
+        : fileNamePrefixFilter
+    }
+    const p: ObjectListParams = {
+      prefix,
+      bucket: pathParams.bucket,
       sortBy: sortField,
       sortDir: sortDirection,
-      offset,
       limit,
+      delimiter: '/',
     }
-    if (fileNamePrefixFilter) {
-      p.prefix = fileNamePrefixFilter.startsWith('/')
-        ? fileNamePrefixFilter.slice(1)
-        : fileNamePrefixFilter
+    if (marker) {
+      p.marker = marker
     }
     return p
   }, [
-    activeDirectoryPath,
     fileNamePrefixFilter,
+    pathParams,
     sortField,
     sortDirection,
-    offset,
+    marker,
     limit,
   ])
 
-  const response = useObjectDirectory({
+  const response = useObjectList({
     disabled: !activeBucketName,
     params,
     config: {
@@ -57,9 +64,9 @@ export function useDataset() {
   const objects = useMemo(
     () => ({
       isValidating: response.isValidating,
-      data: response.data?.entries,
+      data: response.data?.objects,
     }),
-    [response.isValidating, response.data?.entries]
+    [response.isValidating, response.data?.objects]
   )
 
   const d = useDatasetGeneric({
@@ -69,7 +76,8 @@ export function useDataset() {
 
   return {
     limit,
-    offset,
+    marker,
+    isMore: response.data?.hasMore,
     response,
     dataset: d.data,
     refresh: response.mutate,

diff --git a/apps/renterd/contexts/filesDirectory/index.tsx b/apps/renterd/contexts/filesDirectory/index.tsx
@@ -16,7 +16,7 @@ function useFilesDirectoryMain() {
     enabledColumns,
   } = useFilesManager()
 
-  const { limit, offset, response, refresh, dataset } = useDataset()
+  const { limit, marker, isMore, response, refresh, dataset } = useDataset()
 
   const {
     onDragEnd,
@@ -104,7 +104,8 @@ function useFilesDirectoryMain() {
     columns: filteredTableColumns,
     refresh,
     limit,
-    offset,
+    marker,
+    isMore,
     datasetPage,
     pageCount: dataset?.length || 0,
     onDragStart,

diff --git a/apps/renterd/contexts/filesFlat/dataset.tsx b/apps/renterd/contexts/filesFlat/dataset.tsx
@@ -1,11 +1,11 @@
-import { ObjectListPayload } from '@siafoundation/renterd-types'
 import { useObjectList } from '@siafoundation/renterd-react'
 import { SortField } from '../filesManager/types'
 import { useDataset as useDatasetGeneric } from '../filesManager/dataset'
 import { useRouter } from 'next/router'
 import { useMemo } from 'react'
 import { useFilesManager } from '../filesManager'
 import { defaultDatasetRefreshInterval } from '../../config/swr'
+import { ObjectListParams } from '@siafoundation/renterd-types'
 
 type Props = {
   sortDirection: 'asc' | 'desc'
@@ -21,22 +21,27 @@ export function useDataset({ sortDirection, sortField }: Props) {
   const marker = router.query.marker as string
 
   const params = useMemo(() => {
-    const p: ObjectListPayload = {
+    let prefix = ''
+    if (fileNamePrefixFilter) {
+      prefix += fileNamePrefixFilter.startsWith('/')
+        ? fileNamePrefixFilter.slice(1)
+        : fileNamePrefixFilter
+    }
+    const p: ObjectListParams = {
+      prefix,
       bucket: activeBucketName,
       sortBy: sortField,
       sortDir: sortDirection,
-      marker,
       limit,
+      delimiter: '',
     }
-    if (fileNamePrefixFilter) {
-      p.prefix = fileNamePrefixFilter.startsWith('/')
-        ? fileNamePrefixFilter
-        : '/' + fileNamePrefixFilter
+    if (marker) {
+      p.marker = marker
     }
     return p
   }, [
-    activeBucketName,
     fileNamePrefixFilter,
+    activeBucketName,
     sortField,
     sortDirection,
     marker,
@@ -45,7 +50,7 @@ export function useDataset({ sortDirection, sortField }: Props) {
 
   const response = useObjectList({
     disabled: !activeBucketName,
-    payload: params,
+    params,
     config: {
       swr: {
         refreshInterval: defaultDatasetRefreshInterval,

diff --git a/apps/renterd/contexts/filesManager/dataset.tsx b/apps/renterd/contexts/filesManager/dataset.tsx
@@ -1,4 +1,4 @@
-import { ObjEntry } from '@siafoundation/renterd-types'
+import { ObjectMetadata } from '@siafoundation/renterd-types'
 import { sortBy, toPairs } from '@technically/lodash'
 import useSWR from 'swr'
 import { useContracts } from '../contracts'
@@ -16,7 +16,7 @@ type Props = {
   id: string
   objects: {
     isValidating: boolean
-    data?: ObjEntry[]
+    data?: ObjectMetadata[]
   }
 }
 
@@ -57,7 +57,7 @@ export function useDataset({ id, objects }: Props) {
           }
         })
       } else if (objects.data || uploadsList.length) {
-        objects.data?.forEach(({ name: key, size, health }) => {
+        objects.data?.forEach(({ key, size, health }) => {
           const path = join(activeBucketName, key)
           const name = getFilename(key)
           dataMap[path] = {

diff --git a/apps/renterd/contexts/filesManager/uploads.tsx b/apps/renterd/contexts/filesManager/uploads.tsx
@@ -1,5 +1,9 @@
 import { triggerErrorToast } from '@siafoundation/design-system'
-import { Bucket } from '@siafoundation/renterd-types'
+import {
+  Bucket,
+  busListObjectsPrefixRoute,
+  busListObjectsRoute,
+} from '@siafoundation/renterd-types'
 import {
   useBuckets,
   useMultipartUploadAbort,
@@ -106,7 +110,7 @@ export function useUploads({ activeDirectoryPath }: Props) {
       const key = getKeyFromPath(path)
       const multipartUpload = new MultipartUpload({
         file: uploadFile,
-        path: key,
+        key,
         bucket: bucket.name,
         api: ref.current,
         partSize: getMultipartUploadPartSize(
@@ -133,7 +137,7 @@ export function useUploads({ activeDirectoryPath }: Props) {
         }, 1000)
       )
       multipartUpload.setOnComplete(async () => {
-        await ref.current.mutate((key) => key.startsWith('/bus/objects'))
+        await ref.current.mutate((key) => key.startsWith(busListObjectsRoute))
         ref.current.removeUpload(uploadId)
         setTimeout(() => {
           ref.current.checkAndStartUploads()

diff --git a/apps/renterd/contexts/uploads/index.tsx b/apps/renterd/contexts/uploads/index.tsx
@@ -46,7 +46,7 @@ function useUploadsMain() {
           await apiBusUploadAbort.post({
             payload: {
               bucket: activeBucket?.name,
-              path: upload.path,
+              key: upload.key,
               uploadID: upload.uploadID,
             },
           })
@@ -59,8 +59,8 @@ function useUploadsMain() {
     return (
       response.data?.uploads?.map((upload) => {
         const id = upload.uploadID
-        const name = getFilename(upload.path)
-        const fullPath = join(activeBucket?.name, upload.path)
+        const name = getFilename(upload.key)
+        const fullPath = join(activeBucket?.name, upload.key)
         const localUpload = uploadsMap[id]
         if (localUpload) {
           {
@@ -83,7 +83,7 @@ function useUploadsMain() {
             await apiBusUploadAbort.post({
               payload: {
                 bucket: activeBucket?.name,
-                path: upload.path,
+                key: upload.key,
                 uploadID: upload.uploadID,
               },
             })