From 30f21e2dc6767e5ac40931c0602b7b3053b135f5 Mon Sep 17 00:00:00 2001 From: Alex Freska Date: Tue, 10 Sep 2024 11:34:30 -0400 Subject: [PATCH] refactor(renterd): migrate objects apis --- .changeset/brown-pianos-travel.md | 7 ++ .changeset/cyan-jokes-prove.md | 5 ++ .changeset/dry-elephants-try.md | 7 ++ .../contexts/filesDirectory/dataset.tsx | 39 ++++++----- apps/renterd/contexts/filesFlat/dataset.tsx | 21 +++--- .../renterd/contexts/filesManager/dataset.tsx | 6 +- .../renterd/contexts/filesManager/uploads.tsx | 10 ++- apps/renterd/contexts/uploads/index.tsx | 8 +-- apps/renterd/lib/multipartUpload.ts | 15 ++--- libs/renterd-js/src/bus.ts | 21 +----- libs/renterd-react/src/bus.ts | 34 +++------- libs/renterd-react/src/worker.ts | 7 +- libs/renterd-types/src/bus.ts | 65 ++++++++----------- libs/renterd-types/src/types.ts | 22 +++++-- 14 files changed, 131 insertions(+), 136 deletions(-) create mode 100644 .changeset/brown-pianos-travel.md create mode 100644 .changeset/cyan-jokes-prove.md create mode 100644 .changeset/dry-elephants-try.md diff --git a/.changeset/brown-pianos-travel.md b/.changeset/brown-pianos-travel.md new file mode 100644 index 000000000..690fba00d --- /dev/null +++ b/.changeset/brown-pianos-travel.md @@ -0,0 +1,7 @@ +--- +'@siafoundation/renterd-js': minor +'@siafoundation/renterd-react': minor +'@siafoundation/renterd-types': minor +--- + +Removed deprecated object search and object directory APIs. diff --git a/.changeset/cyan-jokes-prove.md b/.changeset/cyan-jokes-prove.md new file mode 100644 index 000000000..46ff3d245 --- /dev/null +++ b/.changeset/cyan-jokes-prove.md @@ -0,0 +1,5 @@ +--- +'renterd': minor +--- + +The files directory and global mode explorers now use the new list objects API. diff --git a/.changeset/dry-elephants-try.md b/.changeset/dry-elephants-try.md new file mode 100644 index 000000000..e1774edfd --- /dev/null +++ b/.changeset/dry-elephants-try.md @@ -0,0 +1,7 @@ +--- +'@siafoundation/renterd-js': minor +'@siafoundation/renterd-react': minor +'@siafoundation/renterd-types': minor +--- + +Updated the multipart upload APIs with the new payloads. diff --git a/apps/renterd/contexts/filesDirectory/dataset.tsx b/apps/renterd/contexts/filesDirectory/dataset.tsx index 3219e507f..2a7eb3c2b 100644 --- a/apps/renterd/contexts/filesDirectory/dataset.tsx +++ b/apps/renterd/contexts/filesDirectory/dataset.tsx @@ -1,5 +1,4 @@ -import { ObjectDirectoryParams } from '@siafoundation/renterd-types' -import { useObjectDirectory } from '@siafoundation/renterd-react' +import { useObjectList } from '@siafoundation/renterd-react' import { useDataset as useDatasetGeneric } from '../filesManager/dataset' import { bucketAndKeyParamsFromPath } from '../../lib/paths' import { useRouter } from 'next/router' @@ -19,32 +18,36 @@ export function useDataset() { } = useFilesManager() const router = useRouter() const limit = Number(router.query.limit || defaultLimit) - const offset = Number(router.query.offset || 0) + const marker = String(router.query.marker) + + const pathParams = bucketAndKeyParamsFromPath(activeDirectoryPath) const params = useMemo(() => { - const p: ObjectDirectoryParams = { - ...bucketAndKeyParamsFromPath(activeDirectoryPath), - sortBy: sortField, - sortDir: sortDirection, - offset, - limit, - } + let prefix = pathParams.key if (fileNamePrefixFilter) { - p.prefix = fileNamePrefixFilter.startsWith('/') + prefix += fileNamePrefixFilter.startsWith('/') ? fileNamePrefixFilter.slice(1) : fileNamePrefixFilter } - return p + return { + prefix, + bucket: pathParams.bucket, + sortBy: sortField, + sortDir: sortDirection, + marker, + limit, + delimiter: '/', + } }, [ - activeDirectoryPath, fileNamePrefixFilter, + pathParams, sortField, sortDirection, - offset, + marker, limit, ]) - const response = useObjectDirectory({ + const response = useObjectList({ disabled: !activeBucketName, params, config: { @@ -57,9 +60,9 @@ export function useDataset() { const objects = useMemo( () => ({ isValidating: response.isValidating, - data: response.data?.entries, + data: response.data?.objects, }), - [response.isValidating, response.data?.entries] + [response.isValidating, response.data?.objects] ) const d = useDatasetGeneric({ @@ -69,7 +72,7 @@ export function useDataset() { return { limit, - offset, + marker, response, dataset: d.data, refresh: response.mutate, diff --git a/apps/renterd/contexts/filesFlat/dataset.tsx b/apps/renterd/contexts/filesFlat/dataset.tsx index 60d73a594..56486d90c 100644 --- a/apps/renterd/contexts/filesFlat/dataset.tsx +++ b/apps/renterd/contexts/filesFlat/dataset.tsx @@ -1,4 +1,3 @@ -import { ObjectListPayload } from '@siafoundation/renterd-types' import { useObjectList } from '@siafoundation/renterd-react' import { SortField } from '../filesManager/types' import { useDataset as useDatasetGeneric } from '../filesManager/dataset' @@ -21,22 +20,24 @@ export function useDataset({ sortDirection, sortField }: Props) { const marker = router.query.marker as string const params = useMemo(() => { - const p: ObjectListPayload = { + let prefix = '' + if (fileNamePrefixFilter) { + prefix += fileNamePrefixFilter.startsWith('/') + ? fileNamePrefixFilter + : '/' + fileNamePrefixFilter + } + return { + prefix, bucket: activeBucketName, sortBy: sortField, sortDir: sortDirection, marker, limit, + delimiter: '', } - if (fileNamePrefixFilter) { - p.prefix = fileNamePrefixFilter.startsWith('/') - ? fileNamePrefixFilter - : '/' + fileNamePrefixFilter - } - return p }, [ - activeBucketName, fileNamePrefixFilter, + activeBucketName, sortField, sortDirection, marker, @@ -45,7 +46,7 @@ export function useDataset({ sortDirection, sortField }: Props) { const response = useObjectList({ disabled: !activeBucketName, - payload: params, + params, config: { swr: { refreshInterval: defaultDatasetRefreshInterval, diff --git a/apps/renterd/contexts/filesManager/dataset.tsx b/apps/renterd/contexts/filesManager/dataset.tsx index c8af717cd..31b1d8506 100644 --- a/apps/renterd/contexts/filesManager/dataset.tsx +++ b/apps/renterd/contexts/filesManager/dataset.tsx @@ -1,4 +1,4 @@ -import { ObjEntry } from '@siafoundation/renterd-types' +import { ObjectMetadata } from '@siafoundation/renterd-types' import { sortBy, toPairs } from '@technically/lodash' import useSWR from 'swr' import { useContracts } from '../contracts' @@ -16,7 +16,7 @@ type Props = { id: string objects: { isValidating: boolean - data?: ObjEntry[] + data?: ObjectMetadata[] } } @@ -57,7 +57,7 @@ export function useDataset({ id, objects }: Props) { } }) } else if (objects.data || uploadsList.length) { - objects.data?.forEach(({ name: key, size, health }) => { + objects.data?.forEach(({ key, size, health }) => { const path = join(activeBucketName, key) const name = getFilename(key) dataMap[path] = { diff --git a/apps/renterd/contexts/filesManager/uploads.tsx b/apps/renterd/contexts/filesManager/uploads.tsx index 5b7f2de96..079e7e287 100644 --- a/apps/renterd/contexts/filesManager/uploads.tsx +++ b/apps/renterd/contexts/filesManager/uploads.tsx @@ -1,5 +1,9 @@ import { triggerErrorToast } from '@siafoundation/design-system' -import { Bucket } from '@siafoundation/renterd-types' +import { + Bucket, + busListObjectsPrefixRoute, + busListObjectsRoute, +} from '@siafoundation/renterd-types' import { useBuckets, useMultipartUploadAbort, @@ -106,7 +110,7 @@ export function useUploads({ activeDirectoryPath }: Props) { const key = getKeyFromPath(path) const multipartUpload = new MultipartUpload({ file: uploadFile, - path: key, + key, bucket: bucket.name, api: ref.current, partSize: getMultipartUploadPartSize( @@ -133,7 +137,7 @@ export function useUploads({ activeDirectoryPath }: Props) { }, 1000) ) multipartUpload.setOnComplete(async () => { - await ref.current.mutate((key) => key.startsWith('/bus/objects')) + await ref.current.mutate((key) => key.startsWith(busListObjectsRoute)) ref.current.removeUpload(uploadId) setTimeout(() => { ref.current.checkAndStartUploads() diff --git a/apps/renterd/contexts/uploads/index.tsx b/apps/renterd/contexts/uploads/index.tsx index 0c2852896..d9ef85d92 100644 --- a/apps/renterd/contexts/uploads/index.tsx +++ b/apps/renterd/contexts/uploads/index.tsx @@ -46,7 +46,7 @@ function useUploadsMain() { await apiBusUploadAbort.post({ payload: { bucket: activeBucket?.name, - path: upload.path, + key: upload.key, uploadID: upload.uploadID, }, }) @@ -59,8 +59,8 @@ function useUploadsMain() { return ( response.data?.uploads?.map((upload) => { const id = upload.uploadID - const name = getFilename(upload.path) - const fullPath = join(activeBucket?.name, upload.path) + const name = getFilename(upload.key) + const fullPath = join(activeBucket?.name, upload.key) const localUpload = uploadsMap[id] if (localUpload) { { @@ -83,7 +83,7 @@ function useUploadsMain() { await apiBusUploadAbort.post({ payload: { bucket: activeBucket?.name, - path: upload.path, + key: upload.key, uploadID: upload.uploadID, }, }) diff --git a/apps/renterd/lib/multipartUpload.ts b/apps/renterd/lib/multipartUpload.ts index c3b15884e..c3d13496a 100644 --- a/apps/renterd/lib/multipartUpload.ts +++ b/apps/renterd/lib/multipartUpload.ts @@ -14,7 +14,7 @@ type ApiBusUploadAbort = ReturnType export type MultipartParams = { bucket: string - path: string + key: string file: File api: { workerUploadPart: ApiWorkerUploadPart @@ -41,7 +41,7 @@ type UploadedPart = { export class MultipartUpload { // params #bucket: string - #path: string + #key: string #file: File #partSize: number #maxConcurrentParts: number @@ -75,7 +75,7 @@ export class MultipartUpload { constructor(options: MultipartParams) { // params this.#bucket = options.bucket - this.#path = options.path + this.#key = options.key this.#partSize = options.partSize || 1024 * 1024 * 5 this.#maxConcurrentParts = Math.min(options.maxConcurrentParts || 5, 15) this.#file = options.file @@ -96,8 +96,7 @@ export class MultipartUpload { public async create() { const createPayload = { bucket: this.#bucket, - generateKey: true, - path: this.#path, + key: this.#key, } const response = await this.#api.busUploadCreate.post({ payload: createPayload, @@ -133,7 +132,7 @@ export class MultipartUpload { await this.#api.busUploadAbort.post({ payload: { bucket: this.#bucket, - path: this.#path, + key: this.#key, uploadID: this.#uploadId, }, }) @@ -235,7 +234,7 @@ export class MultipartUpload { try { const payload = { bucket: this.#bucket, - path: this.#path, + key: this.#key, uploadID: this.#uploadId, parts: this.#uploadedParts.sort((a, b) => a.partNumber - b.partNumber), } @@ -280,7 +279,7 @@ export class MultipartUpload { try { const response = await this.#api.workerUploadPart.put({ params: { - key: this.#path.slice(1), + key: this.#key.slice(1), bucket: this.#bucket, uploadid: this.#uploadId, offset: partOffset, diff --git a/libs/renterd-js/src/bus.ts b/libs/renterd-js/src/bus.ts index 0a1e814b2..503d7dbea 100644 --- a/libs/renterd-js/src/bus.ts +++ b/libs/renterd-js/src/bus.ts @@ -121,9 +121,6 @@ import { ObjectDeleteParams, ObjectDeletePayload, ObjectDeleteResponse, - ObjectDirectoryParams, - ObjectDirectoryPayload, - ObjectDirectoryResponse, ObjectListParams, ObjectListPayload, ObjectListResponse, @@ -133,9 +130,6 @@ import { ObjectRenamePayload, ObjectRenameResponse, ObjectResponse, - ObjectSearchParams, - ObjectSearchPayload, - ObjectSearchResponse, ObjectsStatsParams, ObjectsStatsPayload, ObjectsStatsResponse, @@ -230,10 +224,8 @@ import { busMultipartListuploadsRoute, busMultipartPartRoute, busObjectsKeyRoute, - busObjectsListRoute, busObjectsRenameRoute, busHostsRoute, - busSearchObjectsRoute, busSettingKeyRoute, busSettingsRoute, busSlabKeyObjectsRoute, @@ -255,6 +247,7 @@ import { AutopilotsPayload, AutopilotsResponse, busWalletEventsRoute, + busListObjectsPrefixRoute, } from '@siafoundation/renterd-types' import { buildRequestHandler, initAxios } from '@siafoundation/request' import { AxiosRequestConfig } from 'axios' @@ -468,26 +461,16 @@ export function Bus({ api, password }: { api: string; password?: string }) { BucketDeletePayload, BucketDeleteResponse >(axios, 'delete', busBucketNameRoute), - objectDirectory: buildRequestHandler< - ObjectDirectoryParams, - ObjectDirectoryPayload, - ObjectDirectoryResponse - >(axios, 'get', busObjectsKeyRoute), objectList: buildRequestHandler< ObjectListParams, ObjectListPayload, ObjectListResponse - >(axios, 'post', busObjectsListRoute), + >(axios, 'get', busListObjectsPrefixRoute), object: buildRequestHandler( axios, 'get', busObjectsKeyRoute ), - objectSearch: buildRequestHandler< - ObjectSearchParams, - ObjectSearchPayload, - ObjectSearchResponse - >(axios, 'get', busSearchObjectsRoute), objectAdd: buildRequestHandler< ObjectAddParams, ObjectAddPayload, diff --git a/libs/renterd-react/src/bus.ts b/libs/renterd-react/src/bus.ts index 05d8821bc..4b9563eda 100644 --- a/libs/renterd-react/src/bus.ts +++ b/libs/renterd-react/src/bus.ts @@ -115,18 +115,13 @@ import { ObjectDeleteParams, ObjectDeletePayload, ObjectDeleteResponse, - ObjectDirectoryParams, - ObjectDirectoryResponse, ObjectListParams, - ObjectListPayload, ObjectListResponse, ObjectParams, ObjectRenameParams, ObjectRenamePayload, ObjectRenameResponse, ObjectResponse, - ObjectSearchParams, - ObjectSearchResponse, ObjectsStatsParams, ObjectsStatsResponse, SettingParams, @@ -189,10 +184,8 @@ import { busHostsHostKeyRoute, busObjectsRoute, busObjectsKeyRoute, - busObjectsListRoute, busObjectsRenameRoute, busHostsRoute, - busSearchObjectsRoute, busSettingKeyRoute, busSettingsRoute, busStateRoute, @@ -244,6 +237,8 @@ import { WalletEventsParams, WalletEventsResponse, busWalletEventsRoute, + busListObjectsPrefixRoute, + busListObjectsRoute, } from '@siafoundation/renterd-types' // state @@ -699,32 +694,16 @@ export function useBucketDelete( ) } -export function useObjectDirectory( - args: HookArgsSwr -) { - return useGetSwr({ ...args, route: busObjectsKeyRoute }) -} - export function useObjectList( - args: HookArgsWithPayloadSwr< - ObjectListParams, - ObjectListPayload, - ObjectListResponse - > + args: HookArgsSwr ) { - return usePostSwr({ ...args, route: busObjectsListRoute }) + return useGetSwr({ ...args, route: busListObjectsPrefixRoute }) } export function useObject(args: HookArgsSwr) { return useGetSwr({ ...args, route: busObjectsKeyRoute }) } -export function useObjectSearch( - args: HookArgsSwr -) { - return useGetSwr({ ...args, route: busSearchObjectsRoute }) -} - export function useObjectAdd( args: HookArgsCallback ) { @@ -751,7 +730,10 @@ export function useObjectDelete( return useDeleteFunc( { ...args, route: busObjectsKeyRoute }, async (mutate) => { - mutate((key) => key.startsWith(busObjectsRoute)) + mutate( + (key) => + key.startsWith(busObjectsRoute) || key.startsWith(busListObjectsRoute) + ) } ) } diff --git a/libs/renterd-react/src/worker.ts b/libs/renterd-react/src/worker.ts index aba185c07..d6749294b 100644 --- a/libs/renterd-react/src/worker.ts +++ b/libs/renterd-react/src/worker.ts @@ -32,6 +32,8 @@ import { workerObjectsKeyRoute, workerRhpScanRoute, workerStateRoute, + busListObjectsRoute, + Host, } from '@siafoundation/renterd-types' // state @@ -77,7 +79,10 @@ export function useObjectUpload( route: workerObjectsKeyRoute, }, async (mutate) => { - mutate((key) => key.startsWith(busObjectsRoute)) + mutate( + (key) => + key.startsWith(busObjectsRoute) || key.startsWith(busListObjectsRoute) + ) } ) } diff --git a/libs/renterd-types/src/bus.ts b/libs/renterd-types/src/bus.ts index 53f933d4e..1d7e45544 100644 --- a/libs/renterd-types/src/bus.ts +++ b/libs/renterd-types/src/bus.ts @@ -16,6 +16,7 @@ import { Host, HostSettings, Obj, + ObjectMetadata, PartialSlab, SlabSlice, } from './types' @@ -60,9 +61,9 @@ export const busBucketsRoute = '/bus/buckets' export const busBucketNameRoute = '/bus/bucket/:name' export const busBucketNamePolicyRoute = '/bus/bucket/:name/policy' export const busObjectsRoute = '/bus/objects' +export const busListObjectsRoute = '/bus/listobjects' +export const busListObjectsPrefixRoute = '/bus/listobjects/:prefix' export const busObjectsKeyRoute = '/bus/objects/:key' -export const busObjectsListRoute = '/bus/objects/list' -export const busSearchObjectsRoute = '/bus/search/objects' export const busObjectsRenameRoute = '/bus/objects/rename' export const busStatsObjectsRoute = '/bus/stats/objects' export const busSettingRoute = '/bus/setting' @@ -388,37 +389,21 @@ export type BucketDeleteParams = { name: string } export type BucketDeletePayload = void export type BucketDeleteResponse = void -export type ObjEntry = { - name: string - size: number - health: number -} - -export type ObjectDirectoryParams = { - key: string - bucket: string - limit?: number +export type ObjectListParams = { prefix?: string - offset?: number - sortBy?: 'name' | 'health' | 'size' - sortDir?: 'asc' | 'desc' -} -export type ObjectDirectoryPayload = void -export type ObjectDirectoryResponse = { hasMore: boolean; entries: ObjEntry[] } - -export type ObjectListParams = void -export type ObjectListPayload = { bucket: string + delimiter?: string limit?: number - prefix?: string marker?: string sortBy?: 'name' | 'health' | 'size' sortDir?: 'asc' | 'desc' + substring?: string } +export type ObjectListPayload = void export type ObjectListResponse = { hasMore: boolean nextMarker: string - objects: ObjEntry[] + objects?: ObjectMetadata[] } export type ObjectParams = { key: string; bucket: string } @@ -432,7 +417,7 @@ export type ObjectSearchParams = { limit: number } export type ObjectSearchPayload = void -export type ObjectSearchResponse = ObjEntry[] +export type ObjectSearchResponse = ObjectMetadata[] export type ObjectAddParams = { key: string; bucket: string } export type ObjectAddPayload = { @@ -521,7 +506,7 @@ export type AlertsDismissResponse = void export type SlabObjectsParams = { key: string } export type SlabObjectsPayload = void -export type SlabObjectsResponse = ObjEntry[] +export type SlabObjectsResponse = ObjectMetadata[] // metrics @@ -609,10 +594,11 @@ export type WalletMetricsResponse = WalletMetric[] export type MultipartUploadCreateParams = void export type MultipartUploadCreatePayload = { - path: string + key: string bucket: string - generateKey: boolean - key?: string + mimeType?: string + metadata?: Record + disableClientSideEncryption?: boolean } export type MultipartUploadCreateResponse = { uploadID: string @@ -625,7 +611,7 @@ export type MultipartUploadCompletePart = { export type MultipartUploadCompleteParams = void export type MultipartUploadCompletePayload = { - path: string + key: string bucket: string uploadID: string parts: MultipartUploadCompletePart[] @@ -634,7 +620,7 @@ export type MultipartUploadCompleteResponse = void export type MultipartUploadAbortParams = void export type MultipartUploadAbortPayload = { - path: string + key: string bucket: string uploadID: string } @@ -643,7 +629,7 @@ export type MultipartUploadAbortResponse = void export type MultipartUploadListPartsParams = void export type MultipartUploadListPartsPayload = { bucket: string - path: string + key: string uploadID: string partNumberMarker?: number limit?: number @@ -663,24 +649,27 @@ export type MultipartUploadListUploadsParams = void export type MultipartUploadListUploadsPayload = { bucket: string prefix?: string - pathMarker?: string + keyMarker?: string uploadIDMarker?: string limit?: number } +export type MultipartUploadListItem = { + bucket: string + key: string + encryptionKey: string + uploadID: string + createdAt: string +} export type MultipartUploadListUploadsResponse = { hasMore: boolean nextMarker: string nextUploadIDMarker: string - uploads?: { - path: string - uploadID: string - createdAt: string - }[] + uploads?: MultipartUploadListItem[] } export type MultipartUploadAddPartParams = void export type MultipartUploadAddPartPayload = { - path: string + key: string bucket: string uploadID: string eTag: string diff --git a/libs/renterd-types/src/types.ts b/libs/renterd-types/src/types.ts index d796724f6..a30f2de90 100644 --- a/libs/renterd-types/src/types.ts +++ b/libs/renterd-types/src/types.ts @@ -108,17 +108,27 @@ export type SlabSlice = { length: number } -export type Obj = { - name: string +export type ObjectObject = { + encryptionKey: EncryptionKey + slabs?: SlabSlice[] +} + +export type ObjectMetadata = { + key: string size: number health: number - key: EncryptionKey - eTag: string - mimeType: string + eTag?: string modTime: string - slabs?: SlabSlice[] + mimeType?: string } +export type ObjectUserMetadata = Record + +export type Obj = ObjectMetadata & + ObjectObject & { + metadata: ObjectUserMetadata + } + export type ContractSetSettings = { default: string }