From bdb1d2053b652810287acb409b86af3749daf888 Mon Sep 17 00:00:00 2001 From: amit <1mitccc@gmail.com> Date: Wed, 18 Dec 2024 16:50:23 +0530 Subject: [PATCH] feat: onedrive: set up parent-child relationships for folder and files --- .../file/services/onedrive/mappers.ts | 1 + .../folder/services/onedrive/index.ts | 75 ++++++++++++------- .../folder/services/onedrive/mappers.ts | 6 +- .../folder/services/onedrive/types.ts | 4 + 4 files changed, 56 insertions(+), 30 deletions(-) diff --git a/packages/api/src/filestorage/file/services/onedrive/mappers.ts b/packages/api/src/filestorage/file/services/onedrive/mappers.ts index fe9694bfa..18e216f4b 100644 --- a/packages/api/src/filestorage/file/services/onedrive/mappers.ts +++ b/packages/api/src/filestorage/file/services/onedrive/mappers.ts @@ -122,6 +122,7 @@ export class OnedriveFileMapper implements IFileMapper { return { remote_id: file.id, remote_data: file, + remote_folder_id: file.parentReference?.id, name: file.name, file_url: file.webUrl, mime_type: file.file.mimeType, diff --git a/packages/api/src/filestorage/folder/services/onedrive/index.ts b/packages/api/src/filestorage/folder/services/onedrive/index.ts index 375ccc09c..a6a9b9f96 100644 --- a/packages/api/src/filestorage/folder/services/onedrive/index.ts +++ b/packages/api/src/filestorage/folder/services/onedrive/index.ts @@ -13,6 +13,7 @@ import { SyncParam } from '@@core/utils/types/interface'; import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { UnifiedFilestorageFileOutput } from '@filestorage/file/types/model.unified'; import { OnedriveFileOutput } from '@filestorage/file/services/onedrive/types'; +import { v4 as uuidv4 } from 'uuid'; @Injectable() export class OnedriveService implements IFolderService { @@ -112,7 +113,17 @@ export class OnedriveService implements IFolderService { let result: OnedriveFolderOutput[] = [rootFolderData.data]; let depth = 0; - let batch: string[] = [remote_folder_id]; + let batch: { + remote_folder_id: string; + internal_id: string | null; + internal_parent_folder_id: string | null; + }[] = [ + { + remote_folder_id: remote_folder_id, + internal_id: uuidv4(), + internal_parent_folder_id: null, + }, + ]; while (batch.length > 0) { if (depth > 50) { @@ -121,37 +132,49 @@ export class OnedriveService implements IFolderService { } const nestedFoldersPromises: Promise[] = - batch.map(async (folder_id: string) => { - const childrenConfig: AxiosRequestConfig = { - method: 'get', - url: `${connection.account_url}/v1.0/me/drive/items/${folder_id}/children`, - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${this.cryptoService.decrypt( - connection.access_token, - )}`, - }, - }; - - const resp: AxiosResponse = await this.makeRequestWithRetry( - childrenConfig, - ); - - const folders: OnedriveFolderOutput[] = resp.data.value.filter( - (driveItem: any) => driveItem.folder, - ); - - return folders; - }); + batch.map( + async (folder: { + remote_folder_id: string; + internal_id: string; + internal_parent_folder_id: string; + }) => { + const childrenConfig: AxiosRequestConfig = { + method: 'get', + url: `${connection.account_url}/v1.0/me/drive/items/${folder.remote_folder_id}/children`, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }; + + const resp: AxiosResponse = await this.makeRequestWithRetry( + childrenConfig, + ); + + const folders: OnedriveFolderOutput[] = resp.data.value.filter( + (driveItem: any) => driveItem.folder, + ); + + return folders.map((f: OnedriveFolderOutput) => ({ + ...f, + internal_id: uuidv4(), + internal_parent_folder_id: folder.internal_id, + })); + }, + ); const nestedFolders: OnedriveFolderOutput[][] = await Promise.all( nestedFoldersPromises, ); result = result.concat(nestedFolders.flat()); - batch = nestedFolders - .flat() - .map((folder: OnedriveFolderOutput) => folder.id); + batch = nestedFolders.flat().map((folder: OnedriveFolderOutput) => ({ + remote_folder_id: folder.id, + internal_id: folder.internal_id, + internal_parent_folder_id: folder.internal_parent_folder_id, + })); this.logger.log(`Batch size: ${batch.length} at depth ${depth}`); depth++; } diff --git a/packages/api/src/filestorage/folder/services/onedrive/mappers.ts b/packages/api/src/filestorage/folder/services/onedrive/mappers.ts index 28182b52b..8e2607fe7 100644 --- a/packages/api/src/filestorage/folder/services/onedrive/mappers.ts +++ b/packages/api/src/filestorage/folder/services/onedrive/mappers.ts @@ -124,16 +124,14 @@ export class OnedriveFolderMapper implements IFolderMapper { } const result = { + id: folder.internal_id ?? null, + parent_folder_id: folder.internal_parent_folder_id ?? null, remote_id: folder.id, remote_data: folder, name: folder.name, folder_url: folder.webUrl, description: folder.description, drive_id: null, - parent_folder_id: await this.utils.getFolderIdFromRemote( - folder.parentReference?.id, - connectionId, - ), // permission: opts.permissions?.[0] || null, permissions: [], size: folder.size.toString(), diff --git a/packages/api/src/filestorage/folder/services/onedrive/types.ts b/packages/api/src/filestorage/folder/services/onedrive/types.ts index 3d9448a19..60b68be1b 100644 --- a/packages/api/src/filestorage/folder/services/onedrive/types.ts +++ b/packages/api/src/filestorage/folder/services/onedrive/types.ts @@ -57,6 +57,10 @@ export interface OnedriveFolderInput { readonly lastModifiedBy?: IdentitySet; /** If this property is non-null, it indicates that the driveItem is the top-most driveItem in the drive. */ readonly root?: any; + + // internal fields + internal_id?: string; + internal_parent_folder_id?: string; } /**