From b21446fe4255f21ed23edefee5bda3ffa51913a5 Mon Sep 17 00:00:00 2001 From: amit <1mitccc@gmail.com> Date: Fri, 20 Dec 2024 10:47:21 +0530 Subject: [PATCH] feat: ondrive: integrate permissions with files and folder sync --- .../unification/ingest-data.service.ts | 1 + .../drive/services/onedrive/types.ts | 2 + .../file/services/onedrive/index.ts | 197 ++++++++++++++---- .../file/services/onedrive/mappers.ts | 36 +--- .../file/services/onedrive/types.ts | 3 + .../folder/services/onedrive/index.ts | 141 +++++++++++++ .../folder/services/onedrive/mappers.ts | 2 +- .../folder/services/onedrive/types.ts | 1 + .../permission/services/onedrive/mappers.ts | 6 +- .../permission/services/onedrive/types.ts | 4 + 10 files changed, 313 insertions(+), 80 deletions(-) diff --git a/packages/api/src/@core/@core-services/unification/ingest-data.service.ts b/packages/api/src/@core/@core-services/unification/ingest-data.service.ts index 504217db6..c78da6f09 100644 --- a/packages/api/src/@core/@core-services/unification/ingest-data.service.ts +++ b/packages/api/src/@core/@core-services/unification/ingest-data.service.ts @@ -139,6 +139,7 @@ export class IngestDataService { ingestParams, ); } catch (syncError) { + console.log(syncError, 'syncError in ingest-data.service.ts'); this.logger.error( `Error syncing ${integrationId} ${commonObject}: ${syncError.message}`, syncError, diff --git a/packages/api/src/filestorage/drive/services/onedrive/types.ts b/packages/api/src/filestorage/drive/services/onedrive/types.ts index 9cc1f591d..f4af5fdb9 100644 --- a/packages/api/src/filestorage/drive/services/onedrive/types.ts +++ b/packages/api/src/filestorage/drive/services/onedrive/types.ts @@ -56,6 +56,8 @@ export interface IdentitySet { readonly phone?: Identity; /** Identity representing a user. */ readonly user?: Identity; + /** Identity representing a group. */ + readonly group?: Identity; } /** diff --git a/packages/api/src/filestorage/file/services/onedrive/index.ts b/packages/api/src/filestorage/file/services/onedrive/index.ts index 52f2c270a..1e7dc05c7 100644 --- a/packages/api/src/filestorage/file/services/onedrive/index.ts +++ b/packages/api/src/filestorage/file/services/onedrive/index.ts @@ -14,6 +14,9 @@ import { IngestDataService } from '@@core/@core-services/unification/ingest-data import { Connection } from '@@core/connections/@utils/types'; import { UnifiedFilestorageFileOutput } from '@filestorage/file/types/model.unified'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { OnedrivePermissionOutput } from '@filestorage/permission/services/onedrive/types'; +import { UnifiedFilestoragePermissionOutput } from '@filestorage/permission/types/model.unified'; + @Injectable() export class OnedriveService implements IFileService { private readonly MAX_RETRIES: number = 6; @@ -109,12 +112,16 @@ export class OnedriveService implements IFileService { } if (files.length > 0) { + // Assign and ingest permissions for the ingested files + await this.ingestPermissionsForFiles(files, connection); + const ingestedFiles = await this.ingestFiles( files, connection, custom_field_mappings, ingestParams, ); + this.logger.log( `Ingested ${ingestedFiles.length} files from OneDrive.`, 'onedrive files ingestion', @@ -137,7 +144,9 @@ export class OnedriveService implements IFileService { ); } } else { - const lastSyncTime = await this.getLastSyncTime(connection); + const lastSyncTime = await this.getLastSyncTime( + connection.id_connection, + ); const deltaLink = lastSyncTime ? `${ connection.account_url @@ -258,29 +267,138 @@ export class OnedriveService implements IFileService { ); } - private async getLastSyncTime(connection: { - id_connection: string; - }): Promise { - const lastSyncTime = await this.prisma.fs_files.findFirst({ - where: { - id_connection: connection.id_connection, - }, - orderBy: { - remote_modified_at: { - sort: 'desc', - nulls: 'last', - }, - }, - select: { - remote_modified_at: true, - }, - }); + /** + * Ingests and assigns permissions for files. + * @param allFiles - Array of OnedriveFileOutput to process. + * @param connection - The connection object. + * @returns The updated array of OnedriveFileOutput with ingested permissions. + */ + private async ingestPermissionsForFiles( + allFiles: OnedriveFileOutput[], + connection: Connection, + ): Promise { + const allPermissions: OnedrivePermissionOutput[] = []; + const fileIdToRemotePermissionIdMap: Map = new Map(); + const batchSize = 100; // simultaneous requests + + const files = allFiles.filter((f) => !f.deleted); + + for (let i = 0; i < files.length; i += batchSize) { + const batch = files.slice(i, i + batchSize); + const permissions = await Promise.all( + batch.map(async (file) => { + const permissionConfig: AxiosRequestConfig = { + timeout: 30000, + method: 'get', + url: `${connection.account_url}/v1.0/me/drive/items/${file.id}/permissions`, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }; - this.logger.log( - `Last file sync time: ${lastSyncTime?.remote_modified_at}`, - 'onedrive files sync', + const resp = await this.makeRequestWithRetry(permissionConfig); + const permissions = resp.data.value; + fileIdToRemotePermissionIdMap.set( + file.id, + permissions.map((p) => p.id), + ); + return permissions; + }), + ); + + allPermissions.push(...permissions.flat()); + } + + const uniquePermissions = Array.from( + new Map( + allPermissions.map((permission) => [permission.id, permission]), + ).values(), ); - return lastSyncTime?.remote_modified_at ?? null; + + await this.assignUserAndGroupIdsToPermissions(uniquePermissions); + + const syncedPermissions = await this.ingestService.ingestData< + UnifiedFilestoragePermissionOutput, + OnedrivePermissionOutput + >( + uniquePermissions, + 'onedrive', + connection.id_connection, + 'filestorage', + 'permission', + ); + + this.logger.log(`Ingested ${allPermissions.length} permissions for files.`); + + const permissionIdMap: Map = new Map( + syncedPermissions.map((permission) => [ + permission.remote_id, + permission.id_fs_permission, + ]), + ); + + files.forEach((file) => { + if (fileIdToRemotePermissionIdMap.has(file.id)) { + file.internal_permissions = fileIdToRemotePermissionIdMap + .get(file.id) + ?.map((permission) => permissionIdMap.get(permission)) + .filter((id) => id !== undefined); + } + }); + + return allFiles; + } + + private async assignUserAndGroupIdsToPermissions( + permissions: OnedrivePermissionOutput[], + ): Promise { + const userLookupCache: Map = new Map(); + const groupLookupCache: Map = new Map(); + + for (const permission of permissions) { + if (permission.grantedToV2?.user?.id) { + const remote_user_id = permission.grantedToV2.user.id; + if (userLookupCache.has(remote_user_id)) { + permission.internal_user_id = userLookupCache.get(remote_user_id); + continue; + } + const user = await this.prisma.fs_users.findFirst({ + where: { + remote_id: remote_user_id, + }, + select: { + id_fs_user: true, + }, + }); + if (user) { + permission.internal_user_id = user.id_fs_user; + userLookupCache.set(remote_user_id, user.id_fs_user); + } + } + + if (permission.grantedToV2?.group?.id) { + const remote_group_id = permission.grantedToV2.group.id; + if (groupLookupCache.has(remote_group_id)) { + permission.internal_group_id = groupLookupCache.get(remote_group_id); + continue; + } + const group = await this.prisma.fs_groups.findFirst({ + where: { + remote_id: remote_group_id, + }, + select: { + id_fs_group: true, + }, + }); + if (group) { + permission.internal_group_id = group.id_fs_group; + groupLookupCache.set(remote_group_id, group.id_fs_group); + } + } + } } private async syncFolder( @@ -306,6 +424,8 @@ export class OnedriveService implements IFileService { (elem: any) => !elem.folder, // files don't have a folder property ); + await this.ingestPermissionsForFiles(files, connection); + return files; } catch (error: any) { if (error.response?.status === 404) { @@ -332,27 +452,15 @@ export class OnedriveService implements IFileService { } throw error; } + } - // Add permissions (shared link is also included in permissions in OneDrive) - // await Promise.all( - // files.map(async (driveItem: OnedriveFileOutput) => { - // const permissionsConfig: AxiosRequestConfig = { - // method: 'get', - // url: `${connection.account_url}/v1.0/me/drive/items/${driveItem.id}/permissions`, - // headers: { - // 'Content-Type': 'application/json', - // Authorization: `Bearer ${this.cryptoService.decrypt( - // connection.access_token, - // )}`, - // }, - // }; - - // const permissionsResp: AxiosResponse = await this.makeRequestWithRetry( - // permissionsConfig, - // ); - // driveItem.permissions = permissionsResp.data.value; - // }), - // ); + private async getLastSyncTime(connectionId: string): Promise { + const lastSync = await this.prisma.fs_files.findFirst({ + where: { id_connection: connectionId }, + orderBy: { remote_modified_at: { sort: 'desc', nulls: 'last' } }, + }); + this.logger.log(`Last sync time: ${lastSync?.remote_modified_at}`); + return lastSync ? lastSync.remote_modified_at : null; } async downloadFile(fileId: string, connection: any): Promise { @@ -435,6 +543,11 @@ export class OnedriveService implements IFileService { continue; } + // handle 410 gone errors + if (error.response?.status === 410 && config.url.includes('delta')) { + // todo: handle 410 gone errors + } + throw error; } } diff --git a/packages/api/src/filestorage/file/services/onedrive/mappers.ts b/packages/api/src/filestorage/file/services/onedrive/mappers.ts index b36b70f13..410e668c8 100644 --- a/packages/api/src/filestorage/file/services/onedrive/mappers.ts +++ b/packages/api/src/filestorage/file/services/onedrive/mappers.ts @@ -88,37 +88,6 @@ export class OnedriveFileMapper implements IFileMapper { } } - const opts: any = {}; - if (file.permissions?.length) { - const permissions = await this.coreUnificationService.unify< - OriginalPermissionOutput[] - >({ - sourceObject: file.permissions, - targetType: FileStorageObject.permission, - providerName: 'onedrive', - vertical: 'filestorage', - connectionId, - customFieldMappings: [], - }); - opts.permissions = permissions; - - // shared link - if (file.permissions.some((p) => p.link)) { - const sharedLinks = - await this.coreUnificationService.unify({ - sourceObject: file.permissions.find((p) => p.link), - targetType: FileStorageObject.sharedlink, - providerName: 'onedrive', - vertical: 'filestorage', - connectionId, - customFieldMappings: [], - }); - opts.shared_links = sharedLinks; - } - } - - // todo: handle folder - return { remote_id: file.id, remote_data: file, @@ -136,9 +105,8 @@ export class OnedriveFileMapper implements IFileMapper { mime_type: file.file.mimeType, size: file.size.toString(), folder_id: null, - // permission: opts.permissions?.[0] || null, - permissions: null, - shared_link: opts.shared_links?.[0] || null, + permissions: file.internal_permissions, + shared_link: null, field_mappings, }; } diff --git a/packages/api/src/filestorage/file/services/onedrive/types.ts b/packages/api/src/filestorage/file/services/onedrive/types.ts index d2b3a12ab..535c433e0 100644 --- a/packages/api/src/filestorage/file/services/onedrive/types.ts +++ b/packages/api/src/filestorage/file/services/onedrive/types.ts @@ -57,6 +57,9 @@ export interface OnedriveFileOutput { readonly video?: Video; /** WebDAV compatible URL for the item. */ readonly webDavUrl?: string; + + // INTERNAL FIELDS + internal_permissions?: string[]; } /** diff --git a/packages/api/src/filestorage/folder/services/onedrive/index.ts b/packages/api/src/filestorage/folder/services/onedrive/index.ts index c9f2cd32c..0497459ab 100644 --- a/packages/api/src/filestorage/folder/services/onedrive/index.ts +++ b/packages/api/src/filestorage/folder/services/onedrive/index.ts @@ -16,6 +16,8 @@ import { OnedriveFileOutput } from '@filestorage/file/services/onedrive/types'; import { v4 as uuidv4 } from 'uuid'; import { Connection } from '@@core/connections/@utils/types'; import { fs_folders } from '@prisma/client'; +import { OnedrivePermissionOutput } from '@filestorage/permission/services/onedrive/types'; +import { UnifiedFilestoragePermissionOutput } from '@filestorage/permission/types/model.unified'; @Injectable() export class OnedriveService implements IFolderService { @@ -117,6 +119,8 @@ export class OnedriveService implements IFolderService { `${folders.length} OneDrive folders synced successfully.`, ); + await this.ingestPermissionsForFolders(folders, connection); + return { data: folders, message: 'OneDrive folders synced', @@ -124,6 +128,7 @@ export class OnedriveService implements IFolderService { }; } catch (error) { this.logger.error('Error in OneDrive sync:', error); + console.log(error, 'error in onedrive service'); throw error; } } @@ -317,6 +322,142 @@ export class OnedriveService implements IFolderService { return foldersToSync; } + /** + * Ingests and assigns permissions for folders. + * @param allFolders - Array of OneDriveFolderOutput to process. + * @param connection - The connection object. + * @returns The updated array of OneDriveFolderOutput with ingested permissions. + */ + private async ingestPermissionsForFolders( + allFolders: OnedriveFolderOutput[], + connection: Connection, + ): Promise { + const allPermissions: OnedrivePermissionOutput[] = []; + const folderIdToRemotePermissionIdMap: Map = new Map(); + const batchSize = 100; // simultaneous requests + + const folders = allFolders.filter((f) => !f.deleted); + + for (let i = 0; i < folders.length; i += batchSize) { + const batch = folders.slice(i, i + batchSize); + const permissions = await Promise.all( + batch.map(async (folder) => { + const permissionConfig: AxiosRequestConfig = { + timeout: 30000, + method: 'get', + url: `${connection.account_url}/v1.0/me/drive/items/${folder.id}/permissions`, + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }; + + const resp = await this.makeRequestWithRetry(permissionConfig); + const permissions = resp.data.value; + folderIdToRemotePermissionIdMap.set( + folder.id, + permissions.map((p) => p.id), + ); + return permissions; + }), + ); + + allPermissions.push(...permissions.flat()); + } + + const uniquePermissions = Array.from( + new Map( + allPermissions.map((permission) => [permission.id, permission]), + ).values(), + ); + + await this.assignUserAndGroupIdsToPermissions(uniquePermissions); + + const syncedPermissions = await this.ingestService.ingestData< + UnifiedFilestoragePermissionOutput, + OnedrivePermissionOutput + >( + uniquePermissions, + 'onedrive', + connection.id_connection, + 'filestorage', + 'permission', + ); + + this.logger.log( + `Ingested ${allPermissions.length} permissions for folders.`, + ); + + const permissionIdMap: Map = new Map( + syncedPermissions.map((permission) => [ + permission.remote_id, + permission.id_fs_permission, + ]), + ); + + folders.forEach((folder) => { + if (folderIdToRemotePermissionIdMap.has(folder.id)) { + folder.internal_permissions = folderIdToRemotePermissionIdMap + .get(folder.id) + ?.map((permission) => permissionIdMap.get(permission)) + .filter((id) => id !== undefined); + } + }); + + return allFolders; + } + + private async assignUserAndGroupIdsToPermissions( + permissions: OnedrivePermissionOutput[], + ): Promise { + const userLookupCache: Map = new Map(); + const groupLookupCache: Map = new Map(); + + for (const permission of permissions) { + if (permission.grantedToV2?.user?.id) { + const remote_user_id = permission.grantedToV2.user.id; + if (userLookupCache.has(remote_user_id)) { + permission.internal_user_id = userLookupCache.get(remote_user_id); + continue; + } + const user = await this.prisma.fs_users.findFirst({ + where: { + remote_id: remote_user_id, + }, + select: { + id_fs_user: true, + }, + }); + if (user) { + permission.internal_user_id = user.id_fs_user; + userLookupCache.set(remote_user_id, user.id_fs_user); + } + } + + if (permission.grantedToV2?.group?.id) { + const remote_group_id = permission.grantedToV2.group.id; + if (groupLookupCache.has(remote_group_id)) { + permission.internal_group_id = groupLookupCache.get(remote_group_id); + continue; + } + const group = await this.prisma.fs_groups.findFirst({ + where: { + remote_id: remote_group_id, + }, + select: { + id_fs_group: true, + }, + }); + if (group) { + permission.internal_group_id = group.id_fs_group; + groupLookupCache.set(remote_group_id, group.id_fs_group); + } + } + } + } + /** * Assigns internal parent IDs to OneDrive folders, ensuring proper parent-child relationships. * @param folders - Array of OneDriveFolderOutput to process. diff --git a/packages/api/src/filestorage/folder/services/onedrive/mappers.ts b/packages/api/src/filestorage/folder/services/onedrive/mappers.ts index c137556ab..a837065a2 100644 --- a/packages/api/src/filestorage/folder/services/onedrive/mappers.ts +++ b/packages/api/src/filestorage/folder/services/onedrive/mappers.ts @@ -142,7 +142,7 @@ export class OnedriveFolderMapper implements IFolderMapper { description: folder.description, drive_id: null, // permission: opts.permissions?.[0] || null, - permissions: [], + permissions: folder.internal_permissions, size: folder.size.toString(), shared_link: opts.shared_links?.[0] || null, field_mappings, diff --git a/packages/api/src/filestorage/folder/services/onedrive/types.ts b/packages/api/src/filestorage/folder/services/onedrive/types.ts index 60b68be1b..837c98f3c 100644 --- a/packages/api/src/filestorage/folder/services/onedrive/types.ts +++ b/packages/api/src/filestorage/folder/services/onedrive/types.ts @@ -61,6 +61,7 @@ export interface OnedriveFolderInput { // internal fields internal_id?: string; internal_parent_folder_id?: string; + internal_permissions?: string[]; } /** diff --git a/packages/api/src/filestorage/permission/services/onedrive/mappers.ts b/packages/api/src/filestorage/permission/services/onedrive/mappers.ts index a78cedb67..04c74707d 100644 --- a/packages/api/src/filestorage/permission/services/onedrive/mappers.ts +++ b/packages/api/src/filestorage/permission/services/onedrive/mappers.ts @@ -80,15 +80,15 @@ export class OnedrivePermissionMapper implements IPermissionMapper { return { remote_id: permission.id, remote_data: permission, - roles: permission.roles.map((role) => role.toUpperCase()), + roles: permission.roles?.map((role) => role.toUpperCase()), type: permission.link?.type === 'edit' ? 'WRITE' : permission.link?.type === 'view' ? 'READ' : permission.link?.type, - user_id: null, - group_id: null, + user_id: permission.internal_user_id, + group_id: permission.internal_group_id, field_mappings, }; } diff --git a/packages/api/src/filestorage/permission/services/onedrive/types.ts b/packages/api/src/filestorage/permission/services/onedrive/types.ts index bda7dfa81..fe9facbda 100644 --- a/packages/api/src/filestorage/permission/services/onedrive/types.ts +++ b/packages/api/src/filestorage/permission/services/onedrive/types.ts @@ -27,6 +27,10 @@ export interface OnedrivePermissionOutput { shareId?: string; /** A format of yyyy-MM-ddTHH:mm:ssZ of DateTimeOffset indicates the expiration time of the permission. DateTime.MinValue indicates there's no expiration set for this permission. Optional. */ expirationDateTime?: string; + + // INTERNAL + internal_user_id?: string; + internal_group_id?: string; } /**