feat: onedrive: request error handling for drive, user and group

packages/api/src/filestorage/drive/services/onedrive/index.ts

@@ -6,14 +6,17 @@ import { SyncParam } from '@@core/utils/types/interface';
 import { FileStorageObject } from '@filestorage/@lib/@types';
 import { IDriveService } from '@filestorage/drive/types';
 import { Injectable } from '@nestjs/common';
-import axios from 'axios';
+import axios, { AxiosRequestConfig, AxiosResponse } from 'axios';
 import { ServiceRegistry } from '../registry.service';
 import { OnedriveDriveOutput } from './types';
 import { DesunifyReturnType } from '@@core/utils/types/desunify.input';
 import { OriginalDriveOutput } from '@@core/utils/types/original/original.file-storage';
 
 @Injectable()
 export class OnedriveService implements IDriveService {
+  private readonly MAX_RETRIES: number = 6;
+  private readonly INITIAL_BACKOFF_MS: number = 1000;
+
   constructor(
     private prisma: PrismaService,
     private logger: LoggerService,
@@ -26,14 +29,29 @@ export class OnedriveService implements IDriveService {
     this.registry.registerService('onedrive', this);
   }
 
+  /**
+   * Adds a new OneDrive drive.
+   * @param driveData - Drive data to add.
+   * @param linkedUserId - ID of the linked user.
+   * @returns API response with the original drive output.
+   */
   async addDrive(
     driveData: DesunifyReturnType,
     linkedUserId: string,
   ): Promise<ApiResponse<OriginalDriveOutput>> {
-    // No API to add drive in onedrive
-    return;
+    // No API to add drive in OneDrive
+    return {
+      data: null,
+      message: 'Add drive not supported for OneDrive.',
+      statusCode: 501,
+    };
   }
 
+  /**
+   * Synchronizes OneDrive drives.
+   * @param data - Synchronization parameters.
+   * @returns API response with an array of OneDrive drive outputs.
+   */
   async sync(data: SyncParam): Promise<ApiResponse<OnedriveDriveOutput[]>> {
     try {
       const { linkedUserId } = data;
@@ -46,25 +64,113 @@ export class OnedriveService implements IDriveService {
         },
       });
 
-      const resp = await axios.get(`${connection.account_url}/v1.0/drives`, {
+      const config: AxiosRequestConfig = {
+        method: 'get',
+        url: `${connection.account_url}/v1.0/drives`,
         headers: {
           'Content-Type': 'application/json',
           Authorization: `Bearer ${this.cryptoService.decrypt(
             connection.access_token,
           )}`,
         },
-      });
+      };
+
+      const resp: AxiosResponse = await this.makeRequestWithRetry(config);
 
       const drives: OnedriveDriveOutput[] = resp.data.value;
-      this.logger.log(`Synced onedrive drives !`);
+      this.logger.log(`Synced OneDrive drives successfully.`);
 
       return {
         data: drives,
-        message: 'Onedrive drives retrived',
+        message: 'OneDrive drives retrieved successfully.',
         statusCode: 200,
       };
-    } catch (error) {
+    } catch (error: any) {
+      this.logger.error(
+        `Error syncing OneDrive drives: ${error.message}`,
+        error,
+      );
       throw error;
     }
   }
+
+  /**
+   * Makes an HTTP request with retry logic for handling 500 and 429 errors.
+   * @param config - Axios request configuration.
+   * @returns Axios response.
+   */
+  private async makeRequestWithRetry(
+    config: AxiosRequestConfig,
+  ): Promise<AxiosResponse> {
+    let attempts = 0;
+    let backoff: number = this.INITIAL_BACKOFF_MS;
+
+    while (attempts < this.MAX_RETRIES) {
+      try {
+        const response: AxiosResponse = await axios(config);
+        return response;
+      } catch (error: any) {
+        attempts++;
+
+        // Handle rate limiting (429) and server errors (500+)
+        if (
+          (error.response && error.response.status === 429) ||
+          (error.response && error.response.status >= 500) ||
+          error.code === 'ECONNABORTED' ||
+          error.code === 'ETIMEDOUT' ||
+          error.response?.code === 'ETIMEDOUT'
+        ) {
+          const retryAfter = this.getRetryAfter(
+            error.response?.headers['retry-after'],
+          );
+          const delayTime: number = Math.max(retryAfter * 1000, backoff);
+
+          this.logger.warn(
+            `Request failed with ${
+              error.code || error.response?.status
+            }. Retrying in ${delayTime}ms (Attempt ${attempts}/${
+              this.MAX_RETRIES
+            })`,
+          );
+
+          await this.delay(delayTime);
+          backoff *= 2; // Exponential backoff
+          continue;
+        }
+
+        // Handle other errors
+        this.logger.error(`Request failed: ${error.message}`, error);
+        throw error;
+      }
+    }
+
+    this.logger.error(
+      'Max retry attempts reached. Request failed.',
+      OnedriveService.name,
+    );
+    throw new Error('Max retry attempts reached.');
+  }
+
+  /**
+   * Parses the Retry-After header to determine the wait time.
+   * @param retryAfterHeader - Value of the Retry-After header.
+   * @returns Retry delay in seconds.
+   */
+  private getRetryAfter(retryAfterHeader: string | undefined): number {
+    if (!retryAfterHeader) {
+      return 1; // Default to 1 second if header is missing
+    }
+
+    const retryAfterSeconds: number = parseInt(retryAfterHeader, 10);
+    return isNaN(retryAfterSeconds) ? 1 : retryAfterSeconds;
+  }
+
+  /**
+   * Delays execution for the specified duration.
+   * @param ms - Duration in milliseconds.
+   * @returns Promise that resolves after the delay.
+   */
+  private delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
 }

packages/api/src/filestorage/file/services/onedrive/index.ts

@@ -112,9 +112,6 @@ export class OnedriveService implements IFileService {
         }
 
         if (files.length > 0) {
-          // Assign and ingest permissions for the ingested files
-          await this.ingestPermissionsForFiles(files, connection);
-
           const ingestedFiles = await this.ingestFiles(
             files,
             connection,
@@ -253,6 +250,11 @@ export class OnedriveService implements IFileService {
       'onedrive files ingestion',
     );
 
+    await this.ingestPermissionsForFiles(
+      Array.from(uniqueFiles.values()),
+      connection,
+    );
+
     return this.ingestService.ingestData<
       UnifiedFilestorageFileOutput,
       OnedriveFileOutput
@@ -279,7 +281,7 @@ export class OnedriveService implements IFileService {
   ): Promise<OnedriveFileOutput[]> {
     const allPermissions: OnedrivePermissionOutput[] = [];
     const fileIdToRemotePermissionIdMap: Map<string, string[]> = new Map();
-    const batchSize = 100; // simultaneous requests
+    const batchSize = 10; // simultaneous requests
 
     const files = allFiles.filter((f) => !f.deleted);
 
@@ -309,6 +311,8 @@ export class OnedriveService implements IFileService {
         }),
       );
 
+      this.delay(1000); // delay to avoid rate limiting
+
       allPermissions.push(...permissions.flat());
     }
 
@@ -331,7 +335,9 @@ export class OnedriveService implements IFileService {
       'permission',
     );
 
-    this.logger.log(`Ingested ${allPermissions.length} permissions for files.`);
+    this.logger.log(
+      `Ingested ${syncedPermissions.length} permissions for files.`,
+    );
 
     const permissionIdMap: Map<string, string> = new Map(
       syncedPermissions.map((permission) => [

packages/api/src/filestorage/folder/services/onedrive/index.ts

@@ -289,8 +289,26 @@ export class OnedriveService implements IFolderService {
       );
     } while (deltaLink);
 
-    const deletedFolders = onedriveFolders.filter((f: any) => f.deleted);
-    const updatedFolders = onedriveFolders.filter((f: any) => !f.deleted);
+    // Sort folders by lastModifiedDateTime in descending order (newest first)
+    const sortedFolders = [...onedriveFolders].sort((a, b) => {
+      const dateA = new Date(a.lastModifiedDateTime).getTime();
+      const dateB = new Date(b.lastModifiedDateTime).getTime();
+      return dateB - dateA;
+    });
+
+    const uniqueFolders = sortedFolders.reduce((acc, folder) => {
+      if (!acc.has(folder.id)) {
+        acc.set(folder.id, folder);
+      }
+      return acc;
+    }, new Map<string, OnedriveFolderOutput>());
+
+    const deletedFolders = Array.from(uniqueFolders.values()).filter(
+      (f: any) => f.deleted,
+    );
+    const updatedFolders = Array.from(uniqueFolders.values()).filter(
+      (f: any) => !f.deleted,
+    );
 
     // handle deleted folders
     await Promise.all(
@@ -334,7 +352,7 @@ export class OnedriveService implements IFolderService {
   ): Promise<OnedriveFolderOutput[]> {
     const allPermissions: OnedrivePermissionOutput[] = [];
     const folderIdToRemotePermissionIdMap: Map<string, string[]> = new Map();
-    const batchSize = 100; // simultaneous requests
+    const batchSize = 10; // simultaneous requests
 
     const folders = allFolders.filter((f) => !f.deleted);
 
@@ -387,7 +405,7 @@ export class OnedriveService implements IFolderService {
     );
 
     this.logger.log(
-      `Ingested ${allPermissions.length} permissions for folders.`,
+      `Ingested ${syncedPermissions.length} permissions for folders.`,
     );
 
     const permissionIdMap: Map<string, string> = new Map(