Skip to content

Commit

Permalink
fix: better handle deleted folder for one drive
Browse files Browse the repository at this point in the history
  • Loading branch information
@amuwal
amuwal committed Dec 19, 2024
1 parent dfa673c commit aa2abb0
Show file tree
Hide file tree
Showing 4 changed files with 228 additions and 26 deletions.
2 changes: 2 additions & 0 deletions packages/api/src/filestorage/file/file.module.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ import { SharepointService } from './services/sharepoint';
import { SharepointFileMapper } from './services/sharepoint/mappers';
import { SyncService } from './sync/sync.service';
import { GoogleDriveQueueProcessor } from './services/googledrive/processor';
import { FolderModule } from '../folder/folder.module';

@Module({
imports: [FolderModule],
controllers: [FileController],
providers: [
FileService,
Expand Down
19 changes: 15 additions & 4 deletions packages/api/src/filestorage/file/services/onedrive/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { Injectable } from '@nestjs/common';
import axios, { AxiosRequestConfig, AxiosResponse } from 'axios';
import { ServiceRegistry } from '../registry.service';
import { OnedriveFileOutput } from './types';

import { OnedriveService as OnedriveFolderService } from '@filestorage/folder/services/onedrive';
@Injectable()
export class OnedriveService implements IFileService {
private readonly MAX_RETRIES: number = 5;
Expand All @@ -21,6 +21,7 @@ export class OnedriveService implements IFileService {
private logger: LoggerService,
private cryptoService: EncryptionService,
private registry: ServiceRegistry,
private onedriveFolderService: OnedriveFolderService,
) {
this.logger.setContext(
`${FileStorageObject.file.toUpperCase()}:${OnedriveService.name}`,
Expand Down Expand Up @@ -96,6 +97,7 @@ export class OnedriveService implements IFileService {
statusCode: 200,
};
} catch (error) {
console.log(error);
this.logger.error(
`Error syncing OneDrive files: ${error.message}`,
error,
Expand Down Expand Up @@ -129,16 +131,25 @@ export class OnedriveService implements IFileService {
return files;
} catch (error: any) {
if (error.response?.status === 404) {
// Folder not found, mark as deleted
await this.prisma.fs_folders.updateMany({
const internalFolder = await this.prisma.fs_folders.findFirst({
where: {
remote_id: folderId,
id_connection: connection.id_connection,
},
data: {
select: {
id_fs_folder: true,
remote_was_deleted: true,
},
});
if (internalFolder && internalFolder.remote_was_deleted) {
this.logger.debug(
`Folder ${internalFolder.id_fs_folder} not found in OneDrive, marking as deleted in internal database.`,
);
await this.onedriveFolderService.handleDeletedFolder(
internalFolder.id_fs_folder,
connection,
);
}
return [];
}
throw error;
Expand Down
2 changes: 1 addition & 1 deletion packages/api/src/filestorage/folder/folder.module.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,6 @@ import { SyncService } from './sync/sync.service';
DropboxFolderMapper,
GoogleDriveFolderService,
],
exports: [SyncService],
exports: [SyncService, OnedriveService],
})
export class FolderModule {}
231 changes: 210 additions & 21 deletions packages/api/src/filestorage/folder/services/onedrive/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import { IngestDataService } from '@@core/@core-services/unification/ingest-data
import { UnifiedFilestorageFileOutput } from '@filestorage/file/types/model.unified';
import { OnedriveFileOutput } from '@filestorage/file/services/onedrive/types';
import { v4 as uuidv4 } from 'uuid';
import { Connection } from '@@core/connections/@utils/types';
import { fs_folders } from '@prisma/client';

@Injectable()
export class OnedriveService implements IFolderService {
Expand Down Expand Up @@ -80,6 +82,29 @@ export class OnedriveService implements IFolderService {
}
}

async sync(data: SyncParam): Promise<ApiResponse<OnedriveFolderOutput[]>> {
try {
this.logger.log('Syncing OneDrive folders');
const { linkedUserId } = data;

const folders: OnedriveFolderOutput[] =
await this.iterativeGetOnedriveFolders('root', linkedUserId);

this.logger.log(
`${folders.length} OneDrive folders synced successfully.`,
);

return {
data: folders,
message: 'OneDrive folders synced',
statusCode: 200,
};
} catch (error) {
this.logger.error('Error in OneDrive sync:', error);
throw error;
}
}

async iterativeGetOnedriveFolders(
remote_folder_id: string,
linkedUserId: string,
Expand Down Expand Up @@ -167,17 +192,19 @@ export class OnedriveService implements IFolderService {
}));
} catch (error: any) {
if (error.response && error.response.status === 404) {
console.log('Folder not found', folder.remote_folder_id);
// Folder not found, mark as deleted
await this.prisma.fs_folders.updateMany({
console.log('Found deleted folder');
const f = await this.prisma.fs_folders.findFirst({
where: {
remote_id: folder.remote_folder_id,
id_connection: connection.id_connection,
},
data: {
remote_was_deleted: true,
select: {
id_fs_folder: true,
},
});
if (f) {
await this.handleDeletedFolder(f.id_fs_folder, connection);
}
return [];
}
throw error;
Expand Down Expand Up @@ -206,25 +233,187 @@ export class OnedriveService implements IFolderService {
}
}

async sync(data: SyncParam): Promise<ApiResponse<OnedriveFolderOutput[]>> {
try {
this.logger.log('Syncing OneDrive folders');
const { linkedUserId } = data;
/**
* Handles the deletion of a folder by marking it and its children as deleted in the database.
* @param folderId - The internal ID of the folder to be marked as deleted.
* @param connection - The connection object containing the connection details.
*/
async handleDeletedFolder(folderId: string, connection: Connection) {
const folder = await this.prisma.fs_folders.findFirst({
where: {
id_fs_folder: folderId,
id_connection: connection.id_connection,
},
select: {
remote_was_deleted: true,
id_fs_folder: true,
parent_folder: true,
},
});

if (!folder || folder.remote_was_deleted) {
this.logger.debug('Folder already marked deleted');
return;
}

const folders: OnedriveFolderOutput[] =
await this.iterativeGetOnedriveFolders('root', linkedUserId);
const highestDeletedPredecessor = await this.findHigestDeletedPredecessor(
folder as fs_folders,
connection,
);

this.logger.log(
`${folders.length} OneDrive folders synced successfully.`,
if (highestDeletedPredecessor === 'not_deleted') {
this.logger.debug(
"Higest deleted predecessor came out to be as 'not_deleted'",
);
return;
}

return {
data: folders,
message: 'OneDrive folders synced',
statusCode: 200,
};
} catch (error) {
this.logger.error('Error in OneDrive sync:', error);
const entitiesDeleted = await this.markChildrenAsDeleted(
highestDeletedPredecessor,
connection,
);

this.logger.debug(
`Deleted ${entitiesDeleted} entities for folder ${folderId}`,
);
}

private async markChildrenAsDeleted(
folderId: string,
connection: Connection,
): Promise<number> {
let entitiesDeleted = 0;

// we need to find all the children of this folder and mark them as deleted
const childFolders = await this.prisma.fs_folders.findMany({
where: {
parent_folder: folderId,
id_connection: connection.id_connection,
},
select: {
id_fs_folder: true,
remote_was_deleted: false,
},
});

const childFiles = await this.prisma.fs_files.findMany({
where: {
id_fs_folder: folderId,
id_connection: connection.id_connection,
},
select: {
id_fs_file: true,
remote_was_deleted: false,
},
});

const childFolderIds = childFolders.map((f) => f.id_fs_folder);
const childFileIds = childFiles.map((f) => f.id_fs_file);

await this.prisma.fs_folders.updateMany({
where: {
id_fs_folder: { in: childFolderIds },
},
data: {
remote_was_deleted: true,
},
});

await this.prisma.fs_files.updateMany({
where: {
id_fs_file: { in: childFileIds },
},
data: {
remote_was_deleted: true,
},
});

entitiesDeleted += childFolderIds.length + childFileIds.length;

const childFolderResults = await Promise.all(
childFolderIds.map(
async (id) => await this.markChildrenAsDeleted(id, connection),
),
);

entitiesDeleted += childFolderResults.reduce((acc, curr) => acc + curr, 0);
return entitiesDeleted;
}

private async findHigestDeletedPredecessor(
folder: fs_folders,
connection: Connection,
): Promise<string | 'not_deleted'> {
// if the folder has no parent, it is the root folder
if (!folder.parent_folder) {
return folder.id_fs_folder;
}

const remoteDeleted = folder.remote_was_deleted
? true
: await this.folderDeletedOnRemote(folder, connection);
if (!remoteDeleted) {
return 'not_deleted';
}

// if the folder is deleted, we need to find the highest deleted predecessor
const parentFolder = await this.prisma.fs_folders.findFirst({
where: {
id_fs_folder: folder.parent_folder,
id_connection: connection.id_connection,
},
select: {
remote_was_deleted: true,
id_fs_folder: true,
parent_folder: true,
},
});

if (!parentFolder) {
return folder.id_fs_folder;
}

const parentResult = await this.findHigestDeletedPredecessor(
parentFolder as fs_folders,
connection,
);

if (parentResult === 'not_deleted') {
return folder.id_fs_folder;
}

return parentResult;
}

/**
* Checks if a folder is deleted on the remote OneDrive service.
* @param folder - The folder to check.
* @param connection - The connection object containing the connection details.
* @returns True if the folder is deleted, false otherwise.
*/
private async folderDeletedOnRemote(
folder: fs_folders,
connection: Connection,
) {
try {
const remoteFolder = await this.makeRequestWithRetry({
timeout: 30000,
method: 'get',
url: `${connection.account_url}/v1.0/me/drive/items/${folder.remote_id}?$select=id,deleted`,
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.cryptoService.decrypt(
connection.access_token,
)}`,
},
});

return remoteFolder && remoteFolder.data.deleted;
} catch (error: any) {
if (error.response?.status === 404) {
// If we get a 404, we assume the folder is deleted
return true;
}
throw error;
}
}
Expand Down Expand Up @@ -303,7 +492,7 @@ export class OnedriveService implements IFolderService {
}

const retryAfterSeconds: number = parseInt(retryAfterHeader, 10);
return isNaN(retryAfterSeconds) ? 1 : retryAfterSeconds;
return isNaN(retryAfterSeconds) ? 1 : retryAfterSeconds + 0.5;
}

/**
Expand Down

0 comments on commit aa2abb0

Please sign in to comment.