Skip to content

Commit

Permalink
feat: basic error handling for onedrive file and folder sync
Browse files Browse the repository at this point in the history
  • Loading branch information
@amuwal
amuwal committed Dec 18, 2024
1 parent 63560e1 commit dfa673c
Show file tree
Hide file tree
Showing 2 changed files with 129 additions and 51 deletions.
79 changes: 59 additions & 20 deletions packages/api/src/filestorage/file/services/onedrive/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ export class OnedriveService implements IFileService {
const folders = await this.prisma.fs_folders.findMany({
where: {
id_connection: connection.id_connection,
remote_was_deleted: false,
},
select: {
remote_id: true,
Expand Down Expand Up @@ -106,22 +107,42 @@ export class OnedriveService implements IFileService {
connection: any,
folderId: string,
): Promise<OnedriveFileOutput[]> {
const config: AxiosRequestConfig = {
method: 'get',
url: `${connection.account_url}/v1.0/me/drive/items/${folderId}/children`,
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.cryptoService.decrypt(
connection.access_token,
)}`,
},
};
try {
const config: AxiosRequestConfig = {
timeout: 30000,
method: 'get',
url: `${connection.account_url}/v1.0/me/drive/items/${folderId}/children`,
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.cryptoService.decrypt(
connection.access_token,
)}`,
},
};

const resp: AxiosResponse = await this.makeRequestWithRetry(config);
const resp: AxiosResponse = await this.makeRequestWithRetry(config);

const files: OnedriveFileOutput[] = resp.data.value.filter(
(elem: any) => !elem.folder, // files don't have a folder property
);
const files: OnedriveFileOutput[] = resp.data.value.filter(
(elem: any) => !elem.folder, // files don't have a folder property
);

return files;
} catch (error: any) {
if (error.response?.status === 404) {
// Folder not found, mark as deleted
await this.prisma.fs_folders.updateMany({
where: {
remote_id: folderId,
id_connection: connection.id_connection,
},
data: {
remote_was_deleted: true,
},
});
return [];
}
throw error;
}

// Add permissions (shared link is also included in permissions in OneDrive)
// await Promise.all(
Expand All @@ -143,9 +164,8 @@ export class OnedriveService implements IFileService {
// driveItem.permissions = permissionsResp.data.value;
// }),
// );

return files;
}

async downloadFile(fileId: string, connection: any): Promise<Buffer> {
const config: AxiosRequestConfig = {
method: 'get',
Expand Down Expand Up @@ -178,8 +198,10 @@ export class OnedriveService implements IFileService {
const response: AxiosResponse = await axios(config);
return response;
} catch (error: any) {
attempts++;

// Handle rate limiting
if (error.response && error.response.status === 429) {
attempts++;
const retryAfter: number = this.getRetryAfter(
error.response.headers['retry-after'],
);
Expand All @@ -191,10 +213,27 @@ export class OnedriveService implements IFileService {

await this.delay(delayTime);
backoff *= 2; // Exponential backoff
} else {
this.logger.error('HTTP request failed:', error);
throw error;
continue;
}

// Handle timeout errors
if (
error.code === 'ECONNABORTED' ||
error.code === 'ETIMEDOUT' ||
error.response?.code === 'ETIMEDOUT'
) {
const delayTime: number = backoff;

this.logger.warn(
`Request timeout. Retrying in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`,
);

await this.delay(delayTime);
backoff *= 2;
continue;
}

throw error;
}
}

Expand Down
101 changes: 70 additions & 31 deletions packages/api/src/filestorage/folder/services/onedrive/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ export class OnedriveService implements IFolderService {
}

const config: AxiosRequestConfig = {
timeout: 30000,
method: 'post',
url: `${connection.account_url}/v1.0/drive/root/children`,
data: {
Expand Down Expand Up @@ -138,30 +139,49 @@ export class OnedriveService implements IFolderService {
internal_id: string;
internal_parent_folder_id: string;
}) => {
const childrenConfig: AxiosRequestConfig = {
method: 'get',
url: `${connection.account_url}/v1.0/me/drive/items/${folder.remote_folder_id}/children`,
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.cryptoService.decrypt(
connection.access_token,
)}`,
},
};

const resp: AxiosResponse = await this.makeRequestWithRetry(
childrenConfig,
);

const folders: OnedriveFolderOutput[] = resp.data.value.filter(
(driveItem: any) => driveItem.folder,
);

return folders.map((f: OnedriveFolderOutput) => ({
...f,
internal_id: uuidv4(),
internal_parent_folder_id: folder.internal_id,
}));
try {
const childrenConfig: AxiosRequestConfig = {
timeout: 30000,
method: 'get',
url: `${connection.account_url}/v1.0/me/drive/items/${folder.remote_folder_id}/children`,
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.cryptoService.decrypt(
connection.access_token,
)}`,
},
};

const resp: AxiosResponse = await this.makeRequestWithRetry(
childrenConfig,
);

const folders: OnedriveFolderOutput[] = resp.data.value.filter(
(driveItem: any) => driveItem.folder,
);

return folders.map((f: OnedriveFolderOutput) => ({
...f,
internal_id: uuidv4(),
internal_parent_folder_id: folder.internal_id,
}));
} catch (error: any) {
if (error.response && error.response.status === 404) {
console.log('Folder not found', folder.remote_folder_id);
// Folder not found, mark as deleted
await this.prisma.fs_folders.updateMany({
where: {
remote_id: folder.remote_folder_id,
id_connection: connection.id_connection,
},
data: {
remote_was_deleted: true,
},
});
return [];
}
throw error;
}
},
);

Expand Down Expand Up @@ -225,23 +245,42 @@ export class OnedriveService implements IFolderService {
const response: AxiosResponse = await axios(config);
return response;
} catch (error: any) {
attempts++;

// Handle rate limiting
if (error.response && error.response.status === 429) {
attempts++;
const retryAfter: number = this.getRetryAfter(
error.response.headers['retry-after'],
);
const delay: number = Math.max(retryAfter * 1000, backoff);
const delayTime: number = Math.max(retryAfter * 1000, backoff);

this.logger.warn(
`Rate limit hit. Retrying request in ${delay}ms (Attempt ${attempts}/${this.MAX_RETRIES})`,
`Rate limit hit. Retrying request in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`,
);

await this.delay(delay);
await this.delay(delayTime);
backoff *= 2; // Exponential backoff
} else {
this.logger.error('HTTP request failed:', error);
throw error;
continue;
}

// Handle timeout errors
if (
error.code === 'ECONNABORTED' ||
error.code === 'ETIMEDOUT' ||
error.response?.code === 'ETIMEDOUT'
) {
const delayTime: number = backoff;

this.logger.warn(
`Request timeout. Retrying in ${delayTime}ms (Attempt ${attempts}/${this.MAX_RETRIES})`,
);

await this.delay(delayTime);
backoff *= 2;
continue;
}

throw error;
}
}

Expand Down

0 comments on commit dfa673c

Please sign in to comment.