Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add integration with: OneDrive #692

Merged
merged 14 commits into from
Sep 9, 2024
Merged

feat: Add integration with: OneDrive #692

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
7e91440
Add drive object for one-drive integration
amuwal Sep 5, 2024
418a022
save state
amuwal Sep 6, 2024
1eee489
Add folder object for one-drive integration
amuwal Sep 6, 2024
67d8780
Update filestorage > folder > onedrive files
amuwal Sep 6, 2024
4422c20
Add mappers.ts and types.ts for one-drive file object
amuwal Sep 6, 2024
b95690b
Add file object to one-drive integration
amuwal Sep 6, 2024
43eef7c
Add user object to one-drive integration
amuwal Sep 6, 2024
753d7e3
Add group object to one-drive integration
amuwal Sep 7, 2024
069fd0b
Add types.ts for sharedLink and permission objects for one-drive inte…
amuwal Sep 7, 2024
a76f31d
Fix connectorUpdate script
amuwal Sep 7, 2024
1da83cf
Add permission object to one-drive integration
amuwal Sep 7, 2024
6a63c5b
Add sharedlink object to one-drive integration
amuwal Sep 7, 2024
6547159
Integrate sharedlink and permissions in file and folder objects in on…
amuwal Sep 8, 2024
e591eb5
Fix small bug
amuwal Sep 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .env.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,9 @@ LINEAR_TICKETING_CLOUD_CLIENT_SECRET=
# Box
BOX_FILESTORAGE_CLOUD_CLIENT_ID=
BOX_FILESTORAGE_CLOUD_CLIENT_SECRET=
# Onedrive
ONEDRIVE_FILESTORAGE_CLOUD_CLIENT_ID=
ONEDRIVE_FILESTORAGE_CLOUD_CLIENT_SECRET=


# ================================================
Expand Down
40 changes: 28 additions & 12 deletions packages/api/prisma/schema.prisma
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1965,18 +1965,19 @@ model linked_users {

/// This model or at least one of its fields has comments in the database, and requires an additional setup for migrations: Read more: https://pris.ly/d/database-comments
model projects {
id_project String @id(map: "pk_projects") @db.Uuid
name String
sync_mode String
pull_frequency BigInt?
redirect_url String?
id_user String @db.Uuid
id_connector_set String @db.Uuid
api_keys api_keys[]
connections connections[]
linked_users linked_users[]
users users @relation(fields: [id_user], references: [id_user], onDelete: NoAction, onUpdate: NoAction, map: "fk_46_1")
connector_sets connector_sets @relation(fields: [id_connector_set], references: [id_connector_set], onDelete: NoAction, onUpdate: NoAction, map: "fk_project_connectorsetid")
id_project String @id(map: "pk_projects") @db.Uuid
name String
sync_mode String
pull_frequency BigInt?
redirect_url String?
id_user String @db.Uuid
id_connector_set String @db.Uuid
api_keys api_keys[]
connections connections[]
linked_users linked_users[]
users users @relation(fields: [id_user], references: [id_user], onDelete: NoAction, onUpdate: NoAction, map: "fk_46_1")
connector_sets connector_sets @relation(fields: [id_connector_set], references: [id_connector_set], onDelete: NoAction, onUpdate: NoAction, map: "fk_project_connectorsetid")
projects_pull_frequency projects_pull_frequency?

@@index([id_connector_set], map: "fk_connectors_sets")
}
Expand Down Expand Up @@ -2128,3 +2129,18 @@ model webhook_delivery_attempts {
@@index([id_event], map: "fk_webhook_delivery_attempt_eventid")
@@index([id_webhooks_reponse], map: "fk_webhook_delivery_attempt_webhook_responseid")
}

model projects_pull_frequency {
id_projects_pull_frequency String @id(map: "pk_projects_pull_frequency") @db.Uuid
crm BigInt?
ats BigInt?
hris BigInt?
accounting BigInt?
filestorage BigInt?
ecommerce BigInt?
ticketing BigInt?
created_at DateTime @default(now()) @db.Timestamptz(6)
modified_at DateTime @default(now()) @db.Timestamptz(6)
id_project String @unique(map: "uq_projects_pull_frequency_project") @db.Uuid
projects projects @relation(fields: [id_project], references: [id_project], onDelete: NoAction, onUpdate: NoAction, map: "fk_projects_pull_frequency_project")
}
42 changes: 27 additions & 15 deletions packages/api/scripts/connectorUpdate.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,10 @@ function updateModuleFileForMapper(moduleFile, newServiceDirs, objectType) {
// Generate and insert new service imports
newServiceDirs.forEach((serviceName) => {
const mapperClass =
serviceName.charAt(0).toUpperCase() + serviceName.slice(1) + objectType + 'Mapper';
serviceName.charAt(0).toUpperCase() +
serviceName.slice(1) +
objectType +
'Mapper';
Comment on lines +268 to +271
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Refactor to use template literals for string construction.

The current string concatenation can be improved for readability and performance by using template literals. Here's the suggested change:

- const mapperClass =
-   serviceName.charAt(0).toUpperCase() +
-   serviceName.slice(1) +
-   objectType +
-   'Mapper';
+ const mapperClass = `${serviceName.charAt(0).toUpperCase()}${serviceName.slice(1)}${objectType}Mapper`;
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
serviceName.charAt(0).toUpperCase() +
serviceName.slice(1) +
objectType +
'Mapper';
const mapperClass = `${serviceName.charAt(0).toUpperCase()}${serviceName.slice(1)}${objectType}Mapper`;
Tools
Biome

[error] 268-271: Template literals are preferred over string concatenation.

Unsafe fix: Use a template literal.

(lint/style/useTemplate)

const importStatement = `import { ${mapperClass} } from './services/${serviceName}/mappers';\n`;
if (!moduleFileContent.includes(importStatement)) {
moduleFileContent = importStatement + moduleFileContent;
Expand Down Expand Up @@ -404,17 +407,20 @@ function updateSeedSQLFile(seedSQLFile, newServiceDirs, vertical) {
fileContent = fileContent.replace(lastMatch[1], newColumnsSection);

// Update each VALUES section
fileContent = fileContent.replace(/INSERT INTO connector_sets \(([^)]+)\) VALUES(.*?);/gs, (match) => {
return match
.replace(/\),\s*\(/g, '),\n (') // Fix line formatting
.replace(/\([^\)]+\)/g, (values, index) => {
if (values.startsWith('(id_connector_set')) {
return values
}
let newValues = newColumns.map(() => 'TRUE').join(', ');
return values.slice(0, -1) + ', ' + newValues + ')';
});
});
fileContent = fileContent.replace(
/INSERT INTO connector_sets \(([^)]+)\) VALUES(.*?);/gs,
(match) => {
return match
.replace(/\),\s*\(/g, '),\n (') // Fix line formatting
.replace(/\([^\)]+\)/g, (values, index) => {
if (values.startsWith('(id_connector_set')) {
return values;
}
let newValues = newColumns.map(() => 'TRUE').join(', ');
return values.slice(0, -1) + ', ' + newValues + ')';
});
},
);
Comment on lines +410 to +423
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Refactor SQL string manipulation to use template literals and improve maintainability.

The manipulation of SQL strings using regex and string concatenation is error-prone and hard to maintain. Consider using template literals for better readability and potentially transitioning to a more robust SQL handling approach, such as parameterized queries or a dedicated library. Here's how you can refactor the string concatenation:

- let newValues = newColumns.map(() => 'TRUE').join(', ');
- return values.slice(0, -1) + ', ' + newValues + ')';
+ let newValues = newColumns.map(() => 'TRUE').join(', ');
+ return `${values.slice(0, -1)}, ${newValues})`;
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
fileContent = fileContent.replace(
/INSERT INTO connector_sets \(([^)]+)\) VALUES(.*?);/gs,
(match) => {
return match
.replace(/\),\s*\(/g, '),\n (') // Fix line formatting
.replace(/\([^\)]+\)/g, (values, index) => {
if (values.startsWith('(id_connector_set')) {
return values;
}
let newValues = newColumns.map(() => 'TRUE').join(', ');
return values.slice(0, -1) + ', ' + newValues + ')';
});
},
);
fileContent = fileContent.replace(
/INSERT INTO connector_sets \(([^)]+)\) VALUES(.*?);/gs,
(match) => {
return match
.replace(/\),\s*\(/g, '),\n (') // Fix line formatting
.replace(/\([^\)]+\)/g, (values, index) => {
if (values.startsWith('(id_connector_set')) {
return values;
}
let newValues = newColumns.map(() => 'TRUE').join(', ');
return `${values.slice(0, -1)}, ${newValues})`;
});
},
);
Tools
Biome

[error] 420-420: Template literals are preferred over string concatenation.

Unsafe fix: Use a template literal.

(lint/style/useTemplate)

Refactor SQL string manipulation to use template literals and improve maintainability.

The manipulation of SQL strings using regex and string concatenation is error-prone and hard to maintain. Consider using template literals for better readability and potentially transitioning to a more robust SQL handling approach, such as parameterized queries or a dedicated library. Here's how you can refactor the string concatenation:

- let newValues = newColumns.map(() => 'TRUE').join(', ');
- return values.slice(0, -1) + ', ' + newValues + ')';
+ let newValues = newColumns.map(() => 'TRUE').join(', ');
+ return `${values.slice(0, -1)}, ${newValues})`;
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
fileContent = fileContent.replace(
/INSERT INTO connector_sets \(([^)]+)\) VALUES(.*?);/gs,
(match) => {
return match
.replace(/\),\s*\(/g, '),\n (') // Fix line formatting
.replace(/\([^\)]+\)/g, (values, index) => {
if (values.startsWith('(id_connector_set')) {
return values;
}
let newValues = newColumns.map(() => 'TRUE').join(', ');
return values.slice(0, -1) + ', ' + newValues + ')';
});
},
);
fileContent = fileContent.replace(
/INSERT INTO connector_sets \(([^)]+)\) VALUES(.*?);/gs,
(match) => {
return match
.replace(/\),\s*\(/g, '),\n (') // Fix line formatting
.replace(/\([^\)]+\)/g, (values, index) => {
if (values.startsWith('(id_connector_set')) {
return values;
}
let newValues = newColumns.map(() => 'TRUE').join(', ');
return `${values.slice(0, -1)}, ${newValues})`;
});
},
);
Tools
Biome

[error] 420-420: Template literals are preferred over string concatenation.

Unsafe fix: Use a template literal.

(lint/style/useTemplate)

}
// Write the modified content back to the file
console.log(fileContent);
Expand All @@ -427,9 +433,15 @@ function updateSeedSQLFile(seedSQLFile, newServiceDirs, vertical) {
function updateObjectTypes(baseDir, objectType, vertical) {
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const servicesDir = path.join(__dirname, baseDir);
const targetFileName =
vertical === 'filestorage'
? 'file-storage'
: vertical === 'marketingautomation'
? 'marketing-automation'
: vertical;
Comment on lines +436 to +441
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Refactor to improve readability in selecting the target file name.

The current use of multiple ternary operators makes the code hard to read. Consider refactoring to use a switch statement or a mapping object for clarity:

const targetFileNameMap = {
  filestorage: 'file-storage',
  marketingautomation: 'marketing-automation'
};
const targetFileName = targetFileNameMap[vertical] || vertical;

const targetFile = path.join(
__dirname,
`../src/@core/utils/types/original/original.${vertical}.ts`,
`../src/@core/utils/types/original/original.${targetFileName}.ts`,
);

const newServiceDirs = scanDirectory(servicesDir);
Expand Down Expand Up @@ -468,7 +480,7 @@ function updateObjectTypes(baseDir, objectType, vertical) {
);

updateModuleFileForService(moduleFile, newServiceDirs);
updateModuleFileForMapper(moduleFile, newServiceDirs, objectType)
updateModuleFileForMapper(moduleFile, newServiceDirs, objectType);

// Path to the mappings file
// const mappingsFile = path.join(
Expand Down Expand Up @@ -522,4 +534,4 @@ if (import.meta.url === process.argv[1]) {

const argv = yargs(hideBin(process.argv)).argv;
const baseDir = `../src/${argv.vertical.toLowerCase()}/${argv.objectType.toLowerCase()}/services`;
updateObjectTypes(baseDir, argv.objectType, argv.vertical);
updateObjectTypes(baseDir, argv.objectType, argv.vertical);
1 change: 1 addition & 0 deletions packages/api/src/@core/sync/sync.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,7 @@ export class CoreSyncService {
try {
await task();
} catch (error) {
console.log(error);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider refining error logging strategy.

The addition of console.log(error); before this.logger.error(...) in the error handling block provides immediate visibility to the error object, which can be beneficial for debugging. However, consider the implications of using console.log in a production environment, where it might lead to performance issues or cluttered log outputs.

It might be more appropriate to integrate this logging into the existing logging framework or to use a conditional logging level that includes these details only in non-production environments.

this.logger.error(`File Task failed: ${error.message}`, error);
}
}
Expand Down
71 changes: 59 additions & 12 deletions packages/api/src/@core/utils/types/original/original.file-storage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,42 @@
import {
BoxSharedLinkInput,
BoxSharedLinkOutput,
} from '@filestorage/sharedlink/services/box/types';
import {
OnedriveSharedLinkInput,
OnedriveSharedLinkOutput,
} from '@filestorage/sharedlink/services/onedrive/types';

import {
OnedrivePermissionInput,
OnedrivePermissionOutput,
} from '@filestorage/permission/services/onedrive/types';

import {
OnedriveGroupInput,
OnedriveGroupOutput,
} from '@filestorage/group/services/onedrive/types';

import {
OnedriveUserInput,
OnedriveUserOutput,
} from '@filestorage/user/services/onedrive/types';

import {
OnedriveFileInput,
OnedriveFileOutput,
} from '@filestorage/file/services/onedrive/types';

import {
OnedriveFolderInput,
OnedriveFolderOutput,
} from '@filestorage/folder/services/onedrive/types';

import {
OnedriveDriveInput,
OnedriveDriveOutput,
} from '@filestorage/drive/services/onedrive/types';

/* INPUT */

import {
Expand All @@ -18,25 +57,25 @@ import {
} from '@filestorage/user/services/box/types';

/* file */
export type OriginalFileInput = BoxFileInput;
export type OriginalFileInput = BoxFileInput | OnedriveFileInput;

/* folder */
export type OriginalFolderInput = BoxFolderInput;
export type OriginalFolderInput = BoxFolderInput | OnedriveFolderInput;

/* permission */
export type OriginalPermissionInput = any;
export type OriginalPermissionInput = any | OnedrivePermissionInput;

/* shared link */
export type OriginalSharedLinkInput = any;

/* drive */
export type OriginalDriveInput = any;
export type OriginalDriveInput = any | OnedriveDriveInput;

/* group */
export type OriginalGroupInput = BoxGroupInput;
export type OriginalGroupInput = BoxGroupInput | OnedriveGroupInput;

/* user */
export type OriginalUserInput = BoxUserInput;
export type OriginalUserInput = BoxUserInput | OnedriveUserInput;

export type FileStorageObjectInput =
| OriginalFileInput
Expand All @@ -50,25 +89,25 @@ export type FileStorageObjectInput =
/* OUTPUT */

/* file */
export type OriginalFileOutput = BoxFileOutput;
export type OriginalFileOutput = BoxFileOutput | OnedriveFileOutput;

/* folder */
export type OriginalFolderOutput = BoxFolderOutput;
export type OriginalFolderOutput = BoxFolderOutput | OnedriveFolderOutput;

/* permission */
export type OriginalPermissionOutput = any;
export type OriginalPermissionOutput = any | OnedrivePermissionOutput;

/* shared link */
export type OriginalSharedLinkOutput = any;

/* drive */
export type OriginalDriveOutput = any;
export type OriginalDriveOutput = any | OnedriveDriveOutput;

/* group */
export type OriginalGroupOutput = BoxGroupOutput;
export type OriginalGroupOutput = BoxGroupOutput | OnedriveGroupOutput;

/* user */
export type OriginalUserOutput = BoxUserOutput;
export type OriginalUserOutput = BoxUserOutput | OnedriveUserOutput;

export type FileStorageObjectOutput =
| OriginalFileOutput
Expand All @@ -78,3 +117,11 @@ export type FileStorageObjectOutput =
| OriginalDriveOutput
| OriginalGroupOutput
| OriginalUserOutput;

export type OriginalSharedlinkInput =
| BoxSharedLinkInput
| OnedriveSharedLinkInput;

export type OriginalSharedlinkOutput =
| BoxSharedLinkOutput
| OnedriveSharedLinkOutput;
4 changes: 4 additions & 0 deletions packages/api/src/filestorage/drive/drive.module.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { OnedriveDriveMapper } from './services/onedrive/mappers';
import { OnedriveService } from './services/onedrive';
import { BullQueueModule } from '@@core/@core-services/queues/queue.module';
import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service';
import { Module } from '@nestjs/common';
Expand All @@ -17,6 +19,8 @@ import { Utils } from '@filestorage/@lib/@utils';
ServiceRegistry,
Utils,
/* PROVIDERS SERVICES */
OnedriveService,
OnedriveDriveMapper,
],
exports: [SyncService],
})
Expand Down
70 changes: 70 additions & 0 deletions packages/api/src/filestorage/drive/services/onedrive/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
import { EncryptionService } from '@@core/@core-services/encryption/encryption.service';
import { LoggerService } from '@@core/@core-services/logger/logger.service';
import { PrismaService } from '@@core/@core-services/prisma/prisma.service';
import { ApiResponse } from '@@core/utils/types';
import { SyncParam } from '@@core/utils/types/interface';
import { FileStorageObject } from '@filestorage/@lib/@types';
import { IDriveService } from '@filestorage/drive/types';
import { Injectable } from '@nestjs/common';
import axios from 'axios';
import { ServiceRegistry } from '../registry.service';
import { OnedriveDriveOutput } from './types';
import { DesunifyReturnType } from '@@core/utils/types/desunify.input';
import { OriginalDriveOutput } from '@@core/utils/types/original/original.file-storage';

@Injectable()
export class OnedriveService implements IDriveService {
constructor(
private prisma: PrismaService,
private logger: LoggerService,
private cryptoService: EncryptionService,
private registry: ServiceRegistry,
) {
this.logger.setContext(
`${FileStorageObject.file.toUpperCase()}:${OnedriveService.name}`,
);
this.registry.registerService('onedrive', this);
}

async addDrive(
driveData: DesunifyReturnType,
linkedUserId: string,
): Promise<ApiResponse<OriginalDriveOutput>> {
// No API to add drive in onedrive
return;
}
Comment on lines +29 to +35
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Clarify or remove the addDrive method.

The addDrive method currently does nothing, with a comment indicating that there is no API to add a drive in OneDrive. If this functionality is not needed, consider removing the method. Otherwise, enhance the documentation to explain why this method is present and what its future intentions might be.


async sync(data: SyncParam): Promise<ApiResponse<OnedriveDriveOutput[]>> {
try {
const { linkedUserId } = data;

const connection = await this.prisma.connections.findFirst({
where: {
id_linked_user: linkedUserId,
provider_slug: 'onedrive',
vertical: 'filestorage',
},
});

const resp = await axios.get(`${connection.account_url}/v1.0/drives`, {
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.cryptoService.decrypt(
connection.access_token,
)}`,
},
});

const drives: OnedriveDriveOutput[] = resp.data.value;
this.logger.log(`Synced onedrive drives !`);

return {
data: drives,
message: 'Onedrive drives retrived',
statusCode: 200,
};
} catch (error) {
throw error;
}
}
Comment on lines +37 to +69
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Optimize error handling in sync method.

The sync method is well-implemented for retrieving OneDrive drives. However, the catch block that simply rethrows the error is redundant and could be removed to clean up the code.

Apply this diff to remove the redundant catch block:

-    } catch (error) {
-      throw error;
-    }
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
async sync(data: SyncParam): Promise<ApiResponse<OnedriveDriveOutput[]>> {
try {
const { linkedUserId } = data;
const connection = await this.prisma.connections.findFirst({
where: {
id_linked_user: linkedUserId,
provider_slug: 'onedrive',
vertical: 'filestorage',
},
});
const resp = await axios.get(`${connection.account_url}/v1.0/drives`, {
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.cryptoService.decrypt(
connection.access_token,
)}`,
},
});
const drives: OnedriveDriveOutput[] = resp.data.value;
this.logger.log(`Synced onedrive drives !`);
return {
data: drives,
message: 'Onedrive drives retrived',
statusCode: 200,
};
} catch (error) {
throw error;
}
}
async sync(data: SyncParam): Promise<ApiResponse<OnedriveDriveOutput[]>> {
const { linkedUserId } = data;
const connection = await this.prisma.connections.findFirst({
where: {
id_linked_user: linkedUserId,
provider_slug: 'onedrive',
vertical: 'filestorage',
},
});
const resp = await axios.get(`${connection.account_url}/v1.0/drives`, {
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.cryptoService.decrypt(
connection.access_token,
)}`,
},
});
const drives: OnedriveDriveOutput[] = resp.data.value;
this.logger.log(`Synced onedrive drives !`);
return {
data: drives,
message: 'Onedrive drives retrived',
statusCode: 200,
};
}
Tools
Biome

[error] 67-67: The catch clause that only rethrows the original error is redundant.

These unnecessary catch clauses can be confusing. It is recommended to remove them.

(lint/complexity/noUselessCatch)

}
Loading
Loading