From 79deb6cd65aecaa59c4c7ae7332442794360c07e Mon Sep 17 00:00:00 2001 From: nael Date: Sat, 22 Jun 2024 22:57:14 +0200 Subject: [PATCH] :bug: Added sync --- .../types/original/original.accounting.ts | 80 ++-- .../utils/types/original/original.ats.ts | 64 +-- .../types/original/original.file-storage.ts | 28 +- .../utils/types/original/original.hris.ts | 56 +-- .../original/original.marketing-automation.ts | 40 +- packages/api/src/ats/@lib/@types/index.ts | 35 -- .../ats/activity/services/activity.service.ts | 6 + .../src/ats/activity/sync/sync.processor.ts | 17 + .../api/src/ats/activity/sync/sync.service.ts | 337 +++++++++++++++- packages/api/src/ats/activity/types/index.ts | 2 +- .../src/ats/activity/types/model.unified.ts | 2 +- .../services/application.service.ts | 6 + .../ats/application/sync/sync.processor.ts | 18 + .../src/ats/application/sync/sync.service.ts | 361 ++++++++++++++++- .../ats/application/types/model.unified.ts | 2 +- .../attachment/services/attachment.service.ts | 6 + .../src/ats/attachment/sync/sync.processor.ts | 18 + .../src/ats/attachment/sync/sync.service.ts | 345 +++++++++++++++- .../src/ats/attachment/types/model.unified.ts | 2 +- .../src/ats/candidate/sync/sync.processor.ts | 18 + .../src/ats/candidate/sync/sync.service.ts | 375 +++++++++++++++++- .../src/ats/candidate/types/model.unified.ts | 59 +-- .../department/services/department.service.ts | 6 + .../src/ats/department/sync/sync.processor.ts | 18 + .../src/ats/department/sync/sync.service.ts | 309 ++++++++++++++- .../src/ats/department/types/model.unified.ts | 2 +- .../src/ats/eeocs/services/eeocs.service.ts | 6 + .../api/src/ats/eeocs/sync/sync.processor.ts | 18 + .../api/src/ats/eeocs/sync/sync.service.ts | 339 +++++++++++++++- .../api/src/ats/eeocs/types/model.unified.ts | 2 +- .../interview/services/interview.service.ts | 6 + .../src/ats/interview/sync/sync.processor.ts | 18 + .../src/ats/interview/sync/sync.service.ts | 369 ++++++++++++++++- .../src/ats/interview/types/model.unified.ts | 2 +- .../api/src/ats/job/services/job.service.ts | 6 + .../api/src/ats/job/sync/sync.processor.ts | 18 + packages/api/src/ats/job/sync/sync.service.ts | 375 +++++++++++++++++- .../api/src/ats/job/types/model.unified.ts | 2 +- .../services/jobinterviewstage.service.ts | 6 + .../jobinterviewstage/sync/sync.processor.ts | 20 + .../jobinterviewstage/sync/sync.service.ts | 333 +++++++++++++++- .../jobinterviewstage/types/model.unified.ts | 2 +- .../src/ats/offer/services/offer.service.ts | 6 + .../api/src/ats/offer/sync/sync.processor.ts | 18 + .../api/src/ats/offer/sync/sync.service.ts | 345 +++++++++++++++- .../api/src/ats/offer/types/model.unified.ts | 2 +- .../src/ats/office/services/office.service.ts | 6 + .../api/src/ats/office/sync/sync.processor.ts | 18 + .../api/src/ats/office/sync/sync.service.ts | 313 ++++++++++++++- .../api/src/ats/office/types/model.unified.ts | 2 +- .../services/rejectreason.service.ts | 6 + .../ats/rejectreason/sync/sync.processor.ts | 18 + .../src/ats/rejectreason/sync/sync.service.ts | 313 ++++++++++++++- .../ats/rejectreason/types/model.unified.ts | 2 +- .../scorecard/services/scorecard.service.ts | 6 + .../src/ats/scorecard/sync/sync.processor.ts | 18 + .../src/ats/scorecard/sync/sync.service.ts | 339 +++++++++++++++- .../src/ats/scorecard/types/model.unified.ts | 2 +- .../screeningquestion/sync/sync.processor.ts | 18 + .../api/src/ats/tag/services/tag.service.ts | 172 ++++++-- .../api/src/ats/tag/sync/sync.processor.ts | 18 + packages/api/src/ats/tag/sync/sync.service.ts | 315 ++++++++++++++- .../api/src/ats/tag/types/model.unified.ts | 66 ++- .../api/src/ats/user/services/user.service.ts | 6 + .../api/src/ats/user/sync/sync.processor.ts | 18 + .../api/src/ats/user/sync/sync.service.ts | 345 +++++++++++++++- .../api/src/ats/user/types/model.unified.ts | 2 +- .../crm/company/services/company.service.ts | 31 +- .../api/src/crm/company/sync/sync.service.ts | 1 - .../src/crm/company/types/model.unified.ts | 6 +- .../crm/contact/services/contact.service.ts | 6 + .../src/crm/contact/types/model.unified.ts | 6 +- packages/api/src/crm/deal/deal.controller.ts | 4 +- .../api/src/crm/deal/services/deal.service.ts | 6 + .../api/src/crm/deal/types/model.unified.ts | 10 +- .../engagement/services/engagement.service.ts | 6 + .../src/crm/engagement/types/model.unified.ts | 14 +- .../api/src/crm/note/services/note.service.ts | 6 + .../api/src/crm/note/types/model.unified.ts | 12 +- .../src/crm/stage/services/stage.service.ts | 6 + .../api/src/crm/stage/types/model.unified.ts | 4 +- .../api/src/crm/task/services/task.service.ts | 6 + .../api/src/crm/task/types/model.unified.ts | 10 +- .../api/src/crm/user/services/user.service.ts | 6 + .../api/src/crm/user/types/model.unified.ts | 4 +- .../api/src/filestorage/@lib/@types/index.ts | 31 +- .../drive/services/drive.service.ts | 6 + .../filestorage/drive/sync/sync.processor.ts | 18 + .../filestorage/drive/sync/sync.service.ts | 320 ++++++++++++++- .../filestorage/drive/types/model.unified.ts | 4 +- .../filestorage/file/services/file.service.ts | 6 + .../filestorage/file/sync/sync.processor.ts | 18 + .../src/filestorage/file/sync/sync.service.ts | 345 +++++++++++++++- .../filestorage/file/types/model.unified.ts | 8 +- .../folder/services/folder.service.ts | 6 + .../filestorage/folder/sync/sync.processor.ts | 18 + .../filestorage/folder/sync/sync.service.ts | 342 +++++++++++++++- .../filestorage/folder/types/model.unified.ts | 10 +- .../group/services/group.service.ts | 6 + .../filestorage/group/sync/sync.processor.ts | 18 + .../filestorage/group/sync/sync.service.ts | 320 ++++++++++++++- .../filestorage/group/types/model.unified.ts | 4 +- .../permission/services/permission.service.ts | 6 + .../permission/sync/sync.processor.ts | 18 + .../permission/sync/sync.service.ts | 326 ++++++++++++++- .../permission/types/model.unified.ts | 8 +- .../sharedlink/services/sharedlink.service.ts | 6 + .../sharedlink/sync/sync.processor.ts | 20 + .../sharedlink/sync/sync.service.ts | 350 +++++++++++++++- .../src/filestorage/sharedlink/types/index.ts | 4 +- .../sharedlink/types/model.unified.ts | 8 +- .../filestorage/user/services/user.service.ts | 6 + .../filestorage/user/sync/sync.processor.ts | 18 + .../src/filestorage/user/sync/sync.service.ts | 320 ++++++++++++++- .../filestorage/user/types/model.unified.ts | 4 +- .../account/services/account.service.ts | 6 + .../ticketing/account/types/model.unified.ts | 4 +- .../attachment/services/attachment.service.ts | 6 + .../attachment/types/model.unified.ts | 6 +- .../collection/services/collection.service.ts | 6 + .../collection/types/model.unified.ts | 4 +- .../comment/services/comment.service.ts | 7 +- .../ticketing/comment/sync/sync.service.ts | 4 +- .../ticketing/comment/types/model.unified.ts | 20 +- .../contact/services/contact.service.ts | 6 + .../ticketing/contact/types/model.unified.ts | 4 +- .../src/ticketing/tag/services/tag.service.ts | 6 + .../src/ticketing/tag/types/model.unified.ts | 4 +- .../ticketing/team/services/team.service.ts | 6 + .../src/ticketing/team/types/model.unified.ts | 4 +- .../ticketing/ticket/services/front/index.ts | 1 - .../ticket/services/ticket.service.ts | 7 +- .../src/ticketing/ticket/sync/sync.service.ts | 2 - .../ticketing/ticket/types/model.unified.ts | 16 +- .../ticketing/user/services/user.service.ts | 6 + .../src/ticketing/user/types/model.unified.ts | 4 +- 136 files changed, 8422 insertions(+), 533 deletions(-) create mode 100644 packages/api/src/ats/activity/sync/sync.processor.ts create mode 100644 packages/api/src/ats/application/sync/sync.processor.ts create mode 100644 packages/api/src/ats/attachment/sync/sync.processor.ts create mode 100644 packages/api/src/ats/candidate/sync/sync.processor.ts create mode 100644 packages/api/src/ats/department/sync/sync.processor.ts create mode 100644 packages/api/src/ats/eeocs/sync/sync.processor.ts create mode 100644 packages/api/src/ats/interview/sync/sync.processor.ts create mode 100644 packages/api/src/ats/job/sync/sync.processor.ts create mode 100644 packages/api/src/ats/jobinterviewstage/sync/sync.processor.ts create mode 100644 packages/api/src/ats/offer/sync/sync.processor.ts create mode 100644 packages/api/src/ats/office/sync/sync.processor.ts create mode 100644 packages/api/src/ats/rejectreason/sync/sync.processor.ts create mode 100644 packages/api/src/ats/scorecard/sync/sync.processor.ts create mode 100644 packages/api/src/ats/screeningquestion/sync/sync.processor.ts create mode 100644 packages/api/src/ats/tag/sync/sync.processor.ts create mode 100644 packages/api/src/ats/user/sync/sync.processor.ts create mode 100644 packages/api/src/filestorage/drive/sync/sync.processor.ts create mode 100644 packages/api/src/filestorage/file/sync/sync.processor.ts create mode 100644 packages/api/src/filestorage/folder/sync/sync.processor.ts create mode 100644 packages/api/src/filestorage/group/sync/sync.processor.ts create mode 100644 packages/api/src/filestorage/permission/sync/sync.processor.ts create mode 100644 packages/api/src/filestorage/sharedlink/sync/sync.processor.ts create mode 100644 packages/api/src/filestorage/user/sync/sync.processor.ts diff --git a/packages/api/src/@core/utils/types/original/original.accounting.ts b/packages/api/src/@core/utils/types/original/original.accounting.ts index a9a4a07c0..9efdb1736 100644 --- a/packages/api/src/@core/utils/types/original/original.accounting.ts +++ b/packages/api/src/@core/utils/types/original/original.accounting.ts @@ -1,64 +1,64 @@ /* INPUT */ /* attachment */ -export type OriginalAttachmentInput = ''; +export type OriginalAttachmentInput = any; /* address */ -export type OriginalAddressInput = ''; +export type OriginalAddressInput = any; /* account */ -export type OriginalAccountInput = ''; +export type OriginalAccountInput = any; /* balancesheet */ -export type OriginalBalanceSheetInput = ''; +export type OriginalBalanceSheetInput = any; /* cashflowstatement */ -export type OriginalCashflowStatementInput = ''; +export type OriginalCashflowStatementInput = any; /* companyinfo */ -export type OriginalCompanyInfoInput = ''; +export type OriginalCompanyInfoInput = any; /* contact */ -export type OriginalContactInput = ''; +export type OriginalContactInput = any; /* creditnote */ -export type OriginalCreditNoteInput = ''; +export type OriginalCreditNoteInput = any; /* expense */ -export type OriginalExpenseInput = ''; +export type OriginalExpenseInput = any; /* incomestatement */ -export type OriginalIncomeStatementInput = ''; +export type OriginalIncomeStatementInput = any; /* invoice */ -export type OriginalInvoiceInput = ''; +export type OriginalInvoiceInput = any; /* item */ -export type OriginalItemInput = ''; +export type OriginalItemInput = any; /* journalentry */ -export type OriginalJournalEntryInput = ''; +export type OriginalJournalEntryInput = any; /* payment */ -export type OriginalPaymentInput = ''; +export type OriginalPaymentInput = any; /* phonenumber */ -export type OriginalPhoneNumberInput = ''; +export type OriginalPhoneNumberInput = any; /* purchaseorder */ -export type OriginalPurchaseOrderInput = ''; +export type OriginalPurchaseOrderInput = any; /* taxrate */ -export type OriginalTaxRateInput = ''; +export type OriginalTaxRateInput = any; /* trackingcategory */ -export type OriginalTrackingCategoryInput = ''; +export type OriginalTrackingCategoryInput = any; /* transaction */ -export type OriginalTransactionInput = ''; +export type OriginalTransactionInput = any; /* vendorcredit */ -export type OriginalVendorCreditInput = ''; +export type OriginalVendorCreditInput = any; export type AccountingObjectInput = | OriginalAttachmentInput @@ -85,64 +85,64 @@ export type AccountingObjectInput = /* OUTPUT */ /* attachment */ -export type OriginalAttachmentOutput = ''; +export type OriginalAttachmentOutput = any; /* address */ -export type OriginalAddressOutput = ''; +export type OriginalAddressOutput = any; /* account */ -export type OriginalAccountOutput = ''; +export type OriginalAccountOutput = any; /* balancesheet */ -export type OriginalBalanceSheetOutput = ''; +export type OriginalBalanceSheetOutput = any; /* cashflowstatement */ -export type OriginalCashflowStatementOutput = ''; +export type OriginalCashflowStatementOutput = any; /* companyinfo */ -export type OriginalCompanyInfoOutput = ''; +export type OriginalCompanyInfoOutput = any; /* contact */ -export type OriginalContactOutput = ''; +export type OriginalContactOutput = any; /* creditnote */ -export type OriginalCreditNoteOutput = ''; +export type OriginalCreditNoteOutput = any; /* expense */ -export type OriginalExpenseOutput = ''; +export type OriginalExpenseOutput = any; /* incomestatement */ -export type OriginalIncomeStatementOutput = ''; +export type OriginalIncomeStatementOutput = any; /* invoice */ -export type OriginalInvoiceOutput = ''; +export type OriginalInvoiceOutput = any; /* item */ -export type OriginalItemOutput = ''; +export type OriginalItemOutput = any; /* journalentry */ -export type OriginalJournalEntryOutput = ''; +export type OriginalJournalEntryOutput = any; /* payment */ -export type OriginalPaymentOutput = ''; +export type OriginalPaymentOutput = any; /* phonenumber */ -export type OriginalPhoneNumberOutput = ''; +export type OriginalPhoneNumberOutput = any; /* purchaseorder */ -export type OriginalPurchaseOrderOutput = ''; +export type OriginalPurchaseOrderOutput = any; /* taxrate */ -export type OriginalTaxRateOutput = ''; +export type OriginalTaxRateOutput = any; /* trackingcategory */ -export type OriginalTrackingCategoryOutput = ''; +export type OriginalTrackingCategoryOutput = any; /* transaction */ -export type OriginalTransactionOutput = ''; +export type OriginalTransactionOutput = any; /* vendorcredit */ -export type OriginalVendorCreditOutput = ''; +export type OriginalVendorCreditOutput = any; export type AccountingObjectOutput = | OriginalAttachmentOutput diff --git a/packages/api/src/@core/utils/types/original/original.ats.ts b/packages/api/src/@core/utils/types/original/original.ats.ts index ddd4d4e19..74e0df290 100644 --- a/packages/api/src/@core/utils/types/original/original.ats.ts +++ b/packages/api/src/@core/utils/types/original/original.ats.ts @@ -1,52 +1,52 @@ /* INPUT */ /* activity */ -export type OriginalActivityInput = ''; +export type OriginalActivityInput = any; /* application */ -export type OriginalApplicationInput = ''; +export type OriginalApplicationInput = any; /* attachment */ -export type OriginalAttachmentInput = ''; +export type OriginalAttachmentInput = any; /* candidate */ -export type OriginalCandidateInput = ''; +export type OriginalCandidateInput = any; /* department */ -export type OriginalDepartmentInput = ''; +export type OriginalDepartmentInput = any; /* interview */ -export type OriginalInterviewInput = ''; +export type OriginalInterviewInput = any; /* jobinterviewstage */ -export type OriginalJobInterviewStageInput = ''; +export type OriginalJobInterviewStageInput = any; /* job */ -export type OriginalJobInput = ''; +export type OriginalJobInput = any; /* offer */ -export type OriginalOfferInput = ''; +export type OriginalOfferInput = any; /* office */ -export type OriginalOfficeInput = ''; +export type OriginalOfficeInput = any; /* rejectreason */ -export type OriginalRejectReasonInput = ''; +export type OriginalRejectReasonInput = any; /* scorecard */ -export type OriginalScoreCardInput = ''; +export type OriginalScoreCardInput = any; /* screeningquestion */ -export type OriginalScreeningQuestionInput = ''; +export type OriginalScreeningQuestionInput = any; /* tag */ -export type OriginalTagInput = ''; +export type OriginalTagInput = any; /* user */ -export type OriginalUserInput = ''; +export type OriginalUserInput = any; /* eeocs */ -export type OriginalEeocsInput = ''; +export type OriginalEeocsInput = any; export type AtsObjectInput = | OriginalActivityInput @@ -69,52 +69,52 @@ export type AtsObjectInput = /* OUTPUT */ /* activity */ -export type OriginalActivityOutput = ''; +export type OriginalActivityOutput = any; /* application */ -export type OriginalApplicationOutput = ''; +export type OriginalApplicationOutput = any; /* attachment */ -export type OriginalAttachmentOutput = ''; +export type OriginalAttachmentOutput = any; /* candidate */ -export type OriginalCandidateOutput = ''; +export type OriginalCandidateOutput = any; /* department */ -export type OriginalDepartmentOutput = ''; +export type OriginalDepartmentOutput = any; /* interview */ -export type OriginalInterviewOutput = ''; +export type OriginalInterviewOutput = any; /* jobinterviewstage */ -export type OriginalJobInterviewStageOutput = ''; +export type OriginalJobInterviewStageOutput = any; /* job */ -export type OriginalJobOutput = ''; +export type OriginalJobOutput = any; /* offer */ -export type OriginalOfferOutput = ''; +export type OriginalOfferOutput = any; /* office */ -export type OriginalOfficeOutput = ''; +export type OriginalOfficeOutput = any; /* rejectreason */ -export type OriginalRejectReasonOutput = ''; +export type OriginalRejectReasonOutput = any; /* scorecard */ -export type OriginalScoreCardOutput = ''; +export type OriginalScoreCardOutput = any; /* screeningquestion */ -export type OriginalScreeningQuestionOutput = ''; +export type OriginalScreeningQuestionOutput = any; /* tag */ -export type OriginalTagOutput = ''; +export type OriginalTagOutput = any; /* user */ -export type OriginalUserOutput = ''; +export type OriginalUserOutput = any; /* eeocs */ -export type OriginalEeocsOutput = ''; +export type OriginalEeocsOutput = any; export type AtsObjectOutput = | OriginalActivityOutput diff --git a/packages/api/src/@core/utils/types/original/original.file-storage.ts b/packages/api/src/@core/utils/types/original/original.file-storage.ts index 77c54b8d3..f593e0804 100644 --- a/packages/api/src/@core/utils/types/original/original.file-storage.ts +++ b/packages/api/src/@core/utils/types/original/original.file-storage.ts @@ -1,25 +1,25 @@ /* INPUT */ /* file */ -export type OriginalFileInput = ''; +export type OriginalFileInput = any; /* folder */ -export type OriginalFolderInput = ''; +export type OriginalFolderInput = any; /* permission */ -export type OriginalPermissionInput = ''; +export type OriginalPermissionInput = any; /* shared link */ -export type OriginalSharedLinkInput = ''; +export type OriginalSharedLinkInput = any; /* drive */ -export type OriginalDriveInput = ''; +export type OriginalDriveInput = any; /* group */ -export type OriginalGroupInput = ''; +export type OriginalGroupInput = any; /* user */ -export type OriginalUserInput = ''; +export type OriginalUserInput = any; export type FileStorageObjectInput = | OriginalFileInput @@ -33,25 +33,25 @@ export type FileStorageObjectInput = /* OUTPUT */ /* file */ -export type OriginalFileOutput = ''; +export type OriginalFileOutput = any; /* folder */ -export type OriginalFolderOutput = ''; +export type OriginalFolderOutput = any; /* permission */ -export type OriginalPermissionOutput = ''; +export type OriginalPermissionOutput = any; /* shared link */ -export type OriginalSharedLinkOutput = ''; +export type OriginalSharedLinkOutput = any; /* drive */ -export type OriginalDriveOutput = ''; +export type OriginalDriveOutput = any; /* group */ -export type OriginalGroupOutput = ''; +export type OriginalGroupOutput = any; /* user */ -export type OriginalUserOutput = ''; +export type OriginalUserOutput = any; export type FileStorageObjectOutput = | OriginalFileOutput diff --git a/packages/api/src/@core/utils/types/original/original.hris.ts b/packages/api/src/@core/utils/types/original/original.hris.ts index 15d362ef4..ccb190529 100644 --- a/packages/api/src/@core/utils/types/original/original.hris.ts +++ b/packages/api/src/@core/utils/types/original/original.hris.ts @@ -1,46 +1,46 @@ /* INPUT */ /* bankinfo */ -export type OriginalBankInfoInput = ''; +export type OriginalBankInfoInput = any; /* benefit */ -export type OriginalBenefitInput = ''; +export type OriginalBenefitInput = any; /* company */ -export type OriginalCompanyInput = ''; +export type OriginalCompanyInput = any; /* dependent */ -export type OriginalDependentInput = ''; +export type OriginalDependentInput = any; /* employee */ -export type OriginalEmployeeInput = ''; +export type OriginalEmployeeInput = any; /* employeepayrollrun */ -export type OriginalEmployeePayrollRunInput = ''; +export type OriginalEmployeePayrollRunInput = any; /* employerbenefit */ -export type OriginalEmployerBenefitInput = ''; +export type OriginalEmployerBenefitInput = any; /* employment */ -export type OriginalEmploymentInput = ''; +export type OriginalEmploymentInput = any; /* group */ -export type OriginalGroupInput = ''; +export type OriginalGroupInput = any; /* location */ -export type OriginalLocationInput = ''; +export type OriginalLocationInput = any; /* paygroup */ -export type OriginalPayGroupInput = ''; +export type OriginalPayGroupInput = any; /* payrollrun */ -export type OriginalPayrollRunInput = ''; +export type OriginalPayrollRunInput = any; /* timeoff */ -export type OriginalTimeoffInput = ''; +export type OriginalTimeoffInput = any; /* timeoffbalance */ -export type OriginalTimeoffBalanceInput = ''; +export type OriginalTimeoffBalanceInput = any; export type HrisObjectInput = | OriginalBankInfoInput @@ -61,46 +61,46 @@ export type HrisObjectInput = /* OUTPUT */ /* bankinfo */ -export type OriginalBankInfoOutput = ''; +export type OriginalBankInfoOutput = any; /* benefit */ -export type OriginalBenefitOutput = ''; +export type OriginalBenefitOutput = any; /* company */ -export type OriginalCompanyOutput = ''; +export type OriginalCompanyOutput = any; /* dependent */ -export type OriginalDependentOutput = ''; +export type OriginalDependentOutput = any; /* employee */ -export type OriginalEmployeeOutput = ''; +export type OriginalEmployeeOutput = any; /* employeepayrollrun */ -export type OriginalEmployeePayrollRunOutput = ''; +export type OriginalEmployeePayrollRunOutput = any; /* employerbenefit */ -export type OriginalEmployerBenefitOutput = ''; +export type OriginalEmployerBenefitOutput = any; /* employment */ -export type OriginalEmploymentOutput = ''; +export type OriginalEmploymentOutput = any; /* group */ -export type OriginalGroupOutput = ''; +export type OriginalGroupOutput = any; /* location */ -export type OriginalLocationOutput = ''; +export type OriginalLocationOutput = any; /* paygroup */ -export type OriginalPayGroupOutput = ''; +export type OriginalPayGroupOutput = any; /* payrollrun */ -export type OriginalPayrollRunOutput = ''; +export type OriginalPayrollRunOutput = any; /* timeoff */ -export type OriginalTimeoffOutput = ''; +export type OriginalTimeoffOutput = any; /* timeoffbalance */ -export type OriginalTimeoffBalanceOutput = ''; +export type OriginalTimeoffBalanceOutput = any; export type HrisObjectOutput = | OriginalBankInfoOutput diff --git a/packages/api/src/@core/utils/types/original/original.marketing-automation.ts b/packages/api/src/@core/utils/types/original/original.marketing-automation.ts index 8fb47b818..c1251657e 100644 --- a/packages/api/src/@core/utils/types/original/original.marketing-automation.ts +++ b/packages/api/src/@core/utils/types/original/original.marketing-automation.ts @@ -1,34 +1,34 @@ /* INPUT */ /* action */ -export type OriginalActionInput = ''; +export type OriginalActionInput = any; /* automation */ -export type OriginalAutomationInput = ''; +export type OriginalAutomationInput = any; /* campaign */ -export type OriginalCampaignInput = ''; +export type OriginalCampaignInput = any; /* contact */ -export type OriginalContactInput = ''; +export type OriginalContactInput = any; /* email */ -export type OriginalEmailInput = ''; +export type OriginalEmailInput = any; /* event */ -export type OriginalEventInput = ''; +export type OriginalEventInput = any; /* list */ -export type OriginalListInput = ''; +export type OriginalListInput = any; /* message */ -export type OriginalMessageInput = ''; +export type OriginalMessageInput = any; /* template */ -export type OriginalTemplateInput = ''; +export type OriginalTemplateInput = any; /* user */ -export type OriginalUserInput = ''; +export type OriginalUserInput = any; export type MarketingAutomationObjectInput = | OriginalActionInput @@ -45,34 +45,34 @@ export type MarketingAutomationObjectInput = /* OUTPUT */ /* action */ -export type OriginalActionOutput = ''; +export type OriginalActionOutput = any; /* automation */ -export type OriginalAutomationOutput = ''; +export type OriginalAutomationOutput = any; /* campaign */ -export type OriginalCampaignOutput = ''; +export type OriginalCampaignOutput = any; /* contact */ -export type OriginalContactOutput = ''; +export type OriginalContactOutput = any; /* email */ -export type OriginalEmailOutput = ''; +export type OriginalEmailOutput = any; /* event */ -export type OriginalEventOutput = ''; +export type OriginalEventOutput = any; /* list */ -export type OriginalListOutput = ''; +export type OriginalListOutput = any; /* message */ -export type OriginalMessageOutput = ''; +export type OriginalMessageOutput = any; /* template */ -export type OriginalTemplateOutput = ''; +export type OriginalTemplateOutput = any; /* user */ -export type OriginalUserOutput = ''; +export type OriginalUserOutput = any; export type MarketingAutomationObjectOutput = | OriginalActionOutput diff --git a/packages/api/src/ats/@lib/@types/index.ts b/packages/api/src/ats/@lib/@types/index.ts index 25246ac9c..8c37609ff 100644 --- a/packages/api/src/ats/@lib/@types/index.ts +++ b/packages/api/src/ats/@lib/@types/index.ts @@ -1,11 +1,9 @@ import { IActivityService } from '@ats/activity/types'; -import { activityUnificationMapping } from '@ats/activity/types/mappingsTypes'; import { UnifiedActivityInput, UnifiedActivityOutput, } from '@ats/activity/types/model.unified'; import { IApplicationService } from '@ats/application/types'; -import { applicationUnificationMapping } from '@ats/application/types/mappingsTypes'; import { UnifiedApplicationInput, UnifiedApplicationOutput, @@ -16,64 +14,50 @@ import { UnifiedAttachmentOutput, } from '@ats/attachment/types/model.unified'; import { ICandidateService } from '@ats/candidate/types'; -import { candidateUnificationMapping } from '@ats/candidate/types/mappingsTypes'; import { UnifiedCandidateInput, UnifiedCandidateOutput, } from '@ats/candidate/types/model.unified'; import { IDepartmentService } from '@ats/department/types'; -import { departmentUnificationMapping } from '@ats/department/types/mappingsTypes'; import { UnifiedDepartmentInput, UnifiedDepartmentOutput, } from '@ats/department/types/model.unified'; import { IEeocsService } from '@ats/eeocs/types'; -import { eeocsUnificationMapping } from '@ats/eeocs/types/mappingsTypes'; import { UnifiedEeocsInput, UnifiedEeocsOutput, } from '@ats/eeocs/types/model.unified'; import { IInterviewService } from '@ats/interview/types'; -import { interviewUnificationMapping } from '@ats/interview/types/mappingsTypes'; import { UnifiedInterviewInput, UnifiedInterviewOutput, } from '@ats/interview/types/model.unified'; import { IJobService } from '@ats/job/types'; -import { jobUnificationMapping } from '@ats/job/types/mappingsTypes'; import { UnifiedJobInput, UnifiedJobOutput, } from '@ats/job/types/model.unified'; import { IOfferService } from '@ats/offer/types'; -import { offerUnificationMapping } from '@ats/offer/types/mappingsTypes'; import { UnifiedOfferInput, UnifiedOfferOutput, } from '@ats/offer/types/model.unified'; import { IOfficeService } from '@ats/office/types'; -import { officeUnificationMapping } from '@ats/office/types/mappingsTypes'; import { UnifiedOfficeInput, UnifiedOfficeOutput, } from '@ats/office/types/model.unified'; -import { scorecardUnificationMapping } from '@ats/scorecard/types/mappingsTypes'; -import { attachmentUnificationMapping } from '@ats/attachment/types/mappingsTypes'; import { ITagService } from '@ats/tag/types'; -import { tagUnificationMapping } from '@ats/tag/types/mappingsTypes'; import { UnifiedTagInput, UnifiedTagOutput, } from '@ats/tag/types/model.unified'; import { IUserService } from '@ats/user/types'; -import { userUnificationMapping } from '@ats/user/types/mappingsTypes'; import { UnifiedUserInput, UnifiedUserOutput, } from '@ats/user/types/model.unified'; -import { jobinterviewstageUnificationMapping } from '@ats/jobinterviewstage/types/mappingsTypes'; -import { rejectreasonUnificationMapping } from '@ats/rejectreason/types/mappingsTypes'; -import { screeningquestionUnificationMapping } from '@ats/screeningquestion/types/mappingsTypes'; import { IJobInterviewStageService } from '@ats/jobinterviewstage/types'; import { UnifiedJobInterviewStageInput, @@ -148,25 +132,6 @@ export type UnifiedAts = | UnifiedEeocsInput | UnifiedEeocsOutput; -/*export const unificationMapping = { - [AtsObject.activity]: activityUnificationMapping, - [AtsObject.application]: applicationUnificationMapping, - [AtsObject.attachment]: attachmentUnificationMapping, - [AtsObject.candidate]: candidateUnificationMapping, - [AtsObject.department]: departmentUnificationMapping, - [AtsObject.interview]: interviewUnificationMapping, - [AtsObject.jobinterviewstage]: jobinterviewstageUnificationMapping, - [AtsObject.job]: jobUnificationMapping, - [AtsObject.offer]: offerUnificationMapping, - [AtsObject.office]: officeUnificationMapping, - [AtsObject.rejectreason]: rejectreasonUnificationMapping, - [AtsObject.scorecard]: scorecardUnificationMapping, - [AtsObject.screeningquestion]: screeningquestionUnificationMapping, - [AtsObject.tag]: tagUnificationMapping, - [AtsObject.user]: userUnificationMapping, - [AtsObject.eeocs]: eeocsUnificationMapping, -};*/ - export type IAtsService = | IActivityService | IApplicationService diff --git a/packages/api/src/ats/activity/services/activity.service.ts b/packages/api/src/ats/activity/services/activity.service.ts index d09b2080c..c74da86af 100644 --- a/packages/api/src/ats/activity/services/activity.service.ts +++ b/packages/api/src/ats/activity/services/activity.service.ts @@ -250,6 +250,9 @@ export class ActivityService { candidate_id: activity.id_ats_candidate, remote_created_at: activity.remote_created_at, field_mappings: field_mappings, + remote_id: activity.remote_id, + created_at: activity.created_at, + modified_at: activity.modified_at, }; let res: UnifiedActivityOutput = unifiedActivity; @@ -346,6 +349,9 @@ export class ActivityService { candidate_id: activity.id_ats_candidate, remote_created_at: activity.remote_created_at, field_mappings: field_mappings, + remote_id: activity.remote_id, + created_at: activity.created_at, + modified_at: activity.modified_at, }; }), ); diff --git a/packages/api/src/ats/activity/sync/sync.processor.ts b/packages/api/src/ats/activity/sync/sync.processor.ts new file mode 100644 index 000000000..29894cc10 --- /dev/null +++ b/packages/api/src/ats/activity/sync/sync.processor.ts @@ -0,0 +1,17 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + @Process('ats-sync-activities') + async handleSyncCompanies(job: Job) { + try { + console.log(`Processing queue -> ats-sync-activities ${job.id}`); + await this.syncService.syncActivities(); + } catch (error) { + console.error('Error syncing ats activities', error); + } + } +} diff --git a/packages/api/src/ats/activity/sync/sync.service.ts b/packages/api/src/ats/activity/sync/sync.service.ts index a76342872..194c82069 100644 --- a/packages/api/src/ats/activity/sync/sync.service.ts +++ b/packages/api/src/ats/activity/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - +import { AtsObject } from '@ats/@lib/@types'; import { WebhookService } from '@@core/webhook/webhook.service'; import { UnifiedActivityOutput } from '../types/model.unified'; import { IActivityService } from '../types'; +import { OriginalActivityOutput } from '@@core/utils/types/original/original.ats'; +import { ats_activities as AtsActivity } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,337 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private utils: Utils, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-activities'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + //function used by sync worker which populate our ats_activities table + //its role is to fetch all activities from providers 3rd parties and save the info inside our db + // @Cron('*/2 * * * *') // every 2 minutes (for testing) + @Cron('0 */8 * * *') // every 8 hours + async syncActivities(user_id?: string) { + try { + this.logger.log(`Syncing activities....`); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncActivitiesForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + //todo: HANDLE DATA REMOVED FROM PROVIDER + async syncActivitiesForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} activities for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping activities syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.activity', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IActivityService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncActivities(linkedUserId, remoteProperties); + + const sourceObject: OriginalActivityOutput[] = resp.data; + + //unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalActivityOutput[] + >({ + sourceObject, + targetType: AtsObject.activity, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedActivityOutput[]; + + //insert the data in the DB with the fieldMappings (value table) + const activities_data = await this.saveActivitysInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.activity.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + activities_data, + 'ats.activity.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveActivitysInDb( + linkedUserId: string, + activities: UnifiedActivityOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let activities_results: AtsActivity[] = []; + for (let i = 0; i < activities.length; i++) { + const activity = activities[i]; + const originId = activity.remote_id; + + if (!originId || originId == '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingActivity = await this.prisma.ats_activities.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_activity_id: string; + + if (existingActivity) { + // Update the existing activity + let data: any = { + modified_at: new Date(), + }; + if (activity.activity_type) { + data = { ...data, activity_type: activity.activity_type }; + } + if (activity.body) { + data = { ...data, activity_type: activity.body }; + } + if (activity.remote_created_at) { + data = { ...data, activity_type: activity.remote_created_at }; + } + if (activity.subject) { + data = { ...data, activity_type: activity.subject }; + } + if (activity.visibility) { + data = { ...data, activity_type: activity.visibility }; + } + if (activity.candidate_id) { + data = { ...data, id_ats_candidate: activity.candidate_id }; + } + const res = await this.prisma.ats_activities.update({ + where: { + id_ats_activity: existingActivity.id_ats_activity, + }, + data: data, + }); + unique_ats_activity_id = res.id_ats_activity; + activities_results = [...activities_results, res]; + } else { + // Create a new activity + this.logger.log('activity not exists'); + const uuid = uuidv4(); + let data: any = { + id_ats_activity: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (activity.activity_type) { + data = { ...data, activity_type: activity.activity_type }; + } + if (activity.body) { + data = { ...data, activity_type: activity.body }; + } + if (activity.remote_created_at) { + data = { ...data, activity_type: activity.remote_created_at }; + } + if (activity.subject) { + data = { ...data, activity_type: activity.subject }; + } + if (activity.visibility) { + data = { ...data, activity_type: activity.visibility }; + } + if (activity.candidate_id) { + data = { ...data, id_ats_candidate: activity.candidate_id }; + } + + const newActivity = await this.prisma.ats_activities.create({ + data: data, + }); + + unique_ats_activity_id = newActivity.id_ats_activity; + activities_results = [...activities_results, newActivity]; + } + + // check duplicate or existing values + if (activity.field_mappings && activity.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_activity_id, + }, + }); + + for (const [slug, value] of Object.entries(activity.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + //insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_activity_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_activity_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return activities_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/activity/types/index.ts b/packages/api/src/ats/activity/types/index.ts index 23c1f2d70..182ab678a 100644 --- a/packages/api/src/ats/activity/types/index.ts +++ b/packages/api/src/ats/activity/types/index.ts @@ -9,7 +9,7 @@ export interface IActivityService { linkedUserId: string, ): Promise>; - syncActivitys( + syncActivities( linkedUserId: string, custom_properties?: string[], ): Promise>; diff --git a/packages/api/src/ats/activity/types/model.unified.ts b/packages/api/src/ats/activity/types/model.unified.ts index f0f68ba4e..883cfd44b 100644 --- a/packages/api/src/ats/activity/types/model.unified.ts +++ b/packages/api/src/ats/activity/types/model.unified.ts @@ -92,7 +92,7 @@ export class UnifiedActivityOutput extends UnifiedActivityInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/application/services/application.service.ts b/packages/api/src/ats/application/services/application.service.ts index 9bb7def36..b51f03bf3 100644 --- a/packages/api/src/ats/application/services/application.service.ts +++ b/packages/api/src/ats/application/services/application.service.ts @@ -268,6 +268,9 @@ export class ApplicationService { candidate_id: application.id_ats_candidate, job_id: application.id_ats_job, field_mappings: field_mappings, + remote_id: application.remote_id, + created_at: application.created_at, + modified_at: application.modified_at, }; let res: UnifiedApplicationOutput = unifiedApplication; @@ -369,6 +372,9 @@ export class ApplicationService { candidate_id: application.id_ats_candidate, job_id: application.id_ats_job, field_mappings: field_mappings, + remote_id: application.remote_id, + created_at: application.created_at, + modified_at: application.modified_at, }; }), ); diff --git a/packages/api/src/ats/application/sync/sync.processor.ts b/packages/api/src/ats/application/sync/sync.processor.ts new file mode 100644 index 000000000..19f9b3307 --- /dev/null +++ b/packages/api/src/ats/application/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-applications') + async handleSyncApplications(job: Job) { + try { + console.log(`Processing queue -> ats-sync-applications ${job.id}`); + await this.syncService.syncApplications(); + } catch (error) { + console.error('Error syncing ats applications', error); + } + } +} diff --git a/packages/api/src/ats/application/sync/sync.service.ts b/packages/api/src/ats/application/sync/sync.service.ts index 139bf017d..7b2944101 100644 --- a/packages/api/src/ats/application/sync/sync.service.ts +++ b/packages/api/src/ats/application/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedApplicationOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IApplicationService } from '../types'; +import { OriginalApplicationOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedApplicationOutput } from '../types/model.unified'; +import { ats_applications as AtsApplication } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,357 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-applications'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncApplications(user_id?: string) { + try { + this.logger.log('Syncing applications...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncApplicationsForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncApplicationsForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} applications for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping applications syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.application', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IApplicationService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncApplications(linkedUserId, remoteProperties); + + const sourceObject: OriginalApplicationOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalApplicationOutput[] + >({ + sourceObject, + targetType: AtsObject.application, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedApplicationOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const applications_data = await this.saveApplicationsInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.application.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + applications_data, + 'ats.application.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveApplicationsInDb( + linkedUserId: string, + applications: UnifiedApplicationOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let applications_results: AtsApplication[] = []; + for (let i = 0; i < applications.length; i++) { + const application = applications[i]; + const originId = application.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingApplication = + await this.prisma.ats_applications.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_application_id: string; + + if (existingApplication) { + // Update the existing application + let data: any = { + modified_at: new Date(), + }; + if (application.applied_at) { + data = { ...data, applied_at: application.applied_at }; + } + if (application.rejected_at) { + data = { ...data, rejected_at: application.rejected_at }; + } + if (application.offers) { + data = { ...data, offers: application.offers }; + } + if (application.source) { + data = { ...data, source: application.source }; + } + if (application.credited_to) { + data = { ...data, credited_to: application.credited_to }; + } + if (application.current_stage) { + data = { ...data, current_stage: application.current_stage }; + } + if (application.reject_reason) { + data = { ...data, reject_reason: application.reject_reason }; + } + if (application.candidate_id) { + data = { ...data, candidate_id: application.candidate_id }; + } + if (application.job_id) { + data = { ...data, job_id: application.job_id }; + } + const res = await this.prisma.ats_applications.update({ + where: { + id_ats_application: existingApplication.id_ats_application, + }, + data: data, + }); + unique_ats_application_id = res.id_ats_application; + applications_results = [...applications_results, res]; + } else { + // Create a new application + this.logger.log('Application does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_application: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (application.applied_at) { + data = { ...data, applied_at: application.applied_at }; + } + if (application.rejected_at) { + data = { ...data, rejected_at: application.rejected_at }; + } + if (application.offers) { + data = { ...data, offers: application.offers }; + } + if (application.source) { + data = { ...data, source: application.source }; + } + if (application.credited_to) { + data = { ...data, credited_to: application.credited_to }; + } + if (application.current_stage) { + data = { ...data, current_stage: application.current_stage }; + } + if (application.reject_reason) { + data = { ...data, reject_reason: application.reject_reason }; + } + if (application.candidate_id) { + data = { ...data, candidate_id: application.candidate_id }; + } + if (application.job_id) { + data = { ...data, job_id: application.job_id }; + } + + const newApplication = await this.prisma.ats_applications.create({ + data: data, + }); + + unique_ats_application_id = newApplication.id_ats_application; + applications_results = [...applications_results, newApplication]; + } + + // check duplicate or existing values + if ( + application.field_mappings && + application.field_mappings.length > 0 + ) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_application_id, + }, + }); + + for (const [slug, value] of Object.entries( + application.field_mappings, + )) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_application_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_application_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return applications_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/application/types/model.unified.ts b/packages/api/src/ats/application/types/model.unified.ts index 261ff673a..d302d5200 100644 --- a/packages/api/src/ats/application/types/model.unified.ts +++ b/packages/api/src/ats/application/types/model.unified.ts @@ -123,7 +123,7 @@ export class UnifiedApplicationOutput extends UnifiedApplicationInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/attachment/services/attachment.service.ts b/packages/api/src/ats/attachment/services/attachment.service.ts index 3587b9b39..16a54bd16 100644 --- a/packages/api/src/ats/attachment/services/attachment.service.ts +++ b/packages/api/src/ats/attachment/services/attachment.service.ts @@ -263,6 +263,9 @@ export class AttachmentService { remote_modified_at: attachment.remote_modified_at, candidate_id: attachment.id_ats_candidate, field_mappings: field_mappings, + remote_id: attachment.remote_id, + created_at: attachment.created_at, + modified_at: attachment.modified_at, }; let res: UnifiedAttachmentOutput = unifiedAttachment; @@ -364,6 +367,9 @@ export class AttachmentService { remote_modified_at: attachment.remote_modified_at, candidate_id: attachment.id_ats_candidate, field_mappings: field_mappings, + remote_id: attachment.remote_id, + created_at: attachment.created_at, + modified_at: attachment.modified_at, }; }), ); diff --git a/packages/api/src/ats/attachment/sync/sync.processor.ts b/packages/api/src/ats/attachment/sync/sync.processor.ts new file mode 100644 index 000000000..ee6311e07 --- /dev/null +++ b/packages/api/src/ats/attachment/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-attachments') + async handleSyncAttachments(job: Job) { + try { + console.log(`Processing queue -> ats-sync-attachments ${job.id}`); + await this.syncService.syncAttachments(); + } catch (error) { + console.error('Error syncing ats attachments', error); + } + } +} diff --git a/packages/api/src/ats/attachment/sync/sync.service.ts b/packages/api/src/ats/attachment/sync/sync.service.ts index 768a2378e..a61777845 100644 --- a/packages/api/src/ats/attachment/sync/sync.service.ts +++ b/packages/api/src/ats/attachment/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedAttachmentOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IAttachmentService } from '../types'; +import { OriginalAttachmentOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedAttachmentOutput } from '../types/model.unified'; +import { ats_attachments as AtsAttachment } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,341 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-attachments'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncAttachments(user_id?: string) { + try { + this.logger.log('Syncing attachments...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncAttachmentsForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncAttachmentsForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} attachments for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping attachments syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.attachment', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IAttachmentService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncAttachments(linkedUserId, remoteProperties); + + const sourceObject: OriginalAttachmentOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalAttachmentOutput[] + >({ + sourceObject, + targetType: AtsObject.attachment, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedAttachmentOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const attachments_data = await this.saveAttachmentsInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.attachment.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + attachments_data, + 'ats.attachment.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveAttachmentsInDb( + linkedUserId: string, + attachments: UnifiedAttachmentOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let attachments_results: AtsAttachment[] = []; + for (let i = 0; i < attachments.length; i++) { + const attachment = attachments[i]; + const originId = attachment.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingAttachment = await this.prisma.ats_attachments.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_attachment_id: string; + + if (existingAttachment) { + // Update the existing attachment + let data: any = { + modified_at: new Date(), + }; + if (attachment.file_url) { + data = { ...data, file_url: attachment.file_url }; + } + if (attachment.file_name) { + data = { ...data, file_name: attachment.file_name }; + } + if (attachment.file_type) { + data = { ...data, file_type: attachment.file_type }; + } + if (attachment.remote_created_at) { + data = { ...data, remote_created_at: attachment.remote_created_at }; + } + if (attachment.remote_modified_at) { + data = { + ...data, + remote_modified_at: attachment.remote_modified_at, + }; + } + if (attachment.candidate_id) { + data = { ...data, candidate_id: attachment.candidate_id }; + } + const res = await this.prisma.ats_attachments.update({ + where: { + id_ats_attachment: existingAttachment.id_ats_attachment, + }, + data: data, + }); + unique_ats_attachment_id = res.id_ats_attachment; + attachments_results = [...attachments_results, res]; + } else { + // Create a new attachment + this.logger.log('Attachment does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_attachment: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (attachment.file_url) { + data = { ...data, file_url: attachment.file_url }; + } + if (attachment.file_name) { + data = { ...data, file_name: attachment.file_name }; + } + if (attachment.file_type) { + data = { ...data, file_type: attachment.file_type }; + } + if (attachment.remote_created_at) { + data = { ...data, remote_created_at: attachment.remote_created_at }; + } + if (attachment.remote_modified_at) { + data = { + ...data, + remote_modified_at: attachment.remote_modified_at, + }; + } + if (attachment.candidate_id) { + data = { ...data, candidate_id: attachment.candidate_id }; + } + + const newAttachment = await this.prisma.ats_attachments.create({ + data: data, + }); + + unique_ats_attachment_id = newAttachment.id_ats_attachment; + attachments_results = [...attachments_results, newAttachment]; + } + + // check duplicate or existing values + if (attachment.field_mappings && attachment.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_attachment_id, + }, + }); + + for (const [slug, value] of Object.entries( + attachment.field_mappings, + )) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_attachment_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_attachment_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return attachments_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/attachment/types/model.unified.ts b/packages/api/src/ats/attachment/types/model.unified.ts index 7a3117048..22da8a01a 100644 --- a/packages/api/src/ats/attachment/types/model.unified.ts +++ b/packages/api/src/ats/attachment/types/model.unified.ts @@ -86,7 +86,7 @@ export class UnifiedAttachmentOutput extends UnifiedAttachmentInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/candidate/sync/sync.processor.ts b/packages/api/src/ats/candidate/sync/sync.processor.ts new file mode 100644 index 000000000..25f0d3d7a --- /dev/null +++ b/packages/api/src/ats/candidate/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-candidates') + async handleSyncCandidates(job: Job) { + try { + console.log(`Processing queue -> ats-sync-candidates ${job.id}`); + await this.syncService.syncCandidates(); + } catch (error) { + console.error('Error syncing ats candidates', error); + } + } +} diff --git a/packages/api/src/ats/candidate/sync/sync.service.ts b/packages/api/src/ats/candidate/sync/sync.service.ts index 3ef7c8ac2..1824261d9 100644 --- a/packages/api/src/ats/candidate/sync/sync.service.ts +++ b/packages/api/src/ats/candidate/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedCandidateOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { ICandidateService } from '../types'; +import { OriginalCandidateOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedCandidateOutput } from '../types/model.unified'; +import { ats_candidates as AtsCandidate } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,371 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-candidates'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncCandidates(user_id?: string) { + try { + this.logger.log('Syncing candidates...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncCandidatesForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncCandidatesForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} candidates for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping candidates syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.candidate', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: ICandidateService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncCandidates(linkedUserId, remoteProperties); + + const sourceObject: OriginalCandidateOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalCandidateOutput[] + >({ + sourceObject, + targetType: AtsObject.candidate, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedCandidateOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const candidates_data = await this.saveCandidatesInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.candidate.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + candidates_data, + 'ats.candidate.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveCandidatesInDb( + linkedUserId: string, + candidates: UnifiedCandidateOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let candidates_results: AtsCandidate[] = []; + for (let i = 0; i < candidates.length; i++) { + const candidate = candidates[i]; + const originId = candidate.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingCandidate = await this.prisma.ats_candidates.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_candidate_id: string; + + if (existingCandidate) { + // Update the existing candidate + let data: any = { + modified_at: new Date(), + }; + if (candidate.first_name) { + data = { ...data, first_name: candidate.first_name }; + } + if (candidate.last_name) { + data = { ...data, last_name: candidate.last_name }; + } + if (candidate.company) { + data = { ...data, company: candidate.company }; + } + if (candidate.title) { + data = { ...data, title: candidate.title }; + } + if (candidate.locations) { + data = { ...data, locations: candidate.locations }; + } + if (candidate.is_private !== undefined) { + data = { ...data, is_private: candidate.is_private }; + } + if (candidate.email_reachable !== undefined) { + data = { ...data, email_reachable: candidate.email_reachable }; + } + if (candidate.remote_created_at) { + data = { ...data, remote_created_at: candidate.remote_created_at }; + } + if (candidate.remote_modified_at) { + data = { + ...data, + remote_modified_at: candidate.remote_modified_at, + }; + } + if (candidate.last_interaction_at) { + data = { + ...data, + last_interaction_at: candidate.last_interaction_at, + }; + } + const res = await this.prisma.ats_candidates.update({ + where: { + id_ats_candidate: existingCandidate.id_ats_candidate, + }, + data: data, + }); + unique_ats_candidate_id = res.id_ats_candidate; + candidates_results = [...candidates_results, res]; + } else { + // Create a new candidate + this.logger.log('Candidate does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_candidate: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (candidate.first_name) { + data = { ...data, first_name: candidate.first_name }; + } + if (candidate.last_name) { + data = { ...data, last_name: candidate.last_name }; + } + if (candidate.company) { + data = { ...data, company: candidate.company }; + } + if (candidate.title) { + data = { ...data, title: candidate.title }; + } + if (candidate.locations) { + data = { ...data, locations: candidate.locations }; + } + if (candidate.is_private !== undefined) { + data = { ...data, is_private: candidate.is_private }; + } + if (candidate.email_reachable !== undefined) { + data = { ...data, email_reachable: candidate.email_reachable }; + } + if (candidate.remote_created_at) { + data = { ...data, remote_created_at: candidate.remote_created_at }; + } + if (candidate.remote_modified_at) { + data = { + ...data, + remote_modified_at: candidate.remote_modified_at, + }; + } + if (candidate.last_interaction_at) { + data = { + ...data, + last_interaction_at: candidate.last_interaction_at, + }; + } + + const newCandidate = await this.prisma.ats_candidates.create({ + data: data, + }); + + unique_ats_candidate_id = newCandidate.id_ats_candidate; + candidates_results = [...candidates_results, newCandidate]; + } + + // check duplicate or existing values + if (candidate.field_mappings && candidate.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_candidate_id, + }, + }); + + for (const [slug, value] of Object.entries( + candidate.field_mappings, + )) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_candidate_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_candidate_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return candidates_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/candidate/types/model.unified.ts b/packages/api/src/ats/candidate/types/model.unified.ts index 1ae772e29..6c6704b94 100644 --- a/packages/api/src/ats/candidate/types/model.unified.ts +++ b/packages/api/src/ats/candidate/types/model.unified.ts @@ -148,7 +148,7 @@ export class UnifiedCandidateInput { export class UnifiedCandidateOutput extends UnifiedCandidateInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the candidate', + description: 'The UUID of the candidate', }) @IsUUID() @IsOptional() @@ -179,67 +179,12 @@ export class UnifiedCandidateOutput extends UnifiedCandidateInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; } -export class UnifiedCandidateTagInput { - @ApiPropertyOptional({ type: String, description: 'The name of the tag' }) - @IsString() - @IsOptional() - name?: string; - - @ApiPropertyOptional({ - type: String, - description: 'The UUID of the candidate', - }) - @IsUUID() - @IsOptional() - id_ats_candidate?: string; - - @ApiPropertyOptional({ - type: String, - format: 'date-time', - description: 'The creation date of the tag', - }) - @IsDateString() - @IsOptional() - created_at?: string; - - @ApiPropertyOptional({ - type: String, - format: 'date-time', - description: 'The modification date of the tag', - }) - @IsDateString() - @IsOptional() - modified_at?: string; -} - -export class UnifiedCandidateTagOutput extends UnifiedCandidateTagInput { - @ApiPropertyOptional({ type: String, description: 'The UUID of the tag' }) - @IsUUID() - @IsOptional() - id?: string; - - @ApiPropertyOptional({ - type: String, - description: 'The remote ID of the tag in the context of the 3rd Party', - }) - @IsString() - @IsOptional() - remote_id?: string; - - @ApiPropertyOptional({ - type: {}, - description: 'The remote data of the tag in the context of the 3rd Party', - }) - @IsOptional() - remote_data?: Record; -} - export class UnifiedCandidateUrlInput { @ApiPropertyOptional({ type: String, description: 'The value of the URL' }) @IsString() diff --git a/packages/api/src/ats/department/services/department.service.ts b/packages/api/src/ats/department/services/department.service.ts index b3a364f97..b76f01167 100644 --- a/packages/api/src/ats/department/services/department.service.ts +++ b/packages/api/src/ats/department/services/department.service.ts @@ -54,6 +54,9 @@ export class DepartmentService { id: department.id_ats_department, name: department.name, field_mappings: field_mappings, + remote_id: department.remote_id, + created_at: department.created_at, + modified_at: department.modified_at, }; let res: UnifiedDepartmentOutput = unifiedDepartment; @@ -124,6 +127,9 @@ export class DepartmentService { id: department.id_ats_department, name: department.name, field_mappings: field_mappings, + remote_id: department.remote_id, + created_at: department.created_at, + modified_at: department.modified_at, }; }), ); diff --git a/packages/api/src/ats/department/sync/sync.processor.ts b/packages/api/src/ats/department/sync/sync.processor.ts new file mode 100644 index 000000000..4850fc609 --- /dev/null +++ b/packages/api/src/ats/department/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-departments') + async handleSyncDepartments(job: Job) { + try { + console.log(`Processing queue -> ats-sync-departments ${job.id}`); + await this.syncService.syncDepartments(); + } catch (error) { + console.error('Error syncing ats departments', error); + } + } +} diff --git a/packages/api/src/ats/department/sync/sync.service.ts b/packages/api/src/ats/department/sync/sync.service.ts index 5926e1aa0..0873d7fec 100644 --- a/packages/api/src/ats/department/sync/sync.service.ts +++ b/packages/api/src/ats/department/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedDepartmentOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IDepartmentService } from '../types'; +import { OriginalDepartmentOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedDepartmentOutput } from '../types/model.unified'; +import { ats_departments as AtsDepartment } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,305 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-departments'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncDepartments(user_id?: string) { + try { + this.logger.log('Syncing departments...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncDepartmentsForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncDepartmentsForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} departments for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping departments syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.department', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IDepartmentService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncDepartments(linkedUserId, remoteProperties); + + const sourceObject: OriginalDepartmentOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalDepartmentOutput[] + >({ + sourceObject, + targetType: AtsObject.department, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedDepartmentOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const departments_data = await this.saveDepartmentsInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.department.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + departments_data, + 'ats.department.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveDepartmentsInDb( + linkedUserId: string, + departments: UnifiedDepartmentOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let departments_results: AtsDepartment[] = []; + for (let i = 0; i < departments.length; i++) { + const department = departments[i]; + const originId = department.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingDepartment = await this.prisma.ats_departments.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_department_id: string; + + if (existingDepartment) { + // Update the existing department + let data: any = { + modified_at: new Date(), + }; + if (department.name) { + data = { ...data, name: department.name }; + } + const res = await this.prisma.ats_departments.update({ + where: { + id_ats_department: existingDepartment.id_ats_department, + }, + data: data, + }); + unique_ats_department_id = res.id_ats_department; + departments_results = [...departments_results, res]; + } else { + // Create a new department + this.logger.log('Department does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_department: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (department.name) { + data = { ...data, name: department.name }; + } + + const newDepartment = await this.prisma.ats_departments.create({ + data: data, + }); + + unique_ats_department_id = newDepartment.id_ats_department; + departments_results = [...departments_results, newDepartment]; + } + + // check duplicate or existing values + if (department.field_mappings && department.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_department_id, + }, + }); + + for (const [slug, value] of Object.entries( + department.field_mappings, + )) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_department_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_department_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return departments_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/department/types/model.unified.ts b/packages/api/src/ats/department/types/model.unified.ts index fd3127b33..a874dcf6b 100644 --- a/packages/api/src/ats/department/types/model.unified.ts +++ b/packages/api/src/ats/department/types/model.unified.ts @@ -54,7 +54,7 @@ export class UnifiedDepartmentOutput extends UnifiedDepartmentInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/eeocs/services/eeocs.service.ts b/packages/api/src/ats/eeocs/services/eeocs.service.ts index c70ce2d50..a9c136547 100644 --- a/packages/api/src/ats/eeocs/services/eeocs.service.ts +++ b/packages/api/src/ats/eeocs/services/eeocs.service.ts @@ -59,6 +59,9 @@ export class EeocsService { veteran_status: eeocs.veteran_status, disability_status: eeocs.disability_status, field_mappings: field_mappings, + remote_id: eeocs.remote_id, + created_at: eeocs.created_at, + modified_at: eeocs.modified_at, }; let res: UnifiedEeocsOutput = unifiedEeocs; @@ -134,6 +137,9 @@ export class EeocsService { veteran_status: eeocs.veteran_status, disability_status: eeocs.disability_status, field_mappings: field_mappings, + remote_id: eeocs.remote_id, + created_at: eeocs.created_at, + modified_at: eeocs.modified_at, }; }), ); diff --git a/packages/api/src/ats/eeocs/sync/sync.processor.ts b/packages/api/src/ats/eeocs/sync/sync.processor.ts new file mode 100644 index 000000000..8c40b41f6 --- /dev/null +++ b/packages/api/src/ats/eeocs/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-eeocs') + async handleSyncEeocs(job: Job) { + try { + console.log(`Processing queue -> ats-sync-eeocs ${job.id}`); + await this.syncService.syncEeocs(); + } catch (error) { + console.error('Error syncing ats eeocs', error); + } + } +} diff --git a/packages/api/src/ats/eeocs/sync/sync.service.ts b/packages/api/src/ats/eeocs/sync/sync.service.ts index 88aeb3dbc..fc1da8853 100644 --- a/packages/api/src/ats/eeocs/sync/sync.service.ts +++ b/packages/api/src/ats/eeocs/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedEeocsOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IEeocsService } from '../types'; +import { OriginalEeocsOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedEeocsOutput } from '../types/model.unified'; +import { ats_eeocs as AtsEeocs } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,335 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-eeocs'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncEeocs(user_id?: string) { + try { + this.logger.log('Syncing EEOCs...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncEeocsForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncEeocsForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} EEOCs for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping EEOCs syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.eeocs', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IEeocsService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = await service.syncEeocs( + linkedUserId, + remoteProperties, + ); + + const sourceObject: OriginalEeocsOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalEeocsOutput[] + >({ + sourceObject, + targetType: AtsObject.eeocs, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedEeocsOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const eeocs_data = await this.saveEeocsInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.eeocs.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + eeocs_data, + 'ats.eeocs.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveEeocsInDb( + linkedUserId: string, + eeocs: UnifiedEeocsOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let eeocs_results: AtsEeocs[] = []; + for (let i = 0; i < eeocs.length; i++) { + const eeoc = eeocs[i]; + const originId = eeoc.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingEeoc = await this.prisma.ats_eeocs.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_eeoc_id: string; + + if (existingEeoc) { + // Update the existing EEOC + let data: any = { + modified_at: new Date(), + }; + if (eeoc.candidate_id) { + data = { ...data, candidate_id: eeoc.candidate_id }; + } + if (eeoc.submitted_at) { + data = { ...data, submitted_at: eeoc.submitted_at }; + } + if (eeoc.race) { + data = { ...data, race: eeoc.race }; + } + if (eeoc.gender) { + data = { ...data, gender: eeoc.gender }; + } + if (eeoc.veteran_status) { + data = { ...data, veteran_status: eeoc.veteran_status }; + } + if (eeoc.disability_status) { + data = { ...data, disability_status: eeoc.disability_status }; + } + const res = await this.prisma.ats_eeocs.update({ + where: { + id_ats_eeoc: existingEeoc.id_ats_eeoc, + }, + data: data, + }); + unique_ats_eeoc_id = res.id_ats_eeoc; + eeocs_results = [...eeocs_results, res]; + } else { + // Create a new EEOC + this.logger.log('EEOC does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_eeoc: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (eeoc.candidate_id) { + data = { ...data, candidate_id: eeoc.candidate_id }; + } + if (eeoc.submitted_at) { + data = { ...data, submitted_at: eeoc.submitted_at }; + } + if (eeoc.race) { + data = { ...data, race: eeoc.race }; + } + if (eeoc.gender) { + data = { ...data, gender: eeoc.gender }; + } + if (eeoc.veteran_status) { + data = { ...data, veteran_status: eeoc.veteran_status }; + } + if (eeoc.disability_status) { + data = { ...data, disability_status: eeoc.disability_status }; + } + + const newEeoc = await this.prisma.ats_eeocs.create({ + data: data, + }); + + unique_ats_eeoc_id = newEeoc.id_ats_eeoc; + eeocs_results = [...eeocs_results, newEeoc]; + } + + // check duplicate or existing values + if (eeoc.field_mappings && eeoc.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_eeoc_id, + }, + }); + + for (const [slug, value] of Object.entries(eeoc.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_eeoc_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_eeoc_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return eeocs_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/eeocs/types/model.unified.ts b/packages/api/src/ats/eeocs/types/model.unified.ts index ea5eff51d..482f7fe88 100644 --- a/packages/api/src/ats/eeocs/types/model.unified.ts +++ b/packages/api/src/ats/eeocs/types/model.unified.ts @@ -90,7 +90,7 @@ export class UnifiedEeocsOutput extends UnifiedEeocsInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/interview/services/interview.service.ts b/packages/api/src/ats/interview/services/interview.service.ts index e9f338c1e..1eacc1864 100644 --- a/packages/api/src/ats/interview/services/interview.service.ts +++ b/packages/api/src/ats/interview/services/interview.service.ts @@ -279,6 +279,9 @@ export class InterviewService { remote_created_at: interview.remote_created_at, remote_updated_at: interview.remote_updated_at, field_mappings: field_mappings, + remote_id: interview.remote_id, + created_at: interview.created_at, + modified_at: interview.modified_at, }; let res: UnifiedInterviewOutput = unifiedInterview; @@ -398,6 +401,9 @@ export class InterviewService { remote_created_at: interview.remote_created_at, remote_updated_at: interview.remote_updated_at, field_mappings: field_mappings, + remote_id: interview.remote_id, + created_at: interview.created_at, + modified_at: interview.modified_at, }; }), ); diff --git a/packages/api/src/ats/interview/sync/sync.processor.ts b/packages/api/src/ats/interview/sync/sync.processor.ts new file mode 100644 index 000000000..6127ccf17 --- /dev/null +++ b/packages/api/src/ats/interview/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-interviews') + async handleSyncInterviews(job: Job) { + try { + console.log(`Processing queue -> ats-sync-interviews ${job.id}`); + await this.syncService.syncInterviews(); + } catch (error) { + console.error('Error syncing ats interviews', error); + } + } +} diff --git a/packages/api/src/ats/interview/sync/sync.service.ts b/packages/api/src/ats/interview/sync/sync.service.ts index b23278cf0..82cab2921 100644 --- a/packages/api/src/ats/interview/sync/sync.service.ts +++ b/packages/api/src/ats/interview/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedInterviewOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IInterviewService } from '../types'; +import { OriginalInterviewOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedInterviewOutput } from '../types/model.unified'; +import { ats_interviews as AtsInterview } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,365 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-interviews'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncInterviews(user_id?: string) { + try { + this.logger.log('Syncing interviews...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncInterviewsForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncInterviewsForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} interviews for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping interviews syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.interview', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IInterviewService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncInterviews(linkedUserId, remoteProperties); + + const sourceObject: OriginalInterviewOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalInterviewOutput[] + >({ + sourceObject, + targetType: AtsObject.interview, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedInterviewOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const interviews_data = await this.saveInterviewsInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.interview.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + interviews_data, + 'ats.interview.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveInterviewsInDb( + linkedUserId: string, + interviews: UnifiedInterviewOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let interviews_results: AtsInterview[] = []; + for (let i = 0; i < interviews.length; i++) { + const interview = interviews[i]; + const originId = interview.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingInterview = await this.prisma.ats_interviews.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_interview_id: string; + + if (existingInterview) { + // Update the existing interview + let data: any = { + modified_at: new Date(), + }; + if (interview.status) { + data = { ...data, status: interview.status }; + } + if (interview.application_id) { + data = { ...data, application_id: interview.application_id }; + } + if (interview.job_interview_stage_id) { + data = { + ...data, + job_interview_stage_id: interview.job_interview_stage_id, + }; + } + if (interview.organized_by) { + data = { ...data, organized_by: interview.organized_by }; + } + if (interview.interviewers) { + data = { ...data, interviewers: interview.interviewers }; + } + if (interview.location) { + data = { ...data, location: interview.location }; + } + if (interview.start_at) { + data = { ...data, start_at: interview.start_at }; + } + if (interview.end_at) { + data = { ...data, end_at: interview.end_at }; + } + if (interview.remote_created_at) { + data = { ...data, remote_created_at: interview.remote_created_at }; + } + if (interview.remote_updated_at) { + data = { ...data, remote_updated_at: interview.remote_updated_at }; + } + const res = await this.prisma.ats_interviews.update({ + where: { + id_ats_interview: existingInterview.id_ats_interview, + }, + data: data, + }); + unique_ats_interview_id = res.id_ats_interview; + interviews_results = [...interviews_results, res]; + } else { + // Create a new interview + this.logger.log('Interview does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_interview: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (interview.status) { + data = { ...data, status: interview.status }; + } + if (interview.application_id) { + data = { ...data, application_id: interview.application_id }; + } + if (interview.job_interview_stage_id) { + data = { + ...data, + job_interview_stage_id: interview.job_interview_stage_id, + }; + } + if (interview.organized_by) { + data = { ...data, organized_by: interview.organized_by }; + } + if (interview.interviewers) { + data = { ...data, interviewers: interview.interviewers }; + } + if (interview.location) { + data = { ...data, location: interview.location }; + } + if (interview.start_at) { + data = { ...data, start_at: interview.start_at }; + } + if (interview.end_at) { + data = { ...data, end_at: interview.end_at }; + } + if (interview.remote_created_at) { + data = { ...data, remote_created_at: interview.remote_created_at }; + } + if (interview.remote_updated_at) { + data = { ...data, remote_updated_at: interview.remote_updated_at }; + } + + const newInterview = await this.prisma.ats_interviews.create({ + data: data, + }); + + unique_ats_interview_id = newInterview.id_ats_interview; + interviews_results = [...interviews_results, newInterview]; + } + + // check duplicate or existing values + if (interview.field_mappings && interview.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_interview_id, + }, + }); + + for (const [slug, value] of Object.entries( + interview.field_mappings, + )) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_interview_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_interview_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return interviews_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/interview/types/model.unified.ts b/packages/api/src/ats/interview/types/model.unified.ts index 1c66d9957..3d23b7b36 100644 --- a/packages/api/src/ats/interview/types/model.unified.ts +++ b/packages/api/src/ats/interview/types/model.unified.ts @@ -136,7 +136,7 @@ export class UnifiedInterviewOutput extends UnifiedInterviewInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/job/services/job.service.ts b/packages/api/src/ats/job/services/job.service.ts index 5a28d069c..d60e4f41d 100644 --- a/packages/api/src/ats/job/services/job.service.ts +++ b/packages/api/src/ats/job/services/job.service.ts @@ -65,6 +65,9 @@ export class JobService { remote_created_at: job.remote_created_at, remote_updated_at: job.remote_updated_at, field_mappings: field_mappings, + remote_id: job.remote_id, + created_at: job.created_at, + modified_at: job.modified_at, }; let res: UnifiedJobOutput = unifiedJob; @@ -146,6 +149,9 @@ export class JobService { remote_created_at: job.remote_created_at, remote_updated_at: job.remote_updated_at, field_mappings: field_mappings, + remote_id: job.remote_id, + created_at: job.created_at, + modified_at: job.modified_at, }; }), ); diff --git a/packages/api/src/ats/job/sync/sync.processor.ts b/packages/api/src/ats/job/sync/sync.processor.ts new file mode 100644 index 000000000..159267786 --- /dev/null +++ b/packages/api/src/ats/job/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-jobs') + async handleSyncJobs(job: Job) { + try { + console.log(`Processing queue -> ats-sync-jobs ${job.id}`); + await this.syncService.syncJobs(); + } catch (error) { + console.error('Error syncing ats jobs', error); + } + } +} diff --git a/packages/api/src/ats/job/sync/sync.service.ts b/packages/api/src/ats/job/sync/sync.service.ts index fc1884c35..0db97bcb8 100644 --- a/packages/api/src/ats/job/sync/sync.service.ts +++ b/packages/api/src/ats/job/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedJobOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IJobService } from '../types'; +import { OriginalJobOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedJobOutput } from '../types/model.unified'; +import { ats_jobs as AtsJob } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,371 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-jobs'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncJobs(user_id?: string) { + try { + this.logger.log('Syncing jobs...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncJobsForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncJobsForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} jobs for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping jobs syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.job', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IJobService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = await service.syncJobs( + linkedUserId, + remoteProperties, + ); + + const sourceObject: OriginalJobOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalJobOutput[] + >({ + sourceObject, + targetType: AtsObject.job, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedJobOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const jobs_data = await this.saveJobsInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.job.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + jobs_data, + 'ats.job.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveJobsInDb( + linkedUserId: string, + jobs: UnifiedJobOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let jobs_results: AtsJob[] = []; + for (let i = 0; i < jobs.length; i++) { + const job = jobs[i]; + const originId = job.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingJob = await this.prisma.ats_jobs.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_job_id: string; + + if (existingJob) { + // Update the existing job + let data: any = { + modified_at: new Date(), + }; + if (job.name) { + data = { ...data, name: job.name }; + } + if (job.description) { + data = { ...data, description: job.description }; + } + if (job.code) { + data = { ...data, code: job.code }; + } + if (job.status) { + data = { ...data, status: job.status }; + } + if (job.type) { + data = { ...data, type: job.type }; + } + if (job.confidential !== undefined) { + data = { ...data, confidential: job.confidential }; + } + if (job.departments) { + data = { ...data, departments: job.departments }; + } + if (job.offices) { + data = { ...data, offices: job.offices }; + } + if (job.managers) { + data = { ...data, managers: job.managers }; + } + if (job.recruiters) { + data = { ...data, recruiters: job.recruiters }; + } + if (job.remote_created_at) { + data = { ...data, remote_created_at: job.remote_created_at }; + } + if (job.remote_updated_at) { + data = { ...data, remote_updated_at: job.remote_updated_at }; + } + const res = await this.prisma.ats_jobs.update({ + where: { + id_ats_job: existingJob.id_ats_job, + }, + data: data, + }); + unique_ats_job_id = res.id_ats_job; + jobs_results = [...jobs_results, res]; + } else { + // Create a new job + this.logger.log('Job does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_job: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (job.name) { + data = { ...data, name: job.name }; + } + if (job.description) { + data = { ...data, description: job.description }; + } + if (job.code) { + data = { ...data, code: job.code }; + } + if (job.status) { + data = { ...data, status: job.status }; + } + if (job.type) { + data = { ...data, type: job.type }; + } + if (job.confidential !== undefined) { + data = { ...data, confidential: job.confidential }; + } + if (job.departments) { + data = { ...data, departments: job.departments }; + } + if (job.offices) { + data = { ...data, offices: job.offices }; + } + if (job.managers) { + data = { ...data, managers: job.managers }; + } + if (job.recruiters) { + data = { ...data, recruiters: job.recruiters }; + } + if (job.remote_created_at) { + data = { ...data, remote_created_at: job.remote_created_at }; + } + if (job.remote_updated_at) { + data = { ...data, remote_updated_at: job.remote_updated_at }; + } + + const newJob = await this.prisma.ats_jobs.create({ + data: data, + }); + + unique_ats_job_id = newJob.id_ats_job; + jobs_results = [...jobs_results, newJob]; + } + + // check duplicate or existing values + if (job.field_mappings && job.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_job_id, + }, + }); + + for (const [slug, value] of Object.entries(job.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_job_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_job_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return jobs_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/job/types/model.unified.ts b/packages/api/src/ats/job/types/model.unified.ts index 114b6217b..cdf603925 100644 --- a/packages/api/src/ats/job/types/model.unified.ts +++ b/packages/api/src/ats/job/types/model.unified.ts @@ -134,7 +134,7 @@ export class UnifiedJobOutput extends UnifiedJobInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/jobinterviewstage/services/jobinterviewstage.service.ts b/packages/api/src/ats/jobinterviewstage/services/jobinterviewstage.service.ts index bcae4b30b..a0850e4e7 100644 --- a/packages/api/src/ats/jobinterviewstage/services/jobinterviewstage.service.ts +++ b/packages/api/src/ats/jobinterviewstage/services/jobinterviewstage.service.ts @@ -58,6 +58,9 @@ export class JobInterviewStageService { stage_order: stage.stage_order, job_id: stage.job_id, field_mappings: field_mappings, + remote_id: stage.remote_id, + created_at: stage.created_at, + modified_at: stage.modified_at, }; let res: UnifiedJobInterviewStageOutput = unifiedJobInterviewStage; @@ -130,6 +133,9 @@ export class JobInterviewStageService { stage_order: stage.stage_order, job_id: stage.job_id, field_mappings: field_mappings, + remote_id: stage.remote_id, + created_at: stage.created_at, + modified_at: stage.modified_at, }; }), ); diff --git a/packages/api/src/ats/jobinterviewstage/sync/sync.processor.ts b/packages/api/src/ats/jobinterviewstage/sync/sync.processor.ts new file mode 100644 index 000000000..b98cdbb20 --- /dev/null +++ b/packages/api/src/ats/jobinterviewstage/sync/sync.processor.ts @@ -0,0 +1,20 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-job-interview-stages') + async handleSyncJobInterviewStages(job: Job) { + try { + console.log( + `Processing queue -> ats-sync-job-interview-stages ${job.id}`, + ); + await this.syncService.syncJobInterviewStages(); + } catch (error) { + console.error('Error syncing ats job interview stages', error); + } + } +} diff --git a/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts b/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts index efbd7b51d..fecda080c 100644 --- a/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts +++ b/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedJobInterviewStageOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IJobInterviewStageService } from '../types'; +import { OriginalJobInterviewStageOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedJobInterviewStageOutput } from '../types/model.unified'; +import { ats_job_interview_stages as AtsJobInterviewStage } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,329 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-job-interview-stages'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncJobInterviewStages(user_id?: string) { + try { + this.logger.log('Syncing job interview stages...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncJobInterviewStagesForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncJobInterviewStagesForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} job interview stages for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping job interview stages syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.job_interview_stage', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IJobInterviewStageService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncJobInterviewStages(linkedUserId, remoteProperties); + + const sourceObject: OriginalJobInterviewStageOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalJobInterviewStageOutput[] + >({ + sourceObject, + targetType: AtsObject.jobinterviewstage, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedJobInterviewStageOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const jobInterviewStages_data = await this.saveJobInterviewStagesInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.job_interview_stage.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + jobInterviewStages_data, + 'ats.job_interview_stage.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveJobInterviewStagesInDb( + linkedUserId: string, + jobInterviewStages: UnifiedJobInterviewStageOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let jobInterviewStages_results: AtsJobInterviewStage[] = []; + for (let i = 0; i < jobInterviewStages.length; i++) { + const jobInterviewStage = jobInterviewStages[i]; + const originId = jobInterviewStage.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingJobInterviewStage = + await this.prisma.ats_job_interview_stages.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_job_interview_stage_id: string; + + if (existingJobInterviewStage) { + // Update the existing job interview stage + let data: any = { + modified_at: new Date(), + }; + if (jobInterviewStage.name) { + data = { ...data, name: jobInterviewStage.name }; + } + if (jobInterviewStage.stage_order) { + data = { ...data, stage_order: jobInterviewStage.stage_order }; + } + if (jobInterviewStage.job_id) { + data = { ...data, job_id: jobInterviewStage.job_id }; + } + const res = await this.prisma.ats_job_interview_stages.update({ + where: { + id_ats_job_interview_stage: + existingJobInterviewStage.id_ats_job_interview_stage, + }, + data: data, + }); + unique_ats_job_interview_stage_id = res.id_ats_job_interview_stage; + jobInterviewStages_results = [...jobInterviewStages_results, res]; + } else { + // Create a new job interview stage + this.logger.log( + 'Job interview stage does not exist, creating a new one', + ); + const uuid = uuidv4(); + let data: any = { + id_ats_job_interview_stage: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (jobInterviewStage.name) { + data = { ...data, name: jobInterviewStage.name }; + } + if (jobInterviewStage.stage_order) { + data = { ...data, stage_order: jobInterviewStage.stage_order }; + } + if (jobInterviewStage.job_id) { + data = { ...data, job_id: jobInterviewStage.job_id }; + } + + const newJobInterviewStage = + await this.prisma.ats_job_interview_stages.create({ + data: data, + }); + + unique_ats_job_interview_stage_id = + newJobInterviewStage.id_ats_job_interview_stage; + jobInterviewStages_results = [ + ...jobInterviewStages_results, + newJobInterviewStage, + ]; + } + + // check duplicate or existing values + if ( + jobInterviewStage.field_mappings && + jobInterviewStage.field_mappings.length > 0 + ) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_job_interview_stage_id, + }, + }); + + for (const [slug, value] of Object.entries( + jobInterviewStage.field_mappings, + )) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_job_interview_stage_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_job_interview_stage_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return jobInterviewStages_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/jobinterviewstage/types/model.unified.ts b/packages/api/src/ats/jobinterviewstage/types/model.unified.ts index 2960ec0a1..b15cd6142 100644 --- a/packages/api/src/ats/jobinterviewstage/types/model.unified.ts +++ b/packages/api/src/ats/jobinterviewstage/types/model.unified.ts @@ -70,7 +70,7 @@ export class UnifiedJobInterviewStageOutput extends UnifiedJobInterviewStageInpu @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/offer/services/offer.service.ts b/packages/api/src/ats/offer/services/offer.service.ts index 5e7878c11..22d1419dd 100644 --- a/packages/api/src/ats/offer/services/offer.service.ts +++ b/packages/api/src/ats/offer/services/offer.service.ts @@ -60,6 +60,9 @@ export class OfferService { status: offer.status, application_id: offer.application_id, field_mappings: field_mappings, + remote_id: offer.remote_id, + created_at: offer.created_at, + modified_at: offer.modified_at, }; let res: UnifiedOfferOutput = unifiedOffer; @@ -137,6 +140,9 @@ export class OfferService { status: offer.status, application_id: offer.application_id, field_mappings: field_mappings, + remote_id: offer.remote_id, + created_at: offer.created_at, + modified_at: offer.modified_at, }; }), ); diff --git a/packages/api/src/ats/offer/sync/sync.processor.ts b/packages/api/src/ats/offer/sync/sync.processor.ts new file mode 100644 index 000000000..2ae7851b1 --- /dev/null +++ b/packages/api/src/ats/offer/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-offers') + async handleSyncOffers(job: Job) { + try { + console.log(`Processing queue -> ats-sync-offers ${job.id}`); + await this.syncService.syncOffers(); + } catch (error) { + console.error('Error syncing ats offers', error); + } + } +} diff --git a/packages/api/src/ats/offer/sync/sync.service.ts b/packages/api/src/ats/offer/sync/sync.service.ts index 60f41e49c..7189b36e0 100644 --- a/packages/api/src/ats/offer/sync/sync.service.ts +++ b/packages/api/src/ats/offer/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedOfferOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IOfferService } from '../types'; +import { OriginalOfferOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedOfferOutput } from '../types/model.unified'; +import { ats_offers as AtsOffer } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,341 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-offers'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncOffers(user_id?: string) { + try { + this.logger.log('Syncing offers...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncOffersForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncOffersForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} offers for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping offers syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.offer', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IOfferService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = await service.syncOffers( + linkedUserId, + remoteProperties, + ); + + const sourceObject: OriginalOfferOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalOfferOutput[] + >({ + sourceObject, + targetType: AtsObject.offer, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedOfferOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const offers_data = await this.saveOffersInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.offer.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + offers_data, + 'ats.offer.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveOffersInDb( + linkedUserId: string, + offers: UnifiedOfferOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let offers_results: AtsOffer[] = []; + for (let i = 0; i < offers.length; i++) { + const offer = offers[i]; + const originId = offer.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingOffer = await this.prisma.ats_offers.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_offer_id: string; + + if (existingOffer) { + // Update the existing offer + let data: any = { + modified_at: new Date(), + }; + if (offer.created_by) { + data = { ...data, created_by: offer.created_by }; + } + if (offer.remote_created_at) { + data = { ...data, remote_created_at: offer.remote_created_at }; + } + if (offer.closed_at) { + data = { ...data, closed_at: offer.closed_at }; + } + if (offer.sent_at) { + data = { ...data, sent_at: offer.sent_at }; + } + if (offer.start_date) { + data = { ...data, start_date: offer.start_date }; + } + if (offer.status) { + data = { ...data, status: offer.status }; + } + if (offer.application_id) { + data = { ...data, application_id: offer.application_id }; + } + const res = await this.prisma.ats_offers.update({ + where: { + id_ats_offer: existingOffer.id_ats_offer, + }, + data: data, + }); + unique_ats_offer_id = res.id_ats_offer; + offers_results = [...offers_results, res]; + } else { + // Create a new offer + this.logger.log('Offer does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_offer: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (offer.created_by) { + data = { ...data, created_by: offer.created_by }; + } + if (offer.remote_created_at) { + data = { ...data, remote_created_at: offer.remote_created_at }; + } + if (offer.closed_at) { + data = { ...data, closed_at: offer.closed_at }; + } + if (offer.sent_at) { + data = { ...data, sent_at: offer.sent_at }; + } + if (offer.start_date) { + data = { ...data, start_date: offer.start_date }; + } + if (offer.status) { + data = { ...data, status: offer.status }; + } + if (offer.application_id) { + data = { ...data, application_id: offer.application_id }; + } + + const newOffer = await this.prisma.ats_offers.create({ + data: data, + }); + + unique_ats_offer_id = newOffer.id_ats_offer; + offers_results = [...offers_results, newOffer]; + } + + // check duplicate or existing values + if (offer.field_mappings && offer.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_offer_id, + }, + }); + + for (const [slug, value] of Object.entries(offer.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_offer_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_offer_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return offers_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/offer/types/model.unified.ts b/packages/api/src/ats/offer/types/model.unified.ts index a75d4b9bf..6547ff9e9 100644 --- a/packages/api/src/ats/offer/types/model.unified.ts +++ b/packages/api/src/ats/offer/types/model.unified.ts @@ -95,7 +95,7 @@ export class UnifiedOfferOutput extends UnifiedOfferInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/office/services/office.service.ts b/packages/api/src/ats/office/services/office.service.ts index cc92bb96e..5a78bb8ab 100644 --- a/packages/api/src/ats/office/services/office.service.ts +++ b/packages/api/src/ats/office/services/office.service.ts @@ -55,6 +55,9 @@ export class OfficeService { name: office.name, location: office.location, field_mappings: field_mappings, + remote_id: office.remote_id, + created_at: office.created_at, + modified_at: office.modified_at, }; let res: UnifiedOfficeOutput = unifiedOffice; @@ -126,6 +129,9 @@ export class OfficeService { name: office.name, location: office.location, field_mappings: field_mappings, + remote_id: office.remote_id, + created_at: office.created_at, + modified_at: office.modified_at, }; }), ); diff --git a/packages/api/src/ats/office/sync/sync.processor.ts b/packages/api/src/ats/office/sync/sync.processor.ts new file mode 100644 index 000000000..7bc5ac853 --- /dev/null +++ b/packages/api/src/ats/office/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-offices') + async handleSyncOffices(job: Job) { + try { + console.log(`Processing queue -> ats-sync-offices ${job.id}`); + await this.syncService.syncOffices(); + } catch (error) { + console.error('Error syncing ats offices', error); + } + } +} diff --git a/packages/api/src/ats/office/sync/sync.service.ts b/packages/api/src/ats/office/sync/sync.service.ts index df94041c2..37ae12cd2 100644 --- a/packages/api/src/ats/office/sync/sync.service.ts +++ b/packages/api/src/ats/office/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedOfficeOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IOfficeService } from '../types'; +import { OriginalOfficeOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedOfficeOutput } from '../types/model.unified'; +import { ats_offices as AtsOffice } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,309 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-offices'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncOffices(user_id?: string) { + try { + this.logger.log('Syncing offices...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncOfficesForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncOfficesForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} offices for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping offices syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.office', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IOfficeService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncOffices(linkedUserId, remoteProperties); + + const sourceObject: OriginalOfficeOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalOfficeOutput[] + >({ + sourceObject, + targetType: AtsObject.office, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedOfficeOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const offices_data = await this.saveOfficesInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.office.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + offices_data, + 'ats.office.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveOfficesInDb( + linkedUserId: string, + offices: UnifiedOfficeOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let offices_results: AtsOffice[] = []; + for (let i = 0; i < offices.length; i++) { + const office = offices[i]; + const originId = office.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingOffice = await this.prisma.ats_offices.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_office_id: string; + + if (existingOffice) { + // Update the existing office + let data: any = { + modified_at: new Date(), + }; + if (office.name) { + data = { ...data, name: office.name }; + } + if (office.location) { + data = { ...data, location: office.location }; + } + const res = await this.prisma.ats_offices.update({ + where: { + id_ats_office: existingOffice.id_ats_office, + }, + data: data, + }); + unique_ats_office_id = res.id_ats_office; + offices_results = [...offices_results, res]; + } else { + // Create a new office + this.logger.log('Office does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_office: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (office.name) { + data = { ...data, name: office.name }; + } + if (office.location) { + data = { ...data, location: office.location }; + } + + const newOffice = await this.prisma.ats_offices.create({ + data: data, + }); + + unique_ats_office_id = newOffice.id_ats_office; + offices_results = [...offices_results, newOffice]; + } + + // check duplicate or existing values + if (office.field_mappings && office.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_office_id, + }, + }); + + for (const [slug, value] of Object.entries(office.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_office_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_office_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return offices_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/office/types/model.unified.ts b/packages/api/src/ats/office/types/model.unified.ts index 59624ca5f..10f6a8257 100644 --- a/packages/api/src/ats/office/types/model.unified.ts +++ b/packages/api/src/ats/office/types/model.unified.ts @@ -55,7 +55,7 @@ export class UnifiedOfficeOutput extends UnifiedOfficeInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/rejectreason/services/rejectreason.service.ts b/packages/api/src/ats/rejectreason/services/rejectreason.service.ts index f8dbb733c..1d02e0a45 100644 --- a/packages/api/src/ats/rejectreason/services/rejectreason.service.ts +++ b/packages/api/src/ats/rejectreason/services/rejectreason.service.ts @@ -56,6 +56,9 @@ export class RejectReasonService { id: rejectReason.id_ats_reject_reason, name: rejectReason.name, field_mappings: field_mappings, + remote_id: rejectReason.remote_id, + created_at: rejectReason.created_at, + modified_at: rejectReason.modified_at, }; let res: UnifiedRejectReasonOutput = unifiedRejectReason; @@ -127,6 +130,9 @@ export class RejectReasonService { id: rejectReason.id_ats_reject_reason, name: rejectReason.name, field_mappings: field_mappings, + remote_id: rejectReason.remote_id, + created_at: rejectReason.created_at, + modified_at: rejectReason.modified_at, }; }), ); diff --git a/packages/api/src/ats/rejectreason/sync/sync.processor.ts b/packages/api/src/ats/rejectreason/sync/sync.processor.ts new file mode 100644 index 000000000..644218792 --- /dev/null +++ b/packages/api/src/ats/rejectreason/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-reject-reasons') + async handleSyncRejectReasons(job: Job) { + try { + console.log(`Processing queue -> ats-sync-reject-reasons ${job.id}`); + await this.syncService.syncRejectReasons(); + } catch (error) { + console.error('Error syncing ats reject reasons', error); + } + } +} diff --git a/packages/api/src/ats/rejectreason/sync/sync.service.ts b/packages/api/src/ats/rejectreason/sync/sync.service.ts index 180cee46e..2212725f8 100644 --- a/packages/api/src/ats/rejectreason/sync/sync.service.ts +++ b/packages/api/src/ats/rejectreason/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedRejectReasonOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IRejectReasonService } from '../types'; +import { OriginalRejectReasonOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedRejectReasonOutput } from '../types/model.unified'; +import { ats_reject_reasons as AtsRejectReason } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,309 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-reject-reasons'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncRejectReasons(user_id?: string) { + try { + this.logger.log('Syncing reject reasons...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncRejectReasonsForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncRejectReasonsForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} reject reasons for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping reject reasons syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.reject_reason', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IRejectReasonService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncRejectReasons(linkedUserId, remoteProperties); + + const sourceObject: OriginalRejectReasonOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalRejectReasonOutput[] + >({ + sourceObject, + targetType: AtsObject.rejectreason, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedRejectReasonOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const rejectReasons_data = await this.saveRejectReasonsInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.reject_reason.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + rejectReasons_data, + 'ats.reject_reason.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveRejectReasonsInDb( + linkedUserId: string, + rejectReasons: UnifiedRejectReasonOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let rejectReasons_results: AtsRejectReason[] = []; + for (let i = 0; i < rejectReasons.length; i++) { + const rejectReason = rejectReasons[i]; + const originId = rejectReason.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingRejectReason = + await this.prisma.ats_reject_reasons.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_reject_reason_id: string; + + if (existingRejectReason) { + // Update the existing reject reason + let data: any = { + modified_at: new Date(), + }; + if (rejectReason.name) { + data = { ...data, name: rejectReason.name }; + } + const res = await this.prisma.ats_reject_reasons.update({ + where: { + id_ats_reject_reason: existingRejectReason.id_ats_reject_reason, + }, + data: data, + }); + unique_ats_reject_reason_id = res.id_ats_reject_reason; + rejectReasons_results = [...rejectReasons_results, res]; + } else { + // Create a new reject reason + this.logger.log('Reject reason does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_reject_reason: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (rejectReason.name) { + data = { ...data, name: rejectReason.name }; + } + + const newRejectReason = await this.prisma.ats_reject_reasons.create({ + data: data, + }); + + unique_ats_reject_reason_id = newRejectReason.id_ats_reject_reason; + rejectReasons_results = [...rejectReasons_results, newRejectReason]; + } + + // check duplicate or existing values + if ( + rejectReason.field_mappings && + rejectReason.field_mappings.length > 0 + ) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_reject_reason_id, + }, + }); + + for (const [slug, value] of Object.entries( + rejectReason.field_mappings, + )) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_reject_reason_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_reject_reason_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return rejectReasons_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/rejectreason/types/model.unified.ts b/packages/api/src/ats/rejectreason/types/model.unified.ts index 54c68e532..8f3952109 100644 --- a/packages/api/src/ats/rejectreason/types/model.unified.ts +++ b/packages/api/src/ats/rejectreason/types/model.unified.ts @@ -54,7 +54,7 @@ export class UnifiedRejectReasonOutput extends UnifiedRejectReasonInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/scorecard/services/scorecard.service.ts b/packages/api/src/ats/scorecard/services/scorecard.service.ts index 0eba18344..5b88731e9 100644 --- a/packages/api/src/ats/scorecard/services/scorecard.service.ts +++ b/packages/api/src/ats/scorecard/services/scorecard.service.ts @@ -58,6 +58,9 @@ export class ScoreCardService { remote_created_at: scorecard.remote_created_at, submitted_at: scorecard.submitted_at, field_mappings: field_mappings, + remote_id: scorecard.remote_id, + created_at: scorecard.created_at, + modified_at: scorecard.modified_at, }; let res: UnifiedScoreCardOutput = unifiedScoreCard; @@ -132,6 +135,9 @@ export class ScoreCardService { remote_created_at: scorecard.remote_created_at, submitted_at: scorecard.submitted_at, field_mappings: field_mappings, + remote_id: scorecard.remote_id, + created_at: scorecard.created_at, + modified_at: scorecard.modified_at, }; }), ); diff --git a/packages/api/src/ats/scorecard/sync/sync.processor.ts b/packages/api/src/ats/scorecard/sync/sync.processor.ts new file mode 100644 index 000000000..4c6eea3e0 --- /dev/null +++ b/packages/api/src/ats/scorecard/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-score-cards') + async handleSyncScoreCards(job: Job) { + try { + console.log(`Processing queue -> ats-sync-score-cards ${job.id}`); + await this.syncService.syncScoreCards(); + } catch (error) { + console.error('Error syncing ats score cards', error); + } + } +} diff --git a/packages/api/src/ats/scorecard/sync/sync.service.ts b/packages/api/src/ats/scorecard/sync/sync.service.ts index 7a6b1e97c..e551a0fd8 100644 --- a/packages/api/src/ats/scorecard/sync/sync.service.ts +++ b/packages/api/src/ats/scorecard/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedScoreCardOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IScoreCardService } from '../types'; +import { OriginalScoreCardOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedScoreCardOutput } from '../types/model.unified'; +import { ats_score_cards as AtsScoreCard } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,335 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-score-cards'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncScoreCards(user_id?: string) { + try { + this.logger.log('Syncing score cards...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncScoreCardsForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncScoreCardsForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} score cards for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping score cards syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.score_card', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IScoreCardService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncScoreCards(linkedUserId, remoteProperties); + + const sourceObject: OriginalScoreCardOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalScoreCardOutput[] + >({ + sourceObject, + targetType: AtsObject.scorecard, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedScoreCardOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const scoreCards_data = await this.saveScoreCardsInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.score_card.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + scoreCards_data, + 'ats.score_card.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveScoreCardsInDb( + linkedUserId: string, + scoreCards: UnifiedScoreCardOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let scoreCards_results: AtsScoreCard[] = []; + for (let i = 0; i < scoreCards.length; i++) { + const scoreCard = scoreCards[i]; + const originId = scoreCard.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingScoreCard = await this.prisma.ats_score_cards.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_score_card_id: string; + + if (existingScoreCard) { + // Update the existing score card + let data: any = { + modified_at: new Date(), + }; + if (scoreCard.overall_recommendation) { + data = { + ...data, + overall_recommendation: scoreCard.overall_recommendation, + }; + } + if (scoreCard.application_id) { + data = { ...data, application_id: scoreCard.application_id }; + } + if (scoreCard.interview_id) { + data = { ...data, interview_id: scoreCard.interview_id }; + } + if (scoreCard.remote_created_at) { + data = { ...data, remote_created_at: scoreCard.remote_created_at }; + } + if (scoreCard.submitted_at) { + data = { ...data, submitted_at: scoreCard.submitted_at }; + } + const res = await this.prisma.ats_score_cards.update({ + where: { + id_ats_score_card: existingScoreCard.id_ats_score_card, + }, + data: data, + }); + unique_ats_score_card_id = res.id_ats_score_card; + scoreCards_results = [...scoreCards_results, res]; + } else { + // Create a new score card + this.logger.log('Score card does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_score_card: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (scoreCard.overall_recommendation) { + data = { + ...data, + overall_recommendation: scoreCard.overall_recommendation, + }; + } + if (scoreCard.application_id) { + data = { ...data, application_id: scoreCard.application_id }; + } + if (scoreCard.interview_id) { + data = { ...data, interview_id: scoreCard.interview_id }; + } + if (scoreCard.remote_created_at) { + data = { ...data, remote_created_at: scoreCard.remote_created_at }; + } + if (scoreCard.submitted_at) { + data = { ...data, submitted_at: scoreCard.submitted_at }; + } + + const newScoreCard = await this.prisma.ats_score_cards.create({ + data: data, + }); + + unique_ats_score_card_id = newScoreCard.id_ats_score_card; + scoreCards_results = [...scoreCards_results, newScoreCard]; + } + + // check duplicate or existing values + if (scoreCard.field_mappings && scoreCard.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_score_card_id, + }, + }); + + for (const [slug, value] of Object.entries( + scoreCard.field_mappings, + )) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_score_card_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_score_card_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return scoreCards_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/scorecard/types/model.unified.ts b/packages/api/src/ats/scorecard/types/model.unified.ts index 3ffca4458..c2a818514 100644 --- a/packages/api/src/ats/scorecard/types/model.unified.ts +++ b/packages/api/src/ats/scorecard/types/model.unified.ts @@ -88,7 +88,7 @@ export class UnifiedScoreCardOutput extends UnifiedScoreCardInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ats/screeningquestion/sync/sync.processor.ts b/packages/api/src/ats/screeningquestion/sync/sync.processor.ts new file mode 100644 index 000000000..25f0d3d7a --- /dev/null +++ b/packages/api/src/ats/screeningquestion/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-candidates') + async handleSyncCandidates(job: Job) { + try { + console.log(`Processing queue -> ats-sync-candidates ${job.id}`); + await this.syncService.syncCandidates(); + } catch (error) { + console.error('Error syncing ats candidates', error); + } + } +} diff --git a/packages/api/src/ats/tag/services/tag.service.ts b/packages/api/src/ats/tag/services/tag.service.ts index 1f85a8ede..b0352977b 100644 --- a/packages/api/src/ats/tag/services/tag.service.ts +++ b/packages/api/src/ats/tag/services/tag.service.ts @@ -1,44 +1,83 @@ import { Injectable } from '@nestjs/common'; import { PrismaService } from '@@core/prisma/prisma.service'; import { LoggerService } from '@@core/logger/logger.service'; -import { v4 as uuidv4 } from 'uuid'; -import { ApiResponse } from '@@core/utils/types'; -import { throwTypedError } from '@@core/utils/errors'; -import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedTagInput, UnifiedTagOutput } from '../types/model.unified'; - -import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; -import { ServiceRegistry } from './registry.service'; -import { OriginalTagOutput } from '@@core/utils/types/original/original.ats'; - -import { ITagService } from '../types'; +import { UnifiedTagOutput } from '../types/model.unified'; @Injectable() export class TagService { - constructor( - private prisma: PrismaService, - private logger: LoggerService, - private webhook: WebhookService, - private fieldMappingService: FieldMappingService, - private serviceRegistry: ServiceRegistry, - ) { + constructor(private prisma: PrismaService, private logger: LoggerService) { this.logger.setContext(TagService.name); } - async addTag( - unifiedTagData: UnifiedTagInput, - integrationId: string, - linkedUserId: string, - remote_data?: boolean, - ): Promise { - return; - } - async getTag( - id_taging_tag: string, + id_ats_tag: string, remote_data?: boolean, ): Promise { - return; + try { + const tag = await this.prisma.ats_tags.findUnique({ + where: { + id_ats_tag: id_ats_tag, + }, + }); + + if (!tag) { + throw new Error(`Tag with ID ${id_ats_tag} not found.`); + } + + // Fetch field mappings for the tag + const values = await this.prisma.value.findMany({ + where: { + entity: { + ressource_owner_id: tag.id_ats_tag, + }, + }, + include: { + attribute: true, + }, + }); + + // Create a map to store unique field mappings + const fieldMappingsMap = new Map(); + + values.forEach((value) => { + fieldMappingsMap.set(value.attribute.slug, value.data); + }); + + // Convert the map to an array of objects + const field_mappings = Array.from(fieldMappingsMap, ([key, value]) => ({ + [key]: value, + })); + + // Transform to UnifiedTagOutput format + const unifiedTag: UnifiedTagOutput = { + id: tag.id_ats_tag, + name: tag.name, + id_ats_candidate: tag.id_ats_candidate, + field_mappings: field_mappings, + remote_id: tag.remote_id, + created_at: tag.created_at, + modified_at: tag.modified_at, + }; + + let res: UnifiedTagOutput = unifiedTag; + if (remote_data) { + const resp = await this.prisma.remote_data.findFirst({ + where: { + ressource_owner_id: tag.id_ats_tag, + }, + }); + const remote_data = JSON.parse(resp.data); + + res = { + ...res, + remote_data: remote_data, + }; + } + + return res; + } catch (error) { + throw error; + } } async getTags( @@ -46,6 +85,79 @@ export class TagService { linkedUserId: string, remote_data?: boolean, ): Promise { - return; + try { + const tags = await this.prisma.ats_tags.findMany({ + where: { + remote_platform: integrationId.toLowerCase(), + id_linked_user: linkedUserId, + }, + }); + + const unifiedTags: UnifiedTagOutput[] = await Promise.all( + tags.map(async (tag) => { + // Fetch field mappings for the tag + const values = await this.prisma.value.findMany({ + where: { + entity: { + ressource_owner_id: tag.id_ats_tag, + }, + }, + include: { + attribute: true, + }, + }); + + // Create a map to store unique field mappings + const fieldMappingsMap = new Map(); + + values.forEach((value) => { + fieldMappingsMap.set(value.attribute.slug, value.data); + }); + + // Convert the map to an array of objects + const field_mappings = Array.from( + fieldMappingsMap, + ([key, value]) => ({ + [key]: value, + }), + ); + + // Transform to UnifiedTagOutput format + return { + id: tag.id_ats_tag, + name: tag.name, + id_ats_candidate: tag.id_ats_candidate, + remote_created_at: tag.remote_created_at, + remote_modified_at: tag.remote_modified_at, + field_mappings: field_mappings, + remote_id: tag.remote_id, + created_at: tag.created_at, + modified_at: tag.modified_at, + }; + }), + ); + + let res: UnifiedTagOutput[] = unifiedTags; + + if (remote_data) { + const remote_array_data: UnifiedTagOutput[] = await Promise.all( + res.map(async (tag) => { + const resp = await this.prisma.remote_data.findFirst({ + where: { + ressource_owner_id: tag.id, + }, + }); + const remote_data = JSON.parse(resp.data); + return { ...tag, remote_data }; + }), + ); + + res = remote_array_data; + } + + return res; + } catch (error) { + throw error; + } } } diff --git a/packages/api/src/ats/tag/sync/sync.processor.ts b/packages/api/src/ats/tag/sync/sync.processor.ts new file mode 100644 index 000000000..730d613e4 --- /dev/null +++ b/packages/api/src/ats/tag/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-tags') + async handleSyncTags(job: Job) { + try { + console.log(`Processing queue -> ats-sync-tags ${job.id}`); + await this.syncService.syncTags(); + } catch (error) { + console.error('Error syncing ats tags', error); + } + } +} diff --git a/packages/api/src/ats/tag/sync/sync.service.ts b/packages/api/src/ats/tag/sync/sync.service.ts index cb9f47ed8..a5d3956be 100644 --- a/packages/api/src/ats/tag/sync/sync.service.ts +++ b/packages/api/src/ats/tag/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedTagOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { ITagService } from '../types'; +import { UnifiedTagOutput } from '../types/model.unified'; +import { ats_candidate_tags as AtsTag } from '@prisma/client'; +import { FILESTORAGE_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; +import { OriginalTagOutput } from '@@core/utils/types/original/original.ats'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,311 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-tags'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncTags(user_id?: string) { + try { + this.logger.log('Syncing tags...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncTagsForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncTagsForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} tags for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping tags syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.tag', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: ITagService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = await service.syncTags( + linkedUserId, + remoteProperties, + ); + + const sourceObject: OriginalTagOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalTagOutput[] + >({ + sourceObject, + targetType: AtsObject.tag, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedTagOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const tags_data = await this.saveTagsInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.tag.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + tags_data, + 'ats.tag.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveTagsInDb( + linkedUserId: string, + tags: UnifiedTagOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let tags_results: AtsTag[] = []; + for (let i = 0; i < tags.length; i++) { + const tag = tags[i]; + const originId = tag.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingTag = await this.prisma.ats_candidate_tags.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_tag_id: string; + + if (existingTag) { + // Update the existing tag + let data: any = { + modified_at: new Date(), + }; + if (tag.name) { + data = { ...data, name: tag.name }; + } + if (tag.id_ats_candidate) { + data = { ...data, id_ats_candidate: tag.id_ats_candidate }; + } + const res = await this.prisma.ats_candidate_tags.update({ + where: { + id_ats_tag: existingTag.id_ats_tag, + }, + data: data, + }); + unique_ats_tag_id = res.id_ats_tag; + tags_results = [...tags_results, res]; + } else { + // Create a new tag + this.logger.log('Tag does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_tag: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (tag.name) { + data = { ...data, name: tag.name }; + } + if (tag.id_ats_candidate) { + data = { ...data, id_ats_candidate: tag.id_ats_candidate }; + } + + const newTag = await this.prisma.ats_candidate_tags.create({ + data: data, + }); + + unique_ats_tag_id = newTag.id_ats_tag; + tags_results = [...tags_results, newTag]; + } + + // check duplicate or existing values + if (tag.field_mappings && tag.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_tag_id, + }, + }); + + for (const [slug, value] of Object.entries(tag.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_tag_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_tag_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return tags_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/tag/types/model.unified.ts b/packages/api/src/ats/tag/types/model.unified.ts index 2701abe60..5af46d23b 100644 --- a/packages/api/src/ats/tag/types/model.unified.ts +++ b/packages/api/src/ats/tag/types/model.unified.ts @@ -1,3 +1,65 @@ -export class UnifiedTagInput {} +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { IsUUID, IsOptional, IsString, IsDateString } from 'class-validator'; -export class UnifiedTagOutput extends UnifiedTagInput {} +export class UnifiedTagInput { + @ApiPropertyOptional({ type: String, description: 'The name of the tag' }) + @IsString() + @IsOptional() + name?: string; + + @ApiPropertyOptional({ + type: String, + description: 'The UUID of the candidate', + }) + @IsUUID() + @IsOptional() + id_ats_candidate?: string; + + @ApiPropertyOptional({ + type: {}, + description: + 'The custom field mappings of the object between the remote 3rd party & Panora', + }) + @IsOptional() + field_mappings?: Record; +} + +export class UnifiedTagOutput extends UnifiedTagInput { + @ApiPropertyOptional({ type: String, description: 'The UUID of the tag' }) + @IsUUID() + @IsOptional() + id?: string; + + @ApiPropertyOptional({ + type: String, + description: 'The remote ID of the tag in the context of the 3rd Party', + }) + @IsString() + @IsOptional() + remote_id?: string; + + @ApiPropertyOptional({ + type: {}, + description: 'The remote data of the tag in the context of the 3rd Party', + }) + @IsOptional() + remote_data?: Record; + + @ApiPropertyOptional({ + type: String, + format: 'date-time', + description: 'The creation date of the tag', + }) + @IsDateString() + @IsOptional() + created_at?: string; + + @ApiPropertyOptional({ + type: String, + format: 'date-time', + description: 'The modification date of the tag', + }) + @IsDateString() + @IsOptional() + modified_at?: string; +} diff --git a/packages/api/src/ats/user/services/user.service.ts b/packages/api/src/ats/user/services/user.service.ts index 5234af60a..31558d2f8 100644 --- a/packages/api/src/ats/user/services/user.service.ts +++ b/packages/api/src/ats/user/services/user.service.ts @@ -60,6 +60,9 @@ export class UserService { remote_created_at: user.remote_created_at, remote_modified_at: user.remote_modified_at, field_mappings: field_mappings, + remote_id: user.remote_id, + created_at: user.created_at, + modified_at: user.modified_at, }; let res: UnifiedUserOutput = unifiedUser; @@ -136,6 +139,9 @@ export class UserService { remote_created_at: user.remote_created_at, remote_modified_at: user.remote_modified_at, field_mappings: field_mappings, + remote_id: user.remote_id, + created_at: user.created_at, + modified_at: user.modified_at, }; }), ); diff --git a/packages/api/src/ats/user/sync/sync.processor.ts b/packages/api/src/ats/user/sync/sync.processor.ts new file mode 100644 index 000000000..750eae927 --- /dev/null +++ b/packages/api/src/ats/user/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('ats-sync-users') + async handleSyncUsers(job: Job) { + try { + console.log(`Processing queue -> ats-sync-users ${job.id}`); + await this.syncService.syncUsers(); + } catch (error) { + console.error('Error syncing ats users', error); + } + } +} diff --git a/packages/api/src/ats/user/sync/sync.service.ts b/packages/api/src/ats/user/sync/sync.service.ts index cef82df8a..844c4527e 100644 --- a/packages/api/src/ats/user/sync/sync.service.ts +++ b/packages/api/src/ats/user/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedUserOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IUserService } from '../types'; +import { OriginalUserOutput } from '@@core/utils/types/original/original.ats'; +import { UnifiedUserOutput } from '../types/model.unified'; +import { ats_users as AtsUser } from '@prisma/client'; +import { ATS_PROVIDERS } from '@panora/shared'; +import { AtsObject } from '@ats/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,341 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'ats-sync-users'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncUsers(user_id?: string) { + try { + this.logger.log('Syncing users...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncUsersForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncUsersForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} users for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'ats', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping users syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'ats.user', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IUserService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = await service.syncUsers( + linkedUserId, + remoteProperties, + ); + + const sourceObject: OriginalUserOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalUserOutput[] + >({ + sourceObject, + targetType: AtsObject.user, + providerName: integrationId, + vertical: 'ats', + customFieldMappings, + })) as UnifiedUserOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const users_data = await this.saveUsersInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'ats.user.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + users_data, + 'ats.user.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveUsersInDb( + linkedUserId: string, + users: UnifiedUserOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let users_results: AtsUser[] = []; + for (let i = 0; i < users.length; i++) { + const user = users[i]; + const originId = user.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingUser = await this.prisma.ats_users.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_ats_user_id: string; + + if (existingUser) { + // Update the existing user + let data: any = { + modified_at: new Date(), + }; + if (user.first_name) { + data = { ...data, first_name: user.first_name }; + } + if (user.last_name) { + data = { ...data, last_name: user.last_name }; + } + if (user.email) { + data = { ...data, email: user.email }; + } + if (user.disabled !== undefined) { + data = { ...data, disabled: user.disabled }; + } + if (user.access_role) { + data = { ...data, access_role: user.access_role }; + } + if (user.remote_created_at) { + data = { ...data, remote_created_at: user.remote_created_at }; + } + if (user.remote_modified_at) { + data = { ...data, remote_modified_at: user.remote_modified_at }; + } + const res = await this.prisma.ats_users.update({ + where: { + id_ats_user: existingUser.id_ats_user, + }, + data: data, + }); + unique_ats_user_id = res.id_ats_user; + users_results = [...users_results, res]; + } else { + // Create a new user + this.logger.log('User does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_ats_user: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (user.first_name) { + data = { ...data, first_name: user.first_name }; + } + if (user.last_name) { + data = { ...data, last_name: user.last_name }; + } + if (user.email) { + data = { ...data, email: user.email }; + } + if (user.disabled !== undefined) { + data = { ...data, disabled: user.disabled }; + } + if (user.access_role) { + data = { ...data, access_role: user.access_role }; + } + if (user.remote_created_at) { + data = { ...data, remote_created_at: user.remote_created_at }; + } + if (user.remote_modified_at) { + data = { ...data, remote_modified_at: user.remote_modified_at }; + } + + const newUser = await this.prisma.ats_users.create({ + data: data, + }); + + unique_ats_user_id = newUser.id_ats_user; + users_results = [...users_results, newUser]; + } + + // check duplicate or existing values + if (user.field_mappings && user.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_ats_user_id, + }, + }); + + for (const [slug, value] of Object.entries(user.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_ats_user_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_ats_user_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return users_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/ats/user/types/model.unified.ts b/packages/api/src/ats/user/types/model.unified.ts index a31de5806..29f906b03 100644 --- a/packages/api/src/ats/user/types/model.unified.ts +++ b/packages/api/src/ats/user/types/model.unified.ts @@ -102,7 +102,7 @@ export class UnifiedUserOutput extends UnifiedUserInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/crm/company/services/company.service.ts b/packages/api/src/crm/company/services/company.service.ts index 8ebfbb5be..c3b42c306 100644 --- a/packages/api/src/crm/company/services/company.service.ts +++ b/packages/api/src/crm/company/services/company.service.ts @@ -14,7 +14,6 @@ import { ServiceRegistry } from './registry.service'; import { OriginalCompanyOutput } from '@@core/utils/types/original/original.crm'; import { ICompanyService } from '../types'; import { Utils } from '@crm/@lib/@utils'; -import { throwTypedError, UnifiedCrmError } from '@@core/utils/errors'; import { CoreUnification } from '@@core/utils/services/core.service'; @Injectable() @@ -31,30 +30,6 @@ export class CompanyService { this.logger.setContext(CompanyService.name); } - async batchAddCompanies( - unifiedCompanyData: UnifiedCompanyInput[], - integrationId: string, - linkedUserId: string, - remote_data?: boolean, - ): Promise { - try { - const responses = await Promise.all( - unifiedCompanyData.map((unifiedData) => - this.addCompany( - unifiedData, - integrationId.toLowerCase(), - linkedUserId, - remote_data, - ), - ), - ); - - return responses; - } catch (error) { - throw error; - } - } - async addCompany( unifiedCompanyData: UnifiedCompanyInput, integrationId: string, @@ -429,6 +404,9 @@ export class CompanyService { addresses: company.crm_addresses.map((addy) => ({ ...addy, })), + remote_id: company.remote_id, + created_at: company.created_at, + modified_at: company.modified_at, }; let res: UnifiedCompanyOutput = { @@ -559,6 +537,9 @@ export class CompanyService { addresses: company.crm_addresses.map((addy) => ({ ...addy, })), + remote_id: company.remote_id, + created_at: company.created_at, + modified_at: company.modified_at, }; }), ); diff --git a/packages/api/src/crm/company/sync/sync.service.ts b/packages/api/src/crm/company/sync/sync.service.ts index c80f69b7b..d804d95f4 100644 --- a/packages/api/src/crm/company/sync/sync.service.ts +++ b/packages/api/src/crm/company/sync/sync.service.ts @@ -15,7 +15,6 @@ import { crm_companies as CrmCompany } from '@prisma/client'; import { CRM_PROVIDERS } from '@panora/shared'; import { Queue } from 'bull'; import { InjectQueue } from '@nestjs/bull'; -import { throwTypedError, SyncError } from '@@core/utils/errors'; import { CoreUnification } from '@@core/utils/services/core.service'; import { Utils } from '@crm/@lib/@utils'; diff --git a/packages/api/src/crm/company/types/model.unified.ts b/packages/api/src/crm/company/types/model.unified.ts index b8d282c68..e61bc341b 100644 --- a/packages/api/src/crm/company/types/model.unified.ts +++ b/packages/api/src/crm/company/types/model.unified.ts @@ -32,7 +32,7 @@ export class UnifiedCompanyInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the user who owns the company', + description: 'The UUID of the user who owns the company', }) @IsOptional() @IsUUID() @@ -69,7 +69,7 @@ export class UnifiedCompanyInput { } export class UnifiedCompanyOutput extends UnifiedCompanyInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the company' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the company' }) @IsUUID() @IsOptional() id?: string; @@ -99,7 +99,7 @@ export class UnifiedCompanyOutput extends UnifiedCompanyInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/crm/contact/services/contact.service.ts b/packages/api/src/crm/contact/services/contact.service.ts index 6d54f0314..80b333dce 100644 --- a/packages/api/src/crm/contact/services/contact.service.ts +++ b/packages/api/src/crm/contact/services/contact.service.ts @@ -436,6 +436,9 @@ export class ContactService { })), user_id: contact.id_crm_user, field_mappings: field_mappings, + remote_id: contact.remote_id, + created_at: contact.created_at, + modified_at: contact.modified_at, }; let res: UnifiedContactOutput = unifiedContact; @@ -565,6 +568,9 @@ export class ContactService { })), user_id: contact.id_crm_user, field_mappings: field_mappings, + remote_id: contact.remote_id, + created_at: contact.created_at, + modified_at: contact.modified_at, }; }), ); diff --git a/packages/api/src/crm/contact/types/model.unified.ts b/packages/api/src/crm/contact/types/model.unified.ts index 9a42e7e9e..9c08b34e6 100644 --- a/packages/api/src/crm/contact/types/model.unified.ts +++ b/packages/api/src/crm/contact/types/model.unified.ts @@ -34,7 +34,7 @@ export class UnifiedContactInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the user who owns the contact', + description: 'The UUID of the user who owns the contact', }) @IsUUID() @IsOptional() @@ -50,7 +50,7 @@ export class UnifiedContactInput { } export class UnifiedContactOutput extends UnifiedContactInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the contact' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the contact' }) @IsUUID() @IsOptional() id?: string; @@ -80,7 +80,7 @@ export class UnifiedContactOutput extends UnifiedContactInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/crm/deal/deal.controller.ts b/packages/api/src/crm/deal/deal.controller.ts index 1f2febf93..973457ded 100644 --- a/packages/api/src/crm/deal/deal.controller.ts +++ b/packages/api/src/crm/deal/deal.controller.ts @@ -64,7 +64,7 @@ export class DealController { connection_token, ); const { remote_data, limit, cursor } = query; - return this.dealService.getDeals( + return await this.dealService.getDeals( remoteSource, linkedUserId, limit, @@ -134,7 +134,7 @@ export class DealController { await this.connectionUtils.getConnectionMetadataFromConnectionToken( connection_token, ); - return this.dealService.addDeal( + return await this.dealService.addDeal( unifiedDealData, remoteSource, linkedUserId, diff --git a/packages/api/src/crm/deal/services/deal.service.ts b/packages/api/src/crm/deal/services/deal.service.ts index 4680ecec6..9924e8fdd 100644 --- a/packages/api/src/crm/deal/services/deal.service.ts +++ b/packages/api/src/crm/deal/services/deal.service.ts @@ -272,6 +272,9 @@ export class DealService { stage_id: deal.id_crm_deals_stage, // uuid of Stage object user_id: deal.id_crm_user, // uuid of User object field_mappings: field_mappings, + remote_id: deal.remote_id, + created_at: deal.created_at, + modified_at: deal.modified_at, }; let res: UnifiedDealOutput = { @@ -388,6 +391,9 @@ export class DealService { stage_id: deal.id_crm_deals_stage, // uuid of Stage object user_id: deal.id_crm_user, // uuid of User object field_mappings: field_mappings, + remote_id: deal.remote_id, + created_at: deal.created_at, + modified_at: deal.modified_at, }; }), ); diff --git a/packages/api/src/crm/deal/types/model.unified.ts b/packages/api/src/crm/deal/types/model.unified.ts index 77b0979ce..d8cdfcc42 100644 --- a/packages/api/src/crm/deal/types/model.unified.ts +++ b/packages/api/src/crm/deal/types/model.unified.ts @@ -16,7 +16,7 @@ export class UnifiedDealInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the user who is on the deal', + description: 'The UUID of the user who is on the deal', }) @IsUUID() @IsOptional() @@ -24,7 +24,7 @@ export class UnifiedDealInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the stage of the deal', + description: 'The UUID of the stage of the deal', }) @IsUUID() @IsOptional() @@ -32,7 +32,7 @@ export class UnifiedDealInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the company tied to the deal', + description: 'The UUID of the company tied to the deal', }) @IsUUID() @IsOptional() @@ -48,7 +48,7 @@ export class UnifiedDealInput { } export class UnifiedDealOutput extends UnifiedDealInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the deal' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the deal' }) @IsUUID() @IsOptional() id?: string; @@ -78,7 +78,7 @@ export class UnifiedDealOutput extends UnifiedDealInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/crm/engagement/services/engagement.service.ts b/packages/api/src/crm/engagement/services/engagement.service.ts index b51baf9a8..f4a9725df 100644 --- a/packages/api/src/crm/engagement/services/engagement.service.ts +++ b/packages/api/src/crm/engagement/services/engagement.service.ts @@ -322,6 +322,9 @@ export class EngagementService { type: engagement.type, company_id: engagement.id_crm_company, field_mappings: field_mappings, + remote_id: engagement.remote_id, + created_at: engagement.created_at, + modified_at: engagement.modified_at, }; let res: UnifiedEngagementOutput = { @@ -446,6 +449,9 @@ export class EngagementService { type: engagement.type, company_id: engagement.id_crm_company, field_mappings: field_mappings, + remote_id: engagement.remote_id, + created_at: engagement.created_at, + modified_at: engagement.modified_at, }; }), ); diff --git a/packages/api/src/crm/engagement/types/model.unified.ts b/packages/api/src/crm/engagement/types/model.unified.ts index 6e84fff04..1437685b5 100644 --- a/packages/api/src/crm/engagement/types/model.unified.ts +++ b/packages/api/src/crm/engagement/types/model.unified.ts @@ -49,7 +49,7 @@ export class UnifiedEngagementInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the user tied to the engagement', + description: 'The UUID of the user tied to the engagement', }) @IsUUID() @IsOptional() @@ -57,18 +57,18 @@ export class UnifiedEngagementInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the company tied to the engagement', + description: 'The UUID of the company tied to the engagement', }) @IsUUID() @IsOptional() - company_id?: string; // uuid of Company object + company_id?: string; // UUID of Company object @ApiPropertyOptional({ type: [String], - description: 'The uuids of contacts tied to the engagement object', + description: 'The UUIDs of contacts tied to the engagement object', }) @IsOptional() - contacts?: string[]; // array of uuids of Engagement Contacts objects + contacts?: string[]; // array of UUIDs of Engagement Contacts objects @ApiPropertyOptional({ type: {}, @@ -82,7 +82,7 @@ export class UnifiedEngagementInput { export class UnifiedEngagementOutput extends UnifiedEngagementInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the engagement', + description: 'The UUID of the engagement', }) @IsUUID() @IsOptional() @@ -113,7 +113,7 @@ export class UnifiedEngagementOutput extends UnifiedEngagementInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/crm/note/services/note.service.ts b/packages/api/src/crm/note/services/note.service.ts index 248853780..bc8a7e77e 100644 --- a/packages/api/src/crm/note/services/note.service.ts +++ b/packages/api/src/crm/note/services/note.service.ts @@ -291,6 +291,9 @@ export class NoteService { deal_id: note.id_crm_deal, // uuid of Contact object user_id: note.id_crm_user, field_mappings: field_mappings, + remote_id: note.remote_id, + created_at: note.created_at, + modified_at: note.modified_at, }; let res: UnifiedNoteOutput = { @@ -407,6 +410,9 @@ export class NoteService { deal_id: note.id_crm_deal, // uuid of Contact object user_id: note.id_crm_user, field_mappings: field_mappings, + remote_id: note.remote_id, + created_at: note.created_at, + modified_at: note.modified_at, }; }), ); diff --git a/packages/api/src/crm/note/types/model.unified.ts b/packages/api/src/crm/note/types/model.unified.ts index ab6b86c3b..1aa974109 100644 --- a/packages/api/src/crm/note/types/model.unified.ts +++ b/packages/api/src/crm/note/types/model.unified.ts @@ -8,7 +8,7 @@ export class UnifiedNoteInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the user tied the note', + description: 'The UUID of the user tied the note', }) @IsUUID() @IsOptional() @@ -16,7 +16,7 @@ export class UnifiedNoteInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the company tied to the note', + description: 'The UUID of the company tied to the note', }) @IsUUID() @IsOptional() @@ -24,7 +24,7 @@ export class UnifiedNoteInput { @ApiPropertyOptional({ type: String, - description: 'The uuid fo the contact tied to the note', + description: 'The UUID fo the contact tied to the note', }) @IsUUID() @IsOptional() @@ -32,7 +32,7 @@ export class UnifiedNoteInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the deal tied to the note', + description: 'The UUID of the deal tied to the note', }) @IsUUID() @IsOptional() @@ -48,7 +48,7 @@ export class UnifiedNoteInput { } export class UnifiedNoteOutput extends UnifiedNoteInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the note' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the note' }) @IsUUID() @IsOptional() id?: string; @@ -79,7 +79,7 @@ export class UnifiedNoteOutput extends UnifiedNoteInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/crm/stage/services/stage.service.ts b/packages/api/src/crm/stage/services/stage.service.ts index 8487cdef7..f1b3a079c 100644 --- a/packages/api/src/crm/stage/services/stage.service.ts +++ b/packages/api/src/crm/stage/services/stage.service.ts @@ -50,6 +50,9 @@ export class StageService { id: stage.id_crm_deals_stage, stage_name: stage.stage_name, field_mappings: field_mappings, + remote_id: stage.remote_id, + created_at: stage.created_at, + modified_at: stage.modified_at, }; let res: UnifiedStageOutput = { @@ -162,6 +165,9 @@ export class StageService { id: stage.id_crm_deals_stage, stage_name: stage.stage_name, field_mappings: field_mappings, + remote_id: stage.remote_id, + created_at: stage.created_at, + modified_at: stage.modified_at, }; }), ); diff --git a/packages/api/src/crm/stage/types/model.unified.ts b/packages/api/src/crm/stage/types/model.unified.ts index cd6e552df..13de9fe9b 100644 --- a/packages/api/src/crm/stage/types/model.unified.ts +++ b/packages/api/src/crm/stage/types/model.unified.ts @@ -16,7 +16,7 @@ export class UnifiedStageInput { } export class UnifiedStageOutput extends UnifiedStageInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the stage' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the stage' }) @IsUUID() @IsOptional() id?: string; @@ -46,7 +46,7 @@ export class UnifiedStageOutput extends UnifiedStageInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/crm/task/services/task.service.ts b/packages/api/src/crm/task/services/task.service.ts index 7977e6a6e..45c62d893 100644 --- a/packages/api/src/crm/task/services/task.service.ts +++ b/packages/api/src/crm/task/services/task.service.ts @@ -297,6 +297,9 @@ export class TaskService { company_id: task.id_crm_company, deal_id: task.id_crm_deal, // uuid of Contact object user_id: task.id_crm_user, // uuid of User object + remote_id: task.remote_id, + created_at: task.created_at, + modified_at: task.modified_at, }; let res: UnifiedTaskOutput = { @@ -415,6 +418,9 @@ export class TaskService { company_id: task.id_crm_company, deal_id: task.id_crm_deal, // uuid of Contact object user_id: task.id_crm_user, // uuid of User object + remote_id: task.remote_id, + created_at: task.created_at, + modified_at: task.modified_at, }; }), ); diff --git a/packages/api/src/crm/task/types/model.unified.ts b/packages/api/src/crm/task/types/model.unified.ts index 2d9d8b4ee..c3ad5d83a 100644 --- a/packages/api/src/crm/task/types/model.unified.ts +++ b/packages/api/src/crm/task/types/model.unified.ts @@ -30,7 +30,7 @@ export class UnifiedTaskInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the user tied to the task', + description: 'The UUID of the user tied to the task', }) @IsUUID() @IsOptional() @@ -38,7 +38,7 @@ export class UnifiedTaskInput { @ApiPropertyOptional({ type: String, - description: 'The uuid fo the company tied to the task', + description: 'The UUID fo the company tied to the task', }) @IsUUID() @IsOptional() @@ -46,7 +46,7 @@ export class UnifiedTaskInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the deal tied to the task', + description: 'The UUID of the deal tied to the task', }) @IsString() @IsOptional() @@ -62,7 +62,7 @@ export class UnifiedTaskInput { } export class UnifiedTaskOutput extends UnifiedTaskInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the task' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the task' }) @IsUUID() @IsOptional() id?: string; @@ -93,7 +93,7 @@ export class UnifiedTaskOutput extends UnifiedTaskInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/crm/user/services/user.service.ts b/packages/api/src/crm/user/services/user.service.ts index 2bccee1b9..215bb9c70 100644 --- a/packages/api/src/crm/user/services/user.service.ts +++ b/packages/api/src/crm/user/services/user.service.ts @@ -60,6 +60,9 @@ export class UserService { name: user.name, email: user.email, field_mappings: field_mappings, + remote_id: user.remote_id, + created_at: user.created_at, + modified_at: user.modified_at, }; let res: UnifiedUserOutput = { @@ -173,6 +176,9 @@ export class UserService { name: user.name, email: user.email, field_mappings: field_mappings, + remote_id: user.remote_id, + created_at: user.created_at, + modified_at: user.modified_at, }; }), ); diff --git a/packages/api/src/crm/user/types/model.unified.ts b/packages/api/src/crm/user/types/model.unified.ts index be734e15b..cbd0c10fa 100644 --- a/packages/api/src/crm/user/types/model.unified.ts +++ b/packages/api/src/crm/user/types/model.unified.ts @@ -20,7 +20,7 @@ export class UnifiedUserInput { } export class UnifiedUserOutput extends UnifiedUserInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the user' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the user' }) @IsUUID() @IsOptional() id?: string; @@ -50,7 +50,7 @@ export class UnifiedUserOutput extends UnifiedUserInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/filestorage/@lib/@types/index.ts b/packages/api/src/filestorage/@lib/@types/index.ts index df0614ee8..15d64668d 100644 --- a/packages/api/src/filestorage/@lib/@types/index.ts +++ b/packages/api/src/filestorage/@lib/@types/index.ts @@ -1,33 +1,38 @@ import { IDriveService } from '@filestorage/drive/types'; -import { driveUnificationMapping } from '@filestorage/drive/types/mappingsTypes'; import { UnifiedDriveInput, UnifiedDriveOutput, } from '@filestorage/drive/types/model.unified'; import { IFileService } from '@filestorage/file/types'; -import { fileUnificationMapping } from '@filestorage/file/types/mappingsTypes'; import { UnifiedFileInput, UnifiedFileOutput, } from '@filestorage/file/types/model.unified'; import { IFolderService } from '@filestorage/folder/types'; -import { folderUnificationMapping } from '@filestorage/folder/types/mappingsTypes'; import { UnifiedFolderInput, UnifiedFolderOutput, } from '@filestorage/folder/types/model.unified'; +import { IGroupService } from '@filestorage/group/types'; +import { + UnifiedGroupInput, + UnifiedGroupOutput, +} from '@filestorage/group/types/model.unified'; import { IPermissionService } from '@filestorage/permission/types'; -import { permissionUnificationMapping } from '@filestorage/permission/types/mappingsTypes'; import { UnifiedPermissionInput, UnifiedPermissionOutput, } from '@filestorage/permission/types/model.unified'; import { ISharedLinkService } from '@filestorage/sharedlink/types'; -import { sharedlinkUnificationMapping } from '@filestorage/sharedlink/types/mappingsTypes'; import { UnifiedSharedLinkInput, UnifiedSharedLinkOutput, } from '@filestorage/sharedlink/types/model.unified'; +import { IUserService } from '@filestorage/user/types'; +import { + UnifiedUserInput, + UnifiedUserOutput, +} from '@filestorage/user/types/model.unified'; export enum FileStorageObject { file = 'file', @@ -35,6 +40,8 @@ export enum FileStorageObject { permission = 'permission', drive = 'drive', sharedlink = 'sharedlink', + group = 'group', + user = 'user', } export type UnifiedFileStorage = @@ -46,20 +53,18 @@ export type UnifiedFileStorage = | UnifiedPermissionOutput | UnifiedDriveInput | UnifiedDriveOutput + | UnifiedGroupInput + | UnifiedGroupOutput + | UnifiedUserInput + | UnifiedUserOutput | UnifiedSharedLinkInput | UnifiedSharedLinkOutput; -/*export const unificationMapping = { - [FileStorageObject.drive]: driveUnificationMapping, - [FileStorageObject.file]: fileUnificationMapping, - [FileStorageObject.folder]: folderUnificationMapping, - [FileStorageObject.permission]: permissionUnificationMapping, - [FileStorageObject.sharedlink]: sharedlinkUnificationMapping, -};*/ - export type IFileStorageService = | IFileService | IFolderService | IDriveService + | IGroupService + | IUserService | IPermissionService | ISharedLinkService; diff --git a/packages/api/src/filestorage/drive/services/drive.service.ts b/packages/api/src/filestorage/drive/services/drive.service.ts index bb8caebdd..a60e925a3 100644 --- a/packages/api/src/filestorage/drive/services/drive.service.ts +++ b/packages/api/src/filestorage/drive/services/drive.service.ts @@ -52,6 +52,9 @@ export class DriveService { name: drive.name, drive_url: drive.drive_url, field_mappings: field_mappings, + remote_id: drive.remote_id, + created_at: drive.created_at, + modified_at: drive.modified_at, }; let res: UnifiedDriveOutput = unifiedDrive; @@ -165,6 +168,9 @@ export class DriveService { name: drive.name, remote_created_at: drive.remote_created_at, field_mappings: field_mappings, + remote_id: drive.remote_id, + created_at: drive.created_at, + modified_at: drive.modified_at, }; }), ); diff --git a/packages/api/src/filestorage/drive/sync/sync.processor.ts b/packages/api/src/filestorage/drive/sync/sync.processor.ts new file mode 100644 index 000000000..eff55de2c --- /dev/null +++ b/packages/api/src/filestorage/drive/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('filestorage-sync-drives') + async handleSyncDrives(job: Job) { + try { + console.log(`Processing queue -> filestorage-sync-drives ${job.id}`); + await this.syncService.syncDrives(); + } catch (error) { + console.error('Error syncing filestorage drives', error); + } + } +} diff --git a/packages/api/src/filestorage/drive/sync/sync.service.ts b/packages/api/src/filestorage/drive/sync/sync.service.ts index 8b729a075..160515ee7 100644 --- a/packages/api/src/filestorage/drive/sync/sync.service.ts +++ b/packages/api/src/filestorage/drive/sync/sync.service.ts @@ -2,14 +2,20 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedDriveOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IDriveService } from '../types'; +import { OriginalDriveOutput } from '@@core/utils/types/original/original.filestorage'; +import { UnifiedDriveOutput } from '../types/model.unified'; +import { fs_drives as FileStorageDrive } from '@prisma/client'; +import { FILESTORAGE_PROVIDERS } from '@panora/shared'; +import { FileStorageObject } from '@filestorage/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -19,13 +25,317 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'filestorage-sync-drives'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); } - // Additional methods and logic + @Cron('0 */8 * * *') // every 8 hours + async syncDrives(user_id?: string) { + try { + this.logger.log('Syncing drives...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncDrivesForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncDrivesForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} drives for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'filestorage', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping drives syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'filestorage.drive', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IDriveService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = await service.syncDrives( + linkedUserId, + remoteProperties, + ); + + const sourceObject: OriginalDriveOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalDriveOutput[] + >({ + sourceObject, + targetType: FileStorageObject.drive, + providerName: integrationId, + vertical: 'filestorage', + customFieldMappings, + })) as UnifiedDriveOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const drives_data = await this.saveDrivesInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'filestorage.drive.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + drives_data, + 'filestorage.drive.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } + } + + async saveDrivesInDb( + linkedUserId: string, + drives: UnifiedDriveOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let drives_results: FileStorageDrive[] = []; + for (let i = 0; i < drives.length; i++) { + const drive = drives[i]; + const originId = drive.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingDrive = await this.prisma.fs_drives.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_fs_drive_id: string; + + if (existingDrive) { + // Update the existing drive + let data: any = { + modified_at: new Date(), + }; + if (drive.name) { + data = { ...data, name: drive.name }; + } + if (drive.remote_created_at) { + data = { ...data, remote_created_at: drive.remote_created_at }; + } + if (drive.drive_url) { + data = { ...data, drive_url: drive.drive_url }; + } + const res = await this.prisma.fs_drives.update({ + where: { + id_fs_drive: existingDrive.id_fs_drive, + }, + data: data, + }); + unique_fs_drive_id = res.id_fs_drive; + drives_results = [...drives_results, res]; + } else { + // Create a new drive + this.logger.log('Drive does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_fs_drive: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (drive.name) { + data = { ...data, name: drive.name }; + } + if (drive.remote_created_at) { + data = { ...data, remote_created_at: drive.remote_created_at }; + } + if (drive.drive_url) { + data = { ...data, drive_url: drive.drive_url }; + } + + const newDrive = await this.prisma.fs_drives.create({ + data: data, + }); + + unique_fs_drive_id = newDrive.id_fs_drive; + drives_results = [...drives_results, newDrive]; + } + + // check duplicate or existing values + if (drive.field_mappings && drive.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_fs_drive_id, + }, + }); + + for (const [slug, value] of Object.entries(drive.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_fs_drive_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_fs_drive_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return drives_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/filestorage/drive/types/model.unified.ts b/packages/api/src/filestorage/drive/types/model.unified.ts index 7b1947fc5..6cfdba765 100644 --- a/packages/api/src/filestorage/drive/types/model.unified.ts +++ b/packages/api/src/filestorage/drive/types/model.unified.ts @@ -27,7 +27,7 @@ export class UnifiedDriveInput { } export class UnifiedDriveOutput extends UnifiedDriveInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the drive' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the drive' }) @IsUUID() @IsOptional() id?: string; @@ -56,7 +56,7 @@ export class UnifiedDriveOutput extends UnifiedDriveInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/filestorage/file/services/file.service.ts b/packages/api/src/filestorage/file/services/file.service.ts index 8d538de3b..89efafd99 100644 --- a/packages/api/src/filestorage/file/services/file.service.ts +++ b/packages/api/src/filestorage/file/services/file.service.ts @@ -263,6 +263,9 @@ export class FileService { folder_id: file.folder_id, permission_id: file.permission_id, field_mappings: field_mappings, + remote_id: file.remote_id, + created_at: file.created_at, + modified_at: file.modified_at, }; let res: UnifiedFileOutput = unifiedFile; @@ -379,6 +382,9 @@ export class FileService { folder_id: file.folder_id, permission_id: file.permission_id, field_mappings: field_mappings, + remote_id: file.remote_id, + created_at: file.created_at, + modified_at: file.modified_at, }; }), ); diff --git a/packages/api/src/filestorage/file/sync/sync.processor.ts b/packages/api/src/filestorage/file/sync/sync.processor.ts new file mode 100644 index 000000000..cb69ba967 --- /dev/null +++ b/packages/api/src/filestorage/file/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('filestorage-sync-files') + async handleSyncFiles(job: Job) { + try { + console.log(`Processing queue -> filestorage-sync-files ${job.id}`); + await this.syncService.syncFiles(); + } catch (error) { + console.error('Error syncing filestorage files', error); + } + } +} diff --git a/packages/api/src/filestorage/file/sync/sync.service.ts b/packages/api/src/filestorage/file/sync/sync.service.ts index 9063a7256..fac3c08a4 100644 --- a/packages/api/src/filestorage/file/sync/sync.service.ts +++ b/packages/api/src/filestorage/file/sync/sync.service.ts @@ -1,16 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; - import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedFileOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IFileService } from '../types'; +import { OriginalFileOutput } from '@@core/utils/types/original/original.filestorage'; +import { UnifiedFileOutput } from '../types/model.unified'; +import { fs_files as FileStorageFile } from '@prisma/client'; +import { FILESTORAGE_PROVIDERS } from '@panora/shared'; +import { FileStorageObject } from '@filestorage/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -20,13 +25,341 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'filestorage-sync-files'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncFiles(user_id?: string) { + try { + this.logger.log('Syncing files...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncFilesForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncFilesForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} files for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'filestorage', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping files syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'filestorage.file', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IFileService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = await service.syncFiles( + linkedUserId, + remoteProperties, + ); + + const sourceObject: OriginalFileOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalFileOutput[] + >({ + sourceObject, + targetType: FileStorageObject.file, + providerName: integrationId, + vertical: 'filestorage', + customFieldMappings, + })) as UnifiedFileOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const files_data = await this.saveFilesInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'filestorage.file.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + files_data, + 'filestorage.file.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } } - // Additional methods and logic + async saveFilesInDb( + linkedUserId: string, + files: UnifiedFileOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let files_results: FileStorageFile[] = []; + for (let i = 0; i < files.length; i++) { + const file = files[i]; + const originId = file.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingFile = await this.prisma.fs_files.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_fs_file_id: string; + + if (existingFile) { + // Update the existing file + let data: any = { + modified_at: new Date(), + }; + if (file.name) { + data = { ...data, name: file.name }; + } + if (file.type) { + data = { ...data, type: file.type }; + } + if (file.file_url) { + data = { ...data, file_url: file.file_url }; + } + if (file.mime_type) { + data = { ...data, mime_type: file.mime_type }; + } + if (file.size) { + data = { ...data, size: file.size }; + } + if (file.folder_id) { + data = { ...data, folder_id: file.folder_id }; + } + if (file.permission_id) { + data = { ...data, permission_id: file.permission_id }; + } + const res = await this.prisma.fs_files.update({ + where: { + id_fs_file: existingFile.id_fs_file, + }, + data: data, + }); + unique_fs_file_id = res.id_fs_file; + files_results = [...files_results, res]; + } else { + // Create a new file + this.logger.log('File does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_fs_file: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (file.name) { + data = { ...data, name: file.name }; + } + if (file.type) { + data = { ...data, type: file.type }; + } + if (file.file_url) { + data = { ...data, file_url: file.file_url }; + } + if (file.mime_type) { + data = { ...data, mime_type: file.mime_type }; + } + if (file.size) { + data = { ...data, size: file.size }; + } + if (file.folder_id) { + data = { ...data, folder_id: file.folder_id }; + } + if (file.permission_id) { + data = { ...data, permission_id: file.permission_id }; + } + + const newFile = await this.prisma.fs_files.create({ + data: data, + }); + + unique_fs_file_id = newFile.id_fs_file; + files_results = [...files_results, newFile]; + } + + // check duplicate or existing values + if (file.field_mappings && file.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_fs_file_id, + }, + }); + + for (const [slug, value] of Object.entries(file.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_fs_file_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_fs_file_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return files_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/filestorage/file/types/model.unified.ts b/packages/api/src/filestorage/file/types/model.unified.ts index 9925f90bb..ab1cd68e7 100644 --- a/packages/api/src/filestorage/file/types/model.unified.ts +++ b/packages/api/src/filestorage/file/types/model.unified.ts @@ -24,14 +24,14 @@ export class UnifiedFileInput { @ApiProperty({ type: String, - description: 'The uuid of the folder tied to the file', + description: 'The UUID of the folder tied to the file', }) @IsString() folder_id: string; @ApiProperty({ type: String, - description: 'The uuid of the permission tied to the file', + description: 'The UUID of the permission tied to the file', }) @IsString() permission_id: string; @@ -46,7 +46,7 @@ export class UnifiedFileInput { } export class UnifiedFileOutput extends UnifiedFileInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the file' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the file' }) @IsUUID() @IsOptional() id?: string; @@ -75,7 +75,7 @@ export class UnifiedFileOutput extends UnifiedFileInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/filestorage/folder/services/folder.service.ts b/packages/api/src/filestorage/folder/services/folder.service.ts index 74e78a361..cb2816853 100644 --- a/packages/api/src/filestorage/folder/services/folder.service.ts +++ b/packages/api/src/filestorage/folder/services/folder.service.ts @@ -272,6 +272,9 @@ export class FolderService { drive_id: folder.id_fs_drive, permission_id: folder.id_fs_permission, field_mappings: field_mappings, + remote_id: folder.remote_id, + created_at: folder.created_at, + modified_at: folder.modified_at, }; let res: UnifiedFolderOutput = unifiedFolder; @@ -388,6 +391,9 @@ export class FolderService { drive_id: folder.id_fs_drive, permission_id: folder.id_fs_permission, field_mappings: field_mappings, + remote_id: folder.remote_id, + created_at: folder.created_at, + modified_at: folder.modified_at, }; }), ); diff --git a/packages/api/src/filestorage/folder/sync/sync.processor.ts b/packages/api/src/filestorage/folder/sync/sync.processor.ts new file mode 100644 index 000000000..c5ff5c145 --- /dev/null +++ b/packages/api/src/filestorage/folder/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('filestorage-sync-folders') + async handleSyncFolders(job: Job) { + try { + console.log(`Processing queue -> filestorage-sync-folders ${job.id}`); + await this.syncService.syncFolders(); + } catch (error) { + console.error('Error syncing filestorage folders', error); + } + } +} diff --git a/packages/api/src/filestorage/folder/sync/sync.service.ts b/packages/api/src/filestorage/folder/sync/sync.service.ts index 3da07367b..128d88b55 100644 --- a/packages/api/src/filestorage/folder/sync/sync.service.ts +++ b/packages/api/src/filestorage/folder/sync/sync.service.ts @@ -2,14 +2,20 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedFolderOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IFolderService } from '../types'; +import { OriginalFolderOutput } from '@@core/utils/types/original/original.filestorage'; +import { UnifiedFolderOutput } from '../types/model.unified'; +import { fs_folders as FileStorageFolder } from '@prisma/client'; +import { FILESTORAGE_PROVIDERS } from '@panora/shared'; +import { FileStorageObject } from '@filestorage/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -19,13 +25,339 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'filestorage-sync-folders'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); } - // Additional methods and logic + @Cron('0 */8 * * *') // every 8 hours + async syncFolders(user_id?: string) { + try { + this.logger.log('Syncing folders...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncFoldersForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncFoldersForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} folders for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'filestorage', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping folders syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'filestorage.folder', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IFolderService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncFolders(linkedUserId, remoteProperties); + + const sourceObject: OriginalFolderOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalFolderOutput[] + >({ + sourceObject, + targetType: FileStorageObject.folder, + providerName: integrationId, + vertical: 'filestorage', + customFieldMappings, + })) as UnifiedFolderOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const folders_data = await this.saveFoldersInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'filestorage.folder.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + folders_data, + 'filestorage.folder.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } + } + + async saveFoldersInDb( + linkedUserId: string, + folders: UnifiedFolderOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let folders_results: FileStorageFolder[] = []; + for (let i = 0; i < folders.length; i++) { + const folder = folders[i]; + const originId = folder.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingFolder = await this.prisma.fs_folders.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_fs_folder_id: string; + + if (existingFolder) { + // Update the existing folder + let data: any = { + modified_at: new Date(), + }; + if (folder.name) { + data = { ...data, name: folder.name }; + } + if (folder.size) { + data = { ...data, size: folder.size }; + } + if (folder.folder_url) { + data = { ...data, folder_url: folder.folder_url }; + } + if (folder.description) { + data = { ...data, description: folder.description }; + } + if (folder.drive_id) { + data = { ...data, drive_id: folder.drive_id }; + } + if (folder.parent_folder_id) { + data = { ...data, parent_folder_id: folder.parent_folder_id }; + } + if (folder.permission_id) { + data = { ...data, permission_id: folder.permission_id }; + } + const res = await this.prisma.fs_folders.update({ + where: { + id_fs_folder: existingFolder.id_fs_folder, + }, + data: data, + }); + unique_fs_folder_id = res.id_fs_folder; + folders_results = [...folders_results, res]; + } else { + // Create a new folder + this.logger.log('Folder does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_fs_folder: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (folder.name) { + data = { ...data, name: folder.name }; + } + if (folder.size) { + data = { ...data, size: folder.size }; + } + if (folder.folder_url) { + data = { ...data, folder_url: folder.folder_url }; + } + if (folder.description) { + data = { ...data, description: folder.description }; + } + if (folder.drive_id) { + data = { ...data, drive_id: folder.drive_id }; + } + if (folder.parent_folder_id) { + data = { ...data, parent_folder_id: folder.parent_folder_id }; + } + if (folder.permission_id) { + data = { ...data, permission_id: folder.permission_id }; + } + + const newFolder = await this.prisma.fs_folders.create({ + data: data, + }); + + unique_fs_folder_id = newFolder.id_fs_folder; + folders_results = [...folders_results, newFolder]; + } + + // check duplicate or existing values + if (folder.field_mappings && folder.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_fs_folder_id, + }, + }); + + for (const [slug, value] of Object.entries(folder.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_fs_folder_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_fs_folder_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return folders_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/filestorage/folder/types/model.unified.ts b/packages/api/src/filestorage/folder/types/model.unified.ts index f518e33a3..e6d7e6018 100644 --- a/packages/api/src/filestorage/folder/types/model.unified.ts +++ b/packages/api/src/filestorage/folder/types/model.unified.ts @@ -20,21 +20,21 @@ export class UnifiedFolderInput { @ApiProperty({ type: String, - description: 'The uuid of the drive tied to the folder', + description: 'The UUID of the drive tied to the folder', }) @IsString() drive_id: string; @ApiProperty({ type: String, - description: 'The uuid of the parent folder', + description: 'The UUID of the parent folder', }) @IsString() parent_folder_id: string; @ApiProperty({ type: String, - description: 'The uuid of the permission tied to the folder', + description: 'The UUID of the permission tied to the folder', }) @IsString() permission_id: string; @@ -49,7 +49,7 @@ export class UnifiedFolderInput { } export class UnifiedFolderOutput extends UnifiedFolderInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the folder' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the folder' }) @IsUUID() @IsOptional() id?: string; @@ -79,7 +79,7 @@ export class UnifiedFolderOutput extends UnifiedFolderInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/filestorage/group/services/group.service.ts b/packages/api/src/filestorage/group/services/group.service.ts index 681673430..a932c0ff7 100644 --- a/packages/api/src/filestorage/group/services/group.service.ts +++ b/packages/api/src/filestorage/group/services/group.service.ts @@ -56,6 +56,9 @@ export class GroupService { users: group.users, remote_was_deleted: group.remote_was_deleted, field_mappings: field_mappings, + remote_id: group.remote_id, + created_at: group.created_at, + modified_at: group.modified_at, }; let res: UnifiedGroupOutput = unifiedGroup; @@ -130,6 +133,9 @@ export class GroupService { users: group.users, remote_was_deleted: group.remote_was_deleted, field_mappings: field_mappings, + remote_id: group.remote_id, + created_at: group.created_at, + modified_at: group.modified_at, }; }), ); diff --git a/packages/api/src/filestorage/group/sync/sync.processor.ts b/packages/api/src/filestorage/group/sync/sync.processor.ts new file mode 100644 index 000000000..ac19645ec --- /dev/null +++ b/packages/api/src/filestorage/group/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('filestorage-sync-groups') + async handleSyncGroups(job: Job) { + try { + console.log(`Processing queue -> filestorage-sync-groups ${job.id}`); + await this.syncService.syncGroups(); + } catch (error) { + console.error('Error syncing filestorage groups', error); + } + } +} diff --git a/packages/api/src/filestorage/group/sync/sync.service.ts b/packages/api/src/filestorage/group/sync/sync.service.ts index 7c8b8960b..356d6ddf6 100644 --- a/packages/api/src/filestorage/group/sync/sync.service.ts +++ b/packages/api/src/filestorage/group/sync/sync.service.ts @@ -2,14 +2,20 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedGroupOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IGroupService } from '../types'; +import { UnifiedGroupOutput } from '../types/model.unified'; +import { fs_groups as FileStorageGroup } from '@prisma/client'; +import { FILESTORAGE_PROVIDERS } from '@panora/shared'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { OriginalGroupOutput } from '@@core/utils/types/original/original.file-storage'; @Injectable() export class SyncService implements OnModuleInit { @@ -19,13 +25,317 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'filestorage-sync-groups'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); } - // Additional methods and logic + @Cron('0 */8 * * *') // every 8 hours + async syncGroups(user_id?: string) { + try { + this.logger.log('Syncing groups...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncGroupsForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncGroupsForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} groups for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'filestorage', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping groups syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'filestorage.group', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IGroupService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = await service.syncGroups( + linkedUserId, + remoteProperties, + ); + + const sourceObject: OriginalGroupOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalGroupOutput[] + >({ + sourceObject, + targetType: FileStorageObject.group, + providerName: integrationId, + vertical: 'filestorage', + customFieldMappings, + })) as UnifiedGroupOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const groups_data = await this.saveGroupsInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'filestorage.group.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + groups_data, + 'filestorage.group.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } + } + + async saveGroupsInDb( + linkedUserId: string, + groups: UnifiedGroupOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let groups_results: FileStorageGroup[] = []; + for (let i = 0; i < groups.length; i++) { + const group = groups[i]; + const originId = group.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingGroup = await this.prisma.fs_groups.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_fs_group_id: string; + + if (existingGroup) { + // Update the existing group + let data: any = { + modified_at: new Date(), + }; + if (group.name) { + data = { ...data, name: group.name }; + } + if (group.users) { + data = { ...data, users: group.users }; + } + if (group.remote_was_deleted) { + data = { ...data, remote_was_deleted: group.remote_was_deleted }; + } + const res = await this.prisma.fs_groups.update({ + where: { + id_fs_group: existingGroup.id_fs_group, + }, + data: data, + }); + unique_fs_group_id = res.id_fs_group; + groups_results = [...groups_results, res]; + } else { + // Create a new group + this.logger.log('Group does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_fs_group: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (group.name) { + data = { ...data, name: group.name }; + } + if (group.users) { + data = { ...data, users: group.users }; + } + if (group.remote_was_deleted) { + data = { ...data, remote_was_deleted: group.remote_was_deleted }; + } + + const newGroup = await this.prisma.fs_groups.create({ + data: data, + }); + + unique_fs_group_id = newGroup.id_fs_group; + groups_results = [...groups_results, newGroup]; + } + + // check duplicate or existing values + if (group.field_mappings && group.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_fs_group_id, + }, + }); + + for (const [slug, value] of Object.entries(group.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_fs_group_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_fs_group_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return groups_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/filestorage/group/types/model.unified.ts b/packages/api/src/filestorage/group/types/model.unified.ts index 839e8257b..150b647a8 100644 --- a/packages/api/src/filestorage/group/types/model.unified.ts +++ b/packages/api/src/filestorage/group/types/model.unified.ts @@ -30,7 +30,7 @@ export class UnifiedGroupInput { export class UnifiedGroupOutput extends UnifiedGroupInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the group', + description: 'The UUID of the group', }) @IsUUID() @IsOptional() @@ -60,7 +60,7 @@ export class UnifiedGroupOutput extends UnifiedGroupInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/filestorage/permission/services/permission.service.ts b/packages/api/src/filestorage/permission/services/permission.service.ts index ad83fc07e..1c752cc58 100644 --- a/packages/api/src/filestorage/permission/services/permission.service.ts +++ b/packages/api/src/filestorage/permission/services/permission.service.ts @@ -261,6 +261,9 @@ export class PermissionService { type: permission.type, roles: permission.roles, field_mappings: field_mappings, + remote_id: permission.remote_id, + created_at: permission.created_at, + modified_at: permission.modified_at, }; let res: UnifiedPermissionOutput = unifiedPermission; @@ -374,6 +377,9 @@ export class PermissionService { type: permission.type, roles: permission.roles, field_mappings: field_mappings, + remote_id: permission.remote_id, + created_at: permission.created_at, + modified_at: permission.modified_at, }; }), ); diff --git a/packages/api/src/filestorage/permission/sync/sync.processor.ts b/packages/api/src/filestorage/permission/sync/sync.processor.ts new file mode 100644 index 000000000..738dd01e1 --- /dev/null +++ b/packages/api/src/filestorage/permission/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('filestorage-sync-permissions') + async handleSyncPermissions(job: Job) { + try { + console.log(`Processing queue -> filestorage-sync-permissions ${job.id}`); + await this.syncService.syncPermissions(); + } catch (error) { + console.error('Error syncing filestorage permissions', error); + } + } +} diff --git a/packages/api/src/filestorage/permission/sync/sync.service.ts b/packages/api/src/filestorage/permission/sync/sync.service.ts index 511861c89..524a83734 100644 --- a/packages/api/src/filestorage/permission/sync/sync.service.ts +++ b/packages/api/src/filestorage/permission/sync/sync.service.ts @@ -2,14 +2,20 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedPermissionOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IPermissionService } from '../types'; +import { OriginalPermissionOutput } from '@@core/utils/types/original/original.file-storage'; +import { UnifiedPermissionOutput } from '../types/model.unified'; +import { fs_permissions as FileStoragePermission } from '@prisma/client'; +import { FILESTORAGE_PROVIDERS } from '@panora/shared'; +import { FileStorageObject } from '@filestorage/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -19,13 +25,323 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'filestorage-sync-permissions'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); } - // Additional methods and logic + @Cron('0 */8 * * *') // every 8 hours + async syncPermissions(user_id?: string) { + try { + this.logger.log('Syncing permissions...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncPermissionsForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncPermissionsForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} permissions for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'filestorage', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping permissions syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'filestorage.permission', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IPermissionService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncPermissions(linkedUserId, remoteProperties); + + const sourceObject: OriginalPermissionOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalPermissionOutput[] + >({ + sourceObject, + targetType: FileStorageObject.permission, + providerName: integrationId, + vertical: 'filestorage', + customFieldMappings, + })) as UnifiedPermissionOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const permissions_data = await this.savePermissionsInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'filestorage.permission.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + permissions_data, + 'filestorage.permission.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } + } + + async savePermissionsInDb( + linkedUserId: string, + permissions: UnifiedPermissionOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let permissions_results: FileStoragePermission[] = []; + for (let i = 0; i < permissions.length; i++) { + const permission = permissions[i]; + const originId = permission.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingPermission = await this.prisma.fs_permissions.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_fs_permission_id: string; + + if (existingPermission) { + // Update the existing permission + let data: any = { + modified_at: new Date(), + }; + if (permission.roles) { + data = { ...data, roles: permission.roles }; + } + if (permission.type) { + data = { ...data, type: permission.type }; + } + if (permission.user_id) { + data = { ...data, user_id: permission.user_id }; + } + if (permission.group_id) { + data = { ...data, group_id: permission.group_id }; + } + const res = await this.prisma.fs_permissions.update({ + where: { + id_fs_permission: existingPermission.id_fs_permission, + }, + data: data, + }); + unique_fs_permission_id = res.id_fs_permission; + permissions_results = [...permissions_results, res]; + } else { + // Create a new permission + this.logger.log('Permission does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_fs_permission: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (permission.roles) { + data = { ...data, roles: permission.roles }; + } + if (permission.type) { + data = { ...data, type: permission.type }; + } + if (permission.user_id) { + data = { ...data, user_id: permission.user_id }; + } + if (permission.group_id) { + data = { ...data, group_id: permission.group_id }; + } + + const newPermission = await this.prisma.fs_permissions.create({ + data: data, + }); + + unique_fs_permission_id = newPermission.id_fs_permission; + permissions_results = [...permissions_results, newPermission]; + } + + // check duplicate or existing values + if (permission.field_mappings && permission.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_fs_permission_id, + }, + }); + + for (const [slug, value] of Object.entries( + permission.field_mappings, + )) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_fs_permission_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_fs_permission_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return permissions_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/filestorage/permission/types/model.unified.ts b/packages/api/src/filestorage/permission/types/model.unified.ts index 8e67f0923..b25f41d7b 100644 --- a/packages/api/src/filestorage/permission/types/model.unified.ts +++ b/packages/api/src/filestorage/permission/types/model.unified.ts @@ -12,14 +12,14 @@ export class UnifiedPermissionInput { @ApiProperty({ type: String, - description: 'The uuid of the user tied to the permission', + description: 'The UUID of the user tied to the permission', }) @IsString() user_id: string; @ApiProperty({ type: String, - description: 'The uuid of the group tied to the permission', + description: 'The UUID of the group tied to the permission', }) @IsString() group_id: string; @@ -36,7 +36,7 @@ export class UnifiedPermissionInput { export class UnifiedPermissionOutput extends UnifiedPermissionInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the permission', + description: 'The UUID of the permission', }) @IsUUID() @IsOptional() @@ -67,7 +67,7 @@ export class UnifiedPermissionOutput extends UnifiedPermissionInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/filestorage/sharedlink/services/sharedlink.service.ts b/packages/api/src/filestorage/sharedlink/services/sharedlink.service.ts index 54ac71b34..89f7799bd 100644 --- a/packages/api/src/filestorage/sharedlink/services/sharedlink.service.ts +++ b/packages/api/src/filestorage/sharedlink/services/sharedlink.service.ts @@ -270,6 +270,9 @@ export class SharedLinkService { password_protected: sharedlink.password_protected, password: sharedlink.password, field_mappings: field_mappings, + remote_id: sharedlink.remote_id, + created_at: sharedlink.created_at, + modified_at: sharedlink.modified_at, }; let res: UnifiedSharedLinkOutput = unifiedSharedLink; @@ -386,6 +389,9 @@ export class SharedLinkService { password_protected: sharedlink.password_protected, password: sharedlink.password, field_mappings: field_mappings, + remote_id: sharedlink.remote_id, + created_at: sharedlink.created_at, + modified_at: sharedlink.modified_at, }; }), ); diff --git a/packages/api/src/filestorage/sharedlink/sync/sync.processor.ts b/packages/api/src/filestorage/sharedlink/sync/sync.processor.ts new file mode 100644 index 000000000..e0e795286 --- /dev/null +++ b/packages/api/src/filestorage/sharedlink/sync/sync.processor.ts @@ -0,0 +1,20 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('filestorage-sync-shared-links') + async handleSyncSharedLinks(job: Job) { + try { + console.log( + `Processing queue -> filestorage-sync-shared-links ${job.id}`, + ); + await this.syncService.syncSharedLinks(); + } catch (error) { + console.error('Error syncing filestorage shared links', error); + } + } +} diff --git a/packages/api/src/filestorage/sharedlink/sync/sync.service.ts b/packages/api/src/filestorage/sharedlink/sync/sync.service.ts index f83798683..40a9b74ab 100644 --- a/packages/api/src/filestorage/sharedlink/sync/sync.service.ts +++ b/packages/api/src/filestorage/sharedlink/sync/sync.service.ts @@ -1,9 +1,21 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; +import { Cron } from '@nestjs/schedule'; +import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; import { WebhookService } from '@@core/webhook/webhook.service'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; +import { ISharedLinkService } from '../types'; +import { OriginalSharedLinkOutput } from '@@core/utils/types/original/original.file-storage'; +import { UnifiedSharedLinkOutput } from '../types/model.unified'; +import { fs_shared_links as FileStorageSharedLink } from '@prisma/client'; +import { FILESTORAGE_PROVIDERS } from '@panora/shared'; +import { FileStorageObject } from '@filestorage/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -13,13 +25,347 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } } - // Additional methods and logic + private async scheduleSyncJob() { + const jobName = 'filestorage-sync-shared-links'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); + } + + @Cron('0 */8 * * *') // every 8 hours + async syncSharedLinks(user_id?: string) { + try { + this.logger.log('Syncing shared links...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncSharedLinksForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncSharedLinksForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} shared links for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'filestorage', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping shared links syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'filestorage.shared_link', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: ISharedLinkService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = + await service.syncSharedLinks(linkedUserId, remoteProperties); + + const sourceObject: OriginalSharedLinkOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalSharedLinkOutput[] + >({ + sourceObject, + targetType: FileStorageObject.sharedlink, + providerName: integrationId, + vertical: 'filestorage', + customFieldMappings, + })) as UnifiedSharedLinkOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const shared_links_data = await this.saveSharedLinksInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'filestorage.shared_link.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + shared_links_data, + 'filestorage.shared_link.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } + } + + async saveSharedLinksInDb( + linkedUserId: string, + sharedLinks: UnifiedSharedLinkOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let shared_links_results: FileStorageSharedLink[] = []; + for (let i = 0; i < sharedLinks.length; i++) { + const sharedLink = sharedLinks[i]; + const originId = sharedLink.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingSharedLink = await this.prisma.fs_shared_links.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_fs_shared_link_id: string; + + if (existingSharedLink) { + // Update the existing shared link + let data: any = { + modified_at: new Date(), + }; + if (sharedLink.url) { + data = { ...data, url: sharedLink.url }; + } + if (sharedLink.download_url) { + data = { ...data, download_url: sharedLink.download_url }; + } + if (sharedLink.folder_id) { + data = { ...data, folder_id: sharedLink.folder_id }; + } + if (sharedLink.file_id) { + data = { ...data, file_id: sharedLink.file_id }; + } + if (sharedLink.scope) { + data = { ...data, scope: sharedLink.scope }; + } + if (sharedLink.password_protected) { + data = { + ...data, + password_protected: sharedLink.password_protected, + }; + } + if (sharedLink.password) { + data = { ...data, password: sharedLink.password }; + } + const res = await this.prisma.fs_shared_links.update({ + where: { + id_fs_shared_link: existingSharedLink.id_fs_shared_link, + }, + data: data, + }); + unique_fs_shared_link_id = res.id_fs_shared_link; + shared_links_results = [...shared_links_results, res]; + } else { + // Create a new shared link + this.logger.log('Shared link does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_fs_shared_link: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (sharedLink.url) { + data = { ...data, url: sharedLink.url }; + } + if (sharedLink.download_url) { + data = { ...data, download_url: sharedLink.download_url }; + } + if (sharedLink.folder_id) { + data = { ...data, folder_id: sharedLink.folder_id }; + } + if (sharedLink.file_id) { + data = { ...data, file_id: sharedLink.file_id }; + } + if (sharedLink.scope) { + data = { ...data, scope: sharedLink.scope }; + } + if (sharedLink.password_protected) { + data = { + ...data, + password_protected: sharedLink.password_protected, + }; + } + if (sharedLink.password) { + data = { ...data, password: sharedLink.password }; + } + + const newSharedLink = await this.prisma.fs_shared_links.create({ + data: data, + }); + + unique_fs_shared_link_id = newSharedLink.id_fs_shared_link; + shared_links_results = [...shared_links_results, newSharedLink]; + } + + // check duplicate or existing values + if (sharedLink.field_mappings && sharedLink.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_fs_shared_link_id, + }, + }); + + for (const [slug, value] of Object.entries( + sharedLink.field_mappings, + )) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_fs_shared_link_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_fs_shared_link_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return shared_links_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/filestorage/sharedlink/types/index.ts b/packages/api/src/filestorage/sharedlink/types/index.ts index babe296e4..40ebde04f 100644 --- a/packages/api/src/filestorage/sharedlink/types/index.ts +++ b/packages/api/src/filestorage/sharedlink/types/index.ts @@ -7,12 +7,12 @@ import { OriginalSharedLinkOutput } from '@@core/utils/types/original/original.f import { ApiResponse } from '@@core/utils/types'; export interface ISharedLinkService { - addSharedlink( + addSharedLink( sharedlinkData: DesunifyReturnType, linkedUserId: string, ): Promise>; - syncSharedlinks( + syncSharedLinks( linkedUserId: string, custom_properties?: string[], ): Promise>; diff --git a/packages/api/src/filestorage/sharedlink/types/model.unified.ts b/packages/api/src/filestorage/sharedlink/types/model.unified.ts index 9b1720a43..6c2dbc4a5 100644 --- a/packages/api/src/filestorage/sharedlink/types/model.unified.ts +++ b/packages/api/src/filestorage/sharedlink/types/model.unified.ts @@ -14,14 +14,14 @@ export class UnifiedSharedLinkInput { @ApiProperty({ type: String, - description: 'The uuid of the folder tied to the shared link', + description: 'The UUID of the folder tied to the shared link', }) @IsString() folder_id: string; @ApiProperty({ type: String, - description: 'The uuid of the file tied to the shared link', + description: 'The UUID of the file tied to the shared link', }) @IsString() file_id: string; @@ -53,7 +53,7 @@ export class UnifiedSharedLinkInput { export class UnifiedSharedLinkOutput extends UnifiedSharedLinkInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the shared link', + description: 'The UUID of the shared link', }) @IsUUID() @IsOptional() @@ -84,7 +84,7 @@ export class UnifiedSharedLinkOutput extends UnifiedSharedLinkInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/filestorage/user/services/user.service.ts b/packages/api/src/filestorage/user/services/user.service.ts index d1b5a4bec..32d3dcf29 100644 --- a/packages/api/src/filestorage/user/services/user.service.ts +++ b/packages/api/src/filestorage/user/services/user.service.ts @@ -56,6 +56,9 @@ export class UserService { email: user.email, is_me: user.is_me, field_mappings: field_mappings, + remote_id: user.remote_id, + created_at: user.created_at, + modified_at: user.modified_at, }; let res: UnifiedUserOutput = unifiedUser; @@ -128,6 +131,9 @@ export class UserService { email: user.email, is_me: user.is_me, field_mappings: field_mappings, + remote_id: user.remote_id, + created_at: user.created_at, + modified_at: user.modified_at, }; }), ); diff --git a/packages/api/src/filestorage/user/sync/sync.processor.ts b/packages/api/src/filestorage/user/sync/sync.processor.ts new file mode 100644 index 000000000..dd77e44d6 --- /dev/null +++ b/packages/api/src/filestorage/user/sync/sync.processor.ts @@ -0,0 +1,18 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { SyncService } from './sync.service'; + +@Processor('syncTasks') +export class SyncProcessor { + constructor(private syncService: SyncService) {} + + @Process('filestorage-sync-users') + async handleSyncUsers(job: Job) { + try { + console.log(`Processing queue -> filestorage-sync-users ${job.id}`); + await this.syncService.syncUsers(); + } catch (error) { + console.error('Error syncing filestorage users', error); + } + } +} diff --git a/packages/api/src/filestorage/user/sync/sync.service.ts b/packages/api/src/filestorage/user/sync/sync.service.ts index fc3919d0d..fdad2327e 100644 --- a/packages/api/src/filestorage/user/sync/sync.service.ts +++ b/packages/api/src/filestorage/user/sync/sync.service.ts @@ -2,14 +2,20 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; - import { WebhookService } from '@@core/webhook/webhook.service'; -import { UnifiedUserOutput } from '../types/model.unified'; +import { Queue } from 'bull'; +import { InjectQueue } from '@nestjs/bull'; +import { CoreUnification } from '@@core/utils/services/core.service'; +import { ApiResponse } from '@@core/utils/types'; import { IUserService } from '../types'; +import { OriginalUserOutput } from '@@core/utils/types/original/original.file-storage'; +import { UnifiedUserOutput } from '../types/model.unified'; +import { fs_users as FileStorageUser } from '@prisma/client'; +import { FILESTORAGE_PROVIDERS } from '@panora/shared'; +import { FileStorageObject } from '@filestorage/@lib/@types'; @Injectable() export class SyncService implements OnModuleInit { @@ -19,13 +25,317 @@ export class SyncService implements OnModuleInit { private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, + private coreUnification: CoreUnification, + @InjectQueue('syncTasks') private syncQueue: Queue, ) { this.logger.setContext(SyncService.name); } async onModuleInit() { - // Initialization logic + try { + await this.scheduleSyncJob(); + } catch (error) { + throw error; + } + } + + private async scheduleSyncJob() { + const jobName = 'filestorage-sync-users'; + + // Remove existing jobs to avoid duplicates in case of application restart + const jobs = await this.syncQueue.getRepeatableJobs(); + for (const job of jobs) { + if (job.name === jobName) { + await this.syncQueue.removeRepeatableByKey(job.key); + } + } + // Add new job to the queue with a CRON expression + await this.syncQueue.add( + jobName, + {}, + { + repeat: { cron: '0 0 * * *' }, // Runs once a day at midnight + }, + ); } - // Additional methods and logic + @Cron('0 */8 * * *') // every 8 hours + async syncUsers(user_id?: string) { + try { + this.logger.log('Syncing users...'); + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const id_project = project.id_project; + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncUsersForLinkedUser( + provider, + linkedUser.id_linked_user, + id_project, + ); + } catch (error) { + throw error; + } + } + } catch (error) { + throw error; + } + }); + } + } + } + } catch (error) { + throw error; + } + } + + async syncUsersForLinkedUser( + integrationId: string, + linkedUserId: string, + id_project: string, + ) { + try { + this.logger.log( + `Syncing ${integrationId} users for linkedUser ${linkedUserId}`, + ); + // check if linkedUser has a connection if not just stop sync + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: integrationId, + vertical: 'filestorage', + }, + }); + if (!connection) { + this.logger.warn( + `Skipping users syncing... No ${integrationId} connection was found for linked user ${linkedUserId} `, + ); + return; + } + // get potential fieldMappings and extract the original properties name + const customFieldMappings = + await this.fieldMappingService.getCustomFieldMappings( + integrationId, + linkedUserId, + 'filestorage.user', + ); + const remoteProperties: string[] = customFieldMappings.map( + (mapping) => mapping.remote_id, + ); + + const service: IUserService = + this.serviceRegistry.getService(integrationId); + const resp: ApiResponse = await service.syncUsers( + linkedUserId, + remoteProperties, + ); + + const sourceObject: OriginalUserOutput[] = resp.data; + + // unify the data according to the target obj wanted + const unifiedObject = (await this.coreUnification.unify< + OriginalUserOutput[] + >({ + sourceObject, + targetType: FileStorageObject.user, + providerName: integrationId, + vertical: 'filestorage', + customFieldMappings, + })) as UnifiedUserOutput[]; + + // insert the data in the DB with the fieldMappings (value table) + const users_data = await this.saveUsersInDb( + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: 'filestorage.user.pulled', + method: 'PULL', + url: '/pull', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.handleWebhook( + users_data, + 'filestorage.user.pulled', + id_project, + event.id_event, + ); + } catch (error) { + throw error; + } + } + + async saveUsersInDb( + linkedUserId: string, + users: UnifiedUserOutput[], + originSource: string, + remote_data: Record[], + ): Promise { + try { + let users_results: FileStorageUser[] = []; + for (let i = 0; i < users.length; i++) { + const user = users[i]; + const originId = user.remote_id; + + if (!originId || originId === '') { + throw new ReferenceError(`Origin id not there, found ${originId}`); + } + + const existingUser = await this.prisma.fs_users.findFirst({ + where: { + remote_id: originId, + remote_platform: originSource, + id_linked_user: linkedUserId, + }, + }); + + let unique_fs_user_id: string; + + if (existingUser) { + // Update the existing user + let data: any = { + modified_at: new Date(), + }; + if (user.name) { + data = { ...data, name: user.name }; + } + if (user.email) { + data = { ...data, email: user.email }; + } + if (user.is_me) { + data = { ...data, is_me: user.is_me }; + } + const res = await this.prisma.fs_users.update({ + where: { + id_fs_user: existingUser.id_fs_user, + }, + data: data, + }); + unique_fs_user_id = res.id_fs_user; + users_results = [...users_results, res]; + } else { + // Create a new user + this.logger.log('User does not exist, creating a new one'); + const uuid = uuidv4(); + let data: any = { + id_fs_user: uuid, + created_at: new Date(), + modified_at: new Date(), + id_linked_user: linkedUserId, + remote_id: originId, + remote_platform: originSource, + }; + + if (user.name) { + data = { ...data, name: user.name }; + } + if (user.email) { + data = { ...data, email: user.email }; + } + if (user.is_me) { + data = { ...data, is_me: user.is_me }; + } + + const newUser = await this.prisma.fs_users.create({ + data: data, + }); + + unique_fs_user_id = newUser.id_fs_user; + users_results = [...users_results, newUser]; + } + + // check duplicate or existing values + if (user.field_mappings && user.field_mappings.length > 0) { + const entity = await this.prisma.entity.create({ + data: { + id_entity: uuidv4(), + ressource_owner_id: unique_fs_user_id, + }, + }); + + for (const [slug, value] of Object.entries(user.field_mappings)) { + const attribute = await this.prisma.attribute.findFirst({ + where: { + slug: slug, + source: originSource, + id_consumer: linkedUserId, + }, + }); + + if (attribute) { + await this.prisma.value.create({ + data: { + id_value: uuidv4(), + data: value || 'null', + attribute: { + connect: { + id_attribute: attribute.id_attribute, + }, + }, + entity: { + connect: { + id_entity: entity.id_entity, + }, + }, + }, + }); + } + } + } + + // insert remote_data in db + await this.prisma.remote_data.upsert({ + where: { + ressource_owner_id: unique_fs_user_id, + }, + create: { + id_remote_data: uuidv4(), + ressource_owner_id: unique_fs_user_id, + format: 'json', + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + update: { + data: JSON.stringify(remote_data[i]), + created_at: new Date(), + }, + }); + } + return users_results; + } catch (error) { + throw error; + } + } } diff --git a/packages/api/src/filestorage/user/types/model.unified.ts b/packages/api/src/filestorage/user/types/model.unified.ts index 99932760a..bcf0470d2 100644 --- a/packages/api/src/filestorage/user/types/model.unified.ts +++ b/packages/api/src/filestorage/user/types/model.unified.ts @@ -32,7 +32,7 @@ export class UnifiedUserInput { export class UnifiedUserOutput extends UnifiedUserInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the user', + description: 'The UUID of the user', }) @IsUUID() @IsOptional() @@ -62,7 +62,7 @@ export class UnifiedUserOutput extends UnifiedUserInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ticketing/account/services/account.service.ts b/packages/api/src/ticketing/account/services/account.service.ts index dc3d8d7cf..f595c03f8 100644 --- a/packages/api/src/ticketing/account/services/account.service.ts +++ b/packages/api/src/ticketing/account/services/account.service.ts @@ -52,6 +52,9 @@ export class AccountService { name: account.name, domains: account.domains, field_mappings: field_mappings, + remote_id: account.remote_id, + created_at: account.created_at, + modified_at: account.modified_at, }; let res: UnifiedAccountOutput = unifiedAccount; @@ -165,6 +168,9 @@ export class AccountService { name: account.name, domains: account.domains, field_mappings: field_mappings, + remote_id: account.remote_id, + created_at: account.created_at, + modified_at: account.modified_at, }; }), ); diff --git a/packages/api/src/ticketing/account/types/model.unified.ts b/packages/api/src/ticketing/account/types/model.unified.ts index 050b2a8a2..d0087f869 100644 --- a/packages/api/src/ticketing/account/types/model.unified.ts +++ b/packages/api/src/ticketing/account/types/model.unified.ts @@ -23,7 +23,7 @@ export class UnifiedAccountInput { } export class UnifiedAccountOutput extends UnifiedAccountInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the account' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the account' }) @IsUUID() @IsOptional() id?: string; @@ -53,7 +53,7 @@ export class UnifiedAccountOutput extends UnifiedAccountInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ticketing/attachment/services/attachment.service.ts b/packages/api/src/ticketing/attachment/services/attachment.service.ts index a28827050..b8ae176de 100644 --- a/packages/api/src/ticketing/attachment/services/attachment.service.ts +++ b/packages/api/src/ticketing/attachment/services/attachment.service.ts @@ -177,6 +177,9 @@ export class AttachmentService { file_url: attachment.file_url, uploader: attachment.uploader, field_mappings: field_mappings, + remote_id: attachment.remote_id, + created_at: attachment.created_at, + modified_at: attachment.modified_at, }; let res: UnifiedAttachmentOutput = unifiedAttachment; @@ -289,6 +292,9 @@ export class AttachmentService { file_url: attachment.file_url, uploader: attachment.uploader, //TODO field_mappings: field_mappings, + remote_id: attachment.remote_id, + created_at: attachment.created_at, + modified_at: attachment.modified_at, }; }), ); diff --git a/packages/api/src/ticketing/attachment/types/model.unified.ts b/packages/api/src/ticketing/attachment/types/model.unified.ts index 47f1ef936..c84a4c64d 100644 --- a/packages/api/src/ticketing/attachment/types/model.unified.ts +++ b/packages/api/src/ticketing/attachment/types/model.unified.ts @@ -12,7 +12,7 @@ export class UnifiedAttachmentInput { @ApiProperty({ type: String, - description: "The uploader's uuid of the attachment", + description: "The uploader's UUID of the attachment", }) @IsString() @IsOptional() @@ -30,7 +30,7 @@ export class UnifiedAttachmentInput { export class UnifiedAttachmentOutput extends UnifiedAttachmentInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the attachment', + description: 'The UUID of the attachment', }) @IsUUID() @IsOptional() @@ -61,7 +61,7 @@ export class UnifiedAttachmentOutput extends UnifiedAttachmentInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ticketing/collection/services/collection.service.ts b/packages/api/src/ticketing/collection/services/collection.service.ts index 333eb015a..8cae671f6 100644 --- a/packages/api/src/ticketing/collection/services/collection.service.ts +++ b/packages/api/src/ticketing/collection/services/collection.service.ts @@ -36,6 +36,9 @@ export class CollectionService { name: collection.name, description: collection.description, collection_type: collection.collection_type, + remote_id: collection.remote_id, + created_at: collection.created_at, + modified_at: collection.modified_at, }; let res: UnifiedCollectionOutput = { @@ -124,6 +127,9 @@ export class CollectionService { name: collection.name, description: collection.description, collection_type: collection.collection_type, + remote_id: collection.remote_id, + created_at: collection.created_at, + modified_at: collection.modified_at, }; }), ); diff --git a/packages/api/src/ticketing/collection/types/model.unified.ts b/packages/api/src/ticketing/collection/types/model.unified.ts index 9b90eaece..8f7b7930d 100644 --- a/packages/api/src/ticketing/collection/types/model.unified.ts +++ b/packages/api/src/ticketing/collection/types/model.unified.ts @@ -32,7 +32,7 @@ export class UnifiedCollectionInput { export class UnifiedCollectionOutput extends UnifiedCollectionInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the collection', + description: 'The UUID of the collection', }) @IsUUID() @IsOptional() @@ -63,7 +63,7 @@ export class UnifiedCollectionOutput extends UnifiedCollectionInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ticketing/comment/services/comment.service.ts b/packages/api/src/ticketing/comment/services/comment.service.ts index cda81a351..b58b9be83 100644 --- a/packages/api/src/ticketing/comment/services/comment.service.ts +++ b/packages/api/src/ticketing/comment/services/comment.service.ts @@ -3,7 +3,6 @@ import { PrismaService } from '@@core/prisma/prisma.service'; import { LoggerService } from '@@core/logger/logger.service'; import { v4 as uuidv4 } from 'uuid'; import { ApiResponse } from '@@core/utils/types'; -import { throwTypedError, UnifiedTicketingError } from '@@core/utils/errors'; import { WebhookService } from '@@core/webhook/webhook.service'; import { UnifiedCommentInput, @@ -327,6 +326,9 @@ export class CommentService { ticket_id: comment.id_tcg_ticket, contact_id: comment.id_tcg_contact, // uuid of Contact object user_id: comment.id_tcg_user, // uuid of User object + remote_id: comment.remote_id, + created_at: comment.created_at, + modified_at: comment.modified_at, }; let res: UnifiedCommentOutput = { @@ -447,6 +449,9 @@ export class CommentService { ticket_id: comment.id_tcg_ticket, contact_id: comment.id_tcg_contact, // uuid of Contact object user_id: comment.id_tcg_user, // uuid of User object + remote_id: comment.remote_id, + created_at: comment.created_at, + modified_at: comment.modified_at, }; }), ); diff --git a/packages/api/src/ticketing/comment/sync/sync.service.ts b/packages/api/src/ticketing/comment/sync/sync.service.ts index 70f0e83e1..0c863a9a8 100644 --- a/packages/api/src/ticketing/comment/sync/sync.service.ts +++ b/packages/api/src/ticketing/comment/sync/sync.service.ts @@ -1,6 +1,5 @@ import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; -import { SyncError, throwTypedError } from '@@core/utils/errors'; import { Injectable, OnModuleInit } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { ApiResponse } from '@@core/utils/types'; @@ -324,7 +323,7 @@ export class SyncService implements OnModuleInit { where: { remote_platform: originSource, id_linked_user: linkedUserId, - file_name: attchmt.file_name, + remote_id: attchmt.id, }, }); @@ -335,7 +334,6 @@ export class SyncService implements OnModuleInit { id_tcg_attachment: existingAttachmt.id_tcg_attachment, }, data: { - remote_id: attchmt.id, file_url: attchmt.file_url, id_tcg_comment: unique_ticketing_comment_id, id_tcg_ticket: id_ticket, diff --git a/packages/api/src/ticketing/comment/types/model.unified.ts b/packages/api/src/ticketing/comment/types/model.unified.ts index 8c7278ba9..5bfdf00d0 100644 --- a/packages/api/src/ticketing/comment/types/model.unified.ts +++ b/packages/api/src/ticketing/comment/types/model.unified.ts @@ -36,40 +36,40 @@ export class UnifiedCommentInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the ticket the comment is tied to', + description: 'The UUID of the ticket the comment is tied to', }) @IsUUID() @IsOptional() - ticket_id?: string; // uuid of Ticket object + ticket_id?: string; // UUID of Ticket object @ApiPropertyOptional({ type: String, description: - 'The uuid of the contact which the comment belongs to (if no user_id specified)', + 'The UUID of the contact which the comment belongs to (if no user_id specified)', }) @IsUUID() @IsOptional() - contact_id?: string; // uuid of Contact object + contact_id?: string; // UUID of Contact object @ApiPropertyOptional({ type: String, description: - 'The uuid of the user which the comment belongs to (if no contact_id specified)', + 'The UUID of the user which the comment belongs to (if no contact_id specified)', }) @IsUUID() @IsOptional() - user_id?: string; // uuid of User object + user_id?: string; // UUID of User object @ApiPropertyOptional({ type: [String], - description: 'The attachements uuids tied to the comment', + description: 'The attachements UUIDs tied to the comment', }) @IsOptional() - attachments?: any[]; //uuids of Attachments objects + attachments?: any[]; //UUIDs of Attachments objects } export class UnifiedCommentOutput extends UnifiedCommentInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the comment' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the comment' }) @IsUUID() @IsOptional() id?: string; @@ -106,7 +106,7 @@ export class UnifiedCommentOutput extends UnifiedCommentInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ticketing/contact/services/contact.service.ts b/packages/api/src/ticketing/contact/services/contact.service.ts index 72bfbad67..e8e1cc079 100644 --- a/packages/api/src/ticketing/contact/services/contact.service.ts +++ b/packages/api/src/ticketing/contact/services/contact.service.ts @@ -54,6 +54,9 @@ export class ContactService { details: contact.details, phone_number: contact.phone_number, field_mappings: field_mappings, + remote_id: contact.remote_id, + created_at: contact.created_at, + modified_at: contact.modified_at, }; let res: UnifiedContactOutput = unifiedContact; @@ -168,6 +171,9 @@ export class ContactService { details: contact.details, phone_number: contact.phone_number, field_mappings: field_mappings, + remote_id: contact.remote_id, + created_at: contact.created_at, + modified_at: contact.modified_at, }; }), ); diff --git a/packages/api/src/ticketing/contact/types/model.unified.ts b/packages/api/src/ticketing/contact/types/model.unified.ts index ec825f415..3dc3761c6 100644 --- a/packages/api/src/ticketing/contact/types/model.unified.ts +++ b/packages/api/src/ticketing/contact/types/model.unified.ts @@ -42,7 +42,7 @@ export class UnifiedContactInput { } export class UnifiedContactOutput extends UnifiedContactInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the contact' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the contact' }) @IsUUID() @IsOptional() id?: string; @@ -72,7 +72,7 @@ export class UnifiedContactOutput extends UnifiedContactInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ticketing/tag/services/tag.service.ts b/packages/api/src/ticketing/tag/services/tag.service.ts index ab6cd93d7..8e039fb15 100644 --- a/packages/api/src/ticketing/tag/services/tag.service.ts +++ b/packages/api/src/ticketing/tag/services/tag.service.ts @@ -51,6 +51,9 @@ export class TagService { id: tag.id_tcg_tag, name: tag.name, field_mappings: field_mappings, + remote_id: tag.remote_id, + created_at: tag.created_at, + modified_at: tag.modified_at, }; let res: UnifiedTagOutput = unifiedTag; @@ -163,6 +166,9 @@ export class TagService { id: tag.id_tcg_tag, name: tag.name, field_mappings: field_mappings, + remote_id: tag.remote_id, + created_at: tag.created_at, + modified_at: tag.modified_at, }; }), ); diff --git a/packages/api/src/ticketing/tag/types/model.unified.ts b/packages/api/src/ticketing/tag/types/model.unified.ts index a3ac71228..cbc193472 100644 --- a/packages/api/src/ticketing/tag/types/model.unified.ts +++ b/packages/api/src/ticketing/tag/types/model.unified.ts @@ -19,7 +19,7 @@ export class UnifiedTagInput { } export class UnifiedTagOutput extends UnifiedTagInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the tag' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the tag' }) @IsUUID() @IsOptional() id?: string; @@ -48,7 +48,7 @@ export class UnifiedTagOutput extends UnifiedTagInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ticketing/team/services/team.service.ts b/packages/api/src/ticketing/team/services/team.service.ts index 0ae938a62..11d05c406 100644 --- a/packages/api/src/ticketing/team/services/team.service.ts +++ b/packages/api/src/ticketing/team/services/team.service.ts @@ -52,6 +52,9 @@ export class TeamService { name: team.name, description: team.description, field_mappings: field_mappings, + remote_id: team.remote_id, + created_at: team.created_at, + modified_at: team.modified_at, }; let res: UnifiedTeamOutput = unifiedTeam; @@ -165,6 +168,9 @@ export class TeamService { name: team.name, description: team.description, field_mappings: field_mappings, + remote_id: team.remote_id, + created_at: team.created_at, + modified_at: team.modified_at, }; }), ); diff --git a/packages/api/src/ticketing/team/types/model.unified.ts b/packages/api/src/ticketing/team/types/model.unified.ts index c24b8c72e..e5b6eee31 100644 --- a/packages/api/src/ticketing/team/types/model.unified.ts +++ b/packages/api/src/ticketing/team/types/model.unified.ts @@ -27,7 +27,7 @@ export class UnifiedTeamInput { } export class UnifiedTeamOutput extends UnifiedTeamInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the team' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the team' }) @IsUUID() @IsOptional() id?: string; @@ -56,7 +56,7 @@ export class UnifiedTeamOutput extends UnifiedTeamInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ticketing/ticket/services/front/index.ts b/packages/api/src/ticketing/ticket/services/front/index.ts index 8267d6d3f..08a4460dc 100644 --- a/packages/api/src/ticketing/ticket/services/front/index.ts +++ b/packages/api/src/ticketing/ticket/services/front/index.ts @@ -6,7 +6,6 @@ import { TicketingObject } from '@ticketing/@lib/@types'; import { ITicketService } from '@ticketing/ticket/types'; import { ApiResponse } from '@@core/utils/types'; import axios from 'axios'; -import { ActionType, handle3rdPartyServiceError } from '@@core/utils/errors'; import { ServiceRegistry } from '../registry.service'; import { FrontTicketInput, FrontTicketOutput } from './types'; import { Utils } from '@ticketing/@lib/@utils'; diff --git a/packages/api/src/ticketing/ticket/services/ticket.service.ts b/packages/api/src/ticketing/ticket/services/ticket.service.ts index ac3cfbb16..b8f11aaa6 100644 --- a/packages/api/src/ticketing/ticket/services/ticket.service.ts +++ b/packages/api/src/ticketing/ticket/services/ticket.service.ts @@ -13,7 +13,6 @@ import { TicketingObject } from '@ticketing/@lib/@types'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { OriginalTicketOutput } from '@@core/utils/types/original/original.ticketing'; import { ServiceRegistry } from './registry.service'; -import { throwTypedError, UnifiedTicketingError } from '@@core/utils/errors'; import { CoreUnification } from '@@core/utils/services/core.service'; @Injectable() @@ -369,6 +368,9 @@ export class TicketService { priority: ticket.priority || '', assigned_to: ticket.assigned_to || [], field_mappings: field_mappings, + remote_id: ticket.remote_id, + created_at: ticket.created_at, + modified_at: ticket.modified_at, }; let res: UnifiedTicketOutput = { @@ -497,6 +499,9 @@ export class TicketService { assigned_to: ticket.assigned_to || [], collections: ticket.collections || [], field_mappings: field_mappings, + remote_id: ticket.remote_id, + created_at: ticket.created_at, + modified_at: ticket.modified_at, }; }), ); diff --git a/packages/api/src/ticketing/ticket/sync/sync.service.ts b/packages/api/src/ticketing/ticket/sync/sync.service.ts index 920d6ae6c..b4b3c4ce1 100644 --- a/packages/api/src/ticketing/ticket/sync/sync.service.ts +++ b/packages/api/src/ticketing/ticket/sync/sync.service.ts @@ -1,12 +1,10 @@ import { LoggerService } from '@@core/logger/logger.service'; import { PrismaService } from '@@core/prisma/prisma.service'; -import { SyncError, throwTypedError } from '@@core/utils/errors'; import { Injectable, OnModuleInit } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; - import { TicketingObject } from '@ticketing/@lib/@types'; import { UnifiedTicketOutput } from '../types/model.unified'; import { WebhookService } from '@@core/webhook/webhook.service'; diff --git a/packages/api/src/ticketing/ticket/types/model.unified.ts b/packages/api/src/ticketing/ticket/types/model.unified.ts index f1c8bf090..26fa8e7d0 100644 --- a/packages/api/src/ticketing/ticket/types/model.unified.ts +++ b/packages/api/src/ticketing/ticket/types/model.unified.ts @@ -48,7 +48,7 @@ export class UnifiedTicketInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the parent ticket', + description: 'The UUID of the parent ticket', }) @IsUUID() @IsOptional() @@ -56,7 +56,7 @@ export class UnifiedTicketInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the collection (project) the ticket belongs to', + description: 'The UUID of the collection (project) the ticket belongs to', }) @IsUUID() @IsOptional() @@ -89,10 +89,10 @@ export class UnifiedTicketInput { @ApiPropertyOptional({ type: [String], - description: 'The users uuids the ticket is assigned to', + description: 'The users UUIDs the ticket is assigned to', }) @IsOptional() - assigned_to?: string[]; //uuid of Users objects ? + assigned_to?: string[]; //UUID of Users objects ? @ApiPropertyOptional({ type: UnifiedCommentInput, @@ -103,7 +103,7 @@ export class UnifiedTicketInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the account which the ticket belongs to', + description: 'The UUID of the account which the ticket belongs to', }) @IsUUID() @IsOptional() @@ -111,7 +111,7 @@ export class UnifiedTicketInput { @ApiPropertyOptional({ type: String, - description: 'The uuid of the contact which the ticket belongs to', + description: 'The UUID of the contact which the ticket belongs to', }) @IsUUID() @IsOptional() @@ -126,7 +126,7 @@ export class UnifiedTicketInput { field_mappings?: Record; } export class UnifiedTicketOutput extends UnifiedTicketInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the ticket' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the ticket' }) @IsUUID() @IsOptional() id?: string; @@ -156,7 +156,7 @@ export class UnifiedTicketOutput extends UnifiedTicketInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any; diff --git a/packages/api/src/ticketing/user/services/user.service.ts b/packages/api/src/ticketing/user/services/user.service.ts index 6be9fea6d..16ebb34d9 100644 --- a/packages/api/src/ticketing/user/services/user.service.ts +++ b/packages/api/src/ticketing/user/services/user.service.ts @@ -55,6 +55,9 @@ export class UserService { name: user.name, teams: user.teams, field_mappings: field_mappings, + remote_id: user.remote_id, + created_at: user.created_at, + modified_at: user.modified_at, }; let res: UnifiedUserOutput = unifiedUser; @@ -168,6 +171,9 @@ export class UserService { name: user.name, teams: user.teams, field_mappings: field_mappings, + remote_id: user.remote_id, + created_at: user.created_at, + modified_at: user.modified_at, }; }), ); diff --git a/packages/api/src/ticketing/user/types/model.unified.ts b/packages/api/src/ticketing/user/types/model.unified.ts index ee91e4e15..1a306d02b 100644 --- a/packages/api/src/ticketing/user/types/model.unified.ts +++ b/packages/api/src/ticketing/user/types/model.unified.ts @@ -41,7 +41,7 @@ export class UnifiedUserInput { } export class UnifiedUserOutput extends UnifiedUserInput { - @ApiPropertyOptional({ type: String, description: 'The uuid of the user' }) + @ApiPropertyOptional({ type: String, description: 'The UUID of the user' }) @IsUUID() @IsOptional() id?: string; @@ -70,7 +70,7 @@ export class UnifiedUserOutput extends UnifiedUserInput { @ApiPropertyOptional({ type: {}, - description: 'The modified date of th object', + description: 'The modified date of the object', }) @IsOptional() modified_at?: any;