4489 timebox finish google calendar full sync (#4615)

* add lodash differenceWith * add awaits * update sync cursor is working * add logs * use isSyncEnabled information to enqueue jobs * add decorator InjectObjectMetadataRepository * fix gmail-full-sync
twentyhq · Mar 22, 2024 · 5665656 · 5665656
1 parent 41aae5b
commit 5665656
Show file tree

Hide file tree

Showing 7 changed files with 180 additions and 60 deletions.
diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json
@@ -43,13 +43,15 @@
     "class-validator": "patch:[email protected]#./patches/class-validator+0.14.0.patch",
     "graphql-middleware": "^6.1.35",
     "jwt-decode": "^4.0.0",
+    "lodash.differencewith": "^4.5.0",
     "passport": "^0.7.0",
     "psl": "^1.9.0",
     "tsconfig-paths": "^4.2.0"
   },
   "devDependencies": {
     "@nestjs/cli": "10.3.0",
     "@nx/js": "17.2.8",
+    "@types/lodash.differencewith": "^4.5.9",
     "@types/lodash.isempty": "^4.4.7",
     "@types/lodash.isequal": "^4.5.8",
     "@types/lodash.isobject": "^3.0.7",

diff --git a/packages/twenty-server/src/modules/calendar/commands/google-calendar-full-sync.command.ts b/packages/twenty-server/src/modules/calendar/commands/google-calendar-full-sync.command.ts
@@ -11,6 +11,8 @@ import {
 } from 'src/modules/calendar/jobs/google-calendar-full-sync.job';
 import { InjectObjectMetadataRepository } from 'src/engine/object-metadata-repository/object-metadata-repository.decorator';
 import { ConnectedAccountObjectMetadata } from 'src/modules/connected-account/standard-objects/connected-account.object-metadata';
+import { CalendarChannelRepository } from 'src/modules/calendar/repositories/calendar-channel.repository';
+import { CalendarChannelObjectMetadata } from 'src/modules/calendar/standard-objects/calendar-channel.object-metadata';
 
 interface GoogleCalendarFullSyncOptions {
   workspaceId: string;
@@ -27,6 +29,8 @@ export class GoogleCalendarFullSyncCommand extends CommandRunner {
     private readonly messageQueueService: MessageQueueService,
     @InjectObjectMetadataRepository(ConnectedAccountObjectMetadata)
     private readonly connectedAccountRepository: ConnectedAccountRepository,
+    @InjectObjectMetadataRepository(CalendarChannelObjectMetadata)
+    private readonly calendarChannelRepository: CalendarChannelRepository,
   ) {
     super();
   }
@@ -54,6 +58,16 @@ export class GoogleCalendarFullSyncCommand extends CommandRunner {
       await this.connectedAccountRepository.getAll(workspaceId);
 
     for (const connectedAccount of connectedAccounts) {
+      const calendarChannel =
+        await this.calendarChannelRepository.getFirstByConnectedAccountId(
+          connectedAccount.id,
+          workspaceId,
+        );
+
+      if (!calendarChannel?.isSyncEnabled) {
+        continue;
+      }
+
       await this.messageQueueService.add<GoogleCalendarFullSyncJobData>(
         GoogleCalendarFullSyncJob.name,
         {

diff --git a/packages/twenty-server/src/modules/calendar/repositories/calendar-channel.repository.ts b/packages/twenty-server/src/modules/calendar/repositories/calendar-channel.repository.ts
@@ -28,36 +28,18 @@ export class CalendarChannelRepository {
     );
   }
 
-  public async getFirstByConnectedAccountIdOrFail(
+  public async getFirstByConnectedAccountId(
     connectedAccountId: string,
     workspaceId: string,
-  ): Promise<ObjectRecord<CalendarChannelObjectMetadata>> {
+  ): Promise<ObjectRecord<CalendarChannelObjectMetadata> | undefined> {
     const calendarChannels = await this.getByConnectedAccountId(
       connectedAccountId,
       workspaceId,
     );
 
-    if (!calendarChannels || calendarChannels.length === 0) {
-      throw new Error(
-        `No calendar channel found for connected account ${connectedAccountId} in workspace ${workspaceId}`,
-      );
-    }
-
     return calendarChannels[0];
   }
 
-  public async getIsContactAutoCreationEnabledByConnectedAccountIdOrFail(
-    connectedAccountId: string,
-    workspaceId: string,
-  ): Promise<boolean> {
-    const calendarChannel = await this.getFirstByConnectedAccountIdOrFail(
-      connectedAccountId,
-      workspaceId,
-    );
-
-    return calendarChannel.isContactAutoCreationEnabled;
-  }
-
   public async getByIds(
     ids: string[],
     workspaceId: string,
@@ -73,4 +55,21 @@ export class CalendarChannelRepository {
       transactionManager,
     );
   }
+
+  public async updateSyncCursor(
+    syncCursor: string,
+    calendarChannelId: string,
+    workspaceId: string,
+    transactionManager?: EntityManager,
+  ): Promise<void> {
+    const dataSourceSchema =
+      this.workspaceDataSourceService.getSchemaName(workspaceId);
+
+    await this.workspaceDataSourceService.executeRawQuery(
+      `UPDATE ${dataSourceSchema}."calendarChannel" SET "syncCursor" = $1 WHERE "id" = $2`,
+      [syncCursor, calendarChannelId],
+      workspaceId,
+      transactionManager,
+    );
+  }
 }
diff --git a/...ges/twenty-server/src/modules/calendar/repositories/calendar-event-attendee.repository.ts b/...ges/twenty-server/src/modules/calendar/repositories/calendar-event-attendee.repository.ts
@@ -1,6 +1,7 @@
 import { Injectable } from '@nestjs/common';
 
 import { EntityManager } from 'typeorm';
+import differenceWith from 'lodash.differencewith';
 
 import { WorkspaceDataSourceService } from 'src/engine/workspace-datasource/workspace-datasource.service';
 import { ObjectRecord } from 'src/engine/workspace-manager/workspace-sync-metadata/types/object-record';
@@ -27,7 +28,7 @@ export class CalendarEventAttendeeRepository {
       this.workspaceDataSourceService.getSchemaName(workspaceId);
 
     return await this.workspaceDataSourceService.executeRawQuery(
-      `SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`,
+      `SELECT * FROM ${dataSourceSchema}."calendarEventAttendee" WHERE "id" = ANY($1)`,
       [calendarEventAttendeeIds],
       workspaceId,
       transactionManager,
@@ -47,7 +48,7 @@ export class CalendarEventAttendeeRepository {
       this.workspaceDataSourceService.getSchemaName(workspaceId);
 
     return await this.workspaceDataSourceService.executeRawQuery(
-      `SELECT * FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "calendarEventId" = ANY($1)`,
+      `SELECT * FROM ${dataSourceSchema}."calendarEventAttendee" WHERE "calendarEventId" = ANY($1)`,
       [calendarEventIds],
       workspaceId,
       transactionManager,
@@ -67,7 +68,7 @@ export class CalendarEventAttendeeRepository {
       this.workspaceDataSourceService.getSchemaName(workspaceId);
 
     await this.workspaceDataSourceService.executeRawQuery(
-      `DELETE FROM ${dataSourceSchema}."calendarEventAttendees" WHERE "id" = ANY($1)`,
+      `DELETE FROM ${dataSourceSchema}."calendarEventAttendee" WHERE "id" = ANY($1)`,
       [calendarEventAttendeeIds],
       workspaceId,
       transactionManager,
@@ -119,6 +120,29 @@ export class CalendarEventAttendeeRepository {
     const dataSourceSchema =
       this.workspaceDataSourceService.getSchemaName(workspaceId);
 
+    const calendarEventIds = Array.from(iCalUIDCalendarEventIdMap.values());
+
+    const existingCalendarEventAttendees = await this.getByCalendarEventIds(
+      calendarEventIds,
+      workspaceId,
+      transactionManager,
+    );
+
+    const calendarEventAttendeesToDelete = differenceWith(
+      existingCalendarEventAttendees,
+      calendarEventAttendees,
+      (existingCalendarEventAttendee, calendarEventAttendee) =>
+        existingCalendarEventAttendee.handle === calendarEventAttendee.handle,
+    );
+
+    await this.deleteByIds(
+      calendarEventAttendeesToDelete.map(
+        (calendarEventAttendee) => calendarEventAttendee.id,
+      ),
+      workspaceId,
+      transactionManager,
+    );
+
     const values = calendarEventAttendees.map((calendarEventAttendee) => ({
       ...calendarEventAttendee,
       calendarEventId: iCalUIDCalendarEventIdMap.get(

diff --git a/packages/twenty-server/src/modules/calendar/services/google-calendar-full-sync.service.ts b/packages/twenty-server/src/modules/calendar/services/google-calendar-full-sync.service.ts
@@ -79,11 +79,15 @@ export class GoogleCalendarFullSyncService {
     }
 
     const calendarChannel =
-      await this.calendarChannelRepository.getFirstByConnectedAccountIdOrFail(
+      await this.calendarChannelRepository.getFirstByConnectedAccountId(
         connectedAccountId,
         workspaceId,
       );
 
+    if (!calendarChannel) {
+      return;
+    }
+
     const calendarChannelId = calendarChannel.id;
 
     const googleCalendarClient =
@@ -109,6 +113,7 @@ export class GoogleCalendarFullSyncService {
       : [];
 
     const blocklistedEmails = blocklist.map((blocklist) => blocklist.handle);
+
     let startTime = Date.now();
 
     const googleCalendarEvents = await googleCalendarClient.events.list({
@@ -206,38 +211,94 @@ export class GoogleCalendarFullSyncService {
           workspaceId,
         );
 
-      dataSourceMetadata?.transaction(async (transactionManager) => {
-        this.calendarEventRepository.saveCalendarEvents(
-          eventsToSave,
-          workspaceId,
-          transactionManager,
-        );
-
-        this.calendarEventRepository.updateCalendarEvents(
-          eventsToUpdate,
-          workspaceId,
-          transactionManager,
-        );
-
-        this.calendarChannelEventAssociationRepository.saveCalendarChannelEventAssociations(
-          calendarChannelEventAssociationsToSave,
-          workspaceId,
-          transactionManager,
+      try {
+        dataSourceMetadata?.transaction(async (transactionManager) => {
+          startTime = Date.now();
+
+          await this.calendarEventRepository.saveCalendarEvents(
+            eventsToSave,
+            workspaceId,
+            transactionManager,
+          );
+
+          endTime = Date.now();
+
+          this.logger.log(
+            `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving events in ${
+              endTime - startTime
+            }ms.`,
+          );
+
+          startTime = Date.now();
+
+          await this.calendarEventRepository.updateCalendarEvents(
+            eventsToUpdate,
+            workspaceId,
+            transactionManager,
+          );
+
+          endTime = Date.now();
+
+          this.logger.log(
+            `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating events in ${
+              endTime - startTime
+            }ms.`,
+          );
+
+          startTime = Date.now();
+
+          await this.calendarChannelEventAssociationRepository.saveCalendarChannelEventAssociations(
+            calendarChannelEventAssociationsToSave,
+            workspaceId,
+            transactionManager,
+          );
+
+          endTime = Date.now();
+
+          this.logger.log(
+            `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving calendar channel event associations in ${
+              endTime - startTime
+            }ms.`,
+          );
+
+          startTime = Date.now();
+
+          await this.calendarEventAttendeesRepository.saveCalendarEventAttendees(
+            attendeesToSave,
+            workspaceId,
+            transactionManager,
+          );
+
+          endTime = Date.now();
+
+          this.logger.log(
+            `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: saving attendees in ${
+              endTime - startTime
+            }ms.`,
+          );
+
+          startTime = Date.now();
+
+          await this.calendarEventAttendeesRepository.updateCalendarEventAttendees(
+            attendeesToUpdate,
+            iCalUIDCalendarEventIdMap,
+            workspaceId,
+            transactionManager,
+          );
+
+          endTime = Date.now();
+
+          this.logger.log(
+            `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: updating attendees in ${
+              endTime - startTime
+            }ms.`,
+          );
+        });
+      } catch (error) {
+        this.logger.error(
+          `Error during google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId}: ${error.message}`,
         );
-
-        this.calendarEventAttendeesRepository.saveCalendarEventAttendees(
-          attendeesToSave,
-          workspaceId,
-          transactionManager,
-        );
-
-        this.calendarEventAttendeesRepository.updateCalendarEventAttendees(
-          attendeesToUpdate,
-          iCalUIDCalendarEventIdMap,
-          workspaceId,
-          transactionManager,
-        );
-      });
+      }
     } else {
       this.logger.log(
         `google calendar full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
@@ -252,11 +313,11 @@ export class GoogleCalendarFullSyncService {
 
     startTime = Date.now();
 
-    // await this.calendarChannelService.updateSyncCursor(
-    //   nextSyncToken,
-    //   connectedAccount.id,
-    //   workspaceId,
-    // );
+    await this.calendarChannelRepository.updateSyncCursor(
+      nextSyncToken,
+      calendarChannel.id,
+      workspaceId,
+    );
 
     endTime = Date.now();
 

diff --git a/...s/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service.ts b/...s/twenty-server/src/modules/messaging/services/gmail-full-sync/gmail-full-sync.service.ts
@@ -211,6 +211,8 @@ export class GmailFullSyncService {
       this.logger.log(
         `gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
       );
+
+      return;
     }
 
     if (errors.length) {

diff --git a/yarn.lock b/yarn.lock
@@ -16536,6 +16536,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/lodash.differencewith@npm:^4.5.9":
+  version: 4.5.9
+  resolution: "@types/lodash.differencewith@npm:4.5.9"
+  dependencies:
+    "@types/lodash": "npm:*"
+  checksum: e2433f6daf390c9c09e32532dbd434f133c6ba9e10248d4655ad6c20208d29e620d467fd8a69efe1c66ae15854fca94a4ae36f5ebe0caeb5355272f71b28639f
+  languageName: node
+  linkType: hard
+
 "@types/lodash.groupby@npm:^4.6.9":
   version: 4.6.9
   resolution: "@types/lodash.groupby@npm:4.6.9"
@@ -33330,6 +33339,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"lodash.differencewith@npm:^4.5.0":
+  version: 4.5.0
+  resolution: "lodash.differencewith@npm:4.5.0"
+  checksum: 685a51f5e78aa4bb8ff1a22f144b9729e9189ee44d498bd94ab6061885a33efb906436f7e828b50ec91f3b86bd9f35e240da526ec062eac8f7edb42b989c15ed
+  languageName: node
+  linkType: hard
+
 "lodash.filter@npm:^4.6.0":
   version: 4.6.0
   resolution: "lodash.filter@npm:4.6.0"
@@ -45748,6 +45764,7 @@ __metadata:
     "@nestjs/graphql": "patch:@nestjs/[email protected]#./patches/@nestjs+graphql+12.1.1.patch"
     "@nx/js": "npm:17.2.8"
     "@ptc-org/nestjs-query-graphql": "patch:@ptc-org/[email protected]#./patches/@ptc-org+nestjs-query-graphql+4.2.0.patch"
+    "@types/lodash.differencewith": "npm:^4.5.9"
     "@types/lodash.isempty": "npm:^4.4.7"
     "@types/lodash.isequal": "npm:^4.5.8"
     "@types/lodash.isobject": "npm:^3.0.7"
@@ -45760,6 +45777,7 @@ __metadata:
     class-validator: "patch:[email protected]#./patches/class-validator+0.14.0.patch"
     graphql-middleware: "npm:^6.1.35"
     jwt-decode: "npm:^4.0.0"
+    lodash.differencewith: "npm:^4.5.0"
     passport: "npm:^0.7.0"
     psl: "npm:^1.9.0"
     rimraf: "npm:^5.0.5"
-Original file line number
+Diff line change
@@ Expand Up / @@ -211,6 +211,8 @@ export class GmailFullSyncService { @@
           this.logger.log(
             `gmail full-sync for workspace ${workspaceId} and account ${connectedAccountId} done with nothing to import.`,
           );
+          return;
         }
         if (errors.length) {
@@ Expand Down @@