Merge pull request #1457 from bcgov/fix/edx-3083

EDX-3083: Delete completed sagas

api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaEventRepository.java

@@ -4,12 +4,7 @@
 import ca.bc.gov.educ.studentdatacollection.api.model.v1.SagaEventStatesEntity;
 import ca.bc.gov.educ.studentdatacollection.api.model.v1.SdcSagaEntity;
 import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.data.jpa.repository.Modifying;
-import org.springframework.data.jpa.repository.Query;
 import org.springframework.stereotype.Repository;
-import org.springframework.transaction.annotation.Transactional;
-
-import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
@@ -38,9 +33,4 @@ public interface SagaEventRepository extends JpaRepository<SagaEventStatesEntity
    */
   Optional<SagaEventStatesEntity> findBySagaAndSagaEventOutcomeAndSagaEventStateAndSagaStepNumber(SdcSagaEntity saga, String eventOutcome, String eventState, int stepNumber);
 
-  @Transactional
-  @Modifying
-  @Query(value = "delete from SDC_SAGA_EVENT_STATES e where e.SAGA_ID in (select s.SAGA_ID from SDC_SAGA s where s.STATUS in :cleanupStatus and s.CREATE_DATE <= :createDate and s.SAGA_ID in :sagaIDsToDelete)", nativeQuery = true)
-  void deleteByStatusAndCreateDateBefore(List<String> cleanupStatus, LocalDateTime createDate, List<UUID> sagaIDsToDelete);
-
 }

api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaRepository.java

@@ -9,7 +9,6 @@
 import org.springframework.stereotype.Repository;
 import org.springframework.transaction.annotation.Transactional;
 
-import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
@@ -30,15 +29,13 @@ public interface SagaRepository extends JpaRepository<SdcSagaEntity, UUID>, JpaS
 
   long countAllByStatusIn(List<String> statuses);
 
-  long countAllByStatusInAndCreateDateBefore(List<String> statuses, LocalDateTime createDate);
-
-  @Query(value = "SELECT s.SAGA_ID FROM SDC_SAGA s WHERE s.STATUS in :cleanupStatus and s.CREATE_DATE <= :createDate LIMIT :batchSize", nativeQuery = true)
-  List<UUID> findByStatusInAndCreateDateBefore(List<String> cleanupStatus, LocalDateTime createDate, int batchSize);
+  @Query(value = "SELECT s.SAGA_ID FROM SDC_SAGA s WHERE s.STATUS in :cleanupStatus LIMIT :batchSize", nativeQuery = true)
+  List<UUID> findByStatusIn(List<String> cleanupStatus, int batchSize);
 
   @Transactional
   @Modifying
-  @Query(value = "DELETE FROM SDC_SAGA saga where saga.STATUS in :cleanupStatus and saga.CREATE_DATE <= :createDate AND saga.SAGA_ID in :sagaIDsToDelete", nativeQuery = true)
-  void deleteByStatusAndCreateDateBefore(List<String> cleanupStatus, LocalDateTime createDate, List<UUID> sagaIDsToDelete);
+  @Query(value = "DELETE FROM SDC_SAGA saga where saga.SAGA_ID in :sagaIDsToDelete", nativeQuery = true)
+  void deleteBySagaIdIn(List<UUID> sagaIDsToDelete);
 
   @Transactional
   @Modifying

api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java

@@ -2,18 +2,15 @@
 
 import ca.bc.gov.educ.studentdatacollection.api.repository.v1.SagaEventRepository;
 import ca.bc.gov.educ.studentdatacollection.api.repository.v1.SagaRepository;
+import ca.bc.gov.educ.studentdatacollection.api.service.v1.events.schedulers.EventTaskSchedulerAsyncService;
 import jakarta.transaction.Transactional;
 import lombok.Getter;
-import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import net.javacrumbs.shedlock.core.LockAssert;
 import net.javacrumbs.shedlock.spring.annotation.SchedulerLock;
-import org.apache.commons.collections4.CollectionUtils;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
-import java.time.LocalDateTime;
 import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
@@ -28,17 +25,14 @@ public class PurgeOldSagaRecordsScheduler {
 
   @Getter(PRIVATE)
   private final SagaEventRepository sagaEventRepository;
+  private final EventTaskSchedulerAsyncService taskSchedulerAsyncService;
 
-  @Value("${purge.records.saga.after.days}")
-  @Setter
-  @Getter
-  Integer sagaRecordStaleInDays;
+  private static final int BATCH_SIZE = 10000;
 
-  private static final int BATCHSIZE = 10000;
-
-  public PurgeOldSagaRecordsScheduler(final SagaRepository sagaRepository, final SagaEventRepository sagaEventRepository) {
+  public PurgeOldSagaRecordsScheduler(final SagaRepository sagaRepository, final SagaEventRepository sagaEventRepository, EventTaskSchedulerAsyncService taskSchedulerAsyncService) {
     this.sagaRepository = sagaRepository;
     this.sagaEventRepository = sagaEventRepository;
+    this.taskSchedulerAsyncService = taskSchedulerAsyncService;
   }
 
 
@@ -47,24 +41,15 @@ public PurgeOldSagaRecordsScheduler(final SagaRepository sagaRepository, final S
    */
   @Scheduled(cron = "${scheduled.jobs.purge.old.saga.records.cron}")
   @SchedulerLock(name = "PurgeOldSagaRecordsLock", lockAtLeastFor = "${scheduled.jobs.purge.old.saga.records.cron.lockAtLeastFor}", lockAtMostFor = "${scheduled.jobs.purge.old.saga.records.cron.lockAtMostFor}")
-
   @Transactional
   public void pollSagaTableAndPurgeOldRecords() {
     LockAssert.assertLocked();
-    final LocalDateTime createDateToCompare = this.calculateCreateDateBasedOnStaleSagaRecordInDays();
     final List<String> cleanupStatus = Collections.singletonList("COMPLETED");
-    final long cleanupRecordsCount = this.sagaRepository.countAllByStatusInAndCreateDateBefore(cleanupStatus, createDateToCompare);
-    for (int i = 0; i < cleanupRecordsCount; i += BATCHSIZE) {
-        List<UUID> sagaIDsToDelete = this.sagaRepository.findByStatusInAndCreateDateBefore(cleanupStatus, createDateToCompare, BATCHSIZE);
-        if (CollectionUtils.isNotEmpty(sagaIDsToDelete)) {
-          this.sagaEventRepository.deleteByStatusAndCreateDateBefore(cleanupStatus, createDateToCompare, sagaIDsToDelete);
-          this.sagaRepository.deleteByStatusAndCreateDateBefore(cleanupStatus, createDateToCompare, sagaIDsToDelete);
-        }
+    final long cleanupRecordsCount = this.sagaRepository.countAllByStatusIn(cleanupStatus);
+    for (int i = 0; i < cleanupRecordsCount; i += BATCH_SIZE) {
+      List<UUID> sagaIDsToDelete = this.sagaRepository.findByStatusIn(cleanupStatus, BATCH_SIZE);
+      taskSchedulerAsyncService.deleteCompletedSagas(sagaIDsToDelete);
      }
     log.info("Purged old saga and event records from EDUC-STUDENT-DATA-COLLECTION-SAGA-API");
   }
-
-  private LocalDateTime calculateCreateDateBasedOnStaleSagaRecordInDays() {
-    return LocalDateTime.now().minusDays(this.getSagaRecordStaleInDays());
-  }
 }

api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/service/v1/events/schedulers/EventTaskSchedulerAsyncService.java

@@ -262,6 +262,11 @@ public void findModifiedSchoolsAndUpdateSdcSchoolCollection() {
     }
   }
 
+  @Transactional(propagation = Propagation.REQUIRES_NEW)
+  public void deleteCompletedSagas(List<UUID> sagaIDsToDelete) {
+    this.sagaRepository.deleteBySagaIdIn(sagaIDsToDelete);
+  }
+
   private List<SdcSchoolCollectionEntity> findAddSchoolsAndUpdateSdcSchoolCollection(List<SchoolTombstone> schoolTombstones, CollectionEntity activeCollection, List<SdcSchoolCollectionEntity> activeSchoolCollections) {
     Set<UUID> existingSchoolIds = activeSchoolCollections.stream()
             .map(SdcSchoolCollectionEntity::getSchoolID).collect(Collectors.toSet());

api/src/main/resources/application.properties

@@ -112,7 +112,6 @@ spring.datasource.hikari.maximum-pool-size=${MAXIMUM_DB_POOL_SIZE}
 spring.datasource.hikari.minimum-idle=${MINIMUM_IDLE_DB_POOL_SIZE}
 
 number.students.process.saga=${NUMBER_OF_STUDENTS_TO_PROCESS_SAGA}
-purge.records.saga.after.days=${PURGE_RECORDS_SAGA_AFTER_DAYS}
 scheduled.jobs.purge.old.saga.records.cron=${SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON}
 scheduled.jobs.purge.old.saga.records.cron.lockAtLeastFor=${SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_LEAST_FOR}
 scheduled.jobs.purge.old.saga.records.cron.lockAtMostFor=${SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_MOST_FOR}

api/src/test/resources/application.properties

@@ -80,7 +80,6 @@ scheduled.jobs.delete.migrated.sdc.students.cron.lockAtLeastFor=PT2M
 scheduled.jobs.delete.migrated.sdc.students.cron.lockAtMostFor=PT2M
 
 school.notification.email.from=abc
-purge.records.saga.after.days=365
 scheduled.jobs.purge.old.saga.records.cron=-
 ches.endpoint.url=https://test
 ches.client.id=ABC