From 15a06873fa47acd039181bbfd1ef92378a69aa9f Mon Sep 17 00:00:00 2001 From: Avisha Sodhi <38086281+SodhiA1@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:54:15 -0800 Subject: [PATCH 1/4] EDX-3083: Delete completed sagas --- .../repository/v1/SagaEventRepository.java | 5 ---- .../api/repository/v1/SagaRepository.java | 10 ++++---- .../PurgeOldSagaRecordsScheduler.java | 24 ++++--------------- api/src/main/resources/application.properties | 1 - api/src/test/resources/application.properties | 1 - tools/config/update-configmap.sh | 5 +--- 6 files changed, 10 insertions(+), 36 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaEventRepository.java b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaEventRepository.java index b7fa9e718..e89d72bde 100644 --- a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaEventRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaEventRepository.java @@ -38,9 +38,4 @@ public interface SagaEventRepository extends JpaRepository findBySagaAndSagaEventOutcomeAndSagaEventStateAndSagaStepNumber(SdcSagaEntity saga, String eventOutcome, String eventState, int stepNumber); - @Transactional - @Modifying - @Query(value = "delete from SDC_SAGA_EVENT_STATES e where e.SAGA_ID in (select s.SAGA_ID from SDC_SAGA s where s.STATUS in :cleanupStatus and s.CREATE_DATE <= :createDate and s.SAGA_ID in :sagaIDsToDelete)", nativeQuery = true) - void deleteByStatusAndCreateDateBefore(List cleanupStatus, LocalDateTime createDate, List sagaIDsToDelete); - } diff --git a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaRepository.java b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaRepository.java index 62fe94b51..b0eed1e06 100644 --- a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaRepository.java @@ -30,15 +30,13 @@ public interface SagaRepository extends JpaRepository, JpaS long countAllByStatusIn(List statuses); - long countAllByStatusInAndCreateDateBefore(List statuses, LocalDateTime createDate); - - @Query(value = "SELECT s.SAGA_ID FROM SDC_SAGA s WHERE s.STATUS in :cleanupStatus and s.CREATE_DATE <= :createDate LIMIT :batchSize", nativeQuery = true) - List findByStatusInAndCreateDateBefore(List cleanupStatus, LocalDateTime createDate, int batchSize); + @Query(value = "SELECT s.SAGA_ID FROM SDC_SAGA s WHERE s.STATUS in :cleanupStatus LIMIT :batchSize", nativeQuery = true) + List findByStatusIn(List cleanupStatus, int batchSize); @Transactional @Modifying - @Query(value = "DELETE FROM SDC_SAGA saga where saga.STATUS in :cleanupStatus and saga.CREATE_DATE <= :createDate AND saga.SAGA_ID in :sagaIDsToDelete", nativeQuery = true) - void deleteByStatusAndCreateDateBefore(List cleanupStatus, LocalDateTime createDate, List sagaIDsToDelete); + @Query(value = "DELETE FROM SDC_SAGA saga where saga.SAGA_ID in :sagaIDsToDelete", nativeQuery = true) + void deleteBySagaIdIn(List sagaIDsToDelete); @Transactional @Modifying diff --git a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java index 5fd1aa7bc..4d34813b0 100644 --- a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java +++ b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java @@ -29,12 +29,7 @@ public class PurgeOldSagaRecordsScheduler { @Getter(PRIVATE) private final SagaEventRepository sagaEventRepository; - @Value("${purge.records.saga.after.days}") - @Setter - @Getter - Integer sagaRecordStaleInDays; - - private static final int BATCHSIZE = 10000; + private static final int BATCH_SIZE = 10000; public PurgeOldSagaRecordsScheduler(final SagaRepository sagaRepository, final SagaEventRepository sagaEventRepository) { this.sagaRepository = sagaRepository; @@ -47,24 +42,15 @@ public PurgeOldSagaRecordsScheduler(final SagaRepository sagaRepository, final S */ @Scheduled(cron = "${scheduled.jobs.purge.old.saga.records.cron}") @SchedulerLock(name = "PurgeOldSagaRecordsLock", lockAtLeastFor = "${scheduled.jobs.purge.old.saga.records.cron.lockAtLeastFor}", lockAtMostFor = "${scheduled.jobs.purge.old.saga.records.cron.lockAtMostFor}") - @Transactional public void pollSagaTableAndPurgeOldRecords() { LockAssert.assertLocked(); - final LocalDateTime createDateToCompare = this.calculateCreateDateBasedOnStaleSagaRecordInDays(); final List cleanupStatus = Collections.singletonList("COMPLETED"); - final long cleanupRecordsCount = this.sagaRepository.countAllByStatusInAndCreateDateBefore(cleanupStatus, createDateToCompare); - for (int i = 0; i < cleanupRecordsCount; i += BATCHSIZE) { - List sagaIDsToDelete = this.sagaRepository.findByStatusInAndCreateDateBefore(cleanupStatus, createDateToCompare, BATCHSIZE); - if (CollectionUtils.isNotEmpty(sagaIDsToDelete)) { - this.sagaEventRepository.deleteByStatusAndCreateDateBefore(cleanupStatus, createDateToCompare, sagaIDsToDelete); - this.sagaRepository.deleteByStatusAndCreateDateBefore(cleanupStatus, createDateToCompare, sagaIDsToDelete); - } + final long cleanupRecordsCount = this.sagaRepository.countAllByStatusIn(cleanupStatus); + for (int i = 0; i < cleanupRecordsCount; i += BATCH_SIZE) { + List sagaIDsToDelete = this.sagaRepository.findByStatusIn(cleanupStatus, BATCH_SIZE); + this.sagaRepository.deleteBySagaIdIn(sagaIDsToDelete); } log.info("Purged old saga and event records from EDUC-STUDENT-DATA-COLLECTION-SAGA-API"); } - - private LocalDateTime calculateCreateDateBasedOnStaleSagaRecordInDays() { - return LocalDateTime.now().minusDays(this.getSagaRecordStaleInDays()); - } } diff --git a/api/src/main/resources/application.properties b/api/src/main/resources/application.properties index a826d47a3..460c546ac 100644 --- a/api/src/main/resources/application.properties +++ b/api/src/main/resources/application.properties @@ -112,7 +112,6 @@ spring.datasource.hikari.maximum-pool-size=${MAXIMUM_DB_POOL_SIZE} spring.datasource.hikari.minimum-idle=${MINIMUM_IDLE_DB_POOL_SIZE} number.students.process.saga=${NUMBER_OF_STUDENTS_TO_PROCESS_SAGA} -purge.records.saga.after.days=${PURGE_RECORDS_SAGA_AFTER_DAYS} scheduled.jobs.purge.old.saga.records.cron=${SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON} scheduled.jobs.purge.old.saga.records.cron.lockAtLeastFor=${SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_LEAST_FOR} scheduled.jobs.purge.old.saga.records.cron.lockAtMostFor=${SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_MOST_FOR} diff --git a/api/src/test/resources/application.properties b/api/src/test/resources/application.properties index 0d98d1d61..27d11d0c0 100644 --- a/api/src/test/resources/application.properties +++ b/api/src/test/resources/application.properties @@ -80,7 +80,6 @@ scheduled.jobs.delete.migrated.sdc.students.cron.lockAtLeastFor=PT2M scheduled.jobs.delete.migrated.sdc.students.cron.lockAtMostFor=PT2M school.notification.email.from=abc -purge.records.saga.after.days=365 scheduled.jobs.purge.old.saga.records.cron=- ches.endpoint.url=https://test ches.client.id=ABC diff --git a/tools/config/update-configmap.sh b/tools/config/update-configmap.sh index fad846540..e738caeaf 100644 --- a/tools/config/update-configmap.sh +++ b/tools/config/update-configmap.sh @@ -270,7 +270,6 @@ MAXIMUM_DB_POOL_SIZE=25 MINIMUM_IDLE_DB_POOL_SIZE=15 NUMBER_OF_STUDENTS_TO_PROCESS_SAGA=500 NUMBER_OF_SCHOOL_COLL_TO_PROCESS_SAGA=10 -PURGE_RECORDS_SAGA_AFTER_DAYS=400 EMAIL_SUBJECT_CLOSURE_NOTIFICATION="1701 Closure Reports" EMAIL_TEMPLATE_CLOSURE_REPORT_NOTIFICATION="1701 Closure ReportsReports are now available in EDX for the Student Level Data Collection.

To view the reports, and open \"Data Collections\" in the menu. Reports can be found in each of the collections log into the Student-Admin platform listed.

The Data Management Unit Team
Ministry of Education and Child Care
educationdataexchange@gov.bc.ca" @@ -280,17 +279,15 @@ if [ "$envValue" = "dev" ] then SCHOOL_NOTIFICATION_EMAIL_FROM="dev.educationdataexchange@gov.bc.ca" EMAIL_CLOSURE_NOTIFICATION_TO="avisha.1.sodhi@gov.bc.ca" - PURGE_RECORDS_SAGA_AFTER_DAYS=2 elif [ "$envValue" = "test" ] then SCHOOL_NOTIFICATION_EMAIL_FROM="test.educationdataexchange@gov.bc.ca" EMAIL_CLOSURE_NOTIFICATION_TO="avisha.1.sodhi@gov.bc.ca" - PURGE_RECORDS_SAGA_AFTER_DAYS=3 fi echo echo Creating config map "$APP_NAME"-config-map -oc create -n "$OPENSHIFT_NAMESPACE"-"$envValue" configmap "$APP_NAME"-config-map --from-literal=TZ=$TZVALUE --from-literal=STUDENT_API_URL="http://student-api-master.$COMMON_NAMESPACE-$envValue.svc.cluster.local:8080/api/v1/student" --from-literal=EDX_API_URL="http://edx-api-master.$OPENSHIFT_NAMESPACE-$envValue.svc.cluster.local:8080/api/v1/edx" --from-literal=SCHOOL_NOTIFICATION_EMAIL_FROM=$SCHOOL_NOTIFICATION_EMAIL_FROM --from-literal=SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON="$SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON" --from-literal=SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON="$SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON" --from-literal=EMAIL_TEMPLATE_COLLECTION_INDEPENDENT_SCHOOL_NOTSUBMITTED_NOTIFICATION="$EMAIL_TEMPLATE_COLLECTION_INDEPENDENT_SCHOOL_NOTSUBMITTED_NOTIFICATION" --from-literal=EMAIL_TEMPLATE_COLLECTION_PROVINCIAL_DUPLICATES_NOTIFICATION="$EMAIL_TEMPLATE_COLLECTION_PROVINCIAL_DUPLICATES_NOTIFICATION" --from-literal=EMAIL_SUBJECT_COLLECTION_PROVINCIAL_DUPLICATES_NOTIFICATION="$EMAIL_SUBJECT_COLLECTION_PROVINCIAL_DUPLICATES_NOTIFICATION" --from-literal=EMAIL_SUBJECT_COLLECTION_INDEPENDENT_SCHOOL_NOTSUBMITTED_NOTIFICATION="$EMAIL_SUBJECT_COLLECTION_INDEPENDENT_SCHOOL_NOTSUBMITTED_NOTIFICATION" --from-literal=EMAIL_TEMPLATE_COLLECTION_INDEPENDENT_SCHOOL_NOACTIVITY_NOTIFICATION="$EMAIL_TEMPLATE_COLLECTION_INDEPENDENT_SCHOOL_NOACTIVITY_NOTIFICATION" --from-literal=EMAIL_SUBJECT_COLLECTION_INDEPENDENT_SCHOOL_NOACTIVITY_NOTIFICATION="$EMAIL_SUBJECT_COLLECTION_INDEPENDENT_SCHOOL_NOACTIVITY_NOTIFICATION" --from-literal=CHES_CLIENT_ID="$CHES_CLIENT_ID" --from-literal=CHES_CLIENT_SECRET="$CHES_CLIENT_SECRET" --from-literal=CHES_TOKEN_URL="$CHES_TOKEN_URL" --from-literal=CHES_ENDPOINT_URL="$CHES_ENDPOINT_URL" --from-literal=JDBC_URL="$DB_JDBC_CONNECT_STRING" --from-literal=PURGE_RECORDS_SAGA_AFTER_DAYS="$PURGE_RECORDS_SAGA_AFTER_DAYS" --from-literal=SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON="$SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON" --from-literal=SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_MOST_FOR" --from-literal=DB_USERNAME="$DB_USER" --from-literal=DB_PASSWORD="$DB_PWD" --from-literal=KEYCLOAK_PUBLIC_KEY="$soamFullPublicKey" --from-literal=SPRING_SECURITY_LOG_LEVEL=INFO --from-literal=SPRING_WEB_LOG_LEVEL=INFO --from-literal=APP_LOG_LEVEL=INFO --from-literal=SPRING_BOOT_AUTOCONFIG_LOG_LEVEL=INFO --from-literal=SPRING_SHOW_REQUEST_DETAILS=false --from-literal=SPRING_JPA_SHOW_SQL="false" --from-literal=TOKEN_ISSUER_URL="https://$SOAM_KC/auth/realms/$SOAM_KC_REALM_ID" --from-literal=NATS_MAX_RECONNECT=60 --from-literal=NATS_URL=$NATS_URL --from-literal=CLIENT_ID="student-data-collection-api-service" --from-literal=CLIENT_SECRET="$SDC_APIServiceClientSecret" --from-literal=THREADS_MIN_SUBSCRIBER="$THREADS_MIN_SUBSCRIBER" --from-literal=THREADS_MAX_SUBSCRIBER="$THREADS_MAX_SUBSCRIBER" --from-literal=SAGAS_MAX_PENDING="$SAGAS_MAX_PENDING" --from-literal=SAGAS_MAX_PARALLEL="$SAGAS_MAX_PARALLEL" --from-literal=TOKEN_URL="https://$SOAM_KC/auth/realms/$SOAM_KC_REALM_ID/protocol/openid-connect/token" --from-literal=SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON="$SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON" --from-literal=SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON="$SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON" --from-literal=SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON="$SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON" --from-literal=SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON_LOCK_AT_MOST_FOR" --from-literal=INSTITUTE_API_URL="http://institute-api-master.$COMMON_NAMESPACE-$envValue.svc.cluster.local:8080/api/v1/institute" --from-literal=MAXIMUM_DB_POOL_SIZE="$MAXIMUM_DB_POOL_SIZE" --from-literal=MINIMUM_IDLE_DB_POOL_SIZE="$MINIMUM_IDLE_DB_POOL_SIZE" --from-literal=NUMBER_OF_STUDENTS_TO_PROCESS_SAGA="$NUMBER_OF_STUDENTS_TO_PROCESS_SAGA" --from-literal=NUMBER_OF_SCHOOL_COLL_TO_PROCESS_SAGA="$NUMBER_OF_SCHOOL_COLL_TO_PROCESS_SAGA" --from-literal=SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON="$SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON" --from-literal=SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON="$SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON" --from-literal=SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON_LOCK_AT_LEAST_FOR" --from-literal=EMAIL_SUBJECT_CLOSURE_NOTIFICATION="$EMAIL_SUBJECT_CLOSURE_NOTIFICATION" --from-literal=EMAIL_TEMPLATE_CLOSURE_REPORT_NOTIFICATION="$EMAIL_TEMPLATE_CLOSURE_REPORT_NOTIFICATION" --from-literal=EMAIL_CLOSURE_NOTIFICATION_TO="$EMAIL_CLOSURE_NOTIFICATION_TO" --from-literal=SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON="$SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON" --from-literal=SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON_LOCK_AT_MOST_FOR" --dry-run -o yaml | oc apply -f - +oc create -n "$OPENSHIFT_NAMESPACE"-"$envValue" configmap "$APP_NAME"-config-map --from-literal=TZ=$TZVALUE --from-literal=STUDENT_API_URL="http://student-api-master.$COMMON_NAMESPACE-$envValue.svc.cluster.local:8080/api/v1/student" --from-literal=EDX_API_URL="http://edx-api-master.$OPENSHIFT_NAMESPACE-$envValue.svc.cluster.local:8080/api/v1/edx" --from-literal=SCHOOL_NOTIFICATION_EMAIL_FROM=$SCHOOL_NOTIFICATION_EMAIL_FROM --from-literal=SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON="$SCHEDULED_JOBS_TARDY_INDY_SCHOOLS_CRON" --from-literal=SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON="$SCHEDULED_JOBS_ADD_SCHOOLS_COLLECTION_CRON" --from-literal=EMAIL_TEMPLATE_COLLECTION_INDEPENDENT_SCHOOL_NOTSUBMITTED_NOTIFICATION="$EMAIL_TEMPLATE_COLLECTION_INDEPENDENT_SCHOOL_NOTSUBMITTED_NOTIFICATION" --from-literal=EMAIL_TEMPLATE_COLLECTION_PROVINCIAL_DUPLICATES_NOTIFICATION="$EMAIL_TEMPLATE_COLLECTION_PROVINCIAL_DUPLICATES_NOTIFICATION" --from-literal=EMAIL_SUBJECT_COLLECTION_PROVINCIAL_DUPLICATES_NOTIFICATION="$EMAIL_SUBJECT_COLLECTION_PROVINCIAL_DUPLICATES_NOTIFICATION" --from-literal=EMAIL_SUBJECT_COLLECTION_INDEPENDENT_SCHOOL_NOTSUBMITTED_NOTIFICATION="$EMAIL_SUBJECT_COLLECTION_INDEPENDENT_SCHOOL_NOTSUBMITTED_NOTIFICATION" --from-literal=EMAIL_TEMPLATE_COLLECTION_INDEPENDENT_SCHOOL_NOACTIVITY_NOTIFICATION="$EMAIL_TEMPLATE_COLLECTION_INDEPENDENT_SCHOOL_NOACTIVITY_NOTIFICATION" --from-literal=EMAIL_SUBJECT_COLLECTION_INDEPENDENT_SCHOOL_NOACTIVITY_NOTIFICATION="$EMAIL_SUBJECT_COLLECTION_INDEPENDENT_SCHOOL_NOACTIVITY_NOTIFICATION" --from-literal=CHES_CLIENT_ID="$CHES_CLIENT_ID" --from-literal=CHES_CLIENT_SECRET="$CHES_CLIENT_SECRET" --from-literal=CHES_TOKEN_URL="$CHES_TOKEN_URL" --from-literal=CHES_ENDPOINT_URL="$CHES_ENDPOINT_URL" --from-literal=JDBC_URL="$DB_JDBC_CONNECT_STRING" --from-literal=SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON="$SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON" --from-literal=SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_PURGE_OLD_SAGA_RECORDS_CRON_LOCK_AT_MOST_FOR" --from-literal=DB_USERNAME="$DB_USER" --from-literal=DB_PASSWORD="$DB_PWD" --from-literal=KEYCLOAK_PUBLIC_KEY="$soamFullPublicKey" --from-literal=SPRING_SECURITY_LOG_LEVEL=INFO --from-literal=SPRING_WEB_LOG_LEVEL=INFO --from-literal=APP_LOG_LEVEL=INFO --from-literal=SPRING_BOOT_AUTOCONFIG_LOG_LEVEL=INFO --from-literal=SPRING_SHOW_REQUEST_DETAILS=false --from-literal=SPRING_JPA_SHOW_SQL="false" --from-literal=TOKEN_ISSUER_URL="https://$SOAM_KC/auth/realms/$SOAM_KC_REALM_ID" --from-literal=NATS_MAX_RECONNECT=60 --from-literal=NATS_URL=$NATS_URL --from-literal=CLIENT_ID="student-data-collection-api-service" --from-literal=CLIENT_SECRET="$SDC_APIServiceClientSecret" --from-literal=THREADS_MIN_SUBSCRIBER="$THREADS_MIN_SUBSCRIBER" --from-literal=THREADS_MAX_SUBSCRIBER="$THREADS_MAX_SUBSCRIBER" --from-literal=SAGAS_MAX_PENDING="$SAGAS_MAX_PENDING" --from-literal=SAGAS_MAX_PARALLEL="$SAGAS_MAX_PARALLEL" --from-literal=TOKEN_URL="https://$SOAM_KC/auth/realms/$SOAM_KC_REALM_ID/protocol/openid-connect/token" --from-literal=SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON="$SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON" --from-literal=SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON="$SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON" --from-literal=SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_EXTRACT_UNCOMPLETED_SAGAS_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_PROCESS_LOADED_SDC_STUDENTS_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON="$SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON" --from-literal=SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_PROCESS_MIGRATED_SDC_STUDENTS_CRON_LOCK_AT_MOST_FOR" --from-literal=INSTITUTE_API_URL="http://institute-api-master.$COMMON_NAMESPACE-$envValue.svc.cluster.local:8080/api/v1/institute" --from-literal=MAXIMUM_DB_POOL_SIZE="$MAXIMUM_DB_POOL_SIZE" --from-literal=MINIMUM_IDLE_DB_POOL_SIZE="$MINIMUM_IDLE_DB_POOL_SIZE" --from-literal=NUMBER_OF_STUDENTS_TO_PROCESS_SAGA="$NUMBER_OF_STUDENTS_TO_PROCESS_SAGA" --from-literal=NUMBER_OF_SCHOOL_COLL_TO_PROCESS_SAGA="$NUMBER_OF_SCHOOL_COLL_TO_PROCESS_SAGA" --from-literal=SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON="$SCHEDULED_JOBS_PROCESS_SCHOOL_COLLECTION_FOR_SUBMISSION_CRON" --from-literal=SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON="$SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON" --from-literal=SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON_LOCK_AT_LEAST_FOR" --from-literal=EMAIL_SUBJECT_CLOSURE_NOTIFICATION="$EMAIL_SUBJECT_CLOSURE_NOTIFICATION" --from-literal=EMAIL_TEMPLATE_CLOSURE_REPORT_NOTIFICATION="$EMAIL_TEMPLATE_CLOSURE_REPORT_NOTIFICATION" --from-literal=EMAIL_CLOSURE_NOTIFICATION_TO="$EMAIL_CLOSURE_NOTIFICATION_TO" --from-literal=SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_PROCESS_UPDATE_STUDENT_DOWNSTREAM_CRON_LOCK_AT_MOST_FOR" --from-literal=SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON="$SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON" --from-literal=SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON_LOCK_AT_LEAST_FOR="$SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON_LOCK_AT_LEAST_FOR" --from-literal=SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON_LOCK_AT_MOST_FOR="$SCHEDULED_JOBS_DELETE_MIGRATED_SDC_STUDENT_SAGAS_CRON_LOCK_AT_MOST_FOR" --dry-run -o yaml | oc apply -f - echo echo Setting environment variables for $APP_NAME-$SOAM_KC_REALM_ID application From 2654be5d40808267cd479a3547c22e0b4247089d Mon Sep 17 00:00:00 2001 From: Avisha Sodhi <38086281+SodhiA1@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:54:45 -0800 Subject: [PATCH 2/4] Update PurgeOldSagaRecordsScheduler.java --- .../api/schedulers/PurgeOldSagaRecordsScheduler.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java index 4d34813b0..b78f6ae38 100644 --- a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java +++ b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java @@ -4,16 +4,12 @@ import ca.bc.gov.educ.studentdatacollection.api.repository.v1.SagaRepository; import jakarta.transaction.Transactional; import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import net.javacrumbs.shedlock.core.LockAssert; import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; -import org.apache.commons.collections4.CollectionUtils; -import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import java.time.LocalDateTime; import java.util.Collections; import java.util.List; import java.util.UUID; From e86720d97ee768c975f5a3f4f7453e7fb84f21b7 Mon Sep 17 00:00:00 2001 From: Avisha Sodhi <38086281+SodhiA1@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:59:31 -0800 Subject: [PATCH 3/4] sonar fixes --- .../api/repository/v1/SagaEventRepository.java | 5 ----- .../api/repository/v1/SagaRepository.java | 1 - 2 files changed, 6 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaEventRepository.java b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaEventRepository.java index e89d72bde..5bf2cfe01 100644 --- a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaEventRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaEventRepository.java @@ -4,12 +4,7 @@ import ca.bc.gov.educ.studentdatacollection.api.model.v1.SagaEventStatesEntity; import ca.bc.gov.educ.studentdatacollection.api.model.v1.SdcSagaEntity; import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; -import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; - -import java.time.LocalDateTime; import java.util.List; import java.util.Optional; import java.util.UUID; diff --git a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaRepository.java b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaRepository.java index b0eed1e06..1dd290aa2 100644 --- a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaRepository.java +++ b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/repository/v1/SagaRepository.java @@ -9,7 +9,6 @@ import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; -import java.time.LocalDateTime; import java.util.List; import java.util.Optional; import java.util.UUID; From d7d7a8a439603dcdd281bf24efcaac8c80ae0c0b Mon Sep 17 00:00:00 2001 From: Avisha Sodhi <38086281+SodhiA1@users.noreply.github.com> Date: Wed, 8 Jan 2025 12:41:32 -0800 Subject: [PATCH 4/4] update to transaction --- .../api/schedulers/PurgeOldSagaRecordsScheduler.java | 9 ++++++--- .../schedulers/EventTaskSchedulerAsyncService.java | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java index b78f6ae38..adfa9397b 100644 --- a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java +++ b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/schedulers/PurgeOldSagaRecordsScheduler.java @@ -2,6 +2,7 @@ import ca.bc.gov.educ.studentdatacollection.api.repository.v1.SagaEventRepository; import ca.bc.gov.educ.studentdatacollection.api.repository.v1.SagaRepository; +import ca.bc.gov.educ.studentdatacollection.api.service.v1.events.schedulers.EventTaskSchedulerAsyncService; import jakarta.transaction.Transactional; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -24,12 +25,14 @@ public class PurgeOldSagaRecordsScheduler { @Getter(PRIVATE) private final SagaEventRepository sagaEventRepository; + private final EventTaskSchedulerAsyncService taskSchedulerAsyncService; private static final int BATCH_SIZE = 10000; - public PurgeOldSagaRecordsScheduler(final SagaRepository sagaRepository, final SagaEventRepository sagaEventRepository) { + public PurgeOldSagaRecordsScheduler(final SagaRepository sagaRepository, final SagaEventRepository sagaEventRepository, EventTaskSchedulerAsyncService taskSchedulerAsyncService) { this.sagaRepository = sagaRepository; this.sagaEventRepository = sagaEventRepository; + this.taskSchedulerAsyncService = taskSchedulerAsyncService; } @@ -44,8 +47,8 @@ public void pollSagaTableAndPurgeOldRecords() { final List cleanupStatus = Collections.singletonList("COMPLETED"); final long cleanupRecordsCount = this.sagaRepository.countAllByStatusIn(cleanupStatus); for (int i = 0; i < cleanupRecordsCount; i += BATCH_SIZE) { - List sagaIDsToDelete = this.sagaRepository.findByStatusIn(cleanupStatus, BATCH_SIZE); - this.sagaRepository.deleteBySagaIdIn(sagaIDsToDelete); + List sagaIDsToDelete = this.sagaRepository.findByStatusIn(cleanupStatus, BATCH_SIZE); + taskSchedulerAsyncService.deleteCompletedSagas(sagaIDsToDelete); } log.info("Purged old saga and event records from EDUC-STUDENT-DATA-COLLECTION-SAGA-API"); } diff --git a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/service/v1/events/schedulers/EventTaskSchedulerAsyncService.java b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/service/v1/events/schedulers/EventTaskSchedulerAsyncService.java index 1ee0d8da7..32712b530 100644 --- a/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/service/v1/events/schedulers/EventTaskSchedulerAsyncService.java +++ b/api/src/main/java/ca/bc/gov/educ/studentdatacollection/api/service/v1/events/schedulers/EventTaskSchedulerAsyncService.java @@ -262,6 +262,11 @@ public void findModifiedSchoolsAndUpdateSdcSchoolCollection() { } } + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void deleteCompletedSagas(List sagaIDsToDelete) { + this.sagaRepository.deleteBySagaIdIn(sagaIDsToDelete); + } + private List findAddSchoolsAndUpdateSdcSchoolCollection(List schoolTombstones, CollectionEntity activeCollection, List activeSchoolCollections) { Set existingSchoolIds = activeSchoolCollections.stream() .map(SdcSchoolCollectionEntity::getSchoolID).collect(Collectors.toSet());