Merge remote-tracking branch 'upstream/master'

i18n/en.pot

@@ -2259,6 +2259,9 @@ msgstr ""
 msgid "Since last execution"
 msgstr ""
 
+msgid "Since last successful execution"
+msgstr ""
+
 msgid "Today"
 msgstr ""
 

i18n/es.po

@@ -1,7 +1,7 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: i18next-conv\n"
-"POT-Creation-Date: 2024-07-17T07:09:52.268Z\n"
+"POT-Creation-Date: 2024-09-05T09:32:03.371Z\n"
 "PO-Revision-Date: 2020-07-10T06:53:30.625Z\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -2263,6 +2263,9 @@ msgstr ""
 msgid "Since last execution"
 msgstr ""
 
+msgid "Since last successful execution"
+msgstr ""
+
 msgid "Today"
 msgstr ""
 

i18n/fr.po

@@ -1,7 +1,7 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: i18next-conv\n"
-"POT-Creation-Date: 2024-07-17T07:09:52.268Z\n"
+"POT-Creation-Date: 2024-09-05T09:32:03.371Z\n"
 "PO-Revision-Date: 2020-07-10T06:53:30.625Z\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -2262,6 +2262,9 @@ msgstr ""
 msgid "Since last execution"
 msgstr ""
 
+msgid "Since last successful execution"
+msgstr ""
+
 msgid "Today"
 msgstr ""
 

i18n/pt.po

@@ -1,7 +1,7 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: i18next-conv\n"
-"POT-Creation-Date: 2024-07-17T07:09:52.268Z\n"
+"POT-Creation-Date: 2024-09-05T09:32:03.371Z\n"
 "PO-Revision-Date: 2020-07-10T06:53:30.625Z\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -2262,6 +2262,9 @@ msgstr ""
 msgid "Since last execution"
 msgstr ""
 
+msgid "Since last successful execution"
+msgstr ""
+
 msgid "Today"
 msgstr ""
 

package.json

@@ -1,7 +1,7 @@
 {
     "name": "metadata-synchronization",
     "description": "Advanced metadata & data synchronization utility",
-    "version": "2.19.1",
+    "version": "2.20.0",
     "license": "GPL-3.0",
     "author": "EyeSeeTea team",
     "homepage": ".",

src/data/aggregated/models/DataSyncPeriodModel.ts

@@ -5,6 +5,7 @@ export const DataSyncPeriodModel: Codec<DataSyncPeriod> = Schema.oneOf([
     Schema.exact("ALL"),
     Schema.exact("FIXED"),
     Schema.exact("SINCE_LAST_EXECUTED_DATE"),
+    Schema.exact("SINCE_LAST_SUCCESSFUL_SYNC"),
     Schema.exact("TODAY"),
     Schema.exact("YESTERDAY"),
     Schema.exact("LAST_7_DAYS"),

src/data/events/EventsD2ApiRepository.ts

@@ -4,6 +4,7 @@ import moment from "moment";
 import {
     DataImportParams,
     DataSynchronizationParams,
+    isDataSynchronizationRequired,
 } from "../../domain/aggregated/entities/DataSynchronizationParams";
 import { buildPeriodFromParams } from "../../domain/aggregated/utils";
 import { EventsPackage } from "../../domain/events/entities/EventsPackage";
@@ -134,7 +135,10 @@ export class EventsD2ApiRepository implements EventsRepository {
                     programStage,
                     orgUnit,
                     startDate: period !== "ALL" ? startDate.format("YYYY-MM-DD") : undefined,
-                    endDate: period !== "ALL" ? endDate.format("YYYY-MM-DD") : undefined,
+                    endDate:
+                        period !== "ALL" && period !== "SINCE_LAST_SUCCESSFUL_SYNC"
+                            ? endDate.format("YYYY-MM-DD")
+                            : undefined,
                     lastUpdated: lastUpdated ? moment(lastUpdated).toISOString() : undefined,
                     fields: { $all: true },
                 })
@@ -158,8 +162,14 @@ export class EventsD2ApiRepository implements EventsRepository {
 
         return _(result)
             .flatten()
-            .map(object => ({ ...object, id: object.event }))
-            .map(object => cleanObjectDefault(object, defaults))
+            .map(object => {
+                const event = { ...object, id: object.event };
+
+                return isDataSynchronizationRequired(params, object.lastUpdated)
+                    ? cleanObjectDefault(event, defaults)
+                    : undefined;
+            })
+            .compact()
             .value();
     }
 

src/data/rules/models/SynchronizationRuleModel.ts

@@ -28,6 +28,7 @@ export const SynchronizationRuleModel: Codec<SynchronizationRuleData> = Schema.e
         enabled: Schema.optionalSafe(Schema.boolean, false),
         lastExecuted: Schema.optional(Schema.date),
         lastExecutedBy: Schema.optional(NamedRefModel),
+        lastSuccessfulSync: Schema.optional(Schema.date),
         frequency: Schema.optional(Schema.string),
         type: SynchronizationTypeModel,
     })

src/data/tracked-entity-instances/TEID2ApiRepository.ts

@@ -2,6 +2,7 @@ import _ from "lodash";
 import {
     DataImportParams,
     DataSynchronizationParams,
+    isDataSynchronizationRequired,
 } from "../../domain/aggregated/entities/DataSynchronizationParams";
 import { buildPeriodFromParams } from "../../domain/aggregated/utils";
 import { Instance } from "../../domain/instance/entities/Instance";
@@ -42,7 +43,10 @@ export class TEID2ApiRepository implements TEIRepository {
             return [...trackedEntityInstances, ..._.flatten(paginatedTEIs)];
         });
 
-        return result.flat();
+        return _(result)
+            .flatten()
+            .filter(object => isDataSynchronizationRequired(params, object.lastUpdated))
+            .value();
     }
 
     async getTEIs(
@@ -73,7 +77,10 @@ export class TEID2ApiRepository implements TEIRepository {
                 ou: orgUnits.join(";"),
                 fields: this.fields,
                 programStartDate: period !== "ALL" ? startDate.format("YYYY-MM-DD") : undefined,
-                programEndDate: period !== "ALL" ? endDate.format("YYYY-MM-DD") : undefined,
+                programEndDate:
+                    period !== "ALL" && period !== "SINCE_LAST_SUCCESSFUL_SYNC"
+                        ? endDate.format("YYYY-MM-DD")
+                        : undefined,
                 totalPages: true,
                 page,
                 pageSize,

src/domain/aggregated/entities/DataSyncPeriod.ts

@@ -2,6 +2,7 @@ export type DataSyncPeriod =
     | "ALL"
     | "FIXED"
     | "SINCE_LAST_EXECUTED_DATE"
+    | "SINCE_LAST_SUCCESSFUL_SYNC"
     | "TODAY"
     | "YESTERDAY"
     | "LAST_7_DAYS"

src/domain/aggregated/entities/DataSynchronizationParams.ts

@@ -1,3 +1,4 @@
+import { buildPeriodFromParams } from "../utils";
 import { DataSyncAggregation } from "./DataSyncAggregation";
 import { DataSyncPeriod } from "./DataSyncPeriod";
 
@@ -36,3 +37,13 @@ export interface DataSynchronizationParams extends DataImportParams {
     analyticsYears?: number;
     ignoreDuplicateExistingValues?: boolean;
 }
+
+export function isDataSynchronizationRequired(params: DataSynchronizationParams, lastUpdated: string): boolean {
+    const { period } = params;
+    const { startDate } = buildPeriodFromParams(params);
+
+    const isUpdatedAfterStartDate = new Date(lastUpdated).toISOString() >= startDate.format();
+    const isLastSuccessfulSync = period === "SINCE_LAST_SUCCESSFUL_SYNC";
+
+    return isUpdatedAfterStartDate || !isLastSuccessfulSync;
+}

src/domain/aggregated/utils.ts

@@ -58,7 +58,7 @@ export function buildPeriodFromParams(params: Pick<DataSynchronizationParams, "p
             startDate: moment(startDate ?? "1970-01-01"),
             endDate: moment(endDate ?? moment().add(1, "years").endOf("year").format("YYYY-MM-DD")),
         };
-    } else if (period === "SINCE_LAST_EXECUTED_DATE") {
+    } else if (period === "SINCE_LAST_EXECUTED_DATE" || period === "SINCE_LAST_SUCCESSFUL_SYNC") {
         return {
             startDate: moment(startDate ?? "1970-01-01"),
             endDate: moment(),

src/domain/rules/entities/SynchronizationRule.ts

@@ -37,6 +37,7 @@ export class SynchronizationRule {
             "frequency",
             "lastExecuted",
             "lastExecutedBy",
+            "lastSuccessfulSync",
             "lastUpdated",
             "lastUpdatedBy",
             "publicAccess",
@@ -52,7 +53,7 @@ export class SynchronizationRule {
 
     public replicate(): SynchronizationRule {
         return this.updateName(`Copy of ${this.syncRule.name}`)
-            .update({ lastExecuted: undefined })
+            .update({ lastExecuted: undefined, lastSuccessfulSync: undefined })
             .updateId(generateUid());
     }
 
@@ -192,6 +193,10 @@ export class SynchronizationRule {
         return this.syncRule.lastExecutedBy?.name;
     }
 
+    public get lastSuccessfulSync(): Date | undefined {
+        return this.syncRule.lastSuccessfulSync;
+    }
+
     public get created(): Date | undefined {
         return this.syncRule.created ? new Date(this.syncRule.created) : undefined;
     }
@@ -287,18 +292,32 @@ export class SynchronizationRule {
 
     public static build(syncRule: SynchronizationRuleData | undefined): SynchronizationRule {
         if (syncRule) {
-            return syncRule.builder?.dataParams?.period === "SINCE_LAST_EXECUTED_DATE"
-                ? new SynchronizationRule({
-                      ...syncRule,
-                      builder: {
-                          ...syncRule.builder,
-                          dataParams: {
-                              ...syncRule.builder.dataParams,
-                              startDate: syncRule.lastExecuted ?? new Date(),
-                          },
-                      },
-                  })
-                : new SynchronizationRule(syncRule);
+            switch (syncRule.builder?.dataParams?.period) {
+                case "SINCE_LAST_EXECUTED_DATE":
+                    return new SynchronizationRule({
+                        ...syncRule,
+                        builder: {
+                            ...syncRule.builder,
+                            dataParams: {
+                                ...syncRule.builder.dataParams,
+                                startDate: syncRule.lastExecuted ?? new Date(),
+                            },
+                        },
+                    });
+                case "SINCE_LAST_SUCCESSFUL_SYNC":
+                    return new SynchronizationRule({
+                        ...syncRule,
+                        builder: {
+                            ...syncRule.builder,
+                            dataParams: {
+                                ...syncRule.builder.dataParams,
+                                startDate: syncRule.lastSuccessfulSync,
+                            },
+                        },
+                    });
+                default:
+                    return new SynchronizationRule(syncRule);
+            }
         } else {
             return this.create();
         }
@@ -588,6 +607,10 @@ export class SynchronizationRule {
         return this.update({ lastExecuted, lastExecutedBy });
     }
 
+    public updateLastSuccessfulSync(lastSuccessfulSync: Date): SynchronizationRule {
+        return this.update({ lastSuccessfulSync });
+    }
+
     public isOnDemand() {
         return this.name === "__MANUAL__";
     }
@@ -744,6 +767,7 @@ export interface SynchronizationRuleData extends SharedRef {
     enabled: boolean;
     lastExecuted?: Date;
     lastExecutedBy?: NamedRef;
+    lastSuccessfulSync?: Date;
     frequency?: string;
     type: SynchronizationType;
     ondemand?: boolean;

src/domain/rules/entities/__tests__/SynchronizationRule.spec.ts

@@ -491,6 +491,17 @@ describe("SyncRule", () => {
 
             expect(editedSyncRule.dataSyncStartDate).toEqual(lastExecuted);
         });
+        it("should have start date as last successfully executed date if the period is set to since last successfully executed and it exists", () => {
+            const syncRule = givenASyncRuleWithoutPeriod();
+
+            const lastSuccessfulSync = new Date();
+
+            const editedSyncRule = syncRule
+                .updateLastSuccessfulSync(lastSuccessfulSync)
+                .updateDataSyncPeriod("SINCE_LAST_SUCCESSFUL_SYNC");
+
+            expect(editedSyncRule.dataSyncStartDate).toEqual(lastSuccessfulSync);
+        });
         it("should has start date as now if change the period to since last executed and does not exist", () => {
             const syncRule = givenASyncRuleWithoutPeriod();
 
@@ -502,6 +513,13 @@ describe("SyncRule", () => {
             expect(editedSyncRule.dataSyncStartDate?.getMonth()).toEqual(now.getMonth());
             expect(editedSyncRule.dataSyncStartDate?.getFullYear()).toEqual(now.getFullYear());
         });
+        it("should have no start date if the period is set to since last successfully executed and it does not exist", () => {
+            const syncRule = givenASyncRuleWithoutPeriod();
+
+            const editedSyncRule = syncRule.updateDataSyncPeriod("SINCE_LAST_SUCCESSFUL_SYNC");
+
+            expect(editedSyncRule.dataSyncStartDate).toEqual(undefined);
+        });
         it("should has start date as last executed after build if the period is since last executed and exist last executed", () => {
             const lastExecuted = new Date();
 
@@ -514,7 +532,19 @@ describe("SyncRule", () => {
 
             expect(syncRule.dataSyncStartDate).toEqual(lastExecuted);
         });
-        it("should has start date as now after build if the period is since last executed and  last executed does not exist", () => {
+        it("should have start date as last successfully executed date after build if the period is set to since last successfully executed and it exists", () => {
+            const lastSuccessfulSync = new Date();
+
+            const syncRuleData = givenASyncRuleWithoutPeriod()
+                .updateLastSuccessfulSync(lastSuccessfulSync)
+                .updateDataSyncPeriod("SINCE_LAST_SUCCESSFUL_SYNC")
+                .toObject();
+
+            const syncRule = SynchronizationRule.build(syncRuleData);
+
+            expect(syncRule.dataSyncStartDate).toEqual(lastSuccessfulSync);
+        });
+        it("should has start date as now after build if the period is since last executed and last executed does not exist", () => {
             const now = new Date();
 
             const syncRuleData = givenASyncRuleWithoutPeriod()
@@ -527,6 +557,15 @@ describe("SyncRule", () => {
             expect(syncRule.dataSyncStartDate?.getMonth()).toEqual(now.getMonth());
             expect(syncRule.dataSyncStartDate?.getFullYear()).toEqual(now.getFullYear());
         });
+        it("should have no start date after build if the period is set to since last successfully executed and it does not exist", () => {
+            const syncRuleData = givenASyncRuleWithoutPeriod()
+                .updateDataSyncPeriod("SINCE_LAST_SUCCESSFUL_SYNC")
+                .toObject();
+
+            const syncRule = SynchronizationRule.build(syncRuleData);
+
+            expect(syncRule.dataSyncStartDate).toEqual(undefined);
+        });
     });
 });
 

src/domain/synchronization/usecases/GenericSyncUseCase.ts

@@ -275,6 +275,18 @@ export abstract class GenericSyncUseCase {
 
         // Phase 5: Update parent task status
         syncReport.setStatus(syncReport.hasErrors() ? "FAILURE" : "DONE");
+
+        // Phase 6: if sync report is DONE, update sync rule last successful sync date
+        if (syncReport.status === "DONE" && syncRule) {
+            const syncRulesRepository = this.repositoryFactory.rulesRepository(this.localInstance);
+            const rule = await syncRulesRepository.getById(syncRule);
+
+            if (rule) {
+                const updatedRule = rule.updateLastSuccessfulSync(new Date());
+                await syncRulesRepository.save([updatedRule]);
+            }
+        }
+
         yield { syncReport, done: true };
 
         return syncReport;

src/models/dhis/mapping.ts

@@ -118,7 +118,7 @@ export class ProgramDataElementModel extends DataElementModel {
     protected static parentMappingType = "eventPrograms";
     protected static groupFilterName = DataElementModel.groupFilterName;
     protected static fields = dataElementFields;
-    protected static isSelectable = false;
+    protected static isSelectable = true;
 
     protected static modelFilters = { domainType: "TRACKER" };
 }

src/presentation/react/core/components/sync-wizard/data/PeriodSelectionStep.tsx

@@ -6,7 +6,7 @@ import { SyncWizardStepProps } from "../Steps";
 
 const PeriodSelectionStep: React.FC<SyncWizardStepProps> = ({ syncRule, onChange }) => {
     const [skipPeriods] = useState<Set<PeriodType> | undefined>(
-        syncRule.ondemand ? new Set(["SINCE_LAST_EXECUTED_DATE"]) : undefined
+        syncRule.ondemand ? new Set(["SINCE_LAST_EXECUTED_DATE", "SINCE_LAST_SUCCESSFUL_SYNC"]) : undefined
     );
 
     const updatePeriod = useCallback(