From df5362c6150ad2cb3c0ec9ab2353c057d30af320 Mon Sep 17 00:00:00 2001 From: Miquel Adell Date: Fri, 25 Aug 2023 10:21:01 +0200 Subject: [PATCH 1/8] Update pull_request_template.md Update pull_request_template.md to add screen capture recommendation --- .github/pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5817633b0..f79e41930 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -6,7 +6,7 @@ - -### :art: Screenshots +### :video_camera: Screenshots/Screen capture ### :fire: Is there anything the reviewer should know to test it? From b6073fdc14e7b5279574d26438e7d544a5cb0ba2 Mon Sep 17 00:00:00 2001 From: Ana Garcia Date: Tue, 2 Apr 2024 14:08:46 +0200 Subject: [PATCH 2/8] Get userAccesses and userGroupAccesses from sharing endpoint instead of directly from metaData because displayName is not in the response --- src/data/storage/StorageDataStoreClient.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/data/storage/StorageDataStoreClient.ts b/src/data/storage/StorageDataStoreClient.ts index 355aced7f..7d97e77ba 100644 --- a/src/data/storage/StorageDataStoreClient.ts +++ b/src/data/storage/StorageDataStoreClient.ts @@ -82,10 +82,14 @@ export class StorageDataStoreClient extends StorageClient { const metadata = await this.getMetadataByKey(key); if (!metadata) return undefined; + const { object } = await this.api.sharing.get({ type: "dataStore", id: metadata.id }).getData(); + + if (!object) return undefined; + return { user: { name: "", ...metadata.user }, - userAccesses: metadata.userAccesses, - userGroupAccesses: metadata.userGroupAccesses, + userAccesses: object.userAccesses || [], + userGroupAccesses: object.userGroupAccesses || [], publicAccess: metadata.publicAccess, externalAccess: metadata.externalAccess, }; From 27282148e524834e768393789d5e3a0cf58b53b6 Mon Sep 17 00:00:00 2001 From: Ana Garcia Date: Tue, 2 Apr 2024 14:57:12 +0200 Subject: [PATCH 3/8] Update tests --- .../integration/local-instance-mapped.spec.ts | 15 ++++++++++ .../integration/sync-aggregated.spec.ts | 30 +++++++++++++++++++ .../__tests__/integration/sync-events.spec.ts | 30 +++++++++++++++++++ .../integration/sync-metadata.spec.ts | 30 +++++++++++++++++++ .../__tests__/integration/helpers.ts | 30 +++++++++++++++++++ 5 files changed, 135 insertions(+) diff --git a/src/data/metadata/__tests__/integration/local-instance-mapped.spec.ts b/src/data/metadata/__tests__/integration/local-instance-mapped.spec.ts index 058299a16..51a225b0c 100644 --- a/src/data/metadata/__tests__/integration/local-instance-mapped.spec.ts +++ b/src/data/metadata/__tests__/integration/local-instance-mapped.spec.ts @@ -147,6 +147,21 @@ describe("Sync local instance mapped", () => { id: "Db5532sXKXT", })); + local.get("/sharing", async () => ({ + meta: { + allowPublicAccess: true, + allowExternalAccess: false, + }, + object: { + id: "Db5532sXKXT", + publicAccess: "rw------", + user: { id: "H4atNsEuKxP" }, + userGroupAccesses: [], + userAccesses: [], + externalAccess: false, + }, + })); + const addAggregatedToDb = async (schema: Schema, request: Request) => { schema.db.dataValueSets.insert(JSON.parse(request.requestBody)); diff --git a/src/data/metadata/__tests__/integration/sync-aggregated.spec.ts b/src/data/metadata/__tests__/integration/sync-aggregated.spec.ts index c2007766a..0615bad31 100644 --- a/src/data/metadata/__tests__/integration/sync-aggregated.spec.ts +++ b/src/data/metadata/__tests__/integration/sync-aggregated.spec.ts @@ -214,6 +214,36 @@ describe("Sync aggregated", () => { id: "Db5532sXKX1", })); + local.get("/sharing", async () => ({ + meta: { + allowPublicAccess: true, + allowExternalAccess: false, + }, + object: { + id: "Db5532sXKXT", + publicAccess: "rw------", + user: { id: "H4atNsEuKxP" }, + userGroupAccesses: [], + userAccesses: [], + externalAccess: false, + }, + })); + + local.get("/sharing", async () => ({ + meta: { + allowPublicAccess: true, + allowExternalAccess: false, + }, + object: { + id: "Db5532sXKX1", + externalAccess: false, + publicAccess: "rw------", + user: { id: "H4atNsEuKxP" }, + userGroupAccesses: [], + userAccesses: [], + }, + })); + const addAggregatedToDb = async (schema: Schema, request: Request) => { schema.db.dataValueSets.insert(JSON.parse(request.requestBody)); diff --git a/src/data/metadata/__tests__/integration/sync-events.spec.ts b/src/data/metadata/__tests__/integration/sync-events.spec.ts index ccd9834b2..a84e7f844 100644 --- a/src/data/metadata/__tests__/integration/sync-events.spec.ts +++ b/src/data/metadata/__tests__/integration/sync-events.spec.ts @@ -227,6 +227,36 @@ describe("Sync events", () => { id: "Db5532sXKX1", })); + local.get("/sharing", async () => ({ + meta: { + allowPublicAccess: true, + allowExternalAccess: false, + }, + object: { + id: "Db5532sXKXT", + publicAccess: "rw------", + user: { id: "H4atNsEuKxP" }, + userGroupAccesses: [], + userAccesses: [], + externalAccess: false, + }, + })); + + local.get("/sharing", async () => ({ + meta: { + allowPublicAccess: true, + allowExternalAccess: false, + }, + object: { + id: "Db5532sXKX1", + externalAccess: false, + publicAccess: "rw------", + user: { id: "H4atNsEuKxP" }, + userGroupAccesses: [], + userAccesses: [], + }, + })); + // local.get("/trackedEntityInstances", async () => ({ // trackedEntityInstances: [], // })); diff --git a/src/data/metadata/__tests__/integration/sync-metadata.spec.ts b/src/data/metadata/__tests__/integration/sync-metadata.spec.ts index 4e4dc1a57..04b186b7e 100644 --- a/src/data/metadata/__tests__/integration/sync-metadata.spec.ts +++ b/src/data/metadata/__tests__/integration/sync-metadata.spec.ts @@ -90,6 +90,36 @@ describe("Sync metadata", () => { id: "Db5532sXKX1", })); + local.get("/sharing", async () => ({ + meta: { + allowPublicAccess: true, + allowExternalAccess: false, + }, + object: { + id: "Db5532sXKXT", + publicAccess: "rw------", + user: { id: "H4atNsEuKxP" }, + userGroupAccesses: [], + userAccesses: [], + externalAccess: false, + }, + })); + + local.get("/sharing", async () => ({ + meta: { + allowPublicAccess: true, + allowExternalAccess: false, + }, + object: { + id: "Db5532sXKX1", + externalAccess: false, + publicAccess: "rw------", + user: { id: "H4atNsEuKxP" }, + userGroupAccesses: [], + userAccesses: [], + }, + })); + const addMetadataToDb = async (schema: Schema, request: Request) => { schema.db.metadata.insert(JSON.parse(request.requestBody)); diff --git a/src/data/transformations/__tests__/integration/helpers.ts b/src/data/transformations/__tests__/integration/helpers.ts index 069eea02c..1e25eb55a 100644 --- a/src/data/transformations/__tests__/integration/helpers.ts +++ b/src/data/transformations/__tests__/integration/helpers.ts @@ -101,6 +101,36 @@ export async function sync({ id: "Db5532sXKX1", })); + local.get("/sharing", async () => ({ + meta: { + allowPublicAccess: true, + allowExternalAccess: false, + }, + object: { + id: "Db5532sXKXT", + publicAccess: "rw------", + user: { id: "H4atNsEuKxP" }, + userGroupAccesses: [], + userAccesses: [], + externalAccess: false, + }, + })); + + local.get("/sharing", async () => ({ + meta: { + allowPublicAccess: true, + allowExternalAccess: false, + }, + object: { + id: "Db5532sXKX1", + externalAccess: false, + publicAccess: "rw------", + user: { id: "H4atNsEuKxP" }, + userGroupAccesses: [], + userAccesses: [], + }, + })); + const addMetadataToDb = async (schema: Schema, request: Request) => { schema.db.metadata.insert(JSON.parse(request.requestBody)); From 11397ae1a032184de099a037a3887c9ce007b176 Mon Sep 17 00:00:00 2001 From: Ana Garcia Date: Tue, 16 Apr 2024 08:03:28 +0200 Subject: [PATCH 4/8] Update translations --- i18n/en.pot | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/i18n/en.pot b/i18n/en.pot index 4c9691291..0b0c8aa59 100644 --- a/i18n/en.pot +++ b/i18n/en.pot @@ -5,8 +5,8 @@ msgstr "" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1)\n" -"POT-Creation-Date: 2023-06-12T22:53:44.873Z\n" -"PO-Revision-Date: 2023-06-12T22:53:44.873Z\n" +"POT-Creation-Date: 2024-04-16T06:03:13.568Z\n" +"PO-Revision-Date: 2024-04-16T06:03:13.568Z\n" msgid "" "THIS NEW RELEASE INCLUDES SHARING SETTINGS PER INSTANCES. FOR THIS VERSION " From 21f45284cb879634e35491c544143bc78007bf17 Mon Sep 17 00:00:00 2001 From: mariaozamiz Date: Tue, 16 Apr 2024 10:33:42 +0200 Subject: [PATCH 5/8] feat: add constantsModel. Update factory and MetadataEntities --- i18n/en.pot | 4 ++-- src/domain/metadata/entities/MetadataEntities.ts | 1 + src/models/dhis/factory.ts | 1 + src/models/dhis/metadata.ts | 5 +++++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/i18n/en.pot b/i18n/en.pot index 4c9691291..16a7a75b7 100644 --- a/i18n/en.pot +++ b/i18n/en.pot @@ -5,8 +5,8 @@ msgstr "" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1)\n" -"POT-Creation-Date: 2023-06-12T22:53:44.873Z\n" -"PO-Revision-Date: 2023-06-12T22:53:44.873Z\n" +"POT-Creation-Date: 2024-04-16T07:08:19.108Z\n" +"PO-Revision-Date: 2024-04-16T07:08:19.108Z\n" msgid "" "THIS NEW RELEASE INCLUDES SHARING SETTINGS PER INSTANCES. FOR THIS VERSION " diff --git a/src/domain/metadata/entities/MetadataEntities.ts b/src/domain/metadata/entities/MetadataEntities.ts index 404876789..26cd6cff0 100644 --- a/src/domain/metadata/entities/MetadataEntities.ts +++ b/src/domain/metadata/entities/MetadataEntities.ts @@ -5065,6 +5065,7 @@ export type MetadataEntities = { categoryOptionCombos: CategoryOptionCombo[]; categoryOptionGroups: CategoryOptionGroup[]; categoryOptionGroupSets: CategoryOptionGroupSet[]; + constants: Constant[]; charts: Chart[]; dashboards: Dashboard[]; dataApprovalLevels: DataApprovalLevel[]; diff --git a/src/models/dhis/factory.ts b/src/models/dhis/factory.ts index fc51a9471..f66f48b8f 100644 --- a/src/models/dhis/factory.ts +++ b/src/models/dhis/factory.ts @@ -13,6 +13,7 @@ export const metadataModels = [ metadataClasses.CategoryOptionComboModel, metadataClasses.CategoryOptionGroupModel, metadataClasses.CategoryOptionGroupSetModel, + metadataClasses.ConstantsModel, metadataClasses.DashboardModel, metadataClasses.DataElementModel, metadataClasses.DataElementGroupModel, diff --git a/src/models/dhis/metadata.ts b/src/models/dhis/metadata.ts index f5c5f6059..2099a8c17 100644 --- a/src/models/dhis/metadata.ts +++ b/src/models/dhis/metadata.ts @@ -243,6 +243,11 @@ export class DataEntryFormModel extends D2Model { protected static collectionName = "dataEntryForms" as const; } +export class ConstantsModel extends D2Model { + protected static metadataType = "constants"; + protected static collectionName = "constants" as const; +} + export class DataSetModel extends D2Model { protected static metadataType = "dataSet"; protected static collectionName = "dataSets" as const; From 1c03796c2274ac6c1c0ddc8cf8333dffff93d657 Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Sun, 28 Apr 2024 16:41:49 -0500 Subject: [PATCH 6/8] fix create sync rule for event charts and reports --- src/data/metadata/MetadataD2ApiRepository.ts | 46 ++++++++++++++++++- .../sync-wizard/common/SummaryStep.tsx | 9 +++- src/utils/d2-utils.ts | 21 +++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/src/data/metadata/MetadataD2ApiRepository.ts b/src/data/metadata/MetadataD2ApiRepository.ts index 8d7815ed8..11d6683c7 100644 --- a/src/data/metadata/MetadataD2ApiRepository.ts +++ b/src/data/metadata/MetadataD2ApiRepository.ts @@ -33,7 +33,7 @@ import { D2Api, D2Model, Id, MetadataResponse, Model, Stats } from "../../types/ import { Dictionary, isNotEmpty, Maybe } from "../../types/utils"; import { cache } from "../../utils/cache"; import { promiseMap } from "../../utils/common"; -import { getD2APiFromInstance } from "../../utils/d2-utils"; +import { getD2APiFromInstance, getInChunks } from "../../utils/d2-utils"; import { debug } from "../../utils/debug"; import { paginate } from "../../utils/pagination"; import { metadataTransformations } from "../transformations/PackageTransformations"; @@ -555,7 +555,47 @@ export class MetadataD2ApiRepository implements MetadataRepository { const response = await Promise.all(promises); const results = _.deepMerge({}, ...response); if (results.system) delete results.system; - return results; + + const metadata = await this.validateEventVisualizationsByIds(results, elements); + return metadata; + } + + private async validateEventVisualizationsByIds(metadata: any, ids: Id[]) { + if (!metadata.eventCharts || !metadata.eventReports) return metadata; + const result = await getInChunks(ids, async idsInChunk => { + const d2Charts = await this.api.models.eventCharts + .get({ filter: { id: { in: idsInChunk } }, fields: { id: true }, paging: true }) + .getData(); + + const d2EventReports = await this.api.models.eventReports + .get({ filter: { id: { in: idsInChunk } }, fields: { id: true }, paging: true }) + .getData(); + + return [{ eventCharts: d2Charts.objects, eventReports: d2EventReports.objects }]; + }); + + const allCharts = _(result) + .flatMap(visualization => visualization.eventCharts) + .value(); + const allEvents = _(result) + .flatMap(visualization => visualization.eventReports) + .value(); + + const chartsMetadata = _(metadata.eventCharts) + .map(eventChart => { + return allCharts.find(d2Chart => d2Chart.id === eventChart.id) ? eventChart : undefined; + }) + .compact() + .value(); + + const eventReportsMetadata = _(metadata.eventReports) + .map(eventReport => { + return allEvents.find(d2Chart => d2Chart.id === eventReport.id) ? eventReport : undefined; + }) + .compact() + .value(); + + return { ...metadata, eventReports: eventReportsMetadata, eventCharts: chartsMetadata }; } private getApiModel(type: keyof MetadataEntities): Model { @@ -660,3 +700,5 @@ function getFilterAsString(filter: FilterBase): string[] { ) ); } + +type D2EventVisualization = { eventCharts: Array<{ id: Id }>; eventReports: Array<{ id: Id }> }; diff --git a/src/presentation/react/core/components/sync-wizard/common/SummaryStep.tsx b/src/presentation/react/core/components/sync-wizard/common/SummaryStep.tsx index 025f64bac..a8cc288e3 100644 --- a/src/presentation/react/core/components/sync-wizard/common/SummaryStep.tsx +++ b/src/presentation/react/core/components/sync-wizard/common/SummaryStep.tsx @@ -217,6 +217,12 @@ export const SummaryStepContent = (props: SummaryStepContentProps) => { {_.keys(metadata).map(metadataType => { + //@ts-ignore + const modelByMetadataType = api.models[metadataType]; + if (!modelByMetadataType) { + console.warn(`Metadata type "${metadataType}" not supported in d2-api`); + return null; + } const itemsByType = metadata[metadataType as keyof MetadataEntities] || []; const items = itemsByType.filter(({ id }) => !syncRule.excludedIds.includes(id)); @@ -225,8 +231,7 @@ export const SummaryStepContent = (props: SummaryStepContentProps) => { items.length > 0 && (
    {items.map(({ id, name }) => ( diff --git a/src/utils/d2-utils.ts b/src/utils/d2-utils.ts index 47bfeab31..eaec0c812 100644 --- a/src/utils/d2-utils.ts +++ b/src/utils/d2-utils.ts @@ -1,6 +1,7 @@ import _ from "lodash"; import { D2Api } from "../types/d2-api"; import { Instance } from "../domain/instance/entities/Instance"; +import { Id } from "../domain/common/entities/Schemas"; export function getMajorVersion(version: string): number { const apiVersion = _.get(version.split("."), 1); @@ -22,3 +23,23 @@ export function getD2APiFromInstance(instance: Instance) { */ return new D2Api({ baseUrl: instance.url, auth: instance.auth, backend: "fetch" }); } + +export async function getInChunks( + ids: Id[], + getter: (idsGroup: Id[]) => Promise, + chunkSize = 100 +): Promise { + const objsCollection = await promiseMap(_.chunk(ids, chunkSize), idsGroup => getter(idsGroup)); + return _.flatten(objsCollection); +} + +export function promiseMap(inputValues: T[], mapper: (value: T) => Promise): Promise { + const reducer = (acc$: Promise, inputValue: T): Promise => + acc$.then((acc: S[]) => + mapper(inputValue).then(result => { + acc.push(result); + return acc; + }) + ); + return inputValues.reduce(reducer, Promise.resolve([])); +} From c1fac226c769bda750c46767e1a1757e3556d39f Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Mon, 29 Apr 2024 13:33:59 -0500 Subject: [PATCH 7/8] verify visualization by TYPE --- src/data/metadata/MetadataD2ApiRepository.ts | 34 ++++++-------------- src/utils/d2-utils.ts | 21 ------------ 2 files changed, 10 insertions(+), 45 deletions(-) diff --git a/src/data/metadata/MetadataD2ApiRepository.ts b/src/data/metadata/MetadataD2ApiRepository.ts index 11d6683c7..2614560ee 100644 --- a/src/data/metadata/MetadataD2ApiRepository.ts +++ b/src/data/metadata/MetadataD2ApiRepository.ts @@ -33,7 +33,7 @@ import { D2Api, D2Model, Id, MetadataResponse, Model, Stats } from "../../types/ import { Dictionary, isNotEmpty, Maybe } from "../../types/utils"; import { cache } from "../../utils/cache"; import { promiseMap } from "../../utils/common"; -import { getD2APiFromInstance, getInChunks } from "../../utils/d2-utils"; +import { getD2APiFromInstance } from "../../utils/d2-utils"; import { debug } from "../../utils/debug"; import { paginate } from "../../utils/pagination"; import { metadataTransformations } from "../transformations/PackageTransformations"; @@ -556,41 +556,23 @@ export class MetadataD2ApiRepository implements MetadataRepository { const results = _.deepMerge({}, ...response); if (results.system) delete results.system; - const metadata = await this.validateEventVisualizationsByIds(results, elements); + const metadata = await this.validateEventVisualizationsByIds(results); return metadata; } - private async validateEventVisualizationsByIds(metadata: any, ids: Id[]) { + private async validateEventVisualizationsByIds(metadata: any) { if (!metadata.eventCharts || !metadata.eventReports) return metadata; - const result = await getInChunks(ids, async idsInChunk => { - const d2Charts = await this.api.models.eventCharts - .get({ filter: { id: { in: idsInChunk } }, fields: { id: true }, paging: true }) - .getData(); - - const d2EventReports = await this.api.models.eventReports - .get({ filter: { id: { in: idsInChunk } }, fields: { id: true }, paging: true }) - .getData(); - - return [{ eventCharts: d2Charts.objects, eventReports: d2EventReports.objects }]; - }); - - const allCharts = _(result) - .flatMap(visualization => visualization.eventCharts) - .value(); - const allEvents = _(result) - .flatMap(visualization => visualization.eventReports) - .value(); const chartsMetadata = _(metadata.eventCharts) .map(eventChart => { - return allCharts.find(d2Chart => d2Chart.id === eventChart.id) ? eventChart : undefined; + return this.isEventReport(eventChart.type) ? undefined : eventChart; }) .compact() .value(); const eventReportsMetadata = _(metadata.eventReports) .map(eventReport => { - return allEvents.find(d2Chart => d2Chart.id === eventReport.id) ? eventReport : undefined; + return this.isEventReport(eventReport.type) ? eventReport : undefined; }) .compact() .value(); @@ -598,6 +580,10 @@ export class MetadataD2ApiRepository implements MetadataRepository { return { ...metadata, eventReports: eventReportsMetadata, eventCharts: chartsMetadata }; } + private isEventReport(visualizationType: D2VisualizationType): boolean { + return visualizationType === "PIVOT_TABLE" || visualizationType === "LINE_LIST"; + } + private getApiModel(type: keyof MetadataEntities): Model { return this.api.models[type]; } @@ -701,4 +687,4 @@ function getFilterAsString(filter: FilterBase): string[] { ); } -type D2EventVisualization = { eventCharts: Array<{ id: Id }>; eventReports: Array<{ id: Id }> }; +type D2VisualizationType = "LINE_LIST" | "PIVOT_TABLE"; diff --git a/src/utils/d2-utils.ts b/src/utils/d2-utils.ts index eaec0c812..47bfeab31 100644 --- a/src/utils/d2-utils.ts +++ b/src/utils/d2-utils.ts @@ -1,7 +1,6 @@ import _ from "lodash"; import { D2Api } from "../types/d2-api"; import { Instance } from "../domain/instance/entities/Instance"; -import { Id } from "../domain/common/entities/Schemas"; export function getMajorVersion(version: string): number { const apiVersion = _.get(version.split("."), 1); @@ -23,23 +22,3 @@ export function getD2APiFromInstance(instance: Instance) { */ return new D2Api({ baseUrl: instance.url, auth: instance.auth, backend: "fetch" }); } - -export async function getInChunks( - ids: Id[], - getter: (idsGroup: Id[]) => Promise, - chunkSize = 100 -): Promise { - const objsCollection = await promiseMap(_.chunk(ids, chunkSize), idsGroup => getter(idsGroup)); - return _.flatten(objsCollection); -} - -export function promiseMap(inputValues: T[], mapper: (value: T) => Promise): Promise { - const reducer = (acc$: Promise, inputValue: T): Promise => - acc$.then((acc: S[]) => - mapper(inputValue).then(result => { - acc.push(result); - return acc; - }) - ); - return inputValues.reduce(reducer, Promise.resolve([])); -} From e35cc5043a188e0c8afe224cf81c461dabad4122 Mon Sep 17 00:00:00 2001 From: Miquel Adell Date: Mon, 6 May 2024 08:30:37 +0200 Subject: [PATCH 8/8] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 307ea80c8..8e59a5bc1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "metadata-synchronization", "description": "Advanced metadata & data synchronization utility", - "version": "2.17.0", + "version": "2.17.1", "license": "GPL-3.0", "author": "EyeSeeTea team", "homepage": ".",