Skip to content

Commit

Permalink
chore: Clean up data values persistence [DHIS2-18223]
Browse files Browse the repository at this point in the history
  • Loading branch information
@enricocolasante
enricocolasante committed Oct 18, 2024
1 parent 40689f3 commit 0120de6
Show file tree
Hide file tree
Showing 4 changed files with 81 additions and 167 deletions.
19 changes: 0 additions & 19 deletions ...vice-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/TrackerObjectsMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,7 @@
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import org.hisp.dhis.category.CategoryOptionCombo;
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.event.EventStatus;
import org.hisp.dhis.eventdatavalue.EventDataValue;
import org.hisp.dhis.note.Note;
import org.hisp.dhis.organisationunit.OrganisationUnit;
import org.hisp.dhis.program.Enrollment;
Expand All @@ -54,7 +52,6 @@
import org.hisp.dhis.relationship.RelationshipType;
import org.hisp.dhis.trackedentity.TrackedEntity;
import org.hisp.dhis.trackedentity.TrackedEntityType;
import org.hisp.dhis.tracker.imports.domain.DataValue;
import org.hisp.dhis.tracker.imports.preheat.TrackerPreheat;
import org.hisp.dhis.tracker.imports.util.RelationshipKeySupport;
import org.hisp.dhis.user.User;
Expand Down Expand Up @@ -240,22 +237,6 @@ private TrackerObjectsMapper() {
assignedUser.ifPresent(dbEvent::setAssignedUser);
}

// TODO(DHIS2-18223): Remove data value mapping and fix changelog logic
for (DataValue dataValue : event.getDataValues()) {
DataElement dataElement = preheat.getDataElement(dataValue.getDataElement());

EventDataValue eventDataValue = new EventDataValue();
eventDataValue.setDataElement(dataElement.getUid());
eventDataValue.setCreated(DateUtils.fromInstant(dataValue.getCreatedAt()));
eventDataValue.setCreatedByUserInfo(UserInfoSnapshot.from(user));
eventDataValue.setValue(dataValue.getValue());
eventDataValue.setLastUpdated(now);
eventDataValue.setProvidedElsewhere(dataValue.isProvidedElsewhere());
eventDataValue.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user));

dbEvent.getEventDataValues().add(eventDataValue);
}

if (isNotEmpty(event.getNotes())) {
dbEvent
.getNotes()
Expand Down
175 changes: 81 additions & 94 deletions ...-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/EventPersister.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,7 @@
*/
package org.hisp.dhis.tracker.imports.bundle.persister;

import static com.google.common.base.Preconditions.checkNotNull;

import jakarta.persistence.EntityManager;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
Expand All @@ -41,15 +38,16 @@
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import lombok.Builder;
import lombok.Data;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import org.apache.commons.lang3.StringUtils;
import org.hisp.dhis.changelog.ChangeLogType;
import org.hisp.dhis.common.UID;
import org.hisp.dhis.dataelement.DataElement;
import org.hisp.dhis.event.EventStatus;
import org.hisp.dhis.eventdatavalue.EventDataValue;
import org.hisp.dhis.program.Event;
import org.hisp.dhis.program.UserInfoSnapshot;
import org.hisp.dhis.reservedvalue.ReservedValueService;
import org.hisp.dhis.tracker.TrackerType;
import org.hisp.dhis.tracker.export.event.EventChangeLogService;
Expand All @@ -62,7 +60,6 @@
import org.hisp.dhis.tracker.imports.job.TrackerNotificationDataBundle;
import org.hisp.dhis.tracker.imports.preheat.TrackerPreheat;
import org.hisp.dhis.user.UserDetails;
import org.hisp.dhis.util.DateUtils;
import org.springframework.stereotype.Component;

/**
Expand Down Expand Up @@ -172,63 +169,94 @@ private void handleDataValues(
payloadDataValues.forEach(
dv -> {
DataElement dataElement = preheat.getDataElement(dv.getDataElement());
checkNotNull(
dataElement,
"Data element should never be NULL here if validation is enforced before commit.");

// EventDataValue.dataElement contains a UID
EventDataValue eventDataValue = dataValueDBMap.get(dataElement.getUid());

ValuesHolder valuesHolder = getAuditAndDateParameters(eventDataValue, dv);
if (isNewDataValue(eventDataValue, dv)) {
eventDataValue = new EventDataValue();
eventDataValue.setCreated(new Date());
eventDataValue.setCreatedByUserInfo(UserInfoSnapshot.from(user));
eventDataValue.setDataElement(dataElement.getUid());
eventDataValue.setStoredBy(user.getUsername());

eventDataValue.setLastUpdated(new Date());
eventDataValue.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user));

eventDataValue.setValue(dv.getValue());
eventDataValue.setProvidedElsewhere(dv.isProvidedElsewhere());

eventDataValue = valuesHolder.getEventDataValue();
if (dataElement.isFileType()) {
assignFileResource(entityManager, preheat, event.getUid(), eventDataValue.getValue());
}

eventDataValue.setDataElement(dataElement.getUid());
eventDataValue.setStoredBy(dv.getStoredBy());
event.getEventDataValues().add(eventDataValue);
logTrackedEntityDataValueHistory(
user.getUsername(),
dataElement,
event,
new Date(),
eventDataValue,
ChangeLogType.CREATE);
} else if (isUpdate(eventDataValue, dv)) {
eventDataValue.setLastUpdated(new Date());
eventDataValue.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user));

if (StringUtils.isEmpty(dv.getValue())) {
if (dataElement.isFileType()) {
unassignFileResource(
entityManager, preheat, event.getUid(), eventDataValue.getValue());
assignFileResource(entityManager, preheat, event.getUid(), dv.getValue());
}

event.getEventDataValues().remove(eventDataValue);
} else {
logTrackedEntityDataValueHistory(
user.getUsername(),
dataElement,
event,
new Date(),
eventDataValue,
ChangeLogType.UPDATE);
eventDataValue.setProvidedElsewhere(dv.isProvidedElsewhere());
eventDataValue.setValue(dv.getValue());

} else if (isDeletion(eventDataValue, dv)) {
if (dataElement.isFileType()) {
assignFileResource(entityManager, preheat, event.getUid(), eventDataValue.getValue());
unassignFileResource(
entityManager, preheat, event.getUid(), eventDataValue.getValue());
}

eventDataValue.setLastUpdated(new Date());
eventDataValue.setLastUpdatedByUserInfo(UserInfoSnapshot.from(user));

logTrackedEntityDataValueHistory(
user.getUsername(),
dataElement,
event,
new Date(),
eventDataValue,
ChangeLogType.DELETE);
eventDataValue.setValue(dv.getValue());
eventDataValue.setProvidedElsewhere(dv.isProvidedElsewhere());
event.getEventDataValues().remove(eventDataValue);
event.getEventDataValues().add(eventDataValue);
}

logTrackedEntityDataValueHistory(
user.getUsername(), dataElement, event, new Date(), valuesHolder);
});
}

private Date getFromOrNewDate(DataValue dv, Function<DataValue, Instant> dateGetter) {
return Optional.of(dv).map(dateGetter).map(DateUtils::fromInstant).orElseGet(Date::new);
}

private void logTrackedEntityDataValueHistory(
String userName, DataElement de, Event event, Date created, ValuesHolder valuesHolder) {
ChangeLogType changeLogType = valuesHolder.getChangeLogType();

if (changeLogType != null) {
TrackedEntityDataValueChangeLog valueAudit = new TrackedEntityDataValueChangeLog();
valueAudit.setEvent(event);
valueAudit.setValue(valuesHolder.getValue());
valueAudit.setAuditType(changeLogType);
valueAudit.setDataElement(de);
valueAudit.setModifiedBy(userName);
valueAudit.setProvidedElsewhere(valuesHolder.isProvidedElseWhere());
valueAudit.setCreated(created);

eventChangeLogService.addTrackedEntityDataValueChangeLog(valueAudit);
}
String userName,
DataElement de,
Event event,
Date created,
EventDataValue eventDataValue,
ChangeLogType changeLogType) {

TrackedEntityDataValueChangeLog valueAudit = new TrackedEntityDataValueChangeLog();
valueAudit.setEvent(event);
valueAudit.setValue(eventDataValue.getValue());
valueAudit.setAuditType(changeLogType);
valueAudit.setDataElement(de);
valueAudit.setModifiedBy(userName);
valueAudit.setProvidedElsewhere(eventDataValue.getProvidedElsewhere());
valueAudit.setCreated(created);

eventChangeLogService.addTrackedEntityDataValueChangeLog(valueAudit);
}

@Override
Expand All @@ -245,61 +273,20 @@ protected String getUpdatedTrackedEntity(Event entity) {
.orElse(null);
}

private boolean isNewDataValue(EventDataValue eventDataValue, DataValue dv) {
return eventDataValue == null
|| (eventDataValue.getCreated() == null && StringUtils.isNotBlank(dv.getValue()));
}

private boolean isDeletion(EventDataValue eventDataValue, DataValue dv) {
return StringUtils.isNotBlank(eventDataValue.getValue()) && StringUtils.isBlank(dv.getValue());
private boolean isNewDataValue(
@CheckForNull EventDataValue eventDataValue, @Nonnull DataValue dv) {
return eventDataValue == null && !StringUtils.isBlank(dv.getValue());
}

private boolean isUpdate(EventDataValue eventDataValue, DataValue dv) {
return !StringUtils.equals(dv.getValue(), eventDataValue.getValue());
private boolean isDeletion(@CheckForNull EventDataValue eventDataValue, @Nonnull DataValue dv) {
return eventDataValue != null
&& StringUtils.isNotBlank(eventDataValue.getValue())
&& StringUtils.isBlank(dv.getValue());
}

private ValuesHolder getAuditAndDateParameters(EventDataValue eventDataValue, DataValue dv) {
String persistedValue;

ChangeLogType changeLogType = null;

if (isNewDataValue(eventDataValue, dv)) {
eventDataValue = new EventDataValue();
eventDataValue.setCreated(getFromOrNewDate(dv, DataValue::getCreatedAt));
eventDataValue.setLastUpdated(getFromOrNewDate(dv, DataValue::getUpdatedAt));
persistedValue = dv.getValue();
changeLogType = ChangeLogType.CREATE;
} else {
persistedValue = eventDataValue.getValue();

if (isUpdate(eventDataValue, dv)) {
changeLogType = ChangeLogType.UPDATE;
eventDataValue.setLastUpdated(getFromOrNewDate(dv, DataValue::getUpdatedAt));
}

if (isDeletion(eventDataValue, dv)) {
changeLogType = ChangeLogType.DELETE;
eventDataValue.setLastUpdated(getFromOrNewDate(dv, DataValue::getUpdatedAt));
}
}

return ValuesHolder.builder()
.value(persistedValue)
.providedElseWhere(dv.isProvidedElsewhere())
.changeLogType(changeLogType)
.eventDataValue(eventDataValue)
.build();
}

@Data
@Builder
static class ValuesHolder {
private final String value;

private final boolean providedElseWhere;

private final ChangeLogType changeLogType;

private final EventDataValue eventDataValue;
private boolean isUpdate(@CheckForNull EventDataValue eventDataValue, @Nonnull DataValue dv) {
return eventDataValue != null
&& !StringUtils.isBlank(dv.getValue())
&& !StringUtils.equals(dv.getValue(), eventDataValue.getValue());
}
}
5 changes: 0 additions & 5 deletions ...es/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/imports/domain/DataValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@

import com.fasterxml.jackson.annotation.JsonProperty;
import java.io.Serializable;
import java.time.Instant;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
Expand All @@ -44,10 +43,6 @@
@AllArgsConstructor
// TODO(DHIS2-18222) Remove unused fields
public class DataValue implements Serializable {
@JsonProperty private Instant createdAt; // remove

@JsonProperty private Instant updatedAt; // remove

@JsonProperty private String storedBy;

@JsonProperty private boolean providedElsewhere;
Expand Down
49 changes: 0 additions & 49 deletions ...ava/org/hisp/dhis/tracker/imports/validation/validator/event/DataValuesValidatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@
import org.hisp.dhis.tracker.imports.preheat.TrackerPreheat;
import org.hisp.dhis.tracker.imports.validation.Reporter;
import org.hisp.dhis.tracker.imports.validation.ValidationCode;
import org.hisp.dhis.util.DateUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
Expand Down Expand Up @@ -148,52 +147,6 @@ void successValidationWhenDataElementIsValid() {
assertIsEmpty(reporter.getErrors());
}

@Test
void successValidationWhenCreatedAtIsNull() {
DataElement dataElement = dataElement();
when(preheat.getDataElement(MetadataIdentifier.ofUid(dataElementUid))).thenReturn(dataElement);

ProgramStage programStage = programStage(dataElement);
when(preheat.getProgramStage(MetadataIdentifier.ofUid(programStageUid)))
.thenReturn(programStage);

DataValue validDataValue = dataValue();
validDataValue.setCreatedAt(null);
Event event =
Event.builder()
.programStage(idSchemes.toMetadataIdentifier(programStage))
.status(EventStatus.ACTIVE)
.dataValues(Set.of(validDataValue))
.build();

validator.validate(reporter, bundle, event);

assertIsEmpty(reporter.getErrors());
}

@Test
void failValidationWhenUpdatedAtIsNull() {
DataElement dataElement = dataElement();
when(preheat.getDataElement(MetadataIdentifier.ofUid(dataElementUid))).thenReturn(dataElement);

ProgramStage programStage = programStage(dataElement);
when(preheat.getProgramStage(MetadataIdentifier.ofUid(programStageUid)))
.thenReturn(programStage);

DataValue validDataValue = dataValue();
validDataValue.setUpdatedAt(null);
Event event =
Event.builder()
.programStage(idSchemes.toMetadataIdentifier(programStage))
.status(EventStatus.ACTIVE)
.dataValues(Set.of(validDataValue))
.build();

validator.validate(reporter, bundle, event);

assertIsEmpty(reporter.getErrors());
}

@Test
void failValidationWhenDataElementIsInvalid() {
DataElement dataElement = dataElement();
Expand Down Expand Up @@ -1128,8 +1081,6 @@ private DataValue dataValue(String value) {

private DataValue dataValue() {
DataValue dataValue = new DataValue();
dataValue.setCreatedAt(DateUtils.instantFromDateAsString("2020-10-10"));
dataValue.setUpdatedAt(DateUtils.instantFromDateAsString("2020-10-10"));
dataValue.setValue("text");
dataValue.setDataElement(MetadataIdentifier.ofUid(dataElementUid));
return dataValue;
Expand Down

0 comments on commit 0120de6

Please sign in to comment.