[BACK-815] Implement datum update with historical tracking

data/datum.go

@@ -18,6 +18,11 @@ type Datum interface {
 	GetOrigin() *origin.Origin
 	GetPayload() *metadata.Metadata
 
+	GetHistory() *[]interface{}
+	SetHistory(*[]interface{})
+	GetID() *string
+	SetID(id *string)
+
 	SetUserID(userID *string)
 	SetDataSetID(dataSetID *string)
 	SetActive(active bool)

data/service/api/v1/datasets_data_update.go

@@ -0,0 +1,173 @@
+package v1
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+
+	"github.com/tidepool-org/platform/data"
+	dataNormalizer "github.com/tidepool-org/platform/data/normalizer"
+	dataService "github.com/tidepool-org/platform/data/service"
+	"github.com/tidepool-org/platform/data/store"
+	dataTypesFactory "github.com/tidepool-org/platform/data/types/factory"
+	"github.com/tidepool-org/platform/permission"
+	"github.com/tidepool-org/platform/request"
+	"github.com/tidepool-org/platform/service"
+	structureParser "github.com/tidepool-org/platform/structure/parser"
+	structureValidator "github.com/tidepool-org/platform/structure/validator"
+)
+
+func DataSetDatumUpdate(dataServiceContext dataService.Context) {
+	ctx := dataServiceContext.Request().Context()
+
+	datumID := dataServiceContext.Request().PathParam("datumId")
+	if datumID == "" {
+		dataServiceContext.RespondWithError(ErrorDatumIDMissing())
+		return
+	}
+	dataSetID := dataServiceContext.Request().PathParam("dataSetId")
+	if dataSetID == "" {
+		dataServiceContext.RespondWithError(ErrorDataSetIDMissing())
+		return
+	}
+
+	dataSet, err := dataServiceContext.DataRepository().GetDataSetByID(ctx, dataSetID)
+	if err != nil {
+		dataServiceContext.RespondWithInternalServerFailure("Unable to get data set by id", err)
+		return
+	}
+	if dataSet == nil {
+		dataServiceContext.RespondWithError(ErrorDataSetIDNotFound(dataSetID))
+		return
+	}
+
+	if details := request.DetailsFromContext(ctx); !details.IsService() {
+		var permissions permission.Permissions
+		permissions, err := dataServiceContext.PermissionClient().GetUserPermissions(ctx, details.UserID(), *dataSet.UserID)
+		if err != nil {
+			if request.IsErrorUnauthorized(err) {
+				dataServiceContext.RespondWithError(service.ErrorUnauthorized())
+			} else {
+				dataServiceContext.RespondWithInternalServerFailure("Unable to get user permissions", err)
+			}
+			return
+		}
+		if _, ok := permissions[permission.Write]; !ok {
+			dataServiceContext.RespondWithError(service.ErrorUnauthorized())
+			return
+		}
+	}
+
+	if dataSet.IsClosed() {
+		dataServiceContext.RespondWithError(ErrorDataSetClosed(dataSetID))
+		return
+	}
+
+	rawDatum, err := dataServiceContext.DataRepository().GetDataSetDatumByID(ctx, dataSetID, datumID)
+	if err != nil {
+		if _, ok := err.(*store.ErrDataNotFound); ok {
+			dataServiceContext.RespondWithError(ErrorDataSetDatumMissing())
+			return
+		}
+		request.MustNewResponder(dataServiceContext.Response(), dataServiceContext.Request()).Error(http.StatusBadRequest, err)
+		return
+	}
+
+	rawDatumArrayDB, err := rawDatumToRawDatumArray(rawDatum)
+	if err != nil {
+		request.MustNewResponder(dataServiceContext.Response(), dataServiceContext.Request()).Error(http.StatusBadRequest, err)
+	}
+
+	parser := structureParser.NewArray(&rawDatumArrayDB)
+	validator := structureValidator.New()
+	normalizer := dataNormalizer.New()
+
+	datumArrayDB := []data.Datum{}
+	for _, reference := range parser.References() {
+		if datum := dataTypesFactory.ParseDatum(parser.WithReferenceObjectParser(reference)); datum != nil && *datum != nil {
+			(*datum).Validate(validator.WithReference(strconv.Itoa(reference)))
+			(*datum).Normalize(normalizer.WithReference(strconv.Itoa(reference)))
+			datumArrayDB = append(datumArrayDB, *datum)
+		}
+	}
+	parser.NotParsed()
+
+	if err = parser.Error(); err != nil {
+		request.MustNewResponder(dataServiceContext.Response(), dataServiceContext.Request()).Error(http.StatusBadRequest, err)
+		return
+	}
+
+	var rawDatumWeb interface{}
+	if err = dataServiceContext.Request().DecodeJsonPayload(&rawDatumWeb); err != nil {
+		dataServiceContext.RespondWithError(service.ErrorJSONMalformed())
+		return
+	}
+	rawDatumArray := []interface{}{rawDatumWeb}
+
+	parser = structureParser.NewArray(&rawDatumArray)
+	validator = structureValidator.New()
+	normalizer = dataNormalizer.New()
+
+	datumArray := []data.Datum{}
+	for _, reference := range parser.References() {
+		if datum := dataTypesFactory.ParseDatum(parser.WithReferenceObjectParser(reference)); datum != nil && *datum != nil {
+			(*datum).Validate(validator.WithReference(strconv.Itoa(reference)))
+			(*datum).Normalize(normalizer.WithReference(strconv.Itoa(reference)))
+			datumArray = append(datumArray, *datum)
+		}
+	}
+	parser.NotParsed()
+
+	if err = parser.Error(); err != nil {
+		request.MustNewResponder(dataServiceContext.Response(), dataServiceContext.Request()).Error(http.StatusBadRequest, err)
+		return
+	}
+	if err = validator.Error(); err != nil {
+		request.MustNewResponder(dataServiceContext.Response(), dataServiceContext.Request()).Error(http.StatusBadRequest, err)
+		return
+	}
+	if err = normalizer.Error(); err != nil {
+		request.MustNewResponder(dataServiceContext.Response(), dataServiceContext.Request()).Error(http.StatusBadRequest, err)
+		return
+	}
+
+	datumArray = append(datumArray, normalizer.Data()...)
+	for _, datum := range datumArray {
+		datum.SetUserID(dataSet.UserID)
+		datum.SetDataSetID(dataSet.UploadID)
+	}
+
+	dbDatum := datumArrayDB[0]
+	datum := datumArray[0]
+
+	// incoming datum replaces the db datum, while the previous dbDatum
+	// and its history get flattened into a history array and added to the new entry
+	existingHistory := *dbDatum.GetHistory()
+	dbDatum.SetHistory(nil)
+	existingHistory = append(existingHistory, dbDatum)
+	datum.SetHistory(&existingHistory)
+	datum.SetID(&datumID)
+
+	err = dataServiceContext.DataRepository().UpdateDataSetDatum(ctx, dataSet, datum)
+	if err != nil {
+		request.MustNewResponder(dataServiceContext.Response(), dataServiceContext.Request()).Error(http.StatusBadRequest, err)
+		return
+	}
+
+	dataServiceContext.RespondWithStatusAndData(http.StatusOK, struct{}{})
+}
+
+// the result coming from the db decoder cannot be parsed properly in the parser
+// but the json marshaled/unmarshaled array can be so doing this extra step for now
+func rawDatumToRawDatumArray(rawDatum interface{}) ([]interface{}, error) {
+	resultsJSON, err := json.Marshal([]interface{}{rawDatum})
+	if err != nil {
+		return nil, err
+	}
+	var rawDatumArrayDB []interface{}
+	err = json.Unmarshal(resultsJSON, &rawDatumArrayDB)
+	if err != nil {
+		return nil, err
+	}
+	return rawDatumArrayDB, nil
+}

data/service/api/v1/errors.go

@@ -16,6 +16,24 @@ func ErrorUserIDMissing() *service.Error {
 	}
 }
 
+func ErrorDatumIDMissing() *service.Error {
+	return &service.Error{
+		Code:   "dataset-datum-id-missing",
+		Status: http.StatusBadRequest,
+		Title:  "dataset datum id is missing",
+		Detail: "dataset datum id is missing",
+	}
+}
+
+func ErrorDataSetDatumMissing() *service.Error {
+	return &service.Error{
+		Code:   "dataset-datum-missing",
+		Status: http.StatusNotFound,
+		Title:  "dataset datum cannot be fetched",
+		Detail: "dataset datum cannot be fetched",
+	}
+}
+
 func ErrorUserIDNotFound(userID string) *service.Error {
 	return &service.Error{
 		Code:   "user-id-not-found",

data/service/api/v1/v1.go

@@ -5,6 +5,7 @@ import "github.com/tidepool-org/platform/data/service"
 func Routes() []service.Route {
 	routes := []service.Route{
 		service.MakeRoute("POST", "/v1/datasets/:dataSetId/data", Authenticate(DataSetsDataCreate)),
+		service.MakeRoute("PUT", "/v1/datasets/:dataSetId/data/:datumId", Authenticate(DataSetDatumUpdate)),
 		service.MakeRoute("DELETE", "/v1/datasets/:dataSetId", Authenticate(DataSetsDelete)),
 		service.MakeRoute("PUT", "/v1/datasets/:dataSetId", Authenticate(DataSetsUpdate)),
 		service.MakeRoute("DELETE", "/v1/users/:userId/data", Authenticate(UsersDataDelete)),

data/store/errors.go

@@ -0,0 +1,11 @@
+package store
+
+import "fmt"
+
+type ErrDataNotFound struct {
+	Err error
+}
+
+func (e *ErrDataNotFound) Error() string {
+	return fmt.Sprintf("Data not found, err %v", e.Err)
+}

data/store/mongo/mongo.go

@@ -368,6 +368,56 @@ func (d *DataRepository) CreateDataSetData(ctx context.Context, dataSet *upload.
 	return nil
 }
 
+func (d *DataRepository) UpdateDataSetDatum(ctx context.Context, dataSet *upload.Upload, datum data.Datum) error {
+	if ctx == nil {
+		return errors.New("context is missing")
+	}
+	if err := validateDataSet(dataSet); err != nil {
+		return err
+	}
+	if datum == nil {
+		return errors.New("data set data is missing")
+	}
+
+	now := time.Now()
+	timestamp := now.Truncate(time.Millisecond).Format(time.RFC3339Nano)
+
+	datum.SetUserID(dataSet.UserID)
+	datum.SetDataSetID(dataSet.UploadID)
+	datum.SetCreatedTime(&timestamp)
+
+	filter := bson.M{"id": *datum.GetID(), "uploadId": *dataSet.UploadID}
+	_, err := d.ReplaceOne(ctx, filter, datum)
+	loggerFields := log.Fields{"dataSetId": dataSet.UploadID, "duration": time.Since(now) / time.Microsecond}
+	log.LoggerFromContext(ctx).WithFields(loggerFields).WithError(err).Debug("CreateDataSetData")
+	if err != nil {
+		return errors.Wrap(err, "unable to create data set data")
+	}
+	return nil
+}
+
+func (d *DataRepository) GetDataSetDatumByID(ctx context.Context, dataSetID string, datumID string) (interface{}, error) {
+	if ctx == nil {
+		return nil, errors.New("context is missing")
+	}
+	if datumID == "" {
+		return nil, errors.New("data id is missing")
+	}
+	if dataSetID == "" {
+		return nil, errors.New("data id is missing")
+	}
+	var result map[string]interface{}
+	res := d.FindOne(ctx, bson.M{"id": datumID, "uploadId": dataSetID})
+	if err := res.Err(); err == mongo.ErrNoDocuments {
+		return nil, &store.ErrDataNotFound{Err: err}
+	}
+	err := res.Decode(&result)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to get data sets for user by id")
+	}
+	return result, nil
+}
+
 func (d *DataRepository) ActivateDataSetData(ctx context.Context, dataSet *upload.Upload, selectors *data.Selectors) error {
 	if ctx == nil {
 		return errors.New("context is missing")

data/store/store.go

@@ -25,6 +25,8 @@ type DataRepository interface {
 	UpdateDataSet(ctx context.Context, id string, update *data.DataSetUpdate) (*upload.Upload, error)
 	DeleteDataSet(ctx context.Context, dataSet *upload.Upload) error
 
+	GetDataSetDatumByID(ctx context.Context, dataSetID string, datumID string) (interface{}, error)
+	UpdateDataSetDatum(ctx context.Context, dataSet *upload.Upload, datum data.Datum) error
 	CreateDataSetData(ctx context.Context, dataSet *upload.Upload, dataSetData []data.Datum) error
 	ActivateDataSetData(ctx context.Context, dataSet *upload.Upload, selectors *data.Selectors) error
 	ArchiveDataSetData(ctx context.Context, dataSet *upload.Upload, selectors *data.Selectors) error

data/store/test/data_repository.go

@@ -223,6 +223,14 @@ func (d *DataRepository) GetDataSetByID(ctx context.Context, dataSetID string) (
 	return output.DataSet, output.Error
 }
 
+func (d *DataRepository) GetDataSetDatumByID(ctx context.Context, dataSetID string, dataID string) (interface{}, error) {
+	return nil, nil
+}
+
+func (d *DataRepository) UpdateDataSetDatum(ctx context.Context, dataSet *upload.Upload, datum data.Datum) error {
+	return nil
+}
+
 func (d *DataRepository) CreateDataSet(ctx context.Context, dataSet *upload.Upload) error {
 	d.CreateDataSetInvocations++
 

data/test/datum.go

@@ -17,6 +17,8 @@ type IdentityFieldsOutput struct {
 type Datum struct {
 	MetaInvocations                      int
 	MetaOutputs                          []interface{}
+	ID                                   *string
+	History                              *[]interface{}
 	ParseInvocations                     int
 	ParseInputs                          []structure.ObjectParser
 	ValidateInvocations                  int
@@ -106,6 +108,29 @@ func (d *Datum) GetPayload() *metadata.Metadata {
 	return output
 }
 
+func (d *Datum) GetID() *string {
+	return d.ID
+}
+
+func (d *Datum) SetID(id *string) {
+	d.ID = id
+}
+
+func (d *Datum) GetHistory() *[]interface{} {
+	if d.History == nil {
+		return &[]interface{}{}
+	}
+	return d.History
+}
+
+func (d *Datum) SetHistory(history *[]interface{}) {
+	if history == nil {
+		d.History = &[]interface{}{}
+		return
+	}
+	d.History = history
+}
+
 func (d *Datum) GetOrigin() *origin.Origin {
 	d.GetOriginInvocations++
 

data/types/base.go

@@ -51,6 +51,7 @@ type Base struct {
 	DeviceTime        *string                       `json:"deviceTime,omitempty" bson:"deviceTime,omitempty"`
 	GUID              *string                       `json:"guid,omitempty" bson:"guid,omitempty"`
 	ID                *string                       `json:"id,omitempty" bson:"id,omitempty"`
+	History           *[]interface{}                `json:"history,omitempty" bson:"history,omitempty"`
 	Location          *location.Location            `json:"location,omitempty" bson:"location,omitempty"`
 	ModifiedTime      *string                       `json:"modifiedTime,omitempty" bson:"modifiedTime,omitempty"`
 	ModifiedUserID    *string                       `json:"modifiedUserId,omitempty" bson:"modifiedUserId,omitempty"`
@@ -92,6 +93,7 @@ func (b *Base) Parse(parser structure.ObjectParser) {
 	b.DeviceID = parser.String("deviceId")
 	b.DeviceTime = parser.String("deviceTime")
 	b.ID = parser.String("id")
+	b.History = parser.Array("history")
 	b.Location = location.ParseLocation(parser.WithReferenceObjectParser("location"))
 	b.Notes = parser.StringArray("notes")
 	b.Origin = origin.ParseOrigin(parser.WithReferenceObjectParser("origin"))
@@ -258,6 +260,29 @@ func (b *Base) IdentityFields() ([]string, error) {
 	return []string{*b.UserID, *b.DeviceID, *b.Time, b.Type}, nil
 }
 
+func (b *Base) GetID() *string {
+	return b.ID
+}
+
+func (b *Base) SetID(id *string) {
+	b.ID = id
+}
+
+func (b *Base) GetHistory() *[]interface{} {
+	if b.History == nil {
+		return &[]interface{}{}
+	}
+	return b.History
+}
+
+func (b *Base) SetHistory(history *[]interface{}) {
+	if history == nil {
+		b.History = &[]interface{}{}
+		return
+	}
+	b.History = history
+}
+
 func (b *Base) GetOrigin() *origin.Origin {
 	return b.Origin
 }