Allow legacy deduplication

data/blood/glucose/glucose.go

@@ -67,3 +67,14 @@ func NormalizeValueForUnits(value *float64, units *string) *float64 {
 	}
 	return value
 }
+
+func NormalizeValueForUnitsWithFullPrecision(value *float64, units *string) *float64 {
+	if value != nil && units != nil {
+		switch *units {
+		case MgdL, Mgdl:
+			floatValue := *value / MmolLToMgdLConversionFactor
+			return &floatValue
+		}
+	}
+	return value
+}

data/blood/glucose/glucose_test.go

@@ -125,4 +125,28 @@ var _ = Describe("Glucose", func() {
 			}
 		})
 	})
+	Context("NormalizeValueForUnitsWithFullPrecision", func() {
+		DescribeTable("given value and units",
+			func(value *float64, units *string, expectedValue *float64) {
+				actualValue := glucose.NormalizeValueForUnitsWithFullPrecision(value, units)
+				if expectedValue == nil {
+					Expect(actualValue).To(BeNil())
+				} else {
+					Expect(actualValue).ToNot(BeNil())
+					Expect(*actualValue).To(Equal(*expectedValue))
+				}
+			},
+			Entry("returns nil for nil value", nil, pointer.FromString("mmol/L"), nil),
+			Entry("returns unchanged value for nil units", pointer.FromFloat64(10.0), nil, pointer.FromFloat64(10.0)),
+			Entry("returns unchanged value for unknown units", pointer.FromFloat64(10.0), pointer.FromString("unknown"), pointer.FromFloat64(10.0)),
+			Entry("returns unchanged value for mmol/L units", pointer.FromFloat64(10.0), pointer.FromString("mmol/L"), pointer.FromFloat64(10.0)),
+			Entry("returns unchanged value for mmol/l units", pointer.FromFloat64(10.0), pointer.FromString("mmol/l"), pointer.FromFloat64(10.0)),
+
+			Entry("returns converted value for mg/dL units", pointer.FromFloat64(140.0), pointer.FromString("mg/dL"), pointer.FromFloat64(7.771047187463747)),
+			Entry("returns converted value for mg/dL units", pointer.FromFloat64(100.0), pointer.FromString("mg/dL"), pointer.FromFloat64(5.550747991045533)),
+			Entry("returns converted value for mg/dl units", pointer.FromFloat64(80.0), pointer.FromString("mg/dl"), pointer.FromFloat64(4.440598392836427)),
+			Entry("returns converted value for mg/dl units", pointer.FromFloat64(69.0), pointer.FromString("mg/dl"), pointer.FromFloat64(3.830016113821418)),
+			Entry("returns converted value for mg/dl units", pointer.FromFloat64(50.0), pointer.FromString("mg/dl"), pointer.FromFloat64(2.7753739955227665)),
+		)
+	})
 })

data/blood/glucose/test/glucose.go

@@ -1,10 +1,31 @@
 package test
 
 import (
+	"github.com/onsi/gomega"
+
 	dataBloodGlucose "github.com/tidepool-org/platform/data/blood/glucose"
+	"github.com/tidepool-org/platform/metadata"
 	"github.com/tidepool-org/platform/test"
 )
 
 func RandomUnits() string {
 	return test.RandomStringFromArray(dataBloodGlucose.Units())
 }
+
+func ExpectRaw(raw *metadata.Metadata, expectedRaw *metadata.Metadata) {
+	if expectedRaw != nil {
+		gomega.Expect(raw).ToNot(gomega.BeNil())
+		if units := expectedRaw.Get("units"); units == nil {
+			gomega.Expect(raw.Get("units")).To(gomega.BeNil())
+		} else {
+			gomega.Expect(raw.Get("units")).To(gomega.Equal(units))
+		}
+		if value := expectedRaw.Get("value"); value == nil {
+			gomega.Expect(raw.Get("value")).To(gomega.BeNil())
+		} else {
+			gomega.Expect(raw.Get("value")).To(gomega.Equal(value))
+		}
+	} else {
+		gomega.Expect(raw).To(gomega.BeNil())
+	}
+}

data/data_set.go

@@ -111,6 +111,7 @@ func (d *DataSetClient) Validate(validator structure.Validator) {
 
 type DataSetFilter struct {
 	ClientName *string
+	LegacyOnly *bool
 	Deleted    *bool
 	DeviceID   *string
 }
@@ -321,6 +322,7 @@ type DataSet struct {
 	Type                string                  `json:"type,omitempty" bson:"type,omitempty"`
 	UploadID            *string                 `json:"uploadId,omitempty" bson:"uploadId,omitempty"`
 	UserID              *string                 `json:"-" bson:"_userId,omitempty"`
+	LegacyGroupID       *string                 `json:"-" bson:"_groupId,omitempty"`
 	Version             *string                 `json:"version,omitempty" bson:"version,omitempty"`
 	VersionInternal     int                     `json:"-" bson:"_version,omitempty"`
 }

data/datum.go

@@ -15,7 +15,7 @@ type Datum interface {
 	Validate(validator structure.Validator)
 	Normalize(normalizer Normalizer)
 
-	IdentityFields() ([]string, error)
+	IdentityFields(version string) ([]string, error)
 
 	GetOrigin() *origin.Origin
 	GetPayload() *metadata.Metadata

data/deduplicator/deduplicator/base.go

@@ -24,8 +24,6 @@ func NewBase(name string, version string) (*Base, error) {
 	}
 	if version == "" {
 		return nil, errors.New("version is missing")
-	} else if !net.IsValidSemanticVersion(version) {
-		return nil, errors.New("version is invalid")
 	}
 
 	return &Base{

data/deduplicator/deduplicator/base_test.go

@@ -25,7 +25,10 @@ var _ = Describe("Base", func() {
 
 	BeforeEach(func() {
 		name = netTest.RandomReverseDomain()
-		version = netTest.RandomSemanticVersion()
+		version = test.RandomStringFromArray([]string{
+			dataDeduplicatorDeduplicator.DeviceDeactivateHashVersionCurrent,
+			dataDeduplicatorDeduplicator.DeviceDeactivateHashVersionLegacy,
+		})
 	})
 
 	Context("NewBase", func() {
@@ -50,13 +53,6 @@ var _ = Describe("Base", func() {
 			Expect(deduplicator).To(BeNil())
 		})
 
-		It("returns an error when version is invalid", func() {
-			version = "invalid"
-			deduplicator, err := dataDeduplicatorDeduplicator.NewBase(name, version)
-			Expect(err).To(MatchError("version is invalid"))
-			Expect(deduplicator).To(BeNil())
-		})
-
 		It("returns successfully", func() {
 			Expect(dataDeduplicatorDeduplicator.NewBase(name, version)).ToNot(BeNil())
 		})

data/deduplicator/deduplicator/device_deactivate_hash.go

@@ -5,8 +5,17 @@ import (
 
 	"github.com/tidepool-org/platform/data"
 	dataStore "github.com/tidepool-org/platform/data/store"
+	"github.com/tidepool-org/platform/data/types"
 	dataTypesUpload "github.com/tidepool-org/platform/data/types/upload"
 	"github.com/tidepool-org/platform/errors"
+	"github.com/tidepool-org/platform/page"
+	"github.com/tidepool-org/platform/pointer"
+)
+
+const (
+	DeviceDeactivateHashVersionUnknown = "unknown-hash"
+	DeviceDeactivateHashVersionLegacy  = "legacy-hash"
+	DeviceDeactivateHashVersionCurrent = "current-hash"
 )
 
 const DeviceDeactivateHashName = "org.tidepool.deduplicator.device.deactivate.hash"
@@ -18,12 +27,36 @@ var DeviceDeactivateHashDeviceManufacturerDeviceModels = map[string][]string{
 	"Trividia Health": {"TRUE METRIX", "TRUE METRIX AIR", "TRUE METRIX GO"},
 }
 
+var DeviceDeactivateLegacyHashManufacturerDeviceModels = map[string][]string{
+	"Arkray":    {"GlucocardExpression"},
+	"Bayer":     {"Contour Next Link", "Contour Next Link 2.4", "Contour Next", "Contour USB", "Contour Next USB", "Contour Next One", "Contour", "Contour Next EZ", "Contour Plus", "Contour Plus Blue"},
+	"Dexcom":    {"G5 touchscreen receiver", "G6 touchscreen receiver"},
+	"GlucoRx":   {"Nexus", "HCT", "Nexus Mini Ultra", "Go"},
+	"i-SENS":    {"CareSens"},
+	"MicroTech": {"Equil"},
+	"Roche":     {"Aviva Connect", "Performa Connect", "Guide", "Instant (single-button)", "Guide Me", "Instant (two-button)", "Instant S (single-button)", "ReliOn Platinum"},
+
+	"Insulet": {"Dash", "Eros", "OmniPod"},
+	"Tandem":  {"1002717", "5602", "5448004", "5448003", "5448001", "5448", "4628003", "4628", "10037177", "1001357", "1000354", "1000096"},
+}
+
 type DeviceDeactivateHash struct {
 	*Base
 }
 
+func NewDeviceDeactivateLegacyHash() (*DeviceDeactivateHash, error) {
+	base, err := NewBase(DeviceDeactivateHashName, DeviceDeactivateHashVersionLegacy)
+	if err != nil {
+		return nil, err
+	}
+
+	return &DeviceDeactivateHash{
+		Base: base,
+	}, nil
+}
+
 func NewDeviceDeactivateHash() (*DeviceDeactivateHash, error) {
-	base, err := NewBase(DeviceDeactivateHashName, "1.1.0")
+	base, err := NewBase(DeviceDeactivateHashName, DeviceDeactivateHashVersionCurrent)
 	if err != nil {
 		return nil, err
 	}
@@ -33,44 +66,70 @@ func NewDeviceDeactivateHash() (*DeviceDeactivateHash, error) {
 	}, nil
 }
 
+func getDeduplicatorVersion(dataSet *dataTypesUpload.Upload) string {
+	if dataSet.Deduplicator != nil {
+		if dataSet.Deduplicator.Name != nil && dataSet.Deduplicator.Version != nil {
+			if *dataSet.Deduplicator.Name == DeviceDeactivateHashName {
+				if types.LegacyIdentityFieldsVersion == *dataSet.Deduplicator.Version {
+					return DeviceDeactivateHashVersionLegacy
+				} else if types.IdentityFieldsVersion == *dataSet.Deduplicator.Version {
+					return DeviceDeactivateHashVersionCurrent
+				}
+			}
+		}
+	}
+	if dataSet.DeviceManufacturers != nil && dataSet.DeviceModel != nil {
+		for _, deviceManufacturer := range *dataSet.DeviceManufacturers {
+			if allowedDeviceModels, found := DeviceDeactivateLegacyHashManufacturerDeviceModels[deviceManufacturer]; found {
+				for _, allowedDeviceModel := range allowedDeviceModels {
+					if allowedDeviceModel == *dataSet.DeviceModel {
+						return DeviceDeactivateHashVersionLegacy
+					}
+				}
+			}
+		}
+		for _, deviceManufacturer := range *dataSet.DeviceManufacturers {
+			if allowedDeviceModels, found := DeviceDeactivateHashDeviceManufacturerDeviceModels[deviceManufacturer]; found {
+				for _, allowedDeviceModel := range allowedDeviceModels {
+					if allowedDeviceModel == *dataSet.DeviceModel {
+						return DeviceDeactivateHashVersionCurrent
+					}
+				}
+			}
+		}
+	}
+	return DeviceDeactivateHashVersionUnknown
+}
+
 func (d *DeviceDeactivateHash) New(ctx context.Context, dataSet *dataTypesUpload.Upload) (bool, error) {
 	if dataSet == nil {
 		return false, errors.New("data set is missing")
 	}
-
 	if !dataSet.HasDataSetTypeNormal() {
 		return false, nil
 	}
 	if dataSet.DeviceID == nil {
 		return false, nil
 	}
-
 	if dataSet.HasDeduplicatorName() {
 		return d.Get(ctx, dataSet)
 	}
+	return getDeduplicatorVersion(dataSet) != DeviceDeactivateHashVersionUnknown, nil
+}
 
-	if dataSet.DeviceManufacturers == nil || dataSet.DeviceModel == nil {
-		return false, nil
+func (d *DeviceDeactivateHash) Get(ctx context.Context, dataSet *dataTypesUpload.Upload) (bool, error) {
+	// NOTE: check legacy first then fallback to other matches
+	if dataSet == nil {
+		return false, errors.New("data set is missing")
 	}
 
-	for _, deviceManufacturer := range *dataSet.DeviceManufacturers {
-		if allowedDeviceModels, found := DeviceDeactivateHashDeviceManufacturerDeviceModels[deviceManufacturer]; found {
-			for _, allowedDeviceModel := range allowedDeviceModels {
-				if allowedDeviceModel == *dataSet.DeviceModel {
-					return true, nil
-				}
-			}
-		}
+	if getDeduplicatorVersion(dataSet) == DeviceDeactivateHashVersionLegacy {
+		return true, nil
 	}
 
-	return false, nil
-}
-
-func (d *DeviceDeactivateHash) Get(ctx context.Context, dataSet *dataTypesUpload.Upload) (bool, error) {
 	if found, err := d.Base.Get(ctx, dataSet); err != nil || found {
 		return found, err
 	}
-
 	return dataSet.HasDeduplicatorNameMatch("org.tidepool.hash-deactivate-old"), nil // TODO: DEPRECATED
 }
 
@@ -88,10 +147,27 @@ func (d *DeviceDeactivateHash) AddData(ctx context.Context, repository dataStore
 		return errors.New("data set data is missing")
 	}
 
-	if err := AssignDataSetDataIdentityHashes(dataSetData); err != nil {
-		return err
+	options := NewDefaultDeviceDeactivateHashOptions()
+
+	if getDeduplicatorVersion(dataSet) == DeviceDeactivateHashVersionLegacy {
+		filter := &data.DataSetFilter{LegacyOnly: pointer.FromBool(true), DeviceID: dataSet.DeviceID}
+		pagination := &page.Pagination{Page: 1, Size: 1}
+
+		uploads, err := repository.ListUserDataSets(ctx, *dataSet.UserID, filter, pagination)
+		if err != nil {
+			return errors.Wrap(err, "error getting datasets for user")
+		}
+		if len(uploads) != 0 {
+			if uploads[0].LegacyGroupID == nil {
+				return errors.New("missing required legacy groupId for the device deactive hash legacy version")
+			}
+			options = NewLegacyHashOptions(*uploads[0].LegacyGroupID)
+		}
 	}
 
+	if err := AssignDataSetDataIdentityHashes(dataSetData, options); err != nil {
+		return err
+	}
 	return d.Base.AddData(ctx, repository, dataSet, dataSetData)
 }