Allow legacy deduplication

data/blood/glucose/glucose.go

@@ -67,3 +67,14 @@ func NormalizeValueForUnits(value *float64, units *string) *float64 {
 	}
 	return value
 }
+
+func NormalizeValueForUnitsWithFullPrecision(value *float64, units *string) *float64 {
+	if value != nil && units != nil {
+		switch *units {
+		case MgdL, Mgdl:
+			floatValue := *value / MmolLToMgdLConversionFactor
+			return &floatValue
+		}
+	}
+	return value
+}

data/blood/glucose/glucose_test.go

@@ -125,4 +125,28 @@ var _ = Describe("Glucose", func() {
 			}
 		})
 	})
+	Context("NormalizeValueForUnitsWithFullPrecision", func() {
+		DescribeTable("given value and units",
+			func(value *float64, units *string, expectedValue *float64) {
+				actualValue := glucose.NormalizeValueForUnitsWithFullPrecision(value, units)
+				if expectedValue == nil {
+					Expect(actualValue).To(BeNil())
+				} else {
+					Expect(actualValue).ToNot(BeNil())
+					Expect(*actualValue).To(Equal(*expectedValue))
+				}
+			},
+			Entry("returns nil for nil value", nil, pointer.FromString("mmol/L"), nil),
+			Entry("returns unchanged value for nil units", pointer.FromFloat64(10.0), nil, pointer.FromFloat64(10.0)),
+			Entry("returns unchanged value for unknown units", pointer.FromFloat64(10.0), pointer.FromString("unknown"), pointer.FromFloat64(10.0)),
+			Entry("returns unchanged value for mmol/L units", pointer.FromFloat64(10.0), pointer.FromString("mmol/L"), pointer.FromFloat64(10.0)),
+			Entry("returns unchanged value for mmol/l units", pointer.FromFloat64(10.0), pointer.FromString("mmol/l"), pointer.FromFloat64(10.0)),
+
+			Entry("returns converted value for mg/dL units", pointer.FromFloat64(140.0), pointer.FromString("mg/dL"), pointer.FromFloat64(7.771047187463747)),
+			Entry("returns converted value for mg/dL units", pointer.FromFloat64(100.0), pointer.FromString("mg/dL"), pointer.FromFloat64(5.550747991045533)),
+			Entry("returns converted value for mg/dl units", pointer.FromFloat64(80.0), pointer.FromString("mg/dl"), pointer.FromFloat64(4.440598392836427)),
+			Entry("returns converted value for mg/dl units", pointer.FromFloat64(69.0), pointer.FromString("mg/dl"), pointer.FromFloat64(3.830016113821418)),
+			Entry("returns converted value for mg/dl units", pointer.FromFloat64(50.0), pointer.FromString("mg/dl"), pointer.FromFloat64(2.7753739955227665)),
+		)
+	})
 })

data/blood/glucose/test/glucose.go

@@ -1,10 +1,31 @@
 package test
 
 import (
+	"github.com/onsi/gomega"
+
 	dataBloodGlucose "github.com/tidepool-org/platform/data/blood/glucose"
+	"github.com/tidepool-org/platform/metadata"
 	"github.com/tidepool-org/platform/test"
 )
 
 func RandomUnits() string {
 	return test.RandomStringFromArray(dataBloodGlucose.Units())
 }
+
+func ExpectRaw(raw *metadata.Metadata, expectedRaw *metadata.Metadata) {
+	if expectedRaw != nil {
+		gomega.Expect(raw).ToNot(gomega.BeNil())
+		if expectedRaw.Get("units") == nil {
+			gomega.Expect(raw.Get("units")).To(gomega.BeNil())
+		} else {
+			gomega.Expect(raw.Get("units")).To(gomega.Equal(expectedRaw.Get("units")))
+		}
+		if expectedRaw.Get("value") == nil {
+			gomega.Expect(raw.Get("value")).To(gomega.BeNil())
+		} else {
+			gomega.Expect(raw.Get("value")).To(gomega.Equal(expectedRaw.Get("value")))
+		}
+	} else {
+		gomega.Expect(raw).To(gomega.BeNil())
+	}
+}

data/data_set.go

@@ -111,6 +111,7 @@ func (d *DataSetClient) Validate(validator structure.Validator) {
 
 type DataSetFilter struct {
 	ClientName *string
+	IsLegacy   *bool
 	Deleted    *bool
 	DeviceID   *string
 }
@@ -321,6 +322,7 @@ type DataSet struct {
 	Type                string                  `json:"type,omitempty" bson:"type,omitempty"`
 	UploadID            *string                 `json:"uploadId,omitempty" bson:"uploadId,omitempty"`
 	UserID              *string                 `json:"-" bson:"_userId,omitempty"`
+	LegacyGroupID       *string                 `json:"-" bson:"_groupId,omitempty"`
 	Version             *string                 `json:"version,omitempty" bson:"version,omitempty"`
 	VersionInternal     int                     `json:"-" bson:"_version,omitempty"`
 }

data/datum.go

@@ -16,6 +16,7 @@ type Datum interface {
 	Normalize(normalizer Normalizer)
 
 	IdentityFields() ([]string, error)
+	LegacyIdentityFields() ([]string, error)
 
 	GetOrigin() *origin.Origin
 	GetPayload() *metadata.Metadata

data/deduplicator/deduplicator/device_deactivate_hash.go

@@ -7,6 +7,16 @@ import (
 	dataStore "github.com/tidepool-org/platform/data/store"
 	dataTypesUpload "github.com/tidepool-org/platform/data/types/upload"
 	"github.com/tidepool-org/platform/errors"
+	"github.com/tidepool-org/platform/page"
+	"github.com/tidepool-org/platform/pointer"
+)
+
+type DeviceDeactivateHashVersion string
+
+const (
+	DeviceDeactivateHashVersionUnkown  DeviceDeactivateHashVersion = ""
+	DeviceDeactivateHashVersionCurrent DeviceDeactivateHashVersion = "1.1.0"
+	DeviceDeactivateHashVersionLegacy  DeviceDeactivateHashVersion = "0.0.0"
 )
 
 const DeviceDeactivateHashName = "org.tidepool.deduplicator.device.deactivate.hash"
@@ -18,12 +28,36 @@ var DeviceDeactivateHashDeviceManufacturerDeviceModels = map[string][]string{
 	"Trividia Health": {"TRUE METRIX", "TRUE METRIX AIR", "TRUE METRIX GO"},
 }
 
+var DeviceDeactivateLegacyHashManufacturerDeviceModels = map[string][]string{
+	"Arkray":    {"GlucocardExpression"},
+	"Bayer":     {"Contour Next Link", "Contour Next Link 2.4", "Contour Next", "Contour USB", "Contour Next USB", "Contour Next One", "Contour", "Contour Next EZ", "Contour Plus", "Contour Plus Blue"},
+	"Dexcom":    {"G5 touchscreen receiver", "G6 touchscreen receiver"},
+	"GlucoRx":   {"Nexus", "HCT", "Nexus Mini Ultra", "Go"},
+	"i-SENS":    {"CareSens"},
+	"MicroTech": {"Equil"},
+	"Roche":     {"Aviva Connect", "Performa Connect", "Guide", "Instant (single-button)", "Guide Me", "Instant (two-button)", "Instant S (single-button)", "ReliOn Platinum"},
+
+	"Insulet": {"Dash", "Eros", "OmniPod"},
+	"Tandem":  {"1002717", "5602", "5448004", "5448003", "5448001", "5448", "4628003", "4628", "10037177", "1001357", "1000354", "1000096"},
+}
+
 type DeviceDeactivateHash struct {
 	*Base
 }
 
+func NewDeviceDeactivateLegacyHash() (*DeviceDeactivateHash, error) {
+	base, err := NewBase(DeviceDeactivateHashName, string(DeviceDeactivateHashVersionLegacy))
+	if err != nil {
+		return nil, err
+	}
+
+	return &DeviceDeactivateHash{
+		Base: base,
+	}, nil
+}
+
 func NewDeviceDeactivateHash() (*DeviceDeactivateHash, error) {
-	base, err := NewBase(DeviceDeactivateHashName, "1.1.0")
+	base, err := NewBase(DeviceDeactivateHashName, string(DeviceDeactivateHashVersionCurrent))
 	if err != nil {
 		return nil, err
 	}
@@ -33,44 +67,70 @@ func NewDeviceDeactivateHash() (*DeviceDeactivateHash, error) {
 	}, nil
 }
 
+func getDeviceDeactivateHashVersion(dataSet *dataTypesUpload.Upload) DeviceDeactivateHashVersion {
+	if dataSet.Deduplicator != nil {
+		if dataSet.Deduplicator.Name != nil && dataSet.Deduplicator.Version != nil {
+			if *dataSet.Deduplicator.Name == DeviceDeactivateHashName {
+				if *dataSet.Deduplicator.Version == string(DeviceDeactivateHashVersionLegacy) {
+					return DeviceDeactivateHashVersionLegacy
+				} else if *dataSet.Deduplicator.Version == string(DeviceDeactivateHashVersionCurrent) {
+					return DeviceDeactivateHashVersionCurrent
+				}
+			}
+		}
+	}
+	if dataSet.DeviceManufacturers != nil && dataSet.DeviceModel != nil {
+		for _, deviceManufacturer := range *dataSet.DeviceManufacturers {
+			if allowedDeviceModels, found := DeviceDeactivateLegacyHashManufacturerDeviceModels[deviceManufacturer]; found {
+				for _, allowedDeviceModel := range allowedDeviceModels {
+					if allowedDeviceModel == *dataSet.DeviceModel {
+						return DeviceDeactivateHashVersionLegacy
+					}
+				}
+			}
+		}
+		for _, deviceManufacturer := range *dataSet.DeviceManufacturers {
+			if allowedDeviceModels, found := DeviceDeactivateHashDeviceManufacturerDeviceModels[deviceManufacturer]; found {
+				for _, allowedDeviceModel := range allowedDeviceModels {
+					if allowedDeviceModel == *dataSet.DeviceModel {
+						return DeviceDeactivateHashVersionCurrent
+					}
+				}
+			}
+		}
+	}
+	return DeviceDeactivateHashVersionUnkown
+}
+
 func (d *DeviceDeactivateHash) New(dataSet *dataTypesUpload.Upload) (bool, error) {
 	if dataSet == nil {
 		return false, errors.New("data set is missing")
 	}
-
 	if !dataSet.HasDataSetTypeNormal() {
 		return false, nil
 	}
 	if dataSet.DeviceID == nil {
 		return false, nil
 	}
-
 	if dataSet.HasDeduplicatorName() {
 		return d.Get(dataSet)
 	}
+	return getDeviceDeactivateHashVersion(dataSet) != DeviceDeactivateHashVersionUnkown, nil
+}
 
-	if dataSet.DeviceManufacturers == nil || dataSet.DeviceModel == nil {
-		return false, nil
+func (d *DeviceDeactivateHash) Get(dataSet *dataTypesUpload.Upload) (bool, error) {
+	// NOTE: check legacy first then fallback to other matches
+	if dataSet == nil {
+		return false, errors.New("data set is missing")
 	}
 
-	for _, deviceManufacturer := range *dataSet.DeviceManufacturers {
-		if allowedDeviceModels, found := DeviceDeactivateHashDeviceManufacturerDeviceModels[deviceManufacturer]; found {
-			for _, allowedDeviceModel := range allowedDeviceModels {
-				if allowedDeviceModel == *dataSet.DeviceModel {
-					return true, nil
-				}
-			}
-		}
+	if getDeviceDeactivateHashVersion(dataSet) == DeviceDeactivateHashVersionLegacy {
+		return true, nil
 	}
 
-	return false, nil
-}
-
-func (d *DeviceDeactivateHash) Get(dataSet *dataTypesUpload.Upload) (bool, error) {
 	if found, err := d.Base.Get(dataSet); err != nil || found {
 		return found, err
 	}
-
 	return dataSet.HasDeduplicatorNameMatch("org.tidepool.hash-deactivate-old"), nil // TODO: DEPRECATED
 }
 
@@ -88,10 +148,26 @@ func (d *DeviceDeactivateHash) AddData(ctx context.Context, repository dataStore
 		return errors.New("data set data is missing")
 	}
 
-	if err := AssignDataSetDataIdentityHashes(dataSetData); err != nil {
-		return err
+	options := NewDefaultDeviceDeactivateHashOptions()
+
+	if getDeviceDeactivateHashVersion(dataSet) == DeviceDeactivateHashVersionLegacy {
+		filter := &data.DataSetFilter{IsLegacy: pointer.FromBool(true), DeviceID: dataSet.DeviceID}
+		pagination := &page.Pagination{Page: 1, Size: 1}
+
+		uploads, err := repository.ListUserDataSets(ctx, *dataSet.UserID, filter, pagination)
+		if err != nil {
+			return errors.Wrap(err, "error getting datasets for user")
+		}
+		if len(uploads) != 0 {
+			if uploads[0].LegacyGroupID != nil {
+				options = NewLegacyDeviceDeactivateHashOptions(*uploads[0].LegacyGroupID)
+			}
+		}
 	}
 
+	if err := AssignDataSetDataIdentityHashes(dataSetData, options); err != nil {
+		return err
+	}
 	return d.Base.AddData(ctx, repository, dataSet, dataSetData)
 }
 

data/deduplicator/deduplicator/device_deactivate_hash_test.go

@@ -64,6 +64,52 @@ var _ = Describe("DeviceDeactivateHash", func() {
 				Expect(deduplicator.New(dataSet)).To(BeFalse())
 			})
 
+			It("returns false when the deduplicator name does not match", func() {
+				dataSet.Deduplicator.Name = pointer.FromString(netTest.RandomReverseDomain())
+				Expect(deduplicator.New(dataSet)).To(BeFalse())
+			})
+
+			DescribeTable("returns true when",
+				func(deviceManufacturer string, deviceModel string) {
+					dataSet.DeviceManufacturers = pointer.FromStringArray([]string{deviceManufacturer})
+					dataSet.DeviceModel = pointer.FromString(deviceModel)
+					Expect(deduplicator.New(dataSet)).To(BeTrue())
+				},
+				Entry("is Abbott FreeStyle Libre", "Abbott", "FreeStyle Libre"),
+				Entry("is LifeScan OneTouch Ultra 2", "LifeScan", "OneTouch Ultra 2"),
+				Entry("is LifeScan OneTouch UltraMini", "LifeScan", "OneTouch UltraMini"),
+				Entry("is LifeScan Verio", "LifeScan", "Verio"),
+				Entry("is LifeScan Verio Flex", "LifeScan", "Verio Flex"),
+				Entry("is Medtronic 523", "Medtronic", "523"),
+				Entry("is Medtronic 523K", "Medtronic", "523K"),
+				Entry("is Medtronic 551", "Medtronic", "551"),
+				Entry("is Medtronic 554", "Medtronic", "554"),
+				Entry("is Medtronic 723", "Medtronic", "723"),
+				Entry("is Medtronic 723K", "Medtronic", "723K"),
+				Entry("is Medtronic 751", "Medtronic", "751"),
+				Entry("is Medtronic 754", "Medtronic", "754"),
+				Entry("is Medtronic 1510", "Medtronic", "1510"),
+				Entry("is Medtronic 1510K", "Medtronic", "1510K"),
+				Entry("is Medtronic 1511", "Medtronic", "1511"),
+				Entry("is Medtronic 1512", "Medtronic", "1512"),
+				Entry("is Medtronic 1580", "Medtronic", "1580"),
+				Entry("is Medtronic 1581", "Medtronic", "1581"),
+				Entry("is Medtronic 1582", "Medtronic", "1582"),
+				Entry("is Medtronic 1710", "Medtronic", "1710"),
+				Entry("is Medtronic 1710K", "Medtronic", "1710K"),
+				Entry("is Medtronic 1711", "Medtronic", "1711"),
+				Entry("is Medtronic 1712", "Medtronic", "1712"),
+				Entry("is Medtronic 1714", "Medtronic", "1714"),
+				Entry("is Medtronic 1714K", "Medtronic", "1714K"),
+				Entry("is Medtronic 1715", "Medtronic", "1715"),
+				Entry("is Medtronic 1780", "Medtronic", "1780"),
+				Entry("is Medtronic 1781", "Medtronic", "1781"),
+				Entry("is Medtronic 1782", "Medtronic", "1782"),
+				Entry("is Trividia Health TRUE METRIX", "Trividia Health", "TRUE METRIX"),
+				Entry("is Trividia Health TRUE METRIX AIR", "Trividia Health", "TRUE METRIX AIR"),
+				Entry("is Trividia Health TRUE METRIX GO", "Trividia Health", "TRUE METRIX GO"),
+			)
+
 			dataSetTypeAssertions := func() {
 				It("returns false when the deduplicator name does not match", func() {
 					dataSet.Deduplicator.Name = pointer.FromString(netTest.RandomReverseDomain())
@@ -213,6 +259,13 @@ var _ = Describe("DeviceDeactivateHash", func() {
 				dataSet.Deduplicator.Name = pointer.FromString("org.tidepool.hash-deactivate-old")
 				Expect(deduplicator.Get(dataSet)).To(BeTrue())
 			})
+
+			It("returns true when the deduplicator name matches and legacy id device and model", func() {
+				dataSet.Deduplicator.Name = pointer.FromString(dataDeduplicatorDeduplicator.DeviceDeactivateHashName)
+				dataSet.DeviceManufacturers = pointer.FromStringArray([]string{"Tandem"})
+				dataSet.DeviceModel = pointer.FromString("1002717")
+				Expect(deduplicator.Get(dataSet)).To(BeTrue())
+			})
 		})
 
 		Context("with context and repository", func() {