From 3acfe5b3499e1e91d3b3ea644a5be88e978ef8b5 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Mon, 12 Aug 2024 15:28:12 +0200
Subject: [PATCH 01/19] stores: remove redundant db types

---
 stores/metadata.go      | 86 -----------------------------------------
 stores/metadata_test.go | 19 ++++-----
 stores/sql_test.go      | 12 ++----
 stores/types.go         | 29 --------------
 4 files changed, 12 insertions(+), 134 deletions(-)

diff --git a/stores/metadata.go b/stores/metadata.go
index 5b666efde..3374c73a5 100644
--- a/stores/metadata.go
+++ b/stores/metadata.go
@@ -15,7 +15,6 @@ import (
 	"go.sia.tech/renterd/object"
 	sql "go.sia.tech/renterd/stores/sql"
 	"go.uber.org/zap"
-	"gorm.io/gorm"
 	"lukechampine.com/frand"
 )
 
@@ -57,29 +56,6 @@ var objectDeleteBatchSizes = []int64{10, 50, 100, 200, 500, 1000, 5000, 10000, 5
 type (
 	contractState uint8
 
-	ContractCommon struct {
-		FCID        fileContractID `gorm:"unique;index;NOT NULL;column:fcid;size:32"`
-		RenewedFrom fileContractID `gorm:"index;size:32"`
-
-		ContractPrice  currency
-		State          contractState `gorm:"index;NOT NULL;default:0"`
-		TotalCost      currency
-		ProofHeight    uint64 `gorm:"index;default:0"`
-		RevisionHeight uint64 `gorm:"index;default:0"`
-		RevisionNumber string `gorm:"NOT NULL;default:'0'"` // string since db can't store math.MaxUint64
-		Size           uint64
-		StartHeight    uint64 `gorm:"index;NOT NULL"`
-		WindowStart    uint64 `gorm:"index;NOT NULL;default:0"`
-		WindowEnd      uint64 `gorm:"index;NOT NULL;default:0"`
-
-		// spending fields
-		UploadSpending      currency
-		DownloadSpending    currency
-		FundAccountSpending currency
-		DeleteSpending      currency
-		ListSpending        currency
-	}
-
 	dbObject struct {
 		Model
 
@@ -157,43 +133,6 @@ type (
 		DBSectorID   uint `gorm:"primaryKey;index"`
 		DBContractID uint `gorm:"primaryKey;index"`
 	}
-
-	// rawObject is used for hydration and is made up of one or many raw sectors.
-	rawObject []rawObjectSector
-
-	// rawObjectRow contains all necessary information to reconstruct the object.
-	rawObjectSector struct {
-		// object
-		ObjectID       uint
-		ObjectIndex    uint64
-		ObjectKey      []byte
-		ObjectName     string
-		ObjectSize     int64
-		ObjectModTime  time.Time
-		ObjectMimeType string
-		ObjectHealth   float64
-		ObjectETag     string
-
-		// slice
-		SliceOffset uint32
-		SliceLength uint32
-
-		// slab
-		SlabBuffered  bool
-		SlabID        uint
-		SlabHealth    float64
-		SlabKey       []byte
-		SlabMinShards uint8
-
-		// sector
-		SectorIndex uint
-		SectorRoot  []byte
-		LatestHost  publicKey
-
-		// contract
-		FCID    fileContractID
-		HostKey publicKey
-	}
 )
 
 func (s *contractState) LoadString(state string) error {
@@ -865,31 +804,6 @@ func (s *SQLStore) ObjectMetadata(ctx context.Context, bucket, path string) (obj
 	return
 }
 
-func (s *SQLStore) objectRaw(txn *gorm.DB, bucket string, path string) (rows rawObject, err error) {
-	// NOTE: we LEFT JOIN here because empty objects are valid and need to be
-	// included in the result set, when we convert the rawObject before
-	// returning it we'll check for SlabID and/or SectorID being 0 and act
-	// accordingly
-	err = txn.
-		Select("o.id as ObjectID, o.health as ObjectHealth, sli.object_index as ObjectIndex, o.key as ObjectKey, o.object_id as ObjectName, o.size as ObjectSize, o.mime_type as ObjectMimeType, o.created_at as ObjectModTime, o.etag as ObjectETag, sli.object_index, sli.offset as SliceOffset, sli.length as SliceLength, sla.id as SlabID, sla.health as SlabHealth, sla.key as SlabKey, sla.min_shards as SlabMinShards, bs.id IS NOT NULL AS SlabBuffered, sec.slab_index as SectorIndex, sec.root as SectorRoot, sec.latest_host as LatestHost, c.fcid as FCID, h.public_key as HostKey").
-		Model(&dbObject{}).
-		Table("objects o").
-		Joins("INNER JOIN buckets b ON o.db_bucket_id = b.id").
-		Joins("LEFT JOIN slices sli ON o.id = sli.`db_object_id`").
-		Joins("LEFT JOIN slabs sla ON sli.db_slab_id = sla.`id`").
-		Joins("LEFT JOIN sectors sec ON sla.id = sec.`db_slab_id`").
-		Joins("LEFT JOIN contract_sectors cs ON sec.id = cs.`db_sector_id`").
-		Joins("LEFT JOIN contracts c ON cs.`db_contract_id` = c.`id`").
-		Joins("LEFT JOIN hosts h ON c.host_id = h.id").
-		Joins("LEFT JOIN buffered_slabs bs ON sla.db_buffered_slab_id = bs.`id`").
-		Where("o.object_id = ? AND b.name = ?", path, bucket).
-		Order("sli.object_index ASC").
-		Order("sec.slab_index ASC").
-		Scan(&rows).
-		Error
-	return
-}
-
 // PackedSlabsForUpload returns up to 'limit' packed slabs that are ready for
 // uploading. They are locked for 'lockingDuration' time before being handed out
 // again.
diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index 05b4e3789..c63827ce3 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -1064,7 +1064,7 @@ func TestSQLMetadataStore(t *testing.T) {
 	one := uint(1)
 	expectedObj := dbObject{
 		DBDirectoryID: 1,
-		DBBucketID:    ss.DefaultBucketID(),
+		DBBucketID:    uint(ss.DefaultBucketID()),
 		Health:        1,
 		ObjectID:      objID,
 		Key:           obj1Key,
@@ -3414,16 +3414,13 @@ func TestBucketObjects(t *testing.T) {
 	}
 
 	// See if we can fetch the object by slab.
-	var ec object.EncryptionKey
-	if obj, err := ss.objectRaw(ss.gormDB, b1, "/bar"); err != nil {
-		t.Fatal(err)
-	} else if err := ec.UnmarshalBinary(obj[0].SlabKey); err != nil {
+	if obj, err := ss.Object(context.Background(), b1, "/bar"); err != nil {
 		t.Fatal(err)
-	} else if objects, err := ss.ObjectsBySlabKey(context.Background(), b1, ec); err != nil {
+	} else if objects, err := ss.ObjectsBySlabKey(context.Background(), b1, obj.Slabs[0].Key); err != nil {
 		t.Fatal(err)
 	} else if len(objects) != 1 {
 		t.Fatal("expected 1 object", len(objects))
-	} else if objects, err := ss.ObjectsBySlabKey(context.Background(), b2, ec); err != nil {
+	} else if objects, err := ss.ObjectsBySlabKey(context.Background(), b2, obj.Slabs[0].Key); err != nil {
 		t.Fatal(err)
 	} else if len(objects) != 0 {
 		t.Fatal("expected 0 objects", len(objects))
@@ -4194,7 +4191,7 @@ func TestSlabCleanup(t *testing.T) {
 	obj1 := dbObject{
 		DBDirectoryID: uint(dirID),
 		ObjectID:      "1",
-		DBBucketID:    ss.DefaultBucketID(),
+		DBBucketID:    uint(ss.DefaultBucketID()),
 		Health:        1,
 	}
 	if err := ss.gormDB.Create(&obj1).Error; err != nil {
@@ -4203,7 +4200,7 @@ func TestSlabCleanup(t *testing.T) {
 	obj2 := dbObject{
 		DBDirectoryID: uint(dirID),
 		ObjectID:      "2",
-		DBBucketID:    ss.DefaultBucketID(),
+		DBBucketID:    uint(ss.DefaultBucketID()),
 		Health:        1,
 	}
 	if err := ss.gormDB.Create(&obj2).Error; err != nil {
@@ -4277,7 +4274,7 @@ func TestSlabCleanup(t *testing.T) {
 	obj3 := dbObject{
 		DBDirectoryID: uint(dirID),
 		ObjectID:      "3",
-		DBBucketID:    ss.DefaultBucketID(),
+		DBBucketID:    uint(ss.DefaultBucketID()),
 		Health:        1,
 	}
 	if err := ss.gormDB.Create(&obj3).Error; err != nil {
@@ -4367,7 +4364,7 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 		t.Fatal(err)
 	} else if dbObj.ID != 1 {
 		t.Fatal("unexpected id", dbObj.ID)
-	} else if dbObj.DBBucketID != ss.DefaultBucketID() {
+	} else if dbObj.DBBucketID != uint(ss.DefaultBucketID()) {
 		t.Fatal("bucket id mismatch", dbObj.DBBucketID)
 	} else if dbObj.ObjectID != "1" {
 		t.Fatal("object id mismatch", dbObj.ObjectID)
diff --git a/stores/sql_test.go b/stores/sql_test.go
index cc09e064b..0bd89710f 100644
--- a/stores/sql_test.go
+++ b/stores/sql_test.go
@@ -219,16 +219,12 @@ func (s *testSQLStore) Close() error {
 	return nil
 }
 
-func (s *testSQLStore) DefaultBucketID() uint {
-	var b dbBucket
-	if err := s.gormDB.
-		Model(&dbBucket{}).
-		Where("name = ?", api.DefaultBucketName).
-		Take(&b).
-		Error; err != nil {
+func (s *testSQLStore) DefaultBucketID() (id int64) {
+	if err := s.DB().QueryRow(context.Background(), "SELECT id FROM buckets WHERE name = ?", api.DefaultBucketName).
+		Scan(&id); err != nil {
 		s.t.Fatal(err)
 	}
-	return b.ID
+	return
 }
 
 func (s *testSQLStore) Reopen() *testSQLStore {
diff --git a/stores/types.go b/stores/types.go
index 753d9cf6d..93da48db0 100644
--- a/stores/types.go
+++ b/stores/types.go
@@ -19,7 +19,6 @@ const (
 
 type (
 	unixTimeMS     time.Time
-	currency       types.Currency
 	bCurrency      types.Currency
 	fileContractID types.FileContractID
 	publicKey      types.PublicKey
@@ -114,34 +113,6 @@ func (fcid fileContractID) Value() (driver.Value, error) {
 	return fcid[:], nil
 }
 
-func (currency) GormDataType() string {
-	return "string"
-}
-
-// Scan scan value into currency, implements sql.Scanner interface.
-func (c *currency) Scan(value interface{}) error {
-	var s string
-	switch value := value.(type) {
-	case string:
-		s = value
-	case []byte:
-		s = string(value)
-	default:
-		return fmt.Errorf("failed to unmarshal currency value: %v %t", value, value)
-	}
-	curr, err := types.ParseCurrency(s)
-	if err != nil {
-		return err
-	}
-	*c = currency(curr)
-	return nil
-}
-
-// Value returns a publicKey value, implements driver.Valuer interface.
-func (c currency) Value() (driver.Value, error) {
-	return types.Currency(c).ExactString(), nil
-}
-
 func (publicKey) GormDataType() string {
 	return "bytes"
 }

From 57ab28d83ab90ef1a3a8e9cfab00bf311f247afa Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Mon, 12 Aug 2024 15:45:58 +0200
Subject: [PATCH 02/19] stores: remove dbContractSector

---
 stores/metadata.go      |  9 -----
 stores/metadata_test.go | 79 ++++++++++++++++++-----------------------
 2 files changed, 35 insertions(+), 53 deletions(-)

diff --git a/stores/metadata.go b/stores/metadata.go
index 3374c73a5..55bd24db7 100644
--- a/stores/metadata.go
+++ b/stores/metadata.go
@@ -127,12 +127,6 @@ type (
 		LatestHost publicKey `gorm:"NOT NULL"`
 		Root       []byte    `gorm:"index;unique;NOT NULL;size:32"`
 	}
-
-	// dbContractSector is a join table between dbContract and dbSector.
-	dbContractSector struct {
-		DBSectorID   uint `gorm:"primaryKey;index"`
-		DBContractID uint `gorm:"primaryKey;index"`
-	}
 )
 
 func (s *contractState) LoadString(state string) error {
@@ -177,9 +171,6 @@ func (s dbSlab) HealthValid() bool {
 // TableName implements the gorm.Tabler interface.
 func (dbBucket) TableName() string { return "buckets" }
 
-// TableName implements the gorm.Tabler interface.
-func (dbContractSector) TableName() string { return "contract_sectors" }
-
 // TableName implements the gorm.Tabler interface.
 func (dbObject) TableName() string { return "objects" }
 
diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index c63827ce3..53e931d2b 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -1940,7 +1940,7 @@ func TestUnhealthySlabsNoContracts(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := ss.gormDB.Table("contract_sectors").Where("TRUE").Delete(&dbContractSector{}).Error; err != nil {
+	if _, err := ss.DB().Exec(context.Background(), "DELETE FROM contract_sectors"); err != nil {
 		t.Fatal(err)
 	}
 
@@ -2078,12 +2078,8 @@ func TestContractSectors(t *testing.T) {
 	}
 
 	// Check the join table. Should be empty.
-	var css []dbContractSector
-	if err := ss.gormDB.Find(&css).Error; err != nil {
-		t.Fatal(err)
-	}
-	if len(css) != 0 {
-		t.Fatal("table should be empty", len(css))
+	if n := ss.Count("contract_sectors"); n != 0 {
+		t.Fatal("table should be empty", n)
 	}
 
 	// Add the contract back.
@@ -2105,12 +2101,8 @@ func TestContractSectors(t *testing.T) {
 	// Delete the sector.
 	if err := ss.gormDB.Delete(&dbSector{Model: Model{ID: 1}}).Error; err != nil {
 		t.Fatal(err)
-	}
-	if err := ss.gormDB.Find(&css).Error; err != nil {
-		t.Fatal(err)
-	}
-	if len(css) != 0 {
-		t.Fatal("table should be empty")
+	} else if n := ss.Count("contract_sectors"); n != 0 {
+		t.Fatal("table should be empty", n)
 	}
 }
 
@@ -2627,11 +2619,6 @@ func TestObjectsStats(t *testing.T) {
 
 	// Get all entries in contract_sectors and store them again with a different
 	// contract id. This should cause the uploaded size to double.
-	var contractSectors []dbContractSector
-	err = ss.gormDB.Find(&contractSectors).Error
-	if err != nil {
-		t.Fatal(err)
-	}
 	var newContractID types.FileContractID
 	frand.Read(newContractID[:])
 	hks, err := ss.addTestHosts(1)
@@ -2644,10 +2631,14 @@ func TestObjectsStats(t *testing.T) {
 		t.Fatal(err)
 	}
 	totalUploadedSize += c.Size
-	for _, contractSector := range contractSectors {
-		if _, err := ss.DB().Exec(context.Background(), "INSERT INTO contract_sectors (db_contract_id, db_sector_id) VALUES ((SELECT id FROM contracts WHERE fcid = ?), ?)", sql.FileContractID(newContractID), contractSector.DBSectorID); err != nil {
-			t.Fatal(err)
-		}
+	if _, err := ss.DB().Exec(context.Background(), `
+		INSERT INTO contract_sectors (db_contract_id, db_sector_id)
+		SELECT (
+			SELECT id FROM contracts WHERE fcid = ?
+		), db_sector_id
+		FROM contract_sectors
+	`, sql.FileContractID(newContractID)); err != nil {
+		t.Fatal(err)
 	}
 
 	// Check sizes.
@@ -3543,12 +3534,7 @@ func TestMarkSlabUploadedAfterRenew(t *testing.T) {
 	})
 	if err != nil {
 		t.Fatal(err)
-	}
-
-	var count int64
-	if err := ss.gormDB.Model(&dbContractSector{}).Count(&count).Error; err != nil {
-		t.Fatal(err)
-	} else if count != 1 {
+	} else if count := ss.Count("contract_sectors"); count != 1 {
 		t.Fatal("expected 1 sector", count)
 	}
 }
@@ -3696,12 +3682,7 @@ func TestDeleteHostSector(t *testing.T) {
 	})
 
 	// Make sure 4 contractSector entries exist.
-	var n int64
-	if err := ss.gormDB.Model(&dbContractSector{}).
-		Count(&n).
-		Error; err != nil {
-		t.Fatal(err)
-	} else if n != 4 {
+	if n := ss.Count("contract_sectors"); n != 4 {
 		t.Fatal("expected 4 contract-sector links", n)
 	}
 
@@ -3713,11 +3694,7 @@ func TestDeleteHostSector(t *testing.T) {
 	}
 
 	// Make sure 2 contractSector entries exist.
-	if err := ss.gormDB.Model(&dbContractSector{}).
-		Count(&n).
-		Error; err != nil {
-		t.Fatal(err)
-	} else if n != 2 {
+	if n := ss.Count("contract_sectors"); n != 2 {
 		t.Fatal("expected 2 contract-sector links", n)
 	}
 
@@ -4531,16 +4508,30 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 		t.Fatal("wrong slab")
 	}
 
-	var contractSectors []dbContractSector
-	if err := ss.gormDB.Find(&contractSectors).Error; err != nil {
+	type contractSector struct {
+		ContractID int64
+		SectorID   int64
+	}
+	var contractSectors []contractSector
+	rows, err := ss.DB().Query(context.Background(), "SELECT db_contract_id, db_sector_id FROM contract_sectors")
+	if err != nil {
 		t.Fatal(err)
-	} else if len(contractSectors) != 3*totalShards {
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var cs contractSector
+		if err := rows.Scan(&cs.ContractID, &cs.SectorID); err != nil {
+			t.Fatal(err)
+		}
+		contractSectors = append(contractSectors, cs)
+	}
+	if len(contractSectors) != 3*totalShards {
 		t.Fatal("invalid number of contract sectors", len(contractSectors))
 	}
 	for i, cs := range contractSectors {
-		if cs.DBContractID != uint(i+1) {
+		if cs.ContractID != int64(i+1) {
 			t.Fatal("invalid contract id")
-		} else if cs.DBSectorID != uint(i+1) {
+		} else if cs.SectorID != int64(i+1) {
 			t.Fatal("invalid sector id")
 		}
 	}

From af89516a088ff56b467a9cb0cd5e1c324d415e12 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Mon, 12 Aug 2024 16:32:54 +0200
Subject: [PATCH 03/19] stores: remove contractState

---
 stores/metadata.go      | 45 -----------------------------------------
 stores/metadata_test.go |  2 +-
 2 files changed, 1 insertion(+), 46 deletions(-)

diff --git a/stores/metadata.go b/stores/metadata.go
index 55bd24db7..a3eeb0dbc 100644
--- a/stores/metadata.go
+++ b/stores/metadata.go
@@ -38,14 +38,6 @@ const (
 	refreshHealthMaxHealthValidity = 72 * time.Hour
 )
 
-const (
-	contractStateInvalid contractState = iota
-	contractStatePending
-	contractStateActive
-	contractStateComplete
-	contractStateFailed
-)
-
 var (
 	pruneSlabsAlertID = frand.Entropy256()
 	pruneDirsAlertID  = frand.Entropy256()
@@ -54,8 +46,6 @@ var (
 var objectDeleteBatchSizes = []int64{10, 50, 100, 200, 500, 1000, 5000, 10000, 50000, 100000}
 
 type (
-	contractState uint8
-
 	dbObject struct {
 		Model
 
@@ -129,41 +119,6 @@ type (
 	}
 )
 
-func (s *contractState) LoadString(state string) error {
-	switch strings.ToLower(state) {
-	case api.ContractStateInvalid:
-		*s = contractStateInvalid
-	case api.ContractStatePending:
-		*s = contractStatePending
-	case api.ContractStateActive:
-		*s = contractStateActive
-	case api.ContractStateComplete:
-		*s = contractStateComplete
-	case api.ContractStateFailed:
-		*s = contractStateFailed
-	default:
-		*s = contractStateInvalid
-	}
-	return nil
-}
-
-func (s contractState) String() string {
-	switch s {
-	case contractStateInvalid:
-		return api.ContractStateInvalid
-	case contractStatePending:
-		return api.ContractStatePending
-	case contractStateActive:
-		return api.ContractStateActive
-	case contractStateComplete:
-		return api.ContractStateComplete
-	case contractStateFailed:
-		return api.ContractStateFailed
-	default:
-		return api.ContractStateUnknown
-	}
-}
-
 func (s dbSlab) HealthValid() bool {
 	return time.Now().Before(time.Unix(s.HealthValidUntil, 0))
 }
diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index 53e931d2b..305c5c351 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -768,7 +768,7 @@ func TestRenewedContract(t *testing.T) {
 		RevisionNumber: 1,
 		Size:           rhpv2.SectorSize,
 		StartHeight:    100,
-		State:          contractStatePending.String(),
+		State:          api.ContractStatePending,
 		TotalCost:      oldContractTotal,
 		WindowStart:    2,
 		WindowEnd:      3,

From 9a3fc4364118a3bfd0012bdd8e3b52b0f57cdb58 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Mon, 12 Aug 2024 16:35:57 +0200
Subject: [PATCH 04/19] stores: remove dbBucket

---
 stores/metadata.go | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/stores/metadata.go b/stores/metadata.go
index a3eeb0dbc..06feb82ac 100644
--- a/stores/metadata.go
+++ b/stores/metadata.go
@@ -51,8 +51,7 @@ type (
 
 		DBDirectoryID uint
 
-		DBBucketID uint `gorm:"index;uniqueIndex:idx_object_bucket;NOT NULL"`
-		DBBucket   dbBucket
+		DBBucketID uint   `gorm:"index;uniqueIndex:idx_object_bucket;NOT NULL"`
 		ObjectID   string `gorm:"index;uniqueIndex:idx_object_bucket"`
 
 		Key      secretKey
@@ -74,13 +73,6 @@ type (
 		Value               string
 	}
 
-	dbBucket struct {
-		Model
-
-		Policy api.BucketPolicy `gorm:"serializer:json"`
-		Name   string           `gorm:"unique;index;NOT NULL"`
-	}
-
 	dbSlice struct {
 		Model
 		DBObjectID        *uint `gorm:"index"`
@@ -123,9 +115,6 @@ func (s dbSlab) HealthValid() bool {
 	return time.Now().Before(time.Unix(s.HealthValidUntil, 0))
 }
 
-// TableName implements the gorm.Tabler interface.
-func (dbBucket) TableName() string { return "buckets" }
-
 // TableName implements the gorm.Tabler interface.
 func (dbObject) TableName() string { return "objects" }
 

From bbeb1c3c09493b0c9ac3e336e215c4ba6714bcc8 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Mon, 12 Aug 2024 16:58:36 +0200
Subject: [PATCH 05/19] stores: update TestSQLMetadataStore

---
 stores/metadata_test.go | 134 +++++++++++++++++++++-------------------
 1 file changed, 70 insertions(+), 64 deletions(-)

diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index 305c5c351..7a262b0e9 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -23,7 +23,6 @@ import (
 	"go.sia.tech/renterd/object"
 	sql "go.sia.tech/renterd/stores/sql"
 	"gorm.io/gorm"
-	"gorm.io/gorm/schema"
 	"lukechampine.com/frand"
 )
 
@@ -1041,55 +1040,74 @@ func TestSQLMetadataStore(t *testing.T) {
 	}
 
 	// Fetch it using get and verify every field.
-	obj, err := ss.dbObject(objID)
+	obj, err := ss.Object(context.Background(), api.DefaultBucketName, objID)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	obj1Key, err := obj1.Key.MarshalBinary()
-	if err != nil {
-		t.Fatal(err)
+	// compare timestamp separately
+	if obj.ModTime.IsZero() {
+		t.Fatal("unexpected", obj.ModTime)
 	}
+	obj.ModTime = api.TimeRFC3339{}
+
 	obj1Slab0Key := obj1.Slabs[0].Key
 	obj1Slab1Key := obj1.Slabs[1].Key
 
-	// Set the Model fields to zero before comparing. These are set by gorm
-	// itself and contain a few timestamps which would make the following
-	// code a lot more verbose.
-	obj.Model = Model{}
-	for i := range obj.Slabs {
-		obj.Slabs[i].Model = Model{}
-	}
-
-	one := uint(1)
-	expectedObj := dbObject{
-		DBDirectoryID: 1,
-		DBBucketID:    uint(ss.DefaultBucketID()),
-		Health:        1,
-		ObjectID:      objID,
-		Key:           obj1Key,
-		Size:          obj1.TotalSize(),
-		Slabs: []dbSlice{
-			{
-				DBObjectID:  &one,
-				DBSlabID:    1,
-				ObjectIndex: 1,
-				Offset:      10,
-				Length:      100,
-			},
-			{
-				DBObjectID:  &one,
-				DBSlabID:    2,
-				ObjectIndex: 2,
-				Offset:      20,
-				Length:      200,
+	expectedObj := api.Object{
+		ObjectMetadata: api.ObjectMetadata{
+			ETag:     testETag,
+			Health:   1,
+			ModTime:  api.TimeRFC3339{},
+			Name:     objID,
+			Size:     obj1.TotalSize(),
+			MimeType: testMimeType,
+		},
+		Metadata: testMetadata,
+		Object: &object.Object{
+			Key: obj1.Key,
+			Slabs: []object.SlabSlice{
+				{
+					Offset: 10,
+					Length: 100,
+					Slab: object.Slab{
+						Health:    1,
+						Key:       obj1Slab0Key,
+						MinShards: 1,
+						Shards: []object.Sector{
+							{
+								LatestHost: hk1,
+								Root:       types.Hash256{1},
+								Contracts: map[types.PublicKey][]types.FileContractID{
+									hk1: {fcid1},
+								},
+							},
+						},
+					},
+				},
+				{
+					Offset: 20,
+					Length: 200,
+					Slab: object.Slab{
+						Health:    1,
+						Key:       obj1Slab1Key,
+						MinShards: 2,
+						Shards: []object.Sector{
+							{
+								LatestHost: hk2,
+								Root:       types.Hash256{2},
+								Contracts: map[types.PublicKey][]types.FileContractID{
+									hk2: {fcid2},
+								},
+							},
+						},
+					},
+				},
 			},
 		},
-		MimeType: testMimeType,
-		Etag:     testETag,
 	}
 	if !reflect.DeepEqual(obj, expectedObj) {
-		t.Fatal("object mismatch", cmp.Diff(obj, expectedObj))
+		t.Fatal("object mismatch", cmp.Diff(obj, expectedObj, cmp.AllowUnexported(object.EncryptionKey{}), cmp.Comparer(api.CompareTimeRFC3339)))
 	}
 
 	// Try to store it again. Should work.
@@ -1098,28 +1116,20 @@ func TestSQLMetadataStore(t *testing.T) {
 	}
 
 	// Fetch it again and verify.
-	obj, err = ss.dbObject(objID)
+	obj, err = ss.Object(context.Background(), api.DefaultBucketName, objID)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// Set the Model fields to zero before comparing. These are set by gorm
-	// itself and contain a few timestamps which would make the following
-	// code a lot more verbose.
-	obj.Model = Model{}
-	for i := range obj.Slabs {
-		obj.Slabs[i].Model = Model{}
+	// compare timestamp separately
+	if obj.ModTime.IsZero() {
+		t.Fatal("unexpected", obj.ModTime)
 	}
+	obj.ModTime = api.TimeRFC3339{}
 
-	// The expected object is the same except for some ids which were
-	// incremented due to the object and slab being overwritten.
-	two := uint(2)
-	expectedObj.Slabs[0].DBObjectID = &two
-	expectedObj.Slabs[0].DBSlabID = 3
-	expectedObj.Slabs[1].DBObjectID = &two
-	expectedObj.Slabs[1].DBSlabID = 4
+	// The expected object is the same.
 	if !reflect.DeepEqual(obj, expectedObj) {
-		t.Fatal("object mismatch", cmp.Diff(obj, expectedObj))
+		t.Fatal("object mismatch", cmp.Diff(obj, expectedObj, cmp.AllowUnexported(object.EncryptionKey{}), cmp.Comparer(api.CompareTimeRFC3339)))
 	}
 
 	// Fetch it and verify again.
@@ -1254,27 +1264,23 @@ func TestSQLMetadataStore(t *testing.T) {
 	// - 1 element in the slices table for the same reason
 	// - 1 element in the sectors table for the same reason
 	countCheck := func(objCount, sliceCount, slabCount, sectorCount int64) error {
-		tableCountCheck := func(table interface{}, tblCount int64) error {
-			var count int64
-			if err := ss.gormDB.Model(table).Count(&count).Error; err != nil {
-				return err
-			}
-			if count != tblCount {
-				return fmt.Errorf("expected %v objects in table %v but got %v", tblCount, table.(schema.Tabler).TableName(), count)
+		tableCountCheck := func(table string, tblCount int64) error {
+			if count := ss.Count(table); count != tblCount {
+				return fmt.Errorf("expected %v objects in table %v but got %v", tblCount, table, count)
 			}
 			return nil
 		}
 		// Check all tables.
-		if err := tableCountCheck(&dbObject{}, objCount); err != nil {
+		if err := tableCountCheck("objects", objCount); err != nil {
 			return err
 		}
-		if err := tableCountCheck(&dbSlice{}, sliceCount); err != nil {
+		if err := tableCountCheck("slices", sliceCount); err != nil {
 			return err
 		}
-		if err := tableCountCheck(&dbSlab{}, slabCount); err != nil {
+		if err := tableCountCheck("slabs", slabCount); err != nil {
 			return err
 		}
-		if err := tableCountCheck(&dbSector{}, sectorCount); err != nil {
+		if err := tableCountCheck("sectors", sectorCount); err != nil {
 			return err
 		}
 		return nil

From 3ac6b32d7eec8ef297962612db6cecba583cbb22 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Mon, 12 Aug 2024 17:09:47 +0200
Subject: [PATCH 06/19] stores: update TestUpdateSlab

---
 stores/metadata_test.go | 43 ++++++++++++-----------------------------
 1 file changed, 12 insertions(+), 31 deletions(-)

diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index 7a262b0e9..3227d8d89 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -2159,13 +2159,9 @@ func TestUpdateSlab(t *testing.T) {
 	}
 
 	// helper to fetch a slab from the database
-	fetchSlab := func() (slab dbSlab) {
+	fetchSlab := func() (slab object.Slab) {
 		t.Helper()
-		if err = ss.gormDB.
-			Where(&dbSlab{Key: key}).
-			Preload("Shards").
-			Take(&slab).
-			Error; err != nil {
+		if slab, err = ss.Slab(ctx, obj.Slabs[0].Key); err != nil {
 			t.Fatal(err)
 		}
 		return
@@ -2202,7 +2198,7 @@ func TestUpdateSlab(t *testing.T) {
 	for i := 0; i < 2; i++ {
 		if cids := contractIds(types.Hash256(inserted.Shards[i].Root)); len(cids) != 1 {
 			t.Fatalf("sector %d was uploaded to unexpected amount of contracts, %v!=1", i+1, len(cids))
-		} else if inserted.Shards[i].LatestHost != publicKey(hks[i]) {
+		} else if inserted.Shards[i].LatestHost != hks[i] {
 			t.Fatalf("sector %d was uploaded to unexpected amount of hosts, %v!=1", i+1, len(hks))
 		}
 	}
@@ -2243,7 +2239,7 @@ func TestUpdateSlab(t *testing.T) {
 		t.Fatalf("sector 1 was uploaded to unexpected amount of contracts, %v!=1", len(cids))
 	} else if types.FileContractID(cids[0]) != fcid1 {
 		t.Fatal("sector 1 was uploaded to unexpected contract", cids[0])
-	} else if updated.Shards[0].LatestHost != publicKey(hks[0]) {
+	} else if updated.Shards[0].LatestHost != hks[0] {
 		t.Fatal("host key was invalid", updated.Shards[0].LatestHost, publicKey(hks[0]))
 	} else if hks[0] != hk1 {
 		t.Fatal("sector 1 was uploaded to unexpected host", hks[0])
@@ -2254,7 +2250,7 @@ func TestUpdateSlab(t *testing.T) {
 		t.Fatalf("sector 1 was uploaded to unexpected amount of contracts, %v!=2", len(cids))
 	} else if types.FileContractID(cids[0]) != fcid2 || types.FileContractID(cids[1]) != fcid3 {
 		t.Fatal("sector 1 was uploaded to unexpected contracts", cids[0], cids[1])
-	} else if updated.Shards[0].LatestHost != publicKey(hks[0]) {
+	} else if updated.Shards[0].LatestHost != hks[0] {
 		t.Fatal("host key was invalid", updated.Shards[0].LatestHost, publicKey(hks[0]))
 	}
 
@@ -2278,30 +2274,27 @@ func TestUpdateSlab(t *testing.T) {
 		t.Fatal("unexpected number of slabs to migrate", len(toMigrate))
 	}
 
-	if obj, err := ss.dbObject(t.Name()); err != nil {
+	if obj, err := ss.Object(context.Background(), api.DefaultBucketName, t.Name()); err != nil {
 		t.Fatal(err)
 	} else if len(obj.Slabs) != 1 {
 		t.Fatalf("unexpected number of slabs, %v != 1", len(obj.Slabs))
-	} else if obj.Slabs[0].ID != updated.ID {
-		t.Fatalf("unexpected slab, %v != %v", obj.Slabs[0].ID, updated.ID)
+	} else if obj.Slabs[0].Key.String() != updated.Key.String() {
+		t.Fatalf("unexpected slab, %v != %v", obj.Slabs[0].Key, updated.Key)
 	}
 
 	// update the slab to change its contract set.
 	if err := ss.SetContractSet(ctx, "other", nil); err != nil {
 		t.Fatal(err)
 	}
-	csID := ss.ContractSetID("other")
 	err = ss.UpdateSlab(ctx, slab, "other")
 	if err != nil {
 		t.Fatal(err)
 	}
-	var s dbSlab
-	if err := ss.gormDB.Where(&dbSlab{Key: key}).
-		Preload("Shards").
-		Take(&s).
-		Error; err != nil {
+	var csID int64
+	if err := ss.DB().QueryRow(context.Background(), "SELECT db_contract_set_id FROM slabs WHERE `key` = ?", key).
+		Scan(&csID); err != nil {
 		t.Fatal(err)
-	} else if s.DBContractSetID != uint(csID) {
+	} else if csID != ss.ContractSetID("other") {
 		t.Fatal("contract set was not updated")
 	}
 }
@@ -3141,18 +3134,6 @@ func TestContractSizes(t *testing.T) {
 	}
 }
 
-// dbObject retrieves a dbObject from the store.
-func (s *SQLStore) dbObject(key string) (dbObject, error) {
-	var obj dbObject
-	tx := s.gormDB.Where(&dbObject{ObjectID: key}).
-		Preload("Slabs").
-		Take(&obj)
-	if errors.Is(tx.Error, gorm.ErrRecordNotFound) {
-		return dbObject{}, api.ErrObjectNotFound
-	}
-	return obj, nil
-}
-
 func TestObjectsBySlabKey(t *testing.T) {
 	ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
 	defer ss.Close()

From 2bf644c971b9c2c37a461f89b28507debfd420ec Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 11:12:54 +0200
Subject: [PATCH 07/19] stores: remove dbObject

---
 stores/metadata.go      |  21 --------
 stores/metadata_test.go | 115 ++++++++++++++++++++++------------------
 2 files changed, 63 insertions(+), 73 deletions(-)

diff --git a/stores/metadata.go b/stores/metadata.go
index 06feb82ac..9b2bfa50c 100644
--- a/stores/metadata.go
+++ b/stores/metadata.go
@@ -46,24 +46,6 @@ var (
 var objectDeleteBatchSizes = []int64{10, 50, 100, 200, 500, 1000, 5000, 10000, 50000, 100000}
 
 type (
-	dbObject struct {
-		Model
-
-		DBDirectoryID uint
-
-		DBBucketID uint   `gorm:"index;uniqueIndex:idx_object_bucket;NOT NULL"`
-		ObjectID   string `gorm:"index;uniqueIndex:idx_object_bucket"`
-
-		Key      secretKey
-		Slabs    []dbSlice              // no CASCADE, slices are deleted via trigger
-		Metadata []dbObjectUserMetadata `gorm:"constraint:OnDelete:CASCADE"` // CASCADE to delete metadata too
-		Health   float64                `gorm:"index;default:1.0; NOT NULL"`
-		Size     int64
-
-		MimeType string `json:"index"`
-		Etag     string `gorm:"index"`
-	}
-
 	dbObjectUserMetadata struct {
 		Model
 
@@ -115,9 +97,6 @@ func (s dbSlab) HealthValid() bool {
 	return time.Now().Before(time.Unix(s.HealthValidUntil, 0))
 }
 
-// TableName implements the gorm.Tabler interface.
-func (dbObject) TableName() string { return "objects" }
-
 // TableName implements the gorm.Tabler interface.
 func (dbObjectUserMetadata) TableName() string { return "object_user_metadata" }
 
diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index 3227d8d89..43b184357 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -4152,22 +4152,20 @@ func TestSlabCleanup(t *testing.T) {
 	}
 
 	// create objects
-	obj1 := dbObject{
-		DBDirectoryID: uint(dirID),
-		ObjectID:      "1",
-		DBBucketID:    uint(ss.DefaultBucketID()),
-		Health:        1,
-	}
-	if err := ss.gormDB.Create(&obj1).Error; err != nil {
+	insertObjStmt, err := ss.DB().Prepare(context.Background(), "INSERT INTO objects (db_directory_id, object_id, db_bucket_id, health) VALUES (?, ?, ?, ?);")
+	if err != nil {
 		t.Fatal(err)
 	}
-	obj2 := dbObject{
-		DBDirectoryID: uint(dirID),
-		ObjectID:      "2",
-		DBBucketID:    uint(ss.DefaultBucketID()),
-		Health:        1,
-	}
-	if err := ss.gormDB.Create(&obj2).Error; err != nil {
+	defer insertObjStmt.Close()
+
+	var obj1ID, obj2ID int64
+	if res, err := insertObjStmt.Exec(context.Background(), dirID, "1", ss.DefaultBucketID(), 1); err != nil {
+		t.Fatal(err)
+	} else if obj1ID, err = res.LastInsertId(); err != nil {
+		t.Fatal(err)
+	} else if res, err := insertObjStmt.Exec(context.Background(), dirID, "2", ss.DefaultBucketID(), 1); err != nil {
+		t.Fatal(err)
+	} else if obj2ID, err = res.LastInsertId(); err != nil {
 		t.Fatal(err)
 	}
 
@@ -4184,15 +4182,16 @@ func TestSlabCleanup(t *testing.T) {
 	}
 
 	// reference the slab
+	obj1UID, obj2UID := uint(obj1ID), uint(obj2ID)
 	slice1 := dbSlice{
-		DBObjectID: &obj1.ID,
+		DBObjectID: &obj1UID,
 		DBSlabID:   slab.ID,
 	}
 	if err := ss.gormDB.Create(&slice1).Error; err != nil {
 		t.Fatal(err)
 	}
 	slice2 := dbSlice{
-		DBObjectID: &obj2.ID,
+		DBObjectID: &obj2UID,
 		DBSlabID:   slab.ID,
 	}
 	if err := ss.gormDB.Create(&slice2).Error; err != nil {
@@ -4200,12 +4199,12 @@ func TestSlabCleanup(t *testing.T) {
 	}
 
 	// delete the object
-	err = ss.RemoveObjectBlocking(context.Background(), api.DefaultBucketName, obj1.ObjectID)
+	err = ss.RemoveObjectBlocking(context.Background(), api.DefaultBucketName, "1")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// check slice count
+	// check slab count
 	var slabCntr int64
 	if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
 		t.Fatal(err)
@@ -4214,7 +4213,7 @@ func TestSlabCleanup(t *testing.T) {
 	}
 
 	// delete second object
-	err = ss.RemoveObjectBlocking(context.Background(), api.DefaultBucketName, obj2.ObjectID)
+	err = ss.RemoveObjectBlocking(context.Background(), api.DefaultBucketName, "2")
 	if err != nil {
 		t.Fatal(err)
 	} else if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
@@ -4235,17 +4234,15 @@ func TestSlabCleanup(t *testing.T) {
 	if err := ss.gormDB.Create(&bufferedSlab).Error; err != nil {
 		t.Fatal(err)
 	}
-	obj3 := dbObject{
-		DBDirectoryID: uint(dirID),
-		ObjectID:      "3",
-		DBBucketID:    uint(ss.DefaultBucketID()),
-		Health:        1,
-	}
-	if err := ss.gormDB.Create(&obj3).Error; err != nil {
+	var obj3ID int64
+	if res, err := insertObjStmt.Exec(context.Background(), dirID, "3", ss.DefaultBucketID(), 1); err != nil {
+		t.Fatal(err)
+	} else if obj3ID, err = res.LastInsertId(); err != nil {
 		t.Fatal(err)
 	}
+	obj3UID := uint(obj3ID)
 	slice := dbSlice{
-		DBObjectID: &obj3.ID,
+		DBObjectID: &obj3UID,
 		DBSlabID:   bufferedSlab.ID,
 	}
 	if err := ss.gormDB.Create(&slice).Error; err != nil {
@@ -4258,7 +4255,7 @@ func TestSlabCleanup(t *testing.T) {
 	}
 
 	// delete third object
-	err = ss.RemoveObjectBlocking(context.Background(), api.DefaultBucketName, obj3.ObjectID)
+	err = ss.RemoveObjectBlocking(context.Background(), api.DefaultBucketName, "3")
 	if err != nil {
 		t.Fatal(err)
 	} else if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
@@ -4322,25 +4319,37 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// helper to fetch relevant fields from an object
+	fetchObj := func(bid int64, oid string) (id, bucketID int64, objectID string, health float64, size int64) {
+		t.Helper()
+		err := ss.DB().QueryRow(context.Background(), `
+			SELECT id, db_bucket_id, object_id, health, size
+			FROM objects
+			WHERE db_bucket_id = ? AND object_id = ?
+		`, bid, oid).Scan(&id, &bucketID, &objectID, &health, &size)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return
+	}
+
 	// fetch the object
-	var dbObj dbObject
-	if err := ss.gormDB.Where("db_bucket_id", ss.DefaultBucketID()).Take(&dbObj).Error; err != nil {
-		t.Fatal(err)
-	} else if dbObj.ID != 1 {
-		t.Fatal("unexpected id", dbObj.ID)
-	} else if dbObj.DBBucketID != uint(ss.DefaultBucketID()) {
-		t.Fatal("bucket id mismatch", dbObj.DBBucketID)
-	} else if dbObj.ObjectID != "1" {
-		t.Fatal("object id mismatch", dbObj.ObjectID)
-	} else if dbObj.Health != 1 {
-		t.Fatal("health mismatch", dbObj.Health)
-	} else if dbObj.Size != obj.TotalSize() {
-		t.Fatal("size mismatch", dbObj.Size)
+	id, bid, oid, health, size := fetchObj(ss.DefaultBucketID(), "1")
+	if id != 1 {
+		t.Fatal("unexpected id", id)
+	} else if bid != ss.DefaultBucketID() {
+		t.Fatal("bucket id mismatch", bid)
+	} else if oid != "1" {
+		t.Fatal("object id mismatch", oid)
+	} else if health != 1 {
+		t.Fatal("health mismatch", health)
+	} else if size != obj.TotalSize() {
+		t.Fatal("size mismatch", size)
 	}
 
 	// fetch its slices
 	var dbSlices []dbSlice
-	if err := ss.gormDB.Where("db_object_id", dbObj.ID).Find(&dbSlices).Error; err != nil {
+	if err := ss.gormDB.Where("db_object_id", id).Find(&dbSlices).Error; err != nil {
 		t.Fatal(err)
 	} else if len(dbSlices) != 2 {
 		t.Fatal("invalid number of slices", len(dbSlices))
@@ -4429,20 +4438,22 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 	}
 
 	// fetch the object
-	var dbObj2 dbObject
-	if err := ss.gormDB.Where("db_bucket_id", ss.DefaultBucketID()).
-		Where("object_id", "2").
-		Take(&dbObj2).Error; err != nil {
-		t.Fatal(err)
-	} else if dbObj2.ID != 2 {
-		t.Fatal("unexpected id", dbObj2.ID)
-	} else if dbObj.Size != obj2.TotalSize() {
-		t.Fatal("size mismatch", dbObj2.Size)
+	id2, bid2, oid2, health2, size2 := fetchObj(ss.DefaultBucketID(), "2")
+	if id2 != 2 {
+		t.Fatal("unexpected id", id)
+	} else if bid2 != ss.DefaultBucketID() {
+		t.Fatal("bucket id mismatch", bid)
+	} else if oid2 != "2" {
+		t.Fatal("object id mismatch", oid)
+	} else if health2 != 1 {
+		t.Fatal("health mismatch", health)
+	} else if size2 != obj.TotalSize() {
+		t.Fatal("size mismatch", size)
 	}
 
 	// fetch its slices
 	var dbSlices2 []dbSlice
-	if err := ss.gormDB.Where("db_object_id", dbObj2.ID).Find(&dbSlices2).Error; err != nil {
+	if err := ss.gormDB.Where("db_object_id", id2).Find(&dbSlices2).Error; err != nil {
 		t.Fatal(err)
 	} else if len(dbSlices2) != 2 {
 		t.Fatal("invalid number of slices", len(dbSlices))

From c952c09f1295006f9f89fc2c84cf96f1ceff29e1 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 11:22:20 +0200
Subject: [PATCH 08/19] stores: remove dbObjectUserMetadata

---
 stores/metadata.go       | 12 ------------
 stores/metadata_test.go  |  9 ++-------
 stores/multipart_test.go | 35 ++++++++++++++++++++++++++---------
 3 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/stores/metadata.go b/stores/metadata.go
index 9b2bfa50c..50a7d92cd 100644
--- a/stores/metadata.go
+++ b/stores/metadata.go
@@ -46,15 +46,6 @@ var (
 var objectDeleteBatchSizes = []int64{10, 50, 100, 200, 500, 1000, 5000, 10000, 50000, 100000}
 
 type (
-	dbObjectUserMetadata struct {
-		Model
-
-		DBObjectID          *uint  `gorm:"index:uniqueIndex:idx_object_user_metadata_key"`
-		DBMultipartUploadID *uint  `gorm:"index:uniqueIndex:idx_object_user_metadata_key"`
-		Key                 string `gorm:"index:uniqueIndex:idx_object_user_metadata_key"`
-		Value               string
-	}
-
 	dbSlice struct {
 		Model
 		DBObjectID        *uint `gorm:"index"`
@@ -97,9 +88,6 @@ func (s dbSlab) HealthValid() bool {
 	return time.Now().Before(time.Unix(s.HealthValidUntil, 0))
 }
 
-// TableName implements the gorm.Tabler interface.
-func (dbObjectUserMetadata) TableName() string { return "object_user_metadata" }
-
 // TableName implements the gorm.Tabler interface.
 func (dbSector) TableName() string { return "sectors" }
 
diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index 43b184357..d8df6c2c8 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -271,10 +271,7 @@ func TestObjectMetadata(t *testing.T) {
 	}
 
 	// assert metadata CASCADE on object delete
-	var cnt int64
-	if err := ss.gormDB.Model(&dbObjectUserMetadata{}).Count(&cnt).Error; err != nil {
-		t.Fatal(err)
-	} else if cnt != 2 {
+	if cnt := ss.Count("object_user_metadata"); cnt != 2 {
 		t.Fatal("unexpected number of metadata entries", cnt)
 	}
 
@@ -284,9 +281,7 @@ func TestObjectMetadata(t *testing.T) {
 	}
 
 	// assert records are gone
-	if err := ss.gormDB.Model(&dbObjectUserMetadata{}).Count(&cnt).Error; err != nil {
-		t.Fatal(err)
-	} else if cnt != 0 {
+	if cnt := ss.Count("object_user_metadata"); cnt != 0 {
 		t.Fatal("unexpected number of metadata entries", cnt)
 	}
 }
diff --git a/stores/multipart_test.go b/stores/multipart_test.go
index 369ee7b03..5c651d478 100644
--- a/stores/multipart_test.go
+++ b/stores/multipart_test.go
@@ -71,15 +71,33 @@ func TestMultipartUploadWithUploadPackingRegression(t *testing.T) {
 		})
 	}
 
+	type oum struct {
+		MultipartUploadID *int64
+		ObjectID          *int64
+	}
+	fetchUserMD := func() (metadatas []oum) {
+		rows, err := ss.DB().Query(context.Background(), "SELECT db_multipart_upload_id, db_object_id FROM object_user_metadata")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var md oum
+			if err := rows.Scan(&md.MultipartUploadID, &md.ObjectID); err != nil {
+				t.Fatal(err)
+			}
+			metadatas = append(metadatas, md)
+		}
+		return
+	}
+
 	// Assert metadata was persisted and is linked to the multipart upload
-	var metadatas []dbObjectUserMetadata
-	if err := ss.gormDB.Model(&dbObjectUserMetadata{}).Find(&metadatas).Error; err != nil {
-		t.Fatal(err)
-	} else if len(metadatas) != len(testMetadata) {
+	metadatas := fetchUserMD()
+	if len(metadatas) != len(testMetadata) {
 		t.Fatal("expected metadata to be persisted")
 	}
 	for _, m := range metadatas {
-		if m.DBMultipartUploadID == nil || m.DBObjectID != nil {
+		if m.MultipartUploadID == nil || m.ObjectID != nil {
 			t.Fatal("unexpected")
 		}
 	}
@@ -115,13 +133,12 @@ func TestMultipartUploadWithUploadPackingRegression(t *testing.T) {
 	}
 
 	// Assert metadata was converted and the multipart upload id was nullified
-	if err := ss.gormDB.Model(&dbObjectUserMetadata{}).Find(&metadatas).Error; err != nil {
-		t.Fatal(err)
-	} else if len(metadatas) != len(testMetadata) {
+	metadatas = fetchUserMD()
+	if len(metadatas) != len(testMetadata) {
 		t.Fatal("expected metadata to be persisted")
 	}
 	for _, m := range metadatas {
-		if m.DBMultipartUploadID != nil || m.DBObjectID == nil {
+		if m.MultipartUploadID != nil || m.ObjectID == nil {
 			t.Fatal("unexpected")
 		}
 	}

From e1915a3d2f824f1b9bbca4573393eb5f3ad9b886 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 11:55:47 +0200
Subject: [PATCH 09/19] stores: remove dbSector

---
 stores/metadata.go      |  14 -----
 stores/metadata_test.go | 117 +++++++++++++++++++++++++++-------------
 2 files changed, 81 insertions(+), 50 deletions(-)

diff --git a/stores/metadata.go b/stores/metadata.go
index 50a7d92cd..0eb68b905 100644
--- a/stores/metadata.go
+++ b/stores/metadata.go
@@ -70,17 +70,6 @@ type (
 		TotalShards      uint8     `gorm:"index"`
 
 		Slices []dbSlice
-		Shards []dbSector `gorm:"constraint:OnDelete:CASCADE"` // CASCADE to delete shards too
-	}
-
-	dbSector struct {
-		Model
-
-		DBSlabID  uint `gorm:"index:idx_sectors_db_slab_id;uniqueIndex:idx_sectors_slab_id_slab_index;NOT NULL"`
-		SlabIndex int  `gorm:"index:idx_sectors_slab_index;uniqueIndex:idx_sectors_slab_id_slab_index;NOT NULL"`
-
-		LatestHost publicKey `gorm:"NOT NULL"`
-		Root       []byte    `gorm:"index;unique;NOT NULL;size:32"`
 	}
 )
 
@@ -88,9 +77,6 @@ func (s dbSlab) HealthValid() bool {
 	return time.Now().Before(time.Unix(s.HealthValidUntil, 0))
 }
 
-// TableName implements the gorm.Tabler interface.
-func (dbSector) TableName() string { return "sectors" }
-
 // TableName implements the gorm.Tabler interface.
 func (dbSlab) TableName() string { return "slabs" }
 
diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index d8df6c2c8..c8a7fe4cf 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -2100,7 +2100,7 @@ func TestContractSectors(t *testing.T) {
 	}
 
 	// Delete the sector.
-	if err := ss.gormDB.Delete(&dbSector{Model: Model{ID: 1}}).Error; err != nil {
+	if _, err := ss.DB().Exec(context.Background(), "DELETE FROM sectors WHERE id = ?", 1); err != nil {
 		t.Fatal(err)
 	} else if n := ss.Count("contract_sectors"); n != 0 {
 		t.Fatal("table should be empty", n)
@@ -3682,12 +3682,10 @@ func TestDeleteHostSector(t *testing.T) {
 
 	// Find the slab. It should have an invalid health.
 	var s dbSlab
-	if err := ss.gormDB.Preload("Shards").Take(&s).Error; err != nil {
+	if err := ss.gormDB.Take(&s).Error; err != nil {
 		t.Fatal(err)
 	} else if s.HealthValid() {
 		t.Fatal("expected health to be invalid")
-	} else if s.Shards[0].LatestHost != publicKey(hk2) {
-		t.Fatal("expected hk2 to be latest host", types.PublicKey(s.Shards[0].LatestHost))
 	}
 
 	sectorContractCnt := func(root types.Hash256) (n int) {
@@ -3704,14 +3702,38 @@ func TestDeleteHostSector(t *testing.T) {
 		return
 	}
 
+	// helper to fetch sectors
+	type sector struct {
+		LatestHost types.PublicKey
+		Root       types.Hash256
+		SlabID     int64
+	}
+	fetchSectors := func() (sectors []sector) {
+		t.Helper()
+		rows, err := ss.DB().Query(context.Background(), "SELECT root, latest_host, db_slab_id FROM sectors")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var s sector
+			if err := rows.Scan((*sql.PublicKey)(&s.Root), (*sql.Hash256)(&s.LatestHost), &s.SlabID); err != nil {
+				t.Fatal(err)
+			}
+			sectors = append(sectors, s)
+		}
+		return
+	}
+
 	// Fetch the sector and assert the contracts association.
-	var sectors []dbSector
-	if err := ss.gormDB.Model(&dbSector{}).Find(&sectors).Error; err != nil {
-		t.Fatal(err)
-	} else if len(sectors) != 1 {
+	if sectors := fetchSectors(); len(sectors) != 1 {
 		t.Fatal("expected 1 sector", len(sectors))
 	} else if cnt := sectorContractCnt(types.Hash256(sectors[0].Root)); cnt != 2 {
 		t.Fatal("expected 2 contracts", cnt)
+	} else if sectors[0].LatestHost != hk2 {
+		t.Fatalf("expected latest host to be hk2, got %v", sectors[0].LatestHost)
+	} else if sectors[0].SlabID != int64(s.ID) {
+		t.Fatalf("expected slab id to be %v, got %v", s.ID, sectors[0].SlabID)
 	}
 
 	hi, err := ss.Host(context.Background(), hk1)
@@ -3748,12 +3770,14 @@ func TestDeleteHostSector(t *testing.T) {
 	}
 
 	// Fetch the sector and check the public key has the default value
-	if err := ss.gormDB.Model(&dbSector{}).Find(&sectors).Error; err != nil {
-		t.Fatal(err)
-	} else if len(sectors) != 1 {
+	if sectors := fetchSectors(); len(sectors) != 1 {
 		t.Fatal("expected 1 sector", len(sectors))
+	} else if cnt := sectorContractCnt(types.Hash256(sectors[0].Root)); cnt != 0 {
+		t.Fatal("expected 0 contracts", cnt)
 	} else if sector := sectors[0]; sector.LatestHost != [32]byte{} {
 		t.Fatal("expected latest host to be empty", sector.LatestHost)
+	} else if sectors[0].SlabID != int64(s.ID) {
+		t.Fatalf("expected slab id to be %v, got %v", s.ID, sectors[0].SlabID)
 	}
 }
 func newTestShards(hk types.PublicKey, fcid types.FileContractID, root types.Hash256) []object.Sector {
@@ -4349,6 +4373,31 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 	} else if len(dbSlices) != 2 {
 		t.Fatal("invalid number of slices", len(dbSlices))
 	}
+
+	// helper to fetch sectors
+	type sector struct {
+		ID         int64
+		SlabID     int64
+		LatestHost types.PublicKey
+		Root       types.Hash256
+	}
+	fetchSectorsBySlabID := func(slabID int64) (sectors []sector) {
+		t.Helper()
+		rows, err := ss.DB().Query(context.Background(), "SELECT id, db_slab_id, root, latest_host FROM sectors WHERE db_slab_id = ?", slabID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var s sector
+			if err := rows.Scan(&s.ID, &s.SlabID, (*sql.PublicKey)(&s.Root), (*sql.Hash256)(&s.LatestHost)); err != nil {
+				t.Fatal(err)
+			}
+			sectors = append(sectors, s)
+		}
+		return
+	}
+
 	for i, dbSlice := range dbSlices {
 		if dbSlice.ID != uint(i+1) {
 			t.Fatal("unexpected id", dbSlice.ID)
@@ -4380,20 +4429,18 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 		}
 
 		// fetch the sectors
-		var dbSectors []dbSector
-		if err := ss.gormDB.Where("db_slab_id", dbSlab.ID).Find(&dbSectors).Error; err != nil {
-			t.Fatal(err)
-		} else if len(dbSectors) != totalShards {
-			t.Fatal("invalid number of sectors", len(dbSectors))
+		sectors := fetchSectorsBySlabID(int64(dbSlab.ID))
+		if len(sectors) != totalShards {
+			t.Fatal("invalid number of sectors", len(sectors))
 		}
-		for j, dbSector := range dbSectors {
-			if dbSector.ID != uint(i*totalShards+j+1) {
-				t.Fatal("invalid id", dbSector.ID)
-			} else if dbSector.DBSlabID != dbSlab.ID {
-				t.Fatal("invalid slab id", dbSector.DBSlabID)
-			} else if dbSector.LatestHost != publicKey(hks[i*totalShards+j]) {
+		for j, sector := range sectors {
+			if sector.ID != int64(i*totalShards+j+1) {
+				t.Fatal("invalid id", sector.ID)
+			} else if sector.SlabID != int64(dbSlab.ID) {
+				t.Fatal("invalid slab id", sector.SlabID)
+			} else if sector.LatestHost != hks[i*totalShards+j] {
 				t.Fatal("invalid host")
-			} else if !bytes.Equal(dbSector.Root, obj.Slabs[i].Shards[j].Root[:]) {
+			} else if sector.Root != obj.Slabs[i].Shards[j].Root {
 				t.Fatal("invalid root")
 			}
 		}
@@ -4478,20 +4525,18 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 	}
 
 	// fetch the sectors
-	var dbSectors2 []dbSector
-	if err := ss.gormDB.Where("db_slab_id", dbSlab2.ID).Find(&dbSectors2).Error; err != nil {
-		t.Fatal(err)
-	} else if len(dbSectors2) != totalShards {
-		t.Fatal("invalid number of sectors", len(dbSectors2))
-	}
-	for j, dbSector := range dbSectors2 {
-		if dbSector.ID != uint((len(obj.Slabs))*totalShards+j+1) {
-			t.Fatal("invalid id", dbSector.ID)
-		} else if dbSector.DBSlabID != dbSlab2.ID {
-			t.Fatal("invalid slab id", dbSector.DBSlabID)
-		} else if dbSector.LatestHost != publicKey(hks[(len(obj.Slabs))*totalShards+j]) {
+	sectors2 := fetchSectorsBySlabID(int64(dbSlab2.ID))
+	if len(sectors2) != totalShards {
+		t.Fatal("invalid number of sectors", len(sectors2))
+	}
+	for j, sector := range sectors2 {
+		if sector.ID != int64((len(obj.Slabs))*totalShards+j+1) {
+			t.Fatal("invalid id", sector.ID)
+		} else if sector.SlabID != int64(dbSlab2.ID) {
+			t.Fatal("invalid slab id", sector.SlabID)
+		} else if sector.LatestHost != hks[(len(obj.Slabs))*totalShards+j] {
 			t.Fatal("invalid host")
-		} else if !bytes.Equal(dbSector.Root, obj2.Slabs[0].Shards[j].Root[:]) {
+		} else if sector.Root != obj2.Slabs[0].Shards[j].Root {
 			t.Fatal("invalid root")
 		}
 	}

From dfec8e49c30a96aa11e34f0cb4555fe7b8ab6832 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 12:09:02 +0200
Subject: [PATCH 10/19] stores: remove dbSlice

---
 stores/metadata.go       |  17 -------
 stores/metadata_test.go  | 106 ++++++++++++++++++++++-----------------
 stores/multipart_test.go |  10 +---
 3 files changed, 61 insertions(+), 72 deletions(-)

diff --git a/stores/metadata.go b/stores/metadata.go
index 0eb68b905..4c545d449 100644
--- a/stores/metadata.go
+++ b/stores/metadata.go
@@ -46,18 +46,6 @@ var (
 var objectDeleteBatchSizes = []int64{10, 50, 100, 200, 500, 1000, 5000, 10000, 50000, 100000}
 
 type (
-	dbSlice struct {
-		Model
-		DBObjectID        *uint `gorm:"index"`
-		ObjectIndex       uint  `gorm:"index:idx_slices_object_index"`
-		DBMultipartPartID *uint `gorm:"index"`
-
-		// Slice related fields.
-		DBSlabID uint `gorm:"index"`
-		Offset   uint32
-		Length   uint32
-	}
-
 	dbSlab struct {
 		Model
 		DBContractSetID  uint `gorm:"index"`
@@ -68,8 +56,6 @@ type (
 		Key              secretKey `gorm:"unique;NOT NULL;size:32"`   // json string
 		MinShards        uint8     `gorm:"index"`
 		TotalShards      uint8     `gorm:"index"`
-
-		Slices []dbSlice
 	}
 )
 
@@ -80,9 +66,6 @@ func (s dbSlab) HealthValid() bool {
 // TableName implements the gorm.Tabler interface.
 func (dbSlab) TableName() string { return "slabs" }
 
-// TableName implements the gorm.Tabler interface.
-func (dbSlice) TableName() string { return "slices" }
-
 func (s *SQLStore) Bucket(ctx context.Context, bucket string) (b api.Bucket, err error) {
 	err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) (err error) {
 		b, err = tx.Bucket(ctx, bucket)
diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index c8a7fe4cf..fecb8004b 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -4200,20 +4200,17 @@ func TestSlabCleanup(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// reference the slab
-	obj1UID, obj2UID := uint(obj1ID), uint(obj2ID)
-	slice1 := dbSlice{
-		DBObjectID: &obj1UID,
-		DBSlabID:   slab.ID,
-	}
-	if err := ss.gormDB.Create(&slice1).Error; err != nil {
+	// statement to reference slabs by inserting a slice for an object
+	insertSlabRefStmt, err := ss.DB().Prepare(context.Background(), "INSERT INTO slices (db_object_id, db_slab_id) VALUES (?, ?);")
+	if err != nil {
 		t.Fatal(err)
 	}
-	slice2 := dbSlice{
-		DBObjectID: &obj2UID,
-		DBSlabID:   slab.ID,
-	}
-	if err := ss.gormDB.Create(&slice2).Error; err != nil {
+	defer insertSlabRefStmt.Close()
+
+	// reference the slab
+	if _, err := insertSlabRefStmt.Exec(context.Background(), obj1ID, slab.ID); err != nil {
+		t.Fatal(err)
+	} else if _, err := insertSlabRefStmt.Exec(context.Background(), obj2ID, slab.ID); err != nil {
 		t.Fatal(err)
 	}
 
@@ -4258,13 +4255,7 @@ func TestSlabCleanup(t *testing.T) {
 		t.Fatal(err)
 	} else if obj3ID, err = res.LastInsertId(); err != nil {
 		t.Fatal(err)
-	}
-	obj3UID := uint(obj3ID)
-	slice := dbSlice{
-		DBObjectID: &obj3UID,
-		DBSlabID:   bufferedSlab.ID,
-	}
-	if err := ss.gormDB.Create(&slice).Error; err != nil {
+	} else if _, err := insertSlabRefStmt.Exec(context.Background(), obj3ID, bufferedSlab.ID); err != nil {
 		t.Fatal(err)
 	}
 	if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
@@ -4366,12 +4357,35 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 		t.Fatal("size mismatch", size)
 	}
 
+	// helper to fetch object's slices
+	type slice struct {
+		ID          int64
+		ObjectIndex int64
+		Offset      int64
+		Length      int64
+		SlabID      int64
+	}
+	fetchSlicesByObjectID := func(oid int64) (slices []slice) {
+		t.Helper()
+		rows, err := ss.DB().Query(context.Background(), "SELECT id, object_index, offset, length, db_slab_id FROM slices WHERE db_object_id = ?", oid)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer rows.Close()
+		for rows.Next() {
+			var s slice
+			if err := rows.Scan(&s.ID, &s.ObjectIndex, &s.Offset, &s.Length, &s.SlabID); err != nil {
+				t.Fatal(err)
+			}
+			slices = append(slices, s)
+		}
+		return
+	}
+
 	// fetch its slices
-	var dbSlices []dbSlice
-	if err := ss.gormDB.Where("db_object_id", id).Find(&dbSlices).Error; err != nil {
-		t.Fatal(err)
-	} else if len(dbSlices) != 2 {
-		t.Fatal("invalid number of slices", len(dbSlices))
+	slices := fetchSlicesByObjectID(id)
+	if len(slices) != 2 {
+		t.Fatal("invalid number of slices", len(slices))
 	}
 
 	// helper to fetch sectors
@@ -4398,19 +4412,19 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 		return
 	}
 
-	for i, dbSlice := range dbSlices {
-		if dbSlice.ID != uint(i+1) {
-			t.Fatal("unexpected id", dbSlice.ID)
-		} else if dbSlice.ObjectIndex != uint(i+1) {
-			t.Fatal("unexpected object index", dbSlice.ObjectIndex)
-		} else if dbSlice.Offset != 0 || dbSlice.Length != uint32(minShards)*rhpv2.SectorSize {
-			t.Fatal("invalid offset/length", dbSlice.Offset, dbSlice.Length)
+	for i, slice := range slices {
+		if slice.ID != int64(i+1) {
+			t.Fatal("unexpected id", slice.ID)
+		} else if slice.ObjectIndex != int64(i+1) {
+			t.Fatal("unexpected object index", slice.ObjectIndex)
+		} else if slice.Offset != 0 || slice.Length != int64(minShards)*rhpv2.SectorSize {
+			t.Fatal("invalid offset/length", slice.Offset, slice.Length)
 		}
 
 		// fetch the slab
 		var dbSlab dbSlab
 		key, _ := obj.Slabs[i].Key.MarshalBinary()
-		if err := ss.gormDB.Where("id", dbSlice.DBSlabID).Take(&dbSlab).Error; err != nil {
+		if err := ss.gormDB.Where("id", slice.SlabID).Take(&dbSlab).Error; err != nil {
 			t.Fatal(err)
 		} else if dbSlab.ID != uint(i+1) {
 			t.Fatal("unexpected id", dbSlab.ID)
@@ -4494,29 +4508,27 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 	}
 
 	// fetch its slices
-	var dbSlices2 []dbSlice
-	if err := ss.gormDB.Where("db_object_id", id2).Find(&dbSlices2).Error; err != nil {
-		t.Fatal(err)
-	} else if len(dbSlices2) != 2 {
-		t.Fatal("invalid number of slices", len(dbSlices))
+	slices2 := fetchSlicesByObjectID(id2)
+	if len(slices2) != 2 {
+		t.Fatal("invalid number of slices", len(slices2))
 	}
 
 	// check the first one
-	dbSlice2 := dbSlices2[0]
-	if dbSlice2.ID != uint(len(dbSlices)+1) {
-		t.Fatal("unexpected id", dbSlice2.ID)
-	} else if dbSlice2.ObjectIndex != uint(1) {
-		t.Fatal("unexpected object index", dbSlice2.ObjectIndex)
-	} else if dbSlice2.Offset != 0 || dbSlice2.Length != uint32(minShards)*rhpv2.SectorSize {
-		t.Fatal("invalid offset/length", dbSlice2.Offset, dbSlice2.Length)
+	slice2 := slices2[0]
+	if slice2.ID != int64(len(slices)+1) {
+		t.Fatal("unexpected id", slice2.ID)
+	} else if slice2.ObjectIndex != 1 {
+		t.Fatal("unexpected object index", slice2.ObjectIndex)
+	} else if slice2.Offset != 0 || slice2.Length != int64(minShards)*rhpv2.SectorSize {
+		t.Fatal("invalid offset/length", slice2.Offset, slice2.Length)
 	}
 
 	// fetch the slab
 	var dbSlab2 dbSlab
 	key, _ := obj2.Slabs[0].Key.MarshalBinary()
-	if err := ss.gormDB.Where("id", dbSlice2.DBSlabID).Take(&dbSlab2).Error; err != nil {
+	if err := ss.gormDB.Where("id", slice2.SlabID).Take(&dbSlab2).Error; err != nil {
 		t.Fatal(err)
-	} else if dbSlab2.ID != uint(len(dbSlices)+1) {
+	} else if dbSlab2.ID != uint(len(slices)+1) {
 		t.Fatal("unexpected id", dbSlab2.ID)
 	} else if dbSlab2.DBContractSetID != 1 {
 		t.Fatal("invalid contract set id", dbSlab2.DBContractSetID)
@@ -4542,7 +4554,7 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 	}
 
 	// the second slab of obj2 should be the same as the first in obj
-	if dbSlices2[1].DBSlabID != 2 {
+	if slices2[1].SlabID != 2 {
 		t.Fatal("wrong slab")
 	}
 
diff --git a/stores/multipart_test.go b/stores/multipart_test.go
index 5c651d478..52dddb050 100644
--- a/stores/multipart_test.go
+++ b/stores/multipart_test.go
@@ -103,17 +103,11 @@ func TestMultipartUploadWithUploadPackingRegression(t *testing.T) {
 	}
 
 	// Complete the upload. Check that the number of slices stays the same.
-	var nSlicesBefore int64
-	var nSlicesAfter int64
-	if err := ss.gormDB.Model(&dbSlice{}).Count(&nSlicesBefore).Error; err != nil {
-		t.Fatal(err)
-	} else if nSlicesBefore == 0 {
+	if nSlicesBefore := ss.Count("slices"); nSlicesBefore == 0 {
 		t.Fatal("expected some slices")
 	} else if _, err = ss.CompleteMultipartUpload(ctx, api.DefaultBucketName, objName, resp.UploadID, parts, api.CompleteMultipartOptions{}); err != nil {
 		t.Fatal(err)
-	} else if err := ss.gormDB.Model(&dbSlice{}).Count(&nSlicesAfter).Error; err != nil {
-		t.Fatal(err)
-	} else if nSlicesBefore != nSlicesAfter {
+	} else if nSlicesAfter := ss.Count("slices"); nSlicesAfter != nSlicesBefore {
 		t.Fatalf("expected number of slices to stay the same, but got %v before and %v after", nSlicesBefore, nSlicesAfter)
 	}
 

From bcf38f9a91385109291fb94e4c7e099b6a5eea28 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 14:03:55 +0200
Subject: [PATCH 11/19] stores: remove dbSlab

---
 stores/metadata.go      |  21 ----
 stores/metadata_test.go | 229 ++++++++++++++++++++--------------------
 stores/types.go         |  29 -----
 3 files changed, 112 insertions(+), 167 deletions(-)

diff --git a/stores/metadata.go b/stores/metadata.go
index 4c545d449..fe26bc29e 100644
--- a/stores/metadata.go
+++ b/stores/metadata.go
@@ -45,27 +45,6 @@ var (
 
 var objectDeleteBatchSizes = []int64{10, 50, 100, 200, 500, 1000, 5000, 10000, 50000, 100000}
 
-type (
-	dbSlab struct {
-		Model
-		DBContractSetID  uint `gorm:"index"`
-		DBBufferedSlabID uint `gorm:"index;default: NULL"`
-
-		Health           float64   `gorm:"index;default:1.0; NOT NULL"`
-		HealthValidUntil int64     `gorm:"index;default:0; NOT NULL"` // unix timestamp
-		Key              secretKey `gorm:"unique;NOT NULL;size:32"`   // json string
-		MinShards        uint8     `gorm:"index"`
-		TotalShards      uint8     `gorm:"index"`
-	}
-)
-
-func (s dbSlab) HealthValid() bool {
-	return time.Now().Before(time.Unix(s.HealthValidUntil, 0))
-}
-
-// TableName implements the gorm.Tabler interface.
-func (dbSlab) TableName() string { return "slabs" }
-
 func (s *SQLStore) Bucket(ctx context.Context, bucket string) (b api.Bucket, err error) {
 	err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) (err error) {
 		b, err = tx.Bucket(ctx, bucket)
diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index fecb8004b..9b56647e0 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -3,6 +3,7 @@ package stores
 import (
 	"bytes"
 	"context"
+	dsql "database/sql"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -2250,10 +2251,7 @@ func TestUpdateSlab(t *testing.T) {
 	}
 
 	// assert there's still only one entry in the dbslab table
-	var cnt int64
-	if err := ss.gormDB.Model(&dbSlab{}).Count(&cnt).Error; err != nil {
-		t.Fatal(err)
-	} else if cnt != 1 {
+	if cnt := ss.Count("slabs"); cnt != 1 {
 		t.Fatalf("unexpected number of entries in dbslab, %v != 1", cnt)
 	}
 
@@ -2759,15 +2757,23 @@ func TestPartialSlab(t *testing.T) {
 
 	type bufferedSlab struct {
 		ID       uint
-		DBSlab   dbSlab `gorm:"foreignKey:DBBufferedSlabID"`
 		Filename string
 	}
-
-	var buffer bufferedSlab
-	sk, _ := slabs[0].Key.MarshalBinary()
-	if err := ss.gormDB.Joins("DBSlab").Take(&buffer, "DBSlab.key = ?", secretKey(sk)).Error; err != nil {
-		t.Fatal(err)
+	fetchBuffer := func(ec object.EncryptionKey) (b bufferedSlab) {
+		t.Helper()
+		if err := ss.DB().QueryRow(context.Background(), `
+			SELECT bs.id, bs.filename
+			FROM buffered_slabs bs
+			INNER JOIN slabs sla ON sla.db_buffered_slab_id = bs.id
+			WHERE sla.key = ?
+		`, sql.EncryptionKey(ec)).
+			Scan(&b.ID, &b.Filename); err != nil && !errors.Is(err, dsql.ErrNoRows) {
+			t.Fatal(err)
+		}
+		return
 	}
+
+	buffer := fetchBuffer(slabs[0].Key)
 	if buffer.Filename == "" {
 		t.Fatal("empty filename")
 	}
@@ -2825,11 +2831,6 @@ func TestPartialSlab(t *testing.T) {
 	} else if !bytes.Equal(data, slab2Data) {
 		t.Fatal("wrong data")
 	}
-	buffer = bufferedSlab{}
-	sk, _ = slabs[0].Key.MarshalBinary()
-	if err := ss.gormDB.Joins("DBSlab").Take(&buffer, "DBSlab.key = ?", secretKey(sk)).Error; err != nil {
-		t.Fatal(err)
-	}
 	assertBuffer(buffer1Name, 4194303, false, false)
 
 	// Create an object again.
@@ -2866,17 +2867,9 @@ func TestPartialSlab(t *testing.T) {
 	} else if !bytes.Equal(slab3Data, append(data1, data2...)) {
 		t.Fatal("wrong data")
 	}
-	buffer = bufferedSlab{}
-	sk, _ = slabs[0].Key.MarshalBinary()
-	if err := ss.gormDB.Joins("DBSlab").Take(&buffer, "DBSlab.key = ?", secretKey(sk)).Error; err != nil {
-		t.Fatal(err)
-	}
 	assertBuffer(buffer1Name, rhpv2.SectorSize, true, false)
-	buffer = bufferedSlab{}
-	sk, _ = slabs[1].Key.MarshalBinary()
-	if err := ss.gormDB.Joins("DBSlab").Take(&buffer, "DBSlab.key = ?", secretKey(sk)).Error; err != nil {
-		t.Fatal(err)
-	}
+
+	buffer = fetchBuffer(slabs[1].Key)
 	buffer2Name := buffer.Filename
 	assertBuffer(buffer2Name, 1, false, false)
 
@@ -2901,13 +2894,9 @@ func TestPartialSlab(t *testing.T) {
 	assertBuffer(buffer1Name, rhpv2.SectorSize, true, true)
 	assertBuffer(buffer2Name, 1, false, false)
 
-	var foo []bufferedSlab
-	if err := ss.gormDB.Find(&foo).Error; err != nil {
-		t.Fatal(err)
-	}
-	buffer = bufferedSlab{}
-	if err := ss.gormDB.Take(&buffer, "id = ?", packedSlabs[0].BufferID).Error; err != nil {
-		t.Fatal(err)
+	buffer = fetchBuffer(packedSlabs[0].Key)
+	if buffer.ID != packedSlabs[0].BufferID {
+		t.Fatalf("wrong buffer id, %v != %v", buffer.ID, packedSlabs[0].BufferID)
 	}
 
 	// Mark slab as uploaded.
@@ -2924,8 +2913,8 @@ func TestPartialSlab(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	buffer = bufferedSlab{}
-	if err := ss.gormDB.Take(&buffer, "id = ?", packedSlabs[0].BufferID).Error; !errors.Is(err, gorm.ErrRecordNotFound) {
+	buffer = fetchBuffer(packedSlabs[0].Key)
+	if buffer != (bufferedSlab{}) {
 		t.Fatal("shouldn't be able to find buffer", err)
 	}
 	assertBuffer(buffer2Name, 1, false, false)
@@ -3681,10 +3670,11 @@ func TestDeleteHostSector(t *testing.T) {
 	}
 
 	// Find the slab. It should have an invalid health.
-	var s dbSlab
-	if err := ss.gormDB.Take(&s).Error; err != nil {
+	var slabID int64
+	var validUntil int64
+	if err := ss.DB().QueryRow(context.Background(), "SELECT id, health_valid_until FROM slabs").Scan(&slabID, &validUntil); err != nil {
 		t.Fatal(err)
-	} else if s.HealthValid() {
+	} else if time.Now().Before(time.Unix(validUntil, 0)) {
 		t.Fatal("expected health to be invalid")
 	}
 
@@ -3732,8 +3722,8 @@ func TestDeleteHostSector(t *testing.T) {
 		t.Fatal("expected 2 contracts", cnt)
 	} else if sectors[0].LatestHost != hk2 {
 		t.Fatalf("expected latest host to be hk2, got %v", sectors[0].LatestHost)
-	} else if sectors[0].SlabID != int64(s.ID) {
-		t.Fatalf("expected slab id to be %v, got %v", s.ID, sectors[0].SlabID)
+	} else if sectors[0].SlabID != slabID {
+		t.Fatalf("expected slab id to be %v, got %v", slabID, sectors[0].SlabID)
 	}
 
 	hi, err := ss.Host(context.Background(), hk1)
@@ -3776,8 +3766,8 @@ func TestDeleteHostSector(t *testing.T) {
 		t.Fatal("expected 0 contracts", cnt)
 	} else if sector := sectors[0]; sector.LatestHost != [32]byte{} {
 		t.Fatal("expected latest host to be empty", sector.LatestHost)
-	} else if sectors[0].SlabID != int64(s.ID) {
-		t.Fatalf("expected slab id to be %v, got %v", s.ID, sectors[0].SlabID)
+	} else if sectors[0].SlabID != slabID {
+		t.Fatalf("expected slab id to be %v, got %v", slabID, sectors[0].SlabID)
 	}
 }
 func newTestShards(hk types.PublicKey, fcid types.FileContractID, root types.Hash256) []object.Sector {
@@ -3870,13 +3860,11 @@ func TestSlabHealthInvalidation(t *testing.T) {
 	assertHealthValid := func(slabKey object.EncryptionKey, expected bool) {
 		t.Helper()
 
-		var slab dbSlab
-		if key, err := slabKey.MarshalBinary(); err != nil {
+		var validUntil int64
+		if err := ss.DB().QueryRow(context.Background(), "SELECT health_valid_until FROM slabs WHERE key = ?", sql.EncryptionKey(slabKey)).Scan(&validUntil); err != nil {
 			t.Fatal(err)
-		} else if err := ss.gormDB.Model(&dbSlab{}).Where(&dbSlab{Key: key}).Take(&slab).Error; err != nil {
-			t.Fatal(err)
-		} else if slab.HealthValid() != expected {
-			t.Fatal("unexpected health valid", slab.HealthValid(), slab.HealthValidUntil, time.Now(), time.Unix(slab.HealthValidUntil, 0))
+		} else if valid := time.Now().Before(time.Unix(validUntil, 0)); valid != expected {
+			t.Fatal("unexpected health valid", valid)
 		}
 	}
 
@@ -4005,19 +3993,17 @@ func TestSlabHealthInvalidation(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// fetch slab
-		var slab dbSlab
-		if key, err := s1.MarshalBinary(); err != nil {
-			t.Fatal(err)
-		} else if err := ss.gormDB.Model(&dbSlab{}).Where(&dbSlab{Key: key}).Take(&slab).Error; err != nil {
+		// fetch health_valid_until
+		var validUntil int64
+		if err := ss.DB().QueryRow(context.Background(), "SELECT health_valid_until FROM slabs").Scan(&validUntil); err != nil {
 			t.Fatal(err)
 		}
 
 		// assert it's validity is within expected bounds
 		minValidity := now.Add(refreshHealthMinHealthValidity).Add(-time.Second) // avoid NDF
 		maxValidity := now.Add(refreshHealthMaxHealthValidity).Add(time.Second)  // avoid NDF
-		validUntil := time.Unix(slab.HealthValidUntil, 0)
-		if !(minValidity.Before(validUntil) && maxValidity.After(validUntil)) {
+		validUntilUnix := time.Unix(validUntil, 0)
+		if !(minValidity.Before(validUntilUnix) && maxValidity.After(validUntilUnix)) {
 			t.Fatal("valid until not in boundaries", minValidity, maxValidity, validUntil, now)
 		}
 	}
@@ -4189,14 +4175,10 @@ func TestSlabCleanup(t *testing.T) {
 	}
 
 	// create a slab
-	ek, _ := object.GenerateEncryptionKey().MarshalBinary()
-	slab := dbSlab{
-		DBContractSetID:  uint(csID),
-		Health:           1,
-		Key:              secretKey(ek),
-		HealthValidUntil: 100,
-	}
-	if err := ss.gormDB.Create(&slab).Error; err != nil {
+	var slabID int64
+	if res, err := ss.DB().Exec(context.Background(), "INSERT INTO slabs (db_contract_set_id, key, health_valid_until) VALUES (?, ?, ?);", csID, sql.EncryptionKey(object.GenerateEncryptionKey()), 100); err != nil {
+		t.Fatal(err)
+	} else if slabID, err = res.LastInsertId(); err != nil {
 		t.Fatal(err)
 	}
 
@@ -4208,9 +4190,9 @@ func TestSlabCleanup(t *testing.T) {
 	defer insertSlabRefStmt.Close()
 
 	// reference the slab
-	if _, err := insertSlabRefStmt.Exec(context.Background(), obj1ID, slab.ID); err != nil {
+	if _, err := insertSlabRefStmt.Exec(context.Background(), obj1ID, slabID); err != nil {
 		t.Fatal(err)
-	} else if _, err := insertSlabRefStmt.Exec(context.Background(), obj2ID, slab.ID); err != nil {
+	} else if _, err := insertSlabRefStmt.Exec(context.Background(), obj2ID, slabID); err != nil {
 		t.Fatal(err)
 	}
 
@@ -4221,10 +4203,7 @@ func TestSlabCleanup(t *testing.T) {
 	}
 
 	// check slab count
-	var slabCntr int64
-	if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
-		t.Fatal(err)
-	} else if slabCntr != 1 {
+	if slabCntr := ss.Count("slabs"); slabCntr != 1 {
 		t.Fatalf("expected 1 slabs, got %v", slabCntr)
 	}
 
@@ -4232,35 +4211,27 @@ func TestSlabCleanup(t *testing.T) {
 	err = ss.RemoveObjectBlocking(context.Background(), api.DefaultBucketName, "2")
 	if err != nil {
 		t.Fatal(err)
-	} else if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
-		t.Fatal(err)
-	} else if slabCntr != 0 {
+	} else if slabCntr := ss.Count("slabs"); slabCntr != 0 {
 		t.Fatalf("expected 0 slabs, got %v", slabCntr)
 	}
 
-	// create another object that references a slab with buffer
-	ek, _ = object.GenerateEncryptionKey().MarshalBinary()
-	bufferedSlab := dbSlab{
-		DBBufferedSlabID: bsID,
-		DBContractSetID:  uint(csID),
-		Health:           1,
-		Key:              ek,
-		HealthValidUntil: 100,
-	}
-	if err := ss.gormDB.Create(&bufferedSlab).Error; err != nil {
+	// create another slab referencing the buffered slab
+	var bufferedSlabID int64
+	if res, err := ss.DB().Exec(context.Background(), "INSERT INTO slabs (db_buffered_slab_id, db_contract_set_id, key, health_valid_until) VALUES (?, ?, ?, ?);", bsID, csID, sql.EncryptionKey(object.GenerateEncryptionKey()), 100); err != nil {
+		t.Fatal(err)
+	} else if bufferedSlabID, err = res.LastInsertId(); err != nil {
 		t.Fatal(err)
 	}
+
 	var obj3ID int64
 	if res, err := insertObjStmt.Exec(context.Background(), dirID, "3", ss.DefaultBucketID(), 1); err != nil {
 		t.Fatal(err)
 	} else if obj3ID, err = res.LastInsertId(); err != nil {
 		t.Fatal(err)
-	} else if _, err := insertSlabRefStmt.Exec(context.Background(), obj3ID, bufferedSlab.ID); err != nil {
+	} else if _, err := insertSlabRefStmt.Exec(context.Background(), obj3ID, bufferedSlabID); err != nil {
 		t.Fatal(err)
 	}
-	if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
-		t.Fatal(err)
-	} else if slabCntr != 1 {
+	if slabCntr := ss.Count("slabs"); slabCntr != 1 {
 		t.Fatalf("expected 1 slabs, got %v", slabCntr)
 	}
 
@@ -4268,9 +4239,7 @@ func TestSlabCleanup(t *testing.T) {
 	err = ss.RemoveObjectBlocking(context.Background(), api.DefaultBucketName, "3")
 	if err != nil {
 		t.Fatal(err)
-	} else if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
-		t.Fatal(err)
-	} else if slabCntr != 1 {
+	} else if slabCntr := ss.Count("slabs"); slabCntr != 1 {
 		t.Fatalf("expected 1 slabs, got %v", slabCntr)
 	}
 }
@@ -4412,6 +4381,22 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 		return
 	}
 
+	// helper type to fetch a slab
+	type slab struct {
+		ID               int64
+		ContractSetID    int64
+		Health           float64
+		HealthValidUntil int64
+		MinShards        uint8
+		TotalShards      uint8
+		Key              object.EncryptionKey
+	}
+	fetchSlabStmt, err := ss.DB().Prepare(context.Background(), "SELECT id, db_contract_set_id, health, health_valid_until, min_shards, total_shards, key FROM slabs WHERE id = ?")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fetchSlabStmt.Close()
+
 	for i, slice := range slices {
 		if slice.ID != int64(i+1) {
 			t.Fatal("unexpected id", slice.ID)
@@ -4422,35 +4407,36 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 		}
 
 		// fetch the slab
-		var dbSlab dbSlab
-		key, _ := obj.Slabs[i].Key.MarshalBinary()
-		if err := ss.gormDB.Where("id", slice.SlabID).Take(&dbSlab).Error; err != nil {
+		var slab slab
+		err = fetchSlabStmt.QueryRow(context.Background(), slice.SlabID).
+			Scan(&slab.ID, &slab.ContractSetID, &slab.Health, &slab.HealthValidUntil, &slab.MinShards, &slab.TotalShards, (*sql.EncryptionKey)(&slab.Key))
+		if err != nil {
 			t.Fatal(err)
-		} else if dbSlab.ID != uint(i+1) {
-			t.Fatal("unexpected id", dbSlab.ID)
-		} else if dbSlab.DBContractSetID != 1 {
-			t.Fatal("invalid contract set id", dbSlab.DBContractSetID)
-		} else if dbSlab.Health != 1 {
-			t.Fatal("invalid health", dbSlab.Health)
-		} else if dbSlab.HealthValidUntil != 0 {
-			t.Fatal("invalid health validity", dbSlab.HealthValidUntil)
-		} else if dbSlab.MinShards != uint8(minShards) {
-			t.Fatal("invalid minShards", dbSlab.MinShards)
-		} else if dbSlab.TotalShards != uint8(totalShards) {
-			t.Fatal("invalid totalShards", dbSlab.TotalShards)
-		} else if !bytes.Equal(dbSlab.Key, key) {
+		} else if slab.ID != int64(i+1) {
+			t.Fatal("unexpected id", slab.ID)
+		} else if slab.ContractSetID != 1 {
+			t.Fatal("invalid contract set id", slab.ContractSetID)
+		} else if slab.Health != 1 {
+			t.Fatal("invalid health", slab.Health)
+		} else if slab.HealthValidUntil != 0 {
+			t.Fatal("invalid health validity", slab.HealthValidUntil)
+		} else if slab.MinShards != uint8(minShards) {
+			t.Fatal("invalid minShards", slab.MinShards)
+		} else if slab.TotalShards != uint8(totalShards) {
+			t.Fatal("invalid totalShards", slab.TotalShards)
+		} else if slab.Key.String() != obj.Slabs[i].Key.String() {
 			t.Fatal("wrong key")
 		}
 
 		// fetch the sectors
-		sectors := fetchSectorsBySlabID(int64(dbSlab.ID))
+		sectors := fetchSectorsBySlabID(int64(slab.ID))
 		if len(sectors) != totalShards {
 			t.Fatal("invalid number of sectors", len(sectors))
 		}
 		for j, sector := range sectors {
 			if sector.ID != int64(i*totalShards+j+1) {
 				t.Fatal("invalid id", sector.ID)
-			} else if sector.SlabID != int64(dbSlab.ID) {
+			} else if sector.SlabID != int64(slab.ID) {
 				t.Fatal("invalid slab id", sector.SlabID)
 			} else if sector.LatestHost != hks[i*totalShards+j] {
 				t.Fatal("invalid host")
@@ -4524,27 +4510,36 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 	}
 
 	// fetch the slab
-	var dbSlab2 dbSlab
-	key, _ := obj2.Slabs[0].Key.MarshalBinary()
-	if err := ss.gormDB.Where("id", slice2.SlabID).Take(&dbSlab2).Error; err != nil {
-		t.Fatal(err)
-	} else if dbSlab2.ID != uint(len(slices)+1) {
-		t.Fatal("unexpected id", dbSlab2.ID)
-	} else if dbSlab2.DBContractSetID != 1 {
-		t.Fatal("invalid contract set id", dbSlab2.DBContractSetID)
-	} else if !bytes.Equal(dbSlab2.Key, key) {
+	var slab2 slab
+	err = fetchSlabStmt.QueryRow(context.Background(), slice2.SlabID).
+		Scan(&slab2.ID, &slab2.ContractSetID, &slab2.Health, &slab2.HealthValidUntil, &slab2.MinShards, &slab2.TotalShards, (*sql.EncryptionKey)(&slab2.Key))
+	if err != nil {
+		t.Fatal(err)
+	} else if slab2.ID != int64(len(slices)+1) {
+		t.Fatal("unexpected id", slab2.ID)
+	} else if slab2.ContractSetID != 1 {
+		t.Fatal("invalid contract set id", slab2.ContractSetID)
+	} else if slab2.Health != 1 {
+		t.Fatal("invalid health", slab2.Health)
+	} else if slab2.HealthValidUntil != 0 {
+		t.Fatal("invalid health validity", slab2.HealthValidUntil)
+	} else if slab2.MinShards != uint8(minShards) {
+		t.Fatal("invalid minShards", slab2.MinShards)
+	} else if slab2.TotalShards != uint8(totalShards) {
+		t.Fatal("invalid totalShards", slab2.TotalShards)
+	} else if slab2.Key.String() != obj2.Slabs[0].Key.String() {
 		t.Fatal("wrong key")
 	}
 
 	// fetch the sectors
-	sectors2 := fetchSectorsBySlabID(int64(dbSlab2.ID))
+	sectors2 := fetchSectorsBySlabID(int64(slab2.ID))
 	if len(sectors2) != totalShards {
 		t.Fatal("invalid number of sectors", len(sectors2))
 	}
 	for j, sector := range sectors2 {
 		if sector.ID != int64((len(obj.Slabs))*totalShards+j+1) {
 			t.Fatal("invalid id", sector.ID)
-		} else if sector.SlabID != int64(dbSlab2.ID) {
+		} else if sector.SlabID != int64(slab2.ID) {
 			t.Fatal("invalid slab id", sector.SlabID)
 		} else if sector.LatestHost != hks[(len(obj.Slabs))*totalShards+j] {
 			t.Fatal("invalid host")
diff --git a/stores/types.go b/stores/types.go
index 93da48db0..6d6ad2c4a 100644
--- a/stores/types.go
+++ b/stores/types.go
@@ -22,7 +22,6 @@ type (
 	bCurrency      types.Currency
 	fileContractID types.FileContractID
 	publicKey      types.PublicKey
-	secretKey      []byte
 	setting        string
 
 	// NOTE: we have to wrap the proof here because Gorm can't scan bytes into
@@ -62,34 +61,6 @@ func (s setting) Value() (driver.Value, error) {
 	return string(s), nil
 }
 
-// GormDataType implements gorm.GormDataTypeInterface.
-func (secretKey) GormDataType() string {
-	return "bytes"
-}
-
-// String implements fmt.Stringer to prevent the key from getting leaked in
-// logs.
-func (k secretKey) String() string {
-	return "*****"
-}
-
-// Scan scans value into key, implements sql.Scanner interface.
-func (k *secretKey) Scan(value interface{}) error {
-	bytes, ok := value.([]byte)
-	if !ok {
-		return errors.New(fmt.Sprint("failed to unmarshal secretKey value:", value))
-	} else if len(bytes) != secretKeySize {
-		return fmt.Errorf("failed to unmarshal secretKey value due to invalid number of bytes %v != %v: %v", len(bytes), secretKeySize, value)
-	}
-	*k = append(secretKey{}, secretKey(bytes)...)
-	return nil
-}
-
-// Value returns an key value, implements driver.Valuer interface.
-func (k secretKey) Value() (driver.Value, error) {
-	return []byte(k), nil
-}
-
 // GormDataType implements gorm.GormDataTypeInterface.
 func (fileContractID) GormDataType() string {
 	return "bytes"

From 2e0831cf2e3112d003ae149e1a88353a738697f6 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 14:03:55 +0200
Subject: [PATCH 12/19] stores: remove dbSlab

---
 stores/metadata.go      |  21 ----
 stores/metadata_test.go | 229 ++++++++++++++++++++--------------------
 stores/types.go         |  29 -----
 stores/types_test.go    |  11 --
 4 files changed, 112 insertions(+), 178 deletions(-)

diff --git a/stores/metadata.go b/stores/metadata.go
index 4c545d449..fe26bc29e 100644
--- a/stores/metadata.go
+++ b/stores/metadata.go
@@ -45,27 +45,6 @@ var (
 
 var objectDeleteBatchSizes = []int64{10, 50, 100, 200, 500, 1000, 5000, 10000, 50000, 100000}
 
-type (
-	dbSlab struct {
-		Model
-		DBContractSetID  uint `gorm:"index"`
-		DBBufferedSlabID uint `gorm:"index;default: NULL"`
-
-		Health           float64   `gorm:"index;default:1.0; NOT NULL"`
-		HealthValidUntil int64     `gorm:"index;default:0; NOT NULL"` // unix timestamp
-		Key              secretKey `gorm:"unique;NOT NULL;size:32"`   // json string
-		MinShards        uint8     `gorm:"index"`
-		TotalShards      uint8     `gorm:"index"`
-	}
-)
-
-func (s dbSlab) HealthValid() bool {
-	return time.Now().Before(time.Unix(s.HealthValidUntil, 0))
-}
-
-// TableName implements the gorm.Tabler interface.
-func (dbSlab) TableName() string { return "slabs" }
-
 func (s *SQLStore) Bucket(ctx context.Context, bucket string) (b api.Bucket, err error) {
 	err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) (err error) {
 		b, err = tx.Bucket(ctx, bucket)
diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index fecb8004b..9b56647e0 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -3,6 +3,7 @@ package stores
 import (
 	"bytes"
 	"context"
+	dsql "database/sql"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -2250,10 +2251,7 @@ func TestUpdateSlab(t *testing.T) {
 	}
 
 	// assert there's still only one entry in the dbslab table
-	var cnt int64
-	if err := ss.gormDB.Model(&dbSlab{}).Count(&cnt).Error; err != nil {
-		t.Fatal(err)
-	} else if cnt != 1 {
+	if cnt := ss.Count("slabs"); cnt != 1 {
 		t.Fatalf("unexpected number of entries in dbslab, %v != 1", cnt)
 	}
 
@@ -2759,15 +2757,23 @@ func TestPartialSlab(t *testing.T) {
 
 	type bufferedSlab struct {
 		ID       uint
-		DBSlab   dbSlab `gorm:"foreignKey:DBBufferedSlabID"`
 		Filename string
 	}
-
-	var buffer bufferedSlab
-	sk, _ := slabs[0].Key.MarshalBinary()
-	if err := ss.gormDB.Joins("DBSlab").Take(&buffer, "DBSlab.key = ?", secretKey(sk)).Error; err != nil {
-		t.Fatal(err)
+	fetchBuffer := func(ec object.EncryptionKey) (b bufferedSlab) {
+		t.Helper()
+		if err := ss.DB().QueryRow(context.Background(), `
+			SELECT bs.id, bs.filename
+			FROM buffered_slabs bs
+			INNER JOIN slabs sla ON sla.db_buffered_slab_id = bs.id
+			WHERE sla.key = ?
+		`, sql.EncryptionKey(ec)).
+			Scan(&b.ID, &b.Filename); err != nil && !errors.Is(err, dsql.ErrNoRows) {
+			t.Fatal(err)
+		}
+		return
 	}
+
+	buffer := fetchBuffer(slabs[0].Key)
 	if buffer.Filename == "" {
 		t.Fatal("empty filename")
 	}
@@ -2825,11 +2831,6 @@ func TestPartialSlab(t *testing.T) {
 	} else if !bytes.Equal(data, slab2Data) {
 		t.Fatal("wrong data")
 	}
-	buffer = bufferedSlab{}
-	sk, _ = slabs[0].Key.MarshalBinary()
-	if err := ss.gormDB.Joins("DBSlab").Take(&buffer, "DBSlab.key = ?", secretKey(sk)).Error; err != nil {
-		t.Fatal(err)
-	}
 	assertBuffer(buffer1Name, 4194303, false, false)
 
 	// Create an object again.
@@ -2866,17 +2867,9 @@ func TestPartialSlab(t *testing.T) {
 	} else if !bytes.Equal(slab3Data, append(data1, data2...)) {
 		t.Fatal("wrong data")
 	}
-	buffer = bufferedSlab{}
-	sk, _ = slabs[0].Key.MarshalBinary()
-	if err := ss.gormDB.Joins("DBSlab").Take(&buffer, "DBSlab.key = ?", secretKey(sk)).Error; err != nil {
-		t.Fatal(err)
-	}
 	assertBuffer(buffer1Name, rhpv2.SectorSize, true, false)
-	buffer = bufferedSlab{}
-	sk, _ = slabs[1].Key.MarshalBinary()
-	if err := ss.gormDB.Joins("DBSlab").Take(&buffer, "DBSlab.key = ?", secretKey(sk)).Error; err != nil {
-		t.Fatal(err)
-	}
+
+	buffer = fetchBuffer(slabs[1].Key)
 	buffer2Name := buffer.Filename
 	assertBuffer(buffer2Name, 1, false, false)
 
@@ -2901,13 +2894,9 @@ func TestPartialSlab(t *testing.T) {
 	assertBuffer(buffer1Name, rhpv2.SectorSize, true, true)
 	assertBuffer(buffer2Name, 1, false, false)
 
-	var foo []bufferedSlab
-	if err := ss.gormDB.Find(&foo).Error; err != nil {
-		t.Fatal(err)
-	}
-	buffer = bufferedSlab{}
-	if err := ss.gormDB.Take(&buffer, "id = ?", packedSlabs[0].BufferID).Error; err != nil {
-		t.Fatal(err)
+	buffer = fetchBuffer(packedSlabs[0].Key)
+	if buffer.ID != packedSlabs[0].BufferID {
+		t.Fatalf("wrong buffer id, %v != %v", buffer.ID, packedSlabs[0].BufferID)
 	}
 
 	// Mark slab as uploaded.
@@ -2924,8 +2913,8 @@ func TestPartialSlab(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	buffer = bufferedSlab{}
-	if err := ss.gormDB.Take(&buffer, "id = ?", packedSlabs[0].BufferID).Error; !errors.Is(err, gorm.ErrRecordNotFound) {
+	buffer = fetchBuffer(packedSlabs[0].Key)
+	if buffer != (bufferedSlab{}) {
 		t.Fatal("shouldn't be able to find buffer", err)
 	}
 	assertBuffer(buffer2Name, 1, false, false)
@@ -3681,10 +3670,11 @@ func TestDeleteHostSector(t *testing.T) {
 	}
 
 	// Find the slab. It should have an invalid health.
-	var s dbSlab
-	if err := ss.gormDB.Take(&s).Error; err != nil {
+	var slabID int64
+	var validUntil int64
+	if err := ss.DB().QueryRow(context.Background(), "SELECT id, health_valid_until FROM slabs").Scan(&slabID, &validUntil); err != nil {
 		t.Fatal(err)
-	} else if s.HealthValid() {
+	} else if time.Now().Before(time.Unix(validUntil, 0)) {
 		t.Fatal("expected health to be invalid")
 	}
 
@@ -3732,8 +3722,8 @@ func TestDeleteHostSector(t *testing.T) {
 		t.Fatal("expected 2 contracts", cnt)
 	} else if sectors[0].LatestHost != hk2 {
 		t.Fatalf("expected latest host to be hk2, got %v", sectors[0].LatestHost)
-	} else if sectors[0].SlabID != int64(s.ID) {
-		t.Fatalf("expected slab id to be %v, got %v", s.ID, sectors[0].SlabID)
+	} else if sectors[0].SlabID != slabID {
+		t.Fatalf("expected slab id to be %v, got %v", slabID, sectors[0].SlabID)
 	}
 
 	hi, err := ss.Host(context.Background(), hk1)
@@ -3776,8 +3766,8 @@ func TestDeleteHostSector(t *testing.T) {
 		t.Fatal("expected 0 contracts", cnt)
 	} else if sector := sectors[0]; sector.LatestHost != [32]byte{} {
 		t.Fatal("expected latest host to be empty", sector.LatestHost)
-	} else if sectors[0].SlabID != int64(s.ID) {
-		t.Fatalf("expected slab id to be %v, got %v", s.ID, sectors[0].SlabID)
+	} else if sectors[0].SlabID != slabID {
+		t.Fatalf("expected slab id to be %v, got %v", slabID, sectors[0].SlabID)
 	}
 }
 func newTestShards(hk types.PublicKey, fcid types.FileContractID, root types.Hash256) []object.Sector {
@@ -3870,13 +3860,11 @@ func TestSlabHealthInvalidation(t *testing.T) {
 	assertHealthValid := func(slabKey object.EncryptionKey, expected bool) {
 		t.Helper()
 
-		var slab dbSlab
-		if key, err := slabKey.MarshalBinary(); err != nil {
+		var validUntil int64
+		if err := ss.DB().QueryRow(context.Background(), "SELECT health_valid_until FROM slabs WHERE key = ?", sql.EncryptionKey(slabKey)).Scan(&validUntil); err != nil {
 			t.Fatal(err)
-		} else if err := ss.gormDB.Model(&dbSlab{}).Where(&dbSlab{Key: key}).Take(&slab).Error; err != nil {
-			t.Fatal(err)
-		} else if slab.HealthValid() != expected {
-			t.Fatal("unexpected health valid", slab.HealthValid(), slab.HealthValidUntil, time.Now(), time.Unix(slab.HealthValidUntil, 0))
+		} else if valid := time.Now().Before(time.Unix(validUntil, 0)); valid != expected {
+			t.Fatal("unexpected health valid", valid)
 		}
 	}
 
@@ -4005,19 +3993,17 @@ func TestSlabHealthInvalidation(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		// fetch slab
-		var slab dbSlab
-		if key, err := s1.MarshalBinary(); err != nil {
-			t.Fatal(err)
-		} else if err := ss.gormDB.Model(&dbSlab{}).Where(&dbSlab{Key: key}).Take(&slab).Error; err != nil {
+		// fetch health_valid_until
+		var validUntil int64
+		if err := ss.DB().QueryRow(context.Background(), "SELECT health_valid_until FROM slabs").Scan(&validUntil); err != nil {
 			t.Fatal(err)
 		}
 
 		// assert it's validity is within expected bounds
 		minValidity := now.Add(refreshHealthMinHealthValidity).Add(-time.Second) // avoid NDF
 		maxValidity := now.Add(refreshHealthMaxHealthValidity).Add(time.Second)  // avoid NDF
-		validUntil := time.Unix(slab.HealthValidUntil, 0)
-		if !(minValidity.Before(validUntil) && maxValidity.After(validUntil)) {
+		validUntilUnix := time.Unix(validUntil, 0)
+		if !(minValidity.Before(validUntilUnix) && maxValidity.After(validUntilUnix)) {
 			t.Fatal("valid until not in boundaries", minValidity, maxValidity, validUntil, now)
 		}
 	}
@@ -4189,14 +4175,10 @@ func TestSlabCleanup(t *testing.T) {
 	}
 
 	// create a slab
-	ek, _ := object.GenerateEncryptionKey().MarshalBinary()
-	slab := dbSlab{
-		DBContractSetID:  uint(csID),
-		Health:           1,
-		Key:              secretKey(ek),
-		HealthValidUntil: 100,
-	}
-	if err := ss.gormDB.Create(&slab).Error; err != nil {
+	var slabID int64
+	if res, err := ss.DB().Exec(context.Background(), "INSERT INTO slabs (db_contract_set_id, key, health_valid_until) VALUES (?, ?, ?);", csID, sql.EncryptionKey(object.GenerateEncryptionKey()), 100); err != nil {
+		t.Fatal(err)
+	} else if slabID, err = res.LastInsertId(); err != nil {
 		t.Fatal(err)
 	}
 
@@ -4208,9 +4190,9 @@ func TestSlabCleanup(t *testing.T) {
 	defer insertSlabRefStmt.Close()
 
 	// reference the slab
-	if _, err := insertSlabRefStmt.Exec(context.Background(), obj1ID, slab.ID); err != nil {
+	if _, err := insertSlabRefStmt.Exec(context.Background(), obj1ID, slabID); err != nil {
 		t.Fatal(err)
-	} else if _, err := insertSlabRefStmt.Exec(context.Background(), obj2ID, slab.ID); err != nil {
+	} else if _, err := insertSlabRefStmt.Exec(context.Background(), obj2ID, slabID); err != nil {
 		t.Fatal(err)
 	}
 
@@ -4221,10 +4203,7 @@ func TestSlabCleanup(t *testing.T) {
 	}
 
 	// check slab count
-	var slabCntr int64
-	if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
-		t.Fatal(err)
-	} else if slabCntr != 1 {
+	if slabCntr := ss.Count("slabs"); slabCntr != 1 {
 		t.Fatalf("expected 1 slabs, got %v", slabCntr)
 	}
 
@@ -4232,35 +4211,27 @@ func TestSlabCleanup(t *testing.T) {
 	err = ss.RemoveObjectBlocking(context.Background(), api.DefaultBucketName, "2")
 	if err != nil {
 		t.Fatal(err)
-	} else if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
-		t.Fatal(err)
-	} else if slabCntr != 0 {
+	} else if slabCntr := ss.Count("slabs"); slabCntr != 0 {
 		t.Fatalf("expected 0 slabs, got %v", slabCntr)
 	}
 
-	// create another object that references a slab with buffer
-	ek, _ = object.GenerateEncryptionKey().MarshalBinary()
-	bufferedSlab := dbSlab{
-		DBBufferedSlabID: bsID,
-		DBContractSetID:  uint(csID),
-		Health:           1,
-		Key:              ek,
-		HealthValidUntil: 100,
-	}
-	if err := ss.gormDB.Create(&bufferedSlab).Error; err != nil {
+	// create another slab referencing the buffered slab
+	var bufferedSlabID int64
+	if res, err := ss.DB().Exec(context.Background(), "INSERT INTO slabs (db_buffered_slab_id, db_contract_set_id, key, health_valid_until) VALUES (?, ?, ?, ?);", bsID, csID, sql.EncryptionKey(object.GenerateEncryptionKey()), 100); err != nil {
+		t.Fatal(err)
+	} else if bufferedSlabID, err = res.LastInsertId(); err != nil {
 		t.Fatal(err)
 	}
+
 	var obj3ID int64
 	if res, err := insertObjStmt.Exec(context.Background(), dirID, "3", ss.DefaultBucketID(), 1); err != nil {
 		t.Fatal(err)
 	} else if obj3ID, err = res.LastInsertId(); err != nil {
 		t.Fatal(err)
-	} else if _, err := insertSlabRefStmt.Exec(context.Background(), obj3ID, bufferedSlab.ID); err != nil {
+	} else if _, err := insertSlabRefStmt.Exec(context.Background(), obj3ID, bufferedSlabID); err != nil {
 		t.Fatal(err)
 	}
-	if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
-		t.Fatal(err)
-	} else if slabCntr != 1 {
+	if slabCntr := ss.Count("slabs"); slabCntr != 1 {
 		t.Fatalf("expected 1 slabs, got %v", slabCntr)
 	}
 
@@ -4268,9 +4239,7 @@ func TestSlabCleanup(t *testing.T) {
 	err = ss.RemoveObjectBlocking(context.Background(), api.DefaultBucketName, "3")
 	if err != nil {
 		t.Fatal(err)
-	} else if err := ss.gormDB.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil {
-		t.Fatal(err)
-	} else if slabCntr != 1 {
+	} else if slabCntr := ss.Count("slabs"); slabCntr != 1 {
 		t.Fatalf("expected 1 slabs, got %v", slabCntr)
 	}
 }
@@ -4412,6 +4381,22 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 		return
 	}
 
+	// helper type to fetch a slab
+	type slab struct {
+		ID               int64
+		ContractSetID    int64
+		Health           float64
+		HealthValidUntil int64
+		MinShards        uint8
+		TotalShards      uint8
+		Key              object.EncryptionKey
+	}
+	fetchSlabStmt, err := ss.DB().Prepare(context.Background(), "SELECT id, db_contract_set_id, health, health_valid_until, min_shards, total_shards, key FROM slabs WHERE id = ?")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fetchSlabStmt.Close()
+
 	for i, slice := range slices {
 		if slice.ID != int64(i+1) {
 			t.Fatal("unexpected id", slice.ID)
@@ -4422,35 +4407,36 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 		}
 
 		// fetch the slab
-		var dbSlab dbSlab
-		key, _ := obj.Slabs[i].Key.MarshalBinary()
-		if err := ss.gormDB.Where("id", slice.SlabID).Take(&dbSlab).Error; err != nil {
+		var slab slab
+		err = fetchSlabStmt.QueryRow(context.Background(), slice.SlabID).
+			Scan(&slab.ID, &slab.ContractSetID, &slab.Health, &slab.HealthValidUntil, &slab.MinShards, &slab.TotalShards, (*sql.EncryptionKey)(&slab.Key))
+		if err != nil {
 			t.Fatal(err)
-		} else if dbSlab.ID != uint(i+1) {
-			t.Fatal("unexpected id", dbSlab.ID)
-		} else if dbSlab.DBContractSetID != 1 {
-			t.Fatal("invalid contract set id", dbSlab.DBContractSetID)
-		} else if dbSlab.Health != 1 {
-			t.Fatal("invalid health", dbSlab.Health)
-		} else if dbSlab.HealthValidUntil != 0 {
-			t.Fatal("invalid health validity", dbSlab.HealthValidUntil)
-		} else if dbSlab.MinShards != uint8(minShards) {
-			t.Fatal("invalid minShards", dbSlab.MinShards)
-		} else if dbSlab.TotalShards != uint8(totalShards) {
-			t.Fatal("invalid totalShards", dbSlab.TotalShards)
-		} else if !bytes.Equal(dbSlab.Key, key) {
+		} else if slab.ID != int64(i+1) {
+			t.Fatal("unexpected id", slab.ID)
+		} else if slab.ContractSetID != 1 {
+			t.Fatal("invalid contract set id", slab.ContractSetID)
+		} else if slab.Health != 1 {
+			t.Fatal("invalid health", slab.Health)
+		} else if slab.HealthValidUntil != 0 {
+			t.Fatal("invalid health validity", slab.HealthValidUntil)
+		} else if slab.MinShards != uint8(minShards) {
+			t.Fatal("invalid minShards", slab.MinShards)
+		} else if slab.TotalShards != uint8(totalShards) {
+			t.Fatal("invalid totalShards", slab.TotalShards)
+		} else if slab.Key.String() != obj.Slabs[i].Key.String() {
 			t.Fatal("wrong key")
 		}
 
 		// fetch the sectors
-		sectors := fetchSectorsBySlabID(int64(dbSlab.ID))
+		sectors := fetchSectorsBySlabID(int64(slab.ID))
 		if len(sectors) != totalShards {
 			t.Fatal("invalid number of sectors", len(sectors))
 		}
 		for j, sector := range sectors {
 			if sector.ID != int64(i*totalShards+j+1) {
 				t.Fatal("invalid id", sector.ID)
-			} else if sector.SlabID != int64(dbSlab.ID) {
+			} else if sector.SlabID != int64(slab.ID) {
 				t.Fatal("invalid slab id", sector.SlabID)
 			} else if sector.LatestHost != hks[i*totalShards+j] {
 				t.Fatal("invalid host")
@@ -4524,27 +4510,36 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 	}
 
 	// fetch the slab
-	var dbSlab2 dbSlab
-	key, _ := obj2.Slabs[0].Key.MarshalBinary()
-	if err := ss.gormDB.Where("id", slice2.SlabID).Take(&dbSlab2).Error; err != nil {
-		t.Fatal(err)
-	} else if dbSlab2.ID != uint(len(slices)+1) {
-		t.Fatal("unexpected id", dbSlab2.ID)
-	} else if dbSlab2.DBContractSetID != 1 {
-		t.Fatal("invalid contract set id", dbSlab2.DBContractSetID)
-	} else if !bytes.Equal(dbSlab2.Key, key) {
+	var slab2 slab
+	err = fetchSlabStmt.QueryRow(context.Background(), slice2.SlabID).
+		Scan(&slab2.ID, &slab2.ContractSetID, &slab2.Health, &slab2.HealthValidUntil, &slab2.MinShards, &slab2.TotalShards, (*sql.EncryptionKey)(&slab2.Key))
+	if err != nil {
+		t.Fatal(err)
+	} else if slab2.ID != int64(len(slices)+1) {
+		t.Fatal("unexpected id", slab2.ID)
+	} else if slab2.ContractSetID != 1 {
+		t.Fatal("invalid contract set id", slab2.ContractSetID)
+	} else if slab2.Health != 1 {
+		t.Fatal("invalid health", slab2.Health)
+	} else if slab2.HealthValidUntil != 0 {
+		t.Fatal("invalid health validity", slab2.HealthValidUntil)
+	} else if slab2.MinShards != uint8(minShards) {
+		t.Fatal("invalid minShards", slab2.MinShards)
+	} else if slab2.TotalShards != uint8(totalShards) {
+		t.Fatal("invalid totalShards", slab2.TotalShards)
+	} else if slab2.Key.String() != obj2.Slabs[0].Key.String() {
 		t.Fatal("wrong key")
 	}
 
 	// fetch the sectors
-	sectors2 := fetchSectorsBySlabID(int64(dbSlab2.ID))
+	sectors2 := fetchSectorsBySlabID(int64(slab2.ID))
 	if len(sectors2) != totalShards {
 		t.Fatal("invalid number of sectors", len(sectors2))
 	}
 	for j, sector := range sectors2 {
 		if sector.ID != int64((len(obj.Slabs))*totalShards+j+1) {
 			t.Fatal("invalid id", sector.ID)
-		} else if sector.SlabID != int64(dbSlab2.ID) {
+		} else if sector.SlabID != int64(slab2.ID) {
 			t.Fatal("invalid slab id", sector.SlabID)
 		} else if sector.LatestHost != hks[(len(obj.Slabs))*totalShards+j] {
 			t.Fatal("invalid host")
diff --git a/stores/types.go b/stores/types.go
index 93da48db0..6d6ad2c4a 100644
--- a/stores/types.go
+++ b/stores/types.go
@@ -22,7 +22,6 @@ type (
 	bCurrency      types.Currency
 	fileContractID types.FileContractID
 	publicKey      types.PublicKey
-	secretKey      []byte
 	setting        string
 
 	// NOTE: we have to wrap the proof here because Gorm can't scan bytes into
@@ -62,34 +61,6 @@ func (s setting) Value() (driver.Value, error) {
 	return string(s), nil
 }
 
-// GormDataType implements gorm.GormDataTypeInterface.
-func (secretKey) GormDataType() string {
-	return "bytes"
-}
-
-// String implements fmt.Stringer to prevent the key from getting leaked in
-// logs.
-func (k secretKey) String() string {
-	return "*****"
-}
-
-// Scan scans value into key, implements sql.Scanner interface.
-func (k *secretKey) Scan(value interface{}) error {
-	bytes, ok := value.([]byte)
-	if !ok {
-		return errors.New(fmt.Sprint("failed to unmarshal secretKey value:", value))
-	} else if len(bytes) != secretKeySize {
-		return fmt.Errorf("failed to unmarshal secretKey value due to invalid number of bytes %v != %v: %v", len(bytes), secretKeySize, value)
-	}
-	*k = append(secretKey{}, secretKey(bytes)...)
-	return nil
-}
-
-// Value returns an key value, implements driver.Valuer interface.
-func (k secretKey) Value() (driver.Value, error) {
-	return []byte(k), nil
-}
-
 // GormDataType implements gorm.GormDataTypeInterface.
 func (fileContractID) GormDataType() string {
 	return "bytes"
diff --git a/stores/types_test.go b/stores/types_test.go
index bc2257afd..972220ef7 100644
--- a/stores/types_test.go
+++ b/stores/types_test.go
@@ -9,17 +9,6 @@ import (
 	"go.sia.tech/core/types"
 )
 
-func TestTypeSetting(t *testing.T) {
-	s1 := setting("some setting")
-	s2 := setting("v4Keypairs")
-
-	if s1.String() != "some setting" {
-		t.Fatal("unexpected string")
-	} else if s2.String() != "*****" {
-		t.Fatal("unexpected string")
-	}
-}
-
 func TestTypeCurrency(t *testing.T) {
 	ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
 	defer ss.Close()

From 7c3f447516fb77fa68b599715b026f6368a556af Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 14:13:37 +0200
Subject: [PATCH 13/19] stores: remove types.go

---
 stores/hostdb_test.go   |   2 +-
 stores/metadata_test.go |  10 +-
 stores/metrics_test.go  |  13 +--
 stores/sql/types.go     |  25 +++++
 stores/types.go         | 211 ----------------------------------------
 stores/types_test.go    |  21 ++--
 6 files changed, 49 insertions(+), 233 deletions(-)
 delete mode 100644 stores/types.go

diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go
index 1f0eff101..b376286a4 100644
--- a/stores/hostdb_test.go
+++ b/stores/hostdb_test.go
@@ -393,7 +393,7 @@ func TestSearchHosts(t *testing.T) {
 	}
 
 	// assert cascade delete on host
-	err = ss.gormDB.Exec("DELETE FROM hosts WHERE public_key = ?", publicKey(types.PublicKey{1})).Error
+	err = ss.gormDB.Exec("DELETE FROM hosts WHERE public_key = ?", sql.PublicKey(types.PublicKey{1})).Error
 	if err != nil {
 		t.Fatal(err)
 	}
diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index 9b56647e0..08de24880 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -903,9 +903,9 @@ func TestArchiveContracts(t *testing.T) {
 	}
 
 	// assert the two others were archived
-	ffcids := make([]fileContractID, 2)
-	ffcids[0] = fileContractID(fcids[1])
-	ffcids[1] = fileContractID(fcids[2])
+	ffcids := make([]sql.FileContractID, 2)
+	ffcids[0] = sql.FileContractID(fcids[1])
+	ffcids[1] = sql.FileContractID(fcids[2])
 	rows, err := ss.DB().Query(context.Background(), "SELECT reason FROM archived_contracts WHERE fcid IN (?, ?)",
 		sql.FileContractID(ffcids[0]), sql.FileContractID(ffcids[1]))
 	if err != nil {
@@ -2236,7 +2236,7 @@ func TestUpdateSlab(t *testing.T) {
 	} else if types.FileContractID(cids[0]) != fcid1 {
 		t.Fatal("sector 1 was uploaded to unexpected contract", cids[0])
 	} else if updated.Shards[0].LatestHost != hks[0] {
-		t.Fatal("host key was invalid", updated.Shards[0].LatestHost, publicKey(hks[0]))
+		t.Fatal("host key was invalid", updated.Shards[0].LatestHost, sql.PublicKey(hks[0]))
 	} else if hks[0] != hk1 {
 		t.Fatal("sector 1 was uploaded to unexpected host", hks[0])
 	}
@@ -2247,7 +2247,7 @@ func TestUpdateSlab(t *testing.T) {
 	} else if types.FileContractID(cids[0]) != fcid2 || types.FileContractID(cids[1]) != fcid3 {
 		t.Fatal("sector 1 was uploaded to unexpected contracts", cids[0], cids[1])
 	} else if updated.Shards[0].LatestHost != hks[0] {
-		t.Fatal("host key was invalid", updated.Shards[0].LatestHost, publicKey(hks[0]))
+		t.Fatal("host key was invalid", updated.Shards[0].LatestHost, sql.PublicKey(hks[0]))
 	}
 
 	// assert there's still only one entry in the dbslab table
diff --git a/stores/metrics_test.go b/stores/metrics_test.go
index 5725f7a65..0e9092cb6 100644
--- a/stores/metrics_test.go
+++ b/stores/metrics_test.go
@@ -11,6 +11,7 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/api"
+	"go.sia.tech/renterd/stores/sql"
 	"lukechampine.com/frand"
 )
 
@@ -149,7 +150,7 @@ func TestContractMetrics(t *testing.T) {
 		}
 		for _, m := range metrics {
 			expectedMetric := fcid2Metric[m.ContractID]
-			expectedMetric.Timestamp = api.TimeRFC3339(normaliseTimestamp(start, interval, unixTimeMS(expectedMetric.Timestamp)))
+			expectedMetric.Timestamp = api.TimeRFC3339(normaliseTimestamp(start, interval, sql.UnixTimeMS(expectedMetric.Timestamp)))
 			if !cmp.Equal(m, expectedMetric, cmp.Comparer(api.CompareTimeRFC3339)) {
 				t.Fatal("unexpected metric", cmp.Diff(m, expectedMetric, cmp.Comparer(api.CompareTimeRFC3339)))
 			}
@@ -181,7 +182,7 @@ func TestContractMetrics(t *testing.T) {
 	}
 	for i, m := range metrics {
 		var expectedMetric api.ContractMetric
-		expectedMetric.Timestamp = api.TimeRFC3339(normaliseTimestamp(start, time.Millisecond, unixTimeMS(metricsTimeAsc[2*i].Timestamp)))
+		expectedMetric.Timestamp = api.TimeRFC3339(normaliseTimestamp(start, time.Millisecond, sql.UnixTimeMS(metricsTimeAsc[2*i].Timestamp)))
 		expectedMetric.ContractID = types.FileContractID{}
 		expectedMetric.HostKey = types.PublicKey{}
 		expectedMetric.RemainingCollateral, _ = metricsTimeAsc[2*i].RemainingCollateral.AddWithOverflow(metricsTimeAsc[2*i+1].RemainingCollateral)
@@ -424,7 +425,7 @@ func TestNormaliseTimestamp(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		if result := time.Time(normaliseTimestamp(test.start, test.interval, unixTimeMS(test.ti))); !result.Equal(test.result) {
+		if result := time.Time(normaliseTimestamp(test.start, test.interval, sql.UnixTimeMS(test.ti))); !result.Equal(test.result) {
 			t.Fatalf("expected %v, got %v", test.result, result)
 		}
 	}
@@ -555,13 +556,13 @@ func TestWalletMetrics(t *testing.T) {
 	}
 }
 
-func normaliseTimestamp(start time.Time, interval time.Duration, t unixTimeMS) unixTimeMS {
+func normaliseTimestamp(start time.Time, interval time.Duration, t sql.UnixTimeMS) sql.UnixTimeMS {
 	startMS := start.UnixMilli()
 	toNormaliseMS := time.Time(t).UnixMilli()
 	intervalMS := interval.Milliseconds()
 	if startMS > toNormaliseMS {
-		return unixTimeMS(start)
+		return sql.UnixTimeMS(start)
 	}
 	normalizedMS := (toNormaliseMS-startMS)/intervalMS*intervalMS + start.UnixMilli()
-	return unixTimeMS(time.UnixMilli(normalizedMS))
+	return sql.UnixTimeMS(time.UnixMilli(normalizedMS))
 }
diff --git a/stores/sql/types.go b/stores/sql/types.go
index bd58ad818..10cf76e42 100644
--- a/stores/sql/types.go
+++ b/stores/sql/types.go
@@ -3,6 +3,7 @@ package sql
 import (
 	"database/sql"
 	"database/sql/driver"
+	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -29,6 +30,7 @@ var (
 
 type (
 	AutopilotConfig api.AutopilotConfig
+	BCurrency       types.Currency
 	BigInt          big.Int
 	BusSetting      string
 	Currency        types.Currency
@@ -52,6 +54,7 @@ type scannerValuer interface {
 
 var (
 	_ scannerValuer = (*AutopilotConfig)(nil)
+	_ scannerValuer = (*BCurrency)(nil)
 	_ scannerValuer = (*BigInt)(nil)
 	_ scannerValuer = (*BusSetting)(nil)
 	_ scannerValuer = (*Currency)(nil)
@@ -86,6 +89,28 @@ func (cfg AutopilotConfig) Value() (driver.Value, error) {
 	return json.Marshal(cfg)
 }
 
+// Scan implements the sql.Scanner interface.
+func (sc *BCurrency) Scan(src any) error {
+	buf, ok := src.([]byte)
+	if !ok {
+		return fmt.Errorf("cannot scan %T to Currency", src)
+	} else if len(buf) != 16 {
+		return fmt.Errorf("cannot scan %d bytes to Currency", len(buf))
+	}
+
+	sc.Hi = binary.BigEndian.Uint64(buf[:8])
+	sc.Lo = binary.BigEndian.Uint64(buf[8:])
+	return nil
+}
+
+// Value implements the driver.Valuer interface.
+func (sc BCurrency) Value() (driver.Value, error) {
+	buf := make([]byte, 16)
+	binary.BigEndian.PutUint64(buf[:8], sc.Hi)
+	binary.BigEndian.PutUint64(buf[8:], sc.Lo)
+	return buf, nil
+}
+
 // Scan scan value into BigInt, implements sql.Scanner interface.
 func (b *BigInt) Scan(value interface{}) error {
 	var s string
diff --git a/stores/types.go b/stores/types.go
deleted file mode 100644
index 6d6ad2c4a..000000000
--- a/stores/types.go
+++ /dev/null
@@ -1,211 +0,0 @@
-package stores
-
-import (
-	"database/sql/driver"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"strconv"
-	"strings"
-	"time"
-
-	"go.sia.tech/core/types"
-)
-
-const (
-	proofHashSize = 32
-	secretKeySize = 32
-)
-
-type (
-	unixTimeMS     time.Time
-	bCurrency      types.Currency
-	fileContractID types.FileContractID
-	publicKey      types.PublicKey
-	setting        string
-
-	// NOTE: we have to wrap the proof here because Gorm can't scan bytes into
-	// multiple slices, all bytes are scanned into the first row
-	merkleProof struct{ proof []types.Hash256 }
-)
-
-// GormDataType implements gorm.GormDataTypeInterface.
-func (setting) GormDataType() string {
-	return "string"
-}
-
-// String implements fmt.Stringer to prevent "s3authentication" settings from
-// getting leaked.
-func (s setting) String() string {
-	if strings.Contains(string(s), "v4Keypairs") {
-		return "*****"
-	}
-	return string(s)
-}
-
-// Scan scans value into the setting
-func (s *setting) Scan(value interface{}) error {
-	switch value := value.(type) {
-	case string:
-		*s = setting(value)
-	case []byte:
-		*s = setting(value)
-	default:
-		return fmt.Errorf("failed to unmarshal setting value from type %t", value)
-	}
-	return nil
-}
-
-// Value returns a setting value, implements driver.Valuer interface.
-func (s setting) Value() (driver.Value, error) {
-	return string(s), nil
-}
-
-// GormDataType implements gorm.GormDataTypeInterface.
-func (fileContractID) GormDataType() string {
-	return "bytes"
-}
-
-// Scan scan value into fileContractID, implements sql.Scanner interface.
-func (fcid *fileContractID) Scan(value interface{}) error {
-	bytes, ok := value.([]byte)
-	if !ok {
-		return errors.New(fmt.Sprint("failed to unmarshal fcid value:", value))
-	}
-	if len(bytes) != len(fileContractID{}) {
-		return fmt.Errorf("failed to unmarshal fcid value due to invalid number of bytes %v != %v: %v", len(bytes), len(fileContractID{}), value)
-	}
-	*fcid = *(*fileContractID)(bytes)
-	return nil
-}
-
-// Value returns a fileContractID value, implements driver.Valuer interface.
-func (fcid fileContractID) Value() (driver.Value, error) {
-	return fcid[:], nil
-}
-
-func (publicKey) GormDataType() string {
-	return "bytes"
-}
-
-// Scan scan value into publicKey, implements sql.Scanner interface.
-func (pk *publicKey) Scan(value interface{}) error {
-	bytes, ok := value.([]byte)
-	if !ok {
-		return errors.New(fmt.Sprint("failed to unmarshal publicKey value:", value))
-	}
-	if len(bytes) != len(types.PublicKey{}) {
-		return fmt.Errorf("failed to unmarshal publicKey value due invalid number of bytes %v != %v: %v", len(bytes), len(publicKey{}), value)
-	}
-	*pk = *(*publicKey)(bytes)
-	return nil
-}
-
-// Value returns a publicKey value, implements driver.Valuer interface.
-func (pk publicKey) Value() (driver.Value, error) {
-	return pk[:], nil
-}
-
-// SQLiteTimestampFormats were taken from github.com/mattn/go-sqlite3 and are
-// used when parsing a string to a date
-var SQLiteTimestampFormats = []string{
-	"2006-01-02 15:04:05.999999999-07:00",
-	"2006-01-02T15:04:05.999999999-07:00",
-	"2006-01-02 15:04:05.999999999",
-	"2006-01-02T15:04:05.999999999",
-	"2006-01-02 15:04:05",
-	"2006-01-02T15:04:05",
-	"2006-01-02 15:04",
-	"2006-01-02T15:04",
-	"2006-01-02",
-}
-
-// GormDataType implements gorm.GormDataTypeInterface.
-func (unixTimeMS) GormDataType() string {
-	return "BIGINT"
-}
-
-// Scan scan value into unixTimeMS, implements sql.Scanner interface.
-func (u *unixTimeMS) Scan(value interface{}) error {
-	var msec int64
-	var err error
-	switch value := value.(type) {
-	case int64:
-		msec = value
-	case []uint8:
-		msec, err = strconv.ParseInt(string(value), 10, 64)
-		if err != nil {
-			return fmt.Errorf("failed to unmarshal unixTimeMS value: %v %T", value, value)
-		}
-	default:
-		return fmt.Errorf("failed to unmarshal unixTimeMS value: %v %T", value, value)
-	}
-
-	*u = unixTimeMS(time.UnixMilli(msec))
-	return nil
-}
-
-// Value returns a int64 value representing a unix timestamp in milliseconds,
-// implements driver.Valuer interface.
-func (u unixTimeMS) Value() (driver.Value, error) {
-	return time.Time(u).UnixMilli(), nil
-}
-
-func (bCurrency) GormDataType() string {
-	return "bytes"
-}
-
-// Scan implements the sql.Scanner interface.
-func (sc *bCurrency) Scan(src any) error {
-	buf, ok := src.([]byte)
-	if !ok {
-		return fmt.Errorf("cannot scan %T to Currency", src)
-	} else if len(buf) != 16 {
-		return fmt.Errorf("cannot scan %d bytes to Currency", len(buf))
-	}
-
-	sc.Hi = binary.BigEndian.Uint64(buf[:8])
-	sc.Lo = binary.BigEndian.Uint64(buf[8:])
-	return nil
-}
-
-// Value implements the driver.Valuer interface.
-func (sc bCurrency) Value() (driver.Value, error) {
-	buf := make([]byte, 16)
-	binary.BigEndian.PutUint64(buf[:8], sc.Hi)
-	binary.BigEndian.PutUint64(buf[8:], sc.Lo)
-	return buf, nil
-}
-
-// GormDataType implements gorm.GormDataTypeInterface.
-func (mp *merkleProof) GormDataType() string {
-	return "bytes"
-}
-
-// Scan scans value into mp, implements sql.Scanner interface.
-func (mp *merkleProof) Scan(value interface{}) error {
-	bytes, ok := value.([]byte)
-	if !ok {
-		return errors.New(fmt.Sprint("failed to unmarshal merkleProof value:", value))
-	} else if len(bytes) == 0 || len(bytes)%proofHashSize != 0 {
-		return fmt.Errorf("failed to unmarshal merkleProof value due to invalid number of bytes %v", len(bytes))
-	}
-
-	n := len(bytes) / proofHashSize
-	mp.proof = make([]types.Hash256, n)
-	for i := 0; i < n; i++ {
-		copy(mp.proof[i][:], bytes[:proofHashSize])
-		bytes = bytes[proofHashSize:]
-	}
-	return nil
-}
-
-// Value returns a merkle proof value, implements driver.Valuer interface.
-func (mp merkleProof) Value() (driver.Value, error) {
-	var i int
-	out := make([]byte, len(mp.proof)*proofHashSize)
-	for _, ph := range mp.proof {
-		i += copy(out[i:], ph[:])
-	}
-	return out, nil
-}
diff --git a/stores/types_test.go b/stores/types_test.go
index 972220ef7..4d9453d18 100644
--- a/stores/types_test.go
+++ b/stores/types_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"go.sia.tech/core/types"
+	"go.sia.tech/renterd/stores/sql"
 )
 
 func TestTypeCurrency(t *testing.T) {
@@ -22,12 +23,12 @@ func TestTypeCurrency(t *testing.T) {
 	}
 
 	// insert currencies in random order
-	if err := ss.gormDB.Exec("INSERT INTO currencies (c) VALUES (?),(?),(?);", bCurrency(types.MaxCurrency), bCurrency(types.NewCurrency64(1)), bCurrency(types.ZeroCurrency)).Error; err != nil {
+	if err := ss.gormDB.Exec("INSERT INTO currencies (c) VALUES (?),(?),(?);", sql.BCurrency(types.MaxCurrency), sql.BCurrency(types.NewCurrency64(1)), sql.BCurrency(types.ZeroCurrency)).Error; err != nil {
 		t.Fatal(err)
 	}
 
 	// fetch currencies and assert they're sorted
-	var currencies []bCurrency
+	var currencies []sql.BCurrency
 	if err := ss.gormDB.Raw(`SELECT c FROM currencies ORDER BY c ASC`).Scan(&currencies).Error; err != nil {
 		t.Fatal(err)
 	} else if !sort.SliceIsSorted(currencies, func(i, j int) bool {
@@ -42,8 +43,8 @@ func TestTypeCurrency(t *testing.T) {
 	cM := currencies[2]
 
 	tests := []struct {
-		a   bCurrency
-		b   bCurrency
+		a   sql.BCurrency
+		b   sql.BCurrency
 		cmp string
 	}{
 		{
@@ -119,25 +120,25 @@ func TestTypeMerkleProof(t *testing.T) {
 	}
 
 	// insert merkle proof
-	mp1 := merkleProof{proof: []types.Hash256{{3}, {1}, {2}}}
-	mp2 := merkleProof{proof: []types.Hash256{{4}}}
+	mp1 := sql.MerkleProof{Hashes: []types.Hash256{{3}, {1}, {2}}}
+	mp2 := sql.MerkleProof{Hashes: []types.Hash256{{4}}}
 	if err := ss.gormDB.Exec("INSERT INTO merkle_proofs (merkle_proof) VALUES (?), (?);", mp1, mp2).Error; err != nil {
 		t.Fatal(err)
 	}
 
 	// fetch first proof
-	var first merkleProof
+	var first sql.MerkleProof
 	if err := ss.gormDB.
 		Raw(`SELECT merkle_proof FROM merkle_proofs`).
 		Take(&first).
 		Error; err != nil {
 		t.Fatal(err)
-	} else if first.proof[0] != (types.Hash256{3}) || first.proof[1] != (types.Hash256{1}) || first.proof[2] != (types.Hash256{2}) {
+	} else if first.Hashes[0] != (types.Hash256{3}) || first.Hashes[1] != (types.Hash256{1}) || first.Hashes[2] != (types.Hash256{2}) {
 		t.Fatalf("unexpected proof %+v", first)
 	}
 
 	// fetch both proofs
-	var both []merkleProof
+	var both []sql.MerkleProof
 	if err := ss.gormDB.
 		Raw(`SELECT merkle_proof FROM merkle_proofs`).
 		Scan(&both).
@@ -145,7 +146,7 @@ func TestTypeMerkleProof(t *testing.T) {
 		t.Fatal(err)
 	} else if len(both) != 2 {
 		t.Fatalf("unexpected number of proofs: %d", len(both))
-	} else if both[1].proof[0] != (types.Hash256{4}) {
+	} else if both[1].Hashes[0] != (types.Hash256{4}) {
 		t.Fatalf("unexpected proof %+v", both)
 	}
 }

From 873baff75c70488118e41e3b79a4f557ddee09f7 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 15:03:26 +0200
Subject: [PATCH 14/19] go.mod: remove gorm

---
 go.mod                       |   7 --
 go.sum                       |  53 ----------
 internal/node/node.go        |  60 +++--------
 internal/test/e2e/cluster.go |  15 ++-
 stores/hostdb_test.go        |  22 +----
 stores/metadata_test.go      |  22 ++---
 stores/settingsdb_test.go    |   2 +-
 stores/sql.go                |  94 +-----------------
 stores/sql_test.go           | 186 +++++++++++++----------------------
 stores/types_test.go         |  67 +++++++------
 webhooks/webhooks.go         |   5 +-
 11 files changed, 143 insertions(+), 390 deletions(-)

diff --git a/go.mod b/go.mod
index 371d83e10..4d6ba2d57 100644
--- a/go.mod
+++ b/go.mod
@@ -22,11 +22,7 @@ require (
 	golang.org/x/sys v0.24.0
 	golang.org/x/term v0.23.0
 	gopkg.in/yaml.v3 v3.0.1
-	gorm.io/driver/mysql v1.5.7
-	gorm.io/driver/sqlite v1.5.6
-	gorm.io/gorm v1.25.11
 	lukechampine.com/frand v1.4.2
-	moul.io/zapgorm2 v1.3.0
 )
 
 require (
@@ -35,13 +31,10 @@ require (
 	github.com/cloudflare/cloudflare-go v0.101.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/go-sql-driver/mysql v1.7.1 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
-	github.com/jinzhu/inflection v1.0.0 // indirect
-	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/julienschmidt/httprouter v1.3.0 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
diff --git a/go.sum b/go.sum
index 334709158..6acedb237 100644
--- a/go.sum
+++ b/go.sum
@@ -2,7 +2,6 @@ github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmH
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
 github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
 github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
-github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/cloudflare/cloudflare-go v0.101.0 h1:SXWNSEDkbdY84iFIZGyTdWQwDfd98ljv0/4UubpleBQ=
 github.com/cloudflare/cloudflare-go v0.101.0/go.mod h1:xXQHnoXKR48JlWbFS42i2al3nVqimVhcYvKnIdXLw9g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -14,9 +13,6 @@ github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8
 github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4=
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
-github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
-github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -30,11 +26,6 @@ github.com/gotd/contrib v0.20.0 h1:1Wc4+HMQiIKYQuGHVwVksIx152HFTP6B5n88dDe0ZYw=
 github.com/gotd/contrib v0.20.0/go.mod h1:P6o8W4niqhDPHLA0U+SA/L7l3BQHYLULpeHfRSePn9o=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
-github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
-github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
-github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
-github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -48,11 +39,8 @@ github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS
 github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/klauspost/reedsolomon v1.12.3 h1:tzUznbfc3OFwJaTebv/QdhnFf2Xvb7gZ24XaHLBPmdc=
 github.com/klauspost/reedsolomon v1.12.3/go.mod h1:3K5rXwABAvzGeR01r6pWZieUALXO/Tq7bFKGIb4m4WI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
@@ -63,7 +51,6 @@ github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYC
 github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
 github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
@@ -77,11 +64,8 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dc
 github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
 github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
 go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
 go.sia.tech/core v0.4.3 h1:XEX7v6X8eJh4zyOkSHYi6FsyD+N/OEKw/NIigaaWPAU=
@@ -100,86 +84,49 @@ go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka
 go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk=
 go.sia.tech/web/renterd v0.57.0 h1:ur1Rslbjv+s5285v5JNQj+RHYpdXomrH8KoRQnZDOdQ=
 go.sia.tech/web/renterd v0.57.0/go.mod h1:SWwKoAJvLxiHjTXsNPKX3RLiQzJb/vxwcpku3F78MO8=
-go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
-go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
 golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
 golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
 golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
 golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
 golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
 golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
 golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
 golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
-gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
-gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE=
-gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4=
-gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
-gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
-gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg=
-gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
 lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw=
 lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s=
-moul.io/zapgorm2 v1.3.0 h1:+CzUTMIcnafd0d/BvBce8T4uPn6DQnpIrz64cyixlkk=
-moul.io/zapgorm2 v1.3.0/go.mod h1:nPVy6U9goFKHR4s+zfSo1xVFaoU7Qgd5DoCdOfzoCqs=
 nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0=
 nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
diff --git a/internal/node/node.go b/internal/node/node.go
index d7fae5b2d..7acd6548f 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -4,11 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"log"
 	"net/http"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 
 	"go.sia.tech/core/consensus"
@@ -30,9 +28,6 @@ import (
 	"go.sia.tech/renterd/worker/s3"
 	"go.uber.org/zap"
 	"golang.org/x/crypto/blake2b"
-	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
-	"moul.io/zapgorm2"
 )
 
 // TODOs:
@@ -72,16 +67,10 @@ var NoopFn = func(context.Context) error { return nil }
 
 func NewBus(cfg BusConfig, dir string, seed types.PrivateKey, logger *zap.Logger) (http.Handler, BusSetupFn, ShutdownFn, *chain.Manager, *chain.ChainSubscriber, error) {
 	// create database connections
-	var dbConn gorm.Dialector
+	var dbMain sql.Database
 	var dbMetrics sql.MetricsDatabase
 	if cfg.Database.MySQL.URI != "" {
 		// create MySQL connections
-		dbConn = stores.NewMySQLConnection(
-			cfg.Database.MySQL.User,
-			cfg.Database.MySQL.Password,
-			cfg.Database.MySQL.URI,
-			cfg.Database.MySQL.Database,
-		)
 		dbm, err := mysql.Open(
 			cfg.Database.MySQL.User,
 			cfg.Database.MySQL.Password,
@@ -91,6 +80,10 @@ func NewBus(cfg BusConfig, dir string, seed types.PrivateKey, logger *zap.Logger
 		if err != nil {
 			return nil, nil, nil, nil, nil, fmt.Errorf("failed to open MySQL metrics database: %w", err)
 		}
+		dbMain, err = mysql.NewMainDatabase(dbm, logger.Named("main").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold)
+		if err != nil {
+			return nil, nil, nil, nil, nil, fmt.Errorf("failed to create MySQL main database: %w", err)
+		}
 		dbMetrics, err = mysql.NewMetricsDatabase(dbm, logger.Named("metrics").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold)
 		if err != nil {
 			return nil, nil, nil, nil, nil, fmt.Errorf("failed to create MySQL metrics database: %w", err)
@@ -103,7 +96,14 @@ func NewBus(cfg BusConfig, dir string, seed types.PrivateKey, logger *zap.Logger
 		}
 
 		// create SQLite connections
-		dbConn = stores.NewSQLiteConnection(filepath.Join(dbDir, "db.sqlite"))
+		db, err := sqlite.Open(filepath.Join(dbDir, "db.sqlite"))
+		if err != nil {
+			return nil, nil, nil, nil, nil, fmt.Errorf("failed to open SQLite main database: %w", err)
+		}
+		dbMain, err = sqlite.NewMainDatabase(db, logger.Named("main").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold)
+		if err != nil {
+			return nil, nil, nil, nil, nil, fmt.Errorf("failed to create SQLite main database: %w", err)
+		}
 
 		dbm, err := sqlite.Open(filepath.Join(dbDir, "metrics.sqlite"))
 		if err != nil {
@@ -115,27 +115,16 @@ func NewBus(cfg BusConfig, dir string, seed types.PrivateKey, logger *zap.Logger
 		}
 	}
 
-	// create database logger
-	dbLogger := zapgorm2.Logger{
-		ZapLogger:                 cfg.Logger.Named("SQL"),
-		LogLevel:                  gormLogLevel(cfg.DatabaseLog),
-		SlowThreshold:             cfg.DatabaseLog.SlowThreshold,
-		SkipCallerLookup:          false,
-		IgnoreRecordNotFoundError: cfg.DatabaseLog.IgnoreRecordNotFoundError,
-		Context:                   nil,
-	}
-
 	alertsMgr := alerts.NewManager()
 	sqlStoreDir := filepath.Join(dir, "partial_slabs")
 	sqlStore, err := stores.NewSQLStore(stores.Config{
-		Conn:                          dbConn,
 		Alerts:                        alerts.WithOrigin(alertsMgr, "bus"),
+		DB:                            dbMain,
 		DBMetrics:                     dbMetrics,
 		PartialSlabDir:                sqlStoreDir,
 		Migrate:                       true,
 		SlabBufferCompletionThreshold: cfg.SlabBufferCompletionThreshold,
 		Logger:                        logger.Sugar(),
-		GormLogger:                    dbLogger,
 		RetryTransactionIntervals:     cfg.RetryTxIntervals,
 		WalletAddress:                 types.StandardUnlockHash(seed.PublicKey()),
 		LongQueryDuration:             cfg.DatabaseLog.SlowThreshold,
@@ -265,24 +254,3 @@ func NewAutopilot(cfg AutopilotConfig, b autopilot.Bus, workers []autopilot.Work
 	}
 	return ap.Handler(), ap.Run, ap.Shutdown, nil
 }
-
-func gormLogLevel(cfg config.DatabaseLog) logger.LogLevel {
-	level := logger.Silent
-	if cfg.Enabled {
-		switch strings.ToLower(cfg.Level) {
-		case "":
-			level = logger.Warn // default to 'warn' if not set
-		case "error":
-			level = logger.Error
-		case "warn":
-			level = logger.Warn
-		case "info":
-			level = logger.Info
-		case "debug":
-			level = logger.Info
-		default:
-			log.Fatalf("invalid log level %q, options are: silent, error, warn, info", cfg.Level)
-		}
-	}
-	return level
-}
diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go
index 7db3657a5..e4d38fc7d 100644
--- a/internal/test/e2e/cluster.go
+++ b/internal/test/e2e/cluster.go
@@ -27,12 +27,11 @@ import (
 	"go.sia.tech/renterd/internal/test"
 	"go.sia.tech/renterd/internal/utils"
 	iworker "go.sia.tech/renterd/internal/worker"
-	"go.sia.tech/renterd/stores"
+	"go.sia.tech/renterd/stores/sql/mysql"
 	"go.sia.tech/renterd/worker/s3"
 	"go.sia.tech/web/renterd"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
-	"gorm.io/gorm"
 	"lukechampine.com/frand"
 
 	"go.sia.tech/renterd/worker"
@@ -249,7 +248,7 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster {
 
 	// Check if we are testing against an external database. If so, we create a
 	// database with a random name first.
-	if mysql := config.MySQLConfigFromEnv(); mysql.URI != "" {
+	if mysqlCfg := config.MySQLConfigFromEnv(); mysqlCfg.URI != "" {
 		// generate a random database name if none are set
 		if busCfg.Database.MySQL.Database == "" {
 			busCfg.Database.MySQL.Database = "db" + hex.EncodeToString(frand.Bytes(16))
@@ -258,13 +257,11 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster {
 			busCfg.Database.MySQL.MetricsDatabase = "db" + hex.EncodeToString(frand.Bytes(16))
 		}
 
-		tmpDB, err := gorm.Open(stores.NewMySQLConnection(mysql.User, mysql.Password, mysql.URI, ""))
+		tmpDB, err := mysql.Open(mysqlCfg.User, mysqlCfg.Password, mysqlCfg.URI, "")
 		tt.OK(err)
-		tt.OK(tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s;", busCfg.Database.MySQL.Database)).Error)
-		tt.OK(tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s;", busCfg.Database.MySQL.MetricsDatabase)).Error)
-		tmpDBB, err := tmpDB.DB()
-		tt.OK(err)
-		tt.OK(tmpDBB.Close())
+		tt.OKAll(tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s;", busCfg.Database.MySQL.Database)))
+		tt.OKAll(tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s;", busCfg.Database.MySQL.MetricsDatabase)))
+		tt.OK(tmpDB.Close())
 	}
 
 	// Prepare individual dirs.
diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go
index b376286a4..061f5d6f4 100644
--- a/stores/hostdb_test.go
+++ b/stores/hostdb_test.go
@@ -393,7 +393,7 @@ func TestSearchHosts(t *testing.T) {
 	}
 
 	// assert cascade delete on host
-	err = ss.gormDB.Exec("DELETE FROM hosts WHERE public_key = ?", sql.PublicKey(types.PublicKey{1})).Error
+	_, err = ss.DB().Exec(context.Background(), "DELETE FROM hosts WHERE public_key = ?", sql.PublicKey(types.PublicKey{1}))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -402,7 +402,7 @@ func TestSearchHosts(t *testing.T) {
 	}
 
 	// assert cascade delete on autopilot
-	err = ss.gormDB.Exec("DELETE FROM autopilots WHERE identifier IN (?,?)", ap1, ap2).Error
+	_, err = ss.DB().Exec(context.Background(), "DELETE FROM autopilots WHERE identifier IN (?,?)", ap1, ap2)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -694,11 +694,7 @@ func TestSQLHostAllowlist(t *testing.T) {
 
 	numRelations := func() (cnt int64) {
 		t.Helper()
-		err := ss.gormDB.Table("host_allowlist_entry_hosts").Count(&cnt).Error
-		if err != nil {
-			t.Fatal(err)
-		}
-		return
+		return ss.Count("host_allowlist_entry_hosts")
 	}
 
 	isAllowed := func(hk types.PublicKey) bool {
@@ -866,20 +862,12 @@ func TestSQLHostBlocklist(t *testing.T) {
 
 	numAllowlistRelations := func() (cnt int64) {
 		t.Helper()
-		err := ss.gormDB.Table("host_allowlist_entry_hosts").Count(&cnt).Error
-		if err != nil {
-			t.Fatal(err)
-		}
-		return
+		return ss.Count("host_allowlist_entry_hosts")
 	}
 
 	numBlocklistRelations := func() (cnt int64) {
 		t.Helper()
-		err := ss.gormDB.Table("host_blocklist_entry_hosts").Count(&cnt).Error
-		if err != nil {
-			t.Fatal(err)
-		}
-		return
+		return ss.Count("host_blocklist_entry_hosts")
 	}
 
 	isBlocked := func(hk types.PublicKey) bool {
diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index 08de24880..d98992458 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -23,7 +23,6 @@ import (
 	"go.sia.tech/renterd/internal/test"
 	"go.sia.tech/renterd/object"
 	sql "go.sia.tech/renterd/stores/sql"
-	"gorm.io/gorm"
 	"lukechampine.com/frand"
 )
 
@@ -115,15 +114,16 @@ func randomMultisigUC() types.UnlockConditions {
 	return uc
 }
 
-func updateAllObjectsHealth(tx *gorm.DB) error {
-	return tx.Exec(`
+func updateAllObjectsHealth(db *isql.DB) error {
+	_, err := db.Exec(context.Background(), `
 UPDATE objects
 SET health = (
 	SELECT COALESCE(MIN(slabs.health), 1)
 	FROM slabs
 	INNER JOIN slices sli ON sli.db_slab_id = slabs.id
 	WHERE sli.db_object_id = objects.id)
-`).Error
+`)
+	return err
 }
 
 // TestObjectBasic tests the hydration of raw objects works when we fetch
@@ -471,11 +471,7 @@ func TestSQLContractStore(t *testing.T) {
 	}
 
 	// Check join table count as well.
-	var count int64
-	if err := ss.gormDB.Table("contract_sectors").Count(&count).Error; err != nil {
-		t.Fatal(err)
-	}
-	if count != 0 {
+	if count := ss.Count("contract_sectors"); count != 0 {
 		t.Fatalf("expected %v objects in contract_sectors but got %v", 0, count)
 	}
 }
@@ -1501,7 +1497,7 @@ func TestObjectEntries(t *testing.T) {
 	}
 
 	// update health of objects to match the overridden health of the slabs
-	if err := updateAllObjectsHealth(ss.gormDB); err != nil {
+	if err := updateAllObjectsHealth(ss.DB()); err != nil {
 		t.Fatal()
 	}
 
@@ -3554,7 +3550,7 @@ func TestListObjects(t *testing.T) {
 	}
 
 	// update health of objects to match the overridden health of the slabs
-	if err := updateAllObjectsHealth(ss.gormDB); err != nil {
+	if err := updateAllObjectsHealth(ss.DB()); err != nil {
 		t.Fatal()
 	}
 
@@ -3983,7 +3979,7 @@ func TestSlabHealthInvalidation(t *testing.T) {
 	// assert the health validity is always updated to a random time in the future that matches the boundaries
 	for i := 0; i < 1e3; i++ {
 		// reset health validity
-		if tx := ss.gormDB.Exec("UPDATE slabs SET health_valid_until = 0;"); tx.Error != nil {
+		if _, err := ss.DB().Exec(context.Background(), "UPDATE slabs SET health_valid_until = 0;"); err != nil {
 			t.Fatal(err)
 		}
 
@@ -4142,7 +4138,7 @@ func TestSlabCleanup(t *testing.T) {
 
 	// create buffered slab
 	bsID := uint(1)
-	if err := ss.gormDB.Exec("INSERT INTO buffered_slabs (filename) VALUES ('foo');").Error; err != nil {
+	if _, err := ss.DB().Exec(context.Background(), "INSERT INTO buffered_slabs (filename) VALUES ('foo');"); err != nil {
 		t.Fatal(err)
 	}
 
diff --git a/stores/settingsdb_test.go b/stores/settingsdb_test.go
index b6708d6cb..cf2582579 100644
--- a/stores/settingsdb_test.go
+++ b/stores/settingsdb_test.go
@@ -55,7 +55,7 @@ func TestSQLSettingStore(t *testing.T) {
 	if err := ss.DeleteSetting(ctx, "foo"); err != nil {
 		t.Fatal(err)
 	} else if _, err := ss.Setting(ctx, "foo"); !errors.Is(err, api.ErrSettingNotFound) {
-		t.Fatal("should fail with gorm.ErrRecordNotFound", err)
+		t.Fatal("should fail with api.ErrSettingNotFound", err)
 	} else if keys, err := ss.Settings(ctx); err != nil {
 		t.Fatal(err)
 	} else if len(keys) != 0 {
diff --git a/stores/sql.go b/stores/sql.go
index c640e800a..e6658f341 100644
--- a/stores/sql.go
+++ b/stores/sql.go
@@ -11,26 +11,13 @@ import (
 	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/alerts"
 	"go.sia.tech/renterd/stores/sql"
-	"go.sia.tech/renterd/stores/sql/mysql"
-	"go.sia.tech/renterd/stores/sql/sqlite"
 	"go.uber.org/zap"
-	gmysql "gorm.io/driver/mysql"
-	gsqlite "gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-	glogger "gorm.io/gorm/logger"
 )
 
 type (
-	// Model defines the common fields of every table. Same as Model
-	// but excludes soft deletion since it breaks cascading deletes.
-	Model struct {
-		ID        uint `gorm:"primarykey"`
-		CreatedAt time.Time
-	}
-
 	// Config contains all params for creating a SQLStore
 	Config struct {
-		Conn                          gorm.Dialector
+		DB                            sql.Database
 		DBMetrics                     sql.MetricsDatabase
 		Alerts                        alerts.Alerter
 		PartialSlabDir                string
@@ -39,7 +26,6 @@ type (
 		WalletAddress                 types.Address
 		SlabBufferCompletionThreshold int64
 		Logger                        *zap.SugaredLogger
-		GormLogger                    glogger.Interface
 		RetryTransactionIntervals     []time.Duration
 		LongQueryDuration             time.Duration
 		LongTxDuration                time.Duration
@@ -72,48 +58,9 @@ type (
 		mu           sync.Mutex
 		lastPrunedAt time.Time
 		closed       bool
-
-		gormDB *gorm.DB // deprecated: don't use
 	}
 )
 
-// NewEphemeralSQLiteConnection creates a connection to an in-memory SQLite DB.
-// NOTE: Use simple names such as a random hex identifier or the filepath.Base
-// of a test's name. Certain symbols will break the cfg string and cause a file
-// to be created on disk.
-//
-//	mode: set to memory for in-memory database
-//	cache: set to shared which is required for in-memory databases
-//	_foreign_keys: enforce foreign_key relations
-func NewEphemeralSQLiteConnection(name string) gorm.Dialector {
-	return gsqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared&_foreign_keys=1", name))
-}
-
-// NewSQLiteConnection opens a sqlite db at the given path.
-//
-//	_busy_timeout: set to prevent concurrent transactions from failing and
-//	  instead have them block
-//	_foreign_keys: enforce foreign_key relations
-//	_journal_mode: set to WAL instead of delete since it's usually the fastest.
-//	  Only downside is that the db won't work on network drives. In that case this
-//	  should be made configurable and set to TRUNCATE or any of the other options.
-//	  For reference see https://github.com/mattn/go-sqlite3#connection-string.
-func NewSQLiteConnection(path string) gorm.Dialector {
-	return gsqlite.Open(fmt.Sprintf("file:%s?_busy_timeout=30000&_foreign_keys=1&_journal_mode=WAL&_secure_delete=false&_cache_size=65536", path))
-}
-
-// NewMetricsSQLiteConnection opens a sqlite db at the given path similarly to
-// NewSQLiteConnection but with weaker consistency guarantees since it's
-// optimised for recording metrics.
-func NewMetricsSQLiteConnection(path string) gorm.Dialector {
-	return gsqlite.Open(fmt.Sprintf("file:%s?_busy_timeout=30000&_foreign_keys=1&_journal_mode=WAL&_synchronous=NORMAL", path))
-}
-
-// NewMySQLConnection creates a connection to a MySQL database.
-func NewMySQLConnection(user, password, addr, dbName string) gorm.Dialector {
-	return gmysql.Open(fmt.Sprintf("%s:%s@tcp(%s)/%s?charset=utf8mb4&parseTime=True&loc=Local&multiStatements=true", user, password, addr, dbName))
-}
-
 // NewSQLStore uses a given Dialector to connect to a SQL database.  NOTE: Only
 // pass migrate=true for the first instance of SQLHostDB if you connect via the
 // same Dialector multiple times.
@@ -121,34 +68,11 @@ func NewSQLStore(cfg Config) (*SQLStore, error) {
 	if err := os.MkdirAll(cfg.PartialSlabDir, 0700); err != nil {
 		return nil, fmt.Errorf("failed to create partial slab dir '%s': %v", cfg.PartialSlabDir, err)
 	}
-	db, err := gorm.Open(cfg.Conn, &gorm.Config{
-		Logger:                   cfg.GormLogger, // custom logger
-		SkipDefaultTransaction:   true,
-		DisableNestedTransaction: true,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to open SQL db")
-	}
 	l := cfg.Logger.Named("sql")
-
-	sqlDB, err := db.DB()
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch db: %v", err)
-	}
-
-	// Print DB version
-	var dbMain sql.Database
+	dbMain := cfg.DB
 	dbMetrics := cfg.DBMetrics
-	var mainErr error
-	if cfg.Conn.Name() == "sqlite" {
-		dbMain, mainErr = sqlite.NewMainDatabase(sqlDB, l, cfg.LongQueryDuration, cfg.LongTxDuration)
-	} else {
-		dbMain, mainErr = mysql.NewMainDatabase(sqlDB, l, cfg.LongQueryDuration, cfg.LongTxDuration)
-	}
-	if mainErr != nil {
-		return nil, fmt.Errorf("failed to create main database: %v", mainErr)
-	}
 
+	// Print DB version
 	dbName, dbVersion, err := dbMain.Version(context.Background())
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch db version: %v", err)
@@ -167,7 +91,6 @@ func NewSQLStore(cfg Config) (*SQLStore, error) {
 	shutdownCtx, shutdownCtxCancel := context.WithCancel(context.Background())
 	ss := &SQLStore{
 		alerts:    cfg.Alerts,
-		gormDB:    db,
 		db:        dbMain,
 		dbMetrics: dbMetrics,
 		logger:    l,
@@ -193,17 +116,6 @@ func NewSQLStore(cfg Config) (*SQLStore, error) {
 	return ss, nil
 }
 
-func isSQLite(db *gorm.DB) bool {
-	switch db.Dialector.(type) {
-	case *gsqlite.Dialector:
-		return true
-	case *gmysql.Dialector:
-		return false
-	default:
-		panic(fmt.Sprintf("unknown dialector: %t", db.Dialector))
-	}
-}
-
 func (s *SQLStore) initSlabPruning() error {
 	// start pruning loop
 	s.wg.Add(1)
diff --git a/stores/sql_test.go b/stores/sql_test.go
index 0bd89710f..ba20bfc53 100644
--- a/stores/sql_test.go
+++ b/stores/sql_test.go
@@ -6,9 +6,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 	"time"
 
@@ -22,11 +20,7 @@ import (
 	"go.sia.tech/renterd/stores/sql/mysql"
 	"go.sia.tech/renterd/stores/sql/sqlite"
 	"go.uber.org/zap"
-	"go.uber.org/zap/zapcore"
-	"gorm.io/gorm"
-	"gorm.io/gorm/logger"
 	"lukechampine.com/frand"
-	"moul.io/zapgorm2"
 )
 
 const (
@@ -63,11 +57,9 @@ func randomDBName() string {
 	return "db" + hex.EncodeToString(frand.Bytes(16))
 }
 
-func (cfg *testSQLStoreConfig) dbConnections() (gorm.Dialector, sql.MetricsDatabase, error) {
-	var connMain gorm.Dialector
-	var dbm *dsql.DB
+func (cfg *testSQLStoreConfig) dbConnections() (sql.Database, sql.MetricsDatabase, error) {
+	var dbMain sql.Database
 	var dbMetrics sql.MetricsDatabase
-	var err error
 	if mysqlCfg := config.MySQLConfigFromEnv(); mysqlCfg.URI != "" {
 		// create MySQL connections if URI is set
 
@@ -84,42 +76,72 @@ func (cfg *testSQLStoreConfig) dbConnections() (gorm.Dialector, sql.MetricsDatab
 			mysqlCfg.MetricsDatabase = cfg.dbMetricsName
 		}
 
-		// use a tmp connection to precreate the two databases
-		if tmpDB, err := gorm.Open(NewMySQLConnection(mysqlCfg.User, mysqlCfg.Password, mysqlCfg.URI, "")); err != nil {
+		// precreate the two databases
+		if tmpDB, err := mysql.Open(mysqlCfg.User, mysqlCfg.Password, mysqlCfg.URI, ""); err != nil {
 			return nil, nil, err
-		} else if err := tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", mysqlCfg.Database)).Error; err != nil {
+		} else if _, err := tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", mysqlCfg.Database)); err != nil {
 			return nil, nil, err
-		} else if err := tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", mysqlCfg.MetricsDatabase)).Error; err != nil {
+		} else if _, err := tmpDB.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", mysqlCfg.MetricsDatabase)); err != nil {
+			return nil, nil, err
+		} else if err := tmpDB.Close(); err != nil {
 			return nil, nil, err
 		}
 
-		connMain = NewMySQLConnection(mysqlCfg.User, mysqlCfg.Password, mysqlCfg.URI, mysqlCfg.Database)
-		dbm, err = mysql.Open(mysqlCfg.User, mysqlCfg.Password, mysqlCfg.URI, mysqlCfg.MetricsDatabase)
+		// create MySQL conns
+		connMain, err := mysql.Open(mysqlCfg.User, mysqlCfg.Password, mysqlCfg.URI, mysqlCfg.Database)
+		if err != nil {
+			return nil, nil, err
+		}
+		connMetrics, err := mysql.Open(mysqlCfg.User, mysqlCfg.Password, mysqlCfg.URI, mysqlCfg.MetricsDatabase)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to open MySQL metrics database: %w", err)
 		}
-		dbMetrics, err = mysql.NewMetricsDatabase(dbm, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond)
+		dbMain, err = mysql.NewMainDatabase(connMain, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to create MySQL main database: %w", err)
+		}
+		dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to create MySQL metrics database: %w", err)
+		}
 	} else if cfg.persistent {
 		// create SQL connections if we want a persistent store
-		connMain = NewSQLiteConnection(filepath.Join(cfg.dir, "db.sqlite"))
-		dbm, err = sqlite.Open(filepath.Join(cfg.dir, "metrics.sqlite"))
+		connMain, err := sqlite.Open(filepath.Join(cfg.dir, "db.sqlite"))
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to open SQLite main database: %w", err)
+		}
+		connMetrics, err := sqlite.Open(filepath.Join(cfg.dir, "metrics.sqlite"))
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to open SQLite metrics database: %w", err)
 		}
-		dbMetrics, err = sqlite.NewMetricsDatabase(dbm, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond)
+		dbMain, err = sqlite.NewMainDatabase(connMain, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to create SQLite main database: %w", err)
+		}
+		dbMetrics, err = sqlite.NewMetricsDatabase(connMetrics, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to create SQLite metrics database: %w", err)
+		}
 	} else {
 		// otherwise return ephemeral connections
-		connMain = NewEphemeralSQLiteConnection(cfg.dbName)
-		dbm, err = sqlite.OpenEphemeral(cfg.dbMetricsName)
+		connMain, err := sqlite.OpenEphemeral(cfg.dbName)
 		if err != nil {
 			return nil, nil, fmt.Errorf("failed to open ephemeral SQLite metrics database: %w", err)
 		}
-		dbMetrics, err = sqlite.NewMetricsDatabase(dbm, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond)
-	}
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to create metrics database: %w", err)
+		connMetrics, err := sqlite.OpenEphemeral(cfg.dbMetricsName)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to open ephemeral SQLite metrics database: %w", err)
+		}
+		dbMain, err = sqlite.NewMainDatabase(connMain, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to create ephemeral SQLite main database: %w", err)
+		}
+		dbMetrics, err = sqlite.NewMetricsDatabase(connMetrics, zap.NewNop().Sugar(), 100*time.Millisecond, 100*time.Millisecond)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to create ephemeral SQLite metrics database: %w", err)
+		}
 	}
-	return connMain, dbMetrics, nil
+	return dbMain, dbMetrics, nil
 }
 
 // newTestSQLStore creates a new SQLStore for testing.
@@ -140,15 +162,15 @@ func newTestSQLStore(t *testing.T, cfg testSQLStoreConfig) *testSQLStore {
 	}
 
 	// create db connections
-	conn, dbMetrics, err := cfg.dbConnections()
+	dbMain, dbMetrics, err := cfg.dbConnections()
 	if err != nil {
 		t.Fatal("failed to create db connections", err)
 	}
 
 	alerts := alerts.WithOrigin(alerts.NewManager(), "test")
 	sqlStore, err := NewSQLStore(Config{
-		Conn:                          conn,
 		Alerts:                        alerts,
+		DB:                            dbMain,
 		DBMetrics:                     dbMetrics,
 		PartialSlabDir:                cfg.dir,
 		Migrate:                       !cfg.skipMigrate,
@@ -156,7 +178,6 @@ func newTestSQLStore(t *testing.T, cfg testSQLStoreConfig) *testSQLStore {
 		Logger:                        zap.NewNop().Sugar(),
 		LongQueryDuration:             100 * time.Millisecond,
 		LongTxDuration:                100 * time.Millisecond,
-		GormLogger:                    newTestLogger(),
 		RetryTransactionIntervals:     []time.Duration{50 * time.Millisecond, 100 * time.Millisecond, 200 * time.Millisecond},
 	})
 	if err != nil {
@@ -200,6 +221,18 @@ func (s *testSQLStore) ExecDBSpecific(sqliteQuery, mysqlQuery string) (dsql.Resu
 	panic("unreachable")
 }
 
+func (s *testSQLStore) QueryRowDBSpecific(sqliteQuery, mysqlQuery string, sqliteArgs, mysqlArgs []any) *isql.LoggedRow {
+	switch db := s.db.(type) {
+	case *sqlite.MainDatabase:
+		return db.DB().QueryRow(context.Background(), sqliteQuery, sqliteArgs...)
+	case *mysql.MainDatabase:
+		return db.DB().QueryRow(context.Background(), mysqlQuery, mysqlArgs...)
+	default:
+		s.t.Fatal("unknown db type", db)
+	}
+	panic("unreachable")
+}
+
 func (s *testSQLStore) DBMetrics() *isql.DB {
 	switch db := s.dbMetrics.(type) {
 	case *sqlite.MetricsDatabase:
@@ -249,22 +282,6 @@ func (s *testSQLStore) Retry(tries int, durationBetweenAttempts time.Duration, f
 	}
 }
 
-// newTestLogger creates a console logger used for testing.
-func newTestLogger() logger.Interface {
-	config := zap.NewProductionEncoderConfig()
-	config.EncodeTime = zapcore.RFC3339TimeEncoder
-	config.EncodeLevel = zapcore.CapitalColorLevelEncoder
-	config.StacktraceKey = ""
-	consoleEncoder := zapcore.NewConsoleEncoder(config)
-
-	l := zap.New(
-		zapcore.NewCore(consoleEncoder, zapcore.AddSync(os.Stdout), zapcore.DebugLevel),
-		zap.AddCaller(),
-		zap.AddStacktrace(zapcore.ErrorLevel),
-	)
-	return zapgorm2.New(l)
-}
-
 func (s *testSQLStore) addTestObject(path string, o object.Object) (api.Object, error) {
 	if err := s.UpdateObjectBlocking(context.Background(), api.DefaultBucketName, path, testContractSet, testETag, testMimeType, testMetadata, o); err != nil {
 		return api.Object{}, err
@@ -306,8 +323,8 @@ func (s *SQLStore) addTestRenewedContract(fcid, renewedFrom types.FileContractID
 	return s.AddRenewedContract(context.Background(), rev, types.ZeroCurrency, types.ZeroCurrency, startHeight, renewedFrom, api.ContractStatePending)
 }
 
-func (s *SQLStore) overrideSlabHealth(objectID string, health float64) (err error) {
-	err = s.gormDB.Exec(fmt.Sprintf(`
+func (s *testSQLStore) overrideSlabHealth(objectID string, health float64) (err error) {
+	_, err = s.DB().Exec(context.Background(), fmt.Sprintf(`
 	UPDATE slabs SET health = %v WHERE id IN (
 		SELECT * FROM (
 			SELECT sla.id
@@ -316,77 +333,6 @@ func (s *SQLStore) overrideSlabHealth(objectID string, health float64) (err erro
 			INNER JOIN slabs sla ON sli.db_slab_id = sla.id
 			WHERE o.object_id = "%s"
 		) AS sub
-	)`, health, objectID)).Error
+	)`, health, objectID))
 	return
 }
-
-type sqliteQueryPlan struct {
-	Detail string `json:"detail"`
-}
-
-func (p sqliteQueryPlan) usesIndex() bool {
-	d := strings.ToLower(p.Detail)
-	return strings.Contains(d, "using index") || strings.Contains(d, "using covering index")
-}
-
-//nolint:tagliatelle
-type mysqlQueryPlan struct {
-	Extra        string `json:"Extra"`
-	PossibleKeys string `json:"possible_keys"`
-}
-
-func (p mysqlQueryPlan) usesIndex() bool {
-	d := strings.ToLower(p.Extra)
-	return strings.Contains(d, "using index") || strings.Contains(p.PossibleKeys, "idx_")
-}
-
-func TestQueryPlan(t *testing.T) {
-	ss := newTestSQLStore(t, defaultTestSQLStoreConfig)
-	defer ss.Close()
-
-	queries := []string{
-		// allow_list
-		"SELECT * FROM host_allowlist_entry_hosts WHERE db_host_id = 1",
-		"SELECT * FROM host_allowlist_entry_hosts WHERE db_allowlist_entry_id = 1",
-
-		// block_list
-		"SELECT * FROM host_blocklist_entry_hosts WHERE db_host_id = 1",
-		"SELECT * FROM host_blocklist_entry_hosts WHERE db_blocklist_entry_id = 1",
-
-		// contract_sectors
-		"SELECT * FROM contract_sectors WHERE db_contract_id = 1",
-		"SELECT * FROM contract_sectors WHERE db_sector_id = 1",
-		"SELECT COUNT(DISTINCT db_sector_id) FROM contract_sectors",
-
-		// contract_set_contracts
-		"SELECT * FROM contract_set_contracts WHERE db_contract_id = 1",
-		"SELECT * FROM contract_set_contracts WHERE db_contract_set_id = 1",
-
-		// slabs
-		"SELECT * FROM slabs WHERE health_valid_until > 0",
-		"SELECT * FROM slabs WHERE health > 0",
-		"SELECT * FROM slabs WHERE db_buffered_slab_id = 1",
-
-		// objects
-		"SELECT * FROM objects WHERE db_bucket_id = 1",
-		"SELECT * FROM objects WHERE etag = ''",
-	}
-
-	for _, query := range queries {
-		if isSQLite(ss.gormDB) {
-			var explain sqliteQueryPlan
-			if err := ss.gormDB.Raw(fmt.Sprintf("EXPLAIN QUERY PLAN %s;", query)).Scan(&explain).Error; err != nil {
-				t.Fatal(err)
-			} else if !explain.usesIndex() {
-				t.Fatalf("query '%s' should use an index, instead the plan was %+v", query, explain)
-			}
-		} else {
-			var explain mysqlQueryPlan
-			if err := ss.gormDB.Raw(fmt.Sprintf("EXPLAIN %s;", query)).Scan(&explain).Error; err != nil {
-				t.Fatal(err)
-			} else if !explain.usesIndex() {
-				t.Fatalf("query '%s' should use an index, instead the plan was %+v", query, explain)
-			}
-		}
-	}
-}
diff --git a/stores/types_test.go b/stores/types_test.go
index 4d9453d18..7518a17d0 100644
--- a/stores/types_test.go
+++ b/stores/types_test.go
@@ -1,9 +1,9 @@
 package stores
 
 import (
+	"context"
 	"fmt"
 	"sort"
-	"strings"
 	"testing"
 
 	"go.sia.tech/core/types"
@@ -23,15 +23,25 @@ func TestTypeCurrency(t *testing.T) {
 	}
 
 	// insert currencies in random order
-	if err := ss.gormDB.Exec("INSERT INTO currencies (c) VALUES (?),(?),(?);", sql.BCurrency(types.MaxCurrency), sql.BCurrency(types.NewCurrency64(1)), sql.BCurrency(types.ZeroCurrency)).Error; err != nil {
+	if _, err := ss.DB().Exec(context.Background(), "INSERT INTO currencies (c) VALUES (?),(?),(?);", sql.BCurrency(types.MaxCurrency), sql.BCurrency(types.NewCurrency64(1)), sql.BCurrency(types.ZeroCurrency)); err != nil {
 		t.Fatal(err)
 	}
 
 	// fetch currencies and assert they're sorted
 	var currencies []sql.BCurrency
-	if err := ss.gormDB.Raw(`SELECT c FROM currencies ORDER BY c ASC`).Scan(&currencies).Error; err != nil {
+	rows, err := ss.DB().Query(context.Background(), "SELECT c FROM currencies ORDER BY c ASC;")
+	if err != nil {
 		t.Fatal(err)
-	} else if !sort.SliceIsSorted(currencies, func(i, j int) bool {
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var c sql.BCurrency
+		if err := rows.Scan(&c); err != nil {
+			t.Fatal(err)
+		}
+		currencies = append(currencies, c)
+	}
+	if !sort.SliceIsSorted(currencies, func(i, j int) bool {
 		return types.Currency(currencies[i]).Cmp(types.Currency(currencies[j])) < 0
 	}) {
 		t.Fatal("currencies not sorted", currencies)
@@ -85,11 +95,12 @@ func TestTypeCurrency(t *testing.T) {
 	}
 	for i, test := range tests {
 		var result bool
-		query := fmt.Sprintf("SELECT ? %s ?", test.cmp)
-		if !isSQLite(ss.gormDB) {
-			query = strings.ReplaceAll(query, "?", "HEX(?)")
-		}
-		if err := ss.gormDB.Raw(query, test.a, test.b).Scan(&result).Error; err != nil {
+		if err := ss.QueryRowDBSpecific(
+			fmt.Sprintf("SELECT ? %s ?", test.cmp),
+			fmt.Sprintf("SELECT HEX(?) %s HEX(?)", test.cmp),
+			[]any{test.a, test.b},
+			[]any{test.a, test.b},
+		).Scan(&result); err != nil {
 			t.Fatal(err)
 		} else if !result {
 			t.Errorf("unexpected result in case %d/%d: expected %v %s %v to be true", i+1, len(tests), types.Currency(test.a).String(), test.cmp, types.Currency(test.b).String())
@@ -108,30 +119,21 @@ func TestTypeMerkleProof(t *testing.T) {
 	defer ss.Close()
 
 	// prepare the table
-	if isSQLite(ss.gormDB) {
-		if err := ss.gormDB.Exec("CREATE TABLE merkle_proofs (id INTEGER PRIMARY KEY AUTOINCREMENT,merkle_proof BLOB);").Error; err != nil {
-			t.Fatal(err)
-		}
-	} else {
-		ss.gormDB.Exec("DROP TABLE IF EXISTS merkle_proofs;")
-		if err := ss.gormDB.Exec("CREATE TABLE merkle_proofs (id INT AUTO_INCREMENT PRIMARY KEY, merkle_proof BLOB);").Error; err != nil {
-			t.Fatal(err)
-		}
-	}
+	ss.ExecDBSpecific(
+		"CREATE TABLE merkle_proofs (id INTEGER PRIMARY KEY AUTOINCREMENT,merkle_proof BLOB);",
+		"CREATE TABLE merkle_proofs (id INT AUTO_INCREMENT PRIMARY KEY, merkle_proof BLOB);",
+	)
 
 	// insert merkle proof
 	mp1 := sql.MerkleProof{Hashes: []types.Hash256{{3}, {1}, {2}}}
 	mp2 := sql.MerkleProof{Hashes: []types.Hash256{{4}}}
-	if err := ss.gormDB.Exec("INSERT INTO merkle_proofs (merkle_proof) VALUES (?), (?);", mp1, mp2).Error; err != nil {
+	if _, err := ss.DB().Exec(context.Background(), "INSERT INTO merkle_proofs (merkle_proof) VALUES (?), (?);", mp1, mp2); err != nil {
 		t.Fatal(err)
 	}
 
 	// fetch first proof
 	var first sql.MerkleProof
-	if err := ss.gormDB.
-		Raw(`SELECT merkle_proof FROM merkle_proofs`).
-		Take(&first).
-		Error; err != nil {
+	if err := ss.DB().QueryRow(context.Background(), "SELECT merkle_proof FROM merkle_proofs").Scan(&first); err != nil {
 		t.Fatal(err)
 	} else if first.Hashes[0] != (types.Hash256{3}) || first.Hashes[1] != (types.Hash256{1}) || first.Hashes[2] != (types.Hash256{2}) {
 		t.Fatalf("unexpected proof %+v", first)
@@ -139,12 +141,19 @@ func TestTypeMerkleProof(t *testing.T) {
 
 	// fetch both proofs
 	var both []sql.MerkleProof
-	if err := ss.gormDB.
-		Raw(`SELECT merkle_proof FROM merkle_proofs`).
-		Scan(&both).
-		Error; err != nil {
+	rows, err := ss.DB().Query(context.Background(), "SELECT merkle_proof FROM merkle_proofs")
+	if err != nil {
 		t.Fatal(err)
-	} else if len(both) != 2 {
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var mp sql.MerkleProof
+		if err := rows.Scan(&mp); err != nil {
+			t.Fatal(err)
+		}
+		both = append(both, mp)
+	}
+	if len(both) != 2 {
 		t.Fatalf("unexpected number of proofs: %d", len(both))
 	} else if both[1].Hashes[0] != (types.Hash256{4}) {
 		t.Fatalf("unexpected proof %+v", both)
diff --git a/webhooks/webhooks.go b/webhooks/webhooks.go
index dae5042c0..99557ec8e 100644
--- a/webhooks/webhooks.go
+++ b/webhooks/webhooks.go
@@ -13,7 +13,6 @@ import (
 	"time"
 
 	"go.uber.org/zap"
-	"gorm.io/gorm"
 )
 
 var ErrWebhookNotFound = errors.New("Webhook not found")
@@ -137,9 +136,7 @@ func (m *Manager) Close() error {
 func (m *Manager) Delete(ctx context.Context, wh Webhook) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	if err := m.store.DeleteWebhook(ctx, wh); errors.Is(err, gorm.ErrRecordNotFound) {
-		return ErrWebhookNotFound
-	} else if err != nil {
+	if err := m.store.DeleteWebhook(ctx, wh); err != nil {
 		return err
 	}
 	delete(m.webhooks, wh.String())

From 4fc4db2ee27f10d39ada26b6581b6a015cc0d528 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 15:18:57 +0200
Subject: [PATCH 15/19] client_test: fix TestClient

---
 stores/sql/sqlite/common.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/stores/sql/sqlite/common.go b/stores/sql/sqlite/common.go
index c45c0eab7..fd46688b8 100644
--- a/stores/sql/sqlite/common.go
+++ b/stores/sql/sqlite/common.go
@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"time"
 
+	_ "github.com/mattn/go-sqlite3"
 	"go.sia.tech/renterd/internal/sql"
 	"go.uber.org/zap"
 )

From 2d5b469020f8d852792cd0c80686e5f89427c947 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 15:24:06 +0200
Subject: [PATCH 16/19] sql: import mysql driver

---
 go.mod                     | 4 +++-
 go.sum                     | 4 ++++
 stores/sql/mysql/common.go | 1 +
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 4d6ba2d57..8a7fadab2 100644
--- a/go.mod
+++ b/go.mod
@@ -4,9 +4,11 @@ go 1.22.5
 
 require (
 	github.com/gabriel-vasile/mimetype v1.4.5
+	github.com/go-sql-driver/mysql v1.8.1
 	github.com/google/go-cmp v0.6.0
 	github.com/gotd/contrib v0.20.0
 	github.com/klauspost/reedsolomon v1.12.3
+	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/minio/minio-go/v7 v7.0.74
 	github.com/montanaflynn/stats v0.7.1
 	github.com/shopspring/decimal v1.4.0
@@ -26,6 +28,7 @@ require (
 )
 
 require (
+	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
 	github.com/aws/aws-sdk-go v1.55.5 // indirect
 	github.com/cloudflare/cloudflare-go v0.101.0 // indirect
@@ -39,7 +42,6 @@ require (
 	github.com/julienschmidt/httprouter v1.3.0 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
-	github.com/mattn/go-sqlite3 v1.14.22 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/rs/xid v1.5.0 // indirect
 	github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
diff --git a/go.sum b/go.sum
index 6acedb237..4b75da4ab 100644
--- a/go.sum
+++ b/go.sum
@@ -1,3 +1,5 @@
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da/go.mod h1:eHEWzANqSiWQsof+nXEI9bUVUyV6F53Fp89EuCh2EAA=
 github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
@@ -13,6 +15,8 @@ github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8
 github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4=
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
diff --git a/stores/sql/mysql/common.go b/stores/sql/mysql/common.go
index fca2749e7..73f6c9dc3 100644
--- a/stores/sql/mysql/common.go
+++ b/stores/sql/mysql/common.go
@@ -6,6 +6,7 @@ import (
 	"embed"
 	"fmt"
 
+	_ "github.com/go-sql-driver/mysql"
 	"go.sia.tech/renterd/internal/sql"
 )
 

From 009e9f922bab4edec1b1a8ea0080b34397fbeec5 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 15:35:08 +0200
Subject: [PATCH 17/19] sql: escape 'key'

---
 stores/metadata_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index d98992458..4ad3a16c6 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -3857,7 +3857,7 @@ func TestSlabHealthInvalidation(t *testing.T) {
 		t.Helper()
 
 		var validUntil int64
-		if err := ss.DB().QueryRow(context.Background(), "SELECT health_valid_until FROM slabs WHERE key = ?", sql.EncryptionKey(slabKey)).Scan(&validUntil); err != nil {
+		if err := ss.DB().QueryRow(context.Background(), "SELECT health_valid_until FROM slabs WHERE `key` = ?", sql.EncryptionKey(slabKey)).Scan(&validUntil); err != nil {
 			t.Fatal(err)
 		} else if valid := time.Now().Before(time.Unix(validUntil, 0)); valid != expected {
 			t.Fatal("unexpected health valid", valid)
@@ -4172,7 +4172,7 @@ func TestSlabCleanup(t *testing.T) {
 
 	// create a slab
 	var slabID int64
-	if res, err := ss.DB().Exec(context.Background(), "INSERT INTO slabs (db_contract_set_id, key, health_valid_until) VALUES (?, ?, ?);", csID, sql.EncryptionKey(object.GenerateEncryptionKey()), 100); err != nil {
+	if res, err := ss.DB().Exec(context.Background(), "INSERT INTO slabs (db_contract_set_id, `key`, health_valid_until) VALUES (?, ?, ?);", csID, sql.EncryptionKey(object.GenerateEncryptionKey()), 100); err != nil {
 		t.Fatal(err)
 	} else if slabID, err = res.LastInsertId(); err != nil {
 		t.Fatal(err)
@@ -4213,7 +4213,7 @@ func TestSlabCleanup(t *testing.T) {
 
 	// create another slab referencing the buffered slab
 	var bufferedSlabID int64
-	if res, err := ss.DB().Exec(context.Background(), "INSERT INTO slabs (db_buffered_slab_id, db_contract_set_id, key, health_valid_until) VALUES (?, ?, ?, ?);", bsID, csID, sql.EncryptionKey(object.GenerateEncryptionKey()), 100); err != nil {
+	if res, err := ss.DB().Exec(context.Background(), "INSERT INTO slabs (db_buffered_slab_id, db_contract_set_id, `key`, health_valid_until) VALUES (?, ?, ?, ?);", bsID, csID, sql.EncryptionKey(object.GenerateEncryptionKey()), 100); err != nil {
 		t.Fatal(err)
 	} else if bufferedSlabID, err = res.LastInsertId(); err != nil {
 		t.Fatal(err)
@@ -4387,7 +4387,7 @@ func TestUpdateObjectReuseSlab(t *testing.T) {
 		TotalShards      uint8
 		Key              object.EncryptionKey
 	}
-	fetchSlabStmt, err := ss.DB().Prepare(context.Background(), "SELECT id, db_contract_set_id, health, health_valid_until, min_shards, total_shards, key FROM slabs WHERE id = ?")
+	fetchSlabStmt, err := ss.DB().Prepare(context.Background(), "SELECT id, db_contract_set_id, health, health_valid_until, min_shards, total_shards, `key` FROM slabs WHERE id = ?")
 	if err != nil {
 		t.Fatal(err)
 	}

From 9c818d92d054ecdea5f6f700c97cb67a6fa80c51 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 16:08:53 +0200
Subject: [PATCH 18/19] node: fix connections in NewBus

---
 internal/node/node.go | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/internal/node/node.go b/internal/node/node.go
index 7acd6548f..5ed023c11 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -71,7 +71,16 @@ func NewBus(cfg BusConfig, dir string, seed types.PrivateKey, logger *zap.Logger
 	var dbMetrics sql.MetricsDatabase
 	if cfg.Database.MySQL.URI != "" {
 		// create MySQL connections
-		dbm, err := mysql.Open(
+		connMain, err := mysql.Open(
+			cfg.Database.MySQL.User,
+			cfg.Database.MySQL.Password,
+			cfg.Database.MySQL.URI,
+			cfg.Database.MySQL.Database,
+		)
+		if err != nil {
+			return nil, nil, nil, nil, nil, fmt.Errorf("failed to open MySQL main database: %w", err)
+		}
+		connMetrics, err := mysql.Open(
 			cfg.Database.MySQL.User,
 			cfg.Database.MySQL.Password,
 			cfg.Database.MySQL.URI,
@@ -80,11 +89,11 @@ func NewBus(cfg BusConfig, dir string, seed types.PrivateKey, logger *zap.Logger
 		if err != nil {
 			return nil, nil, nil, nil, nil, fmt.Errorf("failed to open MySQL metrics database: %w", err)
 		}
-		dbMain, err = mysql.NewMainDatabase(dbm, logger.Named("main").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold)
+		dbMain, err = mysql.NewMainDatabase(connMain, logger.Named("main").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold)
 		if err != nil {
 			return nil, nil, nil, nil, nil, fmt.Errorf("failed to create MySQL main database: %w", err)
 		}
-		dbMetrics, err = mysql.NewMetricsDatabase(dbm, logger.Named("metrics").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold)
+		dbMetrics, err = mysql.NewMetricsDatabase(connMetrics, logger.Named("metrics").Sugar(), cfg.DatabaseLog.SlowThreshold, cfg.DatabaseLog.SlowThreshold)
 		if err != nil {
 			return nil, nil, nil, nil, nil, fmt.Errorf("failed to create MySQL metrics database: %w", err)
 		}

From 56b70accdd313ea1f15fdb1cb8011b6a1b2ae558 Mon Sep 17 00:00:00 2001
From: Chris Schinnerl <chris@sia.tech>
Date: Tue, 13 Aug 2024 16:39:33 +0200
Subject: [PATCH 19/19] sql: add missing Close

---
 stores/metadata_test.go | 2 ++
 stores/sql/metrics.go   | 1 +
 2 files changed, 3 insertions(+)

diff --git a/stores/metadata_test.go b/stores/metadata_test.go
index 4ad3a16c6..a5c9d1325 100644
--- a/stores/metadata_test.go
+++ b/stores/metadata_test.go
@@ -2540,6 +2540,7 @@ func TestRenameObjects(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer rows.Close()
 	var i int
 	for rows.Next() {
 		var dir row
@@ -4744,6 +4745,7 @@ func TestDirectories(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer rows.Close()
 	var nDirs int
 	for i := 0; rows.Next(); i++ {
 		var dir row
diff --git a/stores/sql/metrics.go b/stores/sql/metrics.go
index 689f98843..510048114 100644
--- a/stores/sql/metrics.go
+++ b/stores/sql/metrics.go
@@ -206,6 +206,7 @@ func RecordContractMetric(ctx context.Context, tx sql.Tx, metrics ...api.Contrac
 	if err != nil {
 		return fmt.Errorf("failed to prepare statement to delete contract metric: %w", err)
 	}
+	defer deleteStmt.Close()
 
 	for _, metric := range metrics {
 		// delete any existing metric for the same contract that has happened