Merge pull request #773 from SiaFoundation/pj/contract-sizes

Optimise Contract Sizes

stores/metadata.go

@@ -912,26 +912,42 @@ func (s *SQLStore) ContractSets(ctx context.Context) ([]string, error) {
 }
 
 func (s *SQLStore) ContractSizes(ctx context.Context) (map[types.FileContractID]api.ContractSize, error) {
-	rows := make([]struct {
+	type size struct {
 		Fcid     fileContractID `json:"fcid"`
 		Size     uint64         `json:"size"`
 		Prunable uint64         `json:"prunable"`
-	}, 0)
+	}
 
-	if err := s.db.
-		Raw(`
-SELECT fcid, MAX(c.size) as size, CASE WHEN MAX(c.size)>COUNT(cs.db_sector_id) * ? THEN MAX(c.size)-(COUNT(cs.db_sector_id) * ?) ELSE 0 END as prunable
-FROM contracts c
-LEFT JOIN contract_sectors cs ON cs.db_contract_id = c.id
-GROUP BY c.fcid
-	`, rhpv2.SectorSize, rhpv2.SectorSize).
-		Scan(&rows).
-		Error; err != nil {
+	var nullContracts []size
+	var dataContracts []size
+	if err := s.db.Transaction(func(tx *gorm.DB) error {
+		// first, we fetch all contracts without sectors and consider their
+		// entire size as prunable
+		if err := tx.
+			Raw(`
+SELECT c.fcid, c.size, c.size as prunable FROM contracts c WHERE NOT EXISTS (SELECT 1 FROM contract_sectors cs WHERE cs.db_contract_id = c.id)`).
+			Scan(&nullContracts).
+			Error; err != nil {
+			return err
+		}
+
+		// second, we fetch how much data can be pruned from all contracts that
+		// do have sectors, we take a two-step approach because it allows us to
+		// use an INNER JOIN on contract_sectors, drastically improving the
+		// performance of the query
+		return tx.
+			Raw(`
+SELECT fcid, contract_size as size, CASE WHEN contract_size > sector_size THEN contract_size - sector_size ELSE 0 END as prunable FROM (
+SELECT c.fcid, MAX(c.size) as contract_size, COUNT(cs.db_sector_id) * ? as sector_size FROM contracts c INNER JOIN contract_sectors cs ON cs.db_contract_id = c.id GROUP BY c.fcid
+) i`, rhpv2.SectorSize).
+			Scan(&dataContracts).
+			Error
+	}); err != nil {
 		return nil, err
 	}
 
 	sizes := make(map[types.FileContractID]api.ContractSize)
-	for _, row := range rows {
+	for _, row := range append(nullContracts, dataContracts...) {
 		if types.FileContractID(row.Fcid) == (types.FileContractID{}) {
 			return nil, errors.New("invalid file contract id")
 		}
@@ -955,11 +971,10 @@ func (s *SQLStore) ContractSize(ctx context.Context, id types.FileContractID) (a
 
 	if err := s.db.
 		Raw(`
-SELECT MAX(c.size) as size, CASE WHEN MAX(c.size)>(COUNT(cs.db_sector_id) * ?) THEN MAX(c.size)-(COUNT(cs.db_sector_id) * ?) ELSE 0 END as prunable
-FROM contracts c
-LEFT JOIN contract_sectors cs ON cs.db_contract_id = c.id
-WHERE c.fcid = ?
-`, rhpv2.SectorSize, rhpv2.SectorSize, fileContractID(id)).
+SELECT contract_size as size, CASE WHEN contract_size > sector_size THEN contract_size - sector_size ELSE 0 END as prunable FROM (
+SELECT MAX(c.size) as contract_size, COUNT(cs.db_sector_id) * ? as sector_size FROM contracts c LEFT JOIN contract_sectors cs ON cs.db_contract_id = c.id WHERE c.fcid = ?
+) i
+`, rhpv2.SectorSize, fileContractID(id)).
 		Take(&size).
 		Error; err != nil {
 		return api.ContractSize{}, err

stores/metadata_test.go

@@ -3777,7 +3777,7 @@ func TestSlabHealthInvalidation(t *testing.T) {
 		}
 
 		// refresh health
-		now := time.Now().Round(time.Second)
+		now := time.Now()
 		if err := ss.RefreshHealth(context.Background()); err != nil {
 			t.Fatal(err)
 		}