From 9bb5972586e6fc26ad0c21758b0c265f8bab76a2 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 09:55:13 +0100 Subject: [PATCH 01/12] stores: migration code --- .../migration_00004_prune_slabs_cascade.sql | 24 +++++++++++ stores/migrations/mysql/main/schema.sql | 40 ++---------------- .../migration_00004_prune_slabs_cascade.sql | 28 +++++++++++++ stores/migrations/sqlite/main/schema.sql | 42 +------------------ 4 files changed, 57 insertions(+), 77 deletions(-) create mode 100644 stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql create mode 100644 stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql diff --git a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql new file mode 100644 index 000000000..48f98a40a --- /dev/null +++ b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql @@ -0,0 +1,24 @@ +-- prune manually before creating trigger +DELETE slabs +FROM slabs +LEFT JOIN slices ON slices.db_slab_id = slabs.id +WHERE slices.db_object_id IS NULL +AND slices.db_multipart_part_id IS NULL +AND slabs.db_buffered_slab_id IS NULL; + +-- add ON DELETE CASCADE to slices +ALTER TABLE slices DROP FOREIGN KEY fk_objects_slabs; +ALTER TABLE slices ADD CONSTRAINT fk_objects_slabs FOREIGN KEY (db_object_id) REFERENCES objects (id) ON DELETE CASCADE, + +ALTER TABLE slices DROP FOREIGN KEY fk_multipart_parts_slabs; +ALTER TABLE slices ADD CONSTRAINT fk_multipart_parts_slabs FOREIGN KEY (db_multipart_part_id) REFERENCES multipart_parts (id) ON DELETE CASCADE, + +-- add ON DELETE CASCADE to multipart_parts +ALTER TABLE multipart_parts DROP FOREIGN KEY fk_multipart_uploads_parts; +ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE + +-- drop triggers +DROP TRIGGER before_delete_on_objects_delete_slices +DROP TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts +DROP TRIGGER before_delete_on_multipart_parts_delete_slices +DROP TRIGGER after_delete_on_slices_delete_slabs \ No newline at end of file diff --git a/stores/migrations/mysql/main/schema.sql b/stores/migrations/mysql/main/schema.sql index 1b39e4669..a5ed86807 100644 --- a/stores/migrations/mysql/main/schema.sql +++ b/stores/migrations/mysql/main/schema.sql @@ -310,7 +310,7 @@ CREATE TABLE `multipart_parts` ( KEY `idx_multipart_parts_etag` (`etag`), KEY `idx_multipart_parts_part_number` (`part_number`), KEY `idx_multipart_parts_db_multipart_upload_id` (`db_multipart_upload_id`), - CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) + CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; -- dbObject @@ -374,8 +374,8 @@ CREATE TABLE `slices` ( KEY `idx_slices_object_index` (`object_index`), KEY `idx_slices_db_multipart_part_id` (`db_multipart_part_id`), KEY `idx_slices_db_slab_id` (`db_slab_id`), - CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts` (`id`), - CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects` (`id`), + CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects` (`id`) ON DELETE CASCADE, CONSTRAINT `fk_slabs_slices` FOREIGN KEY (`db_slab_id`) REFERENCES `slabs` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; @@ -421,39 +421,5 @@ CREATE TABLE `object_user_metadata` ( CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; --- dbObject trigger to delete from slices -CREATE TRIGGER before_delete_on_objects_delete_slices -BEFORE DELETE -ON objects FOR EACH ROW -DELETE FROM slices -WHERE slices.db_object_id = OLD.id; - --- dbMultipartUpload trigger to delete from dbMultipartPart -CREATE TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts -BEFORE DELETE -ON multipart_uploads FOR EACH ROW -DELETE FROM multipart_parts -WHERE multipart_parts.db_multipart_upload_id = OLD.id; - --- dbMultipartPart trigger to delete from slices -CREATE TRIGGER before_delete_on_multipart_parts_delete_slices -BEFORE DELETE -ON multipart_parts FOR EACH ROW -DELETE FROM slices -WHERE slices.db_multipart_part_id = OLD.id; - --- dbSlices trigger to prune slabs -CREATE TRIGGER after_delete_on_slices_delete_slabs -AFTER DELETE -ON slices FOR EACH ROW -DELETE FROM slabs -WHERE slabs.id = OLD.db_slab_id -AND slabs.db_buffered_slab_id IS NULL -AND NOT EXISTS ( - SELECT 1 - FROM slices - WHERE slices.db_slab_id = OLD.db_slab_id -); - -- create default bucket INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); \ No newline at end of file diff --git a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql new file mode 100644 index 000000000..2cd633c11 --- /dev/null +++ b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql @@ -0,0 +1,28 @@ +PRAGMA foreign_keys=off; +-- update constraints on slices +CREATE TABLE `slices_temp` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer,`object_index` integer,`db_multipart_part_id` integer,`db_slab_id` integer,`offset` integer,`length` integer,CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_slabs_slices` FOREIGN KEY (`db_slab_id`) REFERENCES `slabs`(`id`)); +INSERT INTO slices_temp SELECT `id`, `created_at`, `db_object_id`, `object_index`, `db_multipart_part_id`, `db_slab_id`, `offset`, `length` FROM slices; +DROP TABLE slices; +ALTER TABLE slices_temp RENAME TO slices; + +CREATE INDEX `idx_slices_object_index` ON `slices`(`object_index`); +CREATE INDEX `idx_slices_db_object_id` ON `slices`(`db_object_id`); +CREATE INDEX `idx_slices_db_slab_id` ON `slices`(`db_slab_id`); +CREATE INDEX `idx_slices_db_multipart_part_id` ON `slices`(`db_multipart_part_id`); + +-- update constraints multipart_parts +CREATE TABLE `multipart_parts_temp` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`etag` text,`part_number` integer,`size` integer,`db_multipart_upload_id` integer NOT NULL,CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads`(`id`) ON DELETE CASCADE); +INSERT INTO multipart_parts_temp SELECT * FROM multipart_parts; +DROP TABLE multipart_parts; +ALTER TABLE multipart_parts_temp RENAME TO multipart_parts; + +CREATE INDEX `idx_multipart_parts_db_multipart_upload_id` ON `multipart_parts`(`db_multipart_upload_id`); +CREATE INDEX `idx_multipart_parts_part_number` ON `multipart_parts`(`part_number`); +CREATE INDEX `idx_multipart_parts_etag` ON `multipart_parts`(`etag`); +PRAGMA foreign_keys=on; + +-- drop triggers +DROP TRIGGER before_delete_on_objects_delete_slices +DROP TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts +DROP TRIGGER before_delete_on_multipart_parts_delete_slices +DROP TRIGGER after_delete_on_slices_delete_slabs \ No newline at end of file diff --git a/stores/migrations/sqlite/main/schema.sql b/stores/migrations/sqlite/main/schema.sql index e6bb2546d..8d7afeaa1 100644 --- a/stores/migrations/sqlite/main/schema.sql +++ b/stores/migrations/sqlite/main/schema.sql @@ -85,13 +85,13 @@ CREATE INDEX `idx_contract_sectors_db_contract_id` ON `contract_sectors`(`db_con CREATE INDEX `idx_contract_sectors_db_sector_id` ON `contract_sectors`(`db_sector_id`); -- dbMultipartPart -CREATE TABLE `multipart_parts` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`etag` text,`part_number` integer,`size` integer,`db_multipart_upload_id` integer NOT NULL,CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads`(`id`)); +CREATE TABLE `multipart_parts` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`etag` text,`part_number` integer,`size` integer,`db_multipart_upload_id` integer NOT NULL,CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads`(`id`) ON DELETE CASCADE); CREATE INDEX `idx_multipart_parts_db_multipart_upload_id` ON `multipart_parts`(`db_multipart_upload_id`); CREATE INDEX `idx_multipart_parts_part_number` ON `multipart_parts`(`part_number`); CREATE INDEX `idx_multipart_parts_etag` ON `multipart_parts`(`etag`); -- dbSlice -CREATE TABLE `slices` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer,`object_index` integer,`db_multipart_part_id` integer,`db_slab_id` integer,`offset` integer,`length` integer,CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects`(`id`),CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts`(`id`),CONSTRAINT `fk_slabs_slices` FOREIGN KEY (`db_slab_id`) REFERENCES `slabs`(`id`)); +CREATE TABLE `slices` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer,`object_index` integer,`db_multipart_part_id` integer,`db_slab_id` integer,`offset` integer,`length` integer,CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_slabs_slices` FOREIGN KEY (`db_slab_id`) REFERENCES `slabs`(`id`)); CREATE INDEX `idx_slices_object_index` ON `slices`(`object_index`); CREATE INDEX `idx_slices_db_object_id` ON `slices`(`db_object_id`); CREATE INDEX `idx_slices_db_slab_id` ON `slices`(`db_slab_id`); @@ -148,43 +148,5 @@ CREATE UNIQUE INDEX `idx_module_event_url` ON `webhooks`(`module`,`event`,`url`) CREATE TABLE `object_user_metadata` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer DEFAULT NULL,`db_multipart_upload_id` integer DEFAULT NULL,`key` text NOT NULL,`value` text, CONSTRAINT `fk_object_user_metadata` FOREIGN KEY (`db_object_id`) REFERENCES `objects` (`id`) ON DELETE CASCADE, CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL); CREATE UNIQUE INDEX `idx_object_user_metadata_key` ON `object_user_metadata`(`db_object_id`,`db_multipart_upload_id`,`key`); --- dbObject trigger to delete from slices -CREATE TRIGGER before_delete_on_objects_delete_slices -BEFORE DELETE ON objects -BEGIN - DELETE FROM slices - WHERE slices.db_object_id = OLD.id; -END; - --- dbMultipartUpload trigger to delete from dbMultipartPart -CREATE TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts -BEFORE DELETE ON multipart_uploads -BEGIN - DELETE FROM multipart_parts - WHERE multipart_parts.db_multipart_upload_id = OLD.id; -END; - --- dbMultipartPart trigger to delete from slices -CREATE TRIGGER before_delete_on_multipart_parts_delete_slices -BEFORE DELETE ON multipart_parts -BEGIN - DELETE FROM slices - WHERE slices.db_multipart_part_id = OLD.id; -END; - --- dbSlices trigger to prune slabs -CREATE TRIGGER after_delete_on_slices_delete_slabs -AFTER DELETE ON slices -BEGIN - DELETE FROM slabs - WHERE slabs.id = OLD.db_slab_id - AND slabs.db_buffered_slab_id IS NULL - AND NOT EXISTS ( - SELECT 1 - FROM slices - WHERE slices.db_slab_id = OLD.db_slab_id - ); -END; - -- create default bucket INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); From 1f004223a11b0c4897438db98584d81460365e7e Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 10:06:21 +0100 Subject: [PATCH 02/12] stores: call pruneSlabs every time an object or multipart object were deleted --- stores/metadata.go | 23 ++++++++++++++++++++--- stores/multipart.go | 9 +++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/stores/metadata.go b/stores/metadata.go index 68947ed95..a9636bf75 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -2685,20 +2685,32 @@ func archiveContracts(ctx context.Context, tx *gorm.DB, contracts []dbContract, return nil } +func pruneSlabs(tx *gorm.DB) error { + // delete slabs without any associated slices or buffers + return tx.Exec(` +DELETE +FROM slabs sla +WHERE NOT EXISTS (SELECT 1 FROM slices sli WHERE sli.db_slab_id = sla.id) +AND sla.db_buffered_slab_id IS NULL +`).Error +} + // deleteObject deletes an object from the store and prunes all slabs which are // without an obect after the deletion. That means in case of packed uploads, // the slab is only deleted when no more objects point to it. -func (s *SQLStore) deleteObject(tx *gorm.DB, bucket string, path string) (numDeleted int64, _ error) { +func (s *SQLStore) deleteObject(tx *gorm.DB, bucket string, path string) (int64, error) { tx = tx.Where("object_id = ? AND ?", path, sqlWhereBucket("objects", bucket)). Delete(&dbObject{}) if tx.Error != nil { return 0, tx.Error } - numDeleted = tx.RowsAffected + numDeleted := tx.RowsAffected if numDeleted == 0 { return 0, nil // nothing to prune if no object was deleted + } else if err := pruneSlabs(tx); err != nil { + return 0, err } - return + return numDeleted, nil } // deleteObjects deletes a batch of objects from the database. The order of @@ -2729,6 +2741,11 @@ func (s *SQLStore) deleteObjects(bucket string, path string) (numDeleted int64, } duration = time.Since(start) rowsAffected = res.RowsAffected + + // prune slabs if we deleted an object + if rowsAffected > 0 { + return pruneSlabs(tx) + } return nil }); err != nil { return 0, fmt.Errorf("failed to delete objects: %w", err) diff --git a/stores/multipart.go b/stores/multipart.go index 18706ed0c..3a5bcd54a 100644 --- a/stores/multipart.go +++ b/stores/multipart.go @@ -295,6 +295,10 @@ func (s *SQLStore) AbortMultipartUpload(ctx context.Context, bucket, path string if err != nil { return fmt.Errorf("failed to delete multipart upload: %w", err) } + // Prune the slabs. + if err := pruneSlabs(tx); err != nil { + return fmt.Errorf("failed to prune slabs: %w", err) + } return nil }) } @@ -435,6 +439,11 @@ func (s *SQLStore) CompleteMultipartUpload(ctx context.Context, bucket, path str if err := tx.Delete(&mu).Error; err != nil { return fmt.Errorf("failed to delete multipart upload: %w", err) } + + // Prune the slabs. + if err := pruneSlabs(tx); err != nil { + return fmt.Errorf("failed to prune slabs: %w", err) + } return nil }) if err != nil { From 671f8da872b0decaf217bd4e4d0119d974b1160d Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 10:19:05 +0100 Subject: [PATCH 03/12] stores: add migration to performMigrations --- stores/metadata.go | 6 +++--- stores/migrations.go | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/stores/metadata.go b/stores/metadata.go index a9636bf75..13f274477 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -2689,9 +2689,9 @@ func pruneSlabs(tx *gorm.DB) error { // delete slabs without any associated slices or buffers return tx.Exec(` DELETE -FROM slabs sla -WHERE NOT EXISTS (SELECT 1 FROM slices sli WHERE sli.db_slab_id = sla.id) -AND sla.db_buffered_slab_id IS NULL +FROM slabs +WHERE NOT EXISTS (SELECT 1 FROM slices WHERE slices.db_slab_id = slabs.id) +AND slabs.db_buffered_slab_id IS NULL `).Error } diff --git a/stores/migrations.go b/stores/migrations.go index bf3916ca4..d89be7ab5 100644 --- a/stores/migrations.go +++ b/stores/migrations.go @@ -44,6 +44,12 @@ func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { return performMigration(tx, dbIdentifier, "00003_idx_objects_size", logger) }, }, + { + ID: "00004_prune_slabs_cascade", + Migrate: func(tx *gorm.DB) error { + return performMigration(tx, dbIdentifier, "00004_prune_slabs_cascade", logger) + }, + }, } // Create migrator. From 0a23dea48598451ab56cb5b836a4f6a3cf4c8240 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 10:57:09 +0100 Subject: [PATCH 04/12] stores: fix TestSlabCleanupTrigger --- stores/metadata.go | 5 ++--- stores/metadata_test.go | 9 ++++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/stores/metadata.go b/stores/metadata.go index 13f274477..2ab2cf5da 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -2739,13 +2739,12 @@ func (s *SQLStore) deleteObjects(bucket string, path string) (numDeleted int64, if err := res.Error; err != nil { return res.Error } - duration = time.Since(start) - rowsAffected = res.RowsAffected - // prune slabs if we deleted an object + rowsAffected = res.RowsAffected if rowsAffected > 0 { return pruneSlabs(tx) } + duration = time.Since(start) return nil }); err != nil { return 0, fmt.Errorf("failed to delete objects: %w", err) diff --git a/stores/metadata_test.go b/stores/metadata_test.go index 4a6102399..d5a0f9531 100644 --- a/stores/metadata_test.go +++ b/stores/metadata_test.go @@ -3941,7 +3941,8 @@ func TestSlabCleanupTrigger(t *testing.T) { } // delete the object - if err := ss.db.Delete(&obj1).Error; err != nil { + err := ss.RemoveObject(context.Background(), api.DefaultBucketName, obj1.ObjectID) + if err != nil { t.Fatal(err) } @@ -3954,7 +3955,8 @@ func TestSlabCleanupTrigger(t *testing.T) { } // delete second object - if err := ss.db.Delete(&obj2).Error; err != nil { + err = ss.RemoveObject(context.Background(), api.DefaultBucketName, obj2.ObjectID) + if err != nil { t.Fatal(err) } @@ -3998,7 +4000,8 @@ func TestSlabCleanupTrigger(t *testing.T) { } // delete third object - if err := ss.db.Delete(&obj3).Error; err != nil { + err = ss.RemoveObject(context.Background(), api.DefaultBucketName, obj3.ObjectID) + if err != nil { t.Fatal(err) } if err := ss.db.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil { From 96d4bba92f350ffc4490ff6f53011e9c989f7e47 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 13:37:50 +0100 Subject: [PATCH 05/12] stores: only drop triggers if they exist --- .../mysql/main/migration_00004_prune_slabs_cascade.sql | 8 ++++---- .../sqlite/main/migration_00004_prune_slabs_cascade.sql | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql index 48f98a40a..125da0ecb 100644 --- a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql @@ -18,7 +18,7 @@ ALTER TABLE multipart_parts DROP FOREIGN KEY fk_multipart_uploads_parts; ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE -- drop triggers -DROP TRIGGER before_delete_on_objects_delete_slices -DROP TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts -DROP TRIGGER before_delete_on_multipart_parts_delete_slices -DROP TRIGGER after_delete_on_slices_delete_slabs \ No newline at end of file +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs \ No newline at end of file diff --git a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql index 2cd633c11..b7f5ab128 100644 --- a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql @@ -22,7 +22,7 @@ CREATE INDEX `idx_multipart_parts_etag` ON `multipart_parts`(`etag`); PRAGMA foreign_keys=on; -- drop triggers -DROP TRIGGER before_delete_on_objects_delete_slices -DROP TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts -DROP TRIGGER before_delete_on_multipart_parts_delete_slices -DROP TRIGGER after_delete_on_slices_delete_slabs \ No newline at end of file +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs \ No newline at end of file From adafec043860b8514c71e535eddec1e3729bb407 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 15:24:05 +0100 Subject: [PATCH 06/12] stores: add missing newline --- .../mysql/main/migration_00004_prune_slabs_cascade.sql | 2 +- .../sqlite/main/migration_00004_prune_slabs_cascade.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql index 125da0ecb..0b1c06994 100644 --- a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql @@ -21,4 +21,4 @@ ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KE DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs \ No newline at end of file +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs diff --git a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql index b7f5ab128..1132dd2f5 100644 --- a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql @@ -25,4 +25,4 @@ PRAGMA foreign_keys=on; DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs \ No newline at end of file +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs From 5b968e7e71ccce04f98a3d36f5c17997a2d17907 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 22 Feb 2024 13:49:23 +0100 Subject: [PATCH 07/12] stores: numDeleted --- stores/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stores/metadata.go b/stores/metadata.go index 2ab2cf5da..eaad76c17 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -2708,7 +2708,7 @@ func (s *SQLStore) deleteObject(tx *gorm.DB, bucket string, path string) (int64, if numDeleted == 0 { return 0, nil // nothing to prune if no object was deleted } else if err := pruneSlabs(tx); err != nil { - return 0, err + return numDeleted, err } return numDeleted, nil } From 10e88fd4a00196b467a5b3d3d0ac26564ec861c6 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 22 Feb 2024 18:15:46 +0100 Subject: [PATCH 08/12] stores: fix mysql migration --- .../migration_00004_prune_slabs_cascade.sql | 22 ++++++------------- .../migration_00004_prune_slabs_cascade.sql | 8 +++---- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql index 0b1c06994..9014582e0 100644 --- a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql @@ -1,24 +1,16 @@ --- prune manually before creating trigger -DELETE slabs -FROM slabs -LEFT JOIN slices ON slices.db_slab_id = slabs.id -WHERE slices.db_object_id IS NULL -AND slices.db_multipart_part_id IS NULL -AND slabs.db_buffered_slab_id IS NULL; - -- add ON DELETE CASCADE to slices ALTER TABLE slices DROP FOREIGN KEY fk_objects_slabs; -ALTER TABLE slices ADD CONSTRAINT fk_objects_slabs FOREIGN KEY (db_object_id) REFERENCES objects (id) ON DELETE CASCADE, +ALTER TABLE slices ADD CONSTRAINT fk_objects_slabs FOREIGN KEY (db_object_id) REFERENCES objects (id) ON DELETE CASCADE; ALTER TABLE slices DROP FOREIGN KEY fk_multipart_parts_slabs; -ALTER TABLE slices ADD CONSTRAINT fk_multipart_parts_slabs FOREIGN KEY (db_multipart_part_id) REFERENCES multipart_parts (id) ON DELETE CASCADE, +ALTER TABLE slices ADD CONSTRAINT fk_multipart_parts_slabs FOREIGN KEY (db_multipart_part_id) REFERENCES multipart_parts (id) ON DELETE CASCADE; -- add ON DELETE CASCADE to multipart_parts ALTER TABLE multipart_parts DROP FOREIGN KEY fk_multipart_uploads_parts; -ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE +ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE; -- drop triggers -DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices -DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts -DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs; diff --git a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql index 1132dd2f5..38cd40199 100644 --- a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql @@ -22,7 +22,7 @@ CREATE INDEX `idx_multipart_parts_etag` ON `multipart_parts`(`etag`); PRAGMA foreign_keys=on; -- drop triggers -DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices -DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts -DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs; From 3e92f08d4fabab1992970797a9200afe3141b568 Mon Sep 17 00:00:00 2001 From: ChrisSchinnerl Date: Fri, 23 Feb 2024 00:08:30 +0000 Subject: [PATCH 09/12] ui: v0.46.0 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e727483d1..0ce29b965 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640 go.sia.tech/mux v1.2.0 go.sia.tech/siad v1.5.10-0.20230228235644-3059c0b930ca - go.sia.tech/web/renterd v0.45.0 + go.sia.tech/web/renterd v0.46.0 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.19.0 golang.org/x/term v0.17.0 diff --git a/go.sum b/go.sum index 006a31ea6..2570e9fc5 100644 --- a/go.sum +++ b/go.sum @@ -255,8 +255,8 @@ go.sia.tech/siad v1.5.10-0.20230228235644-3059c0b930ca h1:aZMg2AKevn7jKx+wlusWQf go.sia.tech/siad v1.5.10-0.20230228235644-3059c0b930ca/go.mod h1:h/1afFwpxzff6/gG5i1XdAgPK7dEY6FaibhK7N5F86Y= go.sia.tech/web v0.0.0-20231213145933-3f175a86abff h1:/nE7nhewDRxzEdtSKT4SkiUwtjPSiy7Xz7CHEW3MaGQ= go.sia.tech/web v0.0.0-20231213145933-3f175a86abff/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= -go.sia.tech/web/renterd v0.45.0 h1:5kSiDnHYRacg3JideH9Cl9qHzcZiKnBR0fWRap169hU= -go.sia.tech/web/renterd v0.45.0/go.mod h1:FgXrdmAnu591a3h96RB/15pMZ74xO9457g902uE06BM= +go.sia.tech/web/renterd v0.46.0 h1:BMVg4i7LxSlc8wZ4T0EG1k3EK4JxVIzCfD3/cjmwH0k= +go.sia.tech/web/renterd v0.46.0/go.mod h1:FgXrdmAnu591a3h96RB/15pMZ74xO9457g902uE06BM= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= From 99014c79fe65c3bffa5c737a768d3905351c7517 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 23 Feb 2024 09:47:48 +0100 Subject: [PATCH 10/12] stores: drop indices first --- .../main/migration_00004_prune_slabs_cascade.sql | 14 +++++++------- .../main/migration_00004_prune_slabs_cascade.sql | 14 ++++++++------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql index 9014582e0..c2efe3467 100644 --- a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql @@ -1,3 +1,9 @@ +-- drop triggers +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs; + -- add ON DELETE CASCADE to slices ALTER TABLE slices DROP FOREIGN KEY fk_objects_slabs; ALTER TABLE slices ADD CONSTRAINT fk_objects_slabs FOREIGN KEY (db_object_id) REFERENCES objects (id) ON DELETE CASCADE; @@ -7,10 +13,4 @@ ALTER TABLE slices ADD CONSTRAINT fk_multipart_parts_slabs FOREIGN KEY (db_multi -- add ON DELETE CASCADE to multipart_parts ALTER TABLE multipart_parts DROP FOREIGN KEY fk_multipart_uploads_parts; -ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE; - --- drop triggers -DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; -DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; -DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs; +ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE; \ No newline at end of file diff --git a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql index 38cd40199..03f006acd 100644 --- a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql @@ -1,5 +1,12 @@ +-- drop triggers +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs; + PRAGMA foreign_keys=off; -- update constraints on slices +DROP TABLE IF EXISTS slices_temp; CREATE TABLE `slices_temp` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer,`object_index` integer,`db_multipart_part_id` integer,`db_slab_id` integer,`offset` integer,`length` integer,CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_slabs_slices` FOREIGN KEY (`db_slab_id`) REFERENCES `slabs`(`id`)); INSERT INTO slices_temp SELECT `id`, `created_at`, `db_object_id`, `object_index`, `db_multipart_part_id`, `db_slab_id`, `offset`, `length` FROM slices; DROP TABLE slices; @@ -11,6 +18,7 @@ CREATE INDEX `idx_slices_db_slab_id` ON `slices`(`db_slab_id`); CREATE INDEX `idx_slices_db_multipart_part_id` ON `slices`(`db_multipart_part_id`); -- update constraints multipart_parts +DROP TABLE IF EXISTS multipart_parts_temp; CREATE TABLE `multipart_parts_temp` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`etag` text,`part_number` integer,`size` integer,`db_multipart_upload_id` integer NOT NULL,CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads`(`id`) ON DELETE CASCADE); INSERT INTO multipart_parts_temp SELECT * FROM multipart_parts; DROP TABLE multipart_parts; @@ -20,9 +28,3 @@ CREATE INDEX `idx_multipart_parts_db_multipart_upload_id` ON `multipart_parts`(` CREATE INDEX `idx_multipart_parts_part_number` ON `multipart_parts`(`part_number`); CREATE INDEX `idx_multipart_parts_etag` ON `multipart_parts`(`etag`); PRAGMA foreign_keys=on; - --- drop triggers -DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; -DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; -DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs; From b45b80f70b20c16b8216c99b1d7dd138f055c224 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Wed, 21 Feb 2024 13:31:44 +0100 Subject: [PATCH 11/12] autopilot: accumulate churn information into single alert --- alerts/alerts.go | 15 ++++++-- autopilot/alerts.go | 58 +++++++++++------------------ autopilot/churn.go | 63 ++++++++++++++++++++++++++++++++ autopilot/contractor.go | 10 ++++- bus/bus.go | 11 +++++- bus/client/alerts.go | 4 +- internal/testing/cluster_test.go | 6 +-- 7 files changed, 117 insertions(+), 50 deletions(-) create mode 100644 autopilot/churn.go diff --git a/alerts/alerts.go b/alerts/alerts.go index b0d4963c6..f11004dbe 100644 --- a/alerts/alerts.go +++ b/alerts/alerts.go @@ -35,6 +35,7 @@ const ( type ( Alerter interface { + Alerts(_ context.Context, opts AlertsOpts) (resp AlertsResponse, err error) RegisterAlert(_ context.Context, a Alert) error DismissAlerts(_ context.Context, ids ...types.Hash256) error } @@ -169,17 +170,18 @@ func (m *Manager) DismissAlerts(ctx context.Context, ids ...types.Hash256) error }) } -// Active returns the host's active alerts. -func (m *Manager) Active(offset, limit int) AlertsResponse { +// Alerts returns the host's active alerts. +func (m *Manager) Alerts(_ context.Context, opts AlertsOpts) (AlertsResponse, error) { m.mu.Lock() defer m.mu.Unlock() + offset, limit := opts.Offset, opts.Limit resp := AlertsResponse{ Total: len(m.alerts), } if offset >= len(m.alerts) { - return resp + return resp, nil } else if limit == -1 { limit = len(m.alerts) } @@ -197,7 +199,7 @@ func (m *Manager) Active(offset, limit int) AlertsResponse { resp.HasMore = true } resp.Alerts = alerts - return resp + return resp, nil } func (m *Manager) RegisterWebhookBroadcaster(b webhooks.Broadcaster) { @@ -231,6 +233,11 @@ func WithOrigin(alerter Alerter, origin string) Alerter { } } +// Alerts implements the Alerter interface. +func (a *originAlerter) Alerts(ctx context.Context, opts AlertsOpts) (resp AlertsResponse, err error) { + return a.alerter.Alerts(ctx, opts) +} + // RegisterAlert implements the Alerter interface. func (a *originAlerter) RegisterAlert(ctx context.Context, alert Alert) error { if alert.Data == nil { diff --git a/autopilot/alerts.go b/autopilot/alerts.go index 292670dc5..f4762c4d4 100644 --- a/autopilot/alerts.go +++ b/autopilot/alerts.go @@ -14,12 +14,13 @@ import ( ) var ( - alertAccountRefillID = frand.Entropy256() // constant until restarted - alertLostSectorsID = frand.Entropy256() // constant until restarted - alertLowBalanceID = frand.Entropy256() // constant until restarted - alertMigrationID = frand.Entropy256() // constant until restarted - alertPruningID = frand.Entropy256() // constant until restarted - alertRenewalFailedID = frand.Entropy256() // constant until restarted + alertAccountRefillID = randomAlertID() // constant until restarted + alertChurnID = randomAlertID() // constant until restarted + alertLostSectorsID = randomAlertID() // constant until restarted + alertLowBalanceID = randomAlertID() // constant until restarted + alertMigrationID = randomAlertID() // constant until restarted + alertPruningID = randomAlertID() // constant until restarted + alertRenewalFailedID = randomAlertID() // constant until restarted ) func alertIDForAccount(alertID [32]byte, id rhpv3.Account) types.Hash256 { @@ -54,6 +55,20 @@ func (ap *Autopilot) DismissAlert(ctx context.Context, ids ...types.Hash256) { } } +func (ap *Autopilot) HasAlert(ctx context.Context, id types.Hash256) bool { + ar, err := ap.alerts.Alerts(ctx, alerts.AlertsOpts{Offset: 0, Limit: -1}) + if err != nil { + ap.logger.Errorf("failed to fetch alerts: %v", err) + return false + } + for _, alert := range ar.Alerts { + if alert.ID == id { + return true + } + } + return false +} + func newAccountLowBalanceAlert(address types.Address, balance, allowance types.Currency, bh, renewWindow, endHeight uint64) alerts.Alert { severity := alerts.SeverityInfo if bh+renewWindow/2 >= endHeight { @@ -137,37 +152,6 @@ func newContractPruningFailedAlert(hk types.PublicKey, version string, fcid type } } -func newContractSetChangeAlert(name string, additions map[types.FileContractID]contractSetAddition, removals map[types.FileContractID]contractSetRemoval) alerts.Alert { - var hint string - if len(removals) > 0 { - hint = "A high churn rate can lead to a lot of unnecessary migrations, it might be necessary to tweak your configuration depending on the reason hosts are being discarded from the set." - } - - removedReasons := make(map[string]string, len(removals)) - for k, v := range removals { - removedReasons[k.String()] = v.Reason - } - - return alerts.Alert{ - ID: randomAlertID(), - Severity: alerts.SeverityInfo, - Message: "Contract set changed", - Data: map[string]any{ - "name": name, - "set_additions": additions, - "set_removals": removals, - "hint": hint, - - // TODO: these fields can be removed on the next major release, they - // contain redundant information - "added": len(additions), - "removed": len(removals), - "removals": removedReasons, - }, - Timestamp: time.Now(), - } -} - func newLostSectorsAlert(hk types.PublicKey, lostSectors uint64) alerts.Alert { return alerts.Alert{ ID: alertIDForHost(alertLostSectorsID, hk), diff --git a/autopilot/churn.go b/autopilot/churn.go new file mode 100644 index 000000000..70c4651c2 --- /dev/null +++ b/autopilot/churn.go @@ -0,0 +1,63 @@ +package autopilot + +import ( + "time" + + "go.sia.tech/core/types" + "go.sia.tech/renterd/alerts" +) + +type ( + accumulatedChurn struct { + additions map[types.FileContractID][]contractSetAddition + removals map[types.FileContractID][]contractSetRemoval + } +) + +func newAccumulatedChurn() *accumulatedChurn { + return &accumulatedChurn{ + additions: make(map[types.FileContractID][]contractSetAddition), + removals: make(map[types.FileContractID][]contractSetRemoval), + } +} + +func (c *accumulatedChurn) Alert(name string) alerts.Alert { + var hint string + if len(c.removals) > 0 { + hint = "A high churn rate can lead to a lot of unnecessary migrations, it might be necessary to tweak your configuration depending on the reason hosts are being discarded from the set." + } + + removedReasons := make(map[string][]string, len(c.removals)) + for fcid, contractRemovals := range c.removals { + for _, removal := range contractRemovals { + removedReasons[fcid.String()] = append(removedReasons[fcid.String()], removal.Reason) + } + } + + return alerts.Alert{ + ID: alertChurnID, + Severity: alerts.SeverityInfo, + Message: "Contract set changed", + Data: map[string]any{ + "name": name, + "set_additions": c.additions, + "set_removals": c.removals, + "hint": hint, + }, + Timestamp: time.Now(), + } +} + +func (c *accumulatedChurn) Apply(additions map[types.FileContractID]contractSetAddition, removals map[types.FileContractID]contractSetRemoval) { + for fcid, addition := range additions { + c.additions[fcid] = append(c.additions[fcid], addition) + } + for fcid, removal := range removals { + c.removals[fcid] = append(c.removals[fcid], removal) + } +} + +func (c *accumulatedChurn) Reset() { + c.additions = make(map[types.FileContractID][]contractSetAddition) + c.removals = make(map[types.FileContractID][]contractSetRemoval) +} diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 092f2a831..7909277f0 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -85,6 +85,7 @@ const ( type ( contractor struct { ap *Autopilot + churn *accumulatedChurn resolver *ipResolver logger *zap.SugaredLogger @@ -130,7 +131,7 @@ type ( contractSetRemoval struct { Size uint64 `json:"size"` HostKey types.PublicKey `json:"hostKey"` - Reason string `json:"reason"` + Reason string `json:"reasons"` } renewal struct { @@ -143,6 +144,7 @@ type ( func newContractor(ap *Autopilot, revisionSubmissionBuffer uint64, revisionBroadcastInterval time.Duration) *contractor { return &contractor{ ap: ap, + churn: newAccumulatedChurn(), logger: ap.logger.Named("contractor"), revisionBroadcastInterval: revisionBroadcastInterval, @@ -536,7 +538,11 @@ func (c *contractor) computeContractSetChanged(ctx context.Context, name string, ) hasChanged := len(setAdditions)+len(setRemovals) > 0 if hasChanged { - c.ap.RegisterAlert(ctx, newContractSetChangeAlert(name, setAdditions, setRemovals)) + if !c.ap.HasAlert(ctx, alertChurnID) { + c.churn.Reset() + } + c.churn.Apply(setAdditions, setRemovals) + c.ap.RegisterAlert(ctx, c.churn.Alert(name)) } return hasChanged } diff --git a/bus/bus.go b/bus/bus.go index 9ee6e1ba2..e7e6ddaac 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -1726,7 +1726,10 @@ func (b *bus) gougingParams(ctx context.Context) (api.GougingParams, error) { } func (b *bus) handleGETAlertsDeprecated(jc jape.Context) { - ar := b.alertMgr.Active(0, -1) + ar, err := b.alertMgr.Alerts(jc.Request.Context(), alerts.AlertsOpts{Offset: 0, Limit: -1}) + if jc.Check("failed to fetch alerts", err) != nil { + return + } jc.Encode(ar.Alerts) } @@ -1744,7 +1747,11 @@ func (b *bus) handleGETAlerts(jc jape.Context) { jc.Error(errors.New("offset must be non-negative"), http.StatusBadRequest) return } - jc.Encode(b.alertMgr.Active(offset, limit)) + ar, err := b.alertMgr.Alerts(jc.Request.Context(), alerts.AlertsOpts{Offset: offset, Limit: limit}) + if jc.Check("failed to fetch alerts", err) != nil { + return + } + jc.Encode(ar) } func (b *bus) handlePOSTAlertsDismiss(jc jape.Context) { diff --git a/bus/client/alerts.go b/bus/client/alerts.go index 7f2bf9aa7..7eceaeaed 100644 --- a/bus/client/alerts.go +++ b/bus/client/alerts.go @@ -10,13 +10,13 @@ import ( ) // Alerts fetches the active alerts from the bus. -func (c *Client) Alerts(opts alerts.AlertsOpts) (resp alerts.AlertsResponse, err error) { +func (c *Client) Alerts(ctx context.Context, opts alerts.AlertsOpts) (resp alerts.AlertsResponse, err error) { values := url.Values{} values.Set("offset", fmt.Sprint(opts.Offset)) if opts.Limit != 0 { values.Set("limit", fmt.Sprint(opts.Limit)) } - err = c.c.GET("/alerts?"+values.Encode(), &resp) + err = c.c.WithContext(ctx).GET("/alerts?"+values.Encode(), &resp) return } diff --git a/internal/testing/cluster_test.go b/internal/testing/cluster_test.go index 4fb62ff31..f30a0906a 100644 --- a/internal/testing/cluster_test.go +++ b/internal/testing/cluster_test.go @@ -1923,7 +1923,7 @@ func TestAlerts(t *testing.T) { tt.OK(b.RegisterAlert(context.Background(), alert)) findAlert := func(id types.Hash256) *alerts.Alert { t.Helper() - ar, err := b.Alerts(alerts.AlertsOpts{}) + ar, err := b.Alerts(context.Background(), alerts.AlertsOpts{}) tt.OK(err) for _, alert := range ar.Alerts { if alert.ID == id { @@ -1960,7 +1960,7 @@ func TestAlerts(t *testing.T) { } // try to find with offset = 1 - ar, err := b.Alerts(alerts.AlertsOpts{Offset: 1}) + ar, err := b.Alerts(context.Background(), alerts.AlertsOpts{Offset: 1}) foundAlerts := ar.Alerts tt.OK(err) if len(foundAlerts) != 1 || foundAlerts[0].ID != alert.ID { @@ -1968,7 +1968,7 @@ func TestAlerts(t *testing.T) { } // try to find with limit = 1 - ar, err = b.Alerts(alerts.AlertsOpts{Limit: 1}) + ar, err = b.Alerts(context.Background(), alerts.AlertsOpts{Limit: 1}) foundAlerts = ar.Alerts tt.OK(err) if len(foundAlerts) != 1 || foundAlerts[0].ID != alert2.ID { From c17252a8f250cc0184918ec93dab5e7adeb31791 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 23 Feb 2024 13:14:30 +0100 Subject: [PATCH 12/12] autopilot: update churn alert to contain timestamp --- autopilot/churn.go | 41 +++++++++++++++++------------- autopilot/contractor.go | 56 +++++++++++++++++++++++++++++------------ 2 files changed, 63 insertions(+), 34 deletions(-) diff --git a/autopilot/churn.go b/autopilot/churn.go index 70c4651c2..fdc1a0f54 100644 --- a/autopilot/churn.go +++ b/autopilot/churn.go @@ -9,15 +9,15 @@ import ( type ( accumulatedChurn struct { - additions map[types.FileContractID][]contractSetAddition - removals map[types.FileContractID][]contractSetRemoval + additions map[types.FileContractID]contractSetAdditions + removals map[types.FileContractID]contractSetRemovals } ) func newAccumulatedChurn() *accumulatedChurn { return &accumulatedChurn{ - additions: make(map[types.FileContractID][]contractSetAddition), - removals: make(map[types.FileContractID][]contractSetRemoval), + additions: make(map[types.FileContractID]contractSetAdditions), + removals: make(map[types.FileContractID]contractSetRemovals), } } @@ -27,13 +27,6 @@ func (c *accumulatedChurn) Alert(name string) alerts.Alert { hint = "A high churn rate can lead to a lot of unnecessary migrations, it might be necessary to tweak your configuration depending on the reason hosts are being discarded from the set." } - removedReasons := make(map[string][]string, len(c.removals)) - for fcid, contractRemovals := range c.removals { - for _, removal := range contractRemovals { - removedReasons[fcid.String()] = append(removedReasons[fcid.String()], removal.Reason) - } - } - return alerts.Alert{ ID: alertChurnID, Severity: alerts.SeverityInfo, @@ -48,16 +41,28 @@ func (c *accumulatedChurn) Alert(name string) alerts.Alert { } } -func (c *accumulatedChurn) Apply(additions map[types.FileContractID]contractSetAddition, removals map[types.FileContractID]contractSetRemoval) { - for fcid, addition := range additions { - c.additions[fcid] = append(c.additions[fcid], addition) +func (c *accumulatedChurn) Apply(additions map[types.FileContractID]contractSetAdditions, removals map[types.FileContractID]contractSetRemovals) { + for fcid, a := range additions { + if _, exists := c.additions[fcid]; !exists { + c.additions[fcid] = a + } else { + additions := c.additions[fcid] + additions.Additions = append(additions.Additions, a.Additions...) + c.additions[fcid] = additions + } } - for fcid, removal := range removals { - c.removals[fcid] = append(c.removals[fcid], removal) + for fcid, r := range removals { + if _, exists := c.removals[fcid]; !exists { + c.removals[fcid] = r + } else { + removals := c.removals[fcid] + removals.Removals = append(removals.Removals, r.Removals...) + c.removals[fcid] = removals + } } } func (c *accumulatedChurn) Reset() { - c.additions = make(map[types.FileContractID][]contractSetAddition) - c.removals = make(map[types.FileContractID][]contractSetRemoval) + c.additions = make(map[types.FileContractID]contractSetAdditions) + c.removals = make(map[types.FileContractID]contractSetRemovals) } diff --git a/autopilot/contractor.go b/autopilot/contractor.go index 7909277f0..9e2b52cca 100644 --- a/autopilot/contractor.go +++ b/autopilot/contractor.go @@ -123,15 +123,25 @@ type ( recoverable bool } + contractSetAdditions struct { + HostKey types.PublicKey `json:"hostKey"` + Additions []contractSetAddition `json:"additions"` + } + contractSetAddition struct { - Size uint64 `json:"size"` - HostKey types.PublicKey `json:"hostKey"` + Size uint64 `json:"size"` + Time api.TimeRFC3339 `json:"time"` + } + + contractSetRemovals struct { + HostKey types.PublicKey `json:"hostKey"` + Removals []contractSetRemoval `json:"removals"` } contractSetRemoval struct { - Size uint64 `json:"size"` - HostKey types.PublicKey `json:"hostKey"` - Reason string `json:"reasons"` + Size uint64 `json:"size"` + Reason string `json:"reasons"` + Time api.TimeRFC3339 `json:"time"` } renewal struct { @@ -455,8 +465,9 @@ func (c *contractor) computeContractSetChanged(ctx context.Context, name string, } // log added and removed contracts - setAdditions := make(map[types.FileContractID]contractSetAddition) - setRemovals := make(map[types.FileContractID]contractSetRemoval) + setAdditions := make(map[types.FileContractID]contractSetAdditions) + setRemovals := make(map[types.FileContractID]contractSetRemovals) + now := api.TimeNow() for _, contract := range oldSet { _, exists := inNewSet[contract.ID] _, renewed := inNewSet[renewalsFromTo[contract.ID]] @@ -466,11 +477,18 @@ func (c *contractor) computeContractSetChanged(ctx context.Context, name string, reason = "unknown" } - setRemovals[contract.ID] = contractSetRemoval{ - Size: contractData[contract.ID], - HostKey: contract.HostKey, - Reason: reason, + if _, exists := setRemovals[contract.ID]; !exists { + setRemovals[contract.ID] = contractSetRemovals{ + HostKey: contract.HostKey, + } } + removals := setRemovals[contract.ID] + removals.Removals = append(removals.Removals, contractSetRemoval{ + Size: contractData[contract.ID], + Reason: reason, + Time: now, + }) + setRemovals[contract.ID] = removals c.logger.Debugf("contract %v was removed from the contract set, size: %v, reason: %v", contract.ID, contractData[contract.ID], reason) } } @@ -478,10 +496,17 @@ func (c *contractor) computeContractSetChanged(ctx context.Context, name string, _, existed := inOldSet[contract.ID] _, renewed := renewalsToFrom[contract.ID] if !existed && !renewed { - setAdditions[contract.ID] = contractSetAddition{ - Size: contractData[contract.ID], - HostKey: contract.HostKey, + if _, exists := setAdditions[contract.ID]; !exists { + setAdditions[contract.ID] = contractSetAdditions{ + HostKey: contract.HostKey, + } } + additions := setAdditions[contract.ID] + additions.Additions = append(additions.Additions, contractSetAddition{ + Size: contractData[contract.ID], + Time: now, + }) + setAdditions[contract.ID] = additions c.logger.Debugf("contract %v was added to the contract set, size: %v", contract.ID, contractData[contract.ID]) } } @@ -501,7 +526,6 @@ func (c *contractor) computeContractSetChanged(ctx context.Context, name string, } // record churn metrics - now := api.TimeNow() var metrics []api.ContractSetChurnMetric for fcid := range setAdditions { metrics = append(metrics, api.ContractSetChurnMetric{ @@ -516,7 +540,7 @@ func (c *contractor) computeContractSetChanged(ctx context.Context, name string, Name: c.ap.state.cfg.Contracts.Set, ContractID: fcid, Direction: api.ChurnDirRemoved, - Reason: removal.Reason, + Reason: removal.Removals[0].Reason, Timestamp: now, }) }