From 9bb5972586e6fc26ad0c21758b0c265f8bab76a2 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 09:55:13 +0100 Subject: [PATCH 1/9] stores: migration code --- .../migration_00004_prune_slabs_cascade.sql | 24 +++++++++++ stores/migrations/mysql/main/schema.sql | 40 ++---------------- .../migration_00004_prune_slabs_cascade.sql | 28 +++++++++++++ stores/migrations/sqlite/main/schema.sql | 42 +------------------ 4 files changed, 57 insertions(+), 77 deletions(-) create mode 100644 stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql create mode 100644 stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql diff --git a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql new file mode 100644 index 000000000..48f98a40a --- /dev/null +++ b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql @@ -0,0 +1,24 @@ +-- prune manually before creating trigger +DELETE slabs +FROM slabs +LEFT JOIN slices ON slices.db_slab_id = slabs.id +WHERE slices.db_object_id IS NULL +AND slices.db_multipart_part_id IS NULL +AND slabs.db_buffered_slab_id IS NULL; + +-- add ON DELETE CASCADE to slices +ALTER TABLE slices DROP FOREIGN KEY fk_objects_slabs; +ALTER TABLE slices ADD CONSTRAINT fk_objects_slabs FOREIGN KEY (db_object_id) REFERENCES objects (id) ON DELETE CASCADE, + +ALTER TABLE slices DROP FOREIGN KEY fk_multipart_parts_slabs; +ALTER TABLE slices ADD CONSTRAINT fk_multipart_parts_slabs FOREIGN KEY (db_multipart_part_id) REFERENCES multipart_parts (id) ON DELETE CASCADE, + +-- add ON DELETE CASCADE to multipart_parts +ALTER TABLE multipart_parts DROP FOREIGN KEY fk_multipart_uploads_parts; +ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE + +-- drop triggers +DROP TRIGGER before_delete_on_objects_delete_slices +DROP TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts +DROP TRIGGER before_delete_on_multipart_parts_delete_slices +DROP TRIGGER after_delete_on_slices_delete_slabs \ No newline at end of file diff --git a/stores/migrations/mysql/main/schema.sql b/stores/migrations/mysql/main/schema.sql index 1b39e4669..a5ed86807 100644 --- a/stores/migrations/mysql/main/schema.sql +++ b/stores/migrations/mysql/main/schema.sql @@ -310,7 +310,7 @@ CREATE TABLE `multipart_parts` ( KEY `idx_multipart_parts_etag` (`etag`), KEY `idx_multipart_parts_part_number` (`part_number`), KEY `idx_multipart_parts_db_multipart_upload_id` (`db_multipart_upload_id`), - CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) + CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; -- dbObject @@ -374,8 +374,8 @@ CREATE TABLE `slices` ( KEY `idx_slices_object_index` (`object_index`), KEY `idx_slices_db_multipart_part_id` (`db_multipart_part_id`), KEY `idx_slices_db_slab_id` (`db_slab_id`), - CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts` (`id`), - CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects` (`id`), + CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects` (`id`) ON DELETE CASCADE, CONSTRAINT `fk_slabs_slices` FOREIGN KEY (`db_slab_id`) REFERENCES `slabs` (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; @@ -421,39 +421,5 @@ CREATE TABLE `object_user_metadata` ( CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; --- dbObject trigger to delete from slices -CREATE TRIGGER before_delete_on_objects_delete_slices -BEFORE DELETE -ON objects FOR EACH ROW -DELETE FROM slices -WHERE slices.db_object_id = OLD.id; - --- dbMultipartUpload trigger to delete from dbMultipartPart -CREATE TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts -BEFORE DELETE -ON multipart_uploads FOR EACH ROW -DELETE FROM multipart_parts -WHERE multipart_parts.db_multipart_upload_id = OLD.id; - --- dbMultipartPart trigger to delete from slices -CREATE TRIGGER before_delete_on_multipart_parts_delete_slices -BEFORE DELETE -ON multipart_parts FOR EACH ROW -DELETE FROM slices -WHERE slices.db_multipart_part_id = OLD.id; - --- dbSlices trigger to prune slabs -CREATE TRIGGER after_delete_on_slices_delete_slabs -AFTER DELETE -ON slices FOR EACH ROW -DELETE FROM slabs -WHERE slabs.id = OLD.db_slab_id -AND slabs.db_buffered_slab_id IS NULL -AND NOT EXISTS ( - SELECT 1 - FROM slices - WHERE slices.db_slab_id = OLD.db_slab_id -); - -- create default bucket INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); \ No newline at end of file diff --git a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql new file mode 100644 index 000000000..2cd633c11 --- /dev/null +++ b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql @@ -0,0 +1,28 @@ +PRAGMA foreign_keys=off; +-- update constraints on slices +CREATE TABLE `slices_temp` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer,`object_index` integer,`db_multipart_part_id` integer,`db_slab_id` integer,`offset` integer,`length` integer,CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_slabs_slices` FOREIGN KEY (`db_slab_id`) REFERENCES `slabs`(`id`)); +INSERT INTO slices_temp SELECT `id`, `created_at`, `db_object_id`, `object_index`, `db_multipart_part_id`, `db_slab_id`, `offset`, `length` FROM slices; +DROP TABLE slices; +ALTER TABLE slices_temp RENAME TO slices; + +CREATE INDEX `idx_slices_object_index` ON `slices`(`object_index`); +CREATE INDEX `idx_slices_db_object_id` ON `slices`(`db_object_id`); +CREATE INDEX `idx_slices_db_slab_id` ON `slices`(`db_slab_id`); +CREATE INDEX `idx_slices_db_multipart_part_id` ON `slices`(`db_multipart_part_id`); + +-- update constraints multipart_parts +CREATE TABLE `multipart_parts_temp` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`etag` text,`part_number` integer,`size` integer,`db_multipart_upload_id` integer NOT NULL,CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads`(`id`) ON DELETE CASCADE); +INSERT INTO multipart_parts_temp SELECT * FROM multipart_parts; +DROP TABLE multipart_parts; +ALTER TABLE multipart_parts_temp RENAME TO multipart_parts; + +CREATE INDEX `idx_multipart_parts_db_multipart_upload_id` ON `multipart_parts`(`db_multipart_upload_id`); +CREATE INDEX `idx_multipart_parts_part_number` ON `multipart_parts`(`part_number`); +CREATE INDEX `idx_multipart_parts_etag` ON `multipart_parts`(`etag`); +PRAGMA foreign_keys=on; + +-- drop triggers +DROP TRIGGER before_delete_on_objects_delete_slices +DROP TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts +DROP TRIGGER before_delete_on_multipart_parts_delete_slices +DROP TRIGGER after_delete_on_slices_delete_slabs \ No newline at end of file diff --git a/stores/migrations/sqlite/main/schema.sql b/stores/migrations/sqlite/main/schema.sql index e6bb2546d..8d7afeaa1 100644 --- a/stores/migrations/sqlite/main/schema.sql +++ b/stores/migrations/sqlite/main/schema.sql @@ -85,13 +85,13 @@ CREATE INDEX `idx_contract_sectors_db_contract_id` ON `contract_sectors`(`db_con CREATE INDEX `idx_contract_sectors_db_sector_id` ON `contract_sectors`(`db_sector_id`); -- dbMultipartPart -CREATE TABLE `multipart_parts` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`etag` text,`part_number` integer,`size` integer,`db_multipart_upload_id` integer NOT NULL,CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads`(`id`)); +CREATE TABLE `multipart_parts` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`etag` text,`part_number` integer,`size` integer,`db_multipart_upload_id` integer NOT NULL,CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads`(`id`) ON DELETE CASCADE); CREATE INDEX `idx_multipart_parts_db_multipart_upload_id` ON `multipart_parts`(`db_multipart_upload_id`); CREATE INDEX `idx_multipart_parts_part_number` ON `multipart_parts`(`part_number`); CREATE INDEX `idx_multipart_parts_etag` ON `multipart_parts`(`etag`); -- dbSlice -CREATE TABLE `slices` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer,`object_index` integer,`db_multipart_part_id` integer,`db_slab_id` integer,`offset` integer,`length` integer,CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects`(`id`),CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts`(`id`),CONSTRAINT `fk_slabs_slices` FOREIGN KEY (`db_slab_id`) REFERENCES `slabs`(`id`)); +CREATE TABLE `slices` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer,`object_index` integer,`db_multipart_part_id` integer,`db_slab_id` integer,`offset` integer,`length` integer,CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_slabs_slices` FOREIGN KEY (`db_slab_id`) REFERENCES `slabs`(`id`)); CREATE INDEX `idx_slices_object_index` ON `slices`(`object_index`); CREATE INDEX `idx_slices_db_object_id` ON `slices`(`db_object_id`); CREATE INDEX `idx_slices_db_slab_id` ON `slices`(`db_slab_id`); @@ -148,43 +148,5 @@ CREATE UNIQUE INDEX `idx_module_event_url` ON `webhooks`(`module`,`event`,`url`) CREATE TABLE `object_user_metadata` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer DEFAULT NULL,`db_multipart_upload_id` integer DEFAULT NULL,`key` text NOT NULL,`value` text, CONSTRAINT `fk_object_user_metadata` FOREIGN KEY (`db_object_id`) REFERENCES `objects` (`id`) ON DELETE CASCADE, CONSTRAINT `fk_multipart_upload_user_metadata` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads` (`id`) ON DELETE SET NULL); CREATE UNIQUE INDEX `idx_object_user_metadata_key` ON `object_user_metadata`(`db_object_id`,`db_multipart_upload_id`,`key`); --- dbObject trigger to delete from slices -CREATE TRIGGER before_delete_on_objects_delete_slices -BEFORE DELETE ON objects -BEGIN - DELETE FROM slices - WHERE slices.db_object_id = OLD.id; -END; - --- dbMultipartUpload trigger to delete from dbMultipartPart -CREATE TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts -BEFORE DELETE ON multipart_uploads -BEGIN - DELETE FROM multipart_parts - WHERE multipart_parts.db_multipart_upload_id = OLD.id; -END; - --- dbMultipartPart trigger to delete from slices -CREATE TRIGGER before_delete_on_multipart_parts_delete_slices -BEFORE DELETE ON multipart_parts -BEGIN - DELETE FROM slices - WHERE slices.db_multipart_part_id = OLD.id; -END; - --- dbSlices trigger to prune slabs -CREATE TRIGGER after_delete_on_slices_delete_slabs -AFTER DELETE ON slices -BEGIN - DELETE FROM slabs - WHERE slabs.id = OLD.db_slab_id - AND slabs.db_buffered_slab_id IS NULL - AND NOT EXISTS ( - SELECT 1 - FROM slices - WHERE slices.db_slab_id = OLD.db_slab_id - ); -END; - -- create default bucket INSERT INTO buckets (created_at, name) VALUES (CURRENT_TIMESTAMP, 'default'); From 1f004223a11b0c4897438db98584d81460365e7e Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 10:06:21 +0100 Subject: [PATCH 2/9] stores: call pruneSlabs every time an object or multipart object were deleted --- stores/metadata.go | 23 ++++++++++++++++++++--- stores/multipart.go | 9 +++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/stores/metadata.go b/stores/metadata.go index 68947ed95..a9636bf75 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -2685,20 +2685,32 @@ func archiveContracts(ctx context.Context, tx *gorm.DB, contracts []dbContract, return nil } +func pruneSlabs(tx *gorm.DB) error { + // delete slabs without any associated slices or buffers + return tx.Exec(` +DELETE +FROM slabs sla +WHERE NOT EXISTS (SELECT 1 FROM slices sli WHERE sli.db_slab_id = sla.id) +AND sla.db_buffered_slab_id IS NULL +`).Error +} + // deleteObject deletes an object from the store and prunes all slabs which are // without an obect after the deletion. That means in case of packed uploads, // the slab is only deleted when no more objects point to it. -func (s *SQLStore) deleteObject(tx *gorm.DB, bucket string, path string) (numDeleted int64, _ error) { +func (s *SQLStore) deleteObject(tx *gorm.DB, bucket string, path string) (int64, error) { tx = tx.Where("object_id = ? AND ?", path, sqlWhereBucket("objects", bucket)). Delete(&dbObject{}) if tx.Error != nil { return 0, tx.Error } - numDeleted = tx.RowsAffected + numDeleted := tx.RowsAffected if numDeleted == 0 { return 0, nil // nothing to prune if no object was deleted + } else if err := pruneSlabs(tx); err != nil { + return 0, err } - return + return numDeleted, nil } // deleteObjects deletes a batch of objects from the database. The order of @@ -2729,6 +2741,11 @@ func (s *SQLStore) deleteObjects(bucket string, path string) (numDeleted int64, } duration = time.Since(start) rowsAffected = res.RowsAffected + + // prune slabs if we deleted an object + if rowsAffected > 0 { + return pruneSlabs(tx) + } return nil }); err != nil { return 0, fmt.Errorf("failed to delete objects: %w", err) diff --git a/stores/multipart.go b/stores/multipart.go index 18706ed0c..3a5bcd54a 100644 --- a/stores/multipart.go +++ b/stores/multipart.go @@ -295,6 +295,10 @@ func (s *SQLStore) AbortMultipartUpload(ctx context.Context, bucket, path string if err != nil { return fmt.Errorf("failed to delete multipart upload: %w", err) } + // Prune the slabs. + if err := pruneSlabs(tx); err != nil { + return fmt.Errorf("failed to prune slabs: %w", err) + } return nil }) } @@ -435,6 +439,11 @@ func (s *SQLStore) CompleteMultipartUpload(ctx context.Context, bucket, path str if err := tx.Delete(&mu).Error; err != nil { return fmt.Errorf("failed to delete multipart upload: %w", err) } + + // Prune the slabs. + if err := pruneSlabs(tx); err != nil { + return fmt.Errorf("failed to prune slabs: %w", err) + } return nil }) if err != nil { From 671f8da872b0decaf217bd4e4d0119d974b1160d Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 10:19:05 +0100 Subject: [PATCH 3/9] stores: add migration to performMigrations --- stores/metadata.go | 6 +++--- stores/migrations.go | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/stores/metadata.go b/stores/metadata.go index a9636bf75..13f274477 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -2689,9 +2689,9 @@ func pruneSlabs(tx *gorm.DB) error { // delete slabs without any associated slices or buffers return tx.Exec(` DELETE -FROM slabs sla -WHERE NOT EXISTS (SELECT 1 FROM slices sli WHERE sli.db_slab_id = sla.id) -AND sla.db_buffered_slab_id IS NULL +FROM slabs +WHERE NOT EXISTS (SELECT 1 FROM slices WHERE slices.db_slab_id = slabs.id) +AND slabs.db_buffered_slab_id IS NULL `).Error } diff --git a/stores/migrations.go b/stores/migrations.go index bf3916ca4..d89be7ab5 100644 --- a/stores/migrations.go +++ b/stores/migrations.go @@ -44,6 +44,12 @@ func performMigrations(db *gorm.DB, logger *zap.SugaredLogger) error { return performMigration(tx, dbIdentifier, "00003_idx_objects_size", logger) }, }, + { + ID: "00004_prune_slabs_cascade", + Migrate: func(tx *gorm.DB) error { + return performMigration(tx, dbIdentifier, "00004_prune_slabs_cascade", logger) + }, + }, } // Create migrator. From 0a23dea48598451ab56cb5b836a4f6a3cf4c8240 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 10:57:09 +0100 Subject: [PATCH 4/9] stores: fix TestSlabCleanupTrigger --- stores/metadata.go | 5 ++--- stores/metadata_test.go | 9 ++++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/stores/metadata.go b/stores/metadata.go index 13f274477..2ab2cf5da 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -2739,13 +2739,12 @@ func (s *SQLStore) deleteObjects(bucket string, path string) (numDeleted int64, if err := res.Error; err != nil { return res.Error } - duration = time.Since(start) - rowsAffected = res.RowsAffected - // prune slabs if we deleted an object + rowsAffected = res.RowsAffected if rowsAffected > 0 { return pruneSlabs(tx) } + duration = time.Since(start) return nil }); err != nil { return 0, fmt.Errorf("failed to delete objects: %w", err) diff --git a/stores/metadata_test.go b/stores/metadata_test.go index 4a6102399..d5a0f9531 100644 --- a/stores/metadata_test.go +++ b/stores/metadata_test.go @@ -3941,7 +3941,8 @@ func TestSlabCleanupTrigger(t *testing.T) { } // delete the object - if err := ss.db.Delete(&obj1).Error; err != nil { + err := ss.RemoveObject(context.Background(), api.DefaultBucketName, obj1.ObjectID) + if err != nil { t.Fatal(err) } @@ -3954,7 +3955,8 @@ func TestSlabCleanupTrigger(t *testing.T) { } // delete second object - if err := ss.db.Delete(&obj2).Error; err != nil { + err = ss.RemoveObject(context.Background(), api.DefaultBucketName, obj2.ObjectID) + if err != nil { t.Fatal(err) } @@ -3998,7 +4000,8 @@ func TestSlabCleanupTrigger(t *testing.T) { } // delete third object - if err := ss.db.Delete(&obj3).Error; err != nil { + err = ss.RemoveObject(context.Background(), api.DefaultBucketName, obj3.ObjectID) + if err != nil { t.Fatal(err) } if err := ss.db.Model(&dbSlab{}).Count(&slabCntr).Error; err != nil { From 96d4bba92f350ffc4490ff6f53011e9c989f7e47 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 13:37:50 +0100 Subject: [PATCH 5/9] stores: only drop triggers if they exist --- .../mysql/main/migration_00004_prune_slabs_cascade.sql | 8 ++++---- .../sqlite/main/migration_00004_prune_slabs_cascade.sql | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql index 48f98a40a..125da0ecb 100644 --- a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql @@ -18,7 +18,7 @@ ALTER TABLE multipart_parts DROP FOREIGN KEY fk_multipart_uploads_parts; ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE -- drop triggers -DROP TRIGGER before_delete_on_objects_delete_slices -DROP TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts -DROP TRIGGER before_delete_on_multipart_parts_delete_slices -DROP TRIGGER after_delete_on_slices_delete_slabs \ No newline at end of file +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs \ No newline at end of file diff --git a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql index 2cd633c11..b7f5ab128 100644 --- a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql @@ -22,7 +22,7 @@ CREATE INDEX `idx_multipart_parts_etag` ON `multipart_parts`(`etag`); PRAGMA foreign_keys=on; -- drop triggers -DROP TRIGGER before_delete_on_objects_delete_slices -DROP TRIGGER before_delete_on_multipart_uploads_delete_multipart_parts -DROP TRIGGER before_delete_on_multipart_parts_delete_slices -DROP TRIGGER after_delete_on_slices_delete_slabs \ No newline at end of file +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs \ No newline at end of file From adafec043860b8514c71e535eddec1e3729bb407 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Tue, 20 Feb 2024 15:24:05 +0100 Subject: [PATCH 6/9] stores: add missing newline --- .../mysql/main/migration_00004_prune_slabs_cascade.sql | 2 +- .../sqlite/main/migration_00004_prune_slabs_cascade.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql index 125da0ecb..0b1c06994 100644 --- a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql @@ -21,4 +21,4 @@ ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KE DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs \ No newline at end of file +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs diff --git a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql index b7f5ab128..1132dd2f5 100644 --- a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql @@ -25,4 +25,4 @@ PRAGMA foreign_keys=on; DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs \ No newline at end of file +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs From 5b968e7e71ccce04f98a3d36f5c17997a2d17907 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 22 Feb 2024 13:49:23 +0100 Subject: [PATCH 7/9] stores: numDeleted --- stores/metadata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stores/metadata.go b/stores/metadata.go index 2ab2cf5da..eaad76c17 100644 --- a/stores/metadata.go +++ b/stores/metadata.go @@ -2708,7 +2708,7 @@ func (s *SQLStore) deleteObject(tx *gorm.DB, bucket string, path string) (int64, if numDeleted == 0 { return 0, nil // nothing to prune if no object was deleted } else if err := pruneSlabs(tx); err != nil { - return 0, err + return numDeleted, err } return numDeleted, nil } From 10e88fd4a00196b467a5b3d3d0ac26564ec861c6 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Thu, 22 Feb 2024 18:15:46 +0100 Subject: [PATCH 8/9] stores: fix mysql migration --- .../migration_00004_prune_slabs_cascade.sql | 22 ++++++------------- .../migration_00004_prune_slabs_cascade.sql | 8 +++---- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql index 0b1c06994..9014582e0 100644 --- a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql @@ -1,24 +1,16 @@ --- prune manually before creating trigger -DELETE slabs -FROM slabs -LEFT JOIN slices ON slices.db_slab_id = slabs.id -WHERE slices.db_object_id IS NULL -AND slices.db_multipart_part_id IS NULL -AND slabs.db_buffered_slab_id IS NULL; - -- add ON DELETE CASCADE to slices ALTER TABLE slices DROP FOREIGN KEY fk_objects_slabs; -ALTER TABLE slices ADD CONSTRAINT fk_objects_slabs FOREIGN KEY (db_object_id) REFERENCES objects (id) ON DELETE CASCADE, +ALTER TABLE slices ADD CONSTRAINT fk_objects_slabs FOREIGN KEY (db_object_id) REFERENCES objects (id) ON DELETE CASCADE; ALTER TABLE slices DROP FOREIGN KEY fk_multipart_parts_slabs; -ALTER TABLE slices ADD CONSTRAINT fk_multipart_parts_slabs FOREIGN KEY (db_multipart_part_id) REFERENCES multipart_parts (id) ON DELETE CASCADE, +ALTER TABLE slices ADD CONSTRAINT fk_multipart_parts_slabs FOREIGN KEY (db_multipart_part_id) REFERENCES multipart_parts (id) ON DELETE CASCADE; -- add ON DELETE CASCADE to multipart_parts ALTER TABLE multipart_parts DROP FOREIGN KEY fk_multipart_uploads_parts; -ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE +ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE; -- drop triggers -DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices -DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts -DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs; diff --git a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql index 1132dd2f5..38cd40199 100644 --- a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql @@ -22,7 +22,7 @@ CREATE INDEX `idx_multipart_parts_etag` ON `multipart_parts`(`etag`); PRAGMA foreign_keys=on; -- drop triggers -DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices -DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts -DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs; From 99014c79fe65c3bffa5c737a768d3905351c7517 Mon Sep 17 00:00:00 2001 From: Chris Schinnerl Date: Fri, 23 Feb 2024 09:47:48 +0100 Subject: [PATCH 9/9] stores: drop indices first --- .../main/migration_00004_prune_slabs_cascade.sql | 14 +++++++------- .../main/migration_00004_prune_slabs_cascade.sql | 14 ++++++++------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql index 9014582e0..c2efe3467 100644 --- a/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/mysql/main/migration_00004_prune_slabs_cascade.sql @@ -1,3 +1,9 @@ +-- drop triggers +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs; + -- add ON DELETE CASCADE to slices ALTER TABLE slices DROP FOREIGN KEY fk_objects_slabs; ALTER TABLE slices ADD CONSTRAINT fk_objects_slabs FOREIGN KEY (db_object_id) REFERENCES objects (id) ON DELETE CASCADE; @@ -7,10 +13,4 @@ ALTER TABLE slices ADD CONSTRAINT fk_multipart_parts_slabs FOREIGN KEY (db_multi -- add ON DELETE CASCADE to multipart_parts ALTER TABLE multipart_parts DROP FOREIGN KEY fk_multipart_uploads_parts; -ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE; - --- drop triggers -DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; -DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; -DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs; +ALTER TABLE multipart_parts ADD CONSTRAINT fk_multipart_uploads_parts FOREIGN KEY (db_multipart_upload_id) REFERENCES multipart_uploads (id) ON DELETE CASCADE; \ No newline at end of file diff --git a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql index 38cd40199..03f006acd 100644 --- a/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql +++ b/stores/migrations/sqlite/main/migration_00004_prune_slabs_cascade.sql @@ -1,5 +1,12 @@ +-- drop triggers +DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; +DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; +DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; +DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs; + PRAGMA foreign_keys=off; -- update constraints on slices +DROP TABLE IF EXISTS slices_temp; CREATE TABLE `slices_temp` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`db_object_id` integer,`object_index` integer,`db_multipart_part_id` integer,`db_slab_id` integer,`offset` integer,`length` integer,CONSTRAINT `fk_objects_slabs` FOREIGN KEY (`db_object_id`) REFERENCES `objects`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_multipart_parts_slabs` FOREIGN KEY (`db_multipart_part_id`) REFERENCES `multipart_parts`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_slabs_slices` FOREIGN KEY (`db_slab_id`) REFERENCES `slabs`(`id`)); INSERT INTO slices_temp SELECT `id`, `created_at`, `db_object_id`, `object_index`, `db_multipart_part_id`, `db_slab_id`, `offset`, `length` FROM slices; DROP TABLE slices; @@ -11,6 +18,7 @@ CREATE INDEX `idx_slices_db_slab_id` ON `slices`(`db_slab_id`); CREATE INDEX `idx_slices_db_multipart_part_id` ON `slices`(`db_multipart_part_id`); -- update constraints multipart_parts +DROP TABLE IF EXISTS multipart_parts_temp; CREATE TABLE `multipart_parts_temp` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`etag` text,`part_number` integer,`size` integer,`db_multipart_upload_id` integer NOT NULL,CONSTRAINT `fk_multipart_uploads_parts` FOREIGN KEY (`db_multipart_upload_id`) REFERENCES `multipart_uploads`(`id`) ON DELETE CASCADE); INSERT INTO multipart_parts_temp SELECT * FROM multipart_parts; DROP TABLE multipart_parts; @@ -20,9 +28,3 @@ CREATE INDEX `idx_multipart_parts_db_multipart_upload_id` ON `multipart_parts`(` CREATE INDEX `idx_multipart_parts_part_number` ON `multipart_parts`(`part_number`); CREATE INDEX `idx_multipart_parts_etag` ON `multipart_parts`(`etag`); PRAGMA foreign_keys=on; - --- drop triggers -DROP TRIGGER IF EXISTS before_delete_on_objects_delete_slices; -DROP TRIGGER IF EXISTS before_delete_on_multipart_uploads_delete_multipart_parts; -DROP TRIGGER IF EXISTS before_delete_on_multipart_parts_delete_slices; -DROP TRIGGER IF EXISTS after_delete_on_slices_delete_slabs;