Complete removal of uploaded file references for external source

NASA-AMMOS · Aug 5, 2024 · c2afd3d · c2afd3d
1 parent 66f6235
commit c2afd3d
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 42 deletions.
diff --git a/deployment/hasura/metadata/databases/tables/merlin/external_source.yaml b/deployment/hasura/metadata/databases/tables/merlin/external_source.yaml
@@ -4,9 +4,6 @@ table:
 configuration:
   custom_name: "external_source"
 object_relationships:
-- name: uploaded_file
-  using:
-    foreign_key_constraint_on: file_id
 - name: derivation_group
   using:
     foreign_key_constraint_on: derivation_group_id
@@ -37,16 +34,16 @@ select_permissions:
 insert_permissions:
   - role: aerie_admin
     permission:
-      columns: [key, file_id, source_type_id, valid_at, start_time, end_time, metadata, derivation_group_id, created_at]
+      columns: [key, source_type_id, valid_at, start_time, end_time, metadata, derivation_group_id, created_at]
       check: {}
   - role: user
     permission:
-      columns: [key, file_id, source_type_id, valid_at, start_time, end_time, metadata, derivation_group_id, created_at]
+      columns: [key, source_type_id, valid_at, start_time, end_time, metadata, derivation_group_id, created_at]
       check: {}
 update_permissions:
   - role: aerie_admin
     permission:
-      columns: [key, file_id, source_type_id, valid_at, start_time, end_time, metadata, derivation_group_id, created_at]
+      columns: [key, source_type_id, valid_at, start_time, end_time, metadata, derivation_group_id, created_at]
       filter: {}
 #  - role: user # justified???
 #    permission:

diff --git a/deployment/hasura/migrations/Aerie/10_derived_events/up.sql b/deployment/hasura/migrations/Aerie/10_derived_events/up.sql
@@ -21,7 +21,6 @@ CREATE OR REPLACE VIEW merlin.derived_events
   AS
   SELECT
       source_id,
-      file_id,
       event_id,
       event_key,
       event_type_id,
@@ -30,7 +29,6 @@ CREATE OR REPLACE VIEW merlin.derived_events
       source_range,
       valid_at
      FROM ( SELECT rule1_3.source_id,
-              rule1_3.file_id,
               rule1_3.event_id,
               rule1_3.event_key,
               rule1_3.event_type_id,
@@ -40,7 +38,6 @@ CREATE OR REPLACE VIEW merlin.derived_events
               rule1_3.valid_at,
               row_number() OVER (PARTITION BY rule1_3.event_key, rule1_3.derivation_group_id ORDER BY rule1_3.valid_at DESC) AS rn
              FROM ( SELECT sub.id AS source_id,
-                      sub.file_id,
                       external_event.id AS event_id,
                       external_event.key AS event_key,
                       external_event.event_type_id,
@@ -51,17 +48,17 @@ CREATE OR REPLACE VIEW merlin.derived_events
                      FROM merlin.external_event
                        JOIN (
   	 					WITH derivation_tb_range AS (
-  							SELECT id, key, file_id, derivation_group_id, tstzmultirange(tstzrange(start_time, end_time)) as dr, valid_at
+  							SELECT id, key, derivation_group_id, tstzmultirange(tstzrange(start_time, end_time)) as dr, valid_at
   								FROM merlin.external_source
   								ORDER BY valid_at ASC
   						),
   						ranges_with_subs AS (
-  							SELECT tr1.id, tr1.file_id, tr1.derivation_group_id, tr1.key, tr1.dr as original_range, COALESCE(array_remove(array_agg(tr2.dr ORDER BY tr2.valid_at) FILTER (WHERE tr1.derivation_group_id = tr2.derivation_group_id), NULL), '{}') as subsequent_ranges, tr1.valid_at
+  							SELECT tr1.id, tr1.derivation_group_id, tr1.key, tr1.dr as original_range, COALESCE(array_remove(array_agg(tr2.dr ORDER BY tr2.valid_at) FILTER (WHERE tr1.derivation_group_id = tr2.derivation_group_id), NULL), '{}') as subsequent_ranges, tr1.valid_at
   								FROM derivation_tb_range tr1
   								LEFT JOIN derivation_tb_range tr2 ON tr1.valid_at < tr2.valid_at
-  								GROUP BY tr1.id, tr1.derivation_group_id, tr1.key, tr1.file_id, tr1.valid_at, tr1.dr
+  								GROUP BY tr1.id, tr1.derivation_group_id, tr1.key, tr1.valid_at, tr1.dr
   						)
-  						SELECT id, file_id, derivation_group_id, key, original_range, subsequent_ranges, merlin.subtract_later_ranges(original_range, subsequent_ranges) as source_range, valid_at
+  						SELECT id, derivation_group_id, key, original_range, subsequent_ranges, merlin.subtract_later_ranges(original_range, subsequent_ranges) as source_range, valid_at
   							FROM ranges_with_subs
   							ORDER BY derivation_group_id DESC, valid_at ASC
   	 				) sub ON sub.id = external_event.source_id

diff --git a/...stgres-init-db/sql/functions/merlin/external_events/cleanup_on_external_source_delete.sql b/...stgres-init-db/sql/functions/merlin/external_events/cleanup_on_external_source_delete.sql
@@ -9,21 +9,6 @@ CREATE OR REPLACE FUNCTION merlin.cleanup_on_external_source_delete()
 	as
 	$$
 	begin
-	  -- STEP 0: DELETE ASSOCIATED UPLOADED_FILE
-	  --          Note: as users are now permitted to delete external_sources, deleting uploaded_files from the
-	  --                existing delete query requires that we give users delete permissions on uploaded_file,
-	  --                which currently doesn't exist and is therefore something we aim to avoid. As such, deletion
-	  --                is handled here.
-	  WITH to_delete AS (
-    	SELECT uploaded_file.id, uploaded_file.name FROM merlin.uploaded_file
-    		LEFT JOIN merlin.external_source ON external_source.file_id = uploaded_file.id
-    		LEFT JOIN merlin.mission_model ON mission_model.jar_id = uploaded_file.id
-    		WHERE key IS NULL AND jar_id IS NULL
-    		ORDER BY uploaded_file.id
-    )
-    DELETE FROM merlin.uploaded_file
-    	WHERE id IN (SELECT id FROM to_delete);
-
 		-- STEP 1: DELETE LINGERING plan->dg links:
 		WITH to_delete AS (
 			SELECT plan_derivation_group.id FROM merlin.plan_derivation_group
@@ -69,4 +54,4 @@ AFTER DELETE ON merlin.external_source
 	FOR EACH ROW EXECUTE FUNCTION cleanup_on_external_source_delete();
 --ALTER TRIGGER cleanup_on_external_source_delete
 --    OWNER TO aerie;
---GRANT EXECUTE ON TRIGGER cleanup_on_external_source_delete TO aerie;
+--GRANT EXECUTE ON TRIGGER cleanup_on_external_source_delete TO aerie;
diff --git a/deployment/postgres-init-db/sql/tables/merlin/external_events/external_source.sql b/deployment/postgres-init-db/sql/tables/merlin/external_events/external_source.sql
@@ -2,7 +2,6 @@
 CREATE TABLE merlin.external_source (
     id integer NOT NULL,
     key text NOT NULL,
-    file_id integer NOT NULL,
     source_type_id integer NOT NULL,
     derivation_group_id integer NOT NULL,
     valid_at timestamp with time zone NOT NULL,
@@ -30,11 +29,6 @@ ALTER TABLE ONLY merlin.external_source ALTER COLUMN id SET DEFAULT nextval('mer
 ALTER TABLE ONLY merlin.external_source
     ADD CONSTRAINT external_source_pkey PRIMARY KEY (id);
 
--- Add foreign key definition for file_id field, linking to uploaded_file table
-ALTER TABLE ONLY merlin.external_source
-    ADD CONSTRAINT "file_id -> uploaded_file" FOREIGN KEY (file_id) REFERENCES merlin.uploaded_file(id);
-
-
 -- Add uniqueness constraint for key/derivation_group_id tuple (we exclude source_type_id as derivation_group inherently addresses that, being a subclass of source types)
 ALTER TABLE ONLY merlin.external_source
     ADD CONSTRAINT logical_source_identifiers UNIQUE (key, derivation_group_id);

diff --git a/deployment/postgres-init-db/sql/views/merlin/derived_events.sql b/deployment/postgres-init-db/sql/views/merlin/derived_events.sql
@@ -3,7 +3,6 @@ CREATE OR REPLACE VIEW merlin.derived_events
   AS
   SELECT
       source_id,
-      file_id,
       event_id,
       event_key,
       event_type_id,
@@ -12,7 +11,6 @@ CREATE OR REPLACE VIEW merlin.derived_events
       source_range,
       valid_at
      FROM ( SELECT rule1_3.source_id,
-              rule1_3.file_id,
               rule1_3.event_id,
               rule1_3.event_key,
               rule1_3.event_type_id,
@@ -22,7 +20,6 @@ CREATE OR REPLACE VIEW merlin.derived_events
               rule1_3.valid_at,
               row_number() OVER (PARTITION BY rule1_3.event_key, rule1_3.derivation_group_id ORDER BY rule1_3.valid_at DESC) AS rn
              FROM ( SELECT sub.id AS source_id,
-                      sub.file_id,
                       external_event.id AS event_id,
                       external_event.key AS event_key,
                       external_event.event_type_id,
@@ -33,17 +30,17 @@ CREATE OR REPLACE VIEW merlin.derived_events
                      FROM merlin.external_event
                        JOIN (
   	 					WITH derivation_tb_range AS (
-  							SELECT id, key, file_id, derivation_group_id, tstzmultirange(tstzrange(start_time, end_time)) as dr, valid_at
+  							SELECT id, key, derivation_group_id, tstzmultirange(tstzrange(start_time, end_time)) as dr, valid_at
   								FROM merlin.external_source
   								ORDER BY valid_at ASC
   						),
   						ranges_with_subs AS (
-  							SELECT tr1.id, tr1.file_id, tr1.derivation_group_id, tr1.key, tr1.dr as original_range, COALESCE(array_remove(array_agg(tr2.dr ORDER BY tr2.valid_at) FILTER (WHERE tr1.derivation_group_id = tr2.derivation_group_id), NULL), '{}') as subsequent_ranges, tr1.valid_at
+  							SELECT tr1.id, tr1.derivation_group_id, tr1.key, tr1.dr as original_range, COALESCE(array_remove(array_agg(tr2.dr ORDER BY tr2.valid_at) FILTER (WHERE tr1.derivation_group_id = tr2.derivation_group_id), NULL), '{}') as subsequent_ranges, tr1.valid_at
   								FROM derivation_tb_range tr1
   								LEFT JOIN derivation_tb_range tr2 ON tr1.valid_at < tr2.valid_at
-  								GROUP BY tr1.id, tr1.derivation_group_id, tr1.key, tr1.file_id, tr1.valid_at, tr1.dr
+  								GROUP BY tr1.id, tr1.derivation_group_id, tr1.key, tr1.valid_at, tr1.dr
   						)
-  						SELECT id, file_id, derivation_group_id, key, original_range, subsequent_ranges, merlin.subtract_later_ranges(original_range, subsequent_ranges) as source_range, valid_at
+  						SELECT id, derivation_group_id, key, original_range, subsequent_ranges, merlin.subtract_later_ranges(original_range, subsequent_ranges) as source_range, valid_at
   							FROM ranges_with_subs
   							ORDER BY derivation_group_id DESC, valid_at ASC
   	 				) sub ON sub.id = external_event.source_id