diff --git a/.github/scripts/compareDatabasesUp.sh b/.github/scripts/compareDatabasesUp.sh index cf68514cae..836a7d6cf0 100755 --- a/.github/scripts/compareDatabasesUp.sh +++ b/.github/scripts/compareDatabasesUp.sh @@ -13,6 +13,7 @@ PGCBADEXPLAIN=./comparison/badexplanations.txt \ PGDB=postgres \ PGBINDIR=/usr/bin \ PGCOMITSCHEMAS="('hdb_catalog'),('pg_catalog'),('information_schema')" \ +PGCEXPLANATIONS=./explanations \ ./pgcmp return_code=$? diff --git a/.github/scripts/explanations b/.github/scripts/explanations_down similarity index 100% rename from .github/scripts/explanations rename to .github/scripts/explanations_down diff --git a/.github/scripts/explanations_up b/.github/scripts/explanations_up new file mode 100644 index 0000000000..8ea3f1581b --- /dev/null +++ b/.github/scripts/explanations_up @@ -0,0 +1 @@ +table permissions ui "ui.seen_sources:aerie_test_username" missing in 1st DB AerieAdminHasImplicitPermissions 2 diff --git a/.github/workflows/pgcmp.yml b/.github/workflows/pgcmp.yml index 25aebae79b..d55accf29b 100644 --- a/.github/workflows/pgcmp.yml +++ b/.github/workflows/pgcmp.yml @@ -271,7 +271,7 @@ jobs: - name: Compare Databases id: dbcmp run: | - cp ./.github/scripts/explanations pgcmp + cp ./.github/scripts/explanations_up pgcmp/explanations cp ./.github/scripts/compareDatabasesUp.sh pgcmp/compareDatabases.sh cd pgcmp ./compareDatabases.sh @@ -333,7 +333,7 @@ jobs: - name: Compare Databases id: dbcmp run: | - cp ./.github/scripts/explanations pgcmp + cp ./.github/scripts/explanations_down pgcmp/explanations cp ./.github/scripts/compareDatabasesDown.sh pgcmp/compareDatabases.sh cd pgcmp ./compareDatabases.sh diff --git a/db-tests/src/test/java/gov/nasa/jpl/aerie/database/ExternalEventTests.java b/db-tests/src/test/java/gov/nasa/jpl/aerie/database/ExternalEventTests.java new file mode 100644 index 0000000000..3b1116c467 --- /dev/null +++ b/db-tests/src/test/java/gov/nasa/jpl/aerie/database/ExternalEventTests.java @@ -0,0 +1,2241 @@ +package gov.nasa.jpl.aerie.database; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import org.postgresql.util.PSQLException; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@SuppressWarnings("SqlSourceToSinkFlow") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class ExternalEventTests { + + //region Generic Database Testing Setup + private DatabaseTestHelper helper; + private MerlinDatabaseTestHelper merlinHelper; + private Connection connection; + + @AfterEach + void afterEach() throws SQLException { + helper.clearSchema("merlin"); + } + + @BeforeAll + void beforeAll() throws SQLException, IOException, InterruptedException { + helper = new DatabaseTestHelper("external_event", "Activity Directive Changelog Tests"); + connection = helper.connection(); + merlinHelper = new MerlinDatabaseTestHelper(connection); + } + + @AfterAll + void afterAll() throws SQLException, IOException, InterruptedException { + helper.clearSchema("merlin"); + helper.close(); + connection = null; + helper = null; + } + //endregion + + + //region Commonly Repeated Variables + final static String SOURCE_TYPE = "Test"; + final static String DERIVATION_GROUP = "Test Default"; + final static String EVENT_TYPE = "Test"; + final static String CREATED_AT = "2024-01-01T00:00:00Z"; + //endregion + + + //region Records + protected record ExternalEvent(String key, String event_type_name, String source_key, String derivation_group_name, String start_time, String duration) { + ExternalEvent(String key, String start_time, String duration, ExternalSource source) { + this(key, EVENT_TYPE, source.key(), source.derivation_group_name(), start_time, duration); + } + } + protected record ExternalSource(String key, String source_type_name, String derivation_group_name, String valid_at, String start_time, String end_time, String created_at){} + protected record DerivedEvent(String key, String event_type_name, String source_key, String derivation_group_name, String start_time, String duration, String source_range, String valid_at){} + protected record PlanDerivationGroup(int plan_id, String derivation_group_name, boolean acknowledged, String last_acknowledged_at){} + //endregion + + + //region Helper Functions + protected void insertExternalEventType(String event_type_name) throws SQLException { + try(final var statement = connection.createStatement()) { + // create the event type + statement.executeUpdate( + // language=sql + """ + INSERT INTO + merlin.external_event_type + VALUES ('%s'); + """.formatted(event_type_name) + ); + } + } + + protected void insertExternalSourceType(String source_type_name) throws SQLException { + try(final var statement = connection.createStatement()) { + // create the source type + statement.executeUpdate( + // language=sql + """ + INSERT INTO + merlin.external_source_type + VALUES ('%s'); + """.formatted(source_type_name) + ); + } + } + + protected void insertDerivationGroup(String derivation_group_name, String source_type_name) throws SQLException { + try(final var statement = connection.createStatement()) { + // create the derivation group + statement.executeUpdate( + // language=sql + """ + INSERT INTO + merlin.derivation_group + VALUES ('%s', '%s'); + """.formatted(derivation_group_name, source_type_name) + ); + } + } + + protected void insertExternalSource(ExternalSource externalSource) throws SQLException { + try(final var statement = connection.createStatement()) { + System.out.println("STARTING " + externalSource); + // create the source + statement.executeUpdate( + // language=sql + """ + INSERT INTO + merlin.external_source + VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s'); + """.formatted( + externalSource.key, + externalSource.source_type_name, + externalSource.derivation_group_name, + externalSource.valid_at, + externalSource.start_time, + externalSource.end_time, + externalSource.created_at + ) + ); + System.out.println("FINISHED " + externalSource); + } + } + + protected void insertExternalEvent(ExternalEvent externalEvent) throws SQLException { + try(final var statement = connection.createStatement()) { + // create the event + statement.executeUpdate( + // language=sql + """ + INSERT INTO + merlin.external_event + VALUES ('%s', '%s', '%s', '%s', '%s', '%s'); + """.formatted( + externalEvent.key, + externalEvent.event_type_name, + externalEvent.source_key, + externalEvent.derivation_group_name, + externalEvent.start_time, + externalEvent.duration + ) + ); + } + } + + protected void associateDerivationGroupWithPlan(int planId, String derivationGroupName) throws SQLException { + try(final var statement = connection.createStatement()) { + // create the event type + statement.executeUpdate( + // language=sql + """ + INSERT INTO + merlin.plan_derivation_group + VALUES ('%s', '%s'); + """.formatted(planId, derivationGroupName) + ); + } + } + + /** + * Get all derived events. + */ + protected List getDerivedEvents() throws SQLException { + List results = new ArrayList<>(); + try(final var statement = connection.createStatement()) { + var res = statement.executeQuery( + // language=sql + """ + SELECT * FROM merlin.derived_events; + """ + ); + while (res.next()) { + results.add(new DerivedEvent( + res.getString("event_key"), + res.getString("event_type_name"), + res.getString("source_key"), + res.getString("derivation_group_name"), + res.getString("start_time"), + res.getString("DURATION"), + res.getString("source_range"), + res.getString("valid_at") + )); + } + } + return results; + } + + protected List getDerivedEvents(String dg_name) throws SQLException { + List results = new ArrayList<>(); + try(final var statement = connection.createStatement()) { + var res = statement.executeQuery( + // language=sql + """ + SELECT * FROM merlin.derived_events + WHERE derivation_group_name = '%s'; + """.formatted(dg_name) + ); + while (res.next()) { + results.add(new DerivedEvent( + res.getString("event_key"), + res.getString("event_type_name"), + res.getString("source_key"), + res.getString("derivation_group_name"), + res.getString("start_time"), + res.getString("DURATION"), + res.getString("source_range"), + res.getString("valid_at") + )); + } + } + return results; + } + + /** + * Repeated function that uploads a source and various events, as well as related types. + * Used in: + * - ExternalEventTests.java: + * + duplicateSource + * + superDerivedEvents + * + associateDerivationGroupWithBasePlan + * + * Data here is based on the SSMO-MPS/mission-data-sandbox/derivation_test examples. + * + * @param dg The derivation group name to use in entries. + * @param skip_types A boolean that should be set to true to skip uploading event and source types, if this is being + * called twice in a given test. + */ + protected void upload_source(String dg, boolean skip_types) throws SQLException { + // First, define the sources. + ExternalSource sourceOne = new ExternalSource( + "Derivation_Test_00.json", + SOURCE_TYPE, + dg, + "2024-01-18 00:00:00+00", + "2024-01-05 00:00:00+00", + "2024-01-11 00:00:00+00", + "2024-08-21 22:36:12.858009+00" + ); + ExternalSource sourceTwo = new ExternalSource( + "Derivation_Test_01.json", + SOURCE_TYPE, + dg, + "2024-01-19 00:00:00+00", + "2024-01-01 00:00:00+00", + "2024-01-07 00:00:00+00", + "2024-08-21 22:36:19.381275+00" + ); + ExternalSource sourceThree = new ExternalSource( + "Derivation_Test_02.json", + SOURCE_TYPE, + dg, + "2024-01-20 00:00:00+00", + "2024-01-03 00:00:00+00", + "2024-01-10 00:00:00+00", + "2024-08-21 22:36:23.340941+00" + ); + ExternalSource sourceFour = new ExternalSource( + "Derivation_Test_03.json", + SOURCE_TYPE, + dg, + "2024-01-21 00:00:00+00", + "2024-01-01 12:00:00+00", + "2024-01-02 12:00:00+00", + "2024-08-21 22:36:28.365244+00" + ); + + // Second, define the events, spaced by sources 1-4 + ExternalEvent twoA = new ExternalEvent("2", "DerivationD", "Derivation_Test_00.json", dg, "2024-01-05 23:00:00+00", "01:10:00"); + ExternalEvent seven = new ExternalEvent("7", "DerivationC", "Derivation_Test_00.json", dg, "2024-01-09 23:00:00+00", "02:00:00"); + ExternalEvent eight = new ExternalEvent("8", "DerivationB", "Derivation_Test_00.json", dg, "2024-01-10 11:00:00+00", "01:05:00"); + + ExternalEvent one = new ExternalEvent("1", "DerivationA", "Derivation_Test_01.json", dg, "2024-01-01 00:00:00+00", "02:10:00"); + ExternalEvent twoB = new ExternalEvent("2", "DerivationA", "Derivation_Test_01.json", dg, "2024-01-01 12:00:00+00", "02:10:00"); + ExternalEvent three = new ExternalEvent("3", "DerivationB", "Derivation_Test_01.json", dg, "2024-01-02 23:00:00+00", "03:00:00"); + ExternalEvent four = new ExternalEvent("4", "DerivationB", "Derivation_Test_01.json", dg, "2024-01-05 21:00:00+00", "03:00:00"); + + ExternalEvent five = new ExternalEvent("5", "DerivationC", "Derivation_Test_02.json", dg, "2024-01-05 23:00:00+00", "01:10:00"); + ExternalEvent six = new ExternalEvent("6", "DerivationC", "Derivation_Test_02.json", dg, "2024-01-06 12:00:00+00", "02:00:00"); + ExternalEvent twoC = new ExternalEvent("2", "DerivationB", "Derivation_Test_02.json", dg, "2024-01-09 11:00:00+00", "01:05:00"); + + ExternalEvent nine = new ExternalEvent("9", "DerivationC", "Derivation_Test_03.json", dg, "2024-01-02 00:00:00+00", "01:00:00"); + + // Third, insert types; this can be skipped in the case of multiple calls to upload_source in a given test + if (!skip_types) { + String[] externalEventTypes = {"DerivationA", "DerivationB", "DerivationC", "DerivationD"}; + for (String eventType : externalEventTypes) { + insertExternalEventType(eventType); + } + insertExternalSourceType(SOURCE_TYPE); + } + + // Fourth, insert derivation group + insertDerivationGroup(dg, SOURCE_TYPE); + + // Then, insert sources + insertExternalSource(sourceOne); + insertExternalSource(sourceTwo); + insertExternalSource(sourceThree); + insertExternalSource(sourceFour); + + // Finally, insert events + insertExternalEvent(twoA); + insertExternalEvent(seven); + insertExternalEvent(eight); + insertExternalEvent(one); + insertExternalEvent(twoB); + insertExternalEvent(three); + insertExternalEvent(four); + insertExternalEvent(five); + insertExternalEvent(six); + insertExternalEvent(twoC); + insertExternalEvent(nine); + } + + protected void insertStandardTypes() throws SQLException { + // insert external event type + insertExternalEventType(EVENT_TYPE); + + // insert external source type + insertExternalSourceType(SOURCE_TYPE); + + // insert derivation group + insertDerivationGroup(DERIVATION_GROUP, SOURCE_TYPE); + } + //endregion + + + //region Derivation Tests + /** + * The focus of this class is to test specifically the derivation function for events. This includes overlapping of + * source windows, as well as reconciling differences inside the sources themselves (by testing each of the 4 + * derivation rules). + */ + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class DerivedEventsTests { + + /** + * This class focuses on testing sources' windows to verify that derivation will work as expected. These are + * separate, extracted tests that don't really evaluate events and instead just that sources play together + * correctly. As such we test "empty" sources to make sure their overlapped windows work correctly. + * We add events that span a very short DURATION simply so that the sources show up in derived_events, but we aren't + * testing any properties of said events. + */ + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class DerivedSourcesTests { + + // Commonly Repeated: + final static String DURATION = "00:00:00.000001"; + + @BeforeEach + void beforeEach() throws SQLException { + // insert generic external event type, source type, and derivation group + insertStandardTypes(); + } + + /** + * The first test is a basic, non-overlapping case. We must ensure that gaps are preserved: + * + * A: ++++++++ + * B: +++++++ + * BBBBBBB AAAAAAAA + */ + @Test + void testSparseCoverage() throws SQLException { + // create our sources and their per-window events + ExternalSource a = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T03:00:00Z", + "2024-01-01T04:00:00Z", + CREATED_AT + ); + ExternalEvent aE = new ExternalEvent(a.key() + "_event", SOURCE_TYPE, a.key(), + DERIVATION_GROUP, a.start_time(), DURATION); + ExternalSource b = new ExternalSource( + "B", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-02T00:00:00Z", + "2024-01-01T01:00:00Z", + "2024-01-01T02:00:00Z", + CREATED_AT + ); + ExternalEvent bE = new ExternalEvent(b.key() + "_event", SOURCE_TYPE, b.key(), + DERIVATION_GROUP, b.start_time(), DURATION); + + // verify the ranges are as expected + // insert sources + insertExternalSource(a); + insertExternalSource(b); + + // insert events + insertExternalEvent(aE); + insertExternalEvent(bE); + + var results = getDerivedEvents(); + + // both ranges should only have a single element and be fully present + final List expectedResults = List.of( + new DerivedEvent( + "A_event", + SOURCE_TYPE, + "A", + DERIVATION_GROUP, + "2024-01-01 03:00:00+00", + "00:00:00.000001", + "{[\"2024-01-01 03:00:00+00\",\"2024-01-01 04:00:00+00\")}", + "2024-01-01 00:00:00+00" + ), + new DerivedEvent( + "B_event", + SOURCE_TYPE, + "B", + DERIVATION_GROUP, + "2024-01-01 01:00:00+00", + "00:00:00.000001", + "{[\"2024-01-01 01:00:00+00\",\"2024-01-01 02:00:00+00\")}", + "2024-01-02 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + + /** + * This test is an overlapping case wherein the source of higher precedence (the more recently valid one - B) + * should shorten the range that A applies over. That is, a source is succeeded before in time by a more valid + * source. + * + * A: ++++++++ + * B: +++++++ + * BBBBBBBAAAA + */ + @Test + void testForwardOverlap() throws SQLException { + // create our sources and their per-window events + ExternalSource a = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:30:00Z", + "2024-01-01T02:00:00Z", + CREATED_AT + ); + ExternalEvent aE = new ExternalEvent( + a.key() + "_event", + SOURCE_TYPE, + a.key(), + DERIVATION_GROUP, + "2024-01-01T1:10:00Z", + DURATION + ); + ExternalSource b = new ExternalSource( + "B", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-02T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + CREATED_AT + ); + ExternalEvent bE = new ExternalEvent(b.key() + "_event", SOURCE_TYPE, b.key(), + DERIVATION_GROUP, b.start_time(), DURATION); + + // verify the ranges are as expected + // insert sources + insertExternalSource(a); + insertExternalSource(b); + + // insert events + insertExternalEvent(aE); + insertExternalEvent(bE); + + var results = getDerivedEvents(); + + // both ranges should only have a single element and be fully present + final List expectedResults = List.of( + new DerivedEvent( + "A_event", + SOURCE_TYPE, + "A", + DERIVATION_GROUP, + "2024-01-01 01:10:00+00", + "00:00:00.000001", + "{[\"2024-01-01 01:00:00+00\",\"2024-01-01 02:00:00+00\")}", + "2024-01-01 00:00:00+00" + ), + new DerivedEvent( + "B_event", + SOURCE_TYPE, + "B", + DERIVATION_GROUP, + "2024-01-01 00:00:00+00", + "00:00:00.000001", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 01:00:00+00\")}", + "2024-01-02 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + + /** + * This test is an overlapping case wherein the source of higher precedence (the more recently valid one - B) + * should shorten the range that A applies over. In this case, a source is succeeded after in time by a more + * valid source. + * + * A: +++++++ + * B: ++++++++ + * AAABBBBBBBB + */ + @Test + void testBackwardOverlap() throws SQLException { + // create our sources and their per-window events + ExternalSource a = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + CREATED_AT + ); + // have to manually pick this + ExternalEvent aE = new ExternalEvent( + a.key() + "_event", + SOURCE_TYPE, + a.key(), + DERIVATION_GROUP, + a.start_time(), + DURATION + ); + ExternalSource b = new ExternalSource( + "B", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-02T00:00:00Z", + "2024-01-01T00:30:00Z", + "2024-01-01T01:30:00Z", + CREATED_AT + ); + ExternalEvent bE = new ExternalEvent(b.key() + "_event", SOURCE_TYPE, b.key(), + DERIVATION_GROUP, b.start_time(), DURATION); + + // verify the ranges are as expected + // insert sources + insertExternalSource(a); + insertExternalSource(b); + + // insert events + insertExternalEvent(aE); + insertExternalEvent(bE); + + var results = getDerivedEvents(); + + // both ranges should only have a single element and be fully present + final List expectedResults = List.of( + new DerivedEvent( + "A_event", + SOURCE_TYPE, + "A", + DERIVATION_GROUP, + "2024-01-01 00:00:00+00", + "00:00:00.000001", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 00:30:00+00\")}", + "2024-01-01 00:00:00+00" + ), + new DerivedEvent( + "B_event", + SOURCE_TYPE, + "B", + DERIVATION_GROUP, + "2024-01-01 00:30:00+00", + "00:00:00.000001", + "{[\"2024-01-01 00:30:00+00\",\"2024-01-01 01:30:00+00\")}", + "2024-01-02 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + + /** + * This test is an overlapping case with three sources. The least recent source (of least precedence, source A) + * covers a larger, more inclusive interval. A is succeeded by other, smaller sources (B, C), such that the + * range over which A applies is chopped into several subintervals: + * + * A: +++++++++++++++++++++ + * B: ++++++ + * C: +++++++ + * BBBBBBAAAAAAAACCCCCCCAAAA + */ + @Test + void testBackground() throws SQLException { + // create our sources and their per-window events + ExternalSource a = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:30:00Z", + "2024-01-01T03:00:00Z", + CREATED_AT + ); + // just need 1 that shows up and source range will still show correctly + ExternalEvent aE = new ExternalEvent( + a.key() + "_event", + SOURCE_TYPE, + a.key(), + DERIVATION_GROUP, + "2024-01-01T01:10:00Z", + DURATION + ); + ExternalSource b = new ExternalSource( + "B", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-02T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + CREATED_AT + ); + ExternalEvent bE = new ExternalEvent(b.key() + "_event", SOURCE_TYPE, b.key(), + DERIVATION_GROUP, b.start_time(), DURATION); + ExternalSource c = new ExternalSource( + "C", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-03T00:00:00Z", + "2024-01-01T01:30:00Z", + "2024-01-01T02:00:00Z", + CREATED_AT + ); + ExternalEvent cE = new ExternalEvent(c.key() + "_event", SOURCE_TYPE, c.key(), + DERIVATION_GROUP, c.start_time(), DURATION); + + // verify the ranges are as expected + // insert sources + insertExternalSource(a); + insertExternalSource(b); + insertExternalSource(c); + + // insert events + insertExternalEvent(aE); + insertExternalEvent(bE); + insertExternalEvent(cE); + + var results = getDerivedEvents(); + + final List expectedResults = List.of( + new DerivedEvent( + "A_event", + SOURCE_TYPE, + "A", + DERIVATION_GROUP, + "2024-01-01 01:10:00+00", + "00:00:00.000001", + "{[\"2024-01-01 01:00:00+00\",\"2024-01-01 01:30:00+00\"),[\"2024-01-01 02:00:00+00\",\"2024-01-01 03:00:00+00\")}", + "2024-01-01 00:00:00+00" + ), + new DerivedEvent( + "B_event", + SOURCE_TYPE, + "B", + DERIVATION_GROUP, + "2024-01-01 00:00:00+00", + "00:00:00.000001", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 01:00:00+00\")}", + "2024-01-02 00:00:00+00" + ), + new DerivedEvent( + "C_event", + SOURCE_TYPE, + "C", + DERIVATION_GROUP, + "2024-01-01 01:30:00+00", + "00:00:00.000001", + "{[\"2024-01-01 01:30:00+00\",\"2024-01-01 02:00:00+00\")}", + "2024-01-03 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + + /** + * The first 4 tests are exhaustive of source window cases. This final test, then, is included as an overall, + * cumulative case: + * A: ++++++++++++ + * B: ++++++ + * C: ++++++ + * D: +++++++ + * E: +++++++ + * F: +++ + * G: + + * BBDDFFFDDAAGAEEEEEEECCC + */ + @Test + void testAmalgamation() throws SQLException { + // create our sources and their per-window events + ExternalSource a = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:03:00Z", + "2024-01-01T00:15:00Z", + CREATED_AT + ); + ExternalEvent aE = new ExternalEvent( + a.key() + "_event", + SOURCE_TYPE, + a.key(), + DERIVATION_GROUP, + "2024-01-01T00:09:10Z", + DURATION + ); + ExternalSource b = new ExternalSource( + "B", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-02T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T00:06:00Z", + CREATED_AT + ); + ExternalEvent bE = new ExternalEvent(b.key() + "_event", SOURCE_TYPE, b.key(), + DERIVATION_GROUP, b.start_time(), DURATION); + ExternalSource c = new ExternalSource( + "C", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-03T00:00:00Z", + "2024-01-01T00:17:00Z", + "2024-01-01T00:23:00Z", + CREATED_AT + ); + ExternalEvent cE = new ExternalEvent( + c.key() + "_event", + SOURCE_TYPE, + c.key(), + DERIVATION_GROUP, + "2024-01-01T00:21:00Z", + DURATION + ); + ExternalSource d = new ExternalSource( + "D", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-04T00:00:00Z", + "2024-01-01T00:02:00Z", + "2024-01-01T00:09:00Z", + CREATED_AT + ); + ExternalEvent dE = new ExternalEvent(d.key() + "_event", SOURCE_TYPE, d.key(), + DERIVATION_GROUP, d.start_time(), DURATION); + ExternalSource e = new ExternalSource( + "E", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-05T00:00:00Z", + "2024-01-01T00:13:00Z", + "2024-01-01T00:20:00Z", + CREATED_AT + ); + ExternalEvent eE = new ExternalEvent(e.key() + "_event", SOURCE_TYPE, e.key(), + DERIVATION_GROUP, e.start_time(), DURATION); + ExternalSource f = new ExternalSource( + "F", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-06T00:00:00Z", + "2024-01-01T00:04:00Z", + "2024-01-01T00:07:00Z", + CREATED_AT + ); + ExternalEvent fE = new ExternalEvent(f.key() + "_event", SOURCE_TYPE, f.key(), + DERIVATION_GROUP, f.start_time(), DURATION); + ExternalSource g = new ExternalSource( + "G", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-07T00:00:00Z", + "2024-01-01T00:11:00Z", + "2024-01-01T00:12:00Z", + CREATED_AT + ); + ExternalEvent gE = new ExternalEvent(g.key() + "_event", SOURCE_TYPE, g.key(), + DERIVATION_GROUP, g.start_time(), DURATION); + + // verify the ranges are as expected + // insert sources + insertExternalSource(a); + insertExternalSource(b); + insertExternalSource(c); + insertExternalSource(d); + insertExternalSource(e); + insertExternalSource(f); + insertExternalSource(g); + + // insert events + insertExternalEvent(aE); + insertExternalEvent(bE); + insertExternalEvent(cE); + insertExternalEvent(dE); + insertExternalEvent(eE); + insertExternalEvent(fE); + insertExternalEvent(gE); + + var results = getDerivedEvents(); + + final List expectedResults = List.of( + new DerivedEvent( + "A_event", + SOURCE_TYPE, + "A", + DERIVATION_GROUP, + "2024-01-01 00:09:10+00", + "00:00:00.000001", + "{[\"2024-01-01 00:09:00+00\",\"2024-01-01 00:11:00+00\"),[\"2024-01-01 00:12:00+00\",\"2024-01-01 00:13:00+00\")}", + "2024-01-01 00:00:00+00" + ), + new DerivedEvent( + "B_event", + SOURCE_TYPE, + "B", + DERIVATION_GROUP, + "2024-01-01 00:00:00+00", + "00:00:00.000001", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 00:02:00+00\")}", + "2024-01-02 00:00:00+00" + ), + new DerivedEvent( + "C_event", + SOURCE_TYPE, + "C", + DERIVATION_GROUP, + "2024-01-01 00:21:00+00", + "00:00:00.000001", + "{[\"2024-01-01 00:20:00+00\",\"2024-01-01 00:23:00+00\")}", + "2024-01-03 00:00:00+00" + ), + new DerivedEvent( + "D_event", + SOURCE_TYPE, + "D", + DERIVATION_GROUP, + "2024-01-01 00:02:00+00", + "00:00:00.000001", + "{[\"2024-01-01 00:02:00+00\",\"2024-01-01 00:04:00+00\"),[\"2024-01-01 00:07:00+00\",\"2024-01-01 00:09:00+00\")}", + "2024-01-04 00:00:00+00" + ), + new DerivedEvent( + "E_event", + SOURCE_TYPE, "E", + DERIVATION_GROUP, "2024-01-01 00:13:00+00", + "00:00:00.000001", + "{[\"2024-01-01 00:13:00+00\",\"2024-01-01 00:20:00+00\")}", + "2024-01-05 00:00:00+00" + ), + new DerivedEvent( + "F_event", + SOURCE_TYPE, "F", + DERIVATION_GROUP, "2024-01-01 00:04:00+00", + "00:00:00.000001", + "{[\"2024-01-01 00:04:00+00\",\"2024-01-01 00:07:00+00\")}", + "2024-01-06 00:00:00+00" + ), + new DerivedEvent( + "G_event", + SOURCE_TYPE, "G", + DERIVATION_GROUP, "2024-01-01 00:11:00+00", + "00:00:00.000001", + "{[\"2024-01-01 00:11:00+00\",\"2024-01-01 00:12:00+00\")}", + "2024-01-07 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + } + + /** + * This class focuses on testing each of the four rules involved in the derivation operation. To do so, new sources + * and events are created, and their presence and intervals are tested to ensure correctness. + * The rules are as follows. Refer to the docs for more details on why they are what they are: + * 1. An External Event superseded by nothing will be present in the final, derived result. + * 2. An External Event partially superseded by a later External Source, but whose start time occurs before + * the start of said External Source(s), will be present in the final, derived result. + * 3. An External Event whose start is superseded by another External Source, even if its end occurs after the + * end of said External Source, will be replaced by the contents of that External Source (whether they are + * blank spaces, or other events). + * 4. An External Event who shares a key with an External Event in a later External Source will always be + * replaced. + */ + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class DerivedEventRuleTests { + + // Commonly Repeated: + @BeforeEach + void beforeEach() throws SQLException { + // insert generic external event type, source type, and derivation group + insertStandardTypes(); + } + + ////////////////////////// RULE 1 ////////////////////////// + + /** + * A solitary event shouldn't be superseded by anything. + * + * A: aa + */ + @Test + void rule1_solitary() throws SQLException { + // insert the event(s) (and their source(s)) + ExternalSource eS = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + CREATED_AT + ); + + ExternalEvent e = new ExternalEvent("A.1", "2024-01-01T00:00:00Z", "01:00:00", eS); + + + // insert sources + insertExternalSource(eS); + + // insert events + insertExternalEvent(e); + + // ensure the result has the right size and keys + final var results = getDerivedEvents(); + + // the range should only have one element + final List expectedResults = List.of( + new DerivedEvent( + "A.1", + SOURCE_TYPE, + "A", + DERIVATION_GROUP, + "2024-01-01 00:00:00+00", + "01:00:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 01:00:00+00\")}", + "2024-01-01 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + + /** + * An event that is in a partially overlapped source but outside of the overlap is not superseded, even if + * the other source "bookends" it (i.e. the end of that source meets the event's start time). + * + * A: +++++aa++++++ + * B: bb+++ + * C: +cc++++ + */ + @Test + void rule1_bookended() throws SQLException { + // insert the event(s) (and their source(s)) + ExternalSource A = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:30:00Z", + "2024-01-01T02:00:00Z", + CREATED_AT + ); + ExternalSource B = new ExternalSource( + "B", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-02T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + CREATED_AT + ); + ExternalSource C = new ExternalSource( + "C", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-03T00:00:00Z", + "2024-01-01T01:30:00Z", + "2024-01-01T03:00:00Z", + CREATED_AT + ); + + ExternalEvent e = new ExternalEvent("a", "2024-01-01T01:10:00Z", "00:10:00", A); + ExternalEvent before = new ExternalEvent("b", "2024-01-01T00:00:00Z", "00:30:00", B); + ExternalEvent after = new ExternalEvent("c", "2024-01-01T01:30:00Z", "01:00:00", C); + + + // insert sources + insertExternalSource(A); + insertExternalSource(B); + insertExternalSource(C); + + // insert events + insertExternalEvent(e); + insertExternalEvent(before); + insertExternalEvent(after); + + // verify the expected keys are included + final var results = getDerivedEvents(); + + // the ranges should only have a single element and be fully present + final List expectedResults = List.of( + new DerivedEvent( + "a", + SOURCE_TYPE, + "A", + DERIVATION_GROUP, + "2024-01-01 01:10:00+00", + "00:10:00", + "{[\"2024-01-01 01:00:00+00\",\"2024-01-01 01:30:00+00\")}", + "2024-01-01 00:00:00+00" + ), + new DerivedEvent( + "b", + SOURCE_TYPE, + "B", + DERIVATION_GROUP, + "2024-01-01 00:00:00+00", + "00:30:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 01:00:00+00\")}", + "2024-01-02 00:00:00+00" + ), + new DerivedEvent( + "c", + SOURCE_TYPE, + "C", + DERIVATION_GROUP, + "2024-01-01 01:30:00+00", + "01:00:00", + "{[\"2024-01-01 01:30:00+00\",\"2024-01-01 03:00:00+00\")}", + "2024-01-03 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + + ////////////////////////// RULE 2 ////////////////////////// + + /** + * An event that starts before a source that partially overlaps it is not superseded by the overlapping + * source. + * + * A: +++aaaaa + * B: b+bb++++ + * (a and both b's should be in result) + */ + @Test + void rule2() throws SQLException { + // insert the event(s) (and their source(s)) + ExternalSource A = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + CREATED_AT + ); + ExternalSource B = new ExternalSource( + "B", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-02T00:00:00Z", + "2024-01-01T00:30:00Z", + "2024-01-01T01:30:00Z", + CREATED_AT + ); + + // spills into B + ExternalEvent e = new ExternalEvent("a", "2024-01-01T00:25:00Z", "00:10:00", A); + ExternalEvent b1 = new ExternalEvent("b1", "2024-01-01T00:30:00Z", "00:10:00", B); + ExternalEvent b2 = new ExternalEvent("b2", "2024-01-01T00:45:00Z", "00:10:00", B); + + + // insert sources + insertExternalSource(A); + insertExternalSource(B); + + // insert events + insertExternalEvent(e); + insertExternalEvent(b1); + insertExternalEvent(b2); + + // verify the expected keys + final var results = getDerivedEvents(); + + // all ranges should only have a single element and be fully present + final List expectedResults = List.of( + new DerivedEvent( + "a", + SOURCE_TYPE, + "A", + DERIVATION_GROUP, + "2024-01-01 00:25:00+00", + "00:10:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 00:30:00+00\")}", + "2024-01-01 00:00:00+00" + ), + new DerivedEvent( + "b1", + SOURCE_TYPE, + "B", + DERIVATION_GROUP, + "2024-01-01 00:30:00+00", + "00:10:00", + "{[\"2024-01-01 00:30:00+00\",\"2024-01-01 01:30:00+00\")}", + "2024-01-02 00:00:00+00" + ), + new DerivedEvent( + "b2", + SOURCE_TYPE, + "B", + DERIVATION_GROUP, + "2024-01-01 00:45:00+00", + "00:10:00", + "{[\"2024-01-01 00:30:00+00\",\"2024-01-01 01:30:00+00\")}", + "2024-01-02 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + + ////////////////////////// RULE 3 ////////////////////////// + + /** + * An event whose start time is overlapped by a newer source will be superseded by the newer source. + * This holds even if the new source is empty at the overlapped event's start time. + * + * A: +a+aaaaa + * B: b+bb++++ + * (only b's should be in result) + */ + @Test + void rule3_basic() throws SQLException { + // insert the event(s) (and their source(s)) + ExternalSource A = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:30:00Z", + "2024-01-01T01:30:00Z", + CREATED_AT + ); + ExternalSource B = new ExternalSource( + "B", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-02T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + CREATED_AT + ); + + // negated by B, very clearly + ExternalEvent e1 = new ExternalEvent( + "a1", + "2024-01-01T00:40:00Z", + "00:10:00", + A + ); + // even empty space in B neg should negate + ExternalEvent e2 = new ExternalEvent( + "a2", + "2024-01-01T00:55:00Z", + "00:35:00", + A + ); + ExternalEvent b1 = new ExternalEvent("b1", "2024-01-01T00:00:00Z", "00:10:00", B); + ExternalEvent b2 = new ExternalEvent("b2", "2024-01-01T00:30:00Z", "00:20:00", B); + + + // insert sources + insertExternalSource(A); + insertExternalSource(B); + + // insert events + insertExternalEvent(e1); + insertExternalEvent(e2); + insertExternalEvent(b1); + insertExternalEvent(b2); + + // verify the expected keys + final var results = getDerivedEvents(); + + final List expectedResults = List.of( + new DerivedEvent( + "b1", + SOURCE_TYPE, + "B", + DERIVATION_GROUP, + "2024-01-01 00:00:00+00", + "00:10:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 01:00:00+00\")}", + "2024-01-02 00:00:00+00" + ), + new DerivedEvent( + "b2", + SOURCE_TYPE, + "B", + DERIVATION_GROUP, + "2024-01-01 00:30:00+00", + "00:20:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 01:00:00+00\")}", + "2024-01-02 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + + /** + * A completely empty source will still supercede earlier sources. + * + * A: +a+aaaaa + * B: ++++++++ + * (empty result) + */ + @Test + void rule3_empty() throws SQLException { + // insert the event(s) (and their source(s)) + ExternalSource A = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:30:00Z", + "2024-01-01T01:30:00Z", + CREATED_AT + ); + ExternalSource B = new ExternalSource( + "B", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-02T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + CREATED_AT + ); + + // negated by empty space + ExternalEvent e1 = new ExternalEvent( + "a1", + "2024-01-01T00:40:00Z", + "00:10:00", + A + ); + // negated by empty space + ExternalEvent e2 = new ExternalEvent( + "a2", + "2024-01-01T00:55:00Z", + "00:35:00", + A + ); + + + // insert sources + insertExternalSource(A); + + // insert events + insertExternalEvent(e1); + insertExternalEvent(e2); + + // insert B as a source + insertExternalSource(B); + + // verify expected keys (none) + final var results = getDerivedEvents(); + + assertEquals(0, results.size()); + } + + ////////////////////////// RULE 4 ////////////////////////// + + /** + * An event that appears in a later source will replace its occurrence in an earlier source, even if the sources + * don't overlap. + * + * A: ++++aaa+++++ + * B: +++++aaaaa+ + * C: +++++aaaa+++++ + * (one A, of specific DURATION) + */ + @Test + void rule4() throws SQLException { + // insert the event(s) (and their source(s)) + ExternalSource A = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T01:30:00Z", + "2024-01-01T02:30:00Z", + CREATED_AT + ); + ExternalSource B = new ExternalSource( + "B", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-02T00:00:00Z", + "2024-01-01T03:00:00Z", + "2024-01-01T04:00:00Z", + CREATED_AT + ); + ExternalSource C = new ExternalSource( + "C", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-03T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + CREATED_AT + ); + + // negated by empty space + ExternalEvent e1 = new ExternalEvent( + "a", + "2024-01-01T01:50:00Z", + "00:10:00", + A + ); + // negated by empty space + ExternalEvent e2 = new ExternalEvent( + "a", + "2024-01-01T03:40:00Z", + "00:15:00", + B + ); + // negated by empty space + ExternalEvent e3 = new ExternalEvent( + "a", + "2024-01-01T00:30:00Z", + "00:20:00", + C + ); + + // insert sources + insertExternalSource(A); + insertExternalSource(B); + insertExternalSource(C); + + // insert events + insertExternalEvent(e1); + insertExternalEvent(e2); + insertExternalEvent(e3); + + // verify expected keys + final var results = getDerivedEvents(); + + // this range should only have a single element + final List expectedResults = List.of( + new DerivedEvent( + "a", + SOURCE_TYPE, + "C", + DERIVATION_GROUP, + "2024-01-01 00:30:00+00", + "00:20:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 01:00:00+00\")}", + "2024-01-03 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + } + + /** + * This class separates a few tests that don't test a particular rule, but rather general properties of the derivation + * process. + */ + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class GeneralDerivationTests { + // GENERAL + /** + * An event that overlaps another event in the same source does not supersede the first event. + */ + @Test + void nEventsAtSameTime() throws SQLException { + // construct the sources and events + ExternalSource A = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T01:30:00Z", + CREATED_AT + ); + + ExternalEvent e1 = new ExternalEvent("a", "2024-01-01T00:00:00Z", "00:10:00", A); + ExternalEvent e2 = new ExternalEvent("b", "2024-01-01T00:00:00Z", "00:05:00", A); + ExternalEvent e3 = new ExternalEvent("c", "2024-01-01T00:00:00Z", "00:15:00", A); + + // insert generic external event type, source type, and derivation group + insertStandardTypes(); + + // insert sources + insertExternalSource(A); + + // insert events + insertExternalEvent(e1); + insertExternalEvent(e2); + insertExternalEvent(e3); + + // all 3 keys should be present! + var results = getDerivedEvents(); + + // all ranges should only have a single element and be fully present + final List expectedResults = List.of( + new DerivedEvent( + "a", + SOURCE_TYPE, + "A", + DERIVATION_GROUP, + "2024-01-01 00:00:00+00", + "00:10:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 01:30:00+00\")}", + "2024-01-01 00:00:00+00" + ), + new DerivedEvent( + "b", + SOURCE_TYPE, + "A", + DERIVATION_GROUP, + "2024-01-01 00:00:00+00", + "00:05:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 01:30:00+00\")}", + "2024-01-01 00:00:00+00" + ), + new DerivedEvent( + "c", + SOURCE_TYPE, + "A", + DERIVATION_GROUP, + "2024-01-01 00:00:00+00", + "00:15:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 01:30:00+00\")}", + "2024-01-01 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + + + /** + * This is a comprehensive test for derived events that manages several derivation groups (which entails + * repeating basicDerivedEvents twice but the second round bears a new DG name. Then, we verify that there is + * no overlap! Note that this test is effectively vacuous but is a good sanity check.). + */ + @Test + void superDerivedEvents() throws SQLException { + // upload all source data, but twice (using different dgs, to prove non overlap in derivation) + String dg2 = DERIVATION_GROUP + "_2"; + + // upload the data once for the first derivation group + upload_source(DERIVATION_GROUP, false); + + // repeat with the second derivation group + upload_source(dg2, true); + + // check that derived events in our prewritten case has the correct keys + // verify everything is present + var results = getDerivedEvents(); + + List expectedResults = List.of( + new DerivedEvent( + "8", + "DerivationB", + "Derivation_Test_00.json", // note - the same source name can be used across different derivation groups + DERIVATION_GROUP + "_2", + "2024-01-10 11:00:00+00", + "01:05:00", + "{[\"2024-01-10 00:00:00+00\",\"2024-01-11 00:00:00+00\")}", + "2024-01-18 00:00:00+00" + ), + new DerivedEvent( + "8", + "DerivationB", + "Derivation_Test_00.json", + DERIVATION_GROUP, + "2024-01-10 11:00:00+00", + "01:05:00", + "{[\"2024-01-10 00:00:00+00\",\"2024-01-11 00:00:00+00\")}", + "2024-01-18 00:00:00+00" + ), + new DerivedEvent( + "3", + "DerivationB", + "Derivation_Test_01.json", + DERIVATION_GROUP + "_2", + "2024-01-02 23:00:00+00", + "03:00:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 12:00:00+00\"),[\"2024-01-02 12:00:00+00\",\"2024-01-03 00:00:00+00\")}", + "2024-01-19 00:00:00+00" + ), + new DerivedEvent( + "1", + "DerivationA", + "Derivation_Test_01.json", + DERIVATION_GROUP + "_2", + "2024-01-01 00:00:00+00", + "02:10:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 12:00:00+00\"),[\"2024-01-02 12:00:00+00\",\"2024-01-03 00:00:00+00\")}", + "2024-01-19 00:00:00+00" + ), + new DerivedEvent( + "1", + "DerivationA", + "Derivation_Test_01.json", + DERIVATION_GROUP, + "2024-01-01 00:00:00+00", + "02:10:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 12:00:00+00\"),[\"2024-01-02 12:00:00+00\",\"2024-01-03 00:00:00+00\")}", + "2024-01-19 00:00:00+00" + ), + new DerivedEvent( + "3", + "DerivationB", + "Derivation_Test_01.json", + DERIVATION_GROUP, + "2024-01-02 23:00:00+00", + "03:00:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 12:00:00+00\"),[\"2024-01-02 12:00:00+00\",\"2024-01-03 00:00:00+00\")}", + "2024-01-19 00:00:00+00" + ), + new DerivedEvent( + "5", + "DerivationC", + "Derivation_Test_02.json", + DERIVATION_GROUP + "_2", + "2024-01-05 23:00:00+00", + "01:10:00", + "{[\"2024-01-03 00:00:00+00\",\"2024-01-10 00:00:00+00\")}", + "2024-01-20 00:00:00+00" + ), + new DerivedEvent( + "6", + "DerivationC", + "Derivation_Test_02.json", + DERIVATION_GROUP + "_2", + "2024-01-06 12:00:00+00", + "02:00:00", + "{[\"2024-01-03 00:00:00+00\",\"2024-01-10 00:00:00+00\")}", + "2024-01-20 00:00:00+00" + ), + new DerivedEvent( + "2", + "DerivationB", + "Derivation_Test_02.json", + DERIVATION_GROUP + "_2", + "2024-01-09 11:00:00+00", + "01:05:00", + "{[\"2024-01-03 00:00:00+00\",\"2024-01-10 00:00:00+00\")}", + "2024-01-20 00:00:00+00" + ), + new DerivedEvent( + "5", + "DerivationC", + "Derivation_Test_02.json", + DERIVATION_GROUP, + "2024-01-05 23:00:00+00", + "01:10:00", + "{[\"2024-01-03 00:00:00+00\",\"2024-01-10 00:00:00+00\")}", + "2024-01-20 00:00:00+00" + ), + new DerivedEvent( + "6", + "DerivationC", + "Derivation_Test_02.json", + DERIVATION_GROUP, + "2024-01-06 12:00:00+00", + "02:00:00", + "{[\"2024-01-03 00:00:00+00\",\"2024-01-10 00:00:00+00\")}", + "2024-01-20 00:00:00+00" + ), + new DerivedEvent( + "2", + "DerivationB", + "Derivation_Test_02.json", + DERIVATION_GROUP, + "2024-01-09 11:00:00+00", + "01:05:00", + "{[\"2024-01-03 00:00:00+00\",\"2024-01-10 00:00:00+00\")}", + "2024-01-20 00:00:00+00" + ), + new DerivedEvent( + "9", + "DerivationC", + "Derivation_Test_03.json", + DERIVATION_GROUP + "_2", + "2024-01-02 00:00:00+00", + "01:00:00", + "{[\"2024-01-01 12:00:00+00\",\"2024-01-02 12:00:00+00\")}", + "2024-01-21 00:00:00+00" + ), + new DerivedEvent( + "9", + "DerivationC", + "Derivation_Test_03.json", + DERIVATION_GROUP, + "2024-01-02 00:00:00+00", + "01:00:00", + "{[\"2024-01-01 12:00:00+00\",\"2024-01-02 12:00:00+00\")}", + "2024-01-21 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + + // verify for a given DERIVATION_GROUP expected keys are correct, no overlap inside DERIVATION_GROUP + results = getDerivedEvents(dg2); + + expectedResults = List.of( + new DerivedEvent( + "1", + "DerivationA", + "Derivation_Test_01.json", + DERIVATION_GROUP + "_2", + "2024-01-01 00:00:00+00", + "02:10:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 12:00:00+00\"),[\"2024-01-02 12:00:00+00\",\"2024-01-03 00:00:00+00\")}", + "2024-01-19 00:00:00+00" + ), + new DerivedEvent( + "9", + "DerivationC", + "Derivation_Test_03.json", + DERIVATION_GROUP + "_2", + "2024-01-02 00:00:00+00", + "01:00:00", + "{[\"2024-01-01 12:00:00+00\",\"2024-01-02 12:00:00+00\")}", + "2024-01-21 00:00:00+00" + ), + new DerivedEvent( + "3", + "DerivationB", + "Derivation_Test_01.json", + DERIVATION_GROUP + "_2", + "2024-01-02 23:00:00+00", + "03:00:00", + "{[\"2024-01-01 00:00:00+00\",\"2024-01-01 12:00:00+00\"),[\"2024-01-02 12:00:00+00\",\"2024-01-03 00:00:00+00\")}", + "2024-01-19 00:00:00+00" + ), + new DerivedEvent( + "5", + "DerivationC", + "Derivation_Test_02.json", + DERIVATION_GROUP + "_2", + "2024-01-05 23:00:00+00", + "01:10:00", + "{[\"2024-01-03 00:00:00+00\",\"2024-01-10 00:00:00+00\")}", + "2024-01-20 00:00:00+00" + ), + new DerivedEvent( + "6", + "DerivationC", + "Derivation_Test_02.json", + DERIVATION_GROUP + "_2", + "2024-01-06 12:00:00+00", + "02:00:00", + "{[\"2024-01-03 00:00:00+00\",\"2024-01-10 00:00:00+00\")}", + "2024-01-20 00:00:00+00" + ), + new DerivedEvent( + "2", + "DerivationB", + "Derivation_Test_02.json", + DERIVATION_GROUP + "_2", + "2024-01-09 11:00:00+00", + "01:05:00", + "{[\"2024-01-03 00:00:00+00\",\"2024-01-10 00:00:00+00\")}", + "2024-01-20 00:00:00+00" + ), + new DerivedEvent( + "8", + "DerivationB", + "Derivation_Test_00.json", + DERIVATION_GROUP + "_2", + "2024-01-10 11:00:00+00", + "01:05:00", + "{[\"2024-01-10 00:00:00+00\",\"2024-01-11 00:00:00+00\")}", + "2024-01-18 00:00:00+00" + ) + ); + assertEquals(expectedResults.size(), results.size()); + assertTrue(results.containsAll(expectedResults)); + } + } + } + //endregion + + + //region Constraint Tests + /** + * The focus of this class is to provide a final set of tests to test primary and significant constraints to prove + * that they work and how they work. These constraints are baked into the schema, so extensive tests of foreign or + * unique keys are not excluded except for common use cases/easily made mistakes, as PSQL guarantees functionality + * for us. + * We seek to demonstrate in this class, and verify as a sanity check. + */ + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class ConstraintTests { + + /** + * A source's end time must be greater than its start time. + */ + @Test + void endTimeGEstartTime() throws SQLException { + // construct the sources + ExternalSource endBeforeStart = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + "2024-01-01T00:30:00Z", + CREATED_AT + ); + ExternalSource endMatchesStart = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + "2024-01-01T01:00:00Z", + CREATED_AT + ); + ExternalSource succeeding = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + "2024-01-01T01:00:00.000001Z", + CREATED_AT + ); + + // add source type and derivation group + // create the source type + insertExternalSourceType(endBeforeStart.source_type_name()); + + // create the derivation_group + insertDerivationGroup(endBeforeStart.derivation_group_name(), endBeforeStart.source_type_name()); + + // if start time > end time, error + final SQLException ex = assertThrows(PSQLException.class, () -> insertExternalSource(endBeforeStart)); + if (!ex.getSQLState().equals("23514") + && !ex.getMessage().contains("new row for relation \"external_source\" violates check constraint " + + "\"external_source_check\"")) { + throw ex; + } + + // if start time = end time, error + final SQLException ex2 = assertThrows(PSQLException.class, () -> insertExternalSource(endMatchesStart)); + if (!ex2.getSQLState().equals("23514") + && !ex2.getMessage().contains("new row for relation \"external_source\" violates check constraint " + + "\"external_source_check\"")) { + throw ex2; + } + + // else, no error + assertDoesNotThrow(() -> insertExternalSource(succeeding)); + } + + /** + * An event must be completely within the bounds of its source. + * + * Source : +++++++++++++++ + * Before1: 1111 + * Before2: 2222 + * After1 : 3333 + * After2 : 4444 + */ + @Test + void externalEventSourceBounds() throws SQLException { + // create sources and events + ExternalSource A = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T01:00:00Z", + "2024-01-01T02:00:00Z", + CREATED_AT); + + // legal. + ExternalEvent legal = new ExternalEvent("a", "2024-01-01T01:00:00Z", "00:10:00", A); + + // illegal! + ExternalEvent completelyBefore = new ExternalEvent("completelyBefore", "2024-01-01T00:00:00Z", "00:10:00", A); + ExternalEvent beforeIntersect = new ExternalEvent("beforeIntersect", "2024-01-01T00:55:00Z", "00:25:00", A); + ExternalEvent afterIntersect = new ExternalEvent("afterIntersect", "2024-01-01T01:45:00Z", "00:30:00", A); + ExternalEvent completelyAfter = new ExternalEvent("completelyAfter", "2024-01-01T02:10:00Z", "00:15:00", A); + + // assert the legal event is okay (in the center of the source) + // insert generic external event type, source type, and derivation group + insertStandardTypes(); + + // insert sources + insertExternalSource(A); + + // insert events + insertExternalEvent(legal); + + // assert out of bounds failures + final SQLException ex = assertThrows(PSQLException.class, () -> insertExternalEvent(completelyBefore)); + if (!ex.getMessage().contains("Event " + completelyBefore.key + " out of bounds of source " + A.key + ".")) { + throw ex; + } + + final SQLException ex2 = assertThrows(PSQLException.class, () -> insertExternalEvent(beforeIntersect)); + if (!ex2.getMessage().contains("Event " + beforeIntersect.key + " out of bounds of source " + A.key + ".")) { + throw ex2; + } + + final SQLException ex3 = assertThrows(PSQLException.class, () -> insertExternalEvent(afterIntersect)); + if (!ex3.getMessage().contains("Event " + afterIntersect.key + " out of bounds of source " + A.key + ".")) { + throw ex3; + } + + final SQLException ex4 = assertThrows(PSQLException.class, () -> insertExternalEvent(completelyAfter)); + if (!ex4.getMessage().contains("Event " + completelyAfter.key + " out of bounds of source " + A.key + ".")) { + throw ex4; + } + } + + /** + * Source names must be unique within a derivation group; they can repeat across derivation groups. + */ + @Test + void duplicateSource() throws SQLException { + // same name and DERIVATION_GROUP + ExternalSource failing = new ExternalSource( + "Derivation_Test_00.json", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-18 00:00:00+00", + "2024-01-05 00:00:00+00", + "2024-01-11 00:00:00+00", + CREATED_AT + ); + // same name, diff DERIVATION_GROUP + ExternalSource succeeding = new ExternalSource( + "Derivation_Test_00.json", + SOURCE_TYPE, + DERIVATION_GROUP + "_2", + "2024-01-18 00:00:00+00", + "2024-01-05 00:00:00+00", + "2024-01-11 00:00:00+00", + CREATED_AT + ); + + // upload general data + upload_source(DERIVATION_GROUP, false); + + // upload a conflicting source (same name in a given DERIVATION_GROUP) + final SQLException ex = assertThrows(PSQLException.class, () -> insertExternalSource(failing)); + if (!ex.getSQLState().equals("23505") + && !ex.getMessage().contains("duplicate key value violates unique constraint \"external_source_pkey\"")) { + throw ex; + } + + // upload a non-conflicting source (same name in a different DERIVATION_GROUP) + insertDerivationGroup(DERIVATION_GROUP + "_2", SOURCE_TYPE); + insertExternalSource(succeeding); + } + + /** + * It is impossible to delete a derivation group if there are any sources that are a part of it. + */ + @Test + void deleteDGwithRemainingSource() throws SQLException { + try(final var statement = connection.createStatement()) { + + // create the source + ExternalSource src = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T00:30:00Z", + CREATED_AT + ); + + // insert the source and all relevant groups and types + // create a source type + insertExternalSourceType(SOURCE_TYPE); + + // create a Derivation Group + insertDerivationGroup(DERIVATION_GROUP, SOURCE_TYPE); + + // create a source + insertExternalSource(src); + + // delete the DG (expect error) + final SQLException ex = assertThrows(PSQLException.class, + () -> statement.executeUpdate( + // language=sql + """ + DELETE FROM merlin.derivation_group WHERE name='%s'; + """.formatted(DERIVATION_GROUP) + ) + ); + if (!ex.getSQLState().equals("23503") && + !ex.getMessage().contains( + "update or delete on table \"derivation_group\" violates foreign key constraint " + + "\"external_source_type_matches_derivation_group\" on table \"external_source\"") + ) { + throw ex; + } + } + } + + /** + * An external source's type MUST match the derivation group's source type. + */ + @Test + void externalSourceTypeMatchDerivationGroup() throws SQLException { + // create a source that matches the derivation group + ExternalSource src = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T00:30:00Z", + CREATED_AT + ); + + // add types + // create a source type + insertExternalSourceType(SOURCE_TYPE); + + // create a Derivation Group + insertDerivationGroup(DERIVATION_GROUP, SOURCE_TYPE); + + // insert the source + insertExternalSource(src); + + // create a source that doesn't match the derivation group (the source type has "_B" appended to it) + ExternalSource src_2 = new ExternalSource( + "B", + SOURCE_TYPE + "_B", + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T00:30:00Z", + CREATED_AT + ); + + // insert the erroneous source (expect error) + final SQLException ex = assertThrows( + SQLException.class, + () -> insertExternalSource(src_2) + ); + if (!ex.getSQLState().equals("23503") && + !ex.getMessage().contains( + "ERROR: External source " + src_2.key + " is being added to a derivation group " + + src_2.derivation_group_name + " where its type " + src_2.source_type_name + + " does not match the derivation group type " + src.source_type_name + ".") + ) { + throw ex; + } + } + + /** + * It is not be possible to delete a source type with a source still associated. + */ + @Test + void deleteSourceTypeWithRemainingSource() throws SQLException { + try(final var statement = connection.createStatement()) { + + // create source + ExternalSource src = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T00:30:00Z", + CREATED_AT + ); + + // add types + // create a source type + insertExternalSourceType(SOURCE_TYPE); + + // create a Derivation Group + insertDerivationGroup(DERIVATION_GROUP, SOURCE_TYPE); + + // create a source + insertExternalSource(src); + + // delete the source type (expect error) + final SQLException ex = assertThrows(PSQLException.class, () -> statement.executeUpdate( + // language=sql + """ + DELETE FROM merlin.external_source_type WHERE name='%s'; + """.formatted(SOURCE_TYPE) + ) + ); + if (!ex.getSQLState().equals("23503") + && !ex.getMessage().contains( + "update or delete on table \"external_source_type\" violates foreign key constraint " + + "\"derivation_group_references_external_source_type\" on table \"derivation_group\"") + ) { + throw ex; + } + } + } + + /** + * It is not be possible to delete an event type with an event still associated. + */ + @Test + void deleteEventTypeWithRemainingEvent() throws SQLException { + try (final var statement = connection.createStatement()) { + + // create source and event + ExternalSource src = new ExternalSource( + "A", + SOURCE_TYPE, + DERIVATION_GROUP, + "2024-01-01T00:00:00Z", + "2024-01-01T00:00:00Z", + "2024-01-01T00:30:00Z", + CREATED_AT + ); + ExternalEvent evt = new ExternalEvent("A_1", "2024-01-01T00:00:00Z", "00:05:0", src); + + // insert the event and her types + // insert generic external event type, source type, and derivation group + insertStandardTypes(); + + // insert sources + insertExternalSource(src); + + // insert events + insertExternalEvent(evt); + + // delete the event type (expect error) + final SQLException ex = assertThrows(PSQLException.class, + () -> statement.executeUpdate( + // language=sql + """ + DELETE FROM merlin.external_event_type WHERE name='%s'; + """.formatted(EVENT_TYPE) + ) + ); + if (!ex.getSQLState().equals("23503") + && !ex.getMessage().contains( + "update or delete on table \"external_event_type\" violates foreign key constraint " + + "\"external_event_references_event_type_name\" on table \"external_event\"") + ) { + throw ex; + } + } + } + } + //endregion + + + //region Derivation Group Association Tests + /** + * The following test Derivation Group/Plan Association behavior. + */ + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class PlanDerivationGroupTests { + List getPlanDerivationGroupAssociations() throws SQLException { + List results = new ArrayList<>(); + + try (final var statement = connection.createStatement()) { + // create the event type + final var res = statement.executeQuery( + // language=sql + """ + SELECT * FROM merlin.plan_derivation_group ORDER BY plan_id, derivation_group_name; + """ + ); + + while (res.next()) { + results.add( + new PlanDerivationGroup( + res.getInt("plan_id"), + res.getString("derivation_group_name"), + res.getBoolean("acknowledged"), + res.getString("last_acknowledged_at") + ) + ); + } + } + + return results; + } + + void assertEqualsAsideFromLastAcknowledged(List expected, List actual) { + assertEquals(expected.size(), actual.size()); + for (int i = 0; i < actual.size(); i++) { + assertEquals(expected.get(i).plan_id(), actual.get(i).plan_id()); + assertEquals(expected.get(i).derivation_group_name(), actual.get(i).derivation_group_name()); + assertEquals(expected.get(i).acknowledged(), actual.get(i).acknowledged()); + } + } + + /** + * This test ensures that a derivation group can be associated with a plan, specifically checking that when the + * "acknowledged" field is set, "last_acknowledged_at" is not updated, but no other derivation groups or plans + * (and their associations are affected) + */ + @Test + void associateDerivationGroupWithBasePlan() throws SQLException { + // upload a mission model + final int fileId = merlinHelper.insertFileUpload(); + final int missionModelId = merlinHelper.insertMissionModel(fileId); + + // upload derivation group A and B + String derivationGroupNameControl = "Control"; + String derivationGroupNameDelta = "Delta"; + upload_source(derivationGroupNameControl, false); + upload_source(derivationGroupNameDelta, true); + + // create plan "control", as well as one that would face an update to an associated derivation group "delta" + final int planIdControl = merlinHelper.insertPlan(missionModelId, merlinHelper.user.name(), "control"); + final int planIdDelta = merlinHelper.insertPlan(missionModelId, merlinHelper.user.name(), "delta"); + + // associate the two derivation groups with it + assertDoesNotThrow(() -> associateDerivationGroupWithPlan(planIdControl, derivationGroupNameControl)); + assertDoesNotThrow(() -> associateDerivationGroupWithPlan(planIdDelta, derivationGroupNameControl)); + assertDoesNotThrow(() -> associateDerivationGroupWithPlan(planIdDelta, derivationGroupNameDelta)); + + // check that acknowledged is true for all entries, and save the last_acknowledged_at field results + List expectedResultsInitial = List.of( + new PlanDerivationGroup(planIdControl, derivationGroupNameControl, true, ""), + new PlanDerivationGroup(planIdDelta, derivationGroupNameControl, true, ""), + new PlanDerivationGroup(planIdDelta, derivationGroupNameDelta, true, "") + ); + final List actualResultsInitial = getPlanDerivationGroupAssociations(); + + // first, check other properties + assertEqualsAsideFromLastAcknowledged(expectedResultsInitial, actualResultsInitial); + + // insert a source to the changing derivation group + insertExternalSource( + new ExternalSource( + "A.json", + SOURCE_TYPE, + derivationGroupNameDelta, + "2024-01-19 00:00:01+00", + "2024-01-01 00:00:00+00", + "2024-01-07 00:00:00+00", + CREATED_AT + ) + ); + + // check that acknowledged is now false for all non control (delta only) entries, true otherwise + List expectedResults = List.of( + new PlanDerivationGroup(planIdControl, derivationGroupNameControl, true, actualResultsInitial.getFirst() + .last_acknowledged_at()), + new PlanDerivationGroup(planIdDelta, derivationGroupNameControl, true, actualResultsInitial.get(1) + .last_acknowledged_at()), + new PlanDerivationGroup(planIdDelta, derivationGroupNameDelta, false, actualResultsInitial.get(2) + .last_acknowledged_at()) + ); + final List actualResults = getPlanDerivationGroupAssociations(); + + // first, check other properties + assertEquals(expectedResults.size(), actualResults.size()); + assertTrue(actualResults.containsAll(expectedResults)); + + // final bit - update acknowledged to true for (planIdDelta, derivationGroupNameDelta) pair, see that + // last_acknowledged_at updates + try (final var statement = connection.createStatement()) { + // update acknowledged field for (planIdDelta, derivationGroupNameDelta) pair + statement.executeUpdate( + // language=sql + """ + UPDATE merlin.plan_derivation_group + SET acknowledged = true + WHERE (plan_id, derivation_group_name) = (%d, '%s'); + """.formatted(planIdDelta, derivationGroupNameDelta) + ); + } + + final List postUpdateResults = getPlanDerivationGroupAssociations(); + + // check timestamps + assertEquals(postUpdateResults.getFirst(), actualResultsInitial.getFirst()); + assertEquals(postUpdateResults.get(1), actualResultsInitial.get(1)); + + // comparing strings. The post update last_acknowledged_at should be later than the pre update one. + assertTrue(postUpdateResults.get(2).last_acknowledged_at().compareTo( + actualResultsInitial.get(2).last_acknowledged_at()) > 0); + + // check the other properties + assertEqualsAsideFromLastAcknowledged(actualResultsInitial, postUpdateResults); + } + } + //endregion +} diff --git a/deployment/hasura/metadata/databases/tables/merlin/derivation_group.yaml b/deployment/hasura/metadata/databases/tables/merlin/derivation_group.yaml new file mode 100644 index 0000000000..295485faf3 --- /dev/null +++ b/deployment/hasura/metadata/databases/tables/merlin/derivation_group.yaml @@ -0,0 +1,62 @@ +table: + name: derivation_group + schema: merlin +configuration: + custom_name: "derivation_group" +object_relationships: + - name: external_source_type + using: + foreign_key_constraint_on: source_type_name +array_relationships: + - name: external_sources + using: + foreign_key_constraint_on: + columns: + - derivation_group_name + table: + name: external_source + schema: merlin + - name: derived_events + using: + manual_configuration: + remote_table: + name: derived_events + schema: merlin + column_mapping: + name: derivation_group_name +select_permissions: + - role: aerie_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: user + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: viewer + permission: + columns: '*' + filter: {} + allow_aggregations: true +insert_permissions: + - role: aerie_admin + permission: + columns: [name, source_type_name] + check: {} + set: + owner: "x-hasura-user-id" + - role: user + permission: + columns: [name, source_type_name] + check: {} + set: + owner: "x-hasura-user-id" +delete_permissions: + - role: aerie_admin + permission: + filter: {} + - role: user + permission: + filter: {"owner":{"_eq":"x-hasura-user-id"}} diff --git a/deployment/hasura/metadata/databases/tables/merlin/derived_events.yaml b/deployment/hasura/metadata/databases/tables/merlin/derived_events.yaml new file mode 100644 index 0000000000..bf2c3a87ef --- /dev/null +++ b/deployment/hasura/metadata/databases/tables/merlin/derived_events.yaml @@ -0,0 +1,42 @@ +table: + name: derived_events + schema: merlin +configuration: + custom_name: "derived_events" +object_relationships: + - name: external_source + using: + manual_configuration: + remote_table: + name: external_source + schema: merlin + column_mapping: + derivation_group_name: derivation_group_name + - name: external_event + using: + manual_configuration: + remote_table: + schema: merlin + name: external_event + insertion_order: null + column_mapping: + event_type_name: event_type_name + event_key: key + source_key: source_key + derivation_group_name: derivation_group_name +select_permissions: + - role: aerie_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: user + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: viewer + permission: + columns: '*' + filter: {} + allow_aggregations: true diff --git a/deployment/hasura/metadata/databases/tables/merlin/external_event.yaml b/deployment/hasura/metadata/databases/tables/merlin/external_event.yaml new file mode 100644 index 0000000000..1912fb779a --- /dev/null +++ b/deployment/hasura/metadata/databases/tables/merlin/external_event.yaml @@ -0,0 +1,43 @@ +table: + name: external_event + schema: merlin +configuration: + custom_name: "external_event" +object_relationships: +- name: external_source + using: + foreign_key_constraint_on: + - source_key + - derivation_group_name +- name: external_event_type + using: + foreign_key_constraint_on: event_type_name +select_permissions: + - role: aerie_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: user + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: viewer + permission: + columns: '*' + filter: {} + allow_aggregations: true +insert_permissions: + - role: aerie_admin + permission: + columns: [key, event_type_name, source_key, derivation_group_name, start_time, duration] + check: {} + - role: user + permission: + columns: [key, event_type_name, source_key, derivation_group_name, start_time, duration] + check: {} +delete_permissions: + - role: aerie_admin + permission: + filter: {} diff --git a/deployment/hasura/metadata/databases/tables/merlin/external_event_type.yaml b/deployment/hasura/metadata/databases/tables/merlin/external_event_type.yaml new file mode 100644 index 0000000000..2c4b6f7d70 --- /dev/null +++ b/deployment/hasura/metadata/databases/tables/merlin/external_event_type.yaml @@ -0,0 +1,42 @@ +table: + name: external_event_type + schema: merlin +configuration: + custom_name: "external_event_type" +array_relationships: + - name: external_events + using: + foreign_key_constraint_on: + column: event_type_name + table: + name: external_event + schema: merlin +select_permissions: + - role: aerie_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: user + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: viewer + permission: + columns: '*' + filter: {} + allow_aggregations: true +insert_permissions: + - role: aerie_admin + permission: + columns: [name] + check: {} + - role: user + permission: + columns: [name] + check: {} +delete_permissions: + - role: aerie_admin + permission: + filter: {} diff --git a/deployment/hasura/metadata/databases/tables/merlin/external_source.yaml b/deployment/hasura/metadata/databases/tables/merlin/external_source.yaml new file mode 100644 index 0000000000..e3d062e70a --- /dev/null +++ b/deployment/hasura/metadata/databases/tables/merlin/external_source.yaml @@ -0,0 +1,63 @@ +table: + name: external_source + schema: merlin +configuration: + custom_name: "external_source" +array_relationships: + - name: external_events + using: + foreign_key_constraint_on: + columns: + - source_key + - derivation_group_name + table: + name: external_event + schema: merlin +object_relationships: + - name: external_source_type + using: + foreign_key_constraint_on: source_type_name + - name: derivation_group + using: + foreign_key_constraint_on: derivation_group_name +select_permissions: + - role: aerie_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: user + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: viewer + permission: + columns: '*' + filter: {} + allow_aggregations: true +insert_permissions: + - role: aerie_admin + permission: + columns: [key, source_type_name, valid_at, start_time, end_time, derivation_group_name, created_at] + check: {} + set: + owner: "x-hasura-user-id" + - role: user + permission: + columns: [key, source_type_name, valid_at, start_time, end_time, derivation_group_name, created_at] + check: {} + set: + owner: "x-hasura-user-id" +delete_permissions: + - role: aerie_admin + permission: + filter: {} + - role: user + permission: + filter: { + "_or": [ + { "owner": { "_eq": "x-hasura-user-id" } }, + { "derivation_group": { "owner": { "_eq": "x-hasura-user-id" } } } + ] + } diff --git a/deployment/hasura/metadata/databases/tables/merlin/external_source_type.yaml b/deployment/hasura/metadata/databases/tables/merlin/external_source_type.yaml new file mode 100644 index 0000000000..0cce0ab66b --- /dev/null +++ b/deployment/hasura/metadata/databases/tables/merlin/external_source_type.yaml @@ -0,0 +1,49 @@ +table: + name: external_source_type + schema: merlin +configuration: + custom_name: "external_source_type" +array_relationships: + - name: external_sources + using: + foreign_key_constraint_on: + column: source_type_name + table: + name: external_source + schema: merlin + - name: derivation_groups + using: + foreign_key_constraint_on: + column: source_type_name + table: + name: derivation_group + schema: merlin +select_permissions: + - role: aerie_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: user + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: viewer + permission: + columns: '*' + filter: {} + allow_aggregations: true +insert_permissions: + - role: aerie_admin + permission: + columns: [name] + check: {} + - role: user + permission: + columns: [name] + check: {} +delete_permissions: + - role: aerie_admin + permission: + filter: {} diff --git a/deployment/hasura/metadata/databases/tables/merlin/plan_derivation_group.yaml b/deployment/hasura/metadata/databases/tables/merlin/plan_derivation_group.yaml new file mode 100644 index 0000000000..24bea9254c --- /dev/null +++ b/deployment/hasura/metadata/databases/tables/merlin/plan_derivation_group.yaml @@ -0,0 +1,86 @@ +table: + name: plan_derivation_group + schema: merlin +configuration: + custom_name: "plan_derivation_group" +object_relationships: +- name: plan + using: + foreign_key_constraint_on: plan_id +- name: derivation_group + using: + foreign_key_constraint_on: derivation_group_name +select_permissions: + - role: aerie_admin + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: user + permission: + columns: '*' + filter: {} + allow_aggregations: true + - role: viewer + permission: + columns: '*' + filter: {} + allow_aggregations: true +insert_permissions: + - role: aerie_admin + permission: + columns: [plan_id, derivation_group_name] + check: {} + - role: user + permission: + columns: [plan_id, derivation_group_name] + check: { + "plan": { + "_or": [ + { + "owner": { "_eq": "X-Hasura-User-Id" } + }, + { + "collaborators": { "collaborator": { "_eq": "X-Hasura-User-Id" } } + } + ] + } + } +update_permissions: + - role: aerie_admin + permission: + columns: [ acknowledged ] + filter: {} + - role: user + permission: + columns: [ acknowledged ] + filter: { + "plan": { + "_or": [ + { + "owner": { "_eq": "X-Hasura-User-Id" } + }, + { + "collaborators": { "collaborator": { "_eq": "X-Hasura-User-Id" } } + } + ] + } + } +delete_permissions: + - role: aerie_admin + permission: + filter: {} + - role: user + permission: + filter: { + "plan": { + "_or": [ + { + "owner": { "_eq": "X-Hasura-User-Id" } + }, + { + "collaborators": { "collaborator": { "_eq": "X-Hasura-User-Id" } } + } + ] + } + } diff --git a/deployment/hasura/metadata/databases/tables/tables.yaml b/deployment/hasura/metadata/databases/tables/tables.yaml index 16d4ac5a14..40b07b3cf3 100644 --- a/deployment/hasura/metadata/databases/tables/tables.yaml +++ b/deployment/hasura/metadata/databases/tables/tables.yaml @@ -97,6 +97,15 @@ # External Datasets - "!include merlin/plan_dataset.yaml" +# External Events +- "!include merlin/derivation_group.yaml" +- "!include merlin/derived_events.yaml" +- "!include merlin/external_event.yaml" +- "!include merlin/external_event_type.yaml" +- "!include merlin/external_source.yaml" +- "!include merlin/external_source_type.yaml" +- "!include merlin/plan_derivation_group.yaml" + # Constraints - "!include merlin/constraints/constraint_metadata.yaml" - "!include merlin/constraints/constraint_definition.yaml" diff --git a/deployment/hasura/migrations/Aerie/11_external_events/down.sql b/deployment/hasura/migrations/Aerie/11_external_events/down.sql new file mode 100644 index 0000000000..5280dfe186 --- /dev/null +++ b/deployment/hasura/migrations/Aerie/11_external_events/down.sql @@ -0,0 +1,33 @@ +-- up.sql creates table and sequence, delete them +drop trigger refresh_derived_events_on_derivation_group on merlin.derivation_group; +drop trigger refresh_derived_events_on_external_source on merlin.external_source; +drop trigger refresh_derived_events_on_external_event on merlin.external_event; +drop function merlin.refresh_derived_events_on_trigger cascade; +drop materialized view merlin.derived_events; + +drop function merlin.subtract_later_ranges cascade; + +drop trigger external_source_pdg_association_delete on merlin.external_source; +drop function merlin.external_source_pdg_association_delete cascade; +drop trigger external_source_pdg_ack_update on merlin.external_source; +drop function merlin.external_source_pdg_ack_update cascade; + +drop trigger pdg_update_ack_at on merlin.plan_derivation_group; +drop function merlin.pdg_update_ack_at cascade; +drop table merlin.plan_derivation_group cascade; + +drop trigger check_external_event_boundaries on merlin.external_event; +drop function merlin.check_external_event_boundaries cascade; + +drop trigger check_external_event_duration_is_nonnegative_trigger on merlin.external_event; +drop table merlin.external_event cascade; + +drop trigger external_source_type_matches_dg_on_add on merlin.external_source; +drop function merlin.external_source_type_matches_dg_on_add; +drop table merlin.external_source cascade; + +drop table merlin.derivation_group cascade; +drop table merlin.external_event_type cascade; +drop table merlin.external_source_type cascade; + +call migrations.mark_migration_rolled_back('11'); diff --git a/deployment/hasura/migrations/Aerie/11_external_events/up.sql b/deployment/hasura/migrations/Aerie/11_external_events/up.sql new file mode 100644 index 0000000000..44dfe85c9e --- /dev/null +++ b/deployment/hasura/migrations/Aerie/11_external_events/up.sql @@ -0,0 +1,419 @@ +create table merlin.external_source_type ( + name text not null, + + constraint external_source_type_pkey + primary key (name) +); + +comment on table merlin.external_source_type is e'' + 'Externally imported event source types (each external source has to be of a certain type).\n' + 'They are also helpful to classify external sources.\n' + 'Derivation groups are a subclass of external source type.'; + +comment on column merlin.external_source_type.name is e'' + 'The identifier for this external_source_type, as well as its name.'; + +create table merlin.external_event_type ( + name text not null, + + constraint external_event_type_pkey + primary key (name) +); + +comment on table merlin.external_event_type is e'' + 'Externally imported event types.'; + +comment on column merlin.external_event_type.name is e'' + 'The identifier for this external_event_type, as well as its name.'; + +create table merlin.derivation_group ( + name text not null, + source_type_name text not null, + owner text, + + constraint derivation_group_pkey + primary key (name), + constraint derivation_group_references_external_source_type + foreign key (source_type_name) + references merlin.external_source_type(name) + on update cascade + on delete restrict, + constraint derivation_group_owner_exists + foreign key (owner) references permissions.users + on update cascade + on delete set null +); + +comment on table merlin.derivation_group is e'' + 'A collection of external sources of the same type that the derivation operation is run against.'; + +comment on column merlin.derivation_group.name is e'' + 'The name and primary key of the derivation group.'; +comment on column merlin.derivation_group.source_type_name is e'' + 'The name of the external_source_type of sources in this derivation group.'; +comment on column merlin.derivation_group.owner is e'' + 'The name of the user that created this derivation_group.'; + +create table merlin.external_source ( + key text not null, + source_type_name text not null, + derivation_group_name text not null, + valid_at timestamp with time zone not null, + start_time timestamp with time zone not null, + end_time timestamp with time zone not null, + CHECK (end_time > start_time), + created_at timestamp with time zone default now() not null, + owner text, + + constraint external_source_pkey + primary key (key, derivation_group_name), + -- a given dg cannot have two sources with the same valid_at! + CONSTRAINT dg_unique_valid_at UNIQUE (derivation_group_name, valid_at), + constraint external_source_references_external_source_type_name + foreign key (source_type_name) + references merlin.external_source_type(name) + on update cascade + on delete restrict, + constraint external_source_type_matches_derivation_group + foreign key (derivation_group_name) + references merlin.derivation_group (name) + on update cascade + on delete restrict, + constraint external_source_owner_exists + foreign key (owner) references permissions.users + on update cascade + on delete set null +); + +comment on table merlin.external_source is e'' + 'Externally imported event sources.'; + +comment on column merlin.external_source.key is e'' + 'The key, or name, of the external_source.\n' + 'Part of the primary key, along with the derivation_group_name'; +comment on column merlin.external_source.source_type_name is e'' + 'The type of this external_source.'; +comment on column merlin.external_source.derivation_group_name is e'' + 'The name of the derivation_group that this external_source is included in.'; +comment on column merlin.external_source.valid_at is e'' + 'The time (in _planner_ time, NOT plan time) at which a source becomes valid.\n' + 'This time helps determine when a source''s events are valid for the span of time it covers.'; +comment on column merlin.external_source.start_time is e'' + 'The start time (in _plan_ time, NOT planner time), of the range that this source describes.'; +comment on column merlin.external_source.end_time is e'' + 'The end time (in _plan_ time, NOT planner time), of the range that this source describes.'; +comment on column merlin.external_source.created_at is e'' + 'The time (in _planner_ time, NOT plan time) that this particular source was created.\n' + 'This column is used primarily for documentation purposes, and has no associated functionality.'; +comment on column merlin.external_source.owner is e'' + 'The user who uploaded the external source.'; + +-- make sure new sources' source_type match that of their derivation group! +create function merlin.external_source_type_matches_dg_on_add() + returns trigger + language plpgsql as $$ +declare + source_type text; +begin + select into source_type derivation_group.source_type_name from merlin.derivation_group where name = new.derivation_group_name; + if source_type is distinct from new.source_type_name then + raise foreign_key_violation + using message='External source ' || new.key || ' is being added to a derivation group ' || new.derivation_group_name + || ' where its type ' || new.source_type_name || ' does not match the derivation group type ' + || source_type || '.' ; + end if; + return new; +end; +$$; + +create trigger external_source_type_matches_dg_on_add +before insert or update on merlin.external_source + for each row execute function merlin.external_source_type_matches_dg_on_add(); + +create table merlin.external_event ( + key text not null, + event_type_name text not null, + source_key text not null, + derivation_group_name text not null, + start_time timestamp with time zone not null, + duration interval not null, + + constraint external_event_pkey + primary key (key, source_key, derivation_group_name, event_type_name), + constraint external_event_references_source_key_derivation_group + foreign key (source_key, derivation_group_name) + references merlin.external_source (key, derivation_group_name) + on update cascade + on delete cascade, + constraint external_event_references_event_type_name + foreign key (event_type_name) + references merlin.external_event_type(name) + on update cascade + on delete restrict +); + +comment on table merlin.external_event is e'' + 'Externally imported events.'; + +comment on column merlin.external_event.key is e'' + 'The key, or name, of the external_event.\n' + 'Part of the primary key, along with the source_key, derivation_group_name, and event_type_name.'; +comment on column merlin.external_event.event_type_name is e'' + 'The type of this external_event.'; +comment on column merlin.external_event.source_key is e'' + 'The key of the external_source that this external_event is included in.\n' + 'Used as a foreign key along with the derivation_group_name to directly identify said source.\n' + 'Part of the primary key along with the key, derivation_group_name, and event_type_name.'; +comment on column merlin.external_event.derivation_group_name is e'' + 'The derivation_group that the external_source bearing this external_event is a part of.'; +comment on column merlin.external_event.start_time is e'' + 'The start time (in _plan_ time, NOT planner time), of the range that this source describes.'; +comment on column merlin.external_event.duration is e'' + 'The span of time of this external event.'; + +create trigger check_external_event_duration_is_nonnegative_trigger +before insert or update on merlin.external_event +for each row +when (new.duration < '0') +execute function util_functions.raise_duration_is_negative(); + +create function merlin.check_external_event_boundaries() +returns trigger +language plpgsql as $$ +declare + source_start timestamp with time zone; + source_end timestamp with time zone; + event_start timestamp with time zone; + event_end timestamp with time zone; +begin + select start_time, end_time into source_start, source_end + from merlin.external_source + where new.source_key = external_source.key + and new.derivation_group_name = external_source.derivation_group_name; + + event_start := new.start_time; + event_end := new.start_time + new.duration; + if event_start < source_start or event_end > source_end then + raise exception 'Event % out of bounds of source %.', new.key, new.source_key; + end if; + return new; +end; +$$; + +comment on function merlin.check_external_event_boundaries() is e'' + 'Checks that an external_event added to the database has a start time and duration that fall in bounds of the associated external_source.'; + +create trigger check_external_event_boundaries +before insert on merlin.external_event + for each row execute function merlin.check_external_event_boundaries(); + +comment on trigger check_external_event_boundaries on merlin.external_event is e'' + 'Fires any time a new external event is added that checks that the span of the event fits in its referenced source.'; + +create table merlin.plan_derivation_group ( + plan_id integer not null, + derivation_group_name text not null, + last_acknowledged_at timestamp with time zone default now() not null, + acknowledged boolean not null default true, + + constraint plan_derivation_group_pkey + primary key (plan_id, derivation_group_name), + constraint pdg_plan_exists + foreign key (plan_id) + references merlin.plan(id) + on delete cascade, + constraint pdg_derivation_group_exists + foreign key (derivation_group_name) + references merlin.derivation_group(name) + on update cascade + on delete restrict +); + +comment on table merlin.plan_derivation_group is e'' + 'Links externally imported event sources & plans.\n' + 'Additionally, tracks the last time a plan owner/contributor(s) have acknowledged additions to the derivation group.\n'; + +comment on column merlin.plan_derivation_group.plan_id is e'' + 'The plan with which the derivation group is associated.'; +comment on column merlin.plan_derivation_group.derivation_group_name is e'' + 'The derivation group being associated with the plan.'; +comment on column merlin.plan_derivation_group.last_acknowledged_at is e'' + 'The time at which changes to the derivation group were last acknowledged.'; + +-- update last_acknowledged whenever acknowledged is set to true +create function merlin.pdg_update_ack_at() + returns trigger + language plpgsql as $$ +begin + if new.acknowledged = true then + new.last_acknowledged_at = now(); + end if; + return new; +end; +$$; + +create trigger pdg_update_ack_at +before update on merlin.plan_derivation_group + for each row execute function merlin.pdg_update_ack_at(); + +-- if an external source is linked to a plan it cannot be deleted +create function merlin.external_source_pdg_association_delete() + returns trigger + language plpgsql as $$ +begin + if exists (select * from merlin.plan_derivation_group pdg where pdg.derivation_group_name = old.derivation_group_name) then + raise foreign_key_violation + using message='External source ' || old.key || ' is part of a derivation group that is associated to a plan.'; + end if; + return old; +end; +$$; + +create trigger external_source_pdg_association_delete +before delete on merlin.external_source + for each row execute function merlin.external_source_pdg_association_delete(); + +-- set acknowledged on merlin.plan_derivation_group false for this derivation group as there are new changes +create function merlin.external_source_pdg_ack_update() + returns trigger + language plpgsql as $$ +begin + update merlin.plan_derivation_group set "acknowledged" = false + where plan_derivation_group.derivation_group_name = NEW.derivation_group_name; + return new; +end; +$$; + +create trigger external_source_pdg_ack_update +after insert on merlin.external_source + for each row execute function merlin.external_source_pdg_ack_update(); + +create function merlin.subtract_later_ranges(curr_date tstzmultirange, later_dates tstzmultirange[]) +returns tstzmultirange +immutable +language plpgsql as $$ + declare + ret tstzmultirange := curr_date; + later_date tstzmultirange; +begin + foreach later_date in array later_dates loop + ret := ret - later_date; + end loop; + return ret; +end +$$; + +comment on function merlin.subtract_later_ranges(curr_date tstzmultirange, later_dates tstzmultirange[]) is e'' + 'Used by the derived_events view that produces from the singular interval of time that a source covers a set of disjoint intervals.\n' + 'The disjointedness arises from where future sources'' spans are subtracted from this one.\n' + 'For example, if a source is valid at t=0, and covers span s=1 to s=5, and there is a source valid at t=1 with a span s=2 to s=3\n' + 'and another valid at t=2 with a span 3 to 4, then this source should have those spans subtracted and should only be valid over [1,2] and [4,5].'; + +-- Rule 1. An External Event superseded by nothing will be present in the final, derived result. +-- Rule 2. An External Event partially superseded by a later External Source, but whose start time occurs before the start of said External Source(s), will be present in the final, derived result. +-- Rule 3. An External Event whose start is superseded by another External Source, even if its end occurs after the end of said External Source, will be replaced by the contents of that External Source (whether they are blank spaces, or other events). +-- Rule 4. An External Event who shares an ID with an External Event in a later External Source will always be replaced. + +create materialized view merlin.derived_events as +-- "distinct on (event_key, derivation_group_name)" and "order by valid_at" satisfies rule 4 +-- (only the most recently valid version of an event is included) +select distinct on (event_key, derivation_group_name) + output.event_key, + output.source_key, + output.derivation_group_name, + output.event_type_name, + output.duration, + output.start_time, + output.source_range, + output.valid_at +from ( + -- select the events from the sources and include them as they fit into the ranges determined by sub + select + s.key as source_key, + ee.key as event_key, + ee.event_type_name, + ee.duration, + s.derivation_group_name, + ee.start_time, + s.source_range, + s.valid_at + from merlin.external_event ee + join ( + with base_ranges as ( + -- base_ranges orders sources by their valid time + -- and extracts the multirange that they are stated to be valid over + select + external_source.key, + external_source.derivation_group_name, + tstzmultirange(tstzrange(external_source.start_time, external_source.end_time)) as range, + external_source.valid_at + from merlin.external_source + order by external_source.valid_at + ), base_and_sub_ranges as ( + -- base_and_sub_ranges takes each of the sources above and compiles a list of all the sources that follow it + -- and their multiranges that they are stated to be valid over + select + base.key, + base.derivation_group_name, + base.range as original_range, + array_remove(array_agg(subsequent.range order by subsequent.valid_at), NULL) as subsequent_ranges, + base.valid_at + from base_ranges base + left join base_ranges subsequent + on base.derivation_group_name = subsequent.derivation_group_name + and base.valid_at < subsequent.valid_at + group by base.key, base.derivation_group_name, base.valid_at, base.range + ) + -- this final selection (s) utilizes the first, as well as merlin.subtract_later_ranges, + -- to produce a sparse multirange that a given source is valid over. + -- See merlin.subtract_later_ranges for further details on subtracted ranges. + select + r.key, + r.derivation_group_name, + merlin.subtract_later_ranges(r.original_range, r.subsequent_ranges) as source_range, + r.valid_at + from base_and_sub_ranges r + order by r.derivation_group_name desc, r.valid_at) s + on s.key = ee.source_key + and s.derivation_group_name = ee.derivation_group_name + where s.source_range @> ee.start_time + order by valid_at desc +) output; + +-- create a unique index, which allows concurrent refreshes +create unique index on merlin.derived_events ( + event_key, + source_key, + derivation_group_name, + event_type_name +); + +-- refresh the materialized view after insertion/update/deletion to external_source and external_event and derivation_group +create function merlin.refresh_derived_events_on_trigger() + returns trigger + language plpgsql as $$ +begin + refresh materialized view concurrently merlin.derived_events; + return new; +end; +$$; + +-- events are the most basic source of information, so update when the set of events changes. +create trigger refresh_derived_events_on_external_event +after insert or update or delete on merlin.external_event + for each statement execute function merlin.refresh_derived_events_on_trigger(); + +-- also trigger on external sources, especially in the case of an empty source, which could still overlap and erase some +-- events in time (see "rule3_empty" test in ExternalEventTests.java). +create trigger refresh_derived_events_on_external_source +after insert or update or delete on merlin.external_source + for each statement execute function merlin.refresh_derived_events_on_trigger(); + +create trigger refresh_derived_events_on_derivation_group +after insert or update or delete on merlin.derivation_group + for each statement execute function merlin.refresh_derived_events_on_trigger(); + +comment on materialized view merlin.derived_events is e'' + 'Derives the final event set for each derivation group.'; + +call migrations.mark_migration_applied('11'); diff --git a/deployment/postgres-init-db/sql/applied_migrations.sql b/deployment/postgres-init-db/sql/applied_migrations.sql index 245dc51deb..9f1d007a71 100644 --- a/deployment/postgres-init-db/sql/applied_migrations.sql +++ b/deployment/postgres-init-db/sql/applied_migrations.sql @@ -13,3 +13,4 @@ call migrations.mark_migration_applied('7'); call migrations.mark_migration_applied('8'); call migrations.mark_migration_applied('9'); call migrations.mark_migration_applied('10'); +call migrations.mark_migration_applied('11'); diff --git a/deployment/postgres-init-db/sql/functions/merlin/external_events/subtract_later_ranges.sql b/deployment/postgres-init-db/sql/functions/merlin/external_events/subtract_later_ranges.sql new file mode 100644 index 0000000000..4c275d2fb4 --- /dev/null +++ b/deployment/postgres-init-db/sql/functions/merlin/external_events/subtract_later_ranges.sql @@ -0,0 +1,20 @@ +create function merlin.subtract_later_ranges(curr_date tstzmultirange, later_dates tstzmultirange[]) +returns tstzmultirange +immutable +language plpgsql as $$ + declare + ret tstzmultirange := curr_date; + later_date tstzmultirange; +begin + foreach later_date in array later_dates loop + ret := ret - later_date; + end loop; + return ret; +end +$$; + +comment on function merlin.subtract_later_ranges(curr_date tstzmultirange, later_dates tstzmultirange[]) is e'' + 'Used by the derived_events view that produces from the singular interval of time that a source covers a set of disjoint intervals.\n' + 'The disjointedness arises from where future sources'' spans are subtracted from this one.\n' + 'For example, if a source is valid at t=0, and covers span s=1 to s=5, and there is a source valid at t=1 with a span s=2 to s=3\n' + 'and another valid at t=2 with a span 3 to 4, then this source should have those spans subtracted and should only be valid over [1,2] and [4,5].'; diff --git a/deployment/postgres-init-db/sql/init_merlin.sql b/deployment/postgres-init-db/sql/init_merlin.sql index e0fdb3835f..eca054c4c5 100644 --- a/deployment/postgres-init-db/sql/init_merlin.sql +++ b/deployment/postgres-init-db/sql/init_merlin.sql @@ -72,9 +72,18 @@ begin; \ir tables/merlin/merging/merge_staging_area.sql \ir tables/merlin/merging/conflicting_activities.sql + -- External Events + \ir tables/merlin/external_events/external_source_type.sql + \ir tables/merlin/external_events/external_event_type.sql + \ir tables/merlin/external_events/derivation_group.sql + \ir tables/merlin/external_events/external_source.sql + \ir tables/merlin/external_events/external_event.sql + \ir tables/merlin/external_events/plan_derivation_group.sql + ------------ -- Functions \ir functions/merlin/reanchoring_functions.sql + \ir functions/merlin/external_events/subtract_later_ranges.sql -- Snapshots \ir functions/merlin/snapshots/create_snapshot.sql @@ -94,4 +103,5 @@ begin; \ir views/merlin/activity_directive_extended.sql \ir views/merlin/simulated_activity.sql \ir views/merlin/resource_profile.sql + \ir views/merlin/derived_events.sql end; diff --git a/deployment/postgres-init-db/sql/tables/merlin/external_events/derivation_group.sql b/deployment/postgres-init-db/sql/tables/merlin/external_events/derivation_group.sql new file mode 100644 index 0000000000..c9c76cfc2d --- /dev/null +++ b/deployment/postgres-init-db/sql/tables/merlin/external_events/derivation_group.sql @@ -0,0 +1,27 @@ +create table merlin.derivation_group ( + name text not null, + source_type_name text not null, + owner text, + + constraint derivation_group_pkey + primary key (name), + constraint derivation_group_references_external_source_type + foreign key (source_type_name) + references merlin.external_source_type(name) + on update cascade + on delete restrict, + constraint derivation_group_owner_exists + foreign key (owner) references permissions.users + on update cascade + on delete set null +); + +comment on table merlin.derivation_group is e'' + 'A collection of external sources of the same type that the derivation operation is run against.'; + +comment on column merlin.derivation_group.name is e'' + 'The name and primary key of the derivation group.'; +comment on column merlin.derivation_group.source_type_name is e'' + 'The name of the external_source_type of sources in this derivation group.'; +comment on column merlin.derivation_group.owner is e'' + 'The name of the user that created this derivation_group.'; diff --git a/deployment/postgres-init-db/sql/tables/merlin/external_events/external_event.sql b/deployment/postgres-init-db/sql/tables/merlin/external_events/external_event.sql new file mode 100644 index 0000000000..b5d23125b3 --- /dev/null +++ b/deployment/postgres-init-db/sql/tables/merlin/external_events/external_event.sql @@ -0,0 +1,79 @@ +create table merlin.external_event ( + key text not null, + event_type_name text not null, + source_key text not null, + derivation_group_name text not null, + start_time timestamp with time zone not null, + duration interval not null, + + constraint external_event_pkey + primary key (key, source_key, derivation_group_name, event_type_name), + constraint external_event_references_source_key_derivation_group + foreign key (source_key, derivation_group_name) + references merlin.external_source (key, derivation_group_name) + on update cascade + on delete cascade, + constraint external_event_references_event_type_name + foreign key (event_type_name) + references merlin.external_event_type(name) + on update cascade + on delete restrict +); + +comment on table merlin.external_event is e'' + 'Externally imported events.'; + +comment on column merlin.external_event.key is e'' + 'The key, or name, of the external_event.\n' + 'Part of the primary key, along with the source_key, derivation_group_name, and event_type_name.'; +comment on column merlin.external_event.event_type_name is e'' + 'The type of this external_event.'; +comment on column merlin.external_event.source_key is e'' + 'The key of the external_source that this external_event is included in.\n' + 'Used as a foreign key along with the derivation_group_name to directly identify said source.\n' + 'Part of the primary key along with the key, derivation_group_name, and event_type_name.'; +comment on column merlin.external_event.derivation_group_name is e'' + 'The derivation_group that the external_source bearing this external_event is a part of.'; +comment on column merlin.external_event.start_time is e'' + 'The start time (in _plan_ time, NOT planner time), of the range that this source describes.'; +comment on column merlin.external_event.duration is e'' + 'The span of time of this external event.'; + +create trigger check_external_event_duration_is_nonnegative_trigger +before insert or update on merlin.external_event +for each row +when (new.duration < '0') +execute function util_functions.raise_duration_is_negative(); + +create function merlin.check_external_event_boundaries() +returns trigger +language plpgsql as $$ +declare + source_start timestamp with time zone; + source_end timestamp with time zone; + event_start timestamp with time zone; + event_end timestamp with time zone; +begin + select start_time, end_time into source_start, source_end + from merlin.external_source + where new.source_key = external_source.key + and new.derivation_group_name = external_source.derivation_group_name; + + event_start := new.start_time; + event_end := new.start_time + new.duration; + if event_start < source_start or event_end > source_end then + raise exception 'Event % out of bounds of source %.', new.key, new.source_key; + end if; + return new; +end; +$$; + +comment on function merlin.check_external_event_boundaries() is e'' + 'Checks that an external_event added to the database has a start time and duration that fall in bounds of the associated external_source.'; + +create trigger check_external_event_boundaries +before insert on merlin.external_event + for each row execute function merlin.check_external_event_boundaries(); + +comment on trigger check_external_event_boundaries on merlin.external_event is e'' + 'Fires any time a new external event is added that checks that the span of the event fits in its referenced source.'; diff --git a/deployment/postgres-init-db/sql/tables/merlin/external_events/external_event_type.sql b/deployment/postgres-init-db/sql/tables/merlin/external_events/external_event_type.sql new file mode 100644 index 0000000000..3e59d7a4ab --- /dev/null +++ b/deployment/postgres-init-db/sql/tables/merlin/external_events/external_event_type.sql @@ -0,0 +1,12 @@ +create table merlin.external_event_type ( + name text not null, + + constraint external_event_type_pkey + primary key (name) +); + +comment on table merlin.external_event_type is e'' + 'Externally imported event types.'; + +comment on column merlin.external_event_type.name is e'' + 'The identifier for this external_event_type, as well as its name.'; diff --git a/deployment/postgres-init-db/sql/tables/merlin/external_events/external_source.sql b/deployment/postgres-init-db/sql/tables/merlin/external_events/external_source.sql new file mode 100644 index 0000000000..35f859c578 --- /dev/null +++ b/deployment/postgres-init-db/sql/tables/merlin/external_events/external_source.sql @@ -0,0 +1,107 @@ +create table merlin.external_source ( + key text not null, + source_type_name text not null, + derivation_group_name text not null, + valid_at timestamp with time zone not null, + start_time timestamp with time zone not null, + end_time timestamp with time zone not null, + CHECK (end_time > start_time), + created_at timestamp with time zone default now() not null, + owner text, + + constraint external_source_pkey + primary key (key, derivation_group_name), + -- a given dg cannot have two sources with the same valid_at! + CONSTRAINT dg_unique_valid_at UNIQUE (derivation_group_name, valid_at), + constraint external_source_references_external_source_type_name + foreign key (source_type_name) + references merlin.external_source_type(name) + on update cascade + on delete restrict, + constraint external_source_type_matches_derivation_group + foreign key (derivation_group_name) + references merlin.derivation_group (name) + on update cascade + on delete restrict, + constraint external_source_owner_exists + foreign key (owner) references permissions.users + on update cascade + on delete set null +); + +comment on table merlin.external_source is e'' + 'Externally imported event sources.'; + +comment on column merlin.external_source.key is e'' + 'The key, or name, of the external_source.\n' + 'Part of the primary key, along with the derivation_group_name'; +comment on column merlin.external_source.source_type_name is e'' + 'The type of this external_source.'; +comment on column merlin.external_source.derivation_group_name is e'' + 'The name of the derivation_group that this external_source is included in.'; +comment on column merlin.external_source.valid_at is e'' + 'The time (in _planner_ time, NOT plan time) at which a source becomes valid.\n' + 'This time helps determine when a source''s events are valid for the span of time it covers.'; +comment on column merlin.external_source.start_time is e'' + 'The start time (in _plan_ time, NOT planner time), of the range that this source describes.'; +comment on column merlin.external_source.end_time is e'' + 'The end time (in _plan_ time, NOT planner time), of the range that this source describes.'; +comment on column merlin.external_source.created_at is e'' + 'The time (in _planner_ time, NOT plan time) that this particular source was created.\n' + 'This column is used primarily for documentation purposes, and has no associated functionality.'; +comment on column merlin.external_source.owner is e'' + 'The user who uploaded the external source.'; + +-- make sure new sources' source_type match that of their derivation group! +create function merlin.external_source_type_matches_dg_on_add() + returns trigger + language plpgsql as $$ +declare + source_type text; +begin + select into source_type derivation_group.source_type_name from merlin.derivation_group where name = new.derivation_group_name; + if source_type is distinct from new.source_type_name then + raise foreign_key_violation + using message='External source ' || new.key || ' is being added to a derivation group ' || new.derivation_group_name + || ' where its type ' || new.source_type_name || ' does not match the derivation group type ' + || source_type || '.' ; + end if; + return new; +end; +$$; + +create trigger external_source_type_matches_dg_on_add +before insert or update on merlin.external_source + for each row execute function merlin.external_source_type_matches_dg_on_add(); + +-- if an external source is linked to a plan it cannot be deleted +create function merlin.external_source_pdg_association_delete() + returns trigger + language plpgsql as $$ +begin + if exists (select * from merlin.plan_derivation_group pdg where pdg.derivation_group_name = old.derivation_group_name) then + raise foreign_key_violation + using message='External source ' || old.key || ' is part of a derivation group that is associated to a plan.'; + end if; + return old; +end; +$$; + +create trigger external_source_pdg_association_delete +before delete on merlin.external_source + for each row execute function merlin.external_source_pdg_association_delete(); + +-- set acknowledged on merlin.plan_derivation_group false for this derivation group as there are new changes +create function merlin.external_source_pdg_ack_update() + returns trigger + language plpgsql as $$ +begin + update merlin.plan_derivation_group set "acknowledged" = false + where plan_derivation_group.derivation_group_name = NEW.derivation_group_name; + return new; +end; +$$; + +create trigger external_source_pdg_ack_update +after insert on merlin.external_source + for each row execute function merlin.external_source_pdg_ack_update(); diff --git a/deployment/postgres-init-db/sql/tables/merlin/external_events/external_source_type.sql b/deployment/postgres-init-db/sql/tables/merlin/external_events/external_source_type.sql new file mode 100644 index 0000000000..00243d4787 --- /dev/null +++ b/deployment/postgres-init-db/sql/tables/merlin/external_events/external_source_type.sql @@ -0,0 +1,14 @@ +create table merlin.external_source_type ( + name text not null, + + constraint external_source_type_pkey + primary key (name) +); + +comment on table merlin.external_source_type is e'' + 'Externally imported event source types (each external source has to be of a certain type).\n' + 'They are also helpful to classify external sources.\n' + 'Derivation groups are a subclass of external source type.'; + +comment on column merlin.external_source_type.name is e'' + 'The identifier for this external_source_type, as well as its name.'; diff --git a/deployment/postgres-init-db/sql/tables/merlin/external_events/plan_derivation_group.sql b/deployment/postgres-init-db/sql/tables/merlin/external_events/plan_derivation_group.sql new file mode 100644 index 0000000000..93c1feb6e8 --- /dev/null +++ b/deployment/postgres-init-db/sql/tables/merlin/external_events/plan_derivation_group.sql @@ -0,0 +1,45 @@ +create table merlin.plan_derivation_group ( + plan_id integer not null, + derivation_group_name text not null, + last_acknowledged_at timestamp with time zone default now() not null, + acknowledged boolean not null default true, + + constraint plan_derivation_group_pkey + primary key (plan_id, derivation_group_name), + constraint pdg_plan_exists + foreign key (plan_id) + references merlin.plan(id) + on delete cascade, + constraint pdg_derivation_group_exists + foreign key (derivation_group_name) + references merlin.derivation_group(name) + on update cascade + on delete restrict +); + +comment on table merlin.plan_derivation_group is e'' + 'Links externally imported event sources & plans.\n' + 'Additionally, tracks the last time a plan owner/contributor(s) have acknowledged additions to the derivation group.\n'; + +comment on column merlin.plan_derivation_group.plan_id is e'' + 'The plan with which the derivation group is associated.'; +comment on column merlin.plan_derivation_group.derivation_group_name is e'' + 'The derivation group being associated with the plan.'; +comment on column merlin.plan_derivation_group.last_acknowledged_at is e'' + 'The time at which changes to the derivation group were last acknowledged.'; + +-- update last_acknowledged whenever acknowledged is set to true +create function merlin.pdg_update_ack_at() + returns trigger + language plpgsql as $$ +begin + if new.acknowledged = true then + new.last_acknowledged_at = now(); + end if; + return new; +end; +$$; + +create trigger pdg_update_ack_at +before update on merlin.plan_derivation_group + for each row execute function merlin.pdg_update_ack_at(); diff --git a/deployment/postgres-init-db/sql/views/merlin/derived_events.sql b/deployment/postgres-init-db/sql/views/merlin/derived_events.sql new file mode 100644 index 0000000000..29ca0139b0 --- /dev/null +++ b/deployment/postgres-init-db/sql/views/merlin/derived_events.sql @@ -0,0 +1,106 @@ +-- Rule 1. An External Event superseded by nothing will be present in the final, derived result. +-- Rule 2. An External Event partially superseded by a later External Source, but whose start time occurs before the start of said External Source(s), will be present in the final, derived result. +-- Rule 3. An External Event whose start is superseded by another External Source, even if its end occurs after the end of said External Source, will be replaced by the contents of that External Source (whether they are blank spaces, or other events). +-- Rule 4. An External Event who shares an ID with an External Event in a later External Source will always be replaced. + +create materialized view merlin.derived_events as +-- "distinct on (event_key, derivation_group_name)" and "order by valid_at" satisfies rule 4 +-- (only the most recently valid version of an event is included) +select distinct on (event_key, derivation_group_name) + output.event_key, + output.source_key, + output.derivation_group_name, + output.event_type_name, + output.duration, + output.start_time, + output.source_range, + output.valid_at +from ( + -- select the events from the sources and include them as they fit into the ranges determined by sub + select + s.key as source_key, + ee.key as event_key, + ee.event_type_name, + ee.duration, + s.derivation_group_name, + ee.start_time, + s.source_range, + s.valid_at + from merlin.external_event ee + join ( + with base_ranges as ( + -- base_ranges orders sources by their valid time + -- and extracts the multirange that they are stated to be valid over + select + external_source.key, + external_source.derivation_group_name, + tstzmultirange(tstzrange(external_source.start_time, external_source.end_time)) as range, + external_source.valid_at + from merlin.external_source + order by external_source.valid_at + ), base_and_sub_ranges as ( + -- base_and_sub_ranges takes each of the sources above and compiles a list of all the sources that follow it + -- and their multiranges that they are stated to be valid over + select + base.key, + base.derivation_group_name, + base.range as original_range, + array_remove(array_agg(subsequent.range order by subsequent.valid_at), NULL) as subsequent_ranges, + base.valid_at + from base_ranges base + left join base_ranges subsequent + on base.derivation_group_name = subsequent.derivation_group_name + and base.valid_at < subsequent.valid_at + group by base.key, base.derivation_group_name, base.valid_at, base.range + ) + -- this final selection (s) utilizes the first, as well as merlin.subtract_later_ranges, + -- to produce a sparse multirange that a given source is valid over. + -- See merlin.subtract_later_ranges for further details on subtracted ranges. + select + r.key, + r.derivation_group_name, + merlin.subtract_later_ranges(r.original_range, r.subsequent_ranges) as source_range, + r.valid_at + from base_and_sub_ranges r + order by r.derivation_group_name desc, r.valid_at) s + on s.key = ee.source_key + and s.derivation_group_name = ee.derivation_group_name + where s.source_range @> ee.start_time + order by valid_at desc +) output; + +-- create a unique index, which allows concurrent refreshes +create unique index on merlin.derived_events ( + event_key, + source_key, + derivation_group_name, + event_type_name +); + +-- refresh the materialized view after insertion/update/deletion to external_source and external_event and derivation_group +create function merlin.refresh_derived_events_on_trigger() + returns trigger + language plpgsql as $$ +begin + refresh materialized view concurrently merlin.derived_events; + return new; +end; +$$; + +-- events are the most basic source of information, so update when the set of events changes. +create trigger refresh_derived_events_on_external_event +after insert or update or delete on merlin.external_event + for each statement execute function merlin.refresh_derived_events_on_trigger(); + +-- also trigger on external sources, especially in the case of an empty source, which could still overlap and erase some +-- events in time (see "rule3_empty" test in ExternalEventTests.java). +create trigger refresh_derived_events_on_external_source +after insert or update or delete on merlin.external_source + for each statement execute function merlin.refresh_derived_events_on_trigger(); + +create trigger refresh_derived_events_on_derivation_group +after insert or update or delete on merlin.derivation_group + for each statement execute function merlin.refresh_derived_events_on_trigger(); + +comment on materialized view merlin.derived_events is e'' + 'Derives the final event set for each derivation group.';