AN-140 Metadata index for cost capping

CHANGELOG.md

@@ -54,6 +54,10 @@ The `IX_WORKFLOW_STORE_ENTRY_WS` index is removed from `WORKFLOW_STORE_ENTRY`.
 
 The index had low cardinality and workflow pickup is faster without it. Migration time depends on workflow store size, but should be very fast for most installations. Terminal workflows are removed from the workflow store, so only running workflows contribute to the cost.
 
+#### Index additions
+
+The `IX_METADATA_ENTRY_WEU_MK` and `IX_METADATA_ENTRY_WEU_CF_JSI_JRA_MK` indexes are added to `METADATA_ENTRY`. In pre-release testing, each index proceeded at about 3 million rows per minute. Please plan downtime accordingly: `minutes = ( METADATA_ENTRY rows / 3,000,000 ) * 2`.
+
 ### Bug fixes
 
 #### Improved `size()` function performance on arrays

database/migration/src/main/resources/metadata_changesets/index_add_workflow_call_scatter_attempt_key.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog objectQuotingStrategy="QUOTE_ALL_OBJECTS"
+                   xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet id="index_add_workflow_call_scatter_attempt_key" author="anichols" dbms="hsqldb,mariadb,mysql,postgresql">
+        <!--
+            This index creates at about 3M rows per minute on MySQL.
+            That would be an impossible multi-day downtime in Terra, so we manually pre-create the index asynchronously.
+            This changeset detects environments where this has been done and immediately marks itself as applied.
+        -->
+        <preConditions onFail="MARK_RAN">
+            <not>
+                <indexExists indexName="IX_METADATA_ENTRY_WEU_CF_JSI_JRA_MK"/>
+            </not>
+        </preConditions>
+        <createIndex indexName="IX_METADATA_ENTRY_WEU_CF_JSI_JRA_MK" tableName="METADATA_ENTRY">
+            <column name="WORKFLOW_EXECUTION_UUID"/>
+            <column name="CALL_FQN"/>
+            <column name="JOB_SCATTER_INDEX"/>
+            <column name="JOB_RETRY_ATTEMPT"/>
+            <column name="METADATA_KEY"/>
+        </createIndex>
+    </changeSet>
+</databaseChangeLog>

database/migration/src/main/resources/metadata_changesets/index_add_workflow_key.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog objectQuotingStrategy="QUOTE_ALL_OBJECTS"
+                   xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet id="index_add_workflow_key" author="anichols" dbms="hsqldb,mariadb,mysql,postgresql">
+        <!--
+            This index creates at about 3M rows per minute on MySQL.
+            That would be an impossible multi-day downtime in Terra, so we manually pre-create the index asynchronously.
+            This changeset detects environments where this has been done and immediately marks itself as applied.
+        -->
+        <preConditions onFail="MARK_RAN">
+            <not>
+                <indexExists indexName="IX_METADATA_ENTRY_WEU_MK"/>
+            </not>
+        </preConditions>
+        <createIndex indexName="IX_METADATA_ENTRY_WEU_MK" tableName="METADATA_ENTRY">
+            <column name="WORKFLOW_EXECUTION_UUID"/>
+            <column name="METADATA_KEY"/>
+        </createIndex>
+    </changeSet>
+</databaseChangeLog>

database/migration/src/main/resources/sql_metadata_changelog.xml

@@ -19,6 +19,8 @@
     <include file="metadata_changesets/remove_non_summarizable_metadata_from_queue.xml" relativeToChangelogFile="true" />
     <include file="metadata_changesets/update_metadata_archive_index.xml" relativeToChangelogFile="true" />
     <include file="metadata_changesets/reset_archive_statuses_to_null.xml" relativeToChangelogFile="true" />
+    <include file="metadata_changesets/index_add_workflow_key.xml" relativeToChangelogFile="true" />
+    <include file="metadata_changesets/index_add_workflow_call_scatter_attempt_key.xml" relativeToChangelogFile="true" />
     <!-- WARNING!
       This changeset should always be last.
       It it always run (and should always run last) to set table ownership correctly.

database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala

@@ -56,7 +56,34 @@
     ) <> (MetadataEntry.tupled, MetadataEntry.unapply)
 
     // TODO: rename index via liquibase
+    // TODO: may be possible to drop once we fully deploy the below indexes with leading UUID
     def ixMetadataEntryWeu = index("METADATA_WORKFLOW_IDX", workflowExecutionUuid, unique = false)
+
+    /**
+      * Index designed to accelerate common key-specific queries across an entire workflow such as:
+      * - Get me workflow-level `outputs%` (no tasks, requireEmptyJobKey = true)
+      * - Get me all `vmStartTime%`, `vmEndTime%`, `vmCostPerHour%` in the workflow (include tasks, requireEmptyJobKey = false)
+      *
+      * It is NOT good, as in may make actively slower, queries that reference a specific job.
+      *
+      * @return A reference to the index
+      */
+    def ixMetadataEntryWeuMk = index("IX_METADATA_ENTRY_WEU_MK", (workflowExecutionUuid, metadataKey), unique = false)
+
+    /**
+      * Index designed to accelerate task/attempt level queries such as:
+      * - Get some keys for a specific attempt
+      * - Get all rows for a specific attempt
+      * - Get all rows for a specific call/scatter, across all attempts
+      * - Get unique call names in a workflow
+      *
+      * @return A reference to the index
+      */
+    def ixMetadataEntryWeuCfJsiJraMk =
+      index("IX_METADATA_ENTRY_WEU_CF_JSI_JRA_MK",
+            (workflowExecutionUuid, callFullyQualifiedName, jobIndex, jobAttempt, metadataKey),
+            unique = false
+      )
   }
 
   val metadataEntries = TableQuery[MetadataEntries]