Merge pull request #1418 from NASA-AMMOS/feat/hasura-events-view

Add Hasura Events Views

.github/workflows/pgcmp.yml

@@ -120,7 +120,12 @@ jobs:
         env:
           AERIE_USERNAME: "${{secrets.AERIE_USERNAME}}"
           AERIE_PASSWORD: "${{secrets.AERIE_PASSWORD}}"
-      - name: Dump Databases
+      - name: Clone PGCMP
+        uses: actions/checkout@v4
+        with:
+          repository: cbbrowne/pgcmp
+          path: pgcmp
+      - name: Dump Migrated Database
         env:
           AERIE_USERNAME: "${{secrets.AERIE_USERNAME}}"
           AERIE_PASSWORD: "${{secrets.AERIE_PASSWORD}}"
@@ -193,7 +198,7 @@ jobs:
         with:
           repository: cbbrowne/pgcmp
           path: pgcmp
-      - name: Dump Databases
+      - name: Dump Current Database
         env:
           AERIE_USERNAME: "${{secrets.AERIE_USERNAME}}"
           AERIE_PASSWORD: "${{secrets.AERIE_PASSWORD}}"
@@ -224,7 +229,7 @@ jobs:
         env:
           AERIE_USERNAME: "${{secrets.AERIE_USERNAME}}"
           AERIE_PASSWORD: "${{secrets.AERIE_PASSWORD}}"
-      - name: Dump Databases
+      - name: Dump Migrated Database
         env:
           AERIE_USERNAME: "${{secrets.AERIE_USERNAME}}"
           AERIE_PASSWORD: "${{secrets.AERIE_PASSWORD}}"

deployment/aerie_db_migration.py

@@ -89,7 +89,7 @@ def step_by_step_migration(db_migration, apply):
       return
   input("Press Enter to continue...")
 
-def bulk_migration(db_migration, apply):
+def bulk_migration(db_migration, apply, current_version):
   # Migrate the database
   exit_with = 0
   if apply:
@@ -98,10 +98,15 @@ def bulk_migration(db_migration, apply):
     if exit_code != 0:
       exit_with = 1
   else:
-    os.system(f'hasura migrate apply --goto 1 --database-name {db_migration.db_name} --dry-run --log-level WARN &&'
-              f'hasura migrate apply --down 1 --database-name {db_migration.db_name} --dry-run --log-level WARN')
-    exit_code = os.system(f'hasura migrate apply --goto 1 --database-name {db_migration.db_name} &&'
-                          f'hasura migrate apply --down 1 --database-name {db_migration.db_name}')
+    # Performing GOTO 1 when the database is at migration 1 will cause Hasura to attempt to reapply migration 1
+    if current_version == 1:
+      os.system(f'hasura migrate apply --down 1 --database-name {db_migration.db_name} --dry-run --log-level WARN')
+      exit_code = os.system(f'hasura migrate apply --down 1 --database-name {db_migration.db_name}')
+    else:
+      os.system(f'hasura migrate apply --goto 1 --database-name {db_migration.db_name} --dry-run --log-level WARN &&'
+                f'hasura migrate apply --down 1 --database-name {db_migration.db_name} --dry-run --log-level WARN')
+      exit_code = os.system(f'hasura migrate apply --goto 1 --database-name {db_migration.db_name} &&'
+                            f'hasura migrate apply --down 1 --database-name {db_migration.db_name}')
     if exit_code != 0:
       exit_with = 1
 
@@ -128,7 +133,9 @@ def mark_current_version(username, password, netloc):
 
   # Mark everything up to that as applied
   for i in range(0, current_schema+1):
-    os.system('hasura migrate apply --skip-execution --version '+str(i)+' --database-name aerie >/dev/null 2>&1')
+    os.system('hasura migrate apply --skip-execution --version '+str(i)+' --database-name Aerie >/dev/null 2>&1')
+
+  return current_schema
 
 def main():
   # Create a cli parser
@@ -227,7 +234,7 @@ def main():
   os.chdir(HASURA_PATH)
 
   # Mark all migrations previously applied to the databases to be updated as such
-  mark_current_version(username, password, args.network_location)
+  current_version = mark_current_version(username, password, args.network_location)
 
   clear_screen()
   print(f'\n###############################'
@@ -238,7 +245,7 @@ def main():
     # Go step-by-step through the migrations available for the selected database
     step_by_step_migration(migration, args.apply)
   else:
-    bulk_migration(migration, args.apply)
+    bulk_migration(migration, args.apply, current_version)
 
 if __name__ == "__main__":
   main()

deployment/hasura/metadata/databases/tables/hasura/refresh_activity_type_logs.yaml

@@ -0,0 +1,30 @@
+table:
+  name: refresh_activity_type_logs
+  schema: hasura
+configuration:
+  custom_name: "refresh_activity_type_logs"
+object_relationships:
+- name: model
+  using:
+    manual_configuration:
+      remote_table:
+        name: mission_model
+        schema: merlin
+      column_mapping:
+        model_id: id
+select_permissions:
+  - role: aerie_admin
+    permission:
+      columns: '*'
+      filter: {}
+      allow_aggregations: true
+  - role: user
+    permission:
+      columns: '*'
+      filter: {}
+      allow_aggregations: true
+  - role: viewer
+    permission:
+      columns: '*'
+      filter: {}
+      allow_aggregations: true

deployment/hasura/metadata/databases/tables/hasura/refresh_model_parameter_logs.yaml

@@ -0,0 +1,30 @@
+table:
+  name: refresh_model_parameter_logs
+  schema: hasura
+configuration:
+  custom_name: "refresh_model_parameter_logs"
+object_relationships:
+- name: model
+  using:
+    manual_configuration:
+      remote_table:
+        name: mission_model
+        schema: merlin
+      column_mapping:
+        model_id: id
+select_permissions:
+  - role: aerie_admin
+    permission:
+      columns: '*'
+      filter: {}
+      allow_aggregations: true
+  - role: user
+    permission:
+      columns: '*'
+      filter: {}
+      allow_aggregations: true
+  - role: viewer
+    permission:
+      columns: '*'
+      filter: {}
+      allow_aggregations: true

deployment/hasura/metadata/databases/tables/hasura/refresh_resource_types_logs.yaml

@@ -0,0 +1,30 @@
+table:
+  name: refresh_resource_type_logs
+  schema: hasura
+configuration:
+  custom_name: "refresh_resource_type_logs"
+object_relationships:
+- name: model
+  using:
+    manual_configuration:
+      remote_table:
+        name: mission_model
+        schema: merlin
+      column_mapping:
+        model_id: id
+select_permissions:
+  - role: aerie_admin
+    permission:
+      columns: '*'
+      filter: {}
+      allow_aggregations: true
+  - role: user
+    permission:
+      columns: '*'
+      filter: {}
+      allow_aggregations: true
+  - role: viewer
+    permission:
+      columns: '*'
+      filter: {}
+      allow_aggregations: true

deployment/hasura/metadata/databases/tables/merlin/mission_model.yaml

@@ -57,6 +57,30 @@ array_relationships:
       table:
         name: scheduling_model_specification_goals
         schema: scheduler
+- name: refresh_activity_type_logs
+  using:
+    manual_configuration:
+      remote_table:
+        name: refresh_activity_type_logs
+        schema: hasura
+      column_mapping:
+        id: model_id
+- name: refresh_model_parameter_logs
+  using:
+    manual_configuration:
+      remote_table:
+        name: refresh_model_parameter_logs
+        schema: hasura
+      column_mapping:
+        id: model_id
+- name: refresh_resource_type_logs
+  using:
+    manual_configuration:
+      remote_table:
+        name: refresh_resource_type_logs
+        schema: hasura
+      column_mapping:
+        id: model_id
 select_permissions:
   - role: aerie_admin
     permission:

deployment/hasura/metadata/databases/tables/tables.yaml

@@ -46,6 +46,9 @@
 - "!include hasura/get_conflicting_activities_return_value.yaml"
 - "!include hasura/get_non_conflicting_activities_return_value.yaml"
 - "!include hasura/get_plan_history_return_value.yaml"
+- "!include hasura/refresh_activity_type_logs.yaml"
+- "!include hasura/refresh_model_parameter_logs.yaml"
+- "!include hasura/refresh_resource_types_logs.yaml"
 - "!include hasura/resource_at_start_offset_return_value.yaml"
 - "!include hasura/withdraw_merge_request_return_value.yaml"
 

deployment/hasura/migrations/Aerie/1_hasura_events_views/down.sql

@@ -0,0 +1,7 @@
+drop view hasura.refresh_resource_type_logs;
+drop view hasura.refresh_model_parameter_logs;
+drop view hasura.refresh_activity_type_logs;
+
+drop function hasura.get_event_logs(_trigger_name text);
+
+call migrations.mark_migration_rolled_back('1');

deployment/hasura/migrations/Aerie/1_hasura_events_views/up.sql

@@ -0,0 +1,59 @@
+create function hasura.get_event_logs(_trigger_name text)
+returns table (
+  model_id int,
+  model_name text,
+  model_version text,
+  triggering_user text,
+  delivered boolean,
+  success boolean,
+  tries int,
+  created_at timestamp,
+  next_retry_at timestamp,
+  status int,
+  error jsonb,
+  error_message text,
+  error_type text
+)
+stable
+security invoker
+language plpgsql as $$
+begin
+  return query (
+    select
+      (el.payload->'data'->'new'->>'id')::int as model_id,
+      el.payload->'data'->'new'->>'name' as model_name,
+      el.payload->'data'->'new'->>'version' as model_version,
+      el.payload->'session_variables'->>'x-hasura-user-id' as triggering_user,
+      el.delivered,
+      eil.status is not distinct from 200 as success, -- is not distinct from to catch `null`
+      el.tries,
+      el.created_at,
+      el.next_retry_at,
+      eil.status,
+      (eil.response -> 'data'->> 'message')::jsonb as error,
+      (eil.response -> 'data'->> 'message')::jsonb->>'message' as error_message,
+      (eil.response -> 'data'->> 'message')::jsonb->>'type' as error_type
+      from hdb_catalog.event_log el
+      join hdb_catalog.event_invocation_logs eil on el.id = eil.event_id
+      where trigger_name = _trigger_name);
+end;
+$$;
+comment on function hasura.get_event_logs(_trigger_name text) is e''
+ 'Get the logs for every run of a Hasura event with the specified trigger name.';
+
+create view hasura.refresh_activity_type_logs as
+  select * from hasura.get_event_logs('refreshActivityTypes');
+comment on view hasura.refresh_activity_type_logs is e''
+ 'View containing logs for every run of the Hasura event `refreshActivityTypes`.';
+
+create view hasura.refresh_model_parameter_logs as
+  select * from hasura.get_event_logs('refreshModelParameters');
+comment on view hasura.refresh_model_parameter_logs is e''
+ 'View containing logs for every run of the Hasura event `refreshModelParameters`.';
+
+create view hasura.refresh_resource_type_logs as
+  select * from hasura.get_event_logs('refreshResourceTypes');
+comment on view hasura.refresh_resource_type_logs is e''
+ 'View containing logs for every run of the Hasura event `refreshResourceTypes`.';
+
+call migrations.mark_migration_applied('1');

deployment/postgres-init-db/sql/applied_migrations.sql

@@ -3,3 +3,4 @@ This file denotes which migrations occur "before" this version of the schema.
 */
 
 call migrations.mark_migration_applied('0');
+call migrations.mark_migration_applied('1');

deployment/postgres-init-db/sql/init_hasura.sql

@@ -13,4 +13,7 @@ begin;
   \ir functions/hasura/plan_branching_functions.sql
   \ir functions/hasura/plan_merge_functions.sql
   \ir functions/hasura/snapshot_functions.sql
+
+  -- Event Views
+  \ir views/hasura/hasura_event_logs.sql
 end;

deployment/postgres-init-db/sql/views/hasura/hasura_event_logs.sql

@@ -0,0 +1,60 @@
+create function hasura.get_event_logs(_trigger_name text)
+returns table (
+  model_id int,
+  model_name text,
+  model_version text,
+  triggering_user text,
+  delivered boolean,
+  success boolean,
+  tries int,
+  created_at timestamp,
+  next_retry_at timestamp,
+  status int,
+  error jsonb,
+  error_message text,
+  error_type text
+)
+stable
+security invoker
+language plpgsql as $$
+begin
+  return query (
+    select
+      (el.payload->'data'->'new'->>'id')::int as model_id,
+      el.payload->'data'->'new'->>'name' as model_name,
+      el.payload->'data'->'new'->>'version' as model_version,
+      el.payload->'session_variables'->>'x-hasura-user-id' as triggering_user,
+      el.delivered,
+      eil.status is not distinct from 200 as success, -- is not distinct from to catch `null`
+      el.tries,
+      el.created_at,
+      el.next_retry_at,
+      eil.status,
+      (eil.response -> 'data'->> 'message')::jsonb as error,
+      (eil.response -> 'data'->> 'message')::jsonb->>'message' as error_message,
+      (eil.response -> 'data'->> 'message')::jsonb->>'type' as error_type
+      from hdb_catalog.event_log el
+      join hdb_catalog.event_invocation_logs eil on el.id = eil.event_id
+      where trigger_name = _trigger_name);
+end;
+$$;
+comment on function hasura.get_event_logs(_trigger_name text) is e''
+ 'Get the logs for every run of a Hasura event with the specified trigger name.';
+
+create view hasura.refresh_activity_type_logs as
+  select * from hasura.get_event_logs('refreshActivityTypes');
+comment on view hasura.refresh_activity_type_logs is e''
+ 'View containing logs for every run of the Hasura event `refreshActivityTypes`.';
+
+create view hasura.refresh_model_parameter_logs as
+  select * from hasura.get_event_logs('refreshModelParameters');
+comment on view hasura.refresh_model_parameter_logs is e''
+ 'View containing logs for every run of the Hasura event `refreshModelParameters`.';
+
+create view hasura.refresh_resource_type_logs as
+  select * from hasura.get_event_logs('refreshResourceTypes');
+comment on view hasura.refresh_resource_type_logs is e''
+ 'View containing logs for every run of the Hasura event `refreshResourceTypes`.';
+
+
+