From 1dd4370e1f84994c810ecb3d6d6f02dbf309b6ae Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Fri, 15 Mar 2024 17:14:37 -0700 Subject: [PATCH 1/4] Improve Hasura Queries used in `simulatedActivityBatchLoader.ts` TODO: correctly sum `start_time` and `end_time` --- .../simulatedActivityBatchLoader.ts | 190 ++++++++++++------ 1 file changed, 127 insertions(+), 63 deletions(-) diff --git a/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts b/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts index 074b7258d5..f3102e3f88 100644 --- a/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts +++ b/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts @@ -12,34 +12,45 @@ export const simulatedActivitiesBatchLoader: BatchLoader< { graphqlClient: GraphQLClient; activitySchemaDataLoader: InferredDataloader } > = opts => async keys => { const result = await opts.graphqlClient.batchRequests< - { - data: { - simulated_activity: GraphQLSimulatedActivityInstance[]; - }; - }[] + { + data: { + simulation_dataset: { + id: number, + simulation: { + plan: { + id: number, + model_id: number + } + }, + simulation_start_time: string, + dataset: { spans: GQLSpan[]}, + } + }; + }[] >( keys.map(key => ({ document: gql` query ($simulationDatasetId: Int!) { - simulated_activity(where: { simulation_dataset_id: { _eq: $simulationDatasetId } }) { + simulation_dataset: simulation_dataset_by_pk(id: $simulationDatasetId) { id - simulation_dataset { - id - simulation { - plan { - model_id - } + simulation { + plan { + model_id + id + } + } + simulation_start_time + dataset { + spans { + id + attributes + start_offset + duration + activity_type_name: type } } - attributes - start_offset - start_time - end_time - duration - activity_type_name } - } - `, + }`, variables: { simulationDatasetId: key.simulationDatasetId, }, @@ -48,18 +59,37 @@ export const simulatedActivitiesBatchLoader: BatchLoader< return Promise.all( keys.map(async ({ simulationDatasetId }) => { - const simulatedActivities = result.find( - res => res.data.simulated_activity[0]?.simulation_dataset.id === simulationDatasetId, - )?.data.simulated_activity; - if (simulatedActivities === undefined) { + const simulation_dataset = result.find( + res => res.data.simulation_dataset.id === simulationDatasetId, + )?.data.simulation_dataset; + if (simulation_dataset === undefined) { return new ErrorWithStatusCode(`No simulation_dataset with id: ${simulationDatasetId}`, 404); } + + const spans = simulation_dataset.dataset.spans; + + const simulatedActivities: GraphQLSimulatedActivityInstance[] = spans.map(span => { + return { + id: span.id, + simulation_dataset_id: simulation_dataset.id, + plan_id: simulation_dataset.simulation.plan.id, + model_id: simulation_dataset.simulation.plan.model_id, + attributes: span.attributes, + duration: span.duration, + start_offset: span.start_offset, + // TODO: Sum this intervals + durations properly + start_time: simulation_dataset.simulation_start_time + span.start_offset, + end_time: simulation_dataset.simulation_start_time + span.start_offset + span.duration, + activity_type_name: span.activity_type_name + } + } + ) return Promise.all( simulatedActivities.map(async simulatedActivity => mapGraphQLActivityInstance( simulatedActivity, await opts.activitySchemaDataLoader.load({ - missionModelId: simulatedActivity.simulation_dataset.simulation.plan.model_id, + missionModelId: simulation_dataset.simulation.plan.model_id, activityTypeName: simulatedActivity.activity_type_name, }), ), @@ -77,35 +107,43 @@ export const simulatedActivityInstanceBySimulatedActivityIdBatchLoader: BatchLoa const result = await opts.graphqlClient.batchRequests< { data: { - simulated_activity: GraphQLSimulatedActivityInstance[]; + simulation_dataset: { + id: number, + simulation_start_time: string, + simulation: { + plan: { + id: number, + model_id: number, + } + }, + dataset: {span: GQLSpan} + }; }; }[] >( keys.map(key => ({ document: gql` query ($simulationDatasetId: Int!, $simulatedActivityId: Int!) { - simulated_activity( - where: { simulation_dataset_id: { _eq: $simulationDatasetId }, id: { _eq: $simulatedActivityId } } - ) { + simulation_dataset: simulation_dataset_by_pk(id: $simulationDatasetId) { id - simulation_dataset { - id - simulation { - plan { - model_id - } - plan_id + simulation_start_time + simulation { + plan { + id + model_id + } + } + dataset { + spans: spans(where: {id: {_eq: $simulatedActivityId}}) { + id + attributes + start_offset + duration + activity_type_name: type } } - attributes - start_offset - start_time - end_time - duration - activity_type_name } - } - `, + }`, variables: { simulationDatasetId: key.simulationDatasetId, simulatedActivityId: key.simulatedActivityId, @@ -115,21 +153,42 @@ export const simulatedActivityInstanceBySimulatedActivityIdBatchLoader: BatchLoa return Promise.all( keys.map(async ({ simulationDatasetId, simulatedActivityId }) => { - const simulatedActivity = result.find( + const simulation_dataset = result.find( res => - res.data.simulated_activity[0]?.simulation_dataset.id === simulationDatasetId && - res.data.simulated_activity[0]?.id === simulatedActivityId, - )?.data.simulated_activity[0]; - if (simulatedActivity === undefined) { + res.data.simulation_dataset.id === simulationDatasetId + )?.data.simulation_dataset; + if (simulation_dataset === undefined) { + return new ErrorWithStatusCode( + `No simulation_dataset with id: ${simulationDatasetId}`, + 404, + ); + } + + const span = simulation_dataset.dataset.span + if(span === undefined) { return new ErrorWithStatusCode( - `No simulation_dataset with id: ${simulationDatasetId} and simulated activity id: ${simulatedActivityId}`, - 404, + `No simulation_dataset with id: ${simulationDatasetId} and simulated activity id: ${simulatedActivityId}`, + 404, ); } + + const simulatedActivity: GraphQLSimulatedActivityInstance = { + id: span.id, + simulation_dataset_id: simulation_dataset.id, + plan_id: simulation_dataset.simulation.plan.id, + model_id: simulation_dataset.simulation.plan.model_id, + attributes: span.attributes, + duration: span.duration, + start_offset: span.start_offset, + // TODO: Sum this intervals + durations properly + start_time: simulation_dataset.simulation_start_time + span.start_offset, + end_time: simulation_dataset.simulation_start_time + span.start_offset + span.duration, + activity_type_name: span.activity_type_name + } return mapGraphQLActivityInstance( simulatedActivity, await opts.activitySchemaDataLoader.load({ - missionModelId: simulatedActivity.simulation_dataset.simulation.plan.model_id, + missionModelId: simulation_dataset.simulation.plan.model_id, activityTypeName: simulatedActivity.activity_type_name, }), ); @@ -173,20 +232,25 @@ export interface SimulatedActivity< activityTypeName: string; } +export interface GQLSpan< + ActivityArguments extends Record = Record, + ActivityComputedAttributes extends Record = Record, +> { + id: number + attributes: GraphQLSimulatedActivityAttributes; + start_offset: string; + duration: string; + activity_type_name: string; +} + export interface GraphQLSimulatedActivityInstance< ActivityArguments extends Record = Record, ActivityComputedAttributes extends Record = Record, > { id: number; - simulation_dataset: { - id: number; - simulation: { - plan: { - model_id: number; - }; - plan_id: number; - }; - }; + simulation_dataset_id: number; + plan_id: number; + model_id: number; attributes: GraphQLSimulatedActivityAttributes; duration: string; start_offset: string; @@ -202,7 +266,7 @@ export function mapGraphQLActivityInstance( return { simulationDataset: { simulation: { - planId: activityInstance.simulation_dataset.simulation.plan_id, + planId: activityInstance.model_id }, }, id: activityInstance.id, @@ -210,7 +274,7 @@ export function mapGraphQLActivityInstance( startOffset: Temporal.Duration.from(parse(activityInstance.start_offset).toISOString()), startTime: Temporal.Instant.from(activityInstance.start_time), endTime: activityInstance.end_time ? Temporal.Instant.from(activityInstance.end_time) : null, - simulationDatasetId: activityInstance.simulation_dataset.id, + simulationDatasetId: activityInstance.simulation_dataset_id, activityTypeName: activityInstance.activity_type_name, attributes: { arguments: Object.entries(activityInstance.attributes.arguments).reduce((acc, [key, value]) => { From eec561b5dbd874179b40ede84fbb7618482ea009 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Fri, 15 Mar 2024 17:16:35 -0700 Subject: [PATCH 2/4] Improve DB Queries in `seqJson.ts` --- sequencing-server/src/routes/seqjson.ts | 66 +++++++++++-------------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/sequencing-server/src/routes/seqjson.ts b/sequencing-server/src/routes/seqjson.ts index 14a3fc2b40..3aeedd93f2 100644 --- a/sequencing-server/src/routes/seqjson.ts +++ b/sequencing-server/src/routes/seqjson.ts @@ -137,25 +137,22 @@ seqjsonRouter.post('/get-seqjson-for-seqid-and-simulation-dataset', async (req, errors: ReturnType[] | null; }>( ` - with joined_table as (select activity_instance_commands.commands, - activity_instance_commands.activity_instance_id, - activity_instance_commands.errors, - activity_instance_commands.expansion_run_id - from sequence - join sequence_to_simulated_activity - on sequence.seq_id = sequence_to_simulated_activity.seq_id and - sequence.simulation_dataset_id = - sequence_to_simulated_activity.simulation_dataset_id - join activity_instance_commands - on sequence_to_simulated_activity.simulated_activity_id = - activity_instance_commands.activity_instance_id - join expansion_run - on activity_instance_commands.expansion_run_id = expansion_run.id - where sequence.seq_id = $2 - and sequence.simulation_dataset_id = $1), - max_values as (select activity_instance_id, max(expansion_run_id) as max_expansion_run_id - from joined_table - group by activity_instance_id) + with joined_table as ( + select aic.commands, + aic.activity_instance_id, + aic.errors, + aic.expansion_run_id + from sequence_to_simulated_activity ssa + join activity_instance_commands aic + on ssa.simulated_activity_id = aic.activity_instance_id + where (ssa.simulation_dataset_id, ssa.seq_id) = ($1, $2)), + max_values as ( + select + activity_instance_id, + max(expansion_run_id) as max_expansion_run_id + from joined_table + group by activity_instance_id + ) select joined_table.commands, joined_table.activity_instance_id, joined_table.errors @@ -283,23 +280,16 @@ seqjsonRouter.post('/bulk-get-seqjson-for-seqid-and-simulation-dataset', async ( with joined_table as ( select - activity_instance_commands.commands, - activity_instance_commands.activity_instance_id, - activity_instance_commands.errors, - activity_instance_commands.expansion_run_id, - sequence.seq_id, - sequence.simulation_dataset_id - from sequence - join sequence_to_simulated_activity - on sequence.seq_id = sequence_to_simulated_activity.seq_id - and sequence.simulation_dataset_id = - sequence_to_simulated_activity.simulation_dataset_id - join activity_instance_commands - on sequence_to_simulated_activity.simulated_activity_id = - activity_instance_commands.activity_instance_id - join expansion_run - on activity_instance_commands.expansion_run_id = expansion_run.id - where (sequence.seq_id, sequence.simulation_dataset_id) in (${pgFormat('%L', inputTuples)}) + aic.commands, + aic.activity_instance_id, + aic.errors, + aic.expansion_run_id, + ssa.seq_id, + ssa.simulation_dataset_id + from sequence_to_simulated_activity ssa + join activity_instance_commands aic + on ssa.simulated_activity_id = aic.activity_instance_id + where (ssa.seq_id, ssa.simulation_dataset_id) in (${pgFormat('%L', inputTuples)}) ), max_values as ( select @@ -328,8 +318,8 @@ seqjsonRouter.post('/bulk-get-seqjson-for-seqid-and-simulation-dataset', async ( }>( ` select metadata, seq_id, simulation_dataset_id - from sequence - where (sequence.seq_id, sequence.simulation_dataset_id) in (${pgFormat('%L', inputTuples)}); + from sequence s + where (s.seq_id, s.simulation_dataset_id) in (${pgFormat('%L', inputTuples)}); `, ), ]); From 0895a6a31515a558d577328c9b59645edb4a216d Mon Sep 17 00:00:00 2001 From: Ryan Goetz Date: Mon, 18 Mar 2024 05:07:59 -1000 Subject: [PATCH 3/4] Adding time calculation and minor tweaks. * Using the span to calculate an activityInstance's 'startTime' and 'endTime' --- .../simulatedActivityBatchLoader.ts | 136 +++++++++--------- 1 file changed, 69 insertions(+), 67 deletions(-) diff --git a/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts b/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts index f3102e3f88..2c741f0fc0 100644 --- a/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts +++ b/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts @@ -12,21 +12,21 @@ export const simulatedActivitiesBatchLoader: BatchLoader< { graphqlClient: GraphQLClient; activitySchemaDataLoader: InferredDataloader } > = opts => async keys => { const result = await opts.graphqlClient.batchRequests< - { - data: { - simulation_dataset: { - id: number, - simulation: { - plan: { - id: number, - model_id: number - } - }, - simulation_start_time: string, - dataset: { spans: GQLSpan[]}, - } + { + data: { + simulation_dataset: { + id: number; + simulation: { + plan: { + id: number; + model_id: number; + }; + }; + simulation_start_time: string; + dataset: { spans: GQLSpan[] }; }; - }[] + }; + }[] >( keys.map(key => ({ document: gql` @@ -50,7 +50,8 @@ export const simulatedActivitiesBatchLoader: BatchLoader< } } } - }`, + } + `, variables: { simulationDatasetId: key.simulationDatasetId, }, @@ -59,9 +60,8 @@ export const simulatedActivitiesBatchLoader: BatchLoader< return Promise.all( keys.map(async ({ simulationDatasetId }) => { - const simulation_dataset = result.find( - res => res.data.simulation_dataset.id === simulationDatasetId, - )?.data.simulation_dataset; + const simulation_dataset = result.find(res => res.data.simulation_dataset.id === simulationDatasetId)?.data + .simulation_dataset; if (simulation_dataset === undefined) { return new ErrorWithStatusCode(`No simulation_dataset with id: ${simulationDatasetId}`, 404); } @@ -69,21 +69,18 @@ export const simulatedActivitiesBatchLoader: BatchLoader< const spans = simulation_dataset.dataset.spans; const simulatedActivities: GraphQLSimulatedActivityInstance[] = spans.map(span => { - return { - id: span.id, - simulation_dataset_id: simulation_dataset.id, - plan_id: simulation_dataset.simulation.plan.id, - model_id: simulation_dataset.simulation.plan.model_id, - attributes: span.attributes, - duration: span.duration, - start_offset: span.start_offset, - // TODO: Sum this intervals + durations properly - start_time: simulation_dataset.simulation_start_time + span.start_offset, - end_time: simulation_dataset.simulation_start_time + span.start_offset + span.duration, - activity_type_name: span.activity_type_name - } - } - ) + return { + id: span.id, + simulation_dataset_id: simulation_dataset.id, + plan_id: simulation_dataset.simulation.plan.id, + model_id: simulation_dataset.simulation.plan.model_id, + attributes: span.attributes, + duration: span.duration, + start_offset: span.start_offset, + simulation_start_time: simulation_dataset.simulation_start_time, + activity_type_name: span.activity_type_name, + }; + }); return Promise.all( simulatedActivities.map(async simulatedActivity => mapGraphQLActivityInstance( @@ -108,15 +105,15 @@ export const simulatedActivityInstanceBySimulatedActivityIdBatchLoader: BatchLoa { data: { simulation_dataset: { - id: number, - simulation_start_time: string, + id: number; + simulation_start_time: string; simulation: { plan: { - id: number, - model_id: number, - } - }, - dataset: {span: GQLSpan} + id: number; + model_id: number; + }; + }; + dataset: { spans: GQLSpan[] }; }; }; }[] @@ -134,7 +131,7 @@ export const simulatedActivityInstanceBySimulatedActivityIdBatchLoader: BatchLoa } } dataset { - spans: spans(where: {id: {_eq: $simulatedActivityId}}) { + spans: spans(where: { id: { _eq: $simulatedActivityId } }) { id attributes start_offset @@ -143,7 +140,8 @@ export const simulatedActivityInstanceBySimulatedActivityIdBatchLoader: BatchLoa } } } - }`, + } + `, variables: { simulationDatasetId: key.simulationDatasetId, simulatedActivityId: key.simulatedActivityId, @@ -153,25 +151,22 @@ export const simulatedActivityInstanceBySimulatedActivityIdBatchLoader: BatchLoa return Promise.all( keys.map(async ({ simulationDatasetId, simulatedActivityId }) => { - const simulation_dataset = result.find( - res => - res.data.simulation_dataset.id === simulationDatasetId + const simulation_dataset = result.find(res => + res.data?.simulation_dataset?.dataset?.spans?.some(span => span.id === simulatedActivityId), )?.data.simulation_dataset; if (simulation_dataset === undefined) { - return new ErrorWithStatusCode( - `No simulation_dataset with id: ${simulationDatasetId}`, - 404, - ); + return new ErrorWithStatusCode(`No simulation_dataset with id: ${simulationDatasetId}`, 404); } - const span = simulation_dataset.dataset.span - if(span === undefined) { + const spans = simulation_dataset?.dataset.spans; + if (spans === undefined || spans.length === 0 || spans[0] === undefined) { return new ErrorWithStatusCode( - `No simulation_dataset with id: ${simulationDatasetId} and simulated activity id: ${simulatedActivityId}`, - 404, + `No simulation_dataset with id: ${simulationDatasetId} and simulated activity id: ${simulatedActivityId}`, + 404, ); } + const span = spans[0]; const simulatedActivity: GraphQLSimulatedActivityInstance = { id: span.id, simulation_dataset_id: simulation_dataset.id, @@ -180,11 +175,9 @@ export const simulatedActivityInstanceBySimulatedActivityIdBatchLoader: BatchLoa attributes: span.attributes, duration: span.duration, start_offset: span.start_offset, - // TODO: Sum this intervals + durations properly - start_time: simulation_dataset.simulation_start_time + span.start_offset, - end_time: simulation_dataset.simulation_start_time + span.start_offset + span.duration, - activity_type_name: span.activity_type_name - } + simulation_start_time: simulation_dataset.simulation_start_time, + activity_type_name: span.activity_type_name, + }; return mapGraphQLActivityInstance( simulatedActivity, await opts.activitySchemaDataLoader.load({ @@ -236,7 +229,7 @@ export interface GQLSpan< ActivityArguments extends Record = Record, ActivityComputedAttributes extends Record = Record, > { - id: number + id: number; attributes: GraphQLSimulatedActivityAttributes; start_offset: string; duration: string; @@ -254,8 +247,7 @@ export interface GraphQLSimulatedActivityInstance< attributes: GraphQLSimulatedActivityAttributes; duration: string; start_offset: string; - start_time: string; - end_time: string; + simulation_start_time: string; activity_type_name: string; } @@ -263,17 +255,27 @@ export function mapGraphQLActivityInstance( activityInstance: GraphQLSimulatedActivityInstance, activitySchema: GraphQLActivitySchema, ): SimulatedActivity { + const duration = activityInstance.duration + ? Temporal.Duration.from(parse(activityInstance.duration).toISOString()) + : null; + const startOffset: Temporal.Duration = Temporal.Duration.from(parse(activityInstance.start_offset).toISOString()); + const startTime: Temporal.Instant = Temporal.Instant.from(activityInstance.simulation_start_time) + .toZonedDateTimeISO('UTC') + .add(startOffset) + .toInstant(); + const endTime = duration ? startTime.toZonedDateTimeISO('UTC').add(duration).toInstant() : null; + return { simulationDataset: { simulation: { - planId: activityInstance.model_id + planId: activityInstance.plan_id, }, }, id: activityInstance.id, - duration: activityInstance.duration ? Temporal.Duration.from(parse(activityInstance.duration).toISOString()) : null, - startOffset: Temporal.Duration.from(parse(activityInstance.start_offset).toISOString()), - startTime: Temporal.Instant.from(activityInstance.start_time), - endTime: activityInstance.end_time ? Temporal.Instant.from(activityInstance.end_time) : null, + duration, + startOffset, + startTime, + endTime, simulationDatasetId: activityInstance.simulation_dataset_id, activityTypeName: activityInstance.activity_type_name, attributes: { @@ -283,7 +285,7 @@ export function mapGraphQLActivityInstance( acc[key] = convertType(value, param.schema); } return acc; - }, {} as { [attributeName: string]: any }), + }, {} as Record), directiveId: activityInstance.attributes.directiveId, computed: activityInstance.attributes.computedAttributes ? convertType(activityInstance.attributes.computedAttributes, activitySchema.computed_attributes_value_schema) From 423034fe2bef0ed7f2da6f3068b56b1b2c6f06cf Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Mon, 18 Mar 2024 12:24:59 -0700 Subject: [PATCH 4/4] Add error if too many spans found --- .../src/lib/batchLoaders/simulatedActivityBatchLoader.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts b/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts index 2c741f0fc0..0e0462ba3f 100644 --- a/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts +++ b/sequencing-server/src/lib/batchLoaders/simulatedActivityBatchLoader.ts @@ -166,6 +166,13 @@ export const simulatedActivityInstanceBySimulatedActivityIdBatchLoader: BatchLoa ); } + if(spans.length > 1) { + return new ErrorWithStatusCode( + `Too many spans with simulated activity id ${simulatedActivityId} found for simulation_dataset with id ${simulationDatasetId}`, + 404, + ); + } + const span = spans[0]; const simulatedActivity: GraphQLSimulatedActivityInstance = { id: span.id,