From d47d72eb325d1c97f8c039d335302d98a565aa98 Mon Sep 17 00:00:00 2001 From: Matthew Dailis Date: Fri, 12 Apr 2024 15:08:07 -0700 Subject: [PATCH 01/20] Introduce StepResult as return value from performJobs --- .../merlin/driver/engine/SimulationEngine.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java index 9b87ae83be..bdd0e1f115 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java @@ -203,8 +203,16 @@ public JobSchedule.Batch extractNextJobs(final Duration maximumTime) { return batch; } + public record ResourceUpdate() {} + + public record StepResult( + List> commits, + ResourceUpdate resourceUpdate, + Optional error + ) {} + /** Performs a collection of tasks concurrently, extending the given timeline by their stateful effects. */ - public Pair, Optional> performJobs( + public StepResult performJobs( final Collection jobs, final LiveCells context, final Duration currentTime, @@ -223,10 +231,10 @@ public Pair, Optional> performJobs( })); if (exception.getValue().isPresent()) { - return Pair.of(tip, exception.getValue()); + return new StepResult(List.of(tip), new ResourceUpdate(), exception.getValue()); } } - return Pair.of(tip, Optional.empty()); + return new StepResult(List.of(tip), new ResourceUpdate(), Optional.empty()); } /** Performs a single job. */ From f5dd7134e534f677c0e899b5710b651adbf5ae74 Mon Sep 17 00:00:00 2001 From: Matthew Dailis Date: Fri, 12 Apr 2024 16:58:25 -0700 Subject: [PATCH 02/20] Thread ResourceUpdate through Engine Job Execution Logic --- .../driver/engine/SimulationEngine.java | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java index bdd0e1f115..42b047648a 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java @@ -37,7 +37,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -203,7 +202,20 @@ public JobSchedule.Batch extractNextJobs(final Duration maximumTime) { return batch; } - public record ResourceUpdate() {} + public record ResourceUpdate(Duration currentTime, Map updates) { + static ResourceUpdate init(Duration currentTime) { + return new ResourceUpdate(currentTime, new LinkedHashMap<>()); + } + public void log(String id, Dynamics dynamics) { + this.updates.put(id, dynamics); + } + public boolean isEmpty() { + return updates.isEmpty(); + } + public int size() { + return updates.size(); + } + } public record StepResult( List> commits, @@ -221,20 +233,21 @@ public StepResult performJobs( if (this.closed) throw new IllegalStateException("Cannot perform jobs on closed simulation engine"); var tip = EventGraph.empty(); Mutable> exception = new MutableObject<>(Optional.empty()); + final var resourceUpdate = ResourceUpdate.init(currentTime); for (final var job$ : jobs) { tip = EventGraph.concurrently(tip, TaskFrame.run(job$, context, (job, frame) -> { try { - this.performJob(job, frame, currentTime, maximumTime); + this.performJob(job, frame, currentTime, maximumTime, resourceUpdate); } catch (Throwable ex) { exception.setValue(Optional.of(ex)); } })); if (exception.getValue().isPresent()) { - return new StepResult(List.of(tip), new ResourceUpdate(), exception.getValue()); + return new StepResult(List.of(tip), resourceUpdate, exception.getValue()); } } - return new StepResult(List.of(tip), new ResourceUpdate(), Optional.empty()); + return new StepResult(List.of(tip), resourceUpdate, Optional.empty()); } /** Performs a single job. */ @@ -242,7 +255,8 @@ public void performJob( final JobId job, final TaskFrame frame, final Duration currentTime, - final Duration maximumTime + final Duration maximumTime, + final ResourceUpdate resourceUpdate ) throws SpanException { if (job instanceof JobId.TaskJobId j) { this.stepTask(j.id(), frame, currentTime); @@ -251,7 +265,7 @@ public void performJob( } else if (job instanceof JobId.ConditionJobId j) { this.updateCondition(j.id(), frame, currentTime, maximumTime); } else if (job instanceof JobId.ResourceJobId j) { - this.updateResource(j.id(), frame, currentTime); + this.updateResource(j.id(), frame, currentTime, resourceUpdate); } else { throw new IllegalArgumentException("Unexpected subtype of %s: %s".formatted(JobId.class, job.getClass())); } @@ -386,11 +400,12 @@ public void updateCondition( public void updateResource( final ResourceId resource, final TaskFrame frame, - final Duration currentTime - ) { + final Duration currentTime, + final ResourceUpdate resourceUpdate) { if (this.closed) throw new IllegalStateException("Cannot update resource on closed simulation engine"); final var querier = new EngineQuerier(frame); - this.resources.get(resource).append(currentTime, querier); + final var dynamics = this.resources.get(resource).getDynamics(querier); + resourceUpdate.log(resource.id(), dynamics); this.waitingResources.subscribeQuery(resource, querier.referencedTopics); From 0d8ec2dab9f97c1912cd799a5c697c64c962635a Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 23 May 2024 13:26:28 -0700 Subject: [PATCH 03/20] Create ResourceProfile, ResourceProfiles, ResourceSegments --- .../foomissionmodel/SimulateMapSchedule.java | 4 +- .../merlin/driver/SimulationResults.java | 10 ++-- .../driver/resources/ResourceProfile.java | 12 ++++ .../driver/resources/ResourceProfiles.java | 11 ++++ .../driver/resources/ResourceSegments.java | 20 +++++++ .../aerie/merlin/driver/CellExpiryTest.java | 2 +- .../merlin/server/http/ProfileParsers.java | 16 +++--- .../merlin/server/models/ProfileSet.java | 47 +++++++-------- .../InMemoryResultsCellRepository.java | 8 +-- .../remotes/postgres/PostProfilesAction.java | 14 +++-- .../remotes/postgres/ProfileRepository.java | 26 ++++----- .../server/services/ConstraintAction.java | 8 +-- .../services/GetSimulationResultsAction.java | 22 +++---- .../SimulationResultsConverter.java | 4 +- .../SimulationResultsComparisonUtils.java | 6 +- .../scheduler/server/models/ProfileSet.java | 53 ++++------------- .../server/models/UnwrappedProfileSet.java | 9 +-- .../server/services/GraphQLMerlinService.java | 57 ++++++++++--------- 18 files changed, 164 insertions(+), 165 deletions(-) create mode 100644 merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceProfile.java create mode 100644 merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceProfiles.java create mode 100644 merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceSegments.java diff --git a/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/SimulateMapSchedule.java b/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/SimulateMapSchedule.java index 767b61d1ae..a5eafc5d2f 100644 --- a/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/SimulateMapSchedule.java +++ b/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/SimulateMapSchedule.java @@ -49,12 +49,12 @@ void simulateWithMapSchedule() { simulationResults.realProfiles.forEach((name, samples) -> { System.out.println(name + ":"); - samples.getRight().forEach(point -> System.out.format("\t%s\t%s\n", point.extent(), point.dynamics())); + samples.segments().forEach(point -> System.out.format("\t%s\t%s\n", point.extent(), point.dynamics())); }); simulationResults.discreteProfiles.forEach((name, samples) -> { System.out.println(name + ":"); - samples.getRight().forEach(point -> System.out.format("\t%s\t%s\n", point.extent(), point.dynamics())); + samples.segments().forEach(point -> System.out.format("\t%s\t%s\n", point.extent(), point.dynamics())); }); simulationResults.simulatedActivities.forEach((name, activity) -> { diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationResults.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationResults.java index 426668496a..b842fd48cc 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationResults.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationResults.java @@ -1,7 +1,7 @@ package gov.nasa.jpl.aerie.merlin.driver; import gov.nasa.jpl.aerie.merlin.driver.engine.EventRecord; -import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.driver.resources.ResourceProfile; import gov.nasa.jpl.aerie.merlin.driver.timeline.EventGraph; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; @@ -18,16 +18,16 @@ public final class SimulationResults { public final Instant startTime; public final Duration duration; - public final Map>>> realProfiles; - public final Map>>> discreteProfiles; + public final Map> realProfiles; + public final Map> discreteProfiles; public final Map simulatedActivities; public final Map unfinishedActivities; public final List> topics; public final Map>> events; public SimulationResults( - final Map>>> realProfiles, - final Map>>> discreteProfiles, + final Map> realProfiles, + final Map> discreteProfiles, final Map simulatedActivities, final Map unfinishedActivities, final Instant startTime, diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceProfile.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceProfile.java new file mode 100644 index 0000000000..b369ca1310 --- /dev/null +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceProfile.java @@ -0,0 +1,12 @@ +package gov.nasa.jpl.aerie.merlin.driver.resources; + +import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; + +import java.util.List; + +public record ResourceProfile (ValueSchema schema, List> segments) { + public static ResourceProfile of(ValueSchema schema, List> segments) { + return new ResourceProfile(schema, segments); + } +} diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceProfiles.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceProfiles.java new file mode 100644 index 0000000000..d4ed356c20 --- /dev/null +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceProfiles.java @@ -0,0 +1,11 @@ +package gov.nasa.jpl.aerie.merlin.driver.resources; + +import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; +import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; + +import java.util.Map; + +public record ResourceProfiles( + Map> realProfiles, + Map> discreteProfiles +) {} diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceSegments.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceSegments.java new file mode 100644 index 0000000000..64b674639d --- /dev/null +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/ResourceSegments.java @@ -0,0 +1,20 @@ +package gov.nasa.jpl.aerie.merlin.driver.resources; + +import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; +import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; + +import java.util.ArrayList; + +record ResourceSegments (ValueSchema valueSchema, ArrayList> segments) { + record Segment (Duration startOffset, T dynamics) {} + + ResourceSegments(ValueSchema valueSchema, int threshold) { + this(valueSchema, new ArrayList<>(threshold)); + } + + public ResourceSegments deepCopy(){ + ArrayList> segmentsCopy = new ArrayList<>(this.segments.size()); + segmentsCopy.addAll(this.segments); + return new ResourceSegments<>(valueSchema, segmentsCopy); + } +} diff --git a/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/CellExpiryTest.java b/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/CellExpiryTest.java index 8db50f1676..7b1eec036c 100644 --- a/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/CellExpiryTest.java +++ b/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/CellExpiryTest.java @@ -38,7 +38,7 @@ public void testResourceProfilingByExpiry() { Duration.SECONDS.times(5), () -> false); - final var actual = results.discreteProfiles.get("/key").getRight(); + final var actual = results.discreteProfiles.get("/key").segments(); final var expected = List.of( new ProfileSegment<>(duration(500, MILLISECONDS), SerializedValue.of("value")), diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/http/ProfileParsers.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/http/ProfileParsers.java index 6a981aa127..a4399273e3 100644 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/http/ProfileParsers.java +++ b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/http/ProfileParsers.java @@ -2,17 +2,15 @@ import gov.nasa.jpl.aerie.json.JsonParser; import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.driver.resources.ResourceProfile; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; -import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; import gov.nasa.jpl.aerie.merlin.server.models.DiscreteProfile; import gov.nasa.jpl.aerie.merlin.server.models.ProfileSet; import gov.nasa.jpl.aerie.merlin.server.models.RealProfile; -import org.apache.commons.lang3.tuple.Pair; import java.util.HashMap; -import java.util.List; import java.util.Optional; import java.util.function.BiFunction; @@ -75,15 +73,15 @@ public final class ProfileParsers { = mapP(chooseP(realProfileP, discreteProfileP)) . map( profiles -> { - final var realProfiles = new HashMap>>>>(); - final var discreteProfiles = new HashMap>>>>(); + final var realProfiles = new HashMap>>(); + final var discreteProfiles = new HashMap>>(); for (final var entry : profiles.entrySet()) { final var name = entry.getKey(); final var profile = entry.getValue(); if (profile instanceof RealProfile p) { - realProfiles.put(name, Pair.of(p.schema(), p.segments())); + realProfiles.put(name, ResourceProfile.of(p.schema(), p.segments())); } else if (profile instanceof DiscreteProfile p) { - discreteProfiles.put(name, Pair.of(p.schema(), p.segments())); + discreteProfiles.put(name, ResourceProfile.of(p.schema(), p.segments())); } else { // If this happens, then the parser must have been updated without updating the mapping code // It should not be possible to reach this point unless a new profile type is introduced and we @@ -98,11 +96,11 @@ public final class ProfileParsers { profileSet .realProfiles() .forEach((name, profile) -> - profiles.put(name, new RealProfile(profile.getLeft(), profile.getRight()))); + profiles.put(name, new RealProfile(profile.schema(), profile.segments()))); profileSet .discreteProfiles() .forEach((name, profile) -> - profiles.put(name, new DiscreteProfile(profile.getLeft(), profile.getRight()))); + profiles.put(name, new DiscreteProfile(profile.schema(), profile.segments()))); return profiles; } ); diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/models/ProfileSet.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/models/ProfileSet.java index 59bc0b18df..448bd8b116 100644 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/models/ProfileSet.java +++ b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/models/ProfileSet.java @@ -1,26 +1,23 @@ package gov.nasa.jpl.aerie.merlin.server.models; import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.driver.resources.ResourceProfile; import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; -import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; import org.apache.commons.lang3.tuple.Pair; -import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; import java.util.stream.Collectors; -import static gov.nasa.jpl.aerie.json.Uncurry.untuple; - public record ProfileSet( - Map>>>> realProfiles, - Map>>>> discreteProfiles + Map>> realProfiles, + Map>> discreteProfiles ) { public static ProfileSet ofNullable( - final Map>>>> realProfiles, - final Map>>>> discreteProfiles + final Map>> realProfiles, + final Map>> discreteProfiles ) { return new ProfileSet( realProfiles, @@ -28,8 +25,8 @@ public static ProfileSet ofNullable( ); } public static ProfileSet of( - final Map>>> realProfiles, - final Map>>> discreteProfiles + final Map> realProfiles, + final Map> discreteProfiles ) { return new ProfileSet( wrapInOptional(realProfiles), @@ -37,39 +34,35 @@ public static ProfileSet of( ); } - public static Map>>>> wrapInOptional( - final Map>>> profileMap + public static Map>> wrapInOptional( + final Map> profileMap ) { return profileMap .entrySet().stream() .map($ -> Pair.of( $.getKey(), - Pair.of( - $.getValue().getLeft(), - $.getValue().getRight() + ResourceProfile.of( + $.getValue().schema(), + $.getValue().segments() .stream() - .map(untuple(segment -> new ProfileSegment<>(segment.extent(), Optional.of(segment.dynamics())))) - .toList() - ) - )) + .map(segment -> new ProfileSegment<>(segment.extent(), Optional.of(segment.dynamics()))) + .toList()))) .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); } - public static Map>>> unwrapOptional( - final Map>>>> profileMap + public static Map> unwrapOptional( + final Map>> profileMap ) throws NoSuchElementException { return profileMap .entrySet().stream() .map($ -> Pair.of( $.getKey(), - Pair.of( - $.getValue().getLeft(), - $.getValue().getRight() + ResourceProfile.of( + $.getValue().schema(), + $.getValue().segments() .stream() .map(segment -> new ProfileSegment<>(segment.extent(), segment.dynamics().get())) - .toList() - ) - )) + .toList()))) .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); } } diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/InMemoryResultsCellRepository.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/InMemoryResultsCellRepository.java index dd9e6a2773..11f842de30 100644 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/InMemoryResultsCellRepository.java +++ b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/InMemoryResultsCellRepository.java @@ -4,18 +4,16 @@ import gov.nasa.jpl.aerie.merlin.driver.SimulatedActivityId; import gov.nasa.jpl.aerie.merlin.driver.SimulationFailure; import gov.nasa.jpl.aerie.merlin.driver.SimulationResults; -import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.driver.resources.ResourceProfile; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; -import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; import gov.nasa.jpl.aerie.merlin.server.ResultsProtocol; import gov.nasa.jpl.aerie.merlin.server.exceptions.NoSuchPlanException; import gov.nasa.jpl.aerie.merlin.server.models.PlanId; import gov.nasa.jpl.aerie.merlin.server.models.ProfileSet; import gov.nasa.jpl.aerie.merlin.server.models.SimulationDatasetId; import gov.nasa.jpl.aerie.merlin.server.models.SimulationResultsHandle; -import org.apache.commons.lang3.tuple.Pair; import java.time.Instant; import java.util.HashMap; @@ -187,8 +185,8 @@ public SimulationResults getSimulationResults() { @Override public ProfileSet getProfiles(final List profileNames) { - final var realProfiles = new HashMap>>>(); - final var discreteProfiles = new HashMap>>>(); + final var realProfiles = new HashMap>(); + final var discreteProfiles = new HashMap>(); for (final var profileName : profileNames) { if (this.simulationResults.realProfiles.containsKey(profileName)) { realProfiles.put(profileName, this.simulationResults.realProfiles.get(profileName)); diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/PostProfilesAction.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/PostProfilesAction.java index 1438e9b14c..798045e12d 100644 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/PostProfilesAction.java +++ b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/PostProfilesAction.java @@ -1,6 +1,7 @@ package gov.nasa.jpl.aerie.merlin.server.remotes.postgres; import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.driver.resources.ResourceProfile; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; @@ -20,6 +21,7 @@ import static gov.nasa.jpl.aerie.merlin.server.remotes.postgres.PostgresParsers.discreteProfileTypeP; import static gov.nasa.jpl.aerie.merlin.server.remotes.postgres.PostgresParsers.realProfileTypeP; + /*package-local*/ final class PostProfilesAction implements AutoCloseable { private final @Language("SQL") String sql = """ insert into merlin.profile (dataset_id, name, type, duration) @@ -33,17 +35,17 @@ public PostProfilesAction(final Connection connection) throws SQLException { public Map apply( final long datasetId, - final Map>>>> realProfiles, - final Map>>>> discreteProfiles + final Map>> realProfiles, + final Map>> discreteProfiles ) throws SQLException { final var resourceNames = new ArrayList(); final var resourceTypes = new ArrayList>(); final var durations = new ArrayList(); for (final var entry : realProfiles.entrySet()) { final var resource = entry.getKey(); - final var schema = entry.getValue().getLeft(); + final var schema = entry.getValue().schema(); final var realResourceType = Pair.of("real", schema); - final var segments = entry.getValue().getRight(); + final var segments = entry.getValue().segments(); final var duration = sumDurations(segments); resourceNames.add(resource); resourceTypes.add(realResourceType); @@ -57,9 +59,9 @@ public Map apply( for (final var entry : discreteProfiles.entrySet()) { final var resource = entry.getKey(); - final var schema = entry.getValue().getLeft(); + final var schema = entry.getValue().schema(); final var resourceType = Pair.of("discrete", schema); - final var segments = entry.getValue().getRight(); + final var segments = entry.getValue().segments(); final var duration = sumDurations(segments); resourceNames.add(resource); resourceTypes.add(resourceType); diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/ProfileRepository.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/ProfileRepository.java index 0de9a137d3..032238c8a9 100644 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/ProfileRepository.java +++ b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/ProfileRepository.java @@ -1,6 +1,7 @@ package gov.nasa.jpl.aerie.merlin.server.remotes.postgres; import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.driver.resources.ResourceProfile; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; @@ -8,7 +9,6 @@ import gov.nasa.jpl.aerie.merlin.server.models.PlanId; import gov.nasa.jpl.aerie.merlin.server.models.ProfileSet; import gov.nasa.jpl.aerie.merlin.server.models.SimulationDatasetId; -import org.apache.commons.lang3.tuple.Pair; import java.sql.Connection; import java.sql.SQLException; @@ -26,22 +26,22 @@ static ProfileSet getProfiles( final Connection connection, final long datasetId ) throws SQLException { - final var realProfiles = new HashMap>>>>(); - final var discreteProfiles = new HashMap>>>>(); + final var realProfiles = new HashMap>> (); + final var discreteProfiles = new HashMap>>(); final var profileRecords = getProfileRecords(connection, datasetId); for (final var record : profileRecords) { switch (record.type().getLeft()) { case "real" -> realProfiles.put( record.name(), - Pair.of( + ResourceProfile.of( record.type().getRight(), getRealProfileSegments(connection, record.datasetId(), record.id(), record.duration()) ) ); case "discrete" -> discreteProfiles.put( record.name(), - Pair.of( + ResourceProfile.of( record.type().getRight(), getDiscreteProfileSegments(connection, record.datasetId(), record.id(), record.duration()) ) @@ -58,22 +58,22 @@ static ProfileSet getProfiles( final long datasetId, final List names ) throws SQLException { - final var realProfiles = new HashMap>>>>(); - final var discreteProfiles = new HashMap>>>>(); + final var realProfiles = new HashMap>>(); + final var discreteProfiles = new HashMap>>(); final var profileRecords = getProfileRecords(connection, datasetId, names); for (final var record : profileRecords) { switch (record.type().getLeft()) { case "real" -> realProfiles.put( record.name(), - Pair.of( + ResourceProfile.of( record.type().getRight(), getRealProfileSegments(connection, record.datasetId(), record.id(), record.duration()) ) ); case "discrete" -> discreteProfiles.put( record.name(), - Pair.of( + ResourceProfile.of( record.type().getRight(), getDiscreteProfileSegments(connection, record.datasetId(), record.id(), record.duration()) ) @@ -236,12 +236,12 @@ private static void postProfileSegments( connection, datasetId, record, - realProfiles.get(resource).getRight()); + realProfiles.get(resource).segments()); case "discrete" -> postDiscreteProfileSegments( connection, datasetId, record, - discreteProfiles.get(resource).getRight()); + discreteProfiles.get(resource).segments()); default -> throw new Error("Unrecognized profile type " + record.type().getLeft()); } } @@ -264,7 +264,7 @@ final var record = records.get(resource); final var newProfileDuration = appendProfileSegmentsAction.apply( datasetId, record, - realProfiles.get(resource).getRight(), + realProfiles.get(resource).segments(), realDynamicsP); updateProfileDurationAction.apply(datasetId, record.id(), newProfileDuration); } @@ -278,7 +278,7 @@ final var record = records.get(resource); final var newProfileDuration = appendProfileSegmentsAction.apply( datasetId, record, - discreteProfiles.get(resource).getRight(), + discreteProfiles.get(resource).segments(), serializedValueP); updateProfileDurationAction.apply(datasetId, record.id(), newProfileDuration); } diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/ConstraintAction.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/ConstraintAction.java index cdcf464832..38105f4684 100644 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/ConstraintAction.java +++ b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/ConstraintAction.java @@ -113,14 +113,14 @@ public Map> getViolations(final PlanId planId, final Opt profile.getKey(), DiscreteProfile.fromExternalProfile( offsetFromSimulationStart, - profile.getValue().getRight())); + profile.getValue().segments())); } for (final var profile : profileSet.realProfiles().entrySet()) { realExternalProfiles.put( profile.getKey(), LinearProfile.fromExternalProfile( offsetFromSimulationStart, - profile.getValue().getRight())); + profile.getValue().segments())); } } @@ -195,7 +195,7 @@ public Map> getViolations(final PlanId planId, final Opt for (final var _entry : ProfileSet.unwrapOptional(newProfiles.realProfiles()).entrySet()) { if (!realProfiles.containsKey(_entry.getKey())) { - realProfiles.put(_entry.getKey(), LinearProfile.fromSimulatedProfile(_entry.getValue().getRight())); + realProfiles.put(_entry.getKey(), LinearProfile.fromSimulatedProfile(_entry.getValue().segments())); } } @@ -203,7 +203,7 @@ public Map> getViolations(final PlanId planId, final Opt if (!discreteProfiles.containsKey(_entry.getKey())) { discreteProfiles.put( _entry.getKey(), - DiscreteProfile.fromSimulatedProfile(_entry.getValue().getRight())); + DiscreteProfile.fromSimulatedProfile(_entry.getValue().segments())); } } } catch (InputMismatchException ex) { diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/GetSimulationResultsAction.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/GetSimulationResultsAction.java index abbb3c6611..6b48c94b04 100644 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/GetSimulationResultsAction.java +++ b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/GetSimulationResultsAction.java @@ -41,17 +41,13 @@ public Response run(final PlanId planId, final boolean forceResim, final HasuraA final var response = this.simulationService.getSimulationResults(planId, forceResim, revisionData, session.hasuraUserId()); - if (response instanceof ResultsProtocol.State.Pending r) { - return new Response.Pending(r.simulationDatasetId()); - } else if (response instanceof ResultsProtocol.State.Incomplete r) { - return new Response.Incomplete(r.simulationDatasetId()); - } else if (response instanceof ResultsProtocol.State.Failed r) { - return new Response.Failed(r.simulationDatasetId(), r.reason()); - } else if (response instanceof ResultsProtocol.State.Success r) { - return new Response.Complete(r.simulationDatasetId()); - } else { - throw new UnexpectedSubtypeError(ResultsProtocol.State.class, response); - } + return switch (response) { + case ResultsProtocol.State.Pending r -> new Response.Pending(r.simulationDatasetId()); + case ResultsProtocol.State.Incomplete r -> new Response.Incomplete(r.simulationDatasetId()); + case ResultsProtocol.State.Failed r -> new Response.Failed(r.simulationDatasetId(), r.reason()); + case ResultsProtocol.State.Success r -> new Response.Complete(r.simulationDatasetId()); + default -> throw new UnexpectedSubtypeError(ResultsProtocol.State.class, response); + }; } public Map>> getResourceSamples(final PlanId planId) @@ -66,7 +62,7 @@ public Map>> getResourceSamples(fin simulationResults.realProfiles.forEach((name, p) -> { var elapsed = Duration.ZERO; - var profile = p.getRight(); + var profile = p.segments(); final var timeline = new ArrayList>(); for (final var piece : profile) { @@ -84,7 +80,7 @@ public Map>> getResourceSamples(fin }); simulationResults.discreteProfiles.forEach((name, p) -> { var elapsed = Duration.ZERO; - var profile = p.getRight(); + var profile = p.segments(); final var timeline = new ArrayList>(); for (final var piece : profile) { diff --git a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationResultsConverter.java b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationResultsConverter.java index 9e253315be..868f912f16 100644 --- a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationResultsConverter.java +++ b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationResultsConverter.java @@ -30,8 +30,8 @@ public static gov.nasa.jpl.aerie.constraints.model.SimulationResults convertToCo driverResults.startTime, Interval.between(Duration.ZERO, driverResults.duration), activities, - Maps.transformValues(driverResults.realProfiles, $ -> LinearProfile.fromSimulatedProfile($.getRight())), - Maps.transformValues(driverResults.discreteProfiles, $ -> DiscreteProfile.fromSimulatedProfile($.getRight())) + Maps.transformValues(driverResults.realProfiles, $ -> LinearProfile.fromSimulatedProfile($.segments())), + Maps.transformValues(driverResults.discreteProfiles, $ -> DiscreteProfile.fromSimulatedProfile($.segments())) ); } diff --git a/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationResultsComparisonUtils.java b/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationResultsComparisonUtils.java index 25da0a83d4..43918e54fa 100644 --- a/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationResultsComparisonUtils.java +++ b/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationResultsComparisonUtils.java @@ -30,15 +30,15 @@ public static void assertEqualsSimulationResults(final SimulationResults expecte assertEqualsTSA(convertSimulatedActivitiesToTree(expected), convertSimulatedActivitiesToTree(simulationResults)); final var differencesDiscrete = new HashMap>(); for(final var discreteProfile: simulationResults.discreteProfiles.entrySet()){ - final var differences = equalsDiscreteProfile(expected.discreteProfiles.get(discreteProfile.getKey()).getRight(), discreteProfile.getValue().getRight()); + final var differences = equalsDiscreteProfile(expected.discreteProfiles.get(discreteProfile.getKey()).segments(), discreteProfile.getValue().segments()); if(!differences.isEmpty()){ differencesDiscrete.put(discreteProfile.getKey(), differences); } } final var differencesReal = new HashMap>(); for(final var realProfile: simulationResults.realProfiles.entrySet()){ - final var profileElements = realProfile.getValue().getRight(); - final var expectedProfileElements = expected.realProfiles.get(realProfile.getKey()).getRight(); + final var profileElements = realProfile.getValue().segments(); + final var expectedProfileElements = expected.realProfiles.get(realProfile.getKey()).segments(); final var differences = equalsRealProfile(expectedProfileElements, profileElements); if(!differences.isEmpty()) { differencesReal.put(realProfile.getKey(), differences); diff --git a/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/models/ProfileSet.java b/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/models/ProfileSet.java index 0c35034e68..08f4a65d42 100644 --- a/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/models/ProfileSet.java +++ b/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/models/ProfileSet.java @@ -1,6 +1,7 @@ package gov.nasa.jpl.aerie.scheduler.server.models; import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.driver.resources.ResourceProfile; import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; @@ -8,28 +9,18 @@ import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Optional; import java.util.stream.Collectors; import static gov.nasa.jpl.aerie.json.Uncurry.untuple; public record ProfileSet( - Map>>>> realProfiles, - Map>>>> discreteProfiles + Map>> realProfiles, + Map>> discreteProfiles ) { - public static ProfileSet ofNullable( - final Map>>>> realProfiles, - final Map>>>> discreteProfiles - ) { - return new ProfileSet( - realProfiles, - discreteProfiles - ); - } public static ProfileSet of( - final Map>>> realProfiles, - final Map>>> discreteProfiles + final Map> realProfiles, + final Map> discreteProfiles ) { return new ProfileSet( wrapInOptional(realProfiles), @@ -37,39 +28,19 @@ public static ProfileSet of( ); } - public static Map>>>> wrapInOptional( - final Map>>> profileMap + public static Map>> wrapInOptional( + final Map> profileMap ) { return profileMap .entrySet().stream() .map($ -> Pair.of( $.getKey(), - Pair.of( - $.getValue().getLeft(), - $.getValue().getRight() + ResourceProfile.of( + $.getValue().schema(), + $.getValue().segments() .stream() - .map(untuple(segment -> new ProfileSegment<>(segment.extent(), Optional.of(segment.dynamics())))) - .toList() - ) - )) + .map(segment -> new ProfileSegment<>(segment.extent(), Optional.of(segment.dynamics()))) + .toList()))) .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); } - - public static Map>>> unwrapOptional( - final Map>>>> profileMap - ) throws NoSuchElementException { - return profileMap - .entrySet().stream() - .map($ -> Pair.of( - $.getKey(), - Pair.of( - $.getValue().getLeft(), - $.getValue().getRight() - .stream() - .map(segment -> new ProfileSegment<>(segment.extent(), segment.dynamics().get())) - .toList() - ) - )) - .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); - } } diff --git a/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/models/UnwrappedProfileSet.java b/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/models/UnwrappedProfileSet.java index 3a5c5deffb..e367b7b984 100644 --- a/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/models/UnwrappedProfileSet.java +++ b/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/models/UnwrappedProfileSet.java @@ -1,15 +1,12 @@ package gov.nasa.jpl.aerie.scheduler.server.models; -import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.driver.resources.ResourceProfile; import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; -import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; -import org.apache.commons.lang3.tuple.Pair; -import java.util.List; import java.util.Map; public record UnwrappedProfileSet( - Map>>> realProfiles, - Map>>> discreteProfiles + Map> realProfiles, + Map> discreteProfiles ){} diff --git a/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/services/GraphQLMerlinService.java b/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/services/GraphQLMerlinService.java index 4c30ad8629..6d6d6ec076 100644 --- a/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/services/GraphQLMerlinService.java +++ b/scheduler-server/src/main/java/gov/nasa/jpl/aerie/scheduler/server/services/GraphQLMerlinService.java @@ -12,6 +12,7 @@ import gov.nasa.jpl.aerie.merlin.driver.UnfinishedActivity; import gov.nasa.jpl.aerie.merlin.driver.engine.EventRecord; import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.driver.resources.ResourceProfile; import gov.nasa.jpl.aerie.merlin.driver.timeline.EventGraph; import gov.nasa.jpl.aerie.merlin.protocol.model.SchedulerModel; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; @@ -102,7 +103,7 @@ public record GraphQLMerlinService(URI merlinGraphqlURI, String hasuraGraphQlAdm */ private static final java.time.Duration httpTimeout = java.time.Duration.ofSeconds(60); - public record DatasetMetadata(DatasetId datasetId, Duration offsetFromPlanStart){}; + public record DatasetMetadata(DatasetId datasetId, Duration offsetFromPlanStart){} private record SimulationId(long id){} @@ -168,9 +169,8 @@ protected Optional postRequest(final String gqlStr) throws IOExcepti } } - protected Optional postRequest(final String query, final JsonObject variables) throws IOException, - MerlinServiceException - { + protected Optional postRequest(final String query, final JsonObject variables) + throws IOException, MerlinServiceException { try { //TODO: (mem optimization) use streams here to avoid several copies of strings final var reqBody = Json @@ -988,13 +988,13 @@ public ExternalProfiles getExternalProfiles(final PlanId planId) realProfiles.put(name, LinearProfile.fromExternalProfile( datasetMetadata.offsetFromPlanStart, - profile.getRight())); + profile.segments())); }); profiles.discreteProfiles().forEach((name, profile) -> { discreteProfiles.put(name, DiscreteProfile.fromExternalProfile( datasetMetadata.offsetFromPlanStart, - profile.getRight())); + profile.segments())); }); resourceTypes.addAll(extractResourceTypes(profiles)); } @@ -1005,10 +1005,10 @@ public ExternalProfiles getExternalProfiles(final PlanId planId) private Collection extractResourceTypes(final ProfileSet profileSet){ final var resourceTypes = new ArrayList(); profileSet.realProfiles().forEach((name, profile) -> { - resourceTypes.add(new ResourceType(name, profile.getLeft())); + resourceTypes.add(new ResourceType(name, profile.schema())); }); profileSet.discreteProfiles().forEach((name, profile) -> { - resourceTypes.add(new ResourceType(name, profile.getLeft())); + resourceTypes.add(new ResourceType(name, profile.schema())); }); return resourceTypes; } @@ -1046,25 +1046,25 @@ private UnwrappedProfileSet unwrapProfiles(final ProfileSet profileSet) throws M return new UnwrappedProfileSet(unwrapProfiles(profileSet.realProfiles()), unwrapProfiles(profileSet.discreteProfiles())); } - private HashMap>>> unwrapProfiles(Map>>>> profiles) - throws MerlinServiceException - { - final var unwrapped = new HashMap>>>(); + private HashMap> unwrapProfiles( + Map>> profiles + ) { + final var unwrapped = new HashMap>(); for(final var profile: profiles.entrySet()) { final var unwrappedSegments = new ArrayList>(); - for (final var segment : profile.getValue().getRight()) { + for (final var segment : profile.getValue().segments()) { if (segment.dynamics().isPresent()) { unwrappedSegments.add(new ProfileSegment<>(segment.extent(), segment.dynamics().get())); } } - unwrapped.put(profile.getKey(), Pair.of(profile.getValue().getLeft(), unwrappedSegments)); + unwrapped.put(profile.getKey(), ResourceProfile.of(profile.getValue().schema(), unwrappedSegments)); } return unwrapped; } private ProfileSet parseProfiles(JsonArray dataset){ - Map>>>> realProfiles = new HashMap<>(); - Map>>>> discreteProfiles = new HashMap<>(); + Map>> realProfiles = new HashMap<>(); + Map>> discreteProfiles = new HashMap<>(); for(final var profile :dataset){ final var name = profile.asJsonObject().getString("name"); final var type = profile.asJsonObject().getJsonObject("type"); @@ -1081,7 +1081,7 @@ private ProfileSet parseProfiles(JsonArray dataset){ return new ProfileSet(realProfiles, discreteProfiles); } - public Pair>>> parseProfile(JsonObject profile, JsonParser dynamicsParser){ + public ResourceProfile> parseProfile(JsonObject profile, JsonParser dynamicsParser){ // Profile segments are stored with their start offset relative to simulation start // We must convert these to durations describing how long each segment lasts final var type = chooseP(discreteValueSchemaTypeP, realValueSchemaTypeP).parse(profile.getJsonObject("type")).getSuccessOrThrow(); @@ -1123,7 +1123,7 @@ public Pair>>> pa segments.add(new ProfileSegment<>(duration, dynamics)); } } - return Pair.of(type, segments); + return ResourceProfile.of(type, segments); } private Map parseSimulatedActivities(JsonArray simulatedActivitiesArray, Instant simulationStart) @@ -1284,10 +1284,11 @@ private static Duration sumDurations(final List>> Duration::plus ); } - private HashMap postResourceProfiles(DatasetId datasetId, - final Map>>>> realProfiles, - final Map>>>> discreteProfiles) - throws MerlinServiceException, IOException + private HashMap postResourceProfiles( + DatasetId datasetId, + final Map>> realProfiles, + final Map>> discreteProfiles + ) throws MerlinServiceException, IOException { final var req = """ mutation($profiles: [profile_insert_input!]!) { @@ -1304,9 +1305,9 @@ private HashMap postResourceProfiles(DatasetId datasetId, final var durations = new ArrayList(); for (final var entry : realProfiles.entrySet()) { final var resource = entry.getKey(); - final var schema = entry.getValue().getLeft(); + final var schema = entry.getValue().schema(); final var realResourceType = Pair.of("real", schema); - final var segments = entry.getValue().getRight(); + final var segments = entry.getValue().segments(); final var duration = sumDurations(segments); resourceNames.add(resource); resourceTypes.add(realResourceType); @@ -1321,9 +1322,9 @@ private HashMap postResourceProfiles(DatasetId datasetId, } for (final var entry : discreteProfiles.entrySet()) { final var resource = entry.getKey(); - final var schema = entry.getValue().getLeft(); + final var schema = entry.getValue().schema(); final var resourceType = Pair.of("discrete", schema); - final var segments = entry.getValue().getRight(); + final var segments = entry.getValue().segments(); final var duration = sumDurations(segments); resourceNames.add(resource); resourceTypes.add(resourceType); @@ -1379,11 +1380,11 @@ private void postProfileSegments( case "real" -> postRealProfileSegments( datasetId, record, - realProfiles.get(resource).getRight()); + realProfiles.get(resource).segments()); case "discrete" -> postDiscreteProfileSegments( datasetId, record, - discreteProfiles.get(resource).getRight()); + discreteProfiles.get(resource).segments()); default -> throw new Error("Unrecognized profile type " + record.type().getLeft()); } } From 15933758c6de46488142c10e3310600c8ab4646c Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 17:46:53 -0700 Subject: [PATCH 04/20] Create SimulationResourceManager --- .../InMemorySimulationResourceManager.java | 167 ++++++++++++++ .../resources/SimulationResourceManager.java | 38 ++++ .../StreamingSimulationResourceManager.java | 210 ++++++++++++++++++ 3 files changed, 415 insertions(+) create mode 100644 merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/InMemorySimulationResourceManager.java create mode 100644 merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/SimulationResourceManager.java create mode 100644 merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/StreamingSimulationResourceManager.java diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/InMemorySimulationResourceManager.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/InMemorySimulationResourceManager.java new file mode 100644 index 0000000000..5ca174e16c --- /dev/null +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/InMemorySimulationResourceManager.java @@ -0,0 +1,167 @@ +package gov.nasa.jpl.aerie.merlin.driver.resources; + +import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; +import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; +import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; +import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * A variant of the SimulationResourceManager that keeps all segments in memory + */ +public class InMemorySimulationResourceManager implements SimulationResourceManager { + private final HashMap> realResourceSegments; + private final HashMap> discreteResourceSegments; + + private Duration lastReceivedTime; + + public InMemorySimulationResourceManager() { + this.realResourceSegments = new HashMap<>(); + this.discreteResourceSegments = new HashMap<>(); + lastReceivedTime = Duration.ZERO; + } + + public InMemorySimulationResourceManager(InMemorySimulationResourceManager other) { + this.realResourceSegments = new HashMap<>(other.realResourceSegments.size()); + this.discreteResourceSegments = new HashMap<>(other.discreteResourceSegments.size()); + + this.lastReceivedTime = other.lastReceivedTime; + + // Deep copy the resource maps + for(final var entry : other.realResourceSegments.entrySet()) { + final var segments = entry.getValue().deepCopy(); + realResourceSegments.put(entry.getKey(), segments); + } + for(final var entry : other.discreteResourceSegments.entrySet()) { + final var segments = entry.getValue().deepCopy(); + discreteResourceSegments.put(entry.getKey(), segments); + } + } + + /** + * Clear out the Resource Manager's cache of Resource Segments + */ + public void clear() { + realResourceSegments.clear(); + discreteResourceSegments.clear(); + } + + /** + * Compute all ProfileSegments stored in this resource manager. + * @param elapsedDuration the amount of time elapsed since the start of simulation. + */ + @Override + public ResourceProfiles computeProfiles(final Duration elapsedDuration) { + final var keySet = new HashSet<>(realResourceSegments.keySet()); + keySet.addAll(discreteResourceSegments.keySet()); + return computeProfiles(elapsedDuration, keySet); + } + + /** + * Compute a subset of the ProfileSegments stored in this resource manager + * @param elapsedDuration the amount of time elapsed since the start of simulation. + * @param resources the set of names of the resources to be computed + */ + @Override + public ResourceProfiles computeProfiles(final Duration elapsedDuration, Set resources) { + final var profiles = new ResourceProfiles(new HashMap<>(), new HashMap<>()); + + // Compute Real Profiles + for(final var resource : realResourceSegments.entrySet()) { + final var name = resource.getKey(); + final var schema = resource.getValue().valueSchema(); + final var segments = resource.getValue().segments(); + + if(!resources.contains(name)) continue; + + profiles.realProfiles().put(name, new ResourceProfile<>(schema, new ArrayList<>())); + final var profile = profiles.realProfiles().get(name).segments(); + + for(int i = 0; i < segments.size()-1; i++) { + final var segment = segments.get(i); + final var nextSegment = segments.get(i+1); + profile.add(new ProfileSegment<>(nextSegment.startOffset().minus(segment.startOffset()), segment.dynamics())); + } + + // Process final segment + final var finalSegment = segments.getLast(); + profile.add(new ProfileSegment<>(elapsedDuration.minus(finalSegment.startOffset()), finalSegment.dynamics())); + } + + // Compute Discrete Profiles + for(final var resource : discreteResourceSegments.entrySet()) { + final var name = resource.getKey(); + final var schema = resource.getValue().valueSchema(); + final var segments = resource.getValue().segments(); + + if(!resources.contains(name)) continue; + + profiles.discreteProfiles().put(name, new ResourceProfile<>(schema, new ArrayList<>())); + final var profile = profiles.discreteProfiles().get(name).segments(); + + for(int i = 0; i < segments.size()-1; i++) { + final var segment = segments.get(i); + final var nextSegment = segments.get(i+1); + profile.add(new ProfileSegment<>(nextSegment.startOffset().minus(segment.startOffset()), segment.dynamics())); + } + + // Process final segment + final var finalSegment = segments.getLast(); + profile.add(new ProfileSegment<>(elapsedDuration.minus(finalSegment.startOffset()), finalSegment.dynamics())); + } + + return profiles; + } + + /** + * Add new segments to this manager's internal store of segments. + * @param elapsedTime the amount of time elapsed since the start of simulation. Must be monotonically increasing on subsequent calls. + * @param realResourceUpdates the set of updates to real resources. Up to one update per resource is permitted. + * @param discreteResourceUpdates the set of updates to discrete resources. Up to one update per resource is permitted. + */ + @Override + public void acceptUpdates( + final Duration elapsedTime, + final Map> realResourceUpdates, + final Map> discreteResourceUpdates + ) { + if(elapsedTime.shorterThan(lastReceivedTime)) { + throw new IllegalArgumentException(("elapsedTime must be monotonically increasing between calls.\n" + + "\telaspedTime: %s,\tlastReceivedTme: %s") + .formatted(elapsedTime, lastReceivedTime)); + } + lastReceivedTime = elapsedTime; + + for(final var e : realResourceUpdates.entrySet()) { + final var resourceName = e.getKey(); + final var resourceSegment = e.getValue(); + + realResourceSegments + .computeIfAbsent( + resourceName, + r -> new ResourceSegments<>(resourceSegment.getLeft(), new ArrayList<>())) + .segments() + .add(new ResourceSegments.Segment<>(elapsedTime, resourceSegment.getRight())); + } + + for(final var e : discreteResourceUpdates.entrySet()) { + final var resourceName = e.getKey(); + final var resourceSegment = e.getValue(); + + discreteResourceSegments + .computeIfAbsent( + resourceName, + r -> new ResourceSegments<>(resourceSegment.getLeft(), new ArrayList<>())) + .segments() + .add(new ResourceSegments.Segment<>(elapsedTime, resourceSegment.getRight())); + } + + } +} diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/SimulationResourceManager.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/SimulationResourceManager.java new file mode 100644 index 0000000000..fb90830f9f --- /dev/null +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/SimulationResourceManager.java @@ -0,0 +1,38 @@ +package gov.nasa.jpl.aerie.merlin.driver.resources; + +import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; +import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; +import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; +import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.Map; +import java.util.Set; + +public interface SimulationResourceManager { + + /** + * Compute all ProfileSegments stored in this resource manager + * @param elapsedDuration the amount of time elapsed since the start of simulation. + */ + ResourceProfiles computeProfiles(final Duration elapsedDuration); + + /** + * Compute a subset of the ProfileSegments stored in this resource manager + * @param elapsedDuration the amount of time elapsed since the start of simulation. + * @param resources the set of names of the resources to be computed + */ + ResourceProfiles computeProfiles(final Duration elapsedDuration, Set resources); + + /** + * Process resource updates for a given time. + * @param elapsedTime the amount of time elapsed since the start of simulation. Must be monotonically increasing on subsequent calls. + * @param realResourceUpdates the set of updates to real resources. Up to one update per resource is permitted. + * @param discreteResourceUpdates the set of updates to discrete resources. Up to one update per resource is permitted. + */ + void acceptUpdates( + final Duration elapsedTime, + final Map> realResourceUpdates, + final Map> discreteResourceUpdates + ); +} diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/StreamingSimulationResourceManager.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/StreamingSimulationResourceManager.java new file mode 100644 index 0000000000..927d0b36f6 --- /dev/null +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/resources/StreamingSimulationResourceManager.java @@ -0,0 +1,210 @@ +package gov.nasa.jpl.aerie.merlin.driver.resources; + +import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; +import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; +import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; +import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; + +/** + * A variant of a SimulationResourceManager that streams resources as needed in order to conserve memory. + * The way it streams resources is determined by the Consumer passed to it during construction + */ +public class StreamingSimulationResourceManager implements SimulationResourceManager { + private final HashMap> realResourceSegments; + private final HashMap> discreteResourceSegments; + + private final Consumer streamer; + + private Duration lastReceivedTime; + + // The threshold controls how many segments the longest resource must have before all completed segments are streamed. + // When streaming occurs, all completed profile segments are streamed, + // not just those belonging to the resource that crossed the threshold. + private static final int DEFAULT_THRESHOLD = 1024; + private final int threshold; + + public StreamingSimulationResourceManager(final Consumer streamer) { + this(streamer, DEFAULT_THRESHOLD); + } + + public StreamingSimulationResourceManager(final Consumer streamer, int threshold) { + realResourceSegments = new HashMap<>(); + discreteResourceSegments = new HashMap<>(); + this.threshold = threshold; + this.streamer = streamer; + this.lastReceivedTime = Duration.ZERO; + } + + /** + * Compute all ProfileSegments stored in this resource manager, and stream them to the database + * @param elapsedDuration the amount of time elapsed since the start of simulation. + */ + @Override + public ResourceProfiles computeProfiles(final Duration elapsedDuration) { + final var profiles = computeProfiles(); + + // Compute final segment for real profiles + for(final var resource : realResourceSegments.entrySet()) { + final var name = resource.getKey(); + final var segments = resource.getValue().segments(); + final var finalSegment = segments.getFirst(); + + profiles.realProfiles() + .get(name) + .segments() + .add(new ProfileSegment<>(elapsedDuration.minus(finalSegment.startOffset()), finalSegment.dynamics())); + + // Remove final segment + segments.clear(); + } + + // Compute final segment for discrete profiles + for(final var resource : discreteResourceSegments.entrySet()) { + final var name = resource.getKey(); + final var segments = resource.getValue().segments(); + final var finalSegment = segments.getFirst(); + + profiles.discreteProfiles() + .get(name) + .segments() + .add(new ProfileSegment<>(elapsedDuration.minus(finalSegment.startOffset()), finalSegment.dynamics())); + + // Remove final segment + segments.clear(); + } + + streamer.accept(profiles); + return profiles; + } + + /** + * This class streams all resources it has as it accepts updates, + * so it cannot only compute a subset of ProfileSegments. + * @throws UnsupportedOperationException + */ + @Override + public ResourceProfiles computeProfiles(final Duration elapsedDuration, Set resources) { + throw new UnsupportedOperationException("StreamingSimulationResourceManager streams ALL resources"); + } + + /** + * Compute only the completed profile segments and remove them from internal ResourceSegment maps + * This is intended to be called while simulation is executing. + */ + private ResourceProfiles computeProfiles() { + final var profiles = new ResourceProfiles(new HashMap<>(), new HashMap<>()); + + // Compute Real Profiles + for(final var resource : realResourceSegments.entrySet()) { + final var name = resource.getKey(); + final var schema = resource.getValue().valueSchema(); + final var segments = resource.getValue().segments(); + + profiles.realProfiles().put(name, new ResourceProfile<>(schema, new ArrayList<>(threshold))); + final var profile = profiles.realProfiles().get(name).segments(); + + for(int i = 0; i < segments.size()-1; i++) { + final var segment = segments.get(i); + final var nextSegment = segments.get(i+1); + profile.add(new ProfileSegment<>(nextSegment.startOffset().minus(segment.startOffset()), segment.dynamics())); + } + + // Remove the completed segments, leaving only the final (incomplete) segment in the current set + final var finalSegment = segments.getLast(); + segments.clear(); + segments.add(finalSegment); + } + + // Compute Discrete Profiles + for(final var resource : discreteResourceSegments.entrySet()) { + final var name = resource.getKey(); + final var schema = resource.getValue().valueSchema(); + final var segments = resource.getValue().segments(); + + profiles.discreteProfiles().put(name, new ResourceProfile<>(schema, new ArrayList<>(threshold))); + final var profile = profiles.discreteProfiles().get(name).segments(); + + for(int i = 0; i < segments.size()-1; i++) { + final var segment = segments.get(i); + final var nextSegment = segments.get(i+1); + profile.add(new ProfileSegment<>(nextSegment.startOffset().minus(segment.startOffset()), segment.dynamics())); + } + + // Remove the completed segments, leaving only the final (incomplete) segment in the current set + final var finalSegment = segments.getLast(); + segments.clear(); + segments.add(finalSegment); + } + + return profiles; + } + + + /** + * Add new segments to this manager's internal store of segments. + * Will stream all held segments should any resource's number of stored segments exceed the streaming threshold. + * @param elapsedTime the amount of time elapsed since the start of simulation. Must be monotonically increasing on subsequent calls. + * @param realResourceUpdates the set of updates to real resources. Up to one update per resource is permitted. + * @param discreteResourceUpdates the set of updates to discrete resources. Up to one update per resource is permitted. + */ + @Override + public void acceptUpdates( + final Duration elapsedTime, + final Map> realResourceUpdates, + final Map> discreteResourceUpdates + ) { + if(elapsedTime.shorterThan(lastReceivedTime)) { + throw new IllegalArgumentException(("elapsedTime must be monotonically increasing between calls.\n" + + "\telaspedTime: %s,\tlastReceivedTme: %s") + .formatted(elapsedTime, lastReceivedTime)); + } + + lastReceivedTime = elapsedTime; + boolean readyToStream = false; + + for(final var e : realResourceUpdates.entrySet()) { + final var resourceName = e.getKey(); + final var resourceSegment = e.getValue(); + + realResourceSegments + .computeIfAbsent( + resourceName, + r -> new ResourceSegments<>(resourceSegment.getLeft(), threshold)) + .segments() + .add(new ResourceSegments.Segment<>(elapsedTime, resourceSegment.getRight())); + + if(realResourceSegments.get(resourceName).segments().size() >= threshold) { + readyToStream = true; + } + } + + for(final var e : discreteResourceUpdates.entrySet()) { + final var resourceName = e.getKey(); + final var resourceSegment = e.getValue(); + + discreteResourceSegments + .computeIfAbsent( + resourceName, + r -> new ResourceSegments<>(resourceSegment.getLeft(), threshold)) + .segments() + .add(new ResourceSegments.Segment<>(elapsedTime, resourceSegment.getRight())); + + if(discreteResourceSegments.get(resourceName).segments().size() >= threshold) { + readyToStream = true; + } + } + + // If ANY resource met the size threshold, stream ALL currently held profiles + if(readyToStream) { + streamer.accept(computeProfiles()); + } + } +} From 1a5c43d82020b08b923e3b65eacbce0a9c998e66 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 17:47:46 -0700 Subject: [PATCH 05/20] Pass resource manager to Simulation --- .../aerie/merlin/driver/SimulationDriver.java | 84 ++++++------------- .../services/LocalMissionModelService.java | 10 ++- .../server/services/MissionModelService.java | 11 ++- .../server/mocks/StubMissionModelService.java | 11 ++- 4 files changed, 48 insertions(+), 68 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java index 900e7da930..ac96f1de86 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java @@ -2,8 +2,8 @@ import gov.nasa.jpl.aerie.merlin.driver.engine.SimulationEngine; import gov.nasa.jpl.aerie.merlin.driver.engine.SpanException; -import gov.nasa.jpl.aerie.merlin.driver.timeline.LiveCells; -import gov.nasa.jpl.aerie.merlin.driver.timeline.TemporalEventSource; +import gov.nasa.jpl.aerie.merlin.driver.resources.InMemorySimulationResourceManager; +import gov.nasa.jpl.aerie.merlin.driver.resources.SimulationResourceManager; import gov.nasa.jpl.aerie.merlin.protocol.driver.Topic; import gov.nasa.jpl.aerie.merlin.protocol.model.Task; import gov.nasa.jpl.aerie.merlin.protocol.model.TaskFactory; @@ -11,10 +11,10 @@ import gov.nasa.jpl.aerie.merlin.protocol.types.InstantiationException; import gov.nasa.jpl.aerie.merlin.protocol.types.Unit; import org.apache.commons.lang3.tuple.Pair; +import java.util.ArrayList; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -22,8 +22,7 @@ import java.util.function.Supplier; public final class SimulationDriver { - public static - SimulationResults simulate( + public static SimulationResults simulate( final MissionModel missionModel, final Map schedule, final Instant simulationStartTime, @@ -31,8 +30,7 @@ SimulationResults simulate( final Instant planStartTime, final Duration planDuration, final Supplier simulationCanceled - ) - { + ) { return simulate( missionModel, schedule, @@ -41,11 +39,11 @@ SimulationResults simulate( planStartTime, planDuration, simulationCanceled, - $ -> {}); + $ -> {}, + new InMemorySimulationResourceManager()); } - public static - SimulationResults simulate( + public static SimulationResults simulate( final MissionModel missionModel, final Map schedule, final Instant simulationStartTime, @@ -53,39 +51,20 @@ SimulationResults simulate( final Instant planStartTime, final Duration planDuration, final Supplier simulationCanceled, - final Consumer simulationExtentConsumer + final Consumer simulationExtentConsumer, + final SimulationResourceManager resourceManager ) { - try (final var engine = new SimulationEngine()) { - /* The top-level simulation timeline. */ - var timeline = new TemporalEventSource(); - var cells = new LiveCells(timeline, missionModel.getInitialCells()); + try (final var engine = new SimulationEngine(missionModel.getInitialCells())) { + /* The current real time. */ var elapsedTime = Duration.ZERO; - simulationExtentConsumer.accept(elapsedTime); - // Begin tracking all resources. - for (final var entry : missionModel.getResources().entrySet()) { - final var name = entry.getKey(); - final var resource = entry.getValue(); - - engine.trackResource(name, resource, elapsedTime); - } - // Specify a topic on which tasks can log the activity they're associated with. final var activityTopic = new Topic(); try { - // Start daemon task(s) immediately, before anything else happens. - engine.scheduleTask(Duration.ZERO, missionModel.getDaemon()); - { - final var batch = engine.extractNextJobs(Duration.MAX_VALUE); - final var commit = engine.performJobs(batch.jobs(), cells, elapsedTime, Duration.MAX_VALUE); - timeline.add(commit.getLeft()); - if(commit.getRight().isPresent()) { - throw commit.getRight().get(); - } - } + engine.init(missionModel.getResources(), missionModel.getDaemon()); // Get all activities as close as possible to absolute time // Schedule all activities. @@ -114,34 +93,25 @@ SimulationResults simulate( // Drive the engine until we're out of time or until simulation is canceled. // TERMINATION: Actually, we might never break if real time never progresses forward. + engineLoop: while (!simulationCanceled.get()) { - final var batch = engine.extractNextJobs(simulationDuration); - - // Increment real time, if necessary. - final var delta = batch.offsetFromStart().minus(elapsedTime); - elapsedTime = batch.offsetFromStart(); - timeline.add(delta); - // TODO: Advance a dense time counter so that future tasks are strictly ordered relative to these, - // even if they occur at the same real time. - - simulationExtentConsumer.accept(elapsedTime); - - if (simulationCanceled.get() || - (batch.jobs().isEmpty() && batch.offsetFromStart().isEqualTo(simulationDuration))) { - break; - } - - // Run the jobs in this batch. - final var commit = engine.performJobs(batch.jobs(), cells, elapsedTime, simulationDuration); - timeline.add(commit.getLeft()); - if (commit.getRight().isPresent()) { - throw commit.getRight().get(); + if(simulationCanceled.get()) break; + final var status = engine.step(simulationDuration); + switch (status) { + case SimulationEngine.Status.NoJobs noJobs: break engineLoop; + case SimulationEngine.Status.AtDuration atDuration: break engineLoop; + case SimulationEngine.Status.Nominal nominal: + elapsedTime = nominal.elapsedTime(); + resourceManager.acceptUpdates(elapsedTime, nominal.realResourceUpdates(), nominal.dynamicResourceUpdates()); + break; } + simulationExtentConsumer.accept(elapsedTime); } + } catch (SpanException ex) { // Swallowing the spanException as the internal `spanId` is not user meaningful info. final var topics = missionModel.getTopics(); - final var directiveId = SimulationEngine.getDirectiveIdFromSpan(engine, activityTopic, timeline, topics, ex.spanId); + final var directiveId = engine.getDirectiveIdFromSpan(activityTopic, topics, ex.spanId); if(directiveId.isPresent()) { throw new SimulationException(elapsedTime, simulationStartTime, directiveId.get(), ex.cause); } @@ -151,7 +121,7 @@ SimulationResults simulate( } final var topics = missionModel.getTopics(); - return SimulationEngine.computeResults(engine, simulationStartTime, elapsedTime, activityTopic, timeline, topics); + return SimulationEngine.computeResults(engine, simulationStartTime, elapsedTime, activityTopic, topics, resourceManager); } } diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/LocalMissionModelService.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/LocalMissionModelService.java index c1add5d963..36ad50e637 100644 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/LocalMissionModelService.java +++ b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/LocalMissionModelService.java @@ -7,6 +7,8 @@ import gov.nasa.jpl.aerie.merlin.driver.SerializedActivity; import gov.nasa.jpl.aerie.merlin.driver.SimulationDriver; import gov.nasa.jpl.aerie.merlin.driver.SimulationResults; +import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.driver.resources.SimulationResourceManager; import gov.nasa.jpl.aerie.merlin.protocol.model.InputType.Parameter; import gov.nasa.jpl.aerie.merlin.protocol.model.InputType.ValidationNotice; import gov.nasa.jpl.aerie.merlin.protocol.model.ModelType; @@ -286,7 +288,8 @@ public Map getModelEffectiveArguments(final String miss public SimulationResults runSimulation( final CreateSimulationMessage message, final Consumer simulationExtentConsumer, - final Supplier canceledListener) + final Supplier canceledListener, + final SimulationResourceManager resourceManager) throws NoSuchMissionModelException { final var config = message.configuration(); @@ -307,7 +310,8 @@ public SimulationResults runSimulation( message.planStartTime(), message.planDuration(), canceledListener, - simulationExtentConsumer); + simulationExtentConsumer, + resourceManager); } @Override @@ -346,7 +350,7 @@ public void refreshActivityTypes(final String missionModelId) @Override public void refreshResourceTypes(final String missionModelId) - throws NoSuchMissionModelException { + throws NoSuchMissionModelException, MissionModelLoadException { try { final var model = this.loadAndInstantiateMissionModel(missionModelId); this.missionModelRepository.updateResourceTypes(missionModelId, model.getResources()); diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/MissionModelService.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/MissionModelService.java index 8d9203fe8c..8cf8535409 100644 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/MissionModelService.java +++ b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/MissionModelService.java @@ -4,6 +4,8 @@ import gov.nasa.jpl.aerie.merlin.driver.MissionModelLoader; import gov.nasa.jpl.aerie.merlin.driver.SerializedActivity; import gov.nasa.jpl.aerie.merlin.driver.SimulationResults; +import gov.nasa.jpl.aerie.merlin.driver.engine.ProfileSegment; +import gov.nasa.jpl.aerie.merlin.driver.resources.SimulationResourceManager; import gov.nasa.jpl.aerie.merlin.protocol.model.InputType.Parameter; import gov.nasa.jpl.aerie.merlin.protocol.model.InputType.ValidationNotice; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; @@ -14,6 +16,7 @@ import gov.nasa.jpl.aerie.merlin.server.models.Constraint; import gov.nasa.jpl.aerie.merlin.server.models.MissionModelJar; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -62,8 +65,12 @@ Map getModelEffectiveArguments(String missionModelId, M LocalMissionModelService.MissionModelLoadException, InstantiationException; - SimulationResults runSimulation(CreateSimulationMessage message, Consumer writer, Supplier canceledListener) - throws NoSuchMissionModelException, MissionModelService.NoSuchActivityTypeException; + SimulationResults runSimulation( + final CreateSimulationMessage message, + final Consumer writer, + final Supplier canceledListener, + final SimulationResourceManager resourceManager + ) throws NoSuchMissionModelException, MissionModelService.NoSuchActivityTypeException; void refreshModelParameters(String missionModelId) throws NoSuchMissionModelException; void refreshActivityTypes(String missionModelId) throws NoSuchMissionModelException; diff --git a/merlin-server/src/test/java/gov/nasa/jpl/aerie/merlin/server/mocks/StubMissionModelService.java b/merlin-server/src/test/java/gov/nasa/jpl/aerie/merlin/server/mocks/StubMissionModelService.java index 9a4cc8fea4..058435f323 100644 --- a/merlin-server/src/test/java/gov/nasa/jpl/aerie/merlin/server/mocks/StubMissionModelService.java +++ b/merlin-server/src/test/java/gov/nasa/jpl/aerie/merlin/server/mocks/StubMissionModelService.java @@ -3,15 +3,13 @@ import gov.nasa.jpl.aerie.merlin.driver.ActivityDirectiveId; import gov.nasa.jpl.aerie.merlin.driver.SerializedActivity; import gov.nasa.jpl.aerie.merlin.driver.SimulationResults; +import gov.nasa.jpl.aerie.merlin.driver.resources.SimulationResourceManager; import gov.nasa.jpl.aerie.merlin.protocol.model.InputType.Parameter; import gov.nasa.jpl.aerie.merlin.protocol.model.InputType.ValidationNotice; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; -import gov.nasa.jpl.aerie.merlin.server.ResultsProtocol; -import gov.nasa.jpl.aerie.merlin.server.models.ActivityDirectiveForValidation; import gov.nasa.jpl.aerie.merlin.server.models.ActivityType; -import gov.nasa.jpl.aerie.merlin.server.models.Constraint; import gov.nasa.jpl.aerie.merlin.server.models.MissionModelJar; import gov.nasa.jpl.aerie.merlin.server.services.CreateSimulationMessage; import gov.nasa.jpl.aerie.merlin.server.services.LocalMissionModelService; @@ -197,9 +195,10 @@ public Map getModelEffectiveArguments( @Override public SimulationResults runSimulation( final CreateSimulationMessage message, - Consumer simulationExtentConsumer, - Supplier canceledListener) - throws NoSuchMissionModelException { + final Consumer simulationExtentConsumer, + final Supplier canceledListener, + final SimulationResourceManager resourceManager + ) throws NoSuchMissionModelException { if (!Objects.equals(message.missionModelId(), EXISTENT_MISSION_MODEL_ID)) { throw new NoSuchMissionModelException(message.missionModelId()); } From a27f137c98783d6cc32bfc3f165fe198523ed336 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 18:32:45 -0700 Subject: [PATCH 06/20] Make compute results nonstatic --- .../aerie/merlin/driver/SimulationDriver.java | 2 +- .../driver/engine/SimulationEngine.java | 116 +++++++++++++----- 2 files changed, 88 insertions(+), 30 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java index ac96f1de86..8ecb46186b 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java @@ -121,7 +121,7 @@ public static SimulationResults simulate( } final var topics = missionModel.getTopics(); - return SimulationEngine.computeResults(engine, simulationStartTime, elapsedTime, activityTopic, topics, resourceManager); + return engine.computeResults(simulationStartTime, activityTopic, topics, resourceManager); } } diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java index 42b047648a..9748078daf 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java @@ -720,8 +720,7 @@ public static SimulationResults computeResults( // TODO: Whatever mechanism replaces `computeResults` also ought to replace `isTaskComplete`. // TODO: Produce results for all tasks, not just those that have completed. // Planners need to be aware of failed or unfinished tasks. - public static SimulationResults computeResults( - final SimulationEngine engine, + public SimulationResults computeResults ( final Instant startTime, final Duration elapsedTime, final Topic activityTopic, @@ -733,35 +732,94 @@ public static SimulationResults computeResults( final var spanInfo = computeSpanInfo(timeline, activityTopic, serializableTopics); // Extract profiles for every resource. - final var realProfiles = new HashMap>>>(); - final var discreteProfiles = new HashMap>>>(); - - for (final var entry : engine.resources.entrySet()) { - final var id = entry.getKey(); - final var state = entry.getValue(); - - final var name = id.id(); - final var resource = state.resource(); - if(!resourceNames.contains(name)) continue; - switch (resource.getType()) { - case "real" -> realProfiles.put( - name, - Pair.of( - resource.getOutputType().getSchema(), - serializeProfile(elapsedTime, state, SimulationEngine::extractRealDynamics))); - - case "discrete" -> discreteProfiles.put( - name, - Pair.of( - resource.getOutputType().getSchema(), - serializeProfile(elapsedTime, state, SimulationEngine::extractDiscreteDynamics))); - - default -> - throw new IllegalArgumentException( - "Resource `%s` has unknown type `%s`".formatted(name, resource.getType())); + final var resourceProfiles = resourceManager.computeProfiles(elapsedTime); + final var realProfiles = resourceProfiles.realProfiles(); + final var discreteProfiles = resourceProfiles.discreteProfiles(); + + final var activityResults = computeActivitySimulationResults(engine, startTime, elapsedTime, spanInfo); + + final List> topics = new ArrayList<>(); + final var serializableTopicToId = new HashMap, Integer>(); + for (final var serializableTopic : serializableTopics) { + serializableTopicToId.put(serializableTopic, topics.size()); + topics.add(Triple.of(topics.size(), serializableTopic.name(), serializableTopic.outputType().getSchema())); + } + + final var spanToActivities = spanToSimulatedActivities(engine,spanInfo); + final var serializedTimeline = new TreeMap>>(); + var time = Duration.ZERO; + for (var point : this.timeline.points()) { + if (point instanceof TemporalEventSource.TimePoint.Delta delta) { + time = time.plus(delta.delta()); + } else if (point instanceof TemporalEventSource.TimePoint.Commit commit) { + final var serializedEventGraph = commit.events().substitute( + event -> { + // TODO can we do this more efficiently? + EventGraph output = EventGraph.empty(); + for (final var serializableTopic : serializableTopics) { + Optional serializedEvent = trySerializeEvent(event, serializableTopic); + if (serializedEvent.isPresent()) { + // If the event's `provenance` has no simulated activity id, search its ancestors to find the nearest + // simulated activity id, if one exists + if (!spanToActivities.containsKey(event.provenance())) { + var spanId = Optional.of(event.provenance()); + + while (true) { + if (spanToActivities.containsKey(spanId.get())) { + spanToActivities.put(event.provenance(), spanToActivities.get(spanId.get())); + break; + } + spanId = engine.getSpan(spanId.get()).parent(); + if (spanId.isEmpty()) { + break; + } + } + } + var activitySpanID = Optional.ofNullable(spanToActivities.get(event.provenance()).id()); + output = EventGraph.concurrently( + output, + EventGraph.atom( + new EventRecord(serializableTopicToId.get(serializableTopic), + activitySpanID, + serializedEvent.get()))); + } + } + return output; + } + ).evaluate(new EventGraph.IdentityTrait<>(), EventGraph::atom); + if (!(serializedEventGraph instanceof EventGraph.Empty)) { + serializedTimeline + .computeIfAbsent(time, x -> new ArrayList<>()) + .add(serializedEventGraph); + } } } + return new SimulationResults(realProfiles, + discreteProfiles, + activityResults.simulatedActivities, + activityResults.unfinishedActivities, + startTime, + elapsedTime, + topics, + serializedTimeline); + } + + public SimulationResults computeResults( + final Instant startTime, + final Topic activityTopic, + final Iterable> serializableTopics, + final SimulationResourceManager resourceManager, + final Set resourceNames + ) { + // Collect per-task information from the event graph. + final var spanInfo = computeSpanInfo(timeline, activityTopic, serializableTopics); + + // Extract profiles for every resource. + final var resourceProfiles = resourceManager.computeProfiles(elapsedTime); + final var realProfiles = resourceProfiles.realProfiles(); + final var discreteProfiles = resourceProfiles.discreteProfiles(); + final var activityResults = computeActivitySimulationResults(engine, startTime, elapsedTime, spanInfo); final List> topics = new ArrayList<>(); @@ -774,7 +832,7 @@ public static SimulationResults computeResults( final var spanToActivities = spanToSimulatedActivities(engine,spanInfo); final var serializedTimeline = new TreeMap>>(); var time = Duration.ZERO; - for (var point : timeline.points()) { + for (var point : this.timeline.points()) { if (point instanceof TemporalEventSource.TimePoint.Delta delta) { time = time.plus(delta.delta()); } else if (point instanceof TemporalEventSource.TimePoint.Commit commit) { From ca742262946249cf64d63e6eb43290a70114e963 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 17:49:32 -0700 Subject: [PATCH 07/20] Inline timeline, cells, and elapsedTime - Add `init` and `step` to engine --- .../driver/CheckpointSimulationDriver.java | 54 ++------ .../aerie/merlin/driver/SimulationDriver.java | 55 ++------ .../SimulationResultsComputerInputs.java | 25 ++-- .../driver/engine/SimulationEngine.java | 118 +++++++++++++++++- 4 files changed, 144 insertions(+), 108 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java index 3b0c263a8f..39429a2c5f 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java @@ -1,11 +1,9 @@ package gov.nasa.jpl.aerie.merlin.driver; import gov.nasa.jpl.aerie.merlin.driver.engine.SimulationEngine; -import gov.nasa.jpl.aerie.merlin.driver.engine.SlabList; import gov.nasa.jpl.aerie.merlin.driver.engine.SpanException; import gov.nasa.jpl.aerie.merlin.driver.engine.SpanId; -import gov.nasa.jpl.aerie.merlin.driver.timeline.LiveCells; -import gov.nasa.jpl.aerie.merlin.driver.timeline.TemporalEventSource; +import gov.nasa.jpl.aerie.merlin.driver.resources.InMemorySimulationResourceManager; import gov.nasa.jpl.aerie.merlin.protocol.driver.Topic; import gov.nasa.jpl.aerie.merlin.protocol.model.Task; import gov.nasa.jpl.aerie.merlin.protocol.model.TaskFactory; @@ -17,7 +15,6 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; @@ -40,21 +37,15 @@ public record CachedSimulationEngine( Duration endsAt, Map activityDirectives, SimulationEngine simulationEngine, - LiveCells cells, - SlabList timePoints, Topic activityTopic, MissionModel missionModel ) { public void freeze() { - cells.freeze(); - timePoints.freeze(); simulationEngine.close(); } public static CachedSimulationEngine empty(final MissionModel missionModel) { final SimulationEngine engine = new SimulationEngine(); - final TemporalEventSource timeline = new TemporalEventSource(); - final LiveCells cells = new LiveCells(timeline, missionModel.getInitialCells()); // Begin tracking all resources. for (final var entry : missionModel.getResources().entrySet()) { @@ -140,27 +131,7 @@ public static Optional Pair.of(cachedSimulationEngine, correspondenceMap)); } - private static TemporalEventSource makeCombinedTimeline(List timelines, TemporalEventSource timeline) { - final TemporalEventSource combinedTimeline = new TemporalEventSource(); - for (final var entry : timelines) { - for (final var timePoint : entry.points()) { - if (timePoint instanceof TemporalEventSource.TimePoint.Delta t) { - combinedTimeline.add(t.delta()); - } else if (timePoint instanceof TemporalEventSource.TimePoint.Commit t) { - combinedTimeline.add(t.events()); - } - } - } - for (final var timePoint : timeline) { - if (timePoint instanceof TemporalEventSource.TimePoint.Delta t) { - combinedTimeline.add(t.delta()); - } else if (timePoint instanceof TemporalEventSource.TimePoint.Commit t) { - combinedTimeline.add(t.events()); - } - } - return combinedTimeline; - } public static Function desiredCheckpoints(final List desiredCheckpoints) { return simulationState -> { @@ -244,13 +215,10 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( final boolean duplicationIsOk = cachedEngineStore.capacity() > 1; final var activityToSpan = new HashMap(); final var activityTopic = cachedEngine.activityTopic(); - final var timelines = new ArrayList(); - timelines.add(new TemporalEventSource(cachedEngine.timePoints)); - var engine = !duplicationIsOk ? cachedEngine.simulationEngine : cachedEngine.simulationEngine.duplicate(); + var engine = duplicationIsOk ? cachedEngine.simulationEngine.duplicate() : cachedEngine.simulationEngine; + final var resourceManager = duplicationIsOk ? new InMemorySimulationResourceManager(cachedEngine.resourceManager) : cachedEngine.resourceManager; engine.unscheduleAfter(cachedEngine.endsAt); - var timeline = new TemporalEventSource(); - var cells = new LiveCells(timeline, cachedEngine.cells()); /* The current real time. */ var elapsedTime = Duration.max(ZERO, cachedEngine.endsAt()); @@ -323,14 +291,11 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( elapsedTime, schedule, engine, - cells, - makeCombinedTimeline(timelines, timeline).points(), activityTopic, missionModel); cachedEngineStore.save( newCachedEngine, configuration); - timelines.add(timeline); } break; } @@ -379,13 +344,12 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( throw new SimulationException(elapsedTime, simulationStartTime, ex); } return new SimulationResultsComputerInputs( - engine, - simulationStartTime, - elapsedTime, - activityTopic, - makeCombinedTimeline(timelines, timeline), - missionModel.getTopics(), - activityToSpan); + engine, + simulationStartTime, + activityTopic, + missionModel.getTopics(), + activityToSpan, + resourceManager); } diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java index 8ecb46186b..c1206a7c07 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java @@ -128,52 +128,23 @@ public static SimulationResults simulate( // This method is used as a helper method for executing unit tests public static void simulateTask(final MissionModel missionModel, final TaskFactory task) { - try (final var engine = new SimulationEngine()) { - /* The top-level simulation timeline. */ - var timeline = new TemporalEventSource(); - var cells = new LiveCells(timeline, missionModel.getInitialCells()); - /* The current real time. */ - var elapsedTime = Duration.ZERO; - - // Begin tracking all resources. - for (final var entry : missionModel.getResources().entrySet()) { - final var name = entry.getKey(); - final var resource = entry.getValue(); - - engine.trackResource(name, resource, elapsedTime); - } - - // Start daemon task(s) immediately, before anything else happens. - engine.scheduleTask(Duration.ZERO, missionModel.getDaemon()); - { - final var batch = engine.extractNextJobs(Duration.MAX_VALUE); - final var commit = engine.performJobs(batch.jobs(), cells, elapsedTime, Duration.MAX_VALUE); - timeline.add(commit.getLeft()); - if(commit.getRight().isPresent()) { - throw new RuntimeException("Exception thrown while starting daemon tasks", commit.getRight().get()); - } + try (final var engine = new SimulationEngine(missionModel.getInitialCells())) { + // Track resources and kick off daemon tasks + try { + engine.init(missionModel.getResources(), missionModel.getDaemon()); + } catch (Throwable t) { + throw new RuntimeException("Exception thrown while starting daemon tasks", t); } - // Schedule all activities. - final var spanId = engine.scheduleTask(elapsedTime, task); + // Schedule the task. + final var spanId = engine.scheduleTask(Duration.ZERO, task); - // Drive the engine until we're out of time. - // TERMINATION: Actually, we might never break if real time never progresses forward. + // Drive the engine until the scheduled task completes. while (!engine.getSpan(spanId).isComplete()) { - final var batch = engine.extractNextJobs(Duration.MAX_VALUE); - - // Increment real time, if necessary. - final var delta = batch.offsetFromStart().minus(elapsedTime); - elapsedTime = batch.offsetFromStart(); - timeline.add(delta); - // TODO: Advance a dense time counter so that future tasks are strictly ordered relative to these, - // even if they occur at the same real time. - - // Run the jobs in this batch. - final var commit = engine.performJobs(batch.jobs(), cells, elapsedTime, Duration.MAX_VALUE); - timeline.add(commit.getLeft()); - if(commit.getRight().isPresent()) { - throw new RuntimeException("Exception thrown while simulating tasks", commit.getRight().get()); + try { + engine.step(Duration.MAX_VALUE); + } catch (Throwable t) { + throw new RuntimeException("Exception thrown while simulating tasks", t); } } } diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationResultsComputerInputs.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationResultsComputerInputs.java index 8a152a2cea..6f34c1504f 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationResultsComputerInputs.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationResultsComputerInputs.java @@ -2,6 +2,7 @@ import gov.nasa.jpl.aerie.merlin.driver.engine.SimulationEngine; import gov.nasa.jpl.aerie.merlin.driver.engine.SpanId; +import gov.nasa.jpl.aerie.merlin.driver.resources.SimulationResourceManager; import gov.nasa.jpl.aerie.merlin.driver.timeline.TemporalEventSource; import gov.nasa.jpl.aerie.merlin.protocol.driver.Topic; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; @@ -13,42 +14,34 @@ public record SimulationResultsComputerInputs( SimulationEngine engine, Instant simulationStartTime, - Duration elapsedTime, Topic activityTopic, - TemporalEventSource timeline, Iterable> serializableTopics, - Map activityDirectiveIdTaskIdMap){ + Map activityDirectiveIdTaskIdMap, + SimulationResourceManager resourceManager){ public SimulationResults computeResults(final Set resourceNames){ - return SimulationEngine.computeResults( - this.engine(), + return engine.computeResults( this.simulationStartTime(), - this.elapsedTime(), this.activityTopic(), - this.timeline(), this.serializableTopics(), + this.resourceManager, resourceNames ); } public SimulationResults computeResults(){ - return SimulationEngine.computeResults( - this.engine(), + return engine.computeResults( this.simulationStartTime(), - this.elapsedTime(), this.activityTopic(), - this.timeline(), - this.serializableTopics() + this.serializableTopics(), + this.resourceManager ); } public SimulationEngine.SimulationActivityExtract computeActivitySimulationResults(){ - return SimulationEngine.computeActivitySimulationResults( - this.engine(), + return engine.computeActivitySimulationResults( this.simulationStartTime(), - this.elapsedTime(), this.activityTopic(), - this.timeline(), this.serializableTopics()); } } diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java index 9748078daf..efefee0b7b 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java @@ -5,6 +5,7 @@ import gov.nasa.jpl.aerie.merlin.driver.SerializedActivity; import gov.nasa.jpl.aerie.merlin.driver.SimulatedActivity; import gov.nasa.jpl.aerie.merlin.driver.SimulatedActivityId; +import gov.nasa.jpl.aerie.merlin.driver.resources.SimulationResourceManager; import gov.nasa.jpl.aerie.merlin.driver.SimulationResults; import gov.nasa.jpl.aerie.merlin.driver.UnfinishedActivity; import gov.nasa.jpl.aerie.merlin.driver.timeline.Event; @@ -25,6 +26,7 @@ import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; import gov.nasa.jpl.aerie.merlin.protocol.types.TaskStatus; +import gov.nasa.jpl.aerie.merlin.protocol.types.Unit; import gov.nasa.jpl.aerie.merlin.protocol.types.ValueSchema; import org.apache.commons.lang3.mutable.Mutable; import org.apache.commons.lang3.mutable.MutableInt; @@ -93,8 +95,17 @@ public static int getNumActiveSimulationEngines() { /** A thread pool that modeled tasks can use to keep track of their state between steps. */ private final ExecutorService executor; - public SimulationEngine() { + /* The top-level simulation timeline. */ + private final TemporalEventSource timeline; + private final LiveCells cells; + private Duration elapsedTime; + + public SimulationEngine(LiveCells initialCells) { numActiveSimulationEngines++; + timeline = new TemporalEventSource(); + cells = new LiveCells(timeline, initialCells); + elapsedTime = Duration.ZERO; + scheduledJobs = new JobSchedule<>(); waitingTasks = new LinkedHashMap<>(); blockedTasks = new LinkedHashMap<>(); @@ -111,6 +122,13 @@ public SimulationEngine() { private SimulationEngine(SimulationEngine other) { numActiveSimulationEngines++; + other.timeline.freeze(); + other.cells.freeze(); + + elapsedTime = other.elapsedTime; + timeline = other.combineTimeline(new TemporalEventSource()); + cells = new LiveCells(timeline, other.cells); + // New Executor allows other SimulationEngine to be closed executor = Executors.newVirtualThreadPerTaskExecutor(); scheduledJobs = other.scheduledJobs.duplicate(); @@ -126,10 +144,7 @@ private SimulationEngine(SimulationEngine other) { tasks.put(entry.getKey(), entry.getValue().duplicate(executor)); } conditions = new LinkedHashMap<>(other.conditions); - resources = new LinkedHashMap<>(); - for (final var entry : other.resources.entrySet()) { - resources.put(entry.getKey(), entry.getValue().duplicate()); - } + resources = new LinkedHashMap<>(other.resources); unstartedTasks = new LinkedHashMap<>(other.unstartedTasks); spans = new LinkedHashMap<>(other.spans); spanContributorCount = new LinkedHashMap<>(); @@ -138,6 +153,96 @@ private SimulationEngine(SimulationEngine other) { } } + /** Initialize the engine by tracking resources and kicking off daemon tasks. **/ + public void init(Map> resources, TaskFactory daemons) throws Throwable { + // Begin tracking all resources. + for (final var entry : resources.entrySet()) { + final var name = entry.getKey(); + final var resource = entry.getValue(); + + this.trackResource(name, resource, elapsedTime); + } + + // Start daemon task(s) immediately, before anything else happens. + this.scheduleTask(Duration.ZERO, daemons); + { + final var batch = this.extractNextJobs(Duration.MAX_VALUE); + final var results = this.performJobs(batch.jobs(), cells, elapsedTime, Duration.MAX_VALUE); + for (final var commit : results.commits()) { + timeline.add(commit); + } + if(results.error.isPresent()) { + throw results.error.get(); + } + } + } + + public sealed interface Status { + record NoJobs() implements Status {} + record AtDuration() implements Status{} + record Nominal(Duration elapsedTime, Map> resourceUpdates) implements Status{} + } + + /** Step the engine forward one batch. **/ + public Status step(Duration simulationDuration) throws Throwable { + final var nextTime = this.peekNextTime().orElse(Duration.MAX_VALUE); + if (nextTime.longerThan(simulationDuration)) { + elapsedTime = Duration.max(elapsedTime, simulationDuration); // avoid lowering elapsed time + return new Status.AtDuration(); + } + + final var batch = this.extractNextJobs(simulationDuration); + + // Increment real time, if necessary. + final var delta = batch.offsetFromStart().minus(elapsedTime); + elapsedTime = batch.offsetFromStart(); + timeline.add(delta); + + // TODO: Advance a dense time counter so that future tasks are strictly ordered relative to these, + // even if they occur at the same real time. + if (batch.jobs().isEmpty()) return new Status.NoJobs(); + + // Run the jobs in this batch. + final var results = this.performJobs(batch.jobs(), cells, elapsedTime, simulationDuration); + for (final var commit : results.commits()) { + timeline.add(commit); + } + if(results.error.isPresent()) { + throw results.error.get(); + } + + /* + for (final var entry : engine.resources.entrySet()) { + final var id = entry.getKey(); + final var state = entry.getValue(); + + final var name = id.id(); + final var resource = state.resource(); + + switch (resource.getType()) { + case "real" -> realProfiles.put( + name, + Pair.of( + resource.getOutputType().getSchema(), + serializeProfile(elapsedTime, state, SimulationEngine::extractRealDynamics))); + + case "discrete" -> discreteProfiles.put( + name, + Pair.of( + resource.getOutputType().getSchema(), + serializeProfile(elapsedTime, state, SimulationEngine::extractDiscreteDynamics))); + + default -> + throw new IllegalArgumentException( + "Resource `%s` has unknown type `%s`".formatted(name, resource.getType())); + } + } + */ + + return new Status.Nominal(elapsedTime, Map.of()); + } + + /** Schedule a new task to be performed at the given time. */ public SpanId scheduleTask(final Duration startTime, final TaskFactory state) { if (this.closed) throw new IllegalStateException("Cannot schedule task on closed simulation engine"); @@ -419,6 +524,9 @@ public void updateResource( @Override public void close() { numActiveSimulationEngines--; + cells.freeze(); + timeline.freeze(); + for (final var task : this.tasks.values()) { task.state().release(); } From 8174903c96701491e86e45dc44e0be843a6d7cb5 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 17:49:51 -0700 Subject: [PATCH 08/20] Update Resource Tracking in Engine Make `computeResults` non-static Replace `if` in `performJobs` with type-switch --- .../driver/engine/SimulationEngine.java | 296 +++++++----------- 1 file changed, 121 insertions(+), 175 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java index efefee0b7b..cfd74cc3db 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java @@ -53,7 +53,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; -import java.util.stream.Collectors; /** * A representation of the work remaining to do during a simulation, and its accumulated results. @@ -82,7 +81,7 @@ public static int getNumActiveSimulationEngines() { /** The getter for each tracked condition. */ private final Map conditions; /** The profiling state for each tracked resource. */ - private final Map> resources; + private final Map> resources; /** Tasks that have been scheduled, but not started */ private final Map unstartedTasks; @@ -180,7 +179,11 @@ public void init(Map> resources, TaskFactory daemons) public sealed interface Status { record NoJobs() implements Status {} record AtDuration() implements Status{} - record Nominal(Duration elapsedTime, Map> resourceUpdates) implements Status{} + record Nominal( + Duration elapsedTime, + Map> realResourceUpdates, + Map> dynamicResourceUpdates + ) implements Status {} } /** Step the engine forward one batch. **/ @@ -211,37 +214,37 @@ public Status step(Duration simulationDuration) throws Throwable { throw results.error.get(); } - /* - for (final var entry : engine.resources.entrySet()) { - final var id = entry.getKey(); - final var state = entry.getValue(); - - final var name = id.id(); - final var resource = state.resource(); - - switch (resource.getType()) { - case "real" -> realProfiles.put( - name, - Pair.of( - resource.getOutputType().getSchema(), - serializeProfile(elapsedTime, state, SimulationEngine::extractRealDynamics))); - - case "discrete" -> discreteProfiles.put( - name, - Pair.of( - resource.getOutputType().getSchema(), - serializeProfile(elapsedTime, state, SimulationEngine::extractDiscreteDynamics))); - - default -> - throw new IllegalArgumentException( - "Resource `%s` has unknown type `%s`".formatted(name, resource.getType())); + // Serialize the resources updated in this batch + final var realResourceUpdates = new HashMap>(); + final var dynamicResourceUpdates = new HashMap>(); + + for (final var update : results.resourceUpdates.updates()){ + final var name = update.resourceId().id(); + final var schema = update.resource().getOutputType().getSchema(); + + switch (update.resource.getType()) { + case "real" -> realResourceUpdates.put(name, Pair.of(schema, SimulationEngine.extractRealDynamics(update))); + case "discrete" -> dynamicResourceUpdates.put(name, Pair.of(schema, SimulationEngine.extractDiscreteDynamics(update))); } } - */ - return new Status.Nominal(elapsedTime, Map.of()); + return new Status.Nominal(elapsedTime, realResourceUpdates, dynamicResourceUpdates); } + private static RealDynamics extractRealDynamics(final ResourceUpdates.ResourceUpdate update) { + final var resource = update.resource; + final var dynamics = update.update.dynamics(); + + final var serializedSegment = resource.getOutputType().serialize(dynamics).asMap().orElseThrow(); + final var initial = serializedSegment.get("initial").asReal().orElseThrow(); + final var rate = serializedSegment.get("rate").asReal().orElseThrow(); + + return RealDynamics.linear(initial, rate); + } + + private static SerializedValue extractDiscreteDynamics(final ResourceUpdates.ResourceUpdate update) { + return update.resource.getOutputType().serialize(update.update.dynamics()); + } /** Schedule a new task to be performed at the given time. */ public SpanId scheduleTask(final Duration startTime, final TaskFactory state) { @@ -267,7 +270,7 @@ void trackResource(final String name, final Resource resource, final D if (this.closed) throw new IllegalStateException("Cannot track resource on closed simulation engine"); final var id = new ResourceId(name); - this.resources.put(id, ProfilingState.create(resource)); + this.resources.put(id, resource); this.scheduledJobs.schedule(JobId.forResource(id), SubInstant.Resources.at(nextQueryTime)); } @@ -307,24 +310,43 @@ public JobSchedule.Batch extractNextJobs(final Duration maximumTime) { return batch; } - public record ResourceUpdate(Duration currentTime, Map updates) { - static ResourceUpdate init(Duration currentTime) { - return new ResourceUpdate(currentTime, new LinkedHashMap<>()); - } - public void log(String id, Dynamics dynamics) { - this.updates.put(id, dynamics); - } + public record ResourceUpdates(List> updates){ public boolean isEmpty() { return updates.isEmpty(); } + public int size() { return updates.size(); } + + ResourceUpdates() { + this(new ArrayList<>()); + } + + public void add(ResourceUpdate update) { + this.updates.add(update); + } + + public record ResourceUpdate( + ResourceId resourceId, + Resource resource, + Update update + ) { + public record Update(Duration startOffset, Dynamics dynamics) {} + + public ResourceUpdate (final Querier querier, + final Duration currentTime, + final ResourceId resourceId, + final Resource resource + ) { + this(resourceId, resource, new Update<>(currentTime, resource.getDynamics(querier))); + } + } } public record StepResult( List> commits, - ResourceUpdate resourceUpdate, + ResourceUpdates resourceUpdates, Optional error ) {} @@ -338,21 +360,21 @@ public StepResult performJobs( if (this.closed) throw new IllegalStateException("Cannot perform jobs on closed simulation engine"); var tip = EventGraph.empty(); Mutable> exception = new MutableObject<>(Optional.empty()); - final var resourceUpdate = ResourceUpdate.init(currentTime); + final var resourceUpdates = new ResourceUpdates(); for (final var job$ : jobs) { tip = EventGraph.concurrently(tip, TaskFrame.run(job$, context, (job, frame) -> { try { - this.performJob(job, frame, currentTime, maximumTime, resourceUpdate); + this.performJob(job, frame, currentTime, maximumTime, resourceUpdates); } catch (Throwable ex) { exception.setValue(Optional.of(ex)); } })); if (exception.getValue().isPresent()) { - return new StepResult(List.of(tip), resourceUpdate, exception.getValue()); + return new StepResult(List.of(tip), resourceUpdates, exception.getValue()); } } - return new StepResult(List.of(tip), resourceUpdate, Optional.empty()); + return new StepResult(List.of(tip), resourceUpdates, Optional.empty()); } /** Performs a single job. */ @@ -361,18 +383,17 @@ public void performJob( final TaskFrame frame, final Duration currentTime, final Duration maximumTime, - final ResourceUpdate resourceUpdate + final ResourceUpdates resourceUpdates ) throws SpanException { - if (job instanceof JobId.TaskJobId j) { - this.stepTask(j.id(), frame, currentTime); - } else if (job instanceof JobId.SignalJobId j) { - this.stepTask(this.waitingTasks.remove(j.id()), frame, currentTime); - } else if (job instanceof JobId.ConditionJobId j) { - this.updateCondition(j.id(), frame, currentTime, maximumTime); - } else if (job instanceof JobId.ResourceJobId j) { - this.updateResource(j.id(), frame, currentTime, resourceUpdate); - } else { - throw new IllegalArgumentException("Unexpected subtype of %s: %s".formatted(JobId.class, job.getClass())); + switch (job) { + case JobId.TaskJobId j -> this.stepTask(j.id(), frame, currentTime); + case JobId.SignalJobId j -> this.stepTask(this.waitingTasks.remove(j.id()), frame, currentTime); + case JobId.ConditionJobId j -> this.updateCondition(j.id(), frame, currentTime, maximumTime); + case JobId.ResourceJobId j -> this.updateResource(j.id(), frame, currentTime, resourceUpdates); + case null -> throw new IllegalArgumentException("Unexpected null value for JobId"); + default -> throw new IllegalArgumentException("Unexpected subtype of %s: %s".formatted( + JobId.class, + job.getClass())); } } @@ -503,20 +524,23 @@ public void updateCondition( /** Get the current behavior of a given resource and accumulate it into the resource's profile. */ public void updateResource( - final ResourceId resource, + final ResourceId resourceId, final TaskFrame frame, final Duration currentTime, - final ResourceUpdate resourceUpdate) { + final ResourceUpdates resourceUpdates) { if (this.closed) throw new IllegalStateException("Cannot update resource on closed simulation engine"); final var querier = new EngineQuerier(frame); - final var dynamics = this.resources.get(resource).getDynamics(querier); - resourceUpdate.log(resource.id(), dynamics); + resourceUpdates.add(new ResourceUpdates.ResourceUpdate<>( + querier, + currentTime, + resourceId, + this.resources.get(resourceId))); - this.waitingResources.subscribeQuery(resource, querier.referencedTopics); + this.waitingResources.subscribeQuery(resourceId, querier.referencedTopics); final var expiry = querier.expiry.map(currentTime::plus); if (expiry.isPresent()) { - this.scheduledJobs.schedule(JobId.forResource(resource), SubInstant.Resources.at(expiry.get())); + this.scheduledJobs.schedule(JobId.forResource(resourceId), SubInstant.Resources.at(expiry.get())); } } @@ -633,16 +657,14 @@ void extractOutput(final SerializableTopic topic, final Event ev, final SpanI /** * Get an Activity Directive Id from a SpanId, if the span is a descendent of a directive. */ - public static Optional getDirectiveIdFromSpan( - final SimulationEngine engine, + public Optional getDirectiveIdFromSpan( final Topic activityTopic, - final TemporalEventSource timeline, final Iterable> serializableTopics, final SpanId spanId ) { // Collect per-span information from the event graph. final var spanInfo = new SpanInfo(); - for (final var point : timeline) { + for (final var point : this.timeline) { if (!(point instanceof TemporalEventSource.TimePoint.Commit p)) continue; final var trait = new SpanInfo.Trait(serializableTopics, activityTopic); @@ -652,7 +674,7 @@ public static Optional getDirectiveIdFromSpan( // Identify the nearest ancestor directive Optional directiveSpanId = Optional.of(spanId); while (directiveSpanId.isPresent() && !spanInfo.isDirective(directiveSpanId.get())) { - directiveSpanId = engine.getSpan(directiveSpanId.get()).parent(); + directiveSpanId = this.getSpan(directiveSpanId.get()).parent(); } return directiveSpanId.map(spanInfo::getDirective); } @@ -663,15 +685,14 @@ public record SimulationActivityExtract( Map simulatedActivities, Map unfinishedActivities){} - private static SpanInfo computeSpanInfo( - final TemporalEventSource timeline, + private SpanInfo computeSpanInfo( final Topic activityTopic, final Iterable> serializableTopics ) { // Collect per-span information from the event graph. final var spanInfo = new SpanInfo(); - for (final var point : timeline) { + for (final var point : this.timeline) { if (!(point instanceof TemporalEventSource.TimePoint.Commit p)) continue; final var trait = new SpanInfo.Trait(serializableTopics, activityTopic); @@ -680,39 +701,32 @@ private static SpanInfo computeSpanInfo( return spanInfo; } - public static SimulationActivityExtract computeActivitySimulationResults( - final SimulationEngine engine, + public SimulationActivityExtract computeActivitySimulationResults( final Instant startTime, - final Duration elapsedTime, final Topic activityTopic, - final TemporalEventSource timeline, final Iterable> serializableTopics ){ return computeActivitySimulationResults( - engine, startTime, - elapsedTime, - computeSpanInfo(timeline, activityTopic, serializableTopics) + computeSpanInfo(activityTopic, serializableTopics) ); } - private static HashMap spanToActivityDirectiveId( - final SimulationEngine engine, + private HashMap spanToActivityDirectiveId( final SpanInfo spanInfo ){ final var activityDirectiveIds = new HashMap(); - engine.spans.forEach((span, state) -> { + this.spans.forEach((span, state) -> { if (!spanInfo.isActivity(span)) return; if (spanInfo.isDirective(span)) activityDirectiveIds.put(span, spanInfo.getDirective(span)); }); return activityDirectiveIds; } - private static HashMap spanToSimulatedActivities( - final SimulationEngine engine, + private HashMap spanToSimulatedActivities( final SpanInfo spanInfo ){ - final var activityDirectiveIds = spanToActivityDirectiveId(engine, spanInfo); + final var activityDirectiveIds = spanToActivityDirectiveId(spanInfo); final var spanToSimulatedActivityId = new HashMap(activityDirectiveIds.size()); final var usedSimulatedActivityIds = new HashSet<>(); for (final var entry : activityDirectiveIds.entrySet()) { @@ -720,7 +734,7 @@ private static HashMap spanToSimulatedActivities( usedSimulatedActivityIds.add(entry.getValue().id()); } long counter = 1L; - for (final var span : engine.spans.keySet()) { + for (final var span : this.spans.keySet()) { if (!spanInfo.isActivity(span)) continue; if (spanToSimulatedActivityId.containsKey(span)) continue; @@ -733,26 +747,21 @@ private static HashMap spanToSimulatedActivities( /** * Computes only activity-related results when resources are not needed */ - public static SimulationActivityExtract computeActivitySimulationResults( - final SimulationEngine engine, + public SimulationActivityExtract computeActivitySimulationResults( final Instant startTime, - final Duration elapsedTime, final SpanInfo spanInfo ){ // Identify the nearest ancestor *activity* (excluding intermediate anonymous tasks). final var activityParents = new HashMap(); - final var activityDirectiveIds = spanToActivityDirectiveId(engine, spanInfo); - engine.spans.forEach((span, state) -> { + final var activityDirectiveIds = spanToActivityDirectiveId(spanInfo); + this.spans.forEach((span, state) -> { if (!spanInfo.isActivity(span)) return; var parent = state.parent(); while (parent.isPresent() && !spanInfo.isActivity(parent.get())) { - parent = engine.spans.get(parent.get()).parent(); - } - - if (parent.isPresent()) { - activityParents.put(span, parent.get()); + parent = this.spans.get(parent.get()).parent(); } + parent.ifPresent(spanId -> activityParents.put(span, spanId)); }); final var activityChildren = new HashMap>(); @@ -761,11 +770,11 @@ public static SimulationActivityExtract computeActivitySimulationResults( }); // Give every task corresponding to a child activity an ID that doesn't conflict with any root activity. - final var spanToSimulatedActivityId = spanToSimulatedActivities(engine, spanInfo); + final var spanToSimulatedActivityId = spanToSimulatedActivities(spanInfo); final var simulatedActivities = new HashMap(); final var unfinishedActivities = new HashMap(); - engine.spans.forEach((span, state) -> { + this.spans.forEach((span, state) -> { if (!spanInfo.isActivity(span)) return; final var activityId = spanToSimulatedActivityId.get(span); @@ -800,26 +809,6 @@ public static SimulationActivityExtract computeActivitySimulationResults( return new SimulationActivityExtract(startTime, elapsedTime, simulatedActivities, unfinishedActivities); } - public static SimulationResults computeResults( - final SimulationEngine engine, - final Instant startTime, - final Duration elapsedTime, - final Topic activityTopic, - final TemporalEventSource timeline, - final Iterable> serializableTopics - ) { - return computeResults( - engine, - startTime, - elapsedTime, - activityTopic, - timeline, - serializableTopics, - engine.resources.keySet() - .stream() - .map(ResourceId::id) - .collect(Collectors.toSet())); - } /** Compute a set of results from the current state of simulation. */ // TODO: Move result extraction out of the SimulationEngine. @@ -830,21 +819,19 @@ public static SimulationResults computeResults( // Planners need to be aware of failed or unfinished tasks. public SimulationResults computeResults ( final Instant startTime, - final Duration elapsedTime, final Topic activityTopic, - final TemporalEventSource timeline, final Iterable> serializableTopics, - final Set resourceNames + final SimulationResourceManager resourceManager ) { // Collect per-task information from the event graph. - final var spanInfo = computeSpanInfo(timeline, activityTopic, serializableTopics); + final var spanInfo = computeSpanInfo(activityTopic, serializableTopics); // Extract profiles for every resource. final var resourceProfiles = resourceManager.computeProfiles(elapsedTime); final var realProfiles = resourceProfiles.realProfiles(); final var discreteProfiles = resourceProfiles.discreteProfiles(); - final var activityResults = computeActivitySimulationResults(engine, startTime, elapsedTime, spanInfo); + final var activityResults = computeActivitySimulationResults(startTime, spanInfo); final List> topics = new ArrayList<>(); final var serializableTopicToId = new HashMap, Integer>(); @@ -853,7 +840,7 @@ public SimulationResults computeResults ( topics.add(Triple.of(topics.size(), serializableTopic.name(), serializableTopic.outputType().getSchema())); } - final var spanToActivities = spanToSimulatedActivities(engine,spanInfo); + final var spanToActivities = spanToSimulatedActivities(spanInfo); final var serializedTimeline = new TreeMap>>(); var time = Duration.ZERO; for (var point : this.timeline.points()) { @@ -877,7 +864,7 @@ public SimulationResults computeResults ( spanToActivities.put(event.provenance(), spanToActivities.get(spanId.get())); break; } - spanId = engine.getSpan(spanId.get()).parent(); + spanId = this.getSpan(spanId.get()).parent(); if (spanId.isEmpty()) { break; } @@ -903,14 +890,15 @@ public SimulationResults computeResults ( } } - return new SimulationResults(realProfiles, - discreteProfiles, - activityResults.simulatedActivities, - activityResults.unfinishedActivities, - startTime, - elapsedTime, - topics, - serializedTimeline); + return new SimulationResults( + realProfiles, + discreteProfiles, + activityResults.simulatedActivities, + activityResults.unfinishedActivities, + startTime, + elapsedTime, + topics, + serializedTimeline); } public SimulationResults computeResults( @@ -921,14 +909,14 @@ public SimulationResults computeResults( final Set resourceNames ) { // Collect per-task information from the event graph. - final var spanInfo = computeSpanInfo(timeline, activityTopic, serializableTopics); + final var spanInfo = computeSpanInfo(activityTopic, serializableTopics); // Extract profiles for every resource. - final var resourceProfiles = resourceManager.computeProfiles(elapsedTime); + final var resourceProfiles = resourceManager.computeProfiles(elapsedTime, resourceNames); final var realProfiles = resourceProfiles.realProfiles(); final var discreteProfiles = resourceProfiles.discreteProfiles(); - final var activityResults = computeActivitySimulationResults(engine, startTime, elapsedTime, spanInfo); + final var activityResults = computeActivitySimulationResults(startTime, spanInfo); final List> topics = new ArrayList<>(); final var serializableTopicToId = new HashMap, Integer>(); @@ -937,7 +925,7 @@ public SimulationResults computeResults( topics.add(Triple.of(topics.size(), serializableTopic.name(), serializableTopic.outputType().getSchema())); } - final var spanToActivities = spanToSimulatedActivities(engine,spanInfo); + final var spanToActivities = spanToSimulatedActivities(spanInfo); final var serializedTimeline = new TreeMap>>(); var time = Duration.ZERO; for (var point : this.timeline.points()) { @@ -961,7 +949,7 @@ public SimulationResults computeResults( spanToActivities.put(event.provenance(), spanToActivities.get(spanId.get())); break; } - spanId = engine.getSpan(spanId.get()).parent(); + spanId = this.getSpan(spanId.get()).parent(); if (spanId.isEmpty()) { break; } @@ -1010,48 +998,6 @@ private interface Translator { Target apply(Resource resource, Dynamics dynamics); } - private static - List> serializeProfile( - final Duration elapsedTime, - final ProfilingState state, - final Translator translator - ) { - final var profile = new ArrayList>(state.profile().segments().size()); - - final var iter = state.profile().segments().iterator(); - if (iter.hasNext()) { - var segment = iter.next(); - while (iter.hasNext()) { - final var nextSegment = iter.next(); - - profile.add(new ProfileSegment<>( - nextSegment.startOffset().minus(segment.startOffset()), - translator.apply(state.resource(), segment.dynamics()))); - segment = nextSegment; - } - - profile.add(new ProfileSegment<>( - elapsedTime.minus(segment.startOffset()), - translator.apply(state.resource(), segment.dynamics()))); - } - - return profile; - } - - private static - RealDynamics extractRealDynamics(final Resource resource, final Dynamics dynamics) { - final var serializedSegment = resource.getOutputType().serialize(dynamics).asMap().orElseThrow(); - final var initial = serializedSegment.get("initial").asReal().orElseThrow(); - final var rate = serializedSegment.get("rate").asReal().orElseThrow(); - - return RealDynamics.linear(initial, rate); - } - - private static - SerializedValue extractDiscreteDynamics(final Resource resource, final Dynamics dynamics) { - return resource.getOutputType().serialize(dynamics); - } - /** A handle for processing requests from a modeled resource or condition. */ private static final class EngineQuerier implements Querier { private final TaskFrame frame; From 96e477db3c07f5a006ce6d9540103e48cdb6a9dc Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 23 May 2024 13:25:27 -0700 Subject: [PATCH 09/20] Remove unneeded methods from Engine --- .../aerie/merlin/driver/engine/SimulationEngine.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java index cfd74cc3db..3c5dd515c7 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java @@ -58,13 +58,8 @@ * A representation of the work remaining to do during a simulation, and its accumulated results. */ public final class SimulationEngine implements AutoCloseable { - private static int numActiveSimulationEngines = 0; private boolean closed = false; - public static int getNumActiveSimulationEngines() { - return numActiveSimulationEngines; - } - /** The set of all jobs waiting for time to pass. */ private final JobSchedule scheduledJobs; /** The set of all jobs waiting on a condition. */ @@ -100,7 +95,6 @@ public static int getNumActiveSimulationEngines() { private Duration elapsedTime; public SimulationEngine(LiveCells initialCells) { - numActiveSimulationEngines++; timeline = new TemporalEventSource(); cells = new LiveCells(timeline, initialCells); elapsedTime = Duration.ZERO; @@ -120,7 +114,6 @@ public SimulationEngine(LiveCells initialCells) { } private SimulationEngine(SimulationEngine other) { - numActiveSimulationEngines++; other.timeline.freeze(); other.cells.freeze(); @@ -547,7 +540,6 @@ public void updateResource( /** Resets all tasks (freeing any held resources). The engine should not be used after being closed. */ @Override public void close() { - numActiveSimulationEngines--; cells.freeze(); timeline.freeze(); @@ -994,10 +986,6 @@ private static Optional trySerializeEvent(Event eve return event.extract(serializableTopic.topic(), serializableTopic.outputType()::serialize); } - private interface Translator { - Target apply(Resource resource, Dynamics dynamics); - } - /** A handle for processing requests from a modeled resource or condition. */ private static final class EngineQuerier implements Querier { private final TaskFrame frame; From bbadaf220ed5c47d0e9adc8ecf24e6e4bd65be2c Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Mon, 15 Jul 2024 13:20:28 -0700 Subject: [PATCH 10/20] Add reference timeline in Engine --- .../driver/engine/SimulationEngine.java | 47 +++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java index 3c5dd515c7..65ee4fa62b 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java @@ -91,11 +91,13 @@ public final class SimulationEngine implements AutoCloseable { /* The top-level simulation timeline. */ private final TemporalEventSource timeline; + private final TemporalEventSource referenceTimeline; private final LiveCells cells; private Duration elapsedTime; public SimulationEngine(LiveCells initialCells) { timeline = new TemporalEventSource(); + referenceTimeline = new TemporalEventSource(); cells = new LiveCells(timeline, initialCells); elapsedTime = Duration.ZERO; @@ -115,11 +117,14 @@ public SimulationEngine(LiveCells initialCells) { private SimulationEngine(SimulationEngine other) { other.timeline.freeze(); + other.referenceTimeline.freeze(); other.cells.freeze(); elapsedTime = other.elapsedTime; - timeline = other.combineTimeline(new TemporalEventSource()); + + timeline = new TemporalEventSource(); cells = new LiveCells(timeline, other.cells); + referenceTimeline = other.combineTimeline(); // New Executor allows other SimulationEngine to be closed executor = Executors.newVirtualThreadPerTaskExecutor(); @@ -679,12 +684,13 @@ public record SimulationActivityExtract( private SpanInfo computeSpanInfo( final Topic activityTopic, - final Iterable> serializableTopics + final Iterable> serializableTopics, + final TemporalEventSource timeline ) { // Collect per-span information from the event graph. final var spanInfo = new SpanInfo(); - for (final var point : this.timeline) { + for (final var point : timeline) { if (!(point instanceof TemporalEventSource.TimePoint.Commit p)) continue; final var trait = new SpanInfo.Trait(serializableTopics, activityTopic); @@ -700,7 +706,7 @@ public SimulationActivityExtract computeActivitySimulationResults( ){ return computeActivitySimulationResults( startTime, - computeSpanInfo(activityTopic, serializableTopics) + computeSpanInfo(activityTopic, serializableTopics, combineTimeline()) ); } @@ -815,8 +821,9 @@ public SimulationResults computeResults ( final Iterable> serializableTopics, final SimulationResourceManager resourceManager ) { + final var combinedTimeline = this.combineTimeline(); // Collect per-task information from the event graph. - final var spanInfo = computeSpanInfo(activityTopic, serializableTopics); + final var spanInfo = computeSpanInfo(activityTopic, serializableTopics, combinedTimeline); // Extract profiles for every resource. final var resourceProfiles = resourceManager.computeProfiles(elapsedTime); @@ -835,7 +842,7 @@ public SimulationResults computeResults ( final var spanToActivities = spanToSimulatedActivities(spanInfo); final var serializedTimeline = new TreeMap>>(); var time = Duration.ZERO; - for (var point : this.timeline.points()) { + for (var point : combinedTimeline.points()) { if (point instanceof TemporalEventSource.TimePoint.Delta delta) { time = time.plus(delta.delta()); } else if (point instanceof TemporalEventSource.TimePoint.Commit commit) { @@ -900,8 +907,9 @@ public SimulationResults computeResults( final SimulationResourceManager resourceManager, final Set resourceNames ) { + final var combinedTimeline = this.combineTimeline(); // Collect per-task information from the event graph. - final var spanInfo = computeSpanInfo(activityTopic, serializableTopics); + final var spanInfo = computeSpanInfo(activityTopic, serializableTopics, combinedTimeline); // Extract profiles for every resource. final var resourceProfiles = resourceManager.computeProfiles(elapsedTime, resourceNames); @@ -920,7 +928,7 @@ public SimulationResults computeResults( final var spanToActivities = spanToSimulatedActivities(spanInfo); final var serializedTimeline = new TreeMap>>(); var time = Duration.ZERO; - for (var point : this.timeline.points()) { + for (var point : combinedTimeline.points()) { if (point instanceof TemporalEventSource.TimePoint.Delta delta) { time = time.plus(delta.delta()); } else if (point instanceof TemporalEventSource.TimePoint.Commit commit) { @@ -1152,4 +1160,27 @@ public SimulationEngine duplicate() { public Optional peekNextTime() { return this.scheduledJobs.peekNextTime(); } + + /** + * Create a timeline that in the output of the engine's reference timeline combined with its expanded timeline. + */ + public TemporalEventSource combineTimeline() { + final TemporalEventSource combinedTimeline = new TemporalEventSource(); + for (final var timePoint : referenceTimeline.points()) { + if (timePoint instanceof TemporalEventSource.TimePoint.Delta t) { + combinedTimeline.add(t.delta()); + } else if (timePoint instanceof TemporalEventSource.TimePoint.Commit t) { + combinedTimeline.add(t.events()); + } + } + + for (final var timePoint : timeline) { + if (timePoint instanceof TemporalEventSource.TimePoint.Delta t) { + combinedTimeline.add(t.delta()); + } else if (timePoint instanceof TemporalEventSource.TimePoint.Commit t) { + combinedTimeline.add(t.events()); + } + } + return combinedTimeline; + } } From 9339bd766f5602fb0bb29bc35d1f0d16a6367cc9 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 18:28:21 -0700 Subject: [PATCH 11/20] Formatting (SimEngine, CheckpointSimFacade/Driver) --- .../driver/CheckpointSimulationDriver.java | 80 ++++--- .../driver/engine/SimulationEngine.java | 215 +++++++++++------- .../CheckpointSimulationFacade.java | 129 ++++++----- 3 files changed, 243 insertions(+), 181 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java index 39429a2c5f..7e9c30a7fa 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java @@ -85,7 +85,8 @@ public static CachedSimulationEngine empty(final MissionModel missionModel) { public static Optional>> bestCachedEngine( final Map schedule, final List cachedEngines, - final Duration planDuration) { + final Duration planDuration + ) { Optional bestCandidate = Optional.empty(); final Map correspondenceMap = new HashMap<>(); final var minimumStartTimes = getMinimumStartTimes(schedule, planDuration); @@ -102,7 +103,7 @@ public static Optional e.getValue().equals(activity.getValue())) .findFirst(); - if(entryToRemove.isPresent()) { + if (entryToRemove.isPresent()) { final var entry = entryToRemove.get(); activityDirectivesInCache.remove(entry.getKey()); correspondenceMap.put(activity.getKey(), entry.getKey()); @@ -127,7 +128,7 @@ public static Optional LOGGER.info("Re-using simulation engine at " - + cachedSimulationEngine.endsAt())); + + cachedSimulationEngine.endsAt())); return bestCandidate.map(cachedSimulationEngine -> Pair.of(cachedSimulationEngine, correspondenceMap)); } @@ -136,7 +137,8 @@ public static Optional desiredCheckpoints(final List desiredCheckpoints) { return simulationState -> { for (final var desiredCheckpoint : desiredCheckpoints) { - if (simulationState.currentTime().noLongerThan(desiredCheckpoint) && simulationState.nextTime().longerThan(desiredCheckpoint)) { + if (simulationState.currentTime().noLongerThan(desiredCheckpoint) && simulationState.nextTime().longerThan( + desiredCheckpoint)) { return true; } } @@ -148,22 +150,25 @@ public static Function checkpointAtEnd(Function stoppingCondition.apply(simulationState) || simulationState.nextTime.isEqualTo(MAX_VALUE); } - private static Map getMinimumStartTimes(final Map schedule, final Duration planDuration){ + private static Map getMinimumStartTimes( + final Map schedule, + final Duration planDuration) + { //For an anchored activity, it's minimum invalidationTime would be the sum of all startOffsets in its anchor chain // (plus or minus the plan duration depending on whether the root is anchored to plan start or plan end). // If it's a start anchor chain (as in, all anchors have anchoredToStart set to true), // this will give you its exact start time, but if there are any end-time anchors, this will give you the minimum time the activity could start at. final var minimumStartTimes = new HashMap(); - for(final var activity : schedule.entrySet()){ + for (final var activity : schedule.entrySet()) { var curInChain = activity; var curSum = ZERO; - while(true){ - if(curInChain.getValue().anchorId() == null){ + while (true) { + if (curInChain.getValue().anchorId() == null) { curSum = curSum.plus(curInChain.getValue().startOffset()); curSum = !curInChain.getValue().anchoredToStart() ? curSum.plus(planDuration) : curSum; minimumStartTimes.put(activity.getKey(), curSum); break; - } else{ + } else { curSum = curSum.plus(curInChain.getValue().startOffset()); curInChain = Map.entry(curInChain.getValue().anchorId(), schedule.get(curInChain.getValue().anchorId())); } @@ -176,9 +181,9 @@ public record SimulationState( Duration currentTime, Duration nextTime, SimulationEngine simulationEngine, - Map schedule, + Map schedule, Map activityDirectiveIdSpanIdMap - ){} + ) {} /** * Simulates a plan/schedule while using and creating simulation checkpoints. @@ -193,7 +198,8 @@ public record SimulationState( * @param cachedEngine the simulation engine that is going to be used * @param shouldTakeCheckpoint a function from state of the simulation to boolean deciding when to take checkpoints * @param stopConditionOnPlan a function from state of the simulation to boolean deciding when to stop simulation - * @param cachedEngineStore a store for simulation engine checkpoints taken. If capacity is 1, the simulation will behave like a resumable simulation. + * @param cachedEngineStore a store for simulation engine checkpoints taken. If capacity is 1, the simulation will + * behave like a resumable simulation. * @param configuration the simulation configuration * @return all the information to compute simulation results if needed */ @@ -210,8 +216,8 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( final Function shouldTakeCheckpoint, final Function stopConditionOnPlan, final CachedEngineStore cachedEngineStore, - final SimulationEngineConfiguration configuration) - { + final SimulationEngineConfiguration configuration + ) { final boolean duplicationIsOk = cachedEngineStore.capacity() > 1; final var activityToSpan = new HashMap(); final var activityTopic = cachedEngine.activityTopic(); @@ -229,8 +235,10 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( // Schedule all activities. // Using HashMap explicitly because it allows `null` as a key. // `null` key means that an activity is not waiting on another activity to finish to know its start time - HashMap>> resolved = new StartOffsetReducer(planDuration, schedule).compute(); - if(!resolved.isEmpty()) { + HashMap>> resolved = new StartOffsetReducer( + planDuration, + schedule).compute(); + if (!resolved.isEmpty()) { resolved.put( null, StartOffsetReducer.adjustStartOffset( @@ -240,12 +248,16 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( Duration.MICROSECONDS))); } // Filter out activities that are before simulationStartTime - resolved = StartOffsetReducer.filterOutStartOffsetBefore(resolved, Duration.max(ZERO, cachedEngine.endsAt().plus(MICROSECONDS))); + resolved = StartOffsetReducer.filterOutStartOffsetBefore( + resolved, + Duration.max( + ZERO, + cachedEngine.endsAt().plus(MICROSECONDS))); final var toSchedule = new LinkedHashSet(); toSchedule.add(null); final HashMap>> finalResolved = resolved; final var activitiesToBeScheduledNow = new HashMap(); - if(finalResolved.get(null) != null) { + if (finalResolved.get(null) != null) { for (final var r : finalResolved.get(null)) { activitiesToBeScheduledNow.put(r.getKey(), schedule.get(r.getKey())); } @@ -336,7 +348,7 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( // Swallowing the spanException as the internal `spanId` is not user meaningful info. final var topics = missionModel.getTopics(); final var directiveId = SimulationEngine.getDirectiveIdFromSpan(engine, activityTopic, timeline, topics, ex.spanId); - if(directiveId.isPresent()) { + if (directiveId.isPresent()) { throw new SimulationException(elapsedTime, simulationStartTime, directiveId.get(), ex.cause); } throw new SimulationException(elapsedTime, simulationStartTime, ex.cause); @@ -355,12 +367,13 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( private static Set getSuccessorsToSchedule( final SimulationEngine engine, - final Map toCheckForDependencyScheduling) { + final Map toCheckForDependencyScheduling + ) { final var toSchedule = new LinkedHashSet(); final var iterator = toCheckForDependencyScheduling.entrySet().iterator(); - while(iterator.hasNext()){ + while (iterator.hasNext()) { final var taskToCheck = iterator.next(); - if(engine.spanIsComplete(taskToCheck.getValue())){ + if (engine.spanIsComplete(taskToCheck.getValue())) { toSchedule.add(taskToCheck.getKey()); iterator.remove(); } @@ -376,10 +389,11 @@ private static Map scheduleActivities( final SimulationEngine engine, final Duration curTime, final Map activityToTask, - final Topic activityTopic){ + final Topic activityTopic + ) { final var toCheckForDependencyScheduling = new HashMap(); - for(final var predecessor: toScheduleNow) { - if(!resolved.containsKey(predecessor)) continue; + for (final var predecessor : toScheduleNow) { + if (!resolved.containsKey(predecessor)) continue; for (final var directivePair : resolved.get(predecessor)) { final var offset = directivePair.getRight(); final var directiveIdToSchedule = directivePair.getLeft(); @@ -410,19 +424,21 @@ private static Map scheduleActivities( return toCheckForDependencyScheduling; } - public static Function onceAllActivitiesAreFinished(){ + public static Function onceAllActivitiesAreFinished() { return simulationState -> simulationState.activityDirectiveIdSpanIdMap() - .values() - .stream() - .allMatch(simulationState.simulationEngine()::spanIsComplete); + .values() + .stream() + .allMatch(simulationState.simulationEngine()::spanIsComplete); } - public static Function noCondition(){ + public static Function noCondition() { return simulationState -> false; } - public static Function stopOnceActivityHasFinished(final ActivityDirectiveId activityDirectiveId){ + public static Function stopOnceActivityHasFinished(final ActivityDirectiveId activityDirectiveId) { return simulationState -> (simulationState.activityDirectiveIdSpanIdMap().containsKey(activityDirectiveId) - && simulationState.simulationEngine.spanIsComplete(simulationState.activityDirectiveIdSpanIdMap().get(activityDirectiveId))); + && simulationState.simulationEngine.spanIsComplete(simulationState + .activityDirectiveIdSpanIdMap() + .get(activityDirectiveId))); } } diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java index 65ee4fa62b..90fa01c84a 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java @@ -168,7 +168,7 @@ public void init(Map> resources, TaskFactory daemons) for (final var commit : results.commits()) { timeline.add(commit); } - if(results.error.isPresent()) { + if (results.error.isPresent()) { throw results.error.get(); } } @@ -207,8 +207,8 @@ public Status step(Duration simulationDuration) throws Throwable { final var results = this.performJobs(batch.jobs(), cells, elapsedTime, simulationDuration); for (final var commit : results.commits()) { timeline.add(commit); - } - if(results.error.isPresent()) { + } + if (results.error.isPresent()) { throw results.error.get(); } @@ -216,13 +216,17 @@ public Status step(Duration simulationDuration) throws Throwable { final var realResourceUpdates = new HashMap>(); final var dynamicResourceUpdates = new HashMap>(); - for (final var update : results.resourceUpdates.updates()){ + for (final var update : results.resourceUpdates.updates()) { final var name = update.resourceId().id(); final var schema = update.resource().getOutputType().getSchema(); switch (update.resource.getType()) { case "real" -> realResourceUpdates.put(name, Pair.of(schema, SimulationEngine.extractRealDynamics(update))); - case "discrete" -> dynamicResourceUpdates.put(name, Pair.of(schema, SimulationEngine.extractDiscreteDynamics(update))); + case "discrete" -> dynamicResourceUpdates.put( + name, + Pair.of( + schema, + SimulationEngine.extractDiscreteDynamics(update))); } } @@ -247,7 +251,8 @@ private static SerializedValue extractDiscreteDynamics(final Resource /** Schedule a new task to be performed at the given time. */ public SpanId scheduleTask(final Duration startTime, final TaskFactory state) { if (this.closed) throw new IllegalStateException("Cannot schedule task on closed simulation engine"); - if (startTime.isNegative()) throw new IllegalArgumentException("Cannot schedule a task before the start time of the simulation"); + if (startTime.isNegative()) throw new IllegalArgumentException( + "Cannot schedule a task before the start time of the simulation"); final var span = SpanId.generate(); this.spans.put(span, new Span(Optional.empty(), startTime, Optional.empty())); @@ -332,10 +337,11 @@ public record ResourceUpdate( ) { public record Update(Duration startOffset, Dynamics dynamics) {} - public ResourceUpdate (final Querier querier, - final Duration currentTime, - final ResourceId resourceId, - final Resource resource + public ResourceUpdate( + final Querier querier, + final Duration currentTime, + final ResourceId resourceId, + final Resource resource ) { this(resourceId, resource, new Update<>(currentTime, resource.getDynamics(querier))); } @@ -396,7 +402,8 @@ public void performJob( } /** Perform the next step of a modeled task. */ - public void stepTask(final TaskId task, final TaskFrame frame, final Duration currentTime) throws SpanException { + public void stepTask(final TaskId task, final TaskFrame frame, final Duration currentTime) + throws SpanException { if (this.closed) throw new IllegalStateException("Cannot step task on closed simulation engine"); this.unstartedTasks.remove(task); // The handler for the next status of the task is responsible @@ -425,73 +432,79 @@ private void stepEffectModel( // TODO: Report which cells this activity read from at this point in time. This is useful insight for any user. // Based on the task's return status, update its execution state and schedule its resumption. - switch (status) { - case TaskStatus.Completed s -> { - // Propagate completion up the span hierarchy. - // TERMINATION: The span hierarchy is a finite tree, so eventually we find a parentless span. - var span = scheduler.span; - while (true) { - if (this.spanContributorCount.get(span).decrementAndGet() > 0) break; - this.spanContributorCount.remove(span); + switch (status) { + case TaskStatus.Completed s -> { + // Propagate completion up the span hierarchy. + // TERMINATION: The span hierarchy is a finite tree, so eventually we find a parentless span. + var span = scheduler.span; + while (true) { + if (this.spanContributorCount.get(span).decrementAndGet() > 0) break; + this.spanContributorCount.remove(span); - this.spans.compute(span, (_id, $) -> $.close(currentTime)); + this.spans.compute(span, (_id, $) -> $.close(currentTime)); - final var span$ = this.spans.get(span).parent; - if (span$.isEmpty()) break; + final var span$ = this.spans.get(span).parent; + if (span$.isEmpty()) break; - span = span$.get(); - } - - // Notify any blocked caller of our completion. - progress.caller().ifPresent($ -> { - if (this.blockedTasks.get($).decrementAndGet() == 0) { - this.blockedTasks.remove($); - this.scheduledJobs.schedule(JobId.forTask($), SubInstant.Tasks.at(currentTime)); - } - }); + span = span$.get(); } - case TaskStatus.Delayed s -> { - if (s.delay().isNegative()) throw new IllegalArgumentException("Cannot schedule a task in the past"); + // Notify any blocked caller of our completion. + progress.caller().ifPresent($ -> { + if (this.blockedTasks.get($).decrementAndGet() == 0) { + this.blockedTasks.remove($); + this.scheduledJobs.schedule(JobId.forTask($), SubInstant.Tasks.at(currentTime)); + } + }); + } - this.tasks.put(task, progress.continueWith(s.continuation())); - this.scheduledJobs.schedule(JobId.forTask(task), SubInstant.Tasks.at(currentTime.plus(s.delay()))); - } + case TaskStatus.Delayed s -> { + if (s.delay().isNegative()) throw new IllegalArgumentException("Cannot schedule a task in the past"); - case TaskStatus.CallingTask s -> { - // Prepare a span for the child task. - final var childSpan = switch (s.childSpan()) { - case Parent -> - scheduler.span; - - case Fresh -> { - final var freshSpan = SpanId.generate(); - SimulationEngine.this.spans.put(freshSpan, new Span(Optional.of(scheduler.span), currentTime, Optional.empty())); - SimulationEngine.this.spanContributorCount.put(freshSpan, new MutableInt(1)); - yield freshSpan; - } - }; + this.tasks.put(task, progress.continueWith(s.continuation())); + this.scheduledJobs.schedule(JobId.forTask(task), SubInstant.Tasks.at(currentTime.plus(s.delay()))); + } - // Spawn the child task. - final var childTask = TaskId.generate(); - SimulationEngine.this.spanContributorCount.get(scheduler.span).increment(); - SimulationEngine.this.tasks.put(childTask, new ExecutionState<>(childSpan, Optional.of(task), s.child().create(this.executor))); - frame.signal(JobId.forTask(childTask)); + case TaskStatus.CallingTask s -> { + // Prepare a span for the child task. + final var childSpan = switch (s.childSpan()) { + case Parent -> scheduler.span; + + case Fresh -> { + final var freshSpan = SpanId.generate(); + SimulationEngine.this.spans.put( + freshSpan, + new Span(Optional.of(scheduler.span), currentTime, Optional.empty())); + SimulationEngine.this.spanContributorCount.put(freshSpan, new MutableInt(1)); + yield freshSpan; + } + }; - // Arrange for the parent task to resume.... later. - SimulationEngine.this.blockedTasks.put(task, new MutableInt(1)); - this.tasks.put(task, progress.continueWith(s.continuation())); - } + // Spawn the child task. + final var childTask = TaskId.generate(); + SimulationEngine.this.spanContributorCount.get(scheduler.span).increment(); + SimulationEngine.this.tasks.put( + childTask, + new ExecutionState<>( + childSpan, + Optional.of(task), + s.child().create(this.executor))); + frame.signal(JobId.forTask(childTask)); + + // Arrange for the parent task to resume.... later. + SimulationEngine.this.blockedTasks.put(task, new MutableInt(1)); + this.tasks.put(task, progress.continueWith(s.continuation())); + } - case TaskStatus.AwaitingCondition s -> { - final var condition = ConditionId.generate(); - this.conditions.put(condition, s.condition()); - this.scheduledJobs.schedule(JobId.forCondition(condition), SubInstant.Conditions.at(currentTime)); + case TaskStatus.AwaitingCondition s -> { + final var condition = ConditionId.generate(); + this.conditions.put(condition, s.condition()); + this.scheduledJobs.schedule(JobId.forCondition(condition), SubInstant.Conditions.at(currentTime)); - this.tasks.put(task, progress.continueWith(s.continuation())); - this.waitingTasks.put(condition, task); - } + this.tasks.put(task, progress.continueWith(s.continuation())); + this.waitingTasks.put(condition, task); } + } } /** Determine when a condition is next true, and schedule a signal to be raised at that time. */ @@ -587,7 +600,9 @@ public ActivityDirectiveId getDirective(SpanId id) { return this.spanToPlannedDirective.get(id); } - public record Trait(Iterable> topics, Topic activityTopic) implements EffectTrait> { + public record Trait(Iterable> topics, Topic activityTopic) + implements EffectTrait> + { @Override public Consumer empty() { return spanInfo -> {}; @@ -595,7 +610,10 @@ public Consumer empty() { @Override public Consumer sequentially(final Consumer prefix, final Consumer suffix) { - return spanInfo -> { prefix.accept(spanInfo); suffix.accept(spanInfo); }; + return spanInfo -> { + prefix.accept(spanInfo); + suffix.accept(spanInfo); + }; } @Override @@ -606,7 +624,10 @@ public Consumer concurrently(final Consumer left, final Cons // Arguably, this is a model-specific analysis anyway, since we're looking for specific events // and inferring model structure from them, and at this time we're only working with models // for which every activity has a span to itself. - return spanInfo -> { left.accept(spanInfo); right.accept(spanInfo); }; + return spanInfo -> { + left.accept(spanInfo); + right.accept(spanInfo); + }; } public Consumer atom(final Event ev) { @@ -680,7 +701,8 @@ public record SimulationActivityExtract( Instant startTime, Duration duration, Map simulatedActivities, - Map unfinishedActivities){} + Map unfinishedActivities + ) {} private SpanInfo computeSpanInfo( final Topic activityTopic, @@ -703,7 +725,7 @@ public SimulationActivityExtract computeActivitySimulationResults( final Instant startTime, final Topic activityTopic, final Iterable> serializableTopics - ){ + ) { return computeActivitySimulationResults( startTime, computeSpanInfo(activityTopic, serializableTopics, combineTimeline()) @@ -712,7 +734,8 @@ public SimulationActivityExtract computeActivitySimulationResults( private HashMap spanToActivityDirectiveId( final SpanInfo spanInfo - ){ + ) + { final var activityDirectiveIds = new HashMap(); this.spans.forEach((span, state) -> { if (!spanInfo.isActivity(span)) return; @@ -723,7 +746,7 @@ private HashMap spanToActivityDirectiveId( private HashMap spanToSimulatedActivities( final SpanInfo spanInfo - ){ + ) { final var activityDirectiveIds = spanToActivityDirectiveId(spanInfo); final var spanToSimulatedActivityId = new HashMap(activityDirectiveIds.size()); final var usedSimulatedActivityIds = new HashSet<>(); @@ -748,7 +771,7 @@ private HashMap spanToSimulatedActivities( public SimulationActivityExtract computeActivitySimulationResults( final Instant startTime, final SpanInfo spanInfo - ){ + ) { // Identify the nearest ancestor *activity* (excluding intermediate anonymous tasks). final var activityParents = new HashMap(); final var activityDirectiveIds = spanToActivityDirectiveId(spanInfo); @@ -788,7 +811,11 @@ public SimulationActivityExtract computeActivitySimulationResults( startTime.plus(state.startOffset().in(Duration.MICROSECONDS), ChronoUnit.MICROS), state.endOffset().get().minus(state.startOffset()), spanToSimulatedActivityId.get(activityParents.get(span)), - activityChildren.getOrDefault(span, Collections.emptyList()).stream().map(spanToSimulatedActivityId::get).toList(), + activityChildren + .getOrDefault(span, Collections.emptyList()) + .stream() + .map(spanToSimulatedActivityId::get) + .toList(), (activityParents.containsKey(span)) ? Optional.empty() : Optional.ofNullable(directiveId), outputAttributes )); @@ -799,7 +826,11 @@ public SimulationActivityExtract computeActivitySimulationResults( inputAttributes.getArguments(), startTime.plus(state.startOffset().in(Duration.MICROSECONDS), ChronoUnit.MICROS), spanToSimulatedActivityId.get(activityParents.get(span)), - activityChildren.getOrDefault(span, Collections.emptyList()).stream().map(spanToSimulatedActivityId::get).toList(), + activityChildren + .getOrDefault(span, Collections.emptyList()) + .stream() + .map(spanToSimulatedActivityId::get) + .toList(), (activityParents.containsKey(span)) ? Optional.empty() : Optional.of(directiveId) )); } @@ -975,14 +1006,15 @@ public SimulationResults computeResults( } } - return new SimulationResults(realProfiles, - discreteProfiles, - activityResults.simulatedActivities, - activityResults.unfinishedActivities, - startTime, - elapsedTime, - topics, - serializedTimeline); + return new SimulationResults( + realProfiles, + discreteProfiles, + activityResults.simulatedActivities, + activityResults.unfinishedActivities, + startTime, + elapsedTime, + topics, + serializedTimeline); } public Span getSpan(SpanId spanId) { @@ -990,7 +1022,10 @@ public Span getSpan(SpanId spanId) { } - private static Optional trySerializeEvent(Event event, SerializableTopic serializableTopic) { + private static Optional trySerializeEvent( + Event event, + SerializableTopic serializableTopic + ) { return event.extract(serializableTopic.topic(), serializableTopic.outputType()::serialize); } @@ -1070,8 +1105,7 @@ public void emit(final EventType event, final Topic topic public void spawn(final InSpan inSpan, final TaskFactory state) { // Prepare a span for the child task final var childSpan = switch (inSpan) { - case Parent -> - this.span; + case Parent -> this.span; case Fresh -> { final var freshSpan = SpanId.generate(); @@ -1083,7 +1117,12 @@ public void spawn(final InSpan inSpan, final TaskFactory state) { final var childTask = TaskId.generate(); SimulationEngine.this.spanContributorCount.get(this.span).increment(); - SimulationEngine.this.tasks.put(childTask, new ExecutionState<>(childSpan, this.caller, state.create(SimulationEngine.this.executor))); + SimulationEngine.this.tasks.put( + childTask, + new ExecutionState<>( + childSpan, + this.caller, + state.create(SimulationEngine.this.executor))); this.frame.signal(JobId.forTask(childTask)); this.caller.ifPresent($ -> SimulationEngine.this.blockedTasks.get($).increment()); diff --git a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacade.java b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacade.java index df66f6a55e..20ce254408 100644 --- a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacade.java +++ b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacade.java @@ -50,7 +50,7 @@ public class CheckpointSimulationFacade implements SimulationFacade { * @param simulationData the initial simulation results */ @Override - public void setInitialSimResults(final SimulationData simulationData){ + public void setInitialSimResults(final SimulationData simulationData) { this.initialSimulationResults = simulationData; } @@ -61,8 +61,9 @@ public CheckpointSimulationFacade( final InMemoryCachedEngineStore cachedEngines, final PlanningHorizon planningHorizon, final SimulationEngineConfiguration simulationEngineConfiguration, - final Supplier canceledListener){ - if(cachedEngines.capacity() > 1) ThreadedTask.CACHE_READS = true; + final Supplier canceledListener) + { + if (cachedEngines.capacity() > 1) ThreadedTask.CACHE_READS = true; this.missionModel = missionModel; this.schedulerModel = schedulerModel; this.cachedEngines = cachedEngines; @@ -77,14 +78,14 @@ public CheckpointSimulationFacade( final PlanningHorizon planningHorizon, final MissionModel missionModel, final SchedulerModel schedulerModel - ){ + ) { this( missionModel, schedulerModel, new InMemoryCachedEngineStore(1), planningHorizon, new SimulationEngineConfiguration(Map.of(), Instant.now(), new MissionModelId(1)), - ()-> false + () -> false ); } @@ -98,16 +99,16 @@ public Duration totalSimulationTime(){ } @Override - public Supplier getCanceledListener(){ + public Supplier getCanceledListener() { return this.canceledListener; } @Override - public void addActivityTypes(final Collection activityTypes){ + public void addActivityTypes(final Collection activityTypes) { activityTypes.forEach(at -> this.activityTypes.put(at.getName(), at)); } - private void replaceValue(final Map map, final V value, final V replacement){ + private void replaceValue(final Map map, final V value, final V replacement) { for (final Map.Entry entry : map.entrySet()) { if (entry.getValue().equals(value)) { entry.setValue(replacement); @@ -118,22 +119,30 @@ private void replaceValue(final Map map, final V value, final V repla private void replaceIds( final PlanSimCorrespondence planSimCorrespondence, - final Map updates){ - for(final var replacements : updates.entrySet()){ - replaceValue(planSimCorrespondence.planActDirectiveIdToSimulationActivityDirectiveId(),replacements.getKey(), replacements.getValue()); - if(planSimCorrespondence.directiveIdActivityDirectiveMap().containsKey(replacements.getKey())){ + final Map updates) + { + for (final var replacements : updates.entrySet()) { + replaceValue( + planSimCorrespondence.planActDirectiveIdToSimulationActivityDirectiveId(), + replacements.getKey(), + replacements.getValue()); + if (planSimCorrespondence.directiveIdActivityDirectiveMap().containsKey(replacements.getKey())) { final var value = planSimCorrespondence.directiveIdActivityDirectiveMap().remove(replacements.getKey()); planSimCorrespondence.directiveIdActivityDirectiveMap().put(replacements.getValue(), value); } //replace the anchor ids in the plan final var replacementMap = new HashMap(); - for(final var act : planSimCorrespondence.directiveIdActivityDirectiveMap().entrySet()){ - if(act.getValue().anchorId() != null && act.getValue().anchorId().equals(replacements.getKey())){ - final var replacementActivity = new ActivityDirective(act.getValue().startOffset(), act.getValue().serializedActivity(), replacements.getValue(), act.getValue().anchoredToStart()); + for (final var act : planSimCorrespondence.directiveIdActivityDirectiveMap().entrySet()) { + if (act.getValue().anchorId() != null && act.getValue().anchorId().equals(replacements.getKey())) { + final var replacementActivity = new ActivityDirective( + act.getValue().startOffset(), + act.getValue().serializedActivity(), + replacements.getValue(), + act.getValue().anchoredToStart()); replacementMap.put(act.getKey(), replacementActivity); } } - for(final var replacement: replacementMap.entrySet()){ + for (final var replacement : replacementMap.entrySet()) { planSimCorrespondence.directiveIdActivityDirectiveMap().remove(replacement.getKey()); planSimCorrespondence.directiveIdActivityDirectiveMap().put(replacement.getKey(), replacement.getValue()); } @@ -153,7 +162,7 @@ public SimulationResultsComputerInputs simulateNoResultsAllActivities(final Plan return simulateNoResults(plan, null, null).simulationResultsComputerInputs(); } - /** + /** * Simulates a plan until the end of one of its activities * Do not use to update the plan as decomposing activities may not finish * @param plan @@ -161,34 +170,31 @@ public SimulationResultsComputerInputs simulateNoResultsAllActivities(final Plan * @return * @throws SimulationException */ - @Override public SimulationResultsComputerInputs simulateNoResultsUntilEndAct( - final Plan plan, - final SchedulingActivityDirective activity) throws SimulationException, SchedulingInterruptedException - { + final Plan plan, + final SchedulingActivityDirective activity) + throws SimulationException, SchedulingInterruptedException { return simulateNoResults(plan, null, activity).simulationResultsComputerInputs(); } - public AugmentedSimulationResultsComputerInputs simulateNoResults( - final Plan plan, - final Duration until) throws SimulationException, SchedulingInterruptedException - { + public AugmentedSimulationResultsComputerInputs simulateNoResults(final Plan plan, final Duration until) + throws SimulationException, SchedulingInterruptedException { return simulateNoResults(plan, until, null); } - /** - * Simulates and updates plan - * @param plan - * @param until can be null - * @param activity can be null - */ + /** + * Simulates and updates plan + * @param plan + * @param until can be null + * @param activity can be null + */ private AugmentedSimulationResultsComputerInputs simulateNoResults( final Plan plan, final Duration until, - final SchedulingActivityDirective activity) throws SimulationException, SchedulingInterruptedException - { + final SchedulingActivityDirective activity) + throws SimulationException, SchedulingInterruptedException { final var planSimCorrespondence = scheduleFromPlan(plan, this.schedulerModel); final var best = CheckpointSimulationDriver.bestCachedEngine( @@ -197,7 +203,7 @@ private AugmentedSimulationResultsComputerInputs simulateNoResults( planningHorizon.getEndAerie()); CheckpointSimulationDriver.CachedSimulationEngine engine = null; Duration from = Duration.ZERO; - if(best.isPresent()){ + if (best.isPresent()) { engine = best.get().getKey(); replaceIds(planSimCorrespondence, best.get().getRight()); from = engine.endsAt(); @@ -209,19 +215,20 @@ private AugmentedSimulationResultsComputerInputs simulateNoResults( Function stoppingCondition; //(1) - if(until != null && activity == null){ + if (until != null && activity == null) { simulationDuration = until; stoppingCondition = CheckpointSimulationDriver.noCondition(); LOGGER.info("Simulation mode: until specific time " + simulationDuration); } //(2) - else if(activity != null && until == null){ + else if (activity != null && until == null) { simulationDuration = planningHorizon.getEndAerie(); stoppingCondition = CheckpointSimulationDriver.stopOnceActivityHasFinished( planSimCorrespondence.planActDirectiveIdToSimulationActivityDirectiveId().get(activity.id())); LOGGER.info("Simulation mode: until activity ends " + activity); + } //(3) - } else if(activity == null && until == null){ + else if (activity == null && until == null) { simulationDuration = planningHorizon.getEndAerie(); stoppingCondition = CheckpointSimulationDriver.onceAllActivitiesAreFinished(); LOGGER.info("Simulation mode: until all activities end "); @@ -229,22 +236,23 @@ else if(activity != null && until == null){ throw new SimulationException("Bad configuration", null); } - if(engine == null) engine = CheckpointSimulationDriver.CachedSimulationEngine.empty(missionModel); + if (engine == null) engine = CheckpointSimulationDriver.CachedSimulationEngine.empty(missionModel, planningHorizon.getStartInstant()); - Function checkpointPolicy = new ResourceAwareSpreadCheckpointPolicy( - cachedEngines.capacity(), - Duration.ZERO, - planningHorizon.getEndAerie(), - Duration.max(engine.endsAt(), Duration.ZERO), - simulationDuration, - 1, - true); + Function checkpointPolicy = + new ResourceAwareSpreadCheckpointPolicy( + cachedEngines.capacity(), + Duration.ZERO, + planningHorizon.getEndAerie(), + Duration.max(engine.endsAt(), Duration.ZERO), + simulationDuration, + 1, + true); - if(stoppingCondition.equals(CheckpointSimulationDriver.onceAllActivitiesAreFinished())){ + if (stoppingCondition.equals(CheckpointSimulationDriver.onceAllActivitiesAreFinished())) { checkpointPolicy = or(checkpointPolicy, onceAllActivitiesAreFinished()); } - if(best.isPresent()) cachedEngines.registerUsed(engine); + if (best.isPresent()) cachedEngines.registerUsed(engine); try { final var simulation = CheckpointSimulationDriver.simulateWithCheckpoints( missionModel, @@ -261,8 +269,8 @@ else if(activity != null && until == null){ cachedEngines, configuration ); - if(canceledListener.get()) throw new SchedulingInterruptedException("simulating"); this.totalSimulationTime = this.totalSimulationTime.plus(simulation.elapsedTime().minus(from)); + if (canceledListener.get()) throw new SchedulingInterruptedException("simulating"); final var activityResults = simulation.computeActivitySimulationResults(); updatePlanWithChildActivities( @@ -290,9 +298,9 @@ else if(activity != null && until == null){ private static Function or( final Function... functions) { - return (simulationState) -> { - for(final var function: functions){ - if(function.apply(simulationState)){ + return (simulationState) -> { + for (final var function : functions) { + if (function.apply(simulationState)) { return true; } } @@ -301,21 +309,21 @@ private static Function or( } - @Override - public SimulationData simulateWithResults( - final Plan plan, - final Duration until) throws SimulationException, SchedulingInterruptedException + @Override + public SimulationData simulateWithResults(final Plan plan, final Duration until) + throws SimulationException, SchedulingInterruptedException { return simulateWithResults(plan, until, missionModel.getResources().keySet()); } @Override public SimulationData simulateWithResults( - final Plan plan, - final Duration until, - final Set resourceNames) throws SimulationException, SchedulingInterruptedException + final Plan plan, + final Duration until, + final Set resourceNames + ) throws SimulationException, SchedulingInterruptedException { - if(this.initialSimulationResults != null) { + if (this.initialSimulationResults != null) { final var inputPlan = scheduleFromPlan(plan, schedulerModel); final var initialPlanA = scheduleFromPlan(this.initialSimulationResults.plan(), schedulerModel); if (initialPlanA.equals(inputPlan)) { @@ -336,5 +344,4 @@ public SimulationData simulateWithResults( public Optional getLatestSimulationData() { return Optional.ofNullable(this.latestSimulationData); } - } From 58d6ac405d82de209e1bd3cdf0e0de434fe81587 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 18:11:46 -0700 Subject: [PATCH 12/20] Remove Profile and ProfilingState --- .../aerie/merlin/driver/engine/Profile.java | 27 ------------------- .../merlin/driver/engine/ProfilingState.java | 21 --------------- 2 files changed, 48 deletions(-) delete mode 100644 merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/Profile.java delete mode 100644 merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/ProfilingState.java diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/Profile.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/Profile.java deleted file mode 100644 index df7ea415b3..0000000000 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/Profile.java +++ /dev/null @@ -1,27 +0,0 @@ -package gov.nasa.jpl.aerie.merlin.driver.engine; - -import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; - -import java.util.Iterator; - -/*package-local*/ record Profile(SlabList> segments) -implements Iterable> { - public record Segment(Duration startOffset, Dynamics dynamics) {} - - public Profile() { - this(new SlabList<>()); - } - - public void append(final Duration currentTime, final Dynamics dynamics) { - this.segments.append(new Segment<>(currentTime, dynamics)); - } - - @Override - public Iterator> iterator() { - return this.segments.iterator(); - } - - public Profile duplicate() { - return new Profile<>(segments.duplicate()); - } -} diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/ProfilingState.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/ProfilingState.java deleted file mode 100644 index d34294e329..0000000000 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/ProfilingState.java +++ /dev/null @@ -1,21 +0,0 @@ -package gov.nasa.jpl.aerie.merlin.driver.engine; - -import gov.nasa.jpl.aerie.merlin.protocol.driver.Querier; -import gov.nasa.jpl.aerie.merlin.protocol.model.Resource; -import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; - -/*package-local*/ -record ProfilingState (Resource resource, Profile profile) { - public static - ProfilingState create(final Resource resource) { - return new ProfilingState<>(resource, new Profile<>()); - } - - public void append(final Duration currentTime, final Querier querier) { - this.profile.append(currentTime, this.resource.getDynamics(querier)); - } - - public ProfilingState duplicate() { - return new ProfilingState<>(resource, profile.duplicate()); - } -} From 99d29abb30d217cc568a5405de70062ebf6f145d Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Mon, 15 Jul 2024 13:46:38 -0700 Subject: [PATCH 13/20] Extract CachedSimulationEngine to its own class --- .../FooSimulationDuplicationTest.java | 11 ++-- .../merlin/driver/CachedEngineStore.java | 4 +- .../merlin/driver/CachedSimulationEngine.java | 52 +++++++++++++++++++ .../driver/CheckpointSimulationDriver.java | 49 ++--------------- .../driver/SimulationDuplicationTest.java | 10 ++-- .../CheckpointSimulationFacade.java | 7 +-- .../simulation/InMemoryCachedEngineStore.java | 17 +++--- .../simulation/AnchorSchedulerTest.java | 3 +- .../InMemoryCachedEngineStoreTest.java | 20 +++---- 9 files changed, 90 insertions(+), 83 deletions(-) create mode 100644 merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CachedSimulationEngine.java diff --git a/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/FooSimulationDuplicationTest.java b/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/FooSimulationDuplicationTest.java index 7e0305b90b..65e92a7d33 100644 --- a/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/FooSimulationDuplicationTest.java +++ b/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/FooSimulationDuplicationTest.java @@ -4,6 +4,7 @@ import gov.nasa.jpl.aerie.merlin.driver.ActivityDirective; import gov.nasa.jpl.aerie.merlin.driver.ActivityDirectiveId; import gov.nasa.jpl.aerie.merlin.driver.CachedEngineStore; +import gov.nasa.jpl.aerie.merlin.driver.CachedSimulationEngine; import gov.nasa.jpl.aerie.merlin.driver.CheckpointSimulationDriver; import gov.nasa.jpl.aerie.merlin.driver.DirectiveTypeRegistry; import gov.nasa.jpl.aerie.merlin.driver.MissionModel; @@ -34,17 +35,17 @@ public class FooSimulationDuplicationTest { CachedEngineStore store; final private class InfiniteCapacityEngineStore implements CachedEngineStore{ - private final Map> store = new HashMap<>(); + private final Map> store = new HashMap<>(); @Override public void save( - final CheckpointSimulationDriver.CachedSimulationEngine cachedSimulationEngine, + final CachedSimulationEngine cachedSimulationEngine, final SimulationEngineConfiguration configuration) { store.computeIfAbsent(configuration, conf -> new ArrayList<>()); store.get(configuration).add(cachedSimulationEngine); } @Override - public List getCachedEngines(final SimulationEngineConfiguration configuration) { + public List getCachedEngines(final SimulationEngineConfiguration configuration) { return store.get(configuration); } @@ -374,7 +375,7 @@ static void assertResultsEqual(SimulationResults expected, SimulationResults act static SimulationResults simulateWithCheckpoints( final MissionModel missionModel, - final CheckpointSimulationDriver.CachedSimulationEngine cachedSimulationEngine, + final CachedSimulationEngine cachedSimulationEngine, final List desiredCheckpoints, final Map schedule, final CachedEngineStore cachedEngineStore, @@ -413,7 +414,7 @@ static SimulationResults simulateWithCheckpoints( Duration.HOUR, $ -> {}, () -> false, - CheckpointSimulationDriver.CachedSimulationEngine.empty(missionModel), + CachedSimulationEngine.empty(missionModel, Instant.EPOCH), CheckpointSimulationDriver.desiredCheckpoints(desiredCheckpoints), CheckpointSimulationDriver.noCondition(), cachedEngineStore, diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CachedEngineStore.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CachedEngineStore.java index c1a069d412..97739ec675 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CachedEngineStore.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CachedEngineStore.java @@ -3,9 +3,9 @@ import java.util.List; public interface CachedEngineStore { - void save(final CheckpointSimulationDriver.CachedSimulationEngine cachedSimulationEngine, + void save(final CachedSimulationEngine cachedSimulationEngine, final SimulationEngineConfiguration configuration); - List getCachedEngines( + List getCachedEngines( final SimulationEngineConfiguration configuration); int capacity(); diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CachedSimulationEngine.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CachedSimulationEngine.java new file mode 100644 index 0000000000..9b706ec425 --- /dev/null +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CachedSimulationEngine.java @@ -0,0 +1,52 @@ +package gov.nasa.jpl.aerie.merlin.driver; + +import gov.nasa.jpl.aerie.merlin.driver.engine.SimulationEngine; +import gov.nasa.jpl.aerie.merlin.driver.engine.SpanException; +import gov.nasa.jpl.aerie.merlin.driver.resources.InMemorySimulationResourceManager; +import gov.nasa.jpl.aerie.merlin.protocol.driver.Topic; +import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; + +import java.time.Instant; +import java.util.Map; + +public record CachedSimulationEngine( + Duration endsAt, + Map activityDirectives, + SimulationEngine simulationEngine, + Topic activityTopic, + MissionModel missionModel, + InMemorySimulationResourceManager resourceManager + ) { + public void freeze() { + simulationEngine.close(); + } + + public static CachedSimulationEngine empty(final MissionModel missionModel, final Instant simulationStartTime) { + final SimulationEngine engine = new SimulationEngine(missionModel.getInitialCells()); + + // Specify a topic on which tasks can log the activity they're associated with. + final var activityTopic = new Topic(); + try { + engine.init(missionModel.getResources(), missionModel.getDaemon()); + + return new CachedSimulationEngine( + Duration.MIN_VALUE, + Map.of(), + engine, + new Topic<>(), + missionModel, + new InMemorySimulationResourceManager() + ); + } catch (SpanException ex) { + // Swallowing the spanException as the internal `spanId` is not user meaningful info. + final var topics = missionModel.getTopics(); + final var directiveId = engine.getDirectiveIdFromSpan(activityTopic, topics, ex.spanId); + if (directiveId.isPresent()) { + throw new SimulationException(Duration.ZERO, simulationStartTime, directiveId.get(), ex.cause); + } + throw new SimulationException(Duration.ZERO, simulationStartTime, ex.cause); + } catch (Throwable ex) { + throw new SimulationException(Duration.ZERO, simulationStartTime, ex); + } + } +} diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java index 7e9c30a7fa..25b8731dcf 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java @@ -33,49 +33,6 @@ public class CheckpointSimulationDriver { private static final Logger LOGGER = LoggerFactory.getLogger(CheckpointSimulationDriver.class); - public record CachedSimulationEngine( - Duration endsAt, - Map activityDirectives, - SimulationEngine simulationEngine, - Topic activityTopic, - MissionModel missionModel - ) { - public void freeze() { - simulationEngine.close(); - } - - public static CachedSimulationEngine empty(final MissionModel missionModel) { - final SimulationEngine engine = new SimulationEngine(); - - // Begin tracking all resources. - for (final var entry : missionModel.getResources().entrySet()) { - final var name = entry.getKey(); - final var resource = entry.getValue(); - - engine.trackResource(name, resource, Duration.ZERO); - } - - { - // Start daemon task(s) immediately, before anything else happens. - engine.scheduleTask(Duration.ZERO, missionModel.getDaemon()); - { - final var batch = engine.extractNextJobs(Duration.MAX_VALUE); - final var commit = engine.performJobs(batch.jobs(), cells, Duration.ZERO, Duration.MAX_VALUE); - timeline.add(commit.getKey()); - } - } - return new CachedSimulationEngine( - Duration.MIN_VALUE, - Map.of(), - engine, - cells, - timeline.points(), - new Topic<>(), - missionModel - ); - } - } - /** * Selects the best cached engine for simulating a given plan. * @param schedule the schedule/plan @@ -221,9 +178,9 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( final boolean duplicationIsOk = cachedEngineStore.capacity() > 1; final var activityToSpan = new HashMap(); final var activityTopic = cachedEngine.activityTopic(); - var engine = duplicationIsOk ? cachedEngine.simulationEngine.duplicate() : cachedEngine.simulationEngine; - final var resourceManager = duplicationIsOk ? new InMemorySimulationResourceManager(cachedEngine.resourceManager) : cachedEngine.resourceManager; - engine.unscheduleAfter(cachedEngine.endsAt); + var engine = duplicationIsOk ? cachedEngine.simulationEngine().duplicate() : cachedEngine.simulationEngine(); + final var resourceManager = duplicationIsOk ? new InMemorySimulationResourceManager(cachedEngine.resourceManager()) : cachedEngine.resourceManager(); + engine.unscheduleAfter(cachedEngine.endsAt()); /* The current real time. */ var elapsedTime = Duration.max(ZERO, cachedEngine.endsAt()); diff --git a/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDuplicationTest.java b/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDuplicationTest.java index f7f28361e5..ed29cdf857 100644 --- a/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDuplicationTest.java +++ b/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDuplicationTest.java @@ -16,10 +16,10 @@ public class SimulationDuplicationTest { CachedEngineStore store; final private class InfiniteCapacityEngineStore implements CachedEngineStore{ - private Map> store = new HashMap<>(); + private Map> store = new HashMap<>(); @Override public void save( - final CheckpointSimulationDriver.CachedSimulationEngine cachedSimulationEngine, + final CachedSimulationEngine cachedSimulationEngine, final SimulationEngineConfiguration configuration) { store.computeIfAbsent(configuration, conf -> new ArrayList<>()); @@ -27,7 +27,7 @@ public void save( } @Override - public List getCachedEngines(final SimulationEngineConfiguration configuration) { + public List getCachedEngines(final SimulationEngineConfiguration configuration) { return store.get(configuration); } @@ -52,7 +52,7 @@ void beforeEach(){ @Test void testDuplicate() { final var results = simulateWithCheckpoints( - CheckpointSimulationDriver.CachedSimulationEngine.empty(TestMissionModel.missionModel()), + CachedSimulationEngine.empty(TestMissionModel.missionModel(), Instant.EPOCH), List.of(Duration.of(5, MINUTES)), store); final SimulationResults expected = SimulationDriver.simulate( @@ -70,7 +70,7 @@ void testDuplicate() { } static SimulationResults simulateWithCheckpoints( - final CheckpointSimulationDriver.CachedSimulationEngine cachedEngine, + final CachedSimulationEngine cachedEngine, final List desiredCheckpoints, final CachedEngineStore engineStore ) { diff --git a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacade.java b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacade.java index 20ce254408..d779ceed27 100644 --- a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacade.java +++ b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacade.java @@ -2,6 +2,7 @@ import gov.nasa.jpl.aerie.merlin.driver.ActivityDirective; import gov.nasa.jpl.aerie.merlin.driver.ActivityDirectiveId; +import gov.nasa.jpl.aerie.merlin.driver.CachedSimulationEngine; import gov.nasa.jpl.aerie.merlin.driver.CheckpointSimulationDriver; import gov.nasa.jpl.aerie.merlin.driver.MissionModel; import gov.nasa.jpl.aerie.merlin.driver.MissionModelId; @@ -201,7 +202,7 @@ private AugmentedSimulationResultsComputerInputs simulateNoResults( planSimCorrespondence.directiveIdActivityDirectiveMap(), cachedEngines.getCachedEngines(configuration), planningHorizon.getEndAerie()); - CheckpointSimulationDriver.CachedSimulationEngine engine = null; + CachedSimulationEngine engine = null; Duration from = Duration.ZERO; if (best.isPresent()) { engine = best.get().getKey(); @@ -236,7 +237,7 @@ else if (activity == null && until == null) { throw new SimulationException("Bad configuration", null); } - if (engine == null) engine = CheckpointSimulationDriver.CachedSimulationEngine.empty(missionModel, planningHorizon.getStartInstant()); + if (engine == null) engine = CachedSimulationEngine.empty(missionModel, planningHorizon.getStartInstant()); Function checkpointPolicy = new ResourceAwareSpreadCheckpointPolicy( @@ -269,7 +270,7 @@ else if (activity == null && until == null) { cachedEngines, configuration ); - this.totalSimulationTime = this.totalSimulationTime.plus(simulation.elapsedTime().minus(from)); + this.totalSimulationTime = this.totalSimulationTime.plus(simulation.engine().getElapsedTime().minus(from)); if (canceledListener.get()) throw new SchedulingInterruptedException("simulating"); final var activityResults = simulation.computeActivitySimulationResults(); diff --git a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStore.java b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStore.java index 9c565bb38f..68aa703a3f 100644 --- a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStore.java +++ b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStore.java @@ -1,11 +1,6 @@ package gov.nasa.jpl.aerie.scheduler.simulation; -import gov.nasa.jpl.aerie.merlin.driver.CachedEngineStore; -import gov.nasa.jpl.aerie.merlin.driver.CheckpointSimulationDriver; -import gov.nasa.jpl.aerie.merlin.driver.SimulationDriver; -import gov.nasa.jpl.aerie.merlin.driver.MissionModel; -import gov.nasa.jpl.aerie.merlin.driver.SimulationDriver; -import gov.nasa.jpl.aerie.merlin.driver.SimulationEngineConfiguration; +import gov.nasa.jpl.aerie.merlin.driver.*; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; import org.apache.commons.collections4.map.ListOrderedMap; @@ -23,7 +18,7 @@ private record CachedEngineMetadata( Instant creationDate){} private static final Logger LOGGER = LoggerFactory.getLogger(InMemoryCachedEngineStore.class); - private final ListOrderedMap cachedEngines; + private final ListOrderedMap cachedEngines; private final int capacity; private Duration savedSimulationTime; @@ -52,7 +47,7 @@ public void close() { * Register a re-use for a saved cached simulation engine. Will decrease likelihood of this engine being deleted. * @param cachedSimulationEngine the simulation engine */ - public void registerUsed(final CheckpointSimulationDriver.CachedSimulationEngine cachedSimulationEngine){ + public void registerUsed(final CachedSimulationEngine cachedSimulationEngine){ final var engineMetadata = this.cachedEngines.remove(cachedSimulationEngine); if(engineMetadata != null){ this.cachedEngines.put(0, cachedSimulationEngine, engineMetadata); @@ -61,7 +56,7 @@ public void registerUsed(final CheckpointSimulationDriver.CachedSimulationEngine } public void save( - final CheckpointSimulationDriver.CachedSimulationEngine engine, + final CachedSimulationEngine engine, final SimulationEngineConfiguration configuration) { if (shouldWeSave(engine, configuration)) { if (cachedEngines.size() + 1 > capacity) { @@ -78,7 +73,7 @@ public int capacity(){ return capacity; } - public List getCachedEngines( + public List getCachedEngines( final SimulationEngineConfiguration configuration){ return cachedEngines .entrySet() @@ -100,7 +95,7 @@ public Optional> getMissionModel( return Optional.empty(); } - private boolean shouldWeSave(final CheckpointSimulationDriver.CachedSimulationEngine engine, + private boolean shouldWeSave(final CachedSimulationEngine engine, final SimulationEngineConfiguration configuration){ //avoid duplicates for(final var cached: cachedEngines.entrySet()){ diff --git a/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/AnchorSchedulerTest.java b/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/AnchorSchedulerTest.java index 8a75f8790f..3f557e3dbd 100644 --- a/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/AnchorSchedulerTest.java +++ b/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/AnchorSchedulerTest.java @@ -2,6 +2,7 @@ import gov.nasa.jpl.aerie.merlin.driver.ActivityDirective; import gov.nasa.jpl.aerie.merlin.driver.ActivityDirectiveId; +import gov.nasa.jpl.aerie.merlin.driver.CachedSimulationEngine; import gov.nasa.jpl.aerie.merlin.driver.CheckpointSimulationDriver; import gov.nasa.jpl.aerie.merlin.driver.DirectiveTypeRegistry; import gov.nasa.jpl.aerie.merlin.driver.MissionModel; @@ -70,7 +71,7 @@ public SimulationResultsComputerInputs simulateActivities( final Map {}, () -> false, - CheckpointSimulationDriver.CachedSimulationEngine.empty(AnchorTestModel), + CachedSimulationEngine.empty(AnchorTestModel, planStart), (a) -> false, CheckpointSimulationDriver.onceAllActivitiesAreFinished(), new InMemoryCachedEngineStore(1), diff --git a/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStoreTest.java b/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStoreTest.java index 52d6afa6d4..b2b48a345f 100644 --- a/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStoreTest.java +++ b/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStoreTest.java @@ -2,7 +2,7 @@ import gov.nasa.jpl.aerie.merlin.driver.ActivityDirective; import gov.nasa.jpl.aerie.merlin.driver.ActivityDirectiveId; -import gov.nasa.jpl.aerie.merlin.driver.CheckpointSimulationDriver; +import gov.nasa.jpl.aerie.merlin.driver.CachedSimulationEngine; import gov.nasa.jpl.aerie.merlin.driver.SimulationEngineConfiguration; import gov.nasa.jpl.aerie.merlin.driver.engine.SimulationEngine; import gov.nasa.jpl.aerie.merlin.driver.engine.SlabList; @@ -37,8 +37,8 @@ void afterEach() { this.store.close(); } - public static CheckpointSimulationDriver.CachedSimulationEngine getCachedEngine1(){ - return new CheckpointSimulationDriver.CachedSimulationEngine( + public static CachedSimulationEngine getCachedEngine1(){ + return new CachedSimulationEngine( Duration.SECOND, Map.of( new ActivityDirectiveId(1), new ActivityDirective(Duration.HOUR, "ActivityType1", Map.of(), null, true), @@ -52,8 +52,8 @@ public static CheckpointSimulationDriver.CachedSimulationEngine getCachedEngine1 ); } - public static CheckpointSimulationDriver.CachedSimulationEngine getCachedEngine2(){ - return new CheckpointSimulationDriver.CachedSimulationEngine( + public static CachedSimulationEngine getCachedEngine2(){ + return new CachedSimulationEngine( Duration.SECOND, Map.of( new ActivityDirectiveId(3), new ActivityDirective(Duration.HOUR, "ActivityType3", Map.of(), null, true), @@ -67,8 +67,8 @@ public static CheckpointSimulationDriver.CachedSimulationEngine getCachedEngine2 ); } - public static CheckpointSimulationDriver.CachedSimulationEngine getCachedEngine3(){ - return new CheckpointSimulationDriver.CachedSimulationEngine( + public static CachedSimulationEngine getCachedEngine3(){ + return new CachedSimulationEngine( Duration.SECOND, Map.of( new ActivityDirectiveId(5), new ActivityDirective(Duration.HOUR, "ActivityType5", Map.of(), null, true), @@ -85,9 +85,9 @@ public static CheckpointSimulationDriver.CachedSimulationEngine getCachedEngine3 @Test public void duplicateTest(){ final var store = new InMemoryCachedEngineStore(2); - store.save(CheckpointSimulationDriver.CachedSimulationEngine.empty(SimulationUtility.getFooMissionModel()), this.simulationEngineConfiguration); - store.save(CheckpointSimulationDriver.CachedSimulationEngine.empty(SimulationUtility.getFooMissionModel()), this.simulationEngineConfiguration); - store.save(CheckpointSimulationDriver.CachedSimulationEngine.empty(SimulationUtility.getFooMissionModel()), this.simulationEngineConfiguration); + store.save(CachedSimulationEngine.empty(SimulationUtility.getFooMissionModel(), this.simulationEngineConfiguration.simStartTime()), this.simulationEngineConfiguration); + store.save(CachedSimulationEngine.empty(SimulationUtility.getFooMissionModel(), this.simulationEngineConfiguration.simStartTime()), this.simulationEngineConfiguration); + store.save(CachedSimulationEngine.empty(SimulationUtility.getFooMissionModel(), this.simulationEngineConfiguration.simStartTime()), this.simulationEngineConfiguration); assertEquals(1, store.getCachedEngines(this.simulationEngineConfiguration).size()); } From eb90fa384222a5525148c198da31de673d369176 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 18:21:21 -0700 Subject: [PATCH 14/20] Update CheckpointSimulationDriver --- .../driver/CheckpointSimulationDriver.java | 122 +++++++++--------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java index 25b8731dcf..60b5a091c0 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java @@ -189,7 +189,6 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( try { // Get all activities as close as possible to absolute time - // Schedule all activities. // Using HashMap explicitly because it allows `null` as a key. // `null` key means that an activity is not waiting on another activity to finish to know its start time HashMap>> resolved = new StartOffsetReducer( @@ -207,15 +206,14 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( // Filter out activities that are before simulationStartTime resolved = StartOffsetReducer.filterOutStartOffsetBefore( resolved, - Duration.max( - ZERO, - cachedEngine.endsAt().plus(MICROSECONDS))); + Duration.max(ZERO, cachedEngine.endsAt().plus(MICROSECONDS))); + + // Schedule all activities. final var toSchedule = new LinkedHashSet(); toSchedule.add(null); - final HashMap>> finalResolved = resolved; final var activitiesToBeScheduledNow = new HashMap(); - if (finalResolved.get(null) != null) { - for (final var r : finalResolved.get(null)) { + if (resolved.get(null) != null) { + for (final var r : resolved.get(null)) { activitiesToBeScheduledNow.put(r.getKey(), schedule.get(r.getKey())); } } @@ -231,80 +229,82 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( // Drive the engine until we're out of time. // TERMINATION: Actually, we might never break if real time never progresses forward. - while (elapsedTime.noLongerThan(simulationDuration) && !simulationCanceled.get()) { + engineLoop: + while (!simulationCanceled.get()) { final var nextTime = engine.peekNextTime().orElse(Duration.MAX_VALUE); - if (duplicationIsOk && shouldTakeCheckpoint.apply(new SimulationState(elapsedTime, nextTime, engine, schedule, activityToSpan))) { - cells.freeze(); - LOGGER.info("Saving a simulation engine in memory at time " + elapsedTime + " (next time: " + nextTime + ")"); - final var newCachedEngine = new CachedSimulationEngine( - elapsedTime, - schedule, - engine, - cells, - makeCombinedTimeline(timelines, timeline).points(), - activityTopic, - missionModel); - newCachedEngine.freeze(); - cachedEngineStore.save( - newCachedEngine, - configuration); - timelines.add(timeline); - engine = engine.duplicate(); - timeline = new TemporalEventSource(); - cells = new LiveCells(timeline, cells); - } - //break before changing the state of the engine - if (simulationCanceled.get() || stopConditionOnPlan.apply(new SimulationState(elapsedTime, nextTime, engine, schedule, activityToSpan))) { - if(!duplicationIsOk){ + if (duplicationIsOk && shouldTakeCheckpoint.apply(new SimulationState( + elapsedTime, + nextTime, + engine, + schedule, + activityToSpan)) + ) { + LOGGER.info("Saving a simulation engine in memory at time " + + elapsedTime + + " (next time: " + + nextTime + + ")"); + final var newCachedEngine = new CachedSimulationEngine( elapsedTime, schedule, engine, activityTopic, - missionModel); + missionModel, + new InMemorySimulationResourceManager(resourceManager) + ); + + newCachedEngine.freeze(); cachedEngineStore.save( newCachedEngine, configuration); - } - break; - } - final var batch = engine.extractNextJobs(simulationDuration); - // Increment real time, if necessary. - final var delta = batch.offsetFromStart().minus(elapsedTime); - elapsedTime = batch.offsetFromStart(); - timeline.add(delta); - // TODO: Advance a dense time counter so that future tasks are strictly ordered relative to these, - // even if they occur at the same real time. + engine = engine.duplicate(); + } - simulationExtentConsumer.accept(elapsedTime); + //break before changing the state of the engine + if (simulationCanceled.get()) break; - //this break depends on the state of the batch: this is the soonest we can exit for that reason - if (batch.jobs().isEmpty() && (batch.offsetFromStart().isEqualTo(simulationDuration))) { + if (stopConditionOnPlan.apply(new SimulationState(elapsedTime, nextTime, engine, schedule, activityToSpan))) { + if (!duplicationIsOk) { + final var newCachedEngine = new CachedSimulationEngine( + elapsedTime, + schedule, + engine, + activityTopic, + missionModel, + resourceManager); + cachedEngineStore.save( + newCachedEngine, + configuration); + } break; } - // Run the jobs in this batch. - final var commit = engine.performJobs(batch.jobs(), cells, elapsedTime, simulationDuration); - timeline.add(commit.getLeft()); - if (commit.getRight().isPresent()) { - throw commit.getRight().get(); + final var status = engine.step(simulationDuration); + switch (status) { + case SimulationEngine.Status.NoJobs noJobs: break engineLoop; + case SimulationEngine.Status.AtDuration atDuration: break engineLoop; + case SimulationEngine.Status.Nominal nominal: + elapsedTime = nominal.elapsedTime(); + resourceManager.acceptUpdates(elapsedTime, nominal.realResourceUpdates(), nominal.dynamicResourceUpdates()); + toCheckForDependencyScheduling.putAll(scheduleActivities( + getSuccessorsToSchedule(engine, toCheckForDependencyScheduling), + schedule, + resolved, + missionModel, + engine, + elapsedTime, + activityToSpan, + activityTopic)); + break; } - - toCheckForDependencyScheduling.putAll(scheduleActivities( - getSuccessorsToSchedule(engine, toCheckForDependencyScheduling), - schedule, - resolved, - missionModel, - engine, - elapsedTime, - activityToSpan, - activityTopic)); + simulationExtentConsumer.accept(elapsedTime); } } catch (SpanException ex) { // Swallowing the spanException as the internal `spanId` is not user meaningful info. final var topics = missionModel.getTopics(); - final var directiveId = SimulationEngine.getDirectiveIdFromSpan(engine, activityTopic, timeline, topics, ex.spanId); + final var directiveId = engine.getDirectiveIdFromSpan(activityTopic, topics, ex.spanId); if (directiveId.isPresent()) { throw new SimulationException(elapsedTime, simulationStartTime, directiveId.get(), ex.cause); } From 1dd40fde325981e10202cc0505e0a4e58a8c8f45 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 12:52:36 -0700 Subject: [PATCH 15/20] Create Profile Streamer - Remove posting resources as part of posting results --- .../PostgresResultsCellRepository.java | 2 - .../postgres/PostgresProfileStreamer.java | 180 ++++++++++++++++++ 2 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/postgres/PostgresProfileStreamer.java diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/PostgresResultsCellRepository.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/PostgresResultsCellRepository.java index 312200f0ed..60d23e6dc7 100644 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/PostgresResultsCellRepository.java +++ b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/remotes/postgres/PostgresResultsCellRepository.java @@ -366,8 +366,6 @@ private static void postSimulationResults( ) throws SQLException, NoSuchSimulationDatasetException { final var simulationStart = new Timestamp(results.startTime); - final var profileSet = ProfileSet.of(results.realProfiles, results.discreteProfiles); - ProfileRepository.postResourceProfiles(connection, datasetId, profileSet); postActivities(connection, datasetId, results.simulatedActivities, results.unfinishedActivities, simulationStart); insertSimulationTopics(connection, datasetId, results.topics); insertSimulationEvents(connection, datasetId, results.events, simulationStart); diff --git a/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/postgres/PostgresProfileStreamer.java b/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/postgres/PostgresProfileStreamer.java new file mode 100644 index 0000000000..b0ebb7d60b --- /dev/null +++ b/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/postgres/PostgresProfileStreamer.java @@ -0,0 +1,180 @@ +package gov.nasa.jpl.aerie.merlin.worker.postgres; + +import gov.nasa.jpl.aerie.json.JsonParser; +import gov.nasa.jpl.aerie.merlin.driver.resources.ResourceProfile; +import gov.nasa.jpl.aerie.merlin.driver.resources.ResourceProfiles; +import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; +import gov.nasa.jpl.aerie.merlin.protocol.types.RealDynamics; +import gov.nasa.jpl.aerie.merlin.protocol.types.SerializedValue; +import gov.nasa.jpl.aerie.merlin.server.remotes.postgres.DatabaseException; +import gov.nasa.jpl.aerie.merlin.server.remotes.postgres.FailedInsertException; +import gov.nasa.jpl.aerie.merlin.server.remotes.postgres.FailedUpdateException; +import gov.nasa.jpl.aerie.merlin.server.remotes.postgres.PreparedStatements; +import org.apache.commons.lang3.tuple.Pair; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.function.Consumer; + +import static gov.nasa.jpl.aerie.merlin.server.remotes.postgres.PostgresParsers.discreteProfileTypeP; +import static gov.nasa.jpl.aerie.merlin.server.remotes.postgres.PostgresParsers.realProfileTypeP; + +import static gov.nasa.jpl.aerie.merlin.driver.json.SerializedValueJsonParser.serializedValueP; +import static gov.nasa.jpl.aerie.merlin.server.http.ProfileParsers.realDynamicsP; + +public class PostgresProfileStreamer implements Consumer, AutoCloseable { + private final Connection connection; + private final HashMap profileIds; + private final HashMap profileDurations; + + private final PreparedStatement postProfileStatement; + private final PreparedStatement postSegmentsStatement; + private final PreparedStatement updateDurationStatement; + + public PostgresProfileStreamer(DataSource dataSource, long datasetId) throws SQLException { + this.connection = dataSource.getConnection(); + profileIds = new HashMap<>(); + profileDurations = new HashMap<>(); + + final String postProfilesSql = + //language=sql + """ + insert into merlin.profile (dataset_id, name, type, duration) + values (%d, ?, ?::jsonb, ?::interval) + on conflict (dataset_id, name) do nothing + """.formatted(datasetId); + final String postSegmentsSql = + //language=sql + """ + insert into merlin.profile_segment (dataset_id, profile_id, start_offset, dynamics, is_gap) + values (%d, ?, ?::interval, ?::jsonb, false) + """.formatted(datasetId); + final String updateDurationSql = + //language=SQL + """ + update merlin.profile + set duration = ?::interval + where (dataset_id, id) = (%d, ?); + """.formatted(datasetId); + + postProfileStatement = connection.prepareStatement(postProfilesSql, PreparedStatement.RETURN_GENERATED_KEYS); + postSegmentsStatement = connection.prepareStatement(postSegmentsSql, PreparedStatement.NO_GENERATED_KEYS); + updateDurationStatement = connection.prepareStatement(updateDurationSql, PreparedStatement.NO_GENERATED_KEYS); + } + + @Override + public void accept(final ResourceProfiles resourceProfiles) { + try { + // Add new profiles to DB + for(final var realEntry : resourceProfiles.realProfiles().entrySet()){ + if(!profileIds.containsKey(realEntry.getKey())){ + addRealProfileToBatch(realEntry.getKey(), realEntry.getValue()); + } + } + for(final var discreteEntry : resourceProfiles.discreteProfiles().entrySet()) { + if(!profileIds.containsKey(discreteEntry.getKey())){ + addDiscreteProfileToBatch(discreteEntry.getKey(), discreteEntry.getValue()); + } + } + postProfiles(); + + // Post Segments + for(final var realEntry : resourceProfiles.realProfiles().entrySet()){ + addProfileSegmentsToBatch(realEntry.getKey(), realEntry.getValue(), realDynamicsP); + } + for(final var discreteEntry : resourceProfiles.discreteProfiles().entrySet()) { + addProfileSegmentsToBatch(discreteEntry.getKey(), discreteEntry.getValue(), serializedValueP); + } + + postProfileSegments(); + updateProfileDurations(); + } catch (SQLException ex) { + throw new DatabaseException("Exception occurred while posting profiles.", ex); + } + } + + private void addRealProfileToBatch(final String name, ResourceProfile profile) throws SQLException { + postProfileStatement.setString(1, name); + postProfileStatement.setString(2, realProfileTypeP.unparse(Pair.of("real", profile.schema())).toString()); + PreparedStatements.setDuration(this.postProfileStatement, 3, Duration.ZERO); + + postProfileStatement.addBatch(); + + profileDurations.put(name, Duration.ZERO); + } + + private void addDiscreteProfileToBatch(final String name, ResourceProfile profile) throws SQLException { + postProfileStatement.setString(1, name); + postProfileStatement.setString(2, discreteProfileTypeP.unparse(Pair.of("discrete", profile.schema())).toString()); + PreparedStatements.setDuration(this.postProfileStatement, 3, Duration.ZERO); + + postProfileStatement.addBatch(); + + profileDurations.put(name, Duration.ZERO); + } + + /** + * Insert the batched profiles and cache their ids for future use. + * + * This method takes advantage of the fact that we're using the Postgres JDBC, + * which returns all columns when executing batches with `getGeneratedKeys`. + */ + private void postProfiles() throws SQLException { + final var results = this.postProfileStatement.executeBatch(); + for (final var result : results) { + if (result == Statement.EXECUTE_FAILED) throw new FailedInsertException("merlin.profile_segment"); + } + + final var resultSet = this.postProfileStatement.getGeneratedKeys(); + while(resultSet.next()){ + profileIds.put(resultSet.getString("name"), resultSet.getInt("id")); + } + } + + private void postProfileSegments() throws SQLException { + final var results = this.postSegmentsStatement.executeBatch(); + for (final var result : results) { + if (result == Statement.EXECUTE_FAILED) throw new FailedInsertException("merlin.profile_segment"); + } + } + + private void updateProfileDurations() throws SQLException { + final var results = this.updateDurationStatement.executeBatch(); + for (final var result : results) { + if (result == Statement.EXECUTE_FAILED) throw new FailedUpdateException("merlin.profile"); + } + } + + private void addProfileSegmentsToBatch(final String name, ResourceProfile profile, JsonParser dynamicsP) throws SQLException { + final var id = profileIds.get(name); + this.postSegmentsStatement.setLong(1, id); + + var newDuration = profileDurations.get(name); + for (final var segment : profile.segments()) { + PreparedStatements.setDuration(this.postSegmentsStatement, 2, newDuration); + final var dynamics = dynamicsP.unparse(segment.dynamics()).toString(); + this.postSegmentsStatement.setString(3, dynamics); + this.postSegmentsStatement.addBatch(); + + newDuration = newDuration.plus(segment.extent()); + } + + this.updateDurationStatement.setLong(2, id); + PreparedStatements.setDuration(this.updateDurationStatement, 1, newDuration); + this.updateDurationStatement.addBatch(); + + profileDurations.put(name, newDuration); + } + + @Override + public void close() throws SQLException { + this.postProfileStatement.close(); + this.postSegmentsStatement.close(); + this.updateDurationStatement.close(); + this.connection.close(); + } +} From 7e8e11a605d9a57ff589912fed0ba99850ae2f85 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Wed, 10 Jul 2024 10:03:12 -0700 Subject: [PATCH 16/20] Remove unused implementations and flatten SimulationAgent - add argument for the resource manager - remove unused interface "Response" --- .../server/services/SimulationAgent.java | 122 ++++++++++++++++- .../services/SynchronousSimulationAgent.java | 127 ------------------ .../services/ThreadedSimulationAgent.java | 95 ------------- .../services/UncachedSimulationService.java | 52 ------- .../merlin/worker/MerlinWorkerAppDriver.java | 15 ++- 5 files changed, 131 insertions(+), 280 deletions(-) delete mode 100644 merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/SynchronousSimulationAgent.java delete mode 100644 merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/ThreadedSimulationAgent.java delete mode 100644 merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/UncachedSimulationService.java diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/SimulationAgent.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/SimulationAgent.java index 742a891443..0023d6a04e 100644 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/SimulationAgent.java +++ b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/SimulationAgent.java @@ -1,10 +1,128 @@ package gov.nasa.jpl.aerie.merlin.server.services; +import gov.nasa.jpl.aerie.merlin.driver.SimulationException; +import gov.nasa.jpl.aerie.merlin.driver.SimulationResults; +import gov.nasa.jpl.aerie.merlin.driver.resources.SimulationResourceManager; +import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; import gov.nasa.jpl.aerie.merlin.server.ResultsProtocol; +import gov.nasa.jpl.aerie.merlin.server.exceptions.NoSuchPlanException; +import gov.nasa.jpl.aerie.merlin.server.http.ResponseSerializers; +import gov.nasa.jpl.aerie.merlin.server.models.Plan; import gov.nasa.jpl.aerie.merlin.server.models.PlanId; +import javax.json.Json; +import java.time.temporal.ChronoUnit; +import java.util.Map; import java.util.function.Supplier; +import java.util.stream.Collectors; -public interface SimulationAgent { - void simulate(PlanId planId, RevisionData revisionData, ResultsProtocol.WriterRole writer, Supplier canceledListener) throws InterruptedException; +public record SimulationAgent ( + PlanService planService, + MissionModelService missionModelService, + long simulationProgressPollPeriod +) { + public void simulate( + final PlanId planId, + final RevisionData revisionData, + final ResultsProtocol.WriterRole writer, + final Supplier canceledListener, + final SimulationResourceManager resourceManager + ) { + final Plan plan; + try { + plan = this.planService.getPlanForSimulation(planId); + + // Validate plan revision + final var currentRevisionData = this.planService.getPlanRevisionData(planId); + final var validationResult = currentRevisionData.matches(revisionData); + if (validationResult instanceof RevisionData.MatchResult.Failure failure) { + writer.failWith(b -> b + .type("SIMULATION_REQUEST_NOT_RELEVANT") + .message("Simulation request no longer relevant: %s".formatted(failure.reason()))); + return; + } + } catch (final NoSuchPlanException ex) { + writer.failWith(b -> b + .type("NO_SUCH_PLAN") + .message(ex.toString()) + .data(ResponseSerializers.serializeNoSuchPlanException(ex)) + .trace(ex)); + return; + } + + final var planDuration = Duration.of( + plan.startTimestamp.toInstant().until(plan.endTimestamp.toInstant(), ChronoUnit.MICROS), + Duration.MICROSECONDS); + final var simDuration = Duration.of( + plan.simulationStartTimestamp.toInstant().until(plan.simulationEndTimestamp.toInstant(), ChronoUnit.MICROS), + Duration.MICROSECONDS); + + final SimulationResults results; + try { + // Validate plan activity construction + final var failures = this.missionModelService.validateActivityInstantiations( + plan.missionModelId, + plan.activityDirectives.entrySet().stream().collect(Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().serializedActivity()))); + + if (!failures.isEmpty()) { + writer.failWith(b -> b + .type("PLAN_CONTAINS_UNCONSTRUCTABLE_ACTIVITIES") + .message("Plan contains unconstructable activities") + .data(ResponseSerializers.serializeUnconstructableActivityFailures(failures))); + return; + } + + try (final var extentListener = FixedRateListener.callAtFixedRate( + writer::reportSimulationExtent, + Duration.ZERO, + simulationProgressPollPeriod) + ) { + results = this.missionModelService.runSimulation( + new CreateSimulationMessage( + plan.missionModelId, + plan.simulationStartTimestamp.toInstant(), + simDuration, + plan.startTimestamp.toInstant(), + planDuration, + plan.activityDirectives, + plan.configuration), + extentListener::updateValue, + canceledListener, + resourceManager); + } + } catch (SimulationException ex) { + final var errorMsgBuilder = Json.createObjectBuilder() + .add("elapsedTime", SimulationException.formatDuration(ex.elapsedTime)) + .add("utcTimeDoy", SimulationException.formatInstant(ex.instant)); + ex.directiveId.ifPresent(directiveId -> errorMsgBuilder.add("executingDirectiveId", directiveId.id())); + writer.failWith(b -> b + .type("SIMULATION_EXCEPTION") + .message(ex.cause.getMessage()) + .data(errorMsgBuilder.build()) + .trace(ex.cause)); + return; + } catch (final MissionModelService.NoSuchMissionModelException ex) { + writer.failWith(b -> b + .type("NO_SUCH_MISSION_MODEL") + .message(ex.toString()) + .data(ResponseSerializers.serializeNoSuchMissionModelException(ex)) + .trace(ex)); + return; + } catch (final MissionModelService.NoSuchActivityTypeException ex) { + writer.failWith(b -> b + .type("NO_SUCH_ACTIVITY_TYPE") + .message("Activity of type `%s` could not be instantiated".formatted(ex.activityTypeId)) + .data(ResponseSerializers.serializeNoSuchActivityTypeException(ex)) + .trace(ex)); + return; + } + + if(canceledListener.get()) { + writer.reportIncompleteResults(results); + } else { + writer.succeedWith(results); + } + } } diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/SynchronousSimulationAgent.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/SynchronousSimulationAgent.java deleted file mode 100644 index 18443082e0..0000000000 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/SynchronousSimulationAgent.java +++ /dev/null @@ -1,127 +0,0 @@ -package gov.nasa.jpl.aerie.merlin.server.services; - -import gov.nasa.jpl.aerie.merlin.driver.SimulationException; -import gov.nasa.jpl.aerie.merlin.driver.SimulationResults; -import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; -import gov.nasa.jpl.aerie.merlin.server.ResultsProtocol; -import gov.nasa.jpl.aerie.merlin.server.exceptions.NoSuchPlanException; -import gov.nasa.jpl.aerie.merlin.server.http.ResponseSerializers; -import gov.nasa.jpl.aerie.merlin.server.models.Plan; -import gov.nasa.jpl.aerie.merlin.server.models.PlanId; - -import javax.json.Json; -import java.time.temporal.ChronoUnit; -import java.util.Map; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -public record SynchronousSimulationAgent ( - PlanService planService, - MissionModelService missionModelService, - long simulationProgressPollPeriod -) implements SimulationAgent { - public sealed interface Response { - record Failed(String reason) implements Response {} - record Success(SimulationResults results) implements Response {} - } - - @Override - public void simulate( - final PlanId planId, - final RevisionData revisionData, - final ResultsProtocol.WriterRole writer, - final Supplier canceledListener) { - final Plan plan; - try { - plan = this.planService.getPlanForSimulation(planId); - - // Validate plan revision - final var currentRevisionData = this.planService.getPlanRevisionData(planId); - final var validationResult = currentRevisionData.matches(revisionData); - if (validationResult instanceof RevisionData.MatchResult.Failure failure) { - writer.failWith(b -> b - .type("SIMULATION_REQUEST_NOT_RELEVANT") - .message("Simulation request no longer relevant: %s".formatted(failure.reason()))); - return; - } - } catch (final NoSuchPlanException ex) { - writer.failWith(b -> b - .type("NO_SUCH_PLAN") - .message(ex.toString()) - .data(ResponseSerializers.serializeNoSuchPlanException(ex)) - .trace(ex)); - return; - } - - final var planDuration = Duration.of( - plan.startTimestamp.toInstant().until(plan.endTimestamp.toInstant(), ChronoUnit.MICROS), - Duration.MICROSECONDS); - final var simDuration = Duration.of( - plan.simulationStartTimestamp.toInstant().until(plan.simulationEndTimestamp.toInstant(), ChronoUnit.MICROS), - Duration.MICROSECONDS); - - final SimulationResults results; - try { - // Validate plan activity construction - final var failures = this.missionModelService.validateActivityInstantiations( - plan.missionModelId, - plan.activityDirectives.entrySet().stream().collect(Collectors.toMap( - Map.Entry::getKey, - e -> e.getValue().serializedActivity()))); - - if (!failures.isEmpty()) { - writer.failWith(b -> b - .type("PLAN_CONTAINS_UNCONSTRUCTABLE_ACTIVITIES") - .message("Plan contains unconstructable activities") - .data(ResponseSerializers.serializeUnconstructableActivityFailures(failures))); - return; - } - - try (final var extentListener = FixedRateListener.callAtFixedRate( - writer::reportSimulationExtent, - Duration.ZERO, - simulationProgressPollPeriod) - ) { - results = this.missionModelService.runSimulation(new CreateSimulationMessage( - plan.missionModelId, - plan.simulationStartTimestamp.toInstant(), - simDuration, - plan.startTimestamp.toInstant(), - planDuration, - plan.activityDirectives, - plan.configuration), extentListener::updateValue, canceledListener); - } - } catch (SimulationException ex) { - final var errorMsgBuilder = Json.createObjectBuilder() - .add("elapsedTime", SimulationException.formatDuration(ex.elapsedTime)) - .add("utcTimeDoy", SimulationException.formatInstant(ex.instant)); - ex.directiveId.ifPresent(directiveId -> errorMsgBuilder.add("executingDirectiveId", directiveId.id())); - writer.failWith(b -> b - .type("SIMULATION_EXCEPTION") - .message(ex.cause.getMessage()) - .data(errorMsgBuilder.build()) - .trace(ex.cause)); - return; - } catch (final MissionModelService.NoSuchMissionModelException ex) { - writer.failWith(b -> b - .type("NO_SUCH_MISSION_MODEL") - .message(ex.toString()) - .data(ResponseSerializers.serializeNoSuchMissionModelException(ex)) - .trace(ex)); - return; - } catch (final MissionModelService.NoSuchActivityTypeException ex) { - writer.failWith(b -> b - .type("NO_SUCH_ACTIVITY_TYPE") - .message("Activity of type `%s` could not be instantiated".formatted(ex.activityTypeId)) - .data(ResponseSerializers.serializeNoSuchActivityTypeException(ex)) - .trace(ex)); - return; - } - - if(canceledListener.get()) { - writer.reportIncompleteResults(results); - } else { - writer.succeedWith(results); - } - } -} diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/ThreadedSimulationAgent.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/ThreadedSimulationAgent.java deleted file mode 100644 index 178375375c..0000000000 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/ThreadedSimulationAgent.java +++ /dev/null @@ -1,95 +0,0 @@ -package gov.nasa.jpl.aerie.merlin.server.services; - -import gov.nasa.jpl.aerie.merlin.server.ResultsProtocol; -import gov.nasa.jpl.aerie.merlin.server.models.PlanId; - -import java.util.Objects; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.function.Supplier; - -public final class ThreadedSimulationAgent implements SimulationAgent { - private /*sealed*/ interface SimulationRequest { - record Simulate(PlanId planId, RevisionData revisionData, ResultsProtocol.WriterRole writer) implements SimulationRequest {} - - record Terminate() implements SimulationRequest {} - } - - - private final BlockingQueue requestQueue; - - private ThreadedSimulationAgent(final BlockingQueue requestQueue) { - this.requestQueue = Objects.requireNonNull(requestQueue); - } - - public static ThreadedSimulationAgent spawn(final String threadName, final SimulationAgent simulationAgent) { - final var requestQueue = new LinkedBlockingQueue(); - - final var thread = new Thread(new Worker(requestQueue, simulationAgent)); - thread.setName(threadName); - thread.start(); - - return new ThreadedSimulationAgent(requestQueue); - } - - @Override - public void simulate( - final PlanId planId, - final RevisionData revisionData, - final ResultsProtocol.WriterRole writer, - final Supplier canceledListener) - throws InterruptedException - { - this.requestQueue.put(new SimulationRequest.Simulate(planId, revisionData, writer)); - } - - public void terminate() throws InterruptedException { - this.requestQueue.put(new SimulationRequest.Terminate()); - } - - - private record Worker(BlockingQueue requestQueue, SimulationAgent simulationAgent) - implements Runnable - { - private Worker( - final BlockingQueue requestQueue, - final SimulationAgent simulationAgent) - { - this.requestQueue = Objects.requireNonNull(requestQueue); - this.simulationAgent = Objects.requireNonNull(simulationAgent); - } - - @Override - public void run() { - while (true) { - try { - final var request = this.requestQueue.take(); - - if (request instanceof SimulationRequest.Simulate req) { - try { - this.simulationAgent.simulate( - req.planId(), - req.revisionData(), - req.writer(), - () -> false); - } catch (final Throwable ex) { - ex.printStackTrace(System.err); - req.writer().failWith(b -> b - .type("UNEXPECTED_SIMULATION_EXCEPTION") - .message("Something went wrong while simulating") - .trace(ex)); - } - // continue - } else if (request instanceof SimulationRequest.Terminate) { - break; - } else { - throw new UnexpectedSubtypeError(SimulationRequest.class, request); - } - } catch (final Exception ex) { - ex.printStackTrace(System.err); - // continue - } - } - } - } -} diff --git a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/UncachedSimulationService.java b/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/UncachedSimulationService.java deleted file mode 100644 index 1350e35aa0..0000000000 --- a/merlin-server/src/main/java/gov/nasa/jpl/aerie/merlin/server/services/UncachedSimulationService.java +++ /dev/null @@ -1,52 +0,0 @@ -package gov.nasa.jpl.aerie.merlin.server.services; - -import gov.nasa.jpl.aerie.merlin.server.ResultsProtocol; -import gov.nasa.jpl.aerie.merlin.server.mocks.InMemoryRevisionData; -import gov.nasa.jpl.aerie.merlin.server.models.PlanId; -import gov.nasa.jpl.aerie.merlin.server.models.SimulationDatasetId; -import gov.nasa.jpl.aerie.merlin.server.models.SimulationResultsHandle; -import gov.nasa.jpl.aerie.merlin.server.remotes.InMemoryResultsCellRepository.InMemoryCell; - -import java.util.Optional; - -public record UncachedSimulationService ( - SimulationAgent agent -) implements SimulationService { - - @Override - public ResultsProtocol.State getSimulationResults(final PlanId planId, final boolean forceResim, final RevisionData revisionData, final String requestedBy) { - if (!(revisionData instanceof InMemoryRevisionData inMemoryRevisionData)) { - throw new Error("UncachedSimulationService only accepts InMemoryRevisionData"); - } - final var cell = new InMemoryCell(planId, inMemoryRevisionData.planRevision()); - - try { - this.agent.simulate(planId, revisionData, cell, () -> false); - } catch (final InterruptedException ex) { - // Do nothing. We'll get the current result (probably an Incomplete) - // and throw away the cell anyway. - - // TODO: Fail back to the common supervisor of this Service and the Agent. - // We don't know if the simulation request was successfully received or not, - // so the interaction between the two is now in an undefined state. - } - - final var result = cell.get(); - cell.cancel(); - - return result; - } - - @Override - public Optional get(final PlanId planId, final RevisionData revisionData) { - return Optional.ofNullable( - getSimulationResults(planId, false, revisionData, null) instanceof ResultsProtocol.State.Success s ? - s.results() : - null); - } - - @Override - public Optional get(PlanId planId, SimulationDatasetId simulationDatasetId) { - throw new UnsupportedOperationException(); // Cannot get a cached result from an uncached service - } -} diff --git a/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java b/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java index 9d4f66b77c..25773acb90 100644 --- a/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java +++ b/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java @@ -2,6 +2,7 @@ import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; +import gov.nasa.jpl.aerie.merlin.driver.resources.StreamingSimulationResourceManager; import gov.nasa.jpl.aerie.merlin.server.ResultsProtocol; import gov.nasa.jpl.aerie.merlin.server.config.PostgresStore; import gov.nasa.jpl.aerie.merlin.server.config.Store; @@ -13,8 +14,9 @@ import gov.nasa.jpl.aerie.merlin.server.remotes.postgres.PostgresResultsCellRepository; import gov.nasa.jpl.aerie.merlin.server.services.LocalMissionModelService; import gov.nasa.jpl.aerie.merlin.server.services.LocalPlanService; -import gov.nasa.jpl.aerie.merlin.server.services.SynchronousSimulationAgent; +import gov.nasa.jpl.aerie.merlin.server.services.SimulationAgent; import gov.nasa.jpl.aerie.merlin.server.services.UnexpectedSubtypeError; +import gov.nasa.jpl.aerie.merlin.worker.postgres.PostgresProfileStreamer; import gov.nasa.jpl.aerie.merlin.worker.postgres.PostgresSimulationNotificationPayload; import io.javalin.Javalin; @@ -57,7 +59,7 @@ public static void main(String[] args) throws InterruptedException { configuration.untruePlanStart() ); final var planController = new LocalPlanService(stores.plans()); - final var simulationAgent = new SynchronousSimulationAgent( + final var simulationAgent = new SimulationAgent( planController, missionModelController, configuration.simulationProgressPollPeriodMillis()); @@ -91,8 +93,13 @@ public static void main(String[] args) throws InterruptedException { notification.simulationRevision(), notification.simulationTemplateRevision()); final ResultsProtocol.WriterRole writer = owner.get(); - try { - simulationAgent.simulate(planId, revisionData, writer, canceledListener); + try(final var streamer = new PostgresProfileStreamer(hikariDataSource, datasetId)) { + simulationAgent.simulate( + planId, + revisionData, + writer, + canceledListener, + new StreamingSimulationResourceManager(streamer)); } catch (final Throwable ex) { ex.printStackTrace(System.err); writer.failWith(b -> b From 158693866ed3994737181925cd989f11842db3a1 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 17:03:36 -0700 Subject: [PATCH 17/20] Expose Engine's Elapsed Time --- .../merlin/driver/CheckpointSimulationDriver.java | 2 ++ .../jpl/aerie/merlin/driver/SimulationDriver.java | 14 ++++++-------- .../merlin/driver/engine/SimulationEngine.java | 6 +++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java index 60b5a091c0..ba8af75fff 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/CheckpointSimulationDriver.java @@ -302,6 +302,7 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( simulationExtentConsumer.accept(elapsedTime); } } catch (SpanException ex) { + elapsedTime = engine.getElapsedTime(); // Swallowing the spanException as the internal `spanId` is not user meaningful info. final var topics = missionModel.getTopics(); final var directiveId = engine.getDirectiveIdFromSpan(activityTopic, topics, ex.spanId); @@ -310,6 +311,7 @@ public static SimulationResultsComputerInputs simulateWithCheckpoints( } throw new SimulationException(elapsedTime, simulationStartTime, ex.cause); } catch (Throwable ex) { + elapsedTime = engine.getElapsedTime(); throw new SimulationException(elapsedTime, simulationStartTime, ex); } return new SimulationResultsComputerInputs( diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java index c1206a7c07..e92e748758 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDriver.java @@ -57,8 +57,7 @@ public static SimulationResults simulate( try (final var engine = new SimulationEngine(missionModel.getInitialCells())) { /* The current real time. */ - var elapsedTime = Duration.ZERO; - simulationExtentConsumer.accept(elapsedTime); + simulationExtentConsumer.accept(Duration.ZERO); // Specify a topic on which tasks can log the activity they're associated with. final var activityTopic = new Topic(); @@ -101,11 +100,10 @@ public static SimulationResults simulate( case SimulationEngine.Status.NoJobs noJobs: break engineLoop; case SimulationEngine.Status.AtDuration atDuration: break engineLoop; case SimulationEngine.Status.Nominal nominal: - elapsedTime = nominal.elapsedTime(); - resourceManager.acceptUpdates(elapsedTime, nominal.realResourceUpdates(), nominal.dynamicResourceUpdates()); + resourceManager.acceptUpdates(nominal.elapsedTime(), nominal.realResourceUpdates(), nominal.dynamicResourceUpdates()); break; } - simulationExtentConsumer.accept(elapsedTime); + simulationExtentConsumer.accept(engine.getElapsedTime()); } } catch (SpanException ex) { @@ -113,11 +111,11 @@ public static SimulationResults simulate( final var topics = missionModel.getTopics(); final var directiveId = engine.getDirectiveIdFromSpan(activityTopic, topics, ex.spanId); if(directiveId.isPresent()) { - throw new SimulationException(elapsedTime, simulationStartTime, directiveId.get(), ex.cause); + throw new SimulationException(engine.getElapsedTime(), simulationStartTime, directiveId.get(), ex.cause); } - throw new SimulationException(elapsedTime, simulationStartTime, ex.cause); + throw new SimulationException(engine.getElapsedTime(), simulationStartTime, ex.cause); } catch (Throwable ex) { - throw new SimulationException(elapsedTime, simulationStartTime, ex); + throw new SimulationException(engine.getElapsedTime(), simulationStartTime, ex); } final var topics = missionModel.getTopics(); diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java index 90fa01c84a..fbb8b6a4c9 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java @@ -184,6 +184,10 @@ record Nominal( ) implements Status {} } + public Duration getElapsedTime() { + return elapsedTime; + } + /** Step the engine forward one batch. **/ public Status step(Duration simulationDuration) throws Throwable { final var nextTime = this.peekNextTime().orElse(Duration.MAX_VALUE); @@ -313,7 +317,7 @@ public JobSchedule.Batch extractNextJobs(final Duration maximumTime) { return batch; } - public record ResourceUpdates(List> updates){ + public record ResourceUpdates(List> updates) { public boolean isEmpty() { return updates.isEmpty(); } From 966e77b35ee4b1c8fc37e6f8539de67618c9e018 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Tue, 23 Jul 2024 10:59:28 -0700 Subject: [PATCH 18/20] Extract Creating Serialized Timeline into method --- .../driver/engine/SimulationEngine.java | 134 +++++++----------- 1 file changed, 52 insertions(+), 82 deletions(-) diff --git a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java index fbb8b6a4c9..585012c67f 100644 --- a/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java +++ b/merlin-driver/src/main/java/gov/nasa/jpl/aerie/merlin/driver/engine/SimulationEngine.java @@ -842,39 +842,11 @@ public SimulationActivityExtract computeActivitySimulationResults( return new SimulationActivityExtract(startTime, elapsedTime, simulatedActivities, unfinishedActivities); } - - /** Compute a set of results from the current state of simulation. */ - // TODO: Move result extraction out of the SimulationEngine. - // The Engine should only need to stream events of interest to a downstream consumer. - // The Engine cannot be cognizant of all downstream needs. - // TODO: Whatever mechanism replaces `computeResults` also ought to replace `isTaskComplete`. - // TODO: Produce results for all tasks, not just those that have completed. - // Planners need to be aware of failed or unfinished tasks. - public SimulationResults computeResults ( - final Instant startTime, - final Topic activityTopic, + private TreeMap>> createSerializedTimeline( + final TemporalEventSource combinedTimeline, final Iterable> serializableTopics, - final SimulationResourceManager resourceManager - ) { - final var combinedTimeline = this.combineTimeline(); - // Collect per-task information from the event graph. - final var spanInfo = computeSpanInfo(activityTopic, serializableTopics, combinedTimeline); - - // Extract profiles for every resource. - final var resourceProfiles = resourceManager.computeProfiles(elapsedTime); - final var realProfiles = resourceProfiles.realProfiles(); - final var discreteProfiles = resourceProfiles.discreteProfiles(); - - final var activityResults = computeActivitySimulationResults(startTime, spanInfo); - - final List> topics = new ArrayList<>(); - final var serializableTopicToId = new HashMap, Integer>(); - for (final var serializableTopic : serializableTopics) { - serializableTopicToId.put(serializableTopic, topics.size()); - topics.add(Triple.of(topics.size(), serializableTopic.name(), serializableTopic.outputType().getSchema())); - } - - final var spanToActivities = spanToSimulatedActivities(spanInfo); + final HashMap spanToActivities, + final HashMap, Integer> serializableTopicToId) { final var serializedTimeline = new TreeMap>>(); var time = Duration.ZERO; for (var point : combinedTimeline.points()) { @@ -904,7 +876,7 @@ public SimulationResults computeResults ( } } } - var activitySpanID = Optional.ofNullable(spanToActivities.get(event.provenance()).id()); + var activitySpanID = Optional.of(spanToActivities.get(event.provenance()).id()); output = EventGraph.concurrently( output, EventGraph.atom( @@ -923,6 +895,47 @@ public SimulationResults computeResults ( } } } + return serializedTimeline; + } + + + /** Compute a set of results from the current state of simulation. */ + // TODO: Move result extraction out of the SimulationEngine. + // The Engine should only need to stream events of interest to a downstream consumer. + // The Engine cannot be cognizant of all downstream needs. + // TODO: Whatever mechanism replaces `computeResults` also ought to replace `isTaskComplete`. + // TODO: Produce results for all tasks, not just those that have completed. + // Planners need to be aware of failed or unfinished tasks. + public SimulationResults computeResults ( + final Instant startTime, + final Topic activityTopic, + final Iterable> serializableTopics, + final SimulationResourceManager resourceManager + ) { + final var combinedTimeline = this.combineTimeline(); + // Collect per-task information from the event graph. + final var spanInfo = computeSpanInfo(activityTopic, serializableTopics, combinedTimeline); + + // Extract profiles for every resource. + final var resourceProfiles = resourceManager.computeProfiles(elapsedTime); + final var realProfiles = resourceProfiles.realProfiles(); + final var discreteProfiles = resourceProfiles.discreteProfiles(); + + final var activityResults = computeActivitySimulationResults(startTime, spanInfo); + + final List> topics = new ArrayList<>(); + final var serializableTopicToId = new HashMap, Integer>(); + for (final var serializableTopic : serializableTopics) { + serializableTopicToId.put(serializableTopic, topics.size()); + topics.add(Triple.of(topics.size(), serializableTopic.name(), serializableTopic.outputType().getSchema())); + } + + final var serializedTimeline = createSerializedTimeline( + combinedTimeline, + serializableTopics, + spanToSimulatedActivities(spanInfo), + serializableTopicToId + ); return new SimulationResults( realProfiles, @@ -960,55 +973,12 @@ public SimulationResults computeResults( topics.add(Triple.of(topics.size(), serializableTopic.name(), serializableTopic.outputType().getSchema())); } - final var spanToActivities = spanToSimulatedActivities(spanInfo); - final var serializedTimeline = new TreeMap>>(); - var time = Duration.ZERO; - for (var point : combinedTimeline.points()) { - if (point instanceof TemporalEventSource.TimePoint.Delta delta) { - time = time.plus(delta.delta()); - } else if (point instanceof TemporalEventSource.TimePoint.Commit commit) { - final var serializedEventGraph = commit.events().substitute( - event -> { - // TODO can we do this more efficiently? - EventGraph output = EventGraph.empty(); - for (final var serializableTopic : serializableTopics) { - Optional serializedEvent = trySerializeEvent(event, serializableTopic); - if (serializedEvent.isPresent()) { - // If the event's `provenance` has no simulated activity id, search its ancestors to find the nearest - // simulated activity id, if one exists - if (!spanToActivities.containsKey(event.provenance())) { - var spanId = Optional.of(event.provenance()); - - while (true) { - if (spanToActivities.containsKey(spanId.get())) { - spanToActivities.put(event.provenance(), spanToActivities.get(spanId.get())); - break; - } - spanId = this.getSpan(spanId.get()).parent(); - if (spanId.isEmpty()) { - break; - } - } - } - var activitySpanID = Optional.ofNullable(spanToActivities.get(event.provenance()).id()); - output = EventGraph.concurrently( - output, - EventGraph.atom( - new EventRecord(serializableTopicToId.get(serializableTopic), - activitySpanID, - serializedEvent.get()))); - } - } - return output; - } - ).evaluate(new EventGraph.IdentityTrait<>(), EventGraph::atom); - if (!(serializedEventGraph instanceof EventGraph.Empty)) { - serializedTimeline - .computeIfAbsent(time, x -> new ArrayList<>()) - .add(serializedEventGraph); - } - } - } + final var serializedTimeline = createSerializedTimeline( + combinedTimeline, + serializableTopics, + spanToSimulatedActivities(spanInfo), + serializableTopicToId + ); return new SimulationResults( realProfiles, From b1fac9b277ab1cead226cf6e0a95a43e7d555470 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 17:05:00 -0700 Subject: [PATCH 19/20] Expand worker pool size Streaming resources requires a third connection to the DB --- .../gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java b/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java index 25773acb90..5b3abdc2a3 100644 --- a/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java +++ b/merlin-worker/src/main/java/gov/nasa/jpl/aerie/merlin/worker/MerlinWorkerAppDriver.java @@ -42,7 +42,7 @@ public static void main(String[] args) throws InterruptedException { hikariConfig.addDataSourceProperty("applicationName", "Merlin Server"); hikariConfig.setUsername(postgresStore.user()); hikariConfig.setPassword(postgresStore.password()); - hikariConfig.setMaximumPoolSize(2); + hikariConfig.setMaximumPoolSize(3); hikariConfig.setConnectionInitSql("set time zone 'UTC'"); From 6861a47d5fc00e5a765822c67852ef9b42dd2f54 Mon Sep 17 00:00:00 2001 From: Theresa Kamerman Date: Thu, 11 Jul 2024 18:00:56 -0700 Subject: [PATCH 20/20] Update Existing Tests - TimeTrackerDaemon: make counter "final" - CheckpointSimFacadeTest: Fix incorrect assertEquals - InMemoryCachedEngineTest: Use ResourceManager, reflect changes to `SimulationEngine` constructor SimulationDuplicationTest: Use ResourceManager - FooSimulationDuplicationTest: Remove empty consumer --- .../models/TimeTrackerDaemon.java | 2 +- .../FooSimulationDuplicationTest.java | 18 +++++-------- .../driver/SimulationDuplicationTest.java | 4 ++- .../CheckpointSimulationFacadeTest.java | 2 +- .../InMemoryCachedEngineStoreTest.java | 25 ++++++++----------- 5 files changed, 21 insertions(+), 30 deletions(-) diff --git a/examples/foo-missionmodel/src/main/java/gov/nasa/jpl/aerie/foomissionmodel/models/TimeTrackerDaemon.java b/examples/foo-missionmodel/src/main/java/gov/nasa/jpl/aerie/foomissionmodel/models/TimeTrackerDaemon.java index 31567a013b..6c3fbb3365 100644 --- a/examples/foo-missionmodel/src/main/java/gov/nasa/jpl/aerie/foomissionmodel/models/TimeTrackerDaemon.java +++ b/examples/foo-missionmodel/src/main/java/gov/nasa/jpl/aerie/foomissionmodel/models/TimeTrackerDaemon.java @@ -8,7 +8,7 @@ * A daemon task that tracks the number of minutes since plan start */ public class TimeTrackerDaemon { - private Counter minutesElapsed; + private final Counter minutesElapsed; public int getMinutesElapsed() { return minutesElapsed.get(); diff --git a/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/FooSimulationDuplicationTest.java b/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/FooSimulationDuplicationTest.java index 65e92a7d33..f183d23824 100644 --- a/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/FooSimulationDuplicationTest.java +++ b/examples/foo-missionmodel/src/test/java/gov/nasa/jpl/aerie/foomissionmodel/FooSimulationDuplicationTest.java @@ -100,8 +100,7 @@ void testCompareCheckpointOnEmptyPlan() { Duration.HOUR, Instant.EPOCH, Duration.HOUR, - () -> false, - $ -> {}); + () -> false); assertResultsEqual(expected, results); } @@ -129,8 +128,7 @@ void testFooNonEmptyPlan() { Duration.HOUR, Instant.EPOCH, Duration.HOUR, - () -> false, - $ -> {}); + () -> false); assertResultsEqual(expected, results); assertEquals(Duration.of(5, MINUTES), store.getCachedEngines(mockConfiguration()).getFirst().endsAt()); @@ -171,8 +169,7 @@ void testFooNonEmptyPlanMultipleResumes() { Duration.HOUR, Instant.EPOCH, Duration.HOUR, - () -> false, - $ -> {}); + () -> false); assertResultsEqual(expected, results); assertEquals(Duration.of(5, MINUTES), store.getCachedEngines(mockConfiguration()).getFirst().endsAt()); @@ -224,8 +221,7 @@ void testFooNonEmptyPlanMultipleCheckpointsMultipleResumes() { Duration.HOUR, Instant.EPOCH, Duration.HOUR, - () -> false, - $ -> {}); + () -> false); assertResultsEqual(expected, results); assertEquals(Duration.of(5, MINUTES), store.getCachedEngines(mockConfiguration()).getFirst().endsAt()); @@ -286,8 +282,7 @@ void testFooNonEmptyPlanMultipleCheckpointsMultipleResumesWithEdits() { Duration.HOUR, Instant.EPOCH, Duration.HOUR, - () -> false, - $ -> {}); + () -> false); final SimulationResults expected2 = SimulationDriver.simulate( missionModel, @@ -296,8 +291,7 @@ void testFooNonEmptyPlanMultipleCheckpointsMultipleResumesWithEdits() { Duration.HOUR, Instant.EPOCH, Duration.HOUR, - () -> false, - $ -> {}); + () -> false); assertResultsEqual(expected1, results); diff --git a/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDuplicationTest.java b/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDuplicationTest.java index ed29cdf857..24286636ab 100644 --- a/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDuplicationTest.java +++ b/merlin-driver/src/test/java/gov/nasa/jpl/aerie/merlin/driver/SimulationDuplicationTest.java @@ -1,5 +1,6 @@ package gov.nasa.jpl.aerie.merlin.driver; +import gov.nasa.jpl.aerie.merlin.driver.resources.InMemorySimulationResourceManager; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -63,7 +64,8 @@ void testDuplicate() { Instant.EPOCH, Duration.HOUR, () -> false, - $ -> {}); + $ -> {}, + new InMemorySimulationResourceManager()); assertEquals(expected, results); final var newResults = simulateWithCheckpoints(store.getCachedEngines(mockConfiguration()).get(0), List.of(), store); assertEquals(expected, newResults); diff --git a/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacadeTest.java b/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacadeTest.java index 983b80b5e8..1753eba5d4 100644 --- a/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacadeTest.java +++ b/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/CheckpointSimulationFacadeTest.java @@ -103,7 +103,7 @@ public void testStopsAtEndOfPlanningHorizon() final var actTypeA = activityTypes.get("ControllableDurationActivity"); plan.add(SchedulingActivityDirective.of(actTypeA, t0, HOUR.times(200), null, true)); final var results = newSimulationFacade.simulateNoResultsAllActivities(plan).computeResults(); - assertEquals(H.getEndAerie(), newSimulationFacade.totalSimulationTime()); + assertEquals(H.getEndAerie(), results.duration); assert(results.unfinishedActivities.size() == 1); } diff --git a/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStoreTest.java b/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStoreTest.java index b2b48a345f..9edf6e229a 100644 --- a/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStoreTest.java +++ b/scheduler-driver/src/test/java/gov/nasa/jpl/aerie/scheduler/simulation/InMemoryCachedEngineStoreTest.java @@ -5,10 +5,8 @@ import gov.nasa.jpl.aerie.merlin.driver.CachedSimulationEngine; import gov.nasa.jpl.aerie.merlin.driver.SimulationEngineConfiguration; import gov.nasa.jpl.aerie.merlin.driver.engine.SimulationEngine; -import gov.nasa.jpl.aerie.merlin.driver.engine.SlabList; import gov.nasa.jpl.aerie.merlin.driver.MissionModelId; -import gov.nasa.jpl.aerie.merlin.driver.timeline.CausalEventSource; -import gov.nasa.jpl.aerie.merlin.driver.timeline.LiveCells; +import gov.nasa.jpl.aerie.merlin.driver.resources.InMemorySimulationResourceManager; import gov.nasa.jpl.aerie.merlin.protocol.types.Duration; import gov.nasa.jpl.aerie.scheduler.SimulationUtility; import org.junit.jupiter.api.AfterEach; @@ -44,11 +42,10 @@ public static CachedSimulationEngine getCachedEngine1(){ new ActivityDirectiveId(1), new ActivityDirective(Duration.HOUR, "ActivityType1", Map.of(), null, true), new ActivityDirectiveId(2), new ActivityDirective(Duration.HOUR, "ActivityType2", Map.of(), null, true) ), - new SimulationEngine(), - new LiveCells(new CausalEventSource()), - new SlabList<>(), + new SimulationEngine(SimulationUtility.getFooMissionModel().getInitialCells()), null, - SimulationUtility.getFooMissionModel() + SimulationUtility.getFooMissionModel(), + new InMemorySimulationResourceManager() ); } @@ -59,11 +56,10 @@ public static CachedSimulationEngine getCachedEngine2(){ new ActivityDirectiveId(3), new ActivityDirective(Duration.HOUR, "ActivityType3", Map.of(), null, true), new ActivityDirectiveId(4), new ActivityDirective(Duration.HOUR, "ActivityType4", Map.of(), null, true) ), - new SimulationEngine(), - new LiveCells(new CausalEventSource()), - new SlabList<>(), + new SimulationEngine(SimulationUtility.getFooMissionModel().getInitialCells()), null, - SimulationUtility.getFooMissionModel() + SimulationUtility.getFooMissionModel(), + new InMemorySimulationResourceManager() ); } @@ -74,11 +70,10 @@ public static CachedSimulationEngine getCachedEngine3(){ new ActivityDirectiveId(5), new ActivityDirective(Duration.HOUR, "ActivityType5", Map.of(), null, true), new ActivityDirectiveId(6), new ActivityDirective(Duration.HOUR, "ActivityType6", Map.of(), null, true) ), - new SimulationEngine(), - new LiveCells(new CausalEventSource()), - new SlabList<>(), + new SimulationEngine(SimulationUtility.getFooMissionModel().getInitialCells()), null, - SimulationUtility.getFooMissionModel() + SimulationUtility.getFooMissionModel(), + new InMemorySimulationResourceManager() ); }