Skip to content

Commit

Permalink
Update e2eTests
Browse files Browse the repository at this point in the history 
- Update and Add GQL queries related to constraints
- Update types
- Add ConstraintsTests for versioning and enabling/disabling constraints
- Remove tests that checked `definitionOutdated`, which doesn't exist anymore
  • Loading branch information
@Mythicaeda
Mythicaeda committed Jan 11, 2024
1 parent 4bbb8a6 commit 71ff624
Show file tree
Hide file tree
Showing 5 changed files with 195 additions and 129 deletions.
191 changes: 101 additions & 90 deletions e2e-tests/src/test/java/gov/nasa/jpl/aerie/e2e/ConstraintsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.microsoft.playwright.Playwright;
import gov.nasa.jpl.aerie.e2e.types.ConstraintError;
import gov.nasa.jpl.aerie.e2e.types.ConstraintRecord;
import gov.nasa.jpl.aerie.e2e.types.ExternalDataset.ProfileInput;
import gov.nasa.jpl.aerie.e2e.types.ExternalDataset.ProfileInput.ProfileSegmentInput;
import gov.nasa.jpl.aerie.e2e.types.ValueSchema;
Expand All @@ -12,6 +13,7 @@
import javax.json.Json;
import javax.json.JsonValue;
import java.io.IOException;
import java.util.Comparator;
import java.util.List;

import static org.junit.jupiter.api.Assertions.assertEquals;
Expand Down Expand Up @@ -109,7 +111,6 @@ void constraintsSucceedOneViolation() throws IOException {
assertTrue(constraintResponse.success());
assertEquals(constraintId, constraintResponse.constraintId());
assertEquals(constraintName, constraintResponse.constraintName());
assertEquals("plan", constraintResponse.type());
// Check the Result
assertTrue(constraintResponse.result().isPresent());
final var constraintResult = constraintResponse.result().get();
Expand Down Expand Up @@ -143,7 +144,6 @@ void constraintsSucceedNoViolations() throws IOException {
assertTrue(constraintResponses.get(0).success());
assertEquals(constraintId, constraintResponses.get(0).constraintId());
assertEquals(constraintName, constraintResponses.get(0).constraintName());
assertEquals("plan", constraintResponses.get(0).type());
assertTrue( constraintResponses.get(0).result().isPresent());
assertEquals(0, constraintResponses.get(0).result().get().violations().size());
}
Expand All @@ -161,7 +161,6 @@ void constraintCachedViolation() throws IOException {

// Check properties
final var cachedRun = cachedRuns.get(0);
assertFalse(cachedRun.definitionOutdated());
assertEquals(constraintId, cachedRun.constraintId());
assertEquals(simDatasetId, cachedRun.simDatasetId());
assertEquals(constraintDefinition, cachedRun.constraintDefinition());
Expand All @@ -179,15 +178,16 @@ void constraintCachedNoViolations() throws IOException {
// Delete activity to avoid violation
hasura.deleteActivity(planId, activityId);
final var simDatasetId = hasura.awaitSimulation(planId).simDatasetId();
hasura.checkConstraints(planId);
final var constraintsResults = hasura.checkConstraints(planId);
final var cachedRuns = hasura.getConstraintRuns(simDatasetId);

// There's the correct number of results
assertEquals(1, cachedRuns.size());
assertEquals(1, constraintsResults.size());

// Check properties
final var cachedRun = cachedRuns.get(0);
assertFalse(cachedRun.definitionOutdated());
assertEquals(constraintsResults.get(0).constraintRevision(), cachedRun.constraintRevision());
assertEquals(constraintId, cachedRun.constraintId());
assertEquals(simDatasetId, cachedRun.simDatasetId());
assertEquals(constraintDefinition, cachedRun.constraintDefinition());
Expand All @@ -202,21 +202,6 @@ void constraintCachedNoViolations() throws IOException {
assertEquals(0,results.gaps().size());
}

@Test
void constraintCacheInvalidatedDefinition() throws IOException {
final int simDatasetId = hasura.awaitSimulation(planId).simDatasetId();
hasura.checkConstraints(planId);

// Updating the constraint definition should mark the constraint run as invalid
final String newDefinition =
"export default (): Constraint => Real.Resource(\"/peel\").equal(Real.Resource(\"/fruit\"))";
hasura.updateConstraint(constraintId, newDefinition);

final var cachedRuns = hasura.getConstraintRuns(simDatasetId);
assertEquals(1, cachedRuns.size());
assertTrue(cachedRuns.get(0).definitionOutdated());
}

/**
* Test that an activity with a duration longer than one month is written to and read back from the database
* successfully
Expand All @@ -228,7 +213,7 @@ void constraintCacheInvalidatedDefinition() throws IOException {
@Test
void constraintsWorkMonthLongActivity() throws IOException {
// Setup
hasura.updateConstraint(
hasura.updateConstraintDefinition(
constraintId,
"export default (): Constraint => Windows.During(ActivityType.ControllableDurationActivity).not()");
hasura.deleteActivity(planId, activityId);
Expand All @@ -247,7 +232,6 @@ void constraintsWorkMonthLongActivity() throws IOException {
assertTrue(constraintResponse.success());
assertEquals(constraintId,constraintResponse.constraintId());
assertEquals(constraintName, constraintResponse.constraintName());
assertEquals("plan", constraintResponse.type());

//Check Result
assertTrue(constraintResponse.result().isPresent());
Expand Down Expand Up @@ -281,7 +265,6 @@ void runConstraintsOnOldSimulation() throws IOException {
assertEquals(1, newConstraintResponses.size());
assertEquals(constraintId, newConstraintResponses.get(0).constraintId());
assertEquals(constraintName, newConstraintResponses.get(0).constraintName());
assertEquals("plan", newConstraintResponses.get(0).type());
assertTrue(newConstraintResponses.get(0).result().isPresent());
final var newConstraintResult = newConstraintResponses.get(0).result().get();
assertTrue(newConstraintResult.violations().isEmpty());
Expand All @@ -296,7 +279,6 @@ void runConstraintsOnOldSimulation() throws IOException {
assertTrue(oldConstraintResponse.success());
assertEquals(constraintId, oldConstraintResponse.constraintId());
assertEquals(constraintName, oldConstraintResponse.constraintName());
assertEquals("plan", oldConstraintResponse.type());
assertTrue(oldConstraintResponse.result().isPresent());
final var constraintResult = oldConstraintResponse.result().get();

Expand All @@ -319,19 +301,103 @@ void runConstraintsOnOldSimulation() throws IOException {
assertTrue(constraintResult.gaps().isEmpty());
}

/**
* If a plan specifies a version of a constraint to use, then that version will be used
* when checking constraints.
*
* If the test fails with a compilation error, that means it used the latest version of the constraint
*/
@Test
void cachedRunsNotOutdatedOnResim() throws IOException {
final int oldSimDatasetId = hasura.awaitSimulation(planId).simDatasetId();
hasura.checkConstraints(planId);

// Delete Activity to make the simulation outdated, then resim
hasura.deleteActivity(planId, activityId);
void constraintVersionLocking() throws IOException {
hasura.awaitSimulation(planId);

// Get the old run
final var cachedRuns = hasura.getConstraintRuns(oldSimDatasetId);
assertEquals(1, cachedRuns.size());
assertFalse(cachedRuns.get(0).definitionOutdated());
// Update the plan's constraint specification to use a specific version
hasura.updatePlanConstraintSpecVersion(planId, constraintId, 0);

// Update definition to have invalid syntax
final int newRevision = hasura.updateConstraintDefinition(
constraintId,
" error :-(");

// Check constraints -- should succeed
final var initResults = hasura.checkConstraints(planId);
assertEquals(1, initResults.size());
final var initConstraint = initResults.get(0);
assertEquals(constraintId, initConstraint.constraintId());
assertEquals(0, initConstraint.constraintRevision());
assertTrue(initConstraint.success());
assertTrue(initConstraint.errors().isEmpty());
assertTrue(initConstraint.result().isPresent());

// Update constraint spec to use invalid definition
hasura.updatePlanConstraintSpecVersion(planId, constraintId, newRevision);

// Check constraints -- should fail
final var badDefinitionResults = hasura.checkConstraints(planId);
assertEquals(1, badDefinitionResults.size());
final var badConstraint = badDefinitionResults.get(0);
assertEquals(constraintId, badConstraint.constraintId());
assertFalse(badConstraint.success());
assertEquals(constraintName, badConstraint.constraintName());
assertEquals(2, badConstraint.errors().size());
assertEquals("Constraint 'fruit_equal_peel' compilation failed:\n" +
" TypeError: TS2306 No default export. Expected a default export function with the signature: " +
"\"(...args: []) => Constraint | Promise<Constraint>\".",
badConstraint.errors().get(0).message());
assertEquals("Constraint 'fruit_equal_peel' compilation failed:\n TypeError: TS1109 Expression expected.",
badConstraint.errors().get(1).message());

// Update constraint spec to use initial definition
hasura.updatePlanConstraintSpecVersion(planId, constraintId, 0);

// Check constraints -- should match
assertEquals(initResults, hasura.checkConstraints(planId));
}

@Test
@DisplayName("Disabled Constraints are not checked")
void constraintIgnoreDisabled() throws IOException {
hasura.awaitSimulation(planId);
// Add a problematic constraint to the spec, then disable it
final String problemConstraintName = "bad constraint";
final int problemConstraintId = hasura.insertPlanConstraint(
problemConstraintName,
planId,
"error :-(",
"constraint that shouldn't compile");
try {
hasura.updatePlanConstraintSpecEnabled(planId, problemConstraintId, false);

// Check constraints -- Validate that only the enabled constraint is included
final var initResults = hasura.checkConstraints(planId);
assertEquals(1, initResults.size());
assertEquals(constraintId, initResults.get(0).constraintId());
assertTrue(initResults.get(0).success());

// Enable disabled constraint
hasura.updatePlanConstraintSpecEnabled(planId, problemConstraintId, true);

// Check constraints -- Validate that the other constraint is present and a failure
final var results = hasura.checkConstraints(planId);
results.sort(Comparator.comparing(ConstraintRecord::constraintId));
assertEquals(2, results.size());
assertEquals(constraintId, results.get(0).constraintId());
assertTrue(results.get(0).success());

final var problemResults = results.get(1);
assertEquals(problemConstraintId, problemResults.constraintId());
assertFalse(problemResults.success());
assertEquals(problemConstraintName, problemResults.constraintName());
assertEquals(2, problemResults.errors().size());
assertEquals("Constraint 'bad constraint' compilation failed:\n" +
" TypeError: TS2306 No default export. Expected a default export function with the signature: " +
"\"(...args: []) => Constraint | Promise<Constraint>\".",
problemResults.errors().get(0).message());
assertEquals("Constraint 'bad constraint' compilation failed:\n TypeError: TS1109 Expression expected.",
problemResults.errors().get(1).message());
} finally {
hasura.deleteConstraint(problemConstraintId);
}
}

@Nested
Expand All @@ -353,7 +419,7 @@ class WithExternalDatasets {
@BeforeEach
void beforeEach() throws IOException {
// Change Constraint to be about MyBoolean
hasura.updateConstraint(
hasura.updateConstraintDefinition(
constraintId,
constraintDefinition);
// Simulate Plan
Expand Down Expand Up @@ -382,7 +448,6 @@ void oneViolationCurrentSimulation() throws IOException {
assertTrue(noDatasetResponses.get(0).success());
assertEquals(constraintId, noDatasetResponses.get(0).constraintId());
assertEquals(constraintName, noDatasetResponses.get(0).constraintName());
assertEquals("plan", noDatasetResponses.get(0).type());
assertTrue(noDatasetResponses.get(0).result().isPresent());
final var nRecordResults = noDatasetResponses.get(0).result().get();

Expand All @@ -392,7 +457,6 @@ void oneViolationCurrentSimulation() throws IOException {
assertTrue(withDatasetResponses.get(0).success());
assertEquals(constraintId, withDatasetResponses.get(0).constraintId());
assertEquals(constraintName, withDatasetResponses.get(0).constraintName());
assertEquals("plan", withDatasetResponses.get(0).type());
assertTrue(withDatasetResponses.get(0).result().isPresent());
final var wRecordResults = withDatasetResponses.get(0).result().get();

Expand Down Expand Up @@ -445,7 +509,6 @@ void oneViolationOutdatedSimIdPassed() throws IOException {
assertTrue(constraintResponse.success());
assertEquals(constraintId, constraintResponse.constraintId());
assertEquals(constraintName, constraintResponse.constraintName());
assertEquals("plan", constraintResponse.type());
assertTrue(constraintResponse.result().isPresent());
final var record = constraintResponse.result().get();

Expand Down Expand Up @@ -493,7 +556,6 @@ void compilationFailsOutdatedSimulationSimDatasetId() throws IOException {
final var constraintResponse = constraintResponses.get(0);
assertEquals(constraintId, constraintResponse.constraintId());
assertEquals(constraintName, constraintResponse.constraintName());
assertEquals("plan", constraintResponse.type());
assertFalse(constraintResponse.success());
assertTrue(constraintResponse.result().isEmpty());
assertEquals(1,constraintResponse.errors().size());
Expand All @@ -517,62 +579,11 @@ void compilationFailsOutdatedSimulationNoSimDataset() throws IOException {
assertFalse(constraintResponse.success());
assertEquals(constraintId, constraintResponse.constraintId());
assertEquals(constraintName, constraintResponse.constraintName());
assertEquals("plan", constraintResponse.type());
assertTrue(constraintResponse.result().isEmpty());
assertEquals(1,constraintResponse.errors().size());
final ConstraintError error = constraintResponse.errors().get(0);
assertEquals("Constraint 'fruit_equal_peel' compilation failed:\n"
+ " TypeError: TS2345 Argument of type '\"/my_boolean\"' is not assignable to parameter of type 'ResourceName'.",error.message());

}

/**
* This test case evaluates the scenario where a user initially has a functioning constraint.
* However, during the modification process, they introduce a compilation error.
* The goal is to confirm that the user can successfully fix the compilation error,
* ensuring that the constraint compiles correctly and returns a valid response.
*/
@Test
@DisplayName("If a constraint is updated and then reverted, Constraints will load the cached value from before the update.")
void constraintInvalidModification() throws IOException {
// Simulate the plan to insure a new simulation set
hasura.awaitSimulation(planId);

// Check the initial constraints responses it should compile
final var constraintsResponses = hasura.checkConstraints(planId);
assertEquals(1, constraintsResponses.size());
assertTrue(constraintsResponses.get(0).success());
assertEquals(constraintId, constraintsResponses.get(0).constraintId());
assertEquals(constraintName, constraintsResponses.get(0).constraintName());
assertEquals("plan", constraintsResponses.get(0).type());
// Attempt to update a constraint with an invalid syntax and capture the error response
hasura.updateConstraint(
constraintId,
WithExternalDatasets.constraintDefinition + "; error");

// Check the constraints response after the invalid modification
final var constraintsErrorResponses = hasura.checkConstraints(planId);
assertEquals(1, constraintsErrorResponses.size());
assertFalse(constraintsErrorResponses.get(0).success());
assertEquals(constraintId, constraintsErrorResponses.get(0).constraintId());
assertEquals(constraintName, constraintsErrorResponses.get(0).constraintName());
assertEquals("plan", constraintsErrorResponses.get(0).type());
assertTrue(constraintsErrorResponses.get(0).result().isEmpty());
assertEquals(1, constraintsErrorResponses.get(0).errors().size());

// Restore the original valid constraint
hasura.updateConstraint(
constraintId,
WithExternalDatasets.constraintDefinition);

// Check the constraints response after reverting to the valid constraint
final var constraintsCacheResponses = hasura.checkConstraints(planId);
assertEquals(1, constraintsCacheResponses.size());
assertTrue(constraintsCacheResponses.get(0).success());

// Ensure that the constraints responses are the same before and after the invalid modification
assertEquals(constraintsResponses, constraintsCacheResponses);
}

}
}
6 changes: 3 additions & 3 deletions e2e-tests/src/test/java/gov/nasa/jpl/aerie/e2e/types/CachedConstraintRun.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@

public record CachedConstraintRun(
int constraintId,
int constraintRevision,
int simDatasetId,
String constraintDefinition,
boolean definitionOutdated,
Optional<ConstraintResult> results
) {
public static CachedConstraintRun fromJSON(JsonObject json) {
return new CachedConstraintRun(
json.getInt("constraint_id"),
json.getInt("constraint_revision"),
json.getInt("simulation_dataset_id"),
json.getString("constraint_definition"),
json.getBoolean("definition_outdated"),
json.getJsonObject("constraint_definition").getString("definition"),
json.getJsonObject("results").isEmpty() ?
Optional.empty() :
Optional.of(ConstraintResult.fromJSON(json.getJsonObject("results")))
Expand Down
4 changes: 2 additions & 2 deletions e2e-tests/src/test/java/gov/nasa/jpl/aerie/e2e/types/ConstraintRecord.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
public record ConstraintRecord(
boolean success,
int constraintId,
int constraintRevision,
String constraintName,
String type,
Optional<ConstraintResult> result,
List<ConstraintError> errors

Expand All @@ -17,8 +17,8 @@ public static ConstraintRecord fromJSON(JsonObject json){
return new ConstraintRecord(
json.getBoolean("success"),
json.getInt("constraintId"),
json.getInt("constraintRevision"),
json.getString("constraintName"),
json.getString("type"),
json.getJsonObject("results").isEmpty() ? Optional.empty() : Optional.of(ConstraintResult.fromJSON(json.getJsonObject("results"))),
json.getJsonArray("errors").getValuesAs(ConstraintError::fromJSON));
}
Expand Down
Loading

0 comments on commit 71ff624

Please sign in to comment.