fix(grader): consider communication TLE as TLE, not ERR (#503)

ia-toki · Aug 28, 2023 · 9d1baf6 · 9d1baf6
1 parent d62304f
commit 9d1baf6
Show file tree

Hide file tree

Showing 4 changed files with 54 additions and 3 deletions.
diff --git a/...st/java/judgels/gabriel/engines/interactive/InteractiveGradingEngineIntegrationTests.java b/...st/java/judgels/gabriel/engines/interactive/InteractiveGradingEngineIntegrationTests.java
@@ -3,18 +3,21 @@
 import static judgels.gabriel.api.Verdict.ACCEPTED;
 import static judgels.gabriel.api.Verdict.COMPILATION_ERROR;
 import static judgels.gabriel.api.Verdict.OK;
+import static judgels.gabriel.api.Verdict.TIME_LIMIT_EXCEEDED;
 import static judgels.gabriel.api.Verdict.WRONG_ANSWER;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import java.util.List;
+import java.util.Optional;
 import judgels.gabriel.api.EvaluationException;
 import judgels.gabriel.api.GradingException;
 import judgels.gabriel.api.GradingResult;
 import judgels.gabriel.api.GradingResultDetails;
 import judgels.gabriel.api.PreparationException;
+import judgels.gabriel.api.SandboxExecutionStatus;
 import judgels.gabriel.api.TestCase;
 import judgels.gabriel.api.TestGroup;
 import judgels.gabriel.engines.BlackboxGradingEngineIntegrationTests;
@@ -114,6 +117,31 @@ void wa_90() throws GradingException {
                         subtaskResult(-1, OK, 90)));
     }
 
+    @Test
+    void tle_when_communication_timed_out() throws GradingException {
+        addSourceFile("source", "trigger-communication-TLE.cpp");
+        assertResult(
+                new InteractiveGradingConfig.Builder().from(CONFIG)
+                        .communicator("communicator-TLE.cpp").build(),
+                TIME_LIMIT_EXCEEDED,
+                0,
+                List.of(
+                        testGroupResult(
+                                0,
+                                testCaseResult(TIME_LIMIT_EXCEEDED, "", Optional.of(SandboxExecutionStatus.TIMED_OUT), 0),
+                                testCaseResult(TIME_LIMIT_EXCEEDED, "", Optional.of(SandboxExecutionStatus.TIMED_OUT), 0),
+                                testCaseResult(TIME_LIMIT_EXCEEDED, "", Optional.of(SandboxExecutionStatus.TIMED_OUT), 0)),
+                        testGroupResult(
+                                -1,
+                                testCaseResult(TIME_LIMIT_EXCEEDED, "0.0", Optional.of(SandboxExecutionStatus.TIMED_OUT), -1),
+                                testCaseResult(TIME_LIMIT_EXCEEDED, "0.0", Optional.of(SandboxExecutionStatus.TIMED_OUT), -1),
+                                testCaseResult(TIME_LIMIT_EXCEEDED, "0.0", Optional.of(SandboxExecutionStatus.TIMED_OUT), -1),
+                                testCaseResult(TIME_LIMIT_EXCEEDED, "0.0", Optional.of(SandboxExecutionStatus.TIMED_OUT), -1),
+                                testCaseResult(TIME_LIMIT_EXCEEDED, "0.0", Optional.of(SandboxExecutionStatus.TIMED_OUT), -1))),
+                List.of(
+                        subtaskResult(-1, TIME_LIMIT_EXCEEDED, 0)));
+    }
+
     @Test
     void ce() throws GradingException {
         addSourceFile("source", "binsearch-CE.cpp");

diff --git a/...ls-grader-engines/src/integTest/resources/engines/interactive/helper/communicator-TLE.cpp b/...ls-grader-engines/src/integTest/resources/engines/interactive/helper/communicator-TLE.cpp
@@ -0,0 +1,13 @@
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+int N;
+
+int main(int argc, char* argv[])
+{
+    FILE* in = fopen(argv[1], "r");
+    fscanf(in, "%d", &N);
+
+    return 10;
+}
diff --git a/...-engines/src/integTest/resources/engines/interactive/source/trigger-communication-TLE.cpp b/...-engines/src/integTest/resources/engines/interactive/source/trigger-communication-TLE.cpp
@@ -0,0 +1,4 @@
+// This solution intentionally TLE to trigger the whole communication to get TLE.
+int main() {
+    return 10;
+}
diff --git a/...dgels-grader-engines/src/main/java/judgels/gabriel/helpers/communicator/Communicator.java b/...dgels-grader-engines/src/main/java/judgels/gabriel/helpers/communicator/Communicator.java
@@ -1,5 +1,6 @@
 package judgels.gabriel.helpers.communicator;
 
+import static judgels.gabriel.api.SandboxExecutionStatus.TIMED_OUT;
 import static judgels.gabriel.api.SandboxExecutionStatus.ZERO_EXIT_CODE;
 
 import com.google.common.collect.ImmutableList;
@@ -172,8 +173,13 @@ public EvaluationResult communicate(File input) throws EvaluationException {
         solutionResult = ignoreSignal13(solutionResult);
         communicatorResult = ignoreSignal13(communicatorResult);
 
-        // If the communicator did not exit successfully, it means there is something wrong with it.
-        if (communicatorResult.getStatus() != ZERO_EXIT_CODE) {
+        SandboxExecutionResult finalResult = solutionResult;
+
+        if (communicatorResult.getStatus() == TIMED_OUT) {
+            // If the interaction timed out, and the communicator sandbox reported TLE, take this result (instead of ERR)
+            finalResult = communicatorResult;
+        } else if (communicatorResult.getStatus() != ZERO_EXIT_CODE) {
+            // Otherwise, if the communicator did not exit successfully, it means there is something wrong with it.
             throw new EvaluationException(String.join(" ", command) + " resulted in " + communicatorResult);
         }
 
@@ -186,7 +192,7 @@ public EvaluationResult communicate(File input) throws EvaluationException {
 
         return new EvaluationResult.Builder()
                 .verdict(verdict.get())
-                .executionResult(solutionResult)
+                .executionResult(finalResult)
                 .build();
     }