Test timeout issues (#683)

Co-authored-by: homersimpsons <[email protected]>
exercism · Apr 21, 2024 · cd8a756 · cd8a756
1 parent cd32698
commit cd8a756
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 3 deletions.
diff --git a/bin/test.sh b/bin/test.sh
@@ -58,7 +58,18 @@ function test {
     cp "${exercise_dir}/.meta/${example_file}" "${outdir}/${exercise_file}.${file_ext}"
   fi
 
-  eval "${PHPUNIT_BIN}" --no-configuration "${outdir}/${test_file}"
+  # `54s` timeout is an approximation to ensure the tests will not timeont in Exercism Test Runner.
+  #
+  # 1. Exercism Test Runner is around 3 times faster than GitHub CI on Ubuntu.
+  #    See: https://forum.exercism.org/t/test-tracks-for-the-20-seconds-limit-on-test-runners/10536/8
+  # 2. Exercism Test Runner should complete in 20s and involves:
+  #    - Starting Docker container (~1s)
+  #    - Running the test suite
+  #    - Processing the results (~1s)
+  #
+  # This gives 18s maximum for the test suite to run in the Exercism Test Runner.
+  # Hence the test suite should complete in less than 18s x 3 = 54s in GitHub CI on Ubuntu.
+  timeout 54s "${PHPUNIT_BIN}" --no-configuration "${outdir}/${test_file}"
 }
 
 function installed {

diff --git a/exercises/practice/robot-name/RobotNameTest.php b/exercises/practice/robot-name/RobotNameTest.php
@@ -73,26 +73,42 @@ public function testResetName(): void
         $this->assertMatchesRegularExpression('/\w{2}\d{3}/', $name2);
     }
 
+    /**
+     * PHPUnit ^10.5 has an issue with
+     *   $this->assertArrayNotHasKey($name, $names, sprintf...
+     * that leads to test timeouts. This is fixed in PHPUnit ^11.
+     * Revert workaround
+     *   $this->assertFalse(isset($names[$name]), sprintf...
+     * when upgrading.
+     */
     public function testNameArentRecycled(): void
     {
         $names = [];
 
         for ($i = 0; $i < 10000; $i++) {
             $name = $this->robot->getName();
-            $this->assertArrayNotHasKey($name, $names, sprintf('Name %s reissued after Reset.', $name));
+            $this->assertFalse(isset($names[$name]), sprintf('Name %s reissued after Reset.', $name));
             $names[$name] = true;
             $this->robot->reset();
         }
     }
 
+    /**
+     * PHPUnit ^10.5 has an issue with
+     *   $this->assertArrayNotHasKey($name, $names, sprintf...
+     * that leads to test timeouts. This is fixed in PHPUnit ^11.
+     * Revert workaround
+     *   $this->assertFalse(isset($names[$name]), sprintf...
+     * when upgrading.
+     */
     // This test is optional.
     public function testNameUniquenessManyRobots(): void
     {
         $names = [];
 
         for ($i = 0; $i < 10000; $i++) {
             $name = (new Robot())->getName();
-            $this->assertArrayNotHasKey($name, $names, sprintf('Name %s reissued after %d robots', $name, $i));
+            $this->assertFalse(isset($names[$name]), sprintf('Name %s reissued after %d robots', $name, $i));
             $names[$name] = true;
         }
     }