patch: handle submitted complilation error

Presently a compilation error was classified as a test failure, however
this causes the website to show all of the dependent test cases which
appear to be passing (despite actually not being run), because the
j-unit output doesn't have a success child element to indicate success,
only the absense of a skipped/failure element indicates success.

Dockerfile

@@ -1,7 +1,7 @@
 FROM php:8.2.7-cli-alpine3.18 AS build
 
 RUN apk update && \
-  apk add --no-cache ca-certificates curl jo zip unzip
+  apk add --no-cache ca-certificates bash curl jo zip:w unzip
 
 WORKDIR /usr/local/bin
 
@@ -15,7 +15,7 @@ RUN curl -L -o phpunit-9.phar https://phar.phpunit.de/phpunit-9.phar && \
 WORKDIR /usr/local/bin/junit-handler/
 COPY --from=composer:2.5.8 /usr/bin/composer /usr/local/bin/composer
 COPY junit-handler/ .
-RUN composer install --no-interaction 
+RUN composer install --no-interaction
 
 FROM php:8.2.7-cli-alpine3.18 AS runtime
 

bin/run-tests.sh

@@ -1,7 +1,7 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
 
 # Synopsis:
-# Test the test runner by running it against a predefined set of solutions 
+# Test the test runner by running it against a predefined set of solutions
 # with an expected output.
 
 # Output:
@@ -11,6 +11,15 @@
 # Example:
 # ./bin/run-tests.sh
 
+specific_test=$1
+
+function installed {
+  cmd=$(command -v "${1}")
+
+  [[ -n "${cmd}" ]] && [[ -f "${cmd}" ]]
+  return ${?}
+}
+
 exit_code=0
 
 # Iterate over all test directories
@@ -21,12 +30,16 @@ for test_dir in tests/*; do
     results_file_path="${output_dir_path}/results.json"
     expected_results_file_path="${test_dir_path}/expected_results.json"
 
+    if [ -n "${specific_test}" ] && [ "${specific_test}" != "${test_dir_name}" ]; then
+        continue
+    fi
+
     mkdir -p "${output_dir_path}"
 
     bin/run.sh "${test_dir_name}" "${test_dir_path}" "${output_dir_path}"
 
     # OPTIONAL: Normalize the results file
-    # If the results.json file contains information that changes between 
+    # If the results.json file contains information that changes between
     # different test runs (e.g. timing information or paths), you should normalize
     # the results file to allow the diff comparison below to work as expected
     # sed -i -E \
@@ -35,7 +48,11 @@ for test_dir in tests/*; do
     #   "${results_file_path}"
 
     echo "${test_dir_name}: comparing results.json to expected_results.json"
-    diff "${results_file_path}" "${expected_results_file_path}"
+    if installed "jq"; then
+        diff <(jq -S . "${results_file_path}") <(jq -S . "${expected_results_file_path}")
+    else
+        diff "${results_file_path}" "${expected_results_file_path}"
+    fi
 
     if [ $? -ne 0 ]; then
         exit_code=1

bin/run.sh

@@ -23,6 +23,9 @@ function main {
   phpunit_exit_code=$?
   set -e
 
+  echo ">>>>>>> TESTING: ${solution_dir}"
+  echo ">>>>>>> EXIT CODE: ${phpunit_exit_code}"
+
   if [[ "${phpunit_exit_code}" -eq 255 ]]; then
     jo version=2 status=error message="${phpunit_output}" tests="[]" > "${output_dir%/}/${JSON_RESULTS}"
     return 0;

junit-handler/src/Handler.php

@@ -48,9 +48,7 @@ public function run(string $xml_path, $json_path): void
 
         $output = [
             'version' => self::VERSION,
-            'status' => ($testcase_error_count !== 0 || $testcase_failure_count !== 0)
-            ? self::STATUS_FAIL
-            : self::STATUS_PASS,
+            'status' => $this->getStatus($testcase_error_count, $testcase_failure_count),
             'tests' =>
             $this->parseTestSuite(
                 $testsuite,
@@ -232,4 +230,19 @@ private function getTestCaseSource(
 
         return implode("\n", $test_lines) . "\n";
     }
+
+    private function getStatus(
+        int $testcase_error_count,
+        int $testcase_failure_count
+    ): string {
+        if ($testcase_error_count > 0) {
+            return self::STATUS_ERROR;
+        }
+
+        if ($testcase_failure_count > 0) {
+            return self::STATUS_FAIL;
+        }
+
+        return self::STATUS_PASS;
+    }
 }

tests/all-fail/expected_results.json

@@ -1 +1 @@
-{"version":3,"status":"fail","tests":[{"name":"testLeapYear","status":"error","test_code":"$this->assertTrue(isLeap(1996));\n","message":"LeapTest::testLeapYear\nParseError: syntax error, unexpected token \"(\", expecting \";\"\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7"},{"name":"testNonLeapYear","status":"pass","test_code":"$this->assertFalse(isLeap(1997));\n"},{"name":"testNonLeapEvenYear","status":"pass","test_code":"$this->assertFalse(isLeap(1998));\n"},{"name":"testCentury","status":"pass","test_code":"$this->assertFalse(isLeap(1900));\n"},{"name":"testFourthCentury","status":"pass","test_code":"$this->assertTrue(isLeap(2400));\n"}]}
+{"version":3,"status":"error","tests":[{"name":"testLeapYear","status":"error","test_code":"$this->assertTrue(isLeap(1996));\n","message":"LeapTest::testLeapYear\nParseError: syntax error, unexpected token \"(\", expecting \";\"\n\n\/opt\/test-runner\/tests\/all-fail\/Leap.php:7"},{"name":"testNonLeapYear","status":"pass","test_code":"$this->assertFalse(isLeap(1997));\n"},{"name":"testNonLeapEvenYear","status":"pass","test_code":"$this->assertFalse(isLeap(1998));\n"},{"name":"testCentury","status":"pass","test_code":"$this->assertFalse(isLeap(1900));\n"},{"name":"testFourthCentury","status":"pass","test_code":"$this->assertTrue(isLeap(2400));\n"}]}

tests/syntax-error/expected_results.json

@@ -1 +1 @@
-{"version":3,"status":"fail","tests":[{"name":"testLeapYear","status":"error","test_code":"$this->assertTrue(isLeap(1996));\n","message":"LeapTest::testLeapYear\nParseError: syntax error, unexpected token \"@\", expecting \"(\"\n\n\/opt\/test-runner\/tests\/syntax-error\/Leap.php:5"},{"name":"testNonLeapYear","status":"pass","test_code":"$this->assertFalse(isLeap(1997));\n"},{"name":"testNonLeapEvenYear","status":"pass","test_code":"$this->assertFalse(isLeap(1998));\n"},{"name":"testCentury","status":"pass","test_code":"$this->assertFalse(isLeap(1900));\n"},{"name":"testFourthCentury","status":"pass","test_code":"$this->assertTrue(isLeap(2400));\n"}]}
+{"version":3,"status":"error","tests":[{"name":"testLeapYear","status":"error","test_code":"$this->assertTrue(isLeap(1996));\n","message":"LeapTest::testLeapYear\nParseError: syntax error, unexpected token \"@\", expecting \"(\"\n\n\/opt\/test-runner\/tests\/syntax-error\/Leap.php:5"},{"name":"testNonLeapYear","status":"pass","test_code":"$this->assertFalse(isLeap(1997));\n"},{"name":"testNonLeapEvenYear","status":"pass","test_code":"$this->assertFalse(isLeap(1998));\n"},{"name":"testCentury","status":"pass","test_code":"$this->assertFalse(isLeap(1900));\n"},{"name":"testFourthCentury","status":"pass","test_code":"$this->assertTrue(isLeap(2400));\n"}]}