diff --git a/test/build_defs/test.build_defs b/test/build_defs/test.build_defs
index c03ee70bd5..4745faba10 100644
--- a/test/build_defs/test.build_defs
+++ b/test/build_defs/test.build_defs
@@ -29,7 +29,7 @@ def please_repo_e2e_test(
         test_cmd += [f'_STR="$(cat {o})" _SUBSTR="{c}" && if [ "${_STR##*$_SUBSTR*}" ]; then echo "$_STR"; exit 1; fi' for o, c in expect_output_contains.items()]
 
     if expect_output_doesnt_contain:
-        test_cmd += [f'_STR="$(cat {o})" _SUBSTR="{c}" && if [ -z "${_STR##*$_SUBSTR*}" ]; then echo "$_STR"; exit 1; fi' for o, c in expect_output_contains.items()]
+        test_cmd += [f'_STR="$(cat {o})" _SUBSTR="{c}" && if [ -z "${_STR##*$_SUBSTR*}" ]; then echo "$_STR"; exit 1; fi' for o, c in expect_output_doesnt_contain.items()]
 
     test_cmd = ' && '.join(test_cmd)
 
diff --git a/test/keep_going/BUILD b/test/keep_going/BUILD
index 024f1dfd07..250c7c9847 100644
--- a/test/keep_going/BUILD
+++ b/test/keep_going/BUILD
@@ -1,9 +1,27 @@
 subinclude("//test/build_defs")
 
 please_repo_e2e_test(
-    name = "build_all",
+    name = "plz_build_all",
     expected_failure = True,
+    expected_output = {
+        "plz-out/gen/package/pass": "",
+        "plz-out/gen/package/dep_pass": "",
+        "plz-out/gen/package/fail_test_pass_dep": "",
+    },
     plz_command = "plz build --keep_going //package:all",
     repo = "test_repo",
 )
 
+please_repo_e2e_test(
+    name = "plz_test_all",
+    expect_output_contains = {
+        "plz-out/log/test_results.xml": "fail_test_pass_dep",
+    },
+    expect_output_doesnt_contain = {
+        "plz-out/log/test_results.xml": "fail_test_fail_dep",
+    },
+    expected_failure = True,
+    plz_command = "plz test --keep_going //package:all",
+    repo = "test_repo",
+)
+
diff --git a/test/keep_going/test_repo/package/BUILD_FILE b/test/keep_going/test_repo/package/BUILD_FILE
index 2b8322b7c0..2a91f436fa 100644
--- a/test/keep_going/test_repo/package/BUILD_FILE
+++ b/test/keep_going/test_repo/package/BUILD_FILE
@@ -39,8 +39,18 @@ build_rule(
 
 build_rule(
     test = True,
-    name = "fail_test",
+    name = "fail_test_pass_dep",
     cmd = "touch $OUTS",
     test_cmd = "exit 1",
-    outs = ["fail_test"],
+    outs = ["fail_test_pass_dep"],
+    deps = [":pass"],
+)
+
+build_rule(
+    test = True,
+    name = "pass_test_fail_dep",
+    cmd = "touch $OUTS",
+    test_cmd = "exit 0",
+    outs = ["pass_test_fail_deps"],
+    deps = [":fail"],
 )