From 09108d88dd5f47c1540d50a24061a892434056d3 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Fri, 5 May 2023 20:22:41 -0500 Subject: [PATCH 1/5] Redo test as a vector of lines instead of a string. This test is difficult to work with as-is because it's one big string, and deep-diff2 (currently) shows only that two strings differ, and not the substrings in particular that cause the mismatch. --- test/unit/kaocha/watch_test.clj | 36 ++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/test/unit/kaocha/watch_test.clj b/test/unit/kaocha/watch_test.clj index e5678cf9..954a46af 100644 --- a/test/unit/kaocha/watch_test.clj +++ b/test/unit/kaocha/watch_test.clj @@ -13,7 +13,9 @@ [kaocha.config :as config] [clojure.test :as t] [clojure.string :as str] - [slingshot.slingshot :refer [try+]]) + [slingshot.slingshot :refer [try+]] + [matcher-combinators.matchers :as matchers] + ) (:import (java.io File))) (require 'matcher-combinators.test) @@ -131,15 +133,29 @@ (w/qput q :finish) (Thread/sleep 100) - (is (str/includes? - @out-str - (str/replace - (str/replace - "[(F)]\n\nFAIL in foo.bar-test/xxx-test (bar_test.clj:1)\nExpected:\n :xxx\nActual:\n -:xxx +:yyy\n1 tests, 1 assertions, 1 failures.\n\nbin/kaocha --config-file PATH --focus 'foo.bar-test/xxx-test'\n\n[watch] Reloading #{foo.bar-test}\n[watch] Re-running failed tests #{:foo.bar-test/xxx-test}\n[(F)]\n\nFAIL in foo.bar-test/xxx-test (bar_test.clj:1)\nExpected:\n :xxx\nActual:\n -:xxx +:zzz" - "foo" - prefix) - "PATH" - (str config-file)))))) + (is (match? + (matchers/embeds + ["[(F)]" + "" + (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) + "Expected:" + " :xxx" + "Actual:" + " -:xxx +:yyy" + "1 tests, 1 assertions, 1 failures." + "" + (format "bin/kaocha --config-file %s --focus '%s.bar-test/xxx-test'" (str config-file) prefix) + "" + (str/replace "[watch] Reloading #{foo.bar-test}" "foo" prefix) + (str/replace "[watch] Re-running failed tests #{:foo.bar-test/xxx-test}" "foo" prefix) + "[(F)]" + "" + (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) + "Expected:" + " :xxx" + "Actual:" + " -:xxx +:zzz"]) + (str/split-lines @out-str))))) (deftest ignore-files-merged (let [{:keys [_config-file test-dir] :as m} (integration/test-dir-setup {})] From acf8fb7644a5ebb3ae3388ac7a80c6ca04bd56ae Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 10 May 2023 21:42:49 -0500 Subject: [PATCH 2/5] Fix up style issue. --- test/unit/kaocha/watch_test.clj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/unit/kaocha/watch_test.clj b/test/unit/kaocha/watch_test.clj index 954a46af..3f587662 100644 --- a/test/unit/kaocha/watch_test.clj +++ b/test/unit/kaocha/watch_test.clj @@ -14,8 +14,7 @@ [clojure.test :as t] [clojure.string :as str] [slingshot.slingshot :refer [try+]] - [matcher-combinators.matchers :as matchers] - ) + [matcher-combinators.matchers :as matchers]) (:import (java.io File))) (require 'matcher-combinators.test) From 0e9a7e392d3207313f10e309d00f94dbed5abdaf Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Fri, 28 Jul 2023 21:50:02 -0500 Subject: [PATCH 3/5] Split test into two. It seems to run into quadratic behavior if I try to compare every line. This is slightly less accurate of a test, though. --- test/unit/kaocha/watch_test.clj | 50 ++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/test/unit/kaocha/watch_test.clj b/test/unit/kaocha/watch_test.clj index 3f587662..28c4d345 100644 --- a/test/unit/kaocha/watch_test.clj +++ b/test/unit/kaocha/watch_test.clj @@ -132,29 +132,33 @@ (w/qput q :finish) (Thread/sleep 100) - (is (match? - (matchers/embeds - ["[(F)]" - "" - (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) - "Expected:" - " :xxx" - "Actual:" - " -:xxx +:yyy" - "1 tests, 1 assertions, 1 failures." - "" - (format "bin/kaocha --config-file %s --focus '%s.bar-test/xxx-test'" (str config-file) prefix) - "" - (str/replace "[watch] Reloading #{foo.bar-test}" "foo" prefix) - (str/replace "[watch] Re-running failed tests #{:foo.bar-test/xxx-test}" "foo" prefix) - "[(F)]" - "" - (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) - "Expected:" - " :xxx" - "Actual:" - " -:xxx +:zzz"]) - (str/split-lines @out-str))))) + (let [split-str (str/split-lines @out-str)] + (is (match? + (matchers/embeds + ["[(F)]" + "" + (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) + "Expected:" + " :xxx" + "Actual:" + " -:xxx +:yyy" + "1 tests, 1 assertions, 1 failures."]) + split-str)) + + (is (match? + (matchers/embeds + [(format "bin/kaocha --config-file %s --focus '%s.bar-test/xxx-test'" (str config-file) prefix) + "" + (str/replace "[watch] Reloading #{foo.bar-test}" "foo" prefix) + (str/replace "[watch] Re-running failed tests #{:foo.bar-test/xxx-test}" "foo" prefix) + "[(F)]" + "" + (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) + "Expected:" + " :xxx" + "Actual:" + " -:xxx +:zzz" ]) + split-str))))) (deftest ignore-files-merged (let [{:keys [_config-file test-dir] :as m} (integration/test-dir-setup {})] From 99f96f539f6a42fca025b9595ad021eb1b8251bb Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Fri, 28 Jul 2023 22:09:32 -0500 Subject: [PATCH 4/5] Switch from embeds to equals to improve messages Artificially dividing it up produced weird errors when there was a mismach. Using equals results in more coherent diffs and only required adding a few lines. I also didn't have that much confidence that a different change in test output wouidn't trigger a long delay. This does make the test slightly more fragile, as any thing added to the beginning of the output or the end of the output will cause this to fail. Also, if there's more lines in the output than in the test, you don't get a great diff either. --- test/unit/kaocha/watch_test.clj | 55 +++++++++++++++++---------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/test/unit/kaocha/watch_test.clj b/test/unit/kaocha/watch_test.clj index 28c4d345..da6ce3d8 100644 --- a/test/unit/kaocha/watch_test.clj +++ b/test/unit/kaocha/watch_test.clj @@ -111,6 +111,7 @@ (assoc-in [:kaocha/cli-options :config-file] (str config-file)) (assoc-in [:kaocha/tests 0 :kaocha/source-paths] []) (assoc-in [:kaocha/tests 0 :kaocha/test-paths] [(str test-dir)])) + ;; _ (println config) prefix (str (gensym "foo")) finish? (atom false) q (w/make-queue) @@ -132,33 +133,33 @@ (w/qput q :finish) (Thread/sleep 100) - (let [split-str (str/split-lines @out-str)] - (is (match? - (matchers/embeds - ["[(F)]" - "" - (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) - "Expected:" - " :xxx" - "Actual:" - " -:xxx +:yyy" - "1 tests, 1 assertions, 1 failures."]) - split-str)) - - (is (match? - (matchers/embeds - [(format "bin/kaocha --config-file %s --focus '%s.bar-test/xxx-test'" (str config-file) prefix) - "" - (str/replace "[watch] Reloading #{foo.bar-test}" "foo" prefix) - (str/replace "[watch] Re-running failed tests #{:foo.bar-test/xxx-test}" "foo" prefix) - "[(F)]" - "" - (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) - "Expected:" - " :xxx" - "Actual:" - " -:xxx +:zzz" ]) - split-str))))) + (is (match? + (matchers/equals + ["[(F)]" + "" + (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) + "Expected:" + " :xxx" + "Actual: " + " -:xxx +:yyy" + "1 tests, 1 assertions, 1 failures." + "" + (format "bin/kaocha --config-file %s --focus '%s.bar-test/xxx-test'" (str config-file) prefix) + "" + (str/replace "[watch] Reloading #{foo.bar-test}" "foo" prefix) + (str/replace "[watch] Re-running failed tests #{:foo.bar-test/xxx-test}" "foo" prefix) + "[(F)]" + "" + (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) + "Expected:" + " :xxx" + "Actual:" + " -:xxx +:zzz" + "1 tests, 1 assertions, 1 failures." + "" + (format "bin/kaocha --config-file %s --focus '%s.bar-test/xxx-test'" (str config-file) prefix) + ]) + (str/split-lines @out-str))))) (deftest ignore-files-merged (let [{:keys [_config-file test-dir] :as m} (integration/test-dir-setup {})] From 2e7480e072df44330e09ef5ba06e98c7300d72c9 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Fri, 28 Jul 2023 22:16:34 -0500 Subject: [PATCH 5/5] Switch from matcher-combinators to regular equality. --- test/unit/kaocha/watch_test.clj | 53 ++++++++++++++++----------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/test/unit/kaocha/watch_test.clj b/test/unit/kaocha/watch_test.clj index da6ce3d8..5ea424fc 100644 --- a/test/unit/kaocha/watch_test.clj +++ b/test/unit/kaocha/watch_test.clj @@ -133,33 +133,32 @@ (w/qput q :finish) (Thread/sleep 100) - (is (match? - (matchers/equals - ["[(F)]" - "" - (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) - "Expected:" - " :xxx" - "Actual: " - " -:xxx +:yyy" - "1 tests, 1 assertions, 1 failures." - "" - (format "bin/kaocha --config-file %s --focus '%s.bar-test/xxx-test'" (str config-file) prefix) - "" - (str/replace "[watch] Reloading #{foo.bar-test}" "foo" prefix) - (str/replace "[watch] Re-running failed tests #{:foo.bar-test/xxx-test}" "foo" prefix) - "[(F)]" - "" - (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) - "Expected:" - " :xxx" - "Actual:" - " -:xxx +:zzz" - "1 tests, 1 assertions, 1 failures." - "" - (format "bin/kaocha --config-file %s --focus '%s.bar-test/xxx-test'" (str config-file) prefix) - ]) - (str/split-lines @out-str))))) + (is (= + ["[(F)]" + "" + (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) + "Expected:" + " :xxx" + "Actual:" + " -:xxx +:yyy" + "1 tests, 1 assertions, 1 failures." + "" + (format "bin/kaocha --config-file %s --focus '%s.bar-test/xxx-test'" (str config-file) prefix) + "" + (str/replace "[watch] Reloading #{foo.bar-test}" "foo" prefix) + (str/replace "[watch] Re-running failed tests #{:foo.bar-test/xxx-test}" "foo" prefix) + "[(F)]" + "" + (str/replace "FAIL in foo.bar-test/xxx-test (bar_test.clj:1)" "foo" prefix) + "Expected:" + " :xxx" + "Actual:" + " -:xxx +:zzz" + "1 tests, 1 assertions, 1 failures." + "" + (format "bin/kaocha --config-file %s --focus '%s.bar-test/xxx-test'" (str config-file) prefix) + ] + (str/split-lines @out-str))))) (deftest ignore-files-merged (let [{:keys [_config-file test-dir] :as m} (integration/test-dir-setup {})]