From 00f21350ccf6f8109b36af1de53cba68248d642b Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 1 Jun 2021 23:13:27 -0500 Subject: [PATCH 01/21] Add data structure foundation. --- src/kaocha/testable.clj | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 236723d7..9017c29e 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -11,8 +11,10 @@ [kaocha.plugin :as plugin] [kaocha.result :as result] [kaocha.specs :refer [assert-spec]] - [kaocha.util :as util]) - (:import [clojure.lang Compiler$CompilerException])) + [kaocha.util :as util] + [kaocha.hierarchy :as hierarchy]) + (:import [clojure.lang Compiler$CompilerException] + [java.util.concurrent ArrayBlockingQueue BlockingQueue])) (def ^:dynamic *fail-fast?* "Should testing terminate immediately upon failure or error?" @@ -211,11 +213,33 @@ (run % test-plan) (plugin/run-hook :kaocha.hooks/post-test % test-plan))))) +(defn f [acc value] + (if (instance? BlockingQueue value) + (.drainTo value acc) + (.put acc value)) + acc) + +(defn f [acc value] (doto acc (.put value))) + +(def q (ArrayBlockingQueue. 1024)) +(def r (ArrayBlockingQueue. 1024)) + +(.put r 5) + + +(reduce f [q 1 2 r]) + (defn run-testables "Run a collection of testables, returning a result collection." [testables test-plan] - (let [load-error? (some ::load-error testables)] - (loop [result [] + (let [load-error? (some ::load-error testables) + ;; results (watch/make-queue) + put-return (fn [acc value] + (if (instance? BlockingQueue value) + (.drainTo value acc) + (.put acc value)) + acc)] + (loop [result (ArrayBlockingQueue. 1024) [test & testables] testables] (if test (let [test (cond-> test @@ -223,8 +247,8 @@ (assoc ::skip true)) r (run-testable test test-plan)] (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) - (reduce into result [[r] testables]) - (recur (conj result r) testables))) + (reduce put-return result [[r] testables]) + (recur (doto result (.put r)) testables))) result)))) (defn test-seq [testable] From b45731b92b475a2e6ed26bbc482245ab7052355a Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 23 Jun 2021 10:25:17 -0500 Subject: [PATCH 02/21] Work from pairing session with Arne. --- src/kaocha/testable.clj | 35 +++++++++++++++++++++++++------ src/kaocha/type/ns.clj | 2 +- test/unit/kaocha/type/ns_test.clj | 35 +++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 9017c29e..820ad35c 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -232,14 +232,33 @@ (defn run-testables "Run a collection of testables, returning a result collection." [testables test-plan] - (let [load-error? (some ::load-error testables) + (let [load-error? (some ::load-error testables)] + (loop [result [] + [test & testables] testables] + (if test + (let [test (cond-> test + (and load-error? (not (::load-error test))) + (assoc ::skip true)) + r (run-testable test test-plan)] + (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) + (reduce into result [[r] testables]) + (recur (conj result r) testables))) + result)))) + + +(defn run-testables-parallel + "Run a collection of testables, returning a result collection." + [testables test-plan] +(let [load-error? (some ::load-error testables) ;; results (watch/make-queue) put-return (fn [acc value] (if (instance? BlockingQueue value) (.drainTo value acc) (.put acc value)) - acc)] - (loop [result (ArrayBlockingQueue. 1024) + acc) + futures (map #(future (run-testable % test-plan)) testables)] + (println "Running in parallel!") + (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] (if test (let [test (cond-> test @@ -247,9 +266,13 @@ (assoc ::skip true)) r (run-testable test test-plan)] (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) - (reduce put-return result [[r] testables]) - (recur (doto result (.put r)) testables))) - result)))) + ;(reduce put-return result [[r] testables]) + (reduce into result [[r] testables]) + ;(recur (doto result (.put r)) testables) + (recur (conj result r) testables))) + result))) + (map deref futures) + )) (defn test-seq [testable] (cond->> (mapcat test-seq (remove ::skip (or (:kaocha/tests testable) diff --git a/src/kaocha/type/ns.clj b/src/kaocha/type/ns.clj index 67994b26..a6cddb39 100644 --- a/src/kaocha/type/ns.clj +++ b/src/kaocha/type/ns.clj @@ -18,7 +18,7 @@ ;; It's not guaranteed the the fixture-fn returns the result of calling the ;; tests function, so we need to put it in a box for reference. (let [result (atom (:kaocha.test-plan/tests testable))] - (fixture-fn #(swap! result testable/run-testables test-plan)) + (fixture-fn #(swap! result testable/run-testables-parallel test-plan)) @result)) (defmethod testable/-load :kaocha.type/ns [testable] diff --git a/test/unit/kaocha/type/ns_test.clj b/test/unit/kaocha/type/ns_test.clj index f1579ed3..d2b9af6a 100644 --- a/test/unit/kaocha/type/ns_test.clj +++ b/test/unit/kaocha/type/ns_test.clj @@ -67,3 +67,38 @@ (:result (with-test-ctx {:fail-fast? true} (testable/run testable testable))))))) + +(require '[kaocha.config :as config]) + +(deftest run-test-parallel ;both tests currently test the parallel version but later... + (classpath/add-classpath "fixtures/f-tests") + + (let [testable (testable/load {:kaocha.testable/type :kaocha.type/clojure.test + :kaocha.testable/id :unit + :kaocha/ns-patterns ["-test$"] + :kaocha/source-paths ["src"] + :kaocha/test-paths ["fixtures/d-tests"] + :kaocha.filter/skip-meta [:kaocha/skip]}) + + #_(testable/load {:kaocha.testable/type :kaocha.type/ns + :kaocha.testable/id :foo.bar-test + :kaocha.testable/desc "foo.bar-test" + :kaocha.ns/name 'foo.bar-test})] + (is (match? {:kaocha.testable/type :kaocha.type/ns + :kaocha.testable/id :foo.bar-test + :kaocha.ns/name 'foo.bar-test + :kaocha.ns/ns ns? + :kaocha.result/tests [{:kaocha.testable/type :kaocha.type/var + :kaocha.testable/id :foo.bar-test/a-test + :kaocha.testable/desc "a-test" + :kaocha.var/name 'foo.bar-test/a-test + :kaocha.var/var var? + :kaocha.var/test fn? + :kaocha.result/count 1 + :kaocha.result/pass 1 + :kaocha.result/error 0 + :kaocha.result/pending 0 + :kaocha.result/fail 0}]} + (:result + (with-test-ctx {:fail-fast? true} + (testable/run testable testable))))))) From f434921d7bcaff53ad57d8d20415b4c97ea1d5f2 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 21 Jul 2021 17:20:24 -0500 Subject: [PATCH 03/21] Add semi-functional version (with :parallel) as the flag. --- src/kaocha/runner.clj | 1 + src/kaocha/test_suite.clj | 20 ++++++++++++++++++-- src/kaocha/testable.clj | 22 +++++++++++++--------- src/kaocha/type/ns.clj | 12 ++++++++++-- test/unit/kaocha/type/ns_test.clj | 8 +++++++- 5 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/kaocha/runner.clj b/src/kaocha/runner.clj index e0ea312d..114ccbd3 100644 --- a/src/kaocha/runner.clj +++ b/src/kaocha/runner.clj @@ -186,6 +186,7 @@ (try+ (System/exit (apply -main* args)) (catch :kaocha/early-exit {exit-code :kaocha/early-exit} + (shutdown-agents) (System/exit exit-code)))) (defn exec-fn diff --git a/src/kaocha/test_suite.clj b/src/kaocha/test_suite.clj index bfe45881..3c712357 100644 --- a/src/kaocha/test_suite.clj +++ b/src/kaocha/test_suite.clj @@ -2,12 +2,28 @@ (:require [clojure.test :as t] [kaocha.testable :as testable])) +(defn deref-recur [testables] + (cond (future? testables) (deref testables) + (vector? testables) (doall (mapv deref-recur testables)) + (seq? testables) (deref-recur (into [] (doall testables))) + (contains? testables :kaocha.test-plan/tests) + (update testables :kaocha.test-plan/tests deref-recur) + (contains? testables :kaocha.result/tests) + (update testables :kaocha.result/tests deref-recur) + :else testables)) + (defn run [testable test-plan] (t/do-report {:type :begin-test-suite}) - (let [results (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) + (let [results (deref-recur (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) ) + ;; _ (println "Done derefing") + ;; __ (println (class results)) + ;; __ (println (class (last results))) + ;; (doall (map #(if (future? %) (deref %) %) + ;; (testable/run-testables (:kaocha.test-plan/tests testable) test-plan))) testable (-> testable (dissoc :kaocha.test-plan/tests) (assoc :kaocha.result/tests results))] (t/do-report {:type :end-test-suite :kaocha/testable testable}) - testable)) + (doto testable tap>) + )) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 820ad35c..18fd81a1 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -232,6 +232,10 @@ (defn run-testables "Run a collection of testables, returning a result collection." [testables test-plan] + (print "run-testables got a collection of size" (count testables) + " the first of which is " + (:kaocha.testable/type (first testables)) + ) (let [load-error? (some ::load-error testables)] (loop [result [] [test & testables] testables] @@ -249,15 +253,16 @@ (defn run-testables-parallel "Run a collection of testables, returning a result collection." [testables test-plan] + + (print "run-testables-parallel got a collection of size" (count testables)) (let [load-error? (some ::load-error testables) ;; results (watch/make-queue) - put-return (fn [acc value] - (if (instance? BlockingQueue value) - (.drainTo value acc) - (.put acc value)) - acc) - futures (map #(future (run-testable % test-plan)) testables)] - (println "Running in parallel!") + ;; put-return (fn [acc value] + ;; (if (instance? BlockingQueue value) + ;; (.drainTo value acc) + ;; (.put acc value)) + ;; acc) + futures (doall (map #(future (do (println "Firing off future!" (Thread/currentThread)) (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] (if test @@ -271,8 +276,7 @@ ;(recur (doto result (.put r)) testables) (recur (conj result r) testables))) result))) - (map deref futures) - )) + futures)) (defn test-seq [testable] (cond->> (mapcat test-seq (remove ::skip (or (:kaocha/tests testable) diff --git a/src/kaocha/type/ns.clj b/src/kaocha/type/ns.clj index a6cddb39..f67afd26 100644 --- a/src/kaocha/type/ns.clj +++ b/src/kaocha/type/ns.clj @@ -17,8 +17,16 @@ (defn run-tests [testable test-plan fixture-fn] ;; It's not guaranteed the the fixture-fn returns the result of calling the ;; tests function, so we need to put it in a box for reference. - (let [result (atom (:kaocha.test-plan/tests testable))] - (fixture-fn #(swap! result testable/run-testables-parallel test-plan)) + ;; (println (keys test-plan)) + ;; (println (:kaocha.testable/meta test-plan)) + (let [result (atom (:kaocha.test-plan/tests testable)) + run-testables-fn (if (:parallel testable/*config*) + testable/run-testables-parallel + testable/run-testables) + #_testable/run-testables + + ] + (fixture-fn #(swap! result run-testables-fn test-plan)) @result)) (defmethod testable/-load :kaocha.type/ns [testable] diff --git a/test/unit/kaocha/type/ns_test.clj b/test/unit/kaocha/type/ns_test.clj index d2b9af6a..2875abbd 100644 --- a/test/unit/kaocha/type/ns_test.clj +++ b/test/unit/kaocha/type/ns_test.clj @@ -77,7 +77,7 @@ :kaocha.testable/id :unit :kaocha/ns-patterns ["-test$"] :kaocha/source-paths ["src"] - :kaocha/test-paths ["fixtures/d-tests"] + :kaocha/test-paths ["fixtures/f-tests"] :kaocha.filter/skip-meta [:kaocha/skip]}) #_(testable/load {:kaocha.testable/type :kaocha.type/ns @@ -101,4 +101,10 @@ :kaocha.result/fail 0}]} (:result (with-test-ctx {:fail-fast? true} + (testable/run testable testable))))) + (is (not (nil? (:result + (binding [testable/*config* (assoc testable/*config* :parallel true)] + (with-test-ctx {:fail-fast? true + :parallel true } (testable/run testable testable))))))) + )) From b7f95edc564b84f35443385932320e3ef5d53757 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Thu, 26 Aug 2021 23:23:37 -0500 Subject: [PATCH 04/21] Add some debugging code for now. --- src/kaocha/testable.clj | 25 +++++++++++++++++++------ test/unit/kaocha/type/ns_test.clj | 4 +--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 18fd81a1..916bcca2 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -59,7 +59,12 @@ (assert-spec :kaocha/testable testable) (let [type (::type testable)] (try-load-third-party-lib type) - (assert-spec type testable))) + (try + (assert-spec type testable) + (catch Exception e + (prn e) + ) + ))) (defmulti -load "Given a testable, load the specified tests, producing a test-plan." @@ -125,8 +130,10 @@ Also performs validation, and lazy loading of the testable type's implementation." [testable test-plan] - (load-type+validate testable) + ;; (println (class testable)) (binding [*current-testable* testable] + ;; (println (:kaocha.testable/id *current-testable*)) + (load-type+validate testable) (let [run (plugin/run-hook :kaocha.hooks/wrap-run -run test-plan) result (run testable test-plan)] (if-let [history history/*history*] @@ -232,7 +239,8 @@ (defn run-testables "Run a collection of testables, returning a result collection." [testables test-plan] - (print "run-testables got a collection of size" (count testables) + (doall testables) + #_(print "run-testables got a collection of size" (count testables) " the first of which is " (:kaocha.testable/type (first testables)) ) @@ -253,8 +261,8 @@ (defn run-testables-parallel "Run a collection of testables, returning a result collection." [testables test-plan] - - (print "run-testables-parallel got a collection of size" (count testables)) + (doall testables) + ;; (print "run-testables-parallel got a collection of size" (count testables)) (let [load-error? (some ::load-error testables) ;; results (watch/make-queue) ;; put-return (fn [acc value] @@ -262,7 +270,12 @@ ;; (.drainTo value acc) ;; (.put acc value)) ;; acc) - futures (doall (map #(future (do (println "Firing off future!" (Thread/currentThread)) (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) testables))] + futures (doall (map #(do + (println (:parallel *config*) \space (.getName (Thread/currentThread))) + (future + ;(do #_(println "Firing off future!" (Thread/currentThread)) ) + (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) + testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] (if test diff --git a/test/unit/kaocha/type/ns_test.clj b/test/unit/kaocha/type/ns_test.clj index 2875abbd..710cc0b5 100644 --- a/test/unit/kaocha/type/ns_test.clj +++ b/test/unit/kaocha/type/ns_test.clj @@ -68,9 +68,7 @@ (with-test-ctx {:fail-fast? true} (testable/run testable testable))))))) -(require '[kaocha.config :as config]) - -(deftest run-test-parallel ;both tests currently test the parallel version but later... +(deftest run-test-parallel (classpath/add-classpath "fixtures/f-tests") (let [testable (testable/load {:kaocha.testable/type :kaocha.type/clojure.test From 93405e5ba21a9d9dfbeb942400c599ff0667062e Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Fri, 27 Aug 2021 00:10:41 -0500 Subject: [PATCH 05/21] Add workarounds for now. --- src/kaocha/testable.clj | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 916bcca2..3fa37c99 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -47,6 +47,13 @@ (try-require (symbol (namespace type)))) (try-require (symbol (name type))))) + +(defn- try-assert-spec [type testable n] + (let [ result (try (assert-spec type testable) (catch Exception _e false))] + (if (or result (<= n 1)) result + (try-assert-spec type testable (dec n))) ;otherwise, retry + )) + (defn- load-type+validate "Try to load a testable type, and validate it both to be a valid generic testable, and a valid instance given the type. @@ -60,11 +67,9 @@ (let [type (::type testable)] (try-load-third-party-lib type) (try - (assert-spec type testable) + (try-assert-spec type testable 3) (catch Exception e - (prn e) - ) - ))) + (output/warn (format "Could not load %s. This is a known bug in parallelization.\n%s" type e)))))) (defmulti -load "Given a testable, load the specified tests, producing a test-plan." @@ -130,10 +135,8 @@ Also performs validation, and lazy loading of the testable type's implementation." [testable test-plan] - ;; (println (class testable)) + (load-type+validate testable) (binding [*current-testable* testable] - ;; (println (:kaocha.testable/id *current-testable*)) - (load-type+validate testable) (let [run (plugin/run-hook :kaocha.hooks/wrap-run -run test-plan) result (run testable test-plan)] (if-let [history history/*history*] @@ -220,6 +223,13 @@ (run % test-plan) (plugin/run-hook :kaocha.hooks/post-test % test-plan))))) + +(defn try-run-testable [test test-plan n] + (let [ result (try (run-testable test test-plan) (catch Exception _e false))] + (if (or result (> n 1)) result ;success or last try, return + (try-run-testable test test-plan (dec n))) ;otherwise retry + )) + (defn f [acc value] (if (instance? BlockingQueue value) (.drainTo value acc) @@ -274,7 +284,7 @@ (println (:parallel *config*) \space (.getName (Thread/currentThread))) (future ;(do #_(println "Firing off future!" (Thread/currentThread)) ) - (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) + (binding [*config* (dissoc *config* :parallel)] (try-run-testable % test-plan 3)))) testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] From 8069a9a7f5fc1facbba902f942f34921b23d3300 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Fri, 27 Aug 2021 00:22:17 -0500 Subject: [PATCH 06/21] Replace workarounds by locking require. --- src/kaocha/testable.clj | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 3fa37c99..54864a20 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -30,13 +30,17 @@ and `:line`." nil) + +(def REQUIRE_LOCK (Object.)) + (defn add-desc [testable description] (assoc testable ::desc (str (name (::id testable)) " (" description ")"))) (defn- try-require [n] (try - (require n) + (locking REQUIRE_LOCK + (require n)) true (catch java.io.FileNotFoundException e false))) @@ -48,10 +52,10 @@ (try-require (symbol (name type))))) -(defn- try-assert-spec [type testable n] +(defn- retry-assert-spec [type testable n] (let [ result (try (assert-spec type testable) (catch Exception _e false))] (if (or result (<= n 1)) result - (try-assert-spec type testable (dec n))) ;otherwise, retry + (retry-assert-spec type testable (dec n))) ;otherwise, retry )) (defn- load-type+validate @@ -67,7 +71,7 @@ (let [type (::type testable)] (try-load-third-party-lib type) (try - (try-assert-spec type testable 3) + (assert-spec type testable) (catch Exception e (output/warn (format "Could not load %s. This is a known bug in parallelization.\n%s" type e)))))) @@ -284,7 +288,7 @@ (println (:parallel *config*) \space (.getName (Thread/currentThread))) (future ;(do #_(println "Firing off future!" (Thread/currentThread)) ) - (binding [*config* (dissoc *config* :parallel)] (try-run-testable % test-plan 3)))) + (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] From 249e657454e8dfd6ae0ae02f3196f07eb0f0cb88 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 19 Oct 2021 16:15:01 -0500 Subject: [PATCH 07/21] Allow parallelization to happen at any level. --- src/kaocha/result.clj | 1 + src/kaocha/test_suite.clj | 12 ++---------- src/kaocha/testable.clj | 38 ++++++++++++++++++++++++++++++++------ src/kaocha/type/ns.clj | 10 ++-------- 4 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/kaocha/result.clj b/src/kaocha/result.clj index 7b491b7b..95fd2963 100644 --- a/src/kaocha/result.clj +++ b/src/kaocha/result.clj @@ -34,6 +34,7 @@ (defn ^:no-gen testable-totals "Return a map of summed up results for a testable, including descendants." [testable] + ;; (prn "TESTABLE: " testable) (if-let [testables (::tests testable)] (merge testable (totals testables)) (merge (sum) testable))) diff --git a/src/kaocha/test_suite.clj b/src/kaocha/test_suite.clj index 3c712357..eb3e66a8 100644 --- a/src/kaocha/test_suite.clj +++ b/src/kaocha/test_suite.clj @@ -2,19 +2,11 @@ (:require [clojure.test :as t] [kaocha.testable :as testable])) -(defn deref-recur [testables] - (cond (future? testables) (deref testables) - (vector? testables) (doall (mapv deref-recur testables)) - (seq? testables) (deref-recur (into [] (doall testables))) - (contains? testables :kaocha.test-plan/tests) - (update testables :kaocha.test-plan/tests deref-recur) - (contains? testables :kaocha.result/tests) - (update testables :kaocha.result/tests deref-recur) - :else testables)) + (defn run [testable test-plan] (t/do-report {:type :begin-test-suite}) - (let [results (deref-recur (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) ) + (let [results (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) ;; _ (println "Done derefing") ;; __ (println (class results)) ;; __ (println (class (last results))) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 54864a20..c978b792 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -58,6 +58,16 @@ (retry-assert-spec type testable (dec n))) ;otherwise, retry )) +(defn deref-recur [testables] + (cond (future? testables) (deref testables) + (vector? testables) (doall (mapv deref-recur testables)) + (seq? testables) (deref-recur (into [] (doall testables))) + (contains? testables :kaocha.test-plan/tests) + (update testables :kaocha.test-plan/tests deref-recur) + (contains? testables :kaocha.result/tests) + (update testables :kaocha.result/tests deref-recur) + :else testables)) + (defn- load-type+validate "Try to load a testable type, and validate it both to be a valid generic testable, and a valid instance given the type. @@ -179,6 +189,7 @@ [file line] (util/compiler-exception-file-and-line error) file (::load-error-file test file) line (::load-error-line test line) + thread (.getName (Thread/currentThread)) m (if-let [message (::load-error-message test)] {:type :error :message message @@ -190,7 +201,8 @@ :kaocha/testable test}) m (cond-> m file (assoc :file file) - line (assoc :line line))] + line (assoc :line line) + thread (assoc :thread thread))] (t/do-report (assoc m :type :kaocha/begin-suite)) (binding [*fail-fast?* false] (t/do-report m)) @@ -250,7 +262,7 @@ (reduce f [q 1 2 r]) -(defn run-testables +(defn run-testables-serial "Run a collection of testables, returning a result collection." [testables test-plan] (doall testables) @@ -284,11 +296,17 @@ ;; (.drainTo value acc) ;; (.put acc value)) ;; acc) + ;; _ (println "nested tests?" (:parallel-test-level *config* false)) + type(:kaocha.testable/type test-plan) + _ (println type) futures (doall (map #(do - (println (:parallel *config*) \space (.getName (Thread/currentThread))) (future - ;(do #_(println "Firing off future!" (Thread/currentThread)) ) - (binding [*config* (dissoc *config* :parallel)] (run-testable % test-plan)))) + (binding [;*config* (if (:parallel-test-level *config* false) + ; (dissoc *config* :parallel) *config* ) + + *config* (update *config* :levels (fn [x] (if (nil? x) 1 (inc x)))) ] + (prn (:levels *config* 0)) + (run-testable % test-plan)))) testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] @@ -303,7 +321,15 @@ ;(recur (doto result (.put r)) testables) (recur (conj result r) testables))) result))) - futures)) + (deref-recur futures))) + + +(defn run-testables + [testables test-plan] + (if (:parallel *config*) + (doall (run-testables-parallel testables test-plan)) + (run-testables-serial testables test-plan))) + (defn test-seq [testable] (cond->> (mapcat test-seq (remove ::skip (or (:kaocha/tests testable) diff --git a/src/kaocha/type/ns.clj b/src/kaocha/type/ns.clj index f67afd26..d4c92683 100644 --- a/src/kaocha/type/ns.clj +++ b/src/kaocha/type/ns.clj @@ -19,14 +19,8 @@ ;; tests function, so we need to put it in a box for reference. ;; (println (keys test-plan)) ;; (println (:kaocha.testable/meta test-plan)) - (let [result (atom (:kaocha.test-plan/tests testable)) - run-testables-fn (if (:parallel testable/*config*) - testable/run-testables-parallel - testable/run-testables) - #_testable/run-testables - - ] - (fixture-fn #(swap! result run-testables-fn test-plan)) + (let [result (atom (:kaocha.test-plan/tests testable))] + (fixture-fn #(swap! result testable/run-testables test-plan)) @result)) (defmethod testable/-load :kaocha.type/ns [testable] From df0732683d0e9a770a66b96f98945348dc95226b Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 19 Oct 2021 16:34:41 -0500 Subject: [PATCH 08/21] Clean up. --- src/kaocha/testable.clj | 48 ++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index c978b792..98018486 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -288,40 +288,30 @@ "Run a collection of testables, returning a result collection." [testables test-plan] (doall testables) - ;; (print "run-testables-parallel got a collection of size" (count testables)) -(let [load-error? (some ::load-error testables) - ;; results (watch/make-queue) - ;; put-return (fn [acc value] - ;; (if (instance? BlockingQueue value) - ;; (.drainTo value acc) - ;; (.put acc value)) - ;; acc) - ;; _ (println "nested tests?" (:parallel-test-level *config* false)) - type(:kaocha.testable/type test-plan) - _ (println type) + (let [load-error? (some ::load-error testables) + types (set (:parallel-children-exclude *config*)) futures (doall (map #(do (future - (binding [;*config* (if (:parallel-test-level *config* false) - ; (dissoc *config* :parallel) *config* ) - - *config* (update *config* :levels (fn [x] (if (nil? x) 1 (inc x)))) ] - (prn (:levels *config* 0)) + (binding [*config* + (cond-> *config* + (contains? types (:kaocha.testable/type %)) (dissoc :parallel) + true (update :levels (fn [x] (if (nil? x) 1 (inc x))))) ] (run-testable % test-plan)))) testables))] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) - [test & testables] testables] - (if test - (let [test (cond-> test - (and load-error? (not (::load-error test))) - (assoc ::skip true)) - r (run-testable test test-plan)] - (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) - ;(reduce put-return result [[r] testables]) - (reduce into result [[r] testables]) - ;(recur (doto result (.put r)) testables) - (recur (conj result r) testables))) - result))) - (deref-recur futures))) + [test & testables] testables] + (if test + (let [test (cond-> test + (and load-error? (not (::load-error test))) + (assoc ::skip true)) + r (run-testable test test-plan)] + (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) + ;(reduce put-return result [[r] testables]) + (reduce into result [[r] testables]) + ;(recur (doto result (.put r)) testables) + (recur (conj result r) testables))) + result))) + (deref-recur futures))) (defn run-testables From c9c4b9512cd81eee22685fecef2293d62ad1c620 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 19 Oct 2021 17:12:50 -0500 Subject: [PATCH 09/21] Clean up. --- src/kaocha/testable.clj | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 98018486..a8b8f030 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -290,14 +290,14 @@ (doall testables) (let [load-error? (some ::load-error testables) types (set (:parallel-children-exclude *config*)) - futures (doall (map #(do - (future - (binding [*config* - (cond-> *config* - (contains? types (:kaocha.testable/type %)) (dissoc :parallel) - true (update :levels (fn [x] (if (nil? x) 1 (inc x))))) ] - (run-testable % test-plan)))) - testables))] + futures (map #(do + (future + (binding [*config* + (cond-> *config* + (contains? types (:kaocha.testable/type %)) (dissoc :parallel) + true (update :levels (fn [x] (if (nil? x) 1 (inc x))))) ] + (run-testable % test-plan)))) + testables)] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) [test & testables] testables] (if test From d52eb38681027054c8cf8c7c91762c0520cb0c27 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Thu, 21 Oct 2021 01:27:11 -0500 Subject: [PATCH 10/21] Add benchmarking. --- repl_sessions/benchmark.clj | 97 +++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 repl_sessions/benchmark.clj diff --git a/repl_sessions/benchmark.clj b/repl_sessions/benchmark.clj new file mode 100644 index 00000000..4a205481 --- /dev/null +++ b/repl_sessions/benchmark.clj @@ -0,0 +1,97 @@ + +(ns benchmark + (:require [criterium.core :as c]) + (:import [java.util.concurrent Executors ]) + ) + +(def thread-pool (Executors/newFixedThreadPool 10)) + +(defn math-direct [] + (+ 1 1)) + +(defn math-future [] + (deref + (future (+ 1 1)))) + +(defn math-thread [] + (let [result (atom nil)] + (doto (Thread. (fn [] (reset! result (+ 1 1)))) + (.start) + (.join)) + @result)) + +(defn math-threadpool [] + (let [result (atom nil)] + (.get (.submit thread-pool (fn [] (reset! result (+ 1 1))) )) + @result)) + +(defn math-threadpool-no-atom [] + (.get (.submit thread-pool (fn [] (+ 1 1)) ))) + + +(c/bench (math-direct) ) +; (out) Evaluation count : 6215391600 in 60 samples of 103589860 calls. +; (out) Execution time mean : 2,015262 ns +; (out) Execution time std-deviation : 0,497743 ns +; (out) Execution time lower quantile : 1,442374 ns ( 2,5%) +; (out) Execution time upper quantile : 3,392990 ns (97,5%) +; (out) Overhead used : 7,915626 ns +; (out) +; (out) Found 5 outliers in 60 samples (8,3333 %) +; (out) low-severe 3 (5,0000 %) +; (out) low-mild 2 (3,3333 %) +; (out) Variance from outliers : 94,6147 % Variance is severely inflated by outliers + +(c/bench (math-future) ) +; (out) Evaluation count : 3735420 in 60 samples of 62257 calls. +; (out) Execution time mean : 16,635809 µs +; (out) Execution time std-deviation : 1,104338 µs +; (out) Execution time lower quantile : 15,397518 µs ( 2,5%) +; (out) Execution time upper quantile : 19,751883 µs (97,5%) +; (out) Overhead used : 7,915626 ns +; (out) +; (out) Found 6 outliers in 60 samples (10,0000 %) +; (out) low-severe 3 (5,0000 %) +; (out) low-mild 3 (5,0000 %) +; (out) Variance from outliers : 50,0892 % Variance is severely inflated by outliers + +(c/bench (math-thread)) + +; (out) Evaluation count : 774420 in 60 samples of 12907 calls. +; (out) Execution time mean : 82,513236 µs +; (out) Execution time std-deviation : 5,706987 µs +; (out) Execution time lower quantile : 75,772237 µs ( 2,5%) +; (out) Execution time upper quantile : 91,971212 µs (97,5%) +; (out) Overhead used : 7,915626 ns +; (out) +; (out) Found 1 outliers in 60 samples (1,6667 %) +; (out) low-severe 1 (1,6667 %) +; (out) Variance from outliers : 51,7849 % Variance is severely inflated by outliers + +(c/bench (math-threadpool)) +; (out) Evaluation count : 3815100 in 60 samples of 63585 calls. +; (out) Execution time mean : 16,910124 µs +; (out) Execution time std-deviation : 2,443261 µs +; (out) Execution time lower quantile : 14,670118 µs ( 2,5%) +; (out) Execution time upper quantile : 23,743868 µs (97,5%) +; (out) Overhead used : 7,915626 ns +; (out) +; (out) Found 3 outliers in 60 samples (5,0000 %) +; (out) low-severe 2 (3,3333 %) +; (out) low-mild 1 (1,6667 %) +; (out) Variance from outliers : 82,4670 % Variance is severely inflated by outliers + + +(c/bench (math-threadpool-no-atom)) + +; (out) Evaluation count : 3794940 in 60 samples of 63249 calls. +; (out) Execution time mean : 16,182655 µs +; (out) Execution time std-deviation : 1,215451 µs +; (out) Execution time lower quantile : 14,729393 µs ( 2,5%) +; (out) Execution time upper quantile : 18,549902 µs (97,5%) +; (out) Overhead used : 7,915626 ns +; (out) +; (out) Found 3 outliers in 60 samples (5,0000 %) +; (out) low-severe 2 (3,3333 %) +; (out) low-mild 1 (1,6667 %) +; (out) Variance from outliers : 56,7625 % Variance is severely inflated by outliers From 0990efca19b908943e6d4845e138aec7e9bc9852 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 8 Mar 2022 13:58:24 -0600 Subject: [PATCH 11/21] Add warning. --- src/kaocha/api.clj | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/kaocha/api.clj b/src/kaocha/api.clj index 69e99c97..2519e0ff 100644 --- a/src/kaocha/api.clj +++ b/src/kaocha/api.clj @@ -110,8 +110,13 @@ (count (testable/test-seq-with-skipped test-plan)))) (output/warn (str "No tests were found. This may be an issue in your Kaocha test configuration." " To investigate, check the :test-paths and :ns-patterns keys in tests.edn."))) - (throw+ {:kaocha/early-exit 0 })) + (throw+ {:kaocha/early-exit 0})) + (when (:parallel config) + (output/warn (str "Parallelization enabled. This is a beta " + "feature. If you encounter errors, try " + "running with the feature disabled and " + "consider filing a PR."))) (when (find-ns 'matcher-combinators.core) (require 'kaocha.matcher-combinators)) From 34456077188bd038f508a76cc5cb4eb2157b414d Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Tue, 8 Mar 2022 15:38:24 -0600 Subject: [PATCH 12/21] Add commandline flag. --- src/kaocha/api.clj | 6 ++--- src/kaocha/config.clj | 5 ++-- src/kaocha/runner.clj | 5 ++-- src/kaocha/testable.clj | 56 ++++++++++++++++++----------------------- 4 files changed, 33 insertions(+), 39 deletions(-) diff --git a/src/kaocha/api.clj b/src/kaocha/api.clj index 2519e0ff..d1dfda31 100644 --- a/src/kaocha/api.clj +++ b/src/kaocha/api.clj @@ -113,10 +113,10 @@ (throw+ {:kaocha/early-exit 0})) (when (:parallel config) - (output/warn (str "Parallelization enabled. This is a beta " - "feature. If you encounter errors, try " + (output/warn (str "Parallelization enabled. This feature is in " + "beta If you encounter errors, try " "running with the feature disabled and " - "consider filing a PR."))) + "consider filing an issue."))) (when (find-ns 'matcher-combinators.core) (require 'kaocha.matcher-combinators)) diff --git a/src/kaocha/config.clj b/src/kaocha/config.clj index 1c84906e..c575fc39 100644 --- a/src/kaocha/config.clj +++ b/src/kaocha/config.clj @@ -27,8 +27,8 @@ (symbol? v)) (update config k vary-meta assoc :replace true) (do - (output/error "Test suite configuration value with key " k " should be a collection or symbol, but got '" v "' of type " (type v)) - (throw+ {:kaocha/early-exit 252})))) + (output/error "Test suite configuration value with key " k " should be a collection or symbol, but got '" v "' of type " (type v)) + (throw+ {:kaocha/early-exit 252})))) config)) (defn merge-config [c1 c2] @@ -198,6 +198,7 @@ (some? (:color options)) (assoc :kaocha/color? (:color options)) (some? (:diff-style options)) (assoc :kaocha/diff-style (:diff-style options)) (:plugin options) (update :kaocha/plugins #(distinct (concat % (:plugin options)))) + (some? (:parallel options)) (assoc :parallel (:parallel options)) true (assoc :kaocha/cli-options options))) (defn apply-cli-args [config args] diff --git a/src/kaocha/runner.clj b/src/kaocha/runner.clj index 114ccbd3..2478455c 100644 --- a/src/kaocha/runner.clj +++ b/src/kaocha/runner.clj @@ -8,6 +8,7 @@ [clojure.spec.alpha :as spec] [clojure.string :as str] [clojure.tools.cli :as cli] + [clojure.pprint :as pp] [expound.alpha :as expound] [kaocha.api :as api] [kaocha.config :as config] @@ -34,6 +35,7 @@ [nil "--[no-]fail-fast" "Stop testing after the first failure."] [nil "--[no-]color" "Enable/disable ANSI color codes in output. Defaults to true."] [nil "--[no-]watch" "Watch filesystem for changes and re-run tests."] + [nil "--[no-]parallel" "Run tests in parallel. Warning: This feature is beta."] [nil "--reporter SYMBOL" "Change the test reporter, can be specified multiple times." :parse-fn (fn [s] (let [sym (symbol s)] @@ -42,8 +44,7 @@ (symbol "kaocha.report" s)))) :assoc-fn accumulate] [nil "--diff-style STYLE" "The style of diff to print on failing tests, either :none or :deep" - :parse-fn parse-kw - ] + :parse-fn parse-kw] [nil "--plugin KEYWORD" "Load the given plugin." :parse-fn (fn [s] (let [kw (parse-kw s)] diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index a8b8f030..87f3fedf 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -11,7 +11,7 @@ [kaocha.plugin :as plugin] [kaocha.result :as result] [kaocha.specs :refer [assert-spec]] - [kaocha.util :as util] + [kaocha.util :as util] [kaocha.hierarchy :as hierarchy]) (:import [clojure.lang Compiler$CompilerException] [java.util.concurrent ArrayBlockingQueue BlockingQueue])) @@ -30,7 +30,6 @@ and `:line`." nil) - (def REQUIRE_LOCK (Object.)) (defn add-desc [testable description] @@ -39,7 +38,7 @@ (defn- try-require [n] (try - (locking REQUIRE_LOCK + (locking REQUIRE_LOCK (require n)) true (catch java.io.FileNotFoundException e @@ -51,12 +50,11 @@ (try-require (symbol (namespace type)))) (try-require (symbol (name type))))) - (defn- retry-assert-spec [type testable n] - (let [ result (try (assert-spec type testable) (catch Exception _e false))] + (let [result (try (assert-spec type testable) (catch Exception _e false))] (if (or result (<= n 1)) result - (retry-assert-spec type testable (dec n))) ;otherwise, retry - )) + (retry-assert-spec type testable (dec n))) ;otherwise, retry +)) (defn deref-recur [testables] (cond (future? testables) (deref testables) @@ -80,7 +78,7 @@ (assert-spec :kaocha/testable testable) (let [type (::type testable)] (try-load-third-party-lib type) - (try + (try (assert-spec type testable) (catch Exception e (output/warn (format "Could not load %s. This is a known bug in parallelization.\n%s" type e)))))) @@ -126,8 +124,8 @@ (throw t))))) (s/fdef load - :args (s/cat :testable :kaocha/testable) - :ret :kaocha.test-plan/testable) + :args (s/cat :testable :kaocha/testable) + :ret :kaocha.test-plan/testable) (defmulti -run "Given a test-plan, perform the tests, returning the test results." @@ -239,18 +237,17 @@ (run % test-plan) (plugin/run-hook :kaocha.hooks/post-test % test-plan))))) - (defn try-run-testable [test test-plan n] - (let [ result (try (run-testable test test-plan) (catch Exception _e false))] + (let [result (try (run-testable test test-plan) (catch Exception _e false))] (if (or result (> n 1)) result ;success or last try, return - (try-run-testable test test-plan (dec n))) ;otherwise retry - )) + (try-run-testable test test-plan (dec n))) ;otherwise retry +)) -(defn f [acc value] - (if (instance? BlockingQueue value) - (.drainTo value acc) - (.put acc value)) - acc) +(defn f [acc value] + (if (instance? BlockingQueue value) + (.drainTo value acc) + (.put acc value)) + acc) (defn f [acc value] (doto acc (.put value))) @@ -259,7 +256,6 @@ (.put r 5) - (reduce f [q 1 2 r]) (defn run-testables-serial @@ -267,9 +263,8 @@ [testables test-plan] (doall testables) #_(print "run-testables got a collection of size" (count testables) - " the first of which is " - (:kaocha.testable/type (first testables)) - ) + " the first of which is " + (:kaocha.testable/type (first testables))) (let [load-error? (some ::load-error testables)] (loop [result [] [test & testables] testables] @@ -283,19 +278,18 @@ (recur (conj result r) testables))) result)))) - (defn run-testables-parallel "Run a collection of testables, returning a result collection." [testables test-plan] (doall testables) (let [load-error? (some ::load-error testables) types (set (:parallel-children-exclude *config*)) - futures (map #(do - (future - (binding [*config* - (cond-> *config* + futures (map #(do + (future + (binding [*config* + (cond-> *config* (contains? types (:kaocha.testable/type %)) (dissoc :parallel) - true (update :levels (fn [x] (if (nil? x) 1 (inc x))))) ] + true (update :levels (fn [x] (if (nil? x) 1 (inc x)))))] (run-testable % test-plan)))) testables)] (comment (loop [result [] ;(ArrayBlockingQueue. 1024) @@ -313,14 +307,12 @@ result))) (deref-recur futures))) - -(defn run-testables +(defn run-testables [testables test-plan] (if (:parallel *config*) (doall (run-testables-parallel testables test-plan)) (run-testables-serial testables test-plan))) - (defn test-seq [testable] (cond->> (mapcat test-seq (remove ::skip (or (:kaocha/tests testable) (:kaocha.test-plan/tests testable) From c1615a4069cf00d0871c3355a2b12ef9d4ab9dfa Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 6 Apr 2022 18:03:08 -0500 Subject: [PATCH 13/21] Clean up. --- src/kaocha/result.clj | 1 - src/kaocha/test_suite.clj | 10 +--------- src/kaocha/testable.clj | 14 -------------- src/kaocha/type/ns.clj | 2 -- test/unit/kaocha/type/ns_test.clj | 11 +++-------- 5 files changed, 4 insertions(+), 34 deletions(-) diff --git a/src/kaocha/result.clj b/src/kaocha/result.clj index 95fd2963..7b491b7b 100644 --- a/src/kaocha/result.clj +++ b/src/kaocha/result.clj @@ -34,7 +34,6 @@ (defn ^:no-gen testable-totals "Return a map of summed up results for a testable, including descendants." [testable] - ;; (prn "TESTABLE: " testable) (if-let [testables (::tests testable)] (merge testable (totals testables)) (merge (sum) testable))) diff --git a/src/kaocha/test_suite.clj b/src/kaocha/test_suite.clj index eb3e66a8..56ffe354 100644 --- a/src/kaocha/test_suite.clj +++ b/src/kaocha/test_suite.clj @@ -2,20 +2,12 @@ (:require [clojure.test :as t] [kaocha.testable :as testable])) - - (defn run [testable test-plan] (t/do-report {:type :begin-test-suite}) (let [results (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) - ;; _ (println "Done derefing") - ;; __ (println (class results)) - ;; __ (println (class (last results))) - ;; (doall (map #(if (future? %) (deref %) %) - ;; (testable/run-testables (:kaocha.test-plan/tests testable) test-plan))) testable (-> testable (dissoc :kaocha.test-plan/tests) (assoc :kaocha.result/tests results))] (t/do-report {:type :end-test-suite :kaocha/testable testable}) - (doto testable tap>) - )) + testable)) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 87f3fedf..a21d9384 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -243,20 +243,6 @@ (try-run-testable test test-plan (dec n))) ;otherwise retry )) -(defn f [acc value] - (if (instance? BlockingQueue value) - (.drainTo value acc) - (.put acc value)) - acc) - -(defn f [acc value] (doto acc (.put value))) - -(def q (ArrayBlockingQueue. 1024)) -(def r (ArrayBlockingQueue. 1024)) - -(.put r 5) - -(reduce f [q 1 2 r]) (defn run-testables-serial "Run a collection of testables, returning a result collection." diff --git a/src/kaocha/type/ns.clj b/src/kaocha/type/ns.clj index d4c92683..67994b26 100644 --- a/src/kaocha/type/ns.clj +++ b/src/kaocha/type/ns.clj @@ -17,8 +17,6 @@ (defn run-tests [testable test-plan fixture-fn] ;; It's not guaranteed the the fixture-fn returns the result of calling the ;; tests function, so we need to put it in a box for reference. - ;; (println (keys test-plan)) - ;; (println (:kaocha.testable/meta test-plan)) (let [result (atom (:kaocha.test-plan/tests testable))] (fixture-fn #(swap! result testable/run-testables test-plan)) @result)) diff --git a/test/unit/kaocha/type/ns_test.clj b/test/unit/kaocha/type/ns_test.clj index 710cc0b5..5000f29a 100644 --- a/test/unit/kaocha/type/ns_test.clj +++ b/test/unit/kaocha/type/ns_test.clj @@ -76,12 +76,8 @@ :kaocha/ns-patterns ["-test$"] :kaocha/source-paths ["src"] :kaocha/test-paths ["fixtures/f-tests"] - :kaocha.filter/skip-meta [:kaocha/skip]}) + :kaocha.filter/skip-meta [:kaocha/skip]})] - #_(testable/load {:kaocha.testable/type :kaocha.type/ns - :kaocha.testable/id :foo.bar-test - :kaocha.testable/desc "foo.bar-test" - :kaocha.ns/name 'foo.bar-test})] (is (match? {:kaocha.testable/type :kaocha.type/ns :kaocha.testable/id :foo.bar-test :kaocha.ns/name 'foo.bar-test @@ -103,6 +99,5 @@ (is (not (nil? (:result (binding [testable/*config* (assoc testable/*config* :parallel true)] (with-test-ctx {:fail-fast? true - :parallel true } - (testable/run testable testable))))))) - )) + :parallel true} + (testable/run testable testable))))))))) From 0c87a167add5b6d502c0c83e445be6b3381706fa Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 6 Apr 2022 18:13:40 -0500 Subject: [PATCH 14/21] Add docs and CHANGELOG entry. --- CHANGELOG.md | 6 +++++- doc/04_running_kaocha_cli.md | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 29dd2b01..5eb4e76c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Added +- You can now parallelize tests by enabling the `:parallel` key or + the `--parallel` flag. This is still a beta feature, but works on a variety + of code bases. + ## Fixed ## Changed @@ -888,4 +892,4 @@ namespace. - The configuration format has changed, you should now start with the `#kaocha {}` tagged reader literal in `tests.edn` to provide defaults. If you want more control then overwrite `tests.edn` with the output of `--print-config` and - tweak. \ No newline at end of file + tweak. diff --git a/doc/04_running_kaocha_cli.md b/doc/04_running_kaocha_cli.md index e4fb4351..f4205fe4 100644 --- a/doc/04_running_kaocha_cli.md +++ b/doc/04_running_kaocha_cli.md @@ -95,6 +95,31 @@ unhelpful output in a particular scenario, you can turn it off using the ![Terminal screenshot showing an expected value of "{:expected-key 1}" and an actual value. ":unexpected-key 1" is in green because it is an extra key not expected and "expected-key 1" is in red because it was expected but not present.](./deep-diff.png) +## Parallelization + +Kaocha allows you to run tests in parallel using the `:parallel` key or +`--parallel` flag. This is primarily useful for I/O heavy tests, but could also +be useful for CPU-bound tests. + +Before enabling parallelization, be sure to test it. Consider using a tool like +`bench` or `hyperfine`. While Kaocha's built-in profiling tools are great for +identifying specific slow tests, but don't repeatedly measure your entire test suite +to account for variation and noise. If you want to test it on CI, test it for CI +specifically. CI runners are often lower powered than even a middle-of-the-road laptop. + +`test.check` tests consist of repeatedly testing a property against random data. +In principle, these tests would be an excellent use case for parallelization. +Because this repeated testing happens within `test.check`, Kaocha sees it as a +single test. If you have many property-based tests that take a similar amount of +time, parallelization is a great fit. However, if you have one or two +property-based tests that take the bulk of the time, parallelization may not +make a significant difference because the work cannot be split up. + +If you want to disable parallelization that's enabled in your configuration, you can +pass `--no-parallel`. If you find yourself frequently reaching for this flag, +it's probably worth reconsidering your configuration---having to frequently +disable parallelization might be negating any time saved by parallelization. + ## Debug information `--version` prints version information, whereas `--test-help` will print the From 0d82ed25f0ef62fdbfc785e2a568c515908c85b2 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 6 Apr 2022 18:54:44 -0500 Subject: [PATCH 15/21] Fix dash. --- doc/04_running_kaocha_cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/04_running_kaocha_cli.md b/doc/04_running_kaocha_cli.md index f4205fe4..6c9cf039 100644 --- a/doc/04_running_kaocha_cli.md +++ b/doc/04_running_kaocha_cli.md @@ -117,7 +117,7 @@ make a significant difference because the work cannot be split up. If you want to disable parallelization that's enabled in your configuration, you can pass `--no-parallel`. If you find yourself frequently reaching for this flag, -it's probably worth reconsidering your configuration---having to frequently +it's probably worth reconsidering your configuration—having to frequently disable parallelization might be negating any time saved by parallelization. ## Debug information From 56c1dc37628a2393bc53fc007af3338b64c596e4 Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 6 Apr 2022 19:00:56 -0500 Subject: [PATCH 16/21] Tweak wording further. --- doc/04_running_kaocha_cli.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/04_running_kaocha_cli.md b/doc/04_running_kaocha_cli.md index 6c9cf039..153be341 100644 --- a/doc/04_running_kaocha_cli.md +++ b/doc/04_running_kaocha_cli.md @@ -101,18 +101,19 @@ Kaocha allows you to run tests in parallel using the `:parallel` key or `--parallel` flag. This is primarily useful for I/O heavy tests, but could also be useful for CPU-bound tests. -Before enabling parallelization, be sure to test it. Consider using a tool like +Before enabling parallelization, strongly consider timing it to ensure it +actually makes a difference. Consider using a tool like `bench` or `hyperfine`. While Kaocha's built-in profiling tools are great for -identifying specific slow tests, but don't repeatedly measure your entire test suite -to account for variation and noise. If you want to test it on CI, test it for CI -specifically. CI runners are often lower powered than even a middle-of-the-road laptop. +identifying specific tests that take a disproportionate amount of time, they don't repeatedly measure your entire test suite +to account for variation and noise. If you want to use parallelization to +speed up continuous integration, try to use the CI service itself or similar hardware. CI runners are often lower powered than even a middle-of-the-road laptop. `test.check` tests consist of repeatedly testing a property against random data. In principle, these tests would be an excellent use case for parallelization. -Because this repeated testing happens within `test.check`, Kaocha sees it as a -single test. If you have many property-based tests that take a similar amount of +However, because this repeated testing happens within `test.check`, Kaocha sees each `defspec` as a +single test. If you have many property-based tests that take a significant amount of time, parallelization is a great fit. However, if you have one or two -property-based tests that take the bulk of the time, parallelization may not +property-based tests that take up the bulk of the runtme time, parallelization may not make a significant difference because the work cannot be split up. If you want to disable parallelization that's enabled in your configuration, you can From 47627b790e63ff51cdd7beb3f47dd6a14e0e5dcb Mon Sep 17 00:00:00 2001 From: Alys Brooks Date: Wed, 21 Jul 2021 17:20:24 -0500 Subject: [PATCH 17/21] Add semi-functional version (with :parallel) as the flag. --- src/kaocha/test_suite.clj | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/kaocha/test_suite.clj b/src/kaocha/test_suite.clj index 56ffe354..f4410c88 100644 --- a/src/kaocha/test_suite.clj +++ b/src/kaocha/test_suite.clj @@ -2,6 +2,16 @@ (:require [clojure.test :as t] [kaocha.testable :as testable])) +(defn deref-recur [testables] + (cond (future? testables) (deref testables) + (vector? testables) (doall (mapv deref-recur testables)) + (seq? testables) (deref-recur (into [] (doall testables))) + (contains? testables :kaocha.test-plan/tests) + (update testables :kaocha.test-plan/tests deref-recur) + (contains? testables :kaocha.result/tests) + (update testables :kaocha.result/tests deref-recur) + :else testables)) + (defn run [testable test-plan] (t/do-report {:type :begin-test-suite}) (let [results (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) From 3c68292963c049508fdee62259f19a6ecb0ee071 Mon Sep 17 00:00:00 2001 From: John Shaffer Date: Tue, 8 Mar 2022 13:50:28 -0800 Subject: [PATCH 18/21] Add :parallel-threads option to set thread pool size --- deps.edn | 1 + src/kaocha/test_suite.clj | 10 -------- src/kaocha/testable.clj | 52 +++++++++++---------------------------- 3 files changed, 16 insertions(+), 47 deletions(-) diff --git a/deps.edn b/deps.edn index cde27a9e..917c53da 100644 --- a/deps.edn +++ b/deps.edn @@ -5,6 +5,7 @@ org.clojure/spec.alpha {:mvn/version "0.3.218"} org.clojure/tools.cli {:mvn/version "1.0.206"} lambdaisland/tools.namespace {:mvn/version "0.1.247"} + com.climate/claypoole {:mvn/version "1.1.4"} lambdaisland/deep-diff {:mvn/version "0.0-47"} org.tcrawley/dynapath {:mvn/version "1.1.0"} slingshot/slingshot {:mvn/version "0.12.2"} diff --git a/src/kaocha/test_suite.clj b/src/kaocha/test_suite.clj index f4410c88..56ffe354 100644 --- a/src/kaocha/test_suite.clj +++ b/src/kaocha/test_suite.clj @@ -2,16 +2,6 @@ (:require [clojure.test :as t] [kaocha.testable :as testable])) -(defn deref-recur [testables] - (cond (future? testables) (deref testables) - (vector? testables) (doall (mapv deref-recur testables)) - (seq? testables) (deref-recur (into [] (doall testables))) - (contains? testables :kaocha.test-plan/tests) - (update testables :kaocha.test-plan/tests deref-recur) - (contains? testables :kaocha.result/tests) - (update testables :kaocha.result/tests deref-recur) - :else testables)) - (defn run [testable test-plan] (t/do-report {:type :begin-test-suite}) (let [results (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index a21d9384..3c5fe0cb 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -4,6 +4,7 @@ [clojure.pprint :as pprint] [clojure.spec.alpha :as s] [clojure.test :as t] + [com.climate.claypoole :as cp] [kaocha.classpath :as classpath] [kaocha.hierarchy :as hierarchy] [kaocha.history :as history] @@ -56,16 +57,6 @@ (retry-assert-spec type testable (dec n))) ;otherwise, retry )) -(defn deref-recur [testables] - (cond (future? testables) (deref testables) - (vector? testables) (doall (mapv deref-recur testables)) - (seq? testables) (deref-recur (into [] (doall testables))) - (contains? testables :kaocha.test-plan/tests) - (update testables :kaocha.test-plan/tests deref-recur) - (contains? testables :kaocha.result/tests) - (update testables :kaocha.result/tests deref-recur) - :else testables)) - (defn- load-type+validate "Try to load a testable type, and validate it both to be a valid generic testable, and a valid instance given the type. @@ -267,33 +258,20 @@ (defn run-testables-parallel "Run a collection of testables, returning a result collection." [testables test-plan] - (doall testables) - (let [load-error? (some ::load-error testables) - types (set (:parallel-children-exclude *config*)) - futures (map #(do - (future - (binding [*config* - (cond-> *config* - (contains? types (:kaocha.testable/type %)) (dissoc :parallel) - true (update :levels (fn [x] (if (nil? x) 1 (inc x)))))] - (run-testable % test-plan)))) - testables)] - (comment (loop [result [] ;(ArrayBlockingQueue. 1024) - [test & testables] testables] - (if test - (let [test (cond-> test - (and load-error? (not (::load-error test))) - (assoc ::skip true)) - r (run-testable test test-plan)] - (if (or (and *fail-fast?* (result/failed? r)) (::skip-remaining? r)) - ;(reduce put-return result [[r] testables]) - (reduce into result [[r] testables]) - ;(recur (doto result (.put r)) testables) - (recur (conj result r) testables))) - result))) - (deref-recur futures))) - -(defn run-testables + (let [num-threads (or (:parallel-threads *config*) (+ 2 (cp/ncpus))) + types (set (:parallel-children-exclude *config*))] + (cp/with-shutdown! [pool (cp/threadpool num-threads :name "kaocha-test-runner")] + (doall + (cp/pmap + pool + #(binding [*config* + (cond-> *config* + (contains? types (:kaocha.testable/type %)) (dissoc :parallel) + true (update :levels (fn [x] (if (nil? x) 1 (inc x)))))] + (run-testable % test-plan)) + testables))))) + +(defn run-testables [testables test-plan] (if (:parallel *config*) (doall (run-testables-parallel testables test-plan)) From ae0863b2932e7b928c9f351a345f46bf087fb42a Mon Sep 17 00:00:00 2001 From: John Shaffer Date: Thu, 28 Apr 2022 11:10:16 -0500 Subject: [PATCH 19/21] Fix multiple threadpools being created concurrently run-testables ends up being called within its own body, which kaocha is parallelizing. We generally want to parallelize at the namespace level, so we use run-test-serial for other testable types. This way we only create one threadpool at a time. --- src/kaocha/testable.clj | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 3c5fe0cb..5f111184 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -1,7 +1,6 @@ (ns kaocha.testable (:refer-clojure :exclude [load]) (:require [clojure.java.io :as io] - [clojure.pprint :as pprint] [clojure.spec.alpha :as s] [clojure.test :as t] [com.climate.claypoole :as cp] @@ -12,10 +11,7 @@ [kaocha.plugin :as plugin] [kaocha.result :as result] [kaocha.specs :refer [assert-spec]] - [kaocha.util :as util] - [kaocha.hierarchy :as hierarchy]) - (:import [clojure.lang Compiler$CompilerException] - [java.util.concurrent ArrayBlockingQueue BlockingQueue])) + [kaocha.util :as util])) (def ^:dynamic *fail-fast?* "Should testing terminate immediately upon failure or error?" @@ -258,23 +254,28 @@ (defn run-testables-parallel "Run a collection of testables, returning a result collection." [testables test-plan] - (let [num-threads (or (:parallel-threads *config*) (+ 2 (cp/ncpus))) - types (set (:parallel-children-exclude *config*))] - (cp/with-shutdown! [pool (cp/threadpool num-threads :name "kaocha-test-runner")] - (doall - (cp/pmap - pool - #(binding [*config* - (cond-> *config* - (contains? types (:kaocha.testable/type %)) (dissoc :parallel) - true (update :levels (fn [x] (if (nil? x) 1 (inc x)))))] - (run-testable % test-plan)) - testables))))) + (let [num-threads (or (:parallel-threads *config*) (* 2 (inc (cp/ncpus)))) + pred #(= :kaocha.type/ns (:kaocha.testable/type %)) + nses (seq (filter pred testables)) + others (seq (remove pred testables))] + (concat + (when others (run-testables-serial others test-plan)) + (when nses + (cp/with-shutdown! [pool (cp/threadpool num-threads :name "kaocha-test-runner")] + (doall + (cp/pmap + pool + #(binding [*config* + (-> *config* + (dissoc :parallel) + (update :levels (fn [x] (if (nil? x) 1 (inc x)))))] + (run-testable % test-plan)) + nses))))))) (defn run-testables [testables test-plan] (if (:parallel *config*) - (doall (run-testables-parallel testables test-plan)) + (run-testables-parallel testables test-plan) (run-testables-serial testables test-plan))) (defn test-seq [testable] From 250263251888b486f2876822b84e92dcf44fa474 Mon Sep 17 00:00:00 2001 From: John Shaffer Date: Fri, 19 Aug 2022 19:23:15 -0500 Subject: [PATCH 20/21] Fix run-test-parallel The previous code references fixtures/f-tests, but there is no such directory. The test case looks unfinished, so I'm not sure that there ever was a working f-tests dir. To fix the test case, I have adapted the code from run-test, just before run-test-parallel. --- test/unit/kaocha/type/ns_test.clj | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/test/unit/kaocha/type/ns_test.clj b/test/unit/kaocha/type/ns_test.clj index 5000f29a..a016e8a9 100644 --- a/test/unit/kaocha/type/ns_test.clj +++ b/test/unit/kaocha/type/ns_test.clj @@ -69,15 +69,12 @@ (testable/run testable testable))))))) (deftest run-test-parallel - (classpath/add-classpath "fixtures/f-tests") - - (let [testable (testable/load {:kaocha.testable/type :kaocha.type/clojure.test - :kaocha.testable/id :unit - :kaocha/ns-patterns ["-test$"] - :kaocha/source-paths ["src"] - :kaocha/test-paths ["fixtures/f-tests"] - :kaocha.filter/skip-meta [:kaocha/skip]})] + (classpath/add-classpath "fixtures/a-tests") + (let [testable (testable/load {:kaocha.testable/type :kaocha.type/ns + :kaocha.testable/id :foo.bar-test + :kaocha.testable/desc "foo.bar-test" + :kaocha.ns/name 'foo.bar-test})] (is (match? {:kaocha.testable/type :kaocha.type/ns :kaocha.testable/id :foo.bar-test :kaocha.ns/name 'foo.bar-test @@ -94,10 +91,10 @@ :kaocha.result/pending 0 :kaocha.result/fail 0}]} (:result - (with-test-ctx {:fail-fast? true} + (with-test-ctx {:fail-fast? true :parallel true} (testable/run testable testable))))) (is (not (nil? (:result - (binding [testable/*config* (assoc testable/*config* :parallel true)] - (with-test-ctx {:fail-fast? true - :parallel true} - (testable/run testable testable))))))))) + (binding [testable/*config* (assoc testable/*config* :parallel true)] + (with-test-ctx {:fail-fast? true + :parallel true} + (testable/run testable testable))))))))) From 152f6f0253218d2cc2840434c6339cfe204c8ac1 Mon Sep 17 00:00:00 2001 From: John Shaffer Date: Fri, 19 Aug 2022 21:17:33 -0500 Subject: [PATCH 21/21] Simplify run-testables-parallel with fnil --- src/kaocha/testable.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kaocha/testable.clj b/src/kaocha/testable.clj index 5f111184..44f7f2be 100644 --- a/src/kaocha/testable.clj +++ b/src/kaocha/testable.clj @@ -268,7 +268,7 @@ #(binding [*config* (-> *config* (dissoc :parallel) - (update :levels (fn [x] (if (nil? x) 1 (inc x)))))] + (update :levels (fnil inc 0)))] (run-testable % test-plan)) nses)))))))