Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add config for zero assertions switcher in config and cli args #404

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

add config for zero assertions switcher in config and cli args #404

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
0c61f15
cljfmt src/kaocha/specs
Mar 16, 2023
dc960ce
remove unnecessary require in src/kaocha/report.clj
Mar 16, 2023
2b8e8da
cljfmt src/kaocha/config.clj
Mar 16, 2023
57dce48
add mute-zero-assertion? global opt
Mar 16, 2023
def4f68
cljfmt src/kaocha/runner.clj
Mar 16, 2023
847aa77
implement config & cli-args to accept mute-zero-assertion?
Mar 16, 2023
87a30e8
add document for the configuration option
Mar 17, 2023
c178ed0
change mute-zero-assertions to warnings
Apr 5, 2023
c6f7fb2
also implement an option to interpret zero tests as failure
Apr 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions doc/clojure_test/assertions.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,12 @@ FAIL in sample-test/my-test (sample_test.clj:4)
Test ran without assertions. Did you forget an (is ...)?
```

### Missing assertions are configurable

There are two ways to disable this detection:

* Add `:kaocha/warnings {:zero-assertions :silent}` into `tests.edn`.
* Add `:kaocha/warnings {:zero-tests :error}` into `tests.edn`.

## Detecting single argument `=`

Expand Down
16 changes: 13 additions & 3 deletions src/kaocha/api.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@
(let [config (plugin/run-hook :kaocha.hooks/config config)
color? (:kaocha/color? config)
fail-fast? (:kaocha/fail-fast? config)
warnings (:kaocha/warnings config)
history (atom [])]
(binding [*active?* true
testable/*fail-fast?* fail-fast?
Expand All @@ -102,6 +103,10 @@
(let [config (resolve-reporter config)]
(let [test-plan (test-plan config)]

(when (= (:zero-assertions warnings) :silent)
(hierarchy/underive! :kaocha.type.var/zero-assertions :kaocha/known-key)
(hierarchy/underive! :kaocha.type.var/zero-assertions :kaocha/fail-type))

(when-not (some #(or (hierarchy/leaf? %)
(::testable/load-error %))
(testable/test-seq test-plan))
Expand All @@ -110,9 +115,14 @@
" Check for misspelled settings in your Kaocha test configuration"
" or incorrect focus or skip filters.")
(count (testable/test-seq-with-skipped test-plan))))
(output/warn (str "No tests were found. This may be an issue in your Kaocha test configuration."
" To investigate, check the :test-paths and :ns-patterns keys in tests.edn.")))
(throw+ {:kaocha/early-exit 0 }))
(if (= (:zero-tests warnings) :error)
(do
(output/error (str "No tests were found. This may be an issue in your Kaocha test configuration."
" To investigate, check the :test-paths and :ns-patterns keys in tests.edn."))
(throw+ {:kaocha/early-exit 253}))
(output/warn (str "No tests were found. This may be an issue in your Kaocha test configuration."
" To investigate, check the :test-paths and :ns-patterns keys in tests.edn."))))
(throw+ {:kaocha/early-exit 0}))

(when (find-ns 'matcher-combinators.core)
(require 'kaocha.matcher-combinators))
Expand Down
25 changes: 13 additions & 12 deletions src/kaocha/config.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
reporter
color?
fail-fast?
warnings
diff-style
randomize?
capture-output?
Expand All @@ -88,14 +89,15 @@
tests (assoc :kaocha/tests (vary-meta tests assoc :replace true))
plugins (assoc :kaocha/plugins plugins)
reporter (assoc :kaocha/reporter (vary-meta reporter assoc :replace true))
warnings (assoc :kaocha/warnings warnings)
bindings (assoc :kaocha/bindings bindings)
(some? color?) (assoc :kaocha/color? color?)
(some? fail-fast?) (assoc :kaocha/fail-fast? fail-fast?)
(some? diff-style) (assoc :kaocha/diff-style diff-style)
(some? watch?) (assoc :kaocha/watch? watch?)
(some? randomize?) (assoc :kaocha.plugin.randomize/randomize? randomize?)
(some? capture-output?) (assoc :kaocha.plugin.capture-output/capture-output? capture-output?)
:-> (merge (dissoc config :tests :plugins :reporter :color? :fail-fast? :watch? :randomize?)))))
:-> (merge (dissoc config :tests :plugins :reporter :warnings :color? :fail-fast? :watch? :randomize?)))))

(defmethod aero/reader 'kaocha [_opts _tag value]
(output/warn (format "The #kaocha reader literal is deprecated, please change it to %s." current-reader))
Expand Down Expand Up @@ -195,7 +197,6 @@
config
(read-config nil opts))))


(defn apply-cli-opts [config options]
(cond-> config
(some? (:fail-fast options)) (assoc :kaocha/fail-fast? (:fail-fast options))
Expand Down Expand Up @@ -224,10 +225,10 @@
"Applies command-line options and arguments to the configuration."
[config cli-opts cli-args]
(cond-> config
cli-opts (apply-cli-opts cli-opts)
cli-args (apply-cli-args cli-args)))
cli-opts (apply-cli-opts cli-opts)
cli-args (apply-cli-args cli-args)))

(defn find-config-and-warn
(defn find-config-and-warn
[config-file]
(let [final-config-file (or config-file "tests.edn")]
(when (not (.exists (io/file (or config-file "tests.edn"))))
Expand All @@ -241,13 +242,13 @@
(defn validate!
"Validates the configuration, printing any warnings and errors and possibly throwing."
[config]
(try
(specs/assert-spec :kaocha/config config)
config
(catch AssertionError e
(output/error "Invalid configuration file:\n"
(.getMessage e))
(throw+ {:kaocha/early-exit 252}))))
(try
(specs/assert-spec :kaocha/config config)
config
(catch AssertionError e
(output/error "Invalid configuration file:\n"
(.getMessage e))
(throw+ {:kaocha/early-exit 252}))))

(defn load-config-for-cli-and-validate
"Loads config from config-file, factoring in profile specified using profile,
Expand Down
5 changes: 5 additions & 0 deletions src/kaocha/hierarchy.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@
[tag parent]
(alter-var-root #'hierarchy derive tag parent))

(defn underive!
"Add a parent/child relationship to kaocha's keyword hierarchy."
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This removes a parent/child relationship, right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then I'd suggest:

Suggested change
"Add a parent/child relationship to kaocha's keyword hierarchy."
"Remove a parent/child relationship from Kaocha's keyword hierarchy."

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you planning to make this change?

[tag parent]
(alter-var-root #'hierarchy underive tag parent))

(derive! :fail :kaocha/fail-type)
(derive! :error :kaocha/fail-type)

Expand Down
2 changes: 1 addition & 1 deletion src/kaocha/report.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,6 @@
[kaocha.plugin.capture-output :as capture]
[kaocha.stacktrace :as stacktrace]
[kaocha.testable :as testable]
[kaocha.testable :as testable]
[kaocha.util :as util]
[slingshot.slingshot :refer [throw+]]))

Expand Down Expand Up @@ -425,6 +424,7 @@

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;


(def dots
"Reporter that prints progress as a sequence of dots and letters."
[dots* result])
Expand Down
7 changes: 3 additions & 4 deletions src/kaocha/runner.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,7 @@
(symbol "kaocha.report" s))))
:assoc-fn accumulate]
[nil "--diff-style STYLE" "The style of diff to print on failing tests, either :none or :deep"
:parse-fn parse-kw
]
:parse-fn parse-kw]
[nil "--plugin KEYWORD" "Load the given plugin."
:parse-fn (fn [s]
(let [kw (parse-kw s)]
Expand Down Expand Up @@ -151,8 +150,8 @@
config-file (config/find-config-and-warn config-file)
;; Initial configuration load to determine plugins.
config (-> (config/load-config config-file (if profile {:profile profile} {}))
(config/apply-cli {} (map parse-kw arguments))
(config/validate!))
(config/apply-cli {} (map parse-kw arguments))
(config/validate!))
plugin-chain (plugin/load-all (concat (:kaocha/plugins config) plugin))
cli-options (plugin/run-hook* plugin-chain :kaocha.hooks/cli-options cli-options)

Expand Down
63 changes: 33 additions & 30 deletions src/kaocha/specs.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,33 +22,36 @@

(spec/def :kaocha/fail-fast? boolean?)

(spec/def :kaocha/warnings (spec/map-of #{:zero-assertions :zero-tests} #{:silent :error}))

(spec/def :kaocha/watch? boolean?)

(spec/def :kaocha/plugins (spec/coll-of keyword?))

(spec/def :kaocha/reporter (spec/or :fn (s-fspec :args (spec/cat :m map?))
:symbol symbol?
:symbols (spec/coll-of symbol? :kind vector?)))
:symbol symbol?
:symbols (spec/coll-of symbol? :kind vector?)))

(spec/def :kaocha/global-opts
(spec/keys :opt [:kaocha/reporter
:kaocha/color?
:kaocha/fail-fast?
:kaocha/watch?
:kaocha/plugins]))
:kaocha/color?
:kaocha/fail-fast?
:kaocha/warnings
:kaocha/watch?
:kaocha/plugins]))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; config

(spec/def :kaocha/config (spec/merge :kaocha/global-opts
(spec/keys :opt [:kaocha/tests])))
(spec/keys :opt [:kaocha/tests])))

(spec/def :kaocha/tests (spec/coll-of :kaocha/testable))

(spec/def :kaocha/testable (spec/keys :req [:kaocha.testable/type
:kaocha.testable/id]
:opt [:kaocha.testable/meta
:kaocha.testable/wrap]))
:kaocha.testable/id]
:opt [:kaocha.testable/meta
:kaocha.testable/wrap]))

(spec/def :kaocha/source-paths (spec/coll-of string?))

Expand Down Expand Up @@ -77,43 +80,43 @@

(spec/def :kaocha/test-plan
(spec/merge :kaocha/global-opts
(spec/keys :opt [:kaocha.test-plan/tests])))
(spec/keys :opt [:kaocha.test-plan/tests])))

(spec/def :kaocha.test-plan/tests (spec/coll-of :kaocha.test-plan/testable))

(spec/def :kaocha.test-plan/testable (spec/merge :kaocha/testable
(spec/keys :req []
:opt [:kaocha.testable/desc
:kaocha.test-plan/tests
:kaocha.testable/load-error])))
(spec/keys :req []
:opt [:kaocha.testable/desc
:kaocha.test-plan/tests
:kaocha.testable/load-error])))

(spec/def :kaocha.testable/load-error (s-with-gen
#(instance? Throwable %)
#(s-gen #{(ex-info "load error" {:oops "not good"})})))
#(instance? Throwable %)
#(s-gen #{(ex-info "load error" {:oops "not good"})})))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; result

(spec/def :kaocha/result
(spec/merge :kaocha/global-opts
(spec/keys :opt [:kaocha.result/tests])))
(spec/keys :opt [:kaocha.result/tests])))

(spec/def :kaocha.result/tests (spec/coll-of :kaocha.result/testable))

(spec/def :kaocha.result/testable (spec/merge :kaocha.test-plan/testable
(spec/keys :opt [:kaocha.result/count
:kaocha.result/tests
:kaocha.result/pass
:kaocha.result/error
:kaocha.result/fail
:kaocha.result/out
:kaocha.result/err
:kaocha.result/time])))
(spec/keys :opt [:kaocha.result/count
:kaocha.result/tests
:kaocha.result/pass
:kaocha.result/error
:kaocha.result/fail
:kaocha.result/out
:kaocha.result/err
:kaocha.result/time])))

(spec/def ::small-int (s-with-gen
nat-int?
(constantly (or (some-> (resolve `clojure.test.check.generatorspec/small-integer) deref)
(s-gen nat-int?)))))
nat-int?
(constantly (or (some-> (resolve `clojure.test.check.generatorspec/small-integer) deref)
(s-gen nat-int?)))))

(spec/def :kaocha.result/count ::small-int)
(spec/def :kaocha.result/pass ::small-int)
Expand Down Expand Up @@ -145,7 +148,7 @@
(spec/def ::stc/num-tests (spec/nilable nat-int?))
(spec/def ::stc/max-size (spec/nilable nat-int?))
(spec/def ::stc/opts (spec/nilable (spec/keys :opt-un [::stc/num-tests
::stc/max-size]))))
::stc/max-size]))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; helpers
Expand Down