diff --git a/VERSION b/VERSION index a918a2a..faef31a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.0 +0.7.0 diff --git a/dev/dev.clj b/dev/dev.clj index 8f9ab96..adb528c 100644 --- a/dev/dev.clj +++ b/dev/dev.clj @@ -14,7 +14,7 @@ [job-streamer.control-bus.system :as system])) (def dev-config - {:app {:middleware [wrap-stacktrace]} + {:app {:middleware [wrap-stacktrace :stacktrace]} :datomic {:uri "datomic:mem://jobstreamer"}}) (def config diff --git a/project.clj b/project.clj index 629ce65..a0a67b9 100644 --- a/project.clj +++ b/project.clj @@ -12,9 +12,10 @@ [net.unit8.wscl/websocket-classloader "0.2.1"] [net.unit8.logback/logback-websocket-appender "0.1.0"] [io.undertow/undertow-websockets-jsr "1.3.22.Final"] - [com.datomic/datomic-free "0.9.5372" :exclusions [org.slf4j/slf4j-api org.slf4j/slf4j-nop + [com.datomic/datomic-free "0.9.5394" :exclusions [org.slf4j/slf4j-api org.slf4j/slf4j-nop com.amazonaws/aws-java-sdk]] [org.jsoup/jsoup "1.9.2"] + [org.clojure/data.json "0.2.6"] [datomic-schema "1.3.0"] [liberator "0.14.1"] [compojure "1.5.1"] diff --git a/src/clj/job_streamer/control_bus/component/agents.clj b/src/clj/job_streamer/control_bus/component/agents.clj index 48fa0be..aaafe42 100644 --- a/src/clj/job_streamer/control_bus/component/agents.clj +++ b/src/clj/job_streamer/control_bus/component/agents.clj @@ -14,7 +14,7 @@ (defn list-resource [{:keys [agents]}] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get] :handle-ok (fn [ctx] (->> (vals @agents) @@ -35,7 +35,7 @@ (defn entry-resource [{:keys [agents datomic]} instance-id & [cmd]] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get :put] :put! (fn [ctx] (when-let [uuid (UUID/fromString instance-id)] diff --git a/src/clj/job_streamer/control_bus/component/apps.clj b/src/clj/job_streamer/control_bus/component/apps.clj index a4f7891..58d959e 100644 --- a/src/clj/job_streamer/control_bus/component/apps.clj +++ b/src/clj/job_streamer/control_bus/component/apps.clj @@ -71,7 +71,7 @@ (defn list-resource [{:keys [datomic applications]}] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get :post] :malformed? #(validate (parse-body %) :application/name v/required @@ -106,7 +106,7 @@ (defn batch-components-resource [{:keys [datomic]} app-name] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get] :handle-ok (fn [ctx] (let [in-app (->> (d/query datomic @@ -126,7 +126,7 @@ (defn stats-resource [{:keys [datomic agents]} app-name] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get] :handle-ok (fn [ctx] {:agents (count (ag/available-agents agents)) diff --git a/src/clj/job_streamer/control_bus/component/calendar.clj b/src/clj/job_streamer/control_bus/component/calendar.clj index daee994..a2a65b1 100644 --- a/src/clj/job_streamer/control_bus/component/calendar.clj +++ b/src/clj/job_streamer/control_bus/component/calendar.clj @@ -20,7 +20,7 @@ (defn list-resource [{:keys [datomic scheduler]}] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get :post] :malformed? #(validate (parse-body %) :calendar/name v/required) @@ -35,6 +35,7 @@ true)) :post! (fn [{cal :edn}] + (println cal) (let [id (or (:db/id cal) (d/tempid :db.part/user))] (if-let [old-id (d/query datomic '{:find [?calendar .] @@ -64,7 +65,7 @@ (defn entry-resource [{:keys [datomic scheduler]} name] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get :put :delete] :malformed? (fn [ctx] (or (validate (parse-body ctx) @@ -77,7 +78,7 @@ '{:find [(pull ?e [:*]) .] :in [$ ?n] :where [[?e :calendar/name ?n]]} name) - + :put! (fn [{cal :edn}] (d/transact datomic [{:db/id (:db/id cal) diff --git a/src/clj/job_streamer/control_bus/component/jobs.clj b/src/clj/job_streamer/control_bus/component/jobs.clj index 0e9108f..172efb7 100644 --- a/src/clj/job_streamer/control_bus/component/jobs.clj +++ b/src/clj/job_streamer/control_bus/component/jobs.clj @@ -2,8 +2,11 @@ (:require [clojure.tools.logging :as log] [clojure.edn :as edn] [com.stuartsierra.component :as component] + [bouncer.core :as b] [bouncer.validators :as v] [liberator.core :as liberator] + [clojure.string :as str] + [clj-time.format :as f] (job-streamer.control-bus [notification :as notification] [validation :refer [validate]] [util :refer [parse-body edn->datoms to-int]]) @@ -12,7 +15,7 @@ [scheduler :as scheduler])) (:import [java.util Date])) -(defn- find-latest-execution +(defn find-latest-execution "Find latest from given executions." [executions] (when executions @@ -68,18 +71,71 @@ [?job :job/name ?job-name]]} app-name job-name)) +(defn- parse-query-since [q] + (let [since (.substring q (count "since:"))] + (when (b/valid? {:since since} + :since [[v/datetime (:date f/formatters)]]) + {:since (f/parse (:date f/formatters) since)}))) + +(defn- parse-query-until [q] + (let [until (.substring q (count "until:"))] + (when (b/valid? {:until until} + :until [[v/datetime (:date f/formatters)]]) + {:until (f/parse (:date f/formatters) until)}))) + +(defn- parse-query-exit-status [q] + (let [exit-status (.substring q (count "exit-status:"))] + (when (b/valid? {:exit-status exit-status} + :exit-status v/required) + {:exit-status exit-status}))) + +(defn parse-query [query] + (when (not-empty query) + (->> (str/split query #"\s") + (map #(cond + (.startsWith % "since:") (parse-query-since %) + (.startsWith % "until:") (parse-query-until %) + (.startsWith % "exit-status:") (parse-query-exit-status %) + :default {:job-name [%]})) + (apply merge-with concat {:job-name nil})))) (defn find-all [{:keys [datomic]} app-name query & [offset limit]] - (let [base-query '{:find [?job] - :in [$ ?app-name ?query] + (let [qmap (parse-query query) + base-query '{:find [?job] + :in [$ ?app-name [?job-name-condition ...] ?since-condition ?until-condition ?exit-status-condition] :where [[?app :application/name ?app-name] [?app :application/jobs ?job]]} jobs (d/query datomic - (if (not-empty query) - (update-in base-query [:where] - conj '[(fulltext $ :job/name ?query) [[?job ?job-name]]]) - base-query) - app-name (or query ""))] + (cond-> base-query + (not-empty (:job-name qmap)) + (update-in [:where] conj + '[?job :job/name ?job-name] + '[(.contains ^String ?job-name ?job-name-condition)]) + (or (:since qmap) (:until qmap) (:exit-status qmap)) + (update-in [:where] conj + '[?job :job/executions ?job-executions] + '[?job-execution :job-execution/create-time ?create-time] + '[(max ?create-time)] + '[?job-executions :job-execution/exit-status ?exit-status] + '[?job-executions :job-execution/end-time ?end-time]) + + (:since qmap) + (update-in [:where] conj + '[(>= ?end-time ?since-condition)]) + + (:until qmap) + (update-in [:where] conj + '[(<= ?end-time ?until-condition)]) + + (:exit-status qmap) + (update-in [:where] conj + '[(.contains ^String ?exit-status ?exit-status-condition)])) + app-name + (:job-name qmap []) + (:since qmap "") + (:until qmap "") + (:exit-status qmap ""))] + {:results (->> jobs (drop (dec (or offset 0))) (take (or limit 20)) @@ -212,7 +268,7 @@ (defn list-resource [{:keys [datomic scheduler] :as jobs} app-name] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get :post] :malformed? (fn [ctx] (validate (parse-body ctx) @@ -268,12 +324,13 @@ :status-notification/exit-status :status-notification/type] (:db/id sn)))) vec)})))))) - vec)))))) + vec)))) + :etag (str (int (/ (System/currentTimeMillis) 10000))))) (defn entry-resource [{:keys [datomic scheduler] :as jobs} app-name job-name] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get :put :delete] :malformed? #(parse-body %) :exists? (when-let [[app-id job-id] (find-by-name jobs app-name job-name)] @@ -325,7 +382,7 @@ (defn job-settings-resource [{:keys [datomic] :as jobs} app-name job-name & [cmd]] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get :delete :put] :malformed? #(parse-body %) :exists? (when-let [[app-id job-id] (find-by-name jobs app-name job-name)] @@ -426,7 +483,7 @@ (defn executions-resource [{:keys [datomic] :as jobs} app-name job-name] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get :post] :malformed? #(parse-body %) :exists? (when-let [[app-id job-id] (find-by-name jobs app-name job-name)] @@ -460,7 +517,7 @@ (defn execution-resource [{:keys [agents scheduler datomic] :as jobs} id & [cmd]] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:get :put] :malformed? #(parse-body %) :exists? (fn [ctx] diff --git a/src/clj/job_streamer/control_bus/component/scheduler.clj b/src/clj/job_streamer/control_bus/component/scheduler.clj index 639d8a7..714876f 100644 --- a/src/clj/job_streamer/control_bus/component/scheduler.clj +++ b/src/clj/job_streamer/control_bus/component/scheduler.clj @@ -134,7 +134,7 @@ (defn entry-resource [{:keys [datomic] :as scheduler} job-id & [cmd]] (liberator/resource - :available-media-types ["application/edn"] + :available-media-types ["application/edn" "application/json"] :allowed-methods [:post :put :delete] :malformed? #(parse-body %) :exists? (fn [ctx] diff --git a/src/clj/job_streamer/control_bus/endpoint/api.clj b/src/clj/job_streamer/control_bus/endpoint/api.clj index d25c7a6..cea1cc8 100644 --- a/src/clj/job_streamer/control_bus/endpoint/api.clj +++ b/src/clj/job_streamer/control_bus/endpoint/api.clj @@ -5,6 +5,7 @@ [clojure.tools.logging :as log] [compojure.core :refer [ANY GET routes]] [bouncer.validators :as v] + [ring.util.response :refer [content-type]] (job-streamer.control-bus.component [apps :as apps] [jobs :as jobs] @@ -75,6 +76,8 @@ (apps/batch-components-resource apps app-name)) (ANY "/:app-name/stats" [app-name] (apps/stats-resource apps app-name)) + (GET "/version" [] (-> {:body (clojure.string/replace (str "\"" (slurp "VERSION") "\"") "\n" "")} + (content-type "text/plain"))) ;; For debug ;(GET "/logs" [] (pr-str (model/query '{:find [[(pull ?log [*]) ...]] :where [[?log :execution-log/level]]}))) diff --git a/src/clj/job_streamer/control_bus/system.clj b/src/clj/job_streamer/control_bus/system.clj index 76227a8..ecea67d 100644 --- a/src/clj/job_streamer/control_bus/system.clj +++ b/src/clj/job_streamer/control_bus/system.clj @@ -23,7 +23,6 @@ [api :refer [api-endpoint]]) [job-streamer.control-bus.endpoint.api :refer [api-endpoint]])) - (defn wrap-same-origin-policy [handler alias] (fn [req] (if (= (:request-method req) :options) @@ -39,6 +38,7 @@ {:app {:middleware [[wrap-not-found :not-found] [wrap-same-origin-policy :same-origin] [wrap-defaults :defaults]] + :not-found "Resource Not Found" :defaults (meta-merge api-defaults {})}}) diff --git a/src/clj/job_streamer/control_bus/util.clj b/src/clj/job_streamer/control_bus/util.clj index 1a8b706..d51a8de 100644 --- a/src/clj/job_streamer/control_bus/util.clj +++ b/src/clj/job_streamer/control_bus/util.clj @@ -3,7 +3,8 @@ [clojure.edn :as edn] [clojure.java.io :as io] [datomic.api :as d] - [ring.util.request :refer [content-type]]) + [ring.util.request :refer [content-type]] + [clojure.data.json :as json]) (:import [org.jsoup Jsoup])) (defn to-int [n default-value] @@ -126,6 +127,18 @@ java.lang.String body (slurp (io/reader body))))) +(defn json-value-reader [key value] + (if (= key :calendar/holidays) + (for [date value] (java.sql.Date/valueOf date)) + value)) + +(defn json->edn + "Convert a format from JSON to edn" + [json] + (json/read-str json + :key-fn keyword + :value-fn json-value-reader)) + (defn parse-body [context] (when (#{:put :post} (get-in context [:request :request-method])) (try @@ -133,6 +146,7 @@ (case (or (content-type (:request context)) (get-in context [:request :content-type])) "application/edn" [false {:edn (edn/read-string body)}] "application/xml" [false {:edn (xml->edn body)}] + "application/json" [false {:edn (json->edn body)}] false) false) (catch Exception e diff --git a/test/clj/job_streamer/control_bus/component/jobs_test.clj b/test/clj/job_streamer/control_bus/component/jobs_test.clj index a76bc27..b25f1b3 100644 --- a/test/clj/job_streamer/control_bus/component/jobs_test.clj +++ b/test/clj/job_streamer/control_bus/component/jobs_test.clj @@ -10,7 +10,8 @@ [meta-merge.core :refer [meta-merge]] [clojure.test :refer :all] [clojure.pprint :refer :all] - [clojure.edn :as edn])) + [clojure.edn :as edn] + [clj-time.format :as f])) @@ -71,3 +72,29 @@ (is (= 201 (-> (handler request) :status)))) (let [res (jobs/find-all (:jobs system) "default" nil)] (is (= 1 (:hits res))))))) + +(deftest parse-query + (testing "parse-query" + (let [result (jobs/parse-query "a b since:2016-09-01 until:2016-09-02 exit-status:COMPLETED")] + (is (= "a" (first (:job-name result)))) + (is (= "2016-09-01" (f/unparse (:date f/formatters) (:since result)))) + (is (= "2016-09-02" (f/unparse (:date f/formatters) (:until result)))) + (is (= "COMPLETED" (:exit-status result))))) + (testing "nil query returns nil" + (let [result (jobs/parse-query nil)] + (is (nil? result)))) + (testing "empty query returns nil" + (let [result (jobs/parse-query "")] + (is (nil? result)))) + + (testing "single simple query" + (let [result (jobs/parse-query "a")] + (is (= {:job-name '("a")} result)))) + + (testing "single simple query" + (let [result (jobs/parse-query "a")] + (is (= {:job-name '("a")} result)))) + + (testing "ignore breaking tokens in a query" + (let [result (jobs/parse-query "a since: until: since:xxx until:yyy")] + (is (= {:job-name '("a")} result)))))