From dcc3b45eae27f00f1ef16e716629b7daa0322e90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ho=C3=A0ng=20Minh=20Th=E1=BA=AFng?= Date: Fri, 3 Sep 2021 02:10:27 +0700 Subject: [PATCH 1/5] add new scenario 5 --- dev/resources/common_unpatched.edn | 11 ++++++++++- dev/resources/core-mysql.edn | 11 ++++++++++- dev/resources/core-postgres.edn | 11 ++++++++++- dev/resources/core-sqlite.edn | 11 ++++++++++- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/dev/resources/common_unpatched.edn b/dev/resources/common_unpatched.edn index 220e8d9c..8d3cd1fa 100644 --- a/dev/resources/common_unpatched.edn +++ b/dev/resources/common_unpatched.edn @@ -1,4 +1,4 @@ -{;; scenario 1 migrations +{ ;; scenario 1 migrations [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-1/create-house-table] {:up ["CREATE TABLE `house` (`index` VARCHAR(32) PRIMARY KEY, `color` TEXT)" "INSERT INTO `house` (`index`, `color`) VALUES ('10', 'black'), ('20', 'brown')"] @@ -61,4 +61,13 @@ {:up ["CREATE TABLE `follow` (`human_1` INTEGER REFERENCES `human`(`number`), `human_2` INTEGER REFERENCES `human`(`number`), `year` INTEGER)" "INSERT INTO `follow` (`human_1`, `human_2`, `year`) VALUES (1, 2, 2015), (1, 3, 2017), (2, 1, 2014), (2, 3, 2015), (1, 4, 2012)"] :down ["DROP TABLE `follow`"]} + ;; scenario 5 migrations + [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-article-table] + {:up ["CREATE TABLE `article` (`id` SERIAL PRIMARY KEY, `title` TEXT, `current_revision` INTEGER)" + "INSERT INTO `article` (`id`, `title`, `current_revision`) VALUES (1, 'introduction', 2), (2, 'hello world', 1)" ] + :down ["DROP TABLE `article`"]} + [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-revision-table] + {:up ["CREATE TABLE `revision` (`id` SERIAL, `content` TEXT, `revision` INTEGER)" + "INSERT INTO `revision` (`id`, `content`, `revision`) VALUES (1, 'tbd', 1), (1, 'this is sparta', 2), (2, 'welcome to my site', 1)" ] + :down ["DROP TABLE `revision`"]} } diff --git a/dev/resources/core-mysql.edn b/dev/resources/core-mysql.edn index 220e8d9c..8d3cd1fa 100644 --- a/dev/resources/core-mysql.edn +++ b/dev/resources/core-mysql.edn @@ -1,4 +1,4 @@ -{;; scenario 1 migrations +{ ;; scenario 1 migrations [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-1/create-house-table] {:up ["CREATE TABLE `house` (`index` VARCHAR(32) PRIMARY KEY, `color` TEXT)" "INSERT INTO `house` (`index`, `color`) VALUES ('10', 'black'), ('20', 'brown')"] @@ -61,4 +61,13 @@ {:up ["CREATE TABLE `follow` (`human_1` INTEGER REFERENCES `human`(`number`), `human_2` INTEGER REFERENCES `human`(`number`), `year` INTEGER)" "INSERT INTO `follow` (`human_1`, `human_2`, `year`) VALUES (1, 2, 2015), (1, 3, 2017), (2, 1, 2014), (2, 3, 2015), (1, 4, 2012)"] :down ["DROP TABLE `follow`"]} + ;; scenario 5 migrations + [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-article-table] + {:up ["CREATE TABLE `article` (`id` SERIAL PRIMARY KEY, `title` TEXT, `current_revision` INTEGER)" + "INSERT INTO `article` (`id`, `title`, `current_revision`) VALUES (1, 'introduction', 2), (2, 'hello world', 1)" ] + :down ["DROP TABLE `article`"]} + [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-revision-table] + {:up ["CREATE TABLE `revision` (`id` SERIAL, `content` TEXT, `revision` INTEGER)" + "INSERT INTO `revision` (`id`, `content`, `revision`) VALUES (1, 'tbd', 1), (1, 'this is sparta', 2), (2, 'welcome to my site', 1)" ] + :down ["DROP TABLE `revision`"]} } diff --git a/dev/resources/core-postgres.edn b/dev/resources/core-postgres.edn index ed17a85c..5f6c5f59 100644 --- a/dev/resources/core-postgres.edn +++ b/dev/resources/core-postgres.edn @@ -1,4 +1,4 @@ -{;; scenario 1 migrations +{ ;; scenario 1 migrations [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-1/create-house-table] {:up ["CREATE TABLE \"house\" (\"index\" VARCHAR(32) PRIMARY KEY, \"color\" TEXT)" "INSERT INTO \"house\" (\"index\", \"color\") VALUES ('10', 'black'), ('20', 'brown')"] @@ -61,4 +61,13 @@ {:up ["CREATE TABLE \"follow\" (\"human_1\" INTEGER REFERENCES \"human\"(\"number\"), \"human_2\" INTEGER REFERENCES \"human\"(\"number\"), \"year\" INTEGER)" "INSERT INTO \"follow\" (\"human_1\", \"human_2\", \"year\") VALUES (1, 2, 2015), (1, 3, 2017), (2, 1, 2014), (2, 3, 2015), (1, 4, 2012)"] :down ["DROP TABLE \"follow\""]} + ;; scenario 5 migrations + [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-article-table] + {:up ["CREATE TABLE \"article\" (\"id\" SERIAL PRIMARY KEY, \"title\" TEXT, \"current_revision\" INTEGER)" + "INSERT INTO \"article\" (\"id\", \"title\", \"current_revision\") VALUES (1, 'introduction', 2), (2, 'hello world', 1)" ] + :down ["DROP TABLE \"article\""]} + [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-revision-table] + {:up ["CREATE TABLE \"revision\" (\"id\" SERIAL, \"content\" TEXT, \"revision\" INTEGER)" + "INSERT INTO \"revision\" (\"id\", \"content\", \"revision\") VALUES (1, 'tbd', 1), (1, 'this is sparta', 2), (2, 'welcome to my site', 1)" ] + :down ["DROP TABLE \"revision\""]} } diff --git a/dev/resources/core-sqlite.edn b/dev/resources/core-sqlite.edn index 6c0c6ed2..cecacb53 100644 --- a/dev/resources/core-sqlite.edn +++ b/dev/resources/core-sqlite.edn @@ -1,4 +1,4 @@ -{;; scenario 1 migrations +{ ;; scenario 1 migrations [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-1/create-house-table] {:up ["CREATE TABLE `house` (`index` VARCHAR(32) PRIMARY KEY, `color` TEXT)" "INSERT INTO `house` (`index`, `color`) VALUES ('10', 'black'), ('20', 'brown')"] @@ -61,4 +61,13 @@ {:up ["CREATE TABLE `follow` (`human_1` INTEGER REFERENCES `human`(`number`), `human_2` INTEGER REFERENCES `human`(`number`), `year` INTEGER)" "INSERT INTO `follow` (`human_1`, `human_2`, `year`) VALUES (1, 2, 2015), (1, 3, 2017), (2, 1, 2014), (2, 3, 2015), (1, 4, 2012)"] :down ["DROP TABLE `follow`"]} + ;; scenario 5 migrations + [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-article-table] + {:up ["CREATE TABLE `article` (`id` SERIAL PRIMARY KEY, `title` TEXT, `current_revision` INTEGER)" + "INSERT INTO `article` (`id`, `title`, `current_revision`) VALUES (1, 'introduction', 2), (2, 'hello world', 1)" ] + :down ["DROP TABLE `article`"]} + [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-revision-table] + {:up ["CREATE TABLE `revision` (`id` SERIAL, `content` TEXT, `revision` INTEGER)" + "INSERT INTO `revision` (`id`, `content`, `revision`) VALUES (1, 'tbd', 1), (1, 'this is sparta', 2), (2, 'welcome to my site', 1)" ] + :down ["DROP TABLE `revision`"]} } From 1866c70fb5bbee3672561a11b860dd5a6c835890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ho=C3=A0ng=20Minh=20Th=E1=BA=AFng?= Date: Fri, 3 Sep 2021 02:11:10 +0700 Subject: [PATCH 2/5] use :reload-all in user ns --- dev/src/user.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/src/user.clj b/dev/src/user.clj index 339d8e1a..d260123a 100644 --- a/dev/src/user.clj +++ b/dev/src/user.clj @@ -29,6 +29,6 @@ (defn dev "Load and switch to the 'dev' namespace." [] - (require 'dev) + (require 'dev :reload-all true) (in-ns 'dev) :loaded) From f5bec6bea4a9dc21ecaf138e231db85c96e1bf26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ho=C3=A0ng=20Minh=20Th=E1=BA=AFng?= Date: Fri, 3 Sep 2021 02:12:10 +0700 Subject: [PATCH 3/5] use keywords instead of symbol in destructuring --- src/walkable/core.cljc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/walkable/core.cljc b/src/walkable/core.cljc index 5a6b2d8c..54eebfd8 100644 --- a/src/walkable/core.cljc +++ b/src/walkable/core.cljc @@ -32,7 +32,7 @@ [build-and-run-query env ast] (->> ast (ast-map-loc (fn [loc] - (let [{::ast/keys [prepared-query]} (z/node loc)] + (let [{:keys [::ast/prepared-query]} (z/node loc)] (if prepared-query (let [parent (last (z/path loc)) ;; TODO: partition entities and concat back @@ -54,8 +54,8 @@ (z/root loc) (recur (z/next - (let [{:keys [children] :as node} (z/node loc) - {::ast/keys [prepared-merge-sub-entities]} node] + (let [{:keys [:children] :as node} (z/node loc) + {:keys [::ast/prepared-merge-sub-entities]} node] (if (or (not prepared-merge-sub-entities) ;; node can be nil (not-empty children)) loc @@ -78,25 +78,25 @@ (defn merge-data [wrap-merge ast] - (loop [{:keys [children] :as root} ast] + (loop [{:keys [:children] :as root} ast] (if (empty? children) (:entities root) (recur (merge-data-in-bottom-branches wrap-merge root))))) (defn ast-resolver* - [{:keys [build-and-run-query floor-plan env wrap-merge ast]}] + [{:keys [:build-and-run-query :floor-plan :env :wrap-merge :ast]}] (->> (ast/prepare-ast floor-plan ast) (fetch-data build-and-run-query env) (merge-data wrap-merge))) (defn prepared-ast-resolver* - [{:keys [build-and-run-query env wrap-merge prepared-ast]}] + [{:keys [:build-and-run-query :env :wrap-merge :prepared-ast]}] (->> prepared-ast (fetch-data build-and-run-query env) (merge-data wrap-merge))) (defn query-resolver* - [{:keys [floor-plan env resolver query]}] + [{:keys [:floor-plan :env :resolver :query]}] (resolver floor-plan env (p/query->ast query))) (defn ast-resolver @@ -158,7 +158,7 @@ ios)) (defn internalize-indexes - [indexes {::pc/keys [sym] :as dynamic-resolver}] + [indexes {:keys [::pc/sym] :as dynamic-resolver}] (-> indexes (update ::pc/index-resolvers (fn [resolvers] @@ -178,7 +178,7 @@ (floor-plan/with-db-type db-type registry) registry))] {::p/wrap-parser2 - (fn [parser {::p/keys [plugins]}] + (fn [parser {:keys [::p/plugins]}] (let [resolve-fn (fn [env _] (resolver compiled-floor-plan query-env env)) all-indexes (-> (compute-indexes resolver-sym inputs-outputs) From 02d3ea11465e05c6ff768ab96aa136bb5338843e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ho=C3=A0ng=20Minh=20Th=E1=BA=AFng?= Date: Fri, 3 Sep 2021 12:49:58 +0700 Subject: [PATCH 4/5] fix wrong require syntax --- dev/src/user.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/src/user.clj b/dev/src/user.clj index d260123a..1b65aa24 100644 --- a/dev/src/user.clj +++ b/dev/src/user.clj @@ -29,6 +29,6 @@ (defn dev "Load and switch to the 'dev' namespace." [] - (require 'dev :reload-all true) + (require 'dev :reload-all) (in-ns 'dev) :loaded) From 97308a7825333c0dcba104fb1b457d4fad20d7e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ho=C3=A0ng=20Minh=20Th=E1=BA=AFng?= Date: Mon, 6 Sep 2021 01:18:17 +0700 Subject: [PATCH 5/5] new join api --- dev/resources/common_unpatched.edn | 2 +- dev/resources/config-mysql.edn | 2 + dev/resources/config-postgres.edn | 2 + dev/resources/config-sqlite.edn | 2 + dev/resources/core-mysql.edn | 2 +- dev/resources/core-postgres.edn | 2 +- dev/resources/core-sqlite.edn | 2 +- .../walkable/integration_test/common.cljc | 55 ++- src/walkable/sql_query_builder/ast.cljc | 55 ++- .../sql_query_builder/floor_plan.cljc | 319 ++++++++++-------- .../sql_query_builder/floor_plan_test.cljc | 12 - 11 files changed, 258 insertions(+), 197 deletions(-) diff --git a/dev/resources/common_unpatched.edn b/dev/resources/common_unpatched.edn index 8d3cd1fa..483269fe 100644 --- a/dev/resources/common_unpatched.edn +++ b/dev/resources/common_unpatched.edn @@ -67,7 +67,7 @@ "INSERT INTO `article` (`id`, `title`, `current_revision`) VALUES (1, 'introduction', 2), (2, 'hello world', 1)" ] :down ["DROP TABLE `article`"]} [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-revision-table] - {:up ["CREATE TABLE `revision` (`id` SERIAL, `content` TEXT, `revision` INTEGER)" + {:up ["CREATE TABLE `revision` (`id` INTEGER, `content` TEXT, `revision` INTEGER)" "INSERT INTO `revision` (`id`, `content`, `revision`) VALUES (1, 'tbd', 1), (1, 'this is sparta', 2), (2, 'welcome to my site', 1)" ] :down ["DROP TABLE `revision`"]} } diff --git a/dev/resources/config-mysql.edn b/dev/resources/config-mysql.edn index d6f36f0d..eb5da562 100644 --- a/dev/resources/config-mysql.edn +++ b/dev/resources/config-mysql.edn @@ -13,6 +13,8 @@ #ig/ref :walkable-demo.migration.scenario-3/create-person_pet-table #ig/ref :walkable-demo.migration.scenario-4/create-human-table #ig/ref :walkable-demo.migration.scenario-4/create-follow-table + #ig/ref :walkable-demo.migration.scenario-5/create-article-table + #ig/ref :walkable-demo.migration.scenario-5/create-revision-table ;; mysql-specific scenarios (starting from 100) ]} } diff --git a/dev/resources/config-postgres.edn b/dev/resources/config-postgres.edn index 41fbef51..277cb0c6 100644 --- a/dev/resources/config-postgres.edn +++ b/dev/resources/config-postgres.edn @@ -26,6 +26,8 @@ #ig/ref :walkable-demo.migration.scenario-3/create-person_pet-table #ig/ref :walkable-demo.migration.scenario-4/create-human-table #ig/ref :walkable-demo.migration.scenario-4/create-follow-table + #ig/ref :walkable-demo.migration.scenario-5/create-article-table + #ig/ref :walkable-demo.migration.scenario-5/create-revision-table ;; postgres-specific scenarios (starting from 100) #ig/ref :walkable-demo.migration.scenario-100/create-land-animal-table diff --git a/dev/resources/config-sqlite.edn b/dev/resources/config-sqlite.edn index 8a55d137..abdd484d 100644 --- a/dev/resources/config-sqlite.edn +++ b/dev/resources/config-sqlite.edn @@ -13,6 +13,8 @@ #ig/ref :walkable-demo.migration.scenario-3/create-person_pet-table #ig/ref :walkable-demo.migration.scenario-4/create-human-table #ig/ref :walkable-demo.migration.scenario-4/create-follow-table + #ig/ref :walkable-demo.migration.scenario-5/create-article-table + #ig/ref :walkable-demo.migration.scenario-5/create-revision-table ;; sqlite-specific scenarios (starting from 100) ]} } diff --git a/dev/resources/core-mysql.edn b/dev/resources/core-mysql.edn index 8d3cd1fa..483269fe 100644 --- a/dev/resources/core-mysql.edn +++ b/dev/resources/core-mysql.edn @@ -67,7 +67,7 @@ "INSERT INTO `article` (`id`, `title`, `current_revision`) VALUES (1, 'introduction', 2), (2, 'hello world', 1)" ] :down ["DROP TABLE `article`"]} [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-revision-table] - {:up ["CREATE TABLE `revision` (`id` SERIAL, `content` TEXT, `revision` INTEGER)" + {:up ["CREATE TABLE `revision` (`id` INTEGER, `content` TEXT, `revision` INTEGER)" "INSERT INTO `revision` (`id`, `content`, `revision`) VALUES (1, 'tbd', 1), (1, 'this is sparta', 2), (2, 'welcome to my site', 1)" ] :down ["DROP TABLE `revision`"]} } diff --git a/dev/resources/core-postgres.edn b/dev/resources/core-postgres.edn index 5f6c5f59..b18ef2f6 100644 --- a/dev/resources/core-postgres.edn +++ b/dev/resources/core-postgres.edn @@ -67,7 +67,7 @@ "INSERT INTO \"article\" (\"id\", \"title\", \"current_revision\") VALUES (1, 'introduction', 2), (2, 'hello world', 1)" ] :down ["DROP TABLE \"article\""]} [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-revision-table] - {:up ["CREATE TABLE \"revision\" (\"id\" SERIAL, \"content\" TEXT, \"revision\" INTEGER)" + {:up ["CREATE TABLE \"revision\" (\"id\" INTEGER, \"content\" TEXT, \"revision\" INTEGER)" "INSERT INTO \"revision\" (\"id\", \"content\", \"revision\") VALUES (1, 'tbd', 1), (1, 'this is sparta', 2), (2, 'welcome to my site', 1)" ] :down ["DROP TABLE \"revision\""]} } diff --git a/dev/resources/core-sqlite.edn b/dev/resources/core-sqlite.edn index cecacb53..46f6c9ad 100644 --- a/dev/resources/core-sqlite.edn +++ b/dev/resources/core-sqlite.edn @@ -67,7 +67,7 @@ "INSERT INTO `article` (`id`, `title`, `current_revision`) VALUES (1, 'introduction', 2), (2, 'hello world', 1)" ] :down ["DROP TABLE `article`"]} [:duct.migrator.ragtime/sql :walkable-demo.migration.scenario-5/create-revision-table] - {:up ["CREATE TABLE `revision` (`id` SERIAL, `content` TEXT, `revision` INTEGER)" + {:up ["CREATE TABLE `revision` (`id` INTEGER, `content` TEXT, `revision` INTEGER)" "INSERT INTO `revision` (`id`, `content`, `revision`) VALUES (1, 'tbd', 1), (1, 'this is sparta', 2), (2, 'welcome to my site', 1)" ] :down ["DROP TABLE `revision`"]} } diff --git a/dev/test/walkable/integration_test/common.cljc b/dev/test/walkable/integration_test/common.cljc index 8ba34525..384754d3 100644 --- a/dev/test/walkable/integration_test/common.cljc +++ b/dev/test/walkable/integration_test/common.cljc @@ -14,12 +14,12 @@ :output [:farmer/name :farmer/house-plus :farmer/house-count :farmer/house]} {:key :farmer/house :type :join - :join-path [:farmer/house-index :house/index] + :join-path [{:farmer/house-index :house/index}] :output [:house/color] :cardinality :one} {:key :house/owner :type :join - :join-path [:house/index :farmer/house-index] + :join-path [{:house/index :farmer/house-index}] :output [:farmer/number] :cardinality :one}]) @@ -30,12 +30,12 @@ :output [:kid/name]} {:key :toy/owner :type :join - :join-path [:toy/owner-number :kid/number] + :join-path [{:toy/owner-number :kid/number}] :output [:kid/name :kid/number] :cardinality :one} {:key :kid/toy :type :join - :join-path [:kid/number :toy/owner-number] + :join-path [{:kid/number :toy/owner-number}] :output [:toy/color :toy/index] :cardinality :one} {:key :kid/number @@ -50,7 +50,8 @@ :output [:human/number :human/name :human/yob :human/age]} {:key :human/follow :type :join - :join-path [:human/number :follow/human-1 :follow/human-2 :human/number] + :join-path [{:human/number :follow/human-1} + {:follow/human-2 :human/number}] :output [:human/number :human/name :human/yob :human/age]}]) (def person-pet-registry @@ -82,8 +83,8 @@ {:key :person/pet :type :join :join-path - [:person/number :person-pet/person-number - :person-pet/pet-index :pet/index] + [{:person/number :person-pet/person-number} + {:person-pet/pet-index :pet/index}] :output [:pet/index :person-pet/adoption-year :pet/name @@ -92,13 +93,14 @@ {:key :pet/owner :type :join :join-path - [:pet/index :person-pet/pet-index :person-pet/person-number :person/number] + [{:pet/index :person-pet/pet-index} + {:person-pet/person-number :person/number}] :output [:person/number]} {:key :person/pet-count :type :join :join-path - [:person/number :person-pet/person-number - :person-pet/pet-index :pet/index] + [{:person/number :person-pet/person-number} + {:person-pet/pet-index :pet/index}] :aggregate true :formula [:count-*]} {:key :pets/by-color @@ -111,6 +113,18 @@ :type :pseudo-column :formula [:count :pet/index]}]) +(def article-revision-registry + [{:key :articles/list + :type :root + :table "article" + :output [:article/id :article/title :article/current-revision]} + {:key :article/revision + :type :join + :cardinality :one + :output [:revision/content] + :join-path [{:article/id :revision/id + :article/current-revision :revision/revision}]}]) + (def common-scenarios {:farmer-house {:registry farmer-house-registry @@ -161,7 +175,7 @@ {:farmer/house [:house/index :house/color]}]}] :expected #:farmers{:farmers [#:farmer{:number 2, :house #:house {:index "20", :color "brown"}}]}}]} - + :kid-toy {:registry kid-toy-registry :test-suite @@ -418,4 +432,21 @@ :person/age]}] :expected #:people {:people [#:person{:number 1, :name "jon", :yob 1980, :age 38} - #:person{:number 2, :name "mary", :yob 1992, :age 26}]}}]}}) + #:person{:number 2, :name "mary", :yob 1992, :age 26}]}}]} + + :article-revision + {:registry article-revision-registry + :test-suite + [{:message "join with multiple column pairs should work" + :query `[{:articles/list [:article/id :article/title + {:article/revision [:revision/content]}]}] + :expected + #:articles{:list + [#:article{:id 1, + :title "introduction", + :revision + #:revision{:content "this is sparta"}} + #:article{:id 2, + :title "hello world", + :revision + #:revision{:content "welcome to my site"}}]}}]}}) diff --git a/src/walkable/sql_query_builder/ast.cljc b/src/walkable/sql_query_builder/ast.cljc index 2a932ff7..f6cd8b35 100644 --- a/src/walkable/sql_query_builder/ast.cljc +++ b/src/walkable/sql_query_builder/ast.cljc @@ -10,13 +10,13 @@ [floor-plan {:keys [dispatch-key]}] (get-in floor-plan [:target-table dispatch-key])) -(defn target-column +(defn target-columns [floor-plan {:keys [dispatch-key]}] - (get-in floor-plan [:target-column dispatch-key])) + (get-in floor-plan [:target-columns dispatch-key])) -(defn source-column +(defn source-columns [floor-plan {:keys [dispatch-key]}] - (get-in floor-plan [:source-column dispatch-key])) + (get-in floor-plan [:source-columns dispatch-key])) (defn result-key [ast] (let [k (:pathom/as (:params ast))] @@ -138,9 +138,9 @@ (defn process-children* "Infers which columns to include in SQL query from child keys in ast" - [floor-plan ast] + [floor-plan ast] (let [all-children (:children ast) - + {:keys [columns joins]} (group-by #(keyword-type floor-plan %) all-children) @@ -148,7 +148,7 @@ (into #{} (map :dispatch-key) columns) child-source-columns - (into #{} (map #(source-column floor-plan %)) joins)] + (into #{} (mapcat #(source-columns floor-plan %)) joins)] {:columns-to-query (set/union child-column-keys child-source-columns)})) (defn process-children @@ -184,16 +184,16 @@ (defn individual-query-template [{:keys [floor-plan ast pagination]}] (let [ident? (vector? (:key ast)) - + {:keys [:columns-to-query]} (process-children floor-plan ast) - target-column (target-column floor-plan ast) + + target-columns (target-columns floor-plan ast) {:keys [:offset :limit :order-by :order-by-columns]} (when-not ident? pagination) columns-to-query - (-> (clojure.set/union columns-to-query order-by-columns) - (conj-some target-column)) + (clojure.set/union columns-to-query order-by-columns (set target-columns)) selection (process-selection floor-plan columns-to-query) @@ -204,17 +204,17 @@ sql-query {:raw-string (emitter/->query-string - {:target-table (target-table floor-plan ast) - :join-statement (join-statement floor-plan ast) - :selection (:raw-string selection) - :conditions (:raw-string conditions) - ;; TODO: - ;; use :raw-string/:params here in case there are variables in group-by columns - :group-by (compiled-group-by floor-plan ast) - :having (:raw-string having) - :offset offset - :limit limit - :order-by order-by}) + {:target-table (target-table floor-plan ast) + :join-statement (join-statement floor-plan ast) + :selection (:raw-string selection) + :conditions (:raw-string conditions) + ;; TODO: + ;; use :raw-string/:params here in case there are variables in group-by columns + :group-by (compiled-group-by floor-plan ast) + :having (:raw-string having) + :offset offset + :limit limit + :order-by order-by}) :params (expressions/combine-params selection conditions having)}] sql-query)) @@ -223,12 +223,7 @@ [shared-query batched-individuals])) #_(defn combine-without-cte [{:keys [batched-individuals]}] - batched-individuals) - -(defn source-column-variable-values - [v] - {:variable-values {`floor-plan/source-column-value - (expressions/compile-to-string {} v)}}) + batched-individuals) (defn compute-graphs [floor-plan env variables] (let [variable->graph-index (variable->graph-index floor-plan) @@ -327,8 +322,8 @@ (let [q (batched-individuals env entities)] (when (not-empty (:raw-string q)) (->> q - (process-query floor-plan env) - eliminate-unknown-variables)))))) + (process-query floor-plan env) + eliminate-unknown-variables)))))) (defn prepare-query [floor-plan ast] diff --git a/src/walkable/sql_query_builder/floor_plan.cljc b/src/walkable/sql_query_builder/floor_plan.cljc index aa4bdbe2..e792cce2 100644 --- a/src/walkable/sql_query_builder/floor_plan.cljc +++ b/src/walkable/sql_query_builder/floor_plan.cljc @@ -9,18 +9,57 @@ [walkable.sql-query-builder.helper :as helper] [walkable.sql-query-builder.pagination :as pagination])) -(s/def ::without-join-table - (s/coll-of ::expressions/namespaced-keyword - :count 2)) +(def floor-plan-ns (namespace ::this)) -(s/def ::with-join-table - (s/coll-of ::expressions/namespaced-keyword - :count 4)) +(defn source-column-symbol [i] + (symbol floor-plan-ns (str "source-column-" i))) -(s/def ::join-path - (s/or - :without-join-table ::without-join-table - :with-join-table ::with-join-table)) +(defn ->join-filter [i target-column] + [:= (source-column-symbol i) target-column]) + +(comment + (= (->join-filter 0 :foo/bar) + [:= `source-column-0 :foo/bar])) + +(defn source-column-variable-values + [values] + {:variable-values + (->> values + (into {} + (map-indexed (fn [i v] + [(source-column-symbol i) + (expressions/compile-to-string {} v)]))))}) + +(defn expand-join-first-hop + [m] + (let [source-columns (vec (keys m)) + target-columns (vec (vals m)) + filters (map ->join-filter (range) target-columns)] + {:source-columns source-columns + :source-column-values (apply juxt source-columns) + :target-columns target-columns + :target-column-values (apply juxt target-columns) + :join-filter (if (< 1 (count filters)) + (into [:and] filters) + (first filters))})) + +(defn join-first-hop + [_registry {:keys [:join-path] :as attr}] + (merge (expand-join-first-hop (first join-path)) + attr)) + +(comment + (let [{f :source-column-values} (expand-join-first-hop {:a/x :b/y})] + (mapv f [{:a/x 1} + {:a/x 2} + {:a/x 5}])) + (let [{f :source-column-values} (expand-join-first-hop {:a/x :b/y})] + (mapv f [{:a/x 1} + {:a/x 2} + {:a/x 5}])) + (expand-join-first-hop {:a/x :b/y}) + (expand-join-first-hop {:a/x :b/y :a/m :b/n}) + (expand-join-first-hop {:a/x :b/y :a/m :b/n :a/t :b/z})) (def prop->func {:table-name emitter/table-name @@ -41,16 +80,18 @@ (let [f (prop->func prop)] (f emitter k)))) -(defn join-statement* +(defn second-hop-join-statement* [registry {:keys [:join-path]}] - (let [[tag] (s/conform ::join-path join-path)] - (when (= :with-join-table tag) - (let [[_source _join-source join-target target] join-path] - (str - " JOIN " (keyword-prop registry target :table-name) - " ON " (keyword-prop registry join-target :column-name) - ;; TODO: what about other operators than `=`? - " = " (keyword-prop registry target :column-name)))))) + ;; TODO: use spec + (when (= 2 (count join-path)) + (let [pairs (second join-path) + table (keyword-prop registry (second (first pairs)) :table-name) + conditions (for [pair pairs + :let [[k v] (mapv #(keyword-prop registry % :column-name) pair)]] + (str k " = " v))] + (when (not-empty conditions) + (str " JOIN " table " ON " + (clojure.string/join " AND " conditions)))))) (defn conditionally-update* [x pred f] (if (pred x) @@ -66,32 +107,38 @@ m (assoc m k (f)))) -(defn join-statement +(defn second-hop-join-statement "Generate JOIN statement if not provided. Only for \"two-hop\" join paths." [registry item] - (derive-missing-key item :join-statement #(join-statement* registry item))) - -(defn join-source-column - [_registry {:keys [:join-path] :as item}] - (derive-missing-key item :source-column #(first join-path))) - -(defn join-target-column - [_registry {:keys [:join-path] :as item}] - (derive-missing-key item :target-column #(second join-path))) + (derive-missing-key item :join-statement #(second-hop-join-statement* registry item))) (defn join-target-table - [registry {:keys [:target-column] :as item}] - (derive-missing-key item :target-table #(keyword-prop registry target-column :table-name))) + [registry {:keys [:target-columns] :as item}] + ;; TODO: check if target-columns are in the same table + (derive-missing-key item :target-table #(keyword-prop registry (first target-columns) :table-name))) + +(defn column-list + [registry columns] + (let [column-names (map #(keyword-prop registry % :column-name) columns)] + (clojure.string/join ", " column-names))) + +(defn row-constructor + [registry columns] + (if (= 1 (count columns)) + (keyword-prop registry (first columns) :column-name) + (str "(" + (column-list registry columns) + ")"))) (defn join-filter-subquery* - [registry {:keys [:join-statement :target-table :source-column :target-column]}] - (str (keyword-prop registry source-column :column-name) - " IN (" - (emitter/->query-string - {:selection (keyword-prop registry target-column :column-name) - :target-table target-table - :join-statement join-statement}) - " WHERE ?)")) + [registry {:keys [:join-statement :source-columns :target-columns :target-table]}] + (str (row-constructor registry source-columns) + " IN (" + (emitter/->query-string + {:selection (column-list registry target-columns) + :target-table target-table + :join-statement join-statement}) + " WHERE ?)")) (defn join-filter-subquery [registry {:keys [:aggregate] :as item}] @@ -102,10 +149,9 @@ (defn compile-join [registry attribute] (->> attribute - (join-statement registry) - (join-source-column registry) - (join-target-column registry) + (join-first-hop registry) (join-target-table registry) + (second-hop-join-statement registry) (join-filter-subquery registry))) (defn compile-joins @@ -115,12 +161,12 @@ (fn [attr] (= :join (:type attr))) (fn [attr] (compile-join registry attr)))) -(defn replace-join-with-source-column-in-outputs +(defn replace-join-with-source-columns-in-outputs [{:keys [:attributes] :as registry}] - (let [join->source-column + (let [join->source-columns (->> attributes (filter #(= :join (:type %))) - (helper/build-index-of :source-column)) + (helper/build-index-of :source-columns)) plain-join-keys (->> attributes @@ -128,14 +174,14 @@ (map :key) set) - join-keys - (set (keys join->source-column))] + join-key-sets + (set (keys join->source-columns))] (update registry :attributes conditionally-update :output (fn [{:keys [:output] :as attr}] (let [joins-in-output - (filter join-keys output) + (filter join-key-sets output) source-column-is-ident? (if (:primary-key attr) @@ -143,7 +189,7 @@ (constantly false)) source-columns - (->> (mapv join->source-column joins-in-output) + (->> (mapcat join->source-columns joins-in-output) (remove source-column-is-ident?) set) @@ -154,7 +200,7 @@ (defn check-duplicate-keys [attrs] ;; TODO: implement with loop/recur to - ;; tell which key is duplicated + ;; tell which key is duplicated (when-not (apply distinct? (map :key attrs)) (throw (ex-info "Duplicate keys" {})))) @@ -431,18 +477,7 @@ :primary-key (fn [attr] (ident-filter registry attr)))) -(defn join-filter - ;; Note: :target-column must exist => derive it first - [_registry {:keys [:target-column] :as item}] - ;; TODO: (if (:use-cte item)) - (derive-missing-key item :join-filter (constantly [:= target-column (expressions/av `source-column-value)]))) - -(defn derive-join-filters - [registry] - (update registry :attributes - conditionally-update - (fn [attr] (= :join (:type attr))) - (fn [attr] (join-filter registry attr)))) +(def derive-join-filters identity) (defn collect-compiled-formulas [{:keys [:attributes] :as registry}] @@ -505,8 +540,13 @@ (defn collect-outputs [attrs] (into #{} (mapcat :output) attrs)) +;; TODO: replace with source-columns, target-columns (both first-hop and second-hop) (defn collect-join-paths [attrs] - (into #{} (mapcat :join-path) attrs)) + (into #{} + (apply concat + (for [m (mapcat :join-path attrs) + [k v] m] + [k v])))) (defn collect-under-key [k] (comp (map k) (remove nil?) (mapcat :params) (filter expressions/atomic-variable?) (map :name))) @@ -537,12 +577,12 @@ (let [{:keys [:found-variables :found-columns]} (collect-atomic-variables attributes) non-true-columns (into #{} (comp (filter #(#{:root :join :pseudo-column} (:type %))) (map :key)) attributes) all-columns (set (set/union found-columns - (collect-outputs attributes) - (collect-join-paths attributes))) + (collect-outputs attributes) + (collect-join-paths attributes))) true-columns (set/difference all-columns non-true-columns)] (merge registry - {:true-columns true-columns - :found-variables found-variables}))) + {:true-columns true-columns + :found-variables found-variables}))) (defn fill-true-column-attributes [{:keys [:attributes :true-columns] :as registry}] @@ -564,16 +604,16 @@ (defn collect-clojuric-names [{:keys [:attributes] :as registry}] (assoc registry :clojuric-names - (helper/build-index-of :clojuric-name (filter :clojuric-name attributes)))) + (helper/build-index-of :clojuric-name (filter :clojuric-name attributes)))) (defn compile-true-columns [{:keys [:emitter] :as registry}] (update registry :attributes - conditionally-update - #(#{:true-column} (:type %)) - ;; TODO: take into account existing prop :table, etc - #(let [inline-form (emitter/column-name emitter (:key %))] - (assoc % :compiled-formula {:raw-string inline-form :params []})))) + conditionally-update + #(#{:true-column} (:type %)) + ;; TODO: take into account existing prop :table, etc + #(let [inline-form (emitter/column-name emitter (:key %))] + (assoc % :compiled-formula {:raw-string inline-form :params []})))) (defn inline-into [k inline-forms] @@ -620,31 +660,34 @@ conditionally-update #(and (= :join (:type %)) (:aggregate %)) (fn [{:keys [:compiled-formula :clojuric-name] :as attr}] - (let [{:keys [:target-column]} attr + (let [{:keys [:target-columns]} attr aggregator-selection (expressions/selection compiled-formula clojuric-name) source-column-selection - (expressions/selection - {:raw-string "?" - :params [(expressions/av `source-column-value)]} - (emitter/clojuric-name emitter target-column))] + (->> target-columns + (map-indexed #(expressions/selection + {:raw-string "?" + :params [(expressions/av (source-column-symbol %1))]} + (emitter/clojuric-name emitter %2))))] (assoc attr :compiled-join-aggregator-selection (expressions/concatenate #(clojure.string/join ", " %) - [source-column-selection aggregator-selection])))))) + (into [aggregator-selection] source-column-selection))))))) (defn compile-join-selection [{:keys [:emitter] :as registry}] (update registry :attributes - conditionally-update - #(= :join (:type %)) - (fn [{:keys [:target-column] :as attr}] - (assoc attr :selection - (expressions/selection - {:raw-string "?" - :params [(expressions/av `source-column-value)]} - (emitter/clojuric-name emitter target-column)))))) + conditionally-update + #(= :join (:type %)) + (fn [{:keys [:target-columns] :as attr}] + (->> target-columns + (map-indexed #(expressions/selection + {:raw-string "?" + :params [(expressions/av (source-column-symbol %1))]} + (emitter/clojuric-name emitter %2))) + (expressions/concatenate #(clojure.string/join ", " %)) + (assoc attr :selection))))) (defn compile-traverse-scheme [{attr-type :type :as attr}] @@ -657,7 +700,7 @@ (#{:true-column :pseudo-column} attr-type) (assoc attr :traverse-scheme :columns) - + :else attr)) @@ -670,16 +713,17 @@ (update registry :attributes conditionally-update #(= :join (:type %)) - (fn [{:keys [:return :target-column :source-column] :as attr}] + (fn [{:keys [:return :target-column-values :source-column-values] :as attr}] (assoc attr :merge-sub-entities (fn ->merge-sub-entities [result-key] (fn merge-sub-entities [entities sub-entities] (if (empty? sub-entities) entities - (let [groups (group-by target-column sub-entities)] - (mapv (fn [entity] (let [source-column-value (get entity source-column)] - (assoc entity result-key (return (get groups source-column-value))))) - entities))))))))) + (let [groups (group-by target-column-values sub-entities)] + (->> entities + (mapv (fn [entity] + (let [scv (source-column-values entity)] + (assoc entity result-key (return (get groups scv))))))))))))))) (defn compile-root-sub-entities [registry] @@ -694,31 +738,26 @@ entities (assoc entities result-key (return sub-entities))))))))) -(defn source-column-variable-values - [v] - {:variable-values {`source-column-value - (expressions/compile-to-string {} v)}}) - (defn compile-query-multiplier [{:keys [batch-query] :as registry}] (update registry :attributes - conditionally-update - #(= :join (:type %)) - (fn [{:keys [:source-column] :as attr}] - (assoc attr :query-multiplier - (fn ->query-multiplier [individual-query-template] - (let [xform (comp (map #(get % source-column)) - (remove nil?) - (map #(-> (expressions/substitute-atomic-variables - (source-column-variable-values %) - individual-query-template) - ;; attach source-column-value as meta data - (with-meta {:source-column-value %}))))] - (fn query-multiplier* [_env entities] - (->> entities - ;; TODO: substitue-atomic-variables per entity - (into [] (comp xform)) - batch-query)))))))) + conditionally-update + #(= :join (:type %)) + (fn [{:keys [:source-column-values] :as attr}] + (assoc attr :query-multiplier + (fn ->query-multiplier [individual-query-template] + (let [xform (comp (map source-column-values) + (remove #(every? nil? %)) + (map #(-> (expressions/substitute-atomic-variables + (source-column-variable-values %) + individual-query-template) + ;; attach source-column-value as meta data + #_(with-meta {:source-column-value %}))))] + (fn query-multiplier* [_env entities] + (->> entities + ;; TODO: substitue-atomic-variables per entity + (into [] (comp xform)) + batch-query)))))))) (defn ident-table* [registry item] @@ -750,15 +789,15 @@ plain-joins (->> attributes (filter #(and (= :join (:type %)) (not (:aggregate %)))) - (mapv (fn [{k :key :keys [:output :source-column]}] - {::pc/input #{source-column} + (mapv (fn [{k :key :keys [:output :source-columns]}] + {::pc/input (into #{} source-columns) ::pc/output [{k output}]}))) join-aggregators (->> attributes (filter #(and (= :join (:type %)) (:aggregate %))) - (mapv (fn [{k :key :keys [:source-column]}] - {::pc/input #{source-column} + (mapv (fn [{k :key :keys [:source-columns]}] + {::pc/input (into #{} source-columns) ::pc/output [k]}))) idents (->> attributes @@ -771,25 +810,27 @@ (defn floor-plan [{:keys [:attributes]}] ;; build a compact version suitable for expressions/compile-to-string (merge - {:target-table (merge (helper/build-index-of :target-table (filter #(= :join (:type %)) attributes)) - (helper/build-index-of :table (filter #(or (= :root (:type %)) (:primary-key %)) attributes)))} - {:target-column (helper/build-index-of :target-column (filter #(= :join (:type %)) attributes))} - {:source-column (helper/build-index-of :source-column (filter #(= :join (:type %)) attributes))} - {:merge-sub-entities (helper/build-index-of :merge-sub-entities (filter #(or (#{:root :join} (:type %)) (:primary-key %)) attributes))} - {:query-multiplier (helper/build-index-of :query-multiplier (filter :query-multiplier attributes))} - {:join-statement (helper/build-index-of :join-statement (filter :join-statement attributes))} - {:compiled-join-selection (helper/build-index-of :compiled-join-selection (filter :compiled-join-selection attributes))} - {:compiled-join-aggregator-selection (helper/build-index-of :compiled-join-aggregator-selection (filter :compiled-join-aggregator-selection attributes))} - {:keyword-type (helper/build-index-of :traverse-scheme (filter #(#{:root :join :true-column :pseudo-column} (:type %)) attributes))} - {:compiled-pagination-fallbacks (helper/build-index-of :compiled-pagination-fallbacks (filter :compiled-pagination-fallbacks attributes))} - {:return (helper/build-index-of :return (filter #(or (#{:root :join} (:type %)) (:primary-key %)) attributes))} - {:aggregator-keywords (into #{} (comp (filter #(:aggregate %)) (map :key)) attributes)} - {:compiled-variable-getter (helper/build-index-of :compiled-variable-getter (filter :compiled-variable-getter attributes))} - {:all-filters (helper/build-index-of :all-filters (filter :all-filters attributes))} - {:compiled-having (helper/build-index-of :compiled-having (filter :compiled-having attributes))} - {:compiled-group-by (helper/build-index-of :compiled-group-by (filter :compiled-group-by attributes))} - {:ident-keywords (into #{} (comp (filter #(:primary-key %)) (map :key)) attributes)} - {:compiled-selection (helper/build-index-of :compiled-selection (filter :compiled-selection attributes))})) + {:target-table (merge (helper/build-index-of :target-table (filter #(= :join (:type %)) attributes)) + (helper/build-index-of :table (filter #(or (= :root (:type %)) (:primary-key %)) attributes)))} + {:target-columns (helper/build-index-of :target-columns (filter #(= :join (:type %)) attributes))} + {:target-column-values (helper/build-index-of :target-column-values (filter #(= :join (:type %)) attributes))} + {:source-columns (helper/build-index-of :source-columns (filter #(= :join (:type %)) attributes))} + {:source-column-values (helper/build-index-of :source-column-values (filter #(= :join (:type %)) attributes))} + {:merge-sub-entities (helper/build-index-of :merge-sub-entities (filter #(or (#{:root :join} (:type %)) (:primary-key %)) attributes))} + {:query-multiplier (helper/build-index-of :query-multiplier (filter :query-multiplier attributes))} + {:join-statement (helper/build-index-of :join-statement (filter :join-statement attributes))} + {:compiled-join-selection (helper/build-index-of :compiled-join-selection (filter :compiled-join-selection attributes))} + {:compiled-join-aggregator-selection (helper/build-index-of :compiled-join-aggregator-selection (filter :compiled-join-aggregator-selection attributes))} + {:keyword-type (helper/build-index-of :traverse-scheme (filter #(#{:root :join :true-column :pseudo-column} (:type %)) attributes))} + {:compiled-pagination-fallbacks (helper/build-index-of :compiled-pagination-fallbacks (filter :compiled-pagination-fallbacks attributes))} + {:return (helper/build-index-of :return (filter #(or (#{:root :join} (:type %)) (:primary-key %)) attributes))} + {:aggregator-keywords (into #{} (comp (filter #(:aggregate %)) (map :key)) attributes)} + {:compiled-variable-getter (helper/build-index-of :compiled-variable-getter (filter :compiled-variable-getter attributes))} + {:all-filters (helper/build-index-of :all-filters (filter :all-filters attributes))} + {:compiled-having (helper/build-index-of :compiled-having (filter :compiled-having attributes))} + {:compiled-group-by (helper/build-index-of :compiled-group-by (filter :compiled-group-by attributes))} + {:ident-keywords (into #{} (comp (filter #(:primary-key %)) (map :key)) attributes)} + {:compiled-selection (helper/build-index-of :compiled-selection (filter :compiled-selection attributes))})) (defn compact [registry] {:floor-plan (merge (select-keys registry [:emitter :operators :join-filter-subqueries :batch-query :compiled-formulas :clojuric-names]) @@ -812,7 +853,7 @@ (expand-nested-pseudo-columns) (expand-pseudo-columns-in-aggregators) (compile-joins) - (replace-join-with-source-column-in-outputs) + (replace-join-with-source-columns-in-outputs) (derive-ident-table) (join-filter-subqueries) (derive-ident-filters) diff --git a/test/walkable/sql_query_builder/floor_plan_test.cljc b/test/walkable/sql_query_builder/floor_plan_test.cljc index adce516b..a8610785 100644 --- a/test/walkable/sql_query_builder/floor_plan_test.cljc +++ b/test/walkable/sql_query_builder/floor_plan_test.cljc @@ -13,15 +13,3 @@ :a/b :table-name) "z"))) -(deftest join-statement*-tests - (is (= (sut/join-statement* - {:emitter emitter/default-emitter} - {:key :person/friend - :type :join - :join-path [:person/id :friendship/first-person :friendship/second-person :person/id]}) - " JOIN \"person\" ON \"friendship\".\"second_person\" = \"person\".\"id\"")) - (is (nil? (sut/join-statement* - {:emitter emitter/default-emitter} - {:key :person/house - :type :join - :join-path [:person/id :house/owner-id]}))))