diff --git a/src/keboola/facebook/extractor/query.clj b/src/keboola/facebook/extractor/query.clj index db5ea50..5d9c39a 100644 --- a/src/keboola/facebook/extractor/query.clj +++ b/src/keboola/facebook/extractor/query.clj @@ -27,19 +27,23 @@ (defn choose-token [id user-token version] (or (retrieve-page-access-token id user-token version) user-token)) -(defn- run-by-id-merge-and-write [token out-dir prefix query version] - (let [ids-str (:ids query) - prepare-query #(assoc query :ids %) - run-query (fn [id] (request/nested-request (choose-token id token version) (prepare-query id) :version version)) - ids-seq (s/split ids-str #",") - all-merged-queries-rows (mapcat #(run-query %) ids-seq) - all-rows (apply concat all-merged-queries-rows)] - (output/write-rows all-rows out-dir prefix))) +(defn- run-by-id-merge-and-write + ([token out-dir prefix query version] (run-by-id-merge-and-write token out-dir prefix query version choose-token)) + ([token out-dir prefix query version choose-token-fn] + (let [ids-str (:ids query) + prepare-query #(assoc query :ids %) + run-query (fn [id] (request/nested-request (choose-token-fn id token version) (prepare-query id) :version version)) + ids-seq (s/split ids-str #",") + all-merged-queries-rows (mapcat #(run-query %) ids-seq) + all-rows (apply concat all-merged-queries-rows)] + (output/write-rows all-rows out-dir prefix)))) -(defn run-nested-query [token out-dir {:keys [name query version]}] +(defn run-nested-query [token out-dir {:keys [name query version run-by-id?]}] (if-let [ids-str (:ids query)] - (doseq [ids (partition-all 50 (s/split ids-str #","))] - (run-and-write token out-dir name (assoc query :ids (s/join "," ids)) version)) + (if run-by-id? + (run-by-id-merge-and-write token out-dir name query version (constantly token)) + (doseq [ids (partition-all 50 (s/split ids-str #","))] + (run-and-write token out-dir name (assoc query :ids (s/join "," ids)) version))) ;else if no ids then run the whole query (run-and-write token out-dir name query version)) (runtime/log-strings "Run query" name "finished")) @@ -79,7 +83,7 @@ (runtime/log-strings "Run query:" query) (let [token (docker-config/get-fb-token credentials) q (check-ids query all-ids) - complete-query {:query (:query q) :name (:name q) :version (:api-version q)} + complete-query {:query (:query q) :name (:name q) :version (:api-version q) :run-by-id? (:run-by-id q)} run-with-page-token #(run-nested-query-with-page-token token out-dir complete-query) run-with-user-token #(run-nested-query token out-dir complete-query) run-query #(if (need-page-token? (:query query)) (run-with-page-token) (run-with-user-token))] diff --git a/src/keboola/http/recording.clj b/src/keboola/http/recording.clj index a85f3df..72cad63 100644 --- a/src/keboola/http/recording.clj +++ b/src/keboola/http/recording.clj @@ -21,7 +21,7 @@ (defn random-str [length] (apply str (take length (repeatedly random-char)))) -(def keywords-to-anonymize #{:name :story :caption :message :description :title :account_name :campaign_name}) +(def keywords-to-anonymize #{:name :story :caption :message :description :title :account_name :campaign_name :ad_name :impressions}) (defn- anonymize-item [item] (if (and (vector? item) diff --git a/test/keboola/snapshots/runbyid/apicalls.clj b/test/keboola/snapshots/runbyid/apicalls.clj new file mode 100644 index 0000000..56dcb51 --- /dev/null +++ b/test/keboola/snapshots/runbyid/apicalls.clj @@ -0,0 +1,33 @@ +(ns keboola.snapshots.runbyid.apicalls) +(def recorded +{ +{:method :get, + :address "https://graph.facebook.com/v4.0/", + :as :json, + :query-params + {:path "", + :fields + "insights.time_range({\"since\":\"2019-09-29\",\"until\":\"2019-09-30\"}).level(ad).time_increment(1){account_id,account_name,ad_name,ad_id,impressions}", + :ids "act_1146726535372240", + :access_token "XXTOKENXX", + :since "", + :until ""}} +(fn [req]{:status 200, + :body + "{\"act_1146726535372240\":{\"insights\":{\"data\":[{\"account_id\":\"1146726535372240\",\"account_name\":\"Rn4f8\",\"ad_name\":\"p84kw\",\"ad_id\":\"6164363071480\",\"impressions\":\"hLbV2\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"bepJ8\",\"ad_name\":\"WKjha\",\"ad_id\":\"6164363071680\",\"impressions\":\"pHulH\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"xTlvB\",\"ad_name\":\"5xRHT\",\"ad_id\":\"6164363072080\",\"impressions\":\"loLpz\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"oUxmM\",\"ad_name\":\"3VhKN\",\"ad_id\":\"6164363072280\",\"impressions\":\"cMF7L\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"qEb8h\",\"ad_name\":\"UyTj0\",\"ad_id\":\"6164363072880\",\"impressions\":\"Mh4jx\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"RiDsp\",\"ad_name\":\"k3L8Z\",\"ad_id\":\"6164363073280\",\"impressions\":\"i6URx\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"3gaaK\",\"ad_name\":\"7SbGa\",\"ad_id\":\"6164363071480\",\"impressions\":\"lQ5Cp\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"eJNaw\",\"ad_name\":\"5M1Bg\",\"ad_id\":\"6164363071680\",\"impressions\":\"Mx0hJ\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"KLzUd\",\"ad_name\":\"RLycB\",\"ad_id\":\"6164363072080\",\"impressions\":\"03zWv\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"bVMN4\",\"ad_name\":\"4hyrr\",\"ad_id\":\"6164363072280\",\"impressions\":\"mYTK4\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"Dyzf7\",\"ad_name\":\"I8lef\",\"ad_id\":\"6164363072880\",\"impressions\":\"RBPma\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"viIso\",\"ad_name\":\"rqbRY\",\"ad_id\":\"6164363073280\",\"impressions\":\"RhNJN\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"}],\"paging\":{\"cursors\":{\"before\":\"MAZDZD\",\"after\":\"MTEZD\"}}},\"id\":\"act_1146726535372240\"}}"} +){:method :get, + :address "https://graph.facebook.com/v4.0/", + :as :json, + :query-params + {:path "", + :fields + "insights.time_range({\"since\":\"2019-09-29\",\"until\":\"2019-09-30\"}).level(ad).time_increment(1){account_id,account_name,ad_name,ad_id,impressions}", + :ids "act_108176966036258", + :access_token "XXTOKENXX", + :since "", + :until ""}} +(fn [req]{:status 200, + :body + "{\"act_108176966036258\":{\"insights\":{\"data\":[{\"account_id\":\"108176966036258\",\"account_name\":\"9OJDi\",\"ad_name\":\"cJ2fD\",\"ad_id\":\"6134702917855\",\"impressions\":\"801nU\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"QTiTQ\",\"ad_name\":\"Oq3jX\",\"ad_id\":\"6134760720255\",\"impressions\":\"Yp7Wh\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"ejkr5\",\"ad_name\":\"9Oh5V\",\"ad_id\":\"6134760720655\",\"impressions\":\"5UmnU\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"GCygF\",\"ad_name\":\"j7oPW\",\"ad_id\":\"6134760721055\",\"impressions\":\"k1Ila\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"xvb11\",\"ad_name\":\"ItqGO\",\"ad_id\":\"6134760721255\",\"impressions\":\"Ftcom\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"8e32I\",\"ad_name\":\"yHrYg\",\"ad_id\":\"6134760721455\",\"impressions\":\"oQSrN\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"KDsFR\",\"ad_name\":\"bJZaK\",\"ad_id\":\"6134760721655\",\"impressions\":\"PiS1l\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"UGHhF\",\"ad_name\":\"MCkrb\",\"ad_id\":\"6134760721855\",\"impressions\":\"ST4rD\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"1itYW\",\"ad_name\":\"V8pJn\",\"ad_id\":\"6134760722255\",\"impressions\":\"LFpYk\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"8uHUi\",\"ad_name\":\"6Cgyi\",\"ad_id\":\"6134702917855\",\"impressions\":\"a5czK\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"1MXgy\",\"ad_name\":\"p2qRE\",\"ad_id\":\"6134760721255\",\"impressions\":\"g2Ude\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"OQ4HK\",\"ad_name\":\"G6SZI\",\"ad_id\":\"6134760721455\",\"impressions\":\"gZYka\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"dwBfc\",\"ad_name\":\"zYoRl\",\"ad_id\":\"6134760721655\",\"impressions\":\"RjeTM\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"rjTMp\",\"ad_name\":\"CIP6S\",\"ad_id\":\"6134760722255\",\"impressions\":\"U7dqW\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"}],\"paging\":{\"cursors\":{\"before\":\"MAZDZD\",\"after\":\"MTMZD\"}}},\"id\":\"act_108176966036258\"}}"} +) +}) \ No newline at end of file diff --git a/test/keboola/snapshots/runbyid/config.json b/test/keboola/snapshots/runbyid/config.json new file mode 100644 index 0000000..10b3642 --- /dev/null +++ b/test/keboola/snapshots/runbyid/config.json @@ -0,0 +1,51 @@ +{ + "storage" : { }, + "parameters" : { + "accounts" : { + "act_108176966036258" : { + "account_id" : "108176966036258", + "business_name" : "account 2", + "currency" : "GBP", + "id" : "act_108176966036258", + "name" : "account 2" + }, + "act_1146726535372240" : { + "account_id" : "1146726535372240", + "business_name" : "", + "currency" : "GBP", + "id" : "act_1146726535372240", + "name" : "account 3" + } + }, + "api-version" : "v4.0", + "queries" : [ { + "name" : "ads", + "type" : "nested-query", + "run-by-id" : true, + "disabled" : false, + "query" : { + "path" : "", + "fields" : "insights.time_range({\"since\":\"2019-09-29\",\"until\":\"2019-09-30\"}).level(ad).time_increment(1){account_id,account_name,ad_name,ad_id,impressions}", + "ids" : "" + } + } ] + }, + "authorization" : { + "oauth_api" : { + "id" : "{OAUTH_API_ID}", + "credentials" : { + "id" : "main", + "authorizedFor" : "Myself", + "creator" : { + "id" : "1234", + "description" : "me@keboola.com" + }, + "created" : "2016-01-31 00:13:30", + "oauthVersion" : "facebook", + "appKey" : "", + "#data" : "{\"token\":\"XXTOKENXX\"}", + "#appSecret" : "" + } + } + } +} \ No newline at end of file diff --git a/test/keboola/snapshots/runbyid/out/tables/accounts b/test/keboola/snapshots/runbyid/out/tables/accounts new file mode 100644 index 0000000..909b817 --- /dev/null +++ b/test/keboola/snapshots/runbyid/out/tables/accounts @@ -0,0 +1,3 @@ +account_id,name,business_name,currency,id +108176966036258,account 2,account 2,GBP,act_108176966036258 +1146726535372240,account 3,,GBP,act_1146726535372240 diff --git a/test/keboola/snapshots/runbyid/out/tables/ads_insights.manifest b/test/keboola/snapshots/runbyid/out/tables/ads_insights.manifest new file mode 100644 index 0000000..540e756 --- /dev/null +++ b/test/keboola/snapshots/runbyid/out/tables/ads_insights.manifest @@ -0,0 +1 @@ +{"columns":["ex-account-id","fb-graph-node","parent-id","account_id","account_name","ad_id","ad_name","date_start","date_stop","impressions"],"incremental":true,"primary_key":["parent_id","account_id","date_start","date_stop","ad_id"]} \ No newline at end of file diff --git a/test/keboola/snapshots/runbyid/out/tables/ads_insights/1570782087866 b/test/keboola/snapshots/runbyid/out/tables/ads_insights/1570782087866 new file mode 100644 index 0000000..99cd107 --- /dev/null +++ b/test/keboola/snapshots/runbyid/out/tables/ads_insights/1570782087866 @@ -0,0 +1,26 @@ +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","viIso","6164363073280","rqbRY","2019-09-30","2019-09-30","RhNJN" +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","Dyzf7","6164363072880","I8lef","2019-09-30","2019-09-30","RBPma" +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","bVMN4","6164363072280","4hyrr","2019-09-30","2019-09-30","mYTK4" +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","KLzUd","6164363072080","RLycB","2019-09-30","2019-09-30","03zWv" +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","eJNaw","6164363071680","5M1Bg","2019-09-30","2019-09-30","Mx0hJ" +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","3gaaK","6164363071480","7SbGa","2019-09-30","2019-09-30","lQ5Cp" +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","RiDsp","6164363073280","k3L8Z","2019-09-29","2019-09-29","i6URx" +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","qEb8h","6164363072880","UyTj0","2019-09-29","2019-09-29","Mh4jx" +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","oUxmM","6164363072280","3VhKN","2019-09-29","2019-09-29","cMF7L" +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","xTlvB","6164363072080","5xRHT","2019-09-29","2019-09-29","loLpz" +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","bepJ8","6164363071680","WKjha","2019-09-29","2019-09-29","pHulH" +"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","Rn4f8","6164363071480","p84kw","2019-09-29","2019-09-29","hLbV2" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","rjTMp","6134760722255","CIP6S","2019-09-30","2019-09-30","U7dqW" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","dwBfc","6134760721655","zYoRl","2019-09-30","2019-09-30","RjeTM" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","OQ4HK","6134760721455","G6SZI","2019-09-30","2019-09-30","gZYka" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","1MXgy","6134760721255","p2qRE","2019-09-30","2019-09-30","g2Ude" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","8uHUi","6134702917855","6Cgyi","2019-09-30","2019-09-30","a5czK" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","1itYW","6134760722255","V8pJn","2019-09-29","2019-09-29","LFpYk" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","UGHhF","6134760721855","MCkrb","2019-09-29","2019-09-29","ST4rD" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","KDsFR","6134760721655","bJZaK","2019-09-29","2019-09-29","PiS1l" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","8e32I","6134760721455","yHrYg","2019-09-29","2019-09-29","oQSrN" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","xvb11","6134760721255","ItqGO","2019-09-29","2019-09-29","Ftcom" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","GCygF","6134760721055","j7oPW","2019-09-29","2019-09-29","k1Ila" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","ejkr5","6134760720655","9Oh5V","2019-09-29","2019-09-29","5UmnU" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","QTiTQ","6134760720255","Oq3jX","2019-09-29","2019-09-29","Yp7Wh" +"act_108176966036258","page_insights","act_108176966036258","108176966036258","9OJDi","6134702917855","cJ2fD","2019-09-29","2019-09-29","801nU" diff --git a/test/keboola/snapshots/runbyid/test_runbyid.clj b/test/keboola/snapshots/runbyid/test_runbyid.clj new file mode 100644 index 0000000..d26d049 --- /dev/null +++ b/test/keboola/snapshots/runbyid/test_runbyid.clj @@ -0,0 +1,23 @@ +(ns keboola.snapshots.runbyid.test-runbyid + (:require [keboola.snapshots.runbyid.apicalls :as apicalls] + [clojure.test :as t :refer :all] + [keboola.snapshots.outdirs-check :as outdirs-check] + [keboola.test-utils.core :as test-utils] + [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] + [keboola.facebook.extractor.output :refer [reset-columns-map]] + [keboola.facebook.extractor.core :refer [prepare-and-run]] + ) + (:use clj-http.fake)) + +(deftest runbyid-test + (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "runbyid"))] + (disable-log-token) + (println "testing dir:" tmp-dir) + (println "expected dir:" "test/keboola/snapshots/runbyid") + (test-utils/copy-config-tmp "test/keboola/snapshots/runbyid" tmp-dir) + (with-global-fake-routes-in-isolation + apicalls/recorded + (reset-columns-map) + (prepare-and-run tmp-dir) + (outdirs-check/is-equal "test/keboola/snapshots/runbyid" tmp-dir) + ))) \ No newline at end of file