support run query by each id separately

src/keboola/facebook/extractor/query.clj

@@ -27,19 +27,23 @@
 (defn choose-token [id user-token version]
   (or (retrieve-page-access-token id user-token version) user-token))
 
-(defn- run-by-id-merge-and-write [token out-dir prefix query version]
-  (let [ids-str (:ids query)
-        prepare-query #(assoc query :ids %)
-        run-query (fn [id] (request/nested-request (choose-token id token version) (prepare-query id) :version version))
-        ids-seq (s/split ids-str #",")
-        all-merged-queries-rows (mapcat #(run-query %) ids-seq)
-        all-rows (apply concat all-merged-queries-rows)]
-    (output/write-rows all-rows out-dir prefix)))
+(defn- run-by-id-merge-and-write
+  ([token out-dir prefix query version] (run-by-id-merge-and-write token out-dir prefix query version choose-token))
+  ([token out-dir prefix query version choose-token-fn]
+   (let [ids-str (:ids query)
+         prepare-query #(assoc query :ids %)
+         run-query (fn [id] (request/nested-request (choose-token-fn id token version) (prepare-query id) :version version))
+         ids-seq (s/split ids-str #",")
+         all-merged-queries-rows (mapcat #(run-query %) ids-seq)
+         all-rows (apply concat all-merged-queries-rows)]
+     (output/write-rows all-rows out-dir prefix))))
 
-(defn run-nested-query [token out-dir {:keys [name query version]}]
+(defn run-nested-query [token out-dir {:keys [name query version run-by-id?]}]
   (if-let [ids-str (:ids query)]
-    (doseq [ids (partition-all 50 (s/split ids-str #","))]
-      (run-and-write token out-dir name (assoc query :ids (s/join "," ids)) version))
+    (if run-by-id?
+      (run-by-id-merge-and-write token out-dir name query version (constantly token))
+      (doseq [ids (partition-all 50 (s/split ids-str #","))]
+        (run-and-write token out-dir name (assoc query :ids (s/join "," ids)) version)))
     ;else if no ids then run the whole query
     (run-and-write token out-dir name query version))
   (runtime/log-strings "Run query" name "finished"))
@@ -79,7 +83,7 @@
   (runtime/log-strings "Run query:" query)
   (let [token (docker-config/get-fb-token credentials)
         q (check-ids query all-ids)
-        complete-query {:query (:query q) :name (:name q) :version (:api-version q)}
+        complete-query {:query (:query q) :name (:name q) :version (:api-version q) :run-by-id? (:run-by-id q)}
         run-with-page-token #(run-nested-query-with-page-token token out-dir complete-query)
         run-with-user-token #(run-nested-query token out-dir complete-query)
         run-query #(if (need-page-token? (:query query)) (run-with-page-token) (run-with-user-token))]

src/keboola/http/recording.clj

@@ -21,7 +21,7 @@
 (defn random-str [length]
   (apply str (take length (repeatedly random-char))))
 
-(def keywords-to-anonymize #{:name :story :caption :message :description :title :account_name :campaign_name})
+(def keywords-to-anonymize #{:name :story :caption :message :description :title :account_name :campaign_name :ad_name :impressions})
 (defn- anonymize-item [item]
   (if (and
        (vector? item)

test/keboola/snapshots/runbyid/apicalls.clj

@@ -0,0 +1,33 @@
+(ns keboola.snapshots.runbyid.apicalls)
+(def recorded 
+{
+{:method :get,
+ :address "https://graph.facebook.com/v4.0/",
+ :as :json,
+ :query-params
+ {:path "",
+  :fields
+  "insights.time_range({\"since\":\"2019-09-29\",\"until\":\"2019-09-30\"}).level(ad).time_increment(1){account_id,account_name,ad_name,ad_id,impressions}",
+  :ids "act_1146726535372240",
+  :access_token "XXTOKENXX",
+  :since "",
+  :until ""}}
+(fn [req]{:status 200,
+ :body
+ "{\"act_1146726535372240\":{\"insights\":{\"data\":[{\"account_id\":\"1146726535372240\",\"account_name\":\"Rn4f8\",\"ad_name\":\"p84kw\",\"ad_id\":\"6164363071480\",\"impressions\":\"hLbV2\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"bepJ8\",\"ad_name\":\"WKjha\",\"ad_id\":\"6164363071680\",\"impressions\":\"pHulH\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"xTlvB\",\"ad_name\":\"5xRHT\",\"ad_id\":\"6164363072080\",\"impressions\":\"loLpz\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"oUxmM\",\"ad_name\":\"3VhKN\",\"ad_id\":\"6164363072280\",\"impressions\":\"cMF7L\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"qEb8h\",\"ad_name\":\"UyTj0\",\"ad_id\":\"6164363072880\",\"impressions\":\"Mh4jx\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"RiDsp\",\"ad_name\":\"k3L8Z\",\"ad_id\":\"6164363073280\",\"impressions\":\"i6URx\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"3gaaK\",\"ad_name\":\"7SbGa\",\"ad_id\":\"6164363071480\",\"impressions\":\"lQ5Cp\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"eJNaw\",\"ad_name\":\"5M1Bg\",\"ad_id\":\"6164363071680\",\"impressions\":\"Mx0hJ\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"KLzUd\",\"ad_name\":\"RLycB\",\"ad_id\":\"6164363072080\",\"impressions\":\"03zWv\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"bVMN4\",\"ad_name\":\"4hyrr\",\"ad_id\":\"6164363072280\",\"impressions\":\"mYTK4\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"Dyzf7\",\"ad_name\":\"I8lef\",\"ad_id\":\"6164363072880\",\"impressions\":\"RBPma\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"viIso\",\"ad_name\":\"rqbRY\",\"ad_id\":\"6164363073280\",\"impressions\":\"RhNJN\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"}],\"paging\":{\"cursors\":{\"before\":\"MAZDZD\",\"after\":\"MTEZD\"}}},\"id\":\"act_1146726535372240\"}}"}
+){:method :get,
+ :address "https://graph.facebook.com/v4.0/",
+ :as :json,
+ :query-params
+ {:path "",
+  :fields
+  "insights.time_range({\"since\":\"2019-09-29\",\"until\":\"2019-09-30\"}).level(ad).time_increment(1){account_id,account_name,ad_name,ad_id,impressions}",
+  :ids "act_108176966036258",
+  :access_token "XXTOKENXX",
+  :since "",
+  :until ""}}
+(fn [req]{:status 200,
+ :body
+ "{\"act_108176966036258\":{\"insights\":{\"data\":[{\"account_id\":\"108176966036258\",\"account_name\":\"9OJDi\",\"ad_name\":\"cJ2fD\",\"ad_id\":\"6134702917855\",\"impressions\":\"801nU\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"QTiTQ\",\"ad_name\":\"Oq3jX\",\"ad_id\":\"6134760720255\",\"impressions\":\"Yp7Wh\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"ejkr5\",\"ad_name\":\"9Oh5V\",\"ad_id\":\"6134760720655\",\"impressions\":\"5UmnU\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"GCygF\",\"ad_name\":\"j7oPW\",\"ad_id\":\"6134760721055\",\"impressions\":\"k1Ila\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"xvb11\",\"ad_name\":\"ItqGO\",\"ad_id\":\"6134760721255\",\"impressions\":\"Ftcom\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"8e32I\",\"ad_name\":\"yHrYg\",\"ad_id\":\"6134760721455\",\"impressions\":\"oQSrN\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"KDsFR\",\"ad_name\":\"bJZaK\",\"ad_id\":\"6134760721655\",\"impressions\":\"PiS1l\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"UGHhF\",\"ad_name\":\"MCkrb\",\"ad_id\":\"6134760721855\",\"impressions\":\"ST4rD\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"1itYW\",\"ad_name\":\"V8pJn\",\"ad_id\":\"6134760722255\",\"impressions\":\"LFpYk\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"8uHUi\",\"ad_name\":\"6Cgyi\",\"ad_id\":\"6134702917855\",\"impressions\":\"a5czK\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"1MXgy\",\"ad_name\":\"p2qRE\",\"ad_id\":\"6134760721255\",\"impressions\":\"g2Ude\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"OQ4HK\",\"ad_name\":\"G6SZI\",\"ad_id\":\"6134760721455\",\"impressions\":\"gZYka\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"dwBfc\",\"ad_name\":\"zYoRl\",\"ad_id\":\"6134760721655\",\"impressions\":\"RjeTM\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"rjTMp\",\"ad_name\":\"CIP6S\",\"ad_id\":\"6134760722255\",\"impressions\":\"U7dqW\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"}],\"paging\":{\"cursors\":{\"before\":\"MAZDZD\",\"after\":\"MTMZD\"}}},\"id\":\"act_108176966036258\"}}"}
+)
+})

test/keboola/snapshots/runbyid/config.json

@@ -0,0 +1,51 @@
+{
+  "storage" : { },
+  "parameters" : {
+    "accounts" : {
+      "act_108176966036258" : {
+        "account_id" : "108176966036258",
+        "business_name" : "account 2",
+        "currency" : "GBP",
+        "id" : "act_108176966036258",
+        "name" : "account 2"
+      },
+      "act_1146726535372240" : {
+        "account_id" : "1146726535372240",
+        "business_name" : "",
+        "currency" : "GBP",
+        "id" : "act_1146726535372240",
+        "name" : "account 3"
+      }
+    },
+    "api-version" : "v4.0",
+    "queries" : [ {
+      "name" : "ads",
+      "type" : "nested-query",
+      "run-by-id" : true,
+      "disabled" : false,
+      "query" : {
+        "path" : "",
+        "fields" : "insights.time_range({\"since\":\"2019-09-29\",\"until\":\"2019-09-30\"}).level(ad).time_increment(1){account_id,account_name,ad_name,ad_id,impressions}",
+        "ids" : ""
+      }
+    } ]
+  },
+  "authorization" : {
+    "oauth_api" : {
+      "id" : "{OAUTH_API_ID}",
+      "credentials" : {
+        "id" : "main",
+        "authorizedFor" : "Myself",
+        "creator" : {
+          "id" : "1234",
+          "description" : "[email protected]"
+        },
+        "created" : "2016-01-31 00:13:30",
+        "oauthVersion" : "facebook",
+        "appKey" : "",
+        "#data" : "{\"token\":\"XXTOKENXX\"}",
+        "#appSecret" : ""
+      }
+    }
+  }
+}

test/keboola/snapshots/runbyid/out/tables/accounts

@@ -0,0 +1,3 @@
+account_id,name,business_name,currency,id
+108176966036258,account 2,account 2,GBP,act_108176966036258
+1146726535372240,account 3,,GBP,act_1146726535372240

test/keboola/snapshots/runbyid/out/tables/ads_insights.manifest

@@ -0,0 +1 @@
+{"columns":["ex-account-id","fb-graph-node","parent-id","account_id","account_name","ad_id","ad_name","date_start","date_stop","impressions"],"incremental":true,"primary_key":["parent_id","account_id","date_start","date_stop","ad_id"]}

test/keboola/snapshots/runbyid/out/tables/ads_insights/1570782087866

@@ -0,0 +1,26 @@
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","viIso","6164363073280","rqbRY","2019-09-30","2019-09-30","RhNJN"
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","Dyzf7","6164363072880","I8lef","2019-09-30","2019-09-30","RBPma"
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","bVMN4","6164363072280","4hyrr","2019-09-30","2019-09-30","mYTK4"
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","KLzUd","6164363072080","RLycB","2019-09-30","2019-09-30","03zWv"
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","eJNaw","6164363071680","5M1Bg","2019-09-30","2019-09-30","Mx0hJ"
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","3gaaK","6164363071480","7SbGa","2019-09-30","2019-09-30","lQ5Cp"
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","RiDsp","6164363073280","k3L8Z","2019-09-29","2019-09-29","i6URx"
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","qEb8h","6164363072880","UyTj0","2019-09-29","2019-09-29","Mh4jx"
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","oUxmM","6164363072280","3VhKN","2019-09-29","2019-09-29","cMF7L"
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","xTlvB","6164363072080","5xRHT","2019-09-29","2019-09-29","loLpz"
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","bepJ8","6164363071680","WKjha","2019-09-29","2019-09-29","pHulH"
+"act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","Rn4f8","6164363071480","p84kw","2019-09-29","2019-09-29","hLbV2"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","rjTMp","6134760722255","CIP6S","2019-09-30","2019-09-30","U7dqW"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","dwBfc","6134760721655","zYoRl","2019-09-30","2019-09-30","RjeTM"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","OQ4HK","6134760721455","G6SZI","2019-09-30","2019-09-30","gZYka"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","1MXgy","6134760721255","p2qRE","2019-09-30","2019-09-30","g2Ude"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","8uHUi","6134702917855","6Cgyi","2019-09-30","2019-09-30","a5czK"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","1itYW","6134760722255","V8pJn","2019-09-29","2019-09-29","LFpYk"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","UGHhF","6134760721855","MCkrb","2019-09-29","2019-09-29","ST4rD"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","KDsFR","6134760721655","bJZaK","2019-09-29","2019-09-29","PiS1l"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","8e32I","6134760721455","yHrYg","2019-09-29","2019-09-29","oQSrN"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","xvb11","6134760721255","ItqGO","2019-09-29","2019-09-29","Ftcom"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","GCygF","6134760721055","j7oPW","2019-09-29","2019-09-29","k1Ila"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","ejkr5","6134760720655","9Oh5V","2019-09-29","2019-09-29","5UmnU"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","QTiTQ","6134760720255","Oq3jX","2019-09-29","2019-09-29","Yp7Wh"
+"act_108176966036258","page_insights","act_108176966036258","108176966036258","9OJDi","6134702917855","cJ2fD","2019-09-29","2019-09-29","801nU"

test/keboola/snapshots/runbyid/test_runbyid.clj

@@ -0,0 +1,23 @@
+(ns keboola.snapshots.runbyid.test-runbyid
+  (:require [keboola.snapshots.runbyid.apicalls :as apicalls]
+            [clojure.test :as t :refer :all]
+            [keboola.snapshots.outdirs-check :as outdirs-check]
+            [keboola.test-utils.core :as test-utils]
+            [keboola.facebook.extractor.sync-actions :refer [disable-log-token]]
+            [keboola.facebook.extractor.output :refer [reset-columns-map]]
+            [keboola.facebook.extractor.core :refer [prepare-and-run]]
+            )
+  (:use clj-http.fake))
+
+(deftest runbyid-test
+  (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "runbyid"))]
+    (disable-log-token)
+    (println "testing dir:" tmp-dir)
+    (println "expected dir:" "test/keboola/snapshots/runbyid")
+    (test-utils/copy-config-tmp "test/keboola/snapshots/runbyid" tmp-dir)
+    (with-global-fake-routes-in-isolation
+      apicalls/recorded
+      (reset-columns-map)
+      (prepare-and-run tmp-dir)
+      (outdirs-check/is-equal "test/keboola/snapshots/runbyid" tmp-dir)
+      )))