Merge pull request #115 from keboola/CM-635-ondra

Data API: add metric and dimension filter

src/Configuration/ConfigDefinition.php

@@ -115,6 +115,8 @@ public function getParametersDefinition(): ArrayNodeDefinition
                                 ->end()
                             ->end()
                         ->end()
+                        ->arrayNode('metricFilter')->ignoreExtraKeys(false)->children()->end()->end()
+                        ->arrayNode('dimensionFilter')->ignoreExtraKeys(false)->children()->end()->end()
                         ->arrayNode('dimensions')
                             ->prototype('array')
                                 ->children()

src/GoogleAnalytics/Client.php

@@ -232,7 +232,7 @@ private function getReportRequest(array $query): array
 
     private function getPropertyReportRequest(array $query): array
     {
-        return [
+        $request = [
             'dateRanges' => array_map(function ($item) {
                 return [
                     'startDate' => date('Y-m-d', (int) strtotime($this->getStartDate($item['startDate']))),
@@ -245,6 +245,22 @@ private function getPropertyReportRequest(array $query): array
             'offset' => $query['offset'] ?? 0,
             'limit' => $query['maxResults'] ?? 5000,
         ];
+
+        $dimensionsList = array_map(fn($v) => $v['name'], $request['dimensions']);
+        if (!empty($query['dimensionFilter']) &&
+            in_array($query['dimensionFilter']['filter']['fieldName'], $dimensionsList)
+        ) {
+            $request['dimensionFilter'] = $query['dimensionFilter'];
+        }
+
+        $metricsList = array_map(fn($v) => $v['name'], $request['metrics']);
+        if (!empty($query['metricFilter']) &&
+            in_array($query['metricFilter']['filter']['fieldName'], $metricsList)
+        ) {
+            $request['metricFilter'] = $query['metricFilter'];
+        }
+
+        return $request;
     }
 
     private function processResponseProperty(array $response, array $query): array

tests/functional/data-api-filter-dimension/expected/data/out/tables/properties.csv

@@ -0,0 +1,2 @@
+"propertyKey","propertyName","accountKey","accountName"
+"properties/255885884","users","accounts/185283969","Keboola"

tests/functional/data-api-filter-dimension/expected/data/out/tables/properties.csv.manifest

@@ -0,0 +1 @@
+{"destination":"in.c-ex-google-analytics-cfg.properties","primary_key":["propertyKey"],"incremental":true}

tests/functional/data-api-filter-dimension/expected/data/out/tables/users.csv

@@ -0,0 +1,18 @@
+"id","idProperty","date","dayOfWeek","totalUsers","itemViews"
+"72194b333e93bc548ef559a2ae554d7a6ee50947","255885884","2023-05-16","2","3","0"
+"6a0c94663242fa1b581c0f9d9f61a6460a1c3a81","255885884","2023-03-07","2","2","0"
+"20915d9158fa409723d6750a34b58519a42f39d9","255885884","2023-04-05","3","2","0"
+"f7630c8c0629aa9ea00f93da0c706ea893606177","255885884","2023-03-14","2","1","0"
+"a7ccfc291b472d36e4462b9dd46963223f62e283","255885884","2023-03-22","3","1","0"
+"c171f9b873901812c083f13fde11aafd67407bc6","255885884","2023-03-29","3","1","0"
+"df787f933da2e54e832d9dd7f212371bef8e6751","255885884","2023-04-04","2","1","0"
+"36d4537d931d1e614cf6874099f5661bc30927df","255885884","2023-04-12","3","1","0"
+"0163c6a7f236f9b805059f93556ba584dbda59e6","255885884","2023-04-19","3","1","0"
+"7388dc29d0ce8fb9836a5b66f48ad0f47ff5efbc","255885884","2023-04-25","2","1","0"
+"14f5cba68cf4083d017cb033223c0515932dc8f6","255885884","2023-05-02","2","1","0"
+"bb72ce4b6f330dab42b05cfde3f4ff4b93aee838","255885884","2023-05-03","3","1","0"
+"6519f3e86e6cdc6d00c20a671b6dbc0b7eef54d7","255885884","2023-05-09","2","1","0"
+"1c61f9de1e727acc7941ee673eb1f3e0d9a1a718","255885884","2023-05-17","3","1","0"
+"2163cb803564c4da700a33273aaf0242e1da5514","255885884","2023-05-24","3","1","0"
+"4b662e07e0a4427599761fc58bc1906a4a43ad6a","255885884","2023-05-31","3","1","0"
+"36d6b7a9b0f5c12aa70b64972afc2f54e7096e28","255885884","2023-06-06","2","1","0"

tests/functional/data-api-filter-dimension/expected/data/out/tables/users.csv.manifest

@@ -0,0 +1 @@
+{"destination":"in.c-ex-google-analytics-cfg.users","incremental":true,"primary_key":["id"]}

tests/functional/data-api-filter-dimension/expected/data/out/usage.json

@@ -0,0 +1 @@
+[{"metric":"API Calls","value":1}]

tests/functional/data-api-filter-dimension/source/data/config.json

@@ -0,0 +1,60 @@
+{
+  "authorization": {
+    "oauth_api": {
+      "credentials": {
+        "appKey": "%env(string:CLIENT_ID)%",
+        "#appSecret": "%env(string:CLIENT_SECRET)%",
+        "#data": "%env(string:CREDENTIALS_DATA)%"
+      }
+    }
+  },
+  "parameters": {
+    "outputBucket": "in.c-ex-google-analytics-cfg",
+    "retriesCount": 1,
+    "properties": [
+      {
+        "accountKey": "accounts/185283969",
+        "accountName": "Keboola",
+        "propertyKey": "properties/255885884",
+        "propertyName": "users"
+      }
+    ],
+    "outputTable": "users",
+    "endpoint": "data-api",
+    "query": {
+      "metrics": [
+        {
+          "name": "totalUsers"
+        },
+        {
+          "name": "itemViews"
+        }
+      ],
+      "dimensions": [
+        {
+          "name": "date"
+        },
+        {
+          "name": "dayOfWeek"
+        }
+      ],
+      "dimensionFilter": {
+        "filter": {
+          "fieldName": "dayOfWeek",
+          "inListFilter": {
+            "values": [
+              "2", "3"
+            ]
+          }
+        }
+      },
+      "filtersExpression": "",
+      "dateRanges": [
+        {
+          "startDate": "-3 months",
+          "endDate": "-1 day"
+        }
+      ]
+    }
+  }
+}

tests/functional/data-api-filter-metrics/expected/data/out/tables/properties.csv

@@ -0,0 +1,2 @@
+"propertyKey","propertyName","accountKey","accountName"
+"properties/255885884","users","accounts/185283969","Keboola"

tests/functional/data-api-filter-metrics/expected/data/out/tables/properties.csv.manifest

@@ -0,0 +1 @@
+{"destination":"in.c-ex-google-analytics-cfg.properties","primary_key":["propertyKey"],"incremental":true}

tests/functional/data-api-filter-metrics/expected/data/out/tables/users.csv

@@ -0,0 +1,2 @@
+"id","idProperty","date","totalUsers","itemViews"
+"eef4262bb69d17cacaf4d12d68ad8eb53e819cfc","255885884","2023-05-16","3","0"

tests/functional/data-api-filter-metrics/expected/data/out/tables/users.csv.manifest

@@ -0,0 +1 @@
+{"destination":"in.c-ex-google-analytics-cfg.users","incremental":true,"primary_key":["id"]}

tests/functional/data-api-filter-metrics/expected/data/out/usage.json

@@ -0,0 +1 @@
+[{"metric":"API Calls","value":1}]

tests/functional/data-api-filter-metrics/source/data/config.json

@@ -0,0 +1,58 @@
+{
+  "authorization": {
+    "oauth_api": {
+      "credentials": {
+        "appKey": "%env(string:CLIENT_ID)%",
+        "#appSecret": "%env(string:CLIENT_SECRET)%",
+        "#data": "%env(string:CREDENTIALS_DATA)%"
+      }
+    }
+  },
+  "parameters": {
+    "outputBucket": "in.c-ex-google-analytics-cfg",
+    "retriesCount": 1,
+    "properties": [
+      {
+        "accountKey": "accounts/185283969",
+        "accountName": "Keboola",
+        "propertyKey": "properties/255885884",
+        "propertyName": "users"
+      }
+    ],
+    "outputTable": "users",
+    "endpoint": "data-api",
+    "query": {
+      "metrics": [
+        {
+          "name": "totalUsers"
+        },
+        {
+          "name": "itemViews"
+        }
+      ],
+      "dimensions": [
+        {
+          "name": "date"
+        }
+      ],
+      "metricFilter": {
+        "filter": {
+          "fieldName": "totalUsers",
+          "numericFilter": {
+            "operation": "GREATER_THAN",
+            "value": {
+              "int64Value": "2"
+            }
+          }
+        }
+      },
+      "filtersExpression": "",
+      "dateRanges": [
+        {
+          "startDate": "-3 months",
+          "endDate": "-1 day"
+        }
+      ]
+    }
+  }
+}