Merge pull request #4 from NorwegianRefugeeCouncil/all_token_usage

Refactor

ckanext/tracking/__init__.py

@@ -1 +1 @@
-__VERSION__ = '0.3.0'
+__VERSION__ = '0.3.2'

ckanext/tracking/blueprints/csv.py

@@ -3,145 +3,73 @@
 from io import StringIO
 
 from flask import Blueprint, Response
-from ckan import model
 from ckan.common import current_user
 from ckan.plugins import toolkit
 
-from ckanext.tracking.queries.api import (
-    get_all_token_usage,
-    get_most_accessed_token,
-    get_most_accessed_dataset_with_token,
+from ckanext.tracking.queries.data import (
+    all_token_usage_data,
+    most_accessed_token_data,
+    most_accessed_dataset_with_token_data,
 )
 
 
 log = logging.getLogger(__name__)
 tracking_csv_blueprint = Blueprint('tracking_csv', __name__, url_prefix='/tracking-csv')
 
 
-@tracking_csv_blueprint.route('/most-accessed-dataset-with-token.csv', methods=["GET"])
-def most_accessed_dataset_with_token_csv():
-    """ Get most accessed (using a API token) datasets """
-
+def _csv_response(auth_fn, data_fn, kwargs, filename):
+    """ general CSV response from data_fn(**kwargs) checking access with auth_fn """
     current_user_name = current_user.name if current_user else None
     context = {'user': current_user_name}
-    toolkit.check_access('most_accessed_dataset_with_token_csv', context)
-    data = get_most_accessed_dataset_with_token(limit=10)
-
+    toolkit.check_access(auth_fn, context)
+    data = data_fn(**kwargs)
+    # If no rows, return empty 204 CSV
+    if len(data) == 0:
+        return Response('', status=204)
     # Create CSV including package details
-    rows = []
-    for row in data:
-        object_id = row['object_id']
-        obj = model.Package.get(object_id)
-        if obj:
-            obj_title = obj.title
-            object_url = toolkit.url_for('dataset.read', id=obj.id, qualified=True)
-        else:
-            obj_title = None
-            object_url = None
-
-        rows.append({
-            'Dataset ID': object_id,
-            'Dataset title': obj_title,
-            'Dataset url': object_url,
-            'total': row['total'],
-        })
-
-    headers = ['Dataset ID', 'Dataset title', 'Dataset url', 'total']
+    headers = data[0].keys()
     buffer = StringIO()
     writer = csv.DictWriter(buffer, fieldnames=headers)
 
     writer.writeheader()
-    for row in rows:
+    for row in data:
         writer.writerow(row)
 
     response = Response(buffer.getvalue(), mimetype='text/csv')
-    filename = 'most-accessed-dataset-with-token.csv'
     response.headers.set("Content-Disposition", "attachment", filename=filename)
     return response
 
 
-@tracking_csv_blueprint.route('/most-accessed-token.csv', methods=["GET"])
-def most_accessed_token_csv():
-    """ Get most accessed tokens """
+@tracking_csv_blueprint.route('/most-accessed-dataset-with-token.csv', methods=["GET"])
+def most_accessed_dataset_with_token_csv():
+    """ Get most accessed (using a API token) datasets """
 
-    current_user_name = current_user.name if current_user else None
-    context = {'user': current_user_name}
-    toolkit.check_access('most_accessed_token_csv', context)
-    data = get_most_accessed_token(limit=10)
-    # Create CSV including package details
-    rows = []
-    for row in data:
-        user_id = row['user_id']
-        user = model.User.get(user_id)
-        if user:
-            user_title = user.fullname
-            user_name = user.name
-            user_url = toolkit.url_for('user.read', id=user.name, qualified=True)
-        else:
-            user_title = None
-            user_name = None
-            user_url = None
-
-        rows.append({
-            'User ID': user_id,
-            'User fullname': user_title,
-            'User name': user_name,
-            'User url': user_url,
-            'total': row['total'],
-        })
-
-    headers = ['User ID', 'User fullname', 'User name', 'User url', 'total']
-    buffer = StringIO()
-    writer = csv.DictWriter(buffer, fieldnames=headers)
+    return _csv_response(
+        'most_accessed_dataset_with_token_csv',
+        most_accessed_dataset_with_token_data,
+        {'limit': 10},
+        'most-accessed-dataset-with-token.csv',
+    )
 
-    writer.writeheader()
-    for row in rows:
-        writer.writerow(row)
 
-    response = Response(buffer.getvalue(), mimetype='text/csv')
-    filename = 'most-accessed-tokens.csv'
-    response.headers.set("Content-Disposition", "attachment", filename=filename)
-    return response
+@tracking_csv_blueprint.route('/most-accessed-token.csv', methods=["GET"])
+def most_accessed_token_csv():
+    """ Get most accessed tokens """
+    return _csv_response(
+        'most_accessed_token_csv',
+        most_accessed_token_data,
+        {'limit': 10},
+        'most-accessed-tokens.csv',
+    )
 
 
 @tracking_csv_blueprint.route('/all-token-usage.csv', methods=["GET"])
 def all_token_usage_csv():
     """ Get all tokens usage """
 
-    current_user_name = current_user.name if current_user else None
-    context = {'user': current_user_name}
-    toolkit.check_access('most_accessed_token_csv', context)
-    data = get_all_token_usage(limit=1000)
-    # Create CSV including package details
-    rows = []
-    for row in data:
-        user_id = row['user_id']
-        user = model.User.get(user_id)
-        user_name = user.name if user else None
-
-        rows.append({
-            'User ID': user_id,
-            'User name': user_name,
-            'Token name': row['token_name'],
-            'Tracking type': row['tracking_type'],
-            'Tracking sub type': row['tracking_sub_type'],
-            'Object type': row['object_type'],
-            'Object ID': row['object_id'],
-            'Timestamp': row['timestamp'],
-        })
-
-    headers = [
-        'User ID', 'User name', 'Token name', 'Tracking type', 'Tracking sub type',
-        'Object type', 'Object ID', 'Timestamp'
-    ]
-    buffer = StringIO()
-    writer = csv.DictWriter(buffer, fieldnames=headers)
-
-    writer.writeheader()
-    for row in rows:
-        writer.writerow(row)
-
-    response = Response(buffer.getvalue(), mimetype='text/csv')
-    filename = 'all-token-usage.csv'
-    response.headers.set("Content-Disposition", "attachment", filename=filename)
-    return response
+    return _csv_response(
+        'all_token_usage_csv',
+        all_token_usage_data,
+        {'limit': 1000},
+        'all-token-usage.csv',
+    )

ckanext/tracking/queries/data/__init__.py

@@ -0,0 +1,5 @@
+# flake8: noqa: F401
+
+from ckanext.tracking.queries.data.all import all_token_usage_data
+from ckanext.tracking.queries.data.dataset import most_accessed_dataset_with_token_data
+from ckanext.tracking.queries.data.token import most_accessed_token_data

ckanext/tracking/queries/data/all.py

@@ -0,0 +1,70 @@
+"""
+Post-preocessed data after the DB queries and before the CSV generation
+"""
+
+import logging
+from ckan import model
+from ckan.plugins import toolkit
+
+from ckanext.tracking.queries.api import get_all_token_usage
+
+
+log = logging.getLogger(__name__)
+
+
+def all_token_usage_data(limit=1000):
+    """ Get all tokens usage """
+
+    data = get_all_token_usage(limit=limit)
+    # Create CSV including package details
+    rows = []
+
+    for row in data:
+        user_id = row['user_id']
+        user = model.User.get(user_id)
+        if user:
+            user_name = user.fullname or user.name
+        else:
+            user_name = None
+        object_id = row['object_id']
+        object_type = row['object_type']
+        organization_url = None
+        organization_title = None
+        obj = None
+        obj_title = None
+        object_url = None
+        if object_id:
+            if object_type == 'dataset':
+                obj = model.Package.get(object_id)
+                obj_title = obj.title
+                object_url = toolkit.url_for('dataset.read', id=obj.id)
+                pkg = toolkit.get_action('package_show')({'ignore_auth': True}, {'id': obj.id})
+                owner_org = pkg.get('organization', {})
+                organization_title = owner_org.get('title')
+                organization_url = toolkit.url_for('organization.read', id=owner_org.get('id'))
+            elif object_type == 'resource':
+                obj = model.Resource.get(object_id)
+                obj_title = obj.name
+                object_url = toolkit.url_for('dataset_resource.read', id=obj.package_id, resource_id=obj.id)
+            elif object_type == 'organization':
+                obj = model.Organization.get(object_id)
+                obj_title = obj.title
+                object_url = toolkit.url_for('organization.read', id=obj.id)
+
+        rows.append({
+            'timestamp': row['timestamp'],
+            'user_id': user_id,
+            'user_name': user_name,
+            'user_fullname': user.fullname if user else None,
+            'token_name': row['token_name'],
+            'tracking_type': row['tracking_type'],
+            'tracking_sub_type': row['tracking_sub_type'],
+            'object_type': object_type,
+            'object_id': object_id,
+            'object_title': obj_title,
+            'object_url': object_url,
+            'organization_url': organization_url,
+            'organization_title': organization_title,
+        })
+
+    return rows

ckanext/tracking/queries/data/dataset.py

@@ -0,0 +1,37 @@
+"""
+Post-preocessed data after the DB queries and before the CSV generation
+"""
+
+import logging
+from ckan import model
+from ckan.plugins import toolkit
+
+from ckanext.tracking.queries.api import get_most_accessed_dataset_with_token
+
+
+log = logging.getLogger(__name__)
+
+
+def most_accessed_dataset_with_token_data(limit=10):
+    data = get_most_accessed_dataset_with_token(limit=limit)
+
+    # Create CSV including package details
+    rows = []
+    for row in data:
+        object_id = row['object_id']
+        obj = model.Package.get(object_id)
+        if obj:
+            obj_title = obj.title
+            object_url = toolkit.url_for('dataset.read', id=obj.name, qualified=True)
+        else:
+            obj_title = None
+            object_url = None
+
+        rows.append({
+            'dataset_id': object_id,
+            'dataset_title': obj_title,
+            'dataset_url': object_url,
+            'total': row['total'],
+        })
+
+    return rows

ckanext/tracking/queries/data/token.py

@@ -0,0 +1,41 @@
+"""
+Post-preocessed data after the DB queries and before the CSV generation
+"""
+
+import logging
+from ckan import model
+from ckan.plugins import toolkit
+
+from ckanext.tracking.queries.api import get_most_accessed_token
+
+
+log = logging.getLogger(__name__)
+
+
+def most_accessed_token_data(limit=10):
+    """ Get most accessed tokens """
+    data = get_most_accessed_token(limit=limit)
+    # Create CSV including package details
+    rows = []
+    for row in data:
+        user_id = row['user_id']
+        user = model.User.get(user_id)
+        if user:
+            user_title = user.fullname
+            user_name = user.name
+            user_url = toolkit.url_for('user.read', id=user.name, qualified=True)
+        else:
+            user_title = None
+            user_name = None
+            user_url = None
+
+        rows.append({
+            'user_id': user_id,
+            'user_fullname': user_title,
+            'user_name': user_name,
+            'user_url': user_url,
+            'token_name': row['token_name'],
+            'total': row['total'],
+        })
+
+    return rows

ckanext/tracking/tests/test_csv.py

@@ -54,19 +54,19 @@ def test_dataset_with_token_csv(self, app, base_data):
         # check the CSV content
         lines = full_response.splitlines()
         header = lines[0].split(',')
-        assert header == ['Dataset ID', 'Dataset title', 'Dataset url', 'total']
+        assert header == ['dataset_id', 'dataset_title', 'dataset_url', 'total']
         rows = lines[1:]
         # They are just two datasets
         assert len(rows) == 2
         for row in rows:
             fields = row.split(',')
             if fields[0] == base_data.dataset1['id']:
                 assert fields[1] == base_data.dataset1['title']
-                assert fields[2] == url_for('dataset.read', id=base_data.dataset1['id'], qualified=True)
+                assert fields[2] == url_for('dataset.read', id=base_data.dataset1['name'], qualified=True)
                 assert fields[3] == '6'
             elif fields[0] == base_data.dataset2['id']:
                 assert fields[1] == base_data.dataset2['title']
-                assert fields[2] == url_for('dataset.read', id=base_data.dataset2['id'], qualified=True)
+                assert fields[2] == url_for('dataset.read', id=base_data.dataset2['name'], qualified=True)
                 assert fields[3] == '3'
             else:
                 assert False, f"Unexpected dataset id: {fields[0]}"