Merge branch 'master' into stable

.circleci/config.yml

@@ -49,7 +49,7 @@ jobs:
   install-and-test:
     machine:
       image: ubuntu-2004:202010-01
-      resource_class: large
+      resource_class: xlarge
     environment:
       DOCKER_REGISTRY: cvisionai
     steps:

Makefile

@@ -446,7 +446,9 @@ FILES = \
     annotation/volume-control.js \
     analytics/analytics-breadcrumbs.js \
     analytics/analytics-settings.js \
-    analytics/dashboard/dashboard.js \
+    analytics/dashboards/dashboard-portal.js \
+    analytics/dashboards/dashboard-summary.js \
+    analytics/dashboards/dashboard.js \
     analytics/localizations/card.js \
     analytics/localizations/gallery.js \
     analytics/localizations/panel-data.js \
@@ -457,8 +459,7 @@ FILES = \
     analytics/collections/gallery.js \
     analytics/collections/collections.js \
     analytics/collections/collections-data.js \
-    analytics/visualization/visualization.js \
-    analytics/reports/reports.js \
+    analytics/portal/portal.js \
     third_party/autocomplete.js \
     third_party/webrtcstreamer.js \
     utilities.js
@@ -554,6 +555,9 @@ python-bindings: tator-image
 	cd scripts/packages/tator-py
 	rm -rf dist
 	python3 setup.py sdist bdist_wheel
+	if [ ! -f dist/*.whl ]; then
+		exit 1
+	fi
 	cd ../../..
 
 .PHONY: r-docs

main/models.py

@@ -1477,6 +1477,20 @@ class AnnouncementToUser(Model):
     announcement = ForeignKey(Announcement, on_delete=CASCADE)
     user = ForeignKey(User, on_delete=CASCADE)
 
+class Dashboard(Model):
+    """ Standalone HTML page shown as a dashboard within a project.
+    """
+    categories = ArrayField(CharField(max_length=128), default=list, null=True)
+    """ List of categories associated with the dashboard. This field is currently ignored. """
+    description = CharField(max_length=1024, blank=True)
+    """ Description of the dashboard. """
+    html_file = FileField(upload_to=ProjectBasedFileLocation, null=True, blank=True)
+    """ Dashboard's HTML file """
+    name = CharField(max_length=128)
+    """ Name of the dashboard """
+    project = ForeignKey(Project, on_delete=CASCADE, db_column='project')
+    """ Project associated with the dashboard """
+
 def type_to_obj(typeObj):
     """Returns a data object for a given type object"""
     _dict = {

main/rest/__init__.py

@@ -17,6 +17,8 @@
 from .bucket import BucketDetailAPI
 from .change_log import ChangeLogListAPI
 from .clone_media import CloneMediaListAPI
+from .dashboard import DashboardListAPI
+from .dashboard import DashboardDetailAPI
 from .download_info import DownloadInfoAPI
 from .email import EmailAPI
 from .favorite import FavoriteListAPI
@@ -63,6 +65,7 @@
 from .permalink import PermalinkAPI
 from .project import ProjectListAPI
 from .project import ProjectDetailAPI
+from .save_html_file import SaveHTMLFileAPI
 from .section import SectionListAPI
 from .section import SectionDetailAPI
 from .section_analysis import SectionAnalysisAPI

main/rest/dashboard.py

@@ -0,0 +1,135 @@
+import datetime
+import logging
+import os
+
+from django.db import transaction
+from django.conf import settings
+
+from ..models import Project
+from ..models import Dashboard
+from ..models import User
+from ..models import database_qs
+from ..schema import DashboardListSchema
+from ..schema import DashboardDetailSchema
+from ..schema import parse
+from ..schema.components.dashboard import dashboard_fields as fields
+
+from ._base_views import BaseListView
+from ._base_views import BaseDetailView
+from ._permissions import ProjectExecutePermission
+
+logger = logging.getLogger(__name__)
+
+class DashboardListAPI(BaseListView):
+    schema = DashboardListSchema()
+    permission_classes = [ProjectExecutePermission]
+    http_method_names = ['get', 'post']
+
+    def _get(self, params: dict) -> dict:
+        qs = Dashboard.objects.filter(project=params['project']).order_by('id')
+        return database_qs(qs)
+
+    def get_queryset(self) -> dict:
+        params = parse(self.request)
+        qs = Dashboard.objects.filter(project__id=params['project'])
+        return qs
+
+    def _post(self, params: dict) -> dict:
+        # Does the project ID exist?
+        project_id = params[fields.project]
+        try:
+            project = Project.objects.get(pk=project_id)
+        except Exception as exc:
+            log_msg = f'Provided project ID ({project_id}) does not exist'
+            logger.error(log_msg)
+            raise exc
+
+        # Gather the dashboard file and verify it exists on the server in the right project
+        dashboard_file = os.path.basename(params[fields.html_file])
+        dashboard_url = os.path.join(str(project_id), dashboard_file)
+        dashboard_path = os.path.join(settings.MEDIA_ROOT, dashboard_url)
+        if not os.path.exists(dashboard_path):
+            log_msg = f'Provided dashboard ({dashboard_file}) does not exist in {settings.MEDIA_ROOT}'
+            logging.error(log_msg)
+            raise ValueError(log_msg)
+
+        # Get the optional fields and to null if need be
+        description = params.get(fields.description, None)
+        categories = params.get(fields.categories, None)
+
+        new_dashboard = Dashboard.objects.create(
+            categories=categories,
+            description=description,
+            html_file=dashboard_path,
+            name=params[fields.name],
+            project=project)
+
+        return {"message": f"Successfully created dashboard {new_dashboard.id}!", "id": new_dashboard.id}
+
+class DashboardDetailAPI(BaseDetailView):
+    schema = DashboardDetailSchema()
+    permission_classes = [ProjectExecutePermission]
+    http_method_names = ['get', 'patch', 'delete']
+
+    def safe_delete(self, path: str) -> None:
+        try:
+            logger.info(f"Deleting {path}")
+            os.remove(path)
+        except:
+            logger.warning(f"Could not remove {path}")
+
+    def _delete(self, params: dict) -> dict:
+        # Grab the dashboard object and delete it from the database
+        dashboard = Dashboard.objects.get(pk=params['id'])
+        html_file = dashboard.html_file
+        dashboard.delete()
+
+        # Delete the correlated file
+        path = os.path.join(settings.MEDIA_ROOT, html_file.name)
+        self.safe_delete(path=path)
+
+        msg = 'Registered dashboard deleted successfully!'
+        return {'message': msg}
+
+    def _get(self, params):
+        return database_qs(Dashboard.objects.filter(pk=params['id']))[0]
+
+    @transaction.atomic
+    def _patch(self, params) -> dict:
+        dashboard_id = params["id"]
+        obj = Dashboard.objects.get(pk=dashboard_id)
+
+        name = params.get(fields.name, None)
+        if name is not None:
+            obj.name = name
+
+        description = params.get(fields.description, None)
+        if description is not None:
+            obj.description = description
+
+        categories = params.get(fields.categories, None)
+        if categories is not None:
+            obj.categories = categories
+
+        html_file = params.get(fields.html_file, None)
+        if html_file is not None:
+            dashboard_file = os.path.basename(html_file)
+            dashboard_url = os.path.join(str(obj.project.id), dashboard_file)
+            dashboard_path = os.path.join(settings.MEDIA_ROOT, dashboard_url)
+            if not os.path.exists(dashboard_path):
+                log_msg = f'Provided dashboard ({dashboard_path}) does not exist'
+                logging.error(log_msg)
+                raise ValueError("Dashboard file does not exist in expected location.")
+
+            delete_path = os.path.join(settings.MEDIA_ROOT, obj.html_file.name)
+            self.safe_delete(path=delete_path)
+            obj.html_file = dashboard_path
+
+        obj.save()
+
+        return {'message': f'Dashboard {dashboard_id} successfully updated!'}
+
+    def get_queryset(self):
+        """ Returns a queryset of all registered dashboard files
+        """
+        return Dashboard.objects.all()

main/rest/save_html_file.py

@@ -0,0 +1,54 @@
+import logging
+import os
+import shutil
+
+from django.conf import settings
+
+from ..schema import SaveHTMLFileSchema
+from ..schema.components.html_file import html_file_fields as fields
+from ..download import download_file
+
+from ._base_views import BaseListView
+from ._permissions import ProjectExecutePermission
+
+logger = logging.getLogger(__name__)
+
+class SaveHTMLFileAPI(BaseListView):
+    """ Saves a HTML file used for reports and dashboards
+    """
+
+    schema = SaveHTMLFileSchema()
+    permission_clases = [ProjectExecutePermission]
+    http_method_names = ['post']
+
+    def _post(self, params: dict) -> dict:
+        """ Saves the uploaded report file into the project's permanent storage
+        """
+
+        # Verify the provided file has been uploaded
+        upload_url = params[fields.upload_url]
+
+        # Move the file to the right location using the provided name
+        project_id = str(params[fields.project])
+        basename = os.path.basename(params[fields.name])
+        filename, extension = os.path.splitext(basename)
+        new_filename = filename + extension
+        final_path = os.path.join(settings.MEDIA_ROOT, project_id, new_filename)
+
+        # Make sure there's not a duplicate of this file.
+        file_index = -1
+        while os.path.exists(final_path):
+            file_index += 1
+            new_filename = f'{filename}_{file_index}{extension}'
+            final_path = os.path.join(settings.MEDIA_ROOT, project_id, new_filename)
+
+        project_folder = os.path.dirname(final_path)
+        os.makedirs(project_folder, exist_ok=True)
+
+        # Download the file to the final path.
+        download_file(upload_url, final_path)
+
+        # Create the response back to the user
+        new_url = os.path.join(project_id, new_filename)
+        response = {fields.url: new_url}
+        return response

main/schema/__init__.py

@@ -18,6 +18,8 @@
 from .bucket import BucketDetailSchema
 from .change_log import ChangeLogListSchema
 from .clone_media import CloneMediaListSchema
+from .dashboard import DashboardListSchema
+from .dashboard import DashboardDetailSchema
 from .download_info import DownloadInfoSchema
 from .email import EmailSchema
 from .favorite import FavoriteListSchema
@@ -65,6 +67,7 @@
 from .permalink import PermalinkSchema
 from .project import ProjectListSchema
 from .project import ProjectDetailSchema
+from .save_html_file import SaveHTMLFileSchema
 from .section import SectionListSchema
 from .section import SectionDetailSchema
 from .section_analysis import SectionAnalysisSchema

main/schema/_generator.py

@@ -57,6 +57,8 @@ def get_schema(self, request=None, public=True, parser=False):
                 'Bucket': bucket,
                 'ChangeLog': change_log,
                 'CloneMediaSpec': clone_media_spec,
+                'Dashboard': dashboard,
+                'DashboardSpec': dashboard_spec,
                 'DownloadInfoSpec': download_info_spec,
                 'DownloadInfo': download_info,
                 'EmailSpec': email_spec,
@@ -68,6 +70,8 @@ def get_schema(self, request=None, public=True, parser=False):
                 'FeedDefinition': feed_definition,
                 'FileDefinition': file_definition,
                 'FloatArrayQuery': float_array_query,
+                'HTMLFile': html_file,
+                'HTMLFileSpec': html_file_spec,
                 'LiveDefinition': live_definition,
                 'LiveUpdateDefinition': live_update_definition,
                 'ImageDefinition': image_definition,

main/schema/components/__init__.py

@@ -27,13 +27,17 @@
 from .bucket import bucket
 from .change_log import change_log
 from .clone_media import clone_media_spec
+from .dashboard import dashboard
+from .dashboard import dashboard_spec
 from .download_info import download_info_spec
 from .download_info import download_info
 from .email import email_spec
 from .email import email_attachment_spec
 from .favorite import favorite_spec
 from .favorite import favorite_update
 from .favorite import favorite
+from .html_file import html_file
+from .html_file import html_file_spec
 from .invitation import (
     invitation_spec,
     invitation_update,

main/schema/components/algorithm.py

@@ -102,7 +102,7 @@
             'type': 'string',
         },
         manifest_fields.upload_url: {
-            'description': 'URL of the uploaded file returned from tus upload',
+            'description': 'URL of the uploaded file',
             'type': 'string',
         },
     },

main/schema/components/dashboard.py

@@ -0,0 +1,54 @@
+from types import SimpleNamespace
+
+dashboard_fields = SimpleNamespace(
+    description="description",
+    html_file="html_file",
+    id="id",
+    name="name",
+    project="project",
+    categories="categories")
+
+dashboard_post_properties = {
+    dashboard_fields.name: {
+        "type": "string",
+        "description": "Name of dashboard"
+    },
+    dashboard_fields.html_file: {
+        "type": "string",
+        "description": "Server URL to dashboard HTML file"
+    },
+    dashboard_fields.description: {
+        "type": "string",
+        "description": "Description of dashboard"
+    },
+    dashboard_fields.categories: {
+        'type': 'array',
+        'description': 'List of categories the dashboard belongs to',
+        'items': {'type': 'string'},
+    },
+}
+
+# Note: While project is required, it's part of the path parameter(s)
+dashboard_spec = {
+    'type': 'object',
+    'description': 'Register dashboard spec.',
+    'properties': {
+        **dashboard_post_properties,
+    },
+}
+
+dashboard = {
+    "type": "object",
+    "description": "Dashboard spec.",
+    "properties": {
+        dashboard_fields.id: {
+            "type": "integer",
+            "description": "Unique integer identifying the dashboard",
+        },
+        dashboard_fields.project: {
+            "type": "integer",
+            "description": "Unique integer identifying the project associated with the dashboard",
+        },
+        **dashboard_post_properties
+    },
+}