Tasks management command

alyx/data/management/commands/files.py

@@ -113,7 +113,7 @@ def handle(self, *args, **options):
                     "files on local server. Exiting now."))
                 return
             if before is None:
-                self.stdout.write(self.style.ERROR("Date beforeshould be specified: use the "
+                self.stdout.write(self.style.ERROR("Date before should be specified: use the "
                                                    "--before=yyyy-mm-dd flag. Exiting now."))
                 return
             dtypes = ['ephysData.raw.ap', 'ephysData.raw.lf', 'ephysData.raw.nidq',

alyx/jobs/management/commands/tasks.py

@@ -0,0 +1,67 @@
+from datetime import date
+
+from django.core.management import BaseCommand
+
+from jobs.models import Task
+
+
+class Command(BaseCommand):
+    """
+        ./manage.py tasks cleanup --before 2020-01-01 --status=20 --dry
+        ./manage.py tasks cleanup --status=Waiting --dry
+        ./manage.py tasks cleanup --status=~Complete --limit=200
+    """
+    help = 'Manage tasks'
+
+    def add_arguments(self, parser):
+        parser.add_argument('action', help='Action')
+        parser.add_argument('--status', help='Only delete tasks with this status')
+        parser.add_argument('--dry', action='store_true', help='dry run')
+        parser.add_argument('--limit', help='limit to a maximum number of tasks')
+        parser.add_argument('--before', help='select tasks before a given date')
+
+    def handle(self, *args, **options):
+        action = options.get('action')
+
+        dry = options.get('dry')
+        before = options.get('before', date.today().isoformat())
+
+        if action != 'cleanup':
+            raise ValueError(f'Action "{action}" not recognized')
+
+        before = date.fromisoformat(before)  # validate
+        tasks = Task.objects.filter(datetime__date__lte=before)
+        # Filter status
+        if status := options.get('status'):
+            if status.startswith('~'):
+                status = status[1:]
+                fcn = tasks.exclude
+            else:
+                fcn = tasks.filter
+            if status.isnumeric():
+                status = int(status)
+                if status not in {s[0] for s in Task.STATUS_DATA_SOURCES}:
+                    raise ValueError(f'Status {status} not recognized')
+            else:  # convert status string to int
+                status = next(
+                    (i for i, s in Task.STATUS_DATA_SOURCES if s.casefold() == status.casefold()),
+                    None
+                )
+                if status is None:
+                    raise ValueError(f'Status "{status}" not recognized')
+            tasks = fcn(status=status)
+
+        # Limit
+        if (limit := options.get('limit')) is not None:
+            limit = int(limit)
+            tasks = tasks.order_by('datetime')[:limit]
+
+        self.stdout.write(self.style.SUCCESS(f'Found {tasks.count()} tasks to delete'))
+        if not dry:
+            if limit is None:
+                tasks.delete()
+            else:
+                pks = tasks.values_list('pk', flat=True)
+                Task.objects.filter(pk__in=pks).delete()
+            self.stdout.write(self.style.SUCCESS('Tasks deleted'))
+

alyx/jobs/tests.py

@@ -1,10 +1,15 @@
+from unittest.mock import patch
+
 from django.contrib.auth import get_user_model
 from django.urls import reverse
 from django.core.management import call_command
+from datetime import datetime, timedelta
 
 from actions.models import Session
 from alyx.base import BaseTests
 from data.models import DataRepository
+from jobs.management.commands import tasks
+from jobs.models import Task
 
 
 class APISubjectsTests(BaseTests):
@@ -32,3 +37,71 @@ def test_brain_regions_rest_filter(self):
                      'arguments': {'titi': 'toto', 'tata': 'tutu'}, 'data_repository': 'myrepo'}
         rep = self.post(reverse('tasks-list'), task_dict)
         self.assertEqual(rep.status_code, 201)
+
+
+class TestManagementTasks(BaseTests):
+    """Tests for the tasks management command."""
+
+    def setUp(self) -> None:
+        """Create some tasks to clean up."""
+        self.n_tasks = 100
+        self.command = tasks.Command()
+        base = datetime.today()
+        # Create equally-spaced task dates
+        date_list = [base - timedelta(days=x) for x in range(self.n_tasks)]
+        with patch('django.db.models.fields.timezone.now') as timezone_mock:
+            for i, date in enumerate(date_list):
+                timezone_mock.return_value = date
+                status = Task.STATUS_DATA_SOURCES[i % len(Task.STATUS_DATA_SOURCES)][0]
+                Task.objects.create(name=f'task_{i}', status=status)
+
+    def test_cleanup(self):
+        """Test for cleanup action."""
+        # First run in dry mode, expect submit_delete to not be called
+        n = self.n_tasks - 10
+        before_date = (datetime.today() - timedelta(days=n)).date()
+        with patch.object(self.command.stdout, 'write') as stdout_mock:
+            self.command.handle(action='cleanup', before=str(before_date), dry=True)
+            stdout_mock.assert_called()
+            self.assertIn(f'Found {10} tasks to delete', stdout_mock.call_args.args[0])
+        # All tasks should still exist
+        self.assertEqual(self.n_tasks, Task.objects.count())
+
+        # Without dry flag, tasks should be removed
+        self.command.handle(action='cleanup', before=str(before_date))
+        # All tasks should still exist
+        self.assertEqual(n, Task.objects.count())
+        self.assertEqual(0, Task.objects.filter(datetime__date__lte=before_date).count())
+
+        # With status filter as int
+        n = Task.objects.count() - Task.objects.filter(status=20).count()
+        self.command.handle(action='cleanup', status='20')
+        self.assertEqual(n, Task.objects.count())
+        self.assertEqual(0, Task.objects.filter(status=20).count())
+
+        # With status filter as string
+        n = Task.objects.count() - Task.objects.filter(status=40).count()
+        self.command.handle(action='cleanup', status='Errored')
+        self.assertEqual(n, Task.objects.count())
+        self.assertEqual(0, Task.objects.filter(status=40).count())
+
+        # With status filter as string and ~
+        n_days = self.n_tasks - 20
+        before_date = (datetime.today() - timedelta(days=n_days)).date()
+        n = Task.objects.count()
+        n -= Task.objects.exclude(status=45).filter(datetime__date__lte=before_date).count()
+        self.command.handle(action='cleanup', status='~Abandoned', before=str(before_date))
+        self.assertEqual(n, Task.objects.count())
+        n_tasks = Task.objects.exclude(status=45).filter(datetime__date__lte=before_date).count()
+        self.assertEqual(0, n_tasks)
+        self.assertTrue(Task.objects.filter(status=45, datetime__date__lte=before_date).count())
+
+        # With status filter as int and ~ with limit
+        n = Task.objects.exclude(status=60).count() - 5
+        self.command.handle(action='cleanup', status='~60', limit='5')
+        self.assertEqual(n, Task.objects.exclude(status=60).count())
+
+        # Error handling
+        self.assertRaises(ValueError, self.command.handle, action='cleanup', status='NotAStatus')
+        self.assertRaises(ValueError, self.command.handle, action='cleanup', status='-1000')
+        self.assertRaises(ValueError, self.command.handle, action='NotAnAction')

docs/gettingstarted.md

@@ -212,50 +212,7 @@ print(local_files)
 We went straight to the point here, which was to create a session and register data, to go further consult the [One documentation](https://int-brain-lab.github.io/ONE/), in the section "Using one in Alyx".
 
 ## Backing up the database
-See [this section](https://docs.google.com/document/d/1cx3XLZiZRh3lUzhhR_p65BggEqTKpXHUDkUDagvf9Kc/edit?tab=t.0#heading=h.dibimc48a9xl) in the Alyx user guide on how to back up and restore the database.
+See [this section](https://docs.google.com/document/d/1cx3XLZiZRh3lUzhhR_p65BggEqTKpXHUDkUDagvf9Kc/edit?tab=t.0#heading=h.dibimc48a9xl) in the Alyx user guide on how to back up and restore the database.  There are scripts in `alyx/scripts/templates/` for exporting the database to a sql file and importing from said file.
 
 ## Updating the database
-The database should be updated each time there is a new Alyx release.
-
-1. Pull the changes from GitHub
-```shell
-cd alyx/  # ensure you are within the Alyx git repository
-git stash  # stash any local changes (if required)
-git checkout master  # ensure you're on the main branch
-git pull  # fetch and merge any new code changes
-git stash pop # restore any local changes (if required)
-```
-
-2. Activate environment - install any new requirements
-```shell
-source ./alyxvenv/bin/activate
-pip install -r requirements.txt
-```
-
-3. Update the database if any scheme changes - we expect no migrations
-```shell
-cd alyx
-./manage.py makemigrations
-./manage.py migrate
-```
-
-4. If new fixtures load in the database:
-```shell
-../scripts/load-init-fixtures.sh
-```
-
-5. If new tables were added, change the postgres permissions
-```shell
-./manage.py set_db_permissions
-./manage.py set_user_permissions
-```
-
-6. If there were updates to the Django version
-```shell
-./manage.py collectstatic --no-input
-```
-
-7. Restart the Apache server
-```shell
-sudo service apache2 restart
-```
+The database should be updated each time there is a new Alyx release.  There is an update script in `alyx/scripts/auto-update.sh`, although you may need to change the source and cd command paths.

scripts/auto-update.sh

@@ -5,13 +5,17 @@ cd /var/www/alyx-main/alyx
 git stash
 git pull
 git stash pop
-# 3/ update database if scheme changes
+# 3/ install any new requirements
+pip install -r requirements.txt
+# 4/ update database if scheme changes
 ./manage.py makemigrations
 ./manage.py migrate
-# 4/ If new fixtures load them in the database
+# 5/ If new fixtures load them in the database
 ../scripts/load-init-fixtures.sh
-# 5/ if new tables change the postgres permissions
+# 6/ if new tables change the postgres permissions
 ./manage.py set_db_permissions
 ./manage.py set_user_permissions
-# 6/ restart the apache server
+# 7/ if there were updates to the Django version collect the static files
+./manage.py collectstatic --no-input
+# 8/ restart the apache server
 sudo service apache2 reload