diff --git a/README.md b/README.md index c18aff6e..3d771524 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,6 @@ this setup will work on other systems. Assumptions made are that you have sudo p - `./manage.py test` test with migrations (recommended if model changes) - NB: When running tests ensure `DEBUG = True` in the settings.py file (specifically `SECURE_SSL_REDIRECT = True` causes REST tests to fail) -``` -$ /manage.py test -n +```shell +./manage.py test -n ``` diff --git a/alyx/actions/serializers.py b/alyx/actions/serializers.py index 0809e779..75da211d 100644 --- a/alyx/actions/serializers.py +++ b/alyx/actions/serializers.py @@ -100,6 +100,13 @@ class SessionDatasetsSerializer(serializers.ModelSerializer): default_revision = serializers.CharField(source='default_dataset') qc = BaseSerializerEnumField(required=False) + def to_representation(self, instance): + """Override the default to_representation method to null the revision field.""" + representation = super().to_representation(instance) + if representation.get('revision') is None: + representation['revision'] = '' + return representation + class Meta: list_serializer_class = FilterDatasetSerializer model = Dataset diff --git a/alyx/data/models.py b/alyx/data/models.py index f8e42909..6b6674c0 100644 --- a/alyx/data/models.py +++ b/alyx/data/models.py @@ -386,7 +386,7 @@ def __str__(self): def save(self, *args, **kwargs): # when a dataset is saved / created make sure the probe insertion is set in the reverse m2m super(Dataset, self).save(*args, **kwargs) - if self.collection is None: + if not self.collection: return self.clean_fields() # Validate collection field from experiments.models import ProbeInsertion, FOV diff --git a/alyx/data/serializers.py b/alyx/data/serializers.py index c455611f..57f24c0d 100644 --- a/alyx/data/serializers.py +++ b/alyx/data/serializers.py @@ -141,7 +141,7 @@ class DatasetSerializer(serializers.HyperlinkedModelSerializer): hash = serializers.CharField(required=False, allow_null=True) version = serializers.CharField(required=False, allow_null=True) file_size = serializers.IntegerField(required=False, allow_null=True) - collection = serializers.CharField(required=False, allow_null=True) + collection = serializers.CharField(required=False, allow_blank=True, allow_null=True) default_dataset = serializers.BooleanField(required=False, allow_null=True) public = serializers.ReadOnlyField() protected = serializers.ReadOnlyField() @@ -178,7 +178,7 @@ def get_experiment_number(self, obj): def create(self, validated_data): # Get out some useful info # revision = validated_data.get('revision', None) - collection = validated_data.get('collection', None) + collection = validated_data.get('collection', '') name = validated_data.get('name', None) default = validated_data.get('default_dataset', None) session = validated_data.get('session', None) @@ -213,6 +213,13 @@ def create(self, validated_data): return super(DatasetSerializer, self).create(validated_data) + def to_representation(self, instance): + """Override the default to_representation method to null the revision field.""" + representation = super().to_representation(instance) + if representation.get('revision') is None: + representation['revision'] = '' + return representation + class Meta: model = Dataset fields = ('url', 'name', 'created_by', 'created_datetime', diff --git a/alyx/data/tests.py b/alyx/data/tests.py index 4c0c80aa..5f4cced0 100644 --- a/alyx/data/tests.py +++ b/alyx/data/tests.py @@ -1,5 +1,5 @@ from unittest import mock -from pathlib import Path +from pathlib import PurePosixPath from uuid import uuid4 from datetime import datetime, timedelta @@ -8,12 +8,15 @@ from django.db import transaction from django.db.utils import IntegrityError from django.db.models import ProtectedError +from rest_framework.response import Response +from one.alf.path import add_uuid_string from data.management.commands import files from data.models import Dataset, DatasetType, Tag, Revision, DataRepository, FileRecord from subjects.models import Subject from actions.models import Session from misc.models import Lab +from data import transfers from data.transfers import get_dataset_type @@ -201,4 +204,105 @@ def _new_delete_client(self, _, gid, **kwargs): @staticmethod def _dataset_uuid_name(dataset): - return f'{Path(dataset.name).stem}.{dataset.pk}{Path(dataset.name).suffix}' + return add_uuid_string(dataset.name, dataset.pk).as_posix() + + +class TestTransfers(TestCase): + """Tests for the data.transfers module.""" + + def setUp(self): + """Create some data repositories and file records to clean up.""" + # Two of these are 'large' datasets that will be removed + dtypes = ['ephysData.raw.ap', 'imaging.frames', 'foo.bar.baz'] + self.dtypes = [DatasetType.objects.create(name=name) for name in dtypes] + # Create two labs + self.labs = [Lab.objects.create(name=f'lab{i}') for i in range(2)] + # Create four repos + repo1 = DataRepository.objects.create( + name='lab0_local0', lab=self.labs[0], globus_is_personal=True, + globus_endpoint_id=uuid4(), globus_path='/mnt/foo/') + repo2 = DataRepository.objects.create( + name='lab0_local1', lab=self.labs[0], globus_is_personal=True, + globus_endpoint_id=uuid4(), globus_path='/mnt/foo/') + repo3 = DataRepository.objects.create( + name='lab1_local', lab=self.labs[1], globus_is_personal=True, + globus_endpoint_id=uuid4(), globus_path='/mnt/foo/') + # NB: name must contain 'flatiron'! + repo_main = DataRepository.objects.create( + name='flatiron', globus_is_personal=False, + globus_endpoint_id=uuid4(), globus_path='/mnt/foo/') + # Create one session per lab + self.subjects = [ + Subject.objects.create( + nickname=f'subject{i}', lab=lab) for i, lab in enumerate(self.labs)] + sessions = [Session.objects.create( + subject=sub, number=1, lab=lab) for lab, sub in zip(self.labs, self.subjects)] + # Create datasets and file records + self.dset_names = ['ephysData.raw.ap.bin', 'imaging.frames.tar.bz2', 'foo.bar.baz'] + self.dsets = [] + for session in sessions: # for one session in each lab, create one of each dataset + self.dsets.extend( + Dataset.objects.create(name=name, session=session, + dataset_type=next(x for x in self.dtypes if x.name in name)) + for name in self.dset_names) + + # Create file record on each lab's local server and main repo + session = 'subject/2020-01-01/001' + self.records = [] # All file records + for d in self.dsets: + for i, repo in enumerate((repo1, repo2, repo3, repo_main)): + if repo.globus_is_personal is False: + rel_path = f'{session}/{TestManagementFiles._dataset_uuid_name(d)}' + if i == 0: + rel_path = 'Data2/' + rel_path + if i == 1: + rel_path = '/' + rel_path + elif repo.lab != d.session.lab: + continue # Don't create file record for dataset if session lab different + else: + rel_path = f'{session}/{d.name}' + self.records.append( + FileRecord.objects.create( + relative_path=rel_path, exists=True, dataset=d, data_repository=repo) + ) + + def test_get_absolute_path(self): + expected = '/mnt/foo/subject/2020-01-01/001/ephysData.raw.ap.bin' + self.assertEqual(expected, transfers._get_absolute_path(self.records[0])) + expected = '/mnt/foo/subject/2020-01-01/001/ephysData.raw.ap.bin' + self.assertEqual(expected, transfers._get_absolute_path(self.records[1])) + + def test_get_name_collection_revision(self): + relative_path = PurePosixPath(self.records[0].relative_path) + info, resp = transfers._get_name_collection_revision( + relative_path.name, relative_path.parent.as_posix()) + self.assertIsNone(resp) + expected = { + 'lab': '', 'subject': 'subject', 'date': '2020-01-01', 'number': '001', + 'collection': '', 'revision': '', 'filename': 'ephysData.raw.ap.bin', + 'full_path': 'subject/2020-01-01/001/ephysData.raw.ap.bin', + 'rel_dir_path': 'subject/2020-01-01/001'} + self.assertDictEqual(info, expected) + relative_path = relative_path.parent / 'alf' / '#2020-10-01#' / relative_path.name + expected.update( + {'collection': 'alf', 'revision': '2020-10-01', + 'full_path': relative_path.as_posix()} + ) + info, resp = transfers._get_name_collection_revision( + relative_path.name, relative_path.parent.as_posix()) + self.assertIsNone(resp) + self.assertDictEqual(info, expected) + + relative_path = relative_path.parent / 'invalid' / relative_path.name + info, resp = transfers._get_name_collection_revision( + relative_path.name, relative_path.parent.as_posix()) + self.assertIsNone(info) + self.assertIsInstance(resp, Response) + self.assertEqual(resp.status_code, 400) + self.assertIn('Invalid ALF path', resp.data['detail']) + info, resp = transfers._get_name_collection_revision( + relative_path.name, 'subject/1-1-03/1/@lf') + self.assertIsNone(info) + self.assertIsInstance(resp, Response) + self.assertEqual(resp.status_code, 400) + self.assertIn('Invalid ALF path', resp.data['detail']) diff --git a/alyx/data/tests_rest.py b/alyx/data/tests_rest.py index 43e3a558..c7e8d2c0 100644 --- a/alyx/data/tests_rest.py +++ b/alyx/data/tests_rest.py @@ -142,8 +142,8 @@ def test_dataset(self): r = self.post(reverse('dataset-list'), data) self.ar(r, 201) # Check collection and revision have been set to default values - self.assertEqual(r.data['revision'], None) - self.assertEqual(r.data['collection'], None) + self.assertEqual(r.data['revision'], '') + self.assertEqual(r.data['collection'], '') # Check that it has been set as the default dataset self.assertEqual(r.data['default_dataset'], True) # Check QC value is NOT_SET by default @@ -170,7 +170,7 @@ def test_dataset(self): r = self.post(reverse('dataset-list'), data) self.ar(r, 201) - self.assertEqual(r.data['revision'], None) + self.assertEqual(r.data['revision'], '') self.assertEqual(r.data['collection'], data['collection']) self.assertEqual(r.data['default_dataset'], True) self.assertEqual(r.data['qc'], 'PASS') @@ -189,7 +189,7 @@ def test_dataset(self): self.assertEqual(r['default_dataset'], False) # Make sure if you specify the default dataset flag to false it is indeed false - data['collection'] = None + data['collection'] = '' data['default_dataset'] = False r = self.post(reverse('dataset-list'), data) self.ar(r, 201) diff --git a/alyx/jobs/admin.py b/alyx/jobs/admin.py index f60f02e7..7c2dad0b 100644 --- a/alyx/jobs/admin.py +++ b/alyx/jobs/admin.py @@ -4,7 +4,6 @@ DropdownFilter, ChoiceDropdownFilter, RelatedDropdownFilter) from jobs.models import Task -from misc.models import Lab from alyx.base import BaseAdmin, get_admin_url