From c704a9b95410c2b852c3115dce7bb7892ad0a16a Mon Sep 17 00:00:00 2001 From: k1o0 Date: Mon, 16 Oct 2023 13:21:06 +0300 Subject: [PATCH] Dev (#813) * add habituation procedure to init fixtures * Tag filter in dataset admin page * Exclude non-personal, non-globus endpoints from local file record delete * mpciROIs.uuids dataset type fixture --------- Co-authored-by: olivier Co-authored-by: github-actions --- .../fixtures/actions.proceduretype.json | 11 ++- alyx/data/admin.py | 1 + alyx/data/fixtures/data.datasettype.json | 11 +++ alyx/data/transfers.py | 25 ++++--- requirements_frozen.txt | 68 +++++++++---------- 5 files changed, 68 insertions(+), 48 deletions(-) diff --git a/alyx/actions/fixtures/actions.proceduretype.json b/alyx/actions/fixtures/actions.proceduretype.json index 41e3a886f..e137094c2 100644 --- a/alyx/actions/fixtures/actions.proceduretype.json +++ b/alyx/actions/fixtures/actions.proceduretype.json @@ -14,7 +14,7 @@ "fields": { "name": "Water restriction", "json": null, - "description": "- Variation in the composition, constituents, quantity and/or availability of the diet and/or drinking water which may cause pain, suffering, distress or lasting harm - withholding of water" + "description": "- Variation in the compositigit con, constituents, quantity and/or availability of the diet and/or drinking water which may cause pain, suffering, distress or lasting harm - withholding of water" } }, { @@ -268,6 +268,15 @@ "json": null, "description": "Covers the acquisition of histological sections, the subsequent registration of slices to a common reference atlas and the eventual recovery of electrophysiology tracks through manual picking." } +}, + { + "model": "actions.proceduretype", + "pk": "e55c9ca1-3208-4b81-936f-75c0cf6e9565", + "fields": { + "name": "handling_habituation", + "json": null, + "description": "Rig habituation protocol for a new task and/or a new subject" + } }, { "model": "actions.proceduretype", diff --git a/alyx/data/admin.py b/alyx/data/admin.py index e2e9df01a..2950c0134 100644 --- a/alyx/data/admin.py +++ b/alyx/data/admin.py @@ -93,6 +93,7 @@ class DatasetAdmin(BaseExperimentalDataAdmin): list_filter = [('created_by', RelatedDropdownFilter), ('created_datetime', DateRangeFilter), ('dataset_type', RelatedDropdownFilter), + ('tags', RelatedDropdownFilter) ] search_fields = ('session__id', 'name', 'collection', 'dataset_type__name', 'dataset_type__filename_pattern', 'version') diff --git a/alyx/data/fixtures/data.datasettype.json b/alyx/data/fixtures/data.datasettype.json index d784f1f3d..9c02f9b82 100644 --- a/alyx/data/fixtures/data.datasettype.json +++ b/alyx/data/fixtures/data.datasettype.json @@ -2198,5 +2198,16 @@ "description": "Image meta-data listing the depths and dimensions of each field of view, acquisition parameters, channels, and the MLAP coordinates of the imaging window center.", "filename_pattern": "_*_rawImagingData.meta*" } + }, + { + "model": "data.datasettype", + "pk": "289cb419-274f-4699-ac43-7612de07382e", + "fields": { + "json": null, + "name": "mpciROIs.uuids", + "created_by": null, + "description": "Unique identifier assigned to each ROI when ALF files created after ROI detection.", + "filename_pattern": "" + } } ] diff --git a/alyx/data/transfers.py b/alyx/data/transfers.py index 523a352b1..01c219b3f 100644 --- a/alyx/data/transfers.py +++ b/alyx/data/transfers.py @@ -659,21 +659,22 @@ def globus_delete_local_datasets(datasets, dry=True, gc=None, label=None): """ For each dataset in the queryset delete the file records belonging to a Globus personal repo only if a server file exists and matches the size. - :param datasets: + :param datasets: data.models.Dataset query set :param label: label for the transfer :param dry: default True :return: """ # first get the list of Globus endpoints concerned - file_records = FileRecord.objects.filter(dataset__in=datasets) + file_records = FileRecord.objects.filter( + dataset__in=datasets, data_repository__globus_is_personal=True) + file_records = file_records.exclude(data_repository__globus_endpoint_id__isnull=True) globus_endpoints = file_records.values_list('data_repository__globus_endpoint_id', flat=True).distinct() label = label or 'alyx globus client' # create a globus delete_client for each globus endpoint gtc = gc or globus_transfer_client() - delete_clients = [] - for ge in globus_endpoints: - delete_clients.append(globus_sdk.DeleteData(gtc, ge, label=label)) + # Map of Globus endpoint UUID -> DeleteData client + delete_clients = {ge: globus_sdk.DeleteData(gtc, ge, label=label) for ge in globus_endpoints} def _ls_globus(file_record, add_uuid=False): N_RETRIES = 3 @@ -694,7 +695,6 @@ def _ls_globus(file_record, add_uuid=False): return [ls for ls in ls_obj['DATA'] if ls['name'] == path.name] # appends each file for deletion fr2delete = [] - del_client = [] for ds in datasets: # check the existence of the server file fr_server = ds.file_records.filter(exists=True, @@ -724,17 +724,16 @@ def _ls_globus(file_record, add_uuid=False): # the files exist local and remote, fr2delete.append(frloc.id) file2del = _filename_from_file_record(frloc) - del_client = [dc for dc in delete_clients if dc['endpoint'] == - str(frloc.data_repository.globus_endpoint_id)][0] + del_client = delete_clients[(gid := frloc.data_repository.globus_endpoint_id)] + assert del_client['endpoint'] == str(gid) del_client.add_item(file2del) logger.info('DELETE: ' + _filename_from_file_record(frloc)) # launch the deletion jobs and remove records from the database if dry: - return del_client - for dc in delete_clients: - # submitting a deletion without data will create an error - if dc['DATA'] == []: - continue + return delete_clients + # NB: filter empty clients as submitting a deletion without data will raise an error + for dc in filter(lambda x: x['DATA'], delete_clients.values()): + logger.info('Submitting delete for %i file(s) on %s', len(dc['DATA']), dc['endpoint']) gtc.submit_delete(dc) # remove file records frecs = FileRecord.objects.filter(id__in=fr2delete).exclude( diff --git a/requirements_frozen.txt b/requirements_frozen.txt index eeca6db44..54ed71496 100644 --- a/requirements_frozen.txt +++ b/requirements_frozen.txt @@ -1,81 +1,81 @@ asgiref==3.7.2 backports.zoneinfo==0.2.1 -boto3==1.28.37 -botocore==1.31.37 +boto3==1.28.63 +botocore==1.31.63 certifi==2023.7.22 -cffi==1.15.1 -charset-normalizer==3.2.0 +cffi==1.16.0 +charset-normalizer==3.3.0 click==8.1.7 colorlog==6.7.0 -contourpy==1.1.0 +contourpy==1.1.1 coreapi==2.3.3 coreschema==0.0.4 coverage==6.5.0 coveralls==3.3.1 -cryptography==41.0.3 -cycler==0.11.0 -Django==4.2.4 +cryptography==41.0.4 +cycler==0.12.1 +Django==4.2.6 django-admin-list-filter-dropdown==1.0.3 -django-admin-rangefilter==0.11.0 +django-admin-rangefilter==0.11.2 django-autocomplete-light==3.9.7 django-cleanup==8.0.0 django-filter==21.1 -django-ipware==5.0.0 +django-ipware==5.0.1 django-js-asset==2.1.0 django-mptt==0.14.0 django-polymorphic==3.1.0 -django-reversion==5.0.4 -django-storages==1.13.2 -django-structlog==5.3.0 +django-reversion==5.0.6 +django-storages==1.14.2 +django-structlog==6.0.0 django-test-without-migrations==0.6 djangorestframework==3.14.0 docopt==0.6.2 docutils==0.20.1 drfdocs==0.0.11 flake8==6.1.0 -fonttools==4.42.1 -globus-cli==3.17.0 -globus-sdk==3.27.0 -iblutil==1.7.0 +fonttools==4.43.1 +globus-cli==3.18.0 +globus-sdk==3.28.0 +iblutil==1.7.1 idna==3.4 importlib-metadata==6.8.0 -importlib-resources==6.0.1 +importlib-resources==6.1.0 itypes==1.2.0 Jinja2==3.1.2 jmespath==1.0.1 kiwisolver==1.4.5 -llvmlite==0.40.1 +llvmlite==0.41.0 lxml==4.9.3 -Markdown==3.4.4 +Markdown==3.5 MarkupSafe==2.1.3 -matplotlib==3.7.2 +matplotlib==3.7.3 mccabe==0.7.0 -numba==0.57.1 +numba==0.58.0 numpy==1.24.4 -ONE-api==2.2.2 -packaging==23.1 +ONE-api==2.3.0 +packaging==23.2 pandas==2.0.3 -Pillow==10.0.0 -psycopg2-binary==2.9.7 +Pillow==10.1.0 +psycopg2-binary==2.9.9 pyarrow==13.0.0 -pycodestyle==2.11.0 +pycodestyle==2.11.1 pycparser==2.21 pyflakes==3.1.0 PyJWT==2.8.0 -pyparsing==3.0.9 +pyparsing==3.1.1 python-dateutil==2.8.2 python-magic==0.4.27 -pytz==2023.3 +pytz==2023.3.post1 PyYAML==6.0.1 requests==2.31.0 -s3transfer==0.6.2 +s3transfer==0.7.0 six==1.16.0 sqlparse==0.4.4 -structlog==23.1.0 +structlog==23.2.0 tqdm==4.66.1 -typing_extensions==4.7.1 +typing_extensions==4.8.0 tzdata==2023.3 uritemplate==4.1.1 -urllib3==1.26.16 +urllib3==1.26.17 webdavclient3==3.14.6 -zipp==3.16.2 +zipp==3.17.0