Merge pull request #755 from cortex-lab/task_data_repo

Task data repo

alyx/data/tests_rest.py

@@ -365,6 +365,121 @@ def _assert_registration(self, r, data):
         self.assertEqual(d1['file_records'][0]['relative_path'],
                          op.join(data['path'], 'a.c.e2'))
 
+    def test_register_existence_options(self):
+
+        self.post(reverse('datarepository-list'), {'name': 'ibl1', 'hostname': 'iblhost1',
+                                                   'globus_is_personal': 'True'})
+        self.post(reverse('datarepository-list'), {'name': 'ibl2', 'hostname': 'iblhost2',
+                                                   'globus_is_personal': 'True'})
+        # server repo
+        self.post(reverse('datarepository-list'), {'name': 'ibl3', 'hostname': 'iblhost3',
+                                                   'globus_is_personal': 'False'})
+
+        self.post(reverse('lab-list'), {'name': 'ibl',
+                                        'repositories': ['ibl1', 'ibl2', 'ibl3']})
+
+        # Case 1, server_only = False, no repo name specified
+        # Expect: 3 file records (one for each repo),
+        # all with exists = False (preserves old behavior)
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.a.e1',  # this is the repository name
+                'server_only': False,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 3)
+        for fr in frs:
+            self.assertEqual(fr['exists'], False)
+
+        # Case 2, server_only = True, no repo name specified
+        # Expect: 1 file record for data repo with globus is personal == True,
+        # with exists = True (preserves old behavior)
+
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.a.e2',  # this is the repository name
+                'server_only': True,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 1)
+        self.assertEqual(frs[0]['exists'], True)
+        self.assertEqual(frs[0]['data_repository'], 'ibl3')
+
+        # Case 3, server_only = False, repo name specified
+        # Expect: 3 file records, specified repo with exists = True,
+        # the others False (preserves old behavior)
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.b.e1',  # this is the repository name
+                'name': 'ibl1',
+                'server_only': False,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 3)
+        for fr in frs:
+            self.assertEqual(fr['exists'], fr['data_repository'] == 'ibl1')
+
+        # # Case 4, server_only = False, repo name specified, exists = False
+        # # Expect: 3 file records, all with exists = False
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.b.e2',  # this is the repository name
+                'name': 'ibl1',
+                'server_only': False,
+                'exists': False,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 3)
+        for fr in frs:
+            self.assertEqual(fr['exists'], False)
+
+        # Case 5, server_only = True, repo name specified
+        # Expect: 2 file records, 1 for specified repo and 1 for server repo,
+        # all with exists = True (preserves old behavior)
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.c.e1',  # this is the repository name
+                'name': 'ibl1',
+                'server_only': True,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 2)
+        for fr in frs:
+            self.assertEqual(fr['exists'], True)
+
+        # # Case 5, server_only = True, repo name specified, exists = False
+        # # Expect: 2 file records, 1 for specified repo and 1 for server repo,
+        # all with exists = False
+        data = {'path': '%s/2018-01-01/002/dir' % self.subject,
+                'filenames': 'a.c.e2',  # this is the repository name
+                'name': 'ibl1',
+                'server_only': True,
+                'exists': False,
+                'labs': ['ibl']
+                }
+
+        r = self.client.post(reverse('register-file'), data)
+        r = self.ar(r, 201)[0]
+        frs = r['file_records']
+        self.assertTrue(len(frs) == 2)
+        for fr in frs:
+            self.assertEqual(fr['exists'], False)
+
     def test_register_with_revision(self):
         self.post(reverse('datarepository-list'), {'name': 'drb2', 'hostname': 'hostb2'})
         self.post(reverse('lab-list'), {'name': 'labb', 'repositories': ['drb2']})

alyx/data/views.py

@@ -416,6 +416,8 @@ def create(self, request):
 
         # flag to discard file records creation on local repositories, defaults to False
         server_only = request.data.get('server_only', False)
+        if isinstance(server_only, str):
+            server_only = server_only == 'True'
 
         default = request.data.get('default', True)
         # Need to explicitly cast string to a bool
@@ -432,6 +434,14 @@ def create(self, request):
         if server_only:
             exists_in = repositories
 
+        # # If exists is specified to be false then we set the exists_in back to None
+        exists = request.data.get('exists', True)
+        # Need to explicitly cast string to a bool
+        if isinstance(exists, str):
+            exists = exists == 'True'
+        if not exists:
+            exists_in = (None,)
+
         session = _get_session(
             subject=subject, date=date, number=session_number, user=user)
         assert session

scripts/sync_ucl/prune_cortexlab.py

@@ -234,7 +234,7 @@
 # finds eids that have tasks on both ibl and cortex lab database
 overlap_eids = set(cortex_eids).intersection(ibl_eids)
 
-dfields = ('id', 'name', 'session', 'arguments')
+dfields = ('id', 'name', 'session')
 task_cortex = Task.objects.using('cortexlab').filter(session__in=overlap_eids).exclude(name__in=task_names_to_exclude)
 cids = task_cortex.values_list(*dfields)