diff --git a/@Alyx/Alyx.m b/@Alyx/Alyx.m index 2549569..8839531 100644 --- a/@Alyx/Alyx.m +++ b/@Alyx/Alyx.m @@ -92,6 +92,7 @@ function obj = set.BaseURL(obj, value) % Drop trailing slash and ensure protocol defined + if isempty(value); obj.BaseURL = ''; return; end % return on empty value = iff(value(1:4)~='http', ['https://' value], value); obj.BaseURL = iff(value(end)=='/', value(1:end-1), value); end @@ -109,7 +110,7 @@ % Checks for and uploads queued data to Alyx [data, statusCode] = flushQueue(obj) % Recovers the full filepath of a file on the repository, given the datasetURL - fullPath = getFile(obj, datasetURL) + [fullPath, exists] = getFile(obj, eid, type) % Query the database for a list of sessions [sessions, eids] = getSessions(obj, varargin) % Lists recorded subjects diff --git a/@Alyx/getFile.m b/@Alyx/getFile.m index 8f4cbbd..e505ea0 100644 --- a/@Alyx/getFile.m +++ b/@Alyx/getFile.m @@ -1,36 +1,58 @@ -function fullPath = getFile(obj, datasetURL) -%GETFILE Recovers the full filepath of a file on the repository, given the datasetURL -% This function is SUPER inefficient because it has to load all -% filerecords from the database, and then search for the specific -% filerecord whose parent dataset matches the one supplied as input. +function [fullPath, exists] = getFile(obj, eid, type) +%GETFILE Returns the full filepath for a given eid. +% Returns the full path of associated data files given a dataset or file +% record URL or eid (experiment ID). Also returns a logical array +% indicating whether files exist for each element in the `eid` array. % -% datasetURL: URL of a dataset on Alyx +% Inputs: +% `eid`: a string or char array containing the full URL(s) or eid(s) of +% dataset(s) / file record(s) on Alyx. +% `type`: a string indicating type of eid(s) ('dataset' (default) or +% 'file') % -% See also ALYX, GETDATA +% Outputs: +% `fullPath`: a cellstring containing the full data file path(s) for +% each element in the `eid` array +% `exists`: a logical array indicating whether files exist for each eid +% in the `eid` array +% +% See also ALYX, GETDATA, GETSESSIONS % % Part of Alyx -% TODO: Create endpoint so this function is no longer inefficient? % 2017 PZH created +% 2019 MW Rewrote +if nargin < 3; type = 'dataset'; end -% Get all file records -filerecords = obj.getData('files'); - -% Extract the datasets which are parent to the filerecords -datasets = cellfun(@(fr) fr.dataset, filerecords, 'uni', 0); - -% Find whichever filerecord has input datasetURL as its parent -idx = contains(datasets, datasetURL); - -if any(idx) - fr = filerecords{idx}; - relPath = fr.relative_path; % Get relative path of file - - repo = obj.getData('data-repository'); % Get absolute path of repository - fullPath = [repo{1}.path relPath]; % Recover the full path of the file on the repository +% Convert array of strings to cell string +if isstring(eid) && ~isscalar(eid) + eid = cellstr(eid); else - error('No filerecords with inputted datasetURL as its parent'); + eid = ensureCell(eid); end +% Validate URL (UUIDs have 36 characters) +assert(all(cellfun(@(str)ischar(str) && length(str) >= 36, eid)), 'Invalid eid') +eid = mapToCell(@(str)str(end-35:end), eid); + +% Get all file records +switch lower(type) + case 'file' + filerecords = mapToCell(@(url)obj.getData(['files/', url]), eid); + repos = obj.getData('data-repository'); + repos = containers.Map({repos.name}, {repos.data_url}); + fullPath = mapToCell(@(s)[repos(s.data_repository) s.relative_path], filerecords); + exists = cellfun(@(s)s.exists, filerecords); + % Return as char if user expects one output + if numel(eid) == 1; fullPath = fullPath{1}; end + case 'dataset' + filerecords = catStructs(mapToCell(@(url)getOr(obj.getData(['datasets/', url]), 'file_records'), eid)); + exists = [filerecords.exists]; + fullPath = {filerecords.data_url}; + % Remove records with empty url field + exists = exists(emptyElems(fullPath)); + fullPath = rmEmpty(fullPath); + otherwise + error('Alyx:GetFile:InvalidType', 'Invalid eid type: must be ''dataset'' or ''file''') end diff --git a/@Alyx/listSubjects.m b/@Alyx/listSubjects.m index 87119d4..fb32ac2 100644 --- a/@Alyx/listSubjects.m +++ b/@Alyx/listSubjects.m @@ -48,11 +48,8 @@ subjects = [{'default'}, subjNames]'; end else - % The master 'main' repository is the reference for the existence of + % The remote 'main' repositories are the reference for the existence of % experiments, as given by the folder structure - mainPath = dat.reposPath('main', 'master'); - - dirs = file.list(mainPath, 'dirs'); - subjects = setdiff(dirs, {'@Recently-Snapshot', '@Recycle'}); %exclude the trash directories + subjects = dat.listSubjects; end end \ No newline at end of file diff --git a/@Alyx/newExp.m b/@Alyx/newExp.m index ca928a2..40b00eb 100644 --- a/@Alyx/newExp.m +++ b/@Alyx/newExp.m @@ -102,8 +102,7 @@ try % Generate JSON path and save jsonParams = obj2json(expParams); - jsonPath = fullfile(fileparts(dat.expFilePath(expRef, 'parameters', 'master')),... - [expRef, '_parameters.json']); + jsonPath = dat.expFilePath(expRef, 'parameters', 'master', 'json'); fid = fopen(jsonPath, 'w'); fprintf(fid, '%s', jsonParams); fclose(fid); % Register our JSON parameter set to Alyx files = [files; {jsonPath}]; diff --git a/README.md b/README.md index 636ff44..8da0d9a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ # Alyx-MATLAB +![Custom badge](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fgladius.serveo.net%2Fcoverage%2Falyx-matlab%2Fdev) +![Custom badge](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fgladius.serveo.net%2Fstatus%2Falyx-matlab%2Fdev) + This repository contains a MATLAB class called Alyx, that facilitates RESTful POST and GET requests to an instance of the [Alyx database](http://alyx.readthedocs.io/en/latest/). ## Getting started @@ -153,4 +156,4 @@ For use with [Rigbox](https://github.com/cortex-lab/Rigbox), use the submodule t * [MD5 in MATLAB](https://uk.mathworks.com/matlabcentral/fileexchange/7919-md5-in-matlab) - For checksums (no longer used) ## Authors -This code is maintained and developed by a number of people at [CortexLab](https://www.ucl.ac.uk/cortexlab). See [contributors](https://github.com/cortex-lab/alyx-matlab/graphs/contributors) list for more info. \ No newline at end of file +This code is maintained and developed by a number of people at [CortexLab](https://www.ucl.ac.uk/cortexlab). See [contributors](https://github.com/cortex-lab/alyx-matlab/graphs/contributors) list for more info. diff --git a/tests/Alyx_test.m b/tests/Alyx_test.m index bc1eee8..c9cd56d 100644 --- a/tests/Alyx_test.m +++ b/tests/Alyx_test.m @@ -3,6 +3,12 @@ Alyx_test < matlab.unittest.TestCase % Test adapted from Oliver Winter's AlyxClient test + properties (ClassSetupParameter) + % Alyx base URL. test is for the main branch, testDev is for the dev + % code + base_url = cellsprintf('https://%s.alyx.internationalbrainlab.org', {'test', 'testDev'}); + end + properties % Test objects % Alyx Instance alyx @@ -15,10 +21,14 @@ end properties % Validation data - subjects = {'IBL_46'; 'ZM_1085'; 'ZM_1087'; 'ZM_1094'; 'ZM_1098'; 'ZM_335'} + subjects = {'KS005'; 'ZM_1743'; 'IBL_46'; ... + 'ZM_1085'; 'ZM_1087'; 'ZM_1094'; 'ZM_1098'; 'ZM_335'} water_types = {'Water', 'Hydrogel'} eids = {'cf264653-2deb-44cb-aa84-89b82507028a', ... '4e0b3320-47b7-416e-b842-c34dc9004cf8'} + dataset_id = 'c41dd877-d511-42cb-90a3-01bb19297117' + file_record_ids = {'00c3df4f-99ab-4cc0-b305-b508bcfb07ab',... + '0b747a70-1309-4f84-98f6-5f3aa9815b4c'} end methods (TestClassSetup) @@ -27,12 +37,12 @@ function checkFixtures(~) assert(endsWith(which('dat.paths'), fullfile('fixtures','+dat','paths.m'))); % Check temp mainRepo folder is empty. An extra safe measure as we % don't won't to delete important folders by accident! - mainRepo = getOr(dat.paths, 'mainRepository'); - assert(~exist(mainRepo, 'dir') || isempty(setdiff(getOr(dir(mainRepo),'name'),{'.','..'})),... + mainRepo = dat.reposPath('main','master'); + assert(~exist(mainRepo, 'dir') || isempty(file.list(mainRepo)),... 'Test experiment repo not empty. Please set another path or manual empty folder'); end - function createObject(testCase) + function createObject(testCase, base_url) % Create a number of Alyx instances and log them in testCase.queueDir = fullfile(fileparts(mfilename('fullpath')),'fixtures','data'); Alyx_test.resetQueue(testCase.queueDir); % Ensure empty before logging in @@ -42,13 +52,15 @@ function createObject(testCase) 'Water 10% Sucrose', 'Water 2% Citric Acid', 'Hydrogel'}; ai = Alyx('',''); + ai.BaseURL = base_url; ai.QueueDir = testCase.queueDir; ai = ai.login(testCase.uname, testCase.pwd); testCase.fatalAssertTrue(ai.IsLoggedIn, ... sprintf('Failed to log into %s', ai.BaseURL)) + fprintf('Logged into %s\n', base_url); testCase.alyx = ai; - dataRepo = getOr(dat.paths, 'mainRepository'); + dataRepo = dat.reposPath('main','master'); assert(exist(dataRepo, 'file') == 0 && exist(dataRepo, 'dir') == 0,... 'Test data direcotry already exists. Please remove and rerun tests') assert(mkdir(dataRepo), 'Failed to create test data directory'); @@ -60,7 +72,7 @@ function createObject(testCase) testCase.fatalAssertTrue(all([testCase.alyx.Headless]==0) && ... all([testCase.alyx.IsLoggedIn]==1),... 'Not all test instances connected') - dataRepo = getOr(dat.paths, 'mainRepository'); + dataRepo = dat.reposPath('main','master'); success = cellfun(@(d)mkdir(d), fullfile(dataRepo, testCase.subjects)); assert(all(success), 'Failed to create tesst subject folders') end @@ -69,8 +81,8 @@ function createObject(testCase) methods(TestMethodTeardown) function methodTaredown(testCase) Alyx_test.resetQueue(testCase.queueDir); - dataRepo = getOr(dat.paths, 'mainRepository'); - assert(rmdir(dataRepo, 's'), 'Failed to remove test data directory') + rm = @(repo)assert(rmdir(repo, 's'), 'Failed to remove test repo %s', repo); + cellfun(@(repo)iff(exist(repo,'dir') == 7, @()rm(repo), @()nop), dat.reposPath('main')); end end @@ -79,18 +91,29 @@ function methodTaredown(testCase) function test_listSubjects(testCase) % Test that the subject list returned by the test database is % accurate - ai = testCase.alyx(1); + ai = testCase.alyx; testCase.verifyTrue(isequal(ai.listSubjects, ... [{'default'}; testCase.subjects]), 'Subject list mismatch') % Test behaviour of empty list - testCase.verifyTrue(strcmp('default', ai.listSubjects(1,1)),... + testCase.verifyTrue(strcmp('default', ai.listSubjects{1,1}),... 'Subject list mismatch') + + % Test functionality when logged out + ai = ai.logout; + testCase.assertTrue(~ai.IsLoggedIn, 'Failed to logout') + testCase.verifyTrue(isequal(ai.listSubjects, ... + sort(testCase.subjects)), 'Subject list mismatch') + % Add new subject to repository to be sure + status = mkdir(fullfile(dat.reposPath('main','m'), 'newSubject')); + testCase.assertTrue(status, 'Failed to create new subject folder') + testCase.verifyTrue(isequal(ai.listSubjects, ... + sort([testCase.subjects; {'newSubject'}])), 'Subject list mismatch'); end function test_makeEndPoint(testCase) % Test validation of base url and endpoints - ai = testCase.alyx(1); + ai = testCase.alyx; sub = ai.getData('subjects/flowers'); % Preceding slash @@ -114,7 +137,7 @@ function test_makeEndPoint(testCase) end function test_login(testCase) - ai = testCase.alyx(1); + ai = testCase.alyx; ai = ai.logout; ai.Headless = true; testCase.verifyWarning(@()ai.login('test_user', 'bAdT0k3N'), ... @@ -124,7 +147,7 @@ function test_login(testCase) function test_getData(testCase) % TODO create webread mock for timeout test % Test retrieval from water-type endpoint - ai = testCase.alyx(1); + ai = testCase.alyx; testCase.verifyTrue(isequal(testCase.water_types, ... {ai.getData('water-type').name})) @@ -139,14 +162,14 @@ function test_getData(testCase) 'Alyx:getData:InvalidToken'); % Test incorrect URL - ai = testCase.alyx(1); + ai = testCase.alyx; ai.BaseURL = 'https://notaurl'; testCase.verifyWarning(@()ai.getData('water-type'),... 'MATLAB:webservices:UnknownHost'); end function test_getSessions(testCase) - ai = testCase.alyx(1); + ai = testCase.alyx; % Test subject search sess = ai.getSessions('subject', 'flowers'); testCase.assertTrue(~isempty(sess), 'No sessions returned'); @@ -158,21 +181,19 @@ function test_getSessions(testCase) testCase.verifyEqual(eid, testCase.eids, 'Inconsistent eids') % Test lab search - sess = ai.getSessions('lab', 'cortexlab'); - testCase.verifyTrue(all(strcmp({sess.lab},'cortexlab')), 'Failed to filter by lab') + sess = ai.getSessions('lab', 'zadorlab'); + expected = {'c34dc9004cf8'}; + actual = @(s)mapToCell(@(url)url(end-11:end),{s.url}); + testCase.verifyEqual(actual(sess), expected, 'Failed to filter by lab') % Test user search sess = ai.getSessions('user', 'olivier'); - correct = cellfun(@(usr)any(strcmp(usr,'olivier')), {sess.users}); - testCase.verifyTrue(all(correct), 'Failed to filter by users') + expected = {'89b82507028a', 'c34dc9004cf8'}; + testCase.verifyEqual(actual(sess), expected, 'Failed to filter by users') % Test dataset search sess = ai.getSessions('data', {'clusters.probes', 'eye.blink'}); - correct = cellfun(... - @(s)any(strcmp({s.dataset_type},'clusters.probes')) && ... - any(strcmp({s.dataset_type},'eye.blink')), ... - {sess.data_dataset_session_related}); - testCase.verifyTrue(all(correct), 'Failed to filter by dataset_type') + testCase.verifyEqual(actual(sess), {'89b82507028a'}, 'Failed to filter by dataset_type') % Test eid and search combo [sess, eid] = ai.getSessions(testCase.eids{1}, ... @@ -205,13 +226,13 @@ function test_postWater(testCase) % Test post while logged in ai = testCase.alyx; subject = testCase.subjects{randi(length(testCase.subjects))}; - waterPost = @()ai.postWater(subject, pi, 7.3740e+05); + waterPost = @()ai.postWater(subject, pi, 7.3760e+05); wa = assertWarningFree(testCase, waterPost,'Alyx:flushQueue:NotConnected'); % Check water record expectedFields = {'date_time', 'water_type', 'subject', 'water_administered'}; testCase.assertTrue(all(ismember(expectedFields,fieldnames(wa))), 'Field names missing') - testCase.verifyEqual(wa.date_time, '2018-12-06T00:00:00', 'date_time incorrect') + testCase.verifyEqual(wa.date_time, '2019-06-24T00:00:00', 'date_time incorrect') testCase.verifyEqual(wa.water_type, 'Water', 'water_type incorrect') testCase.verifyEqual(wa.subject, subject, 'subject incorrect') testCase.verifyTrue(wa.water_administered == 3.142, 'Unexpected water volume'); @@ -232,7 +253,7 @@ function test_postWater(testCase) % When headless or not connected, should save post as JSON and % issue warning ai = ai.logout; - waterPost = @()ai.postWater(subject, pi, 7.3740e+05, 'Hydrogel'); + waterPost = @()ai.postWater(subject, pi, 7.3760e+05, 'Hydrogel'); verifyWarning(testCase, waterPost, 'Alyx:flushQueue:NotConnected'); % Check post was saved savedPost = dir([ai.QueueDir filesep '*.post']); @@ -240,26 +261,53 @@ function test_postWater(testCase) fn = @()Alyx_test.loadPost(fullfile(savedPost(1).folder, savedPost(1).name)); [jsonData, endpnt] = testCase.fatalAssertWarningFree(fn); testCase.verifyMatches(endpnt, 'water-administrations', 'Incorrect endpoint') - expected = ['{"date_time":"2018-12-06T00:00:00","water_type":"Hydrogel","subject":"'... + expected = ['{"date_time":"2019-06-24T00:00:00","water_type":"Hydrogel","subject":"'... subject '","water_administered":3.142}']; testCase.verifyMatches(jsonData, expected, 'JSON data incorrect') end function test_getFile(testCase) - %TODO Add test for getFile method + ai = testCase.alyx; + % Test paths from dataset eid + [fullPath, exist] = ai.getFile(testCase.dataset_id); + expected = ['http://ibl.flatironinstitute.org/mainenlab/Subjects/'... + 'clns0730/2018-08-24/1/clusters.probes.c41dd877-d511-42cb-90a3-01bb19297117.npy']; + testCase.verifyEqual(fullPath{1}, expected, 'Unexpected path returned') + testCase.verifyEqual(numel(fullPath), numel(exist)); + + % Test using full URL + url = ai.makeEndpoint(['datasets/', testCase.dataset_id]); + fullPath = ai.getFile(url); + expected = ['http://ibl.flatironinstitute.org/mainenlab/Subjects/'... + 'clns0730/2018-08-24/1/clusters.probes.c41dd877-d511-42cb-90a3-01bb19297117.npy']; + testCase.verifyEqual(fullPath{1}, expected, 'Unexpected path returned') + + % Test file record + [fullPath, exist] = ai.getFile(testCase.file_record_ids{1}, 'file'); + expected = ['http://ibl.flatironinstitute.org/mainenlab/Subjects/'... + 'clns0730/2018-08-24/1/clusters.probes.npy']; + testCase.verifyEqual(fullPath, expected, 'Unexpected path returned') + testCase.verifyEqual(numel(ensureCell(fullPath)), numel(exist)); + + % Test cell array + [fullPath, exist] = ai.getFile(testCase.file_record_ids, 'file'); + expected = {'clusters.probes.npy', 'clusters.depths.npy'}; + testCase.verifyTrue(all(cellfun(@endsWith, fullPath, expected)), ... + 'Unexpected paths returned') + testCase.verifyEqual(numel(ensureCell(fullPath)), numel(exist)); end function test_postWeight(testCase) % Test post while logged in ai = testCase.alyx; subject = testCase.subjects{randi(length(testCase.subjects))}; - weightPost = @()ai.postWeight(25.1, subject, 7.3740e+05); + weightPost = @()ai.postWeight(25.1, subject, 7.3760e+05); wa = assertWarningFree(testCase, weightPost,'Alyx:flushQueue:NotConnected'); % Check water record expectedFields = {'date_time', 'weight', 'subject', 'user', 'url'}; testCase.assertTrue(all(ismember(expectedFields,fieldnames(wa))), 'Field names missing') - testCase.verifyEqual(wa.date_time, '2018-12-06T00:00:00', 'date_time incorrect') + testCase.verifyEqual(wa.date_time, '2019-06-24T00:00:00', 'date_time incorrect') testCase.verifyEqual(wa.weight, 25.1, 'weight incorrect') testCase.verifyEqual(wa.subject, subject, 'subject incorrect') testCase.verifyEqual(wa.user, ai.User, 'Unexpected water volume'); @@ -274,7 +322,7 @@ function test_postWeight(testCase) % When headless or not connected, should save post as JSON and % issue warning ai = ai.logout; - weightPost = @()ai.postWeight(25.1, subject, 7.3740e+05); + weightPost = @()ai.postWeight(25.1, subject, 7.3760e+05); verifyWarning(testCase, weightPost, 'Alyx:flushQueue:NotConnected'); % Check post was saved savedPost = dir([ai.QueueDir filesep '*.post']); @@ -282,13 +330,13 @@ function test_postWeight(testCase) fn = @()Alyx_test.loadPost(fullfile(savedPost(1).folder, savedPost(1).name)); [jsonData, endpnt] = testCase.fatalAssertWarningFree(fn); testCase.verifyEqual(endpnt, 'weighings/', 'Incorrect endpoint') - expected = ['{"date_time":"2018-12-06T00:00:00","subject":"' ... + expected = ['{"date_time":"2019-06-24T00:00:00","subject":"' ... subject '","weight":25.1}']; testCase.verifyMatches(jsonData, expected, 'JSON data incorrect') end function test_newExp(testCase) - ai = testCase.alyx(1); + ai = testCase.alyx; subject = testCase.subjects{end}; newExp_fn = @()newExp(ai, subject); wrnID = 'Alyx:registerFile:InvalidRepoPath'; @@ -313,7 +361,7 @@ function test_postData(testCase) % NB: Standard post tested in other test methods. DELETE and PUT % cannot be tested as these are no longer not allowed by the API. % Test PATCH method for sessions endpoint - ai = testCase.alyx(1); + ai = testCase.alyx; url = ['sessions/' testCase.eids{1}]; d = struct(... 'end_time', ai.datestr(now),... @@ -340,7 +388,7 @@ function test_postData(testCase) end function test_updateNarrative(testCase) - ai = testCase.alyx(1); + ai = testCase.alyx; url = ['sessions/' testCase.eids{1}]; comments = ' this is \r a test\n comment\t...'; data = testCase.verifyWarningFree(@()ai.updateNarrative(comments, url)); @@ -348,7 +396,7 @@ function test_updateNarrative(testCase) end function test_save_loadobj(testCase) - ai = testCase.alyx(1); + ai = testCase.alyx; s = saveobj(ai); % Test options were removed testCase.verifyEmpty(s.WebOptions, 'WebOptions not removed'); @@ -373,7 +421,7 @@ function test_datestr_datenum(testCase) function test_parseAlyxInstance(testCase) ref = '2019-01-01_1_fake'; - ai = testCase.alyx(1); + ai = testCase.alyx; json = testCase.assertWarningFree(@()Alyx.parseAlyxInstance(ref, ai)); [ref2, ai2] = testCase.assertWarningFree(@()Alyx.parseAlyxInstance(json));