Dependency reduction

+dat/addLogEntry.m

@@ -26,6 +26,7 @@
 %% create and store entry
 e = entry(nextidx);
 log(nextidx) = e;
+% Store an instance of Alyx for narrative registration
 if nargin > 5; e.AlyxInstance = AlyxInstance; end
 
 %% store updated log to *all* repos locations

+dat/constructExpRef.m

@@ -1,6 +1,6 @@
 function ref = constructExpRef(subjectRef, expDate, expSequence)
 %DAT.CONSTRUCTEXPREF Constructs an experiment reference string
-%   ref = DAT.CONSTRUCTEXPREF(subject, dat, seq) constructs and returns a
+%   ref = DAT.CONSTRUCTEXPREF(subject, date, seq) constructs and returns a
 %   standard format string reference, for the experiment using the 'subject',
 %   the 'date' of the experiment (a MATLAB datenum), and the daily sequence
 %   number of the experiment, 'seq' (must be an integer).

+dat/delParamProfile.m

@@ -8,7 +8,7 @@ function delParamProfile(expType, profileName)
 
 %path to repositories
 fn = 'parameterProfiles.mat';
-repos = fullfile(dat.reposPath('expInfo'), fn);
+repos = fullfile(dat.reposPath('main'), fn);
 
 %load existing profiles for specified expType
 profiles = dat.loadParamProfiles(expType);

+dat/expExists.m

@@ -1,6 +1,11 @@
 function b = expExists(expRef)
 %DAT.EXPEXISTS Confirm existence of experiment(s) with reference
-%   b = DAT.EXPEXISTS(expRef) TODO
+%   b = DAT.EXPEXISTS(expRef) Returns true is expRef exists, where expRef
+%   is an experiment reference string or cell array thereof.  For an
+%   experiment to exist the correct folder structure must be present in the
+%   main repository's master location.
+%
+% See Also DAT.LISTEXPS, DAT.PATHS
 %
 % Part of Rigbox
 
@@ -14,7 +19,7 @@
 
   function b = check(expRef)
     % ensure the standard folder given the reference exists
-    b = file.exists(dat.expPath(expRef, 'expInfo', 'master'));
+    b = file.exists(dat.expPath(expRef, 'main', 'master'));
   end
 
 end

+dat/expFilePath.m

@@ -7,105 +7,128 @@
 %   e.g. to get the paths for an experiments 2 photon TIFF movie:
 %   DAT.EXPFILEPATH('mouse1', datenum(2013, 01, 01), 1, '2p-raw');
 %
-%   [full, filename] = expFilePath(ref, type, [reposlocation])
+%   [full, filename] = expFilePath(ref, type, [reposlocation, ext])
 %
-%   [full, filename] = expFilePath(subject, date, seq, type, [reposlocation])
+%   [full, filename] = expFilePath(subject, date, seq, type, [reposlocation, ext])
 %
-%   Options for reposlocation are: 'local' or 'master'
+%   Options for reposlocation are: 'all' (default), 'local', 'master' and 'remote'
 %   Many options for type, e.g. 'block', '2p-raw', 'eyetracking', etc
+%   If ext is specified, the path returned has the extention ext, otherwise
+%   the default for that type is used.
 %
 % Part of Rigbox
 
 % 2013-03 CB created
 
-if nargin == 3 || nargin == 5
-  % repos argument was passed, save the value and remove from varargin
-  location = varargin(end);
-  varargin(end) = [];
-elseif nargin < 2
-  error('Not enough arguments supplied.');
-else
-  % repos argument not passed
-  location = {};
+assert(length(varargin) > 1, 'Rigbox:dat:expFilePath:NotEnoughInputs',...
+  'Not enough input arguments.')
+
+parsed = catStructs(regexp(varargin{1}, dat.expRefRegExp, 'names'));
+if isempty(parsed) % Subject, not ref
+  if nargin < 3
+    error('Rigbox:dat:expFilePath:NotEnoughInputs', ...
+      ['Not enough input arguments; check expRef formatted correcly ' ...
+      'or enter subject, date and sequence as separate arguments'])
+  elseif nargin == 3
+    error('Rigbox:dat:expFilePath:NotEnoughInputs', ...
+      'Not enough input arguments; missing file type')
+  elseif nargin > 4
+    location = varargin{5};
+    varargin(5) = [];
+  else
+    location = {};
+  end
+  typeIdx = 4;
+else % Ref, not subject
+  if nargin < 2
+    error('Rigbox:dat:expFilePath:NotEnoughInputs', ...
+      'Not enough input arguments; missing file type')
+  elseif nargin > 2
+    location = varargin{3};
+    varargin(3) = [];
+  else
+    location = {};
+  end
+  typeIdx = 2;
 end
 
 % tabulate the args to get complete rows
 [varargin{1:end}, singleArgs] = tabulateArgs(varargin{:});
 
-% last argument is the file type
-fileType = varargin{end};
+fileType = varargin{typeIdx};
+extention = iff(any(numel(varargin) == [3,5]), varargin{end},...
+  cell(1,length(varargin{1})));
+if any(numel(varargin) == [3,5]); varargin(end) = []; end
+
 % convert file types to file suffixes
-[repos, suffix, dateLevel] = mapToCell(@typeInfo, fileType);
+[repos, suffix, dateLevel] = mapToCell(@typeInfo, fileType(:), extention(:));
 
 reposArgs = cat(2, {repos}, location);
 
 % and the rest are for the experiment reference
 [expPath, expRef] = dat.expPath(varargin{1:end - 1}, reposArgs{:});
 
-  function [repos, suff, dateLevel] = typeInfo(type)
+  function [repos, suff, dateLevel] = typeInfo(type, newExt)
     % whether this repository is at the date level or otherwise deeper at the sequence
-    % level (default)
+    % level (default). FIXME: Date level doesn't work, perhaps this should
+    % be modified to work with deeper sequences also? E.g.
+    % default\2018-05-04\1\2
     dateLevel = false;
+    repos = 'main';
+    ext = '.mat'; 
     switch lower(type)
       case 'block' % MAT-file with info about each set of trials
-        repos = 'expInfo';
-        suff = '_Block.mat';
+        suff = '_Block';
       case 'hw-info' % MAT-file with info about the hardware used for an experiment
-        repos = 'expInfo';
-        suff = '_hardwareInfo.mat';
+        suff = '_hardwareInfo';
       case '2p-raw' % TIFF with 2-photon raw fluorescence movies
-        repos = 'twoPhoton';
         suff = '_2P.tif';
+        ext = '.tif';
       case 'calcium-preview'
-        repos = 'twoPhoton';
-        suff = '_2P_CalciumPreview.tif';
+        suff = '_2P_CalciumPreview';
+        ext = '.tif';
       case 'calcium-reg'
-        repos = 'twoPhoton';
         suff = '_2P_CalciumReg';
+        ext = '';
       case 'calcium-regframe'
-        repos = 'twoPhoton';
-        suff = '_2P_CalciumRegFrame.tif';
+        suff = '_2P_CalciumRegFrame';
+        ext = '.tif';
       case 'timeline' % MAT-file with acquired timing information
-        repos = 'expInfo';
-        suff = '_Timeline.mat';
+        suff = '_Timeline';
       case 'calcium-roi'
-        repos = 'twoPhoton';
-        suff = '_ROI.mat';
+        suff = '_ROI';
       case 'calcium-fc' % minimally filtered fractional change frames
-        repos = 'twoPhoton';
         suff = '_2P_CalciumFC';
+        ext = '';
       case 'calcium-ffc' % ROI filtered fractional change frames
-        repos = 'twoPhoton';
         suff = '_2P_CalciumFFC';
+        ext = '';
       case 'calcium-widefield-svd'
-        repos = 'widefield';
         suff = '_SVD';
+        ext = '';
       case 'eyetracking'
-        repos = 'eyeTracking';
         suff = '_eye';
+        ext = '';
       case 'parameters' % MAT-file with parameters used for experiment
-        repos = 'expInfo';
-        suff = '_parameters.mat';
+        suff = '_parameters';
       case 'lasermanip'
-        repos = 'expInfo';
-        suff = '_laserManip.mat';
+        suff = '_laserManip';
       case 'img-info'
-        repos = 'twoPhoton';
-        suff = '_imgInfo.mat';
+        suff = '_imgInfo';
       case 'tmaze'
-        repos = 'expInfo';
-        suff = '_TMaze.mat';
+        suff = '_TMaze';
       case 'expdeffun'
-        repos = 'expInfo';
-        suff = '_expDef.m';
+        suff = '_expDef';
+        ext = '.m';
       case 'svdspatialcomps'
         dateLevel = true;
-        %               expPath = mapToCell(@fileparts, expPath);
-        %                 repos = 'expInfo';
-        %                 suff = '_expDef.m';
       otherwise
         error('"%s" is not a valid file type', type);
     end
+    % Append extention to suffix
+    ext = iff(isempty(newExt)&&~ischar(newExt), ext, newExt);
+    suff = iff((isempty(ext)&&ischar(ext))||(~isempty(ext)&&ext(1)=='.'),...
+      [suff, ext], [suff, '.', ext]);
   end
 
 % generate a filename for each experiment
@@ -120,5 +143,4 @@
   filename = filename{1};
 end
 
-end
-
+end

+dat/expLogRequest.m

@@ -6,7 +6,7 @@
 
 if nargin < 2
   args = struct;
-elseif nargin == 2 && isstruct(varargin{1});  
+elseif nargin == 2 && isstruct(varargin{1})
 else
   args = varargin2struct(varargin{:});
 end

+dat/expParams.m

@@ -7,25 +7,29 @@
 %   each experiment. Any experiments without saved parameters will return
 %   empty, [].
 %
+%   If there are multiple remote 'main' repositories, all are searched
+%   precedence is given to experiments on the master repository.
+%
+% See also DAT.LOADBLOCK, DAT.EXPFILEPATH
+%
 % Part of Rigbox
 
 % 2013-03 CB created
 
 %If ref is not an array, wrap it so code below works generally
 [ref, singleArg] = ensureCell(ref);
 %Get the paths where parameters for each experiment will be, if any
-files = dat.expFilePath(ref, 'parameters', 'master');
+files = dat.expFilePath(ref, 'parameters', 'remote');
 %Check which param files exist and load those into results array
-p = cell(size(ref));
-present = file.exists(files);
-p(present) = loadVar(files(present), 'parameters');
+matching = @(p) file.exists(ensureCell(p));
+seq = cellfun(@(f)sequence(ensureCell(f)),files);
+p = mapToCell(@(p)iff(isempty(p), [], @() loadVar(p,'parameters')), ...
+  fun.map(@(p)p.filter(matching).first, seq)); %mu = 0.0490; std = 0.0075
+% p = mapToCell(@loadFun, files); %mu = 0.0443; std = 0.0035
 
 if singleArg
   %If single arg was passed in (i.e. not a cell array, but just a ref,
   %make sure we return a single result (i.e. a single parameter variable,
   %not a cell array of them.
   p = p{1};
-end
-
-end
-
+end

+dat/expPath.m

@@ -9,10 +9,10 @@
 %   sames as the above, but returns paths for an experiment with a
 %   specified 'subject', on a particular 'date', and numbered 'seq'.
 %
-% e.g. to get the paths for the 'expInfo' repository, for the first
+% e.g. to get the paths for the 'main' repository, for the first
 % experiment of the day for 'SUBJECTA':
 % 
-% paths = DAT.EXPPATH('SUBJECTA', now, 1, 'expInfo');
+% paths = DAT.EXPPATH('SUBJECTA', now, 1, 'main');
 %
 % Part of Rigbox
 
@@ -23,6 +23,9 @@
   reposArgs = varargin(end);
   varargin = varargin(1:end - 1);
 else
+  % Check for minimum inputs
+  assert(nargin > 2, ...
+    'Rigbox:dat:expPath:NotEnoughInputs', 'Must provide repo location')
   reposArgs = varargin((end - 1):end);
   varargin = varargin(1:end - 2);
 end

+dat/listExps.m

@@ -7,33 +7,36 @@
 
 % 2013-03 CB created
 
-% The master 'expInfo' repository is the reference for the existence of
+% The master 'main' repository is the reference for the existence of
 % experiments, as given by the folder structure
-expInfoPath = dat.reposPath('expInfo', 'master');
+mainPath = ensureCell(dat.reposPath('main', 'remote'));
 
   function [expRef, expDate, expSeq] = subjectExps(subject)
     % finds experiments for individual subjects
     % experiment dates correpsond to date formated folders in subject's
     % folder
-    subjectPath = fullfile(expInfoPath, subject);
-    subjectDirs = file.list(subjectPath, 'dirs');
+    subjectPath = fullfile(mainPath, subject);
+    subjectDirs = cellflat(rmEmpty(file.list(subjectPath, 'dirs')));
     dateRegExp = '^(?<year>\d\d\d\d)\-?(?<month>\d\d)\-?(?<day>\d\d)$';
     dateMatch = regexp(subjectDirs, dateRegExp, 'names');
-    dateStrs = subjectDirs(~emptyElems(dateMatch));
+    dateStrs = unique(subjectDirs(~emptyElems(dateMatch)));
     [expDate, expSeq] = mapToCell(@(d) expsForDate(subjectPath, d), dateStrs);
     expDate = cat(1, expDate{:});
     expSeq = cat(1, expSeq{:});
     expRef = dat.constructExpRef(repmat({subject}, size(expDate)), expDate, expSeq);
     %sort them by date first then sequence number
     [~,  isorted] = sort(cellsprintf('%.0d-%03i', expDate, expSeq));
+    %remove duplicates that may exist in alternate repos
+    [~,ia] = unique(expRef);
+    isorted = intersect(isorted,ia,'stable');
     expRef = expRef(isorted);
     expDate = expDate(isorted);
     expSeq = expSeq(isorted);
   end
 
   function [dates, seqs] = expsForDate(subjectPath, dateStr)
-    dateDirs = file.list(fullfile(subjectPath, dateStr), 'dirs');
-    seqMatch = cell2mat(regexp(dateDirs, '(?<seq>\d+)', 'names'));
+    dateDirs = rmEmpty(file.list(fullfile(subjectPath, dateStr), 'dirs'));
+    seqMatch = cell2mat(regexp(cellflat(dateDirs), '(?<seq>\d+)', 'names'));
     if numel(seqMatch) > 0
       seqs = str2double({seqMatch.seq}');
     else

+dat/listSubjects.m

@@ -1,17 +1,17 @@
 function subjects = listSubjects()
 %DAT.LISTSUBJECTS Lists recorded subjects
 %   subjects = DAT.LISTSUBJECTS() Lists the experimental subjects present
-%   in experiment info repository ('expInfo').
+%   in main experiment repository ('mainRepository').
+%
+% See also ALYX.LISTSUBJECTS
 %
 % Part of Rigbox
 
 % 2013-03 CB created
 
-% The master 'expInfo' repository is the reference for the existence of
+% The master 'main' repository is the reference for the existence of
 % experiments, as given by the folder structure
-expInfoPath = dat.reposPath('expInfo', 'master');
-
-dirs = file.list(expInfoPath, 'dirs');
-subjects = setdiff(dirs, {'misc'}); %exclude the misc directory
+mainPath = dat.reposPath('main', 'remote');
 
-end
+dirs = unique(cellflat(rmEmpty(file.list(mainPath, 'dirs'))));
+subjects = dirs(~cellfun(@(d)startsWith(d, '@'), dirs)); % exclude misc directories