Skip to content
Snippets Groups Projects
Commit 0d9f6f86 authored by Nicola Vigano's avatar Nicola Vigano
Browse files

Cleanups and small fixes 


Signed-off-by: default avatarNicola Vigano <nicola.vigano@esrf.fr>

git-svn-id: https://svn.code.sf.net/p/dct/code/trunk@643 4c865b51-4357-4376-afb4-474e03ccb993
parent 714a3436
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
2_difspot/gtPostSegmentation.m
+ 16
− 14
View file @ 0d9f6f86
......@@ -20,13 +20,13 @@ function gtPostSegmentation()
% load('parameters.mat'); file to look for flags
parameters=[];
parameters = [];
load('parameters.mat');
gtDBConnect();
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% should gtRemoveOverlapping difspots be run?
if isfield(parameters.seg, 'overlaps_removed') && parameters.seg.overlaps_removed==0
if isfield(parameters.seg, 'overlaps_removed') && (parameters.seg.overlaps_removed == 0)
% automatic case
disp('Running gtRemoveOverlappingDifblobs -- this may take a few minutes')
pause(2);
......@@ -38,7 +38,7 @@ else
disp('to deal with blobs split between OAR jobs.')
disp('Double threshold segmentation does not need this.')
% modify default response according to parameters file flags
if isfield(parameters.seg, 'overlaps_removed') && parameters.seg.overlaps_removed==1
if isfield(parameters.seg, 'overlaps_removed') && (parameters.seg.overlaps_removed == 1)
check=inputwdefault('Run gtRemoveOverlappingDifspots? [y/n]', 'n');
else
check=inputwdefault('Run gtRemoveOverlappingDifspots? [y/n]', 'y');
......@@ -58,7 +58,7 @@ disp(' ')
disp(' ')
% get values for OAR jobs
gtDBConnect();
nblobs=mym(['select max(difblobid) from ' parameters.acq.name 'difblob']);
nblobs = mym(['select max(difblobid) from ' parameters.acq.name 'difblob']);
if nblobs > 500
njobs=min(round(nblobs/500), 20);
else
......@@ -66,12 +66,12 @@ else
end
% check if difspots have already been created
test=mym(['select count(*) from ' parameters.acq.name 'difspot']);
if test~=0
test = mym(['select count(*) from ' parameters.acq.name 'difspot']);
if (test ~= 0)
% check if we want to drop and re-create the tables, and run again
check=inputwdefault('Data exists in difspot table! Drop table and re-create data? [y/n]', 'n');
check = inputwdefault('Data exists in difspot table! Drop table and re-create data? [y/n]', 'n');
if strcmpi(check,'y')
overwrite_flag=true;
overwrite_flag = true;
gtDBCreateDifspotTable(parameters.acq.name, overwrite_flag);
else
disp('Keeping difspot data')
......@@ -80,12 +80,14 @@ end
% Otherwise, continue...
% ask user whether to write out difspots
check=inputwdefault('Launch gtDBBlob2SpotTable with OAR? [y/n]', 'y');
if strcmpi(check,'y')
check2=inputwdefault('Write out difspots as .edf files? [y/n]', 'n');
if strcmpi(check2,'y')
disp('Removing previous spots images...')
delete([parameters.acq.dir '/2_difspot/difspot*.edf']);
check = inputwdefault('Launch gtDBBlob2SpotTable with OAR? [y/n]', 'y');
if strcmpi(check, 'y')
check2 = inputwdefault('Write out difspots as .edf files? [y/n]', 'n');
if strcmpi(check2, 'y')
deadFiles = fullfile(parameters.acq.dir, '2_difspot', 'difspot*.edf');
fprintf('Removing previous spots images (from: %s)...\n', deadFiles);
delete(deadFiles);
fprintf('Done.\n')
OAR_make('gtDBBlob2SpotTable', 1, nblobs, njobs, [parameters.acq.dir, ' 1'], true)
else
......
zUtil_OAR/OAR_make.m
+ 19
− 35
View file @ 0d9f6f86
function OAR_make(executable,first,last,njobs,otherparameters,submitflag,varargin)
% OAR_MAKE Submit jobs to OAR.
% OAR_MAKE Submit jobs to OAR.
% OAR_MAKE(executable,first,last,njobs,otherparameters,submitflag,varargin)
% -------------------------------------------------------------------------
%
%
% This function would replace condor_make, using the new cluster
% OAR, resource manager and batch scheduler for NICE
%
% INPUT:
% INPUT:
% executable = '/full/path/to/executable.ext' (if not .m)
% if executable is not in the pwd;
% 'executable.ext' if executable is in the pwd;
......@@ -21,7 +21,7 @@ function OAR_make(executable,first,last,njobs,otherparameters,submitflag,varargi
% i.e. 'walltime','3:00:00','mem',4000, ...
%
% Available directives (default values are shown)
%
%
% NAME VALUE OAR SYNTAX
% name executable -n <text> / --name=<text>
% walltime '2:00:00' -l walltime=<walltime>
......@@ -32,7 +32,7 @@ function OAR_make(executable,first,last,njobs,otherparameters,submitflag,varargi
% core 1 -l /core=<ncore>
% host 'htc' -p host like '<name>%'
% checkpoint 600 --checkpoint <checkpoint>
% signal 2 --signal <signal>
% signal 2 --signal <signal>
% type 'idempotent' -t <type> / --type=<type>
% resubmit 0 --resubmit=<job_id>
%
......@@ -46,7 +46,7 @@ function OAR_make(executable,first,last,njobs,otherparameters,submitflag,varargi
% make_oar_matlab2.sh files
%
% Version 009 08-11-2011 by LNervo
% Add more OAR directives
% Add more OAR directives
% Use singleton.sh in the zUtil_OAR folder.
% Use of make_oar_matlab2.sh
%
......@@ -87,7 +87,7 @@ app = parse_pv_pairs(app, varargin);
oarpars = app;
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% title
% title
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
disp(' ')
disp(' ****************************************')
......@@ -174,23 +174,7 @@ paramfile = 'oarparameters.mat';
oarpars.stderr = logerr;
oarpars.stdout = logout;
% sort OAR directives
names = fieldnames(oarpars);
names = sort(names);
size = length(names);
for ii = 1:size
oarparssort.(char(names(ii))) = oarpars.(char(names(ii)));
if ~ischar( oarparssort.(char(names(ii))) )
str = num2str(oarparssort.(char(names(ii))));
else
str = oarparssort.(char(names(ii)));
end
end
oarpars2 = oarparssort;
%% Setting the parameters struct
set.launchdir = pwd;
set.matparfile = paramfile;
set.oarsubfile = tname;
......@@ -206,14 +190,14 @@ set.oarpars = oarpars;
%% Load of parameters file (if exists)
cd(set.launchdir);
if (exist(fullfile(pwd,paramfile), 'file'))
if (exist(fullfile(pwd, paramfile), 'file'))
out.odisp('Loading the oarparameters file...')
load(fullfile(pwd,paramfile),'oarparameters');
ismat=true;
load(fullfile(pwd, paramfile),'oarparameters');
ismat = true;
else
out.odisp('Creating a new structure called oarparameters with two fields: set and stat')
oarparameters=struct('set','','stat','');
ismat=false;
oarparameters = struct('set','','stat','');
ismat = false;
end
out.fprintf('done!\n')
......@@ -229,16 +213,16 @@ end
out.fprintf('\nSaving the oarparameters.mat file...')
%cd(set.launchdir);
if ismat
save(fullfile(pwd,paramfile), 'oarparameters', '-append');
save(fullfile(pwd, paramfile), 'oarparameters', '-append');
else
save(fullfile(pwd,paramfile), 'oarparameters');
save(fullfile(pwd, paramfile), 'oarparameters');
end
out.fprintf('done!\n')
%% Create OAR parameters file .oar
% create temporary file
out.fprintf('\nCreating OAR directives file')
% get OAR parameters
% get OAR parameters
script_loc = make_oar_matlab(set.launchdir, subdir, tmpname, oarpars);
set.oardirectives = script_loc;
[~, msg] = unix(['chmod 755 ' set.oardirectives]); disp(msg)
......@@ -312,8 +296,8 @@ out.fprintf('Saving parameters in the .log file...')
fid = fopen(set.logname, 'w');
fprintf(fid, '# General information\n');
names = fieldnames(set);
size = length(names);
for ii = 1:size
numOarDirectives = length(names);
for ii = 1:numOarDirectives
if ~strcmp(names{ii}, 'oarpars')
value = set.(char(names(ii)));
if ~ischar(value)
......@@ -325,8 +309,8 @@ for ii = 1:size
end
fprintf(fid, '\n# Setting of OAR directives\n');
names = fieldnames(oarpars);
size = length(names);
for ii = 1:size
numOarDirectives = length(names);
for ii = 1:numOarDirectives
value = oarpars.(char(names(ii)));
if ~ischar(value)
fprintf(fid, '%s = %s\n', char(names(ii)), num2str(value));
......
zUtil_OAR/updateOar.m
+ 44
− 42
View file @ 0d9f6f86
......@@ -36,10 +36,10 @@ end
if ~isempty(fieldnames(oarparameters))
names = fieldnames(oarparameters);
error_ids.terminated = [];
error_ids.error = [];
if (lookfunc)
%%%%%%%%%%%%%%%%%%%%%%%%%%
% look for function name %
......@@ -50,7 +50,7 @@ if ~isempty(fieldnames(oarparameters))
jj = 0;
while (~isempty(match) && (jj < length(names)))
jj = jj + 1;
if isempty(match{jj})
names(jj) = [];
k = jj;
......@@ -59,33 +59,33 @@ if ~isempty(fieldnames(oarparameters))
end
end
end
if (strcmpi(action, 'delete'))
% loop over functions
for jj = 1:length(names)
disp(['Deleting function ' names{jj} '...']);
arrayField = fieldnames(oarparameters.(names{jj}));
questionStr = ['Are you sure to remove the function ' names{jj} ' from OAR and oarparameters.mat? [y/n]'];
check = inputwdefault(questionStr, 'y');
if strcmpi(check, 'y')
fprintf('Deleting jobs from OAR... ')
cellfun(@(array) deleteOarArray(array(6:end)), arrayField);
fprintf('Done.\nRemoving information about the jobs... ')
oarparameters = rmfield(oarparameters, names{jj});
save('oarparameters.mat', 'oarparameters', '-v7.3');
disp('...done!')
disp('Deleting jobs from OAR...')
cellfun(@(array) deleteOarArray(array(6:end)), arrayField);
disp('...done!')
fprintf('Done.\n')
end
end
save('oarparameters.mat','oarparameters', '-v7.3');
elseif (strcmpi(action, 'update'))
% loop over functions
for jj = 1:length(names)
arrays = fieldnames(oarparameters.(names{jj}));
arrayName = names{jj};
for k = 1:length(arrays)
arrayField = arrays{k};
[oarparameters, error_ids] = updateArray(oarparameters, arrayName, arrayField, error_ids);
......@@ -102,60 +102,60 @@ if ~isempty(fieldnames(oarparameters))
disp(['Action ' action ' is not valid! Exiting...'])
return;
end
else
%%%%%%%%%%%%%%%%%%%%%%%%%
% look for array number %
%%%%%%%%%%%%%%%%%%%%%%%%%
if any( cellfun(@(logarray) strcmpi(action, logarray), {'err','out','log','params','oar'}) )
disp(['Looking for job number ' num2str(arrayid) '...'])
output = logviewOar(arrayid, action);
print_structure(output,'output',false,true);
else
% loop over functions
disp(['Looking for array number ' num2str(arrayid) '...'])
for jj = 1:length(names)
arrayName = names{jj};
% search array id(s) for each function
arrays = logviewOar(arrayName, [], false);
% check if arrayid exists
if ~isempty(find(arrays.(arrayName).arrays == arrayid, 1))
arrayField = sprintf('array%d', arrayid);
if strcmpi(action, 'delete')
disp(['Deleting array n. ' arrayField '...']);
questionStr = ['Are you sure to remove array ' num2str(arrayid) ' from oarparameters.mat? [y/n]'];
check = inputwdefault(questionStr, 'y');
if strcmpi(check, 'y')
oarparameters.(arrayName) = rmfield(oarparameters.(arrayName), arrayField);
save('oarparameters.mat','oarparameters');
disp('...done!')
disp('Deleting jobs from OAR...')
fprintf('Deleting jobs from OAR... ')
deleteOarArray(arrayField);
disp('...done!')
fprintf('Done.\nRemoving information about the jobs... ')
oarparameters.(arrayName) = rmfield(oarparameters.(arrayName), arrayField);
save('oarparameters.mat', 'oarparameters', '-v7.3');
fprintf('Done.\n')
end
elseif strcmpi(action, 'update')
[oarparameters, error_ids] = updateArray(oarparameters, arrayName, arrayField, error_ids);
else
disp(['Action ' action ' is not valid! Exiting...'])
return;
end
end % end if arrayid exists
end % end for jj
end
end % end lookfunc
save error_ids error_ids
end % end if
end % end of function
......@@ -167,12 +167,14 @@ function deleteOarArray(array)
% DELETEOARARRAY
% deleteOarArray(array)
if ischar(array)
[~, ~] = unix(['oardel --array ' array]);
else
[~, ~] = unix(['oardel --array ' num2str(array)]);
end
if ~ischar(array)
array = num2str(array);
end
[~, message] = unix(['oardel --array ' array]);
if strfind(message, '[ERROR]')
error('OAR:error_in_deleting', message);
end
end
function [oarparameters, error_ids] = updateArray(oarparameters, arrayName, arrayField, error_ids)
......@@ -194,13 +196,13 @@ else
jobs = oarparameters.(arrayName).(arrayField).job;
njobs = length(jobs);
oarStats = gtOarGetStats([jobs.jobId], vars);
newJobs = struct(jobs(1:0));
h_name = length(arrayName)+2;
h_job = length(num2str(jobs(1).jobId))+2;
h_other = length(datestr(now(),'yyyy-mm-dd HH:MM:SS'))+2;
%%%%%%%%%%%%%%%%%%%%%%%
% loop through jobs
for m = 1 : njobs
......@@ -210,27 +212,27 @@ else
fprintf('%*s ', h_other, jobs(m).(vars{l}))
end
fprintf('\n')
state = jobs(m).('state');
code = jobs(m).('exit_code');
arrayid = jobs(1).('jobId');
jobid = jobs(m).('jobId');
% Remove terminated good jobs
if strcmpi(state, 'Terminated') && strcmpi(code, '0 (0,0,0)')
disp(['Removed terminated job ' num2str(jobid) ' from array ' num2str(arrayid) '...'])
else
newJobs(end+1) = jobs(m); %#ok<AGROW>
end
if ~strcmpi(code, '0 (0,0,0)') && strcmpi(state, 'Terminated')
error_ids.terminated = [error_ids.terminated; jobid];
elseif strcmpi(state, 'Error')
error_ids.error = [error_ids.error; jobid];
end
end
oarparameters.(arrayName).(arrayField).job = newJobs;
save('oarparameters.mat', 'oarparameters', '-v7.3');
end % end if
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment