Skip to content
Snippets Groups Projects
Commit 10f4d3f8 authored by Nicola Vigano's avatar Nicola Vigano
Browse files

OAR/Batch: simplified logviewOar 


Signed-off-by: default avatarNicola Vigano <nicola.vigano@esrf.fr>
parent 31764b09
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
zUtil_OAR/gtBatch.m
+ 22
− 11
View file @ 10f4d3f8
......@@ -9,7 +9,7 @@ function gtBatch(action, varargin)
% action = { 'update' } : updates fields and shows them
% | 'delete' : deletes entries and oar jobs
% | 'clean' : cleans entries in file (implied by 'update')
% | 'view' : calls gtBetchLogView (disabled for now)
% | 'view' : shows logs (not available for the moment)
%
% varargin = 'name' : expects a string which is a regular expression
% | 'array' : expects a list of array ids
......@@ -17,6 +17,17 @@ function gtBatch(action, varargin)
% | <string> : implies 'name'
% | [<double>]: implies 'job'
% | {nothing} : implies 'name' with string 'gt*'
%
% Notes about arguments for action 'view':
% The full argument structure is required (without implied arguments)
% and it requires an extra argument being:
%
% view_arg = 'log' : display the submission '.log' file for the array
% | 'err' : display the '.err' file for a selected job
% | 'out' : display the '.out' file for a selected job
% | 'oar' : display the submission '.oar' file for the array
% | 'params' : display the submission '.params' file for the array
%
if (~exist('action', 'var'))
action = 'update';
......@@ -57,13 +68,13 @@ end
function oarparams = gtBatchUpdate(oarparams, records)
vars = getQueryVars();
fprintf('Quering OAR to get fresh information...')
newOarStats = gtOarGetStats([records(:).jobId], vars);
new_stats = gtOarGetStats([records(:).jobId], vars);
fprintf(' Done.\n')
if (numel(newOarStats) < numel(records))
if (numel(new_stats) < numel(records))
fprintf('-> Some jobs are not recorded by OAR any more! Skipping them!\n')
r_j_ids = [records(:).jobId];
o_j_ids = [newOarStats(:).Job_Id];
o_j_ids = [new_stats(:).Job_Id];
keep = false(size(r_j_ids));
for ii_k = 1:numel(keep)
......@@ -87,12 +98,12 @@ function oarparams = gtBatchUpdate(oarparams, records)
%%%%%%%%%%%%%%%%%%%%%%%
% loop through jobs
for m = 1 : njobs
jobId = newOarStats(m).('Job_Id');
arrayid = str2double(newOarStats(m).('job_array_id'));
funcName = newOarStats(m).('name');
jobId = new_stats(m).('Job_Id');
arrayid = str2double(new_stats(m).('job_array_id'));
funcName = new_stats(m).('name');
% Checking presence of fields
if (~hasAllFields(newOarStats(m), vars))
if (~hasAllFields(new_stats(m), vars))
fprintf('Skipping incomplete record for Job: %d (consider deleting it)\n', jobId)
continue
end
......@@ -107,9 +118,9 @@ function oarparams = gtBatchUpdate(oarparams, records)
fprintf('%*s %*d ', h_name, funcName, h_job, records(m).jobId)
for l = 1:length(display_vars)
temp = newOarStats(m).(display_vars{l});
temp = new_stats(m).(display_vars{l});
jobs(indx).(display_vars{l}) = temp;
fprintf('%*s ', h_other, records(m).(display_vars{l}))
fprintf('%*s ', h_other, temp)
end
fprintf('\n')
oarparams.(funcName).(arrayName).job = jobs;
......@@ -245,7 +256,7 @@ function [records, selectionType] = getRecords(oarparams, args)
selectionValue = args{1};
end
else
selectionType = 'job';
selectionType = 'array';
selectionValue = args{1};
end
case 2
......
zUtil_OAR/logviewOar.m
+ 110
− 116
View file @ 10f4d3f8
function idout=logviewOar(queryname,typefile,look,debug)
function logviewOar(queryname, file_type)
% LOGVIEWOAR Read log file from OAR
% idout=logviewOar(queryname,[type],[look],[debug])
% logviewOar(queryname, [file_type])
% -------------------------------------------------
%
% Version 002 17-11-2011 by LNervo
......@@ -10,134 +10,128 @@ function idout=logviewOar(queryname,typefile,look,debug)
% Read .log file for given function or job array id
%
% INPUT:
% queryname = function name <string>
% arrayid <double> (if not known it will ask which one
% queryname = function name <string>
% arrayid <double> (if not known it will ask which one
% from the existing id(s))
% jobid <double>
%
% typefile = type of logfile
% file_type = type of logfile
% <string> [{'log'} | 'err' | 'out' | 'oar' | 'params']
%
% look = display file content or not
% <logical> [{true} | false]
%
% debug = print or not comments [{false} | true]
%
% OUTPUT:
% arrayid = array of array id(s) from the function specified with
% funcname
%
if ~exist('typefile', 'var') || isempty(typefile)
typefile = 'log';
end
if ~exist('look', 'var') || isempty(look)
look = true;
end
if ~exist('debug', 'var') || isempty(debug)
debug = false;
end
if (~exist('file_type', 'var') || isempty(file_type))
file_type = 'log';
end
if (~ismember(file_type, {'log', 'err', 'out', 'oar', 'params'}))
error('logviewOar:wrong_file_type', ...
['File type %s not allowed! It should be chosen among: ' ...
'[ ''err'' | ''out'' | ''log'' | ''params'' | ''oar'' ]'], ...
file_type);
end
if (~exist('oarparameters.mat', 'file'))
error('OAR:no_parameters_found', ...
'You must run this function from the directory in which there is the oarparameters.mat file.')
end
out = GtConditionalOutput(debug);
log_file_dir = fullfile('/tmp_14_days','oar','log');
% Load of parameters file (if exists)
oarparameters = [];
if exist('oarparameters.mat', 'file')
out.fprintf('Loading the oarparameters file...');
% Load of parameters file (if exists)
oarparameters = [];
load('oarparameters.mat');
out.odisp('done!');
else
error('OAR:no_parameters_found', ...
'You must run this function from the directory in which there is the oarparameters.mat file.')
end
ask = false;
if ischar(queryname)
funcname = queryname;
ask = true;
out.odisp(['Looking for function ' funcname])
else
queryid = queryname;
oarStats = gtOarGetStats(queryid, {'name','job_array_id','cpuset_name'});
funcname = oarStats(1).('name');
idarray = str2double(char(oarStats(1).('job_array_id')));
% We match "_[numbers]" because some users have numbers in their usernames
% (experiments) at the ESRF
idjob = regexp(oarStats(1).('cpuset_name'), '_\d+', 'match');
idjob = str2double(idjob{1}(2:end));
out.odisp(['Looking for arrayid [' num2str(idarray) '] of the function [' funcname '] with jobid [' num2str(queryid) ']'])
end
ask = false;
if (ischar(queryname))
func_name = queryname;
ask = true;
if (ismember(file_type, {'err', 'out'}))
error('logviewOar:wrong_file_type', ...
'Can''t use (''err'', ''out'') options when only using function name')
end
else
query_id = queryname;
oar_stats = gtOarGetStats(query_id, {'name','job_array_id','cpuset_name'});
func_name = oar_stats(1).('name');
selected_array_id = str2double(char(oar_stats(1).('job_array_id')));
% We match "_[numbers]" because some users have numbers in their usernames
% (experiments) at the ESRF
job_id = regexp(oar_stats(1).('cpuset_name'), '_\d+', 'match');
job_id = str2double(job_id{1}(2:end));
end
array_ids = get_arrays_from_function(oarparameters, func_name);
oar_stats = gtOarGetStats(array_ids, ...
{'job_array_id', 'cpuset_name', 'job_array_index', ...
'submissionTime', 'resubmit_job_id'});
if (ask) % idarray is not known
selected_array_id = select_id('array', func_name, array_ids);
end
if ~isempty(fieldnames(oarparameters))
names = fieldnames(oarparameters);
match = strfind(names,funcname);
idout.fname = {};
jj = 0;
% XXX - While loops are terrible and should be avoided like hell
while (~isempty(match) && (jj < length(names)))
jj = jj + 1;
if ~isempty(match{jj})
fname = names{jj};
idout.(fname).fname = fname;
idout.fname{length(idout.fname)+1} = fname;
out.odisp(['Found a function named ' fname])
subnames = fieldnames(oarparameters.(fname));
arrayid = cellfun(@(name) str2double(name(6:end)), subnames);
if ~isempty(arrayid)
out.odisp(['Found ' length(arrayid) ' arrayid(s) for ' fname ':'])
out.odisp(arrayid)
idout.(fname).arrays = arrayid'; % arrays found for the same function
idout.(fname).logs = fullfile('/tmp_14_days','oar','log', ['oar.' fname '.' num2str(arrayid(end)) '.log']);
oarStats = gtOarGetStats(arrayid, {'job_array_id','cpuset_name','job_array_index','submissionTime','resubmit_job_id'});
else
out.odisp(['No array id(s) found for ' fname '!'])
idout.(fname).arrays = [];
idout.(fname).logs = [];
look = false;
end
if look
if ask % idarray is not known
disp(['Found ' length(arrayid) ' arrayid(s) for ' fname ':'])
disp(arrayid)
idarray = inputwdefaultnumeric('Which arrayid do you want to look at? ', arrayid(end));
end
[~, ind] = intersect(arrayid, idarray);
if (~isempty(ind))
if strcmpi(typefile,'log')
logname = fullfile('/tmp_14_days','oar','log',['oar.' fname '-' getenv('USER') '_' oarStats(ind).('resubmit_job_id') '.' num2str(idarray) '.' typefile]);
elseif (strcmpi(typefile,'err') || strcmpi(typefile,'out'))
logname = fullfile('/tmp_14_days','oar','log',['oar.' fname '-' getenv('USER') '_' oarStats(ind).('resubmit_job_id') '.' num2str(idjob) '.' typefile]);
elseif strcmpi(typefile,'oar') || strcmpi(typefile,'params')
logname = [fname '-' getenv('USER') '_' oarStats(ind).('resubmit_job_id') '.' typefile];
end
if (exist(logname, 'file'))
disp(['Reading file ' logname '...'])
disp('-----------------------------------------------------------------------------------------')
type(logname);
disp('-----------------------------------------------------------------------------------------')
idout.(fname).logs = logname;
idout.(fname).date = oarStats(ind).('submissionTime');
else
fprintf('File not found: %s\n', logname)
end
end
end % end if look
if ~isempty(arrayid)
out.odisp('**************')
out.odisp(['Array(s) saved in the oarparameters file for the function ' fname])
out.odisp(arrayid)
out.odisp('**************')
end
end % end if
end % end while
end % end if
ind = find(array_ids == selected_array_id);
if (~isempty(ind))
switch (lower(file_type))
case 'log'
file_name = get_log_file(log_file_dir, func_name, oar_stats(ind), file_type, selected_array_id);
case {'err', 'out'}
file_name = get_log_file(log_file_dir, func_name, oar_stats(ind), file_type, job_id);
case {'oar', 'params'}
file_name = get_submission_file(func_name, oar_stats(ind), file_type);
end
if (~exist(file_name, 'file'))
error('logviewOar:wrong_file_name', ...
'File not found: %s\n', file_name)
end
read_file(file_name, func_name, oar_stats(ind))
end
end % end of function
function read_file(file_name, func_name, oar_stats)
disp(' ')
disp('-----------------------------------------------------------------------------------------')
fprintf('Reading:\n - file: %s\n - array: %s\n - function: %s\n - submitted: %s\n', ...
file_name, oar_stats.('job_array_id'), func_name, oar_stats.('submissionTime'))
disp('-----------------------------------------------------------------------------------------')
type(file_name);
disp('-----------------------------------------------------------------------------------------')
disp(' ')
end
function base_user_file_name = get_base_filename(fname, oar_stats)
base_user_file_name = [fname '-' getenv('USER') '_' oar_stats.('resubmit_job_id')];
end
function filename = get_log_file(log_file_dir, func_name, oar_stats, file_type, id)
base_user_file_name = get_base_filename(func_name, oar_stats);
filename = fullfile(log_file_dir, sprintf('oar.%s.%d.%s', base_user_file_name, id, file_type));
end
function filename = get_submission_file(func_name, oar_stats, file_type)
base_user_file_name = get_base_filename(func_name, oar_stats);
filename = sprintf('%s.%s', base_user_file_name, file_type);
end
function array_ids = get_arrays_from_function(oar_parameters, func_name)
if (~isfield(oar_parameters, func_name))
error('logviewOar:wrong_function_name', 'Function %s was not found in OAR parameters', func_name);
end
subnames = fieldnames(oar_parameters.(func_name));
array_ids = cellfun(@(x)str2double(x(6:end)), subnames);
if (isempty(array_ids))
error('logviewOar:wrong_function', ...
'No array id(s) found for %s!', func_name)
end
end
function selected = select_id(id_type, func_name, list_of_ids)
fprintf('Found %d %s_id(s) for %s:\n', numel(list_of_ids), id_type, func_name)
fprintf(' - %d\n', list_of_ids)
selected = inputwdefaultnumeric( ...
sprintf('Which %s_id do you want to look at? ', id_type), ...
num2str(list_of_ids(end)));
end
zUtil_OAR/updateOar.m
+ 2
− 4
View file @ 10f4d3f8
......@@ -77,8 +77,7 @@ function updateOar(queryname, action)
end % end for k
end % end for j
elseif ismember(lower(action), {'err','out','log','params','oar'})
output = logviewOar(funcname, action);
print_structure(output,'output',false,true);
logviewOar(funcname, action);
else
% Fail :)
gtError( 'OAR:wrong_argument', ...
......@@ -90,8 +89,7 @@ function updateOar(queryname, action)
%%%%%%%%%%%%%%%%%%%%%%%%%
if (ismember(lower(action), {'err', 'out', 'log', 'params', 'oar'}))
fprintf('Looking for job number "%d"...\n', arrayid)
output = logviewOar(arrayid, action);
print_structure(output, 'output', false, true);
logviewOar(arrayid, action);
else
% loop over functions
fprintf('Looking for array number "%d"...\n', arrayid)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment