Skip to content
Snippets Groups Projects
Commit 10f4d3f8 authored by Nicola Vigano's avatar Nicola Vigano
Browse files

OAR/Batch: simplified logviewOar 


Signed-off-by: default avatarNicola Vigano <nicola.vigano@esrf.fr>
parent 31764b09
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
zUtil_OAR/gtBatch.m
+ 22
− 11
View file @ 10f4d3f8
...@@ -9,7 +9,7 @@ function gtBatch(action, varargin) ...@@ -9,7 +9,7 @@ function gtBatch(action, varargin)
% action = { 'update' } : updates fields and shows them % action = { 'update' } : updates fields and shows them
% | 'delete' : deletes entries and oar jobs % | 'delete' : deletes entries and oar jobs
% | 'clean' : cleans entries in file (implied by 'update') % | 'clean' : cleans entries in file (implied by 'update')
% | 'view' : calls gtBetchLogView (disabled for now) % | 'view' : shows logs (not available for the moment)
% %
% varargin = 'name' : expects a string which is a regular expression % varargin = 'name' : expects a string which is a regular expression
% | 'array' : expects a list of array ids % | 'array' : expects a list of array ids
...@@ -17,6 +17,17 @@ function gtBatch(action, varargin) ...@@ -17,6 +17,17 @@ function gtBatch(action, varargin)
% | <string> : implies 'name' % | <string> : implies 'name'
% | [<double>]: implies 'job' % | [<double>]: implies 'job'
% | {nothing} : implies 'name' with string 'gt*' % | {nothing} : implies 'name' with string 'gt*'
%
% Notes about arguments for action 'view':
% The full argument structure is required (without implied arguments)
% and it requires an extra argument being:
%
% view_arg = 'log' : display the submission '.log' file for the array
% | 'err' : display the '.err' file for a selected job
% | 'out' : display the '.out' file for a selected job
% | 'oar' : display the submission '.oar' file for the array
% | 'params' : display the submission '.params' file for the array
%
if (~exist('action', 'var')) if (~exist('action', 'var'))
action = 'update'; action = 'update';
...@@ -57,13 +68,13 @@ end ...@@ -57,13 +68,13 @@ end
function oarparams = gtBatchUpdate(oarparams, records) function oarparams = gtBatchUpdate(oarparams, records)
vars = getQueryVars(); vars = getQueryVars();
fprintf('Quering OAR to get fresh information...') fprintf('Quering OAR to get fresh information...')
newOarStats = gtOarGetStats([records(:).jobId], vars); new_stats = gtOarGetStats([records(:).jobId], vars);
fprintf(' Done.\n') fprintf(' Done.\n')
if (numel(newOarStats) < numel(records)) if (numel(new_stats) < numel(records))
fprintf('-> Some jobs are not recorded by OAR any more! Skipping them!\n') fprintf('-> Some jobs are not recorded by OAR any more! Skipping them!\n')
r_j_ids = [records(:).jobId]; r_j_ids = [records(:).jobId];
o_j_ids = [newOarStats(:).Job_Id]; o_j_ids = [new_stats(:).Job_Id];
keep = false(size(r_j_ids)); keep = false(size(r_j_ids));
for ii_k = 1:numel(keep) for ii_k = 1:numel(keep)
...@@ -87,12 +98,12 @@ function oarparams = gtBatchUpdate(oarparams, records) ...@@ -87,12 +98,12 @@ function oarparams = gtBatchUpdate(oarparams, records)
%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%
% loop through jobs % loop through jobs
for m = 1 : njobs for m = 1 : njobs
jobId = newOarStats(m).('Job_Id'); jobId = new_stats(m).('Job_Id');
arrayid = str2double(newOarStats(m).('job_array_id')); arrayid = str2double(new_stats(m).('job_array_id'));
funcName = newOarStats(m).('name'); funcName = new_stats(m).('name');
% Checking presence of fields % Checking presence of fields
if (~hasAllFields(newOarStats(m), vars)) if (~hasAllFields(new_stats(m), vars))
fprintf('Skipping incomplete record for Job: %d (consider deleting it)\n', jobId) fprintf('Skipping incomplete record for Job: %d (consider deleting it)\n', jobId)
continue continue
end end
...@@ -107,9 +118,9 @@ function oarparams = gtBatchUpdate(oarparams, records) ...@@ -107,9 +118,9 @@ function oarparams = gtBatchUpdate(oarparams, records)
fprintf('%*s %*d ', h_name, funcName, h_job, records(m).jobId) fprintf('%*s %*d ', h_name, funcName, h_job, records(m).jobId)
for l = 1:length(display_vars) for l = 1:length(display_vars)
temp = newOarStats(m).(display_vars{l}); temp = new_stats(m).(display_vars{l});
jobs(indx).(display_vars{l}) = temp; jobs(indx).(display_vars{l}) = temp;
fprintf('%*s ', h_other, records(m).(display_vars{l})) fprintf('%*s ', h_other, temp)
end end
fprintf('\n') fprintf('\n')
oarparams.(funcName).(arrayName).job = jobs; oarparams.(funcName).(arrayName).job = jobs;
...@@ -245,7 +256,7 @@ function [records, selectionType] = getRecords(oarparams, args) ...@@ -245,7 +256,7 @@ function [records, selectionType] = getRecords(oarparams, args)
selectionValue = args{1}; selectionValue = args{1};
end end
else else
selectionType = 'job'; selectionType = 'array';
selectionValue = args{1}; selectionValue = args{1};
end end
case 2 case 2
......
zUtil_OAR/logviewOar.m
+ 110
− 116
View file @ 10f4d3f8
function idout=logviewOar(queryname,typefile,look,debug) function logviewOar(queryname, file_type)
% LOGVIEWOAR Read log file from OAR % LOGVIEWOAR Read log file from OAR
% idout=logviewOar(queryname,[type],[look],[debug]) % logviewOar(queryname, [file_type])
% ------------------------------------------------- % -------------------------------------------------
% %
% Version 002 17-11-2011 by LNervo % Version 002 17-11-2011 by LNervo
...@@ -10,134 +10,128 @@ function idout=logviewOar(queryname,typefile,look,debug) ...@@ -10,134 +10,128 @@ function idout=logviewOar(queryname,typefile,look,debug)
% Read .log file for given function or job array id % Read .log file for given function or job array id
% %
% INPUT: % INPUT:
% queryname = function name <string> % queryname = function name <string>
% arrayid <double> (if not known it will ask which one % arrayid <double> (if not known it will ask which one
% from the existing id(s)) % from the existing id(s))
% jobid <double> % jobid <double>
% %
% typefile = type of logfile % file_type = type of logfile
% <string> [{'log'} | 'err' | 'out' | 'oar' | 'params'] % <string> [{'log'} | 'err' | 'out' | 'oar' | 'params']
% %
% look = display file content or not
% <logical> [{true} | false]
%
% debug = print or not comments [{false} | true]
%
% OUTPUT:
% arrayid = array of array id(s) from the function specified with
% funcname
%
if ~exist('typefile', 'var') || isempty(typefile) if (~exist('file_type', 'var') || isempty(file_type))
typefile = 'log'; file_type = 'log';
end end
if ~exist('look', 'var') || isempty(look) if (~ismember(file_type, {'log', 'err', 'out', 'oar', 'params'}))
look = true; error('logviewOar:wrong_file_type', ...
end ['File type %s not allowed! It should be chosen among: ' ...
if ~exist('debug', 'var') || isempty(debug) '[ ''err'' | ''out'' | ''log'' | ''params'' | ''oar'' ]'], ...
debug = false; file_type);
end end
if (~exist('oarparameters.mat', 'file'))
error('OAR:no_parameters_found', ...
'You must run this function from the directory in which there is the oarparameters.mat file.')
end
out = GtConditionalOutput(debug); log_file_dir = fullfile('/tmp_14_days','oar','log');
% Load of parameters file (if exists) % Load of parameters file (if exists)
oarparameters = []; oarparameters = [];
if exist('oarparameters.mat', 'file')
out.fprintf('Loading the oarparameters file...');
load('oarparameters.mat'); load('oarparameters.mat');
out.odisp('done!');
else
error('OAR:no_parameters_found', ...
'You must run this function from the directory in which there is the oarparameters.mat file.')
end
ask = false; ask = false;
if ischar(queryname) if (ischar(queryname))
funcname = queryname; func_name = queryname;
ask = true; ask = true;
out.odisp(['Looking for function ' funcname])
else if (ismember(file_type, {'err', 'out'}))
queryid = queryname; error('logviewOar:wrong_file_type', ...
oarStats = gtOarGetStats(queryid, {'name','job_array_id','cpuset_name'}); 'Can''t use (''err'', ''out'') options when only using function name')
funcname = oarStats(1).('name'); end
idarray = str2double(char(oarStats(1).('job_array_id'))); else
% We match "_[numbers]" because some users have numbers in their usernames query_id = queryname;
% (experiments) at the ESRF oar_stats = gtOarGetStats(query_id, {'name','job_array_id','cpuset_name'});
idjob = regexp(oarStats(1).('cpuset_name'), '_\d+', 'match'); func_name = oar_stats(1).('name');
idjob = str2double(idjob{1}(2:end)); selected_array_id = str2double(char(oar_stats(1).('job_array_id')));
out.odisp(['Looking for arrayid [' num2str(idarray) '] of the function [' funcname '] with jobid [' num2str(queryid) ']']) % We match "_[numbers]" because some users have numbers in their usernames
end % (experiments) at the ESRF
job_id = regexp(oar_stats(1).('cpuset_name'), '_\d+', 'match');
job_id = str2double(job_id{1}(2:end));
end
array_ids = get_arrays_from_function(oarparameters, func_name);
oar_stats = gtOarGetStats(array_ids, ...
{'job_array_id', 'cpuset_name', 'job_array_index', ...
'submissionTime', 'resubmit_job_id'});
if (ask) % idarray is not known
selected_array_id = select_id('array', func_name, array_ids);
end
if ~isempty(fieldnames(oarparameters)) ind = find(array_ids == selected_array_id);
names = fieldnames(oarparameters);
match = strfind(names,funcname);
idout.fname = {};
jj = 0;
% XXX - While loops are terrible and should be avoided like hell
while (~isempty(match) && (jj < length(names)))
jj = jj + 1;
if ~isempty(match{jj})
fname = names{jj};
idout.(fname).fname = fname;
idout.fname{length(idout.fname)+1} = fname;
out.odisp(['Found a function named ' fname])
subnames = fieldnames(oarparameters.(fname));
arrayid = cellfun(@(name) str2double(name(6:end)), subnames);
if ~isempty(arrayid)
out.odisp(['Found ' length(arrayid) ' arrayid(s) for ' fname ':'])
out.odisp(arrayid)
idout.(fname).arrays = arrayid'; % arrays found for the same function
idout.(fname).logs = fullfile('/tmp_14_days','oar','log', ['oar.' fname '.' num2str(arrayid(end)) '.log']);
oarStats = gtOarGetStats(arrayid, {'job_array_id','cpuset_name','job_array_index','submissionTime','resubmit_job_id'});
else
out.odisp(['No array id(s) found for ' fname '!'])
idout.(fname).arrays = [];
idout.(fname).logs = [];
look = false;
end
if look
if ask % idarray is not known
disp(['Found ' length(arrayid) ' arrayid(s) for ' fname ':'])
disp(arrayid)
idarray = inputwdefaultnumeric('Which arrayid do you want to look at? ', arrayid(end));
end
[~, ind] = intersect(arrayid, idarray);
if (~isempty(ind))
if strcmpi(typefile,'log')
logname = fullfile('/tmp_14_days','oar','log',['oar.' fname '-' getenv('USER') '_' oarStats(ind).('resubmit_job_id') '.' num2str(idarray) '.' typefile]);
elseif (strcmpi(typefile,'err') || strcmpi(typefile,'out'))
logname = fullfile('/tmp_14_days','oar','log',['oar.' fname '-' getenv('USER') '_' oarStats(ind).('resubmit_job_id') '.' num2str(idjob) '.' typefile]);
elseif strcmpi(typefile,'oar') || strcmpi(typefile,'params')
logname = [fname '-' getenv('USER') '_' oarStats(ind).('resubmit_job_id') '.' typefile];
end
if (exist(logname, 'file'))
disp(['Reading file ' logname '...'])
disp('-----------------------------------------------------------------------------------------')
type(logname);
disp('-----------------------------------------------------------------------------------------')
idout.(fname).logs = logname;
idout.(fname).date = oarStats(ind).('submissionTime');
else
fprintf('File not found: %s\n', logname)
end
end
end % end if look
if ~isempty(arrayid)
out.odisp('**************')
out.odisp(['Array(s) saved in the oarparameters file for the function ' fname])
out.odisp(arrayid)
out.odisp('**************')
end
end % end if
end % end while
end % end if
if (~isempty(ind))
switch (lower(file_type))
case 'log'
file_name = get_log_file(log_file_dir, func_name, oar_stats(ind), file_type, selected_array_id);
case {'err', 'out'}
file_name = get_log_file(log_file_dir, func_name, oar_stats(ind), file_type, job_id);
case {'oar', 'params'}
file_name = get_submission_file(func_name, oar_stats(ind), file_type);
end
if (~exist(file_name, 'file'))
error('logviewOar:wrong_file_name', ...
'File not found: %s\n', file_name)
end
read_file(file_name, func_name, oar_stats(ind))
end
end % end of function end % end of function
function read_file(file_name, func_name, oar_stats)
disp(' ')
disp('-----------------------------------------------------------------------------------------')
fprintf('Reading:\n - file: %s\n - array: %s\n - function: %s\n - submitted: %s\n', ...
file_name, oar_stats.('job_array_id'), func_name, oar_stats.('submissionTime'))
disp('-----------------------------------------------------------------------------------------')
type(file_name);
disp('-----------------------------------------------------------------------------------------')
disp(' ')
end
function base_user_file_name = get_base_filename(fname, oar_stats)
base_user_file_name = [fname '-' getenv('USER') '_' oar_stats.('resubmit_job_id')];
end
function filename = get_log_file(log_file_dir, func_name, oar_stats, file_type, id)
base_user_file_name = get_base_filename(func_name, oar_stats);
filename = fullfile(log_file_dir, sprintf('oar.%s.%d.%s', base_user_file_name, id, file_type));
end
function filename = get_submission_file(func_name, oar_stats, file_type)
base_user_file_name = get_base_filename(func_name, oar_stats);
filename = sprintf('%s.%s', base_user_file_name, file_type);
end
function array_ids = get_arrays_from_function(oar_parameters, func_name)
if (~isfield(oar_parameters, func_name))
error('logviewOar:wrong_function_name', 'Function %s was not found in OAR parameters', func_name);
end
subnames = fieldnames(oar_parameters.(func_name));
array_ids = cellfun(@(x)str2double(x(6:end)), subnames);
if (isempty(array_ids))
error('logviewOar:wrong_function', ...
'No array id(s) found for %s!', func_name)
end
end
function selected = select_id(id_type, func_name, list_of_ids)
fprintf('Found %d %s_id(s) for %s:\n', numel(list_of_ids), id_type, func_name)
fprintf(' - %d\n', list_of_ids)
selected = inputwdefaultnumeric( ...
sprintf('Which %s_id do you want to look at? ', id_type), ...
num2str(list_of_ids(end)));
end
zUtil_OAR/updateOar.m
+ 2
− 4
View file @ 10f4d3f8
...@@ -77,8 +77,7 @@ function updateOar(queryname, action) ...@@ -77,8 +77,7 @@ function updateOar(queryname, action)
end % end for k end % end for k
end % end for j end % end for j
elseif ismember(lower(action), {'err','out','log','params','oar'}) elseif ismember(lower(action), {'err','out','log','params','oar'})
output = logviewOar(funcname, action); logviewOar(funcname, action);
print_structure(output,'output',false,true);
else else
% Fail :) % Fail :)
gtError( 'OAR:wrong_argument', ... gtError( 'OAR:wrong_argument', ...
...@@ -90,8 +89,7 @@ function updateOar(queryname, action) ...@@ -90,8 +89,7 @@ function updateOar(queryname, action)
%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%
if (ismember(lower(action), {'err', 'out', 'log', 'params', 'oar'})) if (ismember(lower(action), {'err', 'out', 'log', 'params', 'oar'}))
fprintf('Looking for job number "%d"...\n', arrayid) fprintf('Looking for job number "%d"...\n', arrayid)
output = logviewOar(arrayid, action); logviewOar(arrayid, action);
print_structure(output, 'output', false, true);
else else
% loop over functions % loop over functions
fprintf('Looking for array number "%d"...\n', arrayid) fprintf('Looking for array number "%d"...\n', arrayid)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment