Python/OAR/GUI: added some more functionality (resubmission / deletion)

Signed-off-by: Nicola Vigano <nicola.vigano@esrf.fr>

zUtil_Python/dct_batch_gui.py

@@ -19,6 +19,8 @@ import datetime
 
 import os
 
+import dct_batch_oar_parameters
+
 class DCTBatchGui(QtGui.QWidget):
     '''
     classdocs
@@ -32,10 +34,12 @@ class DCTBatchGui(QtGui.QWidget):
         super(DCTBatchGui, self).__init__()
         self.qapi = oar_api.OarAPI(api_type = "CmdAPI")
         self.arrays = {};
+        self.records = dct_batch_oar_parameters.DCTBatchOarParameters(self)
         if cwd == "":
             cwd = os.getcwd()
         self._init_gui()
         self.set_current_cwd(cwd)
+        self._pbutt_load_params()
 
     def _init_gui(self):
         self.resize(1024, 768)
@@ -83,6 +87,17 @@ class DCTBatchGui(QtGui.QWidget):
         pbutt_view_oar_err.clicked.connect(self._pbutt_open_oar_err_file)
         vbox_array_control.addWidget(pbutt_view_oar_err)
 
+        spacer = QtGui.QSpacerItem(30, 30)
+        vbox_array_control.insertSpacerItem(-1, spacer)
+
+        pbutt_view_oar_resub = QtGui.QPushButton("Resubmit")
+        pbutt_view_oar_resub.clicked.connect(self._pbutt_resubmit_job)
+        vbox_array_control.addWidget(pbutt_view_oar_resub)
+
+        pbutt_view_oar_del = QtGui.QPushButton("Delete")
+        pbutt_view_oar_del.clicked.connect(self._pbutt_delete_job)
+        vbox_array_control.addWidget(pbutt_view_oar_del)
+
         hbox_array_control.addLayout(vbox_array_control)
 
         hbox_cwd = QtGui.QHBoxLayout()
@@ -109,6 +124,20 @@ class DCTBatchGui(QtGui.QWidget):
         pbutt_load_file = QtGui.QPushButton("Load file..")
         pbutt_load_file.clicked.connect(self._pbutt_load_file)
         hbox_actions.addWidget(pbutt_load_file)
+        pbutt_load_pars = QtGui.QPushButton("Load local parameters..")
+        pbutt_load_pars.clicked.connect(self._pbutt_load_params)
+        hbox_actions.addWidget(pbutt_load_pars)
+
+        spacer = QtGui.QSpacerItem(30, 30)
+        hbox_actions.insertSpacerItem(-1, spacer)
+
+        self.combo_load_updt = QtGui.QComboBox(self)
+        self.combo_load_updt.addItem("Arrays")
+        self.combo_load_updt.addItem("Jobs")
+        hbox_actions.addWidget(self.combo_load_updt)
+        pbutt_load_updt = QtGui.QPushButton("Update!")
+        pbutt_load_updt.clicked.connect(self._pbutt_update_params)
+        hbox_actions.addWidget(pbutt_load_updt)
 
         vbox = QtGui.QVBoxLayout()
         vbox.addLayout(hbox_cwd)
@@ -136,6 +165,20 @@ class DCTBatchGui(QtGui.QWidget):
         if not file_path == "":
             self.load_file(file_path)
 
+    def _pbutt_load_params(self):
+        file_path = os.sep.join((self.cwd, "oarparameters.xml"))
+        if os.path.isfile(file_path):
+            self.load_oar_pars_xml_file(file_path)
+        else:
+            raise EnvironmentError("No such file: %s", file_path)
+
+    def _pbutt_update_params(self):
+        if self.combo_load_updt.currentText() == "Arrays":
+            self.records.update_content_arrays()
+        else:
+            self.records.update_content_jobs()
+        self.update_display_arrays()
+
     def _pbutt_query_dialog(self):
         dial = DCTBatchOARQueryDialog(self)
         ret_code = dial.exec_()
@@ -143,19 +186,16 @@ class DCTBatchGui(QtGui.QWidget):
             print("Not doing anything")
             return
 
-        [sel_id, succed] = QtCore.QString.toInt(dial.edit.text())
+        [sel_id, _] = QtCore.QString.toInt(dial.edit.text())
         if dial.combo.currentText() == "Array":
-            print("Query Array ID: %d" % sel_id)
-            res = self.qapi.query(array = sel_id)
+            self.records.load_content_from_arrayids_query_oar([sel_id])
         else:
-            print("Query Job ID: %d" % sel_id)
-            res = self.qapi.query(jobs = [sel_id])
-        print("Loading content")
-        self.load_content(res)
-        print("Done.")
+            self.records.load_content_from_jobids_query_oar([sel_id])
+
+        self.update_display_arrays()
 
     def _pbutt_open_oar_submit_file(self):
-        (array_id, job_info) = self.get_current_array_and_job()
+        (_, job_info) = self.get_current_array_and_job()
         if job_info is None:
             raise ValueError("Click on a job please!")
 
@@ -165,7 +205,7 @@ class DCTBatchGui(QtGui.QWidget):
         dial.set_text(job_info, full_file)
 
     def _pbutt_open_oar_params_file(self):
-        (array_id, job_info) = self.get_current_array_and_job()
+        (_, job_info) = self.get_current_array_and_job()
         if job_info is None:
             raise ValueError("Click on a job please!")
 
@@ -175,7 +215,7 @@ class DCTBatchGui(QtGui.QWidget):
         dial.set_text(job_info, full_file)
 
     def _pbutt_open_oar_out_file(self):
-        (array_id, job_info) = self.get_current_array_and_job()
+        (_, job_info) = self.get_current_array_and_job()
         if job_info is None:
             raise ValueError("Click on a job please!")
 
@@ -184,7 +224,7 @@ class DCTBatchGui(QtGui.QWidget):
         dial.set_text(job_info, full_file)
 
     def _pbutt_open_oar_err_file(self):
-        (array_id, job_info) = self.get_current_array_and_job()
+        (_, job_info) = self.get_current_array_and_job()
         if job_info is None:
             raise ValueError("Click on a job please!")
 
@@ -193,52 +233,79 @@ class DCTBatchGui(QtGui.QWidget):
         dial.set_text(job_info, full_file)
 
     def _pbutt_open_job_info_dialog(self):
-        (array_id, job_info) = self.get_current_array_and_job()
+        (_, job_info) = self.get_current_array_and_job()
         if job_info is None:
             raise ValueError("Click on a job please!")
 
         DCTBatchDialogJobInfo(self, job_info)
 
+    def _pbutt_resubmit_job(self):
+        (_, job_info) = self.get_current_array_and_job()
+        if job_info is None:
+            raise ValueError("Click on a job please!")
+
+        orig_job_id = job_info['Job_Id']
+
+        dial = QtGui.QMessageBox(QtGui.QMessageBox.Question, \
+                                 "Resubmission", \
+                                 "Do you want to resubmit job: %d ?" % (orig_job_id, ), \
+                                 buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No )
+        if dial.exec_() == QtGui.QMessageBox.Yes:
+            new_job_id = self.qapi.resubmit(orig_job_id)
+            dial = QtGui.QMessageBox(QtGui.QMessageBox.Information, \
+                                     "Resubmission", \
+                                     "Job %d, resubmitted with id: %d" % (orig_job_id, new_job_id))
+            dial.exec_()
+
+            self.update_display_arrays()
+
+    def _pbutt_delete_job(self):
+        (array_id, job_info) = self.get_current_array_and_job()
+        if job_info is None:
+            dial = QtGui.QMessageBox(QtGui.QMessageBox.Question, \
+                                     "Deletion", \
+                                     "Do you want to delete array: %d ?" % (array_id, ), \
+                                     buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No )
+
+            if dial.exec_() == QtGui.QMessageBox.Yes:
+                self.qapi.delete(array = array_id)
+
+                self.update_display_arrays()
+        else:
+            orig_job_id = job_info['Job_Id']
+
+            dial = QtGui.QMessageBox(QtGui.QMessageBox.Question, \
+                                     "Deletion", \
+                                     "Do you want to delete job: %d ?" % (orig_job_id, ), \
+                                     buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No )
+            if dial.exec_() == QtGui.QMessageBox.Yes:
+                self.qapi.delete(jobs = [orig_job_id])
+
+                self.update_display_arrays()
+
     def _get_base_log_file_name(self, job_info):
         return "%s-%s_%d" % (job_info['name'], job_info['job_user'], job_info['resubmit_job_id'])
 
     def get_current_array_and_job(self):
-        array_ids = [x for x in self.arrays.iterkeys()]
-
         selected = self.array_list.currentIndex()
-        if selected.row() == -1:
-            return (None, None)
+        return self.records.get_current_array_and_job(selected)
 
-        parent = selected.parent()
+    def load_oar_pars_xml_file(self, file_path):
+        self.records.load_content_from_oarparameters(open(file_path))
 
-        if parent.row() == -1:
-            array_id = array_ids[selected.row()]
-            return (array_id, None)
-        else:
-            array_id = array_ids[parent.row()]
-            return (array_id, self.arrays[array_id][selected.row()])
+        self.update_display_arrays()
 
     def load_file(self, file_path):
         f = open(file_path)
         content = oar_api.load(f, Loader = oar_api.YamlLoader)
-        self.load_content(content)
-
-    def load_content(self, content):
-        # Content is supposed to be in structures, where the Job ID is the main
-        # thing
-        array_ids = (x for x in set((job_info['array_id'] for job_info in content.itervalues())))
-        self.arrays = {};
-        for aid in array_ids:
-            self.arrays[aid] = []
-        for job_info in content.itervalues():
-            type(job_info['array_id'])
-            self.arrays[job_info['array_id']].append(job_info)
+        self.records.load_content_from_structure(content)
 
         self.update_display_arrays()
 
+
     def update_display_arrays(self):
         self.array_list.clear()
-        for (array_key, array_content) in self.arrays.iteritems():
+        for (array_key, array_content) in self.records.arrays.iteritems():
             array_item = QtGui.QTreeWidgetItem(["Array: %d" % array_key, ])
             for job in array_content:
                 if job['exit_code'] is None:
@@ -263,6 +330,7 @@ class DCTBatchGui(QtGui.QWidget):
                             stop_time ]
                 job_item = QtGui.QTreeWidgetItem(QtCore.QStringList(job_info))
                 array_item.addChild(job_item)
+            array_item.setExpanded(True)
             self.array_list.addTopLevelItem(array_item)
 
     def set_current_cwd(self, cwd):

zUtil_Python/dct_batch_oar_parameters.py

+'''
+Created on Nov 20, 2014
+
+@author: vigano
+'''
+
+from PyQt4 import QtXml
+
+class DCTBatchOarParameters(object):
+    '''
+    classdocs
+    '''
+
+    def __init__(self, parent, stream = None):
+        '''
+        Constructor
+        '''
+        self.parent = parent
+        self.arrays = {}
+
+        if stream is not None:
+            self.load_content_from_oarparameters(stream)
+
+    def _sort_jobs(self, field = 'submissionTime'):
+        for aid in self.arrays.keys():
+            self.arrays[aid] = self._sort_by_field(self.arrays[aid], field)
+
+    def _sort_by_field(self, in_list, field):
+        contents = [x[field] for x in in_list]
+        indices = sorted(range(len(contents)), key=contents.__getitem__)
+        return [in_list[x] for x in indices]
+
+    def _get_arrayids_from_xml_file(self, stream):
+        array_ids = []
+
+        doc = QtXml.QDomDocument()
+        doc.setContent(stream.read())
+        xmlroot = doc.firstChildElement()
+        func_nodes = xmlroot.elementsByTagName("function")
+        for func_n in range(func_nodes.count()):
+            func = func_nodes.at(func_n)
+            if func.isElement():
+                func = func.toElement()
+                array_nodes = func.elementsByTagName("array")
+                for array_n in range(array_nodes.count()):
+                    array = array_nodes.at(array_n)
+                    if array.isElement():
+                        array = array.toElement()
+                        (array_id, _) = array.attribute("id").toInt()
+                        array_ids.append(array_id)
+        return array_ids
+
+    def load_content_from_arrayids_query_oar(self, array_ids):
+        if array_ids is None:
+            print("Not doing anything")
+            return
+
+        arrays = {}
+
+        arrays_content = {}
+        for aid in array_ids:
+            arrays[aid] = []
+
+            print("Query Array ID: %d" % aid)
+            content = self.parent.qapi.query(array = aid)
+
+            arrays_content.update(content)
+
+        self.load_content_from_structure(arrays_content)
+
+    def load_content_from_jobids_query_oar(self, jobs):
+        if jobs is None:
+            print("Not doing anything")
+            return
+
+        self.arrays = {}
+
+        print("Query Job IDs: %s" % ", ".join(("%d" % x for x in jobs)))
+        content = self.parent.qapi.query(jobs = jobs)
+
+        self.load_content_from_structure(content)
+
+    def load_content_from_oarparameters(self, stream):
+        array_ids = self._get_arrayids_from_xml_file(stream)
+        self.load_content_from_arrayids_query_oar(array_ids)
+
+    def load_content_from_structure(self, content):
+        # Content is supposed to be in structures, where the Job ID is the main
+        # thing
+        jobs = [job for job in content.itervalues()]
+        self._sort_by_field(jobs, 'submissionTime')
+        array_ids = (x for x in set((job['array_id'] for job in jobs)))
+        self.arrays = {};
+        for aid in array_ids:
+            self.arrays[aid] = []
+        for job in jobs:
+            type(job['array_id'])
+            self.arrays[job['array_id']].append(job)
+
+        self._sort_jobs()
+
+    def get_current_array_and_job(self, selected):
+        if selected.row() == -1:
+            return (None, None)
+
+        parent = selected.parent()
+
+        array_ids = [x for x in self.arrays.iterkeys()]
+        if parent.row() == -1:
+            array_id = array_ids[selected.row()]
+            return (array_id, None)
+        else:
+            array_id = array_ids[parent.row()]
+            return (array_id, self.arrays[array_id][selected.row()])
+
+    def update_content_arrays(self):
+        self.load_content_from_arrayids_query_oar(self.arrays.iterkeys())
+
+    def update_content_jobs(self):
+        jobs = []
+
+        for jobs_list in self.arrays.itervalues():
+            jobs.extend(job_info['Job_Id'] for job_info in jobs_list)
+
+        self.load_content_from_jobids_query_oar(jobs)
+
+

zUtil_Python/oar/oar_api.py

@@ -8,7 +8,7 @@ import subprocess
 import getpass
 
 from yaml import load, load_all
-import cmd
+
 try:
     from yaml import CLoader as YamlLoader
 except ImportError as exc:
@@ -77,7 +77,7 @@ class OarAPI(object):
         elif len(jobs) == 0 and array is None:
             raise ValueError('OAR: no jobs or array specified')
         else:
-            self.backend.delete(jobs, array) 
+            self.backend.delete(jobs, array)
 
     def submit(self, command_path):
         """
@@ -88,6 +88,11 @@ class OarAPI(object):
         job_ids = [ elem['job_id'] for elem in output ]
         return {'array_id': job_ids[0], 'job_ids': job_ids}
 
+    def resubmit(self, prev_job_id):
+        """
+        job_id = resubmit(prev_job_id)
+        """
+        return self.backend.resubmit(prev_job_id)
 
 class OarBackendAPI(object):
 
@@ -100,9 +105,12 @@ class OarBackendAPI(object):
     def submit(self, command_path):
         raise NotImplementedError('Use one of the derived classes!')
 
+    def resubmit(self, prev_job_id):
+        raise NotImplementedError('Use one of the derived classes!')
+
 
 class OarBackendRestAPI(OarBackendAPI):
-    
+
     headers = {'content-type': 'text/yaml'}
 
     @staticmethod
@@ -114,7 +122,7 @@ class OarBackendRestAPI(OarBackendAPI):
             content = r.headers['content-type'].split("; ")
             if content[0] != OarBackendRestAPI.headers['content-type']:
                 raise HTTPError("Wrong content type returned from the server")
- 
+
             return True
         except (ConnectionError, HTTPError, URLError) as exc:
             print(exc)
@@ -137,7 +145,7 @@ class OarBackendRestAPI(OarBackendAPI):
         else:
             if array is not None:
                 params.update({"array" : array})
-            
+
             if full_details is True:
                 full_url = "/".join([self.base_url, "oarapi", "jobs", "details.yaml"])
             else:
@@ -145,7 +153,7 @@ class OarBackendRestAPI(OarBackendAPI):
 
             if username != "":
                 params.update({"user" : username})
-    
+
             r = requests.get(full_url, params=params, headers=self.headers, auth=self.auth)
             r.raise_for_status()
             return r.text
@@ -230,3 +238,18 @@ class OarBackendCmdAPI(OarBackendAPI):
 
         return '\n'.join(result)
 
+    def resubmit(self, prev_job_id):
+        cmd = ['oarsub', '--yaml', '--resubmit=%d' % prev_job_id]
+
+        output = self._call_oar(cmd)
+        print(output)
+
+        import re
+        matches = re.search("OAR_JOB_ID=.*", output)
+        if matches is None:
+            raise ValueError("Wrong output from RESUBMIT: %s" % output)
+
+        return int(matches.group().split("=")[1])
+
+
+