Compiled matlab functions / Batching: introduced XML based record, and faster matlab invocation

Signed-off-by: Nicola Vigano <nicola.vigano@esrf.fr>

zUtil_Conf/batch-dev-conf.xml

+<batch>
+  <options>
+    <binaries>
+      <path absolute="false">bin/compiled/</path>
+    </binaries>
+    <log>
+      <path absolute="true">/tmp_14_days/oar/log/</path>
+    </log>
+    <submit>
+      <path absolute="true">/tmp_14_days/oar/submit/</path>
+    </submit>
+  </options>
+  <functions>
+    <function name="gtDBBlob2SpotTable">
+      <path absolute="false">2_difspot/gtDBBlob2SpotTable.m</path>
+    </function>
+    <function name="gtSeedThreshold_doublethr">
+      <path absolute="false">2_difspot/gtSeedThreshold_doublethr.m</path>
+    </function>
+    <function name="gtSegmentDiffractionBlobs_doublethr">
+      <path absolute="false">2_difspot/gtSegmentDiffractionBlobs_doublethr.m</path>
+    </function>
+    <function name="gtCopyCorrectUndistortCondor">
+      <path absolute="false">1_preprocessing/gtCopyCorrectUndistortCondor.m</path>
+    </function>
+    <function name="gtChangeThreshold3D">
+      <path absolute="false">5_reconstruction/gtChangeThreshold3D.m</path>
+    </function>
+    <function name="gtSequenceMedianRefs">
+      <path absolute="false">1_preprocessing/gtSequenceMedianRefs.m</path>
+    </function>
+    <function name="gtCreateAbsLive">
+      <path absolute="false">1_preprocessing/gtCreateAbsLive.m</path>
+    </function>
+    <function name="gtAbsMedianLive">
+      <path absolute="false">1_preprocessing/gtAbsMedianLive.m</path>
+    </function>
+    <function name="gtMovingMedianLive">
+      <path absolute="false">1_preprocessing/gtMovingMedianLive.m</path>
+    </function>
+    <function name="gtCreateFullLive">
+      <path absolute="false">1_preprocessing/gtCreateFullLive.m</path>
+    </function>
+    <function name="gtSegmentDiffractionBlobs">
+      <path absolute="false">2_difspot/gtSegmentDiffractionBlobs.m</path>
+    </function>
+    <function name="gtReconstructGrains">
+      <path absolute="false">5_reconstruction/gtReconstructGrains.m</path>
+    </function>
+    <function name="gtForwardSimulate_v2">
+      <path absolute="false">5_reconstruction/gtForwardSimulate_v2.m</path>
+    </function>
+  </functions>
+</batch>

zUtil_Python/dct_batch.py

+'''
+Created on May 2, 2013
+
+@author: ben
+'''
+
+import re
+import fnmatch
+import os
+
+import dct_io_xml
+
+class DCTBatchConf(dct_io_xml.DCTXMLBase):
+
+    @staticmethod
+    def getNewFile(xml_file):
+        tree = dct_io_xml.ET.Element("batch")
+        etree = dct_io_xml.ET.ElementTree(tree)
+        etree.write(xml_file, pretty_print = True)
+        conf = DCTBatchConf(xml_file, read_only = False)
+        conf.fixStructure()
+        return conf
+
+    def __init__(self, xml_file = os.path.join("zUtil_Conf", "batch-dev-conf.xml"), read_only = True):
+        dct_io_xml.DCTXMLBase.__init__(self, xml_file)
+        self.read_only = read_only
+
+    def fixStructure(self):
+        if self.tree.find("options") is None:
+            dct_io_xml.ET.SubElement(self.tree.getroot(), "options")
+        if self.tree.find("functions") is None:
+            dct_io_xml.ET.SubElement(self.tree.getroot(), "functions")
+
+    def getPath(self, xml_path, base_dir, base_node = None):
+        if base_node is None:
+            node = self.tree.find(xml_path)
+        else:
+            node = base_node.find(xml_path)
+        if node is not None:
+            if node.attrib.get("absolute") == "false":
+                return os.path.join(base_dir, node.text)
+            else:
+                return node.text
+        else:
+            raise ValueError("No path defined for '%s" % xml_path)
+
+    def getBinDir(self, base_dir):
+        return self.getPath("options/binaries/path", base_dir)
+
+    def addFunction(self, name, path, is_absolute = False):
+        if self.read_only is True:
+            raise ValueError("File %s is open in read only mode" % self.xmlFile)
+
+        func = self.tree.find("functions/function[@name='%s']" % name)
+        if func is None:
+            funcs = self.tree.find("functions")
+            func = dct_io_xml.ET.SubElement(funcs, "function", {"name" : name})
+
+        pathNode = func.find("path")
+        if pathNode is None:
+            pathNode = dct_io_xml.ET.SubElement(func, "path")
+
+        pathNode.text = path
+        if is_absolute is True:
+            pathNode.attrib["absolute"] = "true"
+        else:
+            pathNode.attrib["absolute"] = "false"
+
+        self.save()
+
+    def getFunctions(self, base_dir):
+        functions = {}
+        nodes = self.tree.findall("functions/function")
+        for node in nodes:
+            functions[node.attrib.get("name")] = self.getPath("path", base_dir, base_node = node)
+        return functions
+
+class DCTBatch(object):
+
+    def __init__(self, dct_dir = "", dev_mode = False):
+        if dct_dir is "":
+            dct_dir = os.getcwd()
+        self.dct_dir = dct_dir
+
+        devel_conf_path = os.path.join(self.dct_dir, "zUtil_Conf", "batch-dev-conf.xml");
+        try:
+            self.dev_conf = DCTBatchConf(xml_file = devel_conf_path, read_only = (not dev_mode) )
+        except:
+            self.dev_conf = DCTBatchConf.getNewFile(devel_conf_path)
+
+        user_conf_path = os.path.join(self.dct_dir, "batch-user-conf.xml");
+        try:
+            self.user_conf = DCTBatchConf(xml_file = user_conf_path, read_only = False)
+            # We assume system configuration is always consistent, so we only
+            # check user configuration
+            self.user_conf.fixStructure()
+        except:
+            self.user_conf = DCTBatchConf.getNewFile(user_conf_path)
+
+    def autoDetectFunctions(self, user = True):
+        simpleMatch = 'OAR_make'
+        searchPattern = re.compile('[^%]*\s*' + simpleMatch + "\s*\(\s*'.*'")
+
+        filesList = []
+        for root, dir_name, file_names in os.walk(self.dct_dir):
+            for file_name in file_names:
+                if fnmatch.fnmatch(file_name, '*.m'):
+                    filesList.append((root, file_name))
+
+        # First creating list of file matching methods name
+        mFiles = {}
+        for dir_path, file_name in filesList:
+            filepath = os.path.join(dir_path, file_name)
+            mFiles[file_name[:-2]] = filepath
+
+        # Then looking for complex regexp inside matching files
+        for dir_path, file_name in mFiles.iteritems():
+            filepath = os.path.join(dir_path, file_name)
+            fid = open(filepath, 'r')
+
+            for line in fid:
+                if simpleMatch in line:
+                    matched = searchPattern.search(line)
+                    if matched:
+                        function_name = matched.group().split("'")[1]
+                        self.addFunction(mFiles[function_name], user)
+
+            fid.close()
+
+    def addFunction(self, path, user = True):
+        name = os.path.basename(path)
+        name = name[0:-2]
+
+        # Let's make the path relative
+        abs_path = os.path.abspath(path)
+        abs_dct_path = os.path.abspath(self.dct_dir)
+        rel_path = abs_path[len(abs_dct_path)+1:]
+
+        if user is True:
+            self.user_conf.addFunction(name, rel_path)
+        else:
+            self.dev_conf.addFunction(name, rel_path)
+
+    def getBinDir(self):
+        try:
+            bin_path = self.user_conf.getBinDir(self.dct_dir)
+        except ValueError:
+            bin_path = self.dev_conf.getBinDir(self.dct_dir)
+        return bin_path
+
+    def getFunctions(self):
+        functions = self.dev_conf.getFunctions(self.dct_dir)
+        functions.update(self.user_conf.getFunctions(self.dct_dir))
+        return functions
+
+if __name__ == '__main__':
+    pass
+

zUtil_Python/dct_compile_matlab_functions.py

 #!/usr/bin/python
 
-import re
-import fnmatch
 import os
 import sys
 
@@ -11,10 +9,6 @@ import dct_matlab_invocation
 
 class FunctionsBuilder(object):
 
-    submitMethods = [ 'OAR_make',
-                    #  'condor_make' # Disabled since we moved to OAR
-                    ]
-
     @staticmethod
     def getInstanceFromArgs(args, dct_dir = ""):
         if dct_dir is "":
@@ -33,6 +27,7 @@ class FunctionsBuilder(object):
         will_compile = True
         will_generate = True
         force_compile = False
+        scan_sources = False
         verbose = 1
 
         mfiles_to_consider = [ ]
@@ -47,6 +42,8 @@ class FunctionsBuilder(object):
                 elif args[index] == "-force-compile":
                     will_compile = True
                     force_compile = True
+                elif args[index] == "-scan-sources":
+                    scan_sources = True
                 elif args[index] == "-c":
                     skip_next = True
                     if len(args) > (index-1):
@@ -74,12 +71,13 @@ class FunctionsBuilder(object):
                                 mfiles_to_consider = mfiles_to_consider, \
                                 do_compile = will_compile, \
                                 do_generate = will_generate, \
+                                scan_sources = scan_sources, \
                                 verboseLevel = verbose)
 
     def __init__(self, matlab_path, dct_dir = os.getcwd(), \
                  script_dir = None, bin_dir = None, force_compile = False, \
                  mfiles_to_consider = [], do_compile = True, \
-                 do_generate = True, verboseLevel = 1):
+                 do_generate = True, scan_sources = False, verboseLevel = 1):
         self.dct_dir = dct_dir
         if script_dir is None:
             script_dir = os.path.join(self.dct_dir, 'bin', 'scripts')
@@ -95,9 +93,8 @@ class FunctionsBuilder(object):
 
         self.do_compile = do_compile
         self.do_generate = do_generate
+        self.scan_sources = scan_sources
 
-        self.call_funcs = { }
-        self.m_files = { }
         self.compile_records = { }
 
         self.out = DCTOutput(verboseLevel = verboseLevel);
@@ -114,69 +111,23 @@ class FunctionsBuilder(object):
         print("  -force-compile : to force the compilation even if not needed")
         print("  -just-compile : to skip the generation of the script")
         print("  -just-generate : to skip the actual compilation")
-
-    def _findFuncs(self):
-        searchPatterns = [ ]
-        for method in FunctionsBuilder.submitMethods:
-            regexpString = '[^%]*\s*' + method + "\s*\(\s*'.*'"
-            #print('Regular expression: "' + regexpString + '"')
-            regexp = re.compile(regexpString)
-            searchPatterns.append(regexp)
-
-        self.out.printSubJob("Finding functions that may be compiled, from: '%s'" % self.dct_dir)
-        filesList = []
-        for root, dir_name, file_names in os.walk(self.dct_dir):
-            for file_name in file_names:
-                if fnmatch.fnmatch(file_name, '*.m'):
-                    filesList.append((root, file_name))
-
-        # First creating list of file matching methods name
-        matchingFilesList = []
-        for dir_path, file_name in filesList:
-            filepath = os.path.join(dir_path, file_name)
-            self.m_files[file_name[:-2]] = filepath
-            for method in FunctionsBuilder.submitMethods:
-                if method in open(filepath).read():
-                    matchingFilesList.append((dir_path, file_name))
-                    break
-
-        # Then looking for complex regexp inside matching files
-        for dir_path, file_name in matchingFilesList:
-            filepath = os.path.join(dir_path, file_name)
-            fid = open(filepath, 'r')
-            filecontent = fid.readlines()
-            fid.close()
-
-            for line in filecontent:
-                for reg in searchPatterns:
-                    matched = reg.search(line)
-                    if matched:
-                        function_name = matched.group().split("'")[1]
-                        if filepath in self.call_funcs:
-                            if function_name not in self.call_funcs[filepath]:
-                                self.call_funcs[filepath].append(function_name)
-                        else:
-                            self.call_funcs[filepath] = [function_name]
-
-        self.out.printSubJob("These functions were found:")
-        for file_name in self.call_funcs:
-            self.out.printSubSubJob('In file', file_name)
-            for func in self.call_funcs[file_name]:
-                self.out.printSubSubSubJob("Function", func)
-        print('')
-
-    def _prepareFuncsList(self):
-        self.out.printSubJob("Adding information about functions that may not need to be compiled..")
-        for caller in self.call_funcs:
-            callerFuncName = os.path.basename(caller)[:-2]
-            for function in self.call_funcs[caller]:
-                if function not in self.compile_records.keys():
-                    outpath = os.path.join(self.bin_dir, function)
-                    try:
-                        self.compile_records[function] = (outpath, self.m_files[function])
-                    except KeyError:
-                        DCTOutput.printWarning("-> function: '%' is NOT a local function (may be from ID19) <-" % function)
-                        self.compile_records[callerFuncName][function] = (' ',' ')
+        print("  -scan-sources : to scan the matlab sources")
+
+    def _buildFuncsList(self):
+        import dct_batch
+        batch = dct_batch.DCTBatch(self.dct_dir)
+        if self.scan_sources is True:
+            self.out.printSubJob("Finding functions that may be compiled, from: '%s'" % self.dct_dir)
+            batch.autoDetectFunctions()
+        self.out.printSubJob("Functions available for compilation:")
+        funcs = batch.getFunctions()
+        for function in funcs:
+            self.out.printSubSubJob(function, funcs[function])
+        bin_dir = batch.getBinDir()
+        self.out.printSubJob("Adding information about functions..")
+        for function in funcs:
+            outpath = os.path.join(bin_dir, function)
+            self.compile_records[function] = (outpath, funcs[function])
 
     def _generateMCompileFile(self):
         if not os.path.exists(self.script_dir):
@@ -229,8 +180,7 @@ class FunctionsBuilder(object):
         print("")
 
     def generateMfile(self):
-        self._findFuncs()
-        self._prepareFuncsList()
+        self._buildFuncsList()
         self._generateMCompileFile()
 
     def buildMatlabFunctions(self):

zUtil_Python/dct_distrib.py

@@ -15,6 +15,7 @@ class DCTMakeInstallBundle(object):
                             'dct_utils_git.py', \
                             'dct_utils_platform.py', \
                             'dct_matlab_invocation.py', \
+                            'dct_batch.py', \
                             'dct_setup.py')
 
     def __init__(self, *args, **kwargs):

zUtil_Python/dct_io_xml.py

@@ -65,7 +65,7 @@ class DCTXMLBase(object):
     def save(self, newName = None):
         if newName is not None:
             self.xmlFile = newName
-        self.tree.write(self.xmlFile)
+        self.tree.write(self.xmlFile, pretty_print = True)
 
 class DCTCheck(object):