Refactored merge.

xsocs/examples/id01_merge.py

@@ -28,10 +28,9 @@ __authors__ = ["D. Naudet"]
 __date__ = "20/04/2016"
 __license__ = "MIT"
 
-import os
 import time
 
-from xsocs.util.id01_spec import merge_scan_data
+from xsocs.process.merge import merge_scan_data
 
 # output directory (some temporary files will also be written there)
 output_dir = '/path/to/output/'
@@ -64,6 +63,12 @@ scan_ids = None
 # number, while in version 1 we pad the value to to 5 digits)
 version = 1
 
+# you can also set the offset and padding with the nr_offset and nr_padding
+# keywords to the merge_scan_data function, or ignore them if you want
+#  to use the default values specified by "version".
+# nr_offset = -1
+# nr_padding = 4
+
 # channels (pix.) per degree (used by xrayutilities when converting to
 # qspace coordinates)
 # (not that this value can also be changed later when calling the
@@ -73,12 +78,6 @@ chan_per_deg = [318., 318.]
 # direct beam position in the detector coordinates
 center_chan = [140, 322]
 
-# the merge will actually create one file per scan, then a "master" file
-# (in the output directory) that will contain links to those files. You can
-# give the master file the name you want (if None, a name will be generated
-# with the prefix found in the spec file)
-master_f = None
-
 # checks if some of the output files already exist
 # set it to True if you dont care about overwriting files
 overwrite = False
@@ -86,15 +85,14 @@ overwrite = False
 t_merge = time.time()
 
 merge_scan_data(output_dir,
-                    spec_f,
-                    beam_energy=beam_energy,
-                    chan_per_deg=chan_per_deg,
-                    center_chan=center_chan,
-                    scan_ids=scan_ids,
-                    master_f=master_f,
-                    img_dir=img_base,
-                    version=version,
-                    overwrite=overwrite)
+                spec_f,
+                beam_energy=beam_energy,
+                chan_per_deg=chan_per_deg,
+                center_chan=center_chan,
+                scan_ids=scan_ids,
+                img_dir=img_base,
+                version=version,
+                overwrite=overwrite)
 
 t_merge = time.time() - t_merge
 print('Total time spent : {0}'.format(t_merge))

xsocs/gui/process/MergeWidget.py

@@ -35,7 +35,8 @@ import shutil
 from types import MethodType
 from functools import partial
 
-from ...util.id01_spec import Id01DataMerger
+from ...process.merge import KmapSpecParser, KmapMerger
+
 from ..widgets.AcqParamsWidget import AcqParamsWidget
 
 from ..widgets.Input import StyledLineEdit
@@ -210,7 +211,7 @@ class _ScansSelectDialog(Qt.QDialog):
                 item = self.__table_widget.item(row, self.SEL_COL)
                 item.setCheckState(Qt.Qt.Unchecked)
 
-    def __showMore(self, *args, **kwargs):
+    def __showMore(self):
         if self.__more_bn.text() == 'More':
             self.__more_bn.setText('Less')
             hide = False
@@ -286,7 +287,7 @@ class _ScansInfoDialog(Qt.QDialog):
 
 
 class _MergeProcessDialog(Qt.QDialog):
-    __sigMergeDone = Qt.Signal(object)
+    __sigMergeDone = Qt.Signal()
 
     def __init__(self, merger, **kwargs):
         super(_MergeProcessDialog, self).__init__(**kwargs)
@@ -351,7 +352,7 @@ class _MergeProcessDialog(Qt.QDialog):
         self.__merger = merger
         self.__status = False
 
-    def __onAccept(self, *args, **kwargs):
+    def __onAccept(self):
         merger = self.__merger
         warn = merger.check_overwrite()
 
@@ -383,7 +384,7 @@ class _MergeProcessDialog(Qt.QDialog):
 
         self.__time = time.time()
 
-    def __onAbort(self, *args, **kwargs):
+    def __onAbort(self):
         abort_diag = Qt.QMessageBox(Qt.QMessageBox.Information,
                                     'Aborting...',
                                     '<b>Cancelling merge.</b>'
@@ -394,11 +395,11 @@ class _MergeProcessDialog(Qt.QDialog):
         abort_diag.setAttribute(Qt.Qt.WA_DeleteOnClose)
         abort_diag.setStandardButtons(Qt.QMessageBox.NoButton)
         abort_diag.show()
-        self.__merger.abort_merge(wait=False)
+        self.__merger.abort(wait=False)
 
-    def __mergeDone(self, result):
+    def __mergeDone(self):
         print('TOTAL : {0}.'.format(time.time() - self.__time))
-        self.__status = result[0]
+        self.__status = self.__merger.status
         self.__qtimer.stop()
         self.__qtimer = None
         self.__onProgress()
@@ -412,8 +413,8 @@ class _MergeProcessDialog(Qt.QDialog):
         else:
             self.__bn_box.rejected.connect(self.reject)
 
-    def __onProgress(self, *args, **kwargs):
-        progress = self.__merger.merge_progress()
+    def __onProgress(self):
+        progress = self.__merger.progress()
         if progress is None:
             return
         tree_wid = self.__tree_widget
@@ -443,7 +444,7 @@ class MergeWidget(Qt.QDialog):
                  'offset': [-1, 0]}
     _defaultVersion = 1
 
-    __sigParseDone = Qt.Signal()
+    __sigParsed = Qt.Signal()
 
     def __init__(self,
                  spec_file=None,
@@ -748,6 +749,7 @@ class MergeWidget(Qt.QDialog):
         parseBn.setEnabled(False)
 
         self.__merger = None
+        self.__parser = None
         self.info_wid = None
 
         if tmp_dir is None:
@@ -768,8 +770,6 @@ class MergeWidget(Qt.QDialog):
 
         print('Using temporary folder : {0}.'.format(tmp_dir))
 
-        self.__sigParseDone.connect(self.__parseSpecDone)
-
         self.__widgetIsSetup = False
 
         self.__xsocs_h5 = None
@@ -780,6 +780,8 @@ class MergeWidget(Qt.QDialog):
 
         self.__resetState()
 
+        self.__sigParsed.connect(self.__slotParsed, Qt.Qt.QueuedConnection)
+
     def showEvent(self, *args, **kwargs):
         if not self.__widgetIsSetup:
             self.__widgetIsSetup = True
@@ -887,15 +889,15 @@ class MergeWidget(Qt.QDialog):
 
             name = 'Prefix'
             prefix = self.__output['prefix']
-            if len(prefix) == 0:
+            if not prefix:
                 raise ValueError('parameter is mandatory.')
-            merger.prefix = prefix
+            merger.prefix = str(prefix)
 
             name = 'Output directory'
             outDir = self.__output['outdir']
-            if len(outDir) == 0:
+            if not outDir:
                 raise ValueError('parameter is mandatory.')
-            merger.output_dir = outDir
+            merger.output_dir = str(outDir)
 
         except Exception as ex:
             Qt.QMessageBox.critical(self, 'Error',
@@ -917,15 +919,16 @@ class MergeWidget(Qt.QDialog):
 
         process_diag = _MergeProcessDialog(merger, parent=self)
         process_diag.setAttribute(Qt.Qt.WA_DeleteOnClose)
-        process_diag.accepted.connect(partial(self.__mergeDone, status=True))
-        process_diag.rejected.connect(partial(self.__mergeDone, status=False))
+        process_diag.accepted.connect(partial(self.__mergeDone))
+        process_diag.rejected.connect(partial(self.__mergeDone))
         process_diag.setModal(True)
         self.__process_diag = process_diag
         process_diag.show()
 
-    def __mergeDone(self, status):
+    def __mergeDone(self):
         self.__process_diag = None
-        if status:
+        status = self.__merger.status
+        if status == KmapMerger.DONE:
             self.__xsocs_h5 = self.__merger.master_file
             self.accept()
 
@@ -1025,7 +1028,7 @@ class MergeWidget(Qt.QDialog):
         if merger is None:
             enable = False
         else:
-            enable = merger.parsed and len(merger.matched_ids) > 0
+            enable = len(merger.matched_ids) > 0
 
         self.__outputGbx.setEnabled(enable)
         if not enable:
@@ -1067,20 +1070,16 @@ class MergeWidget(Qt.QDialog):
         padding = self.__input['padding']
         offset = self.__input['offset']
 
+        spec_h5 = os.path.join(self.__tmp_dir_merge, 'temp_spec.h5')
+
         try:
-            if self.__merger is None:
-                self.__merger = Id01DataMerger(specFile,
-                                               self.__tmp_dir_merge,
-                                               img_dir=imgDir,
-                                               # version=version,
-                                               nr_offset=offset,
-                                               nr_padding=padding)
-            else:
-                self.__merger.reset(str(specFile),
-                                    img_dir=imgDir,
-                                    # version=version,
-                                    nr_offset=offset,
-                                    nr_padding=padding)
+            self.__parser = KmapSpecParser(specFile,
+                                           spec_h5,
+                                           img_dir=imgDir,
+                                           # version=version,
+                                           nr_offset=offset,
+                                           nr_padding=padding,
+                                           callback=self.__sigParsed.emit)
         except Exception as ex:
             msg = ('Parsing failed: {0}.\n'
                    'Message : {1}.'
@@ -1104,10 +1103,9 @@ class MergeWidget(Qt.QDialog):
         info_wid.setStandardButtons(Qt.QMessageBox.NoButton)
         info_wid.show()
         self.info_wid = info_wid
-        self.__merger.parse(blocking=False,
-                            callback=self.__sigParseDone.emit)
+        self.__parser.parse(blocking=False)
 
-    def __parseSpecDone(self, *args, **kwargs):
+    def __slotParsed(self):
 
         self.info_wid.done(0)
         self.info_wid = None
@@ -1115,6 +1113,15 @@ class MergeWidget(Qt.QDialog):
         self.__parseBn.setEnabled(True)
         self.__parseBn.setText('Parse file.')
 
+        self.__merger = KmapMerger(self.__parser.results.spec_h5,
+                                   self.__parser.results,
+                                   output_dir=self.__output['outdir'])
+
+        self.__output['prefix'] = self.__merger.prefix
+        blocked = self.__prefixEdit.blockSignals(True)
+        self.__prefixEdit.setText(self.__merger.prefix)
+        self.__prefixEdit.blockSignals(blocked)
+
         self.__updateScansInfos()
         self.__updateOutputGroupBox()
 
@@ -1137,7 +1144,7 @@ class MergeWidget(Qt.QDialog):
             selected_ids = merger.selected_ids
             no_match_ids = merger.no_match_ids
             no_img_ids = merger.no_img_ids
-            enable = merger.parsed
+            enable = True
             prefix = merger.prefix
 
         nTotal = len(matched_ids)

xsocs/process/merge/KmapSpecParser.py

+#!/usr/bin/python
+# coding: utf8
+# /*##########################################################################
+#
+# Copyright (c) 2015-2016 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+
+__authors__ = ["D. Naudet"]
+__date__ = "20/04/2016"
+__license__ = "MIT"
+
+import re
+import glob
+import os.path
+from collections import namedtuple
+
+from threading import Thread
+
+import h5py
+from silx.io import spectoh5
+
+
+# regular expression matching the imageFile comment line
+_IMAGEFILE_LINE_PATTERN = ('^#C imageFile '
+                           'dir\[(?P<dir>[^\]]*)\] '
+                           'prefix\[(?P<prefix>[^\]]*)\] '
+                           '(idxFmt\[(?P<idxFmt>[^\]]*)\] ){0,1}'
+                           'nextNr\[(?P<nextNr>[^\]]*)\] '
+                           'suffix\[(?P<suffix>[^\]]*)\]$')
+
+
+# #######################################################################
+# #######################################################################
+
+
+KmapParseResults = namedtuple('KmapParseResults',
+                              ['img_info', 'no_img_info', 'error', 'spec_h5'])
+
+KmapMatchResults = namedtuple('KmapMatchResults',
+                              ['matched', 'not_matched',
+                               'no_img_info', 'error',
+                               'spec_h5'])
+
+
+# #######################################################################
+# #######################################################################
+
+
+class KmapSpecParser(object):
+    (READY, RUNNING, DONE,
+     ERROR, CANCELED, UNKNOWN) = __STATUSES = range(6)
+    """ Available status codes """
+
+    status = property(lambda self: self.__status)
+    """ Current status code of this instance """
+
+    statusMsg = property(lambda self: self.__status_msg)
+    """ Status message if any, or None """
+
+    results = property(lambda self: self.__results)
+    """ Parse results. KmapParseResults instance. """
+
+    def __init__(self,
+                 spec_fname,
+                 spec_h5,
+                 img_dir=None,
+                 version=1,
+                 nr_padding=None,
+                 nr_offset=None,
+                 callback=None):
+        """
+        Parser for the Kmap SPEC files. This loads a spec file, converts
+            it to HDF5 and then tries to match scans and edf image files.
+        :param spec_fname: Path to the spec file.
+        :param spec_h5: Name of the HDF5 file that will be created and filled
+            with the contents of the SPEC file
+        .. seealso : silx.io.convert_spec_h5.convert
+        :param img_dir: directory where the images are stored, if they are not
+        stored where the path (#C imageFile line) in the SPEC file points to.
+        :param version: version of the spec file. It is currently used to get
+            the offset and padding to apply to the nextNr value found in the
+            spec scan headers. This nextNr is then used to generate the image
+            file name. Set it to 0 if you are merging data generated
+            before April 2016 (TBC).
+        :param nr_padding: zero padding to apply to the nextNr number found
+            in the SPEC file.
+        :param nr_offset: offset to apply to the nextNr number found
+            in the SPEC file.
+        :param callback: callback to call when the parsing is done
+        """
+        super(KmapSpecParser, self).__init__()
+
+        self.__status = None
+
+        self.__set_status(self.UNKNOWN, 'Init')
+
+        self.__results = None
+
+        self.__thread = None
+
+        self.__spec_fname = spec_fname
+        self.__spec_h5 = spec_h5
+        self.__callback = callback
+        self.__img_dir = img_dir
+        self.__version = version
+
+        self.__nr_padding = nr_padding
+        self.__nr_offset = nr_offset
+
+        self.__set_status(self.READY)
+
+    def __set_status(self, status, msg=None):
+        """
+        Sets the status of this instance.
+        :param status:
+        :param msg:
+        :return:
+        """
+        assert status in self.__STATUSES
+        self.__status = status
+        self.__status_msg = msg
+
+    def parse(self, blocking=True):
+        """
+        Starts the parsing.
+        :param blocking: if False, the parsing will be done in a separate
+         thread and this method will return immediately.
+        :return:
+        """
+
+        if self.__thread and self.__thread.is_alive():
+            raise RuntimeError('This KmapSpecParser instance is already '
+                               'parsing.')
+
+        self.__results = None
+
+        if blocking:
+            self.__run_parse()
+            if self.__callback:
+                self.__callback()
+        else:
+            thread = self.__thread = Thread(target=self.__run_parse)
+            thread.start()
+
+    def __run_parse(self):
+        """
+        Runs the parsing.
+        :return:
+        """
+
+        self.__set_status(self.RUNNING)
+
+        try:
+            _spec_to_h5(self.__spec_fname, self.__spec_h5)
+
+            self.__results = _find_scan_img_files(self.__spec_h5,
+                                                  img_dir=self.__img_dir,
+                                                  version=self.__version,
+                                                  nr_padding=self.__nr_padding,
+                                                  nr_offset=self.__nr_offset)
+            self.__set_status(self.DONE)
+        except Exception as ex:
+            self.__set_status(self.ERROR, str(ex))
+
+        # TODO : catch exception?
+        if self.__callback:
+            self.__callback()
+
+        return self.__results
+
+    def wait(self):
+        """
+        Waits until parsing is done, or returns if it is not running.
+        :return:
+        """
+        if self.__thread:
+            self.__thread.join()
+
+
+# #######################################################################
+# #######################################################################
+
+
+def _spec_to_h5(spec_filename, h5_filename):
+    """
+    Converts a spec file into a HDF5 file.
+
+    :param spec_filename: name of the spec file to convert to HDF5.
+    :type spec_filename: str
+
+    :param h5_filename: name of the HDF5 file to create.
+    :type h5_filename: str
+
+    .. seealso : silx.io.convert_spec_h5.convert
+    """
+    spectoh5.convert(spec_filename,
+                     h5_filename,
+                     mode='w')
+
+
+# ########################################################################
+# ########################################################################
+
+
+def _spec_get_img_filenames(spec_h5_filename):
+    """
+    Parsed spec scans headers to retrieve the associated image files.
+
+    :param spec_h5_filename: name of the HDF5 file containing spec data.
+    :type spec_h5_filename: str
+
+    .. todo : can we suppose that there's
+        only one scan per scan number?
+        i.e : no 0.2, 1.2, ...
+    .. todo : expecting only one imageFile comment line per scan. Is this
+        always true?
+
+    :return: 3 elements tuple : a dict containing the scans that have valid
+        image file info, a list with the scans that dont have any image files,
+        and a list of the scans that have more that one image files.
+    :rtype: *list* (*dict*, *list*, *list*)
+    """
+    with h5py.File(spec_h5_filename, 'r') as h5_f:
+
+        # scans for which a file name was found
+        with_file = {}
+
+        # scans for which no file name was found
+        without_file = []
+
+        # scans for which more than one file name was found
+        # -> this case is not expected/handled atm
+        error_scans = []
+
+        # regular expression to find the imagefile line
+        regx = re.compile(_IMAGEFILE_LINE_PATTERN)
+
+        for k_scan, v_scan in h5_f.items():
+            header = v_scan['instrument/specfile/scan_header'][()]
+            header = header.decode().split('\n')
+            imgfile_match = [m for line in header
+                             if line.startswith('#C imageFile')
+                             for m in [regx.match(line.strip())] if m]
+
+            # TODO : provide some more info
+            # expecting only one imagefile line per scan
+            if len(imgfile_match) > 1:
+                error_scans.append(k_scan)
+                continue
+
+            # if no imagefile line
+            if len(imgfile_match) == 0:
+                without_file.append(k_scan)
+                continue
+
+            # extracting the named subgroups
+            imgfile_grpdict = imgfile_match[0].groupdict()
+            with_file[k_scan] = imgfile_grpdict
+
+        return KmapParseResults(img_info=with_file,
+                                no_img_info=without_file,
+                                error=error_scans,
+                                spec_h5=spec_h5_filename)
+
+
+# ########################################################################
+# ########################################################################
+
+
+def _find_scan_img_files(spec_h5_filename,
+                         img_dir=None,
+                         version=1,
+                         nr_padding=None,
+                         nr_offset=None):
+    """
+    Parses the provided "*spec*" HDF5 file and tries to find the edf file
+    associated  with each scans. will look for the files in img_dir if
+    provided (instead of looking for the files in the path written
+    in the spec file).
+
+    :param spec_h5_filename: name of the HDF5 file containing spec data.
+    :type spec_h5_filename: str
+
+    :param img_dir: directory path. If provided the image files will be
+        looked for into that folder instead of the one found in the scan
+        headers.
+    :type img_dir: *optional* str
+
+    :param version: version of the spec file. It is currently used to get
+    the offset and padding to apply to the nextNr value found in the spec scan
+    headers. This nextNr is then used to generate the image file name. Set it
+    to 0 if you are merging data generated before April 2016 (TBC).
+    :type img_dir: *optional* int
+
+    :returns: 4 elements tuple : a dict containing the scans whose image file
+        has been found, a dict containing the scans that have that have
+        valid image file info in the scan header but whose image file has not
+        been found, a list with the scans that dont have any image file info,
+        and a list of the scans that have more that one image file info line.
+    :rtype: *list* (*dict*, *dict*, *list*, *list*)
+    """
+
+    if img_dir and not os.path.exists(img_dir):
+        raise ValueError('Image folder not found : {0}'
+                         ''.format(img_dir))
+
+    imgfile_info = _spec_get_img_filenames(spec_h5_filename)
+    with_files = imgfile_info.img_info
+    complete_scans = {}
+    incomplete_scans = {}
+
+    if version == 0:
+        nextnr_ofst = -1
+        nextnr_pattern = '{0:0>4}'
+    else:
+        nextnr_ofst = 0
+        nextnr_pattern = '{0:0>5}'
+
+    if nr_padding is not None: