rework 'guess_index_frm_file_name' to use basename instead of numeric directly.

Manage several ref and serie of ref

tomoscan/esrf/edfscan.py

@@ -31,6 +31,7 @@ __date__ = "10/10/2019"
 import os
 import re
 import fabio
+import copy
 from lxml import etree
 import json
 import io
@@ -258,10 +259,11 @@ class EDFTomoScan(TomoScanBase):
             for f in os.listdir(scan):
                 if EDFTomoScan.is_a_proj_path(fileName=f, scanID=scan):
                     gfile = os.path.join(scan, f)
-                    index = EDFTomoScan.guess_index_frm_file_name(gfile)
+                    index = EDFTomoScan.guess_index_frm_file_name(gfile,
+                                                                  basename=os.path.basename(scan))
                     urls.update(extract_urls_from_edf(start_index=index,
-                                                       file_=gfile,
-                                                       n_frames=n_frames))
+                                                      file_=gfile,
+                                                      n_frames=n_frames))
 
         return urls
 
@@ -289,18 +291,48 @@ class EDFTomoScan(TomoScanBase):
         return False
 
     @staticmethod
-    def guess_index_frm_file_name(_file: str) -> Union[None, int]:
-        name = _file.rstrip('.edf')
-        ic = []
-        while name[-1].isdigit():
-            ic.append(name[-1])
-            name = name[:-1]
-
-        if len(ic) is 0:
-            return None
+    def guess_index_frm_file_name(_file: str, basename: str) -> Union[None, int]:
+        """
+        Guess the index of the file. Index is most of the an integer but can be
+        a float for 'ref' for example if several are taken.
+
+        :param _file:
+        :param basename:
+        """
+
+        def extract_index(my_str, type_):
+            res = []
+            modified_str = copy.copy(my_str)
+
+            while modified_str[-1].isdigit():
+                res.append(modified_str[-1])
+                modified_str = modified_str[:-1]
+            if len(res) == 0:
+                return None, modified_str
+            else:
+                orignalOrder = res[::-1]
+                if type_ is int:
+                    return int(''.join(orignalOrder)), modified_str
+                else:
+                    return float('.'.join(('0', ''.join(orignalOrder)))), modified_str
+
+        if _file.endswith('.edf'):
+            name = _file.replace(basename, '', 1)
+            name = name.rstrip('.edf')
+
+            part_1, name = extract_index(name, type_=int)
+            if name.endswith('_'):
+                name = name.rstrip('_')
+                part_2, name = extract_index(name, type_=float)
+            else:
+                part_2 = None
+
+            if part_2 is None:
+                return int(part_1)
+            else:
+                return float(part_1) + part_2
         else:
-            orignalOrder = ic[::-1]
-            return int(''.join(orignalOrder))
+            raise ValueError('only edf files are managed')
 
     @staticmethod
     def get_tomo_n(scan: str) -> Union[None, int]:
@@ -410,7 +442,8 @@ class EDFTomoScan(TomoScanBase):
                 # darkHSTXXXX remains...
                 file_fp = file_.lstrip(_prefix).rstrip(file_ext).lstrip('HST')
                 if file_fp == '' or file_fp.isnumeric() is True:
-                    index = EDFTomoScan.guess_index_frm_file_name(file_)
+                    index = EDFTomoScan.guess_index_frm_file_name(_file=file_,
+                                                                  basename=os.path.basename(scan_path))
                     urls = extract_urls_from_edf(os.path.join(scan_path, file_),
                                                  start_index=index)
                     res.update(urls)
@@ -437,7 +470,8 @@ class EDFTomoScan(TomoScanBase):
 
         for file_ in os.listdir(scan_path):
             if file_.startswith(prefix) and file_.endswith(file_ext):
-                index = EDFTomoScan.guess_index_frm_file_name(file_)
+                index = EDFTomoScan.guess_index_frm_file_name(_file=file_,
+                                                              basename=os.path.basename(scan_path))
                 file_fp = os.path.join(scan_path, file_)
                 urls = extract_urls_from_edf(start_index=index, file_=file_fp)
                 res.update(urls)

tomoscan/esrf/test/test_edfscan.py

@@ -37,6 +37,7 @@ from tomoscan.esrf.mock import MockEDF
 from tomoscan.esrf.edfscan import EDFTomoScan
 from tomoscan.scanbase import TomoScanBase
 from tomoscan.scanfactory import ScanFactory
+from silx.io.url import DataUrl
 import collections
 import json
 import shutil
@@ -126,6 +127,7 @@ class TestDarksFlats(unittest.TestCase):
         )
 
     def testFlats(self):
+        """test that refHST flats are correctly found"""
         # check one flat file with two ref
         self.assertEqual(len(self.acquiOneFlat.flats), 1)
         self.assertTrue(isinstance(self.acquiOneFlat.flats[0], silx.io.url.DataUrl))
@@ -145,6 +147,81 @@ class TestDarksFlats(unittest.TestCase):
         self.assertTrue(12 not in self.acquiTwoFlats.flats)
 
 
+class TestOriDarksFlats(unittest.TestCase):
+    """
+    Test that original ref* and dark* before creation of refHST  and darkHST
+    are correctly found
+    """
+    def setUp(self) -> None:
+        unittest.TestCase.setUp(self)
+        self.pixel_size = 0.05
+        self.det_width = 10
+        self.det_height = 10
+
+        self.folder = tempfile.mkdtemp()
+        # create 2 series of ref with 20 frames each
+        for ref_index in ('0000', '1500'):
+            for ref_sub_index in range(20):
+                file_desc = fabio.edfimage.EdfImage(data=numpy.random.random((self.det_height,self.det_width)))
+                file_name = '_'.join(('ref', str(ref_sub_index).zfill(4), str(ref_index).zfill(4) + '.edf'))
+                file_name = os.path.join(self.folder, file_name)
+                file_desc.write(file_name)
+        # create one dark with 10 frames
+        file_desc = fabio.edfimage.EdfImage(
+            data=numpy.random.random((self.det_height, self.det_width)))
+        for frame in range(9):
+            file_desc.append_frame(data=numpy.random.random((self.det_height, self.det_width)))
+        self._dark_file_name = os.path.join(self.folder, 'darkend0000.edf')
+        file_desc.write(self._dark_file_name)
+
+        self.scan = EDFTomoScan(self.folder)
+        # create a .info for mocking the acquisition
+        self.write_metadata(n_radio=0, scan_range=360, ref_n=20, dark_n=1)
+
+    def get_info_file(self):
+        return os.path.join(self.scan.path,
+                            os.path.basename(self.scan.path) + '.info')
+
+    def write_metadata(self, n_radio, scan_range, ref_n, dark_n):
+        info_file = self.get_info_file()
+        if not os.path.exists(info_file):
+            # write the info file
+            with open(self.get_info_file(), 'w') as info_file:
+                info_file.write('TOMO_N=    ' + str(n_radio) + '\n')
+                info_file.write('ScanRange= ' + str(scan_range) + '\n')
+                info_file.write('REF_N=     ' + str(ref_n) + '\n')
+                info_file.write('REF_ON=    ' + str(n_radio) + '\n')
+                info_file.write('DARK_N=    ' + str(dark_n) + '\n')
+                info_file.write('Dim_1=     ' + str(self.det_width) + '\n')
+                info_file.write('Dim_2=     ' + str(self.det_height) + '\n')
+                info_file.write('Col_beg=    0' + '\n')
+                info_file.write('Col_end=   ' + str(self.det_width) + '\n')
+                info_file.write('Row_beg=    0' + '\n')
+                info_file.write('Row_end=    ' + str(self.det_height) + '\n')
+                info_file.write('PixelSize=  ' + str(self.pixel_size) + '\n')
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.folder)
+        unittest.TestCase.tearDown(self)
+
+    def testFlatsOri(self):
+        ref_urls = (self.scan.get_refs_url(prefix='ref', scan_path=self.scan.path))
+        self.assertEqual(len(ref_urls), 40)
+        # checks some random files
+        file_paths = [url.file_path() for url in ref_urls.values()]
+        for ref_file in ('ref_0000_1500.edf', 'ref_0004_0000.edf', 'ref_0019_1500.edf'):
+            ref_full_path = os.path.join(self.scan.path, ref_file)
+            self.assertTrue(ref_full_path in file_paths)
+
+    def testDarksOri(self):
+        darks = self.scan.get_darks_url(prefix='darkend',
+                                        scan_path=self.scan.path)
+        self.assertEqual(len(darks), 10)
+        # check one random url
+        url = DataUrl(file_path=self._dark_file_name,  data_slice=[4,], scheme='fabio')
+        self.assertTrue(url in darks.values())
+
+
 class TestProjections(unittest.TestCase):
     """Test that the """
     def setUp(self) -> None:
@@ -379,7 +456,7 @@ def suite():
     test_suite = unittest.TestSuite()
     for ui in (TestScanFactory, TestDarksFlats,
                TestScanValidatorFindFiles, TestRadioPath, TestProjections,
-               TestTomoBaseHashable):
+               TestTomoBaseHashable, TestOriDarksFlats):
         test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(ui))
     return test_suite