Merge branch 'frm_tomwer_to_tomoscan2' into 'master'

Frm tomwer to tomoscan2

Closes #3

See merge request !22

requirements.txt

 silx>=0.9
-lxml
\ No newline at end of file
+lxml
+numpy
\ No newline at end of file

tomoscan/esrf/edfscan.py

@@ -93,6 +93,7 @@ class EDFTomoScan(TomoScanBase):
 
     @docstring(TomoScanBase.clear_caches)
     def clear_caches(self):
+        super().clear_caches()
         self._darks = None
         self._flats = None
         self._projections = None

tomoscan/esrf/hdf5scan.py

@@ -43,8 +43,8 @@ from tomoscan.io import HDF5File
 from silx.io.utils import get_data
 from ..unitsystem import metricsystem
 from .utils import get_compacted_dataslices
-import logging
 import typing
+import logging
 
 _logger = logging.getLogger(__name__)
 
@@ -188,6 +188,7 @@ class HDF5TomoScan(TomoScanBase):
 
     @docstring(TomoScanBase.clear_caches)
     def clear_caches(self) -> None:
+        super().clear_caches()
         self._projections = None
         self._projections_compacted = None
         self._flats = None

tomoscan/esrf/mock.py

@@ -92,26 +92,6 @@ class _ScanMock:
             n_radio=n_radio, scan_range=scan_range, ref_n=ref_n, dark_n=dark_n
         )
 
-        if n_ini_radio is None:
-            n_ini_radio = n_radio + n_extra_radio
-            call_acqui_end = True
-        else:
-            call_acqui_end = False
-
-        for i_radio in range(n_ini_radio):
-            self.add_radio(i_radio)
-        for i_extra_radio in range(n_extra_radio):
-            self.add_radio(n_radio + i_extra_radio)
-        for i_recons in range(n_recons):
-            self.add_reconstruction(i_recons)
-        for i_recons in range(n_pag_recons):
-            self.add_pag_reconstruction(i_recons)
-        if recons_vol is True:
-            self.add_recons_vol()
-
-        if call_acqui_end is True:
-            self.end_acquisition()
-
     def add_radio(self, index=None):
         raise NotImplementedError("Base class")
 
@@ -233,6 +213,9 @@ class MockHDF5(_ScanMock):
             self.add_initial_dark()
         if create_ini_ref:
             self.add_initial_ref()
+        if n_ini_proj is not None:
+            for i_radio in range(n_ini_proj):
+                self.add_radio(index=i_radio)
         if create_final_ref:
             self.add_final_ref()
         self.scan = HDF5TomoScan(scan=self.scan_master_file, entry="entry")
@@ -438,7 +421,7 @@ class MockEDF(_ScanMock):
             for i_radio in range(n_ini_radio):
                 self.add_radio(i_radio)
         for i_extra_radio in range(n_extra_radio):
-            self.add_radio(i_extra_radio)
+            self.add_radio(i_extra_radio + n_ini_radio)
         for i_recons in range(n_recons):
             self.add_reconstruction(i_recons)
         for i_recons in range(n_pag_recons):

tomoscan/esrf/test/__init__.py

@@ -25,18 +25,20 @@
 
 __authors__ = ["H.Payno"]
 __license__ = "MIT"
-__date__ = "16/09/2019"
+__date__ = "21/08/2020"
 
 
 import unittest
 
 from .test_hdf5scan import suite as h5_test_suite
-from .test_edfscan import suite as scan_test_suite
+from .test_edfscan import suite as edf_test_suite
+from .test_mock import suite as mock_test_suite
 
 
 def suite():
     test_suite = unittest.TestSuite()
     test_suite.addTest(h5_test_suite())
-    test_suite.addTest(scan_test_suite())
+    test_suite.addTest(edf_test_suite())
+    test_suite.addTest(mock_test_suite())
 
     return test_suite

tomoscan/esrf/test/test_edfscan.py

@@ -37,6 +37,7 @@ from tomoscan.esrf.mock import MockEDF
 from tomoscan.esrf.edfscan import EDFTomoScan
 from tomoscan.scanbase import TomoScanBase
 from tomoscan.scanfactory import ScanFactory
+from silx.io.utils import get_data
 from silx.io.url import DataUrl
 import collections
 import json
@@ -272,7 +273,9 @@ class TestProjections(unittest.TestCase):
         shutil.rmtree(self.folder)
 
     def testProjectionNoExtra(self):
-        mock = MockEDF(scan_path=self.folder, n_radio=10, n_extra_radio=0)
+        mock = MockEDF(
+            scan_path=self.folder, n_radio=10, n_ini_radio=10, n_extra_radio=0
+        )
         mock.end_acquisition()
         scan = EDFTomoScan(scan=self.folder)
         self.assertEqual(len(scan.projections), 10)
@@ -291,7 +294,11 @@ class TestProjections(unittest.TestCase):
 
     def testProjectionWithExtraRadio(self):
         mock = MockEDF(
-            scan_path=self.folder, n_radio=11, n_extra_radio=2, scan_range=180
+            scan_path=self.folder,
+            n_radio=11,
+            n_ini_radio=11,
+            n_extra_radio=2,
+            scan_range=180,
         )
         mock.end_acquisition()
         scan = EDFTomoScan(scan=self.folder)
@@ -305,6 +312,103 @@ class TestProjections(unittest.TestCase):
         self.assertTrue(360 not in proj_angle_dict)
 
 
+class TestFlatFieldCorrection(unittest.TestCase):
+    """Test the flat field correction"""
+
+    def setUp(self) -> None:
+        self.folder = tempfile.mkdtemp()
+        mock = MockEDF(
+            scan_path=self.folder, n_radio=30, n_ini_radio=30, n_extra_radio=0, dim=20
+        )
+        mock.end_acquisition()
+        self.scan = EDFTomoScan(scan=self.folder)
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.folder)
+
+    def testFlatAndDarksSet(self):
+        """Test no error is log if `normed` dark and flat are provided"""
+        self.scan.set_normed_flats(
+            {
+                1: numpy.random.random(20 * 20).reshape((20, 20)),
+                21: numpy.random.random(20 * 20).reshape((20, 20)),
+            }
+        )
+        self.scan.set_normed_darks({0: numpy.random.random(20 * 20).reshape((20, 20))})
+
+        projs = []
+        proj_indexes = []
+        for proj_index, proj in self.scan.projections.items():
+            projs.append(proj)
+            proj_indexes.append(proj_index)
+        normed_proj = self.scan.flat_field_correction(
+            projs=projs, proj_indexes=proj_indexes
+        )
+        self.assertEqual(len(normed_proj), len(self.scan.projections))
+        raw_data = get_data(projs[10])
+        self.assertFalse(numpy.array_equal(raw_data, normed_proj[10]))
+
+    def testNoFlatOrDarkSet(self):
+        """Test an error is log if `normed` dark and flat aren't provided"""
+        projs = []
+        proj_indexes = []
+        for proj_index, proj in self.scan.projections.items():
+            projs.append(proj)
+            proj_indexes.append(proj_index)
+        with self.assertLogs("tomoscan", level="ERROR"):
+            normed_proj = self.scan.flat_field_correction(
+                projs=projs, proj_indexes=proj_indexes
+            )
+        self.assertEqual(len(normed_proj), len(self.scan.projections))
+        raw_data = get_data(projs[10])
+        self.assertTrue(numpy.array_equal(raw_data, normed_proj[10]))
+
+
+class TestGetSinogram(unittest.TestCase):
+    """Test the get_sinogram function"""
+
+    def setUp(self) -> None:
+        self.folder = tempfile.mkdtemp()
+        mock = MockEDF(
+            scan_path=self.folder, n_radio=30, n_ini_radio=30, n_extra_radio=0, dim=20
+        )
+        mock.end_acquisition()
+        self.scan = EDFTomoScan(scan=self.folder)
+        self.scan.set_normed_flats(
+            {
+                21: numpy.random.random(20 * 20).reshape((20, 20)),
+                1520: numpy.random.random(20 * 20).reshape((20, 20)),
+            }
+        )
+        self.scan.set_normed_darks({0: numpy.random.random(20 * 20).reshape((20, 20))})
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.folder)
+
+    def testGetSinogram1(self):
+        sinogram = self.scan.get_sinogram(line=12, subsampling=1)
+        self.assertEqual(sinogram.shape, (30, 20))
+
+    def testGetSinogram2(self):
+        """Test if subsampling is negative"""
+        with self.assertRaises(ValueError):
+            self.scan.get_sinogram(line=0, subsampling=-1)
+
+    def testGetSinogram3(self):
+        sinogram = self.scan.get_sinogram(line=0, subsampling=3)
+        self.assertEqual(sinogram.shape, (10, 20))
+
+    def testGetSinogram4(self):
+        """Test if line is not in the projection"""
+        with self.assertRaises(ValueError):
+            self.scan.get_sinogram(line=-1, subsampling=1)
+
+    def testGetSinogram5(self):
+        """Test if line is not in the projection"""
+        with self.assertRaises(ValueError):
+            self.scan.get_sinogram(line=35, subsampling=1)
+
+
 class TestScanValidatorFindFiles(unittest.TestCase):
     """Function testing the getReconstructionsPaths function is correctly
     functioning"""
@@ -506,6 +610,8 @@ def suite():
         TestProjections,
         TestTomoBaseHashable,
         TestOriDarksFlats,
+        TestFlatFieldCorrection,
+        TestGetSinogram,
     ):
         test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(ui))
     return test_suite

tomoscan/esrf/test/test_hdf5scan.py

@@ -217,16 +217,99 @@ class TestHDF5Scan(HDF5TestBaseClass):
     def testCompactedProjs(self):
         projs_compacted = self.scan.projections_compacted
         self.assertTrue(projs_compacted.keys() == self.scan.projections.keys())
-        # ~ print(set(map(str, list(projs_compacted.values()))))
         for i in range(22, 1520 + 1):
             self.assertTrue(projs_compacted[i].data_slice() == slice(22, 1521, None))
         for i in range(1542, 1543):
             self.assertTrue(projs_compacted[i].data_slice() == slice(1542, 1543, None))
 
 
+class TestFlatFieldCorrection(HDF5TestBaseClass):
+    """Test the flat field correction"""
+
+    def setUp(self) -> None:
+        super(TestFlatFieldCorrection, self).setUp()
+        self.dataset_file = self.get_dataset("frm_edftomomill_twoentries.nx")
+        self.scan = HDF5TomoScan(scan=self.dataset_file)
+
+    def testFlatAndDarksSet(self):
+        self.scan.set_normed_flats(
+            {
+                21: numpy.random.random(20 * 20).reshape((20, 20)),
+                1520: numpy.random.random(20 * 20).reshape((20, 20)),
+            }
+        )
+        self.scan.set_normed_darks({0: numpy.random.random(20 * 20).reshape((20, 20))})
+
+        projs = []
+        proj_indexes = []
+        for proj_index, proj in self.scan.projections.items():
+            projs.append(proj)
+            proj_indexes.append(proj_index)
+        normed_proj = self.scan.flat_field_correction(
+            projs=projs, proj_indexes=proj_indexes
+        )
+        self.assertEqual(len(normed_proj), len(self.scan.projections))
+        raw_data = get_data(projs[50])
+        self.assertFalse(numpy.array_equal(raw_data, normed_proj[50]))
+
+    def testNoFlatOrDarkSet(self):
+        projs = []
+        proj_indexes = []
+        for proj_index, proj in self.scan.projections.items():
+            projs.append(proj)
+            proj_indexes.append(proj_index)
+        with self.assertLogs("tomoscan", level="ERROR"):
+            normed_proj = self.scan.flat_field_correction(
+                projs=projs, proj_indexes=proj_indexes
+            )
+        self.assertEqual(len(normed_proj), len(self.scan.projections))
+        raw_data = get_data(projs[50])
+        self.assertTrue(numpy.array_equal(raw_data, normed_proj[50]))
+
+
+class TestGetSinogram(HDF5TestBaseClass):
+    """Test the get_sinogram function"""
+
+    def setUp(self) -> None:
+        super(TestGetSinogram, self).setUp()
+        self.dataset_file = self.get_dataset("frm_edftomomill_twoentries.nx")
+        self.scan = HDF5TomoScan(scan=self.dataset_file)
+        # set some random dark and flats
+        self.scan.set_normed_flats(
+            {
+                21: numpy.random.random(20 * 20).reshape((20, 20)),
+                1520: numpy.random.random(20 * 20).reshape((20, 20)),
+            }
+        )
+        self.scan.set_normed_darks({0: numpy.random.random(20 * 20).reshape((20, 20))})
+
+    def testGetSinogram1(self):
+        sinogram = self.scan.get_sinogram(line=12, subsampling=1)
+        self.assertEqual(sinogram.shape, (1500, 20))
+
+    def testGetSinogram2(self):
+        """Test if subsampling is negative"""
+        with self.assertRaises(ValueError):
+            self.scan.get_sinogram(line=0, subsampling=-1)
+
+    def testGetSinogram3(self):
+        sinogram = self.scan.get_sinogram(line=0, subsampling=3)
+        self.assertEqual(sinogram.shape, (500, 20))
+
+    def testGetSinogram4(self):
+        """Test if line is not in the projection"""
+        with self.assertRaises(ValueError):
+            self.scan.get_sinogram(line=-1, subsampling=1)
+
+    def testGetSinogram5(self):
+        """Test if line is not in the projection"""
+        with self.assertRaises(ValueError):
+            self.scan.get_sinogram(line=25, subsampling=1)
+
+
 def suite():
     test_suite = unittest.TestSuite()
-    for ui in (TestHDF5Scan,):
+    for ui in (TestHDF5Scan, TestFlatFieldCorrection, TestGetSinogram):
         test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(ui))
     return test_suite
 

tomoscan/esrf/test/test_mock.py

+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2017 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+
+__authors__ = ["H. Payno"]
+__license__ = "MIT"
+__date__ = "21/08/2020"
+
+import unittest
+import os
+import tempfile
+from tomoscan.esrf.mock import MockHDF5, MockEDF
+from tomoscan.esrf.hdf5scan import HDF5TomoScan
+import shutil
+
+
+class TestMockEDFScan(unittest.TestCase):
+    """Test that mock scan are adapted to other unit test"""
+
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+        self.scan_id = os.path.join(self.tmpdir, "myscan")
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+
+    def testScanEvolution360(self):
+        """Test get scan evolution from a mock scan with a 360 range and no
+        extra radio"""
+        scan = MockEDF.mockScan(
+            scanID=self.scan_id,
+            nRadio=5,
+            nRecons=1,
+            nPagRecons=1,
+            dim=10,
+            scan_range=360,
+        )
+        scan_dynamic = scan.get_proj_angle_url()
+        self.assertEqual(len(scan_dynamic), 5)
+        self.assertTrue(0 in scan_dynamic)
+        self.assertEqual(
+            scan_dynamic[0].file_path(),
+            os.path.join(self.tmpdir, "myscan", "myscan_0000.edf"),
+        )
+        self.assertTrue(180 in scan_dynamic)
+        self.assertEqual(
+            scan_dynamic[180].file_path(),
+            os.path.join(self.tmpdir, "myscan", "myscan_0002.edf"),
+        )
+        self.assertTrue(360 in scan_dynamic)
+        self.assertEqual(
+            scan_dynamic[360].file_path(),
+            os.path.join(self.tmpdir, "myscan", "myscan_0004.edf"),
+        )
+
+    def testScanEvolution180(self):
+        """Test get scan evolution from a mock scan with a 180 range and no
+        extra radio"""
+        scan = MockEDF.mockScan(
+            scanID=self.scan_id,
+            nRadio=8,
+            nRecons=0,
+            nPagRecons=0,
+            dim=10,
+            scan_range=180,
+        )
+        scan_dynamic = scan.get_proj_angle_url()
+        self.assertEqual(len(scan_dynamic), 8)
+        self.assertTrue(0 in scan_dynamic)
+        self.assertEqual(
+            scan_dynamic[0].file_path(),
+            os.path.join(self.tmpdir, "myscan", "myscan_0000.edf"),
+        )
+        self.assertTrue(180 in scan_dynamic)
+        self.assertEqual(
+            scan_dynamic[180].file_path(),
+            os.path.join(self.tmpdir, "myscan", "myscan_0007.edf"),
+        )
+        self.assertFalse(360 in scan_dynamic)
+
+    def testScanEvolution360Extra4(self):
+        """Test get scan evolution from a mock scan with a 360 range and 4
+        extra radio"""
+        scan = MockEDF.mockScan(
+            scanID=self.scan_id,
+            nRadio=21,
+            nRecons=0,
+            nPagRecons=0,
+            dim=10,
+            scan_range=360,
+            n_extra_radio=4,
+        )
+        scan_dynamic = scan.get_proj_angle_url()
+        self.assertEqual(len(scan_dynamic), 21 + 4)
+        self.assertTrue(0 in scan_dynamic)
+        self.assertEqual(
+            scan_dynamic[0].file_path(),
+            os.path.join(self.tmpdir, "myscan", "myscan_0000.edf"),
+        )
+        self.assertTrue(180 in scan_dynamic)
+        self.assertEqual(
+            scan_dynamic[180].file_path(),
+            os.path.join(self.tmpdir, "myscan", "myscan_0010.edf"),
+        )
+        self.assertTrue(360 in scan_dynamic)
+        self.assertEqual(
+            scan_dynamic[360].file_path(),
+            os.path.join(self.tmpdir, "myscan", "myscan_0020.edf"),
+        )
+        extra_angles = (0, 90, 180, 270)  # note: 0(1) is the last file acquire
+        for iAngle, angle in enumerate(extra_angles):
+            angle_id = str(angle) + "(1)"
+            self.assertTrue(angle_id in scan_dynamic)
+            file_name = os.path.join(
+                self.tmpdir, "myscan", "myscan_%04d.edf" % (21 + 4 - 1 - iAngle)
+            )
+            self.assertEqual(scan_dynamic[angle_id].file_path(), file_name)
+
+    def testScanEvolution180Extra2(self):
+        """Test get scan evolution from a mock scan with a 360 range and 3
+        extra radios"""
+        scan = MockEDF.mockScan(
+            scanID=self.scan_id,
+            nRadio=4,
+            nRecons=2,
+            nPagRecons=2,
+            dim=10,
+            scan_range=180,
+            n_extra_radio=2,
+        )
+        scan_dynamic = scan.get_proj_angle_url()
+        self.assertEqual(len(scan_dynamic), 4 + 2)
+        self.assertTrue(0 in scan_dynamic)
+        self.assertEqual(
+            scan_dynamic[0].file_path(),
+            os.path.join(self.tmpdir, "myscan", "myscan_0000.edf"),
+        )
+        self.assertTrue(180 in scan_dynamic)
+        self.assertEqual(
+            scan_dynamic[180].file_path(),
+            os.path.join(self.tmpdir, "myscan", "myscan_0003.edf"),
+        )
+        self.assertTrue(360 not in scan_dynamic)
+        extra_angles = (0, 90)  # note: 0(1) is the last file acquire
+        for iAngle, angle in enumerate(extra_angles):
+            angle_id = str(angle) + "(1)"
+            self.assertTrue(angle_id in scan_dynamic)
+            file_name = os.path.join(
+                self.tmpdir, "myscan", "myscan_%04d.edf" % (4 + 2 - 1 - iAngle)
+            )
+            self.assertEqual(scan_dynamic[angle_id].file_path(), file_name)
+
+
+class TestMockHDF5(unittest.TestCase):
+    """Test the MockHDF5 file"""
+
+    def setUp(self) -> None:
+        self.folder = tempfile.mkdtemp()
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.folder)
+
+    def test_creation(self):
+        mock = MockHDF5(scan_path=self.folder, n_proj=10, n_ini_proj=10)
+        self.assertEqual(
+            mock.scan_master_file,
+            os.path.join(self.folder, os.path.basename(self.folder) + ".h5"),
+        )
+        tomoScan = HDF5TomoScan(mock.scan_path, entry=mock.scan_entry)
+        self.assertEqual(len(HDF5TomoScan.get_valid_entries(mock.scan_master_file)), 1)
+        tomoScan.update()
+        self.assertEqual(tomoScan.scan_range, 360)
+        self.assertEqual(len(tomoScan.projections), 10)
+
+    def testSimpleMockCreationOneCall(self):
+        """Test mock of an acquisition starting by one dark, then 10 ref,
+        then 20 radios, then 10 'final' ref and 2 alignment radio"""
+        mock = MockHDF5(
+            scan_path=self.folder,
+            n_proj=20,
+            n_ini_proj=20,
+            n_alignement_proj=2,
+            create_ini_dark=True,
+            create_ini_ref=True,
+            create_final_ref=True,
+            n_refs=10,
+        )
+        self.assertTrue(0 in mock.scan.darks.keys())
+        self.assertTrue(1 in mock.scan.flats.keys())
+        self.assertTrue(10 in mock.scan.flats.keys())
+
+
+def suite():
+    test_suite = unittest.TestSuite()
+    for ui in (TestMockHDF5, TestMockEDFScan):
+        test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(ui))
+    return test_suite
+
+
+if __name__ == "__main__":
+    unittest.main(defaultTest="suite")

tomoscan/progress.py

+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,