diff --git a/tomoscan/esrf/__init__.py b/tomoscan/esrf/__init__.py index 2f6a26d10f85902354031b5dcf01b8cfe88d8e7a..bea7f00e15a18129ab37e813f1e94c26402d86ed 100644 --- a/tomoscan/esrf/__init__.py +++ b/tomoscan/esrf/__init__.py @@ -1,7 +1,8 @@ """module dedicated to esrf scans""" from .scan.edfscan import EDFTomoScan # noqa F401 -from .scan.fluoscan import FluoTomoScan # noqa F401 +from .scan.fluoscan import FluoTomoScan3D # noqa F401 +from .scan.fluoscan import FluoTomoScan2D # noqa F401 from .scan.nxtomoscan import NXtomoScan # noqa F401 from .scan.nxtomoscan import HDF5TomoScan # noqa F401 from .volume.edfvolume import EDFVolume # noqa F401 diff --git a/tomoscan/esrf/scan/fluoscan.py b/tomoscan/esrf/scan/fluoscan.py index 59279a518a10fe688bd3b26139ad5fc8dc32c29c..d173554bc3e42a23463499771c5ce1318bf6a11f 100644 --- a/tomoscan/esrf/scan/fluoscan.py +++ b/tomoscan/esrf/scan/fluoscan.py @@ -15,6 +15,7 @@ from tomoscan.utils import docstring from dataclasses import dataclass, field +import numpy as np from numpy.typing import NDArray, DTypeLike from silx.io.utils import h5py_read_dataset @@ -38,13 +39,11 @@ except ImportError as e: else: has_imageio = True -__all__ = [ - "FluoTomoScan", -] +__all__ = ["FluoTomoScanBase", "FluoTomoScan3D", "FluoTomoScan2D"] @dataclass -class FluoTomoScan: +class FluoTomoScanBase: """Dataset manipulation class.""" scan: str @@ -59,10 +58,11 @@ class FluoTomoScan: el_lines: dict[str, list[str]] = field(default_factory=dict) pixel_size: float | None = None energy: float | None = None + detected_folders: list[str] = field(default_factory=list) def __post_init__(self): self.detected_detectors = tuple(self.detect_detectors()) - self.detect_pixel_size_and_energy() + self.get_common_metadata_from_h5_file() _logger.info(f"Detectors: {self.detected_detectors}") if len(self.detectors) == 0: self.detectors = self.detected_detectors @@ -73,12 +73,6 @@ class FluoTomoScan: ) self.detect_elements() - if self.skip_angle_inds is not None: - self.skip_angle_inds = numpy.array(self.skip_angle_inds) - - if self.angles is None: - self.detect_rot_angles() - @property def rot_angles_deg(self) -> NDArray: if self.angles is None: @@ -96,11 +90,8 @@ class FluoTomoScan: def detect_rot_angles(self): tmp_angles = [] proj_ind = 0 - while True: - proj_ind += 1 - prj_dir = os.path.join( - self.scan, "fluofit", self.dataset_basename + "_%03d" % proj_ind - ) + for f in self.detected_folders: + prj_dir = os.path.join(self.scan, "fluofit", f) info_file = os.path.join(prj_dir, "info.txt") if not os.path.exists(info_file): _logger.debug( @@ -115,49 +106,37 @@ class FluoTomoScan: ) self.angles = numpy.array(tmp_angles, ndmin=1, dtype=numpy.float32) - def detect_pixel_size_and_energy(self): - pixel_size_path = os.path.join(self.scan, self.dataset_basename + "_%03d" % 1) - h5_files = glob.glob1(pixel_size_path, "*.h5") + def get_common_metadata_from_h5_file(self): + h5_path = os.path.join(self.scan, self.detected_folders[0]) + h5_files = glob.glob1(h5_path, "*.h5") if len(h5_files) > 1: raise ValueError( "More than one hdf5 file in scan directory. Expect only ONE to pick pixel size." ) elif len(h5_files) == 0: - pattern = os.path.join(pixel_size_path, "*.h5") + pattern = os.path.join(h5_path, "*.h5") raise ValueError( f"Unable to find the hdf5 file in scan directory to pick pixel size. RegExp used is {pattern}" ) else: - try: - if "3DXRF" in h5_files[0]: - sample_name = h5_files[0].split("_3DXRF_")[0].split("-")[1] + with h5py.File(os.path.join(h5_path, h5_files[0]), "r") as f: + if len(list(f.keys())) != 1: + raise ValueError( + f"H5 file should contain only one entry, found {len(list(f.keys()))}" + ) else: - sample_name = h5_files[0].split("_XRF_")[0].split("-")[1] - except IndexError: - raise ValueError( - f"unable to deduce sample name from {h5_files[0]}. Expected synthax is 'proposal-samplename_XRF_XXX.h5'" - ) - entry_name = ( - "entry_0000: " - + sample_name - + " - " - + self.dataset_basename - + "_%03d" % 1 - ) - with h5py.File(os.path.join(pixel_size_path, h5_files[0]), "r") as f: - self.pixel_size = ( - h5py_read_dataset(f[entry_name]["FLUO"]["pixelSize"]) * 1e-6 - ) - self.energy = float( - h5py_read_dataset( - f[entry_name]["instrument"]["monochromator"]["energy"] - ) - ) + entry_name = list(f.keys())[0] + self.pixel_size = ( + h5py_read_dataset(f[entry_name]["FLUO"]["pixelSize"]) * 1e-6 + ) + self.energy = float( + h5py_read_dataset( + f[entry_name]["instrument"]["monochromator"]["energy"] + ) + ) def detect_detectors(self): - proj_1_dir = os.path.join( - self.scan, "fluofit", self.dataset_basename + "_%03d" % 1 - ) + proj_1_dir = os.path.join(self.scan, "fluofit", self.detected_folders[0]) detected_detectors = [] file_names = glob.glob1(proj_1_dir, "IMG_*area_density_ngmm2.tif") for file in file_names: @@ -167,9 +146,7 @@ class FluoTomoScan: return detected_detectors def detect_elements(self): - proj_1_dir = os.path.join( - self.scan, "fluofit", self.dataset_basename + "_%03d" % 1 - ) + proj_1_dir = os.path.join(self.scan, "fluofit", self.detected_folders[0]) detector = self.detectors[0] file_names = glob.glob1(proj_1_dir, f"IMG_{detector}*area_density_ngmm2.tif") for file in file_names: @@ -182,6 +159,76 @@ class FluoTomoScan: except KeyError: self.el_lines[element] = [line] + @staticmethod + def from_identifier(identifier): + """Return the Dataset from a identifier""" + raise NotImplementedError("Not implemented for fluo-tomo yet.") + + @docstring(TomoScanBase) + def get_identifier(self) -> ScanIdentifier: + raise NotImplementedError("Not implemented for fluo-tomo yet.") + + +@dataclass +class FluoTomoScan3D(FluoTomoScanBase): + """Dataset manipulation class.""" + + scan: str + dataset_basename: str + detectors: tuple = () + + skip_angle_inds: Sequence[int] | NDArray | None = None + dtype: DTypeLike = numpy.float32 + verbose: bool = False + + angles: NDArray | None = None + el_lines: dict[str, list[str]] = field(default_factory=dict) + pixel_size: float | None = None + energy: float | None = None + + def __post_init__(self): + self.detected_folders = self.detect_folders() + super().__post_init__() + if self.skip_angle_inds is not None: + self.skip_angle_inds = numpy.array(self.skip_angle_inds) + + if self.angles is None: + self.detect_rot_angles() + + def detect_folders(self): + fit_folders = glob.glob1( + os.path.join(self.scan, "fluofit"), rf"{self.dataset_basename}_projection*" + ) + + if len(fit_folders) == 0: + raise FileNotFoundError( + "No projection was found in the fluofit folder. The searched for pattern is <scan_dir>/fluofit/<dataset_basename>_projection*'." + ) + elif not os.path.isdir(os.path.join(self.scan, fit_folders[0])): + raise FileNotFoundError( + "Found fitted data folders but not the corresponding raw data folder." + ) + else: + return fit_folders + + def detect_rot_angles(self): + tmp_angles = [] + for f in self.detected_folders: + prj_dir = os.path.join(self.scan, "fluofit", f) + info_file = os.path.join(prj_dir, "info.txt") + if not os.path.exists(info_file): + _logger.debug( + f"{info_file} doesn't exist, while expected to be present in each projection folder." + ) + break + with open(info_file, "r") as f: + info_str = f.read() + tmp_angles.append(float(info_str.split(" ")[2])) + _logger.info( + f"Found angle information in info file for {len(tmp_angles)} projections." + ) + self.angles = numpy.array(tmp_angles, ndmin=1, dtype=numpy.float32) + def load_data(self, det: str, element: str, line_ind: int = 0): if not has_imageio: raise RuntimeError("imageio not install. Cannot load data.") @@ -215,7 +262,140 @@ class FluoTomoScan: continue proj_dir = os.path.join( - self.scan, "fluofit", self.dataset_basename + "_%03d" % (ii_i + 1) + self.scan, + "fluofit", + self.dataset_basename + "_projection_%03d" % (ii_i + 1), + ) + img_path = os.path.join( + proj_dir, f"IMG_{det}_{element}-{line}_area_density_ngmm2.tif" + ) + + if self.verbose: + _logger.info(f"Loading {ii_i+1}/{len(self.angles)}: {img_path}") + + img = iio.imread(img_path) + data_det.append(numpy.nan_to_num(numpy.array(img, dtype=self.dtype))) + + data = numpy.array(data_det) + return numpy.ascontiguousarray(data) + + @staticmethod + def from_identifier(identifier): + """Return the Dataset from a identifier""" + raise NotImplementedError("Not implemented for fluo-tomo yet.") + + @docstring(TomoScanBase) + def get_identifier(self) -> ScanIdentifier: + raise NotImplementedError("Not implemented for fluo-tomo yet.") + + +@dataclass +class FluoTomoScan2D(FluoTomoScanBase): + """Dataset manipulation class.""" + + scan: str + dataset_basename: str + detectors: tuple = () + + skip_angle_inds: Sequence[int] | NDArray | None = None + dtype: DTypeLike = numpy.float32 + verbose: bool = False + + angles: NDArray | None = None + el_lines: dict[str, list[str]] = field(default_factory=dict) + pixel_size: float | None = None + energy: float | None = None + + def __post_init__(self): + self.detected_folders = self.detect_folders() + super().__post_init__() + self.get_metadata_from_h5_file() + if self.skip_angle_inds is not None: + self.skip_angle_inds = numpy.array(self.skip_angle_inds) + + if self.angles is None: + self.angles = self.detect_rot_angles() + + def get_metadata_from_h5_file(self): + h5_path = os.path.join(self.scan, self.detected_folders[0]) + h5_files = glob.glob1(h5_path, "*.h5") + if len(h5_files) > 1: + raise ValueError( + "More than one hdf5 file in scan directory. Expect only ONE to pick pixel size." + ) + elif len(h5_files) == 0: + pattern = os.path.join(h5_path, "*.h5") + raise ValueError( + f"Unable to find the hdf5 file in scan directory to pick pixel size. RegExp used is {pattern}" + ) + else: + with h5py.File(os.path.join(h5_path, h5_files[0]), "r") as f: + if len(list(f.keys())) != 1: + raise ValueError( + f"H5 file should contain only one entry, found {len(list(f.keys()))}" + ) + else: + entry_name = list(f.keys())[0] + self.scanRange_2 = float( + h5py_read_dataset(f[entry_name]["FLUO"]["scanRange_2"]) + ) + self.scanDim_2 = int( + h5py_read_dataset(f[entry_name]["FLUO"]["scanDim_2"]) + ) + + def detect_rot_angles(self): + nb_projs = self.scanDim_2 + angular_coverage = self.scanRange_2 + return np.linspace(0, angular_coverage, nb_projs, endpoint=True) + + def detect_folders(self): + fit_folder = os.path.join(self.scan, "fluofit", self.dataset_basename) + + if not os.path.isdir(fit_folder): + raise FileNotFoundError( + f"No folder {fit_folder} was found in the fluofit folder. The searched for pattern is <scan_dir>/fluofit/<dataset_basename>_projection*'." + ) + elif not os.path.isdir(os.path.join(self.scan, self.dataset_basename)): + raise FileNotFoundError( + "Found fitted data folders but not the corresponding raw data folder." + ) + else: + return [ + self.dataset_basename, + ] + + def load_data(self, det: str, element: str, line_ind: int = 0): + if not has_imageio: + raise RuntimeError("imageio not install. Cannot load data.") + if det not in self.detectors: + raise RuntimeError( + f"The detector {det} is invalid. Valid ones are {self.detectors}" + ) + + if self.detectors is None: + raise RuntimeError("Detectors not initialized") + + line = self.el_lines[element][line_ind] + + data_det = [] + + description = f"Loading images of {element}-{line} ({det}): " + + if has_tqdm: + slice_iterator = tqdm( + range(len(self.detected_folders)), + disable=self.verbose, + desc=description, + ) + else: + slice_iterator = range(len(self.detected_folders)) + + for ii_i in slice_iterator: + + proj_dir = os.path.join( + self.scan, + "fluofit", + self.dataset_basename, # WARNING: dataset_basename is ONE SINGLE sinogram. ) img_path = os.path.join( proj_dir, f"IMG_{det}_{element}-{line}_area_density_ngmm2.tif" diff --git a/tomoscan/esrf/scan/tests/test_fluoscan2D.py b/tomoscan/esrf/scan/tests/test_fluoscan2D.py new file mode 100644 index 0000000000000000000000000000000000000000..11860ba8ff4a7489c5484c9159b4432686c915a8 --- /dev/null +++ b/tomoscan/esrf/scan/tests/test_fluoscan2D.py @@ -0,0 +1,66 @@ +# coding: utf-8 + +import logging + +import pytest +import numpy + +from tomoscan.esrf.scan.fluoscan import FluoTomoScan2D +from tomoscan.tests.datasets import GitlabDataset + +logging.disable(logging.INFO) + + +@pytest.fixture(scope="class") +def fluodata2D(request): + cls = request.cls + cls.scan_dir = GitlabDataset.get_dataset("fluo_datasets2D") + cls.dataset_basename = "CONT2_p2_600nm_FT02_slice_0" + cls.scan = FluoTomoScan2D( + scan=cls.scan_dir, + dataset_basename=cls.dataset_basename, + detectors=(), + ) + + +@pytest.mark.usefixtures("fluodata2D") +class TestFluo2D: + def test_all_detectors(self): + assert ( + len(self.scan.el_lines) == 2 + ), f"Number of elements found should be 2 and is {len(self.scan.el_lines)}." + assert set(self.scan.detectors) == set( + ["fluo1", "corrweighted"] + ), f"There should be 2 'detectors' (fluo1 and corrweighted), {len(self.detectors)} were found." + + def test_one_detector(self): + scan = FluoTomoScan2D( + scan=self.scan_dir, + dataset_basename=self.dataset_basename, + detectors=("corrweighted",), + ) + assert ( + len(scan.el_lines) == 2 + ), f"Number of elements found should be 2 and is {len(scan.el_lines)}." + assert ( + len(scan.detectors) == 1 + ), f"There should be 1 detector (corrweighted), {len(scan.detectors)} were found." + + # One ghost detector (no corresponding files) + # test general section setters + with pytest.raises(ValueError): + scan = FluoTomoScan2D( + scan=self.scan_dir, + dataset_basename=self.dataset_basename, + detectors=("toto",), + ) + + def test_load_data(self): + data = self.scan.load_data("corrweighted", "Ca", 0) + assert data.shape == (1, 251, 1000) + + def test_load_energy_and_pixel_size(self): + assert self.scan.energy == 17.1 + assert numpy.allclose( + self.scan.pixel_size, 6e-10, atol=1e-4 + ) # Tolerance:0.1nm (since pixel_size is expected in um). diff --git a/tomoscan/esrf/scan/tests/test_fluoscan.py b/tomoscan/esrf/scan/tests/test_fluoscan3D.py similarity index 85% rename from tomoscan/esrf/scan/tests/test_fluoscan.py rename to tomoscan/esrf/scan/tests/test_fluoscan3D.py index d632abf3ad57d1abb1d18ece431a5a7faf682f65..cb49a5287b442be1e7ef929af0955e8bd973bb6c 100644 --- a/tomoscan/esrf/scan/tests/test_fluoscan.py +++ b/tomoscan/esrf/scan/tests/test_fluoscan3D.py @@ -4,26 +4,26 @@ import logging import pytest -from tomoscan.esrf.scan.fluoscan import FluoTomoScan +from tomoscan.esrf.scan.fluoscan import FluoTomoScan3D from tomoscan.tests.datasets import GitlabDataset logging.disable(logging.INFO) @pytest.fixture(scope="class") -def fluodata(request): +def fluodata3D(request): cls = request.cls cls.scan_dir = GitlabDataset.get_dataset("fluo_datasets") - cls.dataset_basename = "CP1_XRD_insitu_top_ft_100nm_projection" - cls.scan = FluoTomoScan( + cls.dataset_basename = "CP1_XRD_insitu_top_ft_100nm" + cls.scan = FluoTomoScan3D( scan=cls.scan_dir, dataset_basename=cls.dataset_basename, detectors=(), ) -@pytest.mark.usefixtures("fluodata") -class TestFluo: +@pytest.mark.usefixtures("fluodata3D") +class TestFluo3D: def test_all_detectors(self): assert ( len(self.scan.el_lines) == 14 @@ -33,7 +33,7 @@ class TestFluo: ), f"There should be 3 'detectors' (xmap, falcon and weighted), {len(self.detectors)} were found." def test_one_detector(self): - scan = FluoTomoScan( + scan = FluoTomoScan3D( scan=self.scan_dir, dataset_basename=self.dataset_basename, detectors=("xmap",), @@ -48,7 +48,7 @@ class TestFluo: # One ghost detector (no corresponding files) # test general section setters with pytest.raises(ValueError): - scan = FluoTomoScan( + scan = FluoTomoScan3D( scan=self.scan_dir, dataset_basename=self.dataset_basename, detectors=("toto",),