Skip to content
GitLab
Commit cf17630e authored by payno's avatar payno
Browse files

second commit.

parent 45fd8c5a
Hide whitespace changes
Inline Side-by-side
tomoscan/.gitlab-ci.yml
View file @ cf17630e
python -m unittest tomoscan.test.suite -v
\ No newline at end of file
stages:
- test_library
- test_notbooks
variables:
http_proxy: http://proxy.esrf.fr:3128
https_proxy: http://proxy.esrf.fr:3128
no_proxy: .esrf.fr,localhost
ORANGE_WEB_LOG: 'False'
.build_template: &test_linux_template
stage: test_library
before_script:
- arch
- export PYTHONPATH="${PYTHONPATH}:/usr/lib/python3/dist-packages/"
- export LD_LIBRARY_PATH=/lib/i386-linux-gnu/:${LD_LIBRARY_PATH}
- export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
- export ORANGE_WEB_LOG='False'
- python --version
- python -m pip install pip --upgrade
- python -m pip install setuptools --upgrade
- python -m pip install - requirements.txt
- python -m pip install .
script:
- python -m unittest tomoscan.test.suite -v
test:python3.5-stretch-pyqt5:
image: docker-registry.esrf.fr/dau/tomwer:python3.5_stretch_pyqt5
<<: *test_linux_template
test:python3.7_stretch_pyqt5:
image: docker-registry.esrf.fr/dau/tomwer:python3.7_stretch_pyqt5
<<: *test_linux_template
test:windows-conda:
stage: test_library
image: python:3-windowsservercore
tags:
- win
before_script:
- SET "PATH=C:\\miniconda3;C:\\miniconda3\\Scripts;%PATH%"
- SET "PATH=C:\\miniconda3\\include;%PATH%"
- SET "PATH=C:\\miniconda3\\python3.7\\site-packages\\numpy\\core\\include;%PATH%"
- SET "PATH=C:\\windows\\system32\\config\\systemprofile\\appdata\\roaming\\python\\python37\\site-packages;%PATH%"
- SET "PATH=C:\\windows\\system32\\config\\systemprofile\\appdata\\roaming\\python\\python37\\site-packages\\numpy\\core\\include;%PATH%"
- SET "ORANGE_WEB_LOG='False'"
- pip install virtualenv
- virtualenv venv
- call venv\\Scripts\\activate.bat
- python --version
- pip install pip --upgrade
- python -m pip install - requirements.txt
- python -m pip install .
script:
- python -m unittest tomoscan.test.suite -v
test:test-tomoscan-tutorials:
stage: test_notbooks
image: docker-registry.esrf.fr/dau/tomwer:python3.7_buster_pyqt5
before_script:
- arch
- export PYTHONPATH="${PYTHONPATH}:/usr/lib/python3/dist-packages/"
- export LD_LIBRARY_PATH=/lib/i386-linux-gnu/:${LD_LIBRARY_PATH}
- export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
- export ORANGE_WEB_LOG='False'
- python --version
- python -m pip install pip --upgrade
- python -m pip install setuptools --upgrade
- python -m pip install numpy --upgrade
- python -m pip install matplotlib
- python -m pip install jupyter_client
- python -m pip install nbconvert
- python -m pip install ipykernel
- python -m pip install -r requirements-dev.txt
- python -m pip install -r requirements-test.txt
- python -m pip install fabio --upgrade --pre
- python -m pip install silx --upgrade --pre
- python -m pip install .
script:
- jupyter nbconvert --to notebook --execute ../doc/tutorials/edf_scan.ipynb
- jupyter nbconvert --to notebook --execute ../doc/tutorials/hdf5_scan.ipynb
tomoscan/esrf/edfscan.py
View file @ cf17630e
......@@ -28,19 +28,14 @@ __license__ = "MIT"
__date__ = "10/10/2019"
import glob
import os
import re
from collections import OrderedDict
import fabio
from lxml import etree
import numpy
import json
import io
from typing import Union
from silx.io.url import DataUrl
from glob import glob
from ..scanbase import TomoScanBase
from .utils import get_parameters_frm_par_or_info, extract_urls_from_edf
from ..unitsystem import metricsystem
......@@ -69,12 +64,13 @@ class EDFTomoScan(TomoScanBase):
_SCHEME = 'fabio'
def __init__(self, scan):
def __init__(self, scan: Union[str, None]):
TomoScanBase.__init__(self, scan=scan, type_=self._TYPE)
# data caches
self._darks = None
self._refs = None
self._flats = None
self._projections = None
self.__tomo_n = None
self.__ref_n = None
self.__dark_n = None
......@@ -86,28 +82,28 @@ class EDFTomoScan(TomoScanBase):
@docstring(TomoScanBase.tomo_n)
@property
def tomo_n(self):
def tomo_n(self) -> Union[None, int]:
if self.__tomo_n is None:
self.__tomo_n = EDFTomoScan.get_tomo_n(scan=self.path)
return self.__tomo_n
@property
@docstring(TomoScanBase.dark_n)
def dark_n(self):
def dark_n(self) -> Union[None, int]:
if self.__dark_n is None:
self.__dark_n = EDFTomoScan.get_dark_n(scan=self.path)
return self.__dark_n
@property
@docstring(TomoScanBase.ref_n)
def ref_n(self):
def ref_n(self) -> Union[None, int]:
if self.__ref_n is None:
self.__ref_n = EDFTomoScan.get_ref_n(scan=self.path)
return self.__ref_n
@property
@docstring(TomoScanBase.pixel_size)
def pixel_size(self):
def pixel_size(self) -> Union[None, int]:
"""
:return: pixel size
......@@ -119,7 +115,7 @@ class EDFTomoScan(TomoScanBase):
@property
@docstring(TomoScanBase.dim_1)
def dim_1(self):
def dim_1(self) -> Union[None, int]:
"""
:return: image dim1
......@@ -131,7 +127,7 @@ class EDFTomoScan(TomoScanBase):
@property
@docstring(TomoScanBase.dim_2)
def dim_2(self):
def dim_2(self) -> Union[None, int]:
"""
:return: image dim2
......@@ -142,54 +138,50 @@ class EDFTomoScan(TomoScanBase):
return self.__dim2
@property
@docstring(TomoScanBase.ref_interval)
def ref_interval(self):
@docstring(TomoScanBase.ff_interval)
def ff_interval(self) -> Union[None, int]:
if self.__ref_on is None:
self.__ref_on = EDFTomoScan.get_ff_interval(scan=self.path)
return self.__ref_on
@property
@docstring(TomoScanBase.scan_range)
def scan_range(self):
def scan_range(self) -> Union[None, int]:
if self.__scan_range is None:
self.__scan_range = EDFTomoScan.get_scan_range(scan=self.path)
return self.__scan_range
@property
@docstring(TomoScanBase.flats)
def flats(self):
if self._refs is None:
self._refs = self.get_refs_url(scan_path=self.path)
return self._refs
def flats(self) -> Union[None, dict]:
"""
flats are given as a dictionary with index as key and DataUrl as
value"""
if self._flats is None:
self._flats = self.get_refs_url(scan_path=self.path)
return self._flats
@docstring(TomoScanBase.is_tomoscan_dir)
@staticmethod
def is_tomoscan_dir(directory, **kwargs):
print('info file is: %s', EDFTomoScan.get_info_file(directory=directory,
kwargs=kwargs))
def is_tomoscan_dir(directory: str, **kwargs) -> bool:
return os.path.isfile(EDFTomoScan.get_info_file(directory=directory,
kwargs=kwargs))
@staticmethod
def get_info_file(directory, **kwargs):
if directory is None:
return None
print('directory is', directory)
def get_info_file(directory: str, **kwargs) -> str:
basename = os.path.basename(directory)
assert basename != ''
print('basename is', basename)
info_file = os.path.join(directory, basename + EDFTomoScan.INFO_EXT)
print('info_file is', info_file)
if 'src_pattern' in kwargs and kwargs['srx_pattern'] is not None:
assert 'dest_pattern' in kwargs
info_file = info_file.replace(kwargs['src_pattern'],
kwargs['dest_pattern'],
1)
kwargs['dest_pattern'],
1)
return info_file
@docstring(TomoScanBase.is_abort)
def is_abort(self, **kwargs):
def is_abort(self, **kwargs) -> bool:
abort_file = os.path.basename(self.path) + self.ABORT_FILE
abort_file = os.path.join(self.path, abort_file)
if 'src_pattern' in kwargs and kwargs['src_pattern' is not None]:
......@@ -200,19 +192,19 @@ class EDFTomoScan(TomoScanBase):
@property
@docstring(TomoScanBase.darks)
def darks(self):
def darks(self) -> dict:
if self._darks is None:
self._darks = self.get_darks_url(scan_path=self.path)
return self._darks
@docstring(TomoScanBase.get_projections_url)
def get_projections_url(self):
@docstring(TomoScanBase.get_proj_angle_url)
def get_proj_angle_url(self) -> dict:
# TODO: we might use fabio.open_serie instead
if self.path is None:
_logger.warning('no path specified for scan, unable to retrieve'
' the projections')
return {}
n_projection = self.tomo_n()
n_projection = self.tomo_n
radios = EDFTomoScan.get_proj_paths(self.path)
sorted_keys = list(radios.keys())
sorted_keys.sort()
......@@ -241,12 +233,12 @@ class EDFTomoScan(TomoScanBase):
@docstring(TomoScanBase.update)
def update(self):
self.projections = EDFTomoScan.get_projections_url(self.path)
self.projections = EDFTomoScan.get_proj_angle_url(self.path)
self._darks = EDFTomoScan.get_darks_url(self.path)
self._refs = EDFTomoScan.get_refs_url(self.path)
self._flats = EDFTomoScan.get_refs_url(self.path)
@docstring(TomoScanBase.load_from_dict)
def load_from_dict(self, desc):
def load_from_dict(self, desc: Union[dict, io.TextIOWrapper]):
if isinstance(desc, io.TextIOWrapper):
data = json.load(desc)
else:
......@@ -259,7 +251,7 @@ class EDFTomoScan(TomoScanBase):
return self
@staticmethod
def get_proj_paths(scan):
def get_proj_paths(scan: str) -> dict:
"""
Return the dict of radios / projection for the given scan.
Keys of the dictionary is the slice number
......@@ -279,12 +271,14 @@ class EDFTomoScan(TomoScanBase):
for f in os.listdir(scan):
if EDFTomoScan.is_a_proj_path(f, scan):
gfile = os.path.join(scan, f)
files[EDFTomoScan.guess_index_frm_file_name(f, scan)] = gfile
index = EDFTomoScan.guess_index_frm_file_name(gfile)
files.update(extract_urls_from_edf(start_index=index,
file_=gfile))
return files
@staticmethod
def is_a_proj_path(fileName, scanID):
def is_a_proj_path(fileName: str, scanID: str) -> bool:
"""Return True if the given fileName can fit to a Radio name
"""
fileBasename = os.path.basename(fileName)
......@@ -307,7 +301,7 @@ class EDFTomoScan(TomoScanBase):
return False
@staticmethod
def guess_index_frm_file_name(_file):
def guess_index_frm_file_name(_file: str) -> Union[None, int]:
name = _file.rstrip('.edf')
ic = []
while name[-1].isdigit():
......@@ -321,7 +315,7 @@ class EDFTomoScan(TomoScanBase):
return int(''.join(orignalOrder))
@staticmethod
def get_tomo_n(scan):
def get_tomo_n(scan: str) -> Union[None, int]:
return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None,
key='TOMO_N',
......@@ -329,7 +323,7 @@ class EDFTomoScan(TomoScanBase):
key_aliases=['tomo_N', 'Tomo_N'])
@staticmethod
def get_dark_n(scan):
def get_dark_n(scan:str) -> Union[None, int]:
return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None,
key='DARK_N',
......@@ -337,7 +331,7 @@ class EDFTomoScan(TomoScanBase):
key_aliases=['dark_N', ])
@staticmethod
def get_ref_n(scan):
def get_ref_n(scan :str) -> Union[None, int]:
return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None,
key='REF_N',
......@@ -345,7 +339,7 @@ class EDFTomoScan(TomoScanBase):
key_aliases=['ref_N', ])
@staticmethod
def get_ff_interval(scan):
def get_ff_interval(scan: str) -> Union[None, int]:
return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None,
key='REF_ON',
......@@ -353,7 +347,7 @@ class EDFTomoScan(TomoScanBase):
key_aliases=['ref_On', ])
@staticmethod
def get_scan_range(scan):
def get_scan_range(scan: str) -> Union[None, int]:
return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None,
key='ScanRange',
......@@ -361,7 +355,7 @@ class EDFTomoScan(TomoScanBase):
key_aliases=['scanRange', ])
@staticmethod
def get_dim1_dim2(scan):
def get_dim1_dim2(scan: str) -> Union[None, tuple]:
_info_file = os.path.join(scan, os.path.basename(scan) + '.info')
d1 = EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None,
......@@ -377,7 +371,7 @@ class EDFTomoScan(TomoScanBase):
return d1, d2
@staticmethod
def get_pixel_size(scan):
def get_pixel_size(scan: str) -> Union[None, int]:
if os.path.isdir(scan) is False:
return None
value = EDFTomoScan.retrieve_information(scan=scan,
......@@ -402,7 +396,8 @@ class EDFTomoScan(TomoScanBase):
return None
@staticmethod
def get_darks_url(scan_path, prefix='dark', file_ext='.edf'):
def get_darks_url(scan_path: str, prefix: str='dark',
file_ext: str='.edf') -> dict:
"""
:param scan_path:
......@@ -413,25 +408,29 @@ class EDFTomoScan(TomoScanBase):
:type: str
:return: list of refs as silx's `DataUrl`
"""
res = []
res = {}
if os.path.isdir(scan_path) is False:
_logger.error(scan_path + ' is not a directory. Cannot extract '
'DarkHST files')
return res
for my_file in os.listdir(scan_path):
for file_ in os.listdir(scan_path):
_prefix = prefix
if prefix.endswith(file_ext):
_prefix = prefix.rstrip(file_ext)
if my_file.startswith(_prefix) and my_file.endswith(file_ext):
_file = my_file.lstrip(_prefix).rstrip(file_ext)
if _file == '' or _file.isnumeric() is True:
urls = extract_urls_from_edf(os.path.join(scan_path, my_file))
res.extend(urls)
assert os.path.isfile(res[-1])
if file_.startswith(_prefix) and file_.endswith(file_ext):
# usuelly the dark file name should be dark.edf, but some
# darkHSTXXXX remains...
file_fp = file_.lstrip(_prefix).rstrip(file_ext).lstrip('HST')
if file_fp == '' or file_fp.isnumeric() is True:
index = EDFTomoScan.guess_index_frm_file_name(file_)
urls = extract_urls_from_edf(os.path.join(scan_path, file_),
start_index=index)
res.update(urls)
return res
@staticmethod
def get_refs_url(scan_path, prefix='refHST', file_ext='.edf'):
def get_refs_url(scan_path: str, prefix: str='refHST',
file_ext: str='.edf') -> dict:
"""
:param scan_path:
......@@ -442,21 +441,24 @@ class EDFTomoScan(TomoScanBase):
:type: str
:return: list of refs as silx's `DataUrl`
"""
res = []
res = {}
if os.path.isdir(scan_path) is False:
_logger.error(scan_path + ' is not a directory. Cannot extract '
'RefHST files')
return res
for file in os.listdir(scan_path):
if file.startswith(prefix) and file.endswith(
file_ext):
res.append(os.path.join(scan_path, file))
assert os.path.isfile(res[-1])
for file_ in os.listdir(scan_path):
if file_.startswith(prefix) and file_.endswith(file_ext):
index = EDFTomoScan.guess_index_frm_file_name(file_)
file_fp = os.path.join(scan_path, file_)
urls = extract_urls_from_edf(start_index=index, file_=file_fp)
res.update(urls)
return res
@staticmethod
def retrieve_information(scan, ref_file, key, type_, key_aliases=None):
def retrieve_information(scan: str, ref_file: Union[str, None], key: str,
type_: type,
key_aliases: Union[list, tuple, None] = None):
"""
Try to retrieve information a .info file, an .xml or a flat field file.
......
tomoscan/esrf/hdf5scan.py
View file @ cf17630e
......@@ -88,6 +88,7 @@ class HDF5TomoScan(TomoScanBase):
# for now the default entry is 1_tomo but should change with time
# data caches
self._projections = None
self._tomo_n = None
# number of projections / radios
self._dark_n = None
......@@ -168,10 +169,10 @@ class HDF5TomoScan(TomoScanBase):
"""
if not os.path.exists(self.master_file):
return
self.projections = self.get_projections_url()
self.projections = self._get_projections_url()
# TODO: update darks and flats too
@docstring(TomoScanBase.get_projections_url)
@docstring(TomoScanBase.get_proj_angle_url)
def _get_projections_url(self):
"""
......@@ -240,9 +241,9 @@ class HDF5TomoScan(TomoScanBase):
self._ref_n = h5_file[self._entry][self._SCAN_META_PATH]['ref_n'][()]
return self._ref_n
@docstring(TomoScanBase.ref_interval)
@docstring(TomoScanBase.ff_interval)
@property
def ref_interval(self):
def ff_interval(self):
if self._ref_on is None and self.master_file and os.path.exists(self.master_file):
with h5py.File(self.master_file, 'r') as h5_file:
self._ref_on = h5_file[self._entry][self._SCAN_META_PATH]['ref_n'][()]
......@@ -278,8 +279,8 @@ class HDF5TomoScan(TomoScanBase):
self._pixel_size = h5_file[self._entry][self._DET_META_PATH]['pixel_size'][()]
return self._pixel_size
@docstring(TomoScanBase.get_projections_url)
def get_projections_url(self):
@docstring(TomoScanBase.get_proj_angle_url)
def get_proj_angle_url(self) -> dict:
can_compute = True
scan_range = self.scan_range
if scan_range is None:
......@@ -292,7 +293,8 @@ class HDF5TomoScan(TomoScanBase):
can_compute = False
tomo_n = self.tomo_n
if tomo_n is None:
_logger.warning('unable find the theoretical number of projection')
_logger.warning('unable to find the theoretical number of'
'projection')
can_compute = False
if can_compute is False:
......
tomoscan/esrf/test/test_edfscan.py
View file @ cf17630e
......@@ -41,6 +41,7 @@ import collections
import json
import shutil
import os
import silx.io.utils
logging.disable(logging.INFO)
......@@ -98,7 +99,7 @@ class FTSerieReconstructionTest(unittest.TestCase):
saveData(self.flatField_600, 'refHST_0600.edf', self.folderTwoFlats)
# save configuration two
saveData(self.darkData, 'darkHST.edf', self.folderOneFlat)
saveData(self.darkData, 'dark.edf', self.folderOneFlat)
saveData(self.flatField, 'refHST.edf', self.folderOneFlat)
self.acquiOneFlat = EDFTomoScan(self.folderOneFlat)
......@@ -109,26 +110,39 @@ class FTSerieReconstructionTest(unittest.TestCase):
shutil.rmtree(f)
unittest.TestCase.tearDown(self)
def testGetDark(self):
self.assertTrue(numpy.array_equal(
self.acquiOneFlat.get_darks(), self.darkData))
self.assertTrue(numpy.array_equal(
self.acquiTwoFlats.get_darks(), self.darkData))
def testDarks(self):
self.assertTrue(isinstance(self.acquiOneFlat.darks, dict))
self.assertEqual(len(self.acquiOneFlat.darks), 1)
self.assertTrue(isinstance(self.acquiTwoFlats.darks, dict))
self.assertEqual(len(self.acquiTwoFlats.darks), 1)
def testGetFlat(self):
# check one flat file with two ref
self.assertTrue(
numpy.array_equal(self.acquiOneFlat.get_flat(), self.flatField))
numpy.array_equal(silx.io.utils.get_data(self.acquiOneFlat.darks[0]),
self.darkData)
)
self.assertTrue(
numpy.array_equal(silx.io.utils.get_data(self.acquiTwoFlats.darks[0]),
self.darkData)
)
def testFlats(self):
# check one flat file with two ref
self.assertEqual(len(self.acquiOneFlat.flats), 1)
self.assertTrue(isinstance(self.acquiOneFlat.flats[0], silx.io.url.DataUrl))
data = silx.io.utils.get_data(self.acquiOneFlat.flats[0])
numpy.array_equal(data, self.flatField)
# check two flat files
self.assertTrue(numpy.array_equal(
self.acquiTwoFlats.get_flat(), numpy.arange(100).reshape(10, 10)))
self.assertTrue(numpy.array_equal(
self.acquiTwoFlats.get_flat(index=300), numpy.arange(100).reshape(10, 10)))
self.assertTrue(numpy.array_equal(
self.acquiTwoFlats.get_flat(index=600), self.flatField_600))
self.assertTrue(numpy.array_equal(
self.acquiTwoFlats.get_flat(index=0), self.flatField_0))
self.assertTrue(
numpy.array_equal(silx.io.utils.get_data(self.acquiTwoFlats.flats[600]),
self.flatField_600)
)
self.assertTrue(
numpy.array_equal(silx.io.utils.get_data(self.acquiTwoFlats.flats[0]),
self.flatField_0)
)
self.assertTrue(12 not in self.acquiTwoFlats.flats)
class TestScanValidatorFindFiles(unittest.TestCase):
......@@ -164,20 +178,6 @@ class TestScanValidatorFindFiles(unittest.TestCase):
nFound = len(EDFTomoScan.get_proj_paths(self.path))
self.assertTrue(nFound == self.N_RADIO)
def testGetReconstructionsPaths(self):
nFound = len(EDFTomoScan.get_recons_paths(self.path))
self.assertTrue(nFound == self.N_RECONS + self.N_PAG_RECONS)
def testGetSliceReconstruction(self):
self.assertTrue(
EDFTomoScan.getIndexReconstructed('dfadf_slice_32.edf', 'dfadf') == 32)
self.assertTrue(
EDFTomoScan.getIndexReconstructed('dfadf_slice_slice_002.edf', 'dfadf') == 2)
self.assertTrue(
EDFTomoScan.getIndexReconstructed('scan_3slice_0050.edf', 'scan_3') == 50)
self.assertTrue(
EDFTomoScan.getIndexReconstructed('scan3slice_012.edf', 'scan3') == 12)
class TestRadioPath(unittest.TestCase):
"""Test static method getRadioPaths for EDFTomoScan"""
......@@ -310,45 +310,6 @@ class TestRadioPath(unittest.TestCase):
self.assertTrue(nbRadio == 105)