Commit cf17630e authored by payno's avatar payno
Browse files

second commit.

parent 45fd8c5a
python -m unittest tomoscan.test.suite -v
\ No newline at end of file stages:
- test_library
- test_notbooks
variables:
http_proxy: http://proxy.esrf.fr:3128
https_proxy: http://proxy.esrf.fr:3128
no_proxy: .esrf.fr,localhost
ORANGE_WEB_LOG: 'False'
.build_template: &test_linux_template
stage: test_library
before_script:
- arch
- export PYTHONPATH="${PYTHONPATH}:/usr/lib/python3/dist-packages/"
- export LD_LIBRARY_PATH=/lib/i386-linux-gnu/:${LD_LIBRARY_PATH}
- export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
- export ORANGE_WEB_LOG='False'
- python --version
- python -m pip install pip --upgrade
- python -m pip install setuptools --upgrade
- python -m pip install - requirements.txt
- python -m pip install .
script:
- python -m unittest tomoscan.test.suite -v
test:python3.5-stretch-pyqt5:
image: docker-registry.esrf.fr/dau/tomwer:python3.5_stretch_pyqt5
<<: *test_linux_template
test:python3.7_stretch_pyqt5:
image: docker-registry.esrf.fr/dau/tomwer:python3.7_stretch_pyqt5
<<: *test_linux_template
test:windows-conda:
stage: test_library
image: python:3-windowsservercore
tags:
- win
before_script:
- SET "PATH=C:\\miniconda3;C:\\miniconda3\\Scripts;%PATH%"
- SET "PATH=C:\\miniconda3\\include;%PATH%"
- SET "PATH=C:\\miniconda3\\python3.7\\site-packages\\numpy\\core\\include;%PATH%"
- SET "PATH=C:\\windows\\system32\\config\\systemprofile\\appdata\\roaming\\python\\python37\\site-packages;%PATH%"
- SET "PATH=C:\\windows\\system32\\config\\systemprofile\\appdata\\roaming\\python\\python37\\site-packages\\numpy\\core\\include;%PATH%"
- SET "ORANGE_WEB_LOG='False'"
- pip install virtualenv
- virtualenv venv
- call venv\\Scripts\\activate.bat
- python --version
- pip install pip --upgrade
- python -m pip install - requirements.txt
- python -m pip install .
script:
- python -m unittest tomoscan.test.suite -v
test:test-tomoscan-tutorials:
stage: test_notbooks
image: docker-registry.esrf.fr/dau/tomwer:python3.7_buster_pyqt5
before_script:
- arch
- export PYTHONPATH="${PYTHONPATH}:/usr/lib/python3/dist-packages/"
- export LD_LIBRARY_PATH=/lib/i386-linux-gnu/:${LD_LIBRARY_PATH}
- export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
- export ORANGE_WEB_LOG='False'
- python --version
- python -m pip install pip --upgrade
- python -m pip install setuptools --upgrade
- python -m pip install numpy --upgrade
- python -m pip install matplotlib
- python -m pip install jupyter_client
- python -m pip install nbconvert
- python -m pip install ipykernel
- python -m pip install -r requirements-dev.txt
- python -m pip install -r requirements-test.txt
- python -m pip install fabio --upgrade --pre
- python -m pip install silx --upgrade --pre
- python -m pip install .
script:
- jupyter nbconvert --to notebook --execute ../doc/tutorials/edf_scan.ipynb
- jupyter nbconvert --to notebook --execute ../doc/tutorials/hdf5_scan.ipynb
...@@ -28,19 +28,14 @@ __license__ = "MIT" ...@@ -28,19 +28,14 @@ __license__ = "MIT"
__date__ = "10/10/2019" __date__ = "10/10/2019"
import glob
import os import os
import re import re
from collections import OrderedDict
import fabio import fabio
from lxml import etree from lxml import etree
import numpy
import json import json
import io import io
from typing import Union
from silx.io.url import DataUrl from silx.io.url import DataUrl
from glob import glob
from ..scanbase import TomoScanBase from ..scanbase import TomoScanBase
from .utils import get_parameters_frm_par_or_info, extract_urls_from_edf from .utils import get_parameters_frm_par_or_info, extract_urls_from_edf
from ..unitsystem import metricsystem from ..unitsystem import metricsystem
...@@ -69,12 +64,13 @@ class EDFTomoScan(TomoScanBase): ...@@ -69,12 +64,13 @@ class EDFTomoScan(TomoScanBase):
_SCHEME = 'fabio' _SCHEME = 'fabio'
def __init__(self, scan): def __init__(self, scan: Union[str, None]):
TomoScanBase.__init__(self, scan=scan, type_=self._TYPE) TomoScanBase.__init__(self, scan=scan, type_=self._TYPE)
# data caches # data caches
self._darks = None self._darks = None
self._refs = None self._flats = None
self._projections = None
self.__tomo_n = None self.__tomo_n = None
self.__ref_n = None self.__ref_n = None
self.__dark_n = None self.__dark_n = None
...@@ -86,28 +82,28 @@ class EDFTomoScan(TomoScanBase): ...@@ -86,28 +82,28 @@ class EDFTomoScan(TomoScanBase):
@docstring(TomoScanBase.tomo_n) @docstring(TomoScanBase.tomo_n)
@property @property
def tomo_n(self): def tomo_n(self) -> Union[None, int]:
if self.__tomo_n is None: if self.__tomo_n is None:
self.__tomo_n = EDFTomoScan.get_tomo_n(scan=self.path) self.__tomo_n = EDFTomoScan.get_tomo_n(scan=self.path)
return self.__tomo_n return self.__tomo_n
@property @property
@docstring(TomoScanBase.dark_n) @docstring(TomoScanBase.dark_n)
def dark_n(self): def dark_n(self) -> Union[None, int]:
if self.__dark_n is None: if self.__dark_n is None:
self.__dark_n = EDFTomoScan.get_dark_n(scan=self.path) self.__dark_n = EDFTomoScan.get_dark_n(scan=self.path)
return self.__dark_n return self.__dark_n
@property @property
@docstring(TomoScanBase.ref_n) @docstring(TomoScanBase.ref_n)
def ref_n(self): def ref_n(self) -> Union[None, int]:
if self.__ref_n is None: if self.__ref_n is None:
self.__ref_n = EDFTomoScan.get_ref_n(scan=self.path) self.__ref_n = EDFTomoScan.get_ref_n(scan=self.path)
return self.__ref_n return self.__ref_n
@property @property
@docstring(TomoScanBase.pixel_size) @docstring(TomoScanBase.pixel_size)
def pixel_size(self): def pixel_size(self) -> Union[None, int]:
""" """
:return: pixel size :return: pixel size
...@@ -119,7 +115,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -119,7 +115,7 @@ class EDFTomoScan(TomoScanBase):
@property @property
@docstring(TomoScanBase.dim_1) @docstring(TomoScanBase.dim_1)
def dim_1(self): def dim_1(self) -> Union[None, int]:
""" """
:return: image dim1 :return: image dim1
...@@ -131,7 +127,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -131,7 +127,7 @@ class EDFTomoScan(TomoScanBase):
@property @property
@docstring(TomoScanBase.dim_2) @docstring(TomoScanBase.dim_2)
def dim_2(self): def dim_2(self) -> Union[None, int]:
""" """
:return: image dim2 :return: image dim2
...@@ -142,54 +138,50 @@ class EDFTomoScan(TomoScanBase): ...@@ -142,54 +138,50 @@ class EDFTomoScan(TomoScanBase):
return self.__dim2 return self.__dim2
@property @property
@docstring(TomoScanBase.ref_interval) @docstring(TomoScanBase.ff_interval)
def ref_interval(self): def ff_interval(self) -> Union[None, int]:
if self.__ref_on is None: if self.__ref_on is None:
self.__ref_on = EDFTomoScan.get_ff_interval(scan=self.path) self.__ref_on = EDFTomoScan.get_ff_interval(scan=self.path)
return self.__ref_on return self.__ref_on
@property @property
@docstring(TomoScanBase.scan_range) @docstring(TomoScanBase.scan_range)
def scan_range(self): def scan_range(self) -> Union[None, int]:
if self.__scan_range is None: if self.__scan_range is None:
self.__scan_range = EDFTomoScan.get_scan_range(scan=self.path) self.__scan_range = EDFTomoScan.get_scan_range(scan=self.path)
return self.__scan_range return self.__scan_range
@property @property
@docstring(TomoScanBase.flats) @docstring(TomoScanBase.flats)
def flats(self): def flats(self) -> Union[None, dict]:
if self._refs is None: """
self._refs = self.get_refs_url(scan_path=self.path) flats are given as a dictionary with index as key and DataUrl as
return self._refs value"""
if self._flats is None:
self._flats = self.get_refs_url(scan_path=self.path)
return self._flats
@docstring(TomoScanBase.is_tomoscan_dir) @docstring(TomoScanBase.is_tomoscan_dir)
@staticmethod @staticmethod
def is_tomoscan_dir(directory, **kwargs): def is_tomoscan_dir(directory: str, **kwargs) -> bool:
print('info file is: %s', EDFTomoScan.get_info_file(directory=directory,
kwargs=kwargs))
return os.path.isfile(EDFTomoScan.get_info_file(directory=directory, return os.path.isfile(EDFTomoScan.get_info_file(directory=directory,
kwargs=kwargs)) kwargs=kwargs))
@staticmethod @staticmethod
def get_info_file(directory, **kwargs): def get_info_file(directory: str, **kwargs) -> str:
if directory is None:
return None
print('directory is', directory)
basename = os.path.basename(directory) basename = os.path.basename(directory)
assert basename != '' assert basename != ''
print('basename is', basename)
info_file = os.path.join(directory, basename + EDFTomoScan.INFO_EXT) info_file = os.path.join(directory, basename + EDFTomoScan.INFO_EXT)
print('info_file is', info_file)
if 'src_pattern' in kwargs and kwargs['srx_pattern'] is not None: if 'src_pattern' in kwargs and kwargs['srx_pattern'] is not None:
assert 'dest_pattern' in kwargs assert 'dest_pattern' in kwargs
info_file = info_file.replace(kwargs['src_pattern'], info_file = info_file.replace(kwargs['src_pattern'],
kwargs['dest_pattern'], kwargs['dest_pattern'],
1) 1)
return info_file return info_file
@docstring(TomoScanBase.is_abort) @docstring(TomoScanBase.is_abort)
def is_abort(self, **kwargs): def is_abort(self, **kwargs) -> bool:
abort_file = os.path.basename(self.path) + self.ABORT_FILE abort_file = os.path.basename(self.path) + self.ABORT_FILE
abort_file = os.path.join(self.path, abort_file) abort_file = os.path.join(self.path, abort_file)
if 'src_pattern' in kwargs and kwargs['src_pattern' is not None]: if 'src_pattern' in kwargs and kwargs['src_pattern' is not None]:
...@@ -200,19 +192,19 @@ class EDFTomoScan(TomoScanBase): ...@@ -200,19 +192,19 @@ class EDFTomoScan(TomoScanBase):
@property @property
@docstring(TomoScanBase.darks) @docstring(TomoScanBase.darks)
def darks(self): def darks(self) -> dict:
if self._darks is None: if self._darks is None:
self._darks = self.get_darks_url(scan_path=self.path) self._darks = self.get_darks_url(scan_path=self.path)
return self._darks return self._darks
@docstring(TomoScanBase.get_projections_url) @docstring(TomoScanBase.get_proj_angle_url)
def get_projections_url(self): def get_proj_angle_url(self) -> dict:
# TODO: we might use fabio.open_serie instead # TODO: we might use fabio.open_serie instead
if self.path is None: if self.path is None:
_logger.warning('no path specified for scan, unable to retrieve' _logger.warning('no path specified for scan, unable to retrieve'
' the projections') ' the projections')
return {} return {}
n_projection = self.tomo_n() n_projection = self.tomo_n
radios = EDFTomoScan.get_proj_paths(self.path) radios = EDFTomoScan.get_proj_paths(self.path)
sorted_keys = list(radios.keys()) sorted_keys = list(radios.keys())
sorted_keys.sort() sorted_keys.sort()
...@@ -241,12 +233,12 @@ class EDFTomoScan(TomoScanBase): ...@@ -241,12 +233,12 @@ class EDFTomoScan(TomoScanBase):
@docstring(TomoScanBase.update) @docstring(TomoScanBase.update)
def update(self): def update(self):
self.projections = EDFTomoScan.get_projections_url(self.path) self.projections = EDFTomoScan.get_proj_angle_url(self.path)
self._darks = EDFTomoScan.get_darks_url(self.path) self._darks = EDFTomoScan.get_darks_url(self.path)
self._refs = EDFTomoScan.get_refs_url(self.path) self._flats = EDFTomoScan.get_refs_url(self.path)
@docstring(TomoScanBase.load_from_dict) @docstring(TomoScanBase.load_from_dict)
def load_from_dict(self, desc): def load_from_dict(self, desc: Union[dict, io.TextIOWrapper]):
if isinstance(desc, io.TextIOWrapper): if isinstance(desc, io.TextIOWrapper):
data = json.load(desc) data = json.load(desc)
else: else:
...@@ -259,7 +251,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -259,7 +251,7 @@ class EDFTomoScan(TomoScanBase):
return self return self
@staticmethod @staticmethod
def get_proj_paths(scan): def get_proj_paths(scan: str) -> dict:
""" """
Return the dict of radios / projection for the given scan. Return the dict of radios / projection for the given scan.
Keys of the dictionary is the slice number Keys of the dictionary is the slice number
...@@ -279,12 +271,14 @@ class EDFTomoScan(TomoScanBase): ...@@ -279,12 +271,14 @@ class EDFTomoScan(TomoScanBase):
for f in os.listdir(scan): for f in os.listdir(scan):
if EDFTomoScan.is_a_proj_path(f, scan): if EDFTomoScan.is_a_proj_path(f, scan):
gfile = os.path.join(scan, f) gfile = os.path.join(scan, f)
files[EDFTomoScan.guess_index_frm_file_name(f, scan)] = gfile index = EDFTomoScan.guess_index_frm_file_name(gfile)
files.update(extract_urls_from_edf(start_index=index,
file_=gfile))
return files return files
@staticmethod @staticmethod
def is_a_proj_path(fileName, scanID): def is_a_proj_path(fileName: str, scanID: str) -> bool:
"""Return True if the given fileName can fit to a Radio name """Return True if the given fileName can fit to a Radio name
""" """
fileBasename = os.path.basename(fileName) fileBasename = os.path.basename(fileName)
...@@ -307,7 +301,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -307,7 +301,7 @@ class EDFTomoScan(TomoScanBase):
return False return False
@staticmethod @staticmethod
def guess_index_frm_file_name(_file): def guess_index_frm_file_name(_file: str) -> Union[None, int]:
name = _file.rstrip('.edf') name = _file.rstrip('.edf')
ic = [] ic = []
while name[-1].isdigit(): while name[-1].isdigit():
...@@ -321,7 +315,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -321,7 +315,7 @@ class EDFTomoScan(TomoScanBase):
return int(''.join(orignalOrder)) return int(''.join(orignalOrder))
@staticmethod @staticmethod
def get_tomo_n(scan): def get_tomo_n(scan: str) -> Union[None, int]:
return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan), return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None, ref_file=None,
key='TOMO_N', key='TOMO_N',
...@@ -329,7 +323,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -329,7 +323,7 @@ class EDFTomoScan(TomoScanBase):
key_aliases=['tomo_N', 'Tomo_N']) key_aliases=['tomo_N', 'Tomo_N'])
@staticmethod @staticmethod
def get_dark_n(scan): def get_dark_n(scan:str) -> Union[None, int]:
return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan), return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None, ref_file=None,
key='DARK_N', key='DARK_N',
...@@ -337,7 +331,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -337,7 +331,7 @@ class EDFTomoScan(TomoScanBase):
key_aliases=['dark_N', ]) key_aliases=['dark_N', ])
@staticmethod @staticmethod
def get_ref_n(scan): def get_ref_n(scan :str) -> Union[None, int]:
return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan), return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None, ref_file=None,
key='REF_N', key='REF_N',
...@@ -345,7 +339,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -345,7 +339,7 @@ class EDFTomoScan(TomoScanBase):
key_aliases=['ref_N', ]) key_aliases=['ref_N', ])
@staticmethod @staticmethod
def get_ff_interval(scan): def get_ff_interval(scan: str) -> Union[None, int]:
return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan), return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None, ref_file=None,
key='REF_ON', key='REF_ON',
...@@ -353,7 +347,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -353,7 +347,7 @@ class EDFTomoScan(TomoScanBase):
key_aliases=['ref_On', ]) key_aliases=['ref_On', ])
@staticmethod @staticmethod
def get_scan_range(scan): def get_scan_range(scan: str) -> Union[None, int]:
return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan), return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None, ref_file=None,
key='ScanRange', key='ScanRange',
...@@ -361,7 +355,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -361,7 +355,7 @@ class EDFTomoScan(TomoScanBase):
key_aliases=['scanRange', ]) key_aliases=['scanRange', ])
@staticmethod @staticmethod
def get_dim1_dim2(scan): def get_dim1_dim2(scan: str) -> Union[None, tuple]:
_info_file = os.path.join(scan, os.path.basename(scan) + '.info') _info_file = os.path.join(scan, os.path.basename(scan) + '.info')
d1 = EDFTomoScan.retrieve_information(scan=os.path.abspath(scan), d1 = EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
ref_file=None, ref_file=None,
...@@ -377,7 +371,7 @@ class EDFTomoScan(TomoScanBase): ...@@ -377,7 +371,7 @@ class EDFTomoScan(TomoScanBase):
return d1, d2 return d1, d2
@staticmethod @staticmethod
def get_pixel_size(scan): def get_pixel_size(scan: str) -> Union[None, int]:
if os.path.isdir(scan) is False: if os.path.isdir(scan) is False:
return None return None
value = EDFTomoScan.retrieve_information(scan=scan, value = EDFTomoScan.retrieve_information(scan=scan,
...@@ -402,7 +396,8 @@ class EDFTomoScan(TomoScanBase): ...@@ -402,7 +396,8 @@ class EDFTomoScan(TomoScanBase):
return None return None
@staticmethod @staticmethod
def get_darks_url(scan_path, prefix='dark', file_ext='.edf'): def get_darks_url(scan_path: str, prefix: str='dark',
file_ext: str='.edf') -> dict:
""" """
:param scan_path: :param scan_path:
...@@ -413,25 +408,29 @@ class EDFTomoScan(TomoScanBase): ...@@ -413,25 +408,29 @@ class EDFTomoScan(TomoScanBase):
:type: str :type: str
:return: list of refs as silx's `DataUrl` :return: list of refs as silx's `DataUrl`
""" """
res = [] res = {}
if os.path.isdir(scan_path) is False: if os.path.isdir(scan_path) is False:
_logger.error(scan_path + ' is not a directory. Cannot extract ' _logger.error(scan_path + ' is not a directory. Cannot extract '
'DarkHST files') 'DarkHST files')
return res return res
for my_file in os.listdir(scan_path): for file_ in os.listdir(scan_path):
_prefix = prefix _prefix = prefix
if prefix.endswith(file_ext): if prefix.endswith(file_ext):
_prefix = prefix.rstrip(file_ext) _prefix = prefix.rstrip(file_ext)
if my_file.startswith(_prefix) and my_file.endswith(file_ext): if file_.startswith(_prefix) and file_.endswith(file_ext):
_file = my_file.lstrip(_prefix).rstrip(file_ext) # usuelly the dark file name should be dark.edf, but some
if _file == '' or _file.isnumeric() is True: # darkHSTXXXX remains...
urls = extract_urls_from_edf(os.path.join(scan_path, my_file)) file_fp = file_.lstrip(_prefix).rstrip(file_ext).lstrip('HST')
res.extend(urls) if file_fp == '' or file_fp.isnumeric() is True:
assert os.path.isfile(res[-1]) index = EDFTomoScan.guess_index_frm_file_name(file_)
urls = extract_urls_from_edf(os.path.join(scan_path, file_),
start_index=index)
res.update(urls)
return res return res
@staticmethod @staticmethod
def get_refs_url(scan_path, prefix='refHST', file_ext='.edf'): def get_refs_url(scan_path: str, prefix: str='refHST',
file_ext: str='.edf') -> dict:
""" """
:param scan_path: :param scan_path:
...@@ -442,21 +441,24 @@ class EDFTomoScan(TomoScanBase): ...@@ -442,21 +441,24 @@ class EDFTomoScan(TomoScanBase):
:type: str :type: str
:return: list of refs as silx's `DataUrl` :return: list of refs as silx's `DataUrl`
""" """
res = [] res = {}
if os.path.isdir(scan_path) is False: if os.path.isdir(scan_path) is False:
_logger.error(scan_path + ' is not a directory. Cannot extract ' _logger.error(scan_path + ' is not a directory. Cannot extract '
'RefHST files') 'RefHST files')
return res return res
for file in os.listdir(scan_path): for file_ in os.listdir(scan_path):
if file.startswith(prefix) and file.endswith( if file_.startswith(prefix) and file_.endswith(file_ext):
file_ext): index = EDFTomoScan.guess_index_frm_file_name(file_)
res.append(os.path.join(scan_path, file)) file_fp = os.path.join(scan_path, file_)
assert os.path.isfile(res[-1]) urls = extract_urls_from_edf(start_index=index, file_=file_fp)