second commit.

tomoscan/.gitlab-ci.yml

-python -m unittest tomoscan.test.suite -v
+
\ No newline at end of file
+stages:
+  - test_library
+  - test_notbooks
+
+variables:
+  http_proxy: http://proxy.esrf.fr:3128
+  https_proxy: http://proxy.esrf.fr:3128
+  no_proxy: .esrf.fr,localhost
+  ORANGE_WEB_LOG: 'False'
+
+
+.build_template: &test_linux_template
+  stage: test_library
+  before_script:
+    - arch
+    - export PYTHONPATH="${PYTHONPATH}:/usr/lib/python3/dist-packages/"
+    - export LD_LIBRARY_PATH=/lib/i386-linux-gnu/:${LD_LIBRARY_PATH}
+    - export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
+    - export ORANGE_WEB_LOG='False'
+    - python --version
+    - python -m pip install pip --upgrade
+    - python -m pip install setuptools --upgrade
+    - python -m pip install - requirements.txt
+    - python -m pip install .
+  script:
+    - python -m unittest tomoscan.test.suite -v
+
+
+test:python3.5-stretch-pyqt5:
+  image: docker-registry.esrf.fr/dau/tomwer:python3.5_stretch_pyqt5
+  <<: *test_linux_template
+
+
+test:python3.7_stretch_pyqt5:
+  image: docker-registry.esrf.fr/dau/tomwer:python3.7_stretch_pyqt5
+  <<: *test_linux_template
+
+
+test:windows-conda:
+  stage: test_library
+  image: python:3-windowsservercore
+  tags:
+    - win
+  before_script:
+    - SET "PATH=C:\\miniconda3;C:\\miniconda3\\Scripts;%PATH%"
+    - SET "PATH=C:\\miniconda3\\include;%PATH%"
+    - SET "PATH=C:\\miniconda3\\python3.7\\site-packages\\numpy\\core\\include;%PATH%"
+    - SET "PATH=C:\\windows\\system32\\config\\systemprofile\\appdata\\roaming\\python\\python37\\site-packages;%PATH%"
+    - SET "PATH=C:\\windows\\system32\\config\\systemprofile\\appdata\\roaming\\python\\python37\\site-packages\\numpy\\core\\include;%PATH%"
+    - SET "ORANGE_WEB_LOG='False'"
+    - pip install virtualenv
+    - virtualenv venv
+    - call venv\\Scripts\\activate.bat
+    - python --version
+    - pip install pip --upgrade
+    - python -m pip install - requirements.txt
+    - python -m pip install .
+  script:
+    - python -m unittest tomoscan.test.suite -v
+
+
+test:test-tomoscan-tutorials:
+  stage: test_notbooks
+  image: docker-registry.esrf.fr/dau/tomwer:python3.7_buster_pyqt5
+  before_script:
+    - arch
+    - export PYTHONPATH="${PYTHONPATH}:/usr/lib/python3/dist-packages/"
+    - export LD_LIBRARY_PATH=/lib/i386-linux-gnu/:${LD_LIBRARY_PATH}
+    - export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
+    - export ORANGE_WEB_LOG='False'
+    - python --version
+    - python -m pip install pip --upgrade
+    - python -m pip install setuptools --upgrade
+    - python -m pip install numpy --upgrade
+    - python -m pip install matplotlib
+    - python -m pip install jupyter_client
+    - python -m pip install nbconvert
+    - python -m pip install ipykernel
+    - python -m pip install -r requirements-dev.txt
+    - python -m pip install -r requirements-test.txt
+    - python -m pip install fabio --upgrade --pre
+    - python -m pip install silx --upgrade --pre
+    - python -m pip install .
+  script:
+    - jupyter nbconvert --to notebook --execute ../doc/tutorials/edf_scan.ipynb
+    - jupyter nbconvert --to notebook --execute ../doc/tutorials/hdf5_scan.ipynb

tomoscan/esrf/edfscan.py

@@ -28,19 +28,14 @@ __license__ = "MIT"
 __date__ = "10/10/2019"
 
 
-import glob
 import os
 import re
-from collections import OrderedDict
-
 import fabio
 from lxml import etree
-import numpy
 import json
 import io
-
+from typing import Union
 from silx.io.url import DataUrl
-from glob import glob
 from ..scanbase import TomoScanBase
 from .utils import get_parameters_frm_par_or_info, extract_urls_from_edf
 from ..unitsystem import metricsystem
@@ -69,12 +64,13 @@ class EDFTomoScan(TomoScanBase):
 
     _SCHEME = 'fabio'
 
-    def __init__(self, scan):
+    def __init__(self, scan: Union[str, None]):
         TomoScanBase.__init__(self, scan=scan, type_=self._TYPE)
 
         # data caches
         self._darks = None
-        self._refs = None
+        self._flats = None
+        self._projections = None
         self.__tomo_n = None
         self.__ref_n = None
         self.__dark_n = None
@@ -86,28 +82,28 @@ class EDFTomoScan(TomoScanBase):
 
     @docstring(TomoScanBase.tomo_n)
     @property
-    def tomo_n(self):
+    def tomo_n(self) -> Union[None, int]:
         if self.__tomo_n is None:
             self.__tomo_n = EDFTomoScan.get_tomo_n(scan=self.path)
         return self.__tomo_n
 
     @property
     @docstring(TomoScanBase.dark_n)
-    def dark_n(self):
+    def dark_n(self) -> Union[None, int]:
         if self.__dark_n is None:
             self.__dark_n = EDFTomoScan.get_dark_n(scan=self.path)
         return self.__dark_n
 
     @property
     @docstring(TomoScanBase.ref_n)
-    def ref_n(self):
+    def ref_n(self) -> Union[None, int]:
         if self.__ref_n is None:
             self.__ref_n = EDFTomoScan.get_ref_n(scan=self.path)
         return self.__ref_n
 
     @property
     @docstring(TomoScanBase.pixel_size)
-    def pixel_size(self):
+    def pixel_size(self) -> Union[None, int]:
         """
 
         :return: pixel size
@@ -119,7 +115,7 @@ class EDFTomoScan(TomoScanBase):
 
     @property
     @docstring(TomoScanBase.dim_1)
-    def dim_1(self):
+    def dim_1(self) -> Union[None, int]:
         """
 
         :return: image dim1
@@ -131,7 +127,7 @@ class EDFTomoScan(TomoScanBase):
 
     @property
     @docstring(TomoScanBase.dim_2)
-    def dim_2(self):
+    def dim_2(self) -> Union[None, int]:
         """
 
         :return: image dim2
@@ -142,54 +138,50 @@ class EDFTomoScan(TomoScanBase):
         return self.__dim2
 
     @property
-    @docstring(TomoScanBase.ref_interval)
-    def ref_interval(self):
+    @docstring(TomoScanBase.ff_interval)
+    def ff_interval(self) -> Union[None, int]:
         if self.__ref_on is None:
             self.__ref_on = EDFTomoScan.get_ff_interval(scan=self.path)
         return self.__ref_on
 
     @property
     @docstring(TomoScanBase.scan_range)
-    def scan_range(self):
+    def scan_range(self) -> Union[None, int]:
         if self.__scan_range is None:
             self.__scan_range = EDFTomoScan.get_scan_range(scan=self.path)
         return self.__scan_range
 
     @property
     @docstring(TomoScanBase.flats)
-    def flats(self):
-        if self._refs is None:
-            self._refs = self.get_refs_url(scan_path=self.path)
-        return self._refs
+    def flats(self) -> Union[None, dict]:
+        """
+        flats are given as a dictionary with index as key and DataUrl as
+        value"""
+        if self._flats is None:
+            self._flats = self.get_refs_url(scan_path=self.path)
+        return self._flats
 
     @docstring(TomoScanBase.is_tomoscan_dir)
     @staticmethod
-    def is_tomoscan_dir(directory, **kwargs):
-        print('info file is: %s', EDFTomoScan.get_info_file(directory=directory,
-                                                        kwargs=kwargs))
+    def is_tomoscan_dir(directory: str, **kwargs) -> bool:
         return os.path.isfile(EDFTomoScan.get_info_file(directory=directory,
                                                         kwargs=kwargs))
 
     @staticmethod
-    def get_info_file(directory, **kwargs):
-        if directory is None:
-            return None
-        print('directory is', directory)
+    def get_info_file(directory: str, **kwargs) -> str:
         basename = os.path.basename(directory)
         assert basename != ''
-        print('basename is', basename)
         info_file = os.path.join(directory, basename + EDFTomoScan.INFO_EXT)
-        print('info_file is', info_file)
 
         if 'src_pattern' in kwargs and kwargs['srx_pattern'] is not None:
             assert 'dest_pattern' in kwargs
             info_file = info_file.replace(kwargs['src_pattern'],
-                                            kwargs['dest_pattern'],
-                                            1)
+                                          kwargs['dest_pattern'],
+                                          1)
         return info_file
 
     @docstring(TomoScanBase.is_abort)
-    def is_abort(self,  **kwargs):
+    def is_abort(self,  **kwargs) -> bool:
         abort_file = os.path.basename(self.path) + self.ABORT_FILE
         abort_file = os.path.join(self.path, abort_file)
         if 'src_pattern' in kwargs and kwargs['src_pattern' is not None]:
@@ -200,19 +192,19 @@ class EDFTomoScan(TomoScanBase):
 
     @property
     @docstring(TomoScanBase.darks)
-    def darks(self):
+    def darks(self) -> dict:
         if self._darks is None:
             self._darks = self.get_darks_url(scan_path=self.path)
         return self._darks
 
-    @docstring(TomoScanBase.get_projections_url)
-    def get_projections_url(self):
+    @docstring(TomoScanBase.get_proj_angle_url)
+    def get_proj_angle_url(self) -> dict:
         # TODO: we might use fabio.open_serie instead
         if self.path is None:
             _logger.warning('no path specified for scan, unable to retrieve'
                             ' the projections')
             return {}
-        n_projection = self.tomo_n()
+        n_projection = self.tomo_n
         radios = EDFTomoScan.get_proj_paths(self.path)
         sorted_keys = list(radios.keys())
         sorted_keys.sort()
@@ -241,12 +233,12 @@ class EDFTomoScan(TomoScanBase):
 
     @docstring(TomoScanBase.update)
     def update(self):
-        self.projections = EDFTomoScan.get_projections_url(self.path)
+        self.projections = EDFTomoScan.get_proj_angle_url(self.path)
         self._darks = EDFTomoScan.get_darks_url(self.path)
-        self._refs = EDFTomoScan.get_refs_url(self.path)
+        self._flats = EDFTomoScan.get_refs_url(self.path)
 
     @docstring(TomoScanBase.load_from_dict)
-    def load_from_dict(self, desc):
+    def load_from_dict(self, desc: Union[dict, io.TextIOWrapper]):
         if isinstance(desc, io.TextIOWrapper):
             data = json.load(desc)
         else:
@@ -259,7 +251,7 @@ class EDFTomoScan(TomoScanBase):
         return self
 
     @staticmethod
-    def get_proj_paths(scan):
+    def get_proj_paths(scan: str) -> dict:
         """
         Return the dict of radios / projection for the given scan.
         Keys of the dictionary is the slice number
@@ -279,12 +271,14 @@ class EDFTomoScan(TomoScanBase):
             for f in os.listdir(scan):
                 if EDFTomoScan.is_a_proj_path(f, scan):
                     gfile = os.path.join(scan, f)
-                    files[EDFTomoScan.guess_index_frm_file_name(f, scan)] = gfile
+                    index = EDFTomoScan.guess_index_frm_file_name(gfile)
+                    files.update(extract_urls_from_edf(start_index=index,
+                                                       file_=gfile))
 
         return files
 
     @staticmethod
-    def is_a_proj_path(fileName, scanID):
+    def is_a_proj_path(fileName: str, scanID: str) -> bool:
         """Return True if the given fileName can fit to a Radio name
         """
         fileBasename = os.path.basename(fileName)
@@ -307,7 +301,7 @@ class EDFTomoScan(TomoScanBase):
         return False
 
     @staticmethod
-    def guess_index_frm_file_name(_file):
+    def guess_index_frm_file_name(_file: str) -> Union[None, int]:
         name = _file.rstrip('.edf')
         ic = []
         while name[-1].isdigit():
@@ -321,7 +315,7 @@ class EDFTomoScan(TomoScanBase):
             return int(''.join(orignalOrder))
 
     @staticmethod
-    def get_tomo_n(scan):
+    def get_tomo_n(scan: str) -> Union[None, int]:
         return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
                                                 ref_file=None,
                                                 key='TOMO_N',
@@ -329,7 +323,7 @@ class EDFTomoScan(TomoScanBase):
                                                 key_aliases=['tomo_N', 'Tomo_N'])
 
     @staticmethod
-    def get_dark_n(scan):
+    def get_dark_n(scan:str) -> Union[None, int]:
         return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
                                                 ref_file=None,
                                                 key='DARK_N',
@@ -337,7 +331,7 @@ class EDFTomoScan(TomoScanBase):
                                                 key_aliases=['dark_N', ])
 
     @staticmethod
-    def get_ref_n(scan):
+    def get_ref_n(scan :str) -> Union[None, int]:
         return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
                                                 ref_file=None,
                                                 key='REF_N',
@@ -345,7 +339,7 @@ class EDFTomoScan(TomoScanBase):
                                                 key_aliases=['ref_N', ])
 
     @staticmethod
-    def get_ff_interval(scan):
+    def get_ff_interval(scan: str) -> Union[None, int]:
         return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
                                                 ref_file=None,
                                                 key='REF_ON',
@@ -353,7 +347,7 @@ class EDFTomoScan(TomoScanBase):
                                                 key_aliases=['ref_On', ])
 
     @staticmethod
-    def get_scan_range(scan):
+    def get_scan_range(scan: str) -> Union[None, int]:
         return EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
                                                 ref_file=None,
                                                 key='ScanRange',
@@ -361,7 +355,7 @@ class EDFTomoScan(TomoScanBase):
                                                 key_aliases=['scanRange', ])
 
     @staticmethod
-    def get_dim1_dim2(scan):
+    def get_dim1_dim2(scan: str) -> Union[None, tuple]:
         _info_file = os.path.join(scan, os.path.basename(scan) + '.info')
         d1 = EDFTomoScan.retrieve_information(scan=os.path.abspath(scan),
                                               ref_file=None,
@@ -377,7 +371,7 @@ class EDFTomoScan(TomoScanBase):
         return d1, d2
 
     @staticmethod
-    def get_pixel_size(scan):
+    def get_pixel_size(scan: str) -> Union[None, int]:
         if os.path.isdir(scan) is False:
             return None
         value = EDFTomoScan.retrieve_information(scan=scan,
@@ -402,7 +396,8 @@ class EDFTomoScan(TomoScanBase):
             return None
 
     @staticmethod
-    def get_darks_url(scan_path, prefix='dark', file_ext='.edf'):
+    def get_darks_url(scan_path: str, prefix: str='dark',
+                      file_ext: str='.edf') -> dict:
         """
 
         :param scan_path:
@@ -413,25 +408,29 @@ class EDFTomoScan(TomoScanBase):
         :type: str
         :return: list of refs as silx's `DataUrl`
         """
-        res = []
+        res = {}
         if os.path.isdir(scan_path) is False:
             _logger.error(scan_path + ' is not a directory. Cannot extract '
                                      'DarkHST files')
             return res
-        for my_file in os.listdir(scan_path):
+        for file_ in os.listdir(scan_path):
             _prefix = prefix
             if prefix.endswith(file_ext):
                 _prefix = prefix.rstrip(file_ext)
-            if my_file.startswith(_prefix) and my_file.endswith(file_ext):
-                _file = my_file.lstrip(_prefix).rstrip(file_ext)
-                if _file == '' or _file.isnumeric() is True:
-                    urls = extract_urls_from_edf(os.path.join(scan_path, my_file))
-                    res.extend(urls)
-                    assert os.path.isfile(res[-1])
+            if file_.startswith(_prefix) and file_.endswith(file_ext):
+                # usuelly the dark file name should be dark.edf, but some
+                # darkHSTXXXX remains...
+                file_fp = file_.lstrip(_prefix).rstrip(file_ext).lstrip('HST')
+                if file_fp == '' or file_fp.isnumeric() is True:
+                    index = EDFTomoScan.guess_index_frm_file_name(file_)
+                    urls = extract_urls_from_edf(os.path.join(scan_path, file_),
+                                                 start_index=index)
+                    res.update(urls)
         return res
 
     @staticmethod
-    def get_refs_url(scan_path,  prefix='refHST', file_ext='.edf'):
+    def get_refs_url(scan_path: str,  prefix: str='refHST',
+                     file_ext: str='.edf') -> dict:
         """
 
         :param scan_path:
@@ -442,21 +441,24 @@ class EDFTomoScan(TomoScanBase):
         :type: str
         :return: list of refs as silx's `DataUrl`
         """
-        res = []
+        res = {}
         if os.path.isdir(scan_path) is False:
             _logger.error(scan_path + ' is not a directory. Cannot extract '
                                       'RefHST files')
             return res
 
-        for file in os.listdir(scan_path):
-            if file.startswith(prefix) and file.endswith(
-                    file_ext):
-                res.append(os.path.join(scan_path, file))
-                assert os.path.isfile(res[-1])
+        for file_ in os.listdir(scan_path):
+            if file_.startswith(prefix) and file_.endswith(file_ext):
+                index = EDFTomoScan.guess_index_frm_file_name(file_)
+                file_fp = os.path.join(scan_path, file_)
+                urls = extract_urls_from_edf(start_index=index, file_=file_fp)