Commit 5c95b3d5 authored by payno's avatar payno
Browse files

Merge branch 'better_experiment_definition' into 'master'

Better experiment definition and deal with several dimension

See merge request !9
parents 79db161d 8d096118
Pipeline #6670 passed with stage
in 2 minutes and 50 seconds
test:python3.5-stretch-pyqt5:
type: test
image: docker-registry.esrf.fr/dau/tomwer:python3.5_stretch_pyqt5
script:
- arch
- export PYTHONPATH="${PYTHONPATH}:/usr/lib/python3/dist-packages/"
- export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
- export http_proxy=http://proxy.esrf.fr:3128/
- export https_proxy=http://proxy.esrf.fr:3128/
- python --version
- python -m pip install pip --upgrade
- python -m pip install setuptools --upgrade
- python -m pip install numpy --upgrade
- python -m pip install matplotlib
- python -m pip install Orange3
- python -m pip install -r requirements.txt
- python -m pip install .
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a id06workflow test -v
......@@ -14,15 +14,13 @@ import logging
import os
import subprocess
import sys
from importlib.machinery import SourceFileLoader
from docutils.parsers.rst.directives.images import Figure
from silx.gui import qt
logging.basicConfig()
_logger = logging.getLogger(__name__)
# TODO:
# - Check if it is needed to patch block_text?
# RST directive ###############################################################
class SnapshotQtDirective(Figure):
......@@ -103,102 +101,61 @@ def setup(app):
return {'version': '0.1'}
# Qt monkey-patch #############################################################
def monkeyPatchQApplication(filename, qt=None):
"""Monkey-patch QApplication to take a snapshot and close the application.
# screenshot function ########################################################
:param str filename: The image filename where to save the snapshot.
:param str qt: The Qt binding to patch.
This MUST be the same as the one used by the script
for which to take a snapshot.
In: 'PyQt4', 'PyQt5', 'PySide2' or None (the default).
If None, it will try to use PyQt4, then PySide2 and
finally PyQt4.
"""
def makescreenshot(script_or_module, filename):
# Probe Qt binding
if qt is None:
try:
import PyQt4.QtCore # noqa
qt = 'PyQt4'
except ImportError:
try:
import PySide2.QtCore # noqa
qt = 'PySide2'
except ImportError:
try:
import PyQt5.QtCore # noqa
qt = 'PyQt5'
except ImportError:
raise RuntimeError('Cannot find any supported Qt binding.')
if qt == 'PyQt4':
from PyQt4.QtGui import QApplication, QPixmap
from PyQt4.QtCore import QTimer
import PyQt4.QtGui as _QApplicationPackage
if qt.BINDING == 'PyQt4':
def grabWindow(winID):
return QPixmap.grabWindow(winID)
elif qt == 'PyQt5':
from PyQt5.QtGui import QPixmap
from PyQt5.QtWidgets import QApplication
from PyQt5.QtCore import QTimer
import PyQt5.QtWidgets as _QApplicationPackage
return qt.QPixmap.grabWindow(winID)
elif qt.BINDING in('PyQt5', 'PySide2'):
def grabWindow(winID):
screen = QApplication.primaryScreen()
screen = qt.QApplication.primaryScreen()
return screen.grabWindow(winID)
elif qt == 'PySide2':
from PySide2.QtGui import QApplication, QPixmap
from PySide2.QtCore import QTimer
import PySide2.QtGui as _QApplicationPackage
def grabWindow(winID):
return QPixmap.grabWindow(winID)
else:
raise ValueError('Unsupported Qt binding: %s' % qt)
_logger.info('Using Qt bindings: %s', qt)
class _QApplication(QApplication):
_TIMEOUT = 1000.
_FILENAME = filename
def _grabActiveWindowAndClose(self):
activeWindow = QApplication.activeWindow()
if activeWindow is not None:
if activeWindow.isVisible():
pixmap = grabWindow(activeWindow.winId())
saveOK = pixmap.save(self._FILENAME)
if not saveOK:
_logger.error(
'Cannot save snapshot to %s', self._FILENAME)
else:
_logger.error('activeWindow is not visible.')
self.quit()
global _count
_count = 5
global _TIMEOUT
_TIMEOUT = 2000.
app = qt.QApplication.instance() or qt.QApplication([])
def _grabActiveWindowAndClose():
global _count
activeWindow = qt.QApplication.activeWindow()
if activeWindow is not None:
if activeWindow.isVisible():
pixmap = grabWindow(activeWindow.winId())
saveOK = pixmap.save(filename)
if not saveOK:
_logger.error(
'Cannot save snapshot to %s', filename)
else:
self._count -= 1
if self._count > 0:
# Only restart a timer if everything is OK
QTimer.singleShot(self._TIMEOUT,
self._grabActiveWindowAndClose)
else:
raise RuntimeError(
'Aborted: It took too long to have an active window.')
def exec_(self):
self._count = 10
QTimer.singleShot(self._TIMEOUT, self._grabActiveWindowAndClose)
return super(_QApplication, self).exec_()
_QApplicationPackage.QApplication = _QApplication
_logger.error('activeWindow is not visible.')
app.quit()
else:
_count -= 1
if _count > 0:
# Only restart a timer if everything is OK
qt.QTimer.singleShot(_TIMEOUT,
_grabActiveWindowAndClose)
else:
app.quit()
# raise('Aborted: It took too long to have an active window.')
_logger.error('Aborted: It took too long to have an active window.')
script_or_module = os.path.abspath(script_or_module)
try:
mod = SourceFileLoader("screenshotmod", script_or_module).load_module()
if hasattr(mod, 'screenshot'):
qt.QTimer.singleShot(_TIMEOUT, _grabActiveWindowAndClose)
mod.screenshot()
else:
_logger.error('no "screenshot" function found in %s' % script_or_module)
except Exception as e:
_logger.error('Fail to import %s : \n %s' % (script_or_module, e))
_logger.info('Using Qt bindings: %s', qt)
# main ########################################################################
......@@ -241,22 +198,9 @@ if __name__ == '__main__':
_logger.error(
'%s: incorrect arguments', os.path.basename(sys.argv[0]))
sys.exit(1)
#
# # Update sys.argv and sys.path
# sys.argv = [script_or_module] + extra_args
# sys.path.insert(0, os.path.abspath(os.path.dirname(script_or_module)))
# Monkey-patch Qt
monkeyPatchQApplication(args.output[0],
args.bindings if args.bindings != 'auto' else None)
# Update sys.argv and sys.path
sys.argv = [script_or_module] + extra_args
sys.path.insert(0, os.path.abspath(os.path.dirname(script_or_module)))
if args.module:
_logger.info('Running module: %s', ' '.join(sys.argv))
runpy.run_module(script_or_module, run_name='__main__')
else:
with open(script_or_module) as f:
code = f.read()
_logger.info('Running script: %s', ' '.join(sys.argv))
exec(code)
makescreenshot(script_or_module, args.output[0])
......@@ -18,8 +18,10 @@ Widgets
widgets/com
widgets/datareduction
widgets/dataselection
widgets/dimension
widgets/geometry
widgets/mapping
widgets/metadata
widgets/noisereduction
widgets/roiselection
widgets/saveexperiment
......
COM
===
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/com.py
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/com_screenshot.py
Signals
......
dimension definition
====================
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/dimension_definition_screenshot.py
Signals
-------
- (Experiment)
**Outputs**:
- (Experiment)
Description
-----------
Used to define the dimension in order to interpret correctly the dataset.
Geometry
========
![image](icons/mywidget.png)
Signals
-------
- (Data)
**Outputs**:
- (Data)
Description
-----------
TODO
Geometry
========
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/geometry_screenshot.py
Signals
-------
- (Experiment)
**Outputs**:
- (Experiment)
Description
-----------
Define the geometry of the experiment.
Used especially to feat dimension and define missing metadata
Mapping
=======
![image](icons/mywidget.png)
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/mapping.py
Signals
......
Mapping
=======
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/metadata_table_screenshot.py
Signals
-------
- (Experiment)
**Outputs**:
- no output
Description
-----------
Display the metadata contained in the dataset
Shift Correction
================
![image](icons/mywidget.png)
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/shift_screenshot.py
Signals
......@@ -11,9 +12,9 @@ Signals
**Outputs**:
- (Data)
- (Data, Image)
Description
-----------
TODO
generate image shift
......@@ -150,6 +150,8 @@ def main(argv):
import id06workflow.test
test_suite = unittest.TestSuite()
test_suite.addTest(id06workflow.test.suite())
import orangecontrib.id06workflow.test
test_suite.addTest(orangecontrib.id06workflow.test.suite())
result = runner.run(test_suite)
if result.wasSuccessful():
......
......@@ -39,12 +39,17 @@ import os
class Dataset(object):
"""
Class used to define a dataset
:param str or None data_files_pattern: pattern to the data files
data_0000.edf for example
:param list or tuple or None dark_files: list of the dark files
:param list or tuple or None ff_files: list of the flat field files
"""
def __init__(self, data_files=None, dark_files=None, ff_files=None):
assert isinstance(data_files, (type(None), list, str, set))
assert isinstance(dark_files, (type(None), list, str, set))
assert isinstance(ff_files, (type(None), list, str, set))
self._data_files = [] if data_files is None else data_files
def __init__(self, data_files_pattern=None, dark_files=None, ff_files=None):
assert isinstance(data_files_pattern, (type(None), str, set))
assert isinstance(dark_files, (type(None), list, set))
assert isinstance(ff_files, (type(None), list, set))
self._data_files_pattern = data_files_pattern
"""data files"""
self._data = None
"""data"""
......@@ -52,7 +57,7 @@ class Dataset(object):
"""dark files"""
self._ff_files = [] if ff_files is None else ff_files
"""flat field files"""
self.__data_has_changed = data_files is not None
self.__data_has_changed = data_files_pattern is not None
"""Flag to avoid loading data from files every time"""
self.__dark_has_changed = dark_files is not None
"""Flag to avoid loading darks from files every time"""
......@@ -66,15 +71,11 @@ class Dataset(object):
@property
def data_files_pattern(self):
return self._data_files
return self._data_files_pattern
@data_files_pattern.setter
def data_files_pattern(self, data_files):
self._data_files = data_files
self.__data_has_changed = True
def addDataFile(self, data_file):
self._data_files.append(data_file)
self._data_files_pattern = data_files
self.__data_has_changed = True
@property
......@@ -108,7 +109,7 @@ class Dataset(object):
raise NotImplementedError('')
def is_valid(self):
return len(self._data_files) > 0 or self._data is not None
return len(self._data_files_pattern) > 0 or self._data is not None
def __eq__(self, other):
if isinstance(other, Dataset) is False:
......@@ -134,8 +135,9 @@ class Dataset(object):
Return a fabio :FileSeries: to iterate over frame.
"""
if self.__data_has_changed is True:
assert os.path.exists(self.data_files_pattern)
# TODO: warning, for now only deal with single frame file
if not os.path.exists(self.data_files_pattern):
raise ValueError('Given file path does not exists (%s)' % self.data_files_pattern)
filenames = filename_series(self.data_files_pattern)
self.__data_series = FileSeries(filenames=filenames,
single_frame=True)
......
......@@ -28,18 +28,41 @@ __authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "01/10/2018"
import logging
from collections import OrderedDict
from silx.io import fabioh5
from silx.io.fabioh5 import FabioReader
import numpy
from id06workflow.core.operation import _BaseOperation
from id06workflow.core.operation.datareduction import DataReduction
from id06workflow.core.Dataset import Dataset
from id06workflow.core.geometry import GeometryBase
from id06workflow.core.geometry.TwoThetaGeometry import TwoThetaGeometry
from id06workflow.core.experiment.operation import _BaseOperation
from id06workflow.core.experiment.operation.roi import RoiOperation
from id06workflow.core.experiment.operation.datareduction import DataReduction
from collections import OrderedDict
import numpy
import logging
from id06workflow.core.operation.roi import RoiOperation
from id06workflow.core.utils import metadata as metadatautils
_logger = logging.getLogger(__file__)
DEFAULT_METADATA = FabioReader.DEFAULT
COUNTER_METADATA = FabioReader.COUNTER
POSITIONER_METADATA = FabioReader.POSITIONER
_METADATA_TYPES = {
'default': DEFAULT_METADATA,
'counter': COUNTER_METADATA,
'positioner': POSITIONER_METADATA,
}
_METADATA_TYPES_I = {}
"""used to retrieve the metadata name (str) for the silx.io.fabioh5 id"""
for key, value in _METADATA_TYPES.items():
assert value not in _METADATA_TYPES_I
_METADATA_TYPES_I[value] = key
class Experiment(object):
"""
......@@ -62,7 +85,7 @@ class Experiment(object):
Which is different from the raw data contained in the dataset"""
self.__metadata = None
"""Metadata associated to the data. One per slice edf file header"""
self.__ndim = 1
self.__dims = AcquisitionDims()
"""Number of dimension of the experiment (motors scanned/rocked)
For now limited to 1
"""
......@@ -95,6 +118,8 @@ class Experiment(object):
@property
def metadata(self):
if self.__metadata is None:
self.data
return self.__metadata
@property
......@@ -108,6 +133,10 @@ class Experiment(object):
else:
raise NotImplementedError('Cannot deal with several ROI yet')
@property
def dims(self):
return self.__dims
def getRawData(self):
# TODO: cache should probably be used in the future to deal wuth data
if self.dataset is None:
......@@ -117,7 +146,10 @@ class Experiment(object):
if reductionStep in (None, []):
reductionStep = [(1, 1, 1)]
if len(reductionStep) is 1:
steps = reductionStep[0].steps
if isinstance(reductionStep[0], DataReduction):
steps = reductionStep[0].steps
else:
steps = reductionStep[0]
else:
raise NotImplementedError('cannot manage several reduction steps for the moment')
......@@ -136,28 +168,47 @@ class Experiment(object):
return data_with_z_reduction[:, ::steps[1], ::steps[0]]
def _loadFileSeries(self, fileseries, z_step):
"""This will only deal with .edf file for now"""
data = []
headers = []
for iFrame in numpy.arange(start=0, stop=fileseries.nframes, step=z_step):
# TODO: deal with motors
frame = fileseries.getframe(iFrame)
data.append(frame.data)
headers.append(frame.header)
headers.append(fabioh5.EdfFabioReader(fabio_image=frame))
return numpy.asarray(data), headers
@property
def data(self):
if self.__dims.ndim > 1:
# TODO: create view or adapt view on raw data
shape = list(self.__dims.shape)
shape.append(self.data_flatten.shape[-2])
shape.append(self.data_flatten.shape[-1])
return self.data_flatten.view().reshape(shape)
else:
return self.data_flatten
@property
def data_flatten(self):
if self.__data is None:
self.__data = self.getRawData()
return self.__data
@data.setter
def data(self, data):
self.__data = data
@data_flatten.setter
def data_flatten(self, data):
assert data.ndim > 2
self.__data = data.reshape(-1, data.shape[-2], data.shape[-1])
@property
def ndim(self):
return self.__ndim
"""
:return: number of dimension in the experiement (so do not include the 2
dimensions form the frame)
:rtype: int
"""
return self.__dims.ndim
@property
def background_subtracted(self):
......@@ -199,7 +250,7 @@ class Experiment(object):
_logger.warning(
'No flat field defined for background, getting it '
'directly from data files')
return numpy.median(numpy.nan_to_num(self.__data))
return numpy.median(numpy.nan_to_num(self.data_flatten))
else:
return None
else:
......@@ -270,3 +321,165 @@ class Experiment(object):
else:
assert self.data.ndim is 3
return self.data.shape[0]
def set_dims(self, dims):
assert isinstance(dims, dict)
self.__dims.clear()
_fail = False, None # register fail status and latest reason
for axis, dim in dims.items():
try:
unique_dim_value = metadatautils.getUnique(self,
kind=dim.kind,
key=dim.name,
relative_prev_val=dim.relative_prev_val,
cycle_length=dim.size,
axis=axis)
except Exception as e:
if _fail[0] is False:
_fail = True, e
else:
if dim.size is None: