Commit 5c95b3d5 authored by payno's avatar payno

Merge branch 'better_experiment_definition' into 'master'

Better experiment definition and deal with several dimension

See merge request !9
parents 79db161d 8d096118
Pipeline #6670 passed with stage
in 2 minutes and 50 seconds
test:python3.5-stretch-pyqt5:
type: test
image: docker-registry.esrf.fr/dau/tomwer:python3.5_stretch_pyqt5
script:
- arch
- export PYTHONPATH="${PYTHONPATH}:/usr/lib/python3/dist-packages/"
- export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
- export http_proxy=http://proxy.esrf.fr:3128/
- export https_proxy=http://proxy.esrf.fr:3128/
- python --version
- python -m pip install pip --upgrade
- python -m pip install setuptools --upgrade
- python -m pip install numpy --upgrade
- python -m pip install matplotlib
- python -m pip install Orange3
- python -m pip install -r requirements.txt
- python -m pip install .
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a id06workflow test -v
......@@ -14,15 +14,13 @@ import logging
import os
import subprocess
import sys
from importlib.machinery import SourceFileLoader
from docutils.parsers.rst.directives.images import Figure
from silx.gui import qt
logging.basicConfig()
_logger = logging.getLogger(__name__)
# TODO:
# - Check if it is needed to patch block_text?
# RST directive ###############################################################
class SnapshotQtDirective(Figure):
......@@ -103,102 +101,61 @@ def setup(app):
return {'version': '0.1'}
# Qt monkey-patch #############################################################
def monkeyPatchQApplication(filename, qt=None):
"""Monkey-patch QApplication to take a snapshot and close the application.
# screenshot function ########################################################
:param str filename: The image filename where to save the snapshot.
:param str qt: The Qt binding to patch.
This MUST be the same as the one used by the script
for which to take a snapshot.
In: 'PyQt4', 'PyQt5', 'PySide2' or None (the default).
If None, it will try to use PyQt4, then PySide2 and
finally PyQt4.
"""
def makescreenshot(script_or_module, filename):
# Probe Qt binding
if qt is None:
try:
import PyQt4.QtCore # noqa
qt = 'PyQt4'
except ImportError:
try:
import PySide2.QtCore # noqa
qt = 'PySide2'
except ImportError:
try:
import PyQt5.QtCore # noqa
qt = 'PyQt5'
except ImportError:
raise RuntimeError('Cannot find any supported Qt binding.')
if qt == 'PyQt4':
from PyQt4.QtGui import QApplication, QPixmap
from PyQt4.QtCore import QTimer
import PyQt4.QtGui as _QApplicationPackage
if qt.BINDING == 'PyQt4':
def grabWindow(winID):
return QPixmap.grabWindow(winID)
elif qt == 'PyQt5':
from PyQt5.QtGui import QPixmap
from PyQt5.QtWidgets import QApplication
from PyQt5.QtCore import QTimer
import PyQt5.QtWidgets as _QApplicationPackage
return qt.QPixmap.grabWindow(winID)
elif qt.BINDING in('PyQt5', 'PySide2'):
def grabWindow(winID):
screen = QApplication.primaryScreen()
screen = qt.QApplication.primaryScreen()
return screen.grabWindow(winID)
elif qt == 'PySide2':
from PySide2.QtGui import QApplication, QPixmap
from PySide2.QtCore import QTimer
import PySide2.QtGui as _QApplicationPackage
def grabWindow(winID):
return QPixmap.grabWindow(winID)
else:
raise ValueError('Unsupported Qt binding: %s' % qt)
_logger.info('Using Qt bindings: %s', qt)
class _QApplication(QApplication):
_TIMEOUT = 1000.
_FILENAME = filename
def _grabActiveWindowAndClose(self):
activeWindow = QApplication.activeWindow()
if activeWindow is not None:
if activeWindow.isVisible():
pixmap = grabWindow(activeWindow.winId())
saveOK = pixmap.save(self._FILENAME)
if not saveOK:
_logger.error(
'Cannot save snapshot to %s', self._FILENAME)
else:
_logger.error('activeWindow is not visible.')
self.quit()
global _count
_count = 5
global _TIMEOUT
_TIMEOUT = 2000.
app = qt.QApplication.instance() or qt.QApplication([])
def _grabActiveWindowAndClose():
global _count
activeWindow = qt.QApplication.activeWindow()
if activeWindow is not None:
if activeWindow.isVisible():
pixmap = grabWindow(activeWindow.winId())
saveOK = pixmap.save(filename)
if not saveOK:
_logger.error(
'Cannot save snapshot to %s', filename)
else:
self._count -= 1
if self._count > 0:
# Only restart a timer if everything is OK
QTimer.singleShot(self._TIMEOUT,
self._grabActiveWindowAndClose)
else:
raise RuntimeError(
'Aborted: It took too long to have an active window.')
def exec_(self):
self._count = 10
QTimer.singleShot(self._TIMEOUT, self._grabActiveWindowAndClose)
return super(_QApplication, self).exec_()
_QApplicationPackage.QApplication = _QApplication
_logger.error('activeWindow is not visible.')
app.quit()
else:
_count -= 1
if _count > 0:
# Only restart a timer if everything is OK
qt.QTimer.singleShot(_TIMEOUT,
_grabActiveWindowAndClose)
else:
app.quit()
# raise('Aborted: It took too long to have an active window.')
_logger.error('Aborted: It took too long to have an active window.')
script_or_module = os.path.abspath(script_or_module)
try:
mod = SourceFileLoader("screenshotmod", script_or_module).load_module()
if hasattr(mod, 'screenshot'):
qt.QTimer.singleShot(_TIMEOUT, _grabActiveWindowAndClose)
mod.screenshot()
else:
_logger.error('no "screenshot" function found in %s' % script_or_module)
except Exception as e:
_logger.error('Fail to import %s : \n %s' % (script_or_module, e))
_logger.info('Using Qt bindings: %s', qt)
# main ########################################################################
......@@ -241,22 +198,9 @@ if __name__ == '__main__':
_logger.error(
'%s: incorrect arguments', os.path.basename(sys.argv[0]))
sys.exit(1)
#
# # Update sys.argv and sys.path
# sys.argv = [script_or_module] + extra_args
# sys.path.insert(0, os.path.abspath(os.path.dirname(script_or_module)))
# Monkey-patch Qt
monkeyPatchQApplication(args.output[0],
args.bindings if args.bindings != 'auto' else None)
# Update sys.argv and sys.path
sys.argv = [script_or_module] + extra_args
sys.path.insert(0, os.path.abspath(os.path.dirname(script_or_module)))
if args.module:
_logger.info('Running module: %s', ' '.join(sys.argv))
runpy.run_module(script_or_module, run_name='__main__')
else:
with open(script_or_module) as f:
code = f.read()
_logger.info('Running script: %s', ' '.join(sys.argv))
exec(code)
makescreenshot(script_or_module, args.output[0])
......@@ -18,8 +18,10 @@ Widgets
widgets/com
widgets/datareduction
widgets/dataselection
widgets/dimension
widgets/geometry
widgets/mapping
widgets/metadata
widgets/noisereduction
widgets/roiselection
widgets/saveexperiment
......
COM
===
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/com.py
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/com_screenshot.py
Signals
......
dimension definition
====================
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/dimension_definition_screenshot.py
Signals
-------
- (Experiment)
**Outputs**:
- (Experiment)
Description
-----------
Used to define the dimension in order to interpret correctly the dataset.
Geometry
========
![image](icons/mywidget.png)
Signals
-------
- (Data)
**Outputs**:
- (Data)
Description
-----------
TODO
Geometry
========
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/geometry_screenshot.py
Signals
-------
- (Experiment)
**Outputs**:
- (Experiment)
Description
-----------
Define the geometry of the experiment.
Used especially to feat dimension and define missing metadata
Mapping
=======
![image](icons/mywidget.png)
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/mapping.py
Signals
......
Mapping
=======
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/metadata_table_screenshot.py
Signals
-------
- (Experiment)
**Outputs**:
- no output
Description
-----------
Display the metadata contained in the dataset
Shift Correction
================
![image](icons/mywidget.png)
.. snapshotqt:: orangecontrib/id06workflow/widgets/screenshots/shift_screenshot.py
Signals
......@@ -11,9 +12,9 @@ Signals
**Outputs**:
- (Data)
- (Data, Image)
Description
-----------
TODO
generate image shift
......@@ -150,6 +150,8 @@ def main(argv):
import id06workflow.test
test_suite = unittest.TestSuite()
test_suite.addTest(id06workflow.test.suite())
import orangecontrib.id06workflow.test
test_suite.addTest(orangecontrib.id06workflow.test.suite())
result = runner.run(test_suite)
if result.wasSuccessful():
......
......@@ -39,12 +39,17 @@ import os
class Dataset(object):
"""
Class used to define a dataset
:param str or None data_files_pattern: pattern to the data files
data_0000.edf for example
:param list or tuple or None dark_files: list of the dark files
:param list or tuple or None ff_files: list of the flat field files
"""
def __init__(self, data_files=None, dark_files=None, ff_files=None):
assert isinstance(data_files, (type(None), list, str, set))
assert isinstance(dark_files, (type(None), list, str, set))
assert isinstance(ff_files, (type(None), list, str, set))
self._data_files = [] if data_files is None else data_files
def __init__(self, data_files_pattern=None, dark_files=None, ff_files=None):
assert isinstance(data_files_pattern, (type(None), str, set))
assert isinstance(dark_files, (type(None), list, set))
assert isinstance(ff_files, (type(None), list, set))
self._data_files_pattern = data_files_pattern
"""data files"""
self._data = None
"""data"""
......@@ -52,7 +57,7 @@ class Dataset(object):
"""dark files"""
self._ff_files = [] if ff_files is None else ff_files
"""flat field files"""
self.__data_has_changed = data_files is not None
self.__data_has_changed = data_files_pattern is not None
"""Flag to avoid loading data from files every time"""
self.__dark_has_changed = dark_files is not None
"""Flag to avoid loading darks from files every time"""
......@@ -66,15 +71,11 @@ class Dataset(object):
@property
def data_files_pattern(self):
return self._data_files
return self._data_files_pattern
@data_files_pattern.setter
def data_files_pattern(self, data_files):
self._data_files = data_files
self.__data_has_changed = True
def addDataFile(self, data_file):
self._data_files.append(data_file)
self._data_files_pattern = data_files
self.__data_has_changed = True
@property
......@@ -108,7 +109,7 @@ class Dataset(object):
raise NotImplementedError('')
def is_valid(self):
return len(self._data_files) > 0 or self._data is not None
return len(self._data_files_pattern) > 0 or self._data is not None
def __eq__(self, other):
if isinstance(other, Dataset) is False:
......@@ -134,8 +135,9 @@ class Dataset(object):
Return a fabio :FileSeries: to iterate over frame.
"""
if self.__data_has_changed is True:
assert os.path.exists(self.data_files_pattern)
# TODO: warning, for now only deal with single frame file
if not os.path.exists(self.data_files_pattern):
raise ValueError('Given file path does not exists (%s)' % self.data_files_pattern)
filenames = filename_series(self.data_files_pattern)
self.__data_series = FileSeries(filenames=filenames,
single_frame=True)
......
This diff is collapsed.
......@@ -61,3 +61,65 @@ def shift_img(data, dx, dy):
res = abs(numpy.fft.ifft2(numpy.fft.fft2(data) * numpy.exp(
1.0j * 2.0 * numpy.pi * (dy * ny / ynum + dx * nx / xnum))))
return res
def guess_shift(data, axis, start=-1.0, stop=1.0, step=0.1):
"""Try to guess the shift from the flatten data"""
def com_1d(ydata):
xdata = numpy.arange(len(ydata))
deno = numpy.sum(ydata).astype(numpy.float32)
if deno == 0.:
return numpy.nan
else:
return numpy.sum(xdata * ydata).astype(numpy.float32) / deno
def com_2d(data, axis):
sum = numpy.sum(data, axis=axis)
deno = numpy.sum(sum)
if deno == 0.0:
return numpy.nan
else:
return sum / deno
def com_var(x_shift, y_shift):
if x_shift is None:
assert y_shift is not None
if y_shift is None:
assert x_shift is not None
coms = []
for iImg, img in enumerate(data):
# if img is too large, reduce size
_img = img
if _img.shape[-1] > 1024:
_img = _img[::2, ::2]
if x_shift is None:
shifted_img = shift_img(_img, dx=0.0, dy=y_shift * iImg)
else:
shifted_img = shift_img(_img, dx=x_shift * iImg, dy=0.0)
if x_shift is None:
com = com_1d(ydata=com_2d(shifted_img, axis=1))
else:
com = com_1d(ydata=com_2d(shifted_img, axis=0))
if img.shape[-1] > 1024:
com = com * 0.5
coms.append(com)
return numpy.var(coms)
assert axis in (0, 1)
assert data.ndim is 3
vars = []
_range = numpy.arange(start=start, stop=stop, step=step)
if axis is 0:
y_shift = None
[vars.append(com_var(x_shift, y_shift)) for x_shift in _range]
elif axis is 1:
x_shift = None
[vars.append(com_var(x_shift, y_shift)) for y_shift in _range]
else:
raise not NotImplementedError('')
return _range[numpy.argmin(vars)]
......@@ -32,10 +32,10 @@ MEAN = 'mean'
VARIANCE = 'variance'
SKENESS = 'skeness'
SKEWNESS = 'skewness'
KURTOSIS = 'kurtosis'
INTENSITY = 'intensity'
MODES = (MEAN, VARIANCE, SKENESS, KURTOSIS, INTENSITY)
MODES = (MEAN, VARIANCE, SKEWNESS, KURTOSIS, INTENSITY)
......@@ -63,6 +63,12 @@ class _BaseOperation(qt.QObject):
data.setflags(write=self._can_overwrite_data)
return data
@property
def data_flatten(self):
data_flatten = self._experiment.data_flatten.view()
data_flatten.setflags(write=self._can_overwrite_data)
return data_flatten
@property
def experiment(self):
return self._experiment
......@@ -97,9 +103,9 @@ class OverwritingOperation(_BaseOperation):
def __init__(self, experiment, name):
_BaseOperation.__init__(self, experiment, name, can_overwrite_data=True)
@_BaseOperation.data.setter
def data(self, data):
self._experiment.data = data
@_BaseOperation.data_flatten.setter
def data_flatten(self, data):
self._experiment.data_flatten = data
def dry_run(self, cache_data=None):
"""
......
......@@ -47,6 +47,7 @@ class COM(AdditiveOperation):
"""
_NAME = 'com calculation'
def __init__(self, map):
assert isinstance(map, _MappingBase)
AdditiveOperation.__init__(self, experiment=map.experiment,
......@@ -61,7 +62,7 @@ class COM(AdditiveOperation):
assert self._map.ndim is 1
self.__maps = []
for dim in self._map.dim:
for dim in self._map.dims:
_map1 = dim[:, :, 0]
_map1 = _map1 - numpy.nan_to_num(_map1.mean())
......
......@@ -29,18 +29,23 @@ __license__ = "MIT"
__date__ = "15/10/2018"
from . import AdditiveOperation
from collections import namedtuple
import numpy
import logging
_logger = logging.getLogger(__file__)
_IMap = namedtuple('_IMap', ['name', 'kind', 'mean', 'variance', 'skewness',
'kurtosis'])
class _MappingBase(AdditiveOperation):
"""
Base class used for mapping
"""
@property
def dim(self):
def dims(self):
raise NotImplementedError('Base class')
@property
......@@ -73,7 +78,8 @@ class IntensityMapping(_MappingBase):
self._threshold = threshold
self.__intensity_map = None
self.__dim = []
self.__dim = {}
"""Associate to each dimension the _IMap"""
def key(self):
return self.KEY
......@@ -85,64 +91,64 @@ class IntensityMapping(_MappingBase):
_logger.warning('Experiment as no data defined, unable to apply'
'mapping')
return None
elif self._experiment.geometry is None:
_logger.warning('No geometry defined to apply mapping')
return None
else:
assert self.data.ndim is 3
assert self.data.ndim > 2
# TODO: avoid recomputing several time the intensity map
self._resetIntensityMap()
self._resetDim()
for x in range(self.data.shape[2]):
for x in range(self.data.shape[-1]):
self.updateProgress(int(x / self.data.shape[2] * 100.0))
for y in range(self.data.shape[1]):
for y in range(self.data.shape[-2]):
# TODO: do this operation using a grid or something close
# but would have to find a way to write at the same position...
reciprocalVol = numpy.squeeze(self.data[:, y, x])
reciprocalVol = numpy.squeeze(self.data_flatten[:, y, x]).astype(numpy.float128)
reciprocalVol = reciprocalVol - reciprocalVol.min()
maxIntensity = reciprocalVol.max()
if maxIntensity > self._threshold:
self.__intensity_map[y, x] = maxIntensity
# Here: TODO, later when can have several dimension (motors scanned/rockers)
# do it for all dims
for iDim in range(self._experiment.ndim):
for axis, dim in self._experiment.dims:
intensity = numpy.squeeze(reciprocalVol.sum())
# TODO: move calculation of angle_mean, angle_variance,
# angle_skewness, angle_kurtosis in :class:`Experiment`
angles = self._experiment.angles
dim_unique_values = dim.unique_values.astype(numpy.float128)
# TODO: why mean is obtained by the sum ?
# angles_mean = (intensity * angles).mean()
angles_mean = (intensity * angles).sum()
angles_variance = (
intensity * (angles - angles_mean) ** 2).sum()
angles_skewness = (
intensity * (angles - angles_mean) ** 3).sum()
angles_kurtosis = (
intensity * (angles - angles_mean) ** 4).sum()
# TODO: pb, this is overwritten at each image if maxIntensity > threshold ???
self.dim[iDim][y, x, 0] = angles_mean
self.dim[iDim][y, x, 1] = angles_variance
self.dim[iDim][y, x, 2] = angles_skewness
self.dim[iDim][y, x, 3] = angles_kurtosis
_mean = (intensity * dim_unique_values).sum()
diff = (dim_unique_values - _mean)
_variance = (intensity * diff ** 2).sum()
_skewness = (intensity * diff ** 3).sum()
_kurtosis = (intensity * diff ** 4).sum()
self.dims[axis].mean[y, x] = _mean
self.dims[axis].variance[y, x] = _variance
self.dims[axis].skewness[y, x] = _skewness
self.dims[axis].kurtosis[y, x] = _kurtosis
self.registerOperation()
return self.dim
return self.dims
def _resetIntensityMap(self):
self.__intensity_map = numpy.zeros(self.data.shape[1:])
self.__intensity_map = numpy.zeros(self.data.shape[-2:], dtype=numpy.float128)
def _resetDim(self):
self.__dim = []
for dim in range(self._experiment.ndim):
self.__dim.append(numpy.zeros((*(self.data.shape[1:]), 4)))
self.__dim = {}
for axis, dim in self._experiment.dims:
_map = _IMap(
name=dim.name,
kind=dim.kind,
mean=numpy.zeros((self.data.shape[-2:])),
variance=numpy.zeros((self.data.shape[-2:])),
skewness=numpy.zeros((self.data.shape[-2:])),
kurtosis=numpy.zeros((self.data.shape[-2:])))
self.__dim[axis] = _map
@property
def intensity_map(self):
return self.__intensity_map
@property
def dim(self):
def dim