Commit b171273a authored by payno's avatar payno
Browse files

Merge branch 'add_gradient_removal' into 'master'

Add gradient removal

Closes #5

See merge request !3
parents ccca49c9 9164dcc0
......@@ -31,8 +31,9 @@ __date__ = "01/10/2018"
try:
from fabio.fabio import file_series
except ImportError:
from id06workflow.third_party.fabio.file_serie import FileSeries
from id06workflow.third_party.fabio.file_serie import FileSeries, filename_series
import numpy
import os
class Dataset(object):
......@@ -59,16 +60,16 @@ class Dataset(object):
"""Flag to avoid loading flat field from files every time"""
"""Region Of Interest if any defined"""
self.__data = None
self.__data_series = None
self.__darks = None
self.__ff = None
@property
def data_files(self):
def data_files_pattern(self):
return self._data_files
@data_files.setter
def data_files(self, data_files):
@data_files_pattern.setter
def data_files_pattern(self, data_files):
self._data_files = data_files
self.__data_has_changed = True
......@@ -112,13 +113,13 @@ class Dataset(object):
def __eq__(self, other):
if isinstance(other, Dataset) is False:
return False
return (sorted(list(self.data_files)) == sorted(list(other.data_files)) and
return (sorted(list(self.data_files_pattern)) == sorted(list(other.data_files_pattern)) and
sorted(list(self.dark_files)) == sorted(list(other.dark_files)) and
sorted(list(self.flat_fields_files)) == sorted(list(other.flat_fields_files)))
def __str__(self):
return "\n".join(
("data_files: " + (str(self.data_files) or ""),
("data_files_pattern: " + (str(self.data_files_pattern) or ""),
"dark_files: " + (str(self.dark_files) or ""),
"flat field images: " + (str(self.flat_fields_files) or "")))
......@@ -128,18 +129,22 @@ class Dataset(object):
self.__ff_has_changed = False
return self.__ff
def getData(self):
def getDataFileSeries(self):
"""
Return a fabio :FileSeries: to iterate over frame.
"""
if self.__data_has_changed is True:
self.__data = self._loadFiles(self.data_files)
assert os.path.exists(self.data_files_pattern)
# TODO: warning, for now only deal with single frame file
filenames = filename_series(self.data_files_pattern)
self.__data_series = FileSeries(filenames=filenames,
single_frame=True)
self.__data_has_changed = False
return self.__data
return self.__data_series
def _loadFiles(self, files_list):
# TODO: add information if the number of frame is fixed...
if len(files_list) is 0:
return None
series = FileSeries(files_list)
framelist = []
for frame in series.iterframes():
framelist.append(frame.data)
return numpy.asarray(framelist)
return self._loadSeries(series)
......@@ -60,6 +60,8 @@ class Experiment(object):
self.__data = None
"""current data. The one updated with treatment, roi...
Which is different from the raw data contained in the dataset"""
self.__metadata = None
"""Metadata associated to the data. One per slice edf file header"""
self.__ndim = 1
"""Number of dimension of the experiment (motors scanned/rocked)
For now limited to 1
......@@ -106,26 +108,38 @@ class Experiment(object):
# TODO: cache should probably be used in the future to deal wuth data
if self.dataset is None:
return None
rawdata = self.dataset.getData()
fileseries = self.dataset.getDataFileSeries()
reductionStep = self._getDataReductionOperations()
if reductionStep in (None, []):
steps = (1, 1, 1)
reductionStep = [(1, 1, 1)]
if len(reductionStep) is 1:
steps = reductionStep[0].steps
else:
if len(reductionStep) is 1:
steps = reductionStep[0]
else:
raise NotImplementedError('cannot manage several reduction steps for the moment')
raise NotImplementedError('cannot manage several reduction steps for the moment')
# load only files we want to keep. Apply the z reduction
data_with_z_reduction, self.__metadata = self._loadFileSeries(fileseries=fileseries,
z_step=steps[2])
if self.roi is not None:
assert isinstance(self.roi, tuple)
origin, size = self.roi
assert rawdata.ndim is 3
assert data_with_z_reduction.ndim is 3
# TODO: make sure origin and size are given as y, x
ymin, ymax = int(origin[1]), int(origin[1] + size[1])
xmin, xmax = int(origin[0]), int(origin[0] + size[0])
return rawdata[::steps[0], ymin:ymax:steps[1], xmin:xmax:steps[2]]
return data_with_z_reduction[:, ymin:ymax:steps[1], xmin:xmax:steps[0]]
else:
return rawdata[::steps[0], ::steps[1], ::steps[2]]
return data_with_z_reduction[:, ::steps[1], ::steps[0]]
def _loadFileSeries(self, fileseries, z_step):
data = []
headers = []
for iFrame in numpy.arange(start=0, stop=fileseries.nframes, step=z_step):
# TODO: deal with motors
frame = fileseries.getframe(iFrame)
data.append(frame.data)
headers.append(frame.header)
return numpy.asarray(data), headers
@property
def data(self):
......@@ -216,6 +230,12 @@ class Experiment(object):
if isinstance(operation, DataReduction):
self._data_reduction_operations.append(operation)
def getOperation(self, id):
if id in self.operations:
return self.operations[id]
else:
return None
def _getRoiOperations(self):
"""
......
......@@ -33,7 +33,11 @@ from . import OverwritingOperation
class DataReduction(OverwritingOperation):
"""
Set the experiment to use only part of the dataset.
This operation is a 'one shot' operation ofr now as it is postpone in the
:class:`Experiment`
"""
def __init__(self, experiment, x_factor, y_factor, z_factor):
"""
Apply a data reduction to the experiment
......@@ -50,25 +54,24 @@ class DataReduction(OverwritingOperation):
self._cache_data = None
def dry_run(self, cache_data=None):
if cache_data is None:
self._cache_data = self.data[...][::self._z_factor, ::self._y_factor, ::self._x_factor]
else:
self._cache_data = cache_data
return self._cache_data
raise NotImplementedError('Not possible for data reduction since'
'managed by the Experiment class')
def compute(self):
self.data = self.data[::self._z_factor, ::self._y_factor, ::self._x_factor]
self.registerOperation()
return self.data
def apply(self):
if self._cache_data is None:
raise ValueError('No data in cache')
self.data = self._cache_data
self.clear_cache()
self.registerOperation()
return self.data
raise NotImplementedError('Not possible for data reduction since'
'managed by the Experiment class')
def clear_cache(self):
self._cache_data = None
def key(self):
return ' '.join((self._name, 'x:', str(self._x_factor), 'y:',
str(self._y_factor), 'z:', str(self._z_factor)))
@property
def steps(self):
return (self._x_factor, self._y_factor, self._z_factor)
......@@ -35,7 +35,24 @@ import logging
_logger = logging.getLogger(__file__)
class IntensityMapping(AdditiveOperation):
class _MappingBase(AdditiveOperation):
"""
Base class used for mapping
"""
@property
def dim(self):
raise NotImplementedError('Base class')
@property
def intensity_map(self):
raise NotImplementedError('Base class')
@property
def ndim(self):
raise NotImplementedError('Base class')
class IntensityMapping(_MappingBase):
"""
for each detector pixel (x,y), each data dimension (motors scanned/rocked)
are evaluated: 1) intensity is summed along all other dimensions than the
......@@ -45,9 +62,10 @@ class IntensityMapping(AdditiveOperation):
:param float or None threshold: if the threshold is None then the mean
value of the current data will be take
"""
KEY = 'intensity mapping'
def __init__(self, experiment, threshold=None):
AdditiveOperation.__init__(self, experiment=experiment,
name='intensity mapping')
_MappingBase.__init__(self, experiment=experiment, name=self.KEY)
self._dim = []
if threshold is None:
self._threshold = numpy.median(self.data)
......@@ -58,7 +76,7 @@ class IntensityMapping(AdditiveOperation):
self.__dim = []
def key(self):
return self._name
return self.KEY
def compute(self):
_logger.info('start computing the dx2d map')
......@@ -107,7 +125,6 @@ class IntensityMapping(AdditiveOperation):
self.dim[iDim][y, x, 2] = angles_skewness
self.dim[iDim][y, x, 3] = angles_kurtosis
print('should register mapping operation')
self.registerOperation()
return self.dim
......@@ -126,3 +143,83 @@ class IntensityMapping(AdditiveOperation):
@property
def dim(self):
return self.__dim
@property
def ndim(self):
return len(self.__dim)
class GradientRemoval(_MappingBase):
"""
Compute gradient for the current intensity and remove it from the initial
intensity calculation
"""
def __init__(self, experiment):
_MappingBase.__init__(self, experiment=experiment,
name='gradient removal')
self._gradients = None
self.__dim = []
self.__intensity_map = None
def dry_run(self, cache_data=None):
raise ValueError('Not available for Gradient removal yet')
def compute(self):
# TODO: why delta = 0.567
delta = 0.567
# TODO: for now we ask experiment for the intensity mapping if computed
mapping = self._experiment.getOperation(IntensityMapping.KEY)
if mapping is None:
_logger.error('unable to process gradient correction, mapping '
'needs to be proceed before.')
return
self.__dim = self.apply_gradient_correction(self.data, delta=delta, mapping=mapping)
self.__intensity_map = mapping.intensity_map[...]
self.registerOperation()
@staticmethod
def apply_gradient_correction(data, delta, mapping):
assert isinstance(mapping, IntensityMapping)
# TODO: number of element: for now only square matrices but if evolve ?
corr2, corr1 = numpy.meshgrid(
numpy.linspace(-delta, delta, data.shape[2]),
numpy.linspace(0, 0, data.shape[1]))
_gradients = []
# TODO: if necessary compute mean...
for iDim in range(mapping.ndim):
gradient = mapping.dim[iDim][...]
# TODO: for now gradient corrcetion is only apply on mean, why ?
# TODO: this should be in enumerate(Mode) but intensity is stored
# in an other struct, why ?
_gradient = numpy.asarray((
gradient[:, :, 0] + corr2,
gradient[:, :, 1] + corr2,
gradient[:, :, 2] + corr2,
gradient[:, :, 3] + corr2)
)
_gradients.append(_gradient)
return _gradients
def gradients(self):
return self._gradients
def key(self):
return self._name
@property
def dim(self):
return self.__dim
@property
def intensity_map(self):
return self.__intensity_map
@property
def ndim(self):
return len(self.__dim)
......@@ -31,61 +31,12 @@ __date__ = "15/10/2018"
import logging
from . import AdditiveOperation, OverwritingOperation
from .mapping import IntensityMapping
import numpy
_logger = logging.getLogger(__file__)
class GradientCalculation(AdditiveOperation):
"""
Compute gradient for the current intensity
"""
def __init__(self, experiment):
AdditiveOperation.__init__(self, experiment=experiment,
name='gradient removal')
self._gradients = None
def compute(self):
try:
# TODO: in the future create the operation if not existing
intensityOperation = self._experiment.getOperation('intensity')
except Exception as e:
_logger.error(e)
return
# TODO: how to get delta ?
delta = 0.567
corr2, corr1 = numpy.meshgrid(
numpy.linspace(-delta, delta, self.data.shape[0]),
numpy.linspace(0, 0,
self.data.shape[0]))
self._gradients = []
# TODO: if necessary compute mean...
for iDim in self._experiment.ndim:
gradient = intensityOperation.dim[iDim][...]
gradient = gradient + corr2
self._gradients.append(gradient)
self.registerOperation()
return self._gradients
def gradients(self):
return self._gradients
def key(self):
return self._name
class GradientRemoval(OverwritingOperation):
"""Apply a gradient removal to the data"""
pass
def key(self):
return self._name
class ThresholdRemoval(OverwritingOperation):
"""
Apply a simple threshold filtering on data
......@@ -129,7 +80,7 @@ class ThresholdRemoval(OverwritingOperation):
@replacement_value.setter
def replacement_value(self, value):
self.replacement_value = value
self._replacement_value = value
def compute(self):
"""
......
......@@ -45,7 +45,7 @@ class RoiOperation(OverwritingOperation):
""" numpy.ndarray used for caching"""
def key(self):
return ' '.join((self._name, 'origin:', str(self._origin), 'origin:',
return ' '.join((self._name, 'origin:', str(self._origin), 'size:',
str(self._size)))
def compute(self):
......
......@@ -57,7 +57,7 @@ class DatasetSelectionDialog(qt.QDialog):
_buttons.rejected.connect(self.reject)
# expose API
self.getDataFiles = self.mainWindow.getDataFiles
self.getDataFilesPattern = self.mainWindow.getDataFilesPattern
self.getDarkFiles = self.mainWindow.getDarkFiles
self.getFlatFieldFiles = self.mainWindow.getFlatFieldFiles
......@@ -68,15 +68,15 @@ class DatasetSelection(qt.QTabWidget):
"""
def __init__(self, parent):
qt.QTabWidget.__init__(self, parent)
self._dataFiles = DataListWidget(parent=self)
self.addTab(self._dataFiles, 'data files')
self._dataFiles = FilePatternWidget(parent=self)
self.addTab(self._dataFiles, 'data files pattern:')
self._darkFiles = DataListWidget(parent=self)
self.addTab(self._darkFiles, 'dark files')
self._ffFiles = DataListWidget(parent=self)
self.addTab(self._ffFiles, 'flat field files')
# expose API
self.getDataFiles = self._dataFiles.getFiles
self.getDataFilesPattern = self._dataFiles.getPattern
self.getDarkFiles = self._darkFiles.getFiles
self.getFlatFieldFiles = self._ffFiles.getFiles
......@@ -86,7 +86,7 @@ class DatasetSelection(qt.QTabWidget):
:return: dataset defined on the GUI
:rtype: :class:`Dataset`
"""
return Dataset(data_files=self.getDataFiles(),
return Dataset(data_files=self.getDataFilesPattern(),
dark_files=self.getDarkFiles(),
ff_files=self.getFlatFieldFiles())
......@@ -97,11 +97,33 @@ class DatasetSelection(qt.QTabWidget):
:param :class:`Dataset` dataset:
"""
assert isinstance(dataset, Dataset)
self._dataFiles.setFiles(dataset.data_files)
self._dataFiles.setFiles(dataset.data_files_pattern)
self._darkFiles.setFiles(dataset.dark_files)
self._ffFiles.setFiles(dataset._ff_files)
class FilePatternWidget(qt.QWidget):
"""A simple interface to select files from a pattern
.. warning: the widget won't check for scan validity and will only
emit the path to folders to the next widgets
:param parent: the parent widget
"""
def __init__(self, parent=None):
qt.QWidget.__init__(self, parent)
self.setLayout(qt.QHBoxLayout())
self.layout().addWidget(qt.QLabel('file pattern:', parent=self))
self._file_pattern = qt.QLineEdit('', parent=self)
self.layout().addWidget(self._file_pattern)
def getPattern(self):
return str(self._file_pattern.text())
def setPattern(self, pattern):
return self._file_pattern.setText(str(pattern))
class DataListWidget(qt.QWidget):
"""A simple list of dataset path.
......@@ -204,7 +226,6 @@ class FileList(qt.QTableWidget):
itemIndex = self.row(item)
self.takeItem(itemIndex, 0)
self._selection.remove(item.text())
# ScanList.remove(self, item.text())
self.removeRow(item.row())
self.setRowCount(self.rowCount() - 1)
self._update()
......
......@@ -32,7 +32,7 @@ from silx.gui import qt
from silx.gui.plot import Plot2D
from id06workflow.gui import icons
from id06workflow.core.mapping import MEAN, VARIANCE, SKENESS, KURTOSIS, MODES, INTENSITY
from id06workflow.core.experiment.operation.mapping import IntensityMapping
from id06workflow.core.experiment.operation.mapping import _MappingBase
from id06workflow.gui.settings import DEFAULT_COLORMAP
from collections import OrderedDict
import functools
......@@ -124,7 +124,7 @@ class MappingPlot(qt.QMainWindow):
self._data[SKENESS] = self._data[KURTOSIS] = None
if operation is not None:
assert isinstance(operation, IntensityMapping)
assert isinstance(operation, _MappingBase)
self._data[INTENSITY] = operation.intensity_map
self._data[MEAN] = operation.dim[0][:, :, 0]
self._data[VARIANCE] = operation.dim[0][:, :, 1]
......
......@@ -99,6 +99,7 @@ class NoiseReductionWidget(qt.QWidget):
threshold = self._paramsWidget.getUpperThreshold()
if threshold is not None:
self._upperThreshold.threshold = threshold
self._upperThreshold.replacement_value = 0.0
self._cache_data = self._upperThreshold.dry_run(self._cache_data)
threshold = self._paramsWidget.getLowerThreshold()
if threshold is not None:
......
......@@ -100,6 +100,8 @@ class ROIOnStackView(qt.QWidget):
size=stack.shape[-2:])
else:
self._clipROI()
# activate ROI edition by default
self._roi.setEditable(True)
def setROI(self, origin, size):
self._roi.setGeometry(origin=origin, size=size)
......
......@@ -75,7 +75,7 @@ class DataReductionOW(OWWidget):
self._widget.layout().addWidget(self._yreduc, 1, 1)
self._widget.layout().addWidget(qt.QLabel('z reduction:'), 2, 0)
self._zreduc = qt.QLineEdit('1', parent=self)
self._zreduc = qt.QLineEdit('10', parent=self)
self._zreduc.setValidator(qt.QIntValidator(parent=self))
self._widget.layout().addWidget(self._zreduc, 2, 1)
layout.addWidget(self._widget)
......@@ -92,12 +92,9 @@ class DataReductionOW(OWWidget):
def _process(self, experiment):
if experiment is None:
return
try:
operation = DataReduction(experiment=experiment,
x_factor=self.getXReduction(),
y_factor=self.getYReduction(),
z_factor=self.getZReduction())
operation.compute()
self.send("data", experiment)
except Exception as e:
_logger.error(e)
operation = DataReduction(experiment=experiment,
x_factor=self.getXReduction(),
y_factor=self.getYReduction(),
z_factor=self.getZReduction())
operation.compute()
self.send("data", experiment)
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "16/10/2018"