Commit 11fa4c3a authored by payno's avatar payno

[refactoring] Add the concept of Operation on the dataset

Add also the list of operation on the save experiment widget.
parent 8d48c1d9
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "01/10/2018"
from id06workflow.core.Dataset import Dataset
from id06workflow.core.geometry import GeometryBase
from id06workflow.core.geometry.TwoThetaGeometry import TwoThetaGeometry
from id06workflow.core.experiment.operation import _BaseOperation
from id06workflow.core.experiment.operation.roi import RoiOperation
from id06workflow.core.experiment.operation.datareduction import DataReduction
from collections import OrderedDict
import numpy
import logging
_logger = logging.getLogger(__file__)
class Experiment(object):
"""
Define all component of an experiment:
- :class:`Dataset`
- :class:`Geometry`
"""
def __init__(self, dataset=None, geometry=None):
if dataset is not None:
assert isinstance(dataset, Dataset)
if geometry is not None:
assert isinstance(geometry, GeometryBase)
self.dataset = dataset
self.geometry = geometry
self.__data = None
"""current data. The one updated with treatment, roi...
Which is different from the raw data contained in the dataset"""
self.__ndim = 1
"""Number of dimension of the experiment (motors scanned/rocked)
For now limited to 1
"""
self._operation_stack = OrderedDict()
"""List all the operations applied don the raw data"""
self._rois_operations = []
"""List all the operations relative to roi on raw data (duplicated
from operation stack)"""
self._data_reduction_operations = []
"""List all the operations relative to data reduction on raw data
(duplicated from operation stack)"""
@property
def dataset(self):
return self._dataset
@dataset.setter
def dataset(self, dataset):
assert dataset is None or isinstance(dataset, Dataset)
self._dataset = dataset
@property
def geometry(self):
return self._geometry
@geometry.setter
def geometry(self, geometry):
assert geometry is None or isinstance(geometry, GeometryBase)
self._geometry = geometry
@property
def roi(self):
# TODO: manage several rois
rois = self._getRoiOperations()
if len(rois) is 0:
return None
elif len(rois) is 1:
return rois[0]._origin, rois[0]._size
else:
raise NotImplementedError('Cannot deal with several ROI yet')
def getRawData(self):
# TODO: cache should probably be used in the future to deal wuth data
if self.dataset is None:
return None
rawdata = self.dataset.getData()
reductionStep = self._getDataReductionOperations()
if reductionStep in (None, []):
steps = (1, 1, 1)
else:
if len(reductionStep) is 1:
steps = reductionStep[0]
else:
raise NotImplementedError('cannot manage several reduction steps for the moment')
if self.roi is not None:
assert isinstance(self.roi, tuple)
origin, size = self.roi
assert rawdata.ndim is 3
# TODO: make sure origin and size are given as y, x
ymin, ymax = int(origin[1]), int(origin[1] + size[1])
xmin, xmax = int(origin[0]), int(origin[0] + size[0])
return rawdata[::steps[0], ymin:ymax:steps[1], xmin:xmax:steps[2]]
else:
return rawdata[::steps[0], ::steps[1], ::steps[2]]
@property
def data(self):
if self.__data is None:
self.__data = self.getRawData()
return self.__data
@data.setter
def data(self, data):
self.__data = data
@property
def ndim(self):
return self.__ndim
@property
def background_subtracted(self):
return self.hasOperation('background substraction')
@background_subtracted.setter
def background_subtracted(self, substracted):
self._background_subtracted = substracted
@property
def angles(self):
# TODO: cache angles array ?
if self.geometry is None:
_logger.warning('No geometry defined')
return None
elif isinstance(self.geometry, TwoThetaGeometry):
# TODO: not sure about the way to compute angles. Different in the matlab script.
return numpy.linspace(start=-self.geometry.twotheta,
stop=self.geometry.twotheta,
num=self.data.shape[0])
else:
raise NotImplementedError('Geometry type not managed')
@property
def operations(self):
return self._operation_stack
def getMedianBackground(self):
"""
:return: the background of the dataset, clamp to the roi if any defined
:rtype single value or numpy.ndarray
"""
background = self.dataset.getBackground()
# case background is the mean value of the current data
if background is None:
if self.__data is not None:
_logger.warning(
'No flat field defined for background, getting it '
'directly from data files')
return numpy.median(numpy.nan_to_num(self.__data))
else:
return None
else:
# case background is the mean of flat fields
reductionStep = self._getDataReductionOperations()
if reductionStep in (None, []):
steps = (1, 1, 1)
else:
if len(reductionStep) is 1:
steps = reductionStep[0]
else:
raise NotImplementedError(
'cannot manage several reduction steps for the moment')
background_img = numpy.median(background, axis=0)
if self.roi is None:
return background_img[::steps[1], ::steps[2]]
else:
origin, size = self.roi
ymin, ymax = int(origin[1]), int(origin[1] + size[1])
xmin, xmax = int(origin[0]), int(origin[0] + size[0])
return background_img[ymin:ymax:steps[1], xmin:xmax:steps[2]]
def addOperation(self, operation):
"""Add the operation to the stack of operations"""
_logger.info('register operation %s' % operation.key())
assert isinstance(operation, _BaseOperation)
if operation.key() in self._operation_stack:
_logger.warning('operation %s has been processed several time' % operation.key())
self._operation_stack[operation.key()] = operation
if isinstance(operation, RoiOperation):
self._rois_operations.append(operation)
if isinstance(operation, DataReduction):
self._data_reduction_operations.append(operation)
def _getRoiOperations(self):
"""
:return: the list of roi operation
"""
return self._rois_operations
def _getDataReductionOperations(self):
"""
:return: the list of roi operation
"""
return self._data_reduction_operations
def hasOperation(self, operationID):
raise NotImplementedError('Not implemented yet')
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "15/10/2018"
import numpy
class _BaseOperation(object):
"""
Operation to be applied on an experiment
"""
# TODO would be nice to have required operations to be processed before this
# one
def __init__(self, experiment, name, can_overwrite_data=False):
"""
:param :class:`Experiment` experiment: experiment concerned by the
operation
:param name: bame of the operation
:param bool can_overwrite_data: If True then can overwrite
experiment.data
:param must_be_unique: If true then can only be applied once to the
experiment
"""
self._name = name
self._experiment = experiment
self._can_overwrite_data = can_overwrite_data
@property
def data(self):
data = self._experiment.data.view()
data.setflags(write=self._can_overwrite_data)
return data
def compute(self):
raise NotImplementedError('Base class')
def key(self):
"""
:return: key used to define the operation
:rtype: str
"""
raise NotImplementedError('Base class')
def registerOperation(self):
self._experiment.addOperation(self)
class OverwritingOperation(_BaseOperation):
"""
Operation that can effect the experiment __data
"""
def __init__(self, experiment, name):
_BaseOperation.__init__(self, experiment, name, can_overwrite_data=True)
@_BaseOperation.data.setter
def data(self, data):
self._experiment.data = data
def dry_run(self, cache_data=None):
"""
Apply the data on a different experiement/ dataset...
To be used before applying it
:param cache_data: some existing cache data if any
"""
raise NotImplementedError('Base class')
def apply(self):
"""
If dry ru nhas been called then can use data in cache to apply the
operation
"""
raise NotImplementedError('Base class')
def clear_cache(self):
"""
Clear the cache used for the dry_run
"""
raise NotImplementedError('Base class')
class AdditiveOperation(_BaseOperation):
"""
Operation that cannot effect the experiment __data
"""
def __init__(self, experiment, name):
_BaseOperation.__init__(self, experiment, name, can_overwrite_data=False)
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "15/10/2018"
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "15/10/2018"
from . import OverwritingOperation
class DataReduction(OverwritingOperation):
def __init__(self, experiment, x_factor, y_factor, z_factor):
"""
Apply a data reduction to the experiment
:param experiment:
:param x_factor:
:param y_factor:
:param z_factor:
"""
OverwritingOperation.__init__(self, experiment, name='data reduction')
self._x_factor = x_factor
self._y_factor = y_factor
self._z_factor = z_factor
self._cache_data = None
def dry_run(self, cache_data=None):
if cache_data is None:
self._cache_data = self.data[...][::self._z_factor, ::self._y_factor, ::self._x_factor]
else:
self._cache_data = cache_data
return self._cache_data
def compute(self):
self.data = self.data[::self._z_factor, ::self._y_factor, ::self._x_factor]
self.registerOperation()
return self.data
def apply(self):
if self._cache_data is None:
raise ValueError('No data in cache')
self.data = self._cache_data
self.clear_cache()
self.registerOperation()
return self.data
def clear_cache(self):
self._cache_data = None
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "15/10/2018"
from . import AdditiveOperation
import numpy
import logging
_logger = logging.getLogger(__file__)
class IntensityMapping(AdditiveOperation):
"""
for each detector pixel (x,y), each data dimension (motors scanned/rocked)
are evaluated: 1) intensity is summed along all other dimensions than the
active one 2) maximum momentum method used to find peak position,
variance and skewness 3) these are saved in Experiment.map
:param float or None threshold: if the threshold is None then the mean
value of the current data will be take
"""
def __init__(self, experiment, threshold=None):
AdditiveOperation.__init__(self, experiment=experiment,
name='intensity mapping')
self._dim = []
if threshold is None:
self._threshold = numpy.median(self.data)
else:
self._threshold = threshold
self.__intensity_map = None
self.__dim = []
def key(self):
return self._name
def compute(self):
_logger.info('start computing the dx2d map')
if self.data is None:
_logger.warning('Experiment as no data defined, unable to apply'
'mapping')
return None
elif self._experiment.geometry is None:
_logger.warning('No geometry defined to apply mapping')
return None
else:
assert self.data.ndim is 3
# TODO: avoid recomputing several time the intensity map
self._resetIntensityMap()
self._resetDim()
for x in range(self.data.shape[2]):
for y in range(self.data.shape[1]):
# TODO: do this operation using a grid or something close
# but would have to find a way to write at the same position...
reciprocalVol = numpy.squeeze(self.data[:, y, x])
reciprocalVol = reciprocalVol - reciprocalVol.min()
maxIntensity = reciprocalVol.max()
if maxIntensity > self._threshold:
self.__intensity_map[y, x] = maxIntensity
# Here: TODO, later when can have several dimension (motors scanned/rockers)
# do it for all dims
for iDim in range(self._experiment.ndim):
intensity = numpy.squeeze(reciprocalVol.sum())
# TODO: move calculation of angle_mean, angle_variance,
# angle_skewness, angle_kurtosis in :class:`Experiment`
angles = self._experiment.angles
# TODO: why mean is obtained by the sum ?
# angles_mean = (intensity * angles).mean()
angles_mean = (intensity * angles).sum()
angles_variance = (
intensity * (angles - angles_mean) ** 2).sum()
angles_skewness = (
intensity * (angles - angles_mean) ** 3).sum()
angles_kurtosis = (
intensity * (angles - angles_mean) ** 4).sum()
# TODO: pb, this is overwritten at each image if maxIntensity > threshold ???
self.dim[iDim][y, x, 0] = angles_mean
self.dim[iDim][y, x, 1] = angles_variance
self.dim[iDim][y, x, 2] = angles_skewness
self.dim[iDim][y, x, 3] = angles_kurtosis
print('should register mapping operation')
self.registerOperation()
return self.dim
def _resetIntensityMap(self):
self.__intensity_map = numpy.zeros(self.data.shape[1:])
def _resetDim(self):
self.__dim = []
for dim in range(self._experiment.ndim):
self.__dim.append(numpy.zeros((*(self.data.shape[1:]), 4)))
@property
def intensity_map(self):
return self.__intensity_map
@property
def dim(self):
return self.__dim
This diff is collapsed.
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "15/10/2018"
from . import OverwritingOperation
class RoiOperation(OverwritingOperation):
def __init__(self, experiment, origin, size):
OverwritingOperation.__init__(self, experiment=experiment, name='roi')
self._origin = origin
self._size = size
"""
roi, for now store as a tuple (origin, size). Only deal with rectangle
roi for now.
"""
self._cache_data = None
""" numpy.ndarray used for caching"""
def key(self):
return ' '.join((self._name, 'origin:', str(self._origin), 'origin:',
str(self._size)))
def compute(self):
self.data = self._compute(self.data)
self.registerOperation()
return self.data
def dry_run(self, cache_data=None):
if cache_data is None:
self._cache_data = self.data[...]
else:
self._cache_data = cache_data
return self._compute(self._cache_data)
def apply(self):
if self._cache_data is None:
raise ValueError('No data in cache')
self.data = self._cache_data
self.clear_cache()
self.registerOperation()
return self.data
def _compute(self, data):
ymin, ymax = int(self._origin[1]), int(self._origin[1] + self._size[1])
xmin, xmax = int(self._origin[0]), int(self._origin[0] + self._size[0])
data = data[:, ymin:ymax, xmin:xmax]
return data
def clear_cache(self):
self._cache_data = None
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "15/10/2018"
import logging
from . import OverwritingOperation