Commit eb765344 authored by payno's avatar payno

[core] rework to fit several dimension and fix unit test

parent 0a80aba7
......@@ -34,7 +34,8 @@ from id06workflow.core.geometry.TwoThetaGeometry import TwoThetaGeometry
from id06workflow.core.experiment.operation import _BaseOperation
from id06workflow.core.experiment.operation.roi import RoiOperation
from id06workflow.core.experiment.operation.datareduction import DataReduction
from collections import OrderedDict, namedtuple
from collections import OrderedDict
from id06workflow.core.utils import metadata as metadatautils
from silx.io.fabioh5 import FabioReader
from silx.io import fabioh5
import numpy
......@@ -48,11 +49,11 @@ COUNTER_METADATA = FabioReader.COUNTER
POSITIONER_METADATA = FabioReader.POSITIONER
_METADATA_TYPES = (DEFAULT_METADATA, COUNTER_METADATA, POSITIONER_METADATA)
_Dim = namedtuple('_Dim', ['kind', 'name', 'size'])
# TODO: define dimension of the Experiment, with a get dim...
_METADATA_TYPES = {
'default': DEFAULT_METADATA,
'counter': COUNTER_METADATA,
'positioner': POSITIONER_METADATA,
}
class Experiment(object):
......@@ -124,6 +125,10 @@ class Experiment(object):
else:
raise NotImplementedError('Cannot deal with several ROI yet')
@property
def dims(self):
return self.__dims
def getRawData(self):
# TODO: cache should probably be used in the future to deal wuth data
if self.dataset is None:
......@@ -169,16 +174,35 @@ class Experiment(object):
def data(self):
if self.__data is None:
self.__data = self.getRawData()
# TODO: create numpy view according to dims...
if self.__dims.ndim > 1:
# TODO: create view or adapt view on raw data
shape = list(self.__dims.shape)
shape.append(self.__data.shape[-2])
shape.append(self.__data.shape[-1])
return self.__data.view().reshape(shape)
else:
return self.__data
@property
def data_flatten(self):
if self.__data is None:
self.__data = self.getRawData()
return self.__data
@data.setter
def data(self, data):
self.__data = data
assert data.ndim > 2
self.__data = data.reshape(-1, data.shape[-2], data.shape[-1])
@property
def ndim(self):
return self.__ndim
"""
:return: number of dimension in the experiement (so do not include the 2
dimensions form the frame)
:rtype: int
"""
return self.__dims.ndim
@property
def background_subtracted(self):
......@@ -292,6 +316,25 @@ class Experiment(object):
assert self.data.ndim is 3
return self.data.shape[0]
def set_dims(self, dims):
assert isinstance(dims, dict)
self.__dims.clear()
for axis, dim in dims.items():
if dim.size is None:
# try to deduce dimension size
try:
unique_dim_value = metadatautils.getUnique(self,
kind=dim.kind,
key=dim.name,
relative_prev_val=dim.relative_prev_val,
cycle_length=dim.size,
axis=axis)
dim.size = len(unique_dim_value)
except Exception as e:
_logger.warning('Fail to deduce size of axis %s because %s' % (axis, e))
print('add dim for axis %s' % axis)
self.__dims.add_dim(axis=axis, dim=dims[axis])
class AcquisitionDims(object):
"""
......@@ -300,14 +343,20 @@ class AcquisitionDims(object):
def __init__(self):
self.__dims = {}
def add_dim(self, axis, kind, name, size=None):
assert type(axis) is int
assert size is None or type(size) is int
self.__dims[axis] = _Dim(name=name, kind=kind, size=size)
def add_dim(self, axis, dim):
assert type(dim) is _Dim
self.__dims[axis] = dim
def clear(self):
self.__dims = {}
@property
def ndim(self):
max_index = -1
for index in self.__dims.keys():
max_index = max(max_index, index)
return max(max_index, 0) + 1
@property
def shape(self):
"""
......@@ -315,10 +364,7 @@ class AcquisitionDims(object):
:return: shape of the currently defined dims
"""
shape = []
max_index = -1
for index in self.__dims.keys():
max_index = max(max_index, index)
for iDim in range(max_index + 1):
for iDim in range(self.ndim):
if iDim not in self.__dims:
shape.append(1)
else:
......@@ -330,6 +376,47 @@ class AcquisitionDims(object):
_logger.error('axis %s is not defined yet, cannot defined a size '
'for it' % axis)
else:
self.__dims[axis] = _Dim(self.__dims[axis].name,
self.__dims[axis].kind,
self.__dims[axis] = _Dim(name=self.__dims[axis].name,
kind=self.__dims[axis].kind,
size=size)
class _Dim(object):
def __init__(self, kind, name, size=None, relative_prev_val=False):
"""
Define a dimension used during the experiment
:param int or str kind: where the ket can be found in fabioh5 mapping
:param str name: name of the dimension (should fit the fabioh5 mapping
for now)
:param int or None size: length of the dimension.
:param relative_prev_val: if the parameter defining the dimension
(kind + key) is defined by a relative value.
'diffry' for example is a relative position.
For dimension we need to retrieve unique
values so in those cases we need to apply
a post processing and get this information
and length of the acquisition (passed
by size here)
:type relative_prev_val: bool
"""
if type(kind) is str:
assert kind in _METADATA_TYPES
self.__kind = _METADATA_TYPES[kind]
else:
self.__kind = kind
self.__name = name
self.size = size
self.__relative_prev_val = relative_prev_val
@property
def kind(self):
return self.__kind
@property
def name(self):
return self.__name
@property
def relative_prev_val(self):
return self.__relative_prev_val
......@@ -98,7 +98,7 @@ class OverwritingOperation(_BaseOperation):
_BaseOperation.__init__(self, experiment, name, can_overwrite_data=True)
@_BaseOperation.data.setter
def data(self, data):
def data_flatten(self, data):
self._experiment.data = data
def dry_run(self, cache_data=None):
......
......@@ -47,6 +47,7 @@ class COM(AdditiveOperation):
"""
_NAME = 'com calculation'
def __init__(self, map):
assert isinstance(map, _MappingBase)
AdditiveOperation.__init__(self, experiment=map.experiment,
......
......@@ -30,8 +30,7 @@ __date__ = "15/10/2018"
import logging
from . import AdditiveOperation, OverwritingOperation
from .mapping import IntensityMapping
from . import OverwritingOperation
import numpy
_logger = logging.getLogger(__file__)
......@@ -108,7 +107,7 @@ class ThresholdRemoval(OverwritingOperation):
def apply(self):
if self._cache_data is None:
raise ValueError('No data in cache')
self.data = self._cache_data
self.data_flatten = self._cache_data
self.registerOperation()
self._cache_data = None
......@@ -157,7 +156,7 @@ class BackgroundSubtraction(OverwritingOperation):
def apply(self):
if self._cache_data is None:
raise ValueError('No data in cache')
self.data = self._cache_data
self.data_flatten = self._cache_data
self.clear_cache()
self.registerOperation()
return self.data
......
......@@ -49,7 +49,7 @@ class RoiOperation(OverwritingOperation):
str(self._size)))
def compute(self):
self.data = self._compute(self.data)
self.data_flatten = self._compute(self.data_flatten)
self.registerOperation()
return self.data
......@@ -63,7 +63,7 @@ class RoiOperation(OverwritingOperation):
def apply(self):
if self._cache_data is None:
raise ValueError('No data in cache')
self.data = self._cache_data
self.data_flatten = self._cache_data
self.clear_cache()
self.registerOperation()
return self.data
......
......@@ -81,12 +81,12 @@ class Shift(OverwritingOperation):
def dz(self):
return self._dz
@dx.setter
@dz.setter
def dz(self, dz):
self._dz = dz
def compute(self):
self.data = self._compute(self.data)
self.data_flatten = self._compute(self.data)
self.registerOperation()
return self.data
......@@ -101,7 +101,7 @@ class Shift(OverwritingOperation):
def apply(self):
if self._cache_data is None:
raise ValueError('No data in cache')
self.data = self._cache_data
self.data_flatten = self._cache_data
self.clear_cache()
self.registerOperation()
return self.data
......
......@@ -29,7 +29,7 @@ __date__ = "09/10/2018"
import unittest
from id06workflow.core.experiment import Experiment, Dataset, AcquisitionDims
from id06workflow.core.experiment import Experiment, Dataset, AcquisitionDims, _Dim, DEFAULT_METADATA
from id06workflow.core.geometry.TwoThetaGeometry import TwoThetaGeometry
from id06workflow.test import utils
import numpy
......@@ -122,16 +122,16 @@ class TestAcquisitionDims(unittest.TestCase):
def test(self):
"""global test of the AcquisitionDims APi"""
acquiDim = AcquisitionDims()
self.assertTrue(acquiDim.shape == tuple())
acquiDim.add_dim(axis=0, kind='0', name='0', size=10)
acquiDim.add_dim(axis=1, kind='1', name='1')
self.assertTrue(acquiDim.shape == (1, ))
acquiDim.add_dim(axis=0, dim=_Dim(kind=DEFAULT_METADATA, name='0', size=10))
acquiDim.add_dim(axis=1, dim=_Dim(kind=DEFAULT_METADATA, name='1'))
self.assertTrue(acquiDim.shape == (10, -1))
acquiDim.set_size(axis=1, size=2)
self.assertTrue(acquiDim.shape == (10, 2))
acquiDim.clear()
self.assertTrue(acquiDim.shape == tuple())
acquiDim.add_dim(axis=1, kind='0', name='0', size=10)
acquiDim.add_dim(axis=4, kind='1', name='1')
self.assertTrue(acquiDim.shape == (1, ))
acquiDim.add_dim(axis=1, dim=_Dim(kind=DEFAULT_METADATA, name='0', size=10))
acquiDim.add_dim(axis=4, dim=_Dim(kind=DEFAULT_METADATA, name='1'))
self.assertTrue(acquiDim.shape == (1, 10, 1, 1, -1))
......
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "20/11/2018"
import unittest
import numpy
from id06workflow.core import image
class TestShift(unittest.TestCase):
"""
Test that RefCopy process is correct
"""
def testShit(self):
"""
Some stupid test to make sur TwoThetaExpSetup is correctly
instanciated
"""
data = numpy.arange(100).reshape(10, 10)
shifted_data = image.shift_img(data, dx=1.0, dy=-1.0)
self.assertTrue(numpy.allclose(data[:-1, 1:], shifted_data[1:, :-1]))
def suite():
test_suite = unittest.TestSuite()
for ui in (TestShift, ):
test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(ui))
return test_suite
if __name__ == '__main__':
unittest.main(defaultTest="suite")
......@@ -64,12 +64,12 @@ class TestOperation(unittest.TestCase):
"""
operationOverwrite = OverwritingOperation(experiment=self.experiment,
name='test')
operationOverwrite.data = 'toto'
operationOverwrite.data_flatten = numpy.random.rand(1, 10, 10)
additiveOperation = AdditiveOperation(experiment=self.experiment,
name='test')
with self.assertRaises(Exception):
additiveOperation.data = 'toto'
additiveOperation.data = numpy.random.rand(1, 10, 10)
class TestCOM(unittest.TestCase):
......
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "21/11/2018"
import unittest
from id06workflow.core.experiment import Experiment, Dataset, POSITIONER_METADATA
from id06workflow.core.utils import metadata as metadatautils
from id06workflow.core.experiment import Experiment, _Dim
import os
class TestOperationStream(unittest.TestCase):
"""Test some default operation streaming"""
def setUp(self):
unittest.TestCase.setUp(self)
root_folder = '/nobackup/linazimov/payno/datasets/id06/strain_scan'
data_file_pattern = os.path.join(root_folder, 'reduced_strain/strain_0000.edf')
data_bb_files = []
bb_folder = os.path.join(root_folder, 'bg_ff_5s_1x1')
for _file in os.listdir(bb_folder):
data_bb_files.append(os.path.join(bb_folder, _file))
self.dataset = Dataset(data_files_pattern=data_file_pattern,
ff_files=data_bb_files)
self.experiment = Experiment(dataset=self.dataset, geometry=None)
def applyGeometry(self):
"""Apply some geometry"""
dim1 = _Dim(kind=POSITIONER_METADATA, name='diffry',
relative_prev_val=True, size=31)
dim2 = _Dim(kind=POSITIONER_METADATA, name='obpitch')
self.experiment.set_dims(dims={0: dim1, 1: dim2})
def testGeometry(self):
"""Make sure geometry is well applied"""
self.assertTrue(len(metadatautils.getUnique(self.experiment,
kind=POSITIONER_METADATA,
key='diffry',
relative_prev_val=True,
cycle_length=31,
axis=0
)) is 31)
self.assertTrue(len(metadatautils.getUnique(self.experiment,
kind=POSITIONER_METADATA,
key='obpitch')) is 2)
self.applyGeometry()
self.assertTrue(self.experiment.dims.ndim is 2)
self.assertTrue(self.experiment.dims.shape == (31, 2))
self.assertTrue(self.experiment.data.shape == (31, 2, 2048, 2048))
def suite():
test_suite = unittest.TestSuite()
for ui in (TestOperationStream, ):
test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(ui))
return test_suite
if __name__ == '__main__':
unittest.main(defaultTest="suite")
......@@ -28,33 +28,53 @@ __license__ = "MIT"
__date__ = "21/11/2018"
from id06workflow.core.experiment import Experiment
import numpy
import logging
_logger = logging.getLogger(__file__)
def getUnique(experiment, key, kind):
def getUnique(experiment, key, kind, relative_prev_val=False, cycle_length=None,
axis=None):
"""Return the set of (unique) value in all the headers
:param str key: the type of value we want
:param Dataset dataset: the dataset to treat
:param :class: Dataset dataset: the dataset to treat
:param bool relative_prev_val: In the case we base the dimension on a
relative value (such as diffry) we can have
identical values representing different
positions. So we need to compute the absolute
value of those. And specify it.
This is also why we need the cycle_length
and the axis
:return set: set of value
"""
assert isinstance(experiment, Experiment)
if relative_prev_val is True:
assert cycle_length is not None and type(cycle_length) is int
assert axis is not None and type(axis) is int
metadata = experiment.metadata
if metadata is None:
_logger.warning('no metadata found in the given experiment, unable to '
'unique value of %s' % key)
return set()
res = set()
for metadata_slice in experiment.metadata:
return numpy.array()
res = list()
for iFrame, metadata_slice in enumerate(experiment.metadata):
if (relative_prev_val is True and axis != 0 and
iFrame % (cycle_length * axis) != 0):
continue
try:
value = metadata_slice.get_value(kind=kind, name=key)
assert isinstance(value, numpy.ndarray)
assert value.size is 1
res.add(value[0])
if not (isinstance(value, numpy.ndarray) and value.size is 1):
raise NotImplementedError('Case with values which are not single'
'values ar enot managed')
else:
if relative_prev_val is True and len(res) > 0:
res.append(value[0] + res[-1])
else:
res.append(value[0])
except Exception as e:
_logger.error("fail to load %s, %s cause %s" % (kind, key, e))
return res
if cycle_length is not None and len(res) >= cycle_length:
break
return numpy.unique(res)
......@@ -102,7 +102,7 @@ class DatasetSelection(qt.QTabWidget):
:param :class:`Dataset` dataset:
"""
assert isinstance(dataset, Dataset)
self._dataFiles.setFiles(dataset.data_files_pattern)
self._dataFiles.setPattern(dataset.data_files_pattern)
self._darkFiles.setFiles(dataset.dark_files)
self._ffFiles.setFiles(dataset._ff_files)
......
......@@ -57,7 +57,7 @@ def createRandomDataset(dims, nb_data_files=10, nb_dark_files=2,
data_file = os.path.join(_dir, 'data_file%04i.edf' % index)
image = fabio.edfimage.EdfImage(data=numpy.random.random(dims))
image.write(data_file)
dataset.addDataFile(data_file)
dataset.data_files_pattern = os.path.join(_dir, 'data_file%04i.edf' % 0)
for index in range(nb_dark_files):
dark_file = os.path.join(_dir, 'dark_file%04i.edf' % index)
......
......@@ -32,10 +32,8 @@ from silx.gui import qt
from Orange.widgets import gui
from Orange.widgets.widget import OWWidget
from Orange.canvas.registry.description import InputSignal, OutputSignal
from id06workflow.gui.mapping import MappingPlot
from id06workflow.core.experiment import Experiment
from id06workflow.core.experiment.operation.mapping import GradientRemoval
from id06workflow.core.types import _Image
from id06workflow.core.experiment import Experiment, _METADATA_TYPES
import numpy
class MetadataTableOW(OWWidget):
......@@ -61,29 +59,65 @@ class MetadataTableOW(OWWidget):
super().__init__()
layout = gui.vBox(self.mainArea, 'metadata table').layout()
self._metadataTypesWidget = qt.QWidget(parent=self)
layout.addWidget(self._metadataTypesWidget)
self._metadataTypesWidget.setLayout(qt.QHBoxLayout())
spacer1 = qt.QWidget(self._metadataTypesWidget)
spacer1.setSizePolicy(qt.QSizePolicy.Expanding,
qt.QSizePolicy.Minimum)
self._metadataTypesWidget.layout().addWidget(spacer1)
self._metadataTypesWidget.layout().addWidget(
qt.QLabel('metadata type:', parent=self._metadataTypesWidget)
)
self._metadataCB = qt.QComboBox(parent=self._metadataTypesWidget)
for _metaDType in _METADATA_TYPES:
self._metadataCB.addItem(_metaDType)
self._metadataTypesWidget.layout().addWidget(self._metadataCB)
default_index = self._metadataCB.findText('default')
assert default_index >= 0
self._metadataCB.setCurrentIndex(default_index)
self._table = qt.QTableWidget(parent=self)
layout.addWidget(self._table)
# connect Signal/SLOT
self._metadataCB.currentTextChanged.connect(self._updateView)
def _process(self, experiment):
if experiment is None:
return
self.__experiment = experiment
self._updateView()
def _updateView(self, metadata_type=None):
if metadata_type is None:
metadata_type = self._metadataCB.currentText()
metadata_type = _METADATA_TYPES[metadata_type]
self._table.clear()
v_header = []
[v_header.append(str(i)) for i in range(experiment.nslices)]
[v_header.append(str(i)) for i in range(self.__experiment.nslices)]
self._table.setRowCount(len(v_header))
self._table.setVerticalHeaderLabels(v_header)
columnCount = None
for row, metadata_slice in enumerate(experiment.metadata):
for row, metadata_slice in enumerate(self.__experiment.metadata):
keys = metadata_slice.get_keys(kind=metadata_type)
if columnCount is None:
columnCount = len(metadata_slice)
self._table.setColumnCount(columnCount)
self._table.setHorizontalHeaderLabels(metadata_slice.keys())
self._table.setColumnCount(len(keys))
self._table.setHorizontalHeaderLabels(keys)
elif columnCount != len(metadata_slice):
raise ValueError('Metadata keys are incoherent')
for column, info in enumerate(metadata_slice):
for column, key in enumerate(keys):
_item = qt.QTableWidgetItem()
_item.setText(metadata_slice[info])
# _item.setFlags(qt.Qt.ItemIsEnabled | qt.Qt.ItemIsSelectable)
txt = metadata_slice.get_value(kind=metadata_type, name=key)
if type(txt) is numpy.ndarray and txt.size is 1:
txt = txt[0]
if hasattr(txt, 'decode'):
txt = txt.decode('utf-8')
else:
txt = str(txt)
_item.setText(txt)
_item.setFlags(qt.Qt.ItemIsEnabled | qt.Qt.ItemIsSelectable)
self._table.setItem(row, column, _item)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment