Commit 7d4251e8 authored by payno's avatar payno

[io] rework the frm_dict and to_dict functions to keep trace about processing

parent d1adc360
......@@ -38,4 +38,5 @@ _logger = logging.getLogger(__name__)
def suite():
test_suite = unittest.TestSuite()
test_suite.addTest(test_exec_suite())
test_suite.addTest(test_reprocessing_suite())
return test_suite
......@@ -81,7 +81,8 @@ def read_frm_file(file_path):
class XASReader(object):
"""Simple reader of a xas file"""
def read_frm_url(self, spectra_url, channel_url, dimensions=None,
@staticmethod
def read_frm_url(spectra_url, channel_url, dimensions=None,
config_url=None):
sp, en, conf = read_xas(spectra_url=spectra_url,
channel_url=channel_url,
......@@ -89,26 +90,27 @@ class XASReader(object):
dimensions=dimensions)
return XASObject(spectra=sp, energy=en, configuration=conf)
def read_from_file(self, file_path):
@staticmethod
def read_from_file(file_path):
"""
:param str file_path:
:return: `.XASObject`
"""
if file_path.endswith('.dat'):
return self.read_frm_url(
return XASReader.read_frm_url(
spectra_url=DataUrl(file_path=file_path,
scheme='PyMca'),
channel_url=DataUrl(file_path=file_path,
scheme='PyMca'))
elif file_path.endswith('.xmu'):
return self.read_frm_url(
return XASReader.read_frm_url(
spectra_url=DataUrl(file_path=file_path,
scheme='larch'),
channel_url=DataUrl(file_path=file_path,
scheme='larch'))
elif h5py.is_hdf5(file_path):
return self.read_frm_url(
return XASReader.read_frm_url(
spectra_url=DataUrl(file_path=file_path,
scheme='silx',
data_path=DEFAULT_SPECTRA_PATH),
......@@ -124,7 +126,6 @@ class XASReader(object):
__call__ = read_from_file
class XASWriter(object):
"""
class to write the output file. In this case we need a class in order to
......
......@@ -35,6 +35,7 @@ import shutil
import tempfile
import unittest
import h5py
import numpy
from silx.io.dictdump import h5todict
from silx.io.url import DataUrl
from est.core.process.roi import xas_roi, _ROI as XASROI
......@@ -115,7 +116,12 @@ class TestStreamSingleSpectrum(unittest.TestCase):
roi = XASROI(origin=(0, 2), size=(5, 1))
xas_obj = XASObject(spectra=spectra, energy=energy,
configuration={'roi': roi.to_dict()})
out = xas_roi(xas_obj.to_dict())
dict_xas_obj = xas_obj.to_dict()
self.assertTrue('spectra' in dict_xas_obj.keys())
self.assertTrue('energy' in dict_xas_obj.keys())
tmp_obj = XASObject.from_dict(dict_xas_obj)
numpy.testing.assert_array_equal(tmp_obj.energy, tmp_obj.spectra[0].energy)
out = xas_roi(dict_xas_obj)
out.configuration = {'EXAFS': self.exafs_configuration, 'SET_KWEIGHT': 0}
out = pymca_normalization(xas_obj=out)
out = pymca_exafs(xas_obj=out)
......@@ -263,7 +269,8 @@ class TestSaveFlowAuto(unittest.TestCase):
# check one configuration
stored_config = h5todict(self.h5_file,
path='/scan1/xas_process_2/configuration')
path='/scan1/xas_process_2/configuration',
asarray=False)
for key in ('KMin', 'KMax'):
self.assertTrue(configuration[key] == stored_config[key])
......
......@@ -123,6 +123,9 @@ class ROIProcess(Process):
for key in xas_obj.spectra_keys():
if isinstance(xas_obj.spectra[0][key], numpy.ndarray):
# there is no processing for the _larch_grp_members case
if key == '_larch_grp_members':
continue
volume = xas_obj._spectra_volume(spectra=xas_obj.spectra,
key=key,
dim_1=xas_obj.dim1,
......
......@@ -32,10 +32,12 @@ import os
import unittest
import tempfile
import h5py
from est.core.types import Spectrum, XASObject
import shutil
from est.core.types import Spectrum, XASObject, Dim
from est.core.utils import spectra as spectra_utils
from est.core.io import read as read_xas
from silx.io.url import DataUrl
import json
import silx.io.utils
try:
import PyMca5
......@@ -92,6 +94,9 @@ class TestXASObject(unittest.TestCase):
ddict = obj.to_dict()
obj2 = XASObject.from_dict(ddict)
self.assertEqual(obj2, obj)
# insure the XASObject is serializable
# import json
# json.dumps(obj2.to_dict())
def test_create_from_several_spectrums(self):
"""check that we can create a XASObject from numpy arrays"""
......@@ -125,9 +130,87 @@ class TestXASObject(unittest.TestCase):
self.assertEqual(obj2, self.xas_obj)
class TestXASObjectSerialization(unittest.TestCase):
def setUp(self) -> None:
self.energy, self.spectra = spectra_utils.create_dataset(
shape=(256, 20, 10))
self.output_dir = tempfile.mkdtemp()
self.spectra_path = '/data/NXdata/data'
self.channel_path = '/data/NXdata/Channel'
self.filename = os.path.join(self.output_dir, 'myfile.h5')
with h5py.File(self.filename, 'a') as f:
f[self.spectra_path] = self.spectra
f[self.channel_path] = self.energy
self.dimensions = (Dim.CHANNEL_ENERGY_DIM, Dim.Y_DIM, Dim.X_DIM)
self.url_spectra = DataUrl(file_path=self.filename,
data_path=self.spectra_path,
scheme='silx')
self.url_energy = DataUrl(file_path=self.filename,
data_path=self.channel_path,
scheme='silx')
self.process_flow_file = os.path.join(self.output_dir, 'process_flow.h5')
def tearDown(self) -> None:
shutil.rmtree(self.output_dir)
def test_serialization_url_auto(self):
"""Make sure the `to_dict` and `from_dict` functions are working
if no url are provided"""
xas_obj = XASObject(spectra=self.spectra, energy=self.energy,
dim1=20, dim2=10, keep_process_flow=False)
# if no h5 file defined, should fail to copy it to a dictionary
with self.assertRaises(ValueError):
xas_obj.to_dict()
xas_obj.link_to_h5(self.process_flow_file)
dict_xas_obj = xas_obj.to_dict()
# make sure it is serializable
json.dumps(dict_xas_obj)
# make sure we find a comparable xas object from it
xas_obj_2 = XASObject.from_dict(dict_xas_obj)
numpy.testing.assert_array_equal(xas_obj.energy, xas_obj_2.energy)
self.assertEqual(xas_obj, xas_obj_2)
# simple test without the process_details
dict_xas_obj = xas_obj.to_dict(with_process_details=False)
# make sure it is serializable
json.dumps(dict_xas_obj)
def test_serialization_url_provided(self):
"""Make sure the `to_dict` and `from_dict` functions are working
if url are provided"""
xas_obj = XASObject(spectra=self.spectra, energy=self.energy,
dim1=20, dim2=10, keep_process_flow=False,
energy_url=self.url_energy,
spectra_url=self.spectra_path)
# if no h5 file defined, should fail to copy it to a dictionary
with self.assertRaises(ValueError):
xas_obj.to_dict()
xas_obj.link_to_h5(self.process_flow_file)
dict_xas_obj = xas_obj.to_dict()
# make sure it is serializable
json.dumps(dict_xas_obj)
# make sure we find a comparable xas object from it
xas_obj_2 = XASObject.from_dict(dict_xas_obj)
numpy.testing.assert_array_equal(xas_obj.energy, xas_obj_2.energy)
self.assertEqual(xas_obj, xas_obj_2)
# simple test without the process_details
dict_xas_obj = xas_obj.to_dict(with_process_details=False)
# make sure it is serializable
json.dumps(dict_xas_obj)
def suite():
test_suite = unittest.TestSuite()
for ui in (TestSpectrum, TestXASObject, ):
for ui in (TestSpectrum, TestXASObject, TestXASObjectSerialization):
test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(ui))
return test_suite
......
This diff is collapsed.
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
"""Define some utils relative to larch"""
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "02/12/2019"
from larch.symboltable import Group
def group_to_dict(group):
"""Convert the larch group to a serializable dictionary
:param group: the group to convert to a serializable dictionary
:type: larch.symboltable.Group
:returns: dictionary corresponding to the given larch.symboltable.Group
:rtype: dictionary
"""
res = {}
for key in group._members():
if isinstance(group._members()[key], Group):
res[key] = group_to_dict(group._members()[key])
else:
res[key] = group._members()[key]
def dict_to_group(dict_, group):
"""Update the given larch group with the content of the dictionary
:param dict_:
:type: dict
:param group:
:type: larch.symboltable.Group
"""
for key in dict_:
group._members()[key] = dict_[key]
......@@ -331,7 +331,7 @@ class SpectrumViewer(qt.QMainWindow):
assert dim1_index >= 0
assert dim2_index >= 0
spectrum = self.xas_obj.getSpectrum(dim1_index, dim2_index)
spectrum = self.xas_obj.get_spectrum(dim1_index, dim2_index)
for operation in self._curveOperations:
curves = [operation(spectrum),]
if silx_plot_has_baseline_feature is True:
......
......@@ -32,7 +32,7 @@ from datetime import datetime
import h5py
import numpy
from silx.io import utils
from silx.io.dictdump import dicttoh5
from silx.io.dictdump import dicttoh5, h5todict
from silx.io.url import DataUrl
from silx.utils.enum import Enum
......@@ -58,6 +58,82 @@ class InputType(Enum):
xmu_spectrum = '*.xmu' # contains one spectrum
def load_data(data_url, name):
"""
Load a specific data from an url. Manage the different scheme (silx, fabio,
numpy, PyMca, xraylarch)
:param data_url: silx DataUrl with path to the data
:type: DataUrl
:param str name: name of the data we want to load. Should be in
('spectra', 'energy', 'configuration')
:return: data loaded
:rtype: Union[None,dict,numpy.ndarray]
"""
if data_url is None:
return None
if data_url.scheme() in ('PyMca', 'PyMca5'):
if has_pymca is False:
_logger.warning('Requires PyMca to load data from '
'%s' % data_url.path())
return None
else:
assert name in ('spectra', 'energy')
energy, mu = pymca_read_spectrum(data_url.file_path())
if name == 'spectra':
return mu.reshape(mu.shape[0], 1, 1)
else:
return energy
elif data_url.scheme() in ('larch', 'xraylarch'):
if has_larch is False:
_logger.warning('Requires larch to load data from '
'%s' % data_url.path())
return None
else:
assert name in ('spectra', 'energy')
energy, mu = larch_read_ascii(xmu_file=data_url.file_path())
if name == 'spectra':
return mu.reshape(mu.shape[0], 1, 1)
else:
return energy
elif data_url.scheme() == 'numpy':
return numpy.load(data_url.file_path())
elif data_url.scheme() == 'est':
assert name == 'spectra'
spectra = []
with h5py.File(data_url.file_path(), 'r') as hdf5:
# get all possible entries
entries = filter(lambda x: isinstance(hdf5[x], h5py.Group) and 'est_saving_pt' in hdf5[x].keys(), hdf5.keys())
entries = list(entries)
if len(entries) == 0:
_logger.error('no spectra dataset found in the file', data_url.file_path())
return
if len(entries) > 1:
_logger.warning('several entry detected, only one will be loaded:', entries[0])
spectra_path = '/'.join((entries[0], 'est_saving_pt', 'spectra'))
node_spectra = hdf5[spectra_path]
spectrum_indexes = list(node_spectra.keys())
spectrum_indexes = list(map(lambda x: int(x), spectrum_indexes))
spectrum_indexes.sort()
from est.core.types import Spectrum
for index in spectrum_indexes:
spectrum_path = '/'.join((spectra_path, str(index)))
dict_ = h5todict(h5file=data_url.file_path(),
path=spectrum_path, asarray=False)
spectrum = Spectrum().load_frm_dict(dict_)
spectra.append(spectrum)
return spectra
else:
if data_url.is_valid():
try:
return utils.get_data(data_url)
except ValueError as e:
_logger.error(e)
else:
_logger.warning('invalid url for', name, ', will not load it')
def read_xas(spectra_url, channel_url, dimensions=None, config_url=None):
"""
Read the given spectra url and the config url if any
......@@ -90,42 +166,6 @@ def read_xas(spectra_url, channel_url, dimensions=None, config_url=None):
if not (_config_url is None or isinstance(_config_url, DataUrl)):
raise TypeError('given input for configuration is invalid')
def load_data(data_url, name):
if data_url is None:
return None
if data_url.scheme() in ('PyMca', 'PyMca5'):
if has_pymca is False:
_logger.warning('Requires PyMca to load data from '
'%s' % data_url.path())
return None
else:
assert name in ('spectra', 'energy')
energy, mu = pymca_read_spectrum(data_url.file_path())
if name == 'spectra':
return mu.reshape(mu.shape[0], 1, 1)
else:
return energy
elif data_url.scheme() in ('larch', 'xraylarch'):
if has_larch is False:
_logger.warning('Requires larch to load data from '
'%s' % data_url.path())
return None
else:
assert name in ('spectra', 'energy')
energy, mu = larch_read_ascii(xmu_file=data_url.file_path())
if name == 'spectra':
return mu.reshape(mu.shape[0], 1, 1)
else:
return energy
else:
if data_url.is_valid():
try:
return utils.get_data(data_url)
except ValueError as e:
_logger.error(e)
else:
_logger.warning('invalid url for', name, ', will not load it')
from est.core.types import Dim # avoid cyclic import
if dimensions is None:
dimensions_ = (Dim.CHANNEL_ENERGY_DIM, Dim.Y_DIM, Dim.X_DIM)
......@@ -299,6 +339,22 @@ def write_xas(h5_file, entry, energy, mu, sample=None, start_time=None,
nx_entry['definition'] = definition
def write_spectrum_saving_pt(h5_file, entry, obj, overwrite=True):
"""Save the current status of an est object
:param str h5_file: path to the hdf5 file
:param str entry: entry name
:param obj: object to save.
:param str obj_name: name of the object to store
:param str data_path:
"""
dicttoh5(obj,
h5file=h5_file,
h5path=entry,
overwrite_data=True,
mode='a')
def get_xasproc(h5_file, entry):
"""
Return the list of all NXxasproc existing at the data_path level
......@@ -321,7 +377,7 @@ def get_xasproc(h5_file, entry):
if key in relevant_keys:
if key == 'configuration':
config_path = '/'.join((h5_group.name, 'configuration'))
res[key] = h5todict(h5_file, config_path)
res[key] = h5todict(h5_file, config_path, asarray=False)
else:
res[key] = h5_group[key][...]
return res
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment