Commit 6987ec2c authored by payno's avatar payno
Browse files

[core] add "_NexusDatasetDef" to be able to specify some units on dataset.

In fact instead of having to repeat those definitions n times we should have some king of a database defining those once for all
parent 1246205b
Pipeline #40566 failed with stages
in 60 minutes and 21 seconds
......@@ -26,6 +26,7 @@
from est.core.types import Spectrum, XASObject
from est.core.process.process import Process
from est.core.process.process import _NexusDatasetDef
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from larch.xafs.autobk import autobk
......@@ -159,7 +160,7 @@ class Larch_autobk(Process):
self._advancement.startProcess()
self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess()
self.register_process(_xas_obj, data_keys=("bkg",))
self.register_process(_xas_obj, data_keys=(_NexusDatasetDef("bkg"),))
return _xas_obj
def _pool_process(self, xas_obj):
......
......@@ -26,6 +26,7 @@
from est.core.types import Spectrum, XASObject
from est.core.process.process import Process
from est.core.process.process import _NexusDatasetDef
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from larch.xafs.mback import mback
......@@ -152,7 +153,13 @@ class Larch_mback(Process):
self._advancement.startProcess()
self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess()
self.register_process(_xas_obj, data_keys=("fpp", "f2"))
self.register_process(
_xas_obj,
data_keys=(
_NexusDatasetDef("fpp"),
_NexusDatasetDef("f2"),
),
)
return _xas_obj
def _pool_process(self, xas_obj):
......
......@@ -26,6 +26,7 @@
from est.core.types import Spectrum, XASObject
from est.core.process.process import Process
from est.core.process.process import _NexusDatasetDef
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from larch.xafs.mback import mback_norm
......@@ -159,7 +160,10 @@ class Larch_mback_norm(Process):
self._advancement.startProcess()
self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess()
data_keys = ["mback_mu", "norm_mback"]
data_keys = [
_NexusDatasetDef("mback_mu"),
_NexusDatasetDef("norm_mback"),
]
if _xas_obj.n_spectrum > 0 and hasattr(_xas_obj.spectra[0], "edge_step"):
data_keys += ["norm"]
self.register_process(_xas_obj, data_keys=data_keys)
......
......@@ -26,9 +26,10 @@
from est.core.types import Spectrum, XASObject
from est.core.process.process import Process
from est.core.process.process import _NexusSpectrumDef
from est.core.process.process import _NexusSpectrumDef, _NexusDatasetDef
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from est.core.utils.symbol import MU_CHAR
from larch.xafs.pre_edge import pre_edge
import multiprocessing
import functools
......@@ -158,20 +159,20 @@ class Larch_pre_edge(Process):
self.register_process(
_xas_obj,
data_keys=(
"flat",
"dmude",
"edge_step_poly",
"norm",
"norm_area",
"post_edge",
"pre_edge_details",
"e0",
"Mu",
"energy",
"mu_ref",
"I0",
"I1",
"I2",
_NexusDatasetDef("flat", "{}(E)".format(MU_CHAR)),
_NexusDatasetDef("dmude"),
_NexusDatasetDef("edge_step_poly"),
_NexusDatasetDef("norm"),
_NexusDatasetDef("norm_area"),
_NexusDatasetDef("post_edge"),
_NexusDatasetDef("pre_edge_details"),
_NexusDatasetDef("e0", "eV"),
_NexusDatasetDef("Mu", "{}(E)".format(MU_CHAR)),
_NexusDatasetDef("energy", "eV"),
_NexusDatasetDef("mu_ref", "{}(E)".format(MU_CHAR)),
_NexusDatasetDef("I0"),
_NexusDatasetDef("I1"),
_NexusDatasetDef("I2"),
),
plots=(
_NexusSpectrumDef(
......
......@@ -27,9 +27,11 @@
from est.core.types import Spectrum, XASObject
from est.core.process.process import Process
from est.core.process.process import _NexusSpectrumDef
from est.core.process.process import _NexusDatasetDef
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from larch.xafs.xafsft import xftf
from est.core.utils.symbol import ANGSTROM_CHAR
import multiprocessing
import functools
import logging
......@@ -173,12 +175,14 @@ class Larch_xftf(Process):
self.register_process(
_xas_obj,
data_keys=(
"chir_im",
"chir_re",
"chir_mag",
"r",
"masked_k",
"masked_chi_weighted_k",
_NexusDatasetDef("chir_im", "{}^(-3)".format(ANGSTROM_CHAR)),
_NexusDatasetDef("chir_re"),
_NexusDatasetDef("chir_mag", "{}^(-3)".format(ANGSTROM_CHAR)),
_NexusDatasetDef("r", "{}".format(ANGSTROM_CHAR)),
_NexusDatasetDef("masked_k", "{}^(-1)".format(ANGSTROM_CHAR)),
_NexusDatasetDef(
"masked_chi_weighted_k", "{}^(-2)".format(ANGSTROM_CHAR)
),
),
plots=(
_NexusSpectrumDef(
......
......@@ -33,6 +33,7 @@ from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from .process import Process
from .process import _NexusSpectrumDef
from .process import _NexusDatasetDef
import scipy.signal
import logging
import numpy
......@@ -191,7 +192,11 @@ class NoiseProcess(Process):
self._advancement.endProcess()
self.register_process(
_xas_obj,
data_keys=("norm_noise_savgol", "noise_savgol", "noise_savgol_energy"),
data_keys=(
_NexusDatasetDef("norm_noise_savgol"),
_NexusDatasetDef("noise_savgol", "raw data noise"),
_NexusDatasetDef("noise_savgol_energy", "eV"),
),
plots=(
_NexusSpectrumDef(
signal="noise_savgol",
......
......@@ -97,6 +97,51 @@ class _NexusSpectrumDef:
return self.__silx_style
class _NexusDatasetDef:
"""Util function to define a Nexus plot"""
def __init__(self, name: str, units=None):
self.__name = None
self.__units = None
assert isinstance(name, str)
self.name = name
self.units = units
@property
def name(self):
return self.__name
@name.setter
def name(self, name):
if not isinstance(name, str):
raise TypeError("name should be an instance of str")
else:
self.__name = name
@property
def units(self):
return self.__units
@units.setter
def units(self, units):
if not isinstance(units, (str, type(None))):
raise TypeError("units should be an instance of str")
else:
self.__units = units
@property
def attrs(self):
attrs = {}
if self.units is not None:
attrs.update({"units": self.units})
return attrs
def __str__(self):
return self.name
class Process(object):
def __init__(self, name):
assert type(name) is str
......@@ -190,7 +235,9 @@ class Process(object):
"""
if xas_obj.has_linked_file():
_data = {}
for key in data_keys:
for data_info in data_keys:
# data_info can be a string or an instance of _NexusDatasetDef
key = str(data_info)
relative_to = None
use = "map_to"
if key in (
......@@ -247,14 +294,14 @@ class Process(object):
if key == "norm_area":
continue
_data[key] = xas_obj.spectra.map_to(
key=key,
data_info=key,
relative_to=relative_to,
)
# if we can display the result as a numpy.array 3d
try:
_data[key] = xas_obj.spectra.map_to(
key=key,
data_info=key,
relative_to=relative_to,
)
except KeyError:
......@@ -277,6 +324,9 @@ class Process(object):
_data[key] = res
else:
raise ValueError()
if isinstance(data_info, _NexusDatasetDef):
for aname, avalue in data_info.attrs.items():
_data[(key, aname)] = avalue
xas_obj.register_processing(process=self, results=_data, plots=plots)
def addCallback(self, callback):
......
......@@ -37,6 +37,7 @@ from PyMca5.PyMcaPhysics.xas.XASClass import e2k, k2e
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from est.core.process.process import Process
from est.core.process.process import _NexusDatasetDef
from est.core.types import XASObject, Spectrum
_logger = logging.getLogger(__name__)
......@@ -172,7 +173,13 @@ class PyMca_exafs(Process):
self._advancement.startProcess()
self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess()
self.register_process(_xas_obj, data_keys=("EXAFSKValues", "EXAFSSignal"))
self.register_process(
_xas_obj,
data_keys=(
_NexusDatasetDef("EXAFSKValues"),
_NexusDatasetDef("EXAFSSignal"),
),
)
return _xas_obj
def _pool_process(self, xas_obj):
......
......@@ -36,6 +36,7 @@ from PyMca5.PyMcaPhysics.xas.XASClass import XASClass
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from est.core.process.process import Process
from est.core.process.process import _NexusDatasetDef
from est.core.types import XASObject, Spectrum
_logger = logging.getLogger(__name__)
......@@ -186,7 +187,12 @@ class PyMca_ft(Process):
assert hasattr(_xas_obj.spectra.data.flat[0].ft, "intensity")
assert hasattr(_xas_obj.spectra.data.flat[0].ft, "imaginary")
self.register_process(
_xas_obj, data_keys=("ft.radius", "ft.intensity", "ft.imaginary")
_xas_obj,
data_keys=(
_NexusDatasetDef("ft.radius"),
_NexusDatasetDef("ft.intensity"),
_NexusDatasetDef("ft.imaginary"),
),
)
return _xas_obj
......
......@@ -35,6 +35,7 @@ from PyMca5.PyMcaPhysics.xas.XASClass import XASClass
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from est.core.process.process import Process, Progress
from est.core.process.process import _NexusDatasetDef
from est.core.types import Spectrum, XASObject
_logger = logging.getLogger(__name__)
......@@ -164,7 +165,12 @@ class PyMca_normalization(Process):
self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess()
self.register_process(
_xas_obj, data_keys=("NormalizedEnergy", "NormalizedMu", "NormalizedSignal")
_xas_obj,
data_keys=(
_NexusDatasetDef("NormalizedEnergy"),
_NexusDatasetDef("NormalizedMu"),
_NexusDatasetDef("NormalizedSignal"),
),
)
return _xas_obj
......
......@@ -142,7 +142,7 @@ class ROIProcess(Process):
# there is no processing for the _larch_grp_members case
if key == "_larch_grp_members":
continue
volume = xas_obj.spectra.map_to(key=key)
volume = xas_obj.spectra.map_to(data_info=key)
volume_res = volume[:, ymin:ymax, xmin:xmax]
volume_res = volume_res.reshape(volume_res.shape[0], -1)
volumes[key] = volume_res
......
......@@ -66,13 +66,13 @@ class TestRoi(unittest.TestCase):
def testApplyRoi(self):
"""Test output of the roi process"""
original_spectra = self.xas_obj.spectra.map_to(key="mu").copy()
original_spectra = self.xas_obj.spectra.map_to(data_info="mu").copy()
self.assertEqual(original_spectra.shape, (16, 100, 30))
roi_dict = {"origin": (20, 50), "size": (10, 20)}
self.xas_obj.configuration = {"roi": roi_dict}
res_xas_obj = xas_roi(self.xas_obj)
self.assertEqual(res_xas_obj.n_spectrum, 20 * 10)
reduces_spectra = res_xas_obj.spectra.map_to(key="mu").copy()
reduces_spectra = res_xas_obj.spectra.map_to(data_info="mu").copy()
assert reduces_spectra.shape == (16, 20, 10)
numpy.testing.assert_array_equal(
original_spectra[:, 50:70, 20:30], reduces_spectra
......
......@@ -142,7 +142,7 @@ class TestXASObject(unittest.TestCase):
)
obj2 = XASObject.from_dict(ddict)
self.assertEqual(self.xas_obj.n_spectrum, obj2.n_spectrum)
obj2_mu_spectra = obj2.spectra.map_to(key="mu")
obj2_mu_spectra = obj2.spectra.map_to(data_info="mu")
numpy.testing.assert_array_equal(original_spectra, obj2_mu_spectra)
self.assertEqual(obj2, self.xas_obj)
......
......@@ -139,7 +139,7 @@ class Spectra:
if len(self.data.flat[0].energy) != len(energy):
_logger.warning("spectra and energy have incoherent dimension")
def map_to(self, key: str, relative_to: str = "energy"):
def map_to(self, data_info: str, relative_to: str = "energy"):
"""
Create a map a specific key of this spectra.
"""
......@@ -167,6 +167,7 @@ class Spectra:
else:
return numpy.zeros(len(self.data.flat))
key = str(data_info)
array = None
for i_spectrum, spectrum in enumerate(self.data.flat):
try:
......
......@@ -113,7 +113,7 @@ class TestXASObject(unittest.TestCase):
)
obj2 = XASObject.from_dict(ddict)
self.assertEqual(self.xas_obj.n_spectrum, obj2.n_spectrum)
obj2_mu_spectra = obj2.spectra.map_to(key="mu")
obj2_mu_spectra = obj2.spectra.map_to(data_info="mu")
numpy.testing.assert_array_equal(original_spectra, obj2_mu_spectra)
self.assertEqual(obj2, self.xas_obj)
......
......@@ -305,7 +305,7 @@ class XASObject(object):
return res
def absorbed_beam(self) -> numpy.ndarray:
return self.spectra.map_to(key="mu")
return self.spectra.map_to(data_info="mu")
def load_frm_dict(self, ddict: dict):
"""load XAS values from a dict"""
......
......@@ -29,3 +29,5 @@ __date__ = "11/12/2020"
MU_CHAR = (b"\xce\xbc").decode("utf-8")
ANGSTROM_CHAR = (b"\xe2\x84\xab").decode("utf-8")
......@@ -242,7 +242,7 @@ class MapViewer(qt.QWidget):
return
# set the map view
spectra_volume = self._xasObj.spectra.map_to(
key=self.getActiveKey(),
data_info=self.getActiveKey(),
)
self._mainWindow.setStack(spectra_volume)
......
......@@ -365,34 +365,57 @@ def write_xas_proc(
return path
# save results
def save_key(key_path, value):
def save_key(key_path, value, attrs):
"""Save the given value to the associated path. Manage numpy arrays
and dictionaries"""
and dictionaries.
"""
if attrs is not None:
assert value is None, "can save value or attribute not both"
if value is not None:
assert attrs is None, "can save value or attribute not both"
key_path = key_path.replace(".", "/")
# save if is dict
if isinstance(value, dict):
h5_path = "/".join((entry, process_name, key_path))
dicttoh5(
value, h5file=h5_file, h5path=h5_path, overwrite_data=True, mode="a"
value,
h5file=h5_file,
h5path=h5_path,
overwrite_data=True,
mode="a",
create_dataset_args=True,
)
else:
with h5py.File(h5_file, "a") as h5f:
nx_process = h5f.require_group(nx_process_path)
try:
nx_process[key_path] = value
except TypeError as e:
_logger.error(
"Unable to write at {} reason is {}"
"".format(str(key_path), str(e))
)
if attrs is None:
try:
nx_process[key_path] = value
except TypeError as e:
_logger.error(
"Unable to write at {} reason is {}"
"".format(str(key_path), str(e))
)
else:
interpretation = get_interpretation(value)
if interpretation:
nx_process[key_path].attrs[
"interpretation"
] = interpretation
else:
interpretation = get_interpretation(value)
if interpretation:
nx_process[key_path].attrs["interpretation"] = interpretation
for key, value in attrs.items():
try:
nx_process[key_path].attrs[key] = value
except Exception as e:
_logger.warning(e)
for key, value in data_.items():
key_path = "/".join(("results", key))
save_key(key_path=key_path, value=value)
if isinstance(key, tuple):
key_path = "/".join(("results", key[0]))
save_key(key_path=key_path, value=None, attrs={key[1]: value})
else:
key_path = "/".join(("results", str(key)))
save_key(key_path=key_path, value=value, attrs=None)
def save_plot(plot_name, plot):
"""save the given plot to an hdf5 group"""
......@@ -414,8 +437,12 @@ def write_xas_proc(
plot_group[name] = h5py.SoftLink(dataset_to_link.name)
elif dataset_to_link.ndim == 2:
plot_group[name] = dataset_to_link[:, 0]
if "units" in dataset_to_link.attrs:
plot_group[name].attrs["units"] = dataset_to_link.attrs["units"]
elif dataset_to_link.ndim == 3:
plot_group[name] = dataset_to_link[:, 0, 0]
if "units" in dataset_to_link.attrs:
plot_group[name].attrs["units"] = dataset_to_link.attrs["units"]
else:
raise ValueError(
"Unable to handle dataset {}".format(dataset_to_link.name)
......
......@@ -60,7 +60,7 @@ class Converter(object):
)
# TODO: prendre normalized_mu and normalized_energy if exists,
# otherwise take mu and energy...
spectra = xas_object.spectra.map_to(key="mu")
spectra = xas_object.spectra.map_to(data_info="mu")
# invert dimensions and axis to fit spectroscopy add-on
X = spectra.reshape((spectra.shape[0], -1))
X = numpy.swapaxes(X, 0, 1)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment