[core] add "_NexusDatasetDef" to be able to specify some units on dataset.

In fact instead of having to repeat those definitions n times we should have some king of a database defining those once for all

est/core/process/larch/autobk.py

@@ -26,6 +26,7 @@
 
 from est.core.types import Spectrum, XASObject
 from est.core.process.process import Process
+from est.core.process.process import _NexusDatasetDef
 from est.core.process.process import _input_desc
 from est.core.process.process import _output_desc
 from larch.xafs.autobk import autobk
@@ -159,7 +160,7 @@ class Larch_autobk(Process):
         self._advancement.startProcess()
         self._pool_process(xas_obj=_xas_obj)
         self._advancement.endProcess()
-        self.register_process(_xas_obj, data_keys=("bkg",))
+        self.register_process(_xas_obj, data_keys=(_NexusDatasetDef("bkg"),))
         return _xas_obj
 
     def _pool_process(self, xas_obj):

est/core/process/larch/mback.py

@@ -26,6 +26,7 @@
 
 from est.core.types import Spectrum, XASObject
 from est.core.process.process import Process
+from est.core.process.process import _NexusDatasetDef
 from est.core.process.process import _input_desc
 from est.core.process.process import _output_desc
 from larch.xafs.mback import mback
@@ -152,7 +153,13 @@ class Larch_mback(Process):
         self._advancement.startProcess()
         self._pool_process(xas_obj=_xas_obj)
         self._advancement.endProcess()
-        self.register_process(_xas_obj, data_keys=("fpp", "f2"))
+        self.register_process(
+            _xas_obj,
+            data_keys=(
+                _NexusDatasetDef("fpp"),
+                _NexusDatasetDef("f2"),
+            ),
+        )
         return _xas_obj
 
     def _pool_process(self, xas_obj):

est/core/process/larch/mback_norm.py

@@ -26,6 +26,7 @@
 
 from est.core.types import Spectrum, XASObject
 from est.core.process.process import Process
+from est.core.process.process import _NexusDatasetDef
 from est.core.process.process import _input_desc
 from est.core.process.process import _output_desc
 from larch.xafs.mback import mback_norm
@@ -159,7 +160,10 @@ class Larch_mback_norm(Process):
         self._advancement.startProcess()
         self._pool_process(xas_obj=_xas_obj)
         self._advancement.endProcess()
-        data_keys = ["mback_mu", "norm_mback"]
+        data_keys = [
+            _NexusDatasetDef("mback_mu"),
+            _NexusDatasetDef("norm_mback"),
+        ]
         if _xas_obj.n_spectrum > 0 and hasattr(_xas_obj.spectra[0], "edge_step"):
             data_keys += ["norm"]
         self.register_process(_xas_obj, data_keys=data_keys)

est/core/process/larch/pre_edge.py

@@ -26,9 +26,10 @@
 
 from est.core.types import Spectrum, XASObject
 from est.core.process.process import Process
-from est.core.process.process import _NexusSpectrumDef
+from est.core.process.process import _NexusSpectrumDef, _NexusDatasetDef
 from est.core.process.process import _input_desc
 from est.core.process.process import _output_desc
+from est.core.utils.symbol import MU_CHAR
 from larch.xafs.pre_edge import pre_edge
 import multiprocessing
 import functools
@@ -158,20 +159,20 @@ class Larch_pre_edge(Process):
         self.register_process(
             _xas_obj,
             data_keys=(
-                "flat",
-                "dmude",
-                "edge_step_poly",
-                "norm",
-                "norm_area",
-                "post_edge",
-                "pre_edge_details",
-                "e0",
-                "Mu",
-                "energy",
-                "mu_ref",
-                "I0",
-                "I1",
-                "I2",
+                _NexusDatasetDef("flat", "{}(E)".format(MU_CHAR)),
+                _NexusDatasetDef("dmude"),
+                _NexusDatasetDef("edge_step_poly"),
+                _NexusDatasetDef("norm"),
+                _NexusDatasetDef("norm_area"),
+                _NexusDatasetDef("post_edge"),
+                _NexusDatasetDef("pre_edge_details"),
+                _NexusDatasetDef("e0", "eV"),
+                _NexusDatasetDef("Mu", "{}(E)".format(MU_CHAR)),
+                _NexusDatasetDef("energy", "eV"),
+                _NexusDatasetDef("mu_ref", "{}(E)".format(MU_CHAR)),
+                _NexusDatasetDef("I0"),
+                _NexusDatasetDef("I1"),
+                _NexusDatasetDef("I2"),
             ),
             plots=(
                 _NexusSpectrumDef(

est/core/process/larch/xftf.py

@@ -27,9 +27,11 @@
 from est.core.types import Spectrum, XASObject
 from est.core.process.process import Process
 from est.core.process.process import _NexusSpectrumDef
+from est.core.process.process import _NexusDatasetDef
 from est.core.process.process import _input_desc
 from est.core.process.process import _output_desc
 from larch.xafs.xafsft import xftf
+from est.core.utils.symbol import ANGSTROM_CHAR
 import multiprocessing
 import functools
 import logging
@@ -173,12 +175,14 @@ class Larch_xftf(Process):
         self.register_process(
             _xas_obj,
             data_keys=(
-                "chir_im",
-                "chir_re",
-                "chir_mag",
-                "r",
-                "masked_k",
-                "masked_chi_weighted_k",
+                _NexusDatasetDef("chir_im", "{}^(-3)".format(ANGSTROM_CHAR)),
+                _NexusDatasetDef("chir_re"),
+                _NexusDatasetDef("chir_mag", "{}^(-3)".format(ANGSTROM_CHAR)),
+                _NexusDatasetDef("r", "{}".format(ANGSTROM_CHAR)),
+                _NexusDatasetDef("masked_k", "{}^(-1)".format(ANGSTROM_CHAR)),
+                _NexusDatasetDef(
+                    "masked_chi_weighted_k", "{}^(-2)".format(ANGSTROM_CHAR)
+                ),
             ),
             plots=(
                 _NexusSpectrumDef(

est/core/process/noise.py

@@ -33,6 +33,7 @@ from est.core.process.process import _input_desc
 from est.core.process.process import _output_desc
 from .process import Process
 from .process import _NexusSpectrumDef
+from .process import _NexusDatasetDef
 import scipy.signal
 import logging
 import numpy
@@ -191,7 +192,11 @@ class NoiseProcess(Process):
         self._advancement.endProcess()
         self.register_process(
             _xas_obj,
-            data_keys=("norm_noise_savgol", "noise_savgol", "noise_savgol_energy"),
+            data_keys=(
+                _NexusDatasetDef("norm_noise_savgol"),
+                _NexusDatasetDef("noise_savgol", "raw data noise"),
+                _NexusDatasetDef("noise_savgol_energy", "eV"),
+            ),
             plots=(
                 _NexusSpectrumDef(
                     signal="noise_savgol",

est/core/process/process.py

@@ -97,6 +97,51 @@ class _NexusSpectrumDef:
             return self.__silx_style
 
 
+class _NexusDatasetDef:
+    """Util function to define a Nexus plot"""
+
+    def __init__(self, name: str, units=None):
+        self.__name = None
+        self.__units = None
+
+        assert isinstance(name, str)
+
+        self.name = name
+        self.units = units
+
+    @property
+    def name(self):
+        return self.__name
+
+    @name.setter
+    def name(self, name):
+        if not isinstance(name, str):
+            raise TypeError("name should be an instance of str")
+        else:
+            self.__name = name
+
+    @property
+    def units(self):
+        return self.__units
+
+    @units.setter
+    def units(self, units):
+        if not isinstance(units, (str, type(None))):
+            raise TypeError("units should be an instance of str")
+        else:
+            self.__units = units
+
+    @property
+    def attrs(self):
+        attrs = {}
+        if self.units is not None:
+            attrs.update({"units": self.units})
+        return attrs
+
+    def __str__(self):
+        return self.name
+
+
 class Process(object):
     def __init__(self, name):
         assert type(name) is str
@@ -190,7 +235,9 @@ class Process(object):
         """
         if xas_obj.has_linked_file():
             _data = {}
-            for key in data_keys:
+            for data_info in data_keys:
+                # data_info can be a string or an instance of _NexusDatasetDef
+                key = str(data_info)
                 relative_to = None
                 use = "map_to"
                 if key in (
@@ -247,14 +294,14 @@ class Process(object):
                     if key == "norm_area":
                         continue
                     _data[key] = xas_obj.spectra.map_to(
-                        key=key,
+                        data_info=key,
                         relative_to=relative_to,
                     )
 
                     # if we can display the result as a numpy.array 3d
                     try:
                         _data[key] = xas_obj.spectra.map_to(
-                            key=key,
+                            data_info=key,
                             relative_to=relative_to,
                         )
                     except KeyError:
@@ -277,6 +324,9 @@ class Process(object):
                     _data[key] = res
                 else:
                     raise ValueError()
+                if isinstance(data_info, _NexusDatasetDef):
+                    for aname, avalue in data_info.attrs.items():
+                        _data[(key, aname)] = avalue
             xas_obj.register_processing(process=self, results=_data, plots=plots)
 
     def addCallback(self, callback):

est/core/process/pymca/exafs.py

@@ -37,6 +37,7 @@ from PyMca5.PyMcaPhysics.xas.XASClass import e2k, k2e
 from est.core.process.process import _input_desc
 from est.core.process.process import _output_desc
 from est.core.process.process import Process
+from est.core.process.process import _NexusDatasetDef
 from est.core.types import XASObject, Spectrum
 
 _logger = logging.getLogger(__name__)
@@ -172,7 +173,13 @@ class PyMca_exafs(Process):
         self._advancement.startProcess()
         self._pool_process(xas_obj=_xas_obj)
         self._advancement.endProcess()
-        self.register_process(_xas_obj, data_keys=("EXAFSKValues", "EXAFSSignal"))
+        self.register_process(
+            _xas_obj,
+            data_keys=(
+                _NexusDatasetDef("EXAFSKValues"),
+                _NexusDatasetDef("EXAFSSignal"),
+            ),
+        )
         return _xas_obj
 
     def _pool_process(self, xas_obj):

est/core/process/pymca/ft.py

@@ -36,6 +36,7 @@ from PyMca5.PyMcaPhysics.xas.XASClass import XASClass
 from est.core.process.process import _input_desc
 from est.core.process.process import _output_desc
 from est.core.process.process import Process
+from est.core.process.process import _NexusDatasetDef
 from est.core.types import XASObject, Spectrum
 
 _logger = logging.getLogger(__name__)
@@ -186,7 +187,12 @@ class PyMca_ft(Process):
         assert hasattr(_xas_obj.spectra.data.flat[0].ft, "intensity")
         assert hasattr(_xas_obj.spectra.data.flat[0].ft, "imaginary")
         self.register_process(
-            _xas_obj, data_keys=("ft.radius", "ft.intensity", "ft.imaginary")
+            _xas_obj,
+            data_keys=(
+                _NexusDatasetDef("ft.radius"),
+                _NexusDatasetDef("ft.intensity"),
+                _NexusDatasetDef("ft.imaginary"),
+            ),
         )
         return _xas_obj
 

est/core/process/pymca/normalization.py

@@ -35,6 +35,7 @@ from PyMca5.PyMcaPhysics.xas.XASClass import XASClass
 from est.core.process.process import _input_desc
 from est.core.process.process import _output_desc
 from est.core.process.process import Process, Progress
+from est.core.process.process import _NexusDatasetDef
 from est.core.types import Spectrum, XASObject
 
 _logger = logging.getLogger(__name__)
@@ -164,7 +165,12 @@ class PyMca_normalization(Process):
         self._pool_process(xas_obj=_xas_obj)
         self._advancement.endProcess()
         self.register_process(
-            _xas_obj, data_keys=("NormalizedEnergy", "NormalizedMu", "NormalizedSignal")
+            _xas_obj,
+            data_keys=(
+                _NexusDatasetDef("NormalizedEnergy"),
+                _NexusDatasetDef("NormalizedMu"),
+                _NexusDatasetDef("NormalizedSignal"),
+            ),
         )
         return _xas_obj
 

est/core/process/roi.py

@@ -142,7 +142,7 @@ class ROIProcess(Process):
                 # there is no processing for the _larch_grp_members case
                 if key == "_larch_grp_members":
                     continue
-                volume = xas_obj.spectra.map_to(key=key)
+                volume = xas_obj.spectra.map_to(data_info=key)
                 volume_res = volume[:, ymin:ymax, xmin:xmax]
                 volume_res = volume_res.reshape(volume_res.shape[0], -1)
                 volumes[key] = volume_res

est/core/process/test/test_roi.py

@@ -66,13 +66,13 @@ class TestRoi(unittest.TestCase):
 
     def testApplyRoi(self):
         """Test output of the roi process"""
-        original_spectra = self.xas_obj.spectra.map_to(key="mu").copy()
+        original_spectra = self.xas_obj.spectra.map_to(data_info="mu").copy()
         self.assertEqual(original_spectra.shape, (16, 100, 30))
         roi_dict = {"origin": (20, 50), "size": (10, 20)}
         self.xas_obj.configuration = {"roi": roi_dict}
         res_xas_obj = xas_roi(self.xas_obj)
         self.assertEqual(res_xas_obj.n_spectrum, 20 * 10)
-        reduces_spectra = res_xas_obj.spectra.map_to(key="mu").copy()
+        reduces_spectra = res_xas_obj.spectra.map_to(data_info="mu").copy()
         assert reduces_spectra.shape == (16, 20, 10)
         numpy.testing.assert_array_equal(
             original_spectra[:, 50:70, 20:30], reduces_spectra

est/core/test/test_types.py

@@ -142,7 +142,7 @@ class TestXASObject(unittest.TestCase):
         )
         obj2 = XASObject.from_dict(ddict)
         self.assertEqual(self.xas_obj.n_spectrum, obj2.n_spectrum)
-        obj2_mu_spectra = obj2.spectra.map_to(key="mu")
+        obj2_mu_spectra = obj2.spectra.map_to(data_info="mu")
 
         numpy.testing.assert_array_equal(original_spectra, obj2_mu_spectra)
         self.assertEqual(obj2, self.xas_obj)

est/core/types/spectra.py

@@ -139,7 +139,7 @@ class Spectra:
             if len(self.data.flat[0].energy) != len(energy):
                 _logger.warning("spectra and energy have incoherent dimension")
 
-    def map_to(self, key: str, relative_to: str = "energy"):
+    def map_to(self, data_info: str, relative_to: str = "energy"):
         """
         Create a map a specific key of this spectra.
         """
@@ -167,6 +167,7 @@ class Spectra:
                 else:
                     return numpy.zeros(len(self.data.flat))
 
+            key = str(data_info)
             array = None
             for i_spectrum, spectrum in enumerate(self.data.flat):
                 try:

est/core/types/test/test_xas_object.py

@@ -113,7 +113,7 @@ class TestXASObject(unittest.TestCase):
         )
         obj2 = XASObject.from_dict(ddict)
         self.assertEqual(self.xas_obj.n_spectrum, obj2.n_spectrum)
-        obj2_mu_spectra = obj2.spectra.map_to(key="mu")
+        obj2_mu_spectra = obj2.spectra.map_to(data_info="mu")
 
         numpy.testing.assert_array_equal(original_spectra, obj2_mu_spectra)
         self.assertEqual(obj2, self.xas_obj)

est/core/types/xasobject.py

@@ -305,7 +305,7 @@ class XASObject(object):
         return res
 
     def absorbed_beam(self) -> numpy.ndarray:
-        return self.spectra.map_to(key="mu")
+        return self.spectra.map_to(data_info="mu")
 
     def load_frm_dict(self, ddict: dict):
         """load XAS values from a dict"""

est/core/utils/symbol.py

@@ -29,3 +29,5 @@ __date__ = "11/12/2020"
 
 
 MU_CHAR = (b"\xce\xbc").decode("utf-8")
+
+ANGSTROM_CHAR = (b"\xe2\x84\xab").decode("utf-8")

est/gui/XasObjectViewer.py

@@ -242,7 +242,7 @@ class MapViewer(qt.QWidget):
             return
         # set the map view
         spectra_volume = self._xasObj.spectra.map_to(
-            key=self.getActiveKey(),
+            data_info=self.getActiveKey(),
         )
         self._mainWindow.setStack(spectra_volume)
 

est/io/io.py

@@ -365,34 +365,57 @@ def write_xas_proc(
         return path
 
     # save results
-    def save_key(key_path, value):
+    def save_key(key_path, value, attrs):
         """Save the given value to the associated path. Manage numpy arrays
-        and dictionaries"""
+        and dictionaries.
+        """
+        if attrs is not None:
+            assert value is None, "can save value or attribute not both"
+        if value is not None:
+            assert attrs is None, "can save value or attribute not both"
         key_path = key_path.replace(".", "/")
         # save if is dict
         if isinstance(value, dict):
             h5_path = "/".join((entry, process_name, key_path))
             dicttoh5(
-                value, h5file=h5_file, h5path=h5_path, overwrite_data=True, mode="a"
+                value,
+                h5file=h5_file,
+                h5path=h5_path,
+                overwrite_data=True,
+                mode="a",
+                create_dataset_args=True,
             )
         else:
             with h5py.File(h5_file, "a") as h5f:
                 nx_process = h5f.require_group(nx_process_path)
-                try:
-                    nx_process[key_path] = value
-                except TypeError as e:
-                    _logger.error(
-                        "Unable to write at {} reason is {}"
-                        "".format(str(key_path), str(e))
-                    )
+                if attrs is None:
+                    try:
+                        nx_process[key_path] = value
+                    except TypeError as e:
+                        _logger.error(
+                            "Unable to write at {} reason is {}"
+                            "".format(str(key_path), str(e))
+                        )
+                    else:
+                        interpretation = get_interpretation(value)
+                        if interpretation:
+                            nx_process[key_path].attrs[
+                                "interpretation"
+                            ] = interpretation
                 else:
-                    interpretation = get_interpretation(value)
-                    if interpretation:
-                        nx_process[key_path].attrs["interpretation"] = interpretation
+                    for key, value in attrs.items():
+                        try:
+                            nx_process[key_path].attrs[key] = value
+                        except Exception as e:
+                            _logger.warning(e)
 
     for key, value in data_.items():
-        key_path = "/".join(("results", key))
-        save_key(key_path=key_path, value=value)
+        if isinstance(key, tuple):
+            key_path = "/".join(("results", key[0]))
+            save_key(key_path=key_path, value=None, attrs={key[1]: value})
+        else:
+            key_path = "/".join(("results", str(key)))
+            save_key(key_path=key_path, value=value, attrs=None)
 
     def save_plot(plot_name, plot):
         """save the given plot to an hdf5 group"""
@@ -414,8 +437,12 @@ def write_xas_proc(
                     plot_group[name] = h5py.SoftLink(dataset_to_link.name)
                 elif dataset_to_link.ndim == 2:
                     plot_group[name] = dataset_to_link[:, 0]
+                    if "units" in dataset_to_link.attrs:
+                        plot_group[name].attrs["units"] = dataset_to_link.attrs["units"]
                 elif dataset_to_link.ndim == 3:
                     plot_group[name] = dataset_to_link[:, 0, 0]
+                    if "units" in dataset_to_link.attrs:
+                        plot_group[name].attrs["units"] = dataset_to_link.attrs["units"]
                 else:
                     raise ValueError(
                         "Unable to handle dataset {}".format(dataset_to_link.name)

orangecontrib/est/utils/converter.py

@@ -60,7 +60,7 @@ class Converter(object):
         )
         # TODO: prendre normalized_mu and normalized_energy if exists,
         # otherwise take mu and energy...
-        spectra = xas_object.spectra.map_to(key="mu")
+        spectra = xas_object.spectra.map_to(data_info="mu")
         # invert dimensions and axis to fit spectroscopy add-on
         X = spectra.reshape((spectra.shape[0], -1))
         X = numpy.swapaxes(X, 0, 1)