normalization.py 8.49 KB
Newer Older
payno's avatar
payno committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/

__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "06/11/2019"


31
import functools
payno's avatar
payno committed
32
import logging
33
34
import multiprocessing
from PyMca5.PyMcaPhysics.xas.XASClass import XASClass
35
36
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
payno's avatar
payno committed
37
from est.core.process.process import Process, Progress
38
from est.core.process.process import _NexusDatasetDef
payno's avatar
payno committed
39
from est.core.types import Spectrum, XASObject
40

payno's avatar
payno committed
41
42
43
_logger = logging.getLogger(__name__)


payno's avatar
payno committed
44
def process_spectr_norm(
payno's avatar
payno committed
45
    spectrum: Spectrum,
payno's avatar
payno committed
46
47
48
49
50
51
    configuration,
    overwrite=True,
    callbacks=None,
    output=None,
    output_dict=None,
):
52
53
    """

54
55
56
57
58
59
    :param spectrum: spectrum to process
    :type: :class:`.Spectrum`
    :param configuration: configuration of the pymca normalization
    :type: dict
    :param overwrite: False if we want to return a new Spectrum instance
    :type: bool
60
    :param callbacks: callback to execute.
61
62
    :param output: list to store the result, needed for pool processing
    :type: multiprocessing.manager.list
63
64
65
    :param output_dict: key is input spectrum, value is index in the output
                        list.
    :type: dict
66
    :return: processed spectrum
67
68
    :rtype: tuple (configuration, spectrum)
    """
payno's avatar
payno committed
69
    assert isinstance(spectrum, Spectrum)
payno's avatar
payno committed
70
    _logger.debug("start normalization on spectrum (%s, %s)" % (spectrum.x, spectrum.y))
71
    if spectrum.energy is None or spectrum.mu is None:
payno's avatar
payno committed
72
73
74
        _logger.error(
            "Energy and or Mu is/are not specified, unable to " "compute exafs"
        )
75
        return None, None
76
    pymca_xas = XASClass()
payno's avatar
payno committed
77
    pymca_xas.setSpectrum(energy=spectrum.energy, mu=spectrum.mu)
78
    if configuration is not None:
payno's avatar
payno committed
79
80
81
        if "e0" in configuration:
            configuration["E0Value"] = configuration["e0"]
            configuration["E0Method"] = "Manual"
82
        pymca_xas.setConfiguration(configuration)
83
    configuration = pymca_xas.getConfiguration()
84
85
86
87
88
89
90
91
    try:
        res = pymca_xas.normalize()
        ddict = spectrum.to_dict()
        ddict.update(res)
        spectrum_ = Spectrum.from_dict(ddict)
    except (IndexError, ValueError) as e:
        _logger.error(e)
        return None, None
92
93
94
    if callbacks:
        for callback in callbacks:
            callback()
95
96
97
98
99
100
101
102
103
104
105
106
107

    def get_output(orignal_spec, res_spec):
        if overwrite:
            orignal_spec.update(res_spec)
            return orignal_spec
        else:
            return res_spec

    if output is not None:
        assert output_dict is not None
        output[output_dict[spectrum]] = get_output(spectrum, spectrum_)

    return configuration, get_output(spectrum, spectrum_)
108
109


110
def pymca_normalization(xas_obj):
payno's avatar
payno committed
111
    """
112

113
114
115
116
    :param xas_obj: object containing the configuration and spectra to process
    :type: Union[XASObject, dict]. If is a dict, should contain configuration or
                                 spectra keys. Otherwise is simply the spectra
    :return: spectra dict
117
    :rtype: dict
payno's avatar
payno committed
118
    """
119
120
    normalization_obj = PyMca_normalization()
    return normalization_obj.process(xas_obj=xas_obj)
121
122


123
_USE_MULTIPROCESSING_POOL = False
payno's avatar
payno committed
124
# note: we cannot use multiprocessing pool with pypushflow for now.
125
126


127
class PyMca_normalization(Process):
128

129
    _INPUT_NAMES = set(["xas_obj"])
130

131
    _OUTPUT_NAMES = set(["xas_obj"])
132

133
134
    def __init__(self, varinfo=None, **inputs):
        Process.__init__(self, "normalization", varinfo=varinfo, **inputs)
135
        self._advancement = Progress(self.name)
136

137
    def set_properties(self, properties):
payno's avatar
payno committed
138
139
        if "_pymcaSettings" in properties:
            self._settings = properties["_pymcaSettings"]
140

141
    def process(self, xas_obj=None):
142
143
        """

144
145
146
        :param xas_obj: object containing the configuration and spectra to process
        :type: Union[XASObject, dict]. If is a dict, should contain configuration or
                                     spectra keys. Otherwise is simply the spectra
147
148
        :return: updated XASObject
        :rtype: :class:`.XASObject`
149
        """
150
151
        if xas_obj is None:
            xas_obj = self.input.xas_obj.value
152
        _xas_obj = self.getXasObject(xas_obj)
153
154

        if _xas_obj.energy is None:
payno's avatar
payno committed
155
            _logger.error("Energy not specified, unable to normalize spectra")
156
157
            return

158
        if self._settings:
payno's avatar
payno committed
159
            _xas_obj.configuration["Normalization"] = self._settings
160
        self._advancement.reset(max_=_xas_obj.n_spectrum)
161
        self._advancement.startProcess()
162
        self._pool_process(xas_obj=_xas_obj)
163
        self._advancement.endProcess()
164
        assert _xas_obj.normalized_energy is not None
payno's avatar
payno committed
165
        self.register_process(
166
167
168
169
170
171
            _xas_obj,
            data_keys=(
                _NexusDatasetDef("NormalizedEnergy"),
                _NexusDatasetDef("NormalizedMu"),
                _NexusDatasetDef("NormalizedSignal"),
            ),
payno's avatar
payno committed
172
        )
173
        self.output.xas_obj.value = _xas_obj.to_dict()
174
175
176
177
        return _xas_obj

    def _pool_process(self, xas_obj):
        """process normalization from a pool"""
178
179
        assert isinstance(xas_obj, XASObject)
        if not _USE_MULTIPROCESSING_POOL:
payno's avatar
payno committed
180
            for spectrum in xas_obj.spectra.data.flat:
payno's avatar
payno committed
181
182
183
184
185
186
                process_spectr_norm(
                    spectrum=spectrum,
                    configuration=xas_obj.configuration,
                    callbacks=self.callbacks,
                    overwrite=True,
                )
187
188
        else:
            from multiprocessing import Manager
payno's avatar
payno committed
189

190
191
192
            manager = Manager()
            output_dict = {}
            res_list = manager.list()
payno's avatar
payno committed
193
            for i_spect, spect in enumerate(xas_obj.spectra.data.flat):
194
195
196
197
                res_list.append(None)
                output_dict[spect] = i_spect

            with multiprocessing.Pool(5) as p:
payno's avatar
payno committed
198
199
200
201
202
203
204
205
                partial_ = functools.partial(
                    process_spectr_norm,
                    configuration=xas_obj.configuration,
                    callbacks=self.callbacks,
                    overwrite=False,
                    output=res_list,
                    output_dict=output_dict,
                )
206
207
208
209
210
211
                p.map(partial_, xas_obj.spectra)

            # then update local spectrum
            for spectrum, res in zip(xas_obj.spectra, res_list):
                spectrum.update(res)

212
213
214
215
216
    def definition(self):
        return "Normalization of the spectrum"

    def program_version(self):
        import PyMca5
payno's avatar
payno committed
217

218
219
        return PyMca5.version()

220
221
    @staticmethod
    def program_name():
payno's avatar
payno committed
222
        return "pymca_normalization"
223
224

    __call__ = process
225
226
227
228
229


if __name__ == "__main__":
    import sys
    import yaml
230
231
    import os

232
    xas_object_yaml_file = sys.argv[1]
233
    working_dir = sys.argv[2]
234
235
236
237
238
    _logger.debug("Load xas object from {}".format(xas_object_yaml_file))
    with open(xas_object_yaml_file, "r") as file:
        ddict = yaml.load(file)["input_data"]
        xas_object = XASObject.from_dict(ddict)
    res_xas_object = pymca_normalization(xas_obj=xas_object)
239
    res_xas_object._create_saving_pt()
240

241
    # dump resulting object in input file
242
243
    with open(xas_object_yaml_file, "w") as file:
        yaml.dump({"input_data": res_xas_object.to_dict()}, file)
244

payno's avatar
payno committed
245
    output_dest = os.path.join(os.getcwd(), "output_normalization.yaml")
246
247
248
    # dump resulting object into output_normalization file
    with open(output_dest, "w") as file:
        yaml.dump({"input_data": res_xas_object.to_dict()}, file)