test_pymca_workflow.py 12.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2017-2019 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
25
"""
Wout De Nolf's avatar
flake8    
Wout De Nolf committed
26
unit test for workflow composed of pymca process
27
"""
28
29
30
31
32
33
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "06/11/2019"


import os
34
35
36
37
import shutil
import tempfile
import unittest
import h5py
38
import numpy
payno's avatar
payno committed
39
from silx.io.utils import h5py_read_dataset
40
from silx.io.dictdump import h5todict
41
from silx.io.url import DataUrl
payno's avatar
payno committed
42
from est.core.process.roi import xas_roi, _ROI as XASROI
payno's avatar
payno committed
43
from est.core.io import XASWriter
payno's avatar
payno committed
44
45
46
from est.core.io import read as read_xas
from est.core.types import XASObject
from est.core.utils import spectra as spectra_utils
payno's avatar
payno committed
47

48
try:
payno's avatar
PEP8    
payno committed
49
    import PyMca5  # noqa
50
51
52
53
54
except ImportError:
    has_pymca = False
else:
    has_pymca = True
    from PyMca5.PyMcaDataDir import PYMCA_DATA_DIR
payno's avatar
payno committed
55
56
57
58
59
    from est.core.process.pymca.exafs import pymca_exafs, PyMca_exafs
    from est.core.process.pymca.ft import pymca_ft, PyMca_ft
    from est.core.process.pymca.k_weight import pymca_k_weight, PyMca_k_weight
    from est.core.process.pymca.normalization import pymca_normalization
    from est.core.process.pymca.normalization import PyMca_normalization
60
61


payno's avatar
payno committed
62
@unittest.skipIf(has_pymca is False, "PyMca5 is not installed")
63
class TestStreamSingleSpectrum(unittest.TestCase):
64
    """Make sure the process have valid io"""
payno's avatar
payno committed
65

66
67
    def setUp(self):
        self.exafs_configuration = {
payno's avatar
payno committed
68
69
70
            "Knots": {"Values": (1, 2, 5), "Number": 3, "Orders": [3, 3, 3]},
            "KMin": 0,
            "KMax": 2.3,
71
72
        }

73
74
    def test_pymca_process(self):
        data_file = os.path.join(PYMCA_DATA_DIR, "EXAFS_Cu.dat")
payno's avatar
payno committed
75
        out = read_xas(
payno's avatar
payno committed
76
77
78
79
80
81
            spectra_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 2"
            ),
            channel_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 1"
            ),
payno's avatar
payno committed
82
83
        )
        out.configuration = {"EXAFS": self.exafs_configuration, "SET_KWEIGHT": 0}
84
        out = pymca_normalization(xas_obj=out)
85
        out = pymca_exafs(xas_obj=out)
86
        out = pymca_k_weight(xas_obj=out)
87
        out = pymca_ft(xas_obj=out)
88
        assert isinstance(out, XASObject)
89
90
91

    def test_pymca_process_with_dict(self):
        data_file = os.path.join(PYMCA_DATA_DIR, "EXAFS_Cu.dat")
payno's avatar
payno committed
92
        out = read_xas(
payno's avatar
payno committed
93
94
95
96
97
98
            spectra_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 2"
            ),
            channel_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 1"
            ),
payno's avatar
payno committed
99
100
        )
        out.configuration = {"EXAFS": self.exafs_configuration, "SET_KWEIGHT": 0}
101
        out = pymca_normalization(xas_obj=out.to_dict())
102
103
104
        out = pymca_exafs(xas_obj=out.to_dict())
        out = pymca_k_weight(xas_obj=out.to_dict())
        out = pymca_ft(xas_obj=out.to_dict())
105
        assert isinstance(out, XASObject)
106

107
108
    def test_pymca_process_with_cls(self):
        data_file = os.path.join(PYMCA_DATA_DIR, "EXAFS_Cu.dat")
payno's avatar
payno committed
109
        out = read_xas(
payno's avatar
payno committed
110
111
112
113
114
115
            spectra_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 2"
            ),
            channel_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 1"
            ),
payno's avatar
payno committed
116
        )
payno's avatar
payno committed
117
118
        out = PyMca_normalization(inputs={"xas_obj": out})()
        exafs_process = PyMca_exafs(inputs={"xas_obj": out})
119
        exafs_process.setConfiguration(configuration=self.exafs_configuration)
120
        out = exafs_process()
payno's avatar
payno committed
121
        kweight_process = PyMca_k_weight(inputs={"xas_obj": out})
payno's avatar
payno committed
122
        kweight_process.setConfiguration({"k_weight": 0})
payno's avatar
payno committed
123
124
        out = kweight_process()
        out = PyMca_ft(inputs={"xas_obj": out})()
125
        assert isinstance(out, XASObject)
payno's avatar
payno committed
126
127
        self.assertTrue(out.spectra.data.flat[0].ft is not None)
        self.assertTrue(len(out.spectra.data.flat[0].ft.intensity) > 0)
128
129

    def test_pymca_with_roi(self):
130
        energy, spectra = spectra_utils.create_dataset(shape=(256, 20, 10))
131
        roi = XASROI(origin=(0, 2), size=(5, 1))
payno's avatar
payno committed
132
133
134
        xas_obj = XASObject(
            spectra=spectra, energy=energy, configuration={"roi": roi.to_dict()}
        )
135
        dict_xas_obj = xas_obj.to_dict()
payno's avatar
payno committed
136
137
        self.assertTrue("spectra" in dict_xas_obj.keys())
        self.assertTrue("energy" in dict_xas_obj.keys())
138
        tmp_obj = XASObject.from_dict(dict_xas_obj)
payno's avatar
payno committed
139
140
141
        numpy.testing.assert_array_equal(
            tmp_obj.energy, tmp_obj.spectra.data.flat[0].energy
        )
142
        out = xas_roi(dict_xas_obj)
payno's avatar
payno committed
143
        out.configuration = {"EXAFS": self.exafs_configuration, "SET_KWEIGHT": 0}
144
        out = pymca_normalization(xas_obj=out)
145
146
147
148
        out = pymca_exafs(xas_obj=out)
        out = pymca_k_weight(xas_obj=out)
        out = pymca_ft(xas_obj=out)
        assert isinstance(out, XASObject)
payno's avatar
payno committed
149
150
        self.assertEqual(out.spectra.shape[0], 1)
        self.assertEqual(out.spectra.shape[1], 5)
payno's avatar
payno committed
151
152
        self.assertTrue(out.spectra.data.flat[0].ft is not None)
        self.assertTrue(len(out.spectra.data.flat[0].ft.intensity) > 0)
153

154

payno's avatar
payno committed
155
@unittest.skipIf(has_pymca is False, "PyMca5 is not installed")
156
157
158
159
class TestWorkflowAndH5LinkedFile(unittest.TestCase):
    """
    Test that the workflow can process and store process in a targeted h5 file
    """
payno's avatar
payno committed
160

161
162
    def setUp(self):
        self.output_dir = tempfile.mkdtemp()
163
        energy, spectra = spectra_utils.create_dataset(shape=(256, 20, 10))
164
        roi = XASROI(origin=(0, 2), size=(5, 1))
payno's avatar
payno committed
165
166
167
168
        self.xas_obj = XASObject(
            spectra=spectra, energy=energy, configuration={"roi": roi.to_dict()}
        )
        self.h5_file = os.path.join(self.output_dir, "output_file.h5")
169
170
171
172
173
174
175
176
        self.xas_obj.link_to_h5(self.h5_file)

    def tearDown(self):
        shutil.rmtree(self.output_dir)

    def test_h5_link_xas_object(self):
        """Test that the processing can be stored continuously on a .h5 file"""
        self.assertTrue(self.xas_obj.linked_h5_file is not None)
payno's avatar
payno committed
177
178
        out = PyMca_normalization(inputs={"xas_obj": self.xas_obj})()
        exafs_process = PyMca_exafs(inputs={"xas_obj": out})
179
        configuration = {
payno's avatar
payno committed
180
181
182
            "Knots": {"Values": (1, 2, 5), "Number": 3, "Orders": [3, 3, 3]},
            "KMin": 0,
            "KMax": 2.3,
183
184
        }
        exafs_process.setConfiguration(configuration=configuration)
185
        out = exafs_process()
payno's avatar
payno committed
186
        k_weight_process = PyMca_k_weight(inputs={"xas_obj": out})
payno's avatar
payno committed
187
        k_weight_process.setConfiguration({"k_weight": 0})
188
        out = k_weight_process()
payno's avatar
payno committed
189
        out = PyMca_ft(inputs={"xas_obj": out})()
190
        out = PyMca_normalization(inputs={"xas_obj": out})()
191
        assert isinstance(out, XASObject)
192
        assert out.linked_h5_file is self.h5_file
193
        # then check all process are correctly registered with the valid id...
194
        processes = self.xas_obj.get_process_flow()
195
        self.assertEqual(len(processes), 5)
payno's avatar
payno committed
196
197
198
199
200
201
202
        self.assertEqual(
            h5py_read_dataset(processes[1]["program"]), "pymca_normalization"
        )
        self.assertEqual(h5py_read_dataset(processes[2]["program"]), "pymca_exafs")
        self.assertEqual(
            h5py_read_dataset(processes[5]["program"]), "pymca_normalization"
        )
203
204
        self.xas_obj.clean_process_flow()
        processes = self.xas_obj.get_process_flow()
205
206
207
208
209
        self.assertEqual(len(processes), 0)

    def test_h5_link_dict(self):
        """Same test as test_h5_link_xas_object but with a dict pass between
        processes"""
210
        self.assertTrue(self.xas_obj.linked_h5_file is not None)
payno's avatar
payno committed
211
212
        out = PyMca_normalization(inputs={"xas_obj": self.xas_obj.to_dict()})()
        exafs_process = PyMca_exafs(inputs={"xas_obj": out})
213
        configuration = {
payno's avatar
payno committed
214
215
216
            "Knots": {"Values": (1, 2, 5), "Number": 3, "Orders": [3, 3, 3]},
            "KMin": 0,
            "KMax": 2.3,
217
218
        }
        exafs_process.setConfiguration(configuration=configuration)
payno's avatar
payno committed
219
        out = exafs_process()
220
        k_weight_process = PyMca_k_weight(inputs={"xas_obj": out.to_dict()})
payno's avatar
payno committed
221
        k_weight_process.setConfiguration({"k_weight": 0})
222
        out = k_weight_process()
payno's avatar
payno committed
223
224
        out = PyMca_ft(inputs={"xas_obj": out.to_dict()})()
        out = PyMca_normalization(inputs={"xas_obj": out.to_dict()})()
225
226
227
228
229
230
        assert isinstance(out, XASObject)
        assert out.linked_h5_file
        assert out.linked_h5_file == self.h5_file
        # then check all process are correctly registered with the valid id...
        processes = self.xas_obj.get_process_flow()
        self.assertEqual(len(processes), 5)
payno's avatar
payno committed
231
232
233
234
235
236
237
        self.assertEqual(
            h5py_read_dataset(processes[1]["program"]), "pymca_normalization"
        )
        self.assertEqual(h5py_read_dataset(processes[2]["program"]), "pymca_exafs")
        self.assertEqual(
            h5py_read_dataset(processes[5]["program"]), "pymca_normalization"
        )
238
239
240
        self.xas_obj.clean_process_flow()
        processes = self.xas_obj.get_process_flow()
        self.assertEqual(len(processes), 0)
241

242

payno's avatar
payno committed
243
@unittest.skipIf(has_pymca is False, "PyMca5 is not installed")
244
245
246
247
class TestSaveFlowAuto(unittest.TestCase):
    """Test that the processing can be stored continuously on a temporary
    .h5 file then dump without lost during call to 'dump_xas' of the XASObject
    """
payno's avatar
payno committed
248

249
250
    def setUp(self):
        self.output_dir = tempfile.mkdtemp()
251
252
253
        energy, spectra = spectra_utils.create_dataset(shape=(100, 10, 10))
        assert spectra.shape == (100, 10, 10)
        assert len(energy) == spectra.shape[0]
payno's avatar
payno committed
254
        self.xas_obj = XASObject(spectra=spectra, energy=energy, dim1=10, dim2=10)
255

payno's avatar
payno committed
256
        self.h5_file = os.path.join(self.output_dir, "output_file.h5")
257
258
259
260
261
262

    def tearDown(self):
        shutil.rmtree(self.output_dir)

    def test(self):
        self.assertTrue(self.xas_obj.linked_h5_file is not None)
payno's avatar
payno committed
263
264
        out = PyMca_normalization(inputs={"xas_obj": self.xas_obj})()
        exafs = PyMca_exafs(inputs={"xas_obj": out})
265
        configuration = {
payno's avatar
payno committed
266
267
268
            "Knots": {"Values": (1, 2, 5), "Number": 3, "Orders": [3, 3, 3]},
            "KMin": 0,
            "KMax": 2.3,
269
270
        }
        exafs.setConfiguration(configuration)
payno's avatar
payno committed
271
272
        out = exafs()
        kweight_process = PyMca_k_weight(inputs={"xas_obj": out})
payno's avatar
payno committed
273
        kweight_process.setConfiguration({"k_weight": 0})
payno's avatar
payno committed
274
275
276
        out = kweight_process()
        out = PyMca_ft(inputs={"xas_obj": out})()
        out = PyMca_normalization(inputs={"xas_obj": out})()
277
278
279

        writer = XASWriter()
        writer.output_file = self.h5_file
280
        writer(out)
281

payno's avatar
payno committed
282
283
284
285
286
287
288
289
290
291
292
293
        with h5py.File(self.h5_file, "r") as hdf:
            self.assertTrue("scan1" in hdf.keys())
            self.assertTrue("data" in hdf["scan1"].keys())
            self.assertTrue("absorbed_beam" in hdf["scan1"].keys())
            self.assertTrue("monochromator" in hdf["scan1"].keys())
            self.assertTrue("xas_process_1" in hdf["scan1"].keys())
            self.assertTrue("xas_process_2" in hdf["scan1"].keys())
            self.assertTrue("xas_process_3" in hdf["scan1"].keys())
            self.assertTrue("xas_process_4" in hdf["scan1"].keys())
            self.assertTrue("xas_process_5" in hdf["scan1"].keys())
            targetted_xas_process = hdf["scan1"]["xas_process_2"]
            self.assertTrue("program" in targetted_xas_process)
payno's avatar
payno committed
294
295
296
            self.assertEqual(
                h5py_read_dataset(targetted_xas_process["program"]), "pymca_exafs"
            )
payno's avatar
payno committed
297
            self.assertTrue("configuration" in targetted_xas_process)
298
299

        # check one configuration
payno's avatar
payno committed
300
301
302
303
        stored_config = h5todict(
            self.h5_file, path="/scan1/xas_process_2/configuration", asarray=False
        )
        for key in ("KMin", "KMax"):
304
            self.assertTrue(configuration[key] == stored_config[key])