test_workflow.py 12.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2017-2019 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
25
26
27
"""
unit test for workflow composed of pymca process 
"""
28
29
30
31
32
33
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "06/11/2019"


import os
34
35
36
37
import shutil
import tempfile
import unittest
import h5py
38
import numpy
payno's avatar
payno committed
39
from silx.io.utils import h5py_read_dataset
40
from silx.io.dictdump import h5todict
41
from silx.io.url import DataUrl
payno's avatar
payno committed
42
from est.core.process.roi import xas_roi, _ROI as XASROI
payno's avatar
payno committed
43
from est.core.io import XASWriter
payno's avatar
payno committed
44
45
46
from est.core.io import read as read_xas
from est.core.types import XASObject
from est.core.utils import spectra as spectra_utils
payno's avatar
fix CI    
payno committed
47
from est.core.types import Dim
payno's avatar
payno committed
48

49
50
51
52
53
54
55
try:
    import PyMca5
except ImportError:
    has_pymca = False
else:
    has_pymca = True
    from PyMca5.PyMcaDataDir import PYMCA_DATA_DIR
payno's avatar
payno committed
56
57
58
59
60
    from est.core.process.pymca.exafs import pymca_exafs, PyMca_exafs
    from est.core.process.pymca.ft import pymca_ft, PyMca_ft
    from est.core.process.pymca.k_weight import pymca_k_weight, PyMca_k_weight
    from est.core.process.pymca.normalization import pymca_normalization
    from est.core.process.pymca.normalization import PyMca_normalization
61
62


payno's avatar
payno committed
63
@unittest.skipIf(has_pymca is False, "PyMca5 is not installed")
64
class TestStreamSingleSpectrum(unittest.TestCase):
65
    """Make sure the process have valid io"""
payno's avatar
payno committed
66

67
68
    def setUp(self):
        self.exafs_configuration = {
payno's avatar
payno committed
69
70
71
            "Knots": {"Values": (1, 2, 5), "Number": 3, "Orders": [3, 3, 3]},
            "KMin": 0,
            "KMax": 2.3,
72
73
        }

74
75
    def test_pymca_process(self):
        data_file = os.path.join(PYMCA_DATA_DIR, "EXAFS_Cu.dat")
payno's avatar
payno committed
76
        out = read_xas(
payno's avatar
payno committed
77
78
79
80
81
82
            spectra_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 2"
            ),
            channel_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 1"
            ),
payno's avatar
payno committed
83
84
        )
        out.configuration = {"EXAFS": self.exafs_configuration, "SET_KWEIGHT": 0}
85
        out = pymca_normalization(xas_obj=out)
86
        out = pymca_exafs(xas_obj=out)
87
        out = pymca_k_weight(xas_obj=out)
88
        out = pymca_ft(xas_obj=out)
89
        assert isinstance(out, XASObject)
90
91
92

    def test_pymca_process_with_dict(self):
        data_file = os.path.join(PYMCA_DATA_DIR, "EXAFS_Cu.dat")
payno's avatar
payno committed
93
        out = read_xas(
payno's avatar
payno committed
94
95
96
97
98
99
            spectra_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 2"
            ),
            channel_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 1"
            ),
payno's avatar
payno committed
100
101
        )
        out.configuration = {"EXAFS": self.exafs_configuration, "SET_KWEIGHT": 0}
102
        out = pymca_normalization(xas_obj=out.to_dict())
103
104
105
        out = pymca_exafs(xas_obj=out.to_dict())
        out = pymca_k_weight(xas_obj=out.to_dict())
        out = pymca_ft(xas_obj=out.to_dict())
106
        assert isinstance(out, XASObject)
107

108
109
    def test_pymca_process_with_cls(self):
        data_file = os.path.join(PYMCA_DATA_DIR, "EXAFS_Cu.dat")
payno's avatar
payno committed
110
        out = read_xas(
payno's avatar
payno committed
111
112
113
114
115
116
            spectra_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 2"
            ),
            channel_url=DataUrl(
                file_path=data_file, scheme="PyMca", data_path="Column 1"
            ),
payno's avatar
payno committed
117
        )
payno's avatar
payno committed
118
119
        out = PyMca_normalization(inputs={"xas_obj": out})()
        exafs_process = PyMca_exafs(inputs={"xas_obj": out})
120
        exafs_process.setConfiguration(configuration=self.exafs_configuration)
121
        out = exafs_process()
payno's avatar
payno committed
122
        kweight_process = PyMca_k_weight(inputs={"xas_obj": out})
payno's avatar
payno committed
123
        kweight_process.setConfiguration({"k_weight": 0})
payno's avatar
payno committed
124
125
        out = kweight_process()
        out = PyMca_ft(inputs={"xas_obj": out})()
126
        assert isinstance(out, XASObject)
payno's avatar
payno committed
127
128
        self.assertTrue(out.spectra.data.flat[0].ft is not None)
        self.assertTrue(len(out.spectra.data.flat[0].ft.intensity) > 0)
129
130

    def test_pymca_with_roi(self):
131
        energy, spectra = spectra_utils.create_dataset(shape=(256, 20, 10))
132
        roi = XASROI(origin=(0, 2), size=(5, 1))
payno's avatar
payno committed
133
134
135
        xas_obj = XASObject(
            spectra=spectra, energy=energy, configuration={"roi": roi.to_dict()}
        )
136
        dict_xas_obj = xas_obj.to_dict()
payno's avatar
payno committed
137
138
        self.assertTrue("spectra" in dict_xas_obj.keys())
        self.assertTrue("energy" in dict_xas_obj.keys())
139
        tmp_obj = XASObject.from_dict(dict_xas_obj)
payno's avatar
payno committed
140
141
142
        numpy.testing.assert_array_equal(
            tmp_obj.energy, tmp_obj.spectra.data.flat[0].energy
        )
143
        out = xas_roi(dict_xas_obj)
payno's avatar
payno committed
144
        out.configuration = {"EXAFS": self.exafs_configuration, "SET_KWEIGHT": 0}
145
        out = pymca_normalization(xas_obj=out)
146
147
148
149
        out = pymca_exafs(xas_obj=out)
        out = pymca_k_weight(xas_obj=out)
        out = pymca_ft(xas_obj=out)
        assert isinstance(out, XASObject)
payno's avatar
payno committed
150
151
        self.assertEqual(out.spectra.shape[0], 1)
        self.assertEqual(out.spectra.shape[1], 5)
payno's avatar
payno committed
152
153
        self.assertTrue(out.spectra.data.flat[0].ft is not None)
        self.assertTrue(len(out.spectra.data.flat[0].ft.intensity) > 0)
154

155

payno's avatar
payno committed
156
@unittest.skipIf(has_pymca is False, "PyMca5 is not installed")
157
158
159
160
class TestWorkflowAndH5LinkedFile(unittest.TestCase):
    """
    Test that the workflow can process and store process in a targeted h5 file
    """
payno's avatar
payno committed
161

162
163
    def setUp(self):
        self.output_dir = tempfile.mkdtemp()
164
        energy, spectra = spectra_utils.create_dataset(shape=(256, 20, 10))
165
        roi = XASROI(origin=(0, 2), size=(5, 1))
payno's avatar
payno committed
166
167
168
169
        self.xas_obj = XASObject(
            spectra=spectra, energy=energy, configuration={"roi": roi.to_dict()}
        )
        self.h5_file = os.path.join(self.output_dir, "output_file.h5")
170
171
172
173
174
175
176
177
        self.xas_obj.link_to_h5(self.h5_file)

    def tearDown(self):
        shutil.rmtree(self.output_dir)

    def test_h5_link_xas_object(self):
        """Test that the processing can be stored continuously on a .h5 file"""
        self.assertTrue(self.xas_obj.linked_h5_file is not None)
payno's avatar
payno committed
178
179
        out = PyMca_normalization(inputs={"xas_obj": self.xas_obj})()
        exafs_process = PyMca_exafs(inputs={"xas_obj": out})
180
        configuration = {
payno's avatar
payno committed
181
182
183
            "Knots": {"Values": (1, 2, 5), "Number": 3, "Orders": [3, 3, 3]},
            "KMin": 0,
            "KMax": 2.3,
184
185
        }
        exafs_process.setConfiguration(configuration=configuration)
186
        out = exafs_process()
payno's avatar
payno committed
187
        k_weight_process = PyMca_k_weight(inputs={"xas_obj": out})
payno's avatar
payno committed
188
        k_weight_process.setConfiguration({"k_weight": 0})
189
        out = k_weight_process()
payno's avatar
payno committed
190
        out = PyMca_ft(inputs={"xas_obj": out})()
191
        out = PyMca_normalization(inputs={"xas_obj": out})()
192
        assert isinstance(out, XASObject)
193
        assert out.linked_h5_file is self.h5_file
194
        # then check all process are correctly registered with the valid id...
195
        processes = self.xas_obj.get_process_flow()
196
        self.assertEqual(len(processes), 5)
payno's avatar
payno committed
197
198
199
200
201
202
203
        self.assertEqual(
            h5py_read_dataset(processes[1]["program"]), "pymca_normalization"
        )
        self.assertEqual(h5py_read_dataset(processes[2]["program"]), "pymca_exafs")
        self.assertEqual(
            h5py_read_dataset(processes[5]["program"]), "pymca_normalization"
        )
204
205
        self.xas_obj.clean_process_flow()
        processes = self.xas_obj.get_process_flow()
206
207
208
209
210
        self.assertEqual(len(processes), 0)

    def test_h5_link_dict(self):
        """Same test as test_h5_link_xas_object but with a dict pass between
        processes"""
211
        self.assertTrue(self.xas_obj.linked_h5_file is not None)
payno's avatar
payno committed
212
213
        out = PyMca_normalization(inputs={"xas_obj": self.xas_obj.to_dict()})()
        exafs_process = PyMca_exafs(inputs={"xas_obj": out})
214
        configuration = {
payno's avatar
payno committed
215
216
217
            "Knots": {"Values": (1, 2, 5), "Number": 3, "Orders": [3, 3, 3]},
            "KMin": 0,
            "KMax": 2.3,
218
219
        }
        exafs_process.setConfiguration(configuration=configuration)
payno's avatar
payno committed
220
        out = exafs_process()
221
        k_weight_process = PyMca_k_weight(inputs={"xas_obj": out.to_dict()})
payno's avatar
payno committed
222
        k_weight_process.setConfiguration({"k_weight": 0})
223
        out = k_weight_process()
payno's avatar
payno committed
224
225
        out = PyMca_ft(inputs={"xas_obj": out.to_dict()})()
        out = PyMca_normalization(inputs={"xas_obj": out.to_dict()})()
226
227
228
229
230
231
        assert isinstance(out, XASObject)
        assert out.linked_h5_file
        assert out.linked_h5_file == self.h5_file
        # then check all process are correctly registered with the valid id...
        processes = self.xas_obj.get_process_flow()
        self.assertEqual(len(processes), 5)
payno's avatar
payno committed
232
233
234
235
236
237
238
        self.assertEqual(
            h5py_read_dataset(processes[1]["program"]), "pymca_normalization"
        )
        self.assertEqual(h5py_read_dataset(processes[2]["program"]), "pymca_exafs")
        self.assertEqual(
            h5py_read_dataset(processes[5]["program"]), "pymca_normalization"
        )
239
240
241
        self.xas_obj.clean_process_flow()
        processes = self.xas_obj.get_process_flow()
        self.assertEqual(len(processes), 0)
242

243

payno's avatar
payno committed
244
@unittest.skipIf(has_pymca is False, "PyMca5 is not installed")
245
246
247
248
class TestSaveFlowAuto(unittest.TestCase):
    """Test that the processing can be stored continuously on a temporary
    .h5 file then dump without lost during call to 'dump_xas' of the XASObject
    """
payno's avatar
payno committed
249

250
251
    def setUp(self):
        self.output_dir = tempfile.mkdtemp()
252
253
254
        energy, spectra = spectra_utils.create_dataset(shape=(100, 10, 10))
        assert spectra.shape == (100, 10, 10)
        assert len(energy) == spectra.shape[0]
payno's avatar
payno committed
255
        self.xas_obj = XASObject(spectra=spectra, energy=energy, dim1=10, dim2=10)
256

payno's avatar
payno committed
257
        self.h5_file = os.path.join(self.output_dir, "output_file.h5")
258
259
260
261
262
263

    def tearDown(self):
        shutil.rmtree(self.output_dir)

    def test(self):
        self.assertTrue(self.xas_obj.linked_h5_file is not None)
payno's avatar
payno committed
264
265
        out = PyMca_normalization(inputs={"xas_obj": self.xas_obj})()
        exafs = PyMca_exafs(inputs={"xas_obj": out})
266
        configuration = {
payno's avatar
payno committed
267
268
269
            "Knots": {"Values": (1, 2, 5), "Number": 3, "Orders": [3, 3, 3]},
            "KMin": 0,
            "KMax": 2.3,
270
271
        }
        exafs.setConfiguration(configuration)
payno's avatar
payno committed
272
273
        out = exafs()
        kweight_process = PyMca_k_weight(inputs={"xas_obj": out})
payno's avatar
payno committed
274
        kweight_process.setConfiguration({"k_weight": 0})
payno's avatar
payno committed
275
276
277
        out = kweight_process()
        out = PyMca_ft(inputs={"xas_obj": out})()
        out = PyMca_normalization(inputs={"xas_obj": out})()
278
279
280

        writer = XASWriter()
        writer.output_file = self.h5_file
281
        writer(out)
282

payno's avatar
payno committed
283
284
285
286
287
288
289
290
291
292
293
294
        with h5py.File(self.h5_file, "r") as hdf:
            self.assertTrue("scan1" in hdf.keys())
            self.assertTrue("data" in hdf["scan1"].keys())
            self.assertTrue("absorbed_beam" in hdf["scan1"].keys())
            self.assertTrue("monochromator" in hdf["scan1"].keys())
            self.assertTrue("xas_process_1" in hdf["scan1"].keys())
            self.assertTrue("xas_process_2" in hdf["scan1"].keys())
            self.assertTrue("xas_process_3" in hdf["scan1"].keys())
            self.assertTrue("xas_process_4" in hdf["scan1"].keys())
            self.assertTrue("xas_process_5" in hdf["scan1"].keys())
            targetted_xas_process = hdf["scan1"]["xas_process_2"]
            self.assertTrue("program" in targetted_xas_process)
payno's avatar
payno committed
295
296
297
            self.assertEqual(
                h5py_read_dataset(targetted_xas_process["program"]), "pymca_exafs"
            )
payno's avatar
payno committed
298
            self.assertTrue("configuration" in targetted_xas_process)
299
300

        # check one configuration
payno's avatar
payno committed
301
302
303
304
        stored_config = h5todict(
            self.h5_file, path="/scan1/xas_process_2/configuration", asarray=False
        )
        for key in ("KMin", "KMax"):
305
            self.assertTrue(configuration[key] == stored_config[key])