Commit 7a7a5db9 authored by payno's avatar payno
Browse files

[core] add structure for larch process + add and test the mback process

parent e781e69d
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
"""wrapper to the larch mback process"""
from xas.core.types import Spectrum, XASObject
from xas.core.process.process import Process
from larch.xafs.mback import mback
import multiprocessing
import functools
import logging
_logger = logging.getLogger(__name__)
_DEBUG = True
if _DEBUG:
from larch.symboltable import Group
def process_spectr_mback(spectrum, configuration, overwrite=True, callback=None,
output=None, output_dict=None):
"""
:param :class:`.Spectrum` spectrum: spectrum to process
:param dict configuration: configuration of the pymca normalization
:param bool overwrite: False if we want to return a new Spectrum instance
:param function pointer callback: callback to execute.
:param output: list to store the result, needed for pool processing
:type: multiprocessing.manager.list
:param dict output_dict: key: input spectrum, value: index in the output
list.
:return: processed spectrum
:rtype: tuple (configuration, spectrum)
"""
assert isinstance(spectrum, Spectrum)
if spectrum.energy is None or spectrum.mu is None:
_logger.error('Energy and or Mu is/are not specified, unable to '
'compute exafs')
return None, None
opts = {}
for opt_name in ('z', 'edge', 'e0', 'pre1', 'pre2', 'norm1', 'norm2',
'order', 'leexiang', 'tables', 'fit_erfc'):
if opt_name in configuration:
opts[opt_name] = configuration[opt_name]
if _DEBUG is True:
assert isinstance(spectrum, Group)
if overwrite:
_spectrum = spectrum
else:
_spectrum = Spectrum().update(spectrum=spectrum)
mback(_spectrum, **opts)
# mback(_spectrum, z=29, edge='K')
return configuration, _spectrum
def larch_mback(xas_obj):
"""
:param xas_obj: object containing the configuration and spectra to process
:type: Union[XASObject, dict]
:return: spectra dict
:rtype: XASObject
"""
mback_obj = Larch_mback()
return mback_obj.process(xas_obj=xas_obj)
_USE_MULTIPROCESSING_POOL = False
# note: we cannot use multiprocessing pool with push workflow for now.
class Larch_mback(Process):
def __init__(self):
Process.__init__(self, name='mback')
def setProperties(self, properties):
if '_pymcaSettings' in properties:
self._settings = properties['_pymcaSettings']
def process(self, xas_obj):
_xas_obj = self.getXasObject(xas_obj=xas_obj)
if self._settings:
_xas_obj.configuration['mback'] = self._settings
self._advancement.reset(max_=_xas_obj.n_spectrum)
self._advancement.startProcess()
self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess()
self.register_process(_xas_obj, data_keys=("fpp",
"f2"))
return _xas_obj
def _pool_process(self, xas_obj):
assert isinstance(xas_obj, XASObject)
if not _USE_MULTIPROCESSING_POOL:
for spectrum in xas_obj.spectra:
process_spectr_mback(spectrum=spectrum,
configuration=xas_obj.configuration,
callback=self._advancement.increaseAdvancement,
overwrite=True)
else:
from multiprocessing import Manager
manager = Manager()
output_dict = {}
res_list = manager.list()
for i_spect, spect in enumerate(xas_obj.spectra):
res_list.append(None)
output_dict[spect] = i_spect
with multiprocessing.Pool(5) as p:
partial_ = functools.partial(process_spectr_mback,
configuration=xas_obj.configuration,
callback=self._advancement.increaseAdvancement,
overwrite=False,
output=res_list,
output_dict=output_dict)
p.map(partial_, xas_obj.spectra)
# then update local spectrum
for spectrum, res in zip(xas_obj.spectra, res_list):
spectrum.update(res)
def definition(self):
return "mback calculation"
def program_version(self):
import larch.version
return larch.version.version_data()['larch']
def program_name(self):
return 'larch_mback'
__call__ = process
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2017-2019 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "07/25/2019"
import os
import unittest
import tempfile
import shutil
import urllib.request
from xas.core.types import Spectrum, XASObject
try:
import larch
except ImportError:
has_larch = False
else:
has_larch = True
from xas.core.process.larch.mback import larch_mback, process_spectr_mback
from xas.io.utils.larch import read_ascii
@unittest.skipIf(has_larch is False, 'xraylarch not installed')
class TestLarchSpectrum(unittest.TestCase):
"""Make sure computation on one spectrum is valid"""
def setUp(self):
self.outputdir = tempfile.mkdtemp()
xmu_url = "https://raw.githubusercontent.com/xraypy/xraylarch/master/examples/xafs/cu_rt01.xmu"
self.data_file = os.path.join(self.outputdir, 'cu_rt01.xmu')
with urllib.request.urlopen(xmu_url) as response, open(self.data_file, 'wb') as out_file:
data = response.read() # a `bytes` object
out_file.write(data)
assert os.path.exists(self.data_file)
energy, mu = read_ascii(self.data_file)
self.spectrum = Spectrum(energy=energy, mu=mu)
self.configuration = {'z': 29}
def tearDown(self):
shutil.rmtree(self.outputdir)
def testProcess(self):
self.assertFalse(hasattr(self.spectrum, 'fpp'))
self.assertFalse(hasattr(self.spectrum, 'f2'))
conf, res_spectrum = process_spectr_mback(self.spectrum,
self.configuration)
self.assertTrue(hasattr(res_spectrum, 'fpp'))
self.assertTrue(hasattr(res_spectrum, 'f2'))
@unittest.skipIf(has_larch is False, 'xraylarch not installed')
class TestLarchSpectra(unittest.TestCase):
"""Make sure computation on spectra is valid (n spectrum)"""
def setUp(self):
self.outputdir = tempfile.mkdtemp()
# Download test file from xraylarch
xmu_url = "https://raw.githubusercontent.com/xraypy/xraylarch/master/examples/xafs/cu_rt01.xmu"
self.data_file = os.path.join(self.outputdir, 'cu_rt01.xmu')
with urllib.request.urlopen(xmu_url) as response, open(self.data_file, 'wb') as out_file:
data = response.read() # a `bytes` object
out_file.write(data)
assert os.path.exists(self.data_file)
# define the xas object
self.configuration = {'z': 29}
energy, mu = read_ascii(self.data_file)
self.xas_object = XASObject(spectra=(Spectrum(energy=energy, mu=mu),),
energy=energy, dim1=1, dim2=1,
configuration=self.configuration)
def tearDown(self):
shutil.rmtree(self.outputdir)
def testProcessXASObject(self):
res = larch_mback(self.xas_object)
spectrum0 = res.spectra[0]
self.assertTrue(hasattr(spectrum0, 'fpp'))
self.assertTrue(hasattr(spectrum0, 'f2'))
def testProcessAsDict(self):
res = larch_mback(self.xas_object.to_dict())
spectrum0 = res.spectra[0]
self.assertTrue(hasattr(spectrum0, 'fpp'))
self.assertTrue(hasattr(spectrum0, 'f2'))
def suite():
test_suite = unittest.TestSuite()
for ui in (TestLarchSpectrum, TestLarchSpectra):
test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(ui))
return test_suite
if __name__ == '__main__':
unittest.main(defaultTest="suite")
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "04/01/2018"
from numpy.distutils.misc_util import Configuration
def configuration(parent_package='', top_path=None):
config = Configuration('pymca', parent_package, top_path)
config.add_subpackage('test')
return config
if __name__ == "__main__":
from numpy.distutils.core import setup
setup(configuration=configuration)
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "07/24/2019"
from larch.io.columnfile import read_ascii as larch_read_ascii
def read_ascii(xmu_file):
"""
:param xmu_file: file containing the spectrum definition
:return: (energy, mu)
:rtype: tuple
"""
larch_group = larch_read_ascii(xmu_file)
return (larch_group.energy, larch_group.mu)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment