Commit f5ccefa2 authored by payno's avatar payno
Browse files

Merge branch 'integrate_ewoks_v2' into 'master'

Integrate ewoks v2

See merge request workflow/est!47
parents f3be7257 e3b4f2bb
Pipeline #56180 failed with stages
in 8 minutes and 46 seconds
...@@ -16,10 +16,10 @@ check_style: ...@@ -16,10 +16,10 @@ check_style:
stage: style stage: style
image: docker-registry.esrf.fr/dau/est:python3.7_stretch_pyqt5_gtk3_wxpython image: docker-registry.esrf.fr/dau/est:python3.7_stretch_pyqt5_gtk3_wxpython
before_script: before_script:
- pip install black flake8 - pip install flake8
script: script:
- LC_ALL=C.UTF-8 black --check --safe . - flake8 est
- flake8 - flake8 orangecontrib
# build # build
...@@ -52,7 +52,6 @@ doc: ...@@ -52,7 +52,6 @@ doc:
only: only:
- master - master
# test # test
.build_template: &test_linux_template_pymca .build_template: &test_linux_template_pymca
stage: test stage: test
...@@ -145,6 +144,30 @@ test:python3.7-stretch-pyqt5_pymca_larch: ...@@ -145,6 +144,30 @@ test:python3.7-stretch-pyqt5_pymca_larch:
image: docker-registry.esrf.fr/dau/est:python3.7_stretch_pyqt5_gtk3_wxpython image: docker-registry.esrf.fr/dau/est:python3.7_stretch_pyqt5_gtk3_wxpython
<<: *test_linux_template_pymca_larch <<: *test_linux_template_pymca_larch
check_lint:
stage: test
image: docker-registry.esrf.fr/dau/ewoks:python_3.8
tags:
- linux
before_script:
- arch
- which python
- python --version
- python -m pip install pip --upgrade
- python -m pip install setuptools --upgrade
- python -m pip install flake8
script:
- python setup.py build
- python -m pip install .[doc]
- python setup.py build_sphinx
- mv build/sphinx/html artifacts/doc
artifacts:
paths:
- artifacts/doc/
when: on_success
expire_in: 2h
only:
- master
test:test-est-tutorials_pymca: test:test-est-tutorials_pymca:
image: docker-registry.esrf.fr/dau/est:python3.7_stretch_pyqt5_gtk3_wxpython image: docker-registry.esrf.fr/dau/est:python3.7_stretch_pyqt5_gtk3_wxpython
......
...@@ -217,7 +217,7 @@ class XASWriter(object): ...@@ -217,7 +217,7 @@ class XASWriter(object):
if not self._output_file: if not self._output_file:
_logger.warning( _logger.warning(
"no output file defined, please give path to the" "output file" "no output file defined, please give path to the output file"
) )
self._output_file = input() self._output_file = input()
......
...@@ -83,15 +83,16 @@ class EnergyROIProcess( ...@@ -83,15 +83,16 @@ class EnergyROIProcess(
xas_obj.spectra.energy >= roi.from_ xas_obj.spectra.energy >= roi.from_
) )
xas_obj.spectra.energy = xas_obj.spectra.energy[mask] xas_obj.spectra.energy = xas_obj.spectra.energy[mask]
for spectrum in xas_obj.spectra.data.flat: n_s = len(xas_obj.spectra.data.flat)
for i_s, spectrum in enumerate(xas_obj.spectra.data.flat):
spectrum.energy = spectrum.energy[mask] spectrum.energy = spectrum.energy[mask]
spectrum.mu = spectrum.mu[mask] spectrum.mu = spectrum.mu[mask]
for attached_key in ("I0", "I1", "I2", "mu_ref"): for attached_key in ("I0", "I1", "I2", "mu_ref"):
if hasattr(spectrum, attached_key): if hasattr(spectrum, attached_key):
values = getattr(spectrum, attached_key)[mask] values = getattr(spectrum, attached_key)[mask]
setattr(spectrum, attached_key, values) setattr(spectrum, attached_key, values)
# print(spectrum.__dict__) self.progress = i_s / n_s * 100.0
# print(xas_obj.__dict__)
return xas_obj return xas_obj
def run(self): def run(self):
...@@ -109,8 +110,10 @@ class EnergyROIProcess( ...@@ -109,8 +110,10 @@ class EnergyROIProcess(
# existing roi is priority. This is the case if called from pushworkflow # existing roi is priority. This is the case if called from pushworkflow
# for example. # for example.
self.progress = 0.0
if self._roi is not None: if self._roi is not None:
xas_obj = self._apply_roi(xas_obj=_xas_obj, roi=self._roi) xas_obj = self._apply_roi(xas_obj=_xas_obj, roi=self._roi)
self.progress = 100.0
self.register_process(_xas_obj, data_keys=("Mu", "energy")) self.register_process(_xas_obj, data_keys=("Mu", "energy"))
self.outputs.xas_obj = _xas_obj.to_dict() self.outputs.xas_obj = _xas_obj.to_dict()
......
...@@ -31,22 +31,24 @@ __date__ = "06/11/2019" ...@@ -31,22 +31,24 @@ __date__ = "06/11/2019"
from .process import Process from .process import Process
from est.core.types import XASObject from est.core.types import XASObject
from est.io.utils.information import InputInformation from est.io.utils.information import InputInformation
from typing import Optional
import pkg_resources import pkg_resources
class DumpXasObject( class DumpXasObject(
Process, name="dump xas object", input_names=["xas_obj"], output_names=["result"] Process, name="dump xas object", input_names=["xas_obj"], output_names=["result"]
): ):
def __init__(self, **kwargs): def __init__(self, missing_file_callback: Optional[str] = None, **kwargs):
self._missing_file_callback = missing_file_callback
super().__init__(**kwargs) super().__init__(**kwargs)
self._output_file = kwargs.get("varinfo", dict()).get("output_file", None) self._output_file = kwargs.get("varinfo", dict()).get("output_file", None)
@property @property
def output_file(self): def output_file(self) -> Optional[str]:
return self._output_file return self._output_file
@output_file.setter @output_file.setter
def output_file(self, output_file): def output_file(self, output_file: Optional[str]):
self._output_file = output_file self._output_file = output_file
def program_name(self): def program_name(self):
...@@ -72,6 +74,8 @@ class DumpXasObject( ...@@ -72,6 +74,8 @@ class DumpXasObject(
"xas_object should be a convertable dict or an" "instance of XASObject" "xas_object should be a convertable dict or an" "instance of XASObject"
) )
if self.output_file is None and self._missing_file_callback is not None:
self.output_file = self._missing_file_callback()
if self.output_file is None: if self.output_file is None:
raise ValueError("output file not provided") raise ValueError("output file not provided")
else: else:
......
...@@ -28,8 +28,6 @@ from est.core.types import Spectrum, XASObject ...@@ -28,8 +28,6 @@ from est.core.types import Spectrum, XASObject
from est.core.process.process import Process from est.core.process.process import Process
from est.core.process.process import _NexusDatasetDef from est.core.process.process import _NexusDatasetDef
from larch.xafs.autobk import autobk from larch.xafs.autobk import autobk
import multiprocessing
import functools
import logging import logging
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -128,10 +126,6 @@ def larch_autobk(xas_obj): ...@@ -128,10 +126,6 @@ def larch_autobk(xas_obj):
return mback_obj.run() return mback_obj.run()
_USE_MULTIPROCESSING_POOL = False
# note: we cannot use multiprocessing pool with pypushflow for now.
class Larch_autobk( class Larch_autobk(
Process, Process,
name="autobk", name="autobk",
...@@ -152,48 +146,24 @@ class Larch_autobk( ...@@ -152,48 +146,24 @@ class Larch_autobk(
self.setConfiguration(self.inputs.autobk) self.setConfiguration(self.inputs.autobk)
_xas_obj.configuration["autobk"] = self.inputs.autobk _xas_obj.configuration["autobk"] = self.inputs.autobk
self._advancement.reset(max_=_xas_obj.n_spectrum) self.progress = 0
self._advancement.startProcess()
self._pool_process(xas_obj=_xas_obj) self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess() self.progress = 100
self.register_process(_xas_obj, data_keys=(_NexusDatasetDef("bkg"),)) self.register_process(_xas_obj, data_keys=(_NexusDatasetDef("bkg"),))
self.outputs.xas_obj = _xas_obj.to_dict() self.outputs.xas_obj = _xas_obj.to_dict()
return _xas_obj return _xas_obj
def _pool_process(self, xas_obj): def _pool_process(self, xas_obj):
assert isinstance(xas_obj, XASObject) assert isinstance(xas_obj, XASObject)
if not _USE_MULTIPROCESSING_POOL: n_s = len(xas_obj.spectra.data.flat)
for spectrum in xas_obj.spectra: for i_s, spectrum in enumerate(xas_obj.spectra):
process_spectr_autobk( process_spectr_autobk(
spectrum=spectrum, spectrum=spectrum,
configuration=xas_obj.configuration, configuration=xas_obj.configuration,
callbacks=self.callbacks, callbacks=self.callbacks,
overwrite=True, overwrite=True,
) )
else: self.progress = i_s / n_s * 100.0
from multiprocessing import Manager
manager = Manager()
output_dict = {}
res_list = manager.list()
for i_spect, spect in enumerate(xas_obj.spectra):
res_list.append(None)
output_dict[spect] = i_spect
with multiprocessing.Pool(5) as p:
partial_ = functools.partial(
process_spectr_autobk,
configuration=xas_obj.configuration,
callbacks=self.callbacks,
overwrite=False,
output=res_list,
output_dict=output_dict,
)
p.map(partial_, xas_obj.spectra)
# then update local spectrum
for spectrum, res in zip(xas_obj.spectra, res_list):
spectrum.update(res)
def definition(self): def definition(self):
return "autobk calculation" return "autobk calculation"
......
...@@ -28,8 +28,6 @@ from est.core.types import Spectrum, XASObject ...@@ -28,8 +28,6 @@ from est.core.types import Spectrum, XASObject
from est.core.process.process import Process from est.core.process.process import Process
from est.core.process.process import _NexusDatasetDef from est.core.process.process import _NexusDatasetDef
from larch.xafs.mback import mback from larch.xafs.mback import mback
import multiprocessing
import functools
import logging import logging
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -119,10 +117,6 @@ def larch_mback(xas_obj): ...@@ -119,10 +117,6 @@ def larch_mback(xas_obj):
return mback_obj.run() return mback_obj.run()
_USE_MULTIPROCESSING_POOL = False
# note: we cannot use multiprocessing pool with pypushflow for now.
class Larch_mback( class Larch_mback(
Process, Process,
name="mback", name="mback",
...@@ -146,10 +140,9 @@ class Larch_mback( ...@@ -146,10 +140,9 @@ class Larch_mback(
elif "mback" not in _xas_obj.configuration: elif "mback" not in _xas_obj.configuration:
_xas_obj.configuration["mback"] = {} _xas_obj.configuration["mback"] = {}
self._advancement.reset(max_=_xas_obj.n_spectrum) self.progress = 0.0
self._advancement.startProcess()
self._pool_process(xas_obj=_xas_obj) self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess() self.progress = 100.0
self.register_process( self.register_process(
_xas_obj, _xas_obj,
data_keys=( data_keys=(
...@@ -162,38 +155,15 @@ class Larch_mback( ...@@ -162,38 +155,15 @@ class Larch_mback(
def _pool_process(self, xas_obj): def _pool_process(self, xas_obj):
assert isinstance(xas_obj, XASObject) assert isinstance(xas_obj, XASObject)
if not _USE_MULTIPROCESSING_POOL: n_s = len(xas_obj.spectra.data.flat)
for spectrum in xas_obj.spectra: for i_s, spectrum in enumerate(xas_obj.spectra):
process_spectr_mback( process_spectr_mback(
spectrum=spectrum, spectrum=spectrum,
configuration=xas_obj.configuration["mback"], configuration=xas_obj.configuration["mback"],
callbacks=self.callbacks, callbacks=self.callbacks,
overwrite=True, overwrite=True,
) )
else: self.progress = i_s / n_s * 100.0
from multiprocessing import Manager
manager = Manager()
output_dict = {}
res_list = manager.list()
for i_spect, spect in enumerate(xas_obj.spectra):
res_list.append(None)
output_dict[spect] = i_spect
with multiprocessing.Pool(5) as p:
partial_ = functools.partial(
process_spectr_mback,
configuration=xas_obj.configuration["mback"],
callbacks=self.callbacks,
overwrite=False,
output=res_list,
output_dict=output_dict,
)
p.map(partial_, xas_obj.spectra)
# then update local spectrum
for spectrum, res in zip(xas_obj.spectra, res_list):
spectrum.update(res)
def definition(self): def definition(self):
return "mback calculation" return "mback calculation"
......
...@@ -29,8 +29,6 @@ from est.core.process.process import Process ...@@ -29,8 +29,6 @@ from est.core.process.process import Process
from est.core.process.process import _NexusDatasetDef from est.core.process.process import _NexusDatasetDef
from larch.xafs.mback import mback_norm from larch.xafs.mback import mback_norm
from larch.xafs.pre_edge import preedge from larch.xafs.pre_edge import preedge
import multiprocessing
import functools
import logging import logging
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -128,10 +126,6 @@ def larch_mback_norm(xas_obj): ...@@ -128,10 +126,6 @@ def larch_mback_norm(xas_obj):
return mback_obj.run() return mback_obj.run()
_USE_MULTIPROCESSING_POOL = False
# note: we cannot use multiprocessing pool with pypushflow for now.
class Larch_mback_norm( class Larch_mback_norm(
Process, Process,
name="mback_norm", name="mback_norm",
...@@ -152,10 +146,9 @@ class Larch_mback_norm( ...@@ -152,10 +146,9 @@ class Larch_mback_norm(
self.setConfiguration(self.inputs.mback_norm) self.setConfiguration(self.inputs.mback_norm)
_xas_obj.configuration["mback_norm"] = self.inputs.mback_norm _xas_obj.configuration["mback_norm"] = self.inputs.mback_norm
self._advancement.reset(max_=_xas_obj.n_spectrum) self.progress = 0.0
self._advancement.startProcess()
self._pool_process(xas_obj=_xas_obj) self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess() self.progress = 100.0
data_keys = [ data_keys = [
_NexusDatasetDef("mback_mu"), _NexusDatasetDef("mback_mu"),
_NexusDatasetDef("norm_mback"), _NexusDatasetDef("norm_mback"),
...@@ -168,38 +161,15 @@ class Larch_mback_norm( ...@@ -168,38 +161,15 @@ class Larch_mback_norm(
def _pool_process(self, xas_obj): def _pool_process(self, xas_obj):
assert isinstance(xas_obj, XASObject) assert isinstance(xas_obj, XASObject)
if not _USE_MULTIPROCESSING_POOL: n_s = len(xas_obj.spectra.data.flat)
for spectrum in xas_obj.spectra: for i_s, spectrum in enumerate(xas_obj.spectra):
process_spectr_mback_norm( process_spectr_mback_norm(
spectrum=spectrum, spectrum=spectrum,
configuration=xas_obj.configuration, configuration=xas_obj.configuration,
callbacks=self.callbacks, callbacks=self.callbacks,
overwrite=True, overwrite=True,
) )
else: self.progress = i_s / n_s * 100.0
from multiprocessing import Manager
manager = Manager()
output_dict = {}
res_list = manager.list()
for i_spect, spect in enumerate(xas_obj.spectra):
res_list.append(None)
output_dict[spect] = i_spect
with multiprocessing.Pool(5) as p:
partial_ = functools.partial(
process_spectr_mback_norm,
configuration=xas_obj.configuration,
callback=self._advancement.increaseAdvancement,
overwrite=False,
output=res_list,
output_dict=output_dict,
)
p.map(partial_, xas_obj.spectra)
# then update local spectrum
for spectrum, res in zip(xas_obj.spectra, res_list):
spectrum.update(res)
def definition(self): def definition(self):
return "mback norm calculation" return "mback norm calculation"
......
...@@ -29,8 +29,6 @@ from est.core.process.process import Process ...@@ -29,8 +29,6 @@ from est.core.process.process import Process
from est.core.process.process import _NexusSpectrumDef, _NexusDatasetDef from est.core.process.process import _NexusSpectrumDef, _NexusDatasetDef
from est.core.utils.symbol import MU_CHAR from est.core.utils.symbol import MU_CHAR
from larch.xafs.pre_edge import pre_edge from larch.xafs.pre_edge import pre_edge
import multiprocessing
import functools
import logging import logging
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -98,7 +96,9 @@ def process_spectr_pre_edge( ...@@ -98,7 +96,9 @@ def process_spectr_pre_edge(
opts[opt_name] = _conf[opt_name] opts[opt_name] = _conf[opt_name]
if _DEBUG is True: if _DEBUG is True:
assert isinstance(spectrum, Group) assert isinstance(
spectrum, Group
), f"spectrum is expected to be an instance of {type(Group)}. Not {type(spectrum)}"
if overwrite: if overwrite:
_spectrum = spectrum _spectrum = spectrum
else: else:
...@@ -124,10 +124,6 @@ def larch_pre_edge(xas_obj): ...@@ -124,10 +124,6 @@ def larch_pre_edge(xas_obj):
return mback_obj.run() return mback_obj.run()
_USE_MULTIPROCESSING_POOL = False
# note: we cannot use multiprocessing pool with pypushflow for now.
class Larch_pre_edge( class Larch_pre_edge(
Process, Process,
name="pre_edge", name="pre_edge",
...@@ -149,10 +145,9 @@ class Larch_pre_edge( ...@@ -149,10 +145,9 @@ class Larch_pre_edge(
self.setConfiguration(self.inputs.pre_edge) self.setConfiguration(self.inputs.pre_edge)
_xas_obj.configuration["pre_edge"] = self.inputs.pre_edge _xas_obj.configuration["pre_edge"] = self.inputs.pre_edge
self._advancement.reset(max_=_xas_obj.n_spectrum) self.progress = 0.0
self._advancement.startProcess()
self._pool_process(xas_obj=_xas_obj) self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess() self.progress = 100.0
self.register_process( self.register_process(
_xas_obj, _xas_obj,
data_keys=( data_keys=(
...@@ -218,38 +213,15 @@ class Larch_pre_edge( ...@@ -218,38 +213,15 @@ class Larch_pre_edge(
def _pool_process(self, xas_obj): def _pool_process(self, xas_obj):
assert isinstance(xas_obj, XASObject) assert isinstance(xas_obj, XASObject)
if not _USE_MULTIPROCESSING_POOL: n_s = len(xas_obj.spectra.data.flat)
for spectrum in xas_obj.spectra: for i_s, spectrum in enumerate(xas_obj.spectra):
process_spectr_pre_edge( process_spectr_pre_edge(
spectrum=spectrum, spectrum=spectrum,
configuration=xas_obj.configuration, configuration=xas_obj.configuration,
callbacks=self.callbacks, callbacks=self.callbacks,
overwrite=True, overwrite=True,
) )
else: self.progress = i_s / n_s * 100.0
from multiprocessing import Manager
manager = Manager()
output_dict = {}
res_list = manager.list()
for i_spect, spect in enumerate(xas_obj.spectra):
res_list.append(None)
output_dict[spect] = i_spect
with multiprocessing.Pool(5) as p:
partial_ = functools.partial(
process_spectr_pre_edge,
configuration=xas_obj.configuration,
callbacks=self.callbacks,
overwrite=False,
output=res_list,
output_dict=output_dict,
)
p.map(partial_, xas_obj.spectra)
# then update local spectrum
for spectrum, res in zip(xas_obj.spectra, res_list):
spectrum.update(res)
def definition(self): def definition(self):
return "pre_edge calculation" return "pre_edge calculation"
......
...@@ -30,8 +30,6 @@ from est.core.process.process import _NexusSpectrumDef ...@@ -30,8 +30,6 @@ from est.core.process.process import _NexusSpectrumDef
from est.core.process.process import _NexusDatasetDef from est.core.process.process import _NexusDatasetDef
from larch.xafs.xafsft import xftf from larch.xafs.xafsft import xftf
from est.core.utils.symbol import ANGSTROM_CHAR from est.core.utils.symbol import ANGSTROM_CHAR
import multiprocessing
import functools