Commit 2ee26319 authored by payno's avatar payno
Browse files

[workflow_concepts] Make it work with the workflow_concept

For now process is always call. When call from workflow_concept it insure the input has benn provided previously. Otherwise you should provide the xas_object. It might evolve with time.

Add some simple unit test to insure an orangecontrib est workflow can be converted to a graph and a graph to an est workflow.
Add test to insure we can execute it as well

Make `Process` inherit from Task
Create a OWESRFWidget which inherit from OWWidget and implement stuff for the task graph.
parent 915509cf
......@@ -76,9 +76,9 @@ def convert_spectra_dims(dims):
def get_unit(current_unit):
if current_unit == "eV":
if current_unit in ("eV", "electron_volt"):
return ur.eV
elif current_unit == "keV":
elif current_unit in ("keV", "kiloelectron_volt"):
return ur.keV
elif current_unit == "J":
return ur.J
......@@ -87,7 +87,7 @@ def get_unit(current_unit):
elif isinstance(current_unit, ur.Quantity):
return current_unit
else:
raise ValueError("{} is not a valid unit for quantity".format(current_unit))
raise ValueError("`{}` is not a valid unit for quantity".format(current_unit))
def get_url(my_str):
......
......@@ -66,8 +66,8 @@ class EnergyROIProcess(Process):
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
def __init__(self):
Process.__init__(self, name="energy roi")
def __init__(self, varinfo=None, **inputs):
Process.__init__(self, name="energy roi", varinfo=varinfo, **inputs)
self._roi = ROI1D(0, 9999999999)
def set_properties(self, properties: dict):
......@@ -132,7 +132,7 @@ class EnergyROIProcess(Process):
return est.version.version
@staticmethod
def registry_name():
def program_name():
return "energy-roi"
__call__ = process
......@@ -2,32 +2,37 @@ from esrftaskgraph import Task
class IgnoreTask(Task):
INPUT_NAMES = ["xas_obj"]
OUTPUT_NAMES = ["xas_obj"]
def process(self):
pass
print("*** ignore task {}".format(self))
class IgnoreInput(IgnoreTask):
@staticmethod
def registry_name():
return "ignored read"
INPUT_NAMES = []
OUTPUT_NAMES = ["xas_obj"]
pass
# @staticmethod
# def registry_name():
# return "orangecontrib.est.widgets.utils.xas_input.XASInputOW"
#
class IgnoreOrangeConversion(IgnoreTask):
@staticmethod
def registry_name():
return "ignored conversion to orange table"
pass
# @staticmethod
# def registry_name():
# return "orangecontrib.est.widgets.utils.converter.ConverterOW"
class IgnoreSavingPoint(IgnoreTask):
@staticmethod
def registry_name():
return "ignored saving point"
pass
# @staticmethod
# def registry_name():
# return "orangecontrib.est.widgets.utils.saving_point.SavingPointOW"
class IgnoreE0Calculation(IgnoreTask):
@staticmethod
def registry_name():
return "ignored E0 calculation"
pass
# @staticmethod
# def registry_name():
# return "orangecontrib.est.widgets.utils.e0calculator.E0calculatorOW"
......@@ -32,22 +32,19 @@ from .process import Process
from est.core.types import XASObject
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from est.io.utils.information import InputInformation
import pkg_resources
class DumpXasObject(Process):
inputs = [
_input_desc(name="xas_obj", type=XASObject, handler="process", doc=""),
]
INPUT_NAMES = ["xas_obj"]
outputs = [
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
OUTPUT_NAMES = ["result"]
def __init__(self):
super().__init__(name="dump xas object")
self._output_file = None
def __init__(self, varinfo=None, **inputs):
super().__init__(name="dump xas object", varinfo=varinfo, **inputs)
self._output_file = inputs.get("output_file", None)
@property
def output_file(self):
......@@ -57,8 +54,7 @@ class DumpXasObject(Process):
def output_file(self, output_file):
self._output_file = output_file
@staticmethod
def registry_name():
def program_name(self):
"""Name of the program used for this processing"""
return "xas writer"
......@@ -70,7 +66,9 @@ class DumpXasObject(Process):
"""definition of the process"""
return "write xas object to a file"
def process(self, xas_object):
def process(self, xas_object=None):
if xas_object is None:
xas_object = self.input.xas_obj.value
if isinstance(xas_object, dict):
xas_object = XASObject.from_dict(xas_object)
if not isinstance(xas_object, XASObject):
......@@ -82,6 +80,7 @@ class DumpXasObject(Process):
raise ValueError("output file not provided")
else:
xas_object.dump(self.output_file)
self.output.result.value = self.output_file
def set_properties(self, properties):
self.setConfiguration(properties)
......@@ -101,3 +100,21 @@ class DumpXasObject(Process):
self.output_file = self._settings["_output_file_setting"]
__call__ = process
class ReadXasObject(Process):
OUTPUT_NAMES = ["xas_obj"]
def __init__(self, varinfo=None, **inputs):
super().__init__(name="read xas", varinfo=varinfo, **inputs)
self.setConfiguration(inputs)
def process(self):
from est.core.io import XASReader
config = self.getConfiguration()
input_information = InputInformation.from_dict(config)
xas_obj = XASReader.read_frm_url(input_information)
self.output.xas_obj.value = xas_obj.to_dict()
return xas_obj
......@@ -136,22 +136,20 @@ _USE_MULTIPROCESSING_POOL = False
class Larch_autobk(Process):
inputs = [
_input_desc(name="xas_obj", type=XASObject, handler="process", doc=""),
]
INPUT_NAMES = ["xas_obj"]
outputs = [
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
OUTPUT_NAMES = ["xas_obj"]
def __init__(self):
Process.__init__(self, name="autobk")
def __init__(self, varinfo=None, **inputs):
Process.__init__(self, name="autobk", varinfo=varinfo, **inputs)
def set_properties(self, properties):
if "_larchSettings" in properties:
self._settings = properties["_larchSettings"]
def process(self, xas_obj):
def process(self, xas_obj=None):
if xas_obj is None:
xas_obj = self.input.xas_obj.value
_xas_obj = self.getXasObject(xas_obj=xas_obj)
if self._settings:
_xas_obj.configuration["autobk"] = self._settings
......@@ -161,6 +159,7 @@ class Larch_autobk(Process):
self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess()
self.register_process(_xas_obj, data_keys=(_NexusDatasetDef("bkg"),))
self.output.xas_obj.value = _xas_obj.to_dict()
return _xas_obj
def _pool_process(self, xas_obj):
......@@ -207,7 +206,7 @@ class Larch_autobk(Process):
return larch.version.version_data()["larch"]
@staticmethod
def registry_name():
def program_name():
return "larch_autobk"
__call__ = process
......@@ -127,22 +127,20 @@ _USE_MULTIPROCESSING_POOL = False
class Larch_mback(Process):
inputs = [
_input_desc(name="xas_obj", type=XASObject, handler="process", doc=""),
]
INPUT_NAMES = ["xas_obj"]
outputs = [
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
OUTPUT_NAMES = ["xas_obj"]
def __init__(self):
Process.__init__(self, name="mback")
def __init__(self, varinfo=None, **inputs):
Process.__init__(self, name="mback", varinfo=varinfo, **inputs)
def set_properties(self, properties):
if "_larchSettings" in properties:
self._settings = properties["_larchSettings"]
def process(self, xas_obj):
def process(self, xas_obj=None):
if xas_obj is None:
xas_obj = self.input.xas_obj.value
_xas_obj = self.getXasObject(xas_obj=xas_obj)
if self._settings:
_xas_obj.configuration["mback"] = self._settings
......@@ -206,7 +204,7 @@ class Larch_mback(Process):
return larch.version.version_data()["larch"]
@staticmethod
def registry_name():
def program_name():
return "larch_mback"
__call__ = process
......@@ -144,8 +144,8 @@ class Larch_mback_norm(Process):
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
def __init__(self):
Process.__init__(self, name="mback_norm")
def __init__(self, varinfo=None, **inputs):
Process.__init__(self, name="mback_norm", varinfo=varinfo, **inputs)
def set_properties(self, properties):
if "_larchSettings" in properties:
......
......@@ -132,23 +132,23 @@ _USE_MULTIPROCESSING_POOL = False
class Larch_pre_edge(Process):
inputs = [
_input_desc(name="xas_obj", type=XASObject, handler="process", doc=""),
]
INPUT_NAMES = ["xas_obj"]
outputs = [
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
OUTPUT_NAMES = ["xas_obj"]
def __init__(self):
Process.__init__(self, name="pre_edge")
def __init__(self, varinfo=None, **inputs):
Process.__init__(self, name="pre_edge", varinfo=varinfo, **inputs)
def set_properties(self, properties):
if "_larchSettings" in properties:
self._settings = properties["_larchSettings"]
def process(self, xas_obj):
_xas_obj = self.getXasObject(xas_obj=xas_obj)
def process(self, xas_obj=None):
if xas_obj is None:
xas_obj = self.input.xas_obj.value
self._xas_obj = self.getXasObject(xas_obj=xas_obj)
assert isinstance(self._xas_obj, XASObject)
_xas_obj = self._xas_obj
if self._settings:
_xas_obj.configuration["pre_edge"] = self._settings
......@@ -216,6 +216,7 @@ class Larch_pre_edge(Process):
),
),
)
self.output.xas_obj.value = _xas_obj.to_dict()
return _xas_obj
def _pool_process(self, xas_obj):
......
......@@ -159,16 +159,12 @@ _USE_MULTIPROCESSING_POOL = False
class Larch_xftf(Process):
inputs = [
_input_desc(name="xas_obj", type=XASObject, handler="process", doc=""),
]
INPUT_NAMES = ["xas_obj"]
outputs = [
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
OUTPUT_NAMES = ["xas_obj"]
def __init__(self):
Process.__init__(self, name="xftf")
def __init__(self, varinfo=None, **inputs):
Process.__init__(self, name="xftf", varinfo=varinfo, **inputs)
self._plot_settings = {}
def set_properties(self, properties):
......@@ -177,7 +173,9 @@ class Larch_xftf(Process):
if "plot_settings" in properties:
self._plot_settings = properties["plot_settings"]
def process(self, xas_obj):
def process(self, xas_obj=None):
if xas_obj is None:
xas_obj = self.input.xas_obj.value
_xas_obj = self.getXasObject(xas_obj=xas_obj)
if self._settings:
_xas_obj.configuration["xftf"] = self._settings
......@@ -249,6 +247,7 @@ class Larch_xftf(Process):
),
),
)
self.output.xas_obj.value = _xas_obj.to_dict()
return _xas_obj
def _pool_process(self, xas_obj):
......
......@@ -159,8 +159,8 @@ class NoiseProcess(Process):
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
def __init__(self):
Process.__init__(self, name="noise")
def __init__(self, varinfo=varinfo, **inputs):
Process.__init__(self, name="noise", varinfo=varinfo, **inputs)
self._window_size = 5
self._polynomial_order = 2
......@@ -253,7 +253,7 @@ class NoiseProcess(Process):
return pkg_resources.get_distribution("est").version
@staticmethod
def registry_name():
def program_name():
return "noise_savgol"
__call__ = process
......@@ -193,10 +193,11 @@ class _NexusDatasetDef:
class Process(Task):
def __init__(self, name):
assert type(name) is str
def __init__(self, name, varinfo, **inputs):
super().__init__(varinfo=varinfo, **inputs)
self._name = name
self._advancement = Progress(name=name)
assert type(self._name) is str
self._advancement = Progress(name=self.name)
self.__stop = False
"""flag to notice when a end of process is required"""
self._settings = {}
......@@ -235,6 +236,9 @@ class Process(Task):
assert isinstance(_xas_obj, XASObject)
return _xas_obj
def program_name(self):
raise NotImplementedError("Base class")
def program_version(self) -> str:
"""version of the program used for this processing"""
raise NotImplementedError("Base class")
......
......@@ -149,22 +149,20 @@ class PyMca_exafs(Process):
"""Process spectra for exafs and get information about the processing
advancement"""
inputs = [
_input_desc(name="xas_obj", type=XASObject, handler="process", doc=""),
]
INPUT_NAMES = ["xas_obj"]
outputs = [
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
OUTPUT_NAMES = ["xas_obj"]
def __init__(self):
Process.__init__(self, name="exafs")
def __init__(self, varinfo=None, **inputs):
Process.__init__(self, name="exafs", varinfo=varinfo, **inputs)
def set_properties(self, properties):
if "_pymcaSettings" in properties:
self.setConfiguration(properties["_pymcaSettings"])
def process(self, xas_obj):
def process(self, xas_obj=None):
if xas_obj is None:
xas_obj = self.input.xas_obj.value
_xas_obj = self.getXasObject(xas_obj=xas_obj)
if self._settings:
_xas_obj.configuration["EXAFS"] = self._settings
......@@ -180,6 +178,7 @@ class PyMca_exafs(Process):
_NexusDatasetDef("EXAFSSignal"),
),
)
self.output.xas_obj.value = _xas_obj.to_dict()
return _xas_obj
def _pool_process(self, xas_obj):
......
......@@ -152,22 +152,18 @@ _USE_MULTIPROCESSING_POOL = False
class PyMca_ft(Process):
inputs = [
_input_desc(name="xas_obj", type=XASObject, handler="process", doc=""),
]
INPUT_NAMES = ["xas_obj"]
outputs = [
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
OUTPUT_NAMES = ["xas_obj"]
def __init__(self):
Process.__init__(self, name="ft")
def __init__(self, varinfo=None, **inputs):
Process.__init__(self, name="ft", varinfo=varinfo, **inputs)
def set_properties(self, properties):
if "_pymcaSettings" in properties:
self._settings = properties["_pymcaSettings"]
def process(self, xas_obj):
def process(self, xas_obj=None):
"""
:param xas_obj: object containing the configuration and spectra to process
......@@ -175,6 +171,8 @@ class PyMca_ft(Process):
:return: spectra dict
:rtype: dict
"""
if xas_obj is None:
xas_obj = self.input.xas_obj.value
_xas_obj = self.getXasObject(xas_obj=xas_obj)
if self._settings:
_xas_obj.configuration["FT"] = self._settings
......@@ -194,6 +192,7 @@ class PyMca_ft(Process):
_NexusDatasetDef("ft.imaginary"),
),
)
self.output.xas_obj.value = _xas_obj.to_dict()
return _xas_obj
def _pool_process(self, xas_obj):
......
......@@ -141,16 +141,12 @@ _USE_MULTIPROCESSING_POOL = False
class PyMca_k_weight(Process):
inputs = [
_input_desc(name="xas_obj", type=XASObject, handler="process", doc=""),
]
INPUT_NAMES = ["xas_obj"]
outputs = [
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
OUTPUT_NAMES = ["xas_obj"]
def __init__(self):
Process.__init__(self, name="k weight")
def __init__(self, varinfo=None, **inputs):
Process.__init__(self, name="k weight", varinfo=varinfo, **inputs)
def set_properties(self, properties):
if "_kWeightSetting" in properties:
......@@ -164,7 +160,7 @@ class PyMca_k_weight(Process):
return self._settings["k_weight"]
return None
def process(self, xas_obj):
def process(self, xas_obj=None):
"""
:param xas_obj: object containing the configuration and spectra to process
......@@ -172,7 +168,8 @@ class PyMca_k_weight(Process):
:return: spectra dict
:rtype: :class:`.XASObject`
"""
assert xas_obj is not None
if xas_obj is None:
xas_obj = self.input.xas_obj.value
_xas_obj = self.getXasObject(xas_obj=xas_obj)
if self._k_weight() is not None:
_xas_obj.configuration["SET_KWEIGHT"] = self._k_weight()
......@@ -202,6 +199,7 @@ class PyMca_k_weight(Process):
self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess()
self.register_process(xas_obj=_xas_obj, data_keys=tuple())
self.output.xas_obj.value = _xas_obj.to_dict()
return _xas_obj
def _pool_process(self, xas_obj):
......
......@@ -126,23 +126,19 @@ _USE_MULTIPROCESSING_POOL = False
class PyMca_normalization(Process):
inputs = [
_input_desc(name="xas_obj", type=XASObject, handler="process", doc=""),
]
INPUT_NAMES = ["xas_obj"]
outputs = [
_output_desc(name="xas_obj", type=XASObject, doc=""),
]
OUTPUT_NAMES = ["xas_obj"]
def __init__(self):
Process.__init__(self, "normalization")
def __init__(self, varinfo=None, **inputs):
Process.__init__(self, "normalization", varinfo=varinfo, **inputs)
self._advancement = Progress(self.name)
def set_properties(self, properties):
if "_pymcaSettings" in properties:
self._settings = properties["_pymcaSettings"]
def process(self, xas_obj):
def process(self, xas_obj=None):
"""
:param xas_obj: object containing the configuration and spectra to process
......@@ -151,6 +147,8 @@ class PyMca_normalization(Process):
:return: updated XASObject
:rtype: :class:`.XASObject`
"""
if xas_obj is None:
xas_obj = self.input.xas_obj.value
_xas_obj = self.getXasObject(xas_obj)
if _xas_obj.energy is None:
......@@ -159,11 +157,11 @@ class PyMca_normalization(Process):
if self._settings:
_xas_obj.configuration["Normalization"] = self._settings
self._advancement.reset(max_=_xas_obj.n_spectrum)
self._advancement.startProcess()
self._pool_process(xas_obj=_xas_obj)
self._advancement.endProcess()
assert _xas_obj.normalized_energy is not None
self.register_process(
_xas_obj,
data_keys=(
......@@ -172,6 +170,7 @@ class PyMca_normalization(Process):
_NexusDatasetDef("NormalizedSignal"),
),
)
self.output.xas_obj.value = _xas_obj.to_dict()
return _xas_obj
def _pool_process(self, xas_obj):
......@@ -236,7 +235,6 @@ if __name__ == "__main__":
with open(xas_object_yaml_file, "r") as file:
ddict = yaml.load(file)["input_data"]
xas_object = XASObject.from_dict(ddict)
print("******* do normalization ********")
res_xas_object = pymca_normalization(xas_obj=xas_object)
res_xas_object._create_saving_pt()
......
......@@ -99,8 +99,8 @@ class ROIProcess(Process):
_output_desc(name="xas_obj", type=XASObject, doc=""),
]