From 2ca166642576ac622272499d805776b3a22a4de0 Mon Sep 17 00:00:00 2001 From: payno Date: Wed, 9 Dec 2020 13:30:04 +0100 Subject: [PATCH 1/6] [exec] add a warning if `output` option is ignored --- est/app/process.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/est/app/process.py b/est/app/process.py index 64e87fd..ab6145b 100644 --- a/est/app/process.py +++ b/est/app/process.py @@ -44,10 +44,17 @@ def exec_( def manage_output(): if output_: + found_output = False # monkey patch the input file for start nodes if an input is given for node in scheme.nodes: if node.properties and "_output_file_setting" in node.properties: node.properties["_output_file_setting"] = output_ + found_output = True + if not found_output: + _logger.warning( + "No node for processing output found. output " + "information provided will be ignored" + ) # TODO: if the input file contains configuration, then overwrite the node # properties... -- GitLab From c7dad41fd67c0f0fb9b985cd312ea973aa03b938 Mon Sep 17 00:00:00 2001 From: payno Date: Wed, 9 Dec 2020 13:34:10 +0100 Subject: [PATCH 2/6] [app][process] add aliases for -i and -o options --- est/app/process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/est/app/process.py b/est/app/process.py index ab6145b..90c8dcd 100644 --- a/est/app/process.py +++ b/est/app/process.py @@ -101,13 +101,13 @@ def main(argv): help="Workflow file providing the workflow description (.ows, .xml)", ) parser.add_argument( - "-i", + "-i", "--input", dest="input_", default=None, help="Input of the workflow. require at most one " "instance of XASInputOW", ) parser.add_argument( - "-o", + "-o", "--output", dest="output_", default=None, help="Output file of the workflow. Require at most one " -- GitLab From a2ed634e149d3a7a0f9a244bf7e28051a3db54b4 Mon Sep 17 00:00:00 2001 From: payno Date: Wed, 9 Dec 2020 14:01:58 +0100 Subject: [PATCH 3/6] [app][exec] move manage_input and manage_output --- est/app/process.py | 69 ++++++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/est/app/process.py b/est/app/process.py index 90c8dcd..1620339 100644 --- a/est/app/process.py +++ b/est/app/process.py @@ -27,38 +27,54 @@ logging.basicConfig(level=logging.DEBUG) _logger = logging.getLogger(__name__) +def _insert_input_in_scheme(scheme, input_): + """update 'starting' node properties to include the provided input""" + # monkey patch the input file for start nodes if an input is given + for node in scheme.nodes: + if node.properties and "_input_file_setting" in node.properties: + if input_: + node.properties["_input_file_setting"] = input_ + + +def _insert_output_in_scheme(scheme, output_): + """update 'starting' node properties to include the provided input""" + found_output = False + # monkey patch the input file for start nodes if an input is given + for node in scheme.nodes: + if node.properties and "_output_file_setting" in node.properties: + node.properties["_output_file_setting"] = output_ + found_output = True + if not found_output: + _logger.warning( + "No node for processing output found. output " + "information provided will be ignored" + ) + + def exec_( scheme: Scheme, input_: Union[str, None, dict] = None, + input_spectra_url: Union[None, DataUrl] = None, + input_spectra_dims: Union[None, tuple] = None, + input_channel_url: Union[None, DataUrl] = None, + input_configuration_url: Union[None, DataUrl] = None, output_: Union[str, None, dict] = None, timeout: Union[int, None] = None, ): - def manage_input(): - if input_: - # monkey patch the input file for start nodes if an input is given - for node in scheme.nodes: - if node.properties and "_input_file_setting" in node.properties: - node.properties["_input_file_setting"] = input_ - - manage_input() - - def manage_output(): - if output_: - found_output = False - # monkey patch the input file for start nodes if an input is given - for node in scheme.nodes: - if node.properties and "_output_file_setting" in node.properties: - node.properties["_output_file_setting"] = output_ - found_output = True - if not found_output: - _logger.warning( - "No node for processing output found. output " - "information provided will be ignored" - ) - - # TODO: if the input file contains configuration, then overwrite the node - # properties... - manage_output() + + has_url_information = ( + input_spectra_url + or input_spectra_dims + or input_channel_url + or input_configuration_url + ) + if input_ is not None and has_url_information: + raise ValueError("You cannot provide an input file and input urls") + + if input_ is not None or has_url_information: + _insert_input_in_scheme(scheme=scheme, input_=input_) + if output_ is not None: + _insert_output_in_scheme(scheme=scheme, output_=output_) workflow = ProcessableWorkflow(scheme=scheme) @@ -70,7 +86,6 @@ def exec_( signal.signal(signal.SIGINT, signal_handler) - # TODO: handle several entries from url on top of .dat file... using read_xas if has_read_spectrum: energy, mu = read_spectrum(input_) spectrum = Spectrum(energy=energy, mu=mu) -- GitLab From c9e06585d5d55b50c07ba45966c53c8776206984 Mon Sep 17 00:00:00 2001 From: payno Date: Wed, 9 Dec 2020 14:02:14 +0100 Subject: [PATCH 4/6] [app][process] add entries for input urls --- est/app/process.py | 47 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/est/app/process.py b/est/app/process.py index 1620339..05fdbd2 100644 --- a/est/app/process.py +++ b/est/app/process.py @@ -11,9 +11,11 @@ except ImportError: has_read_spectrum = False from est.core.types import Spectrum, XASObject from est.core.io import read_xas +from est.core.types import Dim import logging import signal from pypushflow.representation.scheme.scheme import Scheme +from silx.io.url import DataUrl try: import h5py @@ -115,14 +117,43 @@ def main(argv): "workflow_file", help="Workflow file providing the workflow description (.ows, .xml)", ) + # single file input option parser.add_argument( - "-i", "--input", + "-i", + "--input", dest="input_", default=None, - help="Input of the workflow. require at most one " "instance of XASInputOW", + help="Input of the workflow. Should be a path to a file", ) + # input url option parser.add_argument( - "-o", "--output", + "--input-spectra", + dest="input_spectra", + default=None, + help="Data url to spectra", + ) + parser.add_argument( + "--input-spectra-dims", + dest="input_spectra_dims", + default=(Dim.X_DIM.value, Dim.Y_DIM.value, Dim.CHANNEL_ENERGY_DIM.value), + help="Data url to spectra", + ) + parser.add_argument( + "--input-channel", + dest="input_channel", + default=None, + help="Data url to channel", + ) + parser.add_argument( + "--input-configuration", + dest="input_configuration", + default=None, + help="Data url to configuration", + ) + # output option + parser.add_argument( + "-o", + "--output", dest="output_", default=None, help="Output file of the workflow. Require at most one " @@ -131,7 +162,15 @@ def main(argv): options = parser.parse_args(argv[1:]) scheme = OwsParser.scheme_load(options.workflow_file, load_handlers=True) - exec_(scheme, options.input_, options.output_) + exec_( + scheme=scheme, + input_=options.input_, + input_spectra_url=options.input_spectra, + input_spectra_dims=options.input_spectra_dims, + input_channel_url=options.input_channel, + input_configuration_url=options.input_configuration, + output_=options.output_, + ) if __name__ == "__main__": -- GitLab From 8b913d3413cbe0c1213820a356ac74ecd4d43bcc Mon Sep 17 00:00:00 2001 From: payno Date: Wed, 9 Dec 2020 14:25:36 +0100 Subject: [PATCH 5/6] [app][process] handle input provided from url --- est/app/process.py | 65 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/est/app/process.py b/est/app/process.py index 05fdbd2..3e2d19e 100644 --- a/est/app/process.py +++ b/est/app/process.py @@ -29,13 +29,30 @@ logging.basicConfig(level=logging.DEBUG) _logger = logging.getLogger(__name__) -def _insert_input_in_scheme(scheme, input_): +def _insert_input_in_scheme( + scheme, + input_, + input_spectra_url, + input_spectra_dims, + input_channel_url, + input_configuration_url, +): """update 'starting' node properties to include the provided input""" # monkey patch the input file for start nodes if an input is given for node in scheme.nodes: if node.properties and "_input_file_setting" in node.properties: - if input_: + if input_ is not None: node.properties["_input_file_setting"] = input_ + if input_spectra_url is not None: + node.properties["_spectra_url_setting"] = input_spectra_url.path() + if input_spectra_dims is not None: + node.properties["_dimensions_setting"] = input_spectra_dims + if input_channel_url is not None: + node.properties["_energy_url_setting"] = input_channel_url.path() + if input_configuration_url is not None: + node.properties[ + "_configuration_url_setting" + ] = input_configuration_url.path() def _insert_output_in_scheme(scheme, output_): @@ -53,6 +70,32 @@ def _insert_output_in_scheme(scheme, output_): ) +def _convert_spectra_dims(dims): + """ + Convert a tuple of dims that can be strings... to a tuple of + est.core.types.Dim + """ + res = [] + for dim in dims: + if dim in ("energy", "channel"): + dim = Dim.CHANNEL_ENERGY_DIM + if isinstance(dim, str): + if dim.lower() == Dim.X_DIM.value.lower(): + dim = Dim.X_DIM + elif dim.lower() == Dim.Y_DIM.value.lower(): + dim = Dim.Y_DIM + + dim = Dim.from_value(dim) + if dim in res: + raise ValueError( + "dimension {} has been provided several time." + "Each dimension should be set once.".format(dim.value()) + ) + else: + res.append(dim) + return tuple(dims) + + def exec_( scheme: Scheme, input_: Union[str, None, dict] = None, @@ -65,16 +108,20 @@ def exec_( ): has_url_information = ( - input_spectra_url - or input_spectra_dims - or input_channel_url - or input_configuration_url + input_spectra_url or input_channel_url or input_configuration_url ) if input_ is not None and has_url_information: raise ValueError("You cannot provide an input file and input urls") if input_ is not None or has_url_information: - _insert_input_in_scheme(scheme=scheme, input_=input_) + _insert_input_in_scheme( + scheme=scheme, + input_=input_, + input_spectra_url=input_spectra_url, + input_spectra_dims=input_spectra_dims, + input_channel_url=input_channel_url, + input_configuration_url=input_configuration_url, + ) if output_ is not None: _insert_output_in_scheme(scheme=scheme, output_=output_) @@ -136,7 +183,7 @@ def main(argv): "--input-spectra-dims", dest="input_spectra_dims", default=(Dim.X_DIM.value, Dim.Y_DIM.value, Dim.CHANNEL_ENERGY_DIM.value), - help="Data url to spectra", + help="spectra dimension. Should be a tuple of three values: (X,Y,channel)", ) parser.add_argument( "--input-channel", @@ -166,7 +213,7 @@ def main(argv): scheme=scheme, input_=options.input_, input_spectra_url=options.input_spectra, - input_spectra_dims=options.input_spectra_dims, + input_spectra_dims=_convert_spectra_dims(options.input_spectra_dims), input_channel_url=options.input_channel, input_configuration_url=options.input_configuration, output_=options.output_, -- GitLab From 225eaf7798931d2b2e0e86f33f3d861d60f6eb1f Mon Sep 17 00:00:00 2001 From: payno Date: Wed, 9 Dec 2020 16:42:29 +0100 Subject: [PATCH 6/6] [app][process] add handling of url, energy unit and dimension --- est/app/process.py | 88 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 20 deletions(-) diff --git a/est/app/process.py b/est/app/process.py index 3e2d19e..fce0608 100644 --- a/est/app/process.py +++ b/est/app/process.py @@ -16,6 +16,7 @@ import logging import signal from pypushflow.representation.scheme.scheme import Scheme from silx.io.url import DataUrl +from est.units import ur try: import h5py @@ -96,8 +97,46 @@ def _convert_spectra_dims(dims): return tuple(dims) +def get_unit(current_unit): + if current_unit == "eV": + return ur.eV + elif current_unit == "keV": + return ur.keV + elif current_unit == "J": + return ur.J + elif current_unit == "kJ": + return ur.kJ + elif isinstance(current_unit, ur.Quantity): + return current_unit + else: + raise ValueError("{} is not a valid unit for quantity".format(current_unit)) + + +def get_url(my_str): + if my_str is None: + return None + else: + assert isinstance(my_str, str) + if "@" in my_str: + try: + entry, file_path = my_str.split("@") + except Exception: + pass + else: + return DataUrl(file_path=file_path, data_path=entry, scheme="silx") + else: + try: + url = DataUrl(path=my_str) + except Exception: + pass + else: + return url + raise ValueError("unrecognized url {}".format(my_str)) + + def exec_( scheme: Scheme, + input_energy_unit=ur.eV, input_: Union[str, None, dict] = None, input_spectra_url: Union[None, DataUrl] = None, input_spectra_dims: Union[None, tuple] = None, @@ -135,12 +174,22 @@ def exec_( signal.signal(signal.SIGINT, signal_handler) - if has_read_spectrum: - energy, mu = read_spectrum(input_) - spectrum = Spectrum(energy=energy, mu=mu) - xas_obj = XASObject(energy=energy, spectra=(spectrum,), dim1=1, dim2=1) + if input_ is not None: + if has_read_spectrum: + energy, mu = read_spectrum(input_, energy_unit=input_energy_unit) + spectrum = Spectrum(energy=energy, mu=mu) + xas_obj = XASObject(energy=energy, spectra=(spectrum,), dim1=1, dim2=1) + else: + raise ValueError("Unable to read spectrum") else: - raise ValueError("Unable to read spectrum") + sp, en, conf = read_xas( + spectra_url=input_spectra_url, + channel_url=input_channel_url, + config_url=input_configuration_url, + dimensions=input_spectra_dims, + energy_unit=input_energy_unit, + ) + xas_obj = XASObject(spectra=sp, energy=en, configuration=conf) workflow._start_actor.trigger(("data", xas_obj.to_dict())) workflow._end_actor.join(timeout) @@ -150,14 +199,6 @@ def exec_( return res -def getinputinfo(): - return "xas process workflow_file_desc.ows [input]" - - -def getInputFrmHdf5(file_path): - raise NotImplementedError() - - def main(argv): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -177,25 +218,31 @@ def main(argv): "--input-spectra", dest="input_spectra", default=None, - help="Data url to spectra", + help="Input spectra url", ) parser.add_argument( "--input-spectra-dims", dest="input_spectra_dims", - default=(Dim.X_DIM.value, Dim.Y_DIM.value, Dim.CHANNEL_ENERGY_DIM.value), + default=(Dim.CHANNEL_ENERGY_DIM.value, Dim.Y_DIM.value, Dim.X_DIM.value), help="spectra dimension. Should be a tuple of three values: (X,Y,channel)", ) parser.add_argument( "--input-channel", dest="input_channel", default=None, - help="Data url to channel", + help="Input channel url (usually energy)", ) parser.add_argument( "--input-configuration", dest="input_configuration", default=None, - help="Data url to configuration", + help="Input configuration url", + ) + parser.add_argument( + "--input-energy-unit", + dest="input_energy_unit", + default="eV", + help="energy unit", ) # output option parser.add_argument( @@ -212,11 +259,12 @@ def main(argv): exec_( scheme=scheme, input_=options.input_, - input_spectra_url=options.input_spectra, + input_spectra_url=get_url(options.input_spectra), input_spectra_dims=_convert_spectra_dims(options.input_spectra_dims), - input_channel_url=options.input_channel, - input_configuration_url=options.input_configuration, + input_channel_url=get_url(options.input_channel), + input_configuration_url=get_url(options.input_configuration), output_=options.output_, + input_energy_unit=get_unit(options.input_energy_unit), ) -- GitLab