Commit e9e8b58a authored by payno's avatar payno
Browse files

[app] add ows-to-script function: allows to convert a .owd file to a python script.

- add `outputs` and `inputs` entries in processing classes used in ppfaddon aliases file.
- add `get_xas_obj` function in est.app.utils module
- mode est.app.utils._convert_spectra_dims to public
parent ea35188f
......@@ -80,6 +80,12 @@ def main():
launcher.add_command(
"test", module_name="est.app.test_", description="Launch est unittest"
)
launcher.add_command(
"ows-to-script",
module_name="est.app.ows_to_script",
description="Convert a .ows file to python script",
)
status = launcher.execute(sys.argv)
return status
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import logging
import sys
import argparse
from pypushflow.representation.scheme.ows_parser import OwsParser
from pypushflow import Workflow
import est.version
import pypushflow.version
import os
logging.basicConfig(level=logging.DEBUG)
_logger = logging.getLogger(__name__)
def _convert(scheme, output_file, overwrite):
"""
:param scheme:
:param scan:
:param timeout:
:return:
"""
_logger.warning("translate {} to {}".format(scheme, output_file))
if os.path.exists(output_file):
if overwrite is True:
os.remove(output_file)
else:
raise ValueError("{} already exists.".format(output_file))
with open(output_file, "w+") as file_:
file_.write(_dump_info_generation())
workflow = Workflow.ProcessableWorkflow(scheme)
converter = Workflow.Converter(workflow=workflow, output_file=output_file)
converter.process()
# set up workflow
with open(output_file, mode="a") as file_:
file_.write(_dump_executable_script_section())
_logger.info(
"translation finished. You can execute python {} [[--input --input-spectra --input-spectra-dims --input-channel --input-energy-unit --input-dimensions]]".format(
output_file
)
)
def _dump_info_generation():
return (
"# This file has been generated automatically using \n"
"# pypushflow {} and est {}\n".format(
pypushflow.version.version, est.version.version
)
)
def _dump_executable_script_section():
return """
from pypushflow.utils import IgnoreProcess
if __name__ == '__main__':
import sys
import argparse
from est.app.utils import get_xas_obj
from est.app.utils import get_url
from est.app.utils import get_unit
from est.app.utils import convert_spectra_dims
from silx.io.url import DataUrl
parser = argparse.ArgumentParser(description=__doc__)
# single file input option
parser.add_argument(
"-i",
"--input",
dest="input_",
default=None,
help="Input of the workflow. Should be a path to a file",
)
# input url option
parser.add_argument(
"--input-spectra",
dest="input_spectra",
default=None,
help="Input spectra url",
)
parser.add_argument(
"--input-spectra-dims",
dest="input_spectra_dims",
default=None,
help="Input spectra dimension. Should be a tuple of three values: "
"(X,Y,channel). If None will take the default dimension "
"according to the input type.",
)
parser.add_argument(
"--input-channel",
dest="input_channel",
default=None,
help="Input channel url (usually energy)",
)
parser.add_argument(
"--input-configuration",
dest="input_configuration",
default=None,
help="Input configuration url",
)
parser.add_argument(
"--input-energy-unit",
dest="input_energy_unit",
default="eV",
help="energy unit",
)
parser.add_argument(
"--input-dimensions",
dest="input_dimensions",
default="None",
help="dimension of the input as (channel, Y, X) for example."
"If None will take default unit according to the input type",
)
options = parser.parse_args(sys.argv[1:])
xas_obj = get_xas_obj(input_energy_unit=get_unit(options.input_energy_unit),
input_=options.input_,
input_spectra_url=get_url(options.input_spectra),
input_dims=convert_spectra_dims(options.input_spectra_dims),
input_channel_url=get_url(options.input_channel),
input_configuration_url=get_url(options.input_configuration)
)
main(input_data=xas_obj, channel="xas_obj")
"""
def main(argv):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"workflow_file",
help="Path to the .ows file defining the workflow to process with the"
"provided scan",
)
parser.add_argument("output_file", help="Output python file")
parser.add_argument(
"--overwrite",
help="Overwrite output file if exists",
default=False,
action="store_true",
)
parser.add_argument(
"--debug",
dest="debug",
action="store_true",
default=False,
help="Set logging system in debug mode",
)
options = parser.parse_args(argv[1:])
if not options.output_file.lower().endswith(".py"):
options.output_file = options.output_file + ".py"
# tune the log level
log_level = logging.INFO
if options.debug is True:
log_level = logging.DEBUG
for log_ in ("est", "pypushflow"):
logging.getLogger(log_).setLevel(log_level)
scheme = OwsParser.scheme_load(options.workflow_file, load_handlers=True)
_convert(
scheme=scheme, output_file=options.output_file, overwrite=options.overwrite
)
if __name__ == "__main__":
main(sys.argv)
# convert an ows file to a script calling est low level processes.
......@@ -2,16 +2,6 @@ import argparse
import sys
from pypushflow.Workflow import ProcessableWorkflow
from pypushflow.representation.scheme.ows_parser import OwsParser
try:
from est.io.utils.pymca import read_spectrum
has_read_spectrum = True
except ImportError:
has_read_spectrum = False
from est.core.types import Spectrum, XASObject
from est.core.io import read_xas
from est.core.types import Dim
import logging
import signal
from pypushflow.representation.scheme.scheme import Scheme
......@@ -19,7 +9,8 @@ from silx.io.url import DataUrl
from est.units import ur
from .utils import get_unit
from .utils import get_url
from .utils import _convert_spectra_dims
from .utils import convert_spectra_dims
from .utils import get_xas_obj
try:
import h5py
......@@ -114,28 +105,12 @@ def exec_(
signal.signal(signal.SIGINT, signal_handler)
if input_ is not None:
if input_dims is None:
input_dims = (Dim.X_DIM, Dim.Y_DIM)
if has_read_spectrum:
energy, mu = read_spectrum(
input_, energy_unit=input_energy_unit, dimensions=input_dims
)
spectrum = Spectrum(energy=energy, mu=mu)
xas_obj = XASObject(energy=energy, spectra=(spectrum,), dim1=1, dim2=1)
else:
raise ValueError("Unable to read spectrum")
else:
if input_dims is None:
input_dims = (Dim.CHANNEL_ENERGY_DIM, Dim.Y_DIM, Dim.Y_DIM)
sp, en, conf = read_xas(
spectra_url=input_spectra_url,
channel_url=input_channel_url,
config_url=input_configuration_url,
dimensions=input_dims,
energy_unit=input_energy_unit,
)
xas_obj = XASObject(spectra=sp, energy=en, configuration=conf)
xas_obj = get_xas_obj(input_energy_unit=input_energy_unit,
input_=input_,
input_spectra_url=input_spectra_url,
input_dims=input_dims,
input_channel_url=input_channel_url,
input_configuration_url=input_configuration_url)
workflow._start_actor.trigger(("data", xas_obj.to_dict()))
workflow._end_actor.join(timeout)
......@@ -215,7 +190,7 @@ def main(argv):
scheme=scheme,
input_=options.input_,
input_spectra_url=get_url(options.input_spectra),
input_dims=_convert_spectra_dims(options.input_spectra_dims),
input_dims=convert_spectra_dims(options.input_spectra_dims),
input_channel_url=get_url(options.input_channel),
input_configuration_url=get_url(options.input_configuration),
output_=options.output_,
......
......@@ -31,9 +31,18 @@ __date__ = "10/12/2020"
from silx.io.url import DataUrl
from est.units import ur
from est.core.types import Dim
from typing import Union
from est.core.types import Spectrum, XASObject
from est.core.io import read_xas
try:
from est.io.utils.pymca import read_spectrum
has_read_spectrum = True
except ImportError:
has_read_spectrum = False
def _convert_spectra_dims(dims):
def convert_spectra_dims(dims):
"""
Convert a tuple of dims that can be strings... to a tuple of
est.core.types.Dim
......@@ -96,3 +105,34 @@ def get_url(my_str):
else:
return url
raise ValueError("unrecognized url {}".format(my_str))
def get_xas_obj(input_energy_unit=ur.eV, input_: Union[str, None, dict] = None,
input_spectra_url: Union[None, DataUrl] = None,
input_dims: Union[None, tuple] = None,
input_channel_url: Union[None, DataUrl] = None,
input_configuration_url: Union[None, DataUrl] = None):
"""load xas object from command line input"""
if input_ is not None:
if input_dims is None:
input_dims = (Dim.X_DIM, Dim.Y_DIM)
if has_read_spectrum:
energy, mu = read_spectrum(
input_, energy_unit=input_energy_unit, dimensions=input_dims
)
spectrum = Spectrum(energy=energy, mu=mu)
return XASObject(energy=energy, spectra=(spectrum,), dim1=1,
dim2=1)
else:
raise ValueError("Unable to read spectrum")
else:
if input_dims is None:
input_dims = (Dim.CHANNEL_ENERGY_DIM, Dim.Y_DIM, Dim.Y_DIM)
sp, en, conf = read_xas(
spectra_url=input_spectra_url,
channel_url=input_channel_url,
config_url=input_configuration_url,
dimensions=input_dims,
energy_unit=input_energy_unit,
)
return XASObject(spectra=sp, energy=en, configuration=conf)
......@@ -7,7 +7,7 @@ import sys
from est.gui.xas_object_definition import XASObjectWindow
from .utils import get_unit
from .utils import get_url
from .utils import _convert_spectra_dims
from .utils import convert_spectra_dims
from silx.gui import qt
from est.core.types import Dim
......@@ -80,7 +80,7 @@ def main(argv):
_plot(
input_=options.input_,
input_spectra_url=get_url(options.input_spectra),
input_spectra_dims=_convert_spectra_dims(options.input_spectra_dims),
input_spectra_dims=convert_spectra_dims(options.input_spectra_dims),
input_channel_url=get_url(options.input_channel),
input_energy_unit=get_unit(options.input_energy_unit),
)
......
......@@ -30,10 +30,25 @@ __date__ = "06/11/2019"
from .process import Process
from est.core.types import XASObject
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
import pkg_resources
class DumpXasObject(Process):
inputs = [
_input_desc(
name="xas_obj", type=XASObject, handler="process", doc=""
),
]
outputs = [
_output_desc(
name="xas_obj", type=XASObject, doc=""
),
]
def __init__(self):
super().__init__(name="dump xas object")
self._output_file = None
......
......@@ -26,6 +26,8 @@
from est.core.types import Spectrum, XASObject
from est.core.process.process import Process
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from larch.xafs.autobk import autobk
import multiprocessing
import functools
......@@ -134,6 +136,19 @@ _USE_MULTIPROCESSING_POOL = False
class Larch_autobk(Process):
inputs = [
_input_desc(
name="xas_obj", type=XASObject, handler="process", doc=""
),
]
outputs = [
_output_desc(
name="xas_obj", type=XASObject, doc=""
),
]
def __init__(self):
Process.__init__(self, name="autobk")
......
......@@ -26,6 +26,8 @@
from est.core.types import Spectrum, XASObject
from est.core.process.process import Process
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from larch.xafs.mback import mback
import multiprocessing
import functools
......@@ -123,6 +125,19 @@ _USE_MULTIPROCESSING_POOL = False
class Larch_mback(Process):
inputs = [
_input_desc(
name="xas_obj", type=XASObject, handler="process", doc=""
),
]
outputs = [
_output_desc(
name="xas_obj", type=XASObject, doc=""
),
]
def __init__(self):
Process.__init__(self, name="mback")
......
......@@ -26,6 +26,8 @@
from est.core.types import Spectrum, XASObject
from est.core.process.process import Process
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from larch.xafs.mback import mback_norm
from larch.xafs.pre_edge import preedge
import multiprocessing
......@@ -132,6 +134,19 @@ _USE_MULTIPROCESSING_POOL = False
class Larch_mback_norm(Process):
inputs = [
_input_desc(
name="xas_obj", type=XASObject, handler="process", doc=""
),
]
outputs = [
_output_desc(
name="xas_obj", type=XASObject, doc=""
),
]
def __init__(self):
Process.__init__(self, name="mback_norm")
......
......@@ -26,6 +26,8 @@
from est.core.types import Spectrum, XASObject
from est.core.process.process import Process
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from larch.xafs.pre_edge import pre_edge
import multiprocessing
import functools
......@@ -121,6 +123,19 @@ _USE_MULTIPROCESSING_POOL = False
class Larch_pre_edge(Process):
inputs = [
_input_desc(
name="xas_obj", type=XASObject, handler="process", doc=""
),
]
outputs = [
_output_desc(
name="xas_obj", type=XASObject, doc=""
),
]
def __init__(self):
Process.__init__(self, name="pre_edge")
......
......@@ -26,6 +26,8 @@
from est.core.types import Spectrum, XASObject
from est.core.process.process import Process
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from larch.xafs.xafsft import xftf
import multiprocessing
import functools
......@@ -124,6 +126,19 @@ _USE_MULTIPROCESSING_POOL = False
class Larch_xftf(Process):
inputs = [
_input_desc(
name="xas_obj", type=XASObject, handler="process", doc=""
),
]
outputs = [
_output_desc(
name="xas_obj", type=XASObject, doc=""
),
]
def __init__(self):
Process.__init__(self, name="xftf")
......
......@@ -32,16 +32,16 @@ __date__ = "07/08/2019"
from .progress import Progress
from est.core.types import XASObject
import logging
import pint
from collections import namedtuple
from typing import Iterable
_logger = logging.getLogger(__name__)
_DEBUG = True
if _DEBUG:
from est.core.types import Spectrum
import numpy
_input_desc = namedtuple("_input_desc", ["name", "type", "handler", "doc"])
_output_desc = namedtuple("_output_desc", ["name", "type", "doc"])
class Process(object):
......
......@@ -34,7 +34,8 @@ import multiprocessing
import numpy
from PyMca5.PyMcaPhysics.xas.XASClass import XASClass
from PyMca5.PyMcaPhysics.xas.XASClass import e2k, k2e
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from est.core.process.process import Process
from est.core.types import XASObject, Spectrum
......@@ -147,6 +148,19 @@ class PyMca_exafs(Process):
"""Process spectra for exafs and get information about the processing
advancement"""
inputs = [
_input_desc(
name="xas_obj", type=XASObject, handler="process", doc=""
),
]
outputs = [
_output_desc(
name="xas_obj", type=XASObject, doc=""
),
]
def __init__(self):
Process.__init__(self, name="exafs")
......
......@@ -33,7 +33,8 @@ import multiprocessing
import numpy
from PyMca5.PyMcaPhysics.xas.XASClass import XASClass
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from est.core.process.process import Process
from est.core.types import XASObject, Spectrum
......@@ -149,6 +150,19 @@ _USE_MULTIPROCESSING_POOL = False
class PyMca_ft(Process):
inputs = [
_input_desc(
name="xas_obj", type=XASObject, handler="process", doc=""
),
]
outputs = [
_output_desc(
name="xas_obj", type=XASObject, doc=""
),
]
def __init__(self):
Process.__init__(self, name="ft")
......
......@@ -32,7 +32,8 @@ import logging
import multiprocessing
from PyMca5.PyMcaPhysics.xas.XASClass import XASClass, e2k
from est.core.process.process import _input_desc
from est.core.process.process import _output_desc
from est.core.process.process import Process
from est.core.process.pymca.exafs import process_spectr_exafs
from est.core.types import XASObject, Spectrum
......@@ -139,6 +140,19 @@ _USE_MULTIPROCESSING_POOL = False
class PyMca_k_weight(Process):
inputs = [
_input_desc(
name="xas_obj", type=XASObject, handler="process", doc=""
),
]
outputs = [
_output_desc(
name="xas_obj", type=XASObject, doc=""
),
]
def __init__(