Commit 4d3ad24a authored by payno's avatar payno

Merge branch 'fix_29' into 'master'

update xas input

Closes #29, #20, and #28

See merge request workflow/est!31
parents fb00da53 23b572b3
Pipeline #39029 failed with stages
......@@ -223,9 +223,10 @@
"source": [
"from est.core.io import read as read_pymca_xas\n",
"from silx.io.url import DataUrl\n",
"from est.core.types import Dim\n",
"spec_url = DataUrl(file_path=data_file, scheme='PyMca')\n",
"print(spec_url.scheme())\n",
"xas_obj = read_pymca_xas(spectra_url=DataUrl(file_path=data_file, scheme='PyMca'), channel_url=DataUrl(file_path=data_file, scheme='PyMca'))\n",
"xas_obj = read_pymca_xas(spectra_url=DataUrl(file_path=data_file, scheme='PyMca'), channel_url=DataUrl(file_path=data_file, scheme='PyMca'), dimensions=(Dim.CHANNEL_ENERGY_DIM, Dim.X_DIM, Dim.Y_DIM))\n",
"assert 'Mu' in xas_obj.spectra[0]"
]
},
......
......@@ -32,7 +32,7 @@ Your environment should provide a command `est`. You can reach help with
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "06/12/2019"
__date__ = "10/12/2020"
import logging
......@@ -70,7 +70,12 @@ def main():
launcher.add_command(
"process",
module_name="est.app.process",
description="process a workflow from a .xml.ows file" "description",
description="process a workflow from a .xml.ows file description",
)
launcher.add_command(
"xas-viewer",
module_name="est.app.xas_viewer",
description="Open a viewer for browsing xas data",
)
launcher.add_command(
"test", module_name="est.app.test_", description="Launch est unittest"
......
......@@ -17,6 +17,9 @@ import signal
from pypushflow.representation.scheme.scheme import Scheme
from silx.io.url import DataUrl
from est.units import ur
from .utils import get_unit
from .utils import get_url
from .utils import _convert_spectra_dims
try:
import h5py
......@@ -71,75 +74,12 @@ def _insert_output_in_scheme(scheme, output_):
)
def _convert_spectra_dims(dims):
"""
Convert a tuple of dims that can be strings... to a tuple of
est.core.types.Dim
"""
res = []
for dim in dims:
if dim in ("energy", "channel"):
dim = Dim.CHANNEL_ENERGY_DIM
if isinstance(dim, str):
if dim.lower() == Dim.X_DIM.value.lower():
dim = Dim.X_DIM
elif dim.lower() == Dim.Y_DIM.value.lower():
dim = Dim.Y_DIM
dim = Dim.from_value(dim)
if dim in res:
raise ValueError(
"dimension {} has been provided several time."
"Each dimension should be set once.".format(dim.value())
)
else:
res.append(dim)
return tuple(dims)
def get_unit(current_unit):
if current_unit == "eV":
return ur.eV
elif current_unit == "keV":
return ur.keV
elif current_unit == "J":
return ur.J
elif current_unit == "kJ":
return ur.kJ
elif isinstance(current_unit, ur.Quantity):
return current_unit
else:
raise ValueError("{} is not a valid unit for quantity".format(current_unit))
def get_url(my_str):
if my_str is None:
return None
else:
assert isinstance(my_str, str)
if "@" in my_str:
try:
entry, file_path = my_str.split("@")
except Exception:
pass
else:
return DataUrl(file_path=file_path, data_path=entry, scheme="silx")
else:
try:
url = DataUrl(path=my_str)
except Exception:
pass
else:
return url
raise ValueError("unrecognized url {}".format(my_str))
def exec_(
scheme: Scheme,
input_energy_unit=ur.eV,
input_: Union[str, None, dict] = None,
input_spectra_url: Union[None, DataUrl] = None,
input_spectra_dims: Union[None, tuple] = None,
input_dims: Union[None, tuple] = None,
input_channel_url: Union[None, DataUrl] = None,
input_configuration_url: Union[None, DataUrl] = None,
output_: Union[str, None, dict] = None,
......@@ -157,7 +97,7 @@ def exec_(
scheme=scheme,
input_=input_,
input_spectra_url=input_spectra_url,
input_spectra_dims=input_spectra_dims,
input_spectra_dims=input_dims,
input_channel_url=input_channel_url,
input_configuration_url=input_configuration_url,
)
......@@ -175,18 +115,24 @@ def exec_(
signal.signal(signal.SIGINT, signal_handler)
if input_ is not None:
if input_dims is None:
input_dims = (Dim.X_DIM, Dim.Y_DIM)
if has_read_spectrum:
energy, mu = read_spectrum(input_, energy_unit=input_energy_unit)
energy, mu = read_spectrum(
input_, energy_unit=input_energy_unit, dimensions=input_dims
)
spectrum = Spectrum(energy=energy, mu=mu)
xas_obj = XASObject(energy=energy, spectra=(spectrum,), dim1=1, dim2=1)
else:
raise ValueError("Unable to read spectrum")
else:
if input_dims is None:
input_dims = (Dim.CHANNEL_ENERGY_DIM, Dim.Y_DIM, Dim.Y_DIM)
sp, en, conf = read_xas(
spectra_url=input_spectra_url,
channel_url=input_channel_url,
config_url=input_configuration_url,
dimensions=input_spectra_dims,
dimensions=input_dims,
energy_unit=input_energy_unit,
)
xas_obj = XASObject(spectra=sp, energy=en, configuration=conf)
......@@ -223,8 +169,10 @@ def main(argv):
parser.add_argument(
"--input-spectra-dims",
dest="input_spectra_dims",
default=(Dim.CHANNEL_ENERGY_DIM.value, Dim.Y_DIM.value, Dim.X_DIM.value),
help="spectra dimension. Should be a tuple of three values: (X,Y,channel)",
default=None,
help="Input spectra dimension. Should be a tuple of three values: "
"(X,Y,channel). If None will take the default dimension "
"according to the input type.",
)
parser.add_argument(
"--input-channel",
......@@ -244,6 +192,13 @@ def main(argv):
default="eV",
help="energy unit",
)
parser.add_argument(
"--input-dimensions",
dest="input_dimensions",
default="None",
help="dimension of the input as (channel, Y, X) for example."
"If None will take default unit according to the input type",
)
# output option
parser.add_argument(
"-o",
......@@ -260,7 +215,7 @@ def main(argv):
scheme=scheme,
input_=options.input_,
input_spectra_url=get_url(options.input_spectra),
input_spectra_dims=_convert_spectra_dims(options.input_spectra_dims),
input_dims=_convert_spectra_dims(options.input_spectra_dims),
input_channel_url=get_url(options.input_channel),
input_configuration_url=get_url(options.input_configuration),
output_=options.output_,
......
......@@ -37,6 +37,7 @@ from est.core.types import XASObject
import urllib.request
from est.core.types import Spectrum
from ..reprocessing import exec_
from est.core.types import Dim
try:
import PyMca5
......@@ -69,7 +70,7 @@ class TestReprocessingPyMca(unittest.TestCase):
def setUp(self):
self.output_dir = tempfile.mkdtemp()
data_file = os.path.join(PYMCA_DATA_DIR, "EXAFS_Cu.dat")
energy, mu = read_spectrum(data_file)
energy, mu = read_spectrum(data_file, dimensions=(Dim.X_DIM, Dim.Y_DIM))
self.spectrum = Spectrum(energy=energy, mu=mu)
self.xas_obj_ref = XASObject(
spectra=(self.spectrum,), energy=energy, dim1=1, dim2=1
......
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "10/12/2020"
from silx.io.url import DataUrl
from est.units import ur
from est.core.types import Dim
def _convert_spectra_dims(dims):
"""
Convert a tuple of dims that can be strings... to a tuple of
est.core.types.Dim
"""
if dims is None:
return None
res = []
for dim in dims:
if dim in ("energy", "channel"):
dim = Dim.CHANNEL_ENERGY_DIM
if isinstance(dim, str):
if dim.lower() == Dim.X_DIM.value.lower():
dim = Dim.X_DIM
elif dim.lower() == Dim.Y_DIM.value.lower():
dim = Dim.Y_DIM
dim = Dim.from_value(dim)
if dim in res:
raise ValueError(
"dimension {} has been provided several time."
"Each dimension should be set once.".format(dim.value())
)
else:
res.append(dim)
return tuple(dims)
def get_unit(current_unit):
if current_unit == "eV":
return ur.eV
elif current_unit == "keV":
return ur.keV
elif current_unit == "J":
return ur.J
elif current_unit == "kJ":
return ur.kJ
elif isinstance(current_unit, ur.Quantity):
return current_unit
else:
raise ValueError("{} is not a valid unit for quantity".format(current_unit))
def get_url(my_str):
if my_str in (None, ""):
return None
else:
assert isinstance(my_str, str)
if "@" in my_str:
try:
entry, file_path = my_str.split("@")
except Exception:
pass
else:
return DataUrl(file_path=file_path, data_path=entry, scheme="silx")
else:
try:
url = DataUrl(path=my_str)
except Exception:
pass
else:
return url
raise ValueError("unrecognized url {}".format(my_str))
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import argparse
import sys
from est.gui.xas_object_definition import XASObjectWindow
from .utils import get_unit
from .utils import get_url
from .utils import _convert_spectra_dims
from silx.gui import qt
from est.core.types import Dim
def _plot(
input_, input_spectra_url, input_spectra_dims, input_channel_url, input_energy_unit
):
has_url_information = input_spectra_url or input_channel_url
if input_ is not None and has_url_information:
raise ValueError("You cannot provide an input file and input urls")
app = qt.QApplication([])
widget = XASObjectWindow(parent=None)
if input_:
widget.setDatFile(input_)
if input_.lower().endswith(".xmu"):
widget.setCurrentType("*.xmu")
elif input_.lower().endswith(".csv"):
widget.setCurrentType("*.csv")
else:
widget.setCurrentType("*.dat")
else:
widget.setSpectraUrl(input_spectra_url)
widget.setEnergyUrl(input_channel_url)
widget.setCurrentType("*.h5")
widget.setEnergyUnit(input_energy_unit)
widget.setDimensions(input_spectra_dims)
widget.loadXasObject()
widget.show()
app.exec_()
def main(argv):
parser = argparse.ArgumentParser(description=__doc__)
# single file input option
parser.add_argument(
"-i",
"--input",
dest="input_",
default=None,
help="Input of the workflow. Should be a path to a file",
)
# input url option
parser.add_argument(
"--input-spectra",
dest="input_spectra",
default=None,
help="Input spectra url",
)
parser.add_argument(
"--input-spectra-dims",
dest="input_spectra_dims",
default=(Dim.CHANNEL_ENERGY_DIM.value, Dim.Y_DIM.value, Dim.X_DIM.value),
help="spectra dimension. Should be a tuple of three values: (X,Y,channel)",
)
parser.add_argument(
"--input-channel",
dest="input_channel",
default=None,
help="Input channel url (usually energy)",
)
parser.add_argument(
"--input-energy-unit",
dest="input_energy_unit",
default="eV",
help="energy unit",
)
options = parser.parse_args(argv[1:])
_plot(
input_=options.input_,
input_spectra_url=get_url(options.input_spectra),
input_spectra_dims=_convert_spectra_dims(options.input_spectra_dims),
input_channel_url=get_url(options.input_channel),
input_energy_unit=get_unit(options.input_energy_unit),
)
if __name__ == "__main__":
main(sys.argv)
......@@ -46,7 +46,7 @@ DEFAULT_CHANNEL_PATH = "/data/NXdata/Channel"
DEFAULT_CONF_PATH = "/configuration"
def read(spectra_url, channel_url, config_url=None, dimensions=None, energy_unit=ur.eV):
def read(spectra_url, channel_url, dimensions, config_url=None, energy_unit=ur.eV):
"""
:param DataUrl spectra_url: data url to the spectra
......@@ -73,16 +73,27 @@ def read(spectra_url, channel_url, config_url=None, dimensions=None, energy_unit
)
def read_frm_file(file_path, energy_unit=ur.eV):
def read_frm_file(
file_path,
energy_unit=ur.eV,
dimensions: tuple = (Dim.CHANNEL_ENERGY_DIM, Dim.Y_DIM, Dim.X_DIM),
):
"""
:param str file_path: path to the file containing the spectra. Must ba a
.dat file that pymca can handle or a .h5py with
default path
:param tuple dimensions: dimensions of the input data. For ASCII file can
be (X, Y) or (Y, X)
:return XasObject created from the input
:rtype: XASObject
"""
if file_path in (None, ""):
return
reader = XASReader()
return reader.read_from_file(file_path=file_path, energy_unit=energy_unit)
return reader.read_from_file(
file_path=file_path, energy_unit=energy_unit, dimensions=dimensions
)
class XASReader(object):
......@@ -102,7 +113,11 @@ class XASReader(object):
return XASObject(spectra=sp, energy=en, configuration=conf)
@staticmethod
def read_from_file(file_path, energy_unit=ur.eV):
def read_from_file(
file_path,
energy_unit=ur.eV,
dimensions=(Dim.CHANNEL_ENERGY_DIM, Dim.Y_DIM, Dim.X_DIM),
):
"""
:param str file_path:
......@@ -113,12 +128,14 @@ class XASReader(object):
spectra_url=DataUrl(file_path=file_path, scheme="PyMca"),
channel_url=DataUrl(file_path=file_path, scheme="PyMca"),
energy_unit=energy_unit,
dimensions=dimensions,
)
elif file_path.endswith(".xmu"):
return XASReader.read_frm_url(
spectra_url=DataUrl(file_path=file_path, scheme="larch"),
channel_url=DataUrl(file_path=file_path, scheme="larch"),
energy_unit=energy_unit,
dimensions=dimensions,
)
elif h5py.is_hdf5(file_path):
return XASReader.read_frm_url(
......@@ -132,9 +149,12 @@ class XASReader(object):
file_path=file_path, scheme="silx", data_path="configuration"
),
energy_unit=energy_unit,
dimensions=dimensions,
)
else:
raise ValueError("file type not managed, unable to load")
raise ValueError(
"file type {} not managed, unable to load".format(file_path)
)
__call__ = read_from_file
......
......@@ -104,7 +104,7 @@ def process_spectr_k(
# update EXAFSNormalized
e0 = pymca_xas.calculateE0()
kValues = e2k(spectrum_.energy.m - e0)
kValues = e2k(spectrum_.energy - e0)
exafs = exafs_res["EXAFSNormalized"]
if "KWeight" in configuration and configuration["KWeight"] is not None:
exafs *= pow(kValues, configuration["KWeight"])
......
......@@ -29,6 +29,7 @@ __date__ = "06/11/2019"
import os
import unittest
from est.core.types import Dim
try:
import PyMca5
......@@ -50,7 +51,7 @@ class TestEXAFSSingleSpectrum(unittest.TestCase):
def setUp(self):
data_file = os.path.join(PYMCA_DATA_DIR, "EXAFS_Cu.dat")
energy, mu = read_spectrum(data_file)
energy, mu = read_spectrum(data_file, dimensions=(Dim.X_DIM, Dim.Y_DIM))
spectrum = Spectrum(energy=energy, mu=mu)
exafs_configuration = {"Knots": {"Values": (1, 2, 5), "Number": 3}}
configuration = {"EXAFS": exafs_configuration}
......
......@@ -30,6 +30,7 @@ __date__ = "06/11/2019"
import os
import unittest
from est.core.types import XASObject, Spectrum
from est.core.types import Dim
try:
import PyMca5
......@@ -51,7 +52,7 @@ class TestFTSingleSpectrum(unittest.TestCase):
def setUp(self):
data_file = os.path.join(PYMCA_DATA_DIR, "EXAFS_Cu.dat")
energy, mu = read_spectrum(data_file)
energy, mu = read_spectrum(data_file, dimensions=(Dim.X_DIM, Dim.Y_DIM))
self.spectrum = Spectrum(energy=energy, mu=mu)
self.xas_obj = XASObject(
energy=energy, spectra=(self.spectrum,), dim1=1, dim2=1
......
......@@ -37,6 +37,7 @@ from est.core.io import read as read_xas
from est.core.utils import spectra as spectra_utils
from est.core.types import XASObject
from silx.io.url import DataUrl
from est.core.types import Dim
try:
import PyMca5
......@@ -63,6 +64,7 @@ class TestReadWrite(unittest.TestCase):
res = read_xas(
spectra_url=DataUrl(file_path=data_file, scheme="PyMca"),
channel_url=DataUrl(file_path=data_file, scheme="PyMca"),
dimensions=(Dim.CHANNEL_ENERGY_DIM, Dim.X_DIM, Dim.Y_DIM),
)
self.assertTrue(isinstance(res, XASObject))
self.assertEqual(res.n_spectrum, 1)
......
......@@ -30,6 +30,7 @@ __date__ = "06/11/2019"
import os
import unittest
from est.core.types import Spectrum, XASObject
from est.core.types import Dim
try:
import PyMca5
......@@ -55,7 +56,7 @@ class TestKWeightSingleSpectrum(unittest.TestCase):
}
data_file = os.path.join(PYMCA_DATA_DIR, "EXAFS_Cu.dat")
energy, mu = read_spectrum(data_file)
energy, mu = read_spectrum(data_file, dimensions=(Dim.X_DIM, Dim.Y_DIM))
self.spectrum = Spectrum(energy=energy, mu=mu)
self.xas_obj = XASObject(
......
......@@ -37,6 +37,7 @@ from silx.io.url import DataUrl
from est.core.io import read as read_xas
from est.core.types import Spectrum, XASObject
from est.core.utils import spectra as spectra_utils
from est.core.types import Dim
try:
import PyMca5
......@@ -55,7 +56,7 @@ class TestNormalizationSingleSpectrum(unittest.TestCase):
def setUp(self):
data_file = os.path.join(PYMCA_DATA_DIR, "EXAFS_Cu.dat")
energy, mu = read_spectrum(data_file)
energy, mu = read_spectrum(data_file, dimensions=(Dim.X_DIM, Dim.Y_DIM))
self.spectrum = Spectrum(energy=energy, mu=mu)
self.xas_obj = XASObject(
spectra=(self.spectrum,), energy=energy, dim1=1, dim2=1
......@@ -100,6 +101,7 @@ class TestNormalizationMultipleSpectrum(unittest.TestCase):
channel_url=DataUrl(
file_path=filename, data_path=channel_path, scheme="silx"
),
dimensions=(Dim.CHANNEL_ENERGY_DIM, Dim.Y_DIM, Dim.X_DIM),
)
def tearDown(self):
......
......@@ -44,6 +44,7 @@ from est.core.io import XASWriter
from est.core.io import read as read_xas
from est.core.types import XASObject
from est.core.utils import spectra as spectra_utils
from est.core.types import Dim
try:
import PyMca5
......@@ -75,6 +76,7 @@ class TestStreamSingleSpectrum(unittest.TestCase):
out = read_xas(
spectra_url=DataUrl(file_path=data_file, scheme="PyMca"),
channel_url=DataUrl(file_path=data_file, scheme="PyMca"),
dimensions=(Dim.CHANNEL_ENERGY_DIM, Dim.X_DIM, Dim.Y_DIM),
)
out.configuration = {"EXAFS": self.exafs_configuration, "SET_KWEIGHT": 0}
out = pymca_normalization(xas_obj=out)
......@@ -88,6 +90,7 @@ class TestStreamSingleSpectrum(unittest.TestCase):
out = read_xas(
spectra_url=DataUrl(file_path=data_file, scheme="PyMca"),
channel_url=DataUrl(file_path=data_file, scheme="PyMca"),
dimensions=(Dim.CHANNEL_ENERGY_DIM, Dim.X_DIM, Dim.Y_DIM),
)
out.configuration = {"EXAFS": self.exafs_configuration, "SET_KWEIGHT": 0}
out = pymca_normalization(xas_obj=out.to_dict())
......@@ -101,6 +104,7 @@ class TestStreamSingleSpectrum(unittest.TestCase):
out = read_xas(
spectra_url=DataUrl(file_path=data_file, scheme="PyMca"),
channel_url=DataUrl(file_path=data_file, scheme="PyMca"),
dimensions=(Dim.CHANNEL_ENERGY_DIM, Dim.X_DIM, Dim.Y_DIM),
)
out = PyMca_normalization()(xas_obj=out)
exafs_process = PyMca_exafs()
......
......@@ -31,14 +31,13 @@ import os
import shutil
import tempfile
import unittest
import h5py
import numpy
from silx.io.url import DataUrl
from est.core.io import read as read_xas
from est.core.process.roi import xas_roi
from est.core.utils import spectra as spectra_utils
from est.core.types import Dim
class TestRoi(unittest.TestCase):
......@@ -59,6 +58,7 @@ class TestRoi(unittest.TestCase):
channel_url=DataUrl(
file_path=filename, data_path=channel_path, scheme="silx"
),
dimensions=(Dim.CHANNEL_ENERGY_DIM, Dim.Y_DIM, Dim.X_DIM),
)
def tearDown(self):
......