Commit 9bf8b613 authored by payno's avatar payno

Merge branch 'fix_20' into 'master'

Fix 20

Closes #23

See merge request !22
parents 9f51ae1b 8f0438eb
Pipeline #19354 passed with stage
in 6 minutes and 59 seconds
......@@ -24,13 +24,11 @@ variables:
- source ./ci/install_scripts.sh
- install_orange3
- install_anyqt 'master'
- python -m pip install -r requirements.txt
- python -m pip install fabio --upgrade --pre
- python -m pip install silx --upgrade --pre
- python -m pip install pymca --upgrade --pre
- git submodule update --init --recursive
- python -m pip install -e .
- python -m est --help
- python -m pip install .
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a est test -v
......@@ -41,6 +39,7 @@ variables:
- export PYTHONPATH="${PYTHONPATH}:/usr/lib/python3/dist-packages/"
- export LD_LIBRARY_PATH=/lib/i386-linux-gnu/:${LD_LIBRARY_PATH}
- export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
- export LD_LIBRARY_PATH='${LD_LIBRARY_PATH}:/usr/local/lib'
- export EST_TEST_PROCESS=False
- python --version
- python -m pip install pip --upgrade
......@@ -51,13 +50,12 @@ variables:
- source ./ci/install_scripts.sh
- install_orange3
- install_anyqt 'master'
- python -m pip install -r requirements.txt
- python -m pip install fabio --upgrade --pre
# - python -m pip install -r requirements.txt
- python -m pip install silx --upgrade --pre
- install_larch
- git submodule update --init --recursive
- python -m pip install -e .
- python -m est --help
- python -m pip install .
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a est test -v
......@@ -68,6 +66,7 @@ variables:
- export PYTHONPATH="${PYTHONPATH}:/usr/lib/python3/dist-packages/"
- export LD_LIBRARY_PATH=/lib/i386-linux-gnu/:${LD_LIBRARY_PATH}
- export LD_LIBRARY_PATH=/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH}
- export LD_LIBRARY_PATH='${LD_LIBRARY_PATH}:/usr/local/lib'
- export EST_TEST_PROCESS=False
- python --version
- python -m pip install pip --upgrade
......@@ -78,14 +77,13 @@ variables:
- source ./ci/install_scripts.sh
- install_orange3
- install_anyqt 'master'
- python -m pip install -r requirements.txt
- python -m pip install fabio --upgrade --pre
# - python -m pip install -r requirements.txt
- python -m pip install silx --upgrade --pre
- python -m pip install pymca --upgrade --pre
- install_larch
- git submodule update --init --recursive
- python -m pip install -e .
- python -m est --help
- python -m pip install .
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a est test -v
......@@ -95,12 +93,12 @@ test:python3.5-stretch-pyqt5_pymca:
test:python3.5-stretch-pyqt5_larch:
image: docker-registry.esrf.fr/dau/est:python3.7_stretch_pyqt5_gtk3
image: docker-registry.esrf.fr/dau/est:python3.7_stretch_pyqt5_gtk3_wxpython
<<: *test_linux_template_larch
test:python3.5-stretch-pyqt5_pymca_larch:
image: docker-registry.esrf.fr/dau/est:python3.7_stretch_pyqt5_gtk3
image: docker-registry.esrf.fr/dau/est:python3.7_stretch_pyqt5_gtk3_wxpython
<<: *test_linux_template_pymca_larch
......@@ -124,9 +122,9 @@ test:test-est-tutorials_pymca:
- python -m pip install ipykernel
- python -m pip install pymca --upgrade --pre
- python -m pip install silx --upgrade --pre
- source ./ci/install_scripts.sh
- git submodule update --init --recursive
- python -m pip install .
- ls doc/tutorials/
script:
- /usr/bin/xvfb-run --server-args="-screen 0 1024x768x24" -a jupyter nbconvert --to notebook --execute doc/tutorials/pymca_xas_process.ipynb
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Bootstrap helps you to test scripts without installing them
by patching your PYTHONPATH on the fly
example: ./bootstrap.py ipython
"""
__authors__ = ["Frédéric-Emmanuel Picca", "Jérôme Kieffer"]
__contact__ = "jerome.kieffer@esrf.eu"
__license__ = "MIT"
__date__ = "02/03/2018"
import sys
import os
import distutils.util
import subprocess
import logging
logging.basicConfig()
logger = logging.getLogger("bootstrap")
def is_debug_python():
"""Returns true if the Python interpreter is in debug mode."""
try:
import sysconfig
except ImportError: # pragma nocover
# Python < 2.7
import distutils.sysconfig as sysconfig
if sysconfig.get_config_var("Py_DEBUG"):
return True
return hasattr(sys, "gettotalrefcount")
def _distutils_dir_name(dname="lib"):
"""
Returns the name of a distutils build directory
"""
platform = distutils.util.get_platform()
architecture = "%s.%s-%i.%i" % (dname, platform,
sys.version_info[0], sys.version_info[1])
if is_debug_python():
architecture += "-pydebug"
return architecture
def _distutils_scripts_name():
"""Return the name of the distrutils scripts sirectory"""
f = "scripts-{version[0]}.{version[1]}"
return f.format(version=sys.version_info)
def _get_available_scripts(path):
res = []
try:
res = " ".join([s.rstrip('.py') for s in os.listdir(path)])
except OSError:
res = ["no script available, did you ran "
"'python setup.py build' before bootstrapping ?"]
return res
if sys.version_info[0] >= 3: # Python3
def execfile(fullpath, globals=None, locals=None):
"Python3 implementation for execfile"
with open(fullpath) as f:
try:
data = f.read()
except UnicodeDecodeError:
raise SyntaxError("Not a Python script")
code = compile(data, fullpath, 'exec')
exec(code, globals, locals)
def run_file(filename, argv):
"""
Execute a script trying first to use execfile, then a subprocess
:param str filename: Script to execute
:param list[str] argv: Arguments passed to the filename
"""
full_args = [filename]
full_args.extend(argv)
try:
logger.info("Execute target using exec")
# execfile is considered as a local call.
# Providing globals() as locals will force to feed the file into
# globals() (for examples imports).
# Without this any function call from the executed file loses imports
try:
old_argv = sys.argv
sys.argv = full_args
logger.info("Patch the sys.argv: %s", sys.argv)
logger.info("Executing %s.main()", filename)
print("########### EXECFILE ###########")
module_globals = globals().copy()
module_globals['__file__'] = filename
execfile(filename, module_globals, module_globals)
finally:
sys.argv = old_argv
except SyntaxError as error:
logger.error(error)
logger.info("Execute target using subprocess")
env = os.environ.copy()
env.update({"PYTHONPATH": LIBPATH + os.pathsep + os.environ.get("PYTHONPATH", ""),
"PATH": os.environ.get("PATH", "")})
print("########### SUBPROCESS ###########")
run = subprocess.Popen(full_args, shell=False, env=env)
run.wait()
def run_entry_point(entry_point, argv):
"""
Execute an entry_point using the current python context
(http://setuptools.readthedocs.io/en/latest/setuptools.html#automatic-script-creation)
:param str entry_point: A string identifying a function from a module
(NAME = PACKAGE.MODULE:FUNCTION)
"""
import importlib
elements = entry_point.split("=")
target_name = elements[0].strip()
elements = elements[1].split(":")
module_name = elements[0].strip()
function_name = elements[1].strip()
logger.info("Execute target %s (function %s from module %s) using importlib", target_name, function_name, module_name)
full_args = [target_name]
full_args.extend(argv)
try:
old_argv = sys.argv
sys.argv = full_args
print("########### IMPORTLIB ###########")
module = importlib.import_module(module_name)
if hasattr(module, function_name):
func = getattr(module, function_name)
func()
else:
logger.info("Function %s not found", function_name)
finally:
sys.argv = old_argv
def find_executable(target):
"""Find a filename from a script name.
- Check the script name as file path,
- Then checks if the name is a target of the setup.py
- Then search the script from the PATH environment variable.
:param str target: Name of the script
:returns: Returns a tuple: kind, name.
"""
if os.path.isfile(target):
return ("path", os.path.abspath(target))
# search the file from setup.py
import setup
config = setup.get_project_configuration(dry_run=True)
# scripts from project configuration
if "scripts" in config:
for script_name in config["scripts"]:
if os.path.basename(script) == target:
return ("path", os.path.abspath(script_name))
# entry-points from project configuration
if "entry_points" in config:
for kind in config["entry_points"]:
for entry_point in config["entry_points"][kind]:
elements = entry_point.split("=")
name = elements[0].strip()
if name == target:
return ("entry_point", entry_point)
# search the file from env PATH
for dirname in os.environ.get("PATH", "").split(os.pathsep):
path = os.path.join(dirname, target)
if os.path.isfile(path):
return ("path", path)
return None, None
home = os.path.dirname(os.path.abspath(__file__))
LIBPATH = os.path.join(home, 'build', _distutils_dir_name('lib'))
cwd = os.getcwd()
os.chdir(home)
build = subprocess.Popen([sys.executable, "setup.py", "build"],
shell=False, cwd=os.path.dirname(os.path.abspath(__file__)))
build_rc = build.wait()
if not os.path.exists(LIBPATH):
logger.warning("`lib` directory does not exist, trying common Python3 lib")
LIBPATH = os.path.join(os.path.split(LIBPATH)[0], "lib")
os.chdir(cwd)
if build_rc == 0:
logger.info("Build process ended.")
else:
logger.error("Build process ended with rc=%s", build_rc)
sys.exit(-1)
if __name__ == "__main__":
if len(sys.argv) < 2:
logger.warning("usage: ./bootstrap.py <script>\n")
script = None
else:
script = sys.argv[1]
if script:
logger.info("Executing %s from source checkout", script)
else:
logging.info("Running iPython by default")
sys.path.insert(0, LIBPATH)
logger.info("Patched sys.path with %s", LIBPATH)
if script:
argv = sys.argv[2:]
kind, target = find_executable(script)
if kind == "path":
run_file(target, argv)
elif kind == "entry_point":
run_entry_point(target, argv)
else:
logger.error("Script %s not found", script)
else:
logger.info("Patch the sys.argv: %s", sys.argv)
sys.path.insert(2, "")
try:
from IPython import embed
except Exception as err:
logger.error("Unable to execute iPython, using normal Python")
logger.error(err)
import code
code.interact()
else:
embed()
......@@ -15,6 +15,7 @@ function install_silx(){
pip install .
export http_proxy=http://proxy.esrf.fr:3128/
export https_proxy=http://proxy.esrf.fr:3128/
cd ..
fi
}
......
......@@ -7,3 +7,5 @@ Pre-edge subtraction and normalization:
- determine E0
- fit a polynomial to the region above the edge
- extrapolate the two curves to E0 to determine the edge
note: you can open a pop up window for computing mean or median E0 value from the dataset using the button at the right of the E0 entry line
\ No newline at end of file
......@@ -4,3 +4,5 @@ normalization
Spectra normalization.
.. image:: ../img/pymca_normalization.png
note: you can open a pop up window for computing mean or median E0 value from the dataset using the button at the right of the E0 entry line
\ No newline at end of file
......@@ -61,6 +61,9 @@ def main():
else:
_version = est._version.version
launcher = Launcher(prog="est", version=_version)
launcher.add_command("canvas",
module_name="est.app.canvas",
description="open the orange-canvas")
launcher.add_command("process",
module_name="est.app.process",
description="process a workflow from a .xml.ows file"
......
import sys
from .canvas_launcher import Launcher
def main(argv):
Launcher().launch(argv)
if __name__ == "__main__":
sys.exit(main(sys.argv))
from .launcher import Launcher
\ No newline at end of file
from silx.gui import qt
from Orange.canvas import config, __main__ as main
from .splash import splash_screen, getIcon
import os, sys
from Orange.misc import environ
import pkg_resources
def version():
return pkg_resources.get_distribution("est").version
class EstConfig(config.Config):
ApplicationName = "est"
ApplicationVersion = version()
@staticmethod
def splash_screen():
return splash_screen()
@staticmethod
def core_packages():
return super(EstConfig, EstConfig).core_packages() + [
"est-add-on",
]
@staticmethod
def application_icon():
return getIcon()
class EstSplashScreen(qt.QSplashScreen):
"""SplashScreen to overwrite the one of Orange"""
def __init__(self, parent=None, pixmap=None, textRect=None,
textFormat=qt.Qt.PlainText, **kwargs):
qt.QSplashScreen.__init__(self, parent, pixmap=pixmap, **kwargs)
def showMessage(self, message, alignment=qt.Qt.AlignLeft, color=qt.Qt.black):
super().showMessage(version(), qt.Qt.AlignCenter | qt.Qt.AlignBottom, qt.QColor("#e68f17"))
class Launcher:
"""Proxy to orange-canvas"""
def launch(self, argv):
config.Config = EstConfig
self.fix_application_dirs()
self.replace_splash_screen()
self.main(argv)
def fix_application_dirs(self):
def data_dir(versioned=True):
"""
Return the platform dependent Orange data directory.
This is ``data_dir_base()``/Orange/__VERSION__/ directory if versioned is
`True` and ``data_dir_base()``/Orange/ otherwise.
"""
base = environ.data_dir_base()
if versioned:
return os.path.join(base, "est", version())
else:
return os.path.join(base, "est")
environ.data_dir = data_dir
def cache_dir(*args):
"""
Return the platform dependent Orange cache directory.
"""
if sys.platform == "darwin":
base = os.path.expanduser("~/Library/Caches")
elif sys.platform == "win32":
base = os.getenv("APPDATA", os.path.expanduser("~/AppData/Local"))
elif os.name == "posix":
base = os.getenv("XDG_CACHE_HOME", os.path.expanduser("~/.cache"))
else:
base = os.path.expanduser("~/.cache")
base = os.path.join(base, "est", version())
if sys.platform == "win32":
# On Windows cache and data dir are the same.
# Microsoft suggest using a Cache subdirectory
return os.path.join(base, "Cache")
else:
return base
environ.cache_dir = cache_dir
def replace_splash_screen(self):
main.SplashScreen = EstSplashScreen
def main(self, argv):
from Orange.canvas.__main__ import main
main(argv)
from silx.gui import qt
from est.gui import icons
def splash_screen():
"""
:return: splash screen for orange-canvas
:rtype: tuple(QPixmap, QRect),
note: QRect is used by orange to define a mask to display the message.
In our case we overwrite the QSplashScreen so we don't need this.
"""
pixmap = icons.getQPixmap('est')
return pixmap, qt.QRect(0, 0, 0, 0)
def getIcon():
"""
:return: application icon
:rtype: QIcon
"""
pixmap = icons.getQPixmap('est')
return qt.QIcon(pixmap)
......@@ -38,4 +38,5 @@ _logger = logging.getLogger(__name__)
def suite():
test_suite = unittest.TestSuite()
test_suite.addTest(test_exec_suite())
test_suite.addTest(test_reprocessing_suite())
return test_suite
......@@ -48,7 +48,7 @@ class TestWorkflowFromOwsFile(unittest.TestCase):
"""test construction of XAS object"""
def setUp(self):
self.outputdir = tempfile.mkdtemp()
file_ = 'pymca_workflow.ows'
file_ = 'pymca_workflow_2.ows'
DownloadDataset(dataset=file_,
output_folder=self.outputdir,
timeout=10.0)
......
......@@ -32,6 +32,7 @@ __date__ = "07/16/2019"
from est.io import read_xas, write_xas, get_xasproc
from est.core.types import XASObject
from silx.io.url import DataUrl
from est.core.types import Dim
import h5py
import logging
......@@ -44,18 +45,26 @@ DEFAULT_CHANNEL_PATH = '/data/NXdata/Channel'
DEFAULT_CONF_PATH = '/configuration'
def read(spectra_url, channel_url, config_url=None):
def read(spectra_url, channel_url, config_url=None, dimensions=None):
"""
:param DataUrl spectra_url: data url to the spectra
:param DataUrl channel_url: data url to the channel / energy
:param DataUrl config_url: data url to the process configuration
:param dimensions: way the data has been stored.
Usually is (X, Y, channels) of (Channels, Y, X).
If None, by default is considered to be (Channels, Y, X)
:type: tuple
:return:
:rtype: XASObject
"""
dimensions_ = dimensions
if dimensions_ is None:
dimensions_ = (Dim.CHANNEL_ENERGY_DIM, Dim.Y_DIM, Dim.X_DIM)
reader = XASReader()
return reader.read_frm_url(spectra_url=spectra_url, channel_url=channel_url,
config_url=config_url)
config_url=config_url, dimensions=dimensions_)
def read_frm_file(file_path):
......@@ -72,32 +81,36 @@ def read_frm_file(file_path):
class XASReader(object):
"""Simple reader of a xas file"""
def read_frm_url(self, spectra_url, channel_url, config_url=None):
@staticmethod
def read_frm_url(spectra_url, channel_url, dimensions=None,
config_url=None):
sp, en, conf = read_xas(spectra_url=spectra_url,
channel_url=channel_url,
config_url=config_url)
config_url=config_url,
dimensions=dimensions)
return XASObject(spectra=sp, energy=en, configuration=conf)
def read_from_file(self, file_path):
@staticmethod
def read_from_file(file_path):
"""
:param str file_path:
:return: `.XASObject`
"""
if file_path.endswith('.dat'):
return self.read_frm_url(
return XASReader.read_frm_url(
spectra_url=DataUrl(file_path=file_path,
scheme='PyMca'),
channel_url=DataUrl(file_path=file_path,
scheme='PyMca'))
elif file_path.endswith('.xmu'):
return self.read_frm_url(
return XASReader.read_frm_url(
spectra_url=DataUrl(file_path=file_path,
scheme='larch'),
channel_url=DataUrl(file_path=file_path,
scheme='larch'))
elif h5py.is_hdf5(file_path):
return self.read_frm_url(
return XASReader.read_frm_url(
spectra_url=DataUrl(file_path=file_path,
scheme='silx',
data_path=DEFAULT_SPECTRA_PATH),
......@@ -113,7 +126,6 @@ class XASReader(object):
__call__ = read_from_file
class XASWriter(object):
"""
class to write the output file. In this case we need a class in order to
......
......@@ -56,6 +56,7 @@ def process_spectr_autobk(spectrum, configuration, overwrite=True,
:return: processed spectrum
:rtype: tuple (configuration, spectrum)
"""
_logger.debug('start autobk on spectrum (%s, %s)' % (spectrum.x, spectrum.y))
assert isinstance(spectrum, Spectrum)
if spectrum.energy is None or spectrum.mu is None:
_logger.error('Energy and or Mu is/are not specified, unable to '
......
......@@ -56,6 +56,7 @@ def process_spectr_mback(spectrum, configuration, overwrite=True, callbacks=None
:return: processed spectrum
:rtype: tuple (configuration, spectrum)
"""
_logger.debug('start mback on spectrum (%s, %s)' % (spectrum.x, spectrum.y))
assert isinstance(spectrum, Spectrum)
if spectrum.energy is None or spectrum.mu is None:
_logger.error('Energy and or Mu is/are not specified, unable to '
......@@ -70,6 +71,8 @@ def process_spectr_mback(spectrum, configuration, overwrite=True, callbacks=None
'order', 'leexiang', 'tables', 'fit_erfc'):
if opt_name in _conf:
opts[opt_name] = _conf[opt_name]
if not 'z' in opts:
raise ValueError('atomic number of the absorber is not specify')
if _DEBUG is True:
assert isinstance(spectrum, Group)
......
......@@ -56,6 +56,7 @@ def process_spectr_mback_norm(spectrum, configuration, overwrite=True,
:return: processed spectrum
:rtype: tuple (configuration, spectrum)
"""
_logger.debug('start mback_norm on spectrum (%s, %s)' % (spectrum.x, spectrum.y))
assert isinstance(spectrum, Spectrum)
if not hasattr(spectrum, 'norm'):
_logger.error('spectrum doesn\'t have norm. Maybe you meed to compute '
......
......@@ -56,6 +56,7 @@ def process_spectr_pre_edge(spectrum, configuration, overwrite=True, callbacks=N
:return: processed spectrum
:rtype: tuple (configuration, spectrum)
"""
_logger.debug('start pre_edge on spectrum (%s, %s)' % (spectrum.x, spectrum.y))
assert isinstance(spectrum, Spectrum)
if spectrum.energy is None or spectrum.mu is None:
_logger.error('Energy and or Mu is/are not specified, unable to '
......
......@@ -63,12 +63,12 @@ class TestLarchSpectrum(unittest.TestCase):
shutil.rmtree(self.outputdir)
def testProcess(self):
self.assertFalse(hasattr(self.spectrum, 'k'))
self.assertFalse(hasattr(self.spectrum, 'chi'))
self.assertTrue(self.spectrum.k is None)
self.assertTrue(self.spectrum.chi is None)
conf, res_spectrum = process_spectr_autobk(self.spectrum,
self.configuration)
self.assertTrue(hasattr(res_spectrum, 'k'))
self.assertTrue(hasattr(res_spectrum, 'chi'))
self.assertFalse(self.spectrum.k is None)
self.assertFalse(self.spectrum.chi is None)
@unittest.skipIf(has_larch is False, 'xraylarch not installed')
......
......@@ -57,7 +57,7 @@ class TestLarchSpectrum(unittest.TestCase):
assert os.path.exists(self.data_file)
energy, mu = read_ascii(self.data_file)
self.spectrum = Spectrum(energy=energy, mu=mu)
self.configuration = {'z': 29}
self.configuration = {'mback': {'z': 29}}
def tearDown(self):
shutil.rmtree(self.outputdir)
......@@ -84,7 +84,7 @@ class TestLarchSpectra(unittest.TestCase):
out_file.write(data)
assert os.path.exists(self.data_file)
# define the xas object
self.configuration = {'z': 29}
self.configuration = {'mback': {'z': 29}}
energy, mu = read_ascii(self.data_file)
self.xas_object = XASObject(spectra=(Spectrum(energy=energy, mu=mu),),
energy=energy, dim1=1, dim2=1,
......
......@@ -56,6 +56,7 @@ def process_spectr_xftf(spectrum, configuration, overwrite=True, callbacks=None,
:return: processed spectrum
:rtype: tuple (configuration, spectrum)
"""
_logger.debug('start xftf on spectrum (%s, %s)' % (spectrum.x, spectrum.y))
assert isinstance(spectrum, Spectrum)
if (not hasattr(spectrum, 'k') or not hasattr(spectrum, 'chi')
or spectrum.k is None or spectrum.chi is None):
......
......@@ -121,7 +121,6 @@ class Process(object):
:param xas_obj: object for which we want to save the treatment
:type: :class:`.XASObject`
:param tuple data_keys: keys of the id to save
:param str relative_to: x value
"""
if xas_obj.has_linked_file():
_data = {}
......@@ -140,6 +139,7 @@ class Process(object):
elif key in ('ft.radius', 'ft.intensity', 'ft.imaginary'):
relative_to = 'radius'
use = '_list_res_ft'
if use == '_spectra_volume':
# if we can display the result as a numpy.array 3d
try:
......@@ -153,10 +153,15 @@ class Process(object):
_logger.error(mess)
elif use == '_list_res_ft':
res = []
# for ft we are force to have a dictionary because it