GitLab will be upgraded on June 23rd evening. During the upgrade the service will be unavailable, sorry for the inconvenience.

Commit 33e6fc65 authored by Wout De Nolf's avatar Wout De Nolf Committed by Matias Guijarro

1125 nexus writer tests

parent 4b53ab67
......@@ -37,6 +37,7 @@ check_lint:
run_tests:source:
stage: tests
image: continuumio/miniconda3:latest
parallel: 2
script:
- >
if [ $CI_COMMIT_REF_NAME != 'master' ]; then
......@@ -55,13 +56,23 @@ run_tests:source:
- conda install pytest-profiling --yes
- pip install .
# run tests on source
- python setup.py test --addopts "--cov bliss --cov nexus_writer_service --cov-report html --cov-report term --profile --durations=30"
- COVDIR="htmlcov$CI_NODE_INDEX"
- >
if [[ $CI_NODE_INDEX == 1 ]]; then
echo "Run all Bliss core tests"
python setup.py test --addopts "--cov bliss --cov-report html:$COVDIR --cov-report term --profile --durations=30"
elif [[ $CI_NODE_INDEX == 2 ]]; then
echo "Run all writer tests"
python setup.py test --addopts "--cov bliss --cov-report html:$COVDIR --cov-report term --profile --durations=30 -m writer --runwritertests"
else
echo "No tests in this group"
fi
after_script:
- python scripts/profiling2txt.py
- sh scripts/print_test_profiling.sh
artifacts:
paths:
- htmlcov/
- $COVDIR/
expire_in: 7 days
variables:
CHANGES: '\.(py|cfg)$|requirements|gitlab-ci|^(bin|extensions|scripts|spec|tests)/'
......@@ -132,6 +143,7 @@ create_doc:user:
run_tests:package:
stage: package_tests
image: continuumio/miniconda3:latest
parallel: 2
script:
# install Xvfb and opengl libraries (needed for test_flint)
- apt-get update && apt-get -y install xvfb libxi6
......@@ -139,10 +151,20 @@ run_tests:package:
- conda create -y --name testenv
- source activate testenv
- conda install bliss==$CI_COMMIT_TAG --file requirements-test-conda.txt --channel file://${CI_PROJECT_DIR}/conda-local-channel
- pytest --cov=bliss --cov-report html --cov-report term
- COVDIR="htmlcov$CI_NODE_INDEX"
- >
if [[ $CI_NODE_INDEX == 1 ]]; then
echo "Run all Bliss core tests"
pytest --cov=bliss --cov-report html:$COVDIR --cov-report term
elif [[ $CI_NODE_INDEX == 2 ]]; then
echo "Run all writer tests"
pytest --cov=bliss --cov-report html:$COVDIR --cov-report term -m writer --runwritertests
else
echo "No tests in this group"
fi
artifacts:
paths:
- htmlcov/
- $COVDIR/
expire_in: 7 days
only:
- tags
......
......@@ -287,7 +287,12 @@ class LimaImageChannelDataNode(DataNode):
file_path = path_format % file_nb
if file_format == "HDF5":
returned_params.append(
(file_path, "/entry_%04d" % 0, image_index_in_file, file_format)
(
file_path,
self._path_in_hdf5(file_path),
image_index_in_file,
file_format,
)
)
else:
returned_params.append(
......@@ -295,6 +300,51 @@ class LimaImageChannelDataNode(DataNode):
)
return returned_params
@staticmethod
def _path_in_hdf5(filename, signal=True):
# TODO:
# same as nexus_writer_service.io.nexus.getDefault
# handle OSError?
def strattr(node, attr, default):
v = node.attrs.get(attr, default)
try:
v = v.decode()
except AttributeError:
pass
return v
path = ""
with h5py.File(filename, mode="r") as f:
default = strattr(f, "default", "")
if default and not default.startswith("/"):
default = "/" + default
while default:
try:
node = f[default]
except KeyError:
break
nxclass = strattr(node, "NX_class", "")
if nxclass == "NXdata":
if signal:
name = strattr(node, "signal", "data")
try:
path = node[name].name
except KeyError:
pass
else:
path = node.name
break
else:
add = strattr(node, "default", "")
if add.startswith("/"):
default = add
elif add:
default += "/" + add
else:
break
return path
def _get_from_file(self, image_nb):
for ref_data in self.ref_data:
values = self._get_filenames(ref_data, image_nb)
......
......@@ -5,6 +5,7 @@
# Copyright (c) 2015-2019 Beamline Control Unit, ESRF
# Distributed under the GNU LGPLv3. See LICENSE for more info.
import os
from bliss.scanning.writer.file import FileWriter
......@@ -37,4 +38,4 @@ class Writer(FileWriter):
@property
def filename(self):
return "<no saving>"
return os.path.join(self.root_path, self.data_filename + ".null")
......@@ -9,6 +9,13 @@ To start a session writer as a process inside an environment where BLISS is inst
$ NexusSessionWriter test_session --log=info
```
To allow for a proper Nexus structure, add these lines to the session's user script (strongly recommended but not absolutely necessary):
```python
from nexus_writer_service import metadata
metadata.register_all_metadata_generators()
```
!!! warning
Currently the external NeXus writer is in an experimental state and the protocol to ensure the completeness for the
saved data still needs to be put in place.
......
......@@ -76,7 +76,7 @@ def add_edf_arguments(filenames, createkwargs=None):
)
if indices:
img = fabio.open(filename)
# EdfImage.getframe returns an EdfImage, not a EdfFrame
# EdfImage.getframe returns an EdfImage, not an EdfFrame
def getframe(img):
return img._frames[img.currentframe]
......
......@@ -134,7 +134,9 @@ def hdf5_sep(func):
def as_os_path(*args, sep="/"):
args = [re.sub(r"[\/]+", os.sep, x) for x in args]
ret = func(*args)
return ret.replace(os.sep, sep)
if isinstance(ret, str):
ret = ret.replace(os.sep, sep)
return ret
return as_os_path
......@@ -166,7 +168,7 @@ def hdf5_dirname(path):
@hdf5_sep
def hdf5_join(*args):
return os.path.join(args)
return os.path.join(*args)
def splitUri(uri):
......@@ -202,6 +204,59 @@ def normUri(uri):
return filename + "::" + path
def dereference(node):
"""
:param h5py.Dataset or h5py.Group:
:returns str: uri
"""
if node.name == "/":
return getUri(node)
try:
lnk = node.parent.get(node.name, default=None, getlink=True)
except (KeyError, RuntimeError):
return getUri(node)
else:
if isinstance(lnk, h5py.SoftLink):
path = lnk.path
if not path.startswith("/"):
path = hdf5_join(node.parent.name, path)
return node.file.filename + "::" + hdf5_normpath(path)
elif isinstance(lnk, h5py.ExternalLink):
return lnk.filename + "::" + lnk.path
else:
return getUri(node)
def dereferenceUri(uri):
"""
Get full URI of dataset or group
:param h5py.Dataset or h5py.Group:
:returns str:
"""
filename, path = splitUri(uri)
uri2 = uri
istart = 1
with h5open(filename, mode="r") as f:
while True:
parts = path.split("/")[1:]
for i in range(istart, len(parts) + 1):
path2 = hdf5_join(*parts[:i])
filename2, path2 = splitUri(dereference(f[path2]))
path2 = hdf5_join(path2, *parts[i:])
uri2 = filename2 + "::" + path2
if uri != uri2:
if filename == filename2:
path = path2
istart = i + 1
break
else:
return dereferenceUri(uri2)
else:
break
return uri2
def getUri(node):
"""
Get full URI of dataset or group
......@@ -346,6 +401,41 @@ def h5Name(h5group):
return h5group.name.split("/")[-1]
def as_str(s):
"""
:param str or bytes s:
:returns str:
"""
try:
return s.decode()
except AttributeError:
return s
def attr_as_str(node, attr, default):
"""
Get attribute value (scalar or sequence) with type `str`.
:param h5py.Group or h5py.Dataset node:
:param str attr:
:param default:
:return str or sequence(str):
"""
v = node.attrs.get(attr, default)
if isinstance(v, str):
return v
elif isinstance(v, bytes):
return v.encode()
elif isinstance(v, tuple):
return tuple(as_str(s) for s in v)
elif isinstance(v, list):
return list(as_str(s) for s in v)
elif isinstance(v, numpy.ndarray):
return numpy.array(list(as_str(s) for s in v))
else:
return v
def nxClass(h5group):
"""
Nexus class of existing h5py.Group (None when no Nexus instance)
......@@ -353,7 +443,7 @@ def nxClass(h5group):
:param h5py.Group h5group:
:returns str or None:
"""
return h5group.attrs.get("NX_class", None)
return attr_as_str(h5group, "NX_class", None)
def isNxClass(h5group, *classes):
......@@ -890,8 +980,8 @@ def nxDataGetSignals(data):
:param h5py.Group data:
:returns list(str): signal names (default first)
"""
signal = data.attrs.get("signal", None)
auxsignals = data.attrs.get("auxiliary_signals", None)
signal = attr_as_str(data, "signal", None)
auxsignals = attr_as_str(data, "auxiliary_signals", None)
if signal is None:
lst = []
else:
......@@ -1266,7 +1356,7 @@ def nxDataAddAxes(data, axes, append=True):
"""
raiseIsNotNxClass(data, u"NXdata")
if append:
names = data.attrs.get("axes", [])
names = attr_as_str(data, "axes", [])
else:
names = []
for name, value, attrs in axes:
......@@ -1397,3 +1487,43 @@ def markDefault(h5node, nxentrylink=True):
break
path = parent
nxclass = parentnxclass
def getDefault(node, signal=True):
"""
:param h5py.Group or h5py.Dataset h5node:
:returns str: path of dataset or NXdata group
"""
path = ""
default = attr_as_str(node, "default", "")
root = node.file
if default and not default.startswith("/"):
if node.name == "/":
default = "/" + default
else:
default = node.name + "/" + default
while default:
try:
node = root[default]
except KeyError:
break
nxclass = attr_as_str(node, "NX_class", "")
if nxclass == "NXdata":
if signal:
name = attr_as_str(node, "signal", "data")
try:
path = node[name].name
except KeyError:
pass
else:
path = node.name
break
else:
add = attr_as_str(node, "default", "")
if add.startswith("/"):
default = add
elif add:
default += "/" + add
else:
break
return path
......@@ -18,5 +18,5 @@
writer_config
writer_config_publish
devices
nx_dataset
dataset_proxy
"""
......@@ -119,16 +119,30 @@ def update_device(devices, fullname, units=None):
return device
def parse_devices(devices, multivalue_positioners=False):
def parse_devices(devices, short_names=True, multivalue_positioners=False):
"""
Determine names and types based on device name and type
:param dict devices:
:param bool short_names:
:param bool multivalue_positioners:
"""
namemap = shortnamemap(list(devices.keys()))
# aliasmap: alias -> fullname
aliasmap = {
info.get("alias", fullname): fullname for fullname, info in devices.items()
}
if len(aliasmap) != len(devices):
aliasmap = {k: k for k in devices}
if short_names:
# namemap: alias -> shortname
namemap = shortnamemap(list(aliasmap.keys()))
# namemap: fullname -> shortname
namemap = {aliasmap[alias]: shortname for alias, shortname in namemap.items()}
else:
# namemap: fullname -> alias
namemap = {fullname: alias for alias, fullname in aliasmap.items()}
for fullname, device in devices.items():
device["device_name"] = namemap[fullname]
device["device_name"] = namemap.get(fullname, fullname)
if device["device_type"] == "mca":
# 'xmap1:xxxxxx_det1'
# device_name = 'xmap1:det1'
......@@ -210,13 +224,14 @@ def parse_devices(devices, multivalue_positioners=False):
device["unique_name"] = device["device_name"] + ":" + device["data_name"]
def device_info(devices, scan_info, multivalue_positioners=False):
def device_info(devices, scan_info, short_names=True, multivalue_positioners=False):
"""
Merge device information from `writer_config_publish.device_info`
and from the scan info published by the Bliss core library.
:param dict devices: as provided by `writer_config_publish.device_info`
:param dict scan_info:
:param bool short_names:
:param bool multivalue_positioners:
:returns dict: subscanname:dict(fullname:dict)
"""
......@@ -225,9 +240,10 @@ def device_info(devices, scan_info, multivalue_positioners=False):
subdevices = ret[subscan] = {}
# These are the "positioners"
dic = subscaninfo["master"]
units = dic["scalars_units"]
units = dic.get("scalars_units", {})
aliasmap = dic.get("display_names", {})
master_index = 0
for fullname in dic["scalars"]:
for fullname in dic.get("scalars", []):
subdevices[fullname] = devices.get(fullname, {})
device = update_device(subdevices, fullname, units)
if fullname.startswith("timer"):
......@@ -236,14 +252,36 @@ def device_info(devices, scan_info, multivalue_positioners=False):
device["device_type"] = "positioner"
device["master_index"] = master_index
master_index += 1
_add_display_name(device, fullname, aliasmap)
# These are the 0D, 1D and 2D "detectors"
dic = subscaninfo
aliasmap = dic.get("display_names", {})
for key in "scalars", "spectra", "images":
units = dic.get(key + "_units", {})
for fullname in dic[key]:
for fullname in dic.get(key, []):
subdevices[fullname] = devices.get(fullname, {})
device = update_device(subdevices, fullname, units)
if fullname.startswith("timer") and key == "scalars":
device["device_type"] = "positionergroup"
parse_devices(subdevices, multivalue_positioners=multivalue_positioners)
_add_display_name(device, fullname, aliasmap)
parse_devices(
subdevices,
short_names=short_names,
multivalue_positioners=multivalue_positioners,
)
return ret
def _add_display_name(device, fullname, display_names):
"""
:param dict device:
:param str fullname:
:param dict display_names:
"""
# REMARK: display_names contain node.name when alias is missing
# while we want node.fullname to avoid collisions.
alias = display_names.get(fullname, None)
# TODO: this does not work if the alias is the same as node.name
missing_alias = fullname == alias or fullname.split(":")[-1] == alias
if not missing_alias:
device["alias"] = alias
......@@ -20,6 +20,7 @@ import datetime
from contextlib import contextmanager
from . import writer_base
from ..io import nexus
from ..utils import scan_utils
default_saveoptions = dict(writer_base.default_saveoptions)
......@@ -54,16 +55,6 @@ class NexusScanWriterConfigurable(writer_base.NexusScanWriterBase):
super(NexusScanWriterConfigurable, self).__init__(*args, **kwargs)
self._applications = {"appxrf": self._save_application_xrf}
def _filename(self, level=0):
"""
HDF5 file name
"""
filenames = self.filenames
try:
return filenames[level]
except IndexError:
return ""
@property
def filenames(self):
"""
......@@ -71,10 +62,7 @@ class NexusScanWriterConfigurable(writer_base.NexusScanWriterBase):
and the others for the masters
"""
if self.save:
lst = self.config_writer.get("filenames", None)
if not lst:
lst = [self._default_filename]
return lst
return scan_utils.scan_filenames(self.scan_node, config=True)
else:
return []
......@@ -83,7 +71,7 @@ class NexusScanWriterConfigurable(writer_base.NexusScanWriterBase):
"""
Writer information not published by the core Bliss library
"""
return self.scan_info_get("external", default={})
return self.scan_info_get("nxwriter", default={})
@property
def instrument_name(self):
......@@ -582,7 +570,7 @@ class NexusScanWriterConfigurable(writer_base.NexusScanWriterBase):
continue
if linkname in nxroot:
continue
self.logger.debug(
self.logger.info(
"Create link {} in master {}".format(
repr(linkname), repr(nxroot.file.filename)
)
......
......@@ -24,7 +24,7 @@ from ..utils import scan_utils
logger = logging.getLogger(__name__)
CATEGORIES = ["EXTERNAL", "INSTRUMENT"]
CATEGORIES = ["NXWRITER", "INSTRUMENT"]
def register_metadata_generators(generators):
......@@ -35,12 +35,12 @@ def register_metadata_generators(generators):
"""
instrument = generators.instrument
instrument.set("positioners", fill_positioners) # start of scan
external = generators.external
external.set("instrument", fill_instrument_name)
external.set("positioners", fill_positioners) # end of scan
external.set("device_info", fill_device_info)
external.set("technique", fill_technique_info)
external.set("filenames", fill_filenames)
nxwriter = generators.nxwriter
nxwriter.set("instrument", fill_instrument_name)
nxwriter.set("positioners", fill_positioners) # end of scan
nxwriter.set("device_info", fill_device_info)
nxwriter.set("technique", fill_technique_info)
nxwriter.set("filenames", fill_filenames)
def fill_positioners(scan):
......@@ -88,7 +88,7 @@ def fill_filenames(scan):
:param bliss.scanning.scan.Scan scan:
"""
logger.debug("fill filename info")
return {"filenames": scan_utils.filenames()}
return {"filenames": scan_utils.current_filenames()}
def fill_device_info(scan):
......@@ -110,6 +110,14 @@ def _mca_device_info(ctr):
return {"type": "mca", "description": description}
def _samplingcounter_device_info(ctr):
"""
:param SamplingCounter ctr:
:returns str:
"""
return {"type": "samplingcounter", "mode": ctr.mode.name}
def _mca_roi_data_info(ctr):
"""
:param RoiMcaCounter ctr:
......@@ -138,87 +146,99 @@ def device_info(scan):
:returns dict:
"""
devices = {}
# This is not all of them
for ctr in global_map.get_counters_iter():
_device_info_add_ctr(devices, ctr)
return devices
def _device_info_add_ctr(devices, ctr):
"""
:param dict devices: str -> dict
:param ctr:
"""
try:
fullname = ctr.fullname.replace(".", ":") # Redis name
# Derived from: bliss.common.counter.BaseCounter
# bliss.common.counter.Counter
# bliss.common.counter.SamplingCounter
# bliss.common.temperature.TempControllerCounter
# bliss.controllers.simulation_diode.SimulationDiodeSamplingCounter
# bliss.scanning.acquisition.mca.BaseMcaCounter
# bliss.scanning.acquisition.mca.SpectrumMcaCounter
# bliss.scanning.acquisition.mca.StatisticsMcaCounter
ctr_classes = [c.__name__ for c in inspect.getmro(ctr.__class__)]
# print(ctr.fullname, type(ctr), type(ctr.controller), ctr_classes)
# controller_classes = [c.__name__ for c in inspect.getmro(ctr.controller.__class__)]
if "SpectrumMcaCounter" in ctr_classes:
device_info = _mca_device_info(ctr)
device = {"device_info": device_info, "device_type": "mca"}
devices[fullname] = device
elif "StatisticsMcaCounter" in ctr_classes:
device_info = _mca_device_info(ctr)
device = {"device_info": device_info, "device_type": "mca"}
devices[fullname] = device
elif "RoiMcaCounter" in ctr_classes:
device_info = _mca_device_info(ctr)
data_info = _mca_roi_data_info(ctr)
device = {
"device_info": device_info,
"data_info": data_info,
"device_type": "mca",
}
devices[fullname] = device
elif "LimaBpmCounter" in ctr_classes:
device_info = {"type": "lima"}
device = {"device_info": device_info, "device_type": "lima"}
devices[fullname] = device