Commit 5746727e authored by Matias Guijarro's avatar Matias Guijarro
Browse files

Merge branch '2769-scan-and-dataset-metadata' into 'master'

Resolve "Scan and dataset metadata"

Closes #2769

See merge request !3724
parents 62fd87ca 96d43466
Pipeline #47526 passed with stages
in 105 minutes and 45 seconds
......@@ -122,11 +122,8 @@ class _Base:
def wait_ready(self):
return self.device.wait_ready()
def fill_meta_at_scan_start(self, scan_meta):
return self.device.fill_meta_at_scan_start(scan_meta)
def fill_meta_at_scan_end(self, scan_meta):
return self.device.fill_meta_at_scan_end(scan_meta)
def get_acquisition_metadata(self, *args, **kw):
return self.device.get_acquisition_metadata(*args, **kw)
def _all_point_rx(self):
"""
......
......@@ -15,6 +15,7 @@ import inspect
import numpy
from bliss.common.utils import autocomplete_property
from bliss.common.protocols import HasMetadataForScan
def add_conversion_function(obj, method_name, function):
......@@ -58,7 +59,7 @@ def _identity(val):
return val
class Counter:
class Counter(HasMetadataForScan):
""" Counter class """
def __init__(self, name, controller, conversion_function=None, unit=None):
......@@ -122,8 +123,8 @@ class Counter:
assert callable(func)
self._conversion_function = func
def get_metadata(self):
return {}
def scan_metadata(self):
return None
def __info__(self, counter_type=None):
info_str = f"'{self.name}` counter info:\n"
......
......@@ -8,10 +8,13 @@
"""
This file groups all protocols managed by bliss
"""
import weakref
from abc import ABC
from collections import namedtuple
from types import SimpleNamespace
from typing import Mapping
from typing import Union
class IterableNamespace(SimpleNamespace):
......@@ -104,28 +107,67 @@ class Scannable(ABC):
raise NotImplementedError
class IcatPublisher(ABC):
class HasMetadataForDataset(ABC):
"""
Any controller that has this interface can be used
for metadata collection based on the `icat-mapping`
tag in the session configuration
Any controller which provides metadata intended to be saved
during a dataset life cycle.
The `dataset_metadata` is called by the Bliss session's icat_mapping
object when the session has such a mapping configured.
"""
def metadata(self) -> dict:
def dataset_metadata(self) -> Union[dict, None]:
"""
Return a dict containing metadata
Returning an empty dictionary means the controller has metadata
but no values. `None` means the controller has no metadata.
"""
raise NotImplementedError
class HasMetadataForScan(ABC):
"""
Any controller which provides metadata during a scan life cycle.
Any controller which provides metadata intended to be saved
during a scan life cycle.
"""
def metadata_when_prepared(self) -> dict:
disabled_controllers = weakref.WeakKeyDictionary()
def disable_scan_metadata(self):
HasMetadataForScan.disabled_controllers[self] = True
@property
def scan_metadata_enabled(self):
return not HasMetadataForScan.disabled_controllers.get(self)
def enable_scan_metadata(self):
try:
HasMetadataForScan.disabled_controllers.pop(self)
except KeyError:
pass
def scan_metadata(self) -> Union[dict, None]:
"""
Return a dict containing metadata when the device was prepared by the
scan.
Returning an empty dictionary means the controller has metadata
but no values. `None` means the controller has no metadata.
"""
raise NotImplementedError
@property
def scan_metadata_name(self) -> Union[str, None]:
"""
Default implementation returns self.name, can be overwritten in derived classes
Returns None when there is no name
"""
try:
return self.name
except AttributeError:
return None
class HasMetadataForScanExclusive(HasMetadataForScan):
"""
Any controller which provides metadata intended to be saved
during a scan life cycle when used in the acquisition chain.
"""
pass
......@@ -92,8 +92,6 @@ def ct(
scan_info=scan_info,
)
s._update_scan_info_with_user_scan_meta = lambda _: None
if run:
s.run()
......
......@@ -37,6 +37,8 @@ from bliss.common.image_tools import (
test_image,
)
from bliss.common.protocols import HasMetadataForScan
# CHAIN OBJECT NOTES:
#
......@@ -396,10 +398,17 @@ class FakeAcquisitionSlave(AcquisitionSlave):
# self.channels.update_from_array(data)
# self.channels.update({self.chname: self.positions})
def fill_meta_at_scan_start(self, scan_meta):
tmp_dict = {}
def get_acquisition_metadata(self, timing=None):
tmp_dict = super().get_acquisition_metadata(timing=timing)
if timing != self.META_TIMING.PREPARED:
return tmp_dict
for cnt in self._counters:
deep_update(tmp_dict, cnt.get_metadata())
if isinstance(cnt, HasMetadataForScan):
mdata = cnt.scan_metadata()
if mdata is not None:
if tmp_dict is None:
tmp_dict = dict()
deep_update(tmp_dict, mdata)
return tmp_dict
def prepare(self):
......
......@@ -15,10 +15,11 @@ import types
import itertools
import functools
import numpy
import collections.abc
import importlib.util
import distutils.util
from collections.abc import MutableMapping, MutableSequence
from collections.abc import Iterable
from collections.abc import Mapping
from collections.abc import MutableMapping
from collections.abc import MutableSequence
import socket
import fnmatch
import contextlib
......@@ -106,7 +107,7 @@ def grouped_with_tail(iterable, n):
def flatten_gen(items):
"""Yield items from any nested iterable; see Reference."""
for x in items:
if isinstance(x, collections.abc.Iterable) and not isinstance(x, (str, bytes)):
if isinstance(x, Iterable) and not isinstance(x, (str, bytes)):
for sub_x in flatten(x):
yield sub_x
else:
......@@ -610,7 +611,7 @@ def deep_update(d, u):
while stack:
d, u = stack.pop(0)
for k, v in u.items():
if not isinstance(v, collections.abc.Mapping):
if not isinstance(v, Mapping):
# u[k] is not a dict, nothing to merge, so just set it,
# regardless if d[k] *was* a dict
d[k] = v
......@@ -621,7 +622,7 @@ def deep_update(d, u):
# exist
dv = d.setdefault(k, {})
if not isinstance(dv, collections.abc.Mapping):
if not isinstance(dv, Mapping):
# d[k] is not a dict, so just set it to u[k],
# overriding whatever it was
d[k] = v
......@@ -720,10 +721,10 @@ def prudent_update(d, u):
def update_node_info(node, d):
"""updates the BaseHashSetting of a DataNode and does a deep update if needed.
parameters: node: DataNode or DataNodeContainer; d: dict"""
assert type(d) == dict
assert isinstance(d, Mapping)
for key, value in d.items():
tmp = node.info.get(key)
if tmp and type(value) == dict and type(tmp) == dict:
if tmp and isinstance(value, Mapping) and isinstance(tmp, Mapping):
deep_update(tmp, value)
node.info[key] = tmp
else:
......
......@@ -6,7 +6,7 @@
# Distributed under the GNU LGPLv3. See LICENSE for more info.
from bliss.icat.definitions import Definitions
from os.path import commonprefix
from bliss.common.protocols import IcatPublisher
from bliss.common.protocols import HasMetadataForDataset
from collections.abc import MutableSequence
......@@ -19,9 +19,9 @@ class ICATmeta:
self.definitions = Definitions()
for key, obj in self.objects.items():
if not isinstance(obj, IcatPublisher):
if not isinstance(obj, HasMetadataForDataset):
raise RuntimeError(
f"{obj.name} ({key}) is not a valid metadata publisher!"
f"{obj.name} ({key}) does not implement the 'HasMetadataForDataset' protocol"
)
def get_metadata(self):
......@@ -37,13 +37,12 @@ class ICATmeta:
instrumentation = self.definitions.instrumentation._asdict()
for key, device in self.objects.items():
assert key in instrumentation, f"{key} is not a known icat field group"
assert hasattr(
device, "metadata"
), f"{device.name} has no metadata function"
prefix = commonprefix(list(instrumentation[key].fields)).strip("_")
# have to deal with cases where there is no tailing `_` in the prefix
# e.g. attenuator positions
obj_meta = device.metadata()
obj_meta = device.dataset_metadata()
if not obj_meta:
continue
for icat_key in instrumentation[key].fields:
obj_key = icat_key.split(prefix)[-1].strip("_")
if obj_key in obj_meta:
......
......@@ -9,6 +9,7 @@ from typing import Iterable
import typeguard
import numpy
from bliss import global_map
from bliss.common.counter import Counter
from bliss.config.beacon_object import BeaconObject
from bliss.common.logtools import log_debug
......@@ -159,8 +160,12 @@ def _to_list(setting, value):
class LimaImageParameters(BeaconObject):
def __init__(self, config, name):
def __init__(self, controller, name):
config = controller._config_node
super().__init__(config, name=name, share_hardware=False, path=["image"])
# properly put in map, to have "parameters" under the corresponding Lima controller node
# (and not in "controllers")
global_map.register(self, parents_list=[controller], tag="image_parameters")
binning = BeaconObject.property_setting(
"binning", default=[1, 1], set_marshalling=_to_list, set_unmarshalling=_to_list
......@@ -231,7 +236,7 @@ class ImageCounter(Counter):
super().__init__("image", controller)
self._image_params = LimaImageParameters(
controller._config_node, f"{controller._name_prefix}:image"
controller, f"{controller._name_prefix}:image"
)
def __info__(self):
......
......@@ -15,7 +15,7 @@ from bliss.common.tango import DeviceProxy, DevFailed, Database, DevState
from bliss.config import settings
from bliss.config.beacon_object import BeaconObject
from bliss.common.logtools import log_debug
from bliss.common.protocols import HasMetadataForScan
from bliss.common.protocols import HasMetadataForScanExclusive
from bliss.controllers.counter import CounterController, counter_namespace
from bliss import current_session
......@@ -81,7 +81,7 @@ class ChangeTangoTimeout(object):
self.__device.set_timeout_millis(self.__back_timeout)
class Lima(CounterController, HasMetadataForScan):
class Lima(CounterController, HasMetadataForScanExclusive):
"""
Lima controller.
Basic configuration:
......@@ -156,7 +156,7 @@ class Lima(CounterController, HasMetadataForScan):
self, parents_list=["lima", "controllers"], children_list=[self._proxy]
)
def metadata_when_prepared(self) -> dict:
def scan_metadata(self) -> dict:
return {"type": "lima"}
@property
......@@ -580,6 +580,9 @@ class Lima(CounterController, HasMetadataForScan):
def image(self):
if self._image is None:
self._image = ImageCounter(self)
global_map.register(
self._image, parents_list=[self], children_list=[self._proxy]
)
return self._image
@autocomplete_property
......
......@@ -421,7 +421,7 @@ class RoiStatCounter(IntegratingCounter):
name = f"{self.roi_name}_{stat.name.lower()}"
super().__init__(name, kwargs.pop("controller"), **kwargs)
def get_metadata(self):
def scan_metadata(self):
return {self.roi_name: self._counter_controller.get(self.roi_name).to_dict()}
def __int__(self):
......@@ -480,7 +480,7 @@ class RoiProfileCounter(IntegratingCounter):
self.roi_name = roi_name
super().__init__(roi_name, controller, conversion_function, unit)
def get_metadata(self):
def scan_metadata(self):
return {self.roi_name: self._counter_controller.get(self.roi_name).to_dict()}
@property
......@@ -508,7 +508,7 @@ class RoiCollectionCounter(IntegratingCounter):
def __init__(self, name, controller):
super().__init__(name, controller)
def get_metadata(self):
def scan_metadata(self):
params = [roi.get_params() for roi in self._counter_controller.get_rois()]
xs, ys, ws, hs = zip(*params)
meta = {"kind": "collection", "x": xs, "y": ys, "width": ws, "height": hs}
......
......@@ -8,6 +8,7 @@ import enum
import gevent
import functools
from tabulate import tabulate
import warnings
from bliss import global_map
from bliss.config.beacon_object import BeaconObject
......@@ -18,17 +19,17 @@ from bliss.common.logtools import user_print
from bliss.common import timedisplay
from bliss.controllers.counter import counter_namespace
from bliss.scanning.scan_meta import get_user_scan_meta
from bliss.scanning.chain import ChainPreset, ChainIterationPreset
from bliss.common import tango
from bliss.common.protocols import IcatPublisher
from bliss.common.protocols import HasMetadataForDataset
from bliss.scanning.scan_meta import HasMetadataForScan
from bliss.controllers.tango_attr_as_counter import (
TangoCounterController,
TangoAttrCounter,
)
class MachInfo(BeaconObject, IcatPublisher):
class MachInfo(BeaconObject, HasMetadataForScan, HasMetadataForDataset):
""" Access to accelerator information.
- SR_Current
- SR_Lifetime
......@@ -135,7 +136,58 @@ class MachInfo(BeaconObject, IcatPublisher):
self.__check = False
user_print("Removing Wait For Refill on scans")
@BeaconObject.property(default=True)
def dataset_metadata(self):
attributes = ["SR_Mode", "SR_Current"]
attributes = {
attr_name: value
for attr_name, value in zip(attributes, self._read_attributes(attributes))
}
return {
"mode": self.SRMODE(attributes["SR_Mode"]).name,
"current": attributes["SR_Current"],
}
@property
def scan_metadata_name(self):
return "machine"
def scan_metadata(self):
attributes = [
"SR_Mode",
"SR_Filling_Mode",
"SR_Single_Bunch_Current",
"SR_Current",
"Automatic_Mode",
"FE_State",
"SR_Refill_Countdown",
"SR_Operator_Mesg",
]
attributes = {
attr_name: value
for attr_name, value in zip(attributes, self._read_attributes(attributes))
}
# Standard:
meta_dict = {
"@NX_class": "NXsource",
"name": "ESRF",
"type": "Synchrotron",
"mode": self.SRMODE(attributes["SR_Mode"]).name,
"current": attributes["SR_Current"],
"current@units": "mA",
}
# Non-standard:
if attributes["SR_Filling_Mode"] == "1 bunch":
meta_dict["single_bunch_current"] = attributes["SR_Single_Bunch_Current"]
meta_dict["single_bunch_current@units"] = "mA"
meta_dict["filling_mode"] = attributes["SR_Filling_Mode"]
meta_dict["automatic_mode"] = attributes["Automatic_Mode"]
meta_dict["front_end"] = attributes["FE_State"]
meta_dict["refill_countdown"] = attributes["SR_Refill_Countdown"]
meta_dict["refill_countdown@units"] = "s"
meta_dict["message"] = attributes["SR_Operator_Mesg"]
return meta_dict
@BeaconObject.property()
def metadata(self):
"""
Insert machine info metadata's for any scans
......@@ -144,50 +196,12 @@ class MachInfo(BeaconObject, IcatPublisher):
@metadata.setter
def metadata(self, flag):
def get_meta(scan):
attributes = [
"SR_Mode",
"SR_Filling_Mode",
"SR_Single_Bunch_Current",
"SR_Current",
"Automatic_Mode",
"FE_State",
"SR_Refill_Countdown",
"SR_Operator_Mesg",
]
attributes = {
attr_name: value
for attr_name, value in zip(
attributes, self._read_attributes(attributes)
)
}
# Standard:
meta_dict = {
"@NX_class": "NXsource",
"name": "ESRF",
"type": "Synchrotron",
"mode": self.SRMODE(attributes["SR_Mode"]).name,
"current": attributes["SR_Current"],
"current@units": "mA",
}
# Non-standard:
if attributes["SR_Filling_Mode"] == "1 bunch":
meta_dict["single_bunch_current"] = attributes[
"SR_Single_Bunch_Current"
]
meta_dict["single_bunch_current@units"] = "mA"
meta_dict["filling_mode"] = attributes["SR_Filling_Mode"]
meta_dict["automatic_mode"] = attributes["Automatic_Mode"]
meta_dict["front_end"] = attributes["FE_State"]
meta_dict["refill_countdown"] = attributes["SR_Refill_Countdown"]
meta_dict["refill_countdown@units"] = "s"
meta_dict["message"] = attributes["SR_Operator_Mesg"]
return {"machine": meta_dict}
if flag:
get_user_scan_meta().instrument.set(self.KEY_NAME, get_meta)
warnings.warn("Use 'MachInfo.enable_scan_metadata' instead", FutureWarning)
self.enable_scan_metadata()
else:
get_user_scan_meta().instrument.remove(self.KEY_NAME)
warnings.warn("Use 'MachInfo.disable_scan_metadata' instead", FutureWarning)
self.disable_scan_metadata()
def iter_wait_for_refill(self, checktime, waittime=0., polling_time=1.):
"""
......
......@@ -19,9 +19,10 @@ import tabulate
import gevent
from bliss import global_map
from bliss.controllers.mca.roi import RoiConfig
from bliss.common.logtools import log_debug
from bliss.common.protocols import HasMetadataForScan
from bliss.common.protocols import HasMetadataForScanExclusive
from bliss.common.utils import autocomplete_property
from bliss.config.beacon_object import BeaconObject
from bliss.controllers.counter import CounterController
......@@ -97,12 +98,14 @@ class MCABeaconObject(BeaconObject):
# Base class
class BaseMCA(CounterController, HasMetadataForScan):
class BaseMCA(CounterController, HasMetadataForScanExclusive):
"""Generic MCA controller."""
# Life cycle
def __init__(self, name, config, beacon_obj_class=MCABeaconObject):
global_map.register(self) # register as controller
CounterController.__init__(self, name)
self.beacon_obj = beacon_obj_class(self, config)
......@@ -113,7 +116,7 @@ class BaseMCA(CounterController, HasMetadataForScan):
self.initialize_attributes()
self.initialize_hardware()
def metadata_when_prepared(self) -> dict:
def scan_metadata(self) -> dict:
return {"type": "mca"}
def get_acquisition_object(self, acq_params, ctrl_params, parent_acq_params):
......
......@@ -508,7 +508,7 @@ class Controller:
class CalcController(Controller):
def __init__(self, *args, **kwargs):
Controller.__init__(self, *args, **kwargs)
super().__init__(*args, **kwargs)
self.axis_settings.config_setting["velocity"] = False
self.axis_settings.config_setting["acceleration"] = False
......
......@@ -51,13 +51,13 @@ import time
import math
import gevent
from bliss.scanning.scan_meta import get_user_scan_meta
from bliss.common.protocols import HasMetadataForScan
from bliss.controllers.motor import Controller
from bliss.common.axis import AxisState
from bliss.common.tango import DevState, DeviceProxy
from bliss import global_map
from bliss.common.logtools import *
from bliss.common.logtools import log_error, user_print
from bliss.shell.cli.user_dialog import (
UserMsg,
......@@ -72,7 +72,7 @@ from bliss.shell.cli.pt_widgets import display, BlissDialog
__author__ = "Jens Meyer / Gilles Berruyer - ESRF ISDD SOFTGROUP BLISS - June 2019"
class esrf_hexapode(Controller):
class esrf_hexapode(Controller, HasMetadataForScan):
""" Class to implement BLISS motor controller of esrf hexapode controlled
via tango device server
"""
......@@ -80,8 +80,6 @@ class esrf_hexapode(Controller):
def __init__(self, *args, **kwargs):
Controller.__init__(self, *args, **kwargs)
global_map.register(self)
self.device = None
self.roles = {}
self.last_read = None
......@@ -95,31 +93,17 @@ class esrf_hexapode(Controller):
log_error(self, _err_msg)
raise RuntimeError(_err_msg)
self._init_meta_data_publishing()
@property
def scan_metadata_name(self):
return self.name
def _init_meta_data_publishing(self):
"""this is about metadata publishing to the h5 file"""
if not self.name:
user_warning(
"to publish metadata the hexapode controller needs a name in config"
)
return
scan_meta_obj = get_user_scan_meta()
scan_meta_obj.instrument.set(self, lambda _: {self.name: self.metadata()})
def scan_metadata(self):
meta_dict = {"@NX_class": "NXhexapode"}
def metadata(self):
"""
this is about metadata publishing to the h5 file AND ICAT
"""