Merge branch '2769-scan-and-dataset-metadata' into 'master'

Resolve "Scan and dataset metadata"

Closes #2769

See merge request !3724

bliss/common/auto_filter/acquisition_objects.py

@@ -122,11 +122,8 @@ class _Base:
     def wait_ready(self):
         return self.device.wait_ready()
 
-    def fill_meta_at_scan_start(self, scan_meta):
-        return self.device.fill_meta_at_scan_start(scan_meta)
-
-    def fill_meta_at_scan_end(self, scan_meta):
-        return self.device.fill_meta_at_scan_end(scan_meta)
+    def get_acquisition_metadata(self, *args, **kw):
+        return self.device.get_acquisition_metadata(*args, **kw)
 
     def _all_point_rx(self):
         """

bliss/common/counter.py

@@ -15,6 +15,7 @@ import inspect
 import numpy
 
 from bliss.common.utils import autocomplete_property
+from bliss.common.protocols import HasMetadataForScan
 
 
 def add_conversion_function(obj, method_name, function):
@@ -58,7 +59,7 @@ def _identity(val):
     return val
 
 
-class Counter:
+class Counter(HasMetadataForScan):
     """ Counter class """
 
     def __init__(self, name, controller, conversion_function=None, unit=None):
@@ -122,8 +123,8 @@ class Counter:
         assert callable(func)
         self._conversion_function = func
 
-    def get_metadata(self):
-        return {}
+    def scan_metadata(self):
+        return None
 
     def __info__(self, counter_type=None):
         info_str = f"'{self.name}` counter info:\n"

bliss/common/protocols.py

@@ -8,10 +8,13 @@
 """
 This file groups all protocols managed by bliss
 """
+import weakref
+
 from abc import ABC
 from collections import namedtuple
 from types import SimpleNamespace
 from typing import Mapping
+from typing import Union
 
 
 class IterableNamespace(SimpleNamespace):
@@ -104,28 +107,67 @@ class Scannable(ABC):
         raise NotImplementedError
 
 
-class IcatPublisher(ABC):
+class HasMetadataForDataset(ABC):
     """
-    Any controller that has this interface can be used
-    for metadata collection based on the `icat-mapping`
-    tag in the session configuration
+    Any controller which provides metadata intended to be saved
+    during a dataset life cycle.
+
+    The `dataset_metadata` is called by the Bliss session's icat_mapping
+    object when the session has such a mapping configured.
     """
 
-    def metadata(self) -> dict:
+    def dataset_metadata(self) -> Union[dict, None]:
         """
-        Return a dict containing metadata
+        Returning an empty dictionary means the controller has metadata
+        but no values. `None` means the controller has no metadata.
         """
         raise NotImplementedError
 
 
 class HasMetadataForScan(ABC):
     """
-    Any controller which provides metadata during a scan life cycle.
+    Any controller which provides metadata intended to be saved
+    during a scan life cycle.
     """
 
-    def metadata_when_prepared(self) -> dict:
+    disabled_controllers = weakref.WeakKeyDictionary()
+
+    def disable_scan_metadata(self):
+        HasMetadataForScan.disabled_controllers[self] = True
+
+    @property
+    def scan_metadata_enabled(self):
+        return not HasMetadataForScan.disabled_controllers.get(self)
+
+    def enable_scan_metadata(self):
+        try:
+            HasMetadataForScan.disabled_controllers.pop(self)
+        except KeyError:
+            pass
+
+    def scan_metadata(self) -> Union[dict, None]:
         """
-        Return a dict containing metadata when the device was prepared by the
-        scan.
+        Returning an empty dictionary means the controller has metadata
+        but no values. `None` means the controller has no metadata.
         """
         raise NotImplementedError
+
+    @property
+    def scan_metadata_name(self) -> Union[str, None]:
+        """
+        Default implementation returns self.name, can be overwritten in derived classes
+        Returns None when there is no name
+        """
+        try:
+            return self.name
+        except AttributeError:
+            return None
+
+
+class HasMetadataForScanExclusive(HasMetadataForScan):
+    """
+    Any controller which provides metadata intended to be saved
+    during a scan life cycle when used in the acquisition chain.
+    """
+
+    pass

bliss/common/scans/ct.py

@@ -92,8 +92,6 @@ def ct(
         scan_info=scan_info,
     )
 
-    s._update_scan_info_with_user_scan_meta = lambda _: None
-
     if run:
         s.run()
 

bliss/common/scans/simulation.py

@@ -37,6 +37,8 @@ from bliss.common.image_tools import (
     test_image,
 )
 
+from bliss.common.protocols import HasMetadataForScan
+
 
 # CHAIN OBJECT NOTES:
 #
@@ -396,10 +398,17 @@ class FakeAcquisitionSlave(AcquisitionSlave):
         # self.channels.update_from_array(data)
         # self.channels.update({self.chname: self.positions})
 
-    def fill_meta_at_scan_start(self, scan_meta):
-        tmp_dict = {}
+    def get_acquisition_metadata(self, timing=None):
+        tmp_dict = super().get_acquisition_metadata(timing=timing)
+        if timing != self.META_TIMING.PREPARED:
+            return tmp_dict
         for cnt in self._counters:
-            deep_update(tmp_dict, cnt.get_metadata())
+            if isinstance(cnt, HasMetadataForScan):
+                mdata = cnt.scan_metadata()
+                if mdata is not None:
+                    if tmp_dict is None:
+                        tmp_dict = dict()
+                    deep_update(tmp_dict, mdata)
         return tmp_dict
 
     def prepare(self):

bliss/common/utils.py

@@ -15,10 +15,11 @@ import types
 import itertools
 import functools
 import numpy
-import collections.abc
 import importlib.util
-import distutils.util
-from collections.abc import MutableMapping, MutableSequence
+from collections.abc import Iterable
+from collections.abc import Mapping
+from collections.abc import MutableMapping
+from collections.abc import MutableSequence
 import socket
 import fnmatch
 import contextlib
@@ -106,7 +107,7 @@ def grouped_with_tail(iterable, n):
 def flatten_gen(items):
     """Yield items from any nested iterable; see Reference."""
     for x in items:
-        if isinstance(x, collections.abc.Iterable) and not isinstance(x, (str, bytes)):
+        if isinstance(x, Iterable) and not isinstance(x, (str, bytes)):
             for sub_x in flatten(x):
                 yield sub_x
         else:
@@ -610,7 +611,7 @@ def deep_update(d, u):
     while stack:
         d, u = stack.pop(0)
         for k, v in u.items():
-            if not isinstance(v, collections.abc.Mapping):
+            if not isinstance(v, Mapping):
                 # u[k] is not a dict, nothing to merge, so just set it,
                 # regardless if d[k] *was* a dict
                 d[k] = v
@@ -621,7 +622,7 @@ def deep_update(d, u):
                 # exist
                 dv = d.setdefault(k, {})
 
-                if not isinstance(dv, collections.abc.Mapping):
+                if not isinstance(dv, Mapping):
                     # d[k] is not a dict, so just set it to u[k],
                     # overriding whatever it was
                     d[k] = v
@@ -720,10 +721,10 @@ def prudent_update(d, u):
 def update_node_info(node, d):
     """updates the BaseHashSetting of a DataNode and does a deep update if needed. 
     parameters: node: DataNode or DataNodeContainer; d: dict"""
-    assert type(d) == dict
+    assert isinstance(d, Mapping)
     for key, value in d.items():
         tmp = node.info.get(key)
-        if tmp and type(value) == dict and type(tmp) == dict:
+        if tmp and isinstance(value, Mapping) and isinstance(tmp, Mapping):
             deep_update(tmp, value)
             node.info[key] = tmp
         else:

bliss/controllers/icatmeta.py

@@ -6,7 +6,7 @@
 # Distributed under the GNU LGPLv3. See LICENSE for more info.
 from bliss.icat.definitions import Definitions
 from os.path import commonprefix
-from bliss.common.protocols import IcatPublisher
+from bliss.common.protocols import HasMetadataForDataset
 from collections.abc import MutableSequence
 
 
@@ -19,9 +19,9 @@ class ICATmeta:
         self.definitions = Definitions()
 
         for key, obj in self.objects.items():
-            if not isinstance(obj, IcatPublisher):
+            if not isinstance(obj, HasMetadataForDataset):
                 raise RuntimeError(
-                    f"{obj.name} ({key}) is not a valid metadata publisher!"
+                    f"{obj.name} ({key}) does not implement the 'HasMetadataForDataset' protocol"
                 )
 
     def get_metadata(self):
@@ -37,13 +37,12 @@ class ICATmeta:
         instrumentation = self.definitions.instrumentation._asdict()
         for key, device in self.objects.items():
             assert key in instrumentation, f"{key} is not a known icat field group"
-            assert hasattr(
-                device, "metadata"
-            ), f"{device.name} has no metadata function"
             prefix = commonprefix(list(instrumentation[key].fields)).strip("_")
             # have to deal with cases where there is no tailing `_` in the prefix
             # e.g. attenuator positions
-            obj_meta = device.metadata()
+            obj_meta = device.dataset_metadata()
+            if not obj_meta:
+                continue
             for icat_key in instrumentation[key].fields:
                 obj_key = icat_key.split(prefix)[-1].strip("_")
                 if obj_key in obj_meta:

bliss/controllers/lima/image.py

@@ -9,6 +9,7 @@ from typing import Iterable
 import typeguard
 import numpy
 
+from bliss import global_map
 from bliss.common.counter import Counter
 from bliss.config.beacon_object import BeaconObject
 from bliss.common.logtools import log_debug
@@ -159,8 +160,12 @@ def _to_list(setting, value):
 
 
 class LimaImageParameters(BeaconObject):
-    def __init__(self, config, name):
+    def __init__(self, controller, name):
+        config = controller._config_node
         super().__init__(config, name=name, share_hardware=False, path=["image"])
+        # properly put in map, to have "parameters" under the corresponding Lima controller node
+        # (and not in "controllers")
+        global_map.register(self, parents_list=[controller], tag="image_parameters")
 
     binning = BeaconObject.property_setting(
         "binning", default=[1, 1], set_marshalling=_to_list, set_unmarshalling=_to_list
@@ -231,7 +236,7 @@ class ImageCounter(Counter):
         super().__init__("image", controller)
 
         self._image_params = LimaImageParameters(
-            controller._config_node, f"{controller._name_prefix}:image"
+            controller, f"{controller._name_prefix}:image"
         )
 
     def __info__(self):

bliss/controllers/lima/lima_base.py

@@ -15,7 +15,7 @@ from bliss.common.tango import DeviceProxy, DevFailed, Database, DevState
 from bliss.config import settings
 from bliss.config.beacon_object import BeaconObject
 from bliss.common.logtools import log_debug
-from bliss.common.protocols import HasMetadataForScan
+from bliss.common.protocols import HasMetadataForScanExclusive
 
 from bliss.controllers.counter import CounterController, counter_namespace
 from bliss import current_session
@@ -81,7 +81,7 @@ class ChangeTangoTimeout(object):
         self.__device.set_timeout_millis(self.__back_timeout)
 
 
-class Lima(CounterController, HasMetadataForScan):
+class Lima(CounterController, HasMetadataForScanExclusive):
     """
     Lima controller.
     Basic configuration:
@@ -156,7 +156,7 @@ class Lima(CounterController, HasMetadataForScan):
             self, parents_list=["lima", "controllers"], children_list=[self._proxy]
         )
 
-    def metadata_when_prepared(self) -> dict:
+    def scan_metadata(self) -> dict:
         return {"type": "lima"}
 
     @property
@@ -580,6 +580,9 @@ class Lima(CounterController, HasMetadataForScan):
     def image(self):
         if self._image is None:
             self._image = ImageCounter(self)
+            global_map.register(
+                self._image, parents_list=[self], children_list=[self._proxy]
+            )
         return self._image
 
     @autocomplete_property

bliss/controllers/lima/roi.py

@@ -421,7 +421,7 @@ class RoiStatCounter(IntegratingCounter):
         name = f"{self.roi_name}_{stat.name.lower()}"
         super().__init__(name, kwargs.pop("controller"), **kwargs)
 
-    def get_metadata(self):
+    def scan_metadata(self):
         return {self.roi_name: self._counter_controller.get(self.roi_name).to_dict()}
 
     def __int__(self):
@@ -480,7 +480,7 @@ class RoiProfileCounter(IntegratingCounter):
         self.roi_name = roi_name
         super().__init__(roi_name, controller, conversion_function, unit)
 
-    def get_metadata(self):
+    def scan_metadata(self):
         return {self.roi_name: self._counter_controller.get(self.roi_name).to_dict()}
 
     @property
@@ -508,7 +508,7 @@ class RoiCollectionCounter(IntegratingCounter):
     def __init__(self, name, controller):
         super().__init__(name, controller)
 
-    def get_metadata(self):
+    def scan_metadata(self):
         params = [roi.get_params() for roi in self._counter_controller.get_rois()]
         xs, ys, ws, hs = zip(*params)
         meta = {"kind": "collection", "x": xs, "y": ys, "width": ws, "height": hs}

bliss/controllers/machinfo.py

@@ -8,6 +8,7 @@ import enum
 import gevent
 import functools
 from tabulate import tabulate
+import warnings
 
 from bliss import global_map
 from bliss.config.beacon_object import BeaconObject
@@ -18,17 +19,17 @@ from bliss.common.logtools import user_print
 from bliss.common import timedisplay
 from bliss.controllers.counter import counter_namespace
 
-from bliss.scanning.scan_meta import get_user_scan_meta
 from bliss.scanning.chain import ChainPreset, ChainIterationPreset
 from bliss.common import tango
-from bliss.common.protocols import IcatPublisher
+from bliss.common.protocols import HasMetadataForDataset
+from bliss.scanning.scan_meta import HasMetadataForScan
 from bliss.controllers.tango_attr_as_counter import (
     TangoCounterController,
     TangoAttrCounter,
 )
 
 
-class MachInfo(BeaconObject, IcatPublisher):
+class MachInfo(BeaconObject, HasMetadataForScan, HasMetadataForDataset):
     """ Access to accelerator information.
     - SR_Current
     - SR_Lifetime
@@ -135,7 +136,58 @@ class MachInfo(BeaconObject, IcatPublisher):
             self.__check = False
             user_print("Removing Wait For Refill on scans")
 
-    @BeaconObject.property(default=True)
+    def dataset_metadata(self):
+        attributes = ["SR_Mode", "SR_Current"]
+        attributes = {
+            attr_name: value
+            for attr_name, value in zip(attributes, self._read_attributes(attributes))
+        }
+        return {
+            "mode": self.SRMODE(attributes["SR_Mode"]).name,
+            "current": attributes["SR_Current"],
+        }
+
+    @property
+    def scan_metadata_name(self):
+        return "machine"
+
+    def scan_metadata(self):
+        attributes = [
+            "SR_Mode",
+            "SR_Filling_Mode",
+            "SR_Single_Bunch_Current",
+            "SR_Current",
+            "Automatic_Mode",
+            "FE_State",
+            "SR_Refill_Countdown",
+            "SR_Operator_Mesg",
+        ]
+        attributes = {
+            attr_name: value
+            for attr_name, value in zip(attributes, self._read_attributes(attributes))
+        }
+        # Standard:
+        meta_dict = {
+            "@NX_class": "NXsource",
+            "name": "ESRF",
+            "type": "Synchrotron",
+            "mode": self.SRMODE(attributes["SR_Mode"]).name,
+            "current": attributes["SR_Current"],
+            "current@units": "mA",
+        }
+        # Non-standard:
+        if attributes["SR_Filling_Mode"] == "1 bunch":
+            meta_dict["single_bunch_current"] = attributes["SR_Single_Bunch_Current"]
+            meta_dict["single_bunch_current@units"] = "mA"
+        meta_dict["filling_mode"] = attributes["SR_Filling_Mode"]
+        meta_dict["automatic_mode"] = attributes["Automatic_Mode"]
+        meta_dict["front_end"] = attributes["FE_State"]
+        meta_dict["refill_countdown"] = attributes["SR_Refill_Countdown"]
+        meta_dict["refill_countdown@units"] = "s"
+        meta_dict["message"] = attributes["SR_Operator_Mesg"]
+        return meta_dict
+
+    @BeaconObject.property()
     def metadata(self):
         """
         Insert machine info metadata's for any scans
@@ -144,50 +196,12 @@ class MachInfo(BeaconObject, IcatPublisher):
 
     @metadata.setter
     def metadata(self, flag):
-        def get_meta(scan):
-            attributes = [
-                "SR_Mode",
-                "SR_Filling_Mode",
-                "SR_Single_Bunch_Current",
-                "SR_Current",
-                "Automatic_Mode",
-                "FE_State",
-                "SR_Refill_Countdown",
-                "SR_Operator_Mesg",
-            ]
-            attributes = {
-                attr_name: value
-                for attr_name, value in zip(
-                    attributes, self._read_attributes(attributes)
-                )
-            }
-            # Standard:
-            meta_dict = {
-                "@NX_class": "NXsource",
-                "name": "ESRF",
-                "type": "Synchrotron",
-                "mode": self.SRMODE(attributes["SR_Mode"]).name,
-                "current": attributes["SR_Current"],
-                "current@units": "mA",
-            }
-            # Non-standard:
-            if attributes["SR_Filling_Mode"] == "1 bunch":
-                meta_dict["single_bunch_current"] = attributes[
-                    "SR_Single_Bunch_Current"
-                ]
-                meta_dict["single_bunch_current@units"] = "mA"
-            meta_dict["filling_mode"] = attributes["SR_Filling_Mode"]
-            meta_dict["automatic_mode"] = attributes["Automatic_Mode"]
-            meta_dict["front_end"] = attributes["FE_State"]
-            meta_dict["refill_countdown"] = attributes["SR_Refill_Countdown"]
-            meta_dict["refill_countdown@units"] = "s"
-            meta_dict["message"] = attributes["SR_Operator_Mesg"]
-            return {"machine": meta_dict}
-
         if flag:
-            get_user_scan_meta().instrument.set(self.KEY_NAME, get_meta)
+            warnings.warn("Use 'MachInfo.enable_scan_metadata' instead", FutureWarning)
+            self.enable_scan_metadata()
         else:
-            get_user_scan_meta().instrument.remove(self.KEY_NAME)
+            warnings.warn("Use 'MachInfo.disable_scan_metadata' instead", FutureWarning)
+            self.disable_scan_metadata()
 
     def iter_wait_for_refill(self, checktime, waittime=0., polling_time=1.):
         """

bliss/controllers/mca/base.py

@@ -19,9 +19,10 @@ import tabulate
 
 import gevent
 
+from bliss import global_map
 from bliss.controllers.mca.roi import RoiConfig
 from bliss.common.logtools import log_debug
-from bliss.common.protocols import HasMetadataForScan
+from bliss.common.protocols import HasMetadataForScanExclusive
 from bliss.common.utils import autocomplete_property
 from bliss.config.beacon_object import BeaconObject
 from bliss.controllers.counter import CounterController
@@ -97,12 +98,14 @@ class MCABeaconObject(BeaconObject):
 
 
 # Base class
-class BaseMCA(CounterController, HasMetadataForScan):
+class BaseMCA(CounterController, HasMetadataForScanExclusive):
     """Generic MCA controller."""
 
     # Life cycle
 
     def __init__(self, name, config, beacon_obj_class=MCABeaconObject):
+        global_map.register(self)  # register as controller
+
         CounterController.__init__(self, name)
 
         self.beacon_obj = beacon_obj_class(self, config)
@@ -113,7 +116,7 @@ class BaseMCA(CounterController, HasMetadataForScan):
         self.initialize_attributes()
         self.initialize_hardware()
 
-    def metadata_when_prepared(self) -> dict:
+    def scan_metadata(self) -> dict:
         return {"type": "mca"}
 
     def get_acquisition_object(self, acq_params, ctrl_params, parent_acq_params):

bliss/controllers/motor.py

@@ -508,7 +508,7 @@ class Controller:
 
 class CalcController(Controller):
     def __init__(self, *args, **kwargs):
-        Controller.__init__(self, *args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         self.axis_settings.config_setting["velocity"] = False
         self.axis_settings.config_setting["acceleration"] = False

bliss/controllers/motors/esrf_hexapode.py

@@ -51,13 +51,13 @@ import time
 import math
 import gevent
 
-from bliss.scanning.scan_meta import get_user_scan_meta
+from bliss.common.protocols import HasMetadataForScan
 from bliss.controllers.motor import Controller
 from bliss.common.axis import AxisState
 from bliss.common.tango import DevState, DeviceProxy
 
 from bliss import global_map
-from bliss.common.logtools import *
+from bliss.common.logtools import log_error, user_print
 
 from bliss.shell.cli.user_dialog import (
     UserMsg,
@@ -72,7 +72,7 @@ from bliss.shell.cli.pt_widgets import display, BlissDialog
 __author__ = "Jens Meyer / Gilles Berruyer - ESRF ISDD SOFTGROUP BLISS - June 2019"
 
 
-class esrf_hexapode(Controller):
+class esrf_hexapode(Controller, HasMetadataForScan):
     """ Class to implement BLISS motor controller of esrf hexapode controlled
     via tango device server
     """
@@ -80,8 +80,6 @@ class esrf_hexapode(Controller):
     def __init__(self, *args, **kwargs):
         Controller.__init__(self, *args, **kwargs)
 
-        global_map.register(self)
-
         self.device = None
         self.roles = {}
         self.last_read = None
@@ -95,31 +93,17 @@ class esrf_hexapode(Controller):
             log_error(self, _err_msg)
             raise RuntimeError(_err_msg)
 
-        self._init_meta_data_publishing()
+    @property
+    def scan_metadata_name(self):
+        return self.name
 
-    def _init_meta_data_publishing(self):
-        """this is about metadata publishing to the h5 file"""
-        if not self.name:
-            user_warning(
-                "to publish metadata the hexapode controller needs a name in config"
-            )
-            return
-        scan_meta_obj = get_user_scan_meta()
-        scan_meta_obj.instrument.set(self, lambda _: {self.name: self.metadata()})
+    def scan_metadata(self):
+        meta_dict = {"@NX_class": "NXhexapode"}
 
-    def metadata(self):
-        """ 
-        this is about metadata publishing to the h5 file AND ICAT
-        """
-