new "get_controllers_scan_meta" function, to return metadata from controllers using the global map

bliss/common/protocols.py

@@ -8,6 +8,8 @@
 """
 This file groups all protocols managed by bliss
 """
+import weakref
+
 from abc import ABC
 from collections import namedtuple
 from types import SimpleNamespace
@@ -131,18 +133,20 @@ class HasMetadataForScan(ABC):
     objects `AcquisitionObject` (directly or indirectly).
     """
 
+    disabled_controllers = weakref.WeakKeyDictionary()
+
     def disable_scan_metadata(self):
-        self.__disabled_scan_metadata = True
+        HasMetadataForScan.disabled_controllers[self] = True
 
     @property
     def scan_metadata_enabled(self):
-        try:
-            return not self.__disabled_scan_metadata
-        except AttributeError:
-            return True
+        return not HasMetadataForScan.disabled_controllers.get(self)
 
     def enable_scan_metadata(self):
-        self.__disabled_scan_metadata = False
+        try:
+            HasMetadataForScan.disabled_controllers.pop(self)
+        except KeyError:
+            pass
 
     def scan_metadata(self) -> Union[dict, None]:
         """
@@ -161,10 +165,3 @@ class HasMetadataForScan(ABC):
             return self.name
         except AttributeError:
             return None
-
-    @property
-    def strict_scan_metadata(self):
-        """
-        Return whether metadata has to be reported only if the controller is involved in the scan
-        """
-        return False

bliss/common/scans/ct.py

@@ -92,7 +92,7 @@ def ct(
         scan_info=scan_info,
     )
 
-    s._update_scan_info_with_user_scan_meta = lambda _: None
+    s._update_scan_info_with_scan_meta = lambda _: None
 
     if run:
         s.run()

bliss/controllers/lima/lima_base.py

@@ -156,10 +156,6 @@ class Lima(CounterController, HasMetadataForScan):
             self, parents_list=["lima", "controllers"], children_list=[self._proxy]
         )
 
-    @property
-    def strict_scan_metadata(self):
-        return True
-
     def scan_metadata(self) -> dict:
         return {"type": "lima"}
 

bliss/controllers/mca/base.py

@@ -116,10 +116,6 @@ class BaseMCA(CounterController, HasMetadataForScan):
         self.initialize_attributes()
         self.initialize_hardware()
 
-    @property
-    def strict_scan_metadata(self):
-        return True
-
     def scan_metadata(self) -> dict:
         return {"type": "mca"}
 

bliss/scanning/chain.py

@@ -461,7 +461,7 @@ class AcquisitionObject:
 
     def fill_meta_at_scan_start(self, scan_meta):
         """
-        In this method, acquisition device should collect and meta data
+        In this method, acquisition device should collect any meta data
         related to this device and prepare it for publishing. it is called 
         during the scan initialization. 
 

bliss/scanning/scan.py

@@ -30,7 +30,11 @@ from bliss.common.cleanup import error_cleanup, axis as cleanup_axis, capture_ex
 from bliss.common.greenlet_utils import KillMask
 from bliss.common import plot as plot_mdl
 from bliss.common.utils import periodic_exec, deep_update
-from bliss.scanning.scan_meta import get_user_scan_meta, META_TIMING
+from bliss.scanning.scan_meta import (
+    META_TIMING,
+    get_user_scan_meta,
+    get_controllers_scan_meta,
+)
 from bliss.common.motor_group import is_motor_group
 from bliss.common.utils import Null, update_node_info, round
 from bliss.controllers.motor import Controller, get_real_axes
@@ -604,6 +608,9 @@ class Scan:
         """
         self.__name = name
         self.__scan_number = None
+        self.__user_scan_meta = None
+        self.__controllers_scan_meta = None
+        self.__scan_meta = None
         self._scan_info = ScanInfo()
 
         self.root_node = None
@@ -810,7 +817,7 @@ class Scan:
         self._scan_info["scan_nb"] = self.__scan_number
 
         # this has to be done when the writer is ready
-        self._prepare_scan_meta()
+        self._scan_info["filename"] = self.writer.filename
 
         start_timestamp = time.time()
         start_time = datetime.datetime.fromtimestamp(start_timestamp)
@@ -1277,7 +1284,8 @@ class Scan:
     def prepare(self, scan_info, devices_tree):
         self._prepare_devices(devices_tree)
         self.writer.prepare(self)
-        self._fill_meta("fill_meta_at_scan_start")
+
+        self._prepare_scan_meta()
 
         # The scan info was updated with device metadata
         self.node.prepared(self._scan_info)
@@ -1363,22 +1371,48 @@ class Scan:
             else:
                 event.connect(dev, "new_data", self._channel_event)
 
-    def _update_scan_info_with_user_scan_meta(self, meta_timing):
-        # be aware: this is patched in ct!
+    @property
+    def user_scan_meta(self):
+        if self.__user_scan_meta is None:
+            self.__user_scan_meta = get_user_scan_meta().copy()
+        return self.__user_scan_meta
+
+    @property
+    def _controllers_scan_meta(self):
+        if self.__controllers_scan_meta is None:
+            filtered_controller_names = []
+            for acq_obj in self.acq_chain.nodes_list:
+                # we do not want to collect controller metadata for controllers
+                # which are involved in the scan, since they will report their
+                # metadata via 'fill_meta' methods. So, we make a list of
+                # controller names to filter out.
+                # Nb: the acquisition object name == underlying device name normally
+                # /!\ this may be different from 'scan_metadata_name' in controllers metadata,
+                # so if someone tries hard it is possible to break the logic here
+                filtered_controller_names.append(acq_obj.name)
+            self.__controllers_scan_meta = get_controllers_scan_meta(
+                filtered_names=filtered_controller_names
+            )
+        return self.__controllers_scan_meta
+
+    def _update_scan_info_with_scan_meta(self, meta_timing):
         with KillMask(masked_kill_nb=1):
+            deep_update(
+                self._scan_info,
+                self._controllers_scan_meta.to_dict(self, timing=meta_timing),
+            )
             deep_update(
                 self._scan_info, self.user_scan_meta.to_dict(self, timing=meta_timing)
             )
         original = set(self._scan_info.get("scan_meta_categories", []))
-        extra = set(self.user_scan_meta.used_categories_names())
-        self._scan_info["scan_meta_categories"] = list(original | extra)
+        extra1 = set(self._controllers_scan_meta.used_categories_names())
+        extra2 = set(self.user_scan_meta.used_categories_names())
+        self._scan_info["scan_meta_categories"] = list(original | extra1 | extra2)
 
-    def _prepare_scan_meta(self):
-        self._scan_info["filename"] = self.writer.filename
-        # User metadata
-        self.user_scan_meta = get_user_scan_meta().copy()
-        self._update_scan_info_with_user_scan_meta(META_TIMING.START)
+        # update scan info in redis
+        update_node_info(self.node, dict(self._scan_info))
 
+    def _prepare_scan_meta(self):
         # Plot metadata
         display_extra = {}
         displayed_channels = self.__scan_display.displayed_channels
@@ -1396,6 +1430,12 @@ class Scan:
         if len(display_extra) > 0:
             self._scan_info["_display_extra"] = display_extra
 
+        # Collect exclusive scan metadata
+        self._fill_meta("fill_meta_at_scan_start")
+
+        # Update with controllers metadata and user metadata, and push to redis
+        self._update_scan_info_with_scan_meta(META_TIMING.START)
+
     def disconnect_all(self):
         for dev in self._devices:
             if isinstance(dev, (AcquisitionSlave, AcquisitionMaster)):
@@ -1433,6 +1473,7 @@ class Scan:
                 node = self.nodes.get(acq_obj)
                 if node is not None:
                     update_node_info(node, metadata)
+                self._controllers_scan_meta.instrument.set(acq_obj.name, metadata)
                 if method_name == "fill_meta_at_scan_start":
                     self._scan_info._set_device_meta(acq_obj, metadata)
 
@@ -1573,11 +1614,7 @@ class Scan:
                 self._fill_meta("fill_meta_at_scan_end")
 
             with capture():
-                self._update_scan_info_with_user_scan_meta(META_TIMING.END)
-
-                with KillMask(masked_kill_nb=1):
-                    # update scan_info in redis
-                    self.node._info.update(self.scan_info)
+                self._update_scan_info_with_scan_meta(META_TIMING.END)
 
             # wait the end of publishing
             # (should be already finished)

bliss/scanning/scan_meta.py

@@ -34,8 +34,6 @@ def get_user_scan_meta():
     global USER_SCAN_META
     if USER_SCAN_META is None:
         USER_SCAN_META = ScanMeta()
-        USER_SCAN_META.positioners.set("positioners", fill_positioners)
-        USER_SCAN_META.positioners.timing = META_TIMING.START | META_TIMING.END
         USER_SCAN_META.instrument.set("@NX_class", {"@NX_class": "NXinstrument"})
         USER_SCAN_META.instrument.timing = META_TIMING.END
         USER_SCAN_META.technique.set("@NX_class", {"@NX_class": "NXcollection"})
@@ -134,6 +132,36 @@ class ScanMetaCategory:
         return f"{self.__class__.__name__}{self.name}: \n " + s
 
 
+def get_controllers_scan_meta(filtered_names=None):
+    scan_meta = ScanMeta()
+    scan_meta.instrument.set("@NX_class", {"@NX_class": "NXinstrument"})
+    scan_meta.positioners.set("positioners", fill_positioners)
+    scan_meta.positioners.timing = META_TIMING.START | META_TIMING.END
+
+    for obj in global_map.instance_iter("controllers"):
+        if isinstance(obj, HasMetadataForScan):
+            if not obj.scan_metadata_enabled:
+                continue
+            if filtered_names and obj.scan_metadata_name in filtered_names:
+                # this object is filtered out
+                continue
+
+            def metadata_generator(scan, obj=obj):
+                """
+                Metadata generator registred with the instrument category
+                of user scan metadata.
+                """
+                metadata_name = obj.scan_metadata_name
+                if not metadata_name:
+                    user_warning(f"{repr(obj)} needs a name to publish scan metadata")
+                    return {}
+                else:
+                    return {metadata_name: obj.scan_metadata()}
+
+            scan_meta.instrument.set(obj, metadata_generator)
+    return scan_meta
+
+
 class ScanMeta:
     """Register metadata for all scans. The `Scan` object will call `ScanMeta.to_dict`
     to generate the metadata.

tests/common/test_logtools.py

@@ -451,7 +451,7 @@ def test_lima_devproxy_logger(default_session, lima_simulator, capsys, caplog):
     # now activate debug and check 7 active loggers
     debugon(lima)
     captured = capsys.readouterr().out
-    assert len(captured.strip().split("\n")) == 7
+    assert len(captured.strip().split("\n")) == 9
 
     # check some log messages for attribute get/set
     val = lima.proxy.acq_expo_time

tests/scans/test_hdf5.py

@@ -238,8 +238,6 @@ def test_scan_info_cleaning(alias_session):
 def test_fill_meta_mechanisms(alias_session, lima_simulator):
     lima_sim = alias_session.config.get("lima_simulator")
     transf = alias_session.config.get("transfocator_simulator")
-    # Register manually because it is not part of the session:
-    transf.enable_scan_metadata()
 
     s = scans.loopscan(3, .1, lima_sim)
     with h5py.File(s.writer.filename, mode="r") as f:

tests/scans/test_publishing.py

@@ -585,10 +585,8 @@ def test_scan_end_timing(
             first_index=first_index, include_filter="scan"
         ):
             if event_type == event_type.END_SCAN:
-                assert node.info.get("instrument") == {
-                    "DummyDevice": "slow",
-                    "some": "text",
-                }
+                assert node.info.get("instrument")["DummyDevice"] == "slow"
+                assert node.info.get("instrument")["some"] == "text"
                 return
 
     # force existance of scan node before starting the scan

tests/scans/test_scan_info.py

@@ -133,7 +133,12 @@ def test_scan_meta_master_and_device(session, scan_meta):
 
     s = Scan(chain, name="my_simple", save=False)
     s.run()
-    assert s.scan_info["instrument"] == {**master_dict, **device_dict}
+    # check scan info 'instrument' contains the scan metadata
+    # (it also contains controllers metadata but we do not check here)
+    for k, v in master_dict.items():
+        assert s.scan_info["instrument"][k] == v
+    for k, v in device_dict.items():
+        assert s.scan_info["instrument"][k] == v
 
 
 def test_positioners_in_scan_info(alias_session):