[writer] add support for notes

nexus_writer_service/io/nexus.py

@@ -718,46 +718,14 @@ def nxEntryInit(
         return parent[name]
 
 
-def nxNoteInit(parent, name, data=None, type=None, raise_on_exists=False):
-    """
-    Initialize NXnote instance
-
-    :param h5py.Group parent:
-    :param str name:
-    :param str data:
-    :param str type:
-    :param bool raise_on_exists:
-    :return h5py.Group:
-    :raises RuntimeError: wrong Nexus class or parent
-                          not an Nexus class instance
-    :raises NexusInstanceExists:
-    """
-    raiseIsNxClass(parent, None)
-    if nxClassInstantiate(parent, name, u"NXnote", raise_on_exists=raise_on_exists):
-        h5group = parent[name]
-        h5group.attrs["NX_class"] = u"NXnote"
-        update = True
-    else:
-        h5group = parent[name]
-        update = False
-    if data is not None:
-        updateDataset(h5group, "data", data)
-        update = True
-    if type is not None:
-        updateDataset(h5group, "type", type)
-        update = True
-    if update:
-        updated(h5group)
-    return h5group
-
-
 def nxProcessConfigurationInit(
-    parent, configdict=None, type="json", indent=2, raise_on_exists=False
+    parent, date=None, configdict=None, type="json", indent=2, raise_on_exists=False
 ):
     """
     Initialize NXnote instance
 
     :param h5py.Group parent:
+    :param datetime date:
     :param dict configdict:
     :param str type: 'json' or 'ini' or None (pprint)
     :param num indent: pretty-string with indent level
@@ -781,12 +749,14 @@ def nxProcessConfigurationInit(
     else:
         data = None
         type = None
-    name = "configuration"
-    group = nxNoteInit(
-        parent, name, data=data, type=type, raise_on_exists=raise_on_exists
+    return nxNote(
+        parent,
+        "configuration",
+        data=data,
+        type=type,
+        date=date,
+        raise_on_exists=raise_on_exists,
     )
-    updated(group)
-    return group
 
 
 def nxProcess(parent, name, configdict=None, raise_on_exists=False, **kwargs):
@@ -886,6 +856,42 @@ class FilePool(SharedLockPool):
 FILEPOOL = FilePool()
 
 
+def nxNote(parent, name, data=None, type=None, date=None, raise_on_exists=False):
+    """
+    Get NXnote instance (initialize when missing)
+
+    :param h5py.Group parent:
+    :param str name:
+    :param str data:
+    :param str type:
+    :param datetime date:
+    :param bool raise_on_exists:
+    :return h5py.Group:
+    :raises RuntimeError: wrong Nexus class or parent
+                          not an Nexus class instance
+    :raises NexusInstanceExists:
+    """
+    raiseIsNxClass(parent, None)
+    if nxClassInstantiate(parent, name, u"NXnote", raise_on_exists=raise_on_exists):
+        h5group = parent[name]
+        h5group.attrs["NX_class"] = u"NXnote"
+        update = True
+    else:
+        h5group = parent[name]
+        update = False
+    if data is not None:
+        updateDataset(h5group, "data", data)
+        update = True
+    if type is not None:
+        updateDataset(h5group, "type", type)
+        update = True
+    if date:
+        updateDataset(h5group, "date", datetime_to_nexus(date))
+    elif update:
+        updated(h5group)
+    return h5group
+
+
 class File(h5py.File):
     def __init__(self, name, mode="r", enable_file_locking=None, swmr=None, **kwargs):
         """

nexus_writer_service/subscribers/scan_writer_base.py

@@ -605,7 +605,7 @@ class NexusScanWriterBase(base_subscriber.BaseSubscriber):
     @contextmanager
     def nxmeasurement(self, subscan):
         """
-        Yields the generic NXdata instance (h5py.Group) or None
+        Yields the measurement instance (h5py.Group) or None
         when NXentry is missing
         """
         with self.nxentry(subscan) as nxentry:
@@ -614,6 +614,18 @@ class NexusScanWriterBase(base_subscriber.BaseSubscriber):
             else:
                 yield nexus.nxCollection(nxentry, "measurement")
 
+    @contextmanager
+    def nxnotes(self, subscan):
+        """
+        Yields the notes instance (h5py.Group) or None
+        when NXentry is missing
+        """
+        with self.nxentry(subscan) as nxentry:
+            if nxentry is None:
+                yield None
+            else:
+                yield nexus.nxCollection(nxentry, "notes")
+
     def _h5missing(self, variable):
         """
         :param str variable:
@@ -648,7 +660,7 @@ class NexusScanWriterBase(base_subscriber.BaseSubscriber):
             plotselect = self.plotselect
             firstplot = None
             plots = self.plots
-            subscan.logger.info("Create plots: {}".format(list(plots.keys())))
+            subscan.logger.info("Create {} plots".format(len(plots)))
             for plotname, plotparams in plots.items():
                 if plotname in nxentry:
                     subscan.logger.warning(
@@ -977,6 +989,8 @@ class NexusScanWriterBase(base_subscriber.BaseSubscriber):
         """
         self._save_positioners(subscan)
         self._create_plots(subscan)
+        self._fetch_subscan_metadata(subscan)
+        self._fetch_subscan_notes(subscan)
 
     @property
     def current_bytes(self):
@@ -1815,6 +1829,7 @@ class NexusScanWriterBase(base_subscriber.BaseSubscriber):
         with self.nxentry(subscan) as parent:
             if parent is None:
                 return
+            subscan.logger.info("Save scan metadata")
             info = self.info
             categories = set(info["scan_meta_categories"])
             categories -= {"positioners", "nexuswriter"}
@@ -1837,3 +1852,26 @@ class NexusScanWriterBase(base_subscriber.BaseSubscriber):
                     subscan.logger.info(
                         "Saved metadata categories: {}".format(list(scan_meta.keys()))
                     )
+
+    def _fetch_subscan_notes(self, subscan):
+        """
+        Save notes for this subscan
+
+        :param Subscan subscan:
+        """
+        notes = self.get_info("comments", [], cache=True)
+        if not notes:
+            return
+        with self.nxnotes(subscan) as parent:
+            if parent is None:
+                return
+            subscan.logger.info("Save scan notes")
+            for i, note in enumerate(notes, 1):
+                date = datetime.datetime.fromtimestamp(note["timestamp"])
+                nexus.nxNote(
+                    parent,
+                    "note_{:02d}".format(i),
+                    data=note["message"],
+                    type="text/plain",
+                    date=date,
+                )

tests/nexus_writer/helpers/nxw_test_data.py

@@ -42,6 +42,7 @@ def validate_scan_data(
     subscan=1,
     masters=None,
     detectors=None,
+    notes=None,
     master_name="timer",
     scan_shape=None,
     config=True,
@@ -54,6 +55,7 @@ def validate_scan_data(
     :param tuple masters: fast axis first by default `master_name` when 1D scan
                           or None otherwise
     :param list(str) detectors: expected detectors (derived from technique when missing)
+    :param list(str) notes:
     :param str master_name: chain master name
     :param tuple scan_shape: fast axis first 0D scan by default
     :param bool config: configurable writer
@@ -106,6 +108,7 @@ def validate_scan_data(
             withpolicy=withpolicy,
             technique=scan_technique,
             detectors=detectors,
+            notes=notes,
             variable_length=variable_length,
         )
         validate_measurement(
@@ -148,6 +151,7 @@ def validate_scan_data(
             save_options=save_options,
             detectors=detectors,
         )
+        validate_notes(nxentry, notes)
 
 
 def validate_scangroup_data(sequence, config=True, **kwargs):
@@ -207,6 +211,7 @@ def validate_nxentry(
     withpolicy=True,
     technique=None,
     detectors=None,
+    notes=None,
     variable_length=None,
 ):
     """
@@ -233,6 +238,8 @@ def validate_nxentry(
             if info["signals"]:
                 expected |= {name, "plotselect"}
     expected |= expected_applications(technique, config=config, withpolicy=withpolicy)
+    if notes:
+        expected.add("notes")
     assert_set_equal(actual, expected)
 
 
@@ -547,6 +554,24 @@ def validate_nxdata(
     assert axes == masters, (axes, masters, scan_shape, ptype, nxdata.name)
 
 
+def validate_notes(nxentry, notes):
+    """
+    :param h5py.Group nxentry:
+    :param list(str) notes:
+    """
+    if not notes:
+        assert "notes" not in nxentry, nxentry.name
+        return
+    group = nxentry["notes"]
+    assert group.attrs["NX_class"] == "NXcollection", group.name
+    for i, data in enumerate(notes, 1):
+        subgroup = group["note_{:02d}".format(i)]
+        assert subgroup.attrs["NX_class"] == "NXnote", subgroup.name
+        assert set(subgroup.keys()) == {"date", "type", "data"}
+        assert subgroup["data"][()] == data
+        assert subgroup["type"][()] == "text/plain"
+
+
 def expected_plots(technique, config=True, withpolicy=True, detectors=None):
     """
     All expected plots for this technique (see nexus_definitions.yml)

tests/nexus_writer/test_nxw_notes.py

+# -*- coding: utf-8 -*-
+#
+# This file is part of the bliss project
+#
+# Copyright (c) 2015-2019 Beamline Control Unit, ESRF
+# Distributed under the GNU LGPLv3. See LICENSE for more info.
+
+from bliss.common import scans
+import nxw_test_utils
+import nxw_test_data
+
+
+def test_nxw_notes(nexus_writer_config):
+    _test_nxw_notes(**nexus_writer_config)
+
+
+@nxw_test_utils.writer_stdout_on_exception
+def _test_nxw_notes(session=None, tmpdir=None, writer=None, **kwargs):
+    scan = scans.ct(.1, run=False, save=True)
+    notes = ["test1", "text2", "text3"]
+    for note in notes:
+        scan.add_comment(note)
+    nxw_test_utils.run_scan(scan)
+    nxw_test_utils.wait_scan_data_finished([scan], writer=writer, **kwargs)
+    nxw_test_data.assert_scan_data(scan, notes=notes, **kwargs)