Merge branch...

Merge branch '11-iter_bliss_scan_data_from_memory-yields-many-lima-arrays-if-counter_names-is-empty' into 'main'

Resolve "iter_bliss_scan_data_from_memory yields many lima arrays if counter_names is empty"

Closes #12 and #11

See merge request !76

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## 0.5.0 (unreleased)
 
+## 0.4.2
+
+Bug fixes:
+
+- `iter_bliss_scan_data_from_memory` was yielding lima data wrong
+
 ## 0.4.1
 
 Bug fixes:

src/ewoksdata/__init__.py

-__version__ = "0.4.1"
+__version__ = "0.4.2"

src/ewoksdata/data/blissdata/blissdatav1.py

 from numbers import Number
+from collections import Counter
 from typing import List, Optional
 
 from numpy.typing import ArrayLike
@@ -44,32 +45,38 @@ def iter_bliss_scan_data_from_memory(
             counter_streams[name] = stream
 
     client = StreamingClient({**lima_streams, **counter_streams})
+    lima_buffer_count = Counter()
 
     while True:
         try:
             output = client.read()
         except EndOfStream:
             break
-        else:
-            for stream, data in output.items():
-                _, payload = data  # drop index
-                if stream.name in lima_streams:
-                    lima_client = lima_clients[stream.name]
-                    prev_len = len(lima_client)
-                    lima_client.update(**payload[-1])
-                    try:
-                        data = lima_client[prev_len - 1 :]
-                    except IndexNoMoreThereError:
-                        continue
-                    buffers[stream.name.split(":")[-2]].extend(data)
-                else:
-                    buffers[stream.name.split(":")[-1]].extend(payload)
+        for stream, (_, payload) in output.items():
+            name_parts = stream.name.split(":")
+            if stream.name in lima_streams:
+                # payload is a sequence of JSON statuses
+                ctr_name = name_parts[-2]
+                last_status = payload[-1]
+                lima_client = lima_clients[stream.name]
+                lima_client.update(**last_status)
+                n_already_read = lima_buffer_count[ctr_name]
+                try:
+                    data = lima_client[n_already_read:]
+                except IndexNoMoreThereError:
+                    continue
+                buffers[ctr_name].extend(data)
+                lima_buffer_count[ctr_name] += len(data)
+            else:
+                # payload is a sequence of data points (0D, 1D, 2D)
+                ctr_name = name_parts[-1]
+                buffers[ctr_name].extend(payload)
 
-            nyield = min(len(v) for v in buffers.values())
-            if nyield:
-                for i in range(nyield):
-                    yield {name: values[i] for name, values in buffers.items()}
-                buffers = {name: values[nyield:] for name, values in buffers.items()}
+        nyield = min(len(v) for v in buffers.values())
+        if nyield:
+            for i in range(nyield):
+                yield {name: values[i] for name, values in buffers.items()}
+            buffers = {name: values[nyield:] for name, values in buffers.items()}
 
 
 def last_lima_image(channel_info: dict) -> ArrayLike:

src/ewoksdata/data/blissdata/blissdatavid31.py

 from numbers import Number
+from collections import Counter
 from typing import List, Optional
 
 from numpy.typing import ArrayLike
@@ -35,39 +36,46 @@ def iter_bliss_scan_data_from_memory(
 
     lima_streams = dict()
     lima_clients = dict()
+    lima_buffer_count = dict()
     counter_streams = dict()
     for name, stream in scan.streams.items():
         if stream.encoding["type"] == "json" and "lima" in stream.info["format"]:
             if name.split(":")[-2] in lima_names:
                 lima_streams[name] = stream
                 lima_clients[name] = lima_client_factory(stream.info)
+                lima_buffer_count[name] = 0
         elif name.split(":")[-1] in counter_names:
             counter_streams[name] = stream
 
     client = StreamingClient({**lima_streams, **counter_streams})
+    lima_buffer_count = Counter()
 
     while True:
         try:
             output = client.read()
         except EndOfStream:
             break
-        else:
-            for stream, data in output.items():
-                _, payload = data  # drop index
-                if stream.name in lima_streams:
-                    lima_client = lima_clients[stream.name]
-                    prev_len = len(lima_client)
-                    lima_client.update(**payload[-1])
-                    data = lima_client[prev_len - 1 :]
-                    buffers[stream.name.split(":")[-2]].extend(data)
-                else:
-                    buffers[stream.name.split(":")[-1]].extend(payload)
-
-            nyield = min(len(v) for v in buffers.values())
-            if nyield:
-                for i in range(nyield):
-                    yield {name: values[i] for name, values in buffers.items()}
-                buffers = {name: values[nyield:] for name, values in buffers.items()}
+        for stream, (_, payload) in output.items():
+            name_parts = stream.name.split(":")
+            if stream.name in lima_streams:
+                # payload is a sequence of JSON statuses
+                ctr_name = name_parts[-2]
+                lima_client = lima_clients[stream.name]
+                lima_client.update(**payload[-1])
+                n_already_read = lima_buffer_count[ctr_name]
+                data = lima_client[n_already_read:]
+                buffers[ctr_name].extend(data)
+                lima_buffer_count[ctr_name] += len(data)
+            else:
+                # payload is a sequence of data points (0D, 1D, 2D)
+                ctr_name = name_parts[-1]
+                buffers[ctr_name].extend(payload)
+
+        nyield = min(len(v) for v in buffers.values())
+        if nyield:
+            for i in range(nyield):
+                yield {name: values[i] for name, values in buffers.items()}
+            buffers = {name: values[nyield:] for name, values in buffers.items()}
 
 
 def last_lima_image(channel_info: dict) -> ArrayLike: