Improve Stream reliability by implementing state machine

* Wait for global header before considering Stream is Armed
* Stream sequence finishes only when the series end header is received
* Stream::stop just sets state to Stopped, ignoring received data messages;
  add abort, which really interrupts Stream sequence
* Add EventCtrlObj, trigger Error Events on _read_zmq_messages exceptions
* Add Camera::State::Armed, remove obsolete Readout

CMakeLists.txt

@@ -86,6 +86,7 @@ add_library(eiger SHARED
   src/EigerInterface.cpp
   src/EigerDetInfoCtrlObj.cpp
   src/EigerSyncCtrlObj.cpp
+  src/EigerEventCtrlObj.cpp
   src/EigerDecompress.cpp
   src/EigerSavingCtrlObj.cpp
   src/EigerStream.cpp

include/EigerCamera.h

@@ -65,7 +65,7 @@ class LIBEIGER Camera : public HwMaxImageSizeCallbackGen, public EventCallbackGe
 
  public:
   enum ApiGeneration { Eiger1, Eiger2 };
-  enum Status { Ready, Initializing, Exposure, Readout, Fault };
+  enum Status { Initializing, Ready, Armed, Exposure, Fault };
   enum CompressionType {NoCompression,LZ4,BSLZ4};
 
   Camera(const std::string& detector_ip, ApiGeneration api = Eiger1);
@@ -148,7 +148,7 @@ class LIBEIGER Camera : public HwMaxImageSizeCallbackGen, public EventCallbackGe
 
   void getCompression(bool&);
   void setCompression(bool);
-  void getCompressionType(CompressionType&) const;
+  void getCompressionType(CompressionType&);
   void setCompressionType(CompressionType);
   void getSerieId(int&);
   void deleteMemoryFiles();
@@ -171,6 +171,7 @@ class LIBEIGER Camera : public HwMaxImageSizeCallbackGen, public EventCallbackGe
   void _synchronize(); /// Used during plug-in initialization
   void _trigger_finished(bool);
   void _initialization_finished(bool ok);
+  void _disarm();
 
   void _updateImageSize();
 
@@ -212,10 +213,11 @@ class LIBEIGER Camera : public HwMaxImageSizeCallbackGen, public EventCallbackGe
 
   InternalStatus            m_initialize_state;
   InternalStatus            m_trigger_state;
+  bool                      m_armed;
   int                       m_serie_id;
   //- EigerAPI stuff
   eigerapi::Requests*       m_requests;
-       
+
   double                    m_temperature;
   double                    m_humidity;
   Cache<double>             m_exp_time;

include/EigerEventCtrlObj.h

+//###########################################################################
+// This file is part of LImA, a Library for Image Acquisition
+//
+// Copyright (C) : 2009-2014
+// European Synchrotron Radiation Facility
+// BP 220, Grenoble 38043
+// FRANCE
+//
+// This is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 3 of the License, or
+// (at your option) any later version.
+//
+// This software is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, see <http://www.gnu.org/licenses/>.
+//###########################################################################
+#ifndef EIGEREVENTCTRLOBJ_H
+#define EIGEREVENTCTRLOBJ_H
+
+#include "lima/HwEventCtrlObj.h"
+#include "EigerCamera.h"
+
+namespace lima
+{
+    namespace Eiger
+    {
+
+/*******************************************************************
+ * \class EventCtrlObj
+ * \brief Control object providing Eiger event interface
+ *******************************************************************/
+
+	class /*LIBEIGER*/ EventCtrlObj : public HwEventCtrlObj
+	{
+	    DEB_CLASS_NAMESPC(DebModCamera, "EventCtrlObj", "Eiger");
+
+	public:
+	    EventCtrlObj(Camera& cam);
+	    virtual ~EventCtrlObj();
+
+	private:
+	    class EventCallback : public lima::EventCallback {
+	    public:
+		EventCallback(EventCtrlObj& ctrl_obj)
+		  : m_ctrl_obj(ctrl_obj) {}
+		virtual void processEvent(Event *event)
+		{ m_ctrl_obj.reportEvent(event); }
+	    private:
+		EventCtrlObj& m_ctrl_obj;
+	    };
+	    Camera& m_cam;
+	    std::unique_ptr<EventCallback> m_cbk;
+	};
+
+    } // namespace Eiger
+} // namespace lima
+
+#endif // EIGEREVENTCTRLOBJ

include/EigerInterface.h

@@ -36,6 +36,7 @@ namespace lima
       class DetInfoCtrlObj;
       class SyncCtrlObj;
       class SavingCtrlObj;
+      class EventCtrlObj;
       class Camera;
       class Stream;
       class StreamInfo;
@@ -73,6 +74,7 @@ namespace lima
 	    DetInfoCtrlObj* m_det_info;
 	    SyncCtrlObj*    m_sync;
 	    SavingCtrlObj*  m_saving;
+	    EventCtrlObj*   m_event;
 	    Stream*	    m_stream;
 	    Decompress*	    m_decompress;
 	};

sip/EigerCamera.sip

@@ -29,7 +29,7 @@ namespace Eiger
   public:
 
     enum ApiGeneration { Eiger1, Eiger2 };
-    enum Status { Ready, Initializing, Exposure, Readout, Fault };
+    enum Status { Initializing, Ready, Armed, Exposure, Fault };
     enum CompressionType {NoCompression,LZ4,BSLZ4};
 
     Camera(const std::string& detector_ip,
@@ -102,7 +102,7 @@ namespace Eiger
 
     void getCompression(bool& /Out/);
     void setCompression(const bool);
-    void getCompressionType(CompressionType&) const;
+    void getCompressionType(CompressionType& /Out/);
     void setCompressionType(CompressionType);   
 
     void getSerieId(int& /Out/);

src/EigerCamera.cpp

@@ -223,6 +223,7 @@ Camera::Camera(const std::string& detector_ip, 	///< [in] Ip address of the dete
                 m_detectorImageType(Bpp16),
 		m_initialize_state(IDLE),
 		m_trigger_state(IDLE),
+		m_armed(false),
 		m_serie_id(0),
                 m_requests(new Requests(detector_ip)),
                 m_exp_time(1.),
@@ -313,8 +314,8 @@ void Camera::prepareAcq()
 {
   DEB_MEMBER_FUNCT();
   AutoMutex aLock(m_cond.mutex());
-  if(m_trigger_state != IDLE)
-    disarm();
+  if(m_armed)
+    THROW_HW_ERROR(Error) << "Camera already armed";
   
   unsigned nb_images, nb_triggers;
   switch(m_trig_mode)
@@ -359,6 +360,7 @@ void Camera::prepareAcq()
       arm_cmd->wait(timeout);
       DEB_TRACE() << "Arm end";
       m_serie_id = arm_cmd->get_serie_id();
+      m_armed = true;
     }
   catch(const eigerapi::EigerException &e)
     {
@@ -377,7 +379,6 @@ void Camera::startAcq()
   DEB_MEMBER_FUNCT();
   AutoMutex lock(m_cond.mutex());
 
-
   if(m_trig_mode == IntTrig ||
      m_trig_mode == IntTrigMult)
     {
@@ -401,6 +402,7 @@ void Camera::startAcq()
 void Camera::stopAcq()
 {
   DEB_MEMBER_FUNCT();
+  AutoMutex lock(m_cond.mutex());
   EIGER_SYNC_CMD(Requests::ABORT);
 }
 
@@ -696,6 +698,8 @@ Camera::Status Camera::getStatus() ///< [out] current camera status
     status = Fault;
   else if(m_trigger_state == RUNNING)
     status = Exposure;
+  else if(m_armed)
+    status = Armed;
   else if(m_initialize_state == RUNNING)
     status = Initializing;
   else
@@ -1177,9 +1181,10 @@ void Camera::setCompression(bool value)
   EIGER_SYNC_SET_PARAM(Requests::FILEWRITER_COMPRESSION,value);
 }
 
-void Camera::getCompressionType(Camera::CompressionType& type) const
+void Camera::getCompressionType(Camera::CompressionType& type)
 {
   DEB_MEMBER_FUNCT();
+  AutoMutex lock(m_cond.mutex());
   type = m_compression_type;
   DEB_RETURN() << DEB_VAR1(type);
 }
@@ -1214,6 +1219,7 @@ void Camera::setCompressionType(Camera::CompressionType type)
 				 << " not allowed";
     
   EIGER_SYNC_SET_PARAM(Requests::COMPRESSION_TYPE, s);
+  AutoMutex lock(m_cond.mutex());
   m_compression_type = type;
 }
 
@@ -1233,7 +1239,19 @@ void Camera::deleteMemoryFiles()
 void Camera::disarm()
 {
   DEB_MEMBER_FUNCT();
-  EIGER_SYNC_CMD(Requests::DISARM);
+  AutoMutex lock(m_cond.mutex());
+  _disarm();
+}
+
+void Camera::_disarm()
+{
+  DEB_MEMBER_FUNCT();
+  DEB_PARAM() << DEB_VAR1(m_armed);
+  if (m_armed) {
+    DEB_TRACE() << "Disarming";
+    EIGER_SYNC_CMD(Requests::DISARM);
+    m_armed = false;
+  }
 }
 
 const std::string& Camera::getDetectorIp() const

src/EigerEventCtrlObj.cpp

+//###########################################################################
+// This file is part of LImA, a Library for Image Acquisition
+//
+// Copyright (C) : 2009-2014
+// European Synchrotron Radiation Facility
+// BP 220, Grenoble 38043
+// FRANCE
+//
+// This is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 3 of the License, or
+// (at your option) any later version.
+//
+// This software is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, see <http://www.gnu.org/licenses/>.
+//###########################################################################
+#include "EigerEventCtrlObj.h"
+
+using namespace lima;
+using namespace lima::Eiger;
+using namespace std;
+
+
+//-----------------------------------------------------
+// @brief Ctor
+//-----------------------------------------------------
+EventCtrlObj::EventCtrlObj(Camera& cam)
+  : m_cam(cam), m_cbk(new EventCallback(*this))
+{
+    DEB_CONSTRUCTOR();
+    m_cam.registerEventCallback(*m_cbk);
+}
+
+//-----------------------------------------------------
+// @brief Dtor
+//-----------------------------------------------------
+EventCtrlObj::~EventCtrlObj()
+{
+    DEB_DESTRUCTOR();
+    m_cam.unregisterEventCallback(*m_cbk);
+}
+

src/EigerInterface.cpp

@@ -23,6 +23,7 @@
 #include "EigerCamera.h"
 #include "EigerDetInfoCtrlObj.h"
 #include "EigerSyncCtrlObj.h"
+#include "EigerEventCtrlObj.h"
 #include "EigerSavingCtrlObj.h"
 #include "EigerStream.h"
 #include "EigerDecompress.h"
@@ -47,6 +48,9 @@ Interface::Interface(Camera& cam) : m_cam(cam)
   m_saving = new SavingCtrlObj(cam);
   m_cap_list.push_back(HwCap(m_saving));
 
+  m_event = new EventCtrlObj(cam);
+  m_cap_list.push_back(HwCap(m_event));
+
   m_stream = new Stream(cam);
   
   HwBufferCtrlObj* buffer = m_stream->getBufferCtrlObj();
@@ -95,6 +99,10 @@ void Interface::reset(ResetLevel reset_level)
 void Interface::prepareAcq()
 {
     DEB_MEMBER_FUNCT();
+
+    if (m_cam.getStatus() == Camera::Armed)
+      m_cam.disarm();
+
     bool use_filewriter = m_saving->isActive(); 
     m_stream->setActive(!use_filewriter);
     m_decompress->setActive(!use_filewriter);
@@ -104,6 +112,10 @@ void Interface::prepareAcq()
     m_cam.prepareAcq();
     int serie_id; m_cam.getSerieId(serie_id);
     m_saving->setSerieId(serie_id);
+    if (!use_filewriter) {
+      double stream_armed_timeout = 5.0;
+      m_stream->waitArmed(stream_armed_timeout);
+    }
 }
 
 //-----------------------------------------------------
@@ -112,11 +124,20 @@ void Interface::prepareAcq()
 void Interface::startAcq()
 {
     DEB_MEMBER_FUNCT();
-    // either we use eiger saving or the raw stream
-    if(m_saving->isActive())
-      m_saving->start();
-    else
-      m_stream->start();
+    // start data retrieval subsystems only in first call
+    if (getNbHwAcquiredFrames() == 0) {
+      // either we use eiger saving or the raw stream
+      if(m_saving->isActive())
+	m_saving->start();
+      else
+	m_stream->start();
+    } else {
+      TrigMode trig_mode;
+      m_cam.getTrigMode(trig_mode);
+      if (trig_mode != IntTrigMult)
+	DEB_WARNING() << "Unexpected start";
+    }
+
     m_cam.startAcq();
 }
 
@@ -173,8 +194,8 @@ void Interface::getStatus(StatusType& status)
         status.set(HwInterface::StatusType::Exposure);
         break;
 
-      case Camera::Readout:
-        status.set(HwInterface::StatusType::Readout);
+      case Camera::Armed:
+        status.set(HwInterface::StatusType::Ready);
         break;
 
       case Camera::Fault:

src/EigerStream.h

@@ -28,6 +28,8 @@
 #include "EigerStreamInfo.h"
 #include "lima/HwBufferMgr.h"
 
+#include <json/json.h>
+
 namespace lima
 {
   namespace Eiger
@@ -37,10 +39,11 @@ namespace lima
       DEB_CLASS_NAMESPC(DebModCamera,"Stream","Eiger");
     public:
       class Message;
-      typedef std::shared_ptr<Stream::Message> MessagePtr;
+      typedef std::shared_ptr<Message> MessagePtr;
       typedef Camera::CompressionType CompressionType;
 
       enum HeaderDetail {ALL,BASIC,OFF};
+      enum State {Init,Idle,Connected,Armed,Running,Failed,Stopped,Aborted};
 
       struct ImageData {
 	MessagePtr msg;
@@ -55,6 +58,7 @@ namespace lima
       void start();
       void stop();
       bool isRunning() const;
+      void waitArmed(double timeout);
 
       void getHeaderDetail(HeaderDetail&) const;
       void setHeaderDetail(HeaderDetail);
@@ -75,9 +79,16 @@ namespace lima
       friend class ImageDataPtr;
 
       typedef std::map<void*,ImageData> Data2Message;
+      typedef std::vector<MessagePtr> MessageList;
+
+      bool _isRunning() const;
 
       static void* _runFunc(void*);
       void _run();
+      void _run_sequence();
+      Json::Value _get_global_header(const Json::Value& stream_header,
+				     MessageList& pending_messages);
+      Json::Value _get_json_header(MessagePtr &msg);
       bool _read_zmq_messages(void *stream_socket);
       void _send_synchro();
 
@@ -87,26 +98,26 @@ namespace lima
       char		m_endianess;
       bool		m_active;
       HeaderDetail	m_header_detail;
+      std::string	m_header_detail_str;
       bool		m_dirty_flag;
+      State		m_state;
 
       mutable Cond	m_cond;
-      bool		m_wait;
-      bool		m_running;
-      bool		m_stop;
+      bool		m_quit;
 
       pthread_t		m_thread_id;
       void*		m_zmq_context;
       int		m_pipes[2];
       Data2Message	m_data_2_msg;
       StreamInfo	m_last_info;
-      Timestamp		m_activate_tstamp;
-      TrigMode		m_trigger_mode;
+      bool		m_ext_trigger;
       CompressionType	m_comp_type;
 
       std::unique_ptr<_BufferCtrlObj>	m_buffer_ctrl_obj;
       StdBufferCbMgr*			m_buffer_mgr;
     };
 
+    std::ostream& operator <<(std::ostream& os, Stream::State state);
     std::ostream& operator <<(std::ostream& os,
 			      const Stream::ImageData& img_data);
   }

tango/Eiger.py

@@ -86,10 +86,10 @@ class Eiger(PyTango.Device_4Impl):
         self.__CompressionType = {'NONE': EigerAcq.Camera.NoCompression,
                                   'LZ4': EigerAcq.Camera.LZ4,
                                   'BSLZ4': EigerAcq.Camera.BSLZ4}
-        self.__Status = {'READY': EigerAcq.Camera.Ready,
-                         'INITIALIZING': EigerAcq.Camera.Initializing,
+        self.__Status = {'INITIALIZING': EigerAcq.Camera.Initializing,
+                         'READY': EigerAcq.Camera.Ready,
+                         'ARMED': EigerAcq.Camera.Armed,
                          'EXPOSURE': EigerAcq.Camera.Exposure,
-                         'READOUT': EigerAcq.Camera.Readout,
                          'FAULT': EigerAcq.Camera.Fault}
 
 #------------------------------------------------------------------